add support for django 4.1 (#208)

* Fixed failing timezone unit test for Django 4.1 (#169)

* Added Django 4.1 tests into test suite

* Fixed datetime issues

* Added skipped timezone tests

* fix date_trunc_sql to work with < 4-digit year

set has_case_insensitive_like to True

Revert "fix date_trunc_sql to work with < 4-digit year"

This reverts commit 492e32c.

Revert "set has_case_insensitive_like to True"

This reverts commit 2bc4f8e.

fix date_trunc_sql to work with < 4-digit year (#188)

* set has_case_insensitive_like to True (#189)

Co-Authored-By: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com>

Co-authored-by: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com>

* bump required django version to include 4.1.x updates (#192)

* add introspected field type for DurationField (#191)

* Add check to see if `FROM` clause exists (#190)

* Make MSSQL introspection raise DatabaseError on nonexistent tables (#194)

* Make MSSQL introspection raise DatabaseError on nonexistent tables

* edit error message to make it specific to MSSQL

* Refix JSONField has_key so it doesn't break previous Django versions (#195)

* Fix removing unique_together constraint if exists primary key/unique constraint on the same field. (#204)

* Fix removing unique_together constraint if
 exists primary key/unique constraint on the same field.

* add django version requirement

* fix jsonfield test in sql server 2022 and newer (#203)

* fix jsonfield test in sql server 2022

* fix sql server version

* fix bulk update tests (#205)

* match native  function updates

* syntax update

* fix `mssql-django` tests

* trim whitespace

* create custom `Window.as_sql` (#207)

Co-Authored-By: Khanh Bui 85855766+khanhmaibui@users.noreply.github.com

* Add skipped tests to Django 4.1 (#199)

Skips the currently unfixable tests added by Django 4.1

Co-authored-by: mShan0 <mark.shan19@gmail.com>

Co-authored-by: jmah8 <59151084+jmah8@users.noreply.github.com>
Co-authored-by: Khanh Bui <khanhmb815@gmail.com>
Co-authored-by: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com>

Author: 3 people

azure-pipelines.yml

@@ -24,6 +24,16 @@ jobs:
 
     strategy:
       matrix:
+        Python3.10 - Django 4.1:
+          python.version: '3.10'
+          tox.env: 'py310-django41'
+        Python 3.9 - Django 4.1:
+          python.version: '3.9'
+          tox.env: 'py39-django41'
+        Python 3.8 - Django 4.1:
+          python.version: '3.8'
+          tox.env: 'py38-django41'
+
         Python3.10 - Django 4.0:
           python.version: '3.10'
           tox.env: 'py310-django40'
@@ -101,6 +111,16 @@ jobs:
 
     strategy:
       matrix:
+        Python3.10 - Django 4.1:
+          python.version: '3.10'
+          tox.env: 'py310-django41'
+        Python 3.9 - Django 4.1:
+          python.version: '3.9'
+          tox.env: 'py39-django41'
+        Python 3.8 - Django 4.1:
+          python.version: '3.8'
+          tox.env: 'py38-django41'
+
         Python3.10 - Django 4.0:
           python.version: '3.10'
           tox.env: 'py310-django40'

mssql/compiler.py

@@ -6,7 +6,7 @@
 
 import django
 from django.db.models.aggregates import Avg, Count, StdDev, Variance
-from django.db.models.expressions import Ref, Subquery, Value
+from django.db.models.expressions import Ref, Subquery, Value, Window
 from django.db.models.functions import (
     Chr, ConcatPair, Greatest, Least, Length, LPad, Random, Repeat, RPad, StrIndex, Substr, Trim
 )
@@ -129,6 +129,41 @@ def _as_sql_variance(self, compiler, connection):
         function = '%sP' % function
     return self.as_sql(compiler, connection, function=function)
 
+def _as_sql_window(self, compiler, connection, template=None):
+    connection.ops.check_expression_support(self)
+    if not connection.features.supports_over_clause:
+        raise NotSupportedError("This backend does not support window expressions.")
+    expr_sql, params = compiler.compile(self.source_expression)
+    window_sql, window_params = [], ()
+
+    if self.partition_by is not None:
+        sql_expr, sql_params = self.partition_by.as_sql(
+            compiler=compiler,
+            connection=connection,
+            template="PARTITION BY %(expressions)s",
+        )
+        window_sql.append(sql_expr)
+        window_params += tuple(sql_params)
+
+    if self.order_by is not None:
+        order_sql, order_params = compiler.compile(self.order_by)
+        window_sql.append(order_sql)
+        window_params += tuple(order_params)
+    else:
+        # MSSQL window functions require an OVER clause with ORDER BY
+        window_sql.append('ORDER BY (SELECT NULL)')
+    
+    if self.frame:
+        frame_sql, frame_params = compiler.compile(self.frame)
+        window_sql.append(frame_sql)
+        window_params += tuple(frame_params)
+
+    template = template or self.template
+
+    return (
+        template % {"expression": expr_sql, "window": " ".join(window_sql).strip()},
+        (*params, *window_params),
+    )
 
 def _cursor_iter(cursor, sentinel, col_count, itersize):
     """
@@ -281,7 +316,9 @@ def as_sql(self, with_limits=True, with_col_aliases=False):
                 if for_update_part and self.connection.features.for_update_after_from:
                     from_.insert(1, for_update_part)
 
-                result += [', '.join(out_cols), 'FROM', *from_]
+                result += [', '.join(out_cols)]
+                if from_:
+                    result += ['FROM', *from_]
                 params.extend(f_params)
 
                 if where:
@@ -422,6 +459,9 @@ def _as_microsoft(self, node):
         if django.VERSION >= (3, 1):
             if isinstance(node, json_KeyTransform):
                 as_microsoft = _as_sql_json_keytransform
+        if django.VERSION >= (4, 1):
+            if isinstance(node, Window):
+                as_microsoft = _as_sql_window
         if as_microsoft:
             node = node.copy()
             node.as_microsoft = types.MethodType(as_microsoft, node)

mssql/features.py

@@ -17,6 +17,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     can_use_chunked_reads = False
     for_update_after_from = True
     greatest_least_ignores_nulls = True
+    has_case_insensitive_like = True
     has_json_object_function = False
     has_json_operators = False
     has_native_json_field = False
@@ -64,3 +65,10 @@ def has_zoneinfo_database(self):
     @cached_property
     def supports_json_field(self):
         return self.connection.sql_server_version >= 2016 or self.connection.to_azure_sql_db
+
+    @cached_property
+    def introspected_field_types(self):
+        return {
+            **super().introspected_field_types,
+            "DurationField": "BigIntegerField",
+        }

mssql/functions.py

@@ -24,6 +24,7 @@
     from django.db.models.functions.math import Random
 
 DJANGO3 = VERSION[0] >= 3
+DJANGO41 = VERSION >= (4, 1)
 
 
 class TryCast(Cast):
@@ -95,7 +96,7 @@ def sqlserver_random(self, compiler, connection, **extra_context):
 
 def sqlserver_window(self, compiler, connection, template=None):
     # MSSQL window functions require an OVER clause with ORDER BY
-    if self.order_by is None:
+    if VERSION < (4, 1) and self.order_by is None:
         self.order_by = Value('SELECT NULL')
     return self.as_sql(compiler, connection, template)
 
@@ -204,8 +205,11 @@ def json_HasKeyLookup(self, compiler, connection):
     else:
         lhs, _ = self.process_lhs(compiler, connection)
         lhs_json_path = '$'
-    sql = lhs + ' IN (SELECT ' + lhs + ' FROM ' + self.lhs.output_field.model._meta.db_table + \
-    ' CROSS APPLY OPENJSON(' + lhs + ') WITH ( [json_path_value] char(1) \'%s\') WHERE [json_path_value] IS NOT NULL)'
+    if connection.sql_server_version >= 2022:
+        sql = "JSON_PATH_EXISTS(%s, '%%s') > 0" % lhs
+    else:
+        sql = lhs + ' IN (SELECT ' + lhs + ' FROM ' + self.lhs.output_field.model._meta.db_table + \
+        ' CROSS APPLY OPENJSON(' + lhs + ') WITH ( [json_path_value] char(1) \'%s\') WHERE [json_path_value] IS NOT NULL)'
     # Process JSON path from the right-hand side.
     rhs = self.rhs
     rhs_params = []
@@ -216,7 +220,13 @@ def json_HasKeyLookup(self, compiler, connection):
             *_, rhs_key_transforms = key.preprocess_lhs(compiler, connection)
         else:
             rhs_key_transforms = [key]
-        rhs_params.append('%s%s' % (
+        if VERSION >= (4, 1):
+            *rhs_key_transforms, final_key = rhs_key_transforms
+            rhs_json_path = compile_json_path(rhs_key_transforms, include_root=False)
+            rhs_json_path += self.compile_json_path_final_key(final_key)
+            rhs_params.append(lhs_json_path + rhs_json_path)
+        else:
+            rhs_params.append('%s%s' % (
             lhs_json_path,
             compile_json_path(rhs_key_transforms, include_root=False),
         ))
@@ -277,11 +287,18 @@ def bulk_update_with_default(self, objs, fields, batch_size=None, default=0):
         raise ValueError('bulk_update() cannot be used with primary key fields.')
     if not objs:
         return 0
+    if DJANGO41:
+        for obj in objs:
+            obj._prepare_related_fields_for_save(
+                operation_name="bulk_update", fields=fields
+            )
     # PK is used twice in the resulting update query, once in the filter
     # and once in the WHEN. Each field will also have one CAST.
-    max_batch_size = connections[self.db].ops.bulk_batch_size(['pk', 'pk'] + fields, objs)
+    self._for_write = True
+    connection = connections[self.db]
+    max_batch_size = connection.ops.bulk_batch_size(['pk', 'pk'] + fields, objs)
     batch_size = min(batch_size, max_batch_size) if batch_size else max_batch_size
-    requires_casting = connections[self.db].features.requires_casted_case_in_updates
+    requires_casting = connection.features.requires_casted_case_in_updates
     batches = (objs[i:i + batch_size] for i in range(0, len(objs), batch_size))
     updates = []
     for batch_objs in batches:
@@ -291,12 +308,12 @@ def bulk_update_with_default(self, objs, fields, batch_size=None, default=0):
             when_statements = []
             for obj in batch_objs:
                 attr = getattr(obj, field.attname)
-                if not isinstance(attr, Expression):
+                if not hasattr(attr, "resolve_expression"):
                     if attr is None:
                         value_none_counter += 1
                     attr = Value(attr, output_field=field)
                 when_statements.append(When(pk=obj.pk, then=attr))
-            if connections[self.db].vendor == 'microsoft' and value_none_counter == len(when_statements):
+            if connection.vendor == 'microsoft' and value_none_counter == len(when_statements):
                 case_statement = Case(*when_statements, output_field=field, default=Value(default))
             else:
                 case_statement = Case(*when_statements, output_field=field)
@@ -305,9 +322,10 @@ def bulk_update_with_default(self, objs, fields, batch_size=None, default=0):
             update_kwargs[field.attname] = case_statement
         updates.append(([obj.pk for obj in batch_objs], update_kwargs))
     rows_updated = 0
+    queryset = self.using(self.db)
     with transaction.atomic(using=self.db, savepoint=False):
         for pks, update_kwargs in updates:
-            rows_updated += self.filter(pk__in=pks).update(**update_kwargs)
+            rows_updated += queryset.filter(pk__in=pks).update(**update_kwargs)
     return rows_updated
 
 

mssql/introspection.py

@@ -1,6 +1,7 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the BSD license.
 
+from django.db import DatabaseError
 import pyodbc as Database
 
 from django import VERSION
@@ -108,6 +109,9 @@ def get_table_description(self, cursor, table_name, identity_check=True):
         # map pyodbc's cursor.columns to db-api cursor description
         columns = [[c[3], c[4], None, c[6], c[6], c[8], c[10], c[12]] for c in cursor.columns(table=table_name)]
 
+        if not columns:
+            raise DatabaseError(f"Table {table_name} does not exist.")
+            
         items = []
         for column in columns:
             if VERSION >= (3, 2):