Merge pull request #78 from microsoft/dev

Prepare for for 1.1 release

Author: absci

README.md

@@ -1,4 +1,4 @@
-# Project
+# SQL Server backend for Django
 
 Welcome to the MSSQL-Django 3rd party backend project!
 
@@ -10,7 +10,7 @@ We hope you enjoy using the MSSQL-Django 3rd party backend.
 
 ## Features
 
--  Supports Django 2.2, 3.0, 3.1 and 3.2
+-  Supports Django 2.2, 3.0, 3.1, 3.2 and 4.0
 -  Tested on Microsoft SQL Server 2016, 2017, 2019
 -  Passes most of the tests of the Django test suite
 -  Compatible with
@@ -153,7 +153,7 @@ Dictionary. Current available keys are:
 -  extra_params
 
    String. Additional parameters for the ODBC connection. The format is
-   ``"param=value;param=value"``, [Azure AD Authentication](https://github.com/microsoft/mssql-django/wiki/Azure-AD-Authentication) can be added to this field.
+   ``"param=value;param=value"``, [Azure AD Authentication](https://github.com/microsoft/mssql-django/wiki/Azure-AD-Authentication) (Service Principal, Interactive, Msi) can be added to this field.
 
 -  collation
 
@@ -225,7 +225,6 @@ The following features are currently not fully supported:
 - Exists function in order_by
 - Righthand power and arithmetic with datatimes
 - Timezones, timedeltas not fully supported
-- `bulk_update` multiple field to null
 - Rename field/model with foreign key constraint
 - Database level constraints
 - Math degrees power or radians

azure-pipelines.yml

@@ -16,11 +16,21 @@ jobs:
     pool:
       name: Django-1ES-pool
       demands:
-      - imageOverride -equals MMS2016
+      - imageOverride -equals JDBC-MMS2019-SQL2019
     timeoutInMinutes: 120
 
     strategy:
       matrix:
+        Python3.10 - Django 4.0:
+          python.version: '3.10'
+          tox.env: 'py310-django40'
+        Python 3.9 - Django 4.0:
+          python.version: '3.9'
+          tox.env: 'py39-django40'
+        Python 3.8 - Django 4.0:
+          python.version: '3.8'
+          tox.env: 'py38-django40'
+
         Python 3.9 - Django 3.2:
           python.version: '3.9'
           tox.env: 'py39-django32'
@@ -85,10 +95,20 @@ jobs:
           Invoke-WebRequest https://download.microsoft.com/download/E/6/B/E6BFDC7A-5BCD-4C51-9912-635646DA801E/en-US/17.5.2.1/x64/msodbcsql.msi -OutFile msodbcsql.msi
           msiexec /quiet /passive /qn /i msodbcsql.msi IACCEPTMSODBCSQLLICENSETERMS=YES
           Get-OdbcDriver
+        displayName: Install ODBC
 
-          docker pull microsoft/mssql-server-windows-developer
-          docker run -e 'ACCEPT_EULA=Y' -e 'SA_PASSWORD=MyPassword42' -p 1433:1433 -d microsoft/mssql-server-windows-developer
-          docker ps
+      - powershell: |
+          Import-Module "sqlps"
+          Invoke-Sqlcmd @"
+              EXEC xp_instance_regwrite N'HKEY_LOCAL_MACHINE', N'Software\Microsoft\MSSQLServer\MSSQLServer', N'LoginMode', REG_DWORD, 2
+              ALTER LOGIN [sa] ENABLE;
+              ALTER LOGIN [sa] WITH PASSWORD = 'MyPassword42', CHECK_POLICY=OFF;
+          "@
+        displayName: Set up SQL Server
+
+      - powershell: |
+          Restart-Service -Name MSSQLSERVER -Force
+        displayName: Restart SQL Server
 
       - powershell: |
           python -m pip install --upgrade pip wheel setuptools
@@ -107,6 +127,16 @@ jobs:
 
     strategy:
       matrix:
+        Python3.10 - Django 4.0:
+          python.version: '3.10'
+          tox.env: 'py310-django40'
+        Python 3.9 - Django 4.0:
+          python.version: '3.9'
+          tox.env: 'py39-django40'
+        Python 3.8 - Django 4.0:
+          python.version: '3.8'
+          tox.env: 'py38-django40'
+
         Python 3.9 - Django 3.2:
           python.version: '3.9'
           tox.env: 'py39-django32'
@@ -186,5 +216,5 @@ jobs:
         displayName: Publish test results via jUnit
         inputs:
           testResultsFormat: 'JUnit'
-          testResultsFiles: 'result.xml'
+          testResultsFiles: 'django/result.xml'
           testRunTitle: 'junit-$(Agent.OS)-$(Agent.OSArchitecture)-$(tox.env)'

docker/Dockerfile

@@ -258,6 +258,7 @@ def get_new_connection(self, conn_params):
         options = conn_params.get('OPTIONS', {})
         driver = options.get('driver', 'ODBC Driver 17 for SQL Server')
         dsn = options.get('dsn', None)
+        options_extra_params = options.get('extra_params', '')
 
         # Microsoft driver names assumed here are:
         # * SQL Server Native Client 10.0/11.0
@@ -291,10 +292,10 @@ def get_new_connection(self, conn_params):
 
         if user:
             cstr_parts['UID'] = user
-            if 'Authentication=ActiveDirectoryInteractive' not in options.get('extra_params', ''):
+            if 'Authentication=ActiveDirectoryInteractive' not in options_extra_params:
                 cstr_parts['PWD'] = password
         else:
-            if ms_drivers.match(driver):
+            if ms_drivers.match(driver) and 'Authentication=ActiveDirectoryMsi' not in options_extra_params:
                 cstr_parts['Trusted_Connection'] = trusted_connection
             else:
                 cstr_parts['Integrated Security'] = 'SSPI'

docker/docker-compose.yml

@@ -302,7 +302,8 @@ def as_sql(self, with_limits=True, with_col_aliases=False):
                     result.append('HAVING %s' % having)
                     params.extend(h_params)
 
-            if self.query.explain_query:
+            explain = self.query.explain_info if django.VERSION >= (4, 0) else self.query.explain_query
+            if explain:
                 result.insert(0, self.connection.ops.explain_query_prefix(
                     self.query.explain_format,
                     **self.query.explain_options

mssql/base.py

@@ -4,6 +4,7 @@
 import binascii
 import os
 
+from django.db.utils import InterfaceError
 from django.db.backends.base.creation import BaseDatabaseCreation
 from django import VERSION as django_version
 
@@ -16,6 +17,25 @@ def cursor(self):
 
         return self.connection._nodb_connection.cursor()
 
+    def _create_test_db(self, verbosity, autoclobber, keepdb=False):
+        """
+        Internal implementation - create the test db tables.
+        """
+
+        # Try to create the test DB, but if we fail due to 28000 (Login failed for user),
+        #   it's probably because the user doesn't have permission to [dbo].[master],
+        #   so we can proceed if we're keeping the DB anyway.
+        # https://github.com/microsoft/mssql-django/issues/61
+        try:
+            return super()._create_test_db(verbosity, autoclobber, keepdb)
+        except InterfaceError as err:
+            if err.args[0] == '28000' and keepdb:
+                self.log('Received error %s, proceeding because keepdb=True' % (
+                    err.args[1],
+                ))
+            else:
+                raise err
+
     def _destroy_test_db(self, test_database_name, verbosity):
         """
         Internal implementation - remove the test db tables.

mssql/compiler.py

@@ -12,6 +12,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     can_introspect_small_integer_field = True
     can_return_columns_from_insert = True
     can_return_id_from_insert = True
+    can_rollback_ddl = True
     can_use_chunked_reads = False
     for_update_after_from = True
     greatest_least_ignores_nulls = True
@@ -47,26 +48,11 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     supports_timezones = False
     supports_transactions = True
     uses_savepoints = True
-
-    @cached_property
-    def has_bulk_insert(self):
-        return self.connection.sql_server_version > 2005
-
-    @cached_property
-    def supports_nullable_unique_constraints(self):
-        return self.connection.sql_server_version > 2005
-
-    @cached_property
-    def supports_partially_nullable_unique_constraints(self):
-        return self.connection.sql_server_version > 2005
-
-    @cached_property
-    def supports_partial_indexes(self):
-        return self.connection.sql_server_version > 2005
-
-    @cached_property
-    def supports_functions_in_partial_indexes(self):
-        return self.connection.sql_server_version > 2005
+    has_bulk_insert = True
+    supports_nullable_unique_constraints = True
+    supports_partially_nullable_unique_constraints = True
+    supports_partial_indexes = True
+    supports_functions_in_partial_indexes = True
 
     @cached_property
     def has_zoneinfo_database(self):
@@ -76,4 +62,4 @@ def has_zoneinfo_database(self):
 
     @cached_property
     def supports_json_field(self):
-        return self.connection.sql_server_version >= 2016
+        return self.connection.sql_server_version >= 2016 or self.connection.to_azure_sql_db

mssql/creation.py

@@ -4,14 +4,17 @@
 import json
 
 from django import VERSION
-from django.db import NotSupportedError
+
+from django.db import NotSupportedError, connections, transaction
 from django.db.models import BooleanField, Value
 from django.db.models.functions import Cast, NthValue
 from django.db.models.functions.math import ATan2, Log, Ln, Mod, Round
-from django.db.models.expressions import Case, Exists, OrderBy, When, Window
+from django.db.models.expressions import Case, Exists, OrderBy, When, Window, Expression
 from django.db.models.lookups import Lookup, In
-from django.db.models import lookups
+from django.db.models import lookups, CheckConstraint
 from django.db.models.fields import BinaryField, Field
+from django.db.models.sql.query import Query
+from django.db.models.query import QuerySet
 from django.core import validators
 
 if VERSION >= (3, 1):
@@ -129,7 +132,8 @@ def split_parameter_list_as_sql(self, compiler, connection):
     with connection.cursor() as cursor:
         cursor.execute("IF OBJECT_ID('tempdb.dbo.#Temp_params', 'U') IS NOT NULL DROP TABLE #Temp_params; ")
         parameter_data_type = self.lhs.field.db_type(connection)
-        cursor.execute(f"CREATE TABLE #Temp_params (params {parameter_data_type})")
+        Temp_table_collation = 'COLLATE DATABASE_DEFAULT' if 'char' in parameter_data_type else ''
+        cursor.execute(f"CREATE TABLE #Temp_params (params {parameter_data_type} {Temp_table_collation})")
         for offset in range(0, len(rhs_params), 1000):
             sqls_params = rhs_params[offset: offset + 1000]
             sqls_params = ", ".join("('{}')".format(item) for item in sqls_params)
@@ -198,6 +202,74 @@ def BinaryField_init(self, *args, **kwargs):
     else:
         self.max_length = 'max'
 
+def _get_check_sql(self, model, schema_editor):
+    if VERSION >= (3, 1):
+        query = Query(model=model, alias_cols=False)
+    else:
+        query = Query(model=model)
+    where = query.build_where(self.check)
+    compiler = query.get_compiler(connection=schema_editor.connection)
+    sql, params = where.as_sql(compiler, schema_editor.connection)
+    try:
+        for p in params: str(p).encode('ascii')
+    except UnicodeEncodeError:
+        sql = sql.replace('%s', 'N%s')
+
+    return sql % tuple(schema_editor.quote_value(p) for p in params)
+
+def bulk_update_with_default(self, objs, fields, batch_size=None, default=0):
+    """
+        Update the given fields in each of the given objects in the database.
+
+        When bulk_update all fields to null,
+        SQL Server require that at least one of the result expressions in a CASE specification must be an expression other than the NULL constant.
+        Patched with a default value 0. The user can also pass a custom default value for CASE statement.
+    """
+    if batch_size is not None and batch_size < 0:
+        raise ValueError('Batch size must be a positive integer.')
+    if not fields:
+        raise ValueError('Field names must be given to bulk_update().')
+    objs = tuple(objs)
+    if any(obj.pk is None for obj in objs):
+        raise ValueError('All bulk_update() objects must have a primary key set.')
+    fields = [self.model._meta.get_field(name) for name in fields]
+    if any(not f.concrete or f.many_to_many for f in fields):
+        raise ValueError('bulk_update() can only be used with concrete fields.')
+    if any(f.primary_key for f in fields):
+        raise ValueError('bulk_update() cannot be used with primary key fields.')
+    if not objs:
+        return
+    # PK is used twice in the resulting update query, once in the filter
+    # and once in the WHEN. Each field will also have one CAST.
+    max_batch_size = connections[self.db].ops.bulk_batch_size(['pk', 'pk'] + fields, objs)
+    batch_size = min(batch_size, max_batch_size) if batch_size else max_batch_size
+    requires_casting = connections[self.db].features.requires_casted_case_in_updates
+    batches = (objs[i:i + batch_size] for i in range(0, len(objs), batch_size))
+    updates = []
+    for batch_objs in batches:
+        update_kwargs = {}
+        for field in fields:
+            value_none_counter = 0
+            when_statements = []
+            for obj in batch_objs:
+                attr = getattr(obj, field.attname)
+                if not isinstance(attr, Expression):
+                    if attr is None:
+                        value_none_counter+=1
+                    attr = Value(attr, output_field=field)
+                when_statements.append(When(pk=obj.pk, then=attr))
+            if(value_none_counter == len(when_statements)):
+                case_statement = Case(*when_statements, output_field=field, default=Value(default))
+            else:
+                case_statement = Case(*when_statements, output_field=field)
+            if requires_casting:
+                case_statement = Cast(case_statement, output_field=field)
+            update_kwargs[field.attname] = case_statement
+        updates.append(([obj.pk for obj in batch_objs], update_kwargs))
+    with transaction.atomic(using=self.db, savepoint=False):
+        for pks, update_kwargs in updates:
+            self.filter(pk__in=pks).update(**update_kwargs)
+
 ATan2.as_microsoft = sqlserver_atan2
 In.split_parameter_list_as_sql = split_parameter_list_as_sql
 if VERSION >= (3, 1):
@@ -211,6 +283,7 @@ def BinaryField_init(self, *args, **kwargs):
 Round.as_microsoft = sqlserver_round
 Window.as_microsoft = sqlserver_window
 BinaryField.__init__ = BinaryField_init
+CheckConstraint._get_check_sql = _get_check_sql
 
 if VERSION >= (3, 2):
     Random.as_microsoft = sqlserver_random
@@ -221,4 +294,4 @@ def BinaryField_init(self, *args, **kwargs):
     Exists.as_microsoft = sqlserver_exists
 
 OrderBy.as_microsoft = sqlserver_orderby
-
+QuerySet.bulk_update = bulk_update_with_default