Prepare for 1.4.1 release

Prepare for 1.4.1 release

README.md

@@ -1,4 +1,4 @@
-# SQL Server backend for Django
+# Microsoft Django backend for SQL Server
 
 Welcome to the MSSQL-Django 3rd party backend project!
 

azure-pipelines.yml

@@ -82,9 +82,6 @@ jobs:
         Python 3.8 - Django 3.2:
           python.version: '3.8'
           tox.env: 'py38-django32'
-        Python 3.7 - Django 3.2:
-          python.version: '3.7'
-          tox.env: 'py37-django32'
 
 
     steps:
@@ -199,9 +196,6 @@ jobs:
         Python 3.8 - Django 3.2:
           python.version: '3.8'
           tox.env: 'py38-django32'
-        Python 3.7 - Django 3.2:
-          python.version: '3.7'
-          tox.env: 'py37-django32'
 
     steps:
       - task: UsePythonVersion@0

mssql/base.py

@@ -9,6 +9,8 @@
 import time
 import struct
 import datetime
+from decimal import Decimal
+from uuid import UUID
 
 from django.core.exceptions import ImproperlyConfigured
 from django.utils.functional import cached_property
@@ -124,7 +126,7 @@ class DatabaseWrapper(BaseDatabaseWrapper):
         'SmallIntegerField': 'smallint',
         'TextField': 'nvarchar(max)',
         'TimeField': 'time',
-        'UUIDField': 'char(32)',
+        'UUIDField': 'uniqueidentifier',
     }
     data_types_suffix = {
         'AutoField': 'IDENTITY (1, 1)',
@@ -376,7 +378,6 @@ def get_new_connection(self, conn_params):
                         break
                 if not need_to_retry:
                     raise
-
         # Handling values from DATETIMEOFFSET columns
         # source: https://github.com/mkleehammer/pyodbc/wiki/Using-an-Output-Converter-function
         conn.add_output_converter(SQL_TIMESTAMP_WITH_TIMEZONE, handle_datetimeoffset)
@@ -431,6 +432,9 @@ def init_connection_state(self):
         if (options.get('return_rows_bulk_insert', False)):
             self.features_class.can_return_rows_from_bulk_insert = True
 
+        if (options.get('has_native_uuid_field', True)):
+            Database.native_uuid = True
+
         val = self.get_system_datetime
         if isinstance(val, str):
             raise ImproperlyConfigured(
@@ -569,6 +573,36 @@ def __init__(self, cursor, connection):
         self.last_sql = ''
         self.last_params = ()
 
+    def _as_sql_type(self, typ, value):
+        if isinstance(value, str):
+            length = len(value)
+            if length == 0:
+                return 'NVARCHAR'
+            elif length > 4000:
+                return 'NVARCHAR(max)'
+            return 'NVARCHAR(%s)' % len(value)
+        elif typ == int:
+            if value < 0x7FFFFFFF and value > -0x7FFFFFFF:
+                return 'INT'
+            else:
+                return 'BIGINT'
+        elif typ == float:
+            return 'DOUBLE PRECISION'
+        elif typ == bool:
+            return 'BIT'
+        elif isinstance(value, Decimal):
+            return 'NUMERIC'
+        elif isinstance(value, datetime.datetime):
+            return 'DATETIME2'
+        elif isinstance(value, datetime.date):
+            return 'DATE'
+        elif isinstance(value, datetime.time):
+            return 'TIME'
+        elif isinstance(value, UUID):
+            return 'uniqueidentifier'
+        else:
+            raise NotImplementedError('Not supported type %s (%s)' % (type(value), repr(value)))
+
     def close(self):
         if self.active:
             self.active = False
@@ -586,6 +620,27 @@ def format_sql(self, sql, params):
 
         return sql
 
+    def format_group_by_params(self, query, params):
+        if params:
+            # Insert None params directly into the query
+            if None in params:
+                null_params = ['NULL' if param is None else '%s' for param in params]
+                query = query % tuple(null_params)
+                params = tuple(p for p in params if p is not None)
+            params = [(param, type(param)) for param in params]
+            params_dict = {param: '@var%d' % i for i, param in enumerate(set(params))}
+            args = [params_dict[param] for param in params]
+
+            variables = []
+            params = []
+            for key, value in params_dict.items():
+                datatype = self._as_sql_type(key[1], key[0])
+                variables.append("%s %s = %%s " % (value, datatype))
+                params.append(key[0])
+            query = ('DECLARE %s \n' % ','.join(variables)) + (query % tuple(args))
+
+        return query, params
+
     def format_params(self, params):
         fp = []
         if params is not None:
@@ -614,6 +669,8 @@ def format_params(self, params):
 
     def execute(self, sql, params=None):
         self.last_sql = sql
+        if 'GROUP BY' in sql:
+            sql, params = self.format_group_by_params(sql, params)
         sql = self.format_sql(sql, params)
         params = self.format_params(params)
         self.last_params = params

mssql/compiler.py

@@ -155,7 +155,7 @@ def _as_sql_window(self, compiler, connection, template=None):
     else:
         # MSSQL window functions require an OVER clause with ORDER BY
         window_sql.append('ORDER BY (SELECT NULL)')
-    
+
     if self.frame:
         frame_sql, frame_params = compiler.compile(self.frame)
         window_sql.append(frame_sql)
@@ -443,7 +443,45 @@ def compile(self, node, *args, **kwargs):
 
     def collapse_group_by(self, expressions, having):
         expressions = super().collapse_group_by(expressions, having)
-        return [e for e in expressions if not isinstance(e, Subquery)]
+        # SQL server does not allow subqueries or constant expressions in the group by
+        # For constants: Each GROUP BY expression must contain at least one column that is not an outer reference.
+        # For subqueries: Cannot use an aggregate or a subquery in an expression used for the group by list of a GROUP BY clause.
+        return self._filter_subquery_and_constant_expressions(expressions)
+
+    def _is_constant_expression(self, expression):
+        if isinstance(expression, Value):
+            return True
+        sub_exprs = expression.get_source_expressions()
+        if not sub_exprs:
+            return False
+        for each in sub_exprs:
+            if not self._is_constant_expression(each):
+                return False
+        return True
+
+
+
+    def _filter_subquery_and_constant_expressions(self, expressions):
+        ret = []
+        for expression in expressions:
+            if self._is_subquery(expression):
+                continue
+            if self._is_constant_expression(expression):
+                continue
+            if not self._has_nested_subquery(expression):
+                ret.append(expression)
+        return ret
+
+    def _has_nested_subquery(self, expression):
+        if self._is_subquery(expression):
+            return True
+        for sub_expr in expression.get_source_expressions():
+            if self._has_nested_subquery(sub_expr):
+                return True
+        return False
+
+    def _is_subquery(self, expression):
+        return isinstance(expression, Subquery)
 
     def _as_microsoft(self, node):
         as_microsoft = None

mssql/features.py

@@ -13,7 +13,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     can_introspect_small_integer_field = True
     can_return_columns_from_insert = True
     can_return_id_from_insert = True
-    can_return_rows_from_bulk_insert = True
+    can_return_rows_from_bulk_insert = False
     can_rollback_ddl = True
     can_use_chunked_reads = False
     for_update_after_from = True
@@ -22,7 +22,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     has_json_object_function = False
     has_json_operators = False
     has_native_json_field = False
-    has_native_uuid_field = False
+    has_native_uuid_field = True
     has_real_datatype = True
     has_select_for_update = True
     has_select_for_update_nowait = True
@@ -33,6 +33,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     requires_literal_defaults = True
     requires_sqlparse_for_splitting = False
     supports_boolean_expr_in_select_clause = False
+    supports_comparing_boolean_expr = False
     supports_comments = True
     supports_covering_indexes = True
     supports_deferrable_unique_constraints = False
@@ -60,6 +61,9 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     supports_default_keyword_in_insert = True
     supports_expression_defaults = True
     supports_default_keyword_in_bulk_insert = True
+    supports_stored_generated_columns = True
+    supports_virtual_generated_columns = True
+
 
     @cached_property
     def has_zoneinfo_database(self):

mssql/functions.py

@@ -7,7 +7,7 @@
 from django.core import validators
 from django.db import NotSupportedError, connections, transaction
 from django.db.models import BooleanField, CheckConstraint, Value
-from django.db.models.expressions import Case, Exists, Expression, OrderBy, When, Window
+from django.db.models.expressions import Case, Exists, OrderBy, When, Window
 from django.db.models.fields import BinaryField, Field
 from django.db.models.functions import Cast, NthValue, MD5, SHA1, SHA224, SHA256, SHA384, SHA512
 from django.db.models.functions.datetime import Now
@@ -196,8 +196,8 @@ def mssql_split_parameter_list_as_sql(self, compiler, connection):
         cursor.execute(f"CREATE TABLE #Temp_params (params {parameter_data_type} {Temp_table_collation})")
         for offset in range(0, len(rhs_params), 1000):
             sqls_params = rhs_params[offset: offset + 1000]
-            sqls_params = ", ".join("('{}')".format(item) for item in sqls_params)
-            cursor.execute("INSERT INTO #Temp_params VALUES %s" % sqls_params)
+            sql = "INSERT INTO [#Temp_params] ([params]) VALUES " + ', '.join(['(%s)'] * len(sqls_params))
+            cursor.execute(sql, sqls_params)
 
     in_clause = lhs + ' IN ' + '(SELECT params from #Temp_params)'
 
@@ -294,7 +294,7 @@ def _get_check_sql(self, model, schema_editor):
     return sql % tuple(schema_editor.quote_value(p) for p in params)
 
 
-def bulk_update_with_default(self, objs, fields, batch_size=None, default=0):
+def bulk_update_with_default(self, objs, fields, batch_size=None, default=None):
     """
         Update the given fields in each of the given objects in the database.
 
@@ -343,7 +343,8 @@ def bulk_update_with_default(self, objs, fields, batch_size=None, default=0):
                     attr = Value(attr, output_field=field)
                 when_statements.append(When(pk=obj.pk, then=attr))
             if connection.vendor == 'microsoft' and value_none_counter == len(when_statements):
-                case_statement = Case(*when_statements, output_field=field, default=Value(default))
+                # We don't need a case statement if we are setting everything to None
+                case_statement = Value(None)
             else:
                 case_statement = Case(*when_statements, output_field=field)
             if requires_casting:

mssql/operations.py

@@ -129,7 +129,7 @@ def convert_floatfield_value(self, value, expression, connection):
 
     def convert_uuidfield_value(self, value, expression, connection):
         if value is not None:
-            value = uuid.UUID(value)
+            value = uuid.UUID(str(value))
         return value
 
     def convert_booleanfield_value(self, value, expression, connection):

mssql/schema.py

@@ -384,6 +384,14 @@ def _delete_deferred_unique_indexes_for_field(self, field):
     def _add_deferred_unique_index_for_field(self, field, statement):
         self._deferred_unique_indexes[str(field)].append(statement)
 
+    def _column_generated_sql(self, field):
+        """Return the SQL to use in a GENERATED ALWAYS clause."""
+        expression_sql, params = field.generated_sql(self.connection)
+        persistency_sql = "PERSISTED" if field.db_persist else ""
+        if params:
+            expression_sql = expression_sql % tuple(self.quote_value(p) for p in params)
+        return f"AS {expression_sql} {persistency_sql}"
+
     def _alter_field(self, model, old_field, new_field, old_type, new_type,
                      old_db_params, new_db_params, strict=False):
         """Actually perform a "physical" (non-ManyToMany) field update."""
@@ -1016,6 +1024,9 @@ def add_field(self, model, field):
         # It might not actually have a column behind it
         if definition is None:
             return
+        # Remove column type from definition if field is generated
+        if (django_version >= (5,0) and field.generated):
+            definition = definition[definition.find('AS'):]
         # Nullable columns with default values require 'WITH VALUES' to set existing rows
         if 'DEFAULT' in definition and field.null:
             definition = definition.replace('NULL', 'WITH VALUES')
@@ -1218,6 +1229,9 @@ def create_model(self, model):
             definition, extra_params = self.column_sql(model, field)
             if definition is None:
                 continue
+            # Remove column type from definition if field is generated
+            if (django_version >= (5,0) and field.generated):
+                definition = definition[definition.find('AS'):]
 
             if (self.connection.features.supports_nullable_unique_constraints and
                     not field.many_to_many and field.null and field.unique):

setup.py

@@ -11,12 +11,11 @@
     "Operating System :: Microsoft :: Windows",
     'Programming Language :: Python',
     'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.6',
-    'Programming Language :: Python :: 3.7',
     'Programming Language :: Python :: 3.8',
     'Programming Language :: Python :: 3.9',
     'Programming Language :: Python :: 3.10',
     'Programming Language :: Python :: 3.11',
+    'Programming Language :: Python :: 3.12',
     'Framework :: Django :: 3.2',
     'Framework :: Django :: 4.0',
     'Framework :: Django :: 4.1',
@@ -30,7 +29,7 @@
 
 setup(
     name='mssql-django',
-    version='1.4',
+    version='1.4.1',
     description='Django backend for Microsoft SQL Server',
     long_description=long_description,
     long_description_content_type='text/markdown',

testapp/migrations/0025_modelwithnullablefieldsofdifferenttypes.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.0.1 on 2024-01-29 14:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('testapp', '0024_publisher_book'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ModelWithNullableFieldsOfDifferentTypes',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('int_value', models.IntegerField(null=True)),
+                ('name', models.CharField(max_length=100, null=True)),
+                ('date', models.DateTimeField(null=True)),
+            ],
+        ),
+    ]

testapp/models.py

@@ -54,6 +54,12 @@ class UUIDModel(models.Model):
     def __str__(self):
         return self.pk
 
+class ModelWithNullableFieldsOfDifferentTypes(models.Model):
+    # Issue https://github.com/microsoft/mssql-django/issues/340
+    # Ensures the integrity of bulk updates with different types 
+    int_value = models.IntegerField(null=True)
+    name = models.CharField(max_length=100, null=True)
+    date = models.DateTimeField(null=True)
 
 class TestUniqueNullableModel(models.Model):
     # Issue https://github.com/ESSolutions/django-mssql-backend/issues/38: