diff --git a/.github/workflows/test_all_versions.yml b/.github/workflows/test_all_versions.yml index d547abb..7afe3b6 100644 --- a/.github/workflows/test_all_versions.yml +++ b/.github/workflows/test_all_versions.yml @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@v4 with: repository: singlestore-labs/django - ref: singlestore-test-4.2.x + ref: singlestore-test-5.0.x path: testrepo - name: Generate matrix JSON id: set-matrix @@ -90,13 +90,13 @@ jobs: uses: actions/checkout@v4 with: repository: singlestore-labs/django - ref: singlestore-test-4.2.x + ref: singlestore-test-5.0.x path: testrepo - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" cache: pip - name: Install test suite and package diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 81409b7..e207003 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v4 with: repository: singlestore-labs/django - ref: singlestore-test-4.2.x + ref: singlestore-test-5.0.x path: testrepo - name: Generate matrix JSON @@ -69,13 +69,13 @@ jobs: uses: actions/checkout@v4 with: repository: singlestore-labs/django - ref: singlestore-test-4.2.x + ref: singlestore-test-5.0.x path: testrepo - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" cache: pip - name: Install test suite and package diff --git a/README.md b/README.md index 15bc57e..0be335c 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,7 @@ def handle_m2m_field(self, obj, field): - `dumpdata` django command does not work if a table or a referenced m2m table does not have `id` column, which is the case for m2m tables created as suggested above (see `queries_paragraph_page` table definition). - Fetching an instance of a custom through model using .objects.get() is not supported. - Case-insensitive filter requires casting the filtered column to a `TextField` with case-insensitive collation, e.g. `utf8mb4_general_ci`. +- Some complex expressions (e.g., subqueries, CASE statements, mathematical expressions with multiple columns) in `DEFAULT` clauses are not supported. SingleStore typically supports simple literal defaults (strings, numbers) and specific time functions like NOW() or CURRENT_TIMESTAMP() for datetime columns. There may be more limitations (and fixes) as the test suite that comes with django is still being processed. diff --git a/django_singlestore/features.py b/django_singlestore/features.py index 120420a..faf368e 100644 --- a/django_singlestore/features.py +++ b/django_singlestore/features.py @@ -334,6 +334,9 @@ class DatabaseFeatures(BaseDatabaseFeatures): # Does the backend support unlimited character columns? supports_unlimited_charfield = False + # Does the backend support database-side default values? + supports_db_default = True + supports_update_conflicts_with_target = False @cached_property @@ -381,6 +384,7 @@ def test_collations(self): "cs": f"{charset}_bin", "non_default": f"{charset}_esperanto_ci", "swedish_ci": f"{charset}_swedish_ci", + "virtual": f"{charset}_unicode_ci", } test_now_utc_template = "UTC_TIMESTAMP" @@ -456,6 +460,9 @@ def django_test_skips(self): "aggregation.tests.AggregateTestCase.test_values_annotation_with_expression", "aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_on_exists", "expressions_case.tests.CaseExpressionTests.test_annotate_with_in_clause", + "admin_filters.tests.ListFiltersTests.test_facets_no_filter", + "admin_filters.tests.ListFiltersTests.test_facets_filter", + "admin_filters.tests.ListFiltersTests.test_facets_always", }, "Feature 'Correlated subselect that can not be transformed and does not match on shard keys' \ is not supported by SingleStore Distributed": @@ -498,7 +505,8 @@ def django_test_skips(self): but certain django functionality requires id column to be present": { "queries.tests.ExcludeTests.test_exclude_subquery", - "queries.tests.ExcludeTests.test_ticket14511", + # "queries.tests.ExcludeTests.test_ticket14511", + "queries.tests.ExcludeTests.test_exclude_m2m_through", "fixtures.tests.CircularReferenceTests.test_circular_reference_natural_key", "fixtures.tests.CircularReferenceTests.test_circular_reference_natural_key", "fixtures.tests.FixtureLoadingTests.test_dumpdata_progressbar", @@ -524,6 +532,8 @@ def django_test_skips(self): "signals.tests.SignalTests.test_delete_signals_origin_model", "signals.tests.SignalTests.test_delete_signals_origin_queryset", "signals.tests.SignalTests.test_save_and_delete_signals_with_m2m", + "backends.base.test_creation.TestDeserializeDbFromString." + \ + "test_serialize_db_to_string_base_manager_with_prefetch_related", }, "LIMIT with UNION affects only the second part of the union": { @@ -599,7 +609,13 @@ def django_test_skips(self): "ALTER TABLE which modifies column * from NULL to NOT NULL is not supported on a columnstore table.": { "schema.tests.SchemaTests.test_alter_null_to_not_null_keeping_default", + "migrations.test_operations.OperationTests." + \ + "test_alter_field_change_blank_nullable_database_default_to_not_null", # noqa: E131 "schema.tests.SchemaTests.test_alter_null_with_default_value_deferred_constraints", + "migrations.test_operations.OperationTests." + \ + "test_alter_field_change_nullable_to_database_default_not_null", # noqa: E131 + "migrations.test_operations.OperationTests." + \ + "test_alter_field_change_nullable_to_decimal_database_default_not_null", # noqa: E131 }, "Feature 'CHANGE which changes the name of a REFERENCE table auto_increment column' is not supported \ by SingleStore": @@ -728,18 +744,15 @@ def django_test_skips(self): "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests." + \ "test_trunc_timezone_applied_before_truncation", # noqa: E131 "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_func_with_timezone", - "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests." + \ - "test_trunc_ambiguous_and_invalid_times", # noqa: E131 "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_none", "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests." + \ "test_extract_iso_year_func_boundaries", # noqa: E131 "db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_iso_year_func_boundaries", "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests." + \ "test_extract_func_with_timezone", # noqa: E131 - "datetimes.tests.DateTimesTests.test_datetimes_ambiguous_and_invalid_times", "datetimes.tests.DateTimesTests.test_21432", }, - "SingleStore doest not support the SHA224 hashing algorithm": + "SingleStore does not support the SHA224 hashing algorithm": { "db_functions.text.test_sha224.SHA224Tests.test_transform", "db_functions.text.test_sha224.SHA224Tests.test_basic", @@ -791,6 +804,15 @@ def django_test_expected_failures(self): # Captured queries were: 1. BEGIN 2. Actual query 3. COMMIT # Instead of 1 and 3 we can have 5 and 7 or other numbers which differ by 2 # Doesn't look like something is not working, maybe check later + "delete_regress.tests.DeleteTests.test_self_reference_with_through_m2m_at_second_level", + "force_insert_update.tests.ForceInsertInheritanceTests.test_force_insert_diamond_mti", + "force_insert_update.tests.ForceInsertInheritanceTests.test_force_insert_false", + "force_insert_update.tests.ForceInsertInheritanceTests.test_force_insert_false_with_existing_parent", + "force_insert_update.tests.ForceInsertInheritanceTests.test_force_insert_parent", + "force_insert_update.tests.ForceInsertInheritanceTests.test_force_insert_with_existing_grandparent", + "force_insert_update.tests.ForceInsertInheritanceTests.test_force_insert_with_grandparent", + "model_inheritance.tests.ModelInheritanceTests.test_create_diamond_mti_common_parent", + "model_inheritance.tests.ModelInheritanceTests.test_create_diamond_mti_default_pk", "order_with_respect_to.tests.OrderWithRespectToBaseTests.test_database_routing", "queries.test_bulk_update.BulkUpdateTests.test_database_routing", "queries.test_bulk_update.BulkUpdateNoteTests.test_simple", @@ -856,6 +878,9 @@ def django_test_expected_failures(self): # Auto increment fields must have BIGINT data type . default is BigAutoField "introspection.tests.IntrospectionTests.test_get_table_description_types", "introspection.tests.IntrospectionTests.test_smallautofield", + # db_default parameter does no support complex functions. + "field_defaults.tests.DefaultTests.test_case_when_db_default_no_returning", + "migrations.test_operations.OperationTests.test_add_field_database_default_function", } return fails diff --git a/django_singlestore/schema.py b/django_singlestore/schema.py index 6fa721e..7818c3a 100644 --- a/django_singlestore/schema.py +++ b/django_singlestore/schema.py @@ -3,6 +3,7 @@ from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.models import Manager from django.db.models import NOT_PROVIDED +from django.db.models.sql import Query class ModelStorageManager(Manager): @@ -67,6 +68,32 @@ def quote_value(self, value): def prepare_default(self, value): return self.quote_value(value) + def db_default_sql(self, field): + """ + Return the sql and params for the field's database default. + + Overridden from BaseDatabaseSchemaEditor because the parent method + uses "(%s)" for non-Value expressions which causes SQL syntax errors + in SingleStore. We use "%s" instead to generate compatible SQL. + """ + from django.db.models.expressions import Value + + db_default = field._db_default_expression + # Changed from parent: using "%s" instead of "(%s)" for SingleStore compatibility + sql = ( + self._column_default_sql(field) if isinstance(db_default, Value) else "%s" + ) + query = Query(model=field.model) + compiler = query.get_compiler(connection=self.connection) + default_sql, params = compiler.compile(db_default) + if self.connection.features.requires_literal_defaults: + # Some databases don't support parameterized defaults (Oracle, + # SQLite). If this is the case, the individual schema backend + # should implement prepare_default(). + default_sql %= tuple(self.prepare_default(p) for p in params) + params = [] + return sql % default_sql, params + def column_sql(self, model, field, include_default=False): """ Return the column definition for a field. The field must already have @@ -85,9 +112,16 @@ def column_sql(self, model, field, include_default=False): result_sql_parts.append(self._collate_sql(collation)) if self.connection.features.supports_comments_inline and field.db_comment: result_sql_parts.append(self._comment_sql(field.db_comment)) - # Include a default value, if requested. - include_default = include_default and not field.null - if include_default: + + # Handle db_default (database-level default) + if hasattr(field, 'db_default') and field.db_default is not NOT_PROVIDED: + # db_default takes precedence over regular default when creating columns + default_sql, default_params = self.db_default_sql(field) + result_sql_parts.append("DEFAULT " + default_sql) + params.extend(default_params) + + # Include a regular default value, if requested and no db_default is set. + elif include_default and not field.null: default_value = self.effective_default(field) if default_value is not None: column_default = "DEFAULT " + self._column_default_sql(field) @@ -221,8 +255,19 @@ def add_field(self, model, field): super().add_field(model, field) # Simulate the effect of a one-off default. + if hasattr(field, 'db_default') and field.db_default is not NOT_PROVIDED: + default_sql, default_params = self.db_default_sql(field) + self.execute( + "UPDATE %(table)s SET %(column)s = %(default_value)s" + % { + "table": self.quote_name(model._meta.db_table), + "column": self.quote_name(field.column), + "default_value": default_sql, + }, + default_params, + ) # field.default may be unhashable, so a set isn't used for "in" check. - if field.default not in (None, NOT_PROVIDED): + elif field.default not in (None, NOT_PROVIDED): effective_default = self.effective_default(field) self.execute( "UPDATE %(table)s SET %(column)s = %%s" @@ -271,3 +316,41 @@ def _alter_column_type_sql( return super()._alter_column_type_sql( model, old_field, new_field, new_type, old_collation, new_collation, ) + + def _alter_column_null_sql(self, model, old_field, new_field): + """ + Generate SQL to change a column's NULL constraint. + + Overridden from BaseDatabaseSchemaEditor because SingleStore's MODIFY + clause requires the complete column definition (including DEFAULT) + when changing NULL constraints on fields with db_default values. + """ + if new_field.db_default is NOT_PROVIDED: + return super()._alter_column_null_sql(model, old_field, new_field) + + # For fields with db_default, use MODIFY with complete column definition + new_db_params = new_field.db_parameters(connection=self.connection) + type_sql = self._set_field_new_type(new_field, new_db_params["type"]) + return ( + "MODIFY %(column)s %(type)s" + % { + "column": self.quote_name(new_field.column), + "type": type_sql, # Includes DEFAULT and NULL/NOT NULL + }, + [], + ) + + def _set_field_new_type(self, field, new_type): + """ + Keep the NULL and DEFAULT properties of the old field. If it has + changed, it will be handled separately. + """ + if field.db_default is not NOT_PROVIDED: + default_sql, params = self.db_default_sql(field) + default_sql %= tuple(self.quote_value(p) for p in params) + new_type += f" DEFAULT {default_sql}" + if field.null: + new_type += " NULL" + else: + new_type += " NOT NULL" + return new_type diff --git a/scripts/setup_sections/backends_setup.sql b/scripts/setup_sections/backends_setup.sql index 846bb04..7e92d13 100644 --- a/scripts/setup_sections/backends_setup.sql +++ b/scripts/setup_sections/backends_setup.sql @@ -16,3 +16,12 @@ CREATE TABLE `backends_object_object` ( KEY (`from_object_id`), KEY (`to_object_id`) ); + +CREATE TABLE `backends_schoolbus_schoolclass` ( + `schoolbus_id` BIGINT NOT NULL, + `schoolclass_id` BIGINT NOT NULL, + SHARD KEY (`schoolbus_id`), + UNIQUE KEY (`schoolbus_id`, `schoolclass_id`), + KEY (`schoolbus_id`), + KEY (`schoolclass_id`) +);