[release/11.0] Fix JSON indexes for SQLite

src/EFCore.Design/Migrations/Design/CSharpMigrationOperationGenerator.cs

@@ -2144,6 +2144,11 @@ protected virtual void Annotations(
     {
         foreach (var annotation in annotations)
         {
+            if (annotation.Name == RelationalAnnotationNames.JsonIndex)
+            {
+                continue;
+            }
+
             // TODO: Give providers an opportunity to render these as provider-specific extension methods
             // Issue #6546
             builder

src/EFCore.Relational/EFCore.Relational.baseline.json

@@ -13230,7 +13230,7 @@
       "Type": "sealed class Microsoft.EntityFrameworkCore.Metadata.RelationalJsonIndex : System.IEquatable<Microsoft.EntityFrameworkCore.Metadata.RelationalJsonIndex>",
       "Methods": [
         {
-          "Member": "RelationalJsonIndex(System.Collections.Generic.IReadOnlyList<Microsoft.EntityFrameworkCore.Metadata.IRelationalJsonElement> elements, System.Collections.Generic.IReadOnlyList<System.Collections.Generic.IReadOnlyList<int?>?>? collectionIndices);"
+          "Member": "RelationalJsonIndex(System.Collections.Generic.IReadOnlyList<Microsoft.EntityFrameworkCore.Metadata.IRelationalJsonElement> elements, System.Collections.Generic.IReadOnlyList<System.Collections.Generic.IReadOnlyList<int?>?>? collectionIndices, System.Collections.Generic.IReadOnlyList<bool>? isDescending = null);"
         },
         {
           "Member": "bool Equals(Microsoft.EntityFrameworkCore.Metadata.RelationalJsonIndex? other);"
@@ -13240,6 +13240,9 @@
         },
         {
           "Member": "override int GetHashCode();"
+        },
+        {
+          "Member": "bool IsElementDescending(int index);"
         }
       ],
       "Properties": [
@@ -13248,6 +13251,9 @@
         },
         {
           "Member": "System.Collections.Generic.IReadOnlyList<Microsoft.EntityFrameworkCore.Metadata.IRelationalJsonElement> Elements { get; }"
+        },
+        {
+          "Member": "System.Collections.Generic.IReadOnlyList<bool>? IsDescending { get; }"
         }
       ]
     },
@@ -16720,6 +16726,9 @@
         {
           "Member": "static string JsonPathIndexElementsCollectionIndicesMismatch(object? elementCount, object? collectionIndicesCount);"
         },
+        {
+          "Member": "static string JsonPathIndexElementsIsDescendingMismatch(object? elementCount, object? isDescendingCount);"
+        },
         {
           "Member": "static string JsonPathIndexPropertiesInDifferentJsonColumns(object? indexProperties, object? entityType, object? firstColumn, object? secondColumn);"
         },

src/EFCore.Relational/Metadata/Internal/TableIndex.cs

@@ -72,8 +72,22 @@ public override bool IsReadOnly
     public virtual bool IsUnique { get; }
 
     /// <inheritdoc />
+    /// <remarks>
+    ///     For a JSON-mapped index this is always <see langword="null" />: since no database supports
+    ///     creating an index on a plain column and its contained JSON members in a single operation, and
+    ///     multiple JSON members may be deduplicated into a single JSON container column (see
+    ///     <see cref="Columns" />), per-element sort order cannot be represented at the column level.
+    ///     It is instead carried per element on <see cref="RelationalAnnotationNames.JsonIndex" />
+    ///     (<see cref="RelationalJsonIndex.IsDescending" />).
+    /// </remarks>
     public virtual IReadOnlyList<bool>? IsDescending
-        => MappedIndexes.First().IsDescending;
+    {
+        get
+        {
+            var mappedIndex = MappedIndexes.First();
+            return mappedIndex.IsJsonIndex() ? null : mappedIndex.IsDescending;
+        }
+    }
 
     /// <inheritdoc />
     public virtual string? Filter

src/EFCore.Relational/Metadata/RelationalAnnotationProvider.cs

@@ -133,7 +133,7 @@ protected virtual RelationalJsonIndex CreateJsonIndex(IIndex modelIndex, ITableI
             elements[i] = FindJsonElement(modelIndex.Properties[i], tableIndex.Table);
         }
 
-        return new RelationalJsonIndex(elements, modelIndex.CollectionIndices);
+        return new RelationalJsonIndex(elements, modelIndex.CollectionIndices, modelIndex.IsDescending);
     }
 
     /// <summary>

src/EFCore.Relational/Metadata/RelationalJsonIndex.cs

@@ -21,6 +21,14 @@ namespace Microsoft.EntityFrameworkCore.Metadata;
 ///         specific element. A <see langword="null" /> top-level entry indicates the property is
 ///         not reached through any complex collection.
 ///     </para>
+///     <para>
+///         The <see cref="IsDescending" /> list, when present, runs parallel to <see cref="Elements" />
+///         and indicates the sort order requested for each indexed JSON element. Sort order for a JSON
+///         index can only be represented per element (since all elements may share the same JSON
+///         container column, which is exposed as a single, deduplicated operation column), so it is
+///         carried here rather than on the containing <see cref="CreateIndexOperation" />/<see cref="ITableIndex" />,
+///         whose column-level <see cref="ITableIndex.IsDescending" /> is <see langword="null" /> for JSON indexes.
+///     </para>
 /// </remarks>
 public sealed class RelationalJsonIndex : IEquatable<RelationalJsonIndex>
 {
@@ -32,9 +40,14 @@ public sealed class RelationalJsonIndex : IEquatable<RelationalJsonIndex>
     ///     The complex-collection indices traversed to reach each indexed property, parallel to
     ///     <paramref name="elements" />.
     /// </param>
+    /// <param name="isDescending">
+    ///     The sort order requested for each indexed JSON element, parallel to <paramref name="elements" />,
+    ///     or <see langword="null" /> if every element uses ascending order.
+    /// </param>
     public RelationalJsonIndex(
         IReadOnlyList<IRelationalJsonElement> elements,
-        IReadOnlyList<IReadOnlyList<int?>?>? collectionIndices)
+        IReadOnlyList<IReadOnlyList<int?>?>? collectionIndices,
+        IReadOnlyList<bool>? isDescending = null)
     {
         Check.NotNull(elements);
 
@@ -45,8 +58,16 @@ public RelationalJsonIndex(
                 nameof(collectionIndices));
         }
 
+        if (isDescending is not null && isDescending.Count != 0 && elements.Count != isDescending.Count)
+        {
+            throw new ArgumentException(
+                RelationalStrings.JsonPathIndexElementsIsDescendingMismatch(elements.Count, isDescending.Count),
+                nameof(isDescending));
+        }
+
         Elements = elements;
         CollectionIndices = collectionIndices;
+        IsDescending = isDescending;
     }
 
     /// <summary>
@@ -59,6 +80,22 @@ public RelationalJsonIndex(
     /// </summary>
     public IReadOnlyList<IReadOnlyList<int?>?>? CollectionIndices { get; }
 
+    /// <summary>
+    ///     Gets the sort order requested for each indexed JSON element, parallel to <see cref="Elements" />,
+    ///     or <see langword="null" /> if every element uses ascending order.
+    /// </summary>
+    public IReadOnlyList<bool>? IsDescending { get; }
+
+    /// <summary>
+    ///     Resolves whether the element at <paramref name="index" /> should be sorted in descending order,
+    ///     honoring the shorthand where an empty <see cref="IsDescending" /> list means every element is
+    ///     descending.
+    /// </summary>
+    /// <param name="index">The index of the element in <see cref="Elements" />.</param>
+    /// <returns><see langword="true" /> if the element should be sorted in descending order.</returns>
+    public bool IsElementDescending(int index)
+        => IsDescending is not null && (IsDescending.Count == 0 || IsDescending[index]);
+
     /// <inheritdoc />
     public bool Equals(RelationalJsonIndex? other)
     {
@@ -69,7 +106,8 @@ public bool Equals(RelationalJsonIndex? other)
 
         if (other is null
             || Elements.Count != other.Elements.Count
-            || (CollectionIndices is null) != (other.CollectionIndices is null))
+            || (CollectionIndices is null) != (other.CollectionIndices is null)
+            || (IsDescending is null) != (other.IsDescending is null))
         {
             return false;
         }
@@ -86,6 +124,11 @@ public bool Equals(RelationalJsonIndex? other)
             {
                 return false;
             }
+
+            if (IsDescending is not null && IsElementDescending(i) != other.IsElementDescending(i))
+            {
+                return false;
+            }
         }
 
         return true;
@@ -121,6 +164,8 @@ public override int GetHashCode()
                     hash.Add(entry.HasValue ? entry.Value : -1);
                 }
             }
+
+            hash.Add(IsDescending is null ? (bool?)null : IsElementDescending(i));
         }
 
         return hash.ToHashCode();

src/EFCore.Relational/Properties/RelationalStrings.resx

@@ -637,6 +637,9 @@
   <data name="JsonPathIndexElementsCollectionIndicesMismatch" xml:space="preserve">
     <value>The number of elements ({elementCount}) must match the number of collection-index entries ({collectionIndicesCount}) when creating a RelationalJsonIndex.</value>
   </data>
+  <data name="JsonPathIndexElementsIsDescendingMismatch" xml:space="preserve">
+    <value>The number of elements ({elementCount}) must match the number of sort-order entries ({isDescendingCount}) when creating a RelationalJsonIndex.</value>
+  </data>
   <data name="JsonPathIndexPropertiesInDifferentJsonColumns" xml:space="preserve">
     <value>The index {indexProperties} on the entity type '{entityType}' cannot be configured because its properties are mapped to different JSON columns ('{firstColumn}' and '{secondColumn}'). All leaves of a JSON-path index (an index whose properties traverse a complex collection) must be contained in a single JSON column.</value>
   </data>

src/EFCore.SqlServer/Migrations/SqlServerMigrationsSqlGenerator.cs

@@ -961,7 +961,11 @@ protected override void Generate(
             return;
         }
 
-        if (operation[RelationalAnnotationNames.JsonIndex] is RelationalJsonIndex jsonIndex)
+        var jsonIndex = operation[RelationalAnnotationNames.JsonIndex] as RelationalJsonIndex
+            ?? model?.GetRelationalModel()
+                .FindTable(operation.Table, operation.Schema)?
+                .Indexes.FirstOrDefault(i => i.Name == operation.Name)?[RelationalAnnotationNames.JsonIndex] as RelationalJsonIndex;
+        if (jsonIndex is not null)
         {
             GenerateJsonIndex(jsonIndex);
             return;

src/EFCore.Sqlite.Core/EFCore.Sqlite.Core.baseline.json

@@ -355,6 +355,9 @@
         },
         {
           "Member": "override void Generate(Microsoft.EntityFrameworkCore.Migrations.Operations.DropSequenceOperation operation, Microsoft.EntityFrameworkCore.Metadata.IModel? model, Microsoft.EntityFrameworkCore.Migrations.MigrationCommandListBuilder builder);"
+        },
+        {
+          "Member": "override void GenerateIndexColumnList(Microsoft.EntityFrameworkCore.Migrations.Operations.CreateIndexOperation operation, Microsoft.EntityFrameworkCore.Metadata.IModel? model, Microsoft.EntityFrameworkCore.Migrations.MigrationCommandListBuilder builder);"
         }
       ]
     },

src/EFCore.Sqlite.Core/Infrastructure/Internal/SqliteModelValidator.cs

@@ -35,6 +35,19 @@ protected override void ValidateEntityType(
         ValidateNoSchema(entityType, logger);
     }
 
+    /// <inheritdoc />
+    protected override void ValidateIndex(
+        IIndex index,
+        IDiagnosticsLogger<DbLoggerCategory.Model.Validation> logger)
+    {
+        base.ValidateIndex(index, logger);
+
+        if (index.CollectionIndices?.Any(indices => indices?.Any(i => i is null) == true) == true)
+        {
+            throw new InvalidOperationException(SqliteStrings.JsonIndexAllElementsNotSupported);
+        }
+    }
+
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
     ///     the same compatibility standards as public APIs. It may be changed or removed without notice in

src/EFCore.Sqlite.Core/Migrations/SqliteMigrationsSqlGenerator.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Globalization;
 using System.Text;
 using Microsoft.EntityFrameworkCore.Sqlite.Internal;
 using Microsoft.EntityFrameworkCore.Sqlite.Metadata.Internal;
@@ -589,6 +590,92 @@ protected override void Generate(
         }
     }
 
+    /// <summary>
+    ///     Returns a SQL fragment for the column list of an index from a <see cref="CreateIndexOperation" />.
+    /// </summary>
+    /// <param name="operation">The operation.</param>
+    /// <param name="model">The target model which may be <see langword="null" /> if the operations exist without a model.</param>
+    /// <param name="builder">The command builder to use to build the commands.</param>
+    protected override void GenerateIndexColumnList(
+        CreateIndexOperation operation,
+        IModel? model,
+        MigrationCommandListBuilder builder)
+    {
+        var jsonIndex = operation[RelationalAnnotationNames.JsonIndex] as RelationalJsonIndex
+            ?? model?.GetRelationalModel()
+                .FindTable(operation.Table, operation.Schema)?
+                .Indexes.FirstOrDefault(i => i.Name == operation.Name)?[RelationalAnnotationNames.JsonIndex] as RelationalJsonIndex;
+        if (jsonIndex is null)
+        {
+            base.GenerateIndexColumnList(operation, model, builder);
+            return;
+        }
+
+        var stringTypeMapping = Dependencies.TypeMappingSource.GetMapping(typeof(string));
+        for (var i = 0; i < jsonIndex.Elements.Count; i++)
+        {
+            if (i > 0)
+            {
+                builder.Append(", ");
+            }
+
+            var element = jsonIndex.Elements[i];
+            builder.Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(element.ContainingColumn.Name));
+
+            var isArray = element is IRelationalJsonArray;
+            if (element.Path.Count > 0 || isArray)
+            {
+                builder.Append(" ->> ");
+
+                if (!isArray
+                    && element.Path is [{ IsArray: false, PropertyName: { } propertyName }])
+                {
+                    builder.Append(
+                        stringTypeMapping.GenerateSqlLiteral(
+                            Dependencies.SqlGenerationHelper.DelimitJsonPathElement(propertyName)));
+                }
+                else if (element.Path.Count == 0
+                         && isArray)
+                {
+                    builder.Append(jsonIndex.CollectionIndices![i]![0]!.Value.ToString(CultureInfo.InvariantCulture));
+                }
+                else
+                {
+                    var path = new StringBuilder("$");
+                    var collectionIndex = 0;
+                    foreach (var segment in element.Path)
+                    {
+                        if (segment.IsArray)
+                        {
+                            path.Append('[')
+                                .Append(jsonIndex.CollectionIndices?[i]?[collectionIndex++]?.ToString() ?? "*")
+                                .Append(']');
+                        }
+                        else
+                        {
+                            path.Append('.')
+                                .Append(Dependencies.SqlGenerationHelper.DelimitJsonPathElement(segment.PropertyName!));
+                        }
+                    }
+
+                    if (isArray)
+                    {
+                        path.Append('[')
+                            .Append(jsonIndex.CollectionIndices![i]![collectionIndex]!.Value)
+                            .Append(']');
+                    }
+
+                    builder.Append(stringTypeMapping.GenerateSqlLiteral(path.ToString()));
+                }
+            }
+
+            if (jsonIndex.IsElementDescending(i))
+            {
+                builder.Append(" DESC");
+            }
+        }
+    }
+
     /// <summary>
     ///     Builds commands for the given <see cref="RenameIndexOperation" />
     ///     by making calls on the given <see cref="MigrationCommandListBuilder" />.

src/EFCore.Sqlite.Core/Properties/SqliteStrings.resx

@@ -138,6 +138,9 @@
   <data name="InvalidMigrationOperation" xml:space="preserve">
     <value>SQLite does not support this migration operation ('{operation}'). See https://go.microsoft.com/fwlink/?LinkId=723262 for more information and examples.</value>
   </data>
+  <data name="JsonIndexAllElementsNotSupported" xml:space="preserve">
+    <value>SQLite does not support indexes over all elements of a JSON array. Specify a collection index in the index property path.</value>
+  </data>
   <data name="LogCompositeKeyWithValueGeneration" xml:space="preserve">
     <value>The entity type '{entityType}' has composite key '{key}' which is configured to use generated values. SQLite does not support generated values on composite keys.</value>
     <comment>Warning SqliteEventId.CompositeKeyWithValueGeneration string? string?</comment>