feat: Sec. indexes on relations (#2670)

## Relevant issue(s)

Resolves #2601 #2578 #2577

## Description

Enables fetching related objects via secondary indexes.

It also fixes a bug with queries that contain multiple aggregates on the
same collection.

client/normal_nil.go

@@ -30,7 +30,7 @@ func NewNormalNil(kind FieldKind) (NormalValue, error) {
 		return NewNormalNillableFloat(immutable.None[float64]()), nil
 	case FieldKind_NILLABLE_DATETIME:
 		return NewNormalNillableTime(immutable.None[time.Time]()), nil
-	case FieldKind_NILLABLE_STRING, FieldKind_NILLABLE_JSON:
+	case FieldKind_NILLABLE_STRING, FieldKind_NILLABLE_JSON, FieldKind_DocID:
 		return NewNormalNillableString(immutable.None[string]()), nil
 	case FieldKind_NILLABLE_BLOB:
 		return NewNormalNillableBytes(immutable.None[[]byte]()), nil

client/normal_value_test.go

@@ -1404,7 +1404,7 @@ func TestNormalValue_NewNormalNil(t *testing.T) {
 			assert.True(t, v.IsNil())
 		} else {
 			_, err := NewNormalNil(kind)
-			require.Error(t, err)
+			require.Error(t, err, "field kind: "+kind.String())
 		}
 	}
 }

client/schema_field_description.go

@@ -104,7 +104,7 @@ func (k ScalarKind) Underlying() string {
 }
 
 func (k ScalarKind) IsNillable() bool {
-	return k != FieldKind_DocID
+	return true
 }
 
 func (k ScalarKind) IsObject() bool {

docs/data_format_changes/i2670-sec-index-on-relations.md

@@ -0,0 +1,3 @@
+# Enable secondary index on relations
+
+This naturally caused some explain metrics to change and change detector complain about it.

internal/db/collection_index.go

@@ -21,6 +21,7 @@ import (
 	"github.com/sourcenetwork/immutable"
 
 	"github.com/sourcenetwork/defradb/client"
+	"github.com/sourcenetwork/defradb/client/request"
 	"github.com/sourcenetwork/defradb/datastore"
 	"github.com/sourcenetwork/defradb/internal/core"
 	"github.com/sourcenetwork/defradb/internal/db/base"
@@ -264,7 +265,7 @@ func (c *collection) createIndex(
 		return nil, err
 	}
 
-	err = c.checkExistingFields(desc.Fields)
+	err = c.checkExistingFieldsAndAdjustRelFieldNames(desc.Fields)
 	if err != nil {
 		return nil, err
 	}
@@ -493,20 +494,19 @@ func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription,
 	return c.Description().Indexes, nil
 }
 
-func (c *collection) checkExistingFields(
+// checkExistingFieldsAndAdjustRelFieldNames checks if the fields in the index description
+// exist in the collection schema.
+// If a field is a relation, it will be adjusted to relation id field name, a.k.a. `field_name + _id`.
+func (c *collection) checkExistingFieldsAndAdjustRelFieldNames(
 	fields []client.IndexedFieldDescription,
 ) error {
-	collectionFields := c.Schema().Fields
-	for _, field := range fields {
-		found := false
-		for _, colField := range collectionFields {
-			if field.Name == colField.Name {
-				found = true
-				break
-			}
-		}
+	for i := range fields {
+		field, found := c.Schema().GetFieldByName(fields[i].Name)
 		if !found {
-			return NewErrNonExistingFieldForIndex(field.Name)
+			return NewErrNonExistingFieldForIndex(fields[i].Name)
+		}
+		if field.Kind.IsObject() {
+			fields[i].Name = fields[i].Name + request.RelatedObjectID
 		}
 	}
 	return nil

internal/db/index.go

@@ -41,7 +41,7 @@ func getValidateIndexFieldFunc(kind client.FieldKind) func(any) bool {
 	}
 
 	switch kind {
-	case client.FieldKind_NILLABLE_STRING:
+	case client.FieldKind_NILLABLE_STRING, client.FieldKind_DocID:
 		return canConvertIndexFieldValue[string]
 	case client.FieldKind_NILLABLE_INT:
 		return canConvertIndexFieldValue[int64]

internal/planner/explain.go

@@ -92,17 +92,17 @@ func buildDebugExplainGraph(source planNode) (map[string]any, error) {
 		var explainGraphBuilder = map[string]any{}
 
 		// If root is not the last child then keep walking and explaining the root graph.
-		if node.root != nil {
-			indexJoinRootExplainGraph, err := buildDebugExplainGraph(node.root)
+		if node.parentSide.plan != nil {
+			indexJoinRootExplainGraph, err := buildDebugExplainGraph(node.parentSide.plan)
 			if err != nil {
 				return nil, err
 			}
 			// Add the explaination of the rest of the explain graph under the "root" graph.
 			explainGraphBuilder[joinRootLabel] = indexJoinRootExplainGraph
 		}
 
-		if node.subType != nil {
-			indexJoinSubTypeExplainGraph, err := buildDebugExplainGraph(node.subType)
+		if node.childSide.plan != nil {
+			indexJoinSubTypeExplainGraph, err := buildDebugExplainGraph(node.childSide.plan)
 			if err != nil {
 				return nil, err
 			}
@@ -117,8 +117,8 @@ func buildDebugExplainGraph(source planNode) (map[string]any, error) {
 		var explainGraphBuilder = map[string]any{}
 
 		// If root is not the last child then keep walking and explaining the root graph.
-		if node.root != nil {
-			indexJoinRootExplainGraph, err := buildDebugExplainGraph(node.root)
+		if node.parentSide.plan != nil {
+			indexJoinRootExplainGraph, err := buildDebugExplainGraph(node.parentSide.plan)
 			if err != nil {
 				return nil, err
 			}
@@ -128,8 +128,8 @@ func buildDebugExplainGraph(source planNode) (map[string]any, error) {
 			explainGraphBuilder[joinRootLabel] = nil
 		}
 
-		if node.subType != nil {
-			indexJoinSubTypeExplainGraph, err := buildDebugExplainGraph(node.subType)
+		if node.childSide.plan != nil {
+			indexJoinSubTypeExplainGraph, err := buildDebugExplainGraph(node.childSide.plan)
 			if err != nil {
 				return nil, err
 			}

internal/planner/mapper/mapper.go

@@ -413,25 +413,23 @@ func resolveAggregates(
 				childMapping = childMapping.CloneWithoutRender()
 				mapping.SetChildAt(index, childMapping)
 
-				if !childIsMapped {
-					filterDependencies, err := resolveFilterDependencies(
-						ctx,
-						store,
-						rootSelectType,
-						childCollectionName,
-						target.filter,
-						mapping.ChildMappings[index],
-						childFields,
-					)
-					if err != nil {
-						return nil, err
-					}
-					childFields = append(childFields, filterDependencies...)
-
-					// If the child was not mapped, the filter will not have been converted yet
-					// so we must do that now.
-					convertedFilter = ToFilter(target.filter.Value(), mapping.ChildMappings[index])
+				filterDependencies, err := resolveFilterDependencies(
+					ctx,
+					store,
+					rootSelectType,
+					childCollectionName,
+					target.filter,
+					mapping.ChildMappings[index],
+					childFields,
+				)
+				if err != nil {
+					return nil, err
 				}
+				childFields = append(childFields, filterDependencies...)
+
+				// If the child was not mapped, the filter will not have been converted yet
+				// so we must do that now.
+				convertedFilter = ToFilter(target.filter.Value(), mapping.ChildMappings[index])
 
 				dummyJoin := &Select{
 					Targetable: Targetable{
@@ -989,6 +987,7 @@ func resolveInnerFilterDependencies(
 				return nil, err
 			}
 
+			childSelect.SkipResolve = true
 			newFields = append(newFields, childSelect)
 		}
 

internal/planner/mapper/select.go

@@ -38,6 +38,11 @@ type Select struct {
 	// These can include stuff such as version information, aggregates, and other
 	// Selects.
 	Fields []Requestable
+
+	// SkipResolve is a flag that indicates that the fields in this Select don't need to be resolved,
+	// i.e. it's value doesn't need to be fetched and provided to the user.
+	// It is used to avoid resolving related objects if they are used only in a filter and not requested in a response.
+	SkipResolve bool
 }
 
 func (s *Select) AsTargetable() (*Targetable, bool) {

internal/planner/planner.go

@@ -357,18 +357,20 @@ func (p *Planner) tryOptimizeJoinDirection(node *invertibleTypeJoin, parentPlan
 		parentPlan.selectNode.filter.Conditions,
 		node.documentMapping,
 	)
-	slct := node.subType.(*selectTopNode).selectNode
+	slct := node.childSide.plan.(*selectTopNode).selectNode
 	desc := slct.collection.Description()
 	for subFieldName, subFieldInd := range filteredSubFields {
 		indexes := desc.GetIndexesOnField(subFieldName)
 		if len(indexes) > 0 && !filter.IsComplex(parentPlan.selectNode.filter) {
-			subInd := node.documentMapping.FirstIndexOfName(node.subTypeName)
-			relatedField := mapper.Field{Name: node.subTypeName, Index: subInd}
+			subInd := node.documentMapping.FirstIndexOfName(node.parentSide.relFieldDef.Name)
+			relatedField := mapper.Field{Name: node.parentSide.relFieldDef.Name, Index: subInd}
 			fieldFilter := filter.UnwrapRelation(filter.CopyField(
 				parentPlan.selectNode.filter,
 				relatedField,
 				mapper.Field{Name: subFieldName, Index: subFieldInd},
 			), relatedField)
+			// At the moment we just take the first index, but later we want to run some kind of analysis to
+			// determine which index is best to use. https://github.com/sourcenetwork/defradb/issues/2680
 			err := node.invertJoinDirectionWithIndex(fieldFilter, indexes[0])
 			if err != nil {
 				return err
@@ -383,15 +385,15 @@ func (p *Planner) tryOptimizeJoinDirection(node *invertibleTypeJoin, parentPlan
 // expandTypeJoin does a plan graph expansion and other optimizations on invertibleTypeJoin.
 func (p *Planner) expandTypeJoin(node *invertibleTypeJoin, parentPlan *selectTopNode) error {
 	if parentPlan.selectNode.filter == nil {
-		return p.expandPlan(node.subType, parentPlan)
+		return p.expandPlan(node.childSide.plan, parentPlan)
 	}
 
 	err := p.tryOptimizeJoinDirection(node, parentPlan)
 	if err != nil {
 		return err
 	}
 
-	return p.expandPlan(node.subType, parentPlan)
+	return p.expandPlan(node.childSide.plan, parentPlan)
 }
 
 func (p *Planner) expandGroupNodePlan(topNodeSelect *selectTopNode) error {

internal/planner/select.go

@@ -315,6 +315,21 @@ func findIndexByFilteringField(scanNode *scanNode) immutable.Option[client.Index
 	return immutable.None[client.IndexDescription]()
 }
 
+func findIndexByFieldName(col client.Collection, fieldName string) immutable.Option[client.IndexDescription] {
+	for _, field := range col.Schema().Fields {
+		if field.Name != fieldName {
+			continue
+		}
+		indexes := col.Description().GetIndexesOnField(field.Name)
+		if len(indexes) > 0 {
+			// At the moment we just take the first index, but later we want to run some kind of analysis to
+			// determine which index is best to use. https://github.com/sourcenetwork/defradb/issues/2680
+			return immutable.Some(indexes[0])
+		}
+	}
+	return immutable.None[client.IndexDescription]()
+}
+
 func (n *selectNode) initFields(selectReq *mapper.Select) ([]aggregateNode, error) {
 	aggregates := []aggregateNode{}
 	// loop over the sub type