[BACK-2895] Cleanup unused indexes, add missing indexes

data/store/mongo/mongo_data.go

@@ -2,7 +2,6 @@ package mongo
 
 import (
 	"context"
-	"slices"
 	"strings"
 	"time"
 
@@ -199,66 +198,6 @@ func (d *DataRepository) mongoClient() *mongo.Client {
 	return d.DatumRepository.Database().Client()
 }
 
-// MergeSortedUploads combines the unique Uploads by UploadID into a new slice.
-func MergeSortedUploads(newUploads, prevUploads []*upload.Upload) []*upload.Upload {
-	combined := make([]*upload.Upload, 0, len(newUploads)+len(prevUploads))
-
-	// Merge the two datasets like the merge step in merge sort. Note we don't
-	// use sort.Slice/sort.SliceStable from the standard library as the
-	// sorting criteria may change (?) in the Repositories in the future.
-	newCounter := 0
-	// Prefer the uploads in prevUploads as that will maintain proper
-	// Pagination in the case that not all records are in the new collection
-	// yet because all existing uploads are already in the old collection but
-	// might not be in the new one.
-	for _, dataSet := range prevUploads {
-		for newCounter < len(newUploads) && *newUploads[newCounter].UploadID < *dataSet.UploadID {
-			combined = append(combined, newUploads[newCounter])
-			newCounter++
-		}
-		// Always add the dataSet/upload in the "old" deviceData collection
-		// because the dataSet/upload may not have been finished migrating
-		// into the new deviceDataSets collection
-		combined = append(combined, dataSet)
-		// Skip duplicate of newUploads in prevUploads if it exists.
-		if newCounter < len(newUploads) && *newUploads[newCounter].UploadID == *dataSet.UploadID {
-			newCounter++
-		}
-	}
-	combined = append(combined, newUploads[newCounter:]...)
-	return combined
-}
-
-// MergeSortedDataSets combines the unique Uploads by UploadID into a new slice.
-func MergeSortedDataSets(newDataSets, prevDataSets data.DataSets) data.DataSets {
-	combined := make(data.DataSets, 0, len(newDataSets)+len(prevDataSets))
-
-	// Merge the two datasets like the merge step in merge sort. Note we don't
-	// use sort.Slice/sort.SliceStable from the standard library as the
-	// sorting criteria may change (?) in the Repositories in the future.
-	newCounter := 0
-	// Prefer the dataSets in prevDataSets as that will maintain proper
-	// Pagination in the case that not all records are in the new collection
-	// yet because all existing DataSets are already in the old collection but
-	// might not be in the new one.
-	for _, dataSet := range prevDataSets {
-		for newCounter < len(newDataSets) && *newDataSets[newCounter].UploadID < *dataSet.UploadID {
-			combined = append(combined, newDataSets[newCounter])
-			newCounter++
-		}
-		// Always add the dataSet/upload in the "old" deviceData collection
-		// because the dataSet/upload may not have been finished migrating
-		// into the new deviceDataSets collection
-		combined = append(combined, dataSet)
-		// Skip duplicate of newDataSets in prevDataSets if it exists.
-		if newCounter < len(newDataSets) && *newDataSets[newCounter].UploadID == *dataSet.UploadID {
-			newCounter++
-		}
-	}
-	combined = append(combined, newDataSets[newCounter:]...)
-	return combined
-}
-
-func isTypeUpload(typ []string) bool {
-	return slices.Contains(typ, strings.ToLower(upload.Type))
+func isTypeUpload(typ string) bool {
+	return strings.ToLower(typ) == strings.ToLower(upload.Type)
 }

data/store/mongo/mongo_data_set.go

@@ -38,7 +38,6 @@ func (d *DataSetRepository) EnsureIndexes() error {
 				{Key: "time", Value: -1},
 			},
 			Options: options.Index().
-				SetBackground(true).
 				SetName("UserIdTypeWeighted_v2"),
 		},
 		{
@@ -49,7 +48,6 @@ func (d *DataSetRepository) EnsureIndexes() error {
 				{Key: "_active", Value: 1},
 			},
 			Options: options.Index().
-				SetBackground(true).
 				SetName("OriginId"),
 		},
 		{
@@ -68,25 +66,33 @@ func (d *DataSetRepository) EnsureIndexes() error {
 				{Key: "_active", Value: 1},
 			},
 			Options: options.Index().
-				SetBackground(true).
 				SetName("UploadId"),
 		},
+		{
+			Keys: bson.D{
+				{Key: "_userId", Value: 1},
+				{Key: "client.name", Value: 1},
+				{Key: "type", Value: 1},
+				{Key: "createdTime", Value: -1},
+			},
+			Options: options.Index().
+				SetName("ListUserDataSets").
+				SetPartialFilterExpression(bson.D{
+					{Key: "_active", Value: true},
+				}),
+		},
 		{
 			Keys: bson.D{
 				{Key: "_userId", Value: 1},
 				{Key: "deviceId", Value: 1},
 				{Key: "type", Value: 1},
-				{Key: "_active", Value: 1},
-				{Key: "_deduplicator.hash", Value: 1},
+				{Key: "createdTime", Value: -1},
 			},
 			Options: options.Index().
-				SetBackground(true).
+				SetName("ListUserDataSetsDeviceId").
 				SetPartialFilterExpression(bson.D{
 					{Key: "_active", Value: true},
-					{Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
-					{Key: "deviceId", Value: bson.D{{Key: "$exists", Value: true}}},
-				}).
-				SetName("DeduplicatorHash"),
+				}),
 		},
 	})
 }

data/store/mongo/mongo_datum.go

@@ -50,7 +50,6 @@ func (d *DatumRepository) EnsureIndexes() error {
 				{Key: "time", Value: -1},
 			},
 			Options: options.Index().
-				SetBackground(true).
 				SetName("UserIdTypeWeighted_v2"),
 		},
 		{
@@ -80,7 +79,6 @@ func (d *DatumRepository) EnsureIndexes() error {
 				{Key: "_active", Value: 1},
 			},
 			Options: options.Index().
-				SetBackground(true).
 				SetName("OriginId"),
 		},
 		{
@@ -91,7 +89,6 @@ func (d *DatumRepository) EnsureIndexes() error {
 				{Key: "_active", Value: 1},
 			},
 			Options: options.Index().
-				SetBackground(true).
 				SetName("UploadId"),
 		},
 		{
@@ -103,7 +100,6 @@ func (d *DatumRepository) EnsureIndexes() error {
 				{Key: "_deduplicator.hash", Value: 1},
 			},
 			Options: options.Index().
-				SetBackground(true).
 				SetPartialFilterExpression(bson.D{
 					{Key: "_active", Value: true},
 					{Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},