From 02a9ec7c8453ac9bd15a6668f18ad30dd0a6baa0 Mon Sep 17 00:00:00 2001 From: Daniel Adam Date: Fri, 17 May 2024 15:40:18 +0200 Subject: [PATCH] fixup! snapshot-service: persistent Condition storage --- certificate-authority/store/mongodb/store.go | 23 +- .../store/mongodb/subscription.go | 40 ++-- identity-store/persistence/mongodb/persist.go | 4 +- identity-store/persistence/mongodb/store.go | 15 +- pkg/mongodb/store.go | 14 +- snapshot-service/pb/README.md | 10 +- snapshot-service/pb/doc.html | 10 +- snapshot-service/pb/service.pb.go | 6 +- snapshot-service/pb/service.proto | 4 +- snapshot-service/pb/service.swagger.json | 16 +- snapshot-service/store/condition.go | 8 +- snapshot-service/store/mongodb/condition.go | 29 ++- .../store/mongodb/condition_test.go | 207 ++++++++++++++++-- snapshot-service/store/mongodb/store.go | 22 +- snapshot-service/store/store.go | 4 +- 15 files changed, 323 insertions(+), 89 deletions(-) diff --git a/certificate-authority/store/mongodb/store.go b/certificate-authority/store/mongodb/store.go index 90b0115cf..72f535d39 100644 --- a/certificate-authority/store/mongodb/store.go +++ b/certificate-authority/store/mongodb/store.go @@ -10,6 +10,7 @@ import ( pkgMongo "github.com/plgd-dev/hub/v2/pkg/mongodb" "github.com/plgd-dev/hub/v2/pkg/security/certManager/client" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" "go.opentelemetry.io/otel/trace" ) @@ -18,18 +19,18 @@ type Store struct { bulkWriter *bulkWriter } -var DeviceIDKeyQueryIndex = bson.D{ - {Key: store.DeviceIDKey, Value: 1}, - {Key: store.OwnerKey, Value: 1}, +var deviceIDKeyQueryIndex = mongo.IndexModel{ + Keys: bson.D{ + {Key: store.DeviceIDKey, Value: 1}, + {Key: store.OwnerKey, Value: 1}, + }, } -var CommonNameKeyQueryIndex = bson.D{ - {Key: store.CommonNameKey, Value: 1}, - {Key: store.OwnerKey, Value: 1}, -} - -var PublicKeyQueryIndex = bson.D{ - {Key: store.CommonNameKey, Value: 1}, +var commonNameKeyQueryIndex = mongo.IndexModel{ + Keys: bson.D{ + {Key: store.CommonNameKey, Value: 1}, + {Key: store.OwnerKey, Value: 1}, + }, } func New(ctx context.Context, cfg *Config, fileWatcher *fsnotify.Watcher, logger log.Logger, tracerProvider trace.TracerProvider) (*Store, error) { @@ -44,7 +45,7 @@ func New(ctx context.Context, cfg *Config, fileWatcher *fsnotify.Watcher, logger } bulkWriter := newBulkWriter(m.Collection(signingRecordsCol), cfg.BulkWrite.DocumentLimit, cfg.BulkWrite.ThrottleTime, cfg.BulkWrite.Timeout, logger) s := Store{Store: m, bulkWriter: bulkWriter} - err = s.EnsureIndex(ctx, signingRecordsCol, CommonNameKeyQueryIndex, DeviceIDKeyQueryIndex) + err = s.EnsureIndex(ctx, signingRecordsCol, commonNameKeyQueryIndex, deviceIDKeyQueryIndex) if err != nil { certManager.Close() return nil, err diff --git a/cloud2cloud-gateway/store/mongodb/subscription.go b/cloud2cloud-gateway/store/mongodb/subscription.go index 8649063ca..5cb324e30 100644 --- a/cloud2cloud-gateway/store/mongodb/subscription.go +++ b/cloud2cloud-gateway/store/mongodb/subscription.go @@ -21,24 +21,32 @@ const ( initializedKey = "initialized" ) -var typeQueryIndex = bson.D{ - {Key: typeKey, Value: 1}, +var typeQueryIndex = mongo.IndexModel{ + Keys: bson.D{ + {Key: typeKey, Value: 1}, + }, } -var typeDeviceIDQueryIndex = bson.D{ - {Key: typeKey, Value: 1}, - {Key: deviceIDKey, Value: 1}, +var typeDeviceIDQueryIndex = mongo.IndexModel{ + Keys: bson.D{ + {Key: typeKey, Value: 1}, + {Key: deviceIDKey, Value: 1}, + }, } -var typeResourceIDQueryIndex = bson.D{ - {Key: typeKey, Value: 1}, - {Key: deviceIDKey, Value: 1}, - {Key: hrefKey, Value: 1}, +var typeResourceIDQueryIndex = mongo.IndexModel{ + Keys: bson.D{ + {Key: typeKey, Value: 1}, + {Key: deviceIDKey, Value: 1}, + {Key: hrefKey, Value: 1}, + }, } -var typeInitializedIDQueryIndex = bson.D{ - {Key: "_id", Value: 1}, - {Key: initializedKey, Value: 1}, +var typeInitializedIDQueryIndex = mongo.IndexModel{ + Keys: bson.D{ + {Key: "_id", Value: 1}, + {Key: initializedKey, Value: 1}, + }, } type DBSub struct { @@ -160,7 +168,7 @@ func (s *Store) SetInitialized(ctx context.Context, subscriptionID string) error } opts := &options.UpdateOptions{} - opts.SetHint(typeInitializedIDQueryIndex) + opts.SetHint(typeInitializedIDQueryIndex.Keys) _, err := col.UpdateOne(ctx, bson.D{{Key: "_id", Value: subscriptionID}, {Key: initializedKey, Value: false}}, bson.M{"$set": bson.M{initializedKey: true}}, opts) if err != nil { return fmt.Errorf("cannot set initialized for %v: %w", subscriptionID, err) @@ -201,7 +209,7 @@ func (s *Store) LoadSubscriptions(ctx context.Context, query store.SubscriptionQ hrefKey: query.Href, } iter, err = col.Find(ctx, q, &options.FindOptions{ - Hint: typeResourceIDQueryIndex, + Hint: typeResourceIDQueryIndex.Keys, }) case query.DeviceID != "": q := bson.M{ @@ -209,14 +217,14 @@ func (s *Store) LoadSubscriptions(ctx context.Context, query store.SubscriptionQ deviceIDKey: query.DeviceID, } iter, err = col.Find(ctx, q, &options.FindOptions{ - Hint: typeDeviceIDQueryIndex, + Hint: typeDeviceIDQueryIndex.Keys, }) default: q := bson.M{ typeKey: query.Type, } iter, err = col.Find(ctx, q, &options.FindOptions{ - Hint: typeQueryIndex, + Hint: typeQueryIndex.Keys, }) } if errors.Is(err, mongo.ErrNilDocument) { diff --git a/identity-store/persistence/mongodb/persist.go b/identity-store/persistence/mongodb/persist.go index 893562f6c..cacfe0102 100644 --- a/identity-store/persistence/mongodb/persist.go +++ b/identity-store/persistence/mongodb/persist.go @@ -48,7 +48,7 @@ func (p *PersistenceTx) Retrieve(deviceID, userID string) (_ *persistence.Author col := p.tx.Client().Database(p.dbname).Collection(userDevicesCName) iter, err := col.Find(p.ctx, bson.M{deviceIDKey: deviceID, ownerKey: userID}, &options.FindOptions{ - Hint: userDeviceQueryIndex, + Hint: userDeviceQueryIndex.Keys, }) if errors.Is(err, mongo.ErrNilDocument) { @@ -113,7 +113,7 @@ func (p *PersistenceTx) RetrieveByOwner(owner string) persistence.Iterator { col := p.tx.Client().Database(p.dbname).Collection(userDevicesCName) iter, err := col.Find(p.ctx, bson.M{ownerKey: owner}, &options.FindOptions{ - Hint: userDevicesQueryIndex, + Hint: userDevicesQueryIndex.Keys, }) if errors.Is(err, mongo.ErrNilDocument) { diff --git a/identity-store/persistence/mongodb/store.go b/identity-store/persistence/mongodb/store.go index a84c48360..5948c71b1 100644 --- a/identity-store/persistence/mongodb/store.go +++ b/identity-store/persistence/mongodb/store.go @@ -9,18 +9,23 @@ import ( pkgMongo "github.com/plgd-dev/hub/v2/pkg/mongodb" "github.com/plgd-dev/hub/v2/pkg/security/certManager/client" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" "go.opentelemetry.io/otel/trace" ) const userDevicesCName = "userdevices" -var userDeviceQueryIndex = bson.D{ - {Key: ownerKey, Value: 1}, - {Key: deviceIDKey, Value: 1}, +var userDeviceQueryIndex = mongo.IndexModel{ + Keys: bson.D{ + {Key: ownerKey, Value: 1}, + {Key: deviceIDKey, Value: 1}, + }, } -var userDevicesQueryIndex = bson.D{ - {Key: ownerKey, Value: 1}, +var userDevicesQueryIndex = mongo.IndexModel{ + Keys: bson.D{ + {Key: ownerKey, Value: 1}, + }, } type Store struct { diff --git a/pkg/mongodb/store.go b/pkg/mongodb/store.go index c404b3aa3..f5756b5f3 100644 --- a/pkg/mongodb/store.go +++ b/pkg/mongodb/store.go @@ -8,7 +8,6 @@ import ( "github.com/hashicorp/go-multierror" "github.com/plgd-dev/hub/v2/pkg/fn" - "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/readpref" @@ -59,7 +58,7 @@ func NewStore(ctx context.Context, cfg *Config, tls *tls.Config, tracerProvider return s, nil } -func NewStoreWithCollection(ctx context.Context, cfg *Config, tls *tls.Config, tracerProvider trace.TracerProvider, collection string, indexes ...bson.D) (*Store, error) { +func NewStoreWithCollection(ctx context.Context, cfg *Config, tls *tls.Config, tracerProvider trace.TracerProvider, collection string, indexes ...mongo.IndexModel) (*Store, error) { s, err := NewStore(ctx, cfg, tls, tracerProvider) if err != nil { return nil, err @@ -76,7 +75,7 @@ func NewStoreWithCollection(ctx context.Context, cfg *Config, tls *tls.Config, t return s, nil } -func (s *Store) EnsureIndex(ctx context.Context, collection string, indexes ...bson.D) error { +func (s *Store) EnsureIndex(ctx context.Context, collection string, indexes ...mongo.IndexModel) error { if len(indexes) == 0 { return nil } @@ -87,13 +86,8 @@ func (s *Store) EnsureIndex(ctx context.Context, collection string, indexes ...b return nil } -func ensureIndex(ctx context.Context, col *mongo.Collection, indexes ...bson.D) error { - for _, keys := range indexes { - opts := &options.IndexOptions{} - index := mongo.IndexModel{ - Keys: keys, - Options: opts, - } +func ensureIndex(ctx context.Context, col *mongo.Collection, indexes ...mongo.IndexModel) error { + for _, index := range indexes { _, err := col.Indexes().CreateOne(ctx, index) if err != nil { if strings.HasPrefix(err.Error(), "(IndexKeySpecsConflict)") { diff --git a/snapshot-service/pb/README.md b/snapshot-service/pb/README.md index e4bbc3699..f4ffafbad 100644 --- a/snapshot-service/pb/README.md +++ b/snapshot-service/pb/README.md @@ -102,16 +102,20 @@ driven by resource change event | ----- | ---- | ----- | ----------- | | id | [string](#string) | | Unique condition ID -@gotags: bson:"_id" | +@gotags: bson:"id" | | version | [uint64](#uint64) | | @gotags: bson:"version" | | name | [string](#string) | | @gotags: bson:"name,omitempty" | | enabled | [bool](#bool) | | @gotags: bson:"enabled,omitempty" | | configuration_id | [string](#string) | | ID of the configuration to be applied when the condition is satisfied @gotags: bson:"configurationId" | -| device_id_filter | [string](#string) | repeated | @gotags: bson:"deviceIdFilter,omitempty" | +| device_id_filter | [string](#string) | repeated | list of device IDs to which the condition applies + +@gotags: bson:"deviceIdFilter,omitempty" | | resource_type_filter | [string](#string) | repeated | @gotags: bson:"resourceTypeFilter,omitempty" | -| resource_href_filter | [string](#string) | repeated | @gotags: bson:"resourceHrefFilter,omitempty" | +| resource_href_filter | [string](#string) | repeated | list of resource hrefs to which the condition applies + +@gotags: bson:"resourceHrefFilter,omitempty" | | jq_expression_filter | [string](#string) | | @gotags: bson:"jqExpressionFilter,omitempty" | | api_access_token | [string](#string) | | Token used to update resources in the configuration diff --git a/snapshot-service/pb/doc.html b/snapshot-service/pb/doc.html index 7f945e7a0..de23d39a5 100644 --- a/snapshot-service/pb/doc.html +++ b/snapshot-service/pb/doc.html @@ -430,7 +430,7 @@

Condition

Unique condition ID -@gotags: bson:"_id"

+@gotags: bson:"id"

@@ -467,7 +467,9 @@

Condition

device_id_filter string repeated -

@gotags: bson:"deviceIdFilter,omitempty"

+

list of device IDs to which the condition applies + +@gotags: bson:"deviceIdFilter,omitempty"

@@ -481,7 +483,9 @@

Condition

resource_href_filter string repeated -

@gotags: bson:"resourceHrefFilter,omitempty"

+

list of resource hrefs to which the condition applies + +@gotags: bson:"resourceHrefFilter,omitempty"

diff --git a/snapshot-service/pb/service.pb.go b/snapshot-service/pb/service.pb.go index ae140bec8..6a6a5eb8c 100644 --- a/snapshot-service/pb/service.pb.go +++ b/snapshot-service/pb/service.pb.go @@ -194,14 +194,16 @@ type Condition struct { unknownFields protoimpl.UnknownFields // Unique condition ID - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty" bson:"_id"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty" bson:"id"` Version uint64 `protobuf:"varint,2,opt,name=version,proto3" json:"version,omitempty" bson:"version"` Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty" bson:"name,omitempty"` Enabled bool `protobuf:"varint,4,opt,name=enabled,proto3" json:"enabled,omitempty" bson:"enabled,omitempty"` // ID of the configuration to be applied when the condition is satisfied - ConfigurationId string `protobuf:"bytes,5,opt,name=configuration_id,json=configurationId,proto3" json:"configuration_id,omitempty" bson:"configurationId"` + ConfigurationId string `protobuf:"bytes,5,opt,name=configuration_id,json=configurationId,proto3" json:"configuration_id,omitempty" bson:"configurationId"` + // list of device IDs to which the condition applies DeviceIdFilter []string `protobuf:"bytes,6,rep,name=device_id_filter,json=deviceIdFilter,proto3" json:"device_id_filter,omitempty" bson:"deviceIdFilter,omitempty"` ResourceTypeFilter []string `protobuf:"bytes,7,rep,name=resource_type_filter,json=resourceTypeFilter,proto3" json:"resource_type_filter,omitempty" bson:"resourceTypeFilter,omitempty"` + // list of resource hrefs to which the condition applies ResourceHrefFilter []string `protobuf:"bytes,8,rep,name=resource_href_filter,json=resourceHrefFilter,proto3" json:"resource_href_filter,omitempty" bson:"resourceHrefFilter,omitempty"` JqExpressionFilter string `protobuf:"bytes,9,opt,name=jq_expression_filter,json=jqExpressionFilter,proto3" json:"jq_expression_filter,omitempty" bson:"jqExpressionFilter,omitempty"` // Token used to update resources in the configuration diff --git a/snapshot-service/pb/service.proto b/snapshot-service/pb/service.proto index 6c8092048..84f97d064 100644 --- a/snapshot-service/pb/service.proto +++ b/snapshot-service/pb/service.proto @@ -50,14 +50,16 @@ message IDFilter { message Condition { // driven by resource change event // Unique condition ID - string id = 1; // @gotags: bson:"_id" + string id = 1; // @gotags: bson:"id" uint64 version = 2; // @gotags: bson:"version" string name = 3; // @gotags: bson:"name,omitempty" bool enabled = 4; // @gotags: bson:"enabled,omitempty" // ID of the configuration to be applied when the condition is satisfied string configuration_id = 5; // @gotags: bson:"configurationId" + // list of device IDs to which the condition applies repeated string device_id_filter = 6; // @gotags: bson:"deviceIdFilter,omitempty" repeated string resource_type_filter = 7; // @gotags: bson:"resourceTypeFilter,omitempty" + // list of resource hrefs to which the condition applies repeated string resource_href_filter = 8; // @gotags: bson:"resourceHrefFilter,omitempty" string jq_expression_filter = 9; // @gotags: bson:"jqExpressionFilter,omitempty" // Token used to update resources in the configuration diff --git a/snapshot-service/pb/service.swagger.json b/snapshot-service/pb/service.swagger.json index bfae974db..ac4e13b80 100644 --- a/snapshot-service/pb/service.swagger.json +++ b/snapshot-service/pb/service.swagger.json @@ -132,7 +132,7 @@ "parameters": [ { "name": "id", - "description": "Unique condition ID\n\n@gotags: bson:\"_id\"", + "description": "Unique condition ID\n\n@gotags: bson:\"id\"", "in": "path", "required": true, "type": "string" @@ -472,7 +472,8 @@ "items": { "type": "string" }, - "title": "@gotags: bson:\"deviceIdFilter,omitempty\"" + "description": "@gotags: bson:\"deviceIdFilter,omitempty\"", + "title": "list of device IDs to which the condition applies" }, "resourceTypeFilter": { "type": "array", @@ -486,7 +487,8 @@ "items": { "type": "string" }, - "title": "@gotags: bson:\"resourceHrefFilter,omitempty\"" + "description": "@gotags: bson:\"resourceHrefFilter,omitempty\"", + "title": "list of resource hrefs to which the condition applies" }, "jqExpressionFilter": { "type": "string", @@ -617,7 +619,7 @@ "properties": { "id": { "type": "string", - "description": "@gotags: bson:\"_id\"", + "description": "@gotags: bson:\"id\"", "title": "Unique condition ID" }, "version": { @@ -643,7 +645,8 @@ "items": { "type": "string" }, - "title": "@gotags: bson:\"deviceIdFilter,omitempty\"" + "description": "@gotags: bson:\"deviceIdFilter,omitempty\"", + "title": "list of device IDs to which the condition applies" }, "resourceTypeFilter": { "type": "array", @@ -657,7 +660,8 @@ "items": { "type": "string" }, - "title": "@gotags: bson:\"resourceHrefFilter,omitempty\"" + "description": "@gotags: bson:\"resourceHrefFilter,omitempty\"", + "title": "list of resource hrefs to which the condition applies" }, "jqExpressionFilter": { "type": "string", diff --git a/snapshot-service/store/condition.go b/snapshot-service/store/condition.go index d595c94d2..970770196 100644 --- a/snapshot-service/store/condition.go +++ b/snapshot-service/store/condition.go @@ -5,8 +5,12 @@ import ( ) const ( - DeviceIDFilterKey = "deviceIdFilter" // must match with pb.Condition.DeviceIdFilter tag - OwnerKey = "owner" // must match with pb.Condition.Owner tag + IDKey = "id" // must match with pb.Condition.Id tag + VersionKey = "version" // must match with pb.Condition.Version tag + DeviceIDFilterKey = "deviceIdFilter" // must match with pb.Condition.DeviceIdFilter tag + ResourceHrefFilterKey = "resourceHrefFilter" // must match with pb.Condition.ResourceHrefFilter tag + ResourceTypeFilterKey = "resourceTypeFilter" // must match with pb.Condition.ResourceTypeFilter tag + OwnerKey = "owner" // must match with pb.Condition.Owner tag ) type Condition = pb.Condition diff --git a/snapshot-service/store/mongodb/condition.go b/snapshot-service/store/mongodb/condition.go index c2ce9857b..dba7af734 100644 --- a/snapshot-service/store/mongodb/condition.go +++ b/snapshot-service/store/mongodb/condition.go @@ -3,6 +3,7 @@ package mongodb import ( "context" "errors" + "slices" "github.com/hashicorp/go-multierror" "github.com/plgd-dev/hub/v2/snapshot-service/store" @@ -36,6 +37,10 @@ func (s *Store) CreateCondition(ctx context.Context, cond *store.Condition) erro if err := cond.Validate(); err != nil { return err } + // ensure that resource type filter is sorted and compacted, so we can query for exact match instead of other more expensive queries + resourceTypeFilter := cond.GetResourceTypeFilter() + slices.Sort(resourceTypeFilter) + cond.ResourceTypeFilter = slices.Compact(resourceTypeFilter) _, err := s.Collection(conditionsCol).InsertOne(ctx, cond) if err != nil { return err @@ -66,6 +71,21 @@ func toDeviceIDQueryFilter(deviceID string) bson.M { }} } +func toResourceHrefQueryFilter(resourceHref string) bson.M { + return bson.M{"$or": []bson.D{ + {{Key: store.ResourceHrefFilterKey, Value: bson.M{"$exists": false}}}, + {{Key: store.ResourceHrefFilterKey, Value: resourceHref}}, + }} +} + +func toResouceTypeQueryFilter(resourceTypeFilter []string) bson.M { + slices.Sort(resourceTypeFilter) + return bson.M{"$or": []bson.D{ + {{Key: store.ResourceTypeFilterKey, Value: bson.M{"$exists": false}}}, + {{Key: store.ResourceTypeFilterKey, Value: resourceTypeFilter}}, + }} +} + func toConditionsQueryFilter(owner string, queries *store.ConditionsQuery) interface{} { filter := make([]interface{}, 0, 2) if owner != "" { @@ -74,6 +94,12 @@ func toConditionsQueryFilter(owner string, queries *store.ConditionsQuery) inter if queries.DeviceID != "" { filter = append(filter, toDeviceIDQueryFilter(queries.DeviceID)) } + if queries.ResourceHref != "" { + filter = append(filter, toResourceHrefQueryFilter(queries.ResourceHref)) + } + if len(queries.ResourceTypeFilter) > 0 { + filter = append(filter, toResouceTypeQueryFilter(queries.ResourceTypeFilter)) + } if len(filter) == 0 { return bson.D{} } @@ -89,10 +115,9 @@ func (s *Store) LoadConditions(ctx context.Context, owner string, query *store.C if errors.Is(err, mongo.ErrNilDocument) { return nil } - if err != nil { + if h == nil || err != nil { return err } - var errors *multierror.Error i := conditionsIterator{ iter: iter, diff --git a/snapshot-service/store/mongodb/condition_test.go b/snapshot-service/store/mongodb/condition_test.go index 8c4ac5551..c08d4d240 100644 --- a/snapshot-service/store/mongodb/condition_test.go +++ b/snapshot-service/store/mongodb/condition_test.go @@ -116,7 +116,7 @@ func TestStoreCreateCondition(t *testing.T) { } } -func TestStoreLoadConditionsByDeviceIDAndOwner(t *testing.T) { +func TestStoreLoadConditions(t *testing.T) { s, cleanUpStore := test.NewMongoStore(t) defer cleanUpStore() @@ -126,6 +126,14 @@ func TestStoreLoadConditionsByDeviceIDAndOwner(t *testing.T) { const deviceID1 = "deviceID1" const deviceID2 = "deviceID2" const deviceID3 = "deviceID3" + const href1 = "/1" + const href2 = "/2" + const href3 = "/3" + const href4 = "/4" + const href5 = "/5" + const type1 = "type1" + const type2 = "type2" + const type3 = "type3" const owner1 = "owner1" const owner2 = "owner2" cond1 := &store.Condition{ @@ -138,45 +146,65 @@ func TestStoreLoadConditionsByDeviceIDAndOwner(t *testing.T) { require.NoError(t, err) cond2 := &store.Condition{ - Id: uuid.New().String(), - Name: "c2", - ConfigurationId: uuid.New().String(), - DeviceIdFilter: []string{deviceID1}, - Owner: owner1, + Id: uuid.New().String(), + Name: "c2", + ConfigurationId: uuid.New().String(), + DeviceIdFilter: []string{deviceID1}, + ResourceHrefFilter: []string{href1, href2, href3}, + ResourceTypeFilter: []string{type1, type2}, + Owner: owner1, } err = s.CreateCondition(ctx, cond2) require.NoError(t, err) cond3 := &store.Condition{ - Id: uuid.New().String(), - Name: "c3", - ConfigurationId: uuid.New().String(), - DeviceIdFilter: []string{deviceID2}, - Owner: owner1, + Id: uuid.New().String(), + Name: "c3", + ConfigurationId: uuid.New().String(), + DeviceIdFilter: []string{deviceID2}, + ResourceHrefFilter: []string{href3, href4, href5}, + ResourceTypeFilter: []string{type3}, + Owner: owner1, } err = s.CreateCondition(ctx, cond3) require.NoError(t, err) cond4 := &store.Condition{ - Id: uuid.New().String(), - Name: "c4", - ConfigurationId: uuid.New().String(), - DeviceIdFilter: []string{deviceID1, deviceID3}, - Owner: owner1, + Id: uuid.New().String(), + Name: "c4", + ConfigurationId: uuid.New().String(), + DeviceIdFilter: []string{deviceID1, deviceID3}, + ResourceHrefFilter: []string{href1, href5}, + ResourceTypeFilter: []string{type3}, + Owner: owner1, } err = s.CreateCondition(ctx, cond4) require.NoError(t, err) cond5 := &store.Condition{ - Id: uuid.New().String(), - Name: "c5", - ConfigurationId: uuid.New().String(), - DeviceIdFilter: []string{deviceID3}, - Owner: owner2, + Id: uuid.New().String(), + Name: "c5", + ConfigurationId: uuid.New().String(), + DeviceIdFilter: []string{deviceID3}, + ResourceHrefFilter: []string{href1, href2}, + ResourceTypeFilter: []string{type2}, + Owner: owner2, } err = s.CreateCondition(ctx, cond5) require.NoError(t, err) + cond6 := &store.Condition{ + Id: uuid.New().String(), + Name: "c6", + ConfigurationId: uuid.New().String(), + DeviceIdFilter: []string{deviceID2, deviceID3}, + ResourceHrefFilter: []string{href2, href3, href4}, + ResourceTypeFilter: []string{type1, type2, type3}, + Owner: owner2, + } + err = s.CreateCondition(ctx, cond6) + require.NoError(t, err) + type args struct { query *store.ConditionsQuery owner string @@ -193,7 +221,7 @@ func TestStoreLoadConditionsByDeviceIDAndOwner(t *testing.T) { args: args{ query: &store.ConditionsQuery{}, }, - want: pb.Conditions{cond1, cond2, cond3, cond4, cond5}, + want: pb.Conditions{cond1, cond2, cond3, cond4, cond5, cond6}, }, { name: "deviceID3", @@ -202,7 +230,7 @@ func TestStoreLoadConditionsByDeviceIDAndOwner(t *testing.T) { DeviceID: deviceID3, }, }, - want: pb.Conditions{cond1, cond4, cond5}, + want: pb.Conditions{cond1, cond4, cond5, cond6}, }, { name: "owner1", @@ -260,7 +288,77 @@ func TestStoreLoadConditionsByDeviceIDAndOwner(t *testing.T) { }, owner: owner2, }, - want: pb.Conditions{cond5}, + want: pb.Conditions{cond5, cond6}, + }, + { + name: "href1", + args: args{ + query: &store.ConditionsQuery{ + ResourceHref: href1, + }, + }, + want: pb.Conditions{cond1, cond2, cond4, cond5}, + }, + { + name: "deviceID1/href3", + args: args{ + query: &store.ConditionsQuery{ + DeviceID: deviceID1, + ResourceHref: href3, + }, + }, + want: pb.Conditions{cond1, cond2}, + }, + { + name: "owner2/href2", + args: args{ + query: &store.ConditionsQuery{ + ResourceHref: href2, + }, + owner: owner2, + }, + want: pb.Conditions{cond5, cond6}, + }, + { + name: "[type2]", + args: args{ + query: &store.ConditionsQuery{ + ResourceTypeFilter: []string{type2}, + }, + }, + want: pb.Conditions{cond1, cond5}, + }, + { + name: "deviceID2/[type3]", + args: args{ + query: &store.ConditionsQuery{ + DeviceID: deviceID2, + ResourceTypeFilter: []string{type3}, + }, + }, + want: pb.Conditions{cond1, cond3}, + }, + { + name: "owner2/[type1,type2,type3}", + args: args{ + query: &store.ConditionsQuery{ + // order should not matter + ResourceTypeFilter: []string{type2, type1, type3}, + }, + owner: owner2, + }, + want: pb.Conditions{cond6}, + }, + { + name: "deviceID1/href5/[type3]", + args: args{ + query: &store.ConditionsQuery{ + DeviceID: deviceID1, + ResourceHref: href5, + ResourceTypeFilter: []string{type3}, + }, + }, + want: pb.Conditions{cond1, cond4}, }, } @@ -293,3 +391,64 @@ func TestStoreLoadConditionsByDeviceIDAndOwner(t *testing.T) { }) } } + +/* + +func fillDatabase(ctx context.Context, s *mongodb.Store, n int) error { + generateDeviceIDFilter := func(n int) []string { + if n == 0 { + return nil + } + deviceIDFilter := make([]string, 0, n) + for i := 0; i <= n; i++ { + deviceIDFilter = append(deviceIDFilter, "deviceID"+strconv.Itoa(i)) + } + return deviceIDFilter + } + generateResourceHrefFilter := func(start, count int) []string { + if count == 0 { + return nil + } + resourceHrefFilter := make([]string, 0, count) + for i := 0; i < count; i++ { + resourceHrefFilter = append(resourceHrefFilter, "/"+strconv.Itoa(start+i)) + } + return resourceHrefFilter + } + for i := 0; i < n; i++ { + err := s.CreateCondition(ctx, &store.Condition{ + Id: hubTest.GenerateIDbyIdx("a", i), + DeviceIdFilter: generateDeviceIDFilter(i % 7), // deviceID{0-6} + ResourceHrefFilter: generateResourceHrefFilter(i%16, i%5), // /{0-19} + Name: "c" + strconv.Itoa(i), + ConfigurationId: hubTest.GenerateIDbyIdx("b", i), + Owner: "owner" + strconv.Itoa(i%3), // owner{0-2} + }) + if err != nil { + return err + } + } + return nil +} + +func BenchmarkLoadConditions(b *testing.B) { + s, cleanUpStore := test.NewMongoStore(b) + defer cleanUpStore() + + ctx := context.Background() + err := fillDatabase(ctx, s, 5000) + require.NoError(b, err) + + b.ResetTimer() + // by owner + err = s.LoadConditions(ctx, "owner1", &store.ConditionsQuery{}, nil) + require.NoError(b, err) + // by deviceD + err = s.LoadConditions(ctx, "", &store.ConditionsQuery{DeviceID: "deviceID1"}, nil) + require.NoError(b, err) + // by deviceID and owner + err = s.LoadConditions(ctx, "owner0", &store.ConditionsQuery{DeviceID: "deviceID3"}, nil) + require.NoError(b, err) +} + +*/ diff --git a/snapshot-service/store/mongodb/store.go b/snapshot-service/store/mongodb/store.go index 5b2c095b3..0959745fc 100644 --- a/snapshot-service/store/mongodb/store.go +++ b/snapshot-service/store/mongodb/store.go @@ -8,6 +8,10 @@ import ( "github.com/plgd-dev/hub/v2/pkg/log" pkgMongo "github.com/plgd-dev/hub/v2/pkg/mongodb" "github.com/plgd-dev/hub/v2/pkg/security/certManager/client" + "github.com/plgd-dev/hub/v2/snapshot-service/store" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" "go.opentelemetry.io/otel/trace" ) @@ -17,12 +21,28 @@ type Store struct { const conditionsCol = "conditions" +var idVersionUniqueIndex = mongo.IndexModel{ + Keys: bson.D{ + {Key: store.IDKey, Value: 1}, + {Key: store.VersionKey, Value: 1}, + }, + Options: options.Index().SetUnique(true), +} + +var deviceIDFilterAndOwnerIndex = mongo.IndexModel{ + Keys: bson.D{ + {Key: store.DeviceIDFilterKey, Value: 1}, + {Key: store.OwnerKey, Value: 1}, + }, +} + func New(ctx context.Context, cfg *Config, fileWatcher *fsnotify.Watcher, logger log.Logger, tracerProvider trace.TracerProvider) (*Store, error) { certManager, err := client.New(cfg.Mongo.TLS, fileWatcher, logger) if err != nil { return nil, fmt.Errorf("could not create cert manager: %w", err) } - m, err := pkgMongo.NewStore(ctx, &cfg.Mongo, certManager.GetTLSConfig(), tracerProvider) + + m, err := pkgMongo.NewStoreWithCollection(ctx, &cfg.Mongo, certManager.GetTLSConfig(), tracerProvider, conditionsCol, idVersionUniqueIndex, deviceIDFilterAndOwnerIndex) if err != nil { certManager.Close() return nil, err diff --git a/snapshot-service/store/store.go b/snapshot-service/store/store.go index f736ee808..80667222e 100644 --- a/snapshot-service/store/store.go +++ b/snapshot-service/store/store.go @@ -7,7 +7,9 @@ import ( type ( ConditionsQuery struct { - DeviceID string + DeviceID string + ResourceHref string + ResourceTypeFilter []string } )