chore: add test for config traversal

query/config_cache.go

@@ -205,38 +205,43 @@ func configRelationCacheKey(id string) string {
 
 var LocalFilter = "deleted_at is NULL AND agent_id = '00000000-0000-0000-0000-000000000000' OR agent_id IS NULL"
 
-func BuildConfigCache(ctx context.Context) {
+func SyncConfigCache(ctx context.Context) error {
 	var configItems []models.ConfigItem
-	// TODO Func singature if error
-	if err := ctx.DB().Table("config_items").Where(LocalFilter).FindInBatches(&configItems, 1000, nil); err != nil {
+
+	if err := ctx.DB().Table("config_items").Where(LocalFilter).FindInBatches(&configItems, 1000, func(*gorm.DB, int) error { return nil }).Error; err != nil {
+		return fmt.Errorf("error querying config items for cache: %w", err)
 	}
 
 	for _, ci := range configItems {
-		configItemCache.Set(ctx, configItemCacheKey(ci.ID.String()), ci, nil)
+		configItemCache.Set(ctx, configItemCacheKey(ci.ID.String()), ci)
 
 		if ci.Type != nil {
 			configTypeKey := configItemTypeCacheKey(*ci.Type)
 			val, _ := configItemTypeCache.Get(ctx, configTypeKey)
 			val = append(val, ci.ID.String())
-			configItemTypeCache.Set(ctx, configTypeKey, val, nil)
+			configItemTypeCache.Set(ctx, configTypeKey, val)
 		}
 	}
 
 	var configRelations []models.ConfigRelationship
-	// TODO Func singature if error
-	if err := ctx.DB().Table("config_relationships").Where(LocalFilter).FindInBatches(&configRelations, 5000, nil); err != nil {
+
+	if err := ctx.DB().Table("config_relationships").Where("deleted_at IS NULL").FindInBatches(&configRelations, 5000, func(_ *gorm.DB, _ int) error { return nil }).Error; err != nil {
+		return fmt.Errorf("error querying config relationships for cache: %w", err)
 	}
 
 	relGroup := make(map[string][]string)
 	for _, ci := range configRelations {
 		relGroup[ci.ConfigID] = append(relGroup[ci.ConfigID], ci.RelatedID)
 	}
 
+	// TODO: Acquire Lock ? Old config IDs can persist in type cache
+	configRelationCache.Clear(ctx)
+
 	for ciD, relIDs := range relGroup {
-		configRelatedKey := "configRelatedIDs:" + ciD
-		configRelationCache.Set(ctx, configRelatedKey, relIDs, nil)
+		configRelationCache.Set(ctx, configRelationCacheKey(ciD), relIDs)
 	}
 
+	return nil
 }
 
 func ConfigIDsByTypeFromCache(ctx context.Context, typ string) ([]string, error) {

query/config_traversal.go

@@ -9,26 +9,21 @@ import (
 	"github.com/samber/lo"
 )
 
-//         - gitops:
-//             repository:  "{{  catalog_traverse(.id, 'Kubernetes::Kustomization/Kubernetes::GitRepository').spec.repository }}"
-//             file: "{{  catalog_traverse(.id, 'Kubernetes::Kustomization').spec.path }}/{{ .config.annotations['config.kubernetes.io/origin'] | YAML | .path }}"
-
-// IN INIT (Setup cache via ID and type)SpecSpecSpecSpec
-
 func TraverseConfig(ctx context.Context, id, relationType string) (models.ConfigItem, error) {
 	var configItem models.ConfigItem
 
 	relationTypeList := strings.Split(relationType, "/")
-	relatedIDs, err := ConfigRelationsFromCache(ctx, id)
-	if err != nil {
-		return configItem, fmt.Errorf("no relations found for config[%s]: %w", id, err)
-	}
-
-	if len(relatedIDs) == 0 {
-		return configItem, fmt.Errorf("no relations found for config[%s]: %w", id, err)
-	}
 
 	for _, relType := range relationTypeList {
+		relatedIDs, err := ConfigRelationsFromCache(ctx, id)
+		if err != nil {
+			return configItem, fmt.Errorf("no relations found for config[%s] in cache: %w", id, err)
+		}
+
+		if len(relatedIDs) == 0 {
+			return configItem, fmt.Errorf("no relations found for config[%s]: %w", id, err)
+		}
+
 		typeIDs, err := ConfigIDsByTypeFromCache(ctx, relType)
 		if err != nil {
 			return configItem, fmt.Errorf("no type %s exists for any config: %w", relType, err)
@@ -39,14 +34,15 @@ func TraverseConfig(ctx context.Context, id, relationType string) (models.Config
 		})
 
 		if !ok {
-			return configItem, fmt.Errorf("no matching type %s found in relations for config[%s]: %w", relType, id, err)
+			return configItem, fmt.Errorf("no matching type %s found in relations for config[%s]", relType, id)
 		}
 
 		configItem, err = ConfigItemFromCache(ctx, configID)
 		if err != nil {
 			return configItem, fmt.Errorf("no config[%s] found in cache: %w", configID, err)
 		}
-		// Updating to set correct error message
+
+		// Updating for next loop iteration
 		id = configItem.ID.String()
 	}
 

tests/config_traversal_test.go

@@ -0,0 +1,57 @@
+package tests
+
+import (
+	"github.com/flanksource/commons/utils"
+	"github.com/flanksource/duty/models"
+	"github.com/flanksource/duty/query"
+	"github.com/google/uuid"
+	ginkgo "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/samber/lo"
+	"gorm.io/gorm/clause"
+)
+
+var _ = ginkgo.Describe("Config traversal", ginkgo.Ordered, func() {
+	ginkgo.It("should be able to traverse config relationships via types", func() {
+		configItems := map[string]models.ConfigItem{
+			"deployment":                 {ID: uuid.New(), Name: utils.Ptr("canary-checker"), Type: utils.Ptr("Kubernetes::Deployment")},
+			"helm-release-of-deployment": {ID: uuid.New(), Name: utils.Ptr("mission-control"), Type: utils.Ptr("Kubernetes::HelmRelease")},
+			"kustomize-of-helm-release":  {ID: uuid.New(), Name: utils.Ptr("aws-demo-infra"), Type: utils.Ptr("Kubernetes::Kustomization")},
+		}
+		ctx := DefaultContext
+		err := ctx.DB().Save(lo.Values(configItems)).Error
+		Expect(err).ToNot(HaveOccurred())
+
+		configRelations := []models.ConfigRelationship{
+			{ConfigID: configItems["deployment"].ID.String(), RelatedID: configItems["helm-release-of-deployment"].ID.String(), Relation: "HelmReleaseDeployment"},
+			{ConfigID: configItems["helm-release-of-deployment"].ID.String(), RelatedID: configItems["kustomize-of-helm-release"].ID.String(), Relation: "KustomizationHelmRelease"},
+		}
+		err = ctx.DB().Clauses(clause.OnConflict{DoNothing: true}).Save(configRelations).Error
+		Expect(err).ToNot(HaveOccurred())
+
+		err = query.SyncConfigCache(DefaultContext)
+		Expect(err).ToNot(HaveOccurred())
+
+		got, err := query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::HelmRelease")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(got.ID.String()).To(Equal(configItems["helm-release-of-deployment"].ID.String()))
+
+		got, err = query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::HelmRelease/Kubernetes::Kustomization")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(got.ID.String()).To(Equal(configItems["kustomize-of-helm-release"].ID.String()))
+
+		_, err = query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::Pod")
+		Expect(err).To(HaveOccurred())
+
+		_, err = query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::HelmRelease/Kubernetes::Node")
+		Expect(err).To(HaveOccurred())
+
+		// Cleanup for normal tests to pass
+		err = ctx.DB().Where("config_id in ?", lo.Map(lo.Values(configItems), func(c models.ConfigItem, _ int) string { return c.ID.String() })).Delete(&models.ConfigRelationship{}).Error
+		Expect(err).ToNot(HaveOccurred())
+
+		err = ctx.DB().Delete(lo.Values(configItems)).Error
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+})