feat: extend ResourceSelector

getters.go

@@ -3,7 +3,6 @@ package duty
 import (
 	"errors"
 	"fmt"
-	"strings"
 	"time"
 
 	"github.com/flanksource/commons/collections"
@@ -16,8 +15,12 @@ import (
 	"gorm.io/gorm"
 )
 
-// getterCache caches the results for all the getters in this file.
-var getterCache = cache.New(time.Second*90, time.Minute*5)
+var (
+	// getterCache caches the results for all the getters in this file.
+	getterCache = cache.New(time.Second*90, time.Minute*5)
+
+	immutableCache = cache.New(cache.NoExpiration, cache.NoExpiration)
+)
 
 func cacheKey[T any](field, key string) string {
 	var v T
@@ -138,39 +141,94 @@ func apply(db *gorm.DB, opts ...FindOption) *gorm.DB {
 }
 
 func FindChecks(ctx context.Context, resourceSelectors types.ResourceSelectors, opts ...FindOption) (components []models.Check, err error) {
-	var uniqueComponents []models.Check
+	for _, rs := range resourceSelectors {
+		if rs.FieldSelector != "" {
+			return nil, fmt.Errorf("field selector is not supported for checks (%s)", rs.FieldSelector)
+		}
+	}
+
+	var uniqueChecks []models.Check
 	for _, resourceSelector := range resourceSelectors {
+		hash := "FindChecks-CachePrefix" + resourceSelector.Hash()
+		cacheToUse := getterCache
+		if resourceSelector.Immutable() {
+			cacheToUse = immutableCache
+		}
+
+		if val, ok := cacheToUse.Get(hash); ok {
+			checks, err := FindChecksByIDs(ctx, val.([]string), opts...)
+			if err != nil {
+				return nil, err
+			}
+
+			uniqueChecks = append(uniqueChecks, checks...)
+			continue
+		}
+
 		if resourceSelector.LabelSelector != "" {
-			labelComponents, err := FindChecksByLabel(ctx, resourceSelector.LabelSelector, opts...)
+			checks, err := FindChecksByLabel(ctx, resourceSelector.LabelSelector, opts...)
 			if err != nil {
 				return nil, fmt.Errorf("error getting checks with label selectors[%s]: %w", resourceSelector.LabelSelector, err)
 			}
-			uniqueComponents = append(uniqueComponents, labelComponents...)
-		}
-		if resourceSelector.FieldSelector != "" {
-			return nil, fmt.Errorf("fieldSelector not supported on checks")
+
+			cacheToUse.SetDefault(hash, lo.Map(checks, func(c models.Check, _ int) string { return c.ID.String() }))
+			uniqueChecks = append(uniqueChecks, checks...)
 		}
 	}
 
-	return lo.UniqBy(uniqueComponents, models.CheckID), nil
+	return lo.UniqBy(uniqueChecks, models.CheckID), nil
 }
 
 func FindComponents(ctx context.Context, resourceSelectors types.ResourceSelectors, opts ...FindOption) (components []models.Component, err error) {
 	var allComponents []models.Component
 	for _, resourceSelector := range resourceSelectors {
+		hash := "FindComponents-CachePrefix" + resourceSelector.Hash()
+		cacheToUse := getterCache
+		if resourceSelector.Immutable() {
+			cacheToUse = immutableCache
+		}
+
+		if val, ok := cacheToUse.Get(hash); ok {
+			components, err := FindComponentsByIDs(ctx, val.([]string), opts...)
+			if err != nil {
+				return nil, err
+			}
+
+			allComponents = append(allComponents, components...)
+			continue
+		}
+
 		var uniqueComponents []models.Component
 		selectorOpts := opts
-		if resourceSelector.Name != "" {
-			nameComponents, err := FindComponentsByName(ctx, resourceSelector.Name, opts...)
-			if err != nil {
-				return nil, fmt.Errorf("error getting components with name selectors[%s]: %w", resourceSelector.Name, err)
+
+		if resourceSelector.Name != "" || resourceSelector.Namespace != "" || resourceSelector.AgentID != "" || len(resourceSelector.Types) != 0 || len(resourceSelector.Statuses) != 0 {
+			query := ctx.DB()
+			if resourceSelector.Name != "" {
+				query = query.Where("name = ?", resourceSelector.Name)
+			}
+			if resourceSelector.Namespace != "" {
+				query = query.Where("namespace = ?", resourceSelector.Namespace)
+			}
+			if resourceSelector.AgentID != "" {
+				query = query.Where("agent_id = ?", resourceSelector.AgentID)
 			}
+			if len(resourceSelector.Types) != 0 {
+				query = query.Where("type IN ?", resourceSelector.Types)
+			}
+			if len(resourceSelector.Statuses) != 0 {
+				query = query.Where("status IN ?", resourceSelector.Statuses)
+			}
+
+			var nameComponents []models.Component
+			if err := apply(query, opts...).Find(&components).Error; err != nil {
+				return nil, fmt.Errorf("error getting components with selectors[%v]: %w", resourceSelector, err)
+			}
+
 			uniqueComponents = nameComponents
 			selectorOpts = append(selectorOpts, WhereClause("id::text in ?", lo.Map(
 				nameComponents,
 				func(c models.Component, _ int) string { return c.ID.String() }),
 			))
-
 		}
 
 		if resourceSelector.LabelSelector != "" {
@@ -192,34 +250,22 @@ func FindComponents(ctx context.Context, resourceSelectors types.ResourceSelecto
 			}
 			uniqueComponents = fieldComponents
 		}
+
+		cacheToUse.SetDefault(hash, lo.Map(uniqueComponents, func(c models.Component, _ int) string { return c.ID.String() }))
+
 		allComponents = append(allComponents, uniqueComponents...)
 	}
 
 	return lo.UniqBy(allComponents, models.ComponentID), nil
 }
 
-func getLabelsFromSelector(selector string) (matchLabels map[string]string) {
-	matchLabels = make(types.JSONStringMap)
-	labels := strings.Split(selector, ",")
-	for _, label := range labels {
-		if strings.Contains(label, "=") {
-			kv := strings.Split(label, "=")
-			if len(kv) == 2 {
-				matchLabels[kv[0]] = kv[1]
-			} else {
-				matchLabels[kv[0]] = ""
-			}
-		}
-	}
-	return
-}
-
 func FindComponentsByLabel(ctx context.Context, labelSelector string, opts ...FindOption) (components []models.Component, err error) {
 	if labelSelector == "" {
 		return nil, nil
 	}
+
 	var items = make(map[string]models.Component)
-	matchLabels := getLabelsFromSelector(labelSelector)
+	matchLabels := collections.SelectorToMap(labelSelector)
 	var labels = make(map[string]string)
 	var onlyKeys []string
 	for k, v := range matchLabels {
@@ -229,6 +275,7 @@ func FindComponentsByLabel(ctx context.Context, labelSelector string, opts ...Fi
 			onlyKeys = append(onlyKeys, k)
 		}
 	}
+
 	var comps []models.Component
 	if err := apply(ctx.DB().Where(LocalFilter).
 		Where("labels @> ?", types.JSONStringMap(labels)), opts...).
@@ -238,6 +285,7 @@ func FindComponentsByLabel(ctx context.Context, labelSelector string, opts ...Fi
 	for _, c := range comps {
 		items[c.ID.String()] = c
 	}
+
 	for _, k := range onlyKeys {
 		var comps []models.Component
 		if err := apply(ctx.DB().Where(LocalFilter).
@@ -250,14 +298,16 @@ func FindComponentsByLabel(ctx context.Context, labelSelector string, opts ...Fi
 			items[c.ID.String()] = c
 		}
 	}
+
 	return lo.Values(items), nil
 }
 
 func FindComponentsByField(ctx context.Context, fieldSelector string, opts ...FindOption) ([]models.Component, error) {
 	if fieldSelector == "" {
 		return nil, nil
 	}
-	matchLabels := getLabelsFromSelector(fieldSelector)
+
+	matchLabels := collections.SelectorToMap(fieldSelector)
 	allowedColumnsAsFields := []string{"type", "status", "topology_type", "owner", "agent_id", "namespace"}
 
 	columnWhereClauses := map[string]string{
@@ -307,6 +357,26 @@ func FindComponentsByField(ctx context.Context, fieldSelector string, opts ...Fi
 	return components, nil
 }
 
+func FindChecksByIDs(ctx DBContext, ids []string, opts ...FindOption) ([]models.Check, error) {
+	if len(ids) == 0 {
+		return nil, nil
+	}
+
+	var checks []models.Check
+	err := apply(ctx.DB().Where(LocalFilter).Where("id IN ?", ids), opts...).Find(&checks).Error
+	return checks, err
+}
+
+func FindComponentsByIDs(ctx DBContext, ids []string, opts ...FindOption) ([]models.Component, error) {
+	if len(ids) == 0 {
+		return nil, nil
+	}
+
+	var components []models.Component
+	err := apply(ctx.DB().Where(LocalFilter).Where("id IN ?", ids), opts...).Find(&components).Error
+	return components, err
+}
+
 func FindComponentsByName(ctx context.Context, name string, opts ...FindOption) ([]models.Component, error) {
 	if name == "" {
 		return nil, nil
@@ -325,8 +395,9 @@ func FindChecksByLabel(ctx context.Context, labelSelector string, opts ...FindOp
 	if labelSelector == "" {
 		return nil, nil
 	}
+
 	var items = make(map[string]models.Check)
-	matchLabels := getLabelsFromSelector(labelSelector)
+	matchLabels := collections.SelectorToMap(labelSelector)
 	var labels = make(map[string]string)
 	var onlyKeys []string
 	for k, v := range matchLabels {

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/eko/gocache/store/go_cache/v4 v4.2.1
 	github.com/exaring/otelpgx v0.5.2
 	github.com/fergusstrange/embedded-postgres v1.25.0
-	github.com/flanksource/commons v1.21.2
+	github.com/flanksource/commons v1.22.0
 	github.com/flanksource/gomplate/v3 v3.20.29
 	github.com/flanksource/kommons v0.31.4
 	github.com/flanksource/postq v0.1.3
@@ -191,3 +191,5 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
+
+// replace "github.com/flanksource/commons" => ../commons

go.sum

@@ -768,8 +768,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fergusstrange/embedded-postgres v1.25.0 h1:sa+k2Ycrtz40eCRPOzI7Ry7TtkWXXJ+YRsxpKMDhxK0=
 github.com/fergusstrange/embedded-postgres v1.25.0/go.mod h1:t/MLs0h9ukYM6FSt99R7InCHs1nW0ordoVCcnzmpTYw=
-github.com/flanksource/commons v1.21.2 h1:Z6ZGeBkTenucalfPLGEnq5qMB6uwSfH3ycBOGutRcfg=
-github.com/flanksource/commons v1.21.2/go.mod h1:GD5+yGvmYFPIW3WMNN+y1JkeDMJY74e05pQAsRbrvwY=
+github.com/flanksource/commons v1.22.0 h1:LI839ZOVJ6qrNuGKqb6Z4JyIxFHXGhfDxJbXUj5VimQ=
+github.com/flanksource/commons v1.22.0/go.mod h1:GD5+yGvmYFPIW3WMNN+y1JkeDMJY74e05pQAsRbrvwY=
 github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc=
 github.com/flanksource/gomplate/v3 v3.20.29 h1:hDrNw1JaQk+gmhSCvqng+nebYAt3a5afhf/Vdmr4CTs=
 github.com/flanksource/gomplate/v3 v3.20.29/go.mod h1:GKmptFMdr2LbOuqwQZrmo9a/UygyZ0pbXffks8MuYhE=

hack/migrate/go.mod

@@ -3,7 +3,7 @@ module github.com/flanksource/duty/hack/migrate
 go 1.20
 
 require (
-	github.com/flanksource/commons v1.21.2
+	github.com/flanksource/commons v1.22.0
 	github.com/flanksource/duty v1.0.180
 	github.com/spf13/cobra v1.7.0
 )

hack/migrate/go.sum

@@ -767,8 +767,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/flanksource/commons v1.21.2 h1:Z6ZGeBkTenucalfPLGEnq5qMB6uwSfH3ycBOGutRcfg=
-github.com/flanksource/commons v1.21.2/go.mod h1:GD5+yGvmYFPIW3WMNN+y1JkeDMJY74e05pQAsRbrvwY=
+github.com/flanksource/commons v1.22.0 h1:LI839ZOVJ6qrNuGKqb6Z4JyIxFHXGhfDxJbXUj5VimQ=
+github.com/flanksource/commons v1.22.0/go.mod h1:GD5+yGvmYFPIW3WMNN+y1JkeDMJY74e05pQAsRbrvwY=
 github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc=
 github.com/flanksource/gomplate/v3 v3.20.29 h1:hDrNw1JaQk+gmhSCvqng+nebYAt3a5afhf/Vdmr4CTs=
 github.com/flanksource/gomplate/v3 v3.20.29/go.mod h1:GKmptFMdr2LbOuqwQZrmo9a/UygyZ0pbXffks8MuYhE=

types/common.go

@@ -1,6 +1,12 @@
 package types
 
-import "encoding/json"
+import (
+	"encoding/json"
+	"sort"
+	"strings"
+
+	"gorm.io/gorm"
+)
 
 // asMap marshals the given struct into a map.
 func asMap(t any, removeFields ...string) map[string]any {
@@ -16,3 +22,88 @@ func asMap(t any, removeFields ...string) map[string]any {
 
 	return m
 }
+
+type Items []string
+
+func (items Items) String() string {
+	return strings.Join(items, ",")
+}
+
+// contains returns true if any of the items in the list match the item
+// negative matches are supported by prefixing the item with a !
+// * matches everything
+func (items Items) Contains(item string) bool {
+	if len(items) == 0 {
+		return true
+	}
+
+	negations := 0
+	for _, i := range items {
+		if strings.HasPrefix(i, "!") {
+			negations++
+			if item == strings.TrimPrefix(i, "!") {
+				return false
+			}
+		}
+	}
+
+	if negations == len(items) {
+		// none of the negations matched
+		return true
+	}
+
+	for _, i := range items {
+		if strings.HasPrefix(i, "!") {
+			continue
+		}
+		if i == "*" || item == i {
+			return true
+		}
+	}
+	return false
+}
+
+func (items Items) WithNegation() []string {
+	var result []string
+	for _, item := range items {
+		if strings.HasPrefix(item, "!") {
+			result = append(result, item[1:])
+		}
+	}
+	return result
+}
+
+// Sort returns a sorted copy
+func (items Items) Sort() Items {
+	copy := items
+	sort.Slice(copy, func(i, j int) bool { return items[i] < items[j] })
+	return copy
+}
+
+func (items Items) WithoutNegation() []string {
+	var result []string
+	for _, item := range items {
+		if !strings.HasPrefix(item, "!") {
+			result = append(result, item)
+		}
+	}
+	return result
+}
+
+func (items Items) Where(query *gorm.DB, col string) *gorm.DB {
+	if items == nil {
+		return query
+	}
+
+	negated := items.WithNegation()
+	if len(negated) > 0 {
+		query = query.Where("NOT "+col+" IN ?", negated)
+	}
+
+	positive := items.WithoutNegation()
+	if len(positive) > 0 {
+		query = query.Where(col+" IN ?", positive)
+	}
+
+	return query
+}