Engine simplification (#37)

Engine simplification by mutating counts directly in matching engines.

Author: tonyhb

engine.go

@@ -2,6 +2,7 @@ package expr
 
 import (
 	"context"
+	"sync"
 
 	"github.com/cespare/xxhash/v2"
 	"github.com/google/uuid"
@@ -18,6 +19,58 @@ const (
 	// EngineTypeART
 )
 
+func NewMatchResult() *MatchResult {
+	return &MatchResult{
+		Result: map[uuid.UUID]map[groupID]int{},
+		Lock:   &sync.Mutex{},
+	}
+}
+
+// MatchResult is a map of evaluable IDs to the groups found, and the number of elements
+// found matching that group.
+type MatchResult struct {
+	Result map[uuid.UUID]map[groupID]int
+	Lock   *sync.Mutex
+}
+
+func (m *MatchResult) Len() int {
+	m.Lock.Lock()
+	defer m.Lock.Unlock()
+	return len(m.Result)
+}
+
+// AddExprs increments the matched counter for the given eval's group ID
+func (m *MatchResult) Add(evalID uuid.UUID, gID groupID) {
+	m.Lock.Lock()
+	defer m.Lock.Unlock()
+	if _, ok := m.Result[evalID]; !ok {
+		m.Result[evalID] = map[groupID]int{}
+	}
+	m.Result[evalID][gID]++
+}
+
+// AddExprs increments the matched counter for each stored expression part.
+func (m *MatchResult) AddExprs(exprs ...*StoredExpressionPart) {
+	m.Lock.Lock()
+	defer m.Lock.Unlock()
+	for _, expr := range exprs {
+		if _, ok := m.Result[expr.EvaluableID]; !ok {
+			m.Result[expr.EvaluableID] = map[groupID]int{}
+		}
+		m.Result[expr.EvaluableID][expr.GroupID]++
+	}
+}
+
+// GroupMatches returns the total lenght of all matches for a given eval's group ID.
+func (m *MatchResult) GroupMatches(evalID uuid.UUID, gID groupID) int {
+	m.Lock.Lock()
+	defer m.Lock.Unlock()
+	if _, ok := m.Result[evalID]; !ok {
+		return 0
+	}
+	return m.Result[evalID][gID]
+}
+
 // MatchingEngine represents an engine (such as a b-tree, radix trie, or
 // simple hash map) which matches a predicate over many expressions.
 type MatchingEngine interface {
@@ -31,7 +84,9 @@ type MatchingEngine interface {
 	// expression parts received.  Some may return false positives, but
 	// each MatchingEngine should NEVER omit ExpressionParts which match
 	// the given input.
-	Match(ctx context.Context, input map[string]any) (matched []*StoredExpressionPart, err error)
+	//
+	// Note that the MatchResult is mutated on input.
+	Match(ctx context.Context, input map[string]any, result *MatchResult) (err error)
 
 	// Add adds a new expression part to the matching engine for future matches.
 	Add(ctx context.Context, p ExpressionPart) error
@@ -46,8 +101,8 @@ type MatchingEngine interface {
 	// granularity of expression parts received.  Some may return false positives by
 	// ignoring the variable name.  Note that each MatchingEngine should NEVER
 	// omit ExpressionParts which match the given input;  false positives are okay,
+	Search(ctx context.Context, variable string, input any, result *MatchResult)
 	// but not returning valid matches must be impossible.
-	Search(ctx context.Context, variable string, input any) (matched []*StoredExpressionPart)
 }
 
 // Leaf represents the leaf within a tree.  This stores all expressions

engine_null.go

@@ -34,10 +34,7 @@ func (n *nullLookup) Type() EngineType {
 	return EngineTypeNullMatch
 }
 
-func (n *nullLookup) Match(ctx context.Context, data map[string]any) (matched []*StoredExpressionPart, err error) {
-	l := &sync.Mutex{}
-	matched = []*StoredExpressionPart{}
-
+func (n *nullLookup) Match(ctx context.Context, data map[string]any, result *MatchResult) (err error) {
 	pool := newErrPool(errPoolOpts{concurrency: n.concurrency})
 
 	for item := range n.paths {
@@ -55,32 +52,26 @@ func (n *nullLookup) Match(ctx context.Context, data map[string]any) (matched []
 				res = []any{nil}
 			}
 
-			// This matches null, nil (as null), and any non-null items.
-			l.Lock()
-
 			// XXX: This engine hasn't been updated with denied items for !=.  It needs consideration
 			// in how to handle these cases appropriately.
-			found := n.Search(ctx, path, res[0])
-			matched = append(matched, found...)
-			l.Unlock()
+			n.Search(ctx, path, res[0], result)
 
 			return nil
 		})
 	}
 
-	return matched, pool.Wait()
+	return pool.Wait()
 }
 
-func (n *nullLookup) Search(ctx context.Context, variable string, input any) (matched []*StoredExpressionPart) {
+func (n *nullLookup) Search(ctx context.Context, variable string, input any, result *MatchResult) {
 	if input == nil {
 		// The input data is null, so the only items that can match are equality
 		// comparisons to null.
-		all := n.null[variable]
-		return all
+		result.AddExprs(n.null[variable]...)
+		return
 	}
 
-	all := n.not[variable]
-	return all
+	result.AddExprs(n.not[variable]...)
 }
 
 func (n *nullLookup) Add(ctx context.Context, p ExpressionPart) error {

engine_number.go

@@ -41,10 +41,7 @@ func (n numbers) Type() EngineType {
 	return EngineTypeBTree
 }
 
-func (n *numbers) Match(ctx context.Context, input map[string]any) (matched []*StoredExpressionPart, err error) {
-	l := &sync.Mutex{}
-	matched = []*StoredExpressionPart{}
-
+func (n *numbers) Match(ctx context.Context, input map[string]any, result *MatchResult) (err error) {
 	pool := newErrPool(errPoolOpts{concurrency: n.concurrency})
 
 	for item := range n.paths {
@@ -61,40 +58,19 @@ func (n *numbers) Match(ctx context.Context, input map[string]any) (matched []*S
 				return nil
 			}
 
-			var val float64
-			switch v := res[0].(type) {
-			case int:
-				val = float64(v)
-			case int64:
-				val = float64(v)
-			case float64:
-				val = v
-			default:
-				return nil
-			}
-
 			// This matches null, nil (as null), and any non-null items.
-			l.Lock()
-			found := n.Search(ctx, path, val)
-			matched = append(matched, found...)
-			l.Unlock()
+			n.Search(ctx, path, res[0], result)
 
 			return nil
 		})
 	}
 
-	return matched, pool.Wait()
+	return pool.Wait()
 }
 
 // Search returns all ExpressionParts which match the given input, ignoring the variable name
 // entirely.
-func (n *numbers) Search(ctx context.Context, variable string, input any) (matched []*StoredExpressionPart) {
-	n.lock.RLock()
-	defer n.lock.RUnlock()
-
-	// initialize matched
-	matched = []*StoredExpressionPart{}
-
+func (n *numbers) Search(ctx context.Context, variable string, input any, result *MatchResult) {
 	var val float64
 
 	switch v := input.(type) {
@@ -105,7 +81,7 @@ func (n *numbers) Search(ctx context.Context, variable string, input any) (match
 	case float64:
 		val = v
 	default:
-		return nil
+		return
 	}
 
 	// First, find exact matches.
@@ -115,8 +91,9 @@ func (n *numbers) Search(ctx context.Context, variable string, input any) (match
 			if m.Ident != nil && *m.Ident != variable {
 				continue
 			}
-			// This is a candidatre.
-			matched = append(matched, m)
+
+			// This is a candidate.
+			result.AddExprs(m)
 		}
 	}
 
@@ -131,8 +108,8 @@ func (n *numbers) Search(ctx context.Context, variable string, input any) (match
 			if m.Ident != nil && *m.Ident != variable {
 				continue
 			}
-			// This is a candidatre.
-			matched = append(matched, m)
+			// This is a candidate.
+			result.AddExprs(m)
 		}
 		return true
 	})
@@ -148,13 +125,11 @@ func (n *numbers) Search(ctx context.Context, variable string, input any) (match
 			if m.Ident != nil && *m.Ident != variable {
 				continue
 			}
-			// This is a candidatre.
-			matched = append(matched, m)
+			// This is a candidate.
+			result.AddExprs(m)
 		}
 		return true
 	})
-
-	return matched
 }
 
 func (n *numbers) Add(ctx context.Context, p ExpressionPart) error {