Support regex: and expr: selectors for dimension and measure names

proto/gen/rill/runtime/v1/runtime.swagger.yaml

@@ -3990,22 +3990,18 @@ definitions:
         type: array
         items:
           type: string
-        title: Dimensions to show
-      dimensionsExclude:
-        type: boolean
-        description: |-
-          If true, the `dimensions` will be inverted during validation to include all dimensions except the ones listed.
-          Since it is processed during validation, this will always be false in `state.valid_spec`.
+        description: Dimensions to show. If `dimensions_selector` is set, this will only be set in `state.valid_spec`.
+      dimensionsSelector:
+        $ref: '#/definitions/v1FieldSelector'
+        description: Dynamic selector for `dimensions`. Will be processed during validation, so it will always be empty in `state.valid_spec`.
       measures:
         type: array
         items:
           type: string
-        title: Measures to show
-      measuresExclude:
-        type: boolean
-        description: |-
-          If true, `measures` will be inverted during validation to include all measures except the ones listed.
-          Since it is processed during validation, this will always be false in `state.valid_spec`.
+        description: Measures to show. If `measures_selector` is set, this will only be set in `state.valid_spec`.
+      measuresSelector:
+        $ref: '#/definitions/v1FieldSelector'
+        description: Dynamic selector for `measures`. Will be processed during validation, so it will always be empty in `state.valid_spec`.
       timeRange:
         type: string
         description: |-
@@ -4034,22 +4030,18 @@ definitions:
         type: array
         items:
           type: string
-        description: Dimensions to show.
-      dimensionsExclude:
-        type: boolean
-        description: |-
-          If true, `dimensions` will be inverted during validation to include all dimensions except the ones listed.
-          Since it is processed during validation, this will always be false in `state.valid_spec`.
+        description: Dimensions to show. If `dimensions_selector` is set, this will only be set in `state.valid_spec`.
+      dimensionsSelector:
+        $ref: '#/definitions/v1FieldSelector'
+        description: Dynamic selector for `dimensions`. Will be processed during validation, so it will always be empty in `state.valid_spec`.
       measures:
         type: array
         items:
           type: string
-        title: Measures available
-      measuresExclude:
-        type: boolean
-        description: |-
-          If true, `measures` will be inverted during validation to include all measures except the ones listed.
-          Since it is processed during validation, this will always be false in `state.valid_spec`.
+        description: Measures to show. If `measures_selector` is set, this will only be set in `state.valid_spec`.
+      measuresSelector:
+        $ref: '#/definitions/v1FieldSelector'
+        description: Dynamic selector for `measures`. Will be processed during validation, so it will always be empty in `state.valid_spec`.
       theme:
         type: string
         title: Theme to use
@@ -4125,6 +4117,27 @@ definitions:
         $ref: '#/definitions/v1Condition'
       subquery:
         $ref: '#/definitions/v1Subquery'
+  v1FieldSelector:
+    type: object
+    properties:
+      invert:
+        type: boolean
+        description: Invert the result such that all fields *except* the selected fields are returned.
+      all:
+        type: boolean
+        description: Select all fields.
+      fields:
+        $ref: '#/definitions/v1StringListValue'
+        description: Select specific fields by name.
+      regex:
+        type: string
+        description: Select fields by a regular expression.
+      duckdbExpression:
+        type: string
+        description: Select fields by a DuckDB SQL SELECT expression. For example "* EXCLUDE (city)".
+    description: |-
+      FieldSelector describes logic for selecting a list of fields.
+      It is useful for dynamically evaluating fields when the list of potential fields is not known at parse time.
   v1FileEvent:
     type: string
     enum:
@@ -5951,6 +5964,13 @@ definitions:
         $ref: '#/definitions/v1SourceSpec'
       state:
         $ref: '#/definitions/v1SourceState'
+  v1StringListValue:
+    type: object
+    properties:
+      values:
+        type: array
+        items:
+          type: string
   v1StructType:
     type: object
     properties:

proto/rill/runtime/v1/resources.proto

@@ -329,16 +329,14 @@ message ExploreSpec {
   string description = 2;
   // The metrics view the explore is based on
   string metrics_view = 3;
-  // Dimensions to show.
+  // Dimensions to show. If `dimensions_selector` is set, this will only be set in `state.valid_spec`.
   repeated string dimensions = 4;
-  // If true, `dimensions` will be inverted during validation to include all dimensions except the ones listed.
-  // Since it is processed during validation, this will always be false in `state.valid_spec`.
-  bool dimensions_exclude = 5;
-  // Measures available
+  // Dynamic selector for `dimensions`. Will be processed during validation, so it will always be empty in `state.valid_spec`.
+  FieldSelector dimensions_selector = 13;
+  // Measures to show. If `measures_selector` is set, this will only be set in `state.valid_spec`.
   repeated string measures = 6;
-  // If true, `measures` will be inverted during validation to include all measures except the ones listed.
-  // Since it is processed during validation, this will always be false in `state.valid_spec`.
-  bool measures_exclude = 7;
+  // Dynamic selector for `measures`. Will be processed during validation, so it will always be empty in `state.valid_spec`.
+  FieldSelector measures_selector = 14;
   // Theme to use
   string theme = 8;
   // List of selectable time ranges with comparison time ranges.
@@ -378,16 +376,14 @@ message ExploreComparisonTimeRange {
 message ExplorePreset {
   // Display name of the preset
   string label = 1;
-  // Dimensions to show
+  // Dimensions to show. If `dimensions_selector` is set, this will only be set in `state.valid_spec`.
   repeated string dimensions = 2;
-  // If true, the `dimensions` will be inverted during validation to include all dimensions except the ones listed.
-  // Since it is processed during validation, this will always be false in `state.valid_spec`.
-  bool dimensions_exclude = 3;
-  // Measures to show
+  // Dynamic selector for `dimensions`. Will be processed during validation, so it will always be empty in `state.valid_spec`.
+  FieldSelector dimensions_selector = 9;
+  // Measures to show. If `measures_selector` is set, this will only be set in `state.valid_spec`.
   repeated string measures = 4;
-  // If true, `measures` will be inverted during validation to include all measures except the ones listed.
-  // Since it is processed during validation, this will always be false in `state.valid_spec`.
-  bool measures_exclude = 5;
+  // Dynamic selector for `measures`. Will be processed during validation, so it will always be empty in `state.valid_spec`.
+  FieldSelector measures_selector = 10;
   // Time range for the explore.
   // It corresponds to the `range` property of the explore's `time_ranges`.
   // If not found in `time_ranges`, it should be added to the list.
@@ -405,6 +401,27 @@ enum ExploreComparisonMode {
   EXPLORE_COMPARISON_MODE_DIMENSION = 3;
 }
 
+// FieldSelector describes logic for selecting a list of fields.
+// It is useful for dynamically evaluating fields when the list of potential fields is not known at parse time.
+message FieldSelector {
+  // Invert the result such that all fields *except* the selected fields are returned.
+  bool invert = 1;
+  oneof selector {
+    // Select all fields.
+    bool all = 2;
+    // Select specific fields by name.
+    StringListValue fields = 3;
+    // Select fields by a regular expression.
+    string regex = 4;
+    // Select fields by a DuckDB SQL SELECT expression. For example "* EXCLUDE (city)".
+    string duckdb_expression = 5;
+  }
+}
+
+message StringListValue {
+  repeated string values = 1;
+}
+
 message Migration {
   MigrationSpec spec = 1;
   MigrationState state = 2;

runtime/compilers/rillv1/parse_explore.go

@@ -16,84 +16,22 @@ type ExploreYAML struct {
 	Title       string                 `yaml:"title"`
 	Description string                 `yaml:"description"`
 	MetricsView string                 `yaml:"metrics_view"`
-	Dimensions  *NamesYAML             `yaml:"dimensions"`
-	Measures    *NamesYAML             `yaml:"measures"`
+	Dimensions  *FieldSelectorYAML     `yaml:"dimensions"`
+	Measures    *FieldSelectorYAML     `yaml:"measures"`
 	Theme       string                 `yaml:"theme"`
 	TimeRanges  []ExploreTimeRangeYAML `yaml:"time_ranges"`
 	TimeZones   []string               `yaml:"time_zones"`
 	Presets     []*struct {
-		Label               string     `yaml:"label"`
-		Dimensions          *NamesYAML `yaml:"dimensions"`
-		Measures            *NamesYAML `yaml:"measures"`
-		TimeRange           string     `yaml:"time_range"`
-		ComparisonMode      string     `yaml:"comparison_mode"`
-		ComparisonDimension string     `yaml:"comparison_dimension"`
+		Label               string             `yaml:"label"`
+		Dimensions          *FieldSelectorYAML `yaml:"dimensions"`
+		Measures            *FieldSelectorYAML `yaml:"measures"`
+		TimeRange           string             `yaml:"time_range"`
+		ComparisonMode      string             `yaml:"comparison_mode"`
+		ComparisonDimension string             `yaml:"comparison_dimension"`
 	} `yaml:"presets"`
 	Security *SecurityPolicyYAML `yaml:"security"`
 }
 
-// NamesYAML parses a list of names with support for a '*' scalar for all names,
-// and support for a nested "exclude:" list for selecting all except the listed names.
-//
-// Note that '*' is represented by setting Exclude to true and leaving Names nil.
-// (Because excluding nothing is the same as including everything.)
-type NamesYAML struct {
-	Names   []string
-	Exclude bool
-}
-
-func (y *NamesYAML) UnmarshalYAML(v *yaml.Node) error {
-	if v == nil {
-		return nil
-	}
-	switch v.Kind {
-	case yaml.ScalarNode:
-		if v.Value == "*" {
-			y.Exclude = true
-			return nil
-		}
-		return fmt.Errorf("unexpected scalar %q", v.Value)
-	case yaml.SequenceNode:
-		y.Names = make([]string, len(v.Content))
-		for i, c := range v.Content {
-			if c.Kind != yaml.ScalarNode {
-				return fmt.Errorf("unexpected non-string list entry on line %d", c.Line)
-			}
-			y.Names[i] = c.Value
-		}
-	case yaml.MappingNode:
-		tmp := &struct {
-			Exclude yaml.Node `yaml:"exclude"`
-		}{}
-		err := v.Decode(tmp)
-		if err != nil {
-			return err
-		}
-		if tmp.Exclude.IsZero() {
-			return errors.New("expected '*', list of names, or `exclude` field")
-		}
-
-		// Exclude should also be '*' or a list of names.
-		// For simpliciy, we can just recurse on it and invert the result.
-		err = y.UnmarshalYAML(&tmp.Exclude)
-		if err != nil {
-			return fmt.Errorf("error parsing `exclude` field: %w", err)
-		}
-		y.Exclude = !y.Exclude // Oh the irony
-	default:
-		return fmt.Errorf("expected '*', list of names, or `exclude` field, got type %q", v.Kind)
-	}
-	return nil
-}
-
-func (y *NamesYAML) Safe() NamesYAML {
-	if y == nil {
-		// If not specified, default to '*' (include all).
-		return NamesYAML{Names: nil, Exclude: true}
-	}
-	return *y
-}
-
 // ExploreTimeRangeYAML represents a time range in an ExploreYAML.
 // It has a custom parser to support a mixed scalar and mapping structure.
 // Example:
@@ -194,6 +132,18 @@ func (p *Parser) parseExplore(node *Node) error {
 	}
 	node.Refs = append(node.Refs, ResourceName{Kind: ResourceKindMetricsView, Name: tmp.MetricsView})
 
+	// Parse the dimensions and measures selectors
+	var dimensionsSelector *runtimev1.FieldSelector
+	dimensions, ok := tmp.Dimensions.TryResolve()
+	if !ok {
+		dimensionsSelector = tmp.Dimensions.Proto()
+	}
+	var measuresSelector *runtimev1.FieldSelector
+	measures, ok := tmp.Measures.TryResolve()
+	if !ok {
+		measuresSelector = tmp.Measures.Proto()
+	}
+
 	// Add theme to refs
 	if tmp.Theme != "" {
 		node.Refs = append(node.Refs, ResourceName{Kind: ResourceKindTheme, Name: tmp.Theme})
@@ -257,12 +207,24 @@ func (p *Parser) parseExplore(node *Node) error {
 			return errors.New("can only set comparison_dimension when comparison_mode is 'dimension'")
 		}
 
+		var presetDimensionsSelector *runtimev1.FieldSelector
+		presetDimensions, ok := p.Dimensions.TryResolve()
+		if !ok {
+			presetDimensionsSelector = p.Dimensions.Proto()
+		}
+
+		var presetMeasuresSelector *runtimev1.FieldSelector
+		presetMeasures, ok := p.Measures.TryResolve()
+		if !ok {
+			presetMeasuresSelector = p.Measures.Proto()
+		}
+
 		presets = append(presets, &runtimev1.ExplorePreset{
 			Label:               p.Label,
-			Dimensions:          p.Dimensions.Safe().Names,
-			DimensionsExclude:   p.Dimensions.Safe().Exclude,
-			Measures:            p.Measures.Safe().Names,
-			MeasuresExclude:     p.Measures.Safe().Exclude,
+			Dimensions:          presetDimensions,
+			DimensionsSelector:  presetDimensionsSelector,
+			Measures:            presetMeasures,
+			MeasuresSelector:    presetMeasuresSelector,
 			TimeRange:           p.TimeRange,
 			ComparisonMode:      mode,
 			ComparisonDimension: p.ComparisonDimension,
@@ -290,10 +252,10 @@ func (p *Parser) parseExplore(node *Node) error {
 	r.ExploreSpec.Title = tmp.Title
 	r.ExploreSpec.Description = tmp.Description
 	r.ExploreSpec.MetricsView = tmp.MetricsView
-	r.ExploreSpec.Dimensions = tmp.Dimensions.Safe().Names
-	r.ExploreSpec.DimensionsExclude = tmp.Dimensions.Safe().Exclude
-	r.ExploreSpec.Measures = tmp.Measures.Safe().Names
-	r.ExploreSpec.MeasuresExclude = tmp.Measures.Safe().Exclude
+	r.ExploreSpec.Dimensions = dimensions
+	r.ExploreSpec.DimensionsSelector = dimensionsSelector
+	r.ExploreSpec.Measures = measures
+	r.ExploreSpec.MeasuresSelector = measuresSelector
 	r.ExploreSpec.Theme = tmp.Theme
 	r.ExploreSpec.TimeRanges = timeRanges
 	r.ExploreSpec.TimeZones = tmp.TimeZones

runtime/compilers/rillv1/parse_metrics_view.go

@@ -835,11 +835,11 @@ func (p *Parser) parseMetricsView(node *Node) error {
 	for _, dim := range spec.Dimensions {
 		e.ExploreSpec.Dimensions = append(e.ExploreSpec.Dimensions, dim.Name)
 	}
-	e.ExploreSpec.DimensionsExclude = false
+	e.ExploreSpec.DimensionsSelector = nil
 	for _, m := range spec.Measures {
 		e.ExploreSpec.Measures = append(e.ExploreSpec.Measures, m.Name)
 	}
-	e.ExploreSpec.MeasuresExclude = false
+	e.ExploreSpec.MeasuresSelector = nil
 	e.ExploreSpec.Theme = spec.DefaultTheme
 	for _, tr := range tmp.AvailableTimeRanges {
 		res := &runtimev1.ExploreTimeRange{Range: tr.Range}
@@ -864,12 +864,20 @@ func (p *Parser) parseMetricsView(node *Node) error {
 	case runtimev1.MetricsViewSpec_COMPARISON_MODE_DIMENSION:
 		exploreComparisonMode = runtimev1.ExploreComparisonMode_EXPLORE_COMPARISON_MODE_DIMENSION
 	}
+
+	var presetDimensionsSelector, presetMeasuresSelector *runtimev1.FieldSelector
+	if len(spec.DefaultDimensions) == 0 {
+		presetDimensionsSelector = &runtimev1.FieldSelector{Selector: &runtimev1.FieldSelector_All{All: true}}
+	}
+	if len(spec.DefaultMeasures) == 0 {
+		presetMeasuresSelector = &runtimev1.FieldSelector{Selector: &runtimev1.FieldSelector_All{All: true}}
+	}
 	e.ExploreSpec.Presets = []*runtimev1.ExplorePreset{{
 		Label:               "Default",
 		Dimensions:          spec.DefaultDimensions,
-		DimensionsExclude:   len(spec.DefaultDimensions) == 0,
+		DimensionsSelector:  presetDimensionsSelector,
 		Measures:            spec.DefaultMeasures,
-		MeasuresExclude:     len(spec.DefaultMeasures) == 0,
+		MeasuresSelector:    presetMeasuresSelector,
 		TimeRange:           spec.DefaultTimeRange,
 		ComparisonMode:      exploreComparisonMode,
 		ComparisonDimension: spec.DefaultComparisonDimension,