From 802985a230159269582c1ac7333565d39fbf4c85 Mon Sep 17 00:00:00 2001 From: Hussain Sultan Date: Sun, 23 Aug 2026 22:13:38 -0400 Subject: [PATCH 1/2] feat: refuse raw output from definition-side semantic expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A semantic model is a definition, not a query. Executing one returned the raw underlying table — every physical column at row grain, undeclared columns included, computed dimensions missing — through every output sink (execute/sql/to_pandas/to_csv/...) and every pre-aggregation chain (filters, joins, order_by/limit, group_by without aggregate, empty query()). Nothing about that result was semantic. Output now requires a completed query: - All 14 output sinks on SemanticTable call _ensure_executable(), which raises QueryError with the intended spellings (group_by/aggregate, query(), or .to_untagged() for explicit raw access) unless the chain contains an aggregation stage, an index, or a materialized result model. - group_by without aggregate gets its own error pointing at .aggregate() (an empty .aggregate() returns distinct grouped values). - query() with neither dimensions nor measures fails eagerly. - .mutate() chained after filter/order_by/limit on an aggregate raises: a query result is a plain table, so row math over it is spelled .to_untagged().mutate(...); .mutate() directly on the aggregate (the measure-path desugar) and calc measures remain the semantic spellings. The _build_post_aggregate_model wrapper and the now-unreachable non-additive-total refusal machinery are removed. - SemanticTableOp gains _materialized_result to distinguish result models (compare_periods output, as_table() over an aggregate — executable) from definition models; the flag survives with_dimensions/with_measures, filter-preserving rebuilds, and root-derived models. - Agent help topics and the four bsl-query-expert doc copies now teach window functions via .to_untagged() and document the two new gotchas; ADR 0001 carries an amendment note. test_execute_guard.py pins the contract (30 tests). Existing tests that inspected raw rows now do so via the explicit .to_untagged() escape hatch. Co-Authored-By: Claude Fable 5 --- docs/adr/0001-drop-semantic-mutate-op.md | 1 + .../query/langchain/tool-query-model.md | 8 +- .../claude-code/bsl-query-expert/SKILL.md | 8 +- docs/md/skills/codex/bsl-query-expert.codex | 8 +- docs/md/skills/cursor/bsl-query-expert.mdc | 8 +- .../agents/help_topics.py | 43 ++-- src/boring_semantic_layer/expr.py | 198 ++++++++++----- src/boring_semantic_layer/measure_scope.py | 45 ---- src/boring_semantic_layer/ops/__init__.py | 2 - src/boring_semantic_layer/ops/_core.py | 81 +----- src/boring_semantic_layer/query.py | 21 +- .../soundness/test_join_lineage_and_grain.py | 6 +- .../tests/test_dimension_validation.py | 5 +- .../tests/test_execute_guard.py | 239 ++++++++++++++++++ .../tests/test_mutate_compositions.py | 55 ++-- src/boring_semantic_layer/tests/test_query.py | 10 +- .../tests/test_stale_dimensions.py | 2 +- .../tests/test_totals_semantics.py | 57 ++++- .../tests/test_unnest.py | 13 +- 19 files changed, 538 insertions(+), 272 deletions(-) create mode 100644 src/boring_semantic_layer/tests/test_execute_guard.py diff --git a/docs/adr/0001-drop-semantic-mutate-op.md b/docs/adr/0001-drop-semantic-mutate-op.md index 5d20ae5e..74c760c5 100644 --- a/docs/adr/0001-drop-semantic-mutate-op.md +++ b/docs/adr/0001-drop-semantic-mutate-op.md @@ -1,6 +1,7 @@ # ADR 0001: Unify calculated measures and post-aggregation `mutate` on a single ibis-expression primitive - **Status:** Implemented — Phases 1+2 landed on `hussain/feat/calc-measure-analyzer` (merged to `main`); Phase 3 landed on `hussain/feat/drop-semantic-mutate-op-phase3`. +- **Amended 2026-08-23:** the base-class fallback that materialized `.mutate()` chained after `filter`/`order_by`/`limit` on an aggregate (the `_build_post_aggregate_model` wrapper) was removed. That spelling now raises `QueryError`: a query result is a plain table, so row math over it is spelled `.to_untagged().mutate(...)`, while `.mutate()` directly on the aggregate (the measure-path desugar this ADR describes) and calc measures on the model remain the semantic spellings. In the same change, output sinks (`execute`/`sql`/`to_pandas`/…) began refusing definition-side expressions (bare models, filters, joins, `group_by` without `aggregate`) — see `test_execute_guard.py`. - **Date:** 2026-05-08 (revised 2026-05-09 to reflect landed work; revised 2026-06-10 for Phase 3 completion) - **Deciders:** BSL maintainers - **Related code (current state):** `src/boring_semantic_layer/calc_analyzer.py` (new), `src/boring_semantic_layer/calc_compiler.py` (new), `src/boring_semantic_layer/nested_compile.py` (new — extracted from deleted `compile_all.py`), `src/boring_semantic_layer/ops.py` (`CalcMeasure`, `_classify_measure`, `_build_aggregation_plan`, `_compile_aggregation`, `_apply_calc_specs`; `SemanticMutateOp` — deleted in Phase 3), `src/boring_semantic_layer/expr.py` (`SemanticMutate` deleted; `.mutate()` survives as a desugaring alias), `src/boring_semantic_layer/measure_scope.py` (`MeasureScope`/`ColumnScope` thin proxies; curated AST removed), `src/boring_semantic_layer/serialization/extract.py` (resolver-tree calc serialization), `src/boring_semantic_layer/tests/test_mutate_compositions.py` (Phase 3 composition pins). diff --git a/docs/md/prompts/query/langchain/tool-query-model.md b/docs/md/prompts/query/langchain/tool-query-model.md index 9fbf4575..5ac768d3 100644 --- a/docs/md/prompts/query/langchain/tool-query-model.md +++ b/docs/md/prompts/query/langchain/tool-query-model.md @@ -89,11 +89,11 @@ model.group_by("category").aggregate("revenue").order_by(ibis.desc("revenue")).l **CRITICAL**: `.limit()` in query limits data **before** calculations. Use `limit` parameter for display-only limiting. ## Window Functions -`.mutate()` for post-aggregation transforms - **MUST** come after `.order_by()`: +Windows run over the query result — drop to ibis first with `.to_untagged()`; the window carries its own ordering. (`.mutate()` after `.order_by()`/`.limit()`/`.filter()` on a result raises.) ```python -model.group_by("week").aggregate("count").order_by("week").mutate( - rolling_avg=lambda t: t.count.mean().over(ibis.window(rows=(-9, 0), order_by="week")) -) +model.group_by("week").aggregate("count").to_untagged().mutate( + rolling_avg=lambda t: t["count"].mean().over(rows=(-9, 0), order_by="week") +).order_by("week") ``` **More**: `get_documentation(topic="windowing")` diff --git a/docs/md/skills/claude-code/bsl-query-expert/SKILL.md b/docs/md/skills/claude-code/bsl-query-expert/SKILL.md index d9e6e989..20c765c3 100644 --- a/docs/md/skills/claude-code/bsl-query-expert/SKILL.md +++ b/docs/md/skills/claude-code/bsl-query-expert/SKILL.md @@ -204,11 +204,11 @@ model.group_by("category").aggregate("revenue").order_by(ibis.desc("revenue")).l **CRITICAL**: `.limit()` in query limits data **before** calculations. Use `limit` parameter for display-only limiting. ## Window Functions -`.mutate()` for post-aggregation transforms - **MUST** come after `.order_by()`: +Windows run over the query result — drop to ibis first with `.to_untagged()`; the window carries its own ordering. (`.mutate()` after `.order_by()`/`.limit()`/`.filter()` on a result raises.) ```python -model.group_by("week").aggregate("count").order_by("week").mutate( - rolling_avg=lambda t: t.count.mean().over(ibis.window(rows=(-9, 0), order_by="week")) -) +model.group_by("week").aggregate("count").to_untagged().mutate( + rolling_avg=lambda t: t["count"].mean().over(rows=(-9, 0), order_by="week") +).order_by("week") ``` **More**: `get_documentation(topic="windowing")` diff --git a/docs/md/skills/codex/bsl-query-expert.codex b/docs/md/skills/codex/bsl-query-expert.codex index 5ac572fd..44a185cd 100644 --- a/docs/md/skills/codex/bsl-query-expert.codex +++ b/docs/md/skills/codex/bsl-query-expert.codex @@ -203,11 +203,11 @@ model.group_by("category").aggregate("revenue").order_by(ibis.desc("revenue")).l **CRITICAL**: `.limit()` in query limits data **before** calculations. Use `limit` parameter for display-only limiting. ## Window Functions -`.mutate()` for post-aggregation transforms - **MUST** come after `.order_by()`: +Windows run over the query result — drop to ibis first with `.to_untagged()`; the window carries its own ordering. (`.mutate()` after `.order_by()`/`.limit()`/`.filter()` on a result raises.) ```python -model.group_by("week").aggregate("count").order_by("week").mutate( - rolling_avg=lambda t: t.count.mean().over(ibis.window(rows=(-9, 0), order_by="week")) -) +model.group_by("week").aggregate("count").to_untagged().mutate( + rolling_avg=lambda t: t["count"].mean().over(rows=(-9, 0), order_by="week") +).order_by("week") ``` **More**: `get_documentation(topic="windowing")` diff --git a/docs/md/skills/cursor/bsl-query-expert.mdc b/docs/md/skills/cursor/bsl-query-expert.mdc index d57c5c9c..ca2d4afc 100644 --- a/docs/md/skills/cursor/bsl-query-expert.mdc +++ b/docs/md/skills/cursor/bsl-query-expert.mdc @@ -205,11 +205,11 @@ model.group_by("category").aggregate("revenue").order_by(ibis.desc("revenue")).l **CRITICAL**: `.limit()` in query limits data **before** calculations. Use `limit` parameter for display-only limiting. ## Window Functions -`.mutate()` for post-aggregation transforms - **MUST** come after `.order_by()`: +Windows run over the query result — drop to ibis first with `.to_untagged()`; the window carries its own ordering. (`.mutate()` after `.order_by()`/`.limit()`/`.filter()` on a result raises.) ```python -model.group_by("week").aggregate("count").order_by("week").mutate( - rolling_avg=lambda t: t.count.mean().over(ibis.window(rows=(-9, 0), order_by="week")) -) +model.group_by("week").aggregate("count").to_untagged().mutate( + rolling_avg=lambda t: t["count"].mean().over(rows=(-9, 0), order_by="week") +).order_by("week") ``` **More**: `get_documentation(topic="windowing")` diff --git a/src/boring_semantic_layer/agents/help_topics.py b/src/boring_semantic_layer/agents/help_topics.py index f1533853..ff98b1af 100644 --- a/src/boring_semantic_layer/agents/help_topics.py +++ b/src/boring_semantic_layer/agents/help_topics.py @@ -312,34 +312,39 @@ KEY DIFFERENCE: .aggregate() computes from raw data - .mutate() transforms already-aggregated results""", + .mutate() folds into the same aggregation (measure path) + +RULES: + - .mutate() goes directly on the aggregate, BEFORE .order_by()/.limit() + - .mutate() after .filter()/.order_by()/.limit() raises — the result of a + query is a plain table; use .to_untagged().mutate(...) for row math there""", }, "window": { "summary": "Window functions (rolling, cumulative, rank)", "content": """\ Window Functions — calculations across ordered rows -Must come AFTER .order_by(), applied via .mutate(): +Windows run over the QUERY RESULT, so drop to ibis first with +.to_untagged(), then use .mutate() with the window carrying its own +ordering: ROLLING AVERAGE: - model.group_by("week").aggregate("count").order_by("week").mutate( - rolling_avg=lambda t: t.count.mean().over( - ibis.window(rows=(-9, 0), order_by="week") + model.group_by("week").aggregate("count").to_untagged().mutate( + rolling_avg=lambda t: t["count"].mean().over( + rows=(-9, 0), order_by="week" ) - ) + ).order_by("week") CUMULATIVE SUM: - .mutate(running_total=lambda t: t.revenue.cumsum()) + .to_untagged().mutate(running_total=lambda t: t.revenue.cumsum()) RANK: - .mutate(rank=lambda t: ibis.rank().over( - ibis.window(order_by=ibis.desc(t.revenue)) - )) + .to_untagged().mutate(rank=lambda t: t.revenue.rank()) LAG / LEAD: - .mutate( - prev_count=lambda t: t.count.lag(1), - next_count=lambda t: t.count.lead(1), + .to_untagged().mutate( + prev_count=lambda t: t["count"].lag(1), + next_count=lambda t: t["count"].lead(1), ) See: bsl docs query mutate""", @@ -690,9 +695,15 @@ model.limit(10).group_by("x").aggregate("y") # WRONG — limits raw data model.group_by("x").aggregate("y").limit(10) # CORRECT — limits results -10. WINDOW WITHOUT ORDER_BY: - .mutate(avg=lambda t: t.x.mean().over(window)) # WRONG — no ordering - .order_by("date").mutate(avg=...) # CORRECT — ordered first""", +10. EXECUTING A MODEL WITHOUT A QUERY: + model.execute() # WRONG — a model is a definition, raises + model.group_by("x").aggregate("y").execute() # CORRECT — execute a query + model.to_untagged().execute() # CORRECT — explicit raw rows + +11. MUTATE AFTER ORDER_BY/LIMIT/FILTER ON A RESULT: + .order_by("date").mutate(avg=...) # WRONG — result is a plain table, raises + .mutate(avg=...).order_by("date") # CORRECT — mutate the aggregate first + .order_by("date").to_untagged().mutate(avg=...) # CORRECT — row math via ibis""", }, "examples": { "summary": "Quick recipes for common patterns", diff --git a/src/boring_semantic_layer/expr.py b/src/boring_semantic_layer/expr.py index a7c88623..760d3a54 100644 --- a/src/boring_semantic_layer/expr.py +++ b/src/boring_semantic_layer/expr.py @@ -16,6 +16,7 @@ GroupedTable, Table, ) +from .errors import QueryError from .measure_scope import MeasureScope from .ops import ( Dimension, @@ -38,6 +39,7 @@ _extract_join_key_columns, _find_all_root_models, _get_merged_fields, + _has_prior_aggregate, _is_deferred, _normalize_join_predicate, _normalize_to_name, @@ -132,6 +134,60 @@ def to_untagged(expr): raise TypeError(f"Cannot convert {type(expr)} to Ibis expression") +def _ensure_executable(expr: SemanticTable) -> None: + """Refuse to sink a definition-side semantic expression to output. + + A semantic model — and any pre-aggregation chain over it (filters, + joins, order_by/limit) — is a definition, not a query. Sinking it to + output would return the raw underlying table: every physical column at + row grain, with computed dimensions missing, bypassing the semantic + layer entirely. Output requires either an aggregation stage in the + chain or a model explicitly materialized from a query result + (``_materialized_result``); everything else raises ``QueryError`` + pointing at the intended spellings. + """ + node = expr.op() + while node is not None: + if isinstance(node, SemanticAggregateOp | SemanticIndexOp): + # An aggregate completes a query; an index IS a query result + # (fieldName/fieldValue/weight summary rows). + return + if isinstance(node, SemanticGroupByOp): + keys = ", ".join(repr(k) for k in node.keys) + raise QueryError( + f".group_by({keys}) has no aggregation yet — complete the query " + "with .aggregate(...) (an empty .aggregate() returns the " + "distinct grouped values)." + ) + if isinstance(node, SemanticTableOp): + if node._materialized_result: + return + break + if isinstance(node, SemanticJoinOp): + break + node = getattr(node, "source", None) + + name = getattr(expr, "name", None) + label = f"Semantic model {name!r}" if name else "This semantic expression" + dims = sorted(expr.get_dimensions()) if hasattr(expr, "get_dimensions") else [] + meas = ( + sorted({*expr.get_measures(), *expr.get_calculated_measures()}) + if hasattr(expr, "get_measures") + else [] + ) + dim_hint = repr(dims[0]) if dims else "..." + meas_hint = repr(meas[0]) if meas else "..." + raise QueryError( + f"{label} is a definition, not a query — executing it would return the " + "raw underlying table (every physical column at row grain), bypassing " + "its dimensions and measures. Build a query first, e.g. " + f".group_by({dim_hint}).aggregate({meas_hint}) or " + ".query(dimensions=[...], measures=[...]), or call .to_untagged() to " + "work with the raw ibis table explicitly." + + (f" Declared dimensions: {dims}; measures: {meas}." if dims or meas else "") + ) + + def _flatten_group_keys(keys: tuple) -> tuple: """Flatten list/tuple arguments so ``group_by(["a", "b"])`` works like ibis.""" flat: list = [] @@ -346,36 +402,30 @@ def aggregate(self, *measure_names, nest: dict[str, Callable] | None = None, **a agg = aggregate def mutate(self, **post): - """Add derived columns (ADR 0001: desugars to the unified measure path). - - Pre-aggregation, the derivations are row-grain expressions — - they register as dimensions on the model (usable as group-by - keys and in downstream queries). Post-aggregation (chained after - ``filter``/``order_by``/``limit`` on an aggregate), the - derivations are resolved against the current result table in - chain order and materialized, preserving the historical - ``.mutate()`` semantics without a dedicated operator node. + """Add derived row-grain columns (ADR 0001: desugars to dimensions). + + The derivations are row-grain expressions — they register as + dimensions on the model (usable as group-by keys and in downstream + queries). Post-aggregation derivations are not part of the semantic + layer: declare a calculated measure for cross-measure math, or drop + to ibis with ``.to_untagged().mutate(...)`` for row math over a + query result. """ from .ops import ( SemanticJoinOp, SemanticTableOp, - _has_prior_aggregate, - _non_additive_result_columns, _resolve_expr, ) if _has_prior_aggregate(self.op()): - tbl = self.op().to_untagged() - # Only the aggregated rows are available here, so t.all() can only - # window-sum them; pass which columns that would misrepresent. - non_additive = _non_additive_result_columns(self.op()) - for name, fn in post.items(): - proxy = MeasureScope( - _tbl=tbl, _known=[], _post_agg=True, _non_additive=non_additive - ) - resolved = _resolve_expr(fn, proxy) - tbl = tbl.mutate(resolved.name(name)) - return _build_post_aggregate_model(self.op(), tbl) + raise QueryError( + ".mutate() on a filtered/ordered/limited query result is not " + "supported: the result is a plain table, not a semantic model. " + "Call .mutate() on the aggregate itself (before " + "filter/order_by/limit — it desugars to the measure path), " + "declare a calculated measure on the model, or drop to ibis " + "with .to_untagged().mutate(...)." + ) def contains_join(node) -> bool: if isinstance(node, SemanticJoinOp): @@ -399,7 +449,7 @@ def contains_join(node) -> bool: return with_dims(**post) # Flat model: materialize the columns in chain order (so they are - # visible on direct execute) and register them as dimensions. + # visible on the untagged table) and register them as dimensions. tbl = self.op().to_untagged() for name, fn in post.items(): resolved = _resolve_expr(fn, tbl) @@ -446,49 +496,63 @@ def execute(self, **kwargs): # Accept kwargs for ibis compatibility (params, limit, etc) from .ops import _rebind_to_canonical_backend + _ensure_executable(self) return _rebind_to_canonical_backend(to_untagged(self)).execute(**kwargs) def compile(self, **kwargs): from .ops import _rebind_to_canonical_backend + _ensure_executable(self) return _rebind_to_canonical_backend(to_untagged(self)).compile(**kwargs) def sql(self, **kwargs): from .ops import _rebind_to_canonical_backend + _ensure_executable(self) return ibis.to_sql(_rebind_to_canonical_backend(to_untagged(self)), **kwargs) def to_pandas(self, **kwargs): + _ensure_executable(self) return self.to_untagged().to_pandas(**kwargs) def to_pyarrow(self, **kwargs): + _ensure_executable(self) return self.to_untagged().to_pyarrow(**kwargs) def to_pyarrow_batches(self, **kwargs): + _ensure_executable(self) return self.to_untagged().to_pyarrow_batches(**kwargs) def to_polars(self, **kwargs): + _ensure_executable(self) return self.to_untagged().to_polars(**kwargs) def to_csv(self, path, **kwargs): + _ensure_executable(self) return self.to_untagged().to_csv(path, **kwargs) def to_parquet(self, path, **kwargs): + _ensure_executable(self) return self.to_untagged().to_parquet(path, **kwargs) def to_parquet_dir(self, path, **kwargs): + _ensure_executable(self) return self.to_untagged().to_parquet_dir(path, **kwargs) def to_json(self, path, **kwargs): + _ensure_executable(self) return self.to_untagged().to_json(path, **kwargs) def to_xlsx(self, path, **kwargs): + _ensure_executable(self) return self.to_untagged().to_xlsx(path, **kwargs) def to_pandas_batches(self, **kwargs): + _ensure_executable(self) return self.to_untagged().to_pandas_batches(**kwargs) def to_sql(self, **kwargs): + _ensure_executable(self) return self.to_untagged().to_sql(**kwargs) @@ -587,13 +651,20 @@ def _build_semantic_model_from_roots( ibis_table: ir.Table, all_roots: tuple, field_filter: set | None = None, + materialized_result: bool = False, ) -> SemanticModel: + # A model derived from a materialized query result stays a result model + # (executable); a model derived from raw sources stays a definition. + materialized = materialized_result or any( + getattr(root, "_materialized_result", False) for root in all_roots + ) if not all_roots: return SemanticModel( table=ibis_table, dimensions={}, measures={}, calc_measures={}, + _materialized_result=materialized, ) all_dims = _get_merged_fields(all_roots, "dimensions") @@ -610,48 +681,10 @@ def _build_semantic_model_from_roots( dimensions=all_dims, measures=all_measures, calc_measures=all_calc, + _materialized_result=materialized, ) -def _build_post_aggregate_model(source_op, ibis_table: ir.Table) -> SemanticModel: - """Wrap a materialized post-aggregate table as a flat semantic model. - - A ``.mutate()`` chained after ``order_by``/``limit``/``filter`` on an - aggregate materializes the result (the new column is computed against - the post-wrapper table, preserving window/rank semantics). The base - model's dimensions reference *source* columns that no longer exist on - the aggregated table, so reattaching them (via roots) makes - ``schema``/``values`` raise when something — e.g. chart introspection — - forces field resolution. Instead expose the materialized columns - directly: group-by keys become identity dimensions (preserving time - metadata), every other column an identity measure. - """ - from .ops import Dimension, SemanticAggregateOp - - agg = source_op - while agg is not None and not isinstance(agg, SemanticAggregateOp): - agg = getattr(agg, "source", None) - key_names = set(agg.keys) if agg is not None else set() - - root_dims = _get_merged_fields(_find_all_root_models(source_op), "dimensions") - - dimensions: dict[str, Dimension] = {} - measures: dict[str, Callable] = {} - for col in ibis_table.columns: - if col in key_names: - rd = root_dims.get(col) - dimensions[col] = Dimension( - expr=(lambda t, c=col: t[c]), - is_time_dimension=getattr(rd, "is_time_dimension", False) if rd else False, - is_event_timestamp=getattr(rd, "is_event_timestamp", False) if rd else False, - smallest_time_grain=getattr(rd, "smallest_time_grain", None) if rd else None, - ) - else: - measures[col] = lambda t, c=col: t[c] - - return SemanticModel(table=ibis_table, dimensions=dimensions, measures=measures) - - def _get_entity_dims(op) -> frozenset[str]: """Return the logical entity grain carried by an operation. @@ -908,6 +941,7 @@ def _replace_metadata_preserving_filters( name=name, description=description, _source_join=source_join, + _materialized_result=getattr(source, "_materialized_result", False), ) for ( predicate, @@ -1033,6 +1067,7 @@ def __init__( name: str | None = None, description: str | None = None, _source_join: Any | None = None, + _materialized_result: bool = False, ) -> None: # Convert ibis → xorq once at the boundary; internal code paths can # then assume xorq-vendored tables when the backend is supported. @@ -1064,6 +1099,7 @@ def __init__( name=derived_name, description=description, _source_join=_source_join, + _materialized_result=_materialized_result, ) super().__init__(op) @@ -1130,6 +1166,7 @@ def with_dimensions(self, **dims) -> SemanticModel: name=self.name, description=self.description, _source_join=self.op()._source_join, + _materialized_result=self.op()._materialized_result, ) def with_measures(self, **meas) -> SemanticModel: @@ -1154,6 +1191,7 @@ def with_measures(self, **meas) -> SemanticModel: name=self.name, description=self.description, _source_join=self.op()._source_join, + _materialized_result=self.op()._materialized_result, ) def join_one( @@ -1426,7 +1464,11 @@ def json_definition(self): def as_table(self) -> SemanticModel: all_roots = _find_all_root_models(self.op()) - return _build_semantic_model_from_roots(self.op().to_untagged(), all_roots) + return _build_semantic_model_from_roots( + self.op().to_untagged(), + all_roots, + materialized_result=_has_prior_aggregate(self.op()), + ) def with_dimensions(self, **dims) -> SemanticModel: """Add or update dimensions.""" @@ -1578,7 +1620,11 @@ def calc_measures(self): def as_table(self) -> SemanticModel: all_roots = _find_all_root_models(self.op().source) - return _build_semantic_model_from_roots(self.op().to_untagged(), all_roots) + return _build_semantic_model_from_roots( + self.op().to_untagged(), + all_roots, + materialized_result=_has_prior_aggregate(self.op()), + ) def with_dimensions(self, **dims) -> SemanticTable: """Add or update dimensions while retaining filter/join lineage.""" @@ -2021,6 +2067,7 @@ def as_table(self) -> SemanticModel: dimensions={}, measures={}, calc_measures={}, + _materialized_result=True, ) @@ -2068,7 +2115,11 @@ def get_calculated_measures(self): def as_table(self) -> SemanticModel: all_roots = _find_all_root_models(self.source) - return _build_semantic_model_from_roots(self.op().to_untagged(), all_roots) + return _build_semantic_model_from_roots( + self.op().to_untagged(), + all_roots, + materialized_result=_has_prior_aggregate(self.op()), + ) class SemanticLimit(SemanticTable): @@ -2117,7 +2168,11 @@ def get_calculated_measures(self): def as_table(self) -> SemanticModel: all_roots = _find_all_root_models(self.source) - return _build_semantic_model_from_roots(self.op().to_untagged(), all_roots) + return _build_semantic_model_from_roots( + self.op().to_untagged(), + all_roots, + materialized_result=_has_prior_aggregate(self.op()), + ) class SemanticUnnest(SemanticTable): @@ -2162,7 +2217,11 @@ def get_calculated_measures(self): def as_table(self) -> SemanticModel: all_roots = _find_all_root_models(self.source) - return _build_semantic_model_from_roots(self.op().to_untagged(), all_roots) + return _build_semantic_model_from_roots( + self.op().to_untagged(), + all_roots, + materialized_result=_has_prior_aggregate(self.op()), + ) def with_dimensions(self, **dims) -> SemanticModel: all_roots = _find_all_root_models(self.source) @@ -2245,5 +2304,8 @@ def schema(self): def as_table(self) -> SemanticModel: all_roots = _find_all_root_models(self.source) return _build_semantic_model_from_roots( - self.op().to_untagged(), all_roots, field_filter=set(self.fields) + self.op().to_untagged(), + all_roots, + field_filter=set(self.fields), + materialized_result=_has_prior_aggregate(self.op()), ) diff --git a/src/boring_semantic_layer/measure_scope.py b/src/boring_semantic_layer/measure_scope.py index c7ebc82c..4ff55e94 100644 --- a/src/boring_semantic_layer/measure_scope.py +++ b/src/boring_semantic_layer/measure_scope.py @@ -116,15 +116,6 @@ class MeasureScope: converter=tuple, alias="_prefer_known", ) - #: Result columns whose window sum is *not* their overall value (means, - #: medians, distinct counts, calc measures). ``t.all()`` refuses these - #: rather than returning a sum with no meaning. Empty means "not - #: classified" — the historical behaviour applies. - non_additive: frozenset[str] = field( - factory=frozenset, - converter=frozenset, - alias="_non_additive", - ) def __attrs_post_init__(self): object.__setattr__(self, "known_set", frozenset(self.known)) @@ -166,53 +157,17 @@ def all(self, ref): from ._xorq import ibis as ibis_mod if isinstance(ref, str): - self._reject_non_additive(ref) return _float_total(self.tbl[ref].sum().over(ibis_mod.window())) if hasattr(ref, "__class__") and "ibis" in str(type(ref).__module__): if "Scalar" in type(ref).__name__: return ref.over(ibis_mod.window()) - self._reject_non_additive(_column_name_of(ref)) return _float_total(ref.sum().over(ibis_mod.window())) raise TypeError( "t.all(...) expects a string column name or an ibis expression", ) - def _reject_non_additive(self, name: str | None) -> None: - """Refuse a total this scope cannot compute correctly. - - Post-aggregation, the only rows available are the grouped ones, so - the total can only be a window sum over them. For a mean, median, - distinct count or ratio that sum is not the overall value, and - returning it silently produced answers that disagreed with the same - formula written as a calc measure. - """ - if name is None or name not in self.non_additive: - return - raise NonAdditiveTotalError( - f"t.all({name!r}) cannot be computed after aggregation: {name!r} is " - "not additive, so summing its per-group values is not its overall " - "value. Define the derivation on the model instead, where the total " - "is computed from the underlying rows:\n" - f" model.with_measures(share=lambda t: t.{name} / t.all(t.{name}))\n" - "or place the .mutate() directly on the aggregate (before " - "filter/order_by/limit), which routes through the same path." - ) - - -class NonAdditiveTotalError(ValueError): - """``t.all()`` was asked for a total that post-aggregation rows can't give.""" - - -def _column_name_of(ref) -> str | None: - """Best-effort column name for an ibis value expression.""" - try: - name = ref.get_name() - except Exception: - return None - return name if isinstance(name, str) else None - def _float_total(total): """Cast an integral total to float so ``measure / total`` is a true ratio. diff --git a/src/boring_semantic_layer/ops/__init__.py b/src/boring_semantic_layer/ops/__init__.py index 39611be3..a6585c20 100644 --- a/src/boring_semantic_layer/ops/__init__.py +++ b/src/boring_semantic_layer/ops/__init__.py @@ -38,7 +38,6 @@ _has_prior_aggregate, _is_deferred, _make_schema, - _non_additive_result_columns, _resolve_expr, _unwrap, make_bare_ref_lambda, @@ -80,7 +79,6 @@ "_has_prior_aggregate", "_is_deferred", "_make_schema", - "_non_additive_result_columns", "_normalize_join_predicate", "_normalize_to_name", "_rebind_to_backend", diff --git a/src/boring_semantic_layer/ops/_core.py b/src/boring_semantic_layer/ops/_core.py index cc652e4b..34a3bbf7 100644 --- a/src/boring_semantic_layer/ops/_core.py +++ b/src/boring_semantic_layer/ops/_core.py @@ -73,10 +73,6 @@ _allocate_temporary_join_names, _RenamedResolver, ) -from ._reductions import ( - _is_mean_expr, - _reagg_op_for_expr, -) from ._tracking import ( _extract_columns_from_callable, _extract_join_key_columns, @@ -962,6 +958,10 @@ class SemanticTableOp(Relation): _source_join: Any = field( default=None, repr=False ) # Track if this wraps a join (SemanticJoinOp) for optimization + # True when the wrapped table IS a query result (e.g. compare_periods + # output, as_table() over an aggregate) rather than a raw source table. + # Result models may be sunk to output; definition models may not. + _materialized_result: bool = False def __init__( self, @@ -972,6 +972,7 @@ def __init__( name: str | None = None, description: str | None = None, _source_join: Any = None, + _materialized_result: bool = False, ) -> None: # Accept both regular ibis and xorq tables without conversion # This allows using regular ibis by default, xorq only when provided @@ -987,6 +988,7 @@ def __init__( name=name, description=description, _source_join=_source_join, + _materialized_result=_materialized_result, ) def __repr__(self) -> str: @@ -4748,77 +4750,6 @@ def _find_all_root_models(node: Any) -> tuple[SemanticTableOp, ...]: return roots -def _non_additive_result_columns(node: Any) -> frozenset[str]: - """Result columns of a prior aggregate that must not be summed to get a total. - - A post-aggregation ``.mutate()`` only sees the aggregated rows, so its - ``t.all(x)`` can only be a window sum over those rows. That equals the - true overall value for SUM/COUNT measures and nothing else: summing - per-group means, medians, min/max or distinct counts gives a number with - no meaning, which is what ``t.all()`` used to return silently. - - Classification resolves each measure against its root's raw table, which - builds an expression but compiles nothing. Measures that cannot be - classified are omitted rather than assumed non-additive — callers keep - their historical behaviour for those instead of failing on a guess. - """ - current = node - agg_op = None - while current is not None: - if isinstance(current, SemanticAggregateOp): - agg_op = current - break - current = getattr(current, "source", None) - if agg_op is None: - return frozenset() - - try: - roots = _find_all_root_models(agg_op.source) - if not roots: - return frozenset() - merged_base = _get_merged_fields(roots, "measures") - merged_calc = _get_merged_fields(roots, "calc_measures") - probes = [] - for root in roots: - raw = getattr(root, "table", None) - if raw is None: - continue - probes.append(raw.to_expr() if hasattr(raw, "to_expr") else raw) - except Exception as exc: - logger.debug("additivity classification unavailable: %s", exc) - return frozenset() - - non_additive: set[str] = set() - for name in agg_op.aggs: - resolved = _resolve_short_name(name, merged_base, merged_calc) - if resolved is None: - continue - if resolved in merged_calc: - # A calculated measure is a ratio/window expression; summing it - # across groups is never the overall value. - non_additive.add(name) - continue - measure = merged_base.get(resolved) - expr = None - for probe in probes: - try: - expr = _resolve_expr(getattr(measure, "expr", measure), probe) - break - except Exception: - continue - if expr is None: - continue - try: - if _is_mean_expr(expr) or _reagg_op_for_expr(expr) != "sum": - non_additive.add(name) - except Exception as exc: - # _reagg_op_for_expr raises "this is a bug" for undecomposed - # mean / undeferred count-distinct: both are non-additive. - logger.debug("treating %r as non-additive: %s", name, exc) - non_additive.add(name) - return frozenset(non_additive) - - def _has_prior_aggregate(node: Any) -> bool: """True when a SemanticAggregateOp sits beneath ``node`` in the chain. diff --git a/src/boring_semantic_layer/query.py b/src/boring_semantic_layer/query.py index 60db9a91..5eed0073 100644 --- a/src/boring_semantic_layer/query.py +++ b/src/boring_semantic_layer/query.py @@ -582,7 +582,6 @@ def compare_periods( ) -> Any: """Compare two time ranges and return current/previous/delta columns.""" from ._xorq import null_safe_equal - from .api import to_semantic_table dimensions = list(dimensions or []) measures = list(measures or []) @@ -702,7 +701,19 @@ def compare_periods( if limit is not None: result_tbl = result_tbl.limit(limit) - return to_semantic_table(result_tbl, name=f"{model_name or 'model'}_period_comparison") + # The comparison table IS a query result (dims + current/previous/delta + # measures), so the wrapper is a materialized-result model: executable, + # unlike a definition-side semantic model. + from .expr import SemanticModel + + return SemanticModel( + table=result_tbl, + dimensions=None, + measures=None, + calc_measures=None, + name=f"{model_name or 'model'}_period_comparison", + _materialized_result=True, + ) def query( @@ -787,6 +798,12 @@ def query( """ from .ops import Dimension + if not dimensions and not measures: + raise QueryError( + "query() requires at least one dimension or measure — an empty " + "query has no semantic result. For raw rows, use .to_untagged()." + ) + result = semantic_table model_name = getattr(result, "name", None) known_dimensions = set(result.get_dimensions()) diff --git a/src/boring_semantic_layer/tests/soundness/test_join_lineage_and_grain.py b/src/boring_semantic_layer/tests/soundness/test_join_lineage_and_grain.py index 00da4d1e..ef1246ac 100644 --- a/src/boring_semantic_layer/tests/soundness/test_join_lineage_and_grain.py +++ b/src/boring_semantic_layer/tests/soundness/test_join_lineage_and_grain.py @@ -740,7 +740,7 @@ def test_preexisting_right_suffix_is_preserved_and_right_fields_stay_bound(con): "id_right", "value_right2", ) - raw = joined.execute().sort_values("id").reset_index(drop=True) + raw = joined.to_untagged().execute().sort_values("id").reset_index(drop=True) assert raw["value_right"].tolist() == [900, 901] assert raw["value_right2"].tolist() == [10, 20] assert str(joined.schema["value_right"]) == "int64" @@ -812,7 +812,7 @@ def test_user_column_with_internal_join_prefix_does_not_change_predicate(con): ) joined = to_semantic_table(left, "left").join_one(to_semantic_table(right, "right"), on="id") - result = joined.execute() + result = joined.to_untagged().execute() assert result.iloc[0].to_dict() == { "id": 1, "__bsl_jk_id": 99, @@ -833,7 +833,7 @@ def test_cross_join_uses_the_same_collision_safe_aliases(con): joined = to_semantic_table(left, "left").join_cross(to_semantic_table(right, "right")) assert tuple(joined.columns) == ("value", "value_right", "value_right2") - assert joined.execute().iloc[0].to_dict() == { + assert joined.to_untagged().execute().iloc[0].to_dict() == { "value": 1, "value_right": 9, "value_right2": 2, diff --git a/src/boring_semantic_layer/tests/test_dimension_validation.py b/src/boring_semantic_layer/tests/test_dimension_validation.py index 5c6c7f9f..f4f58102 100644 --- a/src/boring_semantic_layer/tests/test_dimension_validation.py +++ b/src/boring_semantic_layer/tests/test_dimension_validation.py @@ -330,8 +330,9 @@ def test_join_dimension_matching_column_name(): # Lambda-based join uses column names directly joined = orders.join_one(customers, on=lambda o, c: o.customer_id == c.customer_id) - # First verify the join executed without error - result = joined.execute() + # First verify the join executed without error (raw rows via the + # explicit escape hatch; a bare join is a definition, not a query) + result = joined.to_untagged().execute() # Verify the join worked correctly assert len(result) == 3 # 3 orders diff --git a/src/boring_semantic_layer/tests/test_execute_guard.py b/src/boring_semantic_layer/tests/test_execute_guard.py new file mode 100644 index 00000000..c61960c6 --- /dev/null +++ b/src/boring_semantic_layer/tests/test_execute_guard.py @@ -0,0 +1,239 @@ +"""A semantic model is a definition, not a query — output requires a query. + +Executing a bare model used to return the raw underlying table: every +physical column at row grain, undeclared columns included, computed +dimensions missing. Nothing about that result is semantic, and the same +hole existed for every output sink (``sql``, ``to_pandas``, ``to_csv``, …) +and for every pre-aggregation chain (filters, joins, order_by/limit, +``group_by`` without ``aggregate``, empty ``query()``). + +The contract now: sinking to output requires an aggregation stage (or an +index, or a model explicitly materialized from a query result). Raw access +stays one explicit call away via ``.to_untagged()``. +""" + +import ibis +import pandas as pd +import pytest + +from boring_semantic_layer import to_semantic_table +from boring_semantic_layer.errors import QueryError + + +@pytest.fixture +def con(): + return ibis.duckdb.connect(":memory:") + + +@pytest.fixture +def model(con): + tbl = con.create_table( + "orders", + pd.DataFrame( + { + "o_id": [1, 2, 3], + "cust": ["a", "b", "a"], + "amt": [10.0, 20.0, 30.0], + "d": pd.to_datetime(["2024-01-01", "2024-02-01", "2024-02-15"]), + } + ), + ) + return ( + to_semantic_table(tbl, name="orders") + .with_dimensions( + cust=lambda t: t.cust, + d={"expr": lambda t: t.d, "is_time_dimension": True}, + ) + .with_measures(revenue=lambda t: t.amt.sum(), n=lambda t: t.count()) + ) + + +@pytest.fixture +def customers(con): + tbl = con.create_table( + "customers", + pd.DataFrame({"cust": ["a", "b"], "region": ["west", "east"]}), + ) + return to_semantic_table(tbl, name="customers").with_dimensions( + cust=lambda t: t.cust, region=lambda t: t.region + ) + + +# --------------------------------------------------------------------------- +# Definition-side expressions refuse every output sink +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "sink", + ["execute", "to_pandas", "to_pyarrow", "to_polars", "to_pandas_batches", "sql", "compile"], +) +def test_bare_model_refuses_output_sinks(model, sink): + with pytest.raises(QueryError, match="definition, not a query"): + getattr(model, sink)() + + +def test_bare_model_refuses_file_sinks(model, tmp_path): + with pytest.raises(QueryError, match="definition, not a query"): + model.to_csv(tmp_path / "out.csv") + with pytest.raises(QueryError, match="definition, not a query"): + model.to_parquet(tmp_path / "out.parquet") + + +def test_error_names_the_model_and_its_fields(model): + with pytest.raises(QueryError) as exc: + model.execute() + msg = str(exc.value) + assert "'orders'" in msg + assert "to_untagged" in msg + assert "cust" in msg and "revenue" in msg + + +def test_filter_chain_is_still_a_definition(model): + with pytest.raises(QueryError, match="definition, not a query"): + model.filter(lambda t: t.amt > 5).execute() + + +def test_order_by_and_limit_do_not_complete_a_query(model): + with pytest.raises(QueryError, match="definition, not a query"): + model.order_by("cust").limit(2).execute() + + +def test_bare_join_is_a_definition(model, customers): + joined = model.join_one(customers, on="cust") + with pytest.raises(QueryError, match="definition, not a query"): + joined.execute() + + +def test_group_by_without_aggregate_is_an_incomplete_query(model): + with pytest.raises(QueryError, match=r"\.aggregate"): + model.group_by("cust").execute() + + +def test_regroup_of_an_aggregate_is_incomplete_again(model): + agg = model.group_by("cust").aggregate("revenue") + with pytest.raises(QueryError, match=r"\.aggregate"): + agg.group_by("cust").execute() + + +def test_empty_query_is_refused_eagerly(model): + with pytest.raises(QueryError, match="at least one dimension or measure"): + model.query() + with pytest.raises(QueryError, match="at least one dimension or measure"): + model.query(filters=[{"field": "cust", "operator": "=", "value": "a"}]) + + +# --------------------------------------------------------------------------- +# The explicit escape hatch and completed queries keep working +# --------------------------------------------------------------------------- + + +def test_to_untagged_is_the_explicit_raw_escape(model): + df = model.to_untagged().execute() + assert len(df) == 3 + assert "o_id" in df.columns # raw access is explicit, so raw columns are fine + + +def test_aggregate_executes(model): + df = model.group_by("cust").aggregate("revenue").execute() + assert sorted(df["revenue"].tolist()) == [20.0, 40.0] + + +def test_empty_aggregate_returns_distinct_group_values(model): + df = model.group_by("cust").aggregate().execute() + assert sorted(df["cust"].tolist()) == ["a", "b"] + + +def test_post_aggregation_chain_executes(model): + df = ( + model.group_by("cust") + .aggregate("revenue") + .filter(lambda t: t.revenue > 5) + .order_by("revenue") + .limit(5) + .execute() + ) + assert len(df) == 2 + + +def test_query_with_fields_executes(model): + df = model.query(dimensions=["cust"], measures=["revenue"]).execute() + assert set(df.columns) == {"cust", "revenue"} + + +def test_join_completed_by_a_query_executes(model, customers): + df = ( + model.join_one(customers, on="cust") + .group_by("customers.region") + .aggregate("orders.revenue") + .execute() + ) + assert len(df) == 2 + + +def test_as_table_over_an_aggregate_is_a_result_model(model): + result = model.group_by("cust").aggregate("revenue").as_table() + df = result.execute() + assert sorted(df["revenue"].tolist()) == [20.0, 40.0] + + +def test_as_table_over_an_ordered_aggregate_is_a_result_model(model): + result = model.group_by("cust").aggregate("revenue").order_by("revenue").as_table() + assert result.execute()["revenue"].tolist() == [20.0, 40.0] + + +def test_as_table_over_a_definition_stays_a_definition(model): + result = model.filter(lambda t: t.amt > 5).as_table() + with pytest.raises(QueryError, match="definition, not a query"): + result.execute() + + +def test_compare_periods_result_executes(model): + df = model.compare_periods( + dimensions=["cust"], + measures=["revenue"], + current_time_range={"start": "2024-02-01", "end": "2024-03-01"}, + previous_time_range={"start": "2024-01-01", "end": "2024-02-01"}, + ).execute() + assert "revenue_delta" in df.columns + + +def test_index_result_operations_execute(model): + df = model.index("cust").filter(lambda t: t.fieldValue == "a").execute() + assert df["fieldValue"].tolist() == ["a"] + + +# --------------------------------------------------------------------------- +# Post-aggregation mutate is refused with guidance +# --------------------------------------------------------------------------- + + +def test_mutate_on_wrapped_aggregate_is_refused(model): + agg = model.group_by("cust").aggregate("revenue") + for wrapped in ( + agg.order_by("revenue"), + agg.filter(lambda t: t.revenue > 5), + agg.limit(2), + ): + with pytest.raises(QueryError, match="to_untagged"): + wrapped.mutate(x=lambda t: t.revenue * 2) + + +def test_mutate_directly_on_aggregate_stays_on_the_measure_path(model): + df = ( + model.group_by("cust") + .aggregate("revenue") + .mutate(share=lambda t: t.revenue / t.all(t.revenue)) + .execute() + ) + assert sorted(round(v, 3) for v in df["share"]) == [0.333, 0.667] + + +def test_pre_aggregation_mutate_registers_a_dimension(model): + df = model.mutate(big=lambda t: t.amt > 15).group_by("big").aggregate("n").execute() + assert sorted(df["n"].tolist()) == [1, 2] + + +def test_pre_aggregation_mutate_result_is_still_a_definition(model): + with pytest.raises(QueryError, match="definition, not a query"): + model.mutate(big=lambda t: t.amt > 15).execute() diff --git a/src/boring_semantic_layer/tests/test_mutate_compositions.py b/src/boring_semantic_layer/tests/test_mutate_compositions.py index 90c4dd2a..bb5ea624 100644 --- a/src/boring_semantic_layer/tests/test_mutate_compositions.py +++ b/src/boring_semantic_layer/tests/test_mutate_compositions.py @@ -275,13 +275,28 @@ def test_mutate_result_usable_as_chart_input(self, flights_model): class TestMutateFilterMutate: - def test_second_mutate_sees_filtered_rows(self, flights_model): - """b after a filter is computed over rows surviving the filter.""" - df = ( + """Post-aggregation ``.mutate()`` on a filtered/ordered/limited result is + refused: the result of a query is a plain table, not a semantic model. + Row math over results is spelled ``.to_untagged().mutate(...)``; the + direct ``aggregate().mutate()`` (measure path) still works and is pinned + by TestMutateAfterAggregate above. + """ + + def test_second_mutate_after_filter_is_refused(self, flights_model): + from boring_semantic_layer.errors import QueryError + + chained = ( flights_model.group_by("carrier") .aggregate("flight_count", "total_distance") .mutate(avg_distance=lambda t: t.total_distance / t.flight_count) .filter(lambda t: t.avg_distance > 200) + ) + with pytest.raises(QueryError, match="to_untagged"): + chained.mutate(rnk=lambda t: t.avg_distance.rank()) + + # The sanctioned spelling: drop to ibis for row math over the result. + df = ( + chained.to_untagged() .mutate(rnk=lambda t: t.avg_distance.rank()) .execute() .sort_values("avg_distance") @@ -290,27 +305,27 @@ def test_second_mutate_sees_filtered_rows(self, flights_model): # Only DL (600) and UA (350) survive; rank computed over the 2 rows. assert df["rnk"].tolist() == [0, 1] - def test_mutate_after_limit(self, flights_model): - """Mutate after limit sees the post-limit table.""" - df = ( - flights_model.group_by("carrier") - .aggregate("flight_count") - .order_by("carrier") - .limit(2) - .mutate(doubled=lambda t: t.flight_count * 2) - .execute() + def test_mutate_after_limit_is_refused(self, flights_model): + from boring_semantic_layer.errors import QueryError + + limited = ( + flights_model.group_by("carrier").aggregate("flight_count").order_by("carrier").limit(2) ) + with pytest.raises(QueryError, match="to_untagged"): + limited.mutate(doubled=lambda t: t.flight_count * 2) + + df = limited.to_untagged().mutate(doubled=lambda t: t.flight_count * 2).execute() assert len(df) == 2 assert df["doubled"].tolist() == (df["flight_count"] * 2).tolist() - def test_mutate_after_order_by(self, flights_model): - df = ( - flights_model.group_by("carrier") - .aggregate("flight_count") - .order_by("carrier") - .mutate(doubled=lambda t: t.flight_count * 2) - .execute() - ) + def test_mutate_after_order_by_is_refused(self, flights_model): + from boring_semantic_layer.errors import QueryError + + ordered = flights_model.group_by("carrier").aggregate("flight_count").order_by("carrier") + with pytest.raises(QueryError, match="to_untagged"): + ordered.mutate(doubled=lambda t: t.flight_count * 2) + + df = ordered.to_untagged().mutate(doubled=lambda t: t.flight_count * 2).execute() assert df["doubled"].tolist() == (df["flight_count"] * 2).tolist() diff --git a/src/boring_semantic_layer/tests/test_query.py b/src/boring_semantic_layer/tests/test_query.py index 16c857cf..2b007d34 100644 --- a/src/boring_semantic_layer/tests/test_query.py +++ b/src/boring_semantic_layer/tests/test_query.py @@ -292,7 +292,7 @@ def flights_st(self): ) def test_filter_lambda_on_second_level_derived(self, flights_st): - result = flights_st.filter(ibis._.d_two > 1000).execute() + result = flights_st.filter(ibis._.d_two > 1000).to_untagged().execute() assert len(result) > 0 assert all(result["d_two"] > 1000) @@ -313,13 +313,15 @@ def test_query_deferred_filter_on_second_level_derived(self, flights_st): assert len(result) > 0 def test_filter_on_first_level_derived_still_works(self, flights_st): - result = flights_st.filter(ibis._.d_one > 1000).execute() + result = flights_st.filter(ibis._.d_one > 1000).to_untagged().execute() assert len(result) > 0 assert all(result["d_one"] > 1000) def test_chained_filters_on_derived_dims(self, flights_st): """Stacked filter().filter() both referencing derived dimensions.""" - result = flights_st.filter(ibis._.d_one > 500).filter(ibis._.d_two > 1000).execute() + result = ( + flights_st.filter(ibis._.d_one > 500).filter(ibis._.d_two > 1000).to_untagged().execute() + ) assert len(result) > 0 assert all(result["d_one"] > 500) assert all(result["d_two"] > 1000) @@ -336,7 +338,7 @@ def test_three_level_derived_dimension_filter(self): d_two=lambda t: t.d_one.add(1), d_three=lambda t: t.d_two.add(1), ) - result = st.filter(ibis._.d_three > 1000).execute() + result = st.filter(ibis._.d_three > 1000).to_untagged().execute() assert len(result) > 0 assert all(result["d_three"] > 1000) diff --git a/src/boring_semantic_layer/tests/test_stale_dimensions.py b/src/boring_semantic_layer/tests/test_stale_dimensions.py index a39c2406..470664dc 100644 --- a/src/boring_semantic_layer/tests/test_stale_dimensions.py +++ b/src/boring_semantic_layer/tests/test_stale_dimensions.py @@ -68,7 +68,7 @@ def test_bracket_filter_after_join_and_aggregate(): ) final = step2.join_one(model_c, lambda s, c: s["orders.customer_id"] == c.customer_id) - df = final.filter(lambda t: t["orders.region"] == "North").execute() + df = final.filter(lambda t: t["orders.region"] == "North").to_untagged().execute() assert df.shape[0] == 2 diff --git a/src/boring_semantic_layer/tests/test_totals_semantics.py b/src/boring_semantic_layer/tests/test_totals_semantics.py index fb135f05..10a10594 100644 --- a/src/boring_semantic_layer/tests/test_totals_semantics.py +++ b/src/boring_semantic_layer/tests/test_totals_semantics.py @@ -16,7 +16,7 @@ import pytest from boring_semantic_layer import to_semantic_table -from boring_semantic_layer.measure_scope import NonAdditiveTotalError +from boring_semantic_layer.errors import QueryError @pytest.fixture @@ -124,39 +124,72 @@ def test_percent_of_total_is_order_independent(): # --------------------------------------------------------------------------- -def test_chain_mutate_total_of_an_additive_measure(model): +def test_chain_mutate_after_order_by_is_refused(model): """``aggregate().order_by().mutate()`` sees only grouped rows. - A window sum over them is the true total for SUM/COUNT, so this must - agree with the same formula written as a calc measure. + A window sum over them equals the true total only for SUM/COUNT + measures, so the spelling is refused outright: post-aggregation row + math belongs on ``.to_untagged()``, and totals belong on the model + (calc measure) or directly on the aggregate, where they are computed + from the underlying rows. """ + with pytest.raises(QueryError, match="to_untagged"): + ( + model.group_by("carrier") + .aggregate("total") + .order_by("carrier") + .mutate(share=lambda t: t.total / t.all(t.total)) + ) + # The surviving spellings agree with each other. + assert _shares(model.with_measures(share=lambda t: t.total / t.all(t.total))) == SUM_SHARE + # The direct-on-the-aggregate spelling, pinned on duckdb: the memtable → + # canonical-backend path truncates this integer ratio to 0.0 (pre-existing + # xorq/DataFusion flavor defect, independent of the mutate desugaring — + # the compiled SQL carries the float cast and duckdb executes it). + con = ibis.duckdb.connect(":memory:") + tbl = con.create_table( + "flights_chain", + {"carrier": ["A", "A", "A", "B"], "distance": [10, 20, 30, 100]}, + ) + duck_model = ( + to_semantic_table(tbl, "flights") + .with_dimensions(carrier=lambda t: t.carrier) + .with_measures(total=lambda t: t.distance.sum()) + ) df = ( - model.group_by("carrier") + duck_model.group_by("carrier") .aggregate("total") - .order_by("carrier") .mutate(share=lambda t: t.total / t.all(t.total)) .execute() ) got = {k: pytest.approx(float(v)) for k, v in zip(df["carrier"], df["share"], strict=True)} assert got == SUM_SHARE - assert got == _shares(model.with_measures(share=lambda t: t.total / t.all(t.total))) def test_chain_mutate_total_of_a_mean_measure_is_refused(model): """Summing per-group means is not the overall mean — refuse, don't guess. - Returning the window sum made this spelling disagree with the identical - calc-measure formula (0.167 vs 0.5) with nothing to indicate which was - right. + The old chain spelling returned a window sum that disagreed with the + identical calc-measure formula (0.167 vs 0.5). The chain spelling is now + refused entirely; the measure path computes the true overall mean from + the underlying rows, so both surviving spellings agree. """ - with pytest.raises(NonAdditiveTotalError, match="not additive"): + with pytest.raises(QueryError, match="to_untagged"): ( model.group_by("carrier") .aggregate("avg") .order_by("carrier") .mutate(share=lambda t: t.avg / t.all(t.avg)) - .execute() ) + assert _shares(model.with_measures(share=lambda t: t.avg / t.all(t.avg))) == MEAN_SHARE + df = ( + model.group_by("carrier") + .aggregate("avg") + .mutate(share=lambda t: t.avg / t.all(t.avg)) + .execute() + ) + got = {k: pytest.approx(float(v)) for k, v in zip(df["carrier"], df["share"], strict=True)} + assert got == MEAN_SHARE # --------------------------------------------------------------------------- diff --git a/src/boring_semantic_layer/tests/test_unnest.py b/src/boring_semantic_layer/tests/test_unnest.py index f5eb139b..3d8cd8e1 100644 --- a/src/boring_semantic_layer/tests/test_unnest.py +++ b/src/boring_semantic_layer/tests/test_unnest.py @@ -31,7 +31,7 @@ def test_basic_unnest(): unnested = semantic.unnest("values") # Execute and check results - result = unnested.execute() + result = unnested.to_untagged().execute() # Should have 6 rows (2 + 1 + 3) assert len(result) == 6 @@ -61,7 +61,7 @@ def test_unnest_with_measures(): unnested = to_semantic_table(tbl, name="ga_sessions").unnest("hit_values") # Execute to verify unnesting worked - result = unnested.execute() + result = unnested.to_untagged().execute() assert len(result) == 6 # 3 + 2 + 1 hits assert result["hit_values"].sum() == 210 # 10+20+30+40+50+60 @@ -86,7 +86,7 @@ def test_unnest_with_groupby(): unnested = to_semantic_table(tbl, name="data").unnest("items") # Execute and verify grouping works - result = unnested.execute() + result = unnested.to_untagged().execute() assert len(result) == 6 # 2 + 3 + 1 @@ -113,7 +113,7 @@ def test_unnest_with_filter(): # Unnest and filter for values > 15 semantic = to_semantic_table(tbl, name="data").unnest("values") - result = semantic.filter(lambda t: t.values > 15).execute() + result = semantic.filter(lambda t: t.values > 15).to_untagged().execute() assert len(result) == 2 # 20 and 30 assert sorted(result["values"].tolist()) == [20, 30] @@ -133,7 +133,7 @@ def test_unnest_preserves_other_columns(): tbl = con.create_table("data", data) unnested = to_semantic_table(tbl, name="data").unnest("codes") - result = unnested.execute() + result = unnested.to_untagged().execute() # Should have 3 rows (2 + 1) assert len(result) == 3 @@ -154,7 +154,7 @@ def test_unnest_error_on_missing_column(): # Try to unnest a column that doesn't exist with pytest.raises(ValueError, match="Column 'nonexistent' not found"): - semantic.unnest("nonexistent").execute() + semantic.unnest("nonexistent").to_untagged() def test_unnest_chaining(): @@ -175,6 +175,7 @@ def test_unnest_chaining(): .unnest("items") .filter(lambda t: t.items > 2) .mutate(doubled=lambda t: t.items * 2) + .to_untagged() .execute() ) From 68e7df681fefb1da8dc63653552b95d1aa443255 Mon Sep 17 00:00:00 2001 From: Hussain Sultan Date: Mon, 24 Aug 2026 02:05:48 -0400 Subject: [PATCH 2/2] fix: align docs, examples, and chart extras with the definition/query contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI surfaced consumers of the old behavior that the core test run missed: - chart/md_parser executor: a doc fence ending with a bare semantic model now renders the "semantic table defined" box (the previously unreachable definition branch) by catching the guard's QueryError from the convert step, instead of erroring — no core privates imported, so the extras boundary ratchet stays green. New test pins the rendering. - chart tests: rolling-window and chart-time-detection regressions rewritten to mutate on the aggregate (the measure path handles windows when the window flavor matches — the window carries its own ordering), then order_by/limit. - examples/window_functions.py: windows over a filtered result drop to ibis via .to_untagged().mutate(...). - docs: windowing.md fences and query-methods.md window example rewritten to the mutate-on-the-aggregate spelling (keyword .over(rows=..., order_by=...)) with .to_untagged() for filtered/limited results; agent help topics and the bsl-query-expert copies teach the same spelling — it keeps .chart() and further order_by/limit working, and avoids constructing ibis.window() objects of the wrong flavor. Docs build now completes with zero query errors. - ruff format for tests/test_query.py (lint job failure). Full tree green: 1777 passed (incl. chart/, agents/, md_parser/ suites), docs build clean, skills check clean, examples run. Co-Authored-By: Claude Fable 5 --- docs/md/doc/query-methods.md | 20 +- docs/md/doc/windowing.md | 79 +- .../query/langchain/tool-query-model.md | 5 +- .../claude-code/bsl-query-expert/SKILL.md | 5 +- docs/md/skills/codex/bsl-query-expert.codex | 5 +- docs/md/skills/cursor/bsl-query-expert.mdc | 5 +- docs/web/public/bsl-data/bucketing.json | 573 ++++-- docs/web/public/bsl-data/builder-agent.json | 2 +- docs/web/public/bsl-data/charting.json | 729 +++++++- docs/web/public/bsl-data/compose.json | 261 ++- docs/web/public/bsl-data/example.json | 208 ++- docs/web/public/bsl-data/getting-started.json | 196 ++- docs/web/public/bsl-data/indexing.json | 1070 ++++++++---- docs/web/public/bsl-data/mcp.json | 2 +- .../web/public/bsl-data/nested-subtotals.json | 1107 ++++++++---- .../web/public/bsl-data/percentage-total.json | 549 ++---- docs/web/public/bsl-data/query-agent-mcp.json | 2 +- docs/web/public/bsl-data/query-methods.json | 1501 ++++++++++++++-- docs/web/public/bsl-data/semantic-table.json | 538 +++++- docs/web/public/bsl-data/sessionized.json | 480 ++++- docs/web/public/bsl-data/windowing.json | 1538 ++++++++++------- docs/web/public/bsl-data/yaml-config.json | 119 +- examples/window_functions.py | 6 +- .../agents/help_topics.py | 31 +- .../chart/md_parser/executor.py | 28 +- .../chart/tests/test_chart.py | 10 +- .../chart/tests/test_md_executor.py | 35 +- src/boring_semantic_layer/tests/test_query.py | 5 +- 28 files changed, 6756 insertions(+), 2353 deletions(-) diff --git a/docs/md/doc/query-methods.md b/docs/md/doc/query-methods.md index 051ee015..23a50825 100644 --- a/docs/md/doc/query-methods.md +++ b/docs/md/doc/query-methods.md @@ -428,29 +428,29 @@ Here's a simple example: ```query_window_example from ibis import _ -# First aggregate to daily level +# First aggregate to origin level daily_flights = ( flights_st .group_by("origin") .aggregate("flight_count", "total_distance") - .order_by("origin") ) -# Then apply window function for cumulative distance -window_spec = xo.window(order_by="origin") - +# Then apply window functions directly on the aggregate — the window +# carries its own ordering, and the result stays a semantic query result = daily_flights.mutate( - cumulative_distance=_.total_distance.cumsum(), + cumulative_distance=lambda t: t.total_distance.sum().over( + rows=(None, 0), order_by="origin" + ), flight_rank=lambda t: xo.rank().over(xo.window(order_by=xo.desc(t.flight_count))) -).limit(10) +).order_by("origin").limit(10) ``` **Key points:** -- Window functions are applied **after** `.aggregate()` using `.mutate()` -- Use `.order_by()` to establish row order for window operations -- Combine with `xo.window()` for advanced sliding window calculations +- Window functions are applied via `.mutate()` **directly on the aggregate** (before `.order_by()`/`.limit()` — after those the result is a plain table and `.mutate()` raises) +- Give each window its own ordering via the keyword form of `.over()` (e.g. `rows=(None, 0), order_by="origin"`) +- For row math over a *filtered* result, drop to ibis explicitly with `.to_untagged().mutate(...)` For comprehensive examples including lag/lead, moving averages, and ranking, see [Window Functions](/advanced/windowing). diff --git a/docs/md/doc/windowing.md b/docs/md/doc/windowing.md index a69e9f02..aa33e5f4 100644 --- a/docs/md/doc/windowing.md +++ b/docs/md/doc/windowing.md @@ -84,16 +84,21 @@ daily_revenue = ( sales_st .group_by("sale_date") .aggregate("total_revenue") - .order_by("sale_date") ) -# Add window functions for lag/lead +# Add window functions for lag/lead — applied directly on the aggregate, +# each window carrying its own ordering result = daily_revenue.mutate( - prev_day_revenue=_.total_revenue.lag(), - next_day_revenue=_.total_revenue.lead(), - day_over_day_change=_.total_revenue - _.total_revenue.lag(), - pct_change=((_.total_revenue - _.total_revenue.lag()) / _.total_revenue.lag() * 100).round(2) -).limit(10) + prev_day_revenue=lambda t: t.total_revenue.lag().over(order_by="sale_date"), + next_day_revenue=lambda t: t.total_revenue.lead().over(order_by="sale_date"), + day_over_day_change=lambda t: ( + t.total_revenue - t.total_revenue.lag().over(order_by="sale_date") + ), + pct_change=lambda t: ( + (t.total_revenue - t.total_revenue.lag().over(order_by="sale_date")) + / t.total_revenue.lag().over(order_by="sale_date") * 100 + ).round(2), +).order_by("sale_date").limit(10) ``` @@ -114,17 +119,15 @@ daily_revenue = ( sales_st .group_by("sale_date") .aggregate("total_revenue") - .order_by("sale_date") ) # Calculate cumulative sum and running average window_unbounded = xo.window(rows=(None, 0), order_by="sale_date") result = daily_revenue.mutate( - cumulative_revenue=_.total_revenue.cumsum(), - days_count=lambda t: t.count().over(window_unbounded), - avg_daily_so_far=lambda t: (t.cumulative_revenue / t.days_count).round(2) -).limit(10) + cumulative_revenue=lambda t: t.total_revenue.sum().over(window_unbounded), + avg_daily_so_far=lambda t: t.total_revenue.mean().over(window_unbounded).round(2), +).order_by("sale_date").limit(10) ``` @@ -141,16 +144,15 @@ daily_revenue = ( sales_st .group_by("sale_date") .aggregate("total_revenue") - .order_by("sale_date") ) # 7-day moving average window_7d = xo.window(rows=(-6, 0), order_by="sale_date") result = daily_revenue.mutate( - ma_7day=_.total_revenue.mean().over(window_7d).round(2), - ma_7day_sum=_.total_revenue.sum().over(window_7d).round(2), -).limit(10) + ma_7day=lambda t: t.total_revenue.mean().over(window_7d).round(2), + ma_7day_sum=lambda t: t.total_revenue.sum().over(window_7d).round(2), +).order_by("sale_date").limit(10) ``` @@ -171,15 +173,14 @@ category_revenue = ( sales_st .group_by("product_category") .aggregate("total_revenue", "sale_count") - .order_by(_.total_revenue.desc()) ) -# Add rank columns +# Add rank columns (each window carries its own ordering) result = category_revenue.mutate( rank=lambda t: xo.rank().over(xo.window(order_by=xo.desc(t.total_revenue))), dense_rank=lambda t: xo.dense_rank().over(xo.window(order_by=xo.desc(t.total_revenue))), row_number=lambda t: xo.row_number().over(xo.window(order_by=xo.desc(t.total_revenue))), -) +).order_by(_.total_revenue.desc()) ``` @@ -201,15 +202,17 @@ weekly_revenue = ( .mutate(week_start=_.sale_date.truncate("W")) .group_by("week_start") .aggregate("total_revenue") - .order_by("week_start") ) # Calculate week-over-week changes result = weekly_revenue.mutate( - prev_week_revenue=_.total_revenue.lag(), - wow_change=_.total_revenue - _.total_revenue.lag(), - wow_pct_change=((_.total_revenue - _.total_revenue.lag()) / _.total_revenue.lag() * 100).round(2) -).limit(10) + prev_week_revenue=lambda t: t.total_revenue.lag().over(order_by="week_start"), + wow_change=lambda t: t.total_revenue - t.total_revenue.lag().over(order_by="week_start"), + wow_pct_change=lambda t: ( + (t.total_revenue - t.total_revenue.lag().over(order_by="week_start")) + / t.total_revenue.lag().over(order_by="week_start") * 100 + ).round(2), +).order_by("week_start").limit(10) ``` @@ -230,11 +233,12 @@ top_days = ( .limit(10) ) -# Calculate cumulative percentage -result = top_days.mutate( - cumulative_revenue=_.total_revenue.cumsum(), - total_top10=_.total_revenue.sum(), - pct_of_top10=(_.total_revenue.cumsum() / _.total_revenue.sum() * 100).round(2) +# A limited query result is a plain table — row math over it drops to +# ibis explicitly via .to_untagged() +result = top_days.to_untagged().mutate( + cumulative_revenue=lambda t: t.total_revenue.cumsum(), + total_top10=lambda t: t.total_revenue.sum(), + pct_of_top10=lambda t: (t.total_revenue.cumsum() / t.total_revenue.sum() * 100).round(2), ) ``` @@ -254,30 +258,29 @@ weekend_revenue = ( .filter(_.is_weekend) .group_by("sale_date") .aggregate("total_revenue") - .order_by("sale_date") ) # 3-weekend moving average window_3 = xo.window(rows=(-2, 0), order_by="sale_date") result = weekend_revenue.mutate( - ma_3weekend=_.total_revenue.mean().over(window_3).round(2), - prev_weekend=_.total_revenue.lag(), - weekend_change=_.total_revenue - _.total_revenue.lag() -).limit(10) + ma_3weekend=lambda t: t.total_revenue.mean().over(window_3).round(2), + prev_weekend=lambda t: t.total_revenue.lag().over(order_by="sale_date"), + weekend_change=lambda t: t.total_revenue - t.total_revenue.lag().over(order_by="sale_date"), +).order_by("sale_date").limit(10) ``` ## Key Takeaways -- **Window functions operate after aggregation**: They work on query results, not raw data -- **Order matters**: Most window functions require `order_by()` for meaningful results +- **Window functions go on the aggregate**: apply `.mutate()` directly on the `aggregate()` result, before `.order_by()`/`.limit()` (after those the result is a plain table and `.mutate()` raises — use `.to_untagged().mutate(...)` there) +- **Each window carries its own ordering**: pass `order_by=` inside the window (e.g. `.over(rows=(-6, 0), order_by="sale_date")` or `.lag().over(order_by="sale_date")`) - **Flexible windows**: Define windows by rows (`rows=(n, m)`) or ranges - **Common patterns**: - `lag()/lead()` for period-over-period comparisons - - `cumsum()` for running totals - - `.over(window)` for moving averages + - `.sum().over(rows=(None, 0), order_by=...)` for running totals + - `.mean().over(window)` for moving averages - `rank()`, `row_number()` for ranking - **Combine with filters**: Focus window calculations on specific subsets diff --git a/docs/md/prompts/query/langchain/tool-query-model.md b/docs/md/prompts/query/langchain/tool-query-model.md index 5ac768d3..934a57e3 100644 --- a/docs/md/prompts/query/langchain/tool-query-model.md +++ b/docs/md/prompts/query/langchain/tool-query-model.md @@ -89,12 +89,13 @@ model.group_by("category").aggregate("revenue").order_by(ibis.desc("revenue")).l **CRITICAL**: `.limit()` in query limits data **before** calculations. Use `limit` parameter for display-only limiting. ## Window Functions -Windows run over the query result — drop to ibis first with `.to_untagged()`; the window carries its own ordering. (`.mutate()` after `.order_by()`/`.limit()`/`.filter()` on a result raises.) +Apply `.mutate()` directly on the aggregate (before `.order_by()`/`.limit()` — after those the result is a plain table and `.mutate()` raises); the window carries its own ordering via the keyword form of `.over()`: ```python -model.group_by("week").aggregate("count").to_untagged().mutate( +model.group_by("week").aggregate("count").mutate( rolling_avg=lambda t: t["count"].mean().over(rows=(-9, 0), order_by="week") ).order_by("week") ``` +On a filtered result, drop to ibis first: `.filter(...).to_untagged().mutate(...)`. **More**: `get_documentation(topic="windowing")` ## Chart diff --git a/docs/md/skills/claude-code/bsl-query-expert/SKILL.md b/docs/md/skills/claude-code/bsl-query-expert/SKILL.md index 20c765c3..a8491e3c 100644 --- a/docs/md/skills/claude-code/bsl-query-expert/SKILL.md +++ b/docs/md/skills/claude-code/bsl-query-expert/SKILL.md @@ -204,12 +204,13 @@ model.group_by("category").aggregate("revenue").order_by(ibis.desc("revenue")).l **CRITICAL**: `.limit()` in query limits data **before** calculations. Use `limit` parameter for display-only limiting. ## Window Functions -Windows run over the query result — drop to ibis first with `.to_untagged()`; the window carries its own ordering. (`.mutate()` after `.order_by()`/`.limit()`/`.filter()` on a result raises.) +Apply `.mutate()` directly on the aggregate (before `.order_by()`/`.limit()` — after those the result is a plain table and `.mutate()` raises); the window carries its own ordering via the keyword form of `.over()`: ```python -model.group_by("week").aggregate("count").to_untagged().mutate( +model.group_by("week").aggregate("count").mutate( rolling_avg=lambda t: t["count"].mean().over(rows=(-9, 0), order_by="week") ).order_by("week") ``` +On a filtered result, drop to ibis first: `.filter(...).to_untagged().mutate(...)`. **More**: `get_documentation(topic="windowing")` ## Chart diff --git a/docs/md/skills/codex/bsl-query-expert.codex b/docs/md/skills/codex/bsl-query-expert.codex index 44a185cd..aa3ab9b6 100644 --- a/docs/md/skills/codex/bsl-query-expert.codex +++ b/docs/md/skills/codex/bsl-query-expert.codex @@ -203,12 +203,13 @@ model.group_by("category").aggregate("revenue").order_by(ibis.desc("revenue")).l **CRITICAL**: `.limit()` in query limits data **before** calculations. Use `limit` parameter for display-only limiting. ## Window Functions -Windows run over the query result — drop to ibis first with `.to_untagged()`; the window carries its own ordering. (`.mutate()` after `.order_by()`/`.limit()`/`.filter()` on a result raises.) +Apply `.mutate()` directly on the aggregate (before `.order_by()`/`.limit()` — after those the result is a plain table and `.mutate()` raises); the window carries its own ordering via the keyword form of `.over()`: ```python -model.group_by("week").aggregate("count").to_untagged().mutate( +model.group_by("week").aggregate("count").mutate( rolling_avg=lambda t: t["count"].mean().over(rows=(-9, 0), order_by="week") ).order_by("week") ``` +On a filtered result, drop to ibis first: `.filter(...).to_untagged().mutate(...)`. **More**: `get_documentation(topic="windowing")` ## Chart diff --git a/docs/md/skills/cursor/bsl-query-expert.mdc b/docs/md/skills/cursor/bsl-query-expert.mdc index ca2d4afc..4774c7ee 100644 --- a/docs/md/skills/cursor/bsl-query-expert.mdc +++ b/docs/md/skills/cursor/bsl-query-expert.mdc @@ -205,12 +205,13 @@ model.group_by("category").aggregate("revenue").order_by(ibis.desc("revenue")).l **CRITICAL**: `.limit()` in query limits data **before** calculations. Use `limit` parameter for display-only limiting. ## Window Functions -Windows run over the query result — drop to ibis first with `.to_untagged()`; the window carries its own ordering. (`.mutate()` after `.order_by()`/`.limit()`/`.filter()` on a result raises.) +Apply `.mutate()` directly on the aggregate (before `.order_by()`/`.limit()` — after those the result is a plain table and `.mutate()` raises); the window carries its own ordering via the keyword form of `.over()`: ```python -model.group_by("week").aggregate("count").to_untagged().mutate( +model.group_by("week").aggregate("count").mutate( rolling_avg=lambda t: t["count"].mean().over(rows=(-9, 0), order_by="week") ).order_by("week") ``` +On a filtered result, drop to ibis first: `.filter(...).to_untagged().mutate(...)`. **More**: `get_documentation(topic="windowing")` ## Chart diff --git a/docs/web/public/bsl-data/bucketing.json b/docs/web/public/bsl-data/bucketing.json index 8eba3923..c3758cdf 100644 --- a/docs/web/public/bsl-data/bucketing.json +++ b/docs/web/public/bsl-data/bucketing.json @@ -137,144 +137,14 @@ } }, "semantic_table_def": { - "code": "from boring_semantic_layer import to_semantic_table\n\ncustomer_st = (\n to_semantic_table(customer_data, name=\"customers\")\n .with_dimensions(\n customer_id=lambda t: t.customer_id,\n age=lambda t: t.age,\n product_category=lambda t: t.product_category\n )\n .with_measures(\n customer_count=lambda t: t.count(),\n total_revenue=lambda t: t.purchase_amount.sum(),\n avg_purchase=lambda t: t.purchase_amount.mean().round(2)\n )\n)", - "sql": "SELECT\n *\nFROM \"ibis_pandas_memtable_5ucshn4zbrczvjw54qyamz32fm\"", - "plan": "SemanticTable: customers\n customer_id [dim]\n age [dim]\n product_category [dim]\n customer_count [measure]\n total_revenue [measure]\n avg_purchase [measure]", - "table": { - "columns": [ - "customer_id", - "age", - "purchase_amount", - "product_category" - ], - "data": [ - [ - 1, - 22, - 45, - "Electronics" - ], - [ - 2, - 28, - 120, - "Clothing" - ], - [ - 3, - 35, - 250, - "Electronics" - ], - [ - 4, - 42, - 180, - "Home" - ], - [ - 5, - 19, - 35, - "Clothing" - ], - [ - 6, - 55, - 520, - "Electronics" - ], - [ - 7, - 31, - 95, - "Clothing" - ], - [ - 8, - 67, - 850, - "Electronics" - ], - [ - 9, - 24, - 65, - "Clothing" - ], - [ - 10, - 38, - 310, - "Home" - ], - [ - 11, - 45, - 190, - "Electronics" - ], - [ - 12, - 29, - 78, - "Clothing" - ], - [ - 13, - 51, - 420, - "Home" - ], - [ - 14, - 33, - 145, - "Clothing" - ], - [ - 15, - 61, - 680, - "Electronics" - ], - [ - 16, - 26, - 88, - "Clothing" - ], - [ - 17, - 48, - 275, - "Home" - ], - [ - 18, - 36, - 165, - "Electronics" - ], - [ - 19, - 58, - 590, - "Electronics" - ], - [ - 20, - 41, - 225, - "Home" - ] - ] - } + "semantic_table": true, + "name": "customers", + "info": "Semantic table definition stored in context" }, "query_top_categories": { "code": "from ibis import _\n\n# Two-stage pipeline: rank then consolidate\nresult = (\n customer_st\n .group_by(\"product_category\")\n .aggregate(\"total_revenue\", \"customer_count\")\n .mutate(\n # Rank categories by revenue\n rank=lambda t: xo.row_number().over(\n xo.window(order_by=xo.desc(t.total_revenue))\n )\n )\n .mutate(\n # Replace non-top categories with \"Other\"\n category_display=lambda t: xo.case()\n .when(t.rank <= 2, t.product_category)\n .else_(\"Other\")\n .end(),\n # Keep original revenue for sorting (only for top categories)\n sort_value=lambda t: xo.case()\n .when(t.rank <= 2, t.total_revenue)\n .else_(0)\n .end()\n )\n .group_by(\"category_display\")\n .aggregate(\n revenue=lambda t: t.total_revenue.sum(),\n customers=lambda t: t.customer_count.sum(),\n sort_helper=lambda t: t.sort_value.max()\n )\n .mutate(\n avg_per_customer=lambda t: (t.revenue / t.customers).round(2)\n )\n .order_by(_.sort_helper.desc())\n)", - "sql": "SELECT\n \"t6\".\"category_display\",\n \"t6\".\"revenue\",\n \"t6\".\"customers\",\n \"t6\".\"sort_helper\",\n ROUND(CAST(\"t6\".\"revenue\" AS DOUBLE PRECISION) / \"t6\".\"customers\", 2) AS \"avg_per_customer\"\nFROM (\n SELECT\n \"t5\".\"category_display\",\n SUM(\"t5\".\"total_revenue\") AS \"revenue\",\n SUM(\"t5\".\"customer_count\") AS \"customers\",\n MAX(\"t5\".\"sort_value\") AS \"sort_helper\"\n FROM (\n SELECT\n \"t5\".\"product_category\",\n \"t5\".\"total_revenue\",\n \"t5\".\"customer_count\",\n \"t5\".\"rank\",\n \"t5\".\"sort_value\",\n \"t5\".\"category_display\"\n FROM (\n SELECT\n \"t4\".\"product_category\",\n \"t4\".\"total_revenue\",\n \"t4\".\"customer_count\",\n \"t4\".\"rank\",\n CASE WHEN \"t4\".\"rank\" <= 2 THEN \"t4\".\"product_category\" ELSE 'Other' END AS \"category_display\",\n CASE WHEN \"t4\".\"rank\" <= 2 THEN \"t4\".\"total_revenue\" ELSE 0 END AS \"sort_value\"\n FROM (\n SELECT\n \"t3\".\"product_category\",\n \"t3\".\"total_revenue\",\n \"t3\".\"customer_count\",\n ROW_NUMBER() OVER (\n ORDER BY \"t3\".\"total_revenue\" DESC NULLS LAST\n ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING\n ) - 1 AS \"rank\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t1\".\"product_category\",\n SUM(\"t1\".\"purchase_amount\") AS \"total_revenue\",\n COUNT(*) AS \"customer_count\"\n FROM (\n SELECT\n \"t1\".\"customer_id\",\n \"t1\".\"age\",\n \"t1\".\"purchase_amount\",\n \"t1\".\"product_category\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_5ucshn4zbrczvjw54qyamz32fm\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"product_category\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS \"t4\"\n ) AS \"t5\"\n ) AS t5\n GROUP BY\n \"t5\".\"category_display\"\n) AS \"t6\"\nORDER BY\n \"t6\".\"sort_helper\" DESC NULLS LAST", - "plan": "SemanticTable: customers\n customer_id [dim]\n age [dim]\n product_category [dim]\n customer_count [measure]\n total_revenue [measure]\n avg_purchase [measure]\n-> GroupBy(product_category)\n-> Aggregate(total_revenue, customer_count)\n-> Mutate(rank)\n-> Mutate(category_display, sort_value)\n-> GroupBy(category_display)\n-> Aggregate(revenue, customers, sort_helper)\n-> Mutate(avg_per_customer)\n-> OrderBy(_CallableWrapper(_fn=_.sort_helper.desc()))", + "sql": "SELECT\n \"t5\".\"category_display\",\n \"t5\".\"revenue\",\n \"t5\".\"customers\",\n \"t5\".\"sort_helper\",\n ROUND(CAST(\"t5\".\"revenue\" AS DOUBLE PRECISION) / \"t5\".\"customers\", 2) AS \"avg_per_customer\"\nFROM (\n SELECT\n \"t4\".\"category_display\",\n SUM(\"t4\".\"total_revenue\") AS \"revenue\",\n SUM(\"t4\".\"customer_count\") AS \"customers\",\n MAX(\"t4\".\"sort_value\") AS \"sort_helper\"\n FROM (\n SELECT\n \"t4\".\"product_category\",\n \"t4\".\"total_revenue\",\n \"t4\".\"customer_count\",\n \"t4\".\"rank\",\n \"t4\".\"sort_value\",\n \"t4\".\"category_display\"\n FROM (\n SELECT\n \"t3\".\"product_category\",\n \"t3\".\"total_revenue\",\n \"t3\".\"customer_count\",\n \"t3\".\"rank\",\n CASE WHEN \"t3\".\"rank\" <= 2 THEN \"t3\".\"product_category\" ELSE 'Other' END AS \"category_display\",\n CASE WHEN \"t3\".\"rank\" <= 2 THEN \"t3\".\"total_revenue\" ELSE 0 END AS \"sort_value\"\n FROM (\n SELECT\n \"t2\".\"product_category\",\n \"t2\".\"total_revenue\",\n \"t2\".\"customer_count\",\n ROW_NUMBER() OVER (ORDER BY \"t2\".\"total_revenue\" DESC NULLS LAST ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) - 1 AS \"rank\"\n FROM (\n SELECT\n \"t1\".\"product_category\",\n SUM(\"t1\".\"purchase_amount\") AS \"total_revenue\",\n COUNT(*) AS \"customer_count\"\n FROM (\n SELECT\n \"t1\".\"customer_id\",\n \"t1\".\"age\",\n \"t1\".\"purchase_amount\",\n \"t1\".\"product_category\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_6kzr6lwjdzerhnkuxyu7brztvi\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"product_category\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS \"t4\"\n ) AS t4\n GROUP BY\n \"t4\".\"category_display\"\n) AS \"t5\"\nORDER BY\n \"t5\".\"sort_helper\" DESC NULLS LAST", + "plan": "SemanticTable: customers\n customer_id [dim]\n age [dim]\n product_category [dim]\n customer_count [measure]\n total_revenue [measure]\n avg_purchase [measure]\n-> GroupBy(product_category)\n-> Aggregate(total_revenue, customer_count, rank, category_display, sort_value)\n-> GroupBy(category_display)\n-> Aggregate(revenue, customers, sort_helper, avg_per_customer)\n-> OrderBy(_CallableWrapper(_fn=_.sort_helper.desc()))", "table": { "columns": [ "category_display", @@ -306,12 +176,103 @@ 89.0 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-ab49cf3817a433557e4ded931ac10a32" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "category_display", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "category_display", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "revenue", + "customers", + "sort_helper", + "avg_per_customer" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-ab49cf3817a433557e4ded931ac10a32": [ + { + "category_display": "Electronics", + "revenue": 3290, + "customers": 8, + "sort_helper": 3290, + "avg_per_customer": 411.0 + }, + { + "category_display": "Home", + "revenue": 1410, + "customers": 5, + "sort_helper": 1410, + "avg_per_customer": 282.0 + }, + { + "category_display": "Clothing", + "revenue": 626, + "customers": 7, + "sort_helper": 626, + "avg_per_customer": 89.0 + } + ] + } + } } }, "query_age_buckets": { "code": "from ibis import _\nresult = (\n customer_st\n .group_by(\"customer_id\", \"age\", \"product_category\")\n .aggregate(\"total_revenue\")\n .mutate(\n age_group=lambda t: xo.case()\n .when(t.age < 25, \"18-24\")\n .when(t.age < 35, \"25-34\")\n .when(t.age < 45, \"35-44\")\n .when(t.age < 55, \"45-54\")\n .else_(\"55+\")\n .end()\n )\n .group_by(\"age_group\")\n .aggregate(\n customers=lambda t: t.count(),\n revenue=lambda t: t.total_revenue.sum()\n )\n .order_by(_.age_group)\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t3\".\"age_group\",\n COUNT(*) AS \"customers\",\n SUM(\"t3\".\"total_revenue\") AS \"revenue\"\n FROM (\n SELECT\n \"t3\".\"customer_id\",\n \"t3\".\"age\",\n \"t3\".\"product_category\",\n \"t3\".\"total_revenue\",\n \"t3\".\"age_group\"\n FROM (\n SELECT\n \"t2\".\"customer_id\",\n \"t2\".\"age\",\n \"t2\".\"product_category\",\n \"t2\".\"total_revenue\",\n CASE\n WHEN \"t2\".\"age\" < 25\n THEN '18-24'\n WHEN \"t2\".\"age\" < 35\n THEN '25-34'\n WHEN \"t2\".\"age\" < 45\n THEN '35-44'\n WHEN \"t2\".\"age\" < 55\n THEN '45-54'\n ELSE '55+'\n END AS \"age_group\"\n FROM (\n SELECT\n \"t1\".\"customer_id\",\n \"t1\".\"age\",\n \"t1\".\"product_category\",\n SUM(\"t1\".\"purchase_amount\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t1\".\"purchase_amount\",\n \"t1\".\"customer_id\",\n \"t1\".\"age\",\n \"t1\".\"product_category\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_5ucshn4zbrczvjw54qyamz32fm\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"customer_id\",\n \"t1\".\"age\",\n \"t1\".\"product_category\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS t3\n GROUP BY\n \"t3\".\"age_group\"\n) AS \"t4\"\nORDER BY\n \"t4\".\"age_group\" ASC", - "plan": "SemanticTable: customers\n customer_id [dim]\n age [dim]\n product_category [dim]\n customer_count [measure]\n total_revenue [measure]\n avg_purchase [measure]\n-> GroupBy(customer_id, age, product_category)\n-> Aggregate(total_revenue)\n-> Mutate(age_group)\n-> GroupBy(age_group)\n-> Aggregate(customers, revenue)\n-> OrderBy(_CallableWrapper(_fn=_.age_group))", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t3\".\"age_group\",\n COUNT(*) AS \"customers\",\n SUM(\"t3\".\"total_revenue\") AS \"revenue\"\n FROM (\n SELECT\n \"t3\".\"customer_id\",\n \"t3\".\"age\",\n \"t3\".\"product_category\",\n \"t3\".\"total_revenue\",\n \"t3\".\"age_group\"\n FROM (\n SELECT\n \"t2\".\"customer_id\",\n \"t2\".\"age\",\n \"t2\".\"product_category\",\n \"t2\".\"total_revenue\",\n CASE\n WHEN \"t2\".\"age\" < 25\n THEN '18-24'\n WHEN \"t2\".\"age\" < 35\n THEN '25-34'\n WHEN \"t2\".\"age\" < 45\n THEN '35-44'\n WHEN \"t2\".\"age\" < 55\n THEN '45-54'\n ELSE '55+'\n END AS \"age_group\"\n FROM (\n SELECT\n \"t1\".\"customer_id\",\n \"t1\".\"age\",\n \"t1\".\"product_category\",\n SUM(\"t1\".\"purchase_amount\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t1\".\"purchase_amount\",\n \"t1\".\"customer_id\",\n \"t1\".\"age\",\n \"t1\".\"product_category\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_6kzr6lwjdzerhnkuxyu7brztvi\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"customer_id\",\n \"t1\".\"age\",\n \"t1\".\"product_category\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS t3\n GROUP BY\n \"t3\".\"age_group\"\n) AS \"t4\"\nORDER BY\n \"t4\".\"age_group\" ASC", + "plan": "SemanticTable: customers\n customer_id [dim]\n age [dim]\n product_category [dim]\n customer_count [measure]\n total_revenue [measure]\n avg_purchase [measure]\n-> GroupBy(customer_id, age, product_category)\n-> Aggregate(total_revenue, age_group)\n-> GroupBy(age_group)\n-> Aggregate(customers, revenue)\n-> OrderBy(_CallableWrapper(_fn=_.age_group))", "table": { "columns": [ "age_group", @@ -345,12 +306,105 @@ 2640 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-9ea38b7e3ee45e5a0a5c2b768afb1e75" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "age_group", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "age_group", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "customers", + "revenue" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-9ea38b7e3ee45e5a0a5c2b768afb1e75": [ + { + "age_group": "18-24", + "customers": 3, + "revenue": 145 + }, + { + "age_group": "25-34", + "customers": 5, + "revenue": 526 + }, + { + "age_group": "35-44", + "customers": 5, + "revenue": 1130 + }, + { + "age_group": "45-54", + "customers": 3, + "revenue": 885 + }, + { + "age_group": "55+", + "customers": 4, + "revenue": 2640 + } + ] + } + } } }, "query_purchase_tiers": { "code": "from ibis import _\nresult = (\n customer_st\n .group_by(\"customer_id\")\n .aggregate(\"total_revenue\")\n .mutate(\n tier=lambda t: xo.case()\n .when(t.total_revenue < 100, \"Small ($0-99)\")\n .when(t.total_revenue < 250, \"Medium ($100-249)\")\n .when(t.total_revenue < 500, \"Large ($250-499)\")\n .else_(\"Premium ($500+)\")\n .end()\n )\n .group_by(\"tier\")\n .aggregate(\n customer_count=lambda t: t.count(),\n total_value=lambda t: t.total_revenue.sum(),\n avg_value=lambda t: t.total_revenue.mean().round(2)\n )\n .order_by(_.total_value.desc())\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t3\".\"tier\",\n COUNT(*) AS \"customer_count\",\n SUM(\"t3\".\"total_revenue\") AS \"total_value\",\n ROUND(CAST(AVG(\"t3\".\"total_revenue\") AS DECIMAL), 2) AS \"avg_value\"\n FROM (\n SELECT\n \"t3\".\"customer_id\",\n \"t3\".\"total_revenue\",\n \"t3\".\"tier\"\n FROM (\n SELECT\n \"t2\".\"customer_id\",\n \"t2\".\"total_revenue\",\n CASE\n WHEN \"t2\".\"total_revenue\" < 100\n THEN 'Small ($0-99)'\n WHEN \"t2\".\"total_revenue\" < 250\n THEN 'Medium ($100-249)'\n WHEN \"t2\".\"total_revenue\" < 500\n THEN 'Large ($250-499)'\n ELSE 'Premium ($500+)'\n END AS \"tier\"\n FROM (\n SELECT\n \"t1\".\"customer_id\",\n SUM(\"t1\".\"purchase_amount\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t1\".\"age\",\n \"t1\".\"purchase_amount\",\n \"t1\".\"product_category\",\n \"t1\".\"customer_id\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_5ucshn4zbrczvjw54qyamz32fm\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"customer_id\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS t3\n GROUP BY\n \"t3\".\"tier\"\n) AS \"t4\"\nORDER BY\n \"t4\".\"total_value\" DESC NULLS LAST", - "plan": "SemanticTable: customers\n customer_id [dim]\n age [dim]\n product_category [dim]\n customer_count [measure]\n total_revenue [measure]\n avg_purchase [measure]\n-> GroupBy(customer_id)\n-> Aggregate(total_revenue)\n-> Mutate(tier)\n-> GroupBy(tier)\n-> Aggregate(customer_count, total_value, avg_value)\n-> OrderBy(_CallableWrapper(_fn=_.total_value.desc()))", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t3\".\"tier\",\n COUNT(*) AS \"customer_count\",\n SUM(\"t3\".\"total_revenue\") AS \"total_value\",\n ROUND(AVG(\"t3\".\"total_revenue\"), 2) AS \"avg_value\"\n FROM (\n SELECT\n \"t3\".\"customer_id\",\n \"t3\".\"total_revenue\",\n \"t3\".\"tier\"\n FROM (\n SELECT\n \"t2\".\"customer_id\",\n \"t2\".\"total_revenue\",\n CASE\n WHEN \"t2\".\"total_revenue\" < 100\n THEN 'Small ($0-99)'\n WHEN \"t2\".\"total_revenue\" < 250\n THEN 'Medium ($100-249)'\n WHEN \"t2\".\"total_revenue\" < 500\n THEN 'Large ($250-499)'\n ELSE 'Premium ($500+)'\n END AS \"tier\"\n FROM (\n SELECT\n \"t1\".\"customer_id\",\n SUM(\"t1\".\"purchase_amount\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t1\".\"age\",\n \"t1\".\"purchase_amount\",\n \"t1\".\"product_category\",\n \"t1\".\"customer_id\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_6kzr6lwjdzerhnkuxyu7brztvi\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"customer_id\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS t3\n GROUP BY\n \"t3\".\"tier\"\n) AS \"t4\"\nORDER BY\n \"t4\".\"total_value\" DESC NULLS LAST", + "plan": "SemanticTable: customers\n customer_id [dim]\n age [dim]\n product_category [dim]\n customer_count [measure]\n total_revenue [measure]\n avg_purchase [measure]\n-> GroupBy(customer_id)\n-> Aggregate(total_revenue, tier)\n-> GroupBy(tier)\n-> Aggregate(customer_count, total_value, avg_value)\n-> OrderBy(_CallableWrapper(_fn=_.total_value.desc()))", "table": { "columns": [ "tier", @@ -384,12 +438,105 @@ 67.67 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-5b19223277d622765747f08ab007e836" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "tier", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "tier", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "customer_count", + "total_value", + "avg_value" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-5b19223277d622765747f08ab007e836": [ + { + "tier": "Premium ($500+)", + "customer_count": 4, + "total_value": 2640, + "avg_value": 660.0 + }, + { + "tier": "Large ($250-499)", + "customer_count": 4, + "total_value": 1255, + "avg_value": 313.75 + }, + { + "tier": "Medium ($100-249)", + "customer_count": 6, + "total_value": 1025, + "avg_value": 170.83 + }, + { + "tier": "Small ($0-99)", + "customer_count": 6, + "total_value": 406, + "avg_value": 67.67 + } + ] + } + } } }, "query_with_other": { "code": "from ibis import _\n\nresult = (\n customer_st\n .group_by(\"product_category\")\n .aggregate(\"total_revenue\", \"customer_count\")\n .mutate(\n # Mark categories with less than 5 customers as \"Other\"\n category_grouped=lambda t: xo.case()\n .when(t.customer_count >= 5, t.product_category)\n .else_(\"Other\")\n .end()\n )\n .group_by(\"category_grouped\")\n .aggregate(\n customers=lambda t: t.customer_count.sum(),\n revenue=lambda t: t.total_revenue.sum()\n )\n .mutate(\n avg_per_customer=lambda t: (t.revenue / t.customers).round(2)\n )\n .order_by(_.revenue.desc())\n)", - "sql": "SELECT\n \"t4\".\"category_grouped\",\n \"t4\".\"customers\",\n \"t4\".\"revenue\",\n ROUND(CAST(\"t4\".\"revenue\" AS DOUBLE PRECISION) / \"t4\".\"customers\", 2) AS \"avg_per_customer\"\nFROM (\n SELECT\n \"t3\".\"category_grouped\",\n SUM(\"t3\".\"customer_count\") AS \"customers\",\n SUM(\"t3\".\"total_revenue\") AS \"revenue\"\n FROM (\n SELECT\n \"t3\".\"product_category\",\n \"t3\".\"total_revenue\",\n \"t3\".\"customer_count\",\n \"t3\".\"category_grouped\"\n FROM (\n SELECT\n \"t2\".\"product_category\",\n \"t2\".\"total_revenue\",\n \"t2\".\"customer_count\",\n CASE WHEN \"t2\".\"customer_count\" >= 5 THEN \"t2\".\"product_category\" ELSE 'Other' END AS \"category_grouped\"\n FROM (\n SELECT\n \"t1\".\"product_category\",\n SUM(\"t1\".\"purchase_amount\") AS \"total_revenue\",\n COUNT(*) AS \"customer_count\"\n FROM (\n SELECT\n \"t1\".\"customer_id\",\n \"t1\".\"age\",\n \"t1\".\"purchase_amount\",\n \"t1\".\"product_category\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_5ucshn4zbrczvjw54qyamz32fm\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"product_category\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS t3\n GROUP BY\n \"t3\".\"category_grouped\"\n) AS \"t4\"\nORDER BY\n \"t4\".\"revenue\" DESC NULLS LAST", - "plan": "SemanticTable: customers\n customer_id [dim]\n age [dim]\n product_category [dim]\n customer_count [measure]\n total_revenue [measure]\n avg_purchase [measure]\n-> GroupBy(product_category)\n-> Aggregate(total_revenue, customer_count)\n-> Mutate(category_grouped)\n-> GroupBy(category_grouped)\n-> Aggregate(customers, revenue)\n-> Mutate(avg_per_customer)\n-> OrderBy(_CallableWrapper(_fn=_.revenue.desc()))", + "sql": "SELECT\n \"t4\".\"category_grouped\",\n \"t4\".\"customers\",\n \"t4\".\"revenue\",\n ROUND(CAST(\"t4\".\"revenue\" AS DOUBLE PRECISION) / \"t4\".\"customers\", 2) AS \"avg_per_customer\"\nFROM (\n SELECT\n \"t3\".\"category_grouped\",\n SUM(\"t3\".\"customer_count\") AS \"customers\",\n SUM(\"t3\".\"total_revenue\") AS \"revenue\"\n FROM (\n SELECT\n \"t3\".\"product_category\",\n \"t3\".\"total_revenue\",\n \"t3\".\"customer_count\",\n \"t3\".\"category_grouped\"\n FROM (\n SELECT\n \"t2\".\"product_category\",\n \"t2\".\"total_revenue\",\n \"t2\".\"customer_count\",\n CASE WHEN \"t2\".\"customer_count\" >= 5 THEN \"t2\".\"product_category\" ELSE 'Other' END AS \"category_grouped\"\n FROM (\n SELECT\n \"t1\".\"product_category\",\n SUM(\"t1\".\"purchase_amount\") AS \"total_revenue\",\n COUNT(*) AS \"customer_count\"\n FROM (\n SELECT\n \"t1\".\"customer_id\",\n \"t1\".\"age\",\n \"t1\".\"purchase_amount\",\n \"t1\".\"product_category\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_6kzr6lwjdzerhnkuxyu7brztvi\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"product_category\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS t3\n GROUP BY\n \"t3\".\"category_grouped\"\n) AS \"t4\"\nORDER BY\n \"t4\".\"revenue\" DESC NULLS LAST", + "plan": "SemanticTable: customers\n customer_id [dim]\n age [dim]\n product_category [dim]\n customer_count [measure]\n total_revenue [measure]\n avg_purchase [measure]\n-> GroupBy(product_category)\n-> Aggregate(total_revenue, customer_count, category_grouped)\n-> GroupBy(category_grouped)\n-> Aggregate(customers, revenue, avg_per_customer)\n-> OrderBy(_CallableWrapper(_fn=_.revenue.desc()))", "table": { "columns": [ "category_grouped", @@ -417,12 +564,99 @@ 89.0 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-12a0f133ab3f8d0586edcf3e313569a8" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "category_grouped", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "category_grouped", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "customers", + "revenue", + "avg_per_customer" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-12a0f133ab3f8d0586edcf3e313569a8": [ + { + "category_grouped": "Electronics", + "customers": 8, + "revenue": 3290, + "avg_per_customer": 411.0 + }, + { + "category_grouped": "Home", + "customers": 5, + "revenue": 1410, + "avg_per_customer": 282.0 + }, + { + "category_grouped": "Clothing", + "customers": 7, + "revenue": 626, + "avg_per_customer": 89.0 + } + ] + } + } } }, "query_combined_buckets": { "code": "from ibis import _\nresult = (\n customer_st\n .group_by(\"customer_id\", \"age\")\n .aggregate(\"total_revenue\")\n .mutate(\n age_group=lambda t: xo.case()\n .when(t.age < 30, \"Young (18-29)\")\n .when(t.age < 50, \"Middle (30-49)\")\n .else_(\"Senior (50+)\")\n .end(),\n value_tier=lambda t: xo.case()\n .when(t.total_revenue < 150, \"Low Value\")\n .when(t.total_revenue < 350, \"Mid Value\")\n .else_(\"High Value\")\n .end()\n )\n .group_by(\"age_group\", \"value_tier\")\n .aggregate(\n customers=lambda t: t.count(),\n revenue=lambda t: t.total_revenue.sum()\n )\n .order_by(_.age_group, _.revenue.desc())\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t3\".\"age_group\",\n \"t3\".\"value_tier\",\n COUNT(*) AS \"customers\",\n SUM(\"t3\".\"total_revenue\") AS \"revenue\"\n FROM (\n SELECT\n \"t3\".\"customer_id\",\n \"t3\".\"age\",\n \"t3\".\"total_revenue\",\n \"t3\".\"age_group\",\n \"t3\".\"value_tier\"\n FROM (\n SELECT\n \"t2\".\"customer_id\",\n \"t2\".\"age\",\n \"t2\".\"total_revenue\",\n CASE\n WHEN \"t2\".\"age\" < 30\n THEN 'Young (18-29)'\n WHEN \"t2\".\"age\" < 50\n THEN 'Middle (30-49)'\n ELSE 'Senior (50+)'\n END AS \"age_group\",\n CASE\n WHEN \"t2\".\"total_revenue\" < 150\n THEN 'Low Value'\n WHEN \"t2\".\"total_revenue\" < 350\n THEN 'Mid Value'\n ELSE 'High Value'\n END AS \"value_tier\"\n FROM (\n SELECT\n \"t1\".\"customer_id\",\n \"t1\".\"age\",\n SUM(\"t1\".\"purchase_amount\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t1\".\"purchase_amount\",\n \"t1\".\"product_category\",\n \"t1\".\"customer_id\",\n \"t1\".\"age\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_5ucshn4zbrczvjw54qyamz32fm\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"customer_id\",\n \"t1\".\"age\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS t3\n GROUP BY\n \"t3\".\"age_group\",\n \"t3\".\"value_tier\"\n) AS \"t4\"\nORDER BY\n \"t4\".\"age_group\" ASC,\n \"t4\".\"revenue\" DESC NULLS LAST", - "plan": "SemanticTable: customers\n customer_id [dim]\n age [dim]\n product_category [dim]\n customer_count [measure]\n total_revenue [measure]\n avg_purchase [measure]\n-> GroupBy(customer_id, age)\n-> Aggregate(total_revenue)\n-> Mutate(age_group, value_tier)\n-> GroupBy(age_group, value_tier)\n-> Aggregate(customers, revenue)\n-> OrderBy(_CallableWrapper(_fn=_.age_group), _CallableWrapper(_fn=_.revenue.desc()))", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t3\".\"age_group\",\n \"t3\".\"value_tier\",\n COUNT(*) AS \"customers\",\n SUM(\"t3\".\"total_revenue\") AS \"revenue\"\n FROM (\n SELECT\n \"t3\".\"customer_id\",\n \"t3\".\"age\",\n \"t3\".\"total_revenue\",\n \"t3\".\"age_group\",\n \"t3\".\"value_tier\"\n FROM (\n SELECT\n \"t2\".\"customer_id\",\n \"t2\".\"age\",\n \"t2\".\"total_revenue\",\n CASE\n WHEN \"t2\".\"age\" < 30\n THEN 'Young (18-29)'\n WHEN \"t2\".\"age\" < 50\n THEN 'Middle (30-49)'\n ELSE 'Senior (50+)'\n END AS \"age_group\",\n CASE\n WHEN \"t2\".\"total_revenue\" < 150\n THEN 'Low Value'\n WHEN \"t2\".\"total_revenue\" < 350\n THEN 'Mid Value'\n ELSE 'High Value'\n END AS \"value_tier\"\n FROM (\n SELECT\n \"t1\".\"customer_id\",\n \"t1\".\"age\",\n SUM(\"t1\".\"purchase_amount\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t1\".\"purchase_amount\",\n \"t1\".\"product_category\",\n \"t1\".\"customer_id\",\n \"t1\".\"age\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_6kzr6lwjdzerhnkuxyu7brztvi\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"customer_id\",\n \"t1\".\"age\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS t3\n GROUP BY\n \"t3\".\"age_group\",\n \"t3\".\"value_tier\"\n) AS \"t4\"\nORDER BY\n \"t4\".\"age_group\" ASC,\n \"t4\".\"revenue\" DESC NULLS LAST", + "plan": "SemanticTable: customers\n customer_id [dim]\n age [dim]\n product_category [dim]\n customer_count [measure]\n total_revenue [measure]\n avg_purchase [measure]\n-> GroupBy(customer_id, age)\n-> Aggregate(total_revenue, age_group, value_tier)\n-> GroupBy(age_group, value_tier)\n-> Aggregate(customers, revenue)\n-> OrderBy(_CallableWrapper(_fn=_.age_group), _CallableWrapper(_fn=_.revenue.desc()))", "table": { "columns": [ "age_group", @@ -456,6 +690,59 @@ 431 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-e53063265ad8373a4153f3a953d1706f" + }, + "mark": { + "type": "text" + }, + "encoding": { + "text": { + "value": "Complex query - consider custom visualization" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-e53063265ad8373a4153f3a953d1706f": [ + { + "age_group": "Middle (30-49)", + "value_tier": "Mid Value", + "customers": 7, + "revenue": 1595 + }, + { + "age_group": "Middle (30-49)", + "value_tier": "Low Value", + "customers": 2, + "revenue": 240 + }, + { + "age_group": "Senior (50+)", + "value_tier": "High Value", + "customers": 5, + "revenue": 3060 + }, + { + "age_group": "Young (18-29)", + "value_tier": "Low Value", + "customers": 6, + "revenue": 431 + } + ] + } + } } } }, diff --git a/docs/web/public/bsl-data/builder-agent.json b/docs/web/public/bsl-data/builder-agent.json index a7cc4218..18435b6b 100644 --- a/docs/web/public/bsl-data/builder-agent.json +++ b/docs/web/public/bsl-data/builder-agent.json @@ -1,5 +1,5 @@ { - "markdown": "# Builder Agent\n\nThe Builder Agent is focused on authoring and evolving semantic tables: defining dimensions, measures, joins, calculated measures, YAML config, and validation patterns. It uses a different Claude skill than the Query Agent because it needs to reason about modeling APIs rather than issuing queries.\n\n## Claude Code Skill\n\n- File: [`docs/md/skills/claude-code/bsl-model-builder/SKILL.md`](../skills/claude-code/bsl-model-builder/SKILL.md)\n- Use it when you want Claude Desktop to help write new semantic tables, add time dimensions, or compose models.\n- The skill includes:\n - Python DSL examples showing `SemanticTable(...)`, `.with_dimensions`, `.with_measures`, `.with_calculated_measures`, and `.join()` patterns.\n - YAML equivalents so you can copy the same logic into declarative configs.\n - Gotchas such as \"measures must aggregate\" and \"join keys must be defined dimensions\".\n\n**Workflow:** Load the skill in Claude Desktop, paste the schema or YAML snippet you are editing, and ask \"Generate a semantic table for flights with avg delay and join to airports\". Claude will respond with both Python and YAML patterns that mirror the documentation.\n\n## Codex Skill\n\nRunning inside the Codex CLI (the environment this assistant uses) already gives you repo access. Pair that with the Builder skill to automate scaffolding:\n\n1. Open `docs/md/doc/semantic-table.md` or the relevant source file in your editor for context.\n2. Ask Codex to \"apply the builder skill\" when drafting new semantic tables. It will reference `bsl-model-building/SKILL.md` to keep the API usage correct.\n3. Use the CLI's `apply_patch` output directly to drop in the generated models or YAML definitions.\n\nThis approach keeps all modeling work version-controlled while still benefiting from the same guard rails the Claude skill enforces.\n\n## Cursor (or other AI IDEs)\n\nIf you prefer Cursor, VS Code Copilot Chat, or another AI-assisted IDE:\n\n1. Store the builder skill text in a snippet (Cursor: *Settings -> Custom Instructions*).\n2. Add quick prompts like \"Use the BSL builder skill\" so the IDE pastes the instructions before generating code.\n3. Point the IDE at your actual data context (DuckDB schema, YAML file) so it can thread the builder guard rails through your request.\n\nRegardless of the host, the Builder Agent should always cite the same modeling patterns. That keeps upstream MCP/Query agents consistent because every semantic table passes through the same validation philosophy.\n", + "markdown": "# Builder Agent\n\nThe Builder Agent is focused on authoring and evolving semantic tables: defining dimensions, measures, joins, calculated measures, YAML config, and validation patterns. It uses a different Claude skill than the Query Agent because it needs to reason about modeling APIs rather than issuing queries.\n\n## Claude Code Skill\n\n- File: [`docs/md/skills/claude-code/bsl-model-builder/SKILL.md`](../skills/claude-code/bsl-model-builder/SKILL.md)\n- Use it when you want Claude Desktop to help write new semantic tables, add time dimensions, or compose models.\n- The skill includes:\n - Python DSL examples showing `to_semantic_table(...)`, `.with_dimensions`, `.with_measures`, `join_one()`, `join_many()`, and `join_cross()` patterns.\n - YAML equivalents so you can copy the same logic into declarative configs.\n - Gotchas such as choosing cardinality from the left side, using source-aware equality predicates, and assigning unique model aliases.\n\n**Workflow:** Load the skill in Claude Desktop, paste the schema or YAML snippet you are editing, and ask \"Generate a semantic table for flights with avg delay and join to airports\". Claude will respond with both Python and YAML patterns that mirror the documentation.\n\n## Codex Skill\n\nRunning inside the Codex CLI (the environment this assistant uses) already gives you repo access. Pair that with the Builder skill to automate scaffolding:\n\n1. Open `docs/md/doc/semantic-table.md` or the relevant source file in your editor for context.\n2. Ask Codex to \"apply the builder skill\" when drafting new semantic tables. It will reference `bsl-model-building/SKILL.md` to keep the API usage correct.\n3. Use the CLI's `apply_patch` output directly to drop in the generated models or YAML definitions.\n\nThis approach keeps all modeling work version-controlled while still benefiting from the same guard rails the Claude skill enforces.\n\n## Cursor (or other AI IDEs)\n\nIf you prefer Cursor, VS Code Copilot Chat, or another AI-assisted IDE:\n\n1. Store the builder skill text in a snippet (Cursor: *Settings -> Custom Instructions*).\n2. Add quick prompts like \"Use the BSL builder skill\" so the IDE pastes the instructions before generating code.\n3. Point the IDE at your actual data context (DuckDB schema, YAML file) so it can thread the builder guard rails through your request.\n\nRegardless of the host, the Builder Agent should always cite the same modeling patterns. That keeps upstream MCP/Query agents consistent because every semantic table passes through the same validation philosophy.\n", "queries": {}, "files": {} } diff --git a/docs/web/public/bsl-data/charting.json b/docs/web/public/bsl-data/charting.json index 7b50d0fb..0af82b6a 100644 --- a/docs/web/public/bsl-data/charting.json +++ b/docs/web/public/bsl-data/charting.json @@ -2,53 +2,9 @@ "markdown": "# Charting\n\nBSL includes built-in support for generating data visualizations from your semantic queries. Create charts directly from query results with automatic chart type detection or full custom control.\n\n## Installation\n\nTo use chart visualization, install with the appropriate backend:\n\n```bash\n# For Altair backend (default)\npip install 'boring-semantic-layer[viz-altair]'\n\n# For Plotly backend\npip install 'boring-semantic-layer[viz-plotly]'\n```\n\n## Quick Start\n\nHere's a simple example showing how to create a chart:\n\n```setup_chart_data\nimport ibis\nfrom boring_semantic_layer import to_semantic_table\n\ncon = ibis.duckdb.connect(\":memory:\")\nflights_data = ibis.memtable({\n \"origin\": [\"JFK\", \"LAX\", \"SFO\", \"ORD\", \"DFW\", \"ATL\", \"DEN\"],\n \"flight_count\": [150, 135, 89, 112, 98, 145, 78],\n \"avg_distance\": [2475, 1850, 1200, 950, 1100, 1650, 900]\n})\nflights_tbl = con.create_table(\"flights\", flights_data)\n\nflights_st = (\n to_semantic_table(flights_tbl, name=\"flights\")\n .with_dimensions(\n origin=lambda t: t.origin\n )\n .with_measures(\n flight_count=lambda t: t.flight_count.sum(),\n avg_distance=lambda t: t.avg_distance.mean()\n )\n)\n```\n\n\n\n```query_basic_chart\n# Query and chart in one fluent chain\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\"flight_count\")\n .order_by(ibis.desc(\"flight_count\"))\n .limit(5)\n)\n\nresult.chart()\n```\n\n\n\n\nThe `.chart()` method is available on query results from `.aggregate()`, `.order_by()`, `.limit()`, and `.mutate()` operations.\n\n\n## Backend Selection\n\nBSL supports two charting backends with different strengths:\n\n### Altair (Default)\n\n**Best for:** Web-native interactive visualizations, declarative specifications, embedding in notebooks and web apps.\n\n```python\n# Use Altair backend (default)\nchart = result.chart()\n# or explicitly\nchart = result.chart(backend=\"altair\")\n```\n\n**Features:**\n- Built on Vega-Lite grammar\n- Declarative JSON specifications\n- Great for interactive web visualizations\n- Excellent notebook integration\n\n### Plotly\n\n**Best for:** Rich interactive dashboards, 3D visualizations, extensive chart types, business intelligence tools.\n\n```python\n# Use Plotly backend\nchart = result.chart(backend=\"plotly\")\n```\n\n**Features:**\n- Extensive chart type library\n- Rich interactivity out of the box\n- Dashboard integration\n- Export to static formats\n\n## Auto-Detection\n\nBSL automatically detects the appropriate chart type based on your query structure:\n\n### Bar Chart (Categorical Data)\n\nSingle dimension + measure \u2192 Bar chart\n\n```query_bar_chart\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\"flight_count\")\n .order_by(ibis.desc(\"flight_count\"))\n)\n\nresult.chart()\n```\n\n\n\n**Auto-detected because:** Single categorical dimension (`origin`) with one measure (`flight_count`)\n\n### Time Series (Temporal Data)\n\nTime dimension + measure \u2192 Line chart with time-aware formatting\n\n```setup_timeseries\nimport ibis\nfrom boring_semantic_layer import to_semantic_table\n\ncon = ibis.duckdb.connect(\":memory:\")\ntimeseries_data = ibis.memtable({\n \"date\": [\"2024-01-01\", \"2024-01-02\", \"2024-01-03\", \"2024-01-04\", \"2024-01-05\", \"2024-01-06\", \"2024-01-07\"],\n \"flight_count\": [145, 152, 148, 139, 156, 161, 143]\n})\ntimeseries_tbl = con.create_table(\"daily_flights\", timeseries_data)\n\ndaily_flights_st = (\n to_semantic_table(timeseries_tbl, name=\"daily_flights\")\n .with_dimensions(\n date={\n \"expr\": lambda t: t.date.cast(\"date\"),\n \"is_time_dimension\": True,\n \"smallest_time_grain\": \"TIME_GRAIN_DAY\"\n }\n )\n .with_measures(\n flight_count=lambda t: t.flight_count.sum()\n )\n)\n```\n\n\n\n```query_timeseries\nresult = (\n daily_flights_st\n .group_by(\"date\")\n .aggregate(\"flight_count\")\n)\nresult.chart()\n```\n\n\n\n**Auto-detected because:** Dimension marked as `is_time_dimension=True`\n\n### Heatmap (Two Dimensions)\n\nTwo categorical dimensions + measure \u2192 Heatmap\n\n```setup_heatmap\nimport ibis\nfrom boring_semantic_layer import to_semantic_table\n\ncon = ibis.duckdb.connect(\":memory:\")\nroute_data = ibis.memtable({\n \"origin\": [\"JFK\", \"JFK\", \"LAX\", \"LAX\", \"SFO\", \"SFO\"],\n \"dest\": [\"LAX\", \"SFO\", \"JFK\", \"SFO\", \"JFK\", \"LAX\"],\n \"flight_count\": [45, 32, 43, 28, 31, 27]\n})\nroute_tbl = con.create_table(\"routes\", route_data)\n\nroutes_st = (\n to_semantic_table(route_tbl, name=\"routes\")\n .with_dimensions(\n origin=lambda t: t.origin,\n dest=lambda t: t.dest\n )\n .with_measures(\n flight_count=lambda t: t.flight_count.sum()\n )\n)\n```\n\n\n\n```query_heatmap\nresult = (\n routes_st\n .group_by(\"origin\", \"dest\")\n .aggregate(\"flight_count\")\n)\nresult.chart()\n```\n\n\n\n**Auto-detected because:** Two categorical dimensions with one measure\n\n### Multi-Series Charts\n\nMultiple measures \u2192 Grouped/overlaid visualization with color encoding\n\n```query_multi_measure\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\"flight_count\", \"avg_distance\")\n .limit(5)\n)\nresult.chart()\n```\n\n\n\n**Auto-detected because:** Multiple measures trigger automatic color encoding by measure name\n\n\n## Custom Specifications\n\nOverride auto-detection with custom specifications:\n\n### Change Mark Type And Add Styling\n\nCustomize the mark type while providing explicit encodings:\n\n```query_custom_mark\nimport ibis\n# Create line chart with custom spec\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\"flight_count\")\n .order_by(ibis.desc(\"flight_count\"))\n .limit(5)\n)\nresult.chart(spec={\n \"mark\": {\"type\": \"line\", \"color\": \"#e74c3c\"}\n})\n```\n\n\n\n\nYou don't need to provide full vega spec: the spec object is merged with the BSL's default one.\n\n\n## Export Formats\n\nExport charts in various formats for different use cases:\n\n```python\n# Interactive chart object (default)\nchart = result.chart()\n\n# JSON specification for web embedding\njson_spec = result.chart(format=\"json\")\n\n# PNG image (requires altair[all] or plotly)\npng_bytes = result.chart(format=\"png\")\n\n# SVG markup (requires altair[all] or plotly)\nsvg_str = result.chart(format=\"svg\")\n\n# Save to file\nwith open(\"my_chart.png\", \"wb\") as f:\n f.write(png_bytes)\n```\n\n**Available formats:**\n- `\"static\"` or `\"interactive\"` - Chart object (default)\n- `\"json\"` - JSON specification\n- `\"png\"` - PNG image bytes\n- `\"svg\"` - SVG markup string\n\n## Next Steps\n\n- Learn about [Query Methods](/querying/methods) to build complex queries\n- Explore [YAML Configuration](/building/yaml) for declarative semantic models\n- See [Compose Models](/building/compose) for joining semantic tables\n", "queries": { "setup_chart_data": { - "code": "import ibis\nfrom boring_semantic_layer import to_semantic_table\n\ncon = ibis.duckdb.connect(\":memory:\")\nflights_data = ibis.memtable({\n \"origin\": [\"JFK\", \"LAX\", \"SFO\", \"ORD\", \"DFW\", \"ATL\", \"DEN\"],\n \"flight_count\": [150, 135, 89, 112, 98, 145, 78],\n \"avg_distance\": [2475, 1850, 1200, 950, 1100, 1650, 900]\n})\nflights_tbl = con.create_table(\"flights\", flights_data)\n\nflights_st = (\n to_semantic_table(flights_tbl, name=\"flights\")\n .with_dimensions(\n origin=lambda t: t.origin\n )\n .with_measures(\n flight_count=lambda t: t.flight_count.sum(),\n avg_distance=lambda t: t.avg_distance.mean()\n )\n)", - "sql": "SELECT\n *\nFROM \"memory\".\"main\".\"flights\"", - "plan": "SemanticTable: flights\n origin [dim]\n flight_count [measure]\n avg_distance [measure]", - "table": { - "columns": [ - "origin", - "flight_count", - "avg_distance" - ], - "data": [ - [ - "JFK", - 150, - 2475 - ], - [ - "LAX", - 135, - 1850 - ], - [ - "SFO", - 89, - 1200 - ], - [ - "ORD", - 112, - 950 - ], - [ - "DFW", - 98, - 1100 - ], - [ - "ATL", - 145, - 1650 - ], - [ - "DEN", - 78, - 900 - ] - ] - } + "semantic_table": true, + "name": "flights", + "info": "Semantic table definition stored in context" }, "query_basic_chart": { "code": "# Query and chart in one fluent chain\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\"flight_count\")\n .order_by(ibis.desc(\"flight_count\"))\n .limit(5)\n)\n\nresult.chart()", @@ -81,6 +37,70 @@ 98 ] ] + }, + "chart": { + "chart_spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-65b589ba3da7959d7f1e1d2c1bdd1c23" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "flight_count", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "y": { + "field": "flight_count", + "type": "quantitative" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-65b589ba3da7959d7f1e1d2c1bdd1c23": [ + { + "origin": "JFK", + "flight_count": 150 + }, + { + "origin": "ATL", + "flight_count": 145 + }, + { + "origin": "LAX", + "flight_count": 135 + }, + { + "origin": "ORD", + "flight_count": 112 + }, + { + "origin": "DFW", + "flight_count": 98 + } + ] + } + } } }, "query_bar_chart": { @@ -122,6 +142,78 @@ 78 ] ] + }, + "chart": { + "chart_spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-614905082e7810dae76519e0046336d6" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "flight_count", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "y": { + "field": "flight_count", + "type": "quantitative" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-614905082e7810dae76519e0046336d6": [ + { + "origin": "JFK", + "flight_count": 150 + }, + { + "origin": "ATL", + "flight_count": 145 + }, + { + "origin": "LAX", + "flight_count": 135 + }, + { + "origin": "ORD", + "flight_count": 112 + }, + { + "origin": "DFW", + "flight_count": 98 + }, + { + "origin": "SFO", + "flight_count": 89 + }, + { + "origin": "DEN", + "flight_count": 78 + } + ] + } + } } }, "setup_timeseries": { @@ -163,6 +255,79 @@ 78 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-614905082e7810dae76519e0046336d6" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "flight_count", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "y": { + "field": "flight_count", + "type": "quantitative" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-614905082e7810dae76519e0046336d6": [ + { + "origin": "JFK", + "flight_count": 150 + }, + { + "origin": "ATL", + "flight_count": 145 + }, + { + "origin": "LAX", + "flight_count": 135 + }, + { + "origin": "ORD", + "flight_count": 112 + }, + { + "origin": "DFW", + "flight_count": 98 + }, + { + "origin": "SFO", + "flight_count": 89 + }, + { + "origin": "DEN", + "flight_count": 78 + } + ] + } + } } }, "query_timeseries": { @@ -176,8 +341,16 @@ ], "data": [ [ - "2024-01-01", - 145 + "2024-01-06", + 161 + ], + [ + "2024-01-07", + 143 + ], + [ + "2024-01-04", + 139 ], [ "2024-01-02", @@ -192,18 +365,85 @@ 156 ], [ - "2024-01-04", - 139 - ], - [ - "2024-01-06", - 161 - ], - [ - "2024-01-07", - 143 + "2024-01-01", + 145 ] ] + }, + "chart": { + "chart_spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-18703a3194e73895f220c3245cf9cd53" + }, + "mark": { + "type": "line" + }, + "encoding": { + "tooltip": [ + { + "field": "date", + "format": "%Y-%m-%d", + "type": "temporal" + }, + { + "field": "flight_count", + "type": "quantitative" + } + ], + "x": { + "axis": { + "labelAngle": -45 + }, + "field": "date", + "type": "temporal" + }, + "y": { + "field": "flight_count", + "type": "quantitative" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-18703a3194e73895f220c3245cf9cd53": [ + { + "date": "2024-01-07", + "flight_count": 143 + }, + { + "date": "2024-01-01", + "flight_count": 145 + }, + { + "date": "2024-01-06", + "flight_count": 161 + }, + { + "date": "2024-01-04", + "flight_count": 139 + }, + { + "date": "2024-01-02", + "flight_count": 152 + }, + { + "date": "2024-01-03", + "flight_count": 148 + }, + { + "date": "2024-01-05", + "flight_count": 156 + } + ] + } + } } }, "setup_heatmap": { @@ -217,8 +457,16 @@ ], "data": [ [ - "2024-01-01", - 145 + "2024-01-06", + 161 + ], + [ + "2024-01-07", + 143 + ], + [ + "2024-01-04", + 139 ], [ "2024-01-02", @@ -233,18 +481,86 @@ 156 ], [ - "2024-01-07", - 143 - ], - [ - "2024-01-04", - 139 - ], - [ - "2024-01-06", - 161 + "2024-01-01", + 145 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-2b30d9f8cbbef1d5cb6971365828d54e" + }, + "mark": { + "type": "line" + }, + "encoding": { + "tooltip": [ + { + "field": "date", + "format": "%Y-%m-%d", + "type": "temporal" + }, + { + "field": "flight_count", + "type": "quantitative" + } + ], + "x": { + "axis": { + "labelAngle": -45 + }, + "field": "date", + "type": "temporal" + }, + "y": { + "field": "flight_count", + "type": "quantitative" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-2b30d9f8cbbef1d5cb6971365828d54e": [ + { + "date": "2024-01-07", + "flight_count": 143 + }, + { + "date": "2024-01-06", + "flight_count": 161 + }, + { + "date": "2024-01-02", + "flight_count": 152 + }, + { + "date": "2024-01-03", + "flight_count": 148 + }, + { + "date": "2024-01-05", + "flight_count": 156 + }, + { + "date": "2024-01-04", + "flight_count": 139 + }, + { + "date": "2024-01-01", + "flight_count": 145 + } + ] + } + } } }, "query_heatmap": { @@ -259,36 +575,119 @@ ], "data": [ [ - "SFO", "JFK", - 31 - ], - [ - "LAX", "SFO", - 28 + 32 ], [ - "JFK", "SFO", - 32 + "JFK", + 31 ], [ "JFK", "LAX", 45 ], + [ + "SFO", + "LAX", + 27 + ], [ "LAX", "JFK", 43 ], [ - "SFO", "LAX", - 27 + "SFO", + 28 ] ] + }, + "chart": { + "chart_spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-f736eab34f3e285c6a383f25c102ee02" + }, + "mark": { + "type": "rect" + }, + "encoding": { + "color": { + "field": "flight_count", + "type": "quantitative" + }, + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "dest", + "type": "nominal" + }, + { + "field": "flight_count", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "y": { + "field": "dest", + "sort": null, + "type": "ordinal" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-f736eab34f3e285c6a383f25c102ee02": [ + { + "origin": "JFK", + "dest": "SFO", + "flight_count": 32 + }, + { + "origin": "SFO", + "dest": "JFK", + "flight_count": 31 + }, + { + "origin": "SFO", + "dest": "LAX", + "flight_count": 27 + }, + { + "origin": "JFK", + "dest": "LAX", + "flight_count": 45 + }, + { + "origin": "LAX", + "dest": "JFK", + "flight_count": 43 + }, + { + "origin": "LAX", + "dest": "SFO", + "flight_count": 28 + } + ] + } + } } }, "query_multi_measure": { @@ -308,9 +707,9 @@ 1650.0 ], [ - "LAX", - 135, - 1850.0 + "JFK", + 150, + 2475.0 ], [ "ORD", @@ -318,9 +717,9 @@ 950.0 ], [ - "DEN", - 78, - 900.0 + "DFW", + 98, + 1100.0 ], [ "SFO", @@ -328,6 +727,98 @@ 1200.0 ] ] + }, + "chart": { + "chart_spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-39f0b2d399d6597cfeabdbfb388597ac" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "avg_distance" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-39f0b2d399d6597cfeabdbfb388597ac": [ + { + "origin": "ATL", + "flight_count": 145, + "avg_distance": 1650.0 + }, + { + "origin": "SFO", + "flight_count": 89, + "avg_distance": 1200.0 + }, + { + "origin": "JFK", + "flight_count": 150, + "avg_distance": 2475.0 + }, + { + "origin": "ORD", + "flight_count": 112, + "avg_distance": 950.0 + }, + { + "origin": "LAX", + "flight_count": 135, + "avg_distance": 1850.0 + } + ] + } + } } }, "query_custom_mark": { @@ -361,6 +852,70 @@ 98 ] ] + }, + "chart": { + "chart_spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-65b589ba3da7959d7f1e1d2c1bdd1c23" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "flight_count", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "y": { + "field": "flight_count", + "type": "quantitative" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-65b589ba3da7959d7f1e1d2c1bdd1c23": [ + { + "origin": "JFK", + "flight_count": 150 + }, + { + "origin": "ATL", + "flight_count": 145 + }, + { + "origin": "LAX", + "flight_count": 135 + }, + { + "origin": "ORD", + "flight_count": 112 + }, + { + "origin": "DFW", + "flight_count": 98 + } + ] + } + } } } }, diff --git a/docs/web/public/bsl-data/compose.json b/docs/web/public/bsl-data/compose.json index 9745ab34..42a939d2 100644 --- a/docs/web/public/bsl-data/compose.json +++ b/docs/web/public/bsl-data/compose.json @@ -31,176 +31,28 @@ } }, "setup_semantic_models": { - "code": "# Create semantic tables\nflights_st = (\n to_semantic_table(flights_tbl, name=\"flights\")\n .with_dimensions(\n flight_id=lambda t: t.flight_id,\n carrier_code=lambda t: t.carrier_code,\n aircraft_id=lambda t: t.aircraft_id\n )\n .with_measures(\n flight_count=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n total_passengers=lambda t: t.passengers.sum()\n )\n)\n\ncarriers_st = (\n to_semantic_table(carriers_tbl, name=\"carriers\")\n .with_dimensions(\n code=lambda t: t.code,\n name=lambda t: t.name,\n country=lambda t: t.country\n )\n .with_measures(\n carrier_count=lambda t: t.count()\n )\n)\n\naircraft_st = (\n to_semantic_table(aircraft_tbl, name=\"aircraft\")\n .with_dimensions(\n id=lambda t: t.id,\n model=lambda t: t.model\n )\n .with_measures(\n aircraft_count=lambda t: t.count(),\n total_capacity=lambda t: t.capacity.sum()\n )\n)", - "sql": "SELECT\n *\nFROM \"memory\".\"main\".\"aircraft\"", - "plan": "SemanticTable: aircraft\n id [dim]\n model [dim]\n aircraft_count [measure]\n total_capacity [measure]", - "table": { - "columns": [ - "id", - "model", - "capacity" - ], - "data": [ - [ - 101, - "Boeing 737", - 180 - ], - [ - 102, - "Airbus A320", - 200 - ], - [ - 103, - "Boeing 777", - 350 - ] - ] - } + "semantic_table": true, + "name": "aircraft", + "info": "Semantic table definition stored in context" }, "level0_dimensions": { - "code": "flights_st.dimensions, flights_st.measures", - "sql": "SELECT\n *\nFROM \"memory\".\"main\".\"aircraft\"", - "plan": "SemanticTable: aircraft\n id [dim]\n model [dim]\n aircraft_count [measure]\n total_capacity [measure]", - "table": { - "columns": [ - "id", - "model", - "capacity" - ], - "data": [ - [ - 101, - "Boeing 737", - 180 - ], - [ - 102, - "Airbus A320", - 200 - ], - [ - 103, - "Boeing 777", - 350 - ] - ] - } + "semantic_table": true, + "name": "aircraft", + "info": "Semantic table definition stored in context" }, "level1_join": { - "code": "# Each flight row matches at most one carrier row\nflights_with_carriers = flights_st.join_one(\n carriers_st,\n lambda f, c: f.carrier_code == c.code\n)\n\n# Inspect dimensions - now includes both flights and carriers\nflights_with_carriers.dimensions, flights_with_carriers.measures", - "sql": "SELECT\n \"t2\".\"flight_id\",\n \"t2\".\"carrier_code\",\n \"t2\".\"aircraft_id\",\n \"t2\".\"distance\",\n \"t2\".\"passengers\",\n \"t3\".\"code\",\n \"t3\".\"name\",\n \"t3\".\"country\"\nFROM \"memory\".\"main\".\"flights\" AS \"t2\"\nLEFT OUTER JOIN \"memory\".\"main\".\"carriers\" AS \"t3\"\n ON \"t2\".\"carrier_code\" = \"t3\".\"code\"", - "plan": "SemanticTable: flights\n flight_id [dim]\n carrier_code [dim]\n aircraft_id [dim]\n flight_count [measure]\n total_distance [measure]\n total_passengers [measure]\n-> Join(left, right=carriers)", - "table": { - "columns": [ - "flight_id", - "carrier_code", - "aircraft_id", - "distance", - "passengers", - "code", - "name", - "country" - ], - "data": [ - [ - 1, - "AA", - 101, - 1000, - 150, - "AA", - "American Airlines", - "USA" - ], - [ - 2, - "UA", - 102, - 1500, - 180, - "UA", - "United Airlines", - "USA" - ], - [ - 3, - "DL", - 103, - 800, - 120, - "DL", - "Delta Air Lines", - "USA" - ] - ] - } + "semantic_table": true, + "name": "unknown", + "info": "Semantic table definition stored in context" }, "level2_join": { - "code": "# Each flight row matches at most one aircraft row\nfull_model = flights_with_carriers.join_one(\n aircraft_st,\n lambda f, a: f.aircraft_id == a.id\n)\n\n# Inspect dimensions - now includes flights, carriers, AND aircraft\nfull_model.dimensions, full_model.measures", - "sql": "SELECT\n \"t3\".\"flight_id\",\n \"t3\".\"carrier_code\",\n \"t3\".\"aircraft_id\",\n \"t3\".\"distance\",\n \"t3\".\"passengers\",\n \"t4\".\"code\",\n \"t4\".\"name\",\n \"t4\".\"country\",\n \"t5\".\"id\",\n \"t5\".\"model\",\n \"t5\".\"capacity\"\nFROM \"memory\".\"main\".\"flights\" AS \"t3\"\nLEFT OUTER JOIN \"memory\".\"main\".\"carriers\" AS \"t4\"\n ON \"t3\".\"carrier_code\" = \"t4\".\"code\"\nLEFT OUTER JOIN \"memory\".\"main\".\"aircraft\" AS \"t5\"\n ON \"t3\".\"aircraft_id\" = \"t5\".\"id\"", - "plan": "SemanticTable: flights\n flight_id [dim]\n carrier_code [dim]\n aircraft_id [dim]\n flight_count [measure]\n total_distance [measure]\n total_passengers [measure]\n-> Join(left, right=carriers)\n-> Join(left, right=aircraft)", - "table": { - "columns": [ - "flight_id", - "carrier_code", - "aircraft_id", - "distance", - "passengers", - "code", - "name", - "country", - "id", - "model", - "capacity" - ], - "data": [ - [ - 1, - "AA", - 101, - 1000, - 150, - "AA", - "American Airlines", - "USA", - 101, - "Boeing 737", - 180 - ], - [ - 2, - "UA", - 102, - 1500, - 180, - "UA", - "United Airlines", - "USA", - 102, - "Airbus A320", - 200 - ], - [ - 3, - "DL", - 103, - 800, - 120, - "DL", - "Delta Air Lines", - "USA", - 103, - "Boeing 777", - 350 - ] - ] - } + "semantic_table": true, + "name": "unknown", + "info": "Semantic table definition stored in context" }, "composed_query": { "code": "# Query using dimensions and measures from all three tables\nresult = (\n full_model\n .group_by( \"aircraft.model\")\n .aggregate(\"flights.flight_count\", \"flights.total_passengers\", \"aircraft.total_capacity\")\n)", - "sql": "WITH \"t10\" AS (\n SELECT\n \"t3\".\"flight_id\",\n \"t3\".\"carrier_code\",\n \"t3\".\"aircraft_id\",\n \"t3\".\"distance\",\n \"t3\".\"passengers\",\n \"t5\".\"code\",\n \"t5\".\"name\",\n \"t5\".\"country\",\n \"t4\".\"id\",\n \"t4\".\"model\",\n \"t4\".\"capacity\"\n FROM \"memory\".\"main\".\"flights\" AS \"t3\"\n LEFT OUTER JOIN \"memory\".\"main\".\"carriers\" AS \"t5\"\n ON \"t3\".\"carrier_code\" = \"t5\".\"code\"\n LEFT OUTER JOIN \"memory\".\"main\".\"aircraft\" AS \"t4\"\n ON \"t3\".\"aircraft_id\" = \"t4\".\"id\"\n)\nSELECT\n *\nFROM (\n SELECT\n \"t21\".\"aircraft.model\",\n \"t21\".\"flights.flight_count\",\n \"t21\".\"flights.total_passengers\",\n \"t20\".\"aircraft.total_capacity\"\n FROM (\n SELECT\n \"t17\".\"aircraft.model\",\n SUM(\"t17\".\"flights.flight_count\") AS \"flights.flight_count\",\n SUM(\"t17\".\"flights.total_passengers\") AS \"flights.total_passengers\"\n FROM (\n SELECT\n \"t15\".\"aircraft.model\",\n \"t15\".\"aircraft_id\",\n \"t15\".\"carrier_code\",\n \"t8\".\"flights.flight_count\",\n \"t8\".\"flights.total_passengers\"\n FROM (\n SELECT DISTINCT\n \"t11\".\"model\" AS \"aircraft.model\",\n \"t11\".\"aircraft_id\",\n \"t11\".\"carrier_code\"\n FROM \"t10\" AS \"t11\"\n ) AS \"t15\"\n LEFT OUTER JOIN (\n SELECT\n \"t0\".\"aircraft_id\",\n \"t0\".\"carrier_code\",\n COUNT(*) AS \"flights.flight_count\",\n SUM(\"t0\".\"passengers\") AS \"flights.total_passengers\"\n FROM \"memory\".\"main\".\"flights\" AS \"t0\"\n GROUP BY\n 1,\n 2\n ) AS \"t8\"\n ON \"t15\".\"aircraft_id\" = \"t8\".\"aircraft_id\"\n AND \"t15\".\"carrier_code\" = \"t8\".\"carrier_code\"\n ) AS \"t17\"\n GROUP BY\n 1\n ) AS \"t21\"\n LEFT OUTER JOIN (\n SELECT\n \"t16\".\"aircraft.model\",\n SUM(\"t16\".\"aircraft.total_capacity\") AS \"aircraft.total_capacity\"\n FROM (\n SELECT\n \"t14\".\"aircraft.model\",\n \"t14\".\"model\",\n \"t9\".\"aircraft.total_capacity\"\n FROM (\n SELECT DISTINCT\n \"t11\".\"model\" AS \"aircraft.model\",\n \"t11\".\"model\"\n FROM \"t10\" AS \"t11\"\n ) AS \"t14\"\n LEFT OUTER JOIN (\n SELECT\n \"t1\".\"model\",\n SUM(\"t1\".\"capacity\") AS \"aircraft.total_capacity\"\n FROM \"memory\".\"main\".\"aircraft\" AS \"t1\"\n GROUP BY\n 1\n ) AS \"t9\"\n ON \"t14\".\"model\" = \"t9\".\"model\"\n ) AS \"t16\"\n GROUP BY\n 1\n ) AS \"t20\"\n ON \"t21\".\"aircraft.model\" = \"t20\".\"aircraft.model\"\n) AS \"t22\"", + "sql": "WITH \"t9\" AS (\n SELECT\n \"t3\".\"flight_id\",\n \"t3\".\"carrier_code\",\n \"t3\".\"aircraft_id\",\n \"t3\".\"distance\",\n \"t3\".\"passengers\",\n \"t4\".\"code\",\n \"t4\".\"name\",\n \"t4\".\"country\",\n \"t5\".\"id\",\n \"t5\".\"model\",\n \"t5\".\"capacity\"\n FROM \"memory\".\"main\".\"flights\" AS \"t3\"\n LEFT OUTER JOIN \"memory\".\"main\".\"carriers\" AS \"t4\"\n ON \"t3\".\"carrier_code\" = \"t4\".\"code\"\n LEFT OUTER JOIN \"memory\".\"main\".\"aircraft\" AS \"t5\"\n ON \"t3\".\"aircraft_id\" = \"t5\".\"id\"\n)\nSELECT\n \"t26\".\"aircraft.model\",\n COALESCE(\"t26\".\"flights.flight_count\", 0) AS \"flights.flight_count\",\n \"t26\".\"flights.total_passengers\",\n \"t26\".\"aircraft.total_capacity\"\nFROM (\n SELECT\n \"t22\".\"aircraft.model\",\n \"t22\".\"flights.flight_count\",\n \"t22\".\"flights.total_passengers\",\n \"t25\".\"aircraft.total_capacity\"\n FROM (\n SELECT\n \"t18\".\"aircraft.model\",\n SUM(\"t18\".\"flights.flight_count\") AS \"flights.flight_count\",\n SUM(\"t18\".\"flights.total_passengers\") AS \"flights.total_passengers\"\n FROM (\n SELECT\n \"t15\".\"aircraft.model\",\n \"t15\".\"aircraft_id\",\n \"t15\".\"carrier_code\",\n \"t8\".\"flights.flight_count\",\n \"t8\".\"flights.total_passengers\"\n FROM (\n SELECT DISTINCT\n \"t10\".\"model\" AS \"aircraft.model\",\n \"t10\".\"aircraft_id\",\n \"t10\".\"carrier_code\"\n FROM \"t9\" AS \"t10\"\n ) AS \"t15\"\n LEFT OUTER JOIN (\n SELECT\n \"t0\".\"aircraft_id\",\n \"t0\".\"carrier_code\",\n COUNT(*) AS \"flights.flight_count\",\n SUM(\"t0\".\"passengers\") AS \"flights.total_passengers\"\n FROM \"memory\".\"main\".\"flights\" AS \"t0\"\n GROUP BY\n 1,\n 2\n ) AS \"t8\"\n ON (\n (\n \"t15\".\"aircraft_id\" = \"t8\".\"aircraft_id\"\n )\n OR (\n (\n \"t15\".\"aircraft_id\" IS NULL\n ) AND (\n \"t8\".\"aircraft_id\" IS NULL\n )\n )\n )\n AND (\n (\n \"t15\".\"carrier_code\" = \"t8\".\"carrier_code\"\n )\n OR (\n (\n \"t15\".\"carrier_code\" IS NULL\n ) AND (\n \"t8\".\"carrier_code\" IS NULL\n )\n )\n )\n ) AS \"t18\"\n GROUP BY\n 1\n ) AS \"t22\"\n LEFT OUTER JOIN (\n SELECT\n \"t23\".\"aircraft.model\",\n SUM(\"t23\".\"aircraft.total_capacity\") AS \"aircraft.total_capacity\"\n FROM (\n SELECT\n \"t14\".\"aircraft.model\",\n \"t14\".\"model\",\n \"t20\".\"aircraft.total_capacity\"\n FROM (\n SELECT DISTINCT\n \"t10\".\"model\" AS \"aircraft.model\",\n \"t10\".\"model\"\n FROM \"t9\" AS \"t10\"\n ) AS \"t14\"\n LEFT OUTER JOIN (\n SELECT\n \"t17\".\"model\",\n SUM(\"t17\".\"capacity\") AS \"aircraft.total_capacity\"\n FROM (\n SELECT\n \"t6\".\"id\",\n \"t6\".\"model\",\n \"t6\".\"capacity\"\n FROM \"memory\".\"main\".\"aircraft\" AS \"t6\"\n INNER JOIN (\n SELECT DISTINCT\n \"t10\".\"id\"\n FROM \"t9\" AS \"t10\"\n ) AS \"t16\"\n ON \"t6\".\"id\" = \"t16\".\"id\"\n ) AS \"t17\"\n GROUP BY\n 1\n ) AS \"t20\"\n ON (\n \"t14\".\"model\" = \"t20\".\"model\"\n )\n OR (\n (\n \"t14\".\"model\" IS NULL\n ) AND (\n \"t20\".\"model\" IS NULL\n )\n )\n ) AS \"t23\"\n GROUP BY\n 1\n ) AS \"t25\"\n ON (\n \"t22\".\"aircraft.model\" = \"t25\".\"aircraft.model\"\n )\n OR (\n (\n \"t22\".\"aircraft.model\" IS NULL\n ) AND (\n \"t25\".\"aircraft.model\" IS NULL\n )\n )\n) AS \"t26\"", "plan": "SemanticTable: flights\n flight_id [dim]\n carrier_code [dim]\n aircraft_id [dim]\n flight_count [measure]\n total_distance [measure]\n total_passengers [measure]\n-> Join(left, right=carriers)\n-> Join(left, right=aircraft)\n-> GroupBy(aircraft.model)\n-> Aggregate(flights.flight_count, flights.total_passengers, aircraft.total_capacity)", "table": { "columns": [ @@ -229,6 +81,93 @@ 200 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-45f971d4ebf6cec4d62aafb35577a7ff" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "aircraft_model", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "aircraft_model", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flights_flight_count", + "flights_total_passengers", + "aircraft_total_capacity" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-45f971d4ebf6cec4d62aafb35577a7ff": [ + { + "aircraft_model": "Boeing 777", + "flights_flight_count": 1, + "flights_total_passengers": 120, + "aircraft_total_capacity": 350 + }, + { + "aircraft_model": "Boeing 737", + "flights_flight_count": 1, + "flights_total_passengers": 150, + "aircraft_total_capacity": 180 + }, + { + "aircraft_model": "Airbus A320", + "flights_flight_count": 1, + "flights_total_passengers": 180, + "aircraft_total_capacity": 200 + } + ] + } + } } } }, diff --git a/docs/web/public/bsl-data/example.json b/docs/web/public/bsl-data/example.json index 61b1ecb1..a0f194d0 100644 --- a/docs/web/public/bsl-data/example.json +++ b/docs/web/public/bsl-data/example.json @@ -3,7 +3,7 @@ "queries": { "revenue_by_customer": { "code": "result = orders_st.group_by(\"customer\").aggregate(\n \"total_orders\",\n \"total_revenue\",\n \"avg_order_value\"\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"customer\",\n COUNT(*) AS \"total_orders\",\n SUM(\"t1\".\"amount\") AS \"total_revenue\",\n AVG(\"t1\".\"amount\") AS \"avg_order_value\"\n FROM (\n SELECT\n \"t1\".\"order_id\",\n \"t1\".\"product\",\n \"t1\".\"amount\",\n \"t1\".\"quantity\",\n \"t1\".\"customer\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_kod4h4nvwbbpfag4osr7gtlqpm\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"customer\"\n) AS \"t2\"", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"customer\",\n COUNT(*) AS \"total_orders\",\n SUM(\"t1\".\"amount\") AS \"total_revenue\",\n AVG(\"t1\".\"amount\") AS \"avg_order_value\"\n FROM (\n SELECT\n \"t1\".\"order_id\",\n \"t1\".\"product\",\n \"t1\".\"amount\",\n \"t1\".\"quantity\",\n \"t1\".\"customer\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_j7tgs6q34bg2xdzzd7srg72ho4\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"customer\"\n) AS \"t2\"", "plan": "SemanticTable: orders\n customer [dim]\n product [dim]\n total_orders [measure]\n total_revenue [measure]\n total_quantity [measure]\n avg_order_value [measure]\n-> GroupBy(customer)\n-> Aggregate(total_orders, total_revenue, avg_order_value)", "table": { "columns": [ @@ -19,12 +19,6 @@ 350, 116.66666666666667 ], - [ - "Bob", - 2, - 250, - 125.0 - ], [ "Charlie", 2, @@ -36,13 +30,112 @@ 1, 100, 100.0 + ], + [ + "Bob", + 2, + 250, + 125.0 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-051431f27a158a793a9e800d4c006ac8" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "customer", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "customer", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "total_orders", + "total_revenue", + "avg_order_value" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-051431f27a158a793a9e800d4c006ac8": [ + { + "customer": "Alice", + "total_orders": 3, + "total_revenue": 350, + "avg_order_value": 116.66666666666667 + }, + { + "customer": "Charlie", + "total_orders": 2, + "total_revenue": 225, + "avg_order_value": 112.5 + }, + { + "customer": "David", + "total_orders": 1, + "total_revenue": 100, + "avg_order_value": 100.0 + }, + { + "customer": "Bob", + "total_orders": 2, + "total_revenue": 250, + "avg_order_value": 125.0 + } + ] + } + } } }, "product_performance": { "code": "result = orders_st.group_by(\"product\").aggregate(\n \"total_orders\",\n \"total_quantity\",\n \"total_revenue\"\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"product\",\n COUNT(*) AS \"total_orders\",\n SUM(\"t1\".\"quantity\") AS \"total_quantity\",\n SUM(\"t1\".\"amount\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t1\".\"order_id\",\n \"t1\".\"customer\",\n \"t1\".\"amount\",\n \"t1\".\"quantity\",\n \"t1\".\"product\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_kod4h4nvwbbpfag4osr7gtlqpm\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"product\"\n) AS \"t2\"", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"product\",\n COUNT(*) AS \"total_orders\",\n SUM(\"t1\".\"quantity\") AS \"total_quantity\",\n SUM(\"t1\".\"amount\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t1\".\"order_id\",\n \"t1\".\"customer\",\n \"t1\".\"amount\",\n \"t1\".\"quantity\",\n \"t1\".\"product\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_j7tgs6q34bg2xdzzd7srg72ho4\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"product\"\n) AS \"t2\"", "plan": "SemanticTable: orders\n customer [dim]\n product [dim]\n total_orders [measure]\n total_revenue [measure]\n total_quantity [measure]\n avg_order_value [measure]\n-> GroupBy(product)\n-> Aggregate(total_orders, total_quantity, total_revenue)", "table": { "columns": [ @@ -52,12 +145,6 @@ "total_revenue" ], "data": [ - [ - "Widget", - 4, - 4, - 400 - ], [ "Doohickey", 1, @@ -69,8 +156,101 @@ 3, 6, 450 + ], + [ + "Widget", + 4, + 4, + 400 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-6991e3d3e667615e632b8261337a1da8" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "product", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "product", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "total_orders", + "total_quantity", + "total_revenue" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-6991e3d3e667615e632b8261337a1da8": [ + { + "product": "Gadget", + "total_orders": 3, + "total_quantity": 6, + "total_revenue": 450 + }, + { + "product": "Widget", + "total_orders": 4, + "total_quantity": 4, + "total_revenue": 400 + }, + { + "product": "Doohickey", + "total_orders": 1, + "total_quantity": 3, + "total_revenue": 75 + } + ] + } + } } } }, diff --git a/docs/web/public/bsl-data/getting-started.json b/docs/web/public/bsl-data/getting-started.json index f4ada0f1..0b3d8e76 100644 --- a/docs/web/public/bsl-data/getting-started.json +++ b/docs/web/public/bsl-data/getting-started.json @@ -3,7 +3,7 @@ "queries": { "query_by_origin": { "code": "# Group flights by origin airport\nresult = flights_st.group_by(\"origin\").aggregate(\n \"flight_count\",\n \"total_distance\",\n \"avg_duration\"\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n SUM(\"t1\".\"distance\") AS \"total_distance\",\n AVG(\"t1\".\"duration\") AS \"avg_duration\"\n FROM (\n SELECT\n \"t1\".\"destination\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_ezhx46n3pnbczjgagwnn24amcu\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n SUM(\"t1\".\"distance\") AS \"total_distance\",\n AVG(\"t1\".\"duration\") AS \"avg_duration\"\n FROM (\n SELECT\n \"t1\".\"destination\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_i6d2okpzpffkllrkd54ddcja3y\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", "plan": "SemanticTable: flights\n origin [dim]\n destination [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy(origin)\n-> Aggregate(flight_count, total_distance, avg_duration)", "table": { "columns": [ @@ -19,24 +19,111 @@ 8480, 340.0 ], - [ - "SFO", - 2, - 3249, - 212.5 - ], [ "LAX", 3, 3483, 153.33333333333334 + ], + [ + "SFO", + 2, + 3249, + 212.5 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-0137bc47e458ebaf211d83de5a1f535d" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "total_distance", + "avg_duration" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-0137bc47e458ebaf211d83de5a1f535d": [ + { + "origin": "NYC", + "flight_count": 3, + "total_distance": 8480, + "avg_duration": 340.0 + }, + { + "origin": "LAX", + "flight_count": 3, + "total_distance": 3483, + "avg_duration": 153.33333333333334 + }, + { + "origin": "SFO", + "flight_count": 2, + "total_distance": 3249, + "avg_duration": 212.5 + } + ] + } + } } }, "query_by_destination": { "code": "# Group flights by destination airport\nresult = flights_st.group_by(\"destination\").aggregate(\n \"flight_count\",\n \"total_distance\"\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"destination\",\n COUNT(*) AS \"flight_count\",\n SUM(\"t1\".\"distance\") AS \"total_distance\"\n FROM (\n SELECT\n \"t1\".\"origin\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"destination\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_ezhx46n3pnbczjgagwnn24amcu\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"destination\"\n) AS \"t2\"", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"destination\",\n COUNT(*) AS \"flight_count\",\n SUM(\"t1\".\"distance\") AS \"total_distance\"\n FROM (\n SELECT\n \"t1\".\"origin\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"destination\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_i6d2okpzpffkllrkd54ddcja3y\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"destination\"\n) AS \"t2\"", "plan": "SemanticTable: flights\n origin [dim]\n destination [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy(destination)\n-> Aggregate(flight_count, total_distance)", "table": { "columns": [ @@ -50,17 +137,100 @@ 3, 3596 ], - [ - "NYC", - 2, - 5691 - ], [ "LAX", 3, 5925 + ], + [ + "NYC", + 2, + 5691 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-4000cbb95248830b8574b5a3a4390b5b" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "destination", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "destination", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "total_distance" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-4000cbb95248830b8574b5a3a4390b5b": [ + { + "destination": "LAX", + "flight_count": 3, + "total_distance": 5925 + }, + { + "destination": "NYC", + "flight_count": 2, + "total_distance": 5691 + }, + { + "destination": "SFO", + "flight_count": 3, + "total_distance": 3596 + } + ] + } + } } } }, diff --git a/docs/web/public/bsl-data/indexing.json b/docs/web/public/bsl-data/indexing.json index 856af25c..8bef1cdd 100644 --- a/docs/web/public/bsl-data/indexing.json +++ b/docs/web/public/bsl-data/indexing.json @@ -2,164 +2,13 @@ "markdown": "# Dimensional Indexing\n\nCreate a searchable catalog of all unique values across your dimensions for data exploration, autocomplete features, and understanding data distributions. Inspired by [Malloy's index pattern](https://docs.malloydata.dev/documentation/patterns/dim_index).\n\n## Overview\n\nDimensional indexing allows you to:\n\n- **Catalog all values**: Extract and count all unique values across dimensions\n- **Search dimensions**: Build autocomplete and search features\n- **Profile data**: Understand cardinality and distributions\n- **Weight by measures**: Find values ranked by custom metrics (e.g., highest revenue cities)\n- **Index across joins**: Search values from related tables\n\nThe `index()` method returns a standardized table with columns:\n- `fieldName`: The dimension name\n- `fieldValue`: The unique value\n- `fieldType`: The data type (string, number, etc.)\n- `weight`: Count or custom measure value for ranking\n\n## Setup\n\nLet's create an airports semantic table for our examples:\n\n```setup_airports\nimport ibis\nfrom boring_semantic_layer import to_semantic_table\n\n# Create synthetic airports data\nairports_data = ibis.memtable({\n \"code\": [\"JFK\", \"LAX\", \"ORD\", \"ATL\", \"DFW\", \"DEN\", \"SFO\", \"LAS\", \"SEA\", \"PHX\",\n \"IAH\", \"MCO\", \"EWR\", \"BOS\", \"MIA\", \"SAN\", \"LGA\", \"PHL\", \"DTW\", \"MSP\"],\n \"city\": [\"NEW YORK\", \"LOS ANGELES\", \"CHICAGO\", \"ATLANTA\", \"DALLAS\", \"DENVER\",\n \"SAN FRANCISCO\", \"LAS VEGAS\", \"SEATTLE\", \"PHOENIX\", \"HOUSTON\", \"ORLANDO\",\n \"NEWARK\", \"BOSTON\", \"MIAMI\", \"SAN DIEGO\", \"NEW YORK\", \"PHILADELPHIA\",\n \"DETROIT\", \"MINNEAPOLIS\"],\n \"state\": [\"NY\", \"CA\", \"IL\", \"GA\", \"TX\", \"CO\", \"CA\", \"NV\", \"WA\", \"AZ\",\n \"TX\", \"FL\", \"NJ\", \"MA\", \"FL\", \"CA\", \"NY\", \"PA\", \"MI\", \"MN\"],\n \"fac_type\": [\"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\",\n \"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\",\n \"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\",\n \"AIRPORT\", \"AIRPORT\"],\n \"elevation\": [13, 128, 672, 1026, 607, 5433, 13, 2181, 433, 1135,\n 97, 96, 18, 19, 8, 17, 21, 36, 645, 841]\n})\n\n# Define semantic table\nairports = (\n to_semantic_table(airports_data, name=\"airports\")\n .with_dimensions(\n code=lambda t: t.code,\n city=lambda t: t.city,\n state=lambda t: t.state,\n fac_type=lambda t: t.fac_type,\n elevation=lambda t: t.elevation,\n )\n .with_measures(\n airport_count=lambda t: t.count(),\n avg_elevation=lambda t: t.elevation.mean(),\n )\n)\n```\n\n\n\n## Basic Index: All Dimensions\n\nIndex all dimensions to see every unique value with its frequency:\n\n```query_index_all\n# Index all dimensions (None means all)\nresult = airports.index(None).limit(10)\n```\n\n\n\nThe `weight` column shows the count for each value. Use this to understand which values are most common across your dataset.\n\n## Index Specific Fields\n\nFocus on specific dimensions by selecting them:\n\n```query_index_specific\n# Index only state and city\nresult = (\n airports.index(lambda t: [t.state, t.city])\n .order_by(lambda t: t.weight.desc())\n .limit(10)\n)\n```\n\n\n\nThis is useful when you only care about certain dimensions, reducing noise and improving performance.\n\n## Search Pattern: Autocomplete\n\nBuild autocomplete features by filtering the index with pattern matching:\n\n```query_autocomplete\n# Get city suggestions starting with \"SAN\"\nresult = (\n airports.index(lambda t: t.city)\n .filter(lambda t: t.fieldValue.like(\"SAN%\"))\n .order_by(lambda t: t.weight.desc())\n .limit(10)\n)\n```\n\n\n\n\nUse pattern matching with `like()` to implement autocomplete, search suggestions, or fuzzy matching features in your application.\n\n\n## Filter by Field Type\n\nAnalyze only string or numeric fields:\n\n```query_by_type\n# Get only string field values\nresult = (\n airports.index(None)\n .filter(lambda t: t.fieldType == \"string\")\n .order_by(lambda t: t.weight.desc())\n .limit(10)\n)\n```\n\n\n\nThis helps when you want to focus on categorical vs. numeric dimensions separately.\n\n## Custom Weights: Rank by Measure\n\nInstead of counting occurrences, weight values by a custom measure:\n\n```query_custom_weight\n# Find states with most airports\nresult = (\n airports.index(lambda t: t.state, by=\"airport_count\")\n .order_by(lambda t: t.weight.desc())\n .limit(10)\n)\n```\n\n\n\n\nThe `by` parameter lets you rank dimension values by any measure. This is powerful for finding \"top cities by revenue\", \"states by average temperature\", etc.\n\n\n## Sampling for Large Datasets\n\nFor very large datasets, use sampling to get quick insights:\n\n```query_sampled\n# Sample 100 rows before indexing\nresult = (\n airports.index(None, sample=100)\n .filter(lambda t: t.fieldType == \"string\")\n .order_by(lambda t: t.weight.desc())\n .limit(10)\n)\n```\n\n\n\nSampling trades perfect accuracy for speed, which is often acceptable for exploratory analysis.\n\n## Index Across Joins\n\nIndex dimensions from joined tables:\n\n```query_index_joins\n# Create synthetic flights data\nflights_data = ibis.memtable({\n \"flight_id\": list(range(1, 31)),\n \"carrier\": [\"AA\", \"UA\", \"DL\", \"WN\", \"B6\", \"AA\", \"UA\", \"DL\", \"WN\", \"B6\"] * 3,\n \"origin\": [\"JFK\", \"LAX\", \"ORD\", \"ATL\", \"DFW\", \"SFO\", \"SEA\", \"DEN\", \"PHX\", \"BOS\"] * 3,\n})\n\nflights = (\n to_semantic_table(flights_data, name=\"flights\")\n .with_dimensions(\n carrier=lambda t: t.carrier,\n origin=lambda t: t.origin,\n )\n .with_measures(\n flight_count=lambda t: t.count(),\n )\n)\n\n# Join flights with airports\nflights_with_origin = flights.join_one(airports, lambda f, a: f.origin == a.code)\n\n# Index across the join\nresult = (\n flights_with_origin.index([\"flights.carrier\", \"airports.state\"])\n .order_by(lambda t: t.weight.desc())\n .limit(10)\n)\n```\n\n\n\n\nWhen referencing dimensions from joined tables in the index, use dot notation with table name prefix: `\"airports.state\"` instead of just `\"state\"`.\n\n\n## Use Cases\n\n**Data Discovery**: Quickly explore what values exist in your dimensions without writing complex group-by queries. Perfect for understanding unfamiliar datasets.\n\n**Autocomplete & Search**: Build type-ahead search features by indexing dimension values and filtering with pattern matching. The weight helps rank suggestions by relevance.\n\n**Data Profiling**: Understand data quality by examining cardinality, common values, and distributions across dimensions. Identify outliers or data entry errors.\n\n**Metric-Weighted Ranking**: Find dimension values that matter most for your metrics - e.g., \"cities with highest revenue\", \"products with most returns\", \"states with longest delivery times\".\n\n**Cross-Table Search**: Index dimensions across joined tables to search related data simultaneously, enabling unified search experiences.\n\n## Key Takeaways\n\n- Use `index(None)` to catalog all dimension values\n- Use `index(lambda t: [t.field1, t.field2])` for specific fields or `index(lambda t: t.field)` for a single field\n- Filter by `fieldType` to focus on strings or numbers\n- Use `by=\"measure_name\"` to weight by custom measures instead of counts\n- Add `sample=N` to analyze large datasets quickly\n- The index works across joins - use `\"table.field\"` syntax for joined dimensions\n- Perfect for building autocomplete, search, and data profiling features\n\n## Next Steps\n\n- Learn about [Nested Subtotals](/advanced/nested-subtotals) for hierarchical data structures\n- Explore [Query Methods](/querying/methods) for more query patterns\n", "queries": { "setup_airports": { - "code": "import ibis\nfrom boring_semantic_layer import to_semantic_table\n\n# Create synthetic airports data\nairports_data = ibis.memtable({\n \"code\": [\"JFK\", \"LAX\", \"ORD\", \"ATL\", \"DFW\", \"DEN\", \"SFO\", \"LAS\", \"SEA\", \"PHX\",\n \"IAH\", \"MCO\", \"EWR\", \"BOS\", \"MIA\", \"SAN\", \"LGA\", \"PHL\", \"DTW\", \"MSP\"],\n \"city\": [\"NEW YORK\", \"LOS ANGELES\", \"CHICAGO\", \"ATLANTA\", \"DALLAS\", \"DENVER\",\n \"SAN FRANCISCO\", \"LAS VEGAS\", \"SEATTLE\", \"PHOENIX\", \"HOUSTON\", \"ORLANDO\",\n \"NEWARK\", \"BOSTON\", \"MIAMI\", \"SAN DIEGO\", \"NEW YORK\", \"PHILADELPHIA\",\n \"DETROIT\", \"MINNEAPOLIS\"],\n \"state\": [\"NY\", \"CA\", \"IL\", \"GA\", \"TX\", \"CO\", \"CA\", \"NV\", \"WA\", \"AZ\",\n \"TX\", \"FL\", \"NJ\", \"MA\", \"FL\", \"CA\", \"NY\", \"PA\", \"MI\", \"MN\"],\n \"fac_type\": [\"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\",\n \"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\",\n \"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\", \"AIRPORT\",\n \"AIRPORT\", \"AIRPORT\"],\n \"elevation\": [13, 128, 672, 1026, 607, 5433, 13, 2181, 433, 1135,\n 97, 96, 18, 19, 8, 17, 21, 36, 645, 841]\n})\n\n# Define semantic table\nairports = (\n to_semantic_table(airports_data, name=\"airports\")\n .with_dimensions(\n code=lambda t: t.code,\n city=lambda t: t.city,\n state=lambda t: t.state,\n fac_type=lambda t: t.fac_type,\n elevation=lambda t: t.elevation,\n )\n .with_measures(\n airport_count=lambda t: t.count(),\n avg_elevation=lambda t: t.elevation.mean(),\n )\n)", - "sql": "SELECT\n *\nFROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\"", - "plan": "SemanticTable: airports\n code [dim]\n city [dim]\n state [dim]\n fac_type [dim]\n elevation [dim]\n airport_count [measure]\n avg_elevation [measure]", - "table": { - "columns": [ - "code", - "city", - "state", - "fac_type", - "elevation" - ], - "data": [ - [ - "JFK", - "NEW YORK", - "NY", - "AIRPORT", - 13 - ], - [ - "LAX", - "LOS ANGELES", - "CA", - "AIRPORT", - 128 - ], - [ - "ORD", - "CHICAGO", - "IL", - "AIRPORT", - 672 - ], - [ - "ATL", - "ATLANTA", - "GA", - "AIRPORT", - 1026 - ], - [ - "DFW", - "DALLAS", - "TX", - "AIRPORT", - 607 - ], - [ - "DEN", - "DENVER", - "CO", - "AIRPORT", - 5433 - ], - [ - "SFO", - "SAN FRANCISCO", - "CA", - "AIRPORT", - 13 - ], - [ - "LAS", - "LAS VEGAS", - "NV", - "AIRPORT", - 2181 - ], - [ - "SEA", - "SEATTLE", - "WA", - "AIRPORT", - 433 - ], - [ - "PHX", - "PHOENIX", - "AZ", - "AIRPORT", - 1135 - ], - [ - "IAH", - "HOUSTON", - "TX", - "AIRPORT", - 97 - ], - [ - "MCO", - "ORLANDO", - "FL", - "AIRPORT", - 96 - ], - [ - "EWR", - "NEWARK", - "NJ", - "AIRPORT", - 18 - ], - [ - "BOS", - "BOSTON", - "MA", - "AIRPORT", - 19 - ], - [ - "MIA", - "MIAMI", - "FL", - "AIRPORT", - 8 - ], - [ - "SAN", - "SAN DIEGO", - "CA", - "AIRPORT", - 17 - ], - [ - "LGA", - "NEW YORK", - "NY", - "AIRPORT", - 21 - ], - [ - "PHL", - "PHILADELPHIA", - "PA", - "AIRPORT", - 36 - ], - [ - "DTW", - "DETROIT", - "MI", - "AIRPORT", - 645 - ], - [ - "MSP", - "MINNEAPOLIS", - "MN", - "AIRPORT", - 841 - ] - ] - } + "semantic_table": true, + "name": "airports", + "info": "Semantic table definition stored in context" }, "query_index_all": { "code": "# Index all dimensions (None means all)\nresult = airports.index(None).limit(10)", - "sql": "SELECT\n *\nFROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n 'code' AS \"fieldName\",\n 'code' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t3\".\"value\" AS \"fieldValue\",\n \"t3\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"code\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t3\"\n ) AS \"t8\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'city' AS \"fieldName\",\n 'city' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t4\".\"value\" AS \"fieldValue\",\n \"t4\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"city\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t4\"\n ) AS \"t9\"\n ) AS \"t10\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'state' AS \"fieldName\",\n 'state' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t2\".\"value\" AS \"fieldValue\",\n \"t2\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"state\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t2\"\n ) AS \"t7\"\n ) AS \"t12\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'fac_type' AS \"fieldName\",\n 'fac_type' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t1\".\"value\" AS \"fieldValue\",\n \"t1\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"fac_type\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t1\"\n ) AS \"t6\"\n) AS \"t13\"\nUNION ALL\nSELECT\n *\nFROM (\n SELECT\n 'elevation' AS \"fieldName\",\n 'elevation' AS \"fieldPath\",\n 'number' AS \"fieldType\",\n CASE\n WHEN CASE\n WHEN CAST(\"t11\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CAST(\"t11\".\"min_val\" AS VARCHAR) || ' to '\n END IS NULL\n OR CAST(\"t11\".\"max_val\" AS VARCHAR) IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CASE\n WHEN CAST(\"t11\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CAST(\"t11\".\"min_val\" AS VARCHAR) || ' to '\n END || CAST(\"t11\".\"max_val\" AS VARCHAR)\n END AS \"fieldValue\",\n \"t11\".\"weight\"\n FROM (\n SELECT\n MIN(\"t5\".\"value\") AS \"min_val\",\n MAX(\"t5\".\"value\") AS \"max_val\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"elevation\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n WHERE\n \"t0\".\"elevation\" IS NOT NULL\n ) AS \"t5\"\n ) AS \"t11\"\n) AS \"t14\"\nLIMIT 10", + "sql": "SELECT\n *\nFROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n 'code' AS \"fieldName\",\n 'code' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t3\".\"value\" AS \"fieldValue\",\n \"t3\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"code\" AS \"value\"\n FROM \"ibis_pandas_memtable_tpgtikqbmbf7ja3i5oiqnevv7e\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t3\"\n ) AS \"t8\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'city' AS \"fieldName\",\n 'city' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t4\".\"value\" AS \"fieldValue\",\n \"t4\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"city\" AS \"value\"\n FROM \"ibis_pandas_memtable_tpgtikqbmbf7ja3i5oiqnevv7e\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t4\"\n ) AS \"t9\"\n ) AS \"t10\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'state' AS \"fieldName\",\n 'state' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t2\".\"value\" AS \"fieldValue\",\n \"t2\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"state\" AS \"value\"\n FROM \"ibis_pandas_memtable_tpgtikqbmbf7ja3i5oiqnevv7e\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t2\"\n ) AS \"t7\"\n ) AS \"t12\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'fac_type' AS \"fieldName\",\n 'fac_type' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t1\".\"value\" AS \"fieldValue\",\n \"t1\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"fac_type\" AS \"value\"\n FROM \"ibis_pandas_memtable_tpgtikqbmbf7ja3i5oiqnevv7e\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t1\"\n ) AS \"t6\"\n) AS \"t13\"\nUNION ALL\nSELECT\n *\nFROM (\n SELECT\n 'elevation' AS \"fieldName\",\n 'elevation' AS \"fieldPath\",\n 'number' AS \"fieldType\",\n CASE\n WHEN CASE\n WHEN CAST(\"t11\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CONCAT(CAST(\"t11\".\"min_val\" AS VARCHAR), ' to ')\n END IS NULL\n OR CAST(\"t11\".\"max_val\" AS VARCHAR) IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CONCAT(\n CASE\n WHEN CAST(\"t11\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CONCAT(CAST(\"t11\".\"min_val\" AS VARCHAR), ' to ')\n END,\n CAST(\"t11\".\"max_val\" AS VARCHAR)\n )\n END AS \"fieldValue\",\n \"t11\".\"weight\"\n FROM (\n SELECT\n MIN(\"t5\".\"value\") AS \"min_val\",\n MAX(\"t5\".\"value\") AS \"max_val\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"elevation\" AS \"value\"\n FROM \"ibis_pandas_memtable_tpgtikqbmbf7ja3i5oiqnevv7e\" AS \"t0\"\n WHERE\n \"t0\".\"elevation\" IS NOT NULL\n ) AS \"t5\"\n ) AS \"t11\"\n) AS \"t14\"\nLIMIT 10", "plan": "SemanticTable: airports\n code [dim]\n city [dim]\n state [dim]\n fac_type [dim]\n elevation [dim]\n airport_count [measure]\n avg_elevation [measure]\n-> Index()\n-> Limit(10)", "table": { "columns": [ @@ -170,83 +19,182 @@ "weight" ], "data": [ - [ - "elevation", - "elevation", - "number", - "8 to 5433", - 20 - ], [ "state", "state", "string", - "CA", - 3 + "GA", + 1 ], [ "state", "state", "string", - "MN", - 1 + "TX", + 2 ], [ "state", "state", "string", - "WA", + "MA", 1 ], [ - "fac_type", - "fac_type", + "city", + "city", "string", - "AIRPORT", - 20 + "NEW YORK", + 2 ], [ "state", "state", "string", - "PA", + "WA", 1 ], [ - "state", - "state", + "city", + "city", "string", - "NV", + "HOUSTON", 1 ], [ "state", "state", "string", - "AZ", + "CO", 1 ], [ - "state", - "state", + "fac_type", + "fac_type", "string", - "FL", - 2 + "AIRPORT", + 20 ], [ - "state", - "state", + "city", + "city", "string", - "IL", + "SEATTLE", + 1 + ], + [ + "city", + "city", + "string", + "DETROIT", 1 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-505aae8f01219de19c9e106a9032b926" + }, + "mark": { + "type": "text" + }, + "encoding": { + "text": { + "value": "Complex query - consider custom visualization" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-505aae8f01219de19c9e106a9032b926": [ + { + "fieldName": "elevation", + "fieldPath": "elevation", + "fieldType": "number", + "fieldValue": "8 to 5433", + "weight": 20 + }, + { + "fieldName": "fac_type", + "fieldPath": "fac_type", + "fieldType": "string", + "fieldValue": "AIRPORT", + "weight": 20 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "NY", + "weight": 2 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "PA", + "weight": 1 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "GA", + "weight": 1 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "TX", + "weight": 2 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "MA", + "weight": 1 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "NJ", + "weight": 1 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "MI", + "weight": 1 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "CO", + "weight": 1 + } + ] + } + } } }, "query_index_specific": { "code": "# Index only state and city\nresult = (\n airports.index(lambda t: [t.state, t.city])\n .order_by(lambda t: t.weight.desc())\n .limit(10)\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n 'code' AS \"fieldName\",\n 'code' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t3\".\"value\" AS \"fieldValue\",\n \"t3\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"code\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t3\"\n ) AS \"t8\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'city' AS \"fieldName\",\n 'city' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t4\".\"value\" AS \"fieldValue\",\n \"t4\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"city\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t4\"\n ) AS \"t9\"\n ) AS \"t10\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'state' AS \"fieldName\",\n 'state' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t2\".\"value\" AS \"fieldValue\",\n \"t2\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"state\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t2\"\n ) AS \"t7\"\n ) AS \"t12\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'fac_type' AS \"fieldName\",\n 'fac_type' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t1\".\"value\" AS \"fieldValue\",\n \"t1\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"fac_type\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t1\"\n ) AS \"t6\"\n ) AS \"t13\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'elevation' AS \"fieldName\",\n 'elevation' AS \"fieldPath\",\n 'number' AS \"fieldType\",\n CASE\n WHEN CASE\n WHEN CAST(\"t11\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CAST(\"t11\".\"min_val\" AS VARCHAR) || ' to '\n END IS NULL\n OR CAST(\"t11\".\"max_val\" AS VARCHAR) IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CASE\n WHEN CAST(\"t11\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CAST(\"t11\".\"min_val\" AS VARCHAR) || ' to '\n END || CAST(\"t11\".\"max_val\" AS VARCHAR)\n END AS \"fieldValue\",\n \"t11\".\"weight\"\n FROM (\n SELECT\n MIN(\"t5\".\"value\") AS \"min_val\",\n MAX(\"t5\".\"value\") AS \"max_val\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"elevation\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n WHERE\n \"t0\".\"elevation\" IS NOT NULL\n ) AS \"t5\"\n ) AS \"t11\"\n ) AS \"t14\"\n) AS \"t15\"\nORDER BY\n \"t15\".\"weight\" DESC NULLS LAST\nLIMIT 10", - "plan": "SemanticTable: airports\n code [dim]\n city [dim]\n state [dim]\n fac_type [dim]\n elevation [dim]\n airport_count [measure]\n avg_elevation [measure]\n-> Index( at 0x77f239220cc0>)\n-> OrderBy(_CallableWrapper(_fn= at 0x77f239283d80>))\n-> Limit(10)", + "sql": "SELECT\n *\nFROM (\n SELECT\n *\n FROM (\n SELECT\n 'state' AS \"fieldName\",\n 'state' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t1\".\"value\" AS \"fieldValue\",\n \"t1\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"state\" AS \"value\"\n FROM \"ibis_pandas_memtable_tpgtikqbmbf7ja3i5oiqnevv7e\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t1\"\n ) AS \"t3\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'city' AS \"fieldName\",\n 'city' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t2\".\"value\" AS \"fieldValue\",\n \"t2\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"city\" AS \"value\"\n FROM \"ibis_pandas_memtable_tpgtikqbmbf7ja3i5oiqnevv7e\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t2\"\n ) AS \"t4\"\n) AS \"t5\"\nORDER BY\n \"t5\".\"weight\" DESC NULLS LAST\nLIMIT 10", + "plan": "SemanticTable: airports\n code [dim]\n city [dim]\n state [dim]\n fac_type [dim]\n elevation [dim]\n airport_count [measure]\n avg_elevation [measure]\n-> Index( at 0x11fa9f100>)\n-> OrderBy(_CallableWrapper(_fn= at 0x11fa9f1a0>))\n-> Limit(10)", "table": { "columns": [ "fieldName", @@ -257,25 +205,25 @@ ], "data": [ [ - "fac_type", - "fac_type", + "state", + "state", "string", - "AIRPORT", - 20 + "CA", + 3 ], [ - "elevation", - "elevation", - "number", - "8 to 5433", - 20 + "state", + "state", + "string", + "NY", + 2 ], [ "state", "state", "string", - "CA", - 3 + "FL", + 2 ], [ "city", @@ -292,47 +240,146 @@ 2 ], [ - "state", - "state", + "city", + "city", "string", - "NY", - 2 + "NEWARK", + 1 ], [ "state", "state", "string", - "FL", - 2 + "PA", + 1 ], [ - "code", - "code", + "city", + "city", "string", - "BOS", + "MINNEAPOLIS", 1 ], [ "city", "city", "string", - "ORLANDO", + "BOSTON", 1 ], [ "city", "city", "string", - "SAN DIEGO", + "CHICAGO", 1 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-fc55fa6f500cf1da85ea0c6f0aefa0ae" + }, + "mark": { + "type": "text" + }, + "encoding": { + "text": { + "value": "Complex query - consider custom visualization" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-fc55fa6f500cf1da85ea0c6f0aefa0ae": [ + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "CA", + "weight": 3 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "NY", + "weight": 2 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "FL", + "weight": 2 + }, + { + "fieldName": "city", + "fieldPath": "city", + "fieldType": "string", + "fieldValue": "NEW YORK", + "weight": 2 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "TX", + "weight": 2 + }, + { + "fieldName": "city", + "fieldPath": "city", + "fieldType": "string", + "fieldValue": "NEWARK", + "weight": 1 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "PA", + "weight": 1 + }, + { + "fieldName": "city", + "fieldPath": "city", + "fieldType": "string", + "fieldValue": "MINNEAPOLIS", + "weight": 1 + }, + { + "fieldName": "city", + "fieldPath": "city", + "fieldType": "string", + "fieldValue": "BOSTON", + "weight": 1 + }, + { + "fieldName": "city", + "fieldPath": "city", + "fieldType": "string", + "fieldValue": "CHICAGO", + "weight": 1 + } + ] + } + } } }, "query_autocomplete": { "code": "# Get city suggestions starting with \"SAN\"\nresult = (\n airports.index(lambda t: t.city)\n .filter(lambda t: t.fieldValue.like(\"SAN%\"))\n .order_by(lambda t: t.weight.desc())\n .limit(10)\n)", - "sql": "SELECT\n 'city' AS \"fieldName\",\n 'city' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t1\".\"value\" AS \"fieldValue\",\n \"t1\".\"weight\"\nFROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"city\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n) AS \"t1\"\nWHERE\n \"t1\".\"value\" LIKE 'SAN%'\nORDER BY\n \"t1\".\"weight\" DESC NULLS LAST\nLIMIT 10", - "plan": "SemanticTable: airports\n code [dim]\n city [dim]\n state [dim]\n fac_type [dim]\n elevation [dim]\n airport_count [measure]\n avg_elevation [measure]\n-> Index( at 0x77f2392207c0>)\n-> Filter(\u03bb )\n-> OrderBy(_CallableWrapper(_fn= at 0x77f239283420>))\n-> Limit(10)", + "sql": "SELECT\n 'city' AS \"fieldName\",\n 'city' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t1\".\"value\" AS \"fieldValue\",\n \"t1\".\"weight\"\nFROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"city\" AS \"value\"\n FROM \"ibis_pandas_memtable_tpgtikqbmbf7ja3i5oiqnevv7e\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n) AS \"t1\"\nWHERE\n \"t1\".\"value\" LIKE 'SAN%'\nORDER BY\n \"t1\".\"weight\" DESC NULLS LAST\nLIMIT 10", + "plan": "SemanticTable: airports\n code [dim]\n city [dim]\n state [dim]\n fac_type [dim]\n elevation [dim]\n airport_count [measure]\n avg_elevation [measure]\n-> Index( at 0x11fa9ee80>)\n-> Filter(\u03bb )\n-> OrderBy(_CallableWrapper(_fn= at 0x11fa9d300>))\n-> Limit(10)", "table": { "columns": [ "fieldName", @@ -357,12 +404,55 @@ 1 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-74617ccf07655ebdcf6519b63e86e8a1" + }, + "mark": { + "type": "text" + }, + "encoding": { + "text": { + "value": "Complex query - consider custom visualization" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-74617ccf07655ebdcf6519b63e86e8a1": [ + { + "fieldName": "city", + "fieldPath": "city", + "fieldType": "string", + "fieldValue": "SAN DIEGO", + "weight": 1 + }, + { + "fieldName": "city", + "fieldPath": "city", + "fieldType": "string", + "fieldValue": "SAN FRANCISCO", + "weight": 1 + } + ] + } + } } }, "query_by_type": { "code": "# Get only string field values\nresult = (\n airports.index(None)\n .filter(lambda t: t.fieldType == \"string\")\n .order_by(lambda t: t.weight.desc())\n .limit(10)\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n 'code' AS \"fieldName\",\n 'code' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t3\".\"value\" AS \"fieldValue\",\n \"t3\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"code\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t3\"\n ) AS \"t8\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'city' AS \"fieldName\",\n 'city' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t4\".\"value\" AS \"fieldValue\",\n \"t4\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"city\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t4\"\n ) AS \"t9\"\n ) AS \"t10\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'state' AS \"fieldName\",\n 'state' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t2\".\"value\" AS \"fieldValue\",\n \"t2\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"state\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t2\"\n ) AS \"t7\"\n ) AS \"t12\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'fac_type' AS \"fieldName\",\n 'fac_type' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t1\".\"value\" AS \"fieldValue\",\n \"t1\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"fac_type\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t1\"\n ) AS \"t6\"\n ) AS \"t13\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'elevation' AS \"fieldName\",\n 'elevation' AS \"fieldPath\",\n 'number' AS \"fieldType\",\n CASE\n WHEN CASE\n WHEN CAST(\"t11\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CAST(\"t11\".\"min_val\" AS VARCHAR) || ' to '\n END IS NULL\n OR CAST(\"t11\".\"max_val\" AS VARCHAR) IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CASE\n WHEN CAST(\"t11\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CAST(\"t11\".\"min_val\" AS VARCHAR) || ' to '\n END || CAST(\"t11\".\"max_val\" AS VARCHAR)\n END AS \"fieldValue\",\n \"t11\".\"weight\"\n FROM (\n SELECT\n MIN(\"t5\".\"value\") AS \"min_val\",\n MAX(\"t5\".\"value\") AS \"max_val\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"elevation\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n WHERE\n \"t0\".\"elevation\" IS NOT NULL\n ) AS \"t5\"\n ) AS \"t11\"\n ) AS \"t14\"\n) AS \"t15\"\nWHERE\n \"t15\".\"fieldType\" = 'string'\nORDER BY\n \"t15\".\"weight\" DESC NULLS LAST\nLIMIT 10", - "plan": "SemanticTable: airports\n code [dim]\n city [dim]\n state [dim]\n fac_type [dim]\n elevation [dim]\n airport_count [measure]\n avg_elevation [measure]\n-> Index()\n-> Filter(\u03bb )\n-> OrderBy(_CallableWrapper(_fn= at 0x77f2392820c0>))\n-> Limit(10)", + "sql": "SELECT\n *\nFROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n 'code' AS \"fieldName\",\n 'code' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t3\".\"value\" AS \"fieldValue\",\n \"t3\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"code\" AS \"value\"\n FROM \"ibis_pandas_memtable_tpgtikqbmbf7ja3i5oiqnevv7e\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t3\"\n ) AS \"t8\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'city' AS \"fieldName\",\n 'city' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t4\".\"value\" AS \"fieldValue\",\n \"t4\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"city\" AS \"value\"\n FROM \"ibis_pandas_memtable_tpgtikqbmbf7ja3i5oiqnevv7e\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t4\"\n ) AS \"t9\"\n ) AS \"t10\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'state' AS \"fieldName\",\n 'state' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t2\".\"value\" AS \"fieldValue\",\n \"t2\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"state\" AS \"value\"\n FROM \"ibis_pandas_memtable_tpgtikqbmbf7ja3i5oiqnevv7e\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t2\"\n ) AS \"t7\"\n ) AS \"t12\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'fac_type' AS \"fieldName\",\n 'fac_type' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t1\".\"value\" AS \"fieldValue\",\n \"t1\".\"weight\"\n FROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"fac_type\" AS \"value\"\n FROM \"ibis_pandas_memtable_tpgtikqbmbf7ja3i5oiqnevv7e\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n ) AS \"t1\"\n ) AS \"t6\"\n ) AS \"t13\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'elevation' AS \"fieldName\",\n 'elevation' AS \"fieldPath\",\n 'number' AS \"fieldType\",\n CASE\n WHEN CASE\n WHEN CAST(\"t11\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CONCAT(CAST(\"t11\".\"min_val\" AS VARCHAR), ' to ')\n END IS NULL\n OR CAST(\"t11\".\"max_val\" AS VARCHAR) IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CONCAT(\n CASE\n WHEN CAST(\"t11\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CONCAT(CAST(\"t11\".\"min_val\" AS VARCHAR), ' to ')\n END,\n CAST(\"t11\".\"max_val\" AS VARCHAR)\n )\n END AS \"fieldValue\",\n \"t11\".\"weight\"\n FROM (\n SELECT\n MIN(\"t5\".\"value\") AS \"min_val\",\n MAX(\"t5\".\"value\") AS \"max_val\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"elevation\" AS \"value\"\n FROM \"ibis_pandas_memtable_tpgtikqbmbf7ja3i5oiqnevv7e\" AS \"t0\"\n WHERE\n \"t0\".\"elevation\" IS NOT NULL\n ) AS \"t5\"\n ) AS \"t11\"\n ) AS \"t14\"\n) AS \"t15\"\nWHERE\n \"t15\".\"fieldType\" = 'string'\nORDER BY\n \"t15\".\"weight\" DESC NULLS LAST\nLIMIT 10", + "plan": "SemanticTable: airports\n code [dim]\n city [dim]\n state [dim]\n fac_type [dim]\n elevation [dim]\n airport_count [measure]\n avg_elevation [measure]\n-> Index()\n-> Filter(\u03bb )\n-> OrderBy(_CallableWrapper(_fn= at 0x11fa9fce0>))\n-> Limit(10)", "table": { "columns": [ "fieldName", @@ -387,68 +477,167 @@ 3 ], [ - "city", - "city", + "state", + "state", "string", - "NEW YORK", + "NY", 2 ], [ "state", "state", "string", - "TX", + "FL", 2 ], [ - "state", - "state", + "city", + "city", "string", - "NY", + "NEW YORK", 2 ], [ "state", "state", "string", - "FL", + "TX", 2 ], [ "code", "code", "string", - "BOS", + "DFW", 1 ], [ "city", "city", "string", - "ORLANDO", + "LAS VEGAS", 1 ], [ - "city", - "city", + "code", + "code", "string", - "SAN DIEGO", + "LAS", 1 ], [ - "state", - "state", + "city", + "city", "string", - "NJ", + "BOSTON", 1 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-713899e2f755e9553ee1e9aaa2cc17ac" + }, + "mark": { + "type": "text" + }, + "encoding": { + "text": { + "value": "Complex query - consider custom visualization" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-713899e2f755e9553ee1e9aaa2cc17ac": [ + { + "fieldName": "fac_type", + "fieldPath": "fac_type", + "fieldType": "string", + "fieldValue": "AIRPORT", + "weight": 20 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "CA", + "weight": 3 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "NY", + "weight": 2 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "FL", + "weight": 2 + }, + { + "fieldName": "city", + "fieldPath": "city", + "fieldType": "string", + "fieldValue": "NEW YORK", + "weight": 2 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "TX", + "weight": 2 + }, + { + "fieldName": "code", + "fieldPath": "code", + "fieldType": "string", + "fieldValue": "DFW", + "weight": 1 + }, + { + "fieldName": "city", + "fieldPath": "city", + "fieldType": "string", + "fieldValue": "BOSTON", + "weight": 1 + }, + { + "fieldName": "code", + "fieldPath": "code", + "fieldType": "string", + "fieldValue": "LAS", + "weight": 1 + }, + { + "fieldName": "city", + "fieldPath": "city", + "fieldType": "string", + "fieldValue": "MIAMI", + "weight": 1 + } + ] + } + } } }, "query_custom_weight": { "code": "# Find states with most airports\nresult = (\n airports.index(lambda t: t.state, by=\"airport_count\")\n .order_by(lambda t: t.weight.desc())\n .limit(10)\n)", - "sql": "SELECT\n 'state' AS \"fieldName\",\n 'state' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t1\".\"value\" AS \"fieldValue\",\n \"t1\".\"weight\"\nFROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"state\" AS \"value\"\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n) AS \"t1\"\nORDER BY\n \"t1\".\"weight\" DESC NULLS LAST\nLIMIT 10", - "plan": "SemanticTable: airports\n code [dim]\n city [dim]\n state [dim]\n fac_type [dim]\n elevation [dim]\n airport_count [measure]\n avg_elevation [measure]\n-> Index( at 0x77f239281260>, by=airport_count)\n-> OrderBy(_CallableWrapper(_fn= at 0x77f239280e00>))\n-> Limit(10)", + "sql": "SELECT\n 'state' AS \"fieldName\",\n 'state' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t1\".\"value\" AS \"fieldValue\",\n \"t1\".\"weight\"\nFROM (\n SELECT\n \"t0\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t0\".\"code\",\n \"t0\".\"city\",\n \"t0\".\"state\",\n \"t0\".\"fac_type\",\n \"t0\".\"elevation\",\n \"t0\".\"state\" AS \"value\"\n FROM \"ibis_pandas_memtable_tpgtikqbmbf7ja3i5oiqnevv7e\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"value\"\n) AS \"t1\"\nORDER BY\n \"t1\".\"weight\" DESC NULLS LAST\nLIMIT 10", + "plan": "SemanticTable: airports\n code [dim]\n city [dim]\n state [dim]\n fac_type [dim]\n elevation [dim]\n airport_count [measure]\n avg_elevation [measure]\n-> Index( at 0x11fa9f6a0>, by=airport_count)\n-> OrderBy(_CallableWrapper(_fn= at 0x11fa9c540>))\n-> Limit(10)", "table": { "columns": [ "fieldName", @@ -469,35 +658,35 @@ "state", "state", "string", - "TX", + "NY", 2 ], [ "state", "state", "string", - "NY", + "FL", 2 ], [ "state", "state", "string", - "FL", + "TX", 2 ], [ "state", "state", "string", - "NJ", + "PA", 1 ], [ "state", "state", "string", - "MN", + "NV", 1 ], [ @@ -518,7 +707,7 @@ "state", "state", "string", - "GA", + "MN", 1 ], [ @@ -529,12 +718,111 @@ 1 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-27f9782d9a18017476bfc3856a947adf" + }, + "mark": { + "type": "text" + }, + "encoding": { + "text": { + "value": "Complex query - consider custom visualization" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-27f9782d9a18017476bfc3856a947adf": [ + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "CA", + "weight": 3 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "NY", + "weight": 2 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "FL", + "weight": 2 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "TX", + "weight": 2 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "PA", + "weight": 1 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "NV", + "weight": 1 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "CO", + "weight": 1 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "WA", + "weight": 1 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "MN", + "weight": 1 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "AZ", + "weight": 1 + } + ] + } + } } }, "query_sampled": { "code": "# Sample 100 rows before indexing\nresult = (\n airports.index(None, sample=100)\n .filter(lambda t: t.fieldType == \"string\")\n .order_by(lambda t: t.weight.desc())\n .limit(10)\n)", - "sql": "WITH \"t1\" AS (\n SELECT\n *\n FROM \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t0\"\n LIMIT 100\n)\nSELECT\n *\nFROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n 'code' AS \"fieldName\",\n 'code' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t5\".\"value\" AS \"fieldValue\",\n \"t5\".\"weight\"\n FROM (\n SELECT\n \"t2\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t2\".\"code\",\n \"t2\".\"city\",\n \"t2\".\"state\",\n \"t2\".\"fac_type\",\n \"t2\".\"elevation\",\n \"t2\".\"code\" AS \"value\"\n FROM \"t1\" AS \"t2\"\n ) AS t2\n GROUP BY\n \"t2\".\"value\"\n ) AS \"t5\"\n ) AS \"t10\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'city' AS \"fieldName\",\n 'city' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t6\".\"value\" AS \"fieldValue\",\n \"t6\".\"weight\"\n FROM (\n SELECT\n \"t2\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t2\".\"code\",\n \"t2\".\"city\",\n \"t2\".\"state\",\n \"t2\".\"fac_type\",\n \"t2\".\"elevation\",\n \"t2\".\"city\" AS \"value\"\n FROM \"t1\" AS \"t2\"\n ) AS t2\n GROUP BY\n \"t2\".\"value\"\n ) AS \"t6\"\n ) AS \"t11\"\n ) AS \"t12\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'state' AS \"fieldName\",\n 'state' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t4\".\"value\" AS \"fieldValue\",\n \"t4\".\"weight\"\n FROM (\n SELECT\n \"t2\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t2\".\"code\",\n \"t2\".\"city\",\n \"t2\".\"state\",\n \"t2\".\"fac_type\",\n \"t2\".\"elevation\",\n \"t2\".\"state\" AS \"value\"\n FROM \"t1\" AS \"t2\"\n ) AS t2\n GROUP BY\n \"t2\".\"value\"\n ) AS \"t4\"\n ) AS \"t9\"\n ) AS \"t14\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'fac_type' AS \"fieldName\",\n 'fac_type' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t3\".\"value\" AS \"fieldValue\",\n \"t3\".\"weight\"\n FROM (\n SELECT\n \"t2\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t2\".\"code\",\n \"t2\".\"city\",\n \"t2\".\"state\",\n \"t2\".\"fac_type\",\n \"t2\".\"elevation\",\n \"t2\".\"fac_type\" AS \"value\"\n FROM \"t1\" AS \"t2\"\n ) AS t2\n GROUP BY\n \"t2\".\"value\"\n ) AS \"t3\"\n ) AS \"t8\"\n ) AS \"t15\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'elevation' AS \"fieldName\",\n 'elevation' AS \"fieldPath\",\n 'number' AS \"fieldType\",\n CASE\n WHEN CASE\n WHEN CAST(\"t13\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CAST(\"t13\".\"min_val\" AS VARCHAR) || ' to '\n END IS NULL\n OR CAST(\"t13\".\"max_val\" AS VARCHAR) IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CASE\n WHEN CAST(\"t13\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CAST(\"t13\".\"min_val\" AS VARCHAR) || ' to '\n END || CAST(\"t13\".\"max_val\" AS VARCHAR)\n END AS \"fieldValue\",\n \"t13\".\"weight\"\n FROM (\n SELECT\n MIN(\"t7\".\"value\") AS \"min_val\",\n MAX(\"t7\".\"value\") AS \"max_val\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t2\".\"elevation\" AS \"value\"\n FROM \"t1\" AS \"t2\"\n WHERE\n \"t2\".\"elevation\" IS NOT NULL\n ) AS \"t7\"\n ) AS \"t13\"\n ) AS \"t16\"\n) AS \"t17\"\nWHERE\n \"t17\".\"fieldType\" = 'string'\nORDER BY\n \"t17\".\"weight\" DESC NULLS LAST\nLIMIT 10", - "plan": "SemanticTable: airports\n code [dim]\n city [dim]\n state [dim]\n fac_type [dim]\n elevation [dim]\n airport_count [measure]\n avg_elevation [measure]\n-> Index(sample=100)\n-> Filter(\u03bb )\n-> OrderBy(_CallableWrapper(_fn= at 0x77f2392207c0>))\n-> Limit(10)", + "sql": "WITH \"t1\" AS (\n SELECT\n *\n FROM \"ibis_pandas_memtable_tpgtikqbmbf7ja3i5oiqnevv7e\" AS \"t0\"\n LIMIT 100\n)\nSELECT\n *\nFROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n 'code' AS \"fieldName\",\n 'code' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t5\".\"value\" AS \"fieldValue\",\n \"t5\".\"weight\"\n FROM (\n SELECT\n \"t2\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t2\".\"code\",\n \"t2\".\"city\",\n \"t2\".\"state\",\n \"t2\".\"fac_type\",\n \"t2\".\"elevation\",\n \"t2\".\"code\" AS \"value\"\n FROM \"t1\" AS \"t2\"\n ) AS t2\n GROUP BY\n \"t2\".\"value\"\n ) AS \"t5\"\n ) AS \"t10\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'city' AS \"fieldName\",\n 'city' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t6\".\"value\" AS \"fieldValue\",\n \"t6\".\"weight\"\n FROM (\n SELECT\n \"t2\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t2\".\"code\",\n \"t2\".\"city\",\n \"t2\".\"state\",\n \"t2\".\"fac_type\",\n \"t2\".\"elevation\",\n \"t2\".\"city\" AS \"value\"\n FROM \"t1\" AS \"t2\"\n ) AS t2\n GROUP BY\n \"t2\".\"value\"\n ) AS \"t6\"\n ) AS \"t11\"\n ) AS \"t12\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'state' AS \"fieldName\",\n 'state' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t4\".\"value\" AS \"fieldValue\",\n \"t4\".\"weight\"\n FROM (\n SELECT\n \"t2\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t2\".\"code\",\n \"t2\".\"city\",\n \"t2\".\"state\",\n \"t2\".\"fac_type\",\n \"t2\".\"elevation\",\n \"t2\".\"state\" AS \"value\"\n FROM \"t1\" AS \"t2\"\n ) AS t2\n GROUP BY\n \"t2\".\"value\"\n ) AS \"t4\"\n ) AS \"t9\"\n ) AS \"t14\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'fac_type' AS \"fieldName\",\n 'fac_type' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t3\".\"value\" AS \"fieldValue\",\n \"t3\".\"weight\"\n FROM (\n SELECT\n \"t2\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t2\".\"code\",\n \"t2\".\"city\",\n \"t2\".\"state\",\n \"t2\".\"fac_type\",\n \"t2\".\"elevation\",\n \"t2\".\"fac_type\" AS \"value\"\n FROM \"t1\" AS \"t2\"\n ) AS t2\n GROUP BY\n \"t2\".\"value\"\n ) AS \"t3\"\n ) AS \"t8\"\n ) AS \"t15\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'elevation' AS \"fieldName\",\n 'elevation' AS \"fieldPath\",\n 'number' AS \"fieldType\",\n CASE\n WHEN CASE\n WHEN CAST(\"t13\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CONCAT(CAST(\"t13\".\"min_val\" AS VARCHAR), ' to ')\n END IS NULL\n OR CAST(\"t13\".\"max_val\" AS VARCHAR) IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CONCAT(\n CASE\n WHEN CAST(\"t13\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CONCAT(CAST(\"t13\".\"min_val\" AS VARCHAR), ' to ')\n END,\n CAST(\"t13\".\"max_val\" AS VARCHAR)\n )\n END AS \"fieldValue\",\n \"t13\".\"weight\"\n FROM (\n SELECT\n MIN(\"t7\".\"value\") AS \"min_val\",\n MAX(\"t7\".\"value\") AS \"max_val\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t2\".\"elevation\" AS \"value\"\n FROM \"t1\" AS \"t2\"\n WHERE\n \"t2\".\"elevation\" IS NOT NULL\n ) AS \"t7\"\n ) AS \"t13\"\n ) AS \"t16\"\n) AS \"t17\"\nWHERE\n \"t17\".\"fieldType\" = 'string'\nORDER BY\n \"t17\".\"weight\" DESC NULLS LAST\nLIMIT 10", + "plan": "SemanticTable: airports\n code [dim]\n city [dim]\n state [dim]\n fac_type [dim]\n elevation [dim]\n airport_count [measure]\n avg_elevation [measure]\n-> Index(sample=100)\n-> Filter(\u03bb )\n-> OrderBy(_CallableWrapper(_fn= at 0x11fa9e020>))\n-> Limit(10)", "table": { "columns": [ "fieldName", @@ -559,68 +847,167 @@ 3 ], [ - "city", - "city", + "state", + "state", "string", - "NEW YORK", + "NY", 2 ], [ "state", "state", "string", - "TX", + "FL", 2 ], [ - "state", - "state", + "city", + "city", "string", - "NY", + "NEW YORK", 2 ], [ "state", "state", "string", - "FL", + "TX", 2 ], [ - "city", - "city", + "code", + "code", "string", - "SAN DIEGO", + "DFW", 1 ], [ "city", "city", "string", - "ORLANDO", + "LAS VEGAS", 1 ], [ "code", "code", "string", - "BOS", + "LAS", 1 ], [ - "state", - "state", + "city", + "city", "string", - "NJ", + "BOSTON", 1 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-d529b7ce7be9fb07af30ecf8a0443a3b" + }, + "mark": { + "type": "text" + }, + "encoding": { + "text": { + "value": "Complex query - consider custom visualization" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-d529b7ce7be9fb07af30ecf8a0443a3b": [ + { + "fieldName": "fac_type", + "fieldPath": "fac_type", + "fieldType": "string", + "fieldValue": "AIRPORT", + "weight": 20 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "CA", + "weight": 3 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "NY", + "weight": 2 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "FL", + "weight": 2 + }, + { + "fieldName": "city", + "fieldPath": "city", + "fieldType": "string", + "fieldValue": "NEW YORK", + "weight": 2 + }, + { + "fieldName": "state", + "fieldPath": "state", + "fieldType": "string", + "fieldValue": "TX", + "weight": 2 + }, + { + "fieldName": "code", + "fieldPath": "code", + "fieldType": "string", + "fieldValue": "LAS", + "weight": 1 + }, + { + "fieldName": "city", + "fieldPath": "city", + "fieldType": "string", + "fieldValue": "BOSTON", + "weight": 1 + }, + { + "fieldName": "code", + "fieldPath": "code", + "fieldType": "string", + "fieldValue": "BOS", + "weight": 1 + }, + { + "fieldName": "city", + "fieldPath": "city", + "fieldType": "string", + "fieldValue": "MIAMI", + "weight": 1 + } + ] + } + } } }, "query_index_joins": { "code": "# Create synthetic flights data\nflights_data = ibis.memtable({\n \"flight_id\": list(range(1, 31)),\n \"carrier\": [\"AA\", \"UA\", \"DL\", \"WN\", \"B6\", \"AA\", \"UA\", \"DL\", \"WN\", \"B6\"] * 3,\n \"origin\": [\"JFK\", \"LAX\", \"ORD\", \"ATL\", \"DFW\", \"SFO\", \"SEA\", \"DEN\", \"PHX\", \"BOS\"] * 3,\n})\n\nflights = (\n to_semantic_table(flights_data, name=\"flights\")\n .with_dimensions(\n carrier=lambda t: t.carrier,\n origin=lambda t: t.origin,\n )\n .with_measures(\n flight_count=lambda t: t.count(),\n )\n)\n\n# Join flights with airports\nflights_with_origin = flights.join_one(airports, lambda f, a: f.origin == a.code)\n\n# Index across the join\nresult = (\n flights_with_origin.index([\"flights.carrier\", \"airports.state\"])\n .order_by(lambda t: t.weight.desc())\n .limit(10)\n)", - "sql": "WITH \"t5\" AS (\n SELECT\n \"t2\".\"flight_id\",\n \"t2\".\"carrier\",\n \"t2\".\"origin\",\n \"t3\".\"code\",\n \"t3\".\"city\",\n \"t3\".\"state\",\n \"t3\".\"fac_type\",\n \"t3\".\"elevation\"\n FROM \"ibis_pandas_memtable_shw7goquencdbjo7t5v4gd64gq\" AS \"t2\"\n LEFT OUTER JOIN \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t3\"\n ON \"t2\".\"origin\" = \"t3\".\"code\"\n), \"t9\" AS (\n SELECT\n \"t7\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t7\".\"flight_id\",\n \"t7\".\"carrier\",\n \"t7\".\"origin\",\n \"t7\".\"code\",\n \"t7\".\"city\",\n \"t7\".\"state\",\n \"t7\".\"fac_type\",\n \"t7\".\"elevation\",\n \"t7\".\"fac_type\" AS \"value\"\n FROM \"t5\" AS \"t7\"\n ) AS t7\n GROUP BY\n \"t7\".\"value\"\n), \"t10\" AS (\n SELECT\n \"t7\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t7\".\"flight_id\",\n \"t7\".\"carrier\",\n \"t7\".\"origin\",\n \"t7\".\"code\",\n \"t7\".\"city\",\n \"t7\".\"state\",\n \"t7\".\"fac_type\",\n \"t7\".\"elevation\",\n \"t7\".\"state\" AS \"value\"\n FROM \"t5\" AS \"t7\"\n ) AS t7\n GROUP BY\n \"t7\".\"value\"\n), \"t11\" AS (\n SELECT\n \"t7\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t7\".\"flight_id\",\n \"t7\".\"carrier\",\n \"t7\".\"origin\",\n \"t7\".\"code\",\n \"t7\".\"city\",\n \"t7\".\"state\",\n \"t7\".\"fac_type\",\n \"t7\".\"elevation\",\n \"t7\".\"city\" AS \"value\"\n FROM \"t5\" AS \"t7\"\n ) AS t7\n GROUP BY\n \"t7\".\"value\"\n), \"t12\" AS (\n SELECT\n \"t7\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t7\".\"flight_id\",\n \"t7\".\"carrier\",\n \"t7\".\"origin\",\n \"t7\".\"code\",\n \"t7\".\"city\",\n \"t7\".\"state\",\n \"t7\".\"fac_type\",\n \"t7\".\"elevation\",\n \"t7\".\"code\" AS \"value\"\n FROM \"t5\" AS \"t7\"\n ) AS t7\n GROUP BY\n \"t7\".\"value\"\n), \"t13\" AS (\n SELECT\n \"t7\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t7\".\"flight_id\",\n \"t7\".\"carrier\",\n \"t7\".\"origin\",\n \"t7\".\"code\",\n \"t7\".\"city\",\n \"t7\".\"state\",\n \"t7\".\"fac_type\",\n \"t7\".\"elevation\",\n \"t7\".\"origin\" AS \"value\"\n FROM \"t5\" AS \"t7\"\n ) AS t7\n GROUP BY\n \"t7\".\"value\"\n), \"t14\" AS (\n SELECT\n \"t7\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t7\".\"flight_id\",\n \"t7\".\"carrier\",\n \"t7\".\"origin\",\n \"t7\".\"code\",\n \"t7\".\"city\",\n \"t7\".\"state\",\n \"t7\".\"fac_type\",\n \"t7\".\"elevation\",\n \"t7\".\"carrier\" AS \"value\"\n FROM \"t5\" AS \"t7\"\n ) AS t7\n GROUP BY\n \"t7\".\"value\"\n), \"t22\" AS (\n SELECT\n MIN(\"t8\".\"value\") AS \"min_val\",\n MAX(\"t8\".\"value\") AS \"max_val\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t3\".\"elevation\" AS \"value\"\n FROM \"ibis_pandas_memtable_shw7goquencdbjo7t5v4gd64gq\" AS \"t2\"\n LEFT OUTER JOIN \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t3\"\n ON \"t2\".\"origin\" = \"t3\".\"code\"\n ) AS \"t4\"\n WHERE\n \"t4\".\"value\" IS NOT NULL\n ) AS \"t8\"\n)\nSELECT\n *\nFROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM (\n SELECT\n 'carrier' AS \"fieldName\",\n 'flights.carrier' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t21\".\"value\" AS \"fieldValue\",\n \"t21\".\"weight\"\n FROM \"t14\" AS \"t21\"\n ) AS \"t34\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'origin' AS \"fieldName\",\n 'flights.origin' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t20\".\"value\" AS \"fieldValue\",\n \"t20\".\"weight\"\n FROM \"t13\" AS \"t20\"\n ) AS \"t32\"\n ) AS \"t37\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'code' AS \"fieldName\",\n 'airports.code' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t19\".\"value\" AS \"fieldValue\",\n \"t19\".\"weight\"\n FROM \"t12\" AS \"t19\"\n ) AS \"t30\"\n ) AS \"t38\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'city' AS \"fieldName\",\n 'airports.city' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t18\".\"value\" AS \"fieldValue\",\n \"t18\".\"weight\"\n FROM \"t11\" AS \"t18\"\n ) AS \"t28\"\n ) AS \"t39\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'state' AS \"fieldName\",\n 'airports.state' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t17\".\"value\" AS \"fieldValue\",\n \"t17\".\"weight\"\n FROM \"t10\" AS \"t17\"\n ) AS \"t26\"\n ) AS \"t40\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'fac_type' AS \"fieldName\",\n 'airports.fac_type' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t16\".\"value\" AS \"fieldValue\",\n \"t16\".\"weight\"\n FROM \"t9\" AS \"t16\"\n ) AS \"t24\"\n ) AS \"t41\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'elevation' AS \"fieldName\",\n 'airports.elevation' AS \"fieldPath\",\n 'number' AS \"fieldType\",\n CASE\n WHEN CASE\n WHEN CAST(\"t36\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CAST(\"t36\".\"min_val\" AS VARCHAR) || ' to '\n END IS NULL\n OR CAST(\"t36\".\"max_val\" AS VARCHAR) IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CASE\n WHEN CAST(\"t36\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CAST(\"t36\".\"min_val\" AS VARCHAR) || ' to '\n END || CAST(\"t36\".\"max_val\" AS VARCHAR)\n END AS \"fieldValue\",\n \"t36\".\"weight\"\n FROM \"t22\" AS \"t36\"\n ) AS \"t44\"\n ) AS \"t45\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'flight_id' AS \"fieldName\",\n 'flight_id' AS \"fieldPath\",\n 'number' AS \"fieldType\",\n CASE\n WHEN CASE\n WHEN CAST(\"t35\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CAST(\"t35\".\"min_val\" AS VARCHAR) || ' to '\n END IS NULL\n OR CAST(\"t35\".\"max_val\" AS VARCHAR) IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CASE\n WHEN CAST(\"t35\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CAST(\"t35\".\"min_val\" AS VARCHAR) || ' to '\n END || CAST(\"t35\".\"max_val\" AS VARCHAR)\n END AS \"fieldValue\",\n \"t35\".\"weight\"\n FROM (\n SELECT\n MIN(\"t15\".\"value\") AS \"min_val\",\n MAX(\"t15\".\"value\") AS \"max_val\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t2\".\"flight_id\" AS \"value\"\n FROM \"ibis_pandas_memtable_shw7goquencdbjo7t5v4gd64gq\" AS \"t2\"\n LEFT OUTER JOIN \"ibis_pandas_memtable_wttenv6q55fldbw4bymbvoi4t4\" AS \"t3\"\n ON \"t2\".\"origin\" = \"t3\".\"code\"\n ) AS \"t6\"\n WHERE\n \"t6\".\"value\" IS NOT NULL\n ) AS \"t15\"\n ) AS \"t35\"\n ) AS \"t42\"\n ) AS \"t46\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'carrier' AS \"fieldName\",\n 'carrier' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t21\".\"value\" AS \"fieldValue\",\n \"t21\".\"weight\"\n FROM \"t14\" AS \"t21\"\n ) AS \"t33\"\n ) AS \"t47\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'origin' AS \"fieldName\",\n 'origin' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t20\".\"value\" AS \"fieldValue\",\n \"t20\".\"weight\"\n FROM \"t13\" AS \"t20\"\n ) AS \"t31\"\n ) AS \"t48\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'code' AS \"fieldName\",\n 'code' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t19\".\"value\" AS \"fieldValue\",\n \"t19\".\"weight\"\n FROM \"t12\" AS \"t19\"\n ) AS \"t29\"\n ) AS \"t49\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'city' AS \"fieldName\",\n 'city' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t18\".\"value\" AS \"fieldValue\",\n \"t18\".\"weight\"\n FROM \"t11\" AS \"t18\"\n ) AS \"t27\"\n ) AS \"t50\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'state' AS \"fieldName\",\n 'state' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t17\".\"value\" AS \"fieldValue\",\n \"t17\".\"weight\"\n FROM \"t10\" AS \"t17\"\n ) AS \"t25\"\n ) AS \"t51\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'fac_type' AS \"fieldName\",\n 'fac_type' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t16\".\"value\" AS \"fieldValue\",\n \"t16\".\"weight\"\n FROM \"t9\" AS \"t16\"\n ) AS \"t23\"\n ) AS \"t52\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'elevation' AS \"fieldName\",\n 'elevation' AS \"fieldPath\",\n 'number' AS \"fieldType\",\n CASE\n WHEN CASE\n WHEN CAST(\"t36\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CAST(\"t36\".\"min_val\" AS VARCHAR) || ' to '\n END IS NULL\n OR CAST(\"t36\".\"max_val\" AS VARCHAR) IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CASE\n WHEN CAST(\"t36\".\"min_val\" AS VARCHAR) IS NULL OR ' to ' IS NULL\n THEN CAST(NULL AS VARCHAR)\n ELSE CAST(\"t36\".\"min_val\" AS VARCHAR) || ' to '\n END || CAST(\"t36\".\"max_val\" AS VARCHAR)\n END AS \"fieldValue\",\n \"t36\".\"weight\"\n FROM \"t22\" AS \"t36\"\n ) AS \"t43\"\n) AS \"t53\"\nORDER BY\n \"t53\".\"weight\" DESC NULLS LAST\nLIMIT 10", - "plan": "SemanticTable: flights\n carrier [dim]\n origin [dim]\n flight_count [measure]\n-> Join(left, right=airports)\n-> Index(flights.carrier, airports.state)\n-> OrderBy(_CallableWrapper(_fn= at 0x77f239283420>))\n-> Limit(10)", + "sql": "WITH \"t4\" AS (\n SELECT\n \"t2\".\"flight_id\",\n \"t2\".\"carrier\",\n \"t2\".\"origin\",\n \"t3\".\"code\",\n \"t3\".\"city\",\n \"t3\".\"state\",\n \"t3\".\"fac_type\",\n \"t3\".\"elevation\"\n FROM \"ibis_pandas_memtable_tb6zngvctfh7dnhmlt54fmitka\" AS \"t2\"\n LEFT OUTER JOIN \"ibis_pandas_memtable_tpgtikqbmbf7ja3i5oiqnevv7e\" AS \"t3\"\n ON \"t2\".\"origin\" = \"t3\".\"code\"\n)\nSELECT\n *\nFROM (\n SELECT\n *\n FROM (\n SELECT\n 'carrier' AS \"fieldName\",\n 'flights.carrier' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t6\".\"value\" AS \"fieldValue\",\n \"t6\".\"weight\"\n FROM (\n SELECT\n \"t5\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t5\".\"flight_id\",\n \"t5\".\"carrier\",\n \"t5\".\"origin\",\n \"t5\".\"code\",\n \"t5\".\"city\",\n \"t5\".\"state\",\n \"t5\".\"fac_type\",\n \"t5\".\"elevation\",\n \"t5\".\"carrier\" AS \"value\"\n FROM \"t4\" AS \"t5\"\n ) AS t5\n GROUP BY\n \"t5\".\"value\"\n ) AS \"t6\"\n ) AS \"t8\"\n UNION ALL\n SELECT\n *\n FROM (\n SELECT\n 'state' AS \"fieldName\",\n 'airports.state' AS \"fieldPath\",\n 'string' AS \"fieldType\",\n \"t7\".\"value\" AS \"fieldValue\",\n \"t7\".\"weight\"\n FROM (\n SELECT\n \"t5\".\"value\",\n COUNT(*) AS \"weight\"\n FROM (\n SELECT\n \"t5\".\"flight_id\",\n \"t5\".\"carrier\",\n \"t5\".\"origin\",\n \"t5\".\"code\",\n \"t5\".\"city\",\n \"t5\".\"state\",\n \"t5\".\"fac_type\",\n \"t5\".\"elevation\",\n \"t5\".\"state\" AS \"value\"\n FROM \"t4\" AS \"t5\"\n ) AS t5\n GROUP BY\n \"t5\".\"value\"\n ) AS \"t7\"\n ) AS \"t9\"\n) AS \"t10\"\nORDER BY\n \"t10\".\"weight\" DESC NULLS LAST\nLIMIT 10", + "plan": "SemanticTable: flights\n carrier [dim]\n origin [dim]\n flight_count [measure]\n-> Join(left, right=airports)\n-> Index(flights.carrier, airports.state)\n-> OrderBy(_CallableWrapper(_fn= at 0x11fa9dee0>))\n-> Limit(10)", "table": { "columns": [ "fieldName", @@ -631,52 +1018,38 @@ ], "data": [ [ - "fac_type", - "airports.fac_type", + "carrier", + "flights.carrier", "string", - "AIRPORT", - 30 - ], - [ - "elevation", - "airports.elevation", - "number", - "13 to 5433", - 30 - ], - [ - "flight_id", - "flight_id", - "number", - "1 to 30", - 30 + "WN", + 6 ], [ - "fac_type", - "fac_type", + "state", + "airports.state", "string", - "AIRPORT", - 30 + "CA", + 6 ], [ - "elevation", - "elevation", - "number", - "13 to 5433", - 30 + "carrier", + "flights.carrier", + "string", + "AA", + 6 ], [ "carrier", "flights.carrier", "string", - "WN", + "DL", 6 ], [ "carrier", "flights.carrier", "string", - "AA", + "B6", 6 ], [ @@ -687,20 +1060,133 @@ 6 ], [ - "carrier", - "carrier", + "state", + "airports.state", "string", - "WN", - 6 + "NY", + 3 ], [ - "carrier", - "flights.carrier", + "state", + "airports.state", "string", - "B6", - 6 + "CO", + 3 + ], + [ + "state", + "airports.state", + "string", + "WA", + 3 + ], + [ + "state", + "airports.state", + "string", + "AZ", + 3 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-78e2860bb41ac718565bceaa32023a9d" + }, + "mark": { + "type": "text" + }, + "encoding": { + "text": { + "value": "Complex query - consider custom visualization" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-78e2860bb41ac718565bceaa32023a9d": [ + { + "fieldName": "carrier", + "fieldPath": "flights.carrier", + "fieldType": "string", + "fieldValue": "WN", + "weight": 6 + }, + { + "fieldName": "state", + "fieldPath": "airports.state", + "fieldType": "string", + "fieldValue": "CA", + "weight": 6 + }, + { + "fieldName": "carrier", + "fieldPath": "flights.carrier", + "fieldType": "string", + "fieldValue": "AA", + "weight": 6 + }, + { + "fieldName": "carrier", + "fieldPath": "flights.carrier", + "fieldType": "string", + "fieldValue": "DL", + "weight": 6 + }, + { + "fieldName": "carrier", + "fieldPath": "flights.carrier", + "fieldType": "string", + "fieldValue": "B6", + "weight": 6 + }, + { + "fieldName": "carrier", + "fieldPath": "flights.carrier", + "fieldType": "string", + "fieldValue": "UA", + "weight": 6 + }, + { + "fieldName": "state", + "fieldPath": "airports.state", + "fieldType": "string", + "fieldValue": "NY", + "weight": 3 + }, + { + "fieldName": "state", + "fieldPath": "airports.state", + "fieldType": "string", + "fieldValue": "CO", + "weight": 3 + }, + { + "fieldName": "state", + "fieldPath": "airports.state", + "fieldType": "string", + "fieldValue": "WA", + "weight": 3 + }, + { + "fieldName": "state", + "fieldPath": "airports.state", + "fieldType": "string", + "fieldValue": "AZ", + "weight": 3 + } + ] + } + } } } }, diff --git a/docs/web/public/bsl-data/mcp.json b/docs/web/public/bsl-data/mcp.json index 848bc123..9e1d4dcf 100644 --- a/docs/web/public/bsl-data/mcp.json +++ b/docs/web/public/bsl-data/mcp.json @@ -1,5 +1,5 @@ { - "markdown": "# Model Context Protocol (MCP) Integration\n\nBSL includes built-in support for the [Model Context Protocol (MCP)](https://github.com/modelcontextprotocol/python-sdk), allowing you to expose your semantic models to Large Language Models like Claude.\n\n\n**Pro tip:** Use [descriptions in dimensions and measures](/building/semantic-tables#with_dimensions) to make your models more AI-friendly. Descriptions help provide context to LLMs, enabling them to understand what each field represents and when to use them.\n\n\n## Installation\n\nTo use MCP functionality, install BSL with the `fastmcp` extra:\n\n```bash\npip install 'boring-semantic-layer[fastmcp]'\n```\n\n## Setting up an MCP Server\n\nCreate an MCP server script that exposes your semantic models:\n\n```python\nimport ibis\nfrom boring_semantic_layer import to_semantic_table, MCPSemanticModel\n\n# Create synthetic flights data\nflights_data = ibis.memtable({\n \"flight_id\": list(range(1, 101)),\n \"origin\": [\"JFK\", \"LAX\", \"ORD\", \"ATL\", \"DFW\"] * 20,\n \"dest\": [\"LAX\", \"JFK\", \"DFW\", \"ORD\", \"ATL\"] * 20,\n \"carrier\": [\"AA\", \"UA\", \"DL\", \"WN\", \"B6\"] * 20,\n \"distance\": [2475, 2475, 801, 606, 732] * 20,\n})\n\n# Define your semantic table with descriptions\nflights = (\n to_semantic_table(flights_data, name=\"flights\")\n .with_dimensions(\n origin={\n \"expr\": lambda t: t.origin,\n \"description\": \"Origin airport code where the flight departed from\"\n },\n destination={\n \"expr\": lambda t: t.dest,\n \"description\": \"Destination airport code where the flight arrived\"\n },\n carrier={\n \"expr\": lambda t: t.carrier,\n \"description\": \"Airline carrier code (e.g., AA, UA, DL)\"\n },\n )\n .with_measures(\n total_flights={\n \"expr\": lambda t: t.count(),\n \"description\": \"Total number of flights\"\n },\n avg_distance={\n \"expr\": lambda t: t.distance.mean(),\n \"description\": \"Average flight distance in miles\"\n },\n )\n)\n\n# Create the MCP server\nmcp_server = MCPSemanticModel(\n models={\"flights\": flights},\n name=\"Flight Data Server\"\n)\n\nif __name__ == \"__main__\":\n mcp_server.run(transport=\"stdio\")\n```\n\nSave this as `example_mcp.py` in your project directory.\n\n## Configuring Claude Desktop\n\nTo use your MCP server with Claude Desktop, add it to your configuration file.\n\n**Configuration file location:**\n- **macOS:** `~/Library/Application Support/Claude/claude_desktop_config.json`\n- **Windows:** `%APPDATA%\\Claude\\claude_desktop_config.json`\n\n**Example configuration:**\n\n```json\n{\n \"mcpServers\": {\n \"flight_sm\": {\n \"command\": \"uv\",\n \"args\": [\n \"--directory\",\n \"/path/to/your/project/\",\n \"run\",\n \"example_mcp.py\"\n ]\n }\n }\n}\n```\n\nReplace `/path/to/your/project/` with the actual path to your project directory.\n\n\nThis example uses [uv](https://docs.astral.sh/uv/) to run the MCP server. You can also use `python` directly if you have BSL installed in your environment:\n\n```json\n{\n \"mcpServers\": {\n \"flight_sm\": {\n \"command\": \"python\",\n \"args\": [\"/path/to/your/project/example_mcp.py\"]\n }\n }\n}\n```\n\n\nAfter updating the configuration:\n1. Restart Claude Desktop\n2. Look for the MCP server indicator in the Claude Desktop interface\n3. You should see \"flight_sm\" listed as an available server\n\n## Available MCP Tools\n\nOnce configured, Claude will have access to these tools for interacting with your semantic models:\n\n### list_models\n\nList all available semantic model names in the MCP server.\n\n**Example usage in Claude:**\n> \"What models are available?\"\n\n**Returns:** Array of model names (e.g., `[\"flights\", \"carriers\"]`)\n\n### get_model\n\nGet detailed information about a specific model including its dimensions, measures, and descriptions.\n\n**Parameters:**\n- `model_name` (str): Name of the model to inspect\n\n**Example usage in Claude:**\n> \"Show me the details of the flights model\"\n\n**Returns:** Model schema including:\n- Model name and description\n- List of dimensions with their descriptions\n- List of measures with their descriptions\n- Available joins (if any)\n\n### get_time_range\n\nGet the available time range for time-series data in a model.\n\n**Parameters:**\n- `model_name` (str): Name of the model\n- `time_dimension` (str): Name of the time dimension\n\n**Example usage in Claude:**\n> \"What's the time range available in the flights model?\"\n\n**Returns:** Dictionary with `min_time` and `max_time` values\n\n### query_model\n\nExecute queries against a semantic model with dimensions, measures, filters, and optional chart specifications.\n\n**Parameters:**\n- `model_name` (str): Name of the model to query\n- `dimensions` (list[str]): List of dimension names to group by\n- `measures` (list[str]): List of measure names to aggregate\n- `filters` (list[str], optional): List of filter expressions (e.g., `[\"origin == 'JFK'\"]`)\n- `limit` (int, optional): Maximum number of rows to return\n- `order_by` (list[str], optional): List of columns to sort by\n- `chart_spec` (dict, optional): Vega-Lite chart specification\n\n**Example usage in Claude:**\n> \"Show me the top 10 origins by flight count\"\n> \"Create a bar chart of average distance by carrier\"\n\n**Returns:**\n- When `chart_spec` is provided: `{\"records\": [...], \"chart\": {...}}`\n- When `chart_spec` is not provided: `{\"records\": [...]}`\n\n### Example Interactions\n\nHere are some example questions you can ask Claude when the MCP server is configured:\n\n**Data Exploration:**\n- \"What models are available in the flight data server?\"\n- \"Show me all dimensions and measures in the flights model\"\n- \"What is the time range covered by the flights data?\"\n\n**Basic Queries:**\n- \"How many flights departed from JFK?\"\n- \"Show me the top 5 destinations by flight count\"\n- \"What's the average flight distance for each carrier?\"\n\n**Filtered Queries:**\n- \"Show me flights from California airports (starting with 'S')\"\n- \"What carriers have an average distance over 1000 miles?\"\n- \"List the top 10 busiest routes\"\n\n**Visualizations:**\n- \"Create a bar chart showing flights by origin airport\"\n- \"Make a line chart of flights over time\"\n- \"Show me a heatmap of routes between origins and destinations\"\n\n## Best Practices\n\n### 1. Add Descriptions to All Fields\n\nDescriptions are crucial for LLMs to understand your data model:\n\n```python\nflights = (\n to_semantic_table(flights_tbl, name=\"flights\")\n .with_dimensions(\n origin={\n \"expr\": lambda t: t.origin,\n \"description\": \"Origin airport code (3-letter IATA code)\"\n }\n )\n .with_measures(\n total_flights={\n \"expr\": lambda t: t.count(),\n \"description\": \"Total number of flights in the dataset\"\n }\n )\n)\n```\n\n### 2. Use Descriptive Model Names\n\nChoose clear, descriptive names for your models:\n\n```python\n# Good\nmcp_server = MCPSemanticModel(\n models={\"flights\": flights, \"carriers\": carriers},\n name=\"Aviation Analytics Server\"\n)\n\n# Less clear\nmcp_server = MCPSemanticModel(\n models={\"f\": flights, \"c\": carriers},\n name=\"Server\"\n)\n```\n\n### 3. Define Time Dimensions for MCP Time-Series Queries\n\nWhen exposing models through MCP, you need to explicitly define time dimensions to enable LLMs to query time ranges and perform time-based aggregations. This is specific to MCP\u2014when using BSL's fluent API directly, you can simply use Ibis functions like `.year()` and `.month()`.\n\nTo define a time dimension, set `is_time_dimension=True` and specify the `smallest_time_grain`:\n\n```python\nfrom boring_semantic_layer import to_semantic_table\n\nflights = (\n to_semantic_table(flights_data, name=\"flights\")\n .with_dimensions(\n arr_time={\n \"expr\": lambda t: t.arr_time,\n \"description\": \"Arrival time of the flight\",\n \"is_time_dimension\": True,\n \"smallest_time_grain\": \"TIME_GRAIN_SECOND\",\n },\n origin={\n \"expr\": lambda t: t.origin,\n \"description\": \"Origin airport code\"\n },\n )\n .with_measures(\n flight_count={\n \"expr\": lambda t: t.count(),\n \"description\": \"Total number of flights\"\n }\n )\n)\n```\n\n**Available time grains:**\n- `TIME_GRAIN_SECOND` - For second-level precision\n- `TIME_GRAIN_MINUTE` - For minute-level precision\n- `TIME_GRAIN_HOUR` - For hourly data\n- `TIME_GRAIN_DAY` - For daily data\n- `TIME_GRAIN_WEEK` - For weekly data\n- `TIME_GRAIN_MONTH` - For monthly data\n- `TIME_GRAIN_QUARTER` - For quarterly data\n- `TIME_GRAIN_YEAR` - For yearly data\n\n\nIf you define multiple time dimensions in your model, the `.query()` method and MCP tools will use the first time dimension that appears in your query's dimensions list.\n\n\n**Example time-based queries:**\n\nWith time dimensions defined, you can use the `.query()` method with time ranges and grains:\n\n```python\n# Query with a specific time range\nresult = flights.query(\n dimensions=[\"origin\"],\n measures=[\"flight_count\"],\n time_range={\"start\": \"2024-01-01\", \"end\": \"2024-12-31\"}\n)\n\n# Query with time grain aggregation\nresult = flights.query(\n dimensions=[\"arr_time\"],\n measures=[\"flight_count\"],\n time_grain=\"TIME_GRAIN_MONTH\"\n)\n```\n\nLLMs can then perform similar queries through MCP:\n```\n> \"What's the time range available in the flights data?\"\n> \"Show me flights from January 2024\"\n> \"Give me monthly flight counts for the last year\"\n```\n\n### 4. Structure Your Data Logically\n\nOrganize related dimensions and measures together, and use joins to connect related models:\n\n```python\n# Flights model focuses on flight operations\nflights = (\n to_semantic_table(flights_tbl, name=\"flights\")\n .with_dimensions(origin=..., destination=..., date=...)\n .with_measures(flight_count=..., avg_delay=...)\n)\n\n# Carriers model focuses on airline information\ncarriers = (\n to_semantic_table(carriers_tbl, name=\"carriers\")\n .with_dimensions(code=..., name=..., country=...)\n .with_measures(carrier_count=...)\n)\n\n# Connect them with joins\nflights_with_carriers = flights.join_one(\n carriers,\n lambda f, c: f.carrier == c.code\n)\n```\n\n## Troubleshooting\n\n### Server Not Appearing in Claude Desktop\n\n1. Check the configuration file path is correct\n2. Verify JSON syntax in `claude_desktop_config.json`\n3. Ensure BSL is installed with MCP support: `pip install 'boring-semantic-layer[fastmcp]'`\n4. Restart Claude Desktop completely\n5. Check Claude Desktop logs for error messages\n\n### Import Errors\n\nIf you see import errors when the server starts:\n\n```bash\n# Ensure all dependencies are installed\npip install 'boring-semantic-layer[fastmcp]'\n\n# Or install specific dependencies\npip install fastmcp ibis-framework\n```\n\n### Path Issues\n\nMake sure file paths in your configuration are absolute paths, not relative:\n\n```json\n{\n \"mcpServers\": {\n \"flight_sm\": {\n \"command\": \"python\",\n \"args\": [\"/Users/username/projects/my-project/example_mcp.py\"]\n }\n }\n}\n```\n\n## Next Steps\n\n- Learn about [YAML Configuration](/building/yaml) for managing multiple models\n- Explore [Query Methods](/querying/methods) to understand what queries LLMs can perform\n- See [Charting](/querying/charting) for visualization capabilities\n- Review the [full API Reference](/reference) for advanced features\n", + "markdown": "# Model Context Protocol (MCP) Integration\n\nBSL includes built-in support for the [Model Context Protocol (MCP)](https://github.com/modelcontextprotocol/python-sdk), allowing you to expose your semantic models to Large Language Models like Claude.\n\n\n**Pro tip:** Use [descriptions in dimensions and measures](/building/semantic-tables#with_dimensions) to make your models more AI-friendly. Descriptions help provide context to LLMs, enabling them to understand what each field represents and when to use them.\n\n\n## Installation\n\nTo use MCP functionality, install BSL with the `fastmcp` extra:\n\n```bash\npip install 'boring-semantic-layer[fastmcp]'\n```\n\n## Setting up an MCP Server\n\nCreate an MCP server script that exposes your semantic models:\n\n```python\nimport ibis\nfrom boring_semantic_layer import to_semantic_table, MCPSemanticModel\n\n# Create synthetic flights data\nflights_data = ibis.memtable({\n \"flight_id\": list(range(1, 101)),\n \"origin\": [\"JFK\", \"LAX\", \"ORD\", \"ATL\", \"DFW\"] * 20,\n \"dest\": [\"LAX\", \"JFK\", \"DFW\", \"ORD\", \"ATL\"] * 20,\n \"carrier\": [\"AA\", \"UA\", \"DL\", \"WN\", \"B6\"] * 20,\n \"distance\": [2475, 2475, 801, 606, 732] * 20,\n})\n\n# Define your semantic table with descriptions\nflights = (\n to_semantic_table(flights_data, name=\"flights\")\n .with_dimensions(\n origin={\n \"expr\": lambda t: t.origin,\n \"description\": \"Origin airport code where the flight departed from\"\n },\n destination={\n \"expr\": lambda t: t.dest,\n \"description\": \"Destination airport code where the flight arrived\"\n },\n carrier={\n \"expr\": lambda t: t.carrier,\n \"description\": \"Airline carrier code (e.g., AA, UA, DL)\"\n },\n )\n .with_measures(\n total_flights={\n \"expr\": lambda t: t.count(),\n \"description\": \"Total number of flights\"\n },\n avg_distance={\n \"expr\": lambda t: t.distance.mean(),\n \"description\": \"Average flight distance in miles\"\n },\n )\n)\n\n# Create the MCP server\nmcp_server = MCPSemanticModel(\n models={\"flights\": flights},\n name=\"Flight Data Server\"\n)\n\nif __name__ == \"__main__\":\n mcp_server.run(transport=\"stdio\")\n```\n\nSave this as `example_mcp.py` in your project directory.\n\n## Configuring Claude Desktop\n\nTo use your MCP server with Claude Desktop, add it to your configuration file.\n\n**Configuration file location:**\n- **macOS:** `~/Library/Application Support/Claude/claude_desktop_config.json`\n- **Windows:** `%APPDATA%\\Claude\\claude_desktop_config.json`\n\n**Example configuration:**\n\n```json\n{\n \"mcpServers\": {\n \"flight_sm\": {\n \"command\": \"uv\",\n \"args\": [\n \"--directory\",\n \"/path/to/your/project/\",\n \"run\",\n \"example_mcp.py\"\n ]\n }\n }\n}\n```\n\nReplace `/path/to/your/project/` with the actual path to your project directory.\n\n\nThis example uses [uv](https://docs.astral.sh/uv/) to run the MCP server. You can also use `python` directly if you have BSL installed in your environment:\n\n```json\n{\n \"mcpServers\": {\n \"flight_sm\": {\n \"command\": \"python\",\n \"args\": [\"/path/to/your/project/example_mcp.py\"]\n }\n }\n}\n```\n\n\nAfter updating the configuration:\n1. Restart Claude Desktop\n2. Look for the MCP server indicator in the Claude Desktop interface\n3. You should see \"flight_sm\" listed as an available server\n\n## Available MCP Tools\n\nOnce configured, Claude will have access to these tools for interacting with your semantic models:\n\n### list_models\n\nList all available semantic model names in the MCP server.\n\n**Example usage in Claude:**\n> \"What models are available?\"\n\n**Returns:** Array of model names (e.g., `[\"flights\", \"carriers\"]`)\n\n### get_model\n\nGet detailed information about a specific model including its dimensions, measures, and descriptions.\n\n**Parameters:**\n- `model_name` (str): Name of the model to inspect\n\n**Example usage in Claude:**\n> \"Show me the details of the flights model\"\n\n**Returns:** Model schema including:\n- Model name and description\n- List of dimensions with their descriptions\n- List of measures with their descriptions\n- Available joins (if any)\n\n### get_time_range\n\nGet the available time range for time-series data in a model.\n\n**Parameters:**\n- `model_name` (str): Name of the model\n- `time_dimension` (str): Name of the time dimension\n\n**Example usage in Claude:**\n> \"What's the time range available in the flights model?\"\n\n**Returns:** Dictionary with `min_time` and `max_time` values\n\n### query_model\n\nExecute queries against a semantic model with dimensions, measures, filters, and optional chart specifications.\n\n**Parameters:**\n- `model_name` (str): Name of the model to query\n- `dimensions` (list[str]): List of dimension names to group by\n- `measures` (list[str]): List of measure names to aggregate\n- `filters` (list[str], optional): List of filter expressions (e.g., `[\"origin == 'JFK'\"]`)\n- `limit` (int, optional): Maximum number of rows to return\n- `order_by` (list[str], optional): List of columns to sort by\n- `chart_spec` (dict, optional): Vega-Lite chart specification\n\n**Example usage in Claude:**\n> \"Show me the top 10 origins by flight count\"\n> \"Create a bar chart of average distance by carrier\"\n\n**Returns:**\n- When `chart_spec` is provided: `{\"records\": [...], \"chart\": {...}}`\n- When `chart_spec` is not provided: `{\"records\": [...]}`\n\n### compare_periods\n\nCompare two explicit time ranges and return `{measure}_current`, `{measure}_previous`, `{measure}_delta`, and `{measure}_pct_change` columns in a single response. This is the recommended MCP pattern for period-over-period chat questions.\n\n**Parameters:**\n- `model_name` (str): Name of the model to query\n- `measures` (list[str]): Measures to compare\n- `current_time_range` (dict): Current period with `start` and `end`\n- `previous_time_range` (dict): Comparison period with `start` and `end`\n- `dimensions` (list[str], optional): Optional grouping dimensions like `carrier` or `store`\n- `filters` (list[dict], optional): Optional filters applied to both periods\n- `time_dimension` (str, optional): Explicit time dimension when a model has more than one\n\n**Example usage in Claude:**\n> \"Compare the last 10 days to the prior 10 days by carrier\"\n> \"Show revenue this month vs last month by store\"\n\n### Example Interactions\n\nHere are some example questions you can ask Claude when the MCP server is configured:\n\n**Data Exploration:**\n- \"What models are available in the flight data server?\"\n- \"Show me all dimensions and measures in the flights model\"\n- \"What is the time range covered by the flights data?\"\n\n**Basic Queries:**\n- \"How many flights departed from JFK?\"\n- \"Show me the top 5 destinations by flight count\"\n- \"What's the average flight distance for each carrier?\"\n\n**Filtered Queries:**\n- \"Show me flights from California airports (starting with 'S')\"\n- \"What carriers have an average distance over 1000 miles?\"\n- \"List the top 10 busiest routes\"\n\n**Visualizations:**\n- \"Create a bar chart showing flights by origin airport\"\n- \"Make a line chart of flights over time\"\n- \"Show me a heatmap of routes between origins and destinations\"\n\n## Best Practices\n\n### 1. Add Descriptions to All Fields\n\nDescriptions are crucial for LLMs to understand your data model:\n\n```python\nflights = (\n to_semantic_table(flights_tbl, name=\"flights\")\n .with_dimensions(\n origin={\n \"expr\": lambda t: t.origin,\n \"description\": \"Origin airport code (3-letter IATA code)\"\n }\n )\n .with_measures(\n total_flights={\n \"expr\": lambda t: t.count(),\n \"description\": \"Total number of flights in the dataset\"\n }\n )\n)\n```\n\n### 2. Use Descriptive Model Names\n\nChoose clear, descriptive names for your models:\n\n```python\n# Good\nmcp_server = MCPSemanticModel(\n models={\"flights\": flights, \"carriers\": carriers},\n name=\"Aviation Analytics Server\"\n)\n\n# Less clear\nmcp_server = MCPSemanticModel(\n models={\"f\": flights, \"c\": carriers},\n name=\"Server\"\n)\n```\n\n### 3. Define Time Dimensions for MCP Time-Series Queries\n\nWhen exposing models through MCP, you need to explicitly define time dimensions to enable LLMs to query time ranges and perform time-based aggregations. This is specific to MCP\u2014when using BSL's fluent API directly, you can simply use Ibis functions like `.year()` and `.month()`.\n\nTo define a time dimension, set `is_time_dimension=True` and specify the `smallest_time_grain`:\n\n```python\nfrom boring_semantic_layer import to_semantic_table\n\nflights = (\n to_semantic_table(flights_data, name=\"flights\")\n .with_dimensions(\n arr_time={\n \"expr\": lambda t: t.arr_time,\n \"description\": \"Arrival time of the flight\",\n \"is_time_dimension\": True,\n \"smallest_time_grain\": \"TIME_GRAIN_SECOND\",\n },\n origin={\n \"expr\": lambda t: t.origin,\n \"description\": \"Origin airport code\"\n },\n )\n .with_measures(\n flight_count={\n \"expr\": lambda t: t.count(),\n \"description\": \"Total number of flights\"\n }\n )\n)\n```\n\n**Available time grains** (short form preferred, long form also accepted):\n- `second` / `TIME_GRAIN_SECOND` - For second-level precision\n- `minute` / `TIME_GRAIN_MINUTE` - For minute-level precision\n- `hour` / `TIME_GRAIN_HOUR` - For hourly data\n- `day` / `TIME_GRAIN_DAY` - For daily data\n- `week` / `TIME_GRAIN_WEEK` - For weekly data\n- `month` / `TIME_GRAIN_MONTH` - For monthly data\n- `quarter` / `TIME_GRAIN_QUARTER` - For quarterly data\n- `year` / `TIME_GRAIN_YEAR` - For yearly data\n\n\nUse `time_grain` to apply one grain to all time dimensions, or `time_grains` (dict) to set different grains per dimension.\n\n\n**Example time-based queries:**\n\nWith time dimensions defined, you can use the `.query()` method with time ranges and grains:\n\n```python\n# Query with a specific time range\nresult = flights.query(\n dimensions=[\"origin\"],\n measures=[\"flight_count\"],\n time_range={\"start\": \"2024-01-01\", \"end\": \"2024-12-31\"}\n)\n\n# Query with time grain aggregation (same grain for all time dims)\nresult = flights.query(\n dimensions=[\"arr_time\"],\n measures=[\"flight_count\"],\n time_grain=\"month\"\n)\n\n# Per-dimension grains (different grain per time dimension)\nresult = orders.query(\n dimensions=[\"order_date\", \"ship_date\"],\n measures=[\"total_sales\"],\n time_grains={\"order_date\": \"month\", \"ship_date\": \"quarter\"}\n)\n```\n\nLLMs can then perform similar queries through MCP:\n```\n> \"What's the time range available in the flights data?\"\n> \"Show me flights from January 2024\"\n> \"Give me monthly flight counts for the last year\"\n```\n\n### 4. Structure Your Data Logically\n\nOrganize related dimensions and measures together, and use joins to connect related models:\n\n```python\n# Flights model focuses on flight operations\nflights = (\n to_semantic_table(flights_tbl, name=\"flights\")\n .with_dimensions(origin=..., destination=..., date=...)\n .with_measures(flight_count=..., avg_delay=...)\n)\n\n# Carriers model focuses on airline information\ncarriers = (\n to_semantic_table(carriers_tbl, name=\"carriers\")\n .with_dimensions(code=..., name=..., country=...)\n .with_measures(carrier_count=...)\n)\n\n# Connect them with joins\nflights_with_carriers = flights.join_one(\n carriers,\n lambda f, c: f.carrier == c.code\n)\n```\n\n## Troubleshooting\n\n### Server Not Appearing in Claude Desktop\n\n1. Check the configuration file path is correct\n2. Verify JSON syntax in `claude_desktop_config.json`\n3. Ensure BSL is installed with MCP support: `pip install 'boring-semantic-layer[fastmcp]'`\n4. Restart Claude Desktop completely\n5. Check Claude Desktop logs for error messages\n\n### Import Errors\n\nIf you see import errors when the server starts:\n\n```bash\n# Ensure all dependencies are installed\npip install 'boring-semantic-layer[fastmcp]'\n\n# Or install specific dependencies\npip install fastmcp ibis-framework\n```\n\n### Path Issues\n\nMake sure file paths in your configuration are absolute paths, not relative:\n\n```json\n{\n \"mcpServers\": {\n \"flight_sm\": {\n \"command\": \"python\",\n \"args\": [\"/Users/username/projects/my-project/example_mcp.py\"]\n }\n }\n}\n```\n\n## Next Steps\n\n- Learn about [YAML Configuration](/building/yaml) for managing multiple models\n- Explore [Query Methods](/querying/methods) to understand what queries LLMs can perform\n- See [Charting](/querying/charting) for visualization capabilities\n- Review the [full API Reference](/reference) for advanced features\n", "queries": {}, "files": {} } diff --git a/docs/web/public/bsl-data/nested-subtotals.json b/docs/web/public/bsl-data/nested-subtotals.json index c223f328..bb323c25 100644 --- a/docs/web/public/bsl-data/nested-subtotals.json +++ b/docs/web/public/bsl-data/nested-subtotals.json @@ -1,235 +1,14 @@ { - "markdown": "# Nested Subtotals\n\nCreate hierarchical aggregations with subtotals at multiple levels using the `nest` parameter. This pattern enables drill-down analysis where each row contains both summary metrics and nested breakdowns.\n\n## Overview\n\nThe nested subtotals pattern allows you to:\n\n- Generate subtotals at each level of a dimensional hierarchy in a single query\n- Create nested structures where each parent row contains child breakdowns\n- Avoid complex self-joins or ROLLUP queries\n- Build hierarchical data suitable for tree views and drill-down UIs\n\n## Setup\n\nCreate a sample order items dataset with temporal and categorical dimensions:\n\n```setup_data\nimport ibis\nfrom ibis import _\nfrom boring_semantic_layer import to_semantic_table\n\n# Create synthetic order items data\norder_items_data = ibis.memtable({\n \"order_id\": [1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010,\n 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020,\n 1021, 1022, 1023, 1024, 1025, 1026, 1027, 1028, 1029, 1030],\n \"sale_price\": [45.99, 89.50, 120.00, 34.99, 67.80, 99.99, 54.50, 78.99, 150.00, 42.00,\n 55.99, 72.50, 88.80, 110.00, 39.99, 95.00, 62.50, 81.99, 125.00, 48.50,\n 66.99, 92.00, 105.50, 73.99, 58.80, 118.00, 84.50, 69.99, 135.00, 51.50],\n \"status\": [\"shipped\", \"delivered\", \"shipped\", \"processing\", \"delivered\",\n \"shipped\", \"cancelled\", \"delivered\", \"shipped\", \"processing\",\n \"delivered\", \"shipped\", \"delivered\", \"processing\", \"shipped\",\n \"cancelled\", \"delivered\", \"shipped\", \"delivered\", \"processing\",\n \"shipped\", \"delivered\", \"shipped\", \"processing\", \"delivered\",\n \"shipped\", \"cancelled\", \"delivered\", \"shipped\", \"processing\"],\n \"created_year\": [2022, 2022, 2022, 2022, 2022, 2022, 2022, 2022, 2022, 2022,\n 2023, 2023, 2023, 2023, 2023, 2023, 2023, 2023, 2023, 2023,\n 2024, 2024, 2024, 2024, 2024, 2024, 2024, 2024, 2024, 2024],\n \"created_month\": [1, 1, 2, 2, 3, 3, 4, 4, 5, 5,\n 1, 1, 2, 2, 3, 3, 4, 4, 5, 5,\n 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]\n})\n\n# Create semantic table with measures\norder_items = to_semantic_table(\n order_items_data,\n name=\"order_items\",\n).with_measures(\n order_count=lambda t: t.count(),\n total_sales=lambda t: t.sale_price.sum(),\n avg_price=lambda t: t.sale_price.mean(),\n)\n```\n\n\n\n## Year with Nested Month Subtotals\n\nCreate yearly totals with monthly breakdowns nested inside each year:\n\n```query_year_with_months\nfrom ibis import _\n\n# First aggregate by year and month to get monthly subtotals\nmonthly_data = (\n order_items\n .group_by(\"created_year\", \"created_month\")\n .aggregate(\"order_count\", \"total_sales\")\n)\n\n# Then nest months within years\nresult = (\n monthly_data\n .group_by(\"created_year\")\n .aggregate(\n year_order_count=lambda t: t.order_count.sum(),\n year_total_sales=lambda t: t.total_sales.sum(),\n nest={\"by_month\": lambda t: t.group_by([\"created_month\", \"order_count\", \"total_sales\"]).order_by(\"created_month\")}\n )\n .order_by(\"created_year\")\n)\n```\n\n\n\n\nEach year row contains a `by_month` array with all monthly subtotals for that year. The pattern is: aggregate at the finest level first, then nest at each parent level.\n\n\n## Year with Nested Status Subtotals\n\nAlternative breakdown: nest order status within each year:\n\n```query_year_with_status\nfrom ibis import _\n\n# First aggregate by year and status\nstatus_data = (\n order_items\n .group_by(\"created_year\", \"status\")\n .aggregate(\"order_count\", \"total_sales\", \"avg_price\")\n)\n\n# Then nest status within years\nresult = (\n status_data\n .group_by(\"created_year\")\n .aggregate(\n year_order_count=lambda t: t.order_count.sum(),\n year_total_sales=lambda t: t.total_sales.sum(),\n nest={\"by_status\": lambda t: t.group_by([\"status\", \"order_count\", \"total_sales\", \"avg_price\"]).order_by(xo.desc(\"total_sales\"))}\n )\n .order_by(\"created_year\")\n)\n```\n\n\n\n## Multi-Level Nesting: Year > Month > Status\n\nCreate three-level hierarchy with nested subtotals:\n\n```query_multi_level\nfrom ibis import _\n\n# First aggregate at the finest level: year, month, status\ndetailed_data = (\n order_items\n .group_by(\"created_year\", \"created_month\", \"status\")\n .aggregate(\"order_count\", \"total_sales\")\n)\n\n# Second level: nest status within month\nmonthly_with_status = (\n detailed_data\n .group_by(\"created_year\", \"created_month\")\n .aggregate(\n month_order_count=lambda t: t.order_count.sum(),\n month_total_sales=lambda t: t.total_sales.sum(),\n nest={\"by_status\": lambda t: t.group_by([\"status\", \"order_count\", \"total_sales\"])}\n )\n)\n\n# Top level: nest months within year\nresult = (\n monthly_with_status\n .group_by(\"created_year\")\n .aggregate(\n year_order_count=lambda t: t.month_order_count.sum(),\n year_total_sales=lambda t: t.month_total_sales.sum(),\n nest={\"by_month\": lambda t: t.group_by([\"created_month\", \"month_order_count\", \"month_total_sales\", \"by_status\"]).order_by(\"created_month\")}\n )\n .order_by(\"created_year\")\n .limit(3)\n)\n```\n\n\n\n## Use Cases\n\n**Financial Reporting**: Create income statements with nested line items - show total revenue with product categories nested inside, each containing individual products.\n\n**Geographic Hierarchies**: Aggregate sales by region, with nested states, with nested cities, all in a single query result.\n\n**Time-Based Drill-Downs**: Show yearly summaries with monthly breakdowns nested inside, perfect for dashboard drill-down interactions.\n\n**Organizational Analysis**: Display department totals with nested team breakdowns, with nested individual employee details.\n\n## Key Takeaways\n\n- Use the `nest` parameter in `.aggregate()` to create hierarchical subtotals\n- Each parent row contains an array column with child-level breakdowns\n- Avoid complex SQL ROLLUP or self-join patterns\n- Nest multiple levels deep for complex hierarchies\n- Perfect for building tree views, expandable tables, and drill-down UIs\n\n## Next Steps\n\n- Learn about [Percentage of Total](/advanced/percentage-total) calculations\n- Explore [Bucketing](/advanced/bucketing) for categorizing continuous values\n", + "markdown": "# Nested Subtotals\n\nCreate hierarchical aggregations with subtotals at multiple levels using the `nest` parameter. This pattern enables drill-down analysis where each row contains both summary metrics and nested breakdowns.\n\n## Overview\n\nThe nested subtotals pattern allows you to:\n\n- Generate subtotals at each level of a dimensional hierarchy in a single query\n- Create nested structures where each parent row contains child breakdowns\n- Avoid complex self-joins or ROLLUP queries\n- Build hierarchical data suitable for tree views and drill-down UIs\n\n## Setup\n\nCreate a sample order items dataset with temporal and categorical dimensions:\n\n```setup_data\nimport ibis\nfrom ibis import _\nfrom boring_semantic_layer import to_semantic_table\n\n# Create synthetic order items data\norder_items_data = ibis.memtable({\n \"order_id\": [1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010,\n 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020,\n 1021, 1022, 1023, 1024, 1025, 1026, 1027, 1028, 1029, 1030],\n \"sale_price\": [45.99, 89.50, 120.00, 34.99, 67.80, 99.99, 54.50, 78.99, 150.00, 42.00,\n 55.99, 72.50, 88.80, 110.00, 39.99, 95.00, 62.50, 81.99, 125.00, 48.50,\n 66.99, 92.00, 105.50, 73.99, 58.80, 118.00, 84.50, 69.99, 135.00, 51.50],\n \"status\": [\"shipped\", \"delivered\", \"shipped\", \"processing\", \"delivered\",\n \"shipped\", \"cancelled\", \"delivered\", \"shipped\", \"processing\",\n \"delivered\", \"shipped\", \"delivered\", \"processing\", \"shipped\",\n \"cancelled\", \"delivered\", \"shipped\", \"delivered\", \"processing\",\n \"shipped\", \"delivered\", \"shipped\", \"processing\", \"delivered\",\n \"shipped\", \"cancelled\", \"delivered\", \"shipped\", \"processing\"],\n \"created_year\": [2022, 2022, 2022, 2022, 2022, 2022, 2022, 2022, 2022, 2022,\n 2023, 2023, 2023, 2023, 2023, 2023, 2023, 2023, 2023, 2023,\n 2024, 2024, 2024, 2024, 2024, 2024, 2024, 2024, 2024, 2024],\n \"created_month\": [1, 1, 2, 2, 3, 3, 4, 4, 5, 5,\n 1, 1, 2, 2, 3, 3, 4, 4, 5, 5,\n 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]\n})\n\n# Create semantic table with measures\norder_items = to_semantic_table(\n order_items_data,\n name=\"order_items\",\n).with_measures(\n order_count=lambda t: t.count(),\n total_sales=lambda t: t.sale_price.sum(),\n avg_price=lambda t: t.sale_price.mean(),\n)\n```\n\n\n\n## Year with Nested Month Subtotals\n\nCreate yearly totals with monthly breakdowns nested inside each year:\n\n```query_year_with_months\nfrom ibis import _\n\n# First aggregate by year and month to get monthly subtotals\nmonthly_data = (\n order_items\n .group_by(\"created_year\", \"created_month\")\n .aggregate(\"order_count\", \"total_sales\")\n)\n\n# Then nest months within years\nresult = (\n monthly_data\n .group_by(\"created_year\")\n .aggregate(\n year_order_count=lambda t: t.order_count.sum(),\n year_total_sales=lambda t: t.total_sales.sum(),\n nest={\n \"by_month\": lambda t: (\n t.group_by(\"created_month\")\n .aggregate(\n order_count=lambda t: t.order_count.sum(),\n total_sales=lambda t: t.total_sales.sum(),\n )\n .order_by(\"created_month\")\n )\n }\n )\n .order_by(\"created_year\")\n)\n```\n\n\n\n\nEach year row contains a `by_month` array with all monthly subtotals for that year. The pattern is: aggregate at the finest level first, then nest at each parent level.\n\n\n## Year with Nested Status Subtotals\n\nAlternative breakdown: nest order status within each year:\n\n```query_year_with_status\nfrom ibis import _\n\n# First aggregate by year and status\nstatus_data = (\n order_items\n .group_by(\"created_year\", \"status\")\n .aggregate(\"order_count\", \"total_sales\", \"avg_price\")\n)\n\n# Then nest status within years\nresult = (\n status_data\n .group_by(\"created_year\")\n .aggregate(\n year_order_count=lambda t: t.order_count.sum(),\n year_total_sales=lambda t: t.total_sales.sum(),\n nest={\n \"by_status\": lambda t: (\n t.group_by(\"status\")\n .aggregate(\n order_count=lambda t: t.order_count.sum(),\n total_sales=lambda t: t.total_sales.sum(),\n avg_price=lambda t: t.avg_price.mean(),\n )\n .order_by(lambda t: t.total_sales.desc())\n )\n }\n )\n .order_by(\"created_year\")\n)\n```\n\n\n\n## Multi-Level Nesting: Year > Month > Status\n\nCreate three-level hierarchy with nested subtotals:\n\n```query_multi_level\nfrom ibis import _\n\n# First aggregate at the finest level: year, month, status\ndetailed_data = (\n order_items\n .group_by(\"created_year\", \"created_month\", \"status\")\n .aggregate(\"order_count\", \"total_sales\")\n)\n\n# Second level: nest status within month\nmonthly_with_status = (\n detailed_data\n .group_by(\"created_year\", \"created_month\")\n .aggregate(\n month_order_count=lambda t: t.order_count.sum(),\n month_total_sales=lambda t: t.total_sales.sum(),\n nest={\n \"by_status\": lambda t: (\n t.group_by(\"status\").aggregate(\n order_count=lambda t: t.order_count.sum(),\n total_sales=lambda t: t.total_sales.sum(),\n )\n )\n }\n )\n)\n\n# Top level: nest months within year\nresult = (\n monthly_with_status\n .group_by(\"created_year\")\n .aggregate(\n year_order_count=lambda t: t.month_order_count.sum(),\n year_total_sales=lambda t: t.month_total_sales.sum(),\n nest={\n \"by_month\": lambda t: t.group_by(\n \"created_month\", \"month_order_count\", \"month_total_sales\", \"by_status\"\n )\n }\n )\n .order_by(\"created_year\")\n .limit(3)\n)\n```\n\n\n\n## Use Cases\n\n**Financial Reporting**: Create income statements with nested line items - show total revenue with product categories nested inside, each containing individual products.\n\n**Geographic Hierarchies**: Aggregate sales by region, with nested states, with nested cities, all in a single query result.\n\n**Time-Based Drill-Downs**: Show yearly summaries with monthly breakdowns nested inside, perfect for dashboard drill-down interactions.\n\n**Organizational Analysis**: Display department totals with nested team breakdowns, with nested individual employee details.\n\n## Key Takeaways\n\n- Use the `nest` parameter in `.aggregate()` to create hierarchical subtotals\n- Each parent row contains an array column with child-level breakdowns\n- Avoid complex SQL ROLLUP or self-join patterns\n- Nest multiple levels deep for complex hierarchies\n- Perfect for building tree views, expandable tables, and drill-down UIs\n\n## Next Steps\n\n- Learn about [Percentage of Total](/advanced/percentage-total) calculations\n- Explore [Bucketing](/advanced/bucketing) for categorizing continuous values\n", "queries": { "setup_data": { - "code": "import ibis\nfrom ibis import _\nfrom boring_semantic_layer import to_semantic_table\n\n# Create synthetic order items data\norder_items_data = ibis.memtable({\n \"order_id\": [1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010,\n 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020,\n 1021, 1022, 1023, 1024, 1025, 1026, 1027, 1028, 1029, 1030],\n \"sale_price\": [45.99, 89.50, 120.00, 34.99, 67.80, 99.99, 54.50, 78.99, 150.00, 42.00,\n 55.99, 72.50, 88.80, 110.00, 39.99, 95.00, 62.50, 81.99, 125.00, 48.50,\n 66.99, 92.00, 105.50, 73.99, 58.80, 118.00, 84.50, 69.99, 135.00, 51.50],\n \"status\": [\"shipped\", \"delivered\", \"shipped\", \"processing\", \"delivered\",\n \"shipped\", \"cancelled\", \"delivered\", \"shipped\", \"processing\",\n \"delivered\", \"shipped\", \"delivered\", \"processing\", \"shipped\",\n \"cancelled\", \"delivered\", \"shipped\", \"delivered\", \"processing\",\n \"shipped\", \"delivered\", \"shipped\", \"processing\", \"delivered\",\n \"shipped\", \"cancelled\", \"delivered\", \"shipped\", \"processing\"],\n \"created_year\": [2022, 2022, 2022, 2022, 2022, 2022, 2022, 2022, 2022, 2022,\n 2023, 2023, 2023, 2023, 2023, 2023, 2023, 2023, 2023, 2023,\n 2024, 2024, 2024, 2024, 2024, 2024, 2024, 2024, 2024, 2024],\n \"created_month\": [1, 1, 2, 2, 3, 3, 4, 4, 5, 5,\n 1, 1, 2, 2, 3, 3, 4, 4, 5, 5,\n 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]\n})\n\n# Create semantic table with measures\norder_items = to_semantic_table(\n order_items_data,\n name=\"order_items\",\n).with_measures(\n order_count=lambda t: t.count(),\n total_sales=lambda t: t.sale_price.sum(),\n avg_price=lambda t: t.sale_price.mean(),\n)", - "sql": "SELECT\n *\nFROM \"ibis_pandas_memtable_pqzxfzd72fhkrizksnexkuj3gm\"", - "plan": "SemanticTable: order_items\n order_count [measure]\n total_sales [measure]\n avg_price [measure]", - "table": { - "columns": [ - "order_id", - "sale_price", - "status", - "created_year", - "created_month" - ], - "data": [ - [ - 1001, - 45.99, - "shipped", - 2022, - 1 - ], - [ - 1002, - 89.5, - "delivered", - 2022, - 1 - ], - [ - 1003, - 120.0, - "shipped", - 2022, - 2 - ], - [ - 1004, - 34.99, - "processing", - 2022, - 2 - ], - [ - 1005, - 67.8, - "delivered", - 2022, - 3 - ], - [ - 1006, - 99.99, - "shipped", - 2022, - 3 - ], - [ - 1007, - 54.5, - "cancelled", - 2022, - 4 - ], - [ - 1008, - 78.99, - "delivered", - 2022, - 4 - ], - [ - 1009, - 150.0, - "shipped", - 2022, - 5 - ], - [ - 1010, - 42.0, - "processing", - 2022, - 5 - ], - [ - 1011, - 55.99, - "delivered", - 2023, - 1 - ], - [ - 1012, - 72.5, - "shipped", - 2023, - 1 - ], - [ - 1013, - 88.8, - "delivered", - 2023, - 2 - ], - [ - 1014, - 110.0, - "processing", - 2023, - 2 - ], - [ - 1015, - 39.99, - "shipped", - 2023, - 3 - ], - [ - 1016, - 95.0, - "cancelled", - 2023, - 3 - ], - [ - 1017, - 62.5, - "delivered", - 2023, - 4 - ], - [ - 1018, - 81.99, - "shipped", - 2023, - 4 - ], - [ - 1019, - 125.0, - "delivered", - 2023, - 5 - ], - [ - 1020, - 48.5, - "processing", - 2023, - 5 - ], - [ - 1021, - 66.99, - "shipped", - 2024, - 1 - ], - [ - 1022, - 92.0, - "delivered", - 2024, - 1 - ], - [ - 1023, - 105.5, - "shipped", - 2024, - 2 - ], - [ - 1024, - 73.99, - "processing", - 2024, - 2 - ], - [ - 1025, - 58.8, - "delivered", - 2024, - 3 - ], - [ - 1026, - 118.0, - "shipped", - 2024, - 3 - ], - [ - 1027, - 84.5, - "cancelled", - 2024, - 4 - ], - [ - 1028, - 69.99, - "delivered", - 2024, - 4 - ], - [ - 1029, - 135.0, - "shipped", - 2024, - 5 - ], - [ - 1030, - 51.5, - "processing", - 2024, - 5 - ] - ] - } + "semantic_table": true, + "name": "order_items", + "info": "Semantic table definition stored in context" }, "query_year_with_months": { - "code": "from ibis import _\n\n# First aggregate by year and month to get monthly subtotals\nmonthly_data = (\n order_items\n .group_by(\"created_year\", \"created_month\")\n .aggregate(\"order_count\", \"total_sales\")\n)\n\n# Then nest months within years\nresult = (\n monthly_data\n .group_by(\"created_year\")\n .aggregate(\n year_order_count=lambda t: t.order_count.sum(),\n year_total_sales=lambda t: t.total_sales.sum(),\n nest={\"by_month\": lambda t: t.group_by([\"created_month\", \"order_count\", \"total_sales\"]).order_by(\"created_month\")}\n )\n .order_by(\"created_year\")\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t2\".\"created_year\",\n SUM(\"t2\".\"order_count\") AS \"year_order_count\",\n SUM(\"t2\".\"total_sales\") AS \"year_total_sales\",\n ARRAY_AGG(\n NAMED_STRUCT(\n 'created_month',\n \"t2\".\"created_month\",\n 'order_count',\n \"t2\".\"order_count\",\n 'total_sales',\n \"t2\".\"total_sales\"\n )\n ) FILTER(WHERE\n NAMED_STRUCT(\n 'created_month',\n \"t2\".\"created_month\",\n 'order_count',\n \"t2\".\"order_count\",\n 'total_sales',\n \"t2\".\"total_sales\"\n ) IS NOT NULL) AS \"by_month\"\n FROM (\n SELECT\n \"t2\".\"created_month\",\n \"t2\".\"order_count\",\n \"t2\".\"total_sales\",\n \"t2\".\"created_year\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t0\".\"created_year\",\n \"t0\".\"created_month\",\n COUNT(*) AS \"order_count\",\n SUM(\"t0\".\"sale_price\") AS \"total_sales\"\n FROM (\n SELECT\n \"t0\".\"order_id\",\n \"t0\".\"sale_price\",\n \"t0\".\"status\",\n \"t0\".\"created_year\",\n \"t0\".\"created_month\"\n FROM \"ibis_pandas_memtable_pqzxfzd72fhkrizksnexkuj3gm\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"created_year\",\n \"t0\".\"created_month\"\n ) AS \"t1\"\n ) AS \"t2\"\n ) AS t2\n GROUP BY\n \"t2\".\"created_year\"\n) AS \"t3\"\nORDER BY\n \"t3\".\"created_year\" ASC", + "code": "from ibis import _\n\n# First aggregate by year and month to get monthly subtotals\nmonthly_data = (\n order_items\n .group_by(\"created_year\", \"created_month\")\n .aggregate(\"order_count\", \"total_sales\")\n)\n\n# Then nest months within years\nresult = (\n monthly_data\n .group_by(\"created_year\")\n .aggregate(\n year_order_count=lambda t: t.order_count.sum(),\n year_total_sales=lambda t: t.total_sales.sum(),\n nest={\n \"by_month\": lambda t: (\n t.group_by(\"created_month\")\n .aggregate(\n order_count=lambda t: t.order_count.sum(),\n total_sales=lambda t: t.total_sales.sum(),\n )\n .order_by(\"created_month\")\n )\n }\n )\n .order_by(\"created_year\")\n)", + "sql": "WITH \"t2\" AS (\n SELECT\n *\n FROM (\n SELECT\n \"t0\".\"created_year\",\n \"t0\".\"created_month\",\n COUNT(*) AS \"order_count\",\n SUM(\"t0\".\"sale_price\") AS \"total_sales\"\n FROM (\n SELECT\n \"t0\".\"order_id\",\n \"t0\".\"sale_price\",\n \"t0\".\"status\",\n \"t0\".\"created_year\",\n \"t0\".\"created_month\"\n FROM \"ibis_pandas_memtable_fbb2qavjh5gd7nl4fyv27mlmmi\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"created_year\",\n \"t0\".\"created_month\"\n ) AS \"t1\"\n)\nSELECT\n *\nFROM (\n SELECT\n \"t8\".\"created_year\",\n \"t8\".\"year_order_count\",\n \"t8\".\"year_total_sales\",\n \"t11\".\"by_month\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t3\".\"created_year\",\n SUM(\"t3\".\"order_count\") AS \"year_order_count\",\n SUM(\"t3\".\"total_sales\") AS \"year_total_sales\"\n FROM (\n SELECT\n \"t3\".\"created_month\",\n \"t3\".\"order_count\",\n \"t3\".\"total_sales\",\n \"t3\".\"created_year\"\n FROM \"t2\" AS \"t3\"\n ) AS t3\n GROUP BY\n \"t3\".\"created_year\"\n ) AS \"t4\"\n ) AS \"t8\"\n LEFT OUTER JOIN (\n SELECT\n \"t9\".\"created_year\" AS \"__bsl_nest_k0__\",\n \"t9\".\"by_month\"\n FROM (\n SELECT\n \"t7\".\"created_year\",\n ARRAY_AGG(\n NAMED_STRUCT(\n 'created_month',\n \"t7\".\"created_month\",\n 'order_count',\n \"t7\".\"order_count\",\n 'total_sales',\n \"t7\".\"total_sales\"\n ) ORDER BY \"t7\".\"created_month\" ASC\n ) FILTER(WHERE\n NAMED_STRUCT(\n 'created_month',\n \"t7\".\"created_month\",\n 'order_count',\n \"t7\".\"order_count\",\n 'total_sales',\n \"t7\".\"total_sales\"\n ) IS NOT NULL) AS \"by_month\"\n FROM (\n SELECT\n \"t7\".\"created_month\",\n \"t7\".\"order_count\",\n \"t7\".\"total_sales\",\n \"t7\".\"created_year\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t3\".\"created_year\",\n \"t3\".\"created_month\",\n SUM(\"t3\".\"order_count\") AS \"order_count\",\n SUM(\"t3\".\"total_sales\") AS \"total_sales\"\n FROM (\n SELECT\n \"t3\".\"order_count\",\n \"t3\".\"total_sales\",\n \"t3\".\"created_year\",\n \"t3\".\"created_month\"\n FROM \"t2\" AS \"t3\"\n ) AS t3\n GROUP BY\n \"t3\".\"created_year\",\n \"t3\".\"created_month\"\n ) AS \"t5\"\n ) AS \"t7\"\n ) AS t7\n GROUP BY\n \"t7\".\"created_year\"\n ) AS \"t9\"\n ) AS \"t11\"\n ON (\n \"t8\".\"created_year\" = \"t11\".\"__bsl_nest_k0__\"\n )\n OR (\n (\n \"t8\".\"created_year\" IS NULL\n ) AND (\n \"t11\".\"__bsl_nest_k0__\" IS NULL\n )\n )\n) AS \"t12\"\nORDER BY\n \"t12\".\"created_year\" ASC", "plan": "SemanticTable: order_items\n order_count [measure]\n total_sales [measure]\n avg_price [measure]\n-> GroupBy(created_year, created_month)\n-> Aggregate(order_count, total_sales)\n-> GroupBy(created_year)\n-> Aggregate(year_order_count, year_total_sales, by_month)\n-> OrderBy(created_year)", "table": { "columns": [ @@ -245,29 +24,29 @@ 783.76, [ { - "created_month": 3, + "created_month": 1, "order_count": 2, - "total_sales": 167.79 + "total_sales": 135.49 }, { - "created_month": 4, + "created_month": 2, "order_count": 2, - "total_sales": 133.49 + "total_sales": 154.99 }, { - "created_month": 2, + "created_month": 3, "order_count": 2, - "total_sales": 154.99 + "total_sales": 167.79 }, { - "created_month": 5, + "created_month": 4, "order_count": 2, - "total_sales": 192.0 + "total_sales": 133.49 }, { - "created_month": 1, + "created_month": 5, "order_count": 2, - "total_sales": 135.49 + "total_sales": 192.0 } ] ], @@ -276,6 +55,11 @@ 10, 780.27, [ + { + "created_month": 1, + "order_count": 2, + "total_sales": 128.49 + }, { "created_month": 2, "order_count": 2, @@ -295,11 +79,6 @@ "created_month": 5, "order_count": 2, "total_sales": 173.5 - }, - { - "created_month": 1, - "order_count": 2, - "total_sales": 128.49 } ] ], @@ -309,14 +88,14 @@ 856.27, [ { - "created_month": 4, + "created_month": 1, "order_count": 2, - "total_sales": 154.49 + "total_sales": 158.99 }, { - "created_month": 1, + "created_month": 2, "order_count": 2, - "total_sales": 158.99 + "total_sales": 179.49 }, { "created_month": 3, @@ -324,9 +103,9 @@ "total_sales": 176.8 }, { - "created_month": 2, + "created_month": 4, "order_count": 2, - "total_sales": 179.49 + "total_sales": 154.49 }, { "created_month": 5, @@ -336,11 +115,176 @@ ] ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-75244d1929137137d7bb60883307a979" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "created_year", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "created_year", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "year_order_count", + "year_total_sales", + "by_month" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-75244d1929137137d7bb60883307a979": [ + { + "created_year": 2022, + "year_order_count": 10, + "year_total_sales": 783.76, + "by_month": [ + { + "created_month": 1, + "order_count": 2, + "total_sales": 135.49 + }, + { + "created_month": 2, + "order_count": 2, + "total_sales": 154.99 + }, + { + "created_month": 3, + "order_count": 2, + "total_sales": 167.79 + }, + { + "created_month": 4, + "order_count": 2, + "total_sales": 133.49 + }, + { + "created_month": 5, + "order_count": 2, + "total_sales": 192.0 + } + ] + }, + { + "created_year": 2023, + "year_order_count": 10, + "year_total_sales": 780.27, + "by_month": [ + { + "created_month": 1, + "order_count": 2, + "total_sales": 128.49 + }, + { + "created_month": 2, + "order_count": 2, + "total_sales": 198.8 + }, + { + "created_month": 3, + "order_count": 2, + "total_sales": 134.99 + }, + { + "created_month": 4, + "order_count": 2, + "total_sales": 144.49 + }, + { + "created_month": 5, + "order_count": 2, + "total_sales": 173.5 + } + ] + }, + { + "created_year": 2024, + "year_order_count": 10, + "year_total_sales": 856.27, + "by_month": [ + { + "created_month": 1, + "order_count": 2, + "total_sales": 158.99 + }, + { + "created_month": 2, + "order_count": 2, + "total_sales": 179.49 + }, + { + "created_month": 3, + "order_count": 2, + "total_sales": 176.8 + }, + { + "created_month": 4, + "order_count": 2, + "total_sales": 154.49 + }, + { + "created_month": 5, + "order_count": 2, + "total_sales": 186.5 + } + ] + } + ] + } + } } }, "query_year_with_status": { - "code": "from ibis import _\n\n# First aggregate by year and status\nstatus_data = (\n order_items\n .group_by(\"created_year\", \"status\")\n .aggregate(\"order_count\", \"total_sales\", \"avg_price\")\n)\n\n# Then nest status within years\nresult = (\n status_data\n .group_by(\"created_year\")\n .aggregate(\n year_order_count=lambda t: t.order_count.sum(),\n year_total_sales=lambda t: t.total_sales.sum(),\n nest={\"by_status\": lambda t: t.group_by([\"status\", \"order_count\", \"total_sales\", \"avg_price\"]).order_by(xo.desc(\"total_sales\"))}\n )\n .order_by(\"created_year\")\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t2\".\"created_year\",\n SUM(\"t2\".\"order_count\") AS \"year_order_count\",\n SUM(\"t2\".\"total_sales\") AS \"year_total_sales\",\n ARRAY_AGG(\n NAMED_STRUCT(\n 'status',\n \"t2\".\"status\",\n 'order_count',\n \"t2\".\"order_count\",\n 'total_sales',\n \"t2\".\"total_sales\",\n 'avg_price',\n \"t2\".\"avg_price\"\n )\n ) FILTER(WHERE\n NAMED_STRUCT(\n 'status',\n \"t2\".\"status\",\n 'order_count',\n \"t2\".\"order_count\",\n 'total_sales',\n \"t2\".\"total_sales\",\n 'avg_price',\n \"t2\".\"avg_price\"\n ) IS NOT NULL) AS \"by_status\"\n FROM (\n SELECT\n \"t2\".\"status\",\n \"t2\".\"order_count\",\n \"t2\".\"total_sales\",\n \"t2\".\"avg_price\",\n \"t2\".\"created_year\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t0\".\"created_year\",\n \"t0\".\"status\",\n COUNT(*) AS \"order_count\",\n SUM(\"t0\".\"sale_price\") AS \"total_sales\",\n AVG(\"t0\".\"sale_price\") AS \"avg_price\"\n FROM (\n SELECT\n \"t0\".\"order_id\",\n \"t0\".\"sale_price\",\n \"t0\".\"created_month\",\n \"t0\".\"created_year\",\n \"t0\".\"status\"\n FROM \"ibis_pandas_memtable_pqzxfzd72fhkrizksnexkuj3gm\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"created_year\",\n \"t0\".\"status\"\n ) AS \"t1\"\n ) AS \"t2\"\n ) AS t2\n GROUP BY\n \"t2\".\"created_year\"\n) AS \"t3\"\nORDER BY\n \"t3\".\"created_year\" ASC", + "code": "from ibis import _\n\n# First aggregate by year and status\nstatus_data = (\n order_items\n .group_by(\"created_year\", \"status\")\n .aggregate(\"order_count\", \"total_sales\", \"avg_price\")\n)\n\n# Then nest status within years\nresult = (\n status_data\n .group_by(\"created_year\")\n .aggregate(\n year_order_count=lambda t: t.order_count.sum(),\n year_total_sales=lambda t: t.total_sales.sum(),\n nest={\n \"by_status\": lambda t: (\n t.group_by(\"status\")\n .aggregate(\n order_count=lambda t: t.order_count.sum(),\n total_sales=lambda t: t.total_sales.sum(),\n avg_price=lambda t: t.avg_price.mean(),\n )\n .order_by(lambda t: t.total_sales.desc())\n )\n }\n )\n .order_by(\"created_year\")\n)", + "sql": "WITH \"t2\" AS (\n SELECT\n *\n FROM (\n SELECT\n \"t0\".\"created_year\",\n \"t0\".\"status\",\n COUNT(*) AS \"order_count\",\n SUM(\"t0\".\"sale_price\") AS \"total_sales\",\n AVG(\"t0\".\"sale_price\") AS \"avg_price\"\n FROM (\n SELECT\n \"t0\".\"order_id\",\n \"t0\".\"sale_price\",\n \"t0\".\"created_month\",\n \"t0\".\"created_year\",\n \"t0\".\"status\"\n FROM \"ibis_pandas_memtable_fbb2qavjh5gd7nl4fyv27mlmmi\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"created_year\",\n \"t0\".\"status\"\n ) AS \"t1\"\n)\nSELECT\n *\nFROM (\n SELECT\n \"t8\".\"created_year\",\n \"t8\".\"year_order_count\",\n \"t8\".\"year_total_sales\",\n \"t11\".\"by_status\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t3\".\"created_year\",\n SUM(\"t3\".\"order_count\") AS \"year_order_count\",\n SUM(\"t3\".\"total_sales\") AS \"year_total_sales\"\n FROM (\n SELECT\n \"t3\".\"status\",\n \"t3\".\"order_count\",\n \"t3\".\"total_sales\",\n \"t3\".\"avg_price\",\n \"t3\".\"created_year\"\n FROM \"t2\" AS \"t3\"\n ) AS t3\n GROUP BY\n \"t3\".\"created_year\"\n ) AS \"t4\"\n ) AS \"t8\"\n LEFT OUTER JOIN (\n SELECT\n \"t9\".\"created_year\" AS \"__bsl_nest_k0__\",\n \"t9\".\"by_status\"\n FROM (\n SELECT\n \"t7\".\"created_year\",\n ARRAY_AGG(\n NAMED_STRUCT(\n 'status',\n \"t7\".\"status\",\n 'order_count',\n \"t7\".\"order_count\",\n 'total_sales',\n \"t7\".\"total_sales\",\n 'avg_price',\n \"t7\".\"avg_price\"\n ) ORDER BY \"t7\".\"total_sales\" DESC NULLS LAST\n ) FILTER(WHERE\n NAMED_STRUCT(\n 'status',\n \"t7\".\"status\",\n 'order_count',\n \"t7\".\"order_count\",\n 'total_sales',\n \"t7\".\"total_sales\",\n 'avg_price',\n \"t7\".\"avg_price\"\n ) IS NOT NULL) AS \"by_status\"\n FROM (\n SELECT\n \"t7\".\"status\",\n \"t7\".\"order_count\",\n \"t7\".\"total_sales\",\n \"t7\".\"avg_price\",\n \"t7\".\"created_year\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t3\".\"created_year\",\n \"t3\".\"status\",\n SUM(\"t3\".\"order_count\") AS \"order_count\",\n SUM(\"t3\".\"total_sales\") AS \"total_sales\",\n AVG(\"t3\".\"avg_price\") AS \"avg_price\"\n FROM (\n SELECT\n \"t3\".\"order_count\",\n \"t3\".\"total_sales\",\n \"t3\".\"avg_price\",\n \"t3\".\"created_year\",\n \"t3\".\"status\"\n FROM \"t2\" AS \"t3\"\n ) AS t3\n GROUP BY\n \"t3\".\"created_year\",\n \"t3\".\"status\"\n ) AS \"t5\"\n ) AS \"t7\"\n ) AS t7\n GROUP BY\n \"t7\".\"created_year\"\n ) AS \"t9\"\n ) AS \"t11\"\n ON (\n \"t8\".\"created_year\" = \"t11\".\"__bsl_nest_k0__\"\n )\n OR (\n (\n \"t8\".\"created_year\" IS NULL\n ) AND (\n \"t11\".\"__bsl_nest_k0__\" IS NULL\n )\n )\n) AS \"t12\"\nORDER BY\n \"t12\".\"created_year\" ASC", "plan": "SemanticTable: order_items\n order_count [measure]\n total_sales [measure]\n avg_price [measure]\n-> GroupBy(created_year, status)\n-> Aggregate(order_count, total_sales, avg_price)\n-> GroupBy(created_year)\n-> Aggregate(year_order_count, year_total_sales, by_status)\n-> OrderBy(created_year)", "table": { "columns": [ @@ -362,10 +306,10 @@ "avg_price": 103.995 }, { - "status": "cancelled", - "order_count": 1, - "total_sales": 54.5, - "avg_price": 54.5 + "status": "delivered", + "order_count": 3, + "total_sales": 236.29000000000002, + "avg_price": 78.76333333333334 }, { "status": "processing", @@ -374,10 +318,10 @@ "avg_price": 38.495000000000005 }, { - "status": "delivered", - "order_count": 3, - "total_sales": 236.29000000000002, - "avg_price": 78.76333333333334 + "status": "cancelled", + "order_count": 1, + "total_sales": 54.5, + "avg_price": 54.5 } ] ], @@ -387,10 +331,10 @@ 780.27, [ { - "status": "processing", - "order_count": 2, - "total_sales": 158.5, - "avg_price": 79.25 + "status": "delivered", + "order_count": 4, + "total_sales": 332.28999999999996, + "avg_price": 83.07249999999999 }, { "status": "shipped", @@ -399,10 +343,10 @@ "avg_price": 64.82666666666667 }, { - "status": "delivered", - "order_count": 4, - "total_sales": 332.28999999999996, - "avg_price": 83.07249999999999 + "status": "processing", + "order_count": 2, + "total_sales": 158.5, + "avg_price": 79.25 }, { "status": "cancelled", @@ -444,11 +388,173 @@ ] ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-c3a57242c76d9fee67baed2dec5037b2" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "created_year", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "created_year", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "year_order_count", + "year_total_sales", + "by_status" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-c3a57242c76d9fee67baed2dec5037b2": [ + { + "created_year": 2022, + "year_order_count": 10, + "year_total_sales": 783.76, + "by_status": [ + { + "status": "shipped", + "order_count": 4, + "total_sales": 415.98, + "avg_price": 103.995 + }, + { + "status": "delivered", + "order_count": 3, + "total_sales": 236.29000000000002, + "avg_price": 78.76333333333334 + }, + { + "status": "processing", + "order_count": 2, + "total_sales": 76.99000000000001, + "avg_price": 38.495000000000005 + }, + { + "status": "cancelled", + "order_count": 1, + "total_sales": 54.5, + "avg_price": 54.5 + } + ] + }, + { + "created_year": 2023, + "year_order_count": 10, + "year_total_sales": 780.27, + "by_status": [ + { + "status": "delivered", + "order_count": 4, + "total_sales": 332.28999999999996, + "avg_price": 83.07249999999999 + }, + { + "status": "shipped", + "order_count": 3, + "total_sales": 194.48000000000002, + "avg_price": 64.82666666666667 + }, + { + "status": "processing", + "order_count": 2, + "total_sales": 158.5, + "avg_price": 79.25 + }, + { + "status": "cancelled", + "order_count": 1, + "total_sales": 95.0, + "avg_price": 95.0 + } + ] + }, + { + "created_year": 2024, + "year_order_count": 10, + "year_total_sales": 856.27, + "by_status": [ + { + "status": "shipped", + "order_count": 4, + "total_sales": 425.49, + "avg_price": 106.3725 + }, + { + "status": "delivered", + "order_count": 3, + "total_sales": 220.79000000000002, + "avg_price": 73.59666666666668 + }, + { + "status": "processing", + "order_count": 2, + "total_sales": 125.49, + "avg_price": 62.745 + }, + { + "status": "cancelled", + "order_count": 1, + "total_sales": 84.5, + "avg_price": 84.5 + } + ] + } + ] + } + } } }, "query_multi_level": { - "code": "from ibis import _\n\n# First aggregate at the finest level: year, month, status\ndetailed_data = (\n order_items\n .group_by(\"created_year\", \"created_month\", \"status\")\n .aggregate(\"order_count\", \"total_sales\")\n)\n\n# Second level: nest status within month\nmonthly_with_status = (\n detailed_data\n .group_by(\"created_year\", \"created_month\")\n .aggregate(\n month_order_count=lambda t: t.order_count.sum(),\n month_total_sales=lambda t: t.total_sales.sum(),\n nest={\"by_status\": lambda t: t.group_by([\"status\", \"order_count\", \"total_sales\"])}\n )\n)\n\n# Top level: nest months within year\nresult = (\n monthly_with_status\n .group_by(\"created_year\")\n .aggregate(\n year_order_count=lambda t: t.month_order_count.sum(),\n year_total_sales=lambda t: t.month_total_sales.sum(),\n nest={\"by_month\": lambda t: t.group_by([\"created_month\", \"month_order_count\", \"month_total_sales\", \"by_status\"]).order_by(\"created_month\")}\n )\n .order_by(\"created_year\")\n .limit(3)\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t4\".\"created_year\",\n SUM(\"t4\".\"month_order_count\") AS \"year_order_count\",\n SUM(\"t4\".\"month_total_sales\") AS \"year_total_sales\",\n ARRAY_AGG(\n NAMED_STRUCT(\n 'created_month',\n \"t4\".\"created_month\",\n 'month_order_count',\n \"t4\".\"month_order_count\",\n 'month_total_sales',\n \"t4\".\"month_total_sales\",\n 'by_status',\n \"t4\".\"by_status\"\n )\n ) FILTER(WHERE\n NAMED_STRUCT(\n 'created_month',\n \"t4\".\"created_month\",\n 'month_order_count',\n \"t4\".\"month_order_count\",\n 'month_total_sales',\n \"t4\".\"month_total_sales\",\n 'by_status',\n \"t4\".\"by_status\"\n ) IS NOT NULL) AS \"by_month\"\n FROM (\n SELECT\n \"t4\".\"created_month\",\n \"t4\".\"month_order_count\",\n \"t4\".\"month_total_sales\",\n \"t4\".\"by_status\",\n \"t4\".\"created_year\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t2\".\"created_year\",\n \"t2\".\"created_month\",\n SUM(\"t2\".\"order_count\") AS \"month_order_count\",\n SUM(\"t2\".\"total_sales\") AS \"month_total_sales\",\n ARRAY_AGG(\n NAMED_STRUCT(\n 'status',\n \"t2\".\"status\",\n 'order_count',\n \"t2\".\"order_count\",\n 'total_sales',\n \"t2\".\"total_sales\"\n )\n ) FILTER(WHERE\n NAMED_STRUCT(\n 'status',\n \"t2\".\"status\",\n 'order_count',\n \"t2\".\"order_count\",\n 'total_sales',\n \"t2\".\"total_sales\"\n ) IS NOT NULL) AS \"by_status\"\n FROM (\n SELECT\n \"t2\".\"status\",\n \"t2\".\"order_count\",\n \"t2\".\"total_sales\",\n \"t2\".\"created_year\",\n \"t2\".\"created_month\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t0\".\"created_year\",\n \"t0\".\"created_month\",\n \"t0\".\"status\",\n COUNT(*) AS \"order_count\",\n SUM(\"t0\".\"sale_price\") AS \"total_sales\"\n FROM (\n SELECT\n \"t0\".\"order_id\",\n \"t0\".\"sale_price\",\n \"t0\".\"created_year\",\n \"t0\".\"created_month\",\n \"t0\".\"status\"\n FROM \"ibis_pandas_memtable_pqzxfzd72fhkrizksnexkuj3gm\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"created_year\",\n \"t0\".\"created_month\",\n \"t0\".\"status\"\n ) AS \"t1\"\n ) AS \"t2\"\n ) AS t2\n GROUP BY\n \"t2\".\"created_year\",\n \"t2\".\"created_month\"\n ) AS \"t3\"\n ) AS \"t4\"\n ) AS t4\n GROUP BY\n \"t4\".\"created_year\"\n) AS \"t5\"\nORDER BY\n \"t5\".\"created_year\" ASC\nLIMIT 3", + "code": "from ibis import _\n\n# First aggregate at the finest level: year, month, status\ndetailed_data = (\n order_items\n .group_by(\"created_year\", \"created_month\", \"status\")\n .aggregate(\"order_count\", \"total_sales\")\n)\n\n# Second level: nest status within month\nmonthly_with_status = (\n detailed_data\n .group_by(\"created_year\", \"created_month\")\n .aggregate(\n month_order_count=lambda t: t.order_count.sum(),\n month_total_sales=lambda t: t.total_sales.sum(),\n nest={\n \"by_status\": lambda t: (\n t.group_by(\"status\").aggregate(\n order_count=lambda t: t.order_count.sum(),\n total_sales=lambda t: t.total_sales.sum(),\n )\n )\n }\n )\n)\n\n# Top level: nest months within year\nresult = (\n monthly_with_status\n .group_by(\"created_year\")\n .aggregate(\n year_order_count=lambda t: t.month_order_count.sum(),\n year_total_sales=lambda t: t.month_total_sales.sum(),\n nest={\n \"by_month\": lambda t: t.group_by(\n \"created_month\", \"month_order_count\", \"month_total_sales\", \"by_status\"\n )\n }\n )\n .order_by(\"created_year\")\n .limit(3)\n)", + "sql": "WITH \"t2\" AS (\n SELECT\n *\n FROM (\n SELECT\n \"t0\".\"created_year\",\n \"t0\".\"created_month\",\n \"t0\".\"status\",\n COUNT(*) AS \"order_count\",\n SUM(\"t0\".\"sale_price\") AS \"total_sales\"\n FROM (\n SELECT\n \"t0\".\"order_id\",\n \"t0\".\"sale_price\",\n \"t0\".\"created_year\",\n \"t0\".\"created_month\",\n \"t0\".\"status\"\n FROM \"ibis_pandas_memtable_fbb2qavjh5gd7nl4fyv27mlmmi\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"created_year\",\n \"t0\".\"created_month\",\n \"t0\".\"status\"\n ) AS \"t1\"\n)\nSELECT\n *\nFROM (\n SELECT\n \"t12\".\"created_year\",\n SUM(\"t12\".\"month_order_count\") AS \"year_order_count\",\n SUM(\"t12\".\"month_total_sales\") AS \"year_total_sales\",\n ARRAY_AGG(\n NAMED_STRUCT(\n 'created_month',\n \"t12\".\"created_month\",\n 'month_order_count',\n \"t12\".\"month_order_count\",\n 'month_total_sales',\n \"t12\".\"month_total_sales\",\n 'by_status',\n \"t12\".\"by_status\"\n )\n ) FILTER(WHERE\n NAMED_STRUCT(\n 'created_month',\n \"t12\".\"created_month\",\n 'month_order_count',\n \"t12\".\"month_order_count\",\n 'month_total_sales',\n \"t12\".\"month_total_sales\",\n 'by_status',\n \"t12\".\"by_status\"\n ) IS NOT NULL) AS \"by_month\"\n FROM (\n SELECT\n \"t12\".\"created_month\",\n \"t12\".\"month_order_count\",\n \"t12\".\"month_total_sales\",\n \"t12\".\"by_status\",\n \"t12\".\"created_year\"\n FROM (\n SELECT\n \"t8\".\"created_year\",\n \"t8\".\"created_month\",\n \"t8\".\"month_order_count\",\n \"t8\".\"month_total_sales\",\n \"t11\".\"by_status\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t3\".\"created_year\",\n \"t3\".\"created_month\",\n SUM(\"t3\".\"order_count\") AS \"month_order_count\",\n SUM(\"t3\".\"total_sales\") AS \"month_total_sales\"\n FROM (\n SELECT\n \"t3\".\"status\",\n \"t3\".\"order_count\",\n \"t3\".\"total_sales\",\n \"t3\".\"created_year\",\n \"t3\".\"created_month\"\n FROM \"t2\" AS \"t3\"\n ) AS t3\n GROUP BY\n \"t3\".\"created_year\",\n \"t3\".\"created_month\"\n ) AS \"t4\"\n ) AS \"t8\"\n LEFT OUTER JOIN (\n SELECT\n \"t9\".\"created_year\" AS \"__bsl_nest_k0__\",\n \"t9\".\"created_month\" AS \"__bsl_nest_k1__\",\n \"t9\".\"by_status\"\n FROM (\n SELECT\n \"t7\".\"created_year\",\n \"t7\".\"created_month\",\n ARRAY_AGG(\n NAMED_STRUCT(\n 'status',\n \"t7\".\"status\",\n 'order_count',\n \"t7\".\"order_count\",\n 'total_sales',\n \"t7\".\"total_sales\"\n )\n ) FILTER(WHERE\n NAMED_STRUCT(\n 'status',\n \"t7\".\"status\",\n 'order_count',\n \"t7\".\"order_count\",\n 'total_sales',\n \"t7\".\"total_sales\"\n ) IS NOT NULL) AS \"by_status\"\n FROM (\n SELECT\n \"t7\".\"status\",\n \"t7\".\"order_count\",\n \"t7\".\"total_sales\",\n \"t7\".\"created_year\",\n \"t7\".\"created_month\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t3\".\"created_year\",\n \"t3\".\"created_month\",\n \"t3\".\"status\",\n SUM(\"t3\".\"order_count\") AS \"order_count\",\n SUM(\"t3\".\"total_sales\") AS \"total_sales\"\n FROM (\n SELECT\n \"t3\".\"order_count\",\n \"t3\".\"total_sales\",\n \"t3\".\"created_year\",\n \"t3\".\"created_month\",\n \"t3\".\"status\"\n FROM \"t2\" AS \"t3\"\n ) AS t3\n GROUP BY\n \"t3\".\"created_year\",\n \"t3\".\"created_month\",\n \"t3\".\"status\"\n ) AS \"t5\"\n ) AS \"t7\"\n ) AS t7\n GROUP BY\n \"t7\".\"created_year\",\n \"t7\".\"created_month\"\n ) AS \"t9\"\n ) AS \"t11\"\n ON (\n (\n \"t8\".\"created_year\" = \"t11\".\"__bsl_nest_k0__\"\n )\n OR (\n (\n \"t8\".\"created_year\" IS NULL\n ) AND (\n \"t11\".\"__bsl_nest_k0__\" IS NULL\n )\n )\n )\n AND (\n (\n \"t8\".\"created_month\" = \"t11\".\"__bsl_nest_k1__\"\n )\n OR (\n (\n \"t8\".\"created_month\" IS NULL\n ) AND (\n \"t11\".\"__bsl_nest_k1__\" IS NULL\n )\n )\n )\n ) AS \"t12\"\n ) AS t12\n GROUP BY\n \"t12\".\"created_year\"\n) AS \"t13\"\nORDER BY\n \"t13\".\"created_year\" ASC\nLIMIT 3", "plan": "SemanticTable: order_items\n order_count [measure]\n total_sales [measure]\n avg_price [measure]\n-> GroupBy(created_year, created_month, status)\n-> Aggregate(order_count, total_sales)\n-> GroupBy(created_year, created_month)\n-> Aggregate(month_order_count, month_total_sales, by_status)\n-> GroupBy(created_year)\n-> Aggregate(year_order_count, year_total_sales, by_month)\n-> OrderBy(created_year)\n-> Limit(3)", "table": { "columns": [ @@ -464,87 +570,87 @@ 783.76, [ { - "created_month": 3, + "created_month": 5, "month_order_count": 2, - "month_total_sales": 167.79, + "month_total_sales": 192.0, "by_status": [ { "status": "shipped", "order_count": 1, - "total_sales": 99.99 + "total_sales": 150.0 }, { - "status": "delivered", + "status": "processing", "order_count": 1, - "total_sales": 67.8 + "total_sales": 42.0 } ] }, { - "created_month": 2, + "created_month": 4, "month_order_count": 2, - "month_total_sales": 154.99, + "month_total_sales": 133.49, "by_status": [ { - "status": "shipped", + "status": "delivered", "order_count": 1, - "total_sales": 120.0 + "total_sales": 78.99 }, { - "status": "processing", + "status": "cancelled", "order_count": 1, - "total_sales": 34.99 + "total_sales": 54.5 } ] }, { - "created_month": 1, + "created_month": 2, "month_order_count": 2, - "month_total_sales": 135.49, + "month_total_sales": 154.99, "by_status": [ { - "status": "shipped", + "status": "processing", "order_count": 1, - "total_sales": 45.99 + "total_sales": 34.99 }, { - "status": "delivered", + "status": "shipped", "order_count": 1, - "total_sales": 89.5 + "total_sales": 120.0 } ] }, { - "created_month": 4, + "created_month": 3, "month_order_count": 2, - "month_total_sales": 133.49, + "month_total_sales": 167.79, "by_status": [ { "status": "delivered", "order_count": 1, - "total_sales": 78.99 + "total_sales": 67.8 }, { - "status": "cancelled", + "status": "shipped", "order_count": 1, - "total_sales": 54.5 + "total_sales": 99.99 } ] }, { - "created_month": 5, + "created_month": 1, "month_order_count": 2, - "month_total_sales": 192.0, + "month_total_sales": 135.49, "by_status": [ { - "status": "processing", + "status": "delivered", "order_count": 1, - "total_sales": 42.0 + "total_sales": 89.5 }, { "status": "shipped", "order_count": 1, - "total_sales": 150.0 + "total_sales": 45.99 } ] } @@ -561,31 +667,31 @@ "month_total_sales": 134.99, "by_status": [ { - "status": "cancelled", + "status": "shipped", "order_count": 1, - "total_sales": 95.0 + "total_sales": 39.99 }, { - "status": "shipped", + "status": "cancelled", "order_count": 1, - "total_sales": 39.99 + "total_sales": 95.0 } ] }, { - "created_month": 2, + "created_month": 1, "month_order_count": 2, - "month_total_sales": 198.8, + "month_total_sales": 128.49, "by_status": [ { "status": "delivered", "order_count": 1, - "total_sales": 88.8 + "total_sales": 55.99 }, { - "status": "processing", + "status": "shipped", "order_count": 1, - "total_sales": 110.0 + "total_sales": 72.5 } ] }, @@ -595,14 +701,14 @@ "month_total_sales": 173.5, "by_status": [ { - "status": "processing", + "status": "delivered", "order_count": 1, - "total_sales": 48.5 + "total_sales": 125.0 }, { - "status": "delivered", + "status": "processing", "order_count": 1, - "total_sales": 125.0 + "total_sales": 48.5 } ] }, @@ -624,19 +730,19 @@ ] }, { - "created_month": 1, + "created_month": 2, "month_order_count": 2, - "month_total_sales": 128.49, + "month_total_sales": 198.8, "by_status": [ { "status": "delivered", "order_count": 1, - "total_sales": 55.99 + "total_sales": 88.8 }, { - "status": "shipped", + "status": "processing", "order_count": 1, - "total_sales": 72.5 + "total_sales": 110.0 } ] } @@ -682,19 +788,19 @@ ] }, { - "created_month": 3, + "created_month": 2, "month_order_count": 2, - "month_total_sales": 176.8, + "month_total_sales": 179.49, "by_status": [ { - "status": "delivered", + "status": "shipped", "order_count": 1, - "total_sales": 58.8 + "total_sales": 105.5 }, { - "status": "shipped", + "status": "processing", "order_count": 1, - "total_sales": 118.0 + "total_sales": 73.99 } ] }, @@ -704,37 +810,382 @@ "month_total_sales": 154.49, "by_status": [ { - "status": "cancelled", + "status": "delivered", "order_count": 1, - "total_sales": 84.5 + "total_sales": 69.99 }, { - "status": "delivered", + "status": "cancelled", "order_count": 1, - "total_sales": 69.99 + "total_sales": 84.5 } ] }, { - "created_month": 2, + "created_month": 3, "month_order_count": 2, - "month_total_sales": 179.49, + "month_total_sales": 176.8, "by_status": [ { - "status": "processing", + "status": "delivered", "order_count": 1, - "total_sales": 73.99 + "total_sales": 58.8 }, { "status": "shipped", "order_count": 1, - "total_sales": 105.5 + "total_sales": 118.0 } ] } ] ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-73f5bebd43c0f207e5d5d7ad0e5b617e" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "created_year", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "created_year", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "year_order_count", + "year_total_sales", + "by_month" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-73f5bebd43c0f207e5d5d7ad0e5b617e": [ + { + "created_year": 2022, + "year_order_count": 10, + "year_total_sales": 783.76, + "by_month": [ + { + "created_month": 1, + "month_order_count": 2, + "month_total_sales": 135.49, + "by_status": [ + { + "status": "delivered", + "order_count": 1, + "total_sales": 89.5 + }, + { + "status": "shipped", + "order_count": 1, + "total_sales": 45.99 + } + ] + }, + { + "created_month": 3, + "month_order_count": 2, + "month_total_sales": 167.79, + "by_status": [ + { + "status": "shipped", + "order_count": 1, + "total_sales": 99.99 + }, + { + "status": "delivered", + "order_count": 1, + "total_sales": 67.8 + } + ] + }, + { + "created_month": 2, + "month_order_count": 2, + "month_total_sales": 154.99, + "by_status": [ + { + "status": "shipped", + "order_count": 1, + "total_sales": 120.0 + }, + { + "status": "processing", + "order_count": 1, + "total_sales": 34.99 + } + ] + }, + { + "created_month": 4, + "month_order_count": 2, + "month_total_sales": 133.49, + "by_status": [ + { + "status": "delivered", + "order_count": 1, + "total_sales": 78.99 + }, + { + "status": "cancelled", + "order_count": 1, + "total_sales": 54.5 + } + ] + }, + { + "created_month": 5, + "month_order_count": 2, + "month_total_sales": 192.0, + "by_status": [ + { + "status": "processing", + "order_count": 1, + "total_sales": 42.0 + }, + { + "status": "shipped", + "order_count": 1, + "total_sales": 150.0 + } + ] + } + ] + }, + { + "created_year": 2023, + "year_order_count": 10, + "year_total_sales": 780.27, + "by_month": [ + { + "created_month": 5, + "month_order_count": 2, + "month_total_sales": 173.5, + "by_status": [ + { + "status": "delivered", + "order_count": 1, + "total_sales": 125.0 + }, + { + "status": "processing", + "order_count": 1, + "total_sales": 48.5 + } + ] + }, + { + "created_month": 3, + "month_order_count": 2, + "month_total_sales": 134.99, + "by_status": [ + { + "status": "shipped", + "order_count": 1, + "total_sales": 39.99 + }, + { + "status": "cancelled", + "order_count": 1, + "total_sales": 95.0 + } + ] + }, + { + "created_month": 1, + "month_order_count": 2, + "month_total_sales": 128.49, + "by_status": [ + { + "status": "delivered", + "order_count": 1, + "total_sales": 55.99 + }, + { + "status": "shipped", + "order_count": 1, + "total_sales": 72.5 + } + ] + }, + { + "created_month": 2, + "month_order_count": 2, + "month_total_sales": 198.8, + "by_status": [ + { + "status": "processing", + "order_count": 1, + "total_sales": 110.0 + }, + { + "status": "delivered", + "order_count": 1, + "total_sales": 88.8 + } + ] + }, + { + "created_month": 4, + "month_order_count": 2, + "month_total_sales": 144.49, + "by_status": [ + { + "status": "shipped", + "order_count": 1, + "total_sales": 81.99 + }, + { + "status": "delivered", + "order_count": 1, + "total_sales": 62.5 + } + ] + } + ] + }, + { + "created_year": 2024, + "year_order_count": 10, + "year_total_sales": 856.27, + "by_month": [ + { + "created_month": 3, + "month_order_count": 2, + "month_total_sales": 176.8, + "by_status": [ + { + "status": "delivered", + "order_count": 1, + "total_sales": 58.8 + }, + { + "status": "shipped", + "order_count": 1, + "total_sales": 118.0 + } + ] + }, + { + "created_month": 1, + "month_order_count": 2, + "month_total_sales": 158.99, + "by_status": [ + { + "status": "shipped", + "order_count": 1, + "total_sales": 66.99 + }, + { + "status": "delivered", + "order_count": 1, + "total_sales": 92.0 + } + ] + }, + { + "created_month": 2, + "month_order_count": 2, + "month_total_sales": 179.49, + "by_status": [ + { + "status": "processing", + "order_count": 1, + "total_sales": 73.99 + }, + { + "status": "shipped", + "order_count": 1, + "total_sales": 105.5 + } + ] + }, + { + "created_month": 4, + "month_order_count": 2, + "month_total_sales": 154.49, + "by_status": [ + { + "status": "delivered", + "order_count": 1, + "total_sales": 69.99 + }, + { + "status": "cancelled", + "order_count": 1, + "total_sales": 84.5 + } + ] + }, + { + "created_month": 5, + "month_order_count": 2, + "month_total_sales": 186.5, + "by_status": [ + { + "status": "processing", + "order_count": 1, + "total_sales": 51.5 + }, + { + "status": "shipped", + "order_count": 1, + "total_sales": 135.0 + } + ] + } + ] + } + ] + } + } } } }, diff --git a/docs/web/public/bsl-data/percentage-total.json b/docs/web/public/bsl-data/percentage-total.json index 784aa531..b240ab5c 100644 --- a/docs/web/public/bsl-data/percentage-total.json +++ b/docs/web/public/bsl-data/percentage-total.json @@ -2,374 +2,13 @@ "markdown": "# Percentage of Total\n\nCalculate percentages relative to total values across different dimensions. Use this pattern when you need to understand market share, contribution ratios, or what proportion each segment represents of the whole.\n\n## Overview\n\nThe percentage of total pattern allows you to:\n\n- Define percentage measures using the `.all()` method\n- Calculate individual segment values as percentages of the grand total\n- Maintain dimensional breakdowns while computing percentage contributions\n- Support multiple aggregation functions (sum, count, average)\n\n## Setup\n\nLet's use the flights dataset with carrier information to demonstrate market share calculations:\n\n```setup_data\nimport ibis\nfrom ibis import _\nfrom boring_semantic_layer import to_semantic_table\n\n# Create synthetic flights data with carrier information\nflights_data = ibis.memtable({\n \"flight_id\": list(range(1, 51)),\n \"carrier\": [\"AA\", \"UA\", \"DL\", \"WN\", \"B6\"] * 10,\n \"nickname\": [\"American Airlines\", \"United Airlines\", \"Delta Air Lines\",\n \"Southwest Airlines\", \"JetBlue Airways\"] * 10,\n \"origin\": [\"JFK\", \"LAX\", \"ORD\", \"ATL\", \"DFW\"] * 10,\n \"distance\": [2475, 1745, 733, 946, 1383, 2475, 1745, 733, 946, 1383,\n 2475, 1745, 733, 946, 1383, 2475, 1745, 733, 946, 1383,\n 2475, 1745, 733, 946, 1383, 2475, 1745, 733, 946, 1383,\n 2475, 1745, 733, 946, 1383, 2475, 1745, 733, 946, 1383,\n 2475, 1745, 733, 946, 1383, 2475, 1745, 733, 946, 1383]\n})\n\n# Create semantic table with measures including percentage calculations\nflights = (\n to_semantic_table(flights_data, name=\"flights\")\n .with_measures(\n flight_count=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n )\n .with_measures(\n market_share=lambda t: t.flight_count / t.all(t.flight_count) * 100,\n distance_share=lambda t: t.total_distance / t.all(t.total_distance) * 100,\n )\n)\n```\n\n\n\n\nThe `.all()` method calculates the grand total across all groups, allowing you to define percentage measures directly in the semantic table. This is more elegant than using window functions in post-processing.\n\n\n## Market Share by Carrier\n\nCalculate each carrier's percentage of total flights:\n\n```query_market_share\nfrom ibis import _\n\nresult = (\n flights.group_by(\"nickname\")\n .aggregate(\"flight_count\", \"market_share\")\n .order_by(_.market_share.desc())\n .limit(10)\n)\n```\n\n\n\n## Market Share by Origin and Carrier\n\nCalculate market share broken down by both origin airport and carrier:\n\n```query_market_share_by_origin\nfrom ibis import _\n\nresult = (\n flights.group_by(\"origin\", \"nickname\")\n .aggregate(\"flight_count\", \"market_share\")\n .order_by(_.market_share.desc())\n .limit(15)\n)\n```\n\n\n\n## Use Cases\n\n**Market Share Analysis**: Calculate each carrier's, product's, or region's share of total volume.\n\n**Traffic Distribution**: Determine what percentage of total website visits or conversions come from each source.\n\n**Resource Allocation**: Understand how resources (budget, time, capacity) are distributed as percentages of the total.\n\n## Key Takeaways\n\n- Define percentage measures using `.all()` to reference the grand total\n- The `.all(measure)` method calculates the total across all groups\n- Percentage measures work seamlessly across different dimensional breakdowns\n- More elegant than post-processing with window functions\n\n## Next Steps\n\n- Learn about [Nested Subtotals](/advanced/nested-subtotals) for hierarchical aggregations\n- Explore [Bucketing](/advanced/bucketing) to group continuous values\n", "queries": { "setup_data": { - "code": "import ibis\nfrom ibis import _\nfrom boring_semantic_layer import to_semantic_table\n\n# Create synthetic flights data with carrier information\nflights_data = ibis.memtable({\n \"flight_id\": list(range(1, 51)),\n \"carrier\": [\"AA\", \"UA\", \"DL\", \"WN\", \"B6\"] * 10,\n \"nickname\": [\"American Airlines\", \"United Airlines\", \"Delta Air Lines\",\n \"Southwest Airlines\", \"JetBlue Airways\"] * 10,\n \"origin\": [\"JFK\", \"LAX\", \"ORD\", \"ATL\", \"DFW\"] * 10,\n \"distance\": [2475, 1745, 733, 946, 1383, 2475, 1745, 733, 946, 1383,\n 2475, 1745, 733, 946, 1383, 2475, 1745, 733, 946, 1383,\n 2475, 1745, 733, 946, 1383, 2475, 1745, 733, 946, 1383,\n 2475, 1745, 733, 946, 1383, 2475, 1745, 733, 946, 1383,\n 2475, 1745, 733, 946, 1383, 2475, 1745, 733, 946, 1383]\n})\n\n# Create semantic table with measures including percentage calculations\nflights = (\n to_semantic_table(flights_data, name=\"flights\")\n .with_measures(\n flight_count=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n )\n .with_measures(\n market_share=lambda t: t.flight_count / t.all(t.flight_count) * 100,\n distance_share=lambda t: t.total_distance / t.all(t.total_distance) * 100,\n )\n)", - "sql": "SELECT\n *\nFROM \"ibis_pandas_memtable_jhatbbyjgngh5ou53h2apqqnaa\"", - "plan": "SemanticTable: flights\n flight_count [measure]\n total_distance [measure]\n market_share [calc]\n distance_share [calc]", - "table": { - "columns": [ - "flight_id", - "carrier", - "nickname", - "origin", - "distance" - ], - "data": [ - [ - 1, - "AA", - "American Airlines", - "JFK", - 2475 - ], - [ - 2, - "UA", - "United Airlines", - "LAX", - 1745 - ], - [ - 3, - "DL", - "Delta Air Lines", - "ORD", - 733 - ], - [ - 4, - "WN", - "Southwest Airlines", - "ATL", - 946 - ], - [ - 5, - "B6", - "JetBlue Airways", - "DFW", - 1383 - ], - [ - 6, - "AA", - "American Airlines", - "JFK", - 2475 - ], - [ - 7, - "UA", - "United Airlines", - "LAX", - 1745 - ], - [ - 8, - "DL", - "Delta Air Lines", - "ORD", - 733 - ], - [ - 9, - "WN", - "Southwest Airlines", - "ATL", - 946 - ], - [ - 10, - "B6", - "JetBlue Airways", - "DFW", - 1383 - ], - [ - 11, - "AA", - "American Airlines", - "JFK", - 2475 - ], - [ - 12, - "UA", - "United Airlines", - "LAX", - 1745 - ], - [ - 13, - "DL", - "Delta Air Lines", - "ORD", - 733 - ], - [ - 14, - "WN", - "Southwest Airlines", - "ATL", - 946 - ], - [ - 15, - "B6", - "JetBlue Airways", - "DFW", - 1383 - ], - [ - 16, - "AA", - "American Airlines", - "JFK", - 2475 - ], - [ - 17, - "UA", - "United Airlines", - "LAX", - 1745 - ], - [ - 18, - "DL", - "Delta Air Lines", - "ORD", - 733 - ], - [ - 19, - "WN", - "Southwest Airlines", - "ATL", - 946 - ], - [ - 20, - "B6", - "JetBlue Airways", - "DFW", - 1383 - ], - [ - 21, - "AA", - "American Airlines", - "JFK", - 2475 - ], - [ - 22, - "UA", - "United Airlines", - "LAX", - 1745 - ], - [ - 23, - "DL", - "Delta Air Lines", - "ORD", - 733 - ], - [ - 24, - "WN", - "Southwest Airlines", - "ATL", - 946 - ], - [ - 25, - "B6", - "JetBlue Airways", - "DFW", - 1383 - ], - [ - 26, - "AA", - "American Airlines", - "JFK", - 2475 - ], - [ - 27, - "UA", - "United Airlines", - "LAX", - 1745 - ], - [ - 28, - "DL", - "Delta Air Lines", - "ORD", - 733 - ], - [ - 29, - "WN", - "Southwest Airlines", - "ATL", - 946 - ], - [ - 30, - "B6", - "JetBlue Airways", - "DFW", - 1383 - ], - [ - 31, - "AA", - "American Airlines", - "JFK", - 2475 - ], - [ - 32, - "UA", - "United Airlines", - "LAX", - 1745 - ], - [ - 33, - "DL", - "Delta Air Lines", - "ORD", - 733 - ], - [ - 34, - "WN", - "Southwest Airlines", - "ATL", - 946 - ], - [ - 35, - "B6", - "JetBlue Airways", - "DFW", - 1383 - ], - [ - 36, - "AA", - "American Airlines", - "JFK", - 2475 - ], - [ - 37, - "UA", - "United Airlines", - "LAX", - 1745 - ], - [ - 38, - "DL", - "Delta Air Lines", - "ORD", - 733 - ], - [ - 39, - "WN", - "Southwest Airlines", - "ATL", - 946 - ], - [ - 40, - "B6", - "JetBlue Airways", - "DFW", - 1383 - ], - [ - 41, - "AA", - "American Airlines", - "JFK", - 2475 - ], - [ - 42, - "UA", - "United Airlines", - "LAX", - 1745 - ], - [ - 43, - "DL", - "Delta Air Lines", - "ORD", - 733 - ], - [ - 44, - "WN", - "Southwest Airlines", - "ATL", - 946 - ], - [ - 45, - "B6", - "JetBlue Airways", - "DFW", - 1383 - ], - [ - 46, - "AA", - "American Airlines", - "JFK", - 2475 - ], - [ - 47, - "UA", - "United Airlines", - "LAX", - 1745 - ], - [ - 48, - "DL", - "Delta Air Lines", - "ORD", - 733 - ], - [ - 49, - "WN", - "Southwest Airlines", - "ATL", - 946 - ], - [ - 50, - "B6", - "JetBlue Airways", - "DFW", - 1383 - ] - ] - } + "semantic_table": true, + "name": "flights", + "info": "Semantic table definition stored in context" }, "query_market_share": { "code": "from ibis import _\n\nresult = (\n flights.group_by(\"nickname\")\n .aggregate(\"flight_count\", \"market_share\")\n .order_by(_.market_share.desc())\n .limit(10)\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t5\".\"nickname\",\n \"t5\".\"flight_count\",\n (\n CAST(\"t5\".\"flight_count\" AS DOUBLE PRECISION) / CAST(\"t5\".\"flight_count_right\" AS DOUBLE PRECISION)\n ) * 100 AS \"market_share\"\n FROM (\n SELECT\n \"t3\".\"nickname\",\n \"t3\".\"flight_count\",\n \"t4\".\"flight_count\" AS \"flight_count_right\"\n FROM (\n SELECT\n \"t0\".\"nickname\",\n COUNT(*) AS \"flight_count\"\n FROM (\n SELECT\n \"t0\".\"flight_id\",\n \"t0\".\"carrier\",\n \"t0\".\"origin\",\n \"t0\".\"distance\",\n \"t0\".\"nickname\"\n FROM \"ibis_pandas_memtable_jhatbbyjgngh5ou53h2apqqnaa\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"nickname\"\n ) AS \"t3\"\n CROSS JOIN (\n SELECT\n COUNT(*) AS \"flight_count\"\n FROM \"ibis_pandas_memtable_jhatbbyjgngh5ou53h2apqqnaa\" AS \"t0\"\n ) AS \"t4\"\n ) AS \"t5\"\n) AS \"t6\"\nORDER BY\n \"t6\".\"market_share\" DESC NULLS LAST\nLIMIT 10", + "sql": "SELECT\n \"t2\".\"nickname\",\n \"t2\".\"flight_count\",\n (\n CAST(\"t2\".\"flight_count\" AS DOUBLE PRECISION) / \"t2\".\"__bsl_totals__flight_count\"\n ) * 100 AS \"market_share\"\nFROM (\n SELECT\n \"t1\".\"nickname\",\n COUNT(*) AS \"flight_count\",\n FIRST_VALUE(\"t1\".\"__bsl_totals__flight_count\") FILTER(WHERE\n NOT \"t1\".\"__bsl_totals__flight_count\" IS NULL) AS \"__bsl_totals__flight_count\"\n FROM (\n SELECT\n \"t1\".\"flight_id\",\n \"t1\".\"carrier\",\n \"t1\".\"origin\",\n \"t1\".\"distance\",\n \"t1\".\"__bsl_totals__flight_count\",\n \"t1\".\"nickname\"\n FROM (\n SELECT\n \"t0\".\"flight_id\",\n \"t0\".\"carrier\",\n \"t0\".\"nickname\",\n \"t0\".\"origin\",\n \"t0\".\"distance\",\n CAST(COUNT(*) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE PRECISION) AS \"__bsl_totals__flight_count\"\n FROM \"ibis_pandas_memtable_pj2vpzby6nb65mpdtxh4ssgece\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"nickname\"\n) AS \"t2\"\nORDER BY\n (\n CAST(\"t2\".\"flight_count\" AS DOUBLE PRECISION) / \"t2\".\"__bsl_totals__flight_count\"\n ) * 100 DESC NULLS LAST\nLIMIT 10", "plan": "SemanticTable: flights\n flight_count [measure]\n total_distance [measure]\n market_share [calc]\n distance_share [calc]\n-> GroupBy(nickname)\n-> Aggregate(flight_count, market_share)\n-> OrderBy(_CallableWrapper(_fn=_.market_share.desc()))\n-> Limit(10)", "table": { "columns": [ @@ -384,12 +23,12 @@ 20.0 ], [ - "Delta Air Lines", + "American Airlines", 10, 20.0 ], [ - "JetBlue Airways", + "Delta Air Lines", 10, 20.0 ], @@ -399,16 +38,109 @@ 20.0 ], [ - "American Airlines", + "JetBlue Airways", 10, 20.0 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-595d70bb1f5af496ada304778efbfd7f" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "nickname", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "nickname", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "market_share" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-595d70bb1f5af496ada304778efbfd7f": [ + { + "nickname": "Southwest Airlines", + "flight_count": 10, + "market_share": 20.0 + }, + { + "nickname": "American Airlines", + "flight_count": 10, + "market_share": 20.0 + }, + { + "nickname": "Delta Air Lines", + "flight_count": 10, + "market_share": 20.0 + }, + { + "nickname": "United Airlines", + "flight_count": 10, + "market_share": 20.0 + }, + { + "nickname": "JetBlue Airways", + "flight_count": 10, + "market_share": 20.0 + } + ] + } + } } }, "query_market_share_by_origin": { "code": "from ibis import _\n\nresult = (\n flights.group_by(\"origin\", \"nickname\")\n .aggregate(\"flight_count\", \"market_share\")\n .order_by(_.market_share.desc())\n .limit(15)\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t5\".\"origin\",\n \"t5\".\"nickname\",\n \"t5\".\"flight_count\",\n (\n CAST(\"t5\".\"flight_count\" AS DOUBLE PRECISION) / CAST(\"t5\".\"flight_count_right\" AS DOUBLE PRECISION)\n ) * 100 AS \"market_share\"\n FROM (\n SELECT\n \"t3\".\"origin\",\n \"t3\".\"nickname\",\n \"t3\".\"flight_count\",\n \"t4\".\"flight_count\" AS \"flight_count_right\"\n FROM (\n SELECT\n \"t0\".\"origin\",\n \"t0\".\"nickname\",\n COUNT(*) AS \"flight_count\"\n FROM (\n SELECT\n \"t0\".\"flight_id\",\n \"t0\".\"carrier\",\n \"t0\".\"distance\",\n \"t0\".\"origin\",\n \"t0\".\"nickname\"\n FROM \"ibis_pandas_memtable_jhatbbyjgngh5ou53h2apqqnaa\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"origin\",\n \"t0\".\"nickname\"\n ) AS \"t3\"\n CROSS JOIN (\n SELECT\n COUNT(*) AS \"flight_count\"\n FROM \"ibis_pandas_memtable_jhatbbyjgngh5ou53h2apqqnaa\" AS \"t0\"\n ) AS \"t4\"\n ) AS \"t5\"\n) AS \"t6\"\nORDER BY\n \"t6\".\"market_share\" DESC NULLS LAST\nLIMIT 15", + "sql": "SELECT\n \"t2\".\"origin\",\n \"t2\".\"nickname\",\n \"t2\".\"flight_count\",\n (\n CAST(\"t2\".\"flight_count\" AS DOUBLE PRECISION) / \"t2\".\"__bsl_totals__flight_count\"\n ) * 100 AS \"market_share\"\nFROM (\n SELECT\n \"t1\".\"origin\",\n \"t1\".\"nickname\",\n COUNT(*) AS \"flight_count\",\n FIRST_VALUE(\"t1\".\"__bsl_totals__flight_count\") FILTER(WHERE\n NOT \"t1\".\"__bsl_totals__flight_count\" IS NULL) AS \"__bsl_totals__flight_count\"\n FROM (\n SELECT\n \"t1\".\"flight_id\",\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"__bsl_totals__flight_count\",\n \"t1\".\"origin\",\n \"t1\".\"nickname\"\n FROM (\n SELECT\n \"t0\".\"flight_id\",\n \"t0\".\"carrier\",\n \"t0\".\"nickname\",\n \"t0\".\"origin\",\n \"t0\".\"distance\",\n CAST(COUNT(*) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE PRECISION) AS \"__bsl_totals__flight_count\"\n FROM \"ibis_pandas_memtable_pj2vpzby6nb65mpdtxh4ssgece\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\",\n \"t1\".\"nickname\"\n) AS \"t2\"\nORDER BY\n (\n CAST(\"t2\".\"flight_count\" AS DOUBLE PRECISION) / \"t2\".\"__bsl_totals__flight_count\"\n ) * 100 DESC NULLS LAST\nLIMIT 15", "plan": "SemanticTable: flights\n flight_count [measure]\n total_distance [measure]\n market_share [calc]\n distance_share [calc]\n-> GroupBy(origin, nickname)\n-> Aggregate(flight_count, market_share)\n-> OrderBy(_CallableWrapper(_fn=_.market_share.desc()))\n-> Limit(15)", "table": { "columns": [ @@ -419,36 +151,95 @@ ], "data": [ [ - "DFW", - "JetBlue Airways", + "ATL", + "Southwest Airlines", 10, 20.0 ], [ - "JFK", - "American Airlines", + "LAX", + "United Airlines", 10, 20.0 ], [ - "ATL", - "Southwest Airlines", + "ORD", + "Delta Air Lines", 10, 20.0 ], [ - "LAX", - "United Airlines", + "JFK", + "American Airlines", 10, 20.0 ], [ - "ORD", - "Delta Air Lines", + "DFW", + "JetBlue Airways", 10, 20.0 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-f32ac0ea05b18a550cab71a81306f74f" + }, + "mark": { + "type": "text" + }, + "encoding": { + "text": { + "value": "Complex query - consider custom visualization" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-f32ac0ea05b18a550cab71a81306f74f": [ + { + "origin": "ATL", + "nickname": "Southwest Airlines", + "flight_count": 10, + "market_share": 20.0 + }, + { + "origin": "LAX", + "nickname": "United Airlines", + "flight_count": 10, + "market_share": 20.0 + }, + { + "origin": "ORD", + "nickname": "Delta Air Lines", + "flight_count": 10, + "market_share": 20.0 + }, + { + "origin": "JFK", + "nickname": "American Airlines", + "flight_count": 10, + "market_share": 20.0 + }, + { + "origin": "DFW", + "nickname": "JetBlue Airways", + "flight_count": 10, + "market_share": 20.0 + } + ] + } + } } } }, diff --git a/docs/web/public/bsl-data/query-agent-mcp.json b/docs/web/public/bsl-data/query-agent-mcp.json index b7680f26..1981e664 100644 --- a/docs/web/public/bsl-data/query-agent-mcp.json +++ b/docs/web/public/bsl-data/query-agent-mcp.json @@ -1,5 +1,5 @@ { - "markdown": "# Query Agent: MCP Server\n\nBSL includes built-in support for the [Model Context Protocol (MCP)](https://github.com/modelcontextprotocol/python-sdk), allowing you to expose your semantic models to Large Language Models like Claude.\n\n\n**Pro tip:** Use [descriptions in dimensions and measures](/building/semantic-tables#with_dimensions) to make your models more AI-friendly. Descriptions help provide context to LLMs, enabling them to understand what each field represents and when to use them.\n\n\n## Installation\n\nTo use MCP functionality, install BSL with the `fastmcp` extra:\n\n```bash\npip install 'boring-semantic-layer[fastmcp]'\n```\n\n## Setting up an MCP Server\n\nCreate an MCP server script that exposes your semantic models:\n\n```python\nimport ibis\nfrom boring_semantic_layer import to_semantic_table, MCPSemanticModel\n\n# Create synthetic flights data\nflights_data = ibis.memtable({\n \"flight_id\": list(range(1, 101)),\n \"origin\": [\"JFK\", \"LAX\", \"ORD\", \"ATL\", \"DFW\"] * 20,\n \"dest\": [\"LAX\", \"JFK\", \"DFW\", \"ORD\", \"ATL\"] * 20,\n \"carrier\": [\"AA\", \"UA\", \"DL\", \"WN\", \"B6\"] * 20,\n \"distance\": [2475, 2475, 801, 606, 732] * 20,\n})\n\n# Define your semantic table with descriptions\nflights = (\n to_semantic_table(flights_data, name=\"flights\")\n .with_dimensions(\n origin={\n \"expr\": lambda t: t.origin,\n \"description\": \"Origin airport code where the flight departed from\"\n },\n destination={\n \"expr\": lambda t: t.dest,\n \"description\": \"Destination airport code where the flight arrived\"\n },\n carrier={\n \"expr\": lambda t: t.carrier,\n \"description\": \"Airline carrier code (e.g., AA, UA, DL)\"\n },\n )\n .with_measures(\n total_flights={\n \"expr\": lambda t: t.count(),\n \"description\": \"Total number of flights\"\n },\n avg_distance={\n \"expr\": lambda t: t.distance.mean(),\n \"description\": \"Average flight distance in miles\"\n },\n )\n)\n\n# Create the MCP server\nmcp_server = MCPSemanticModel(\n models={\"flights\": flights},\n name=\"Flight Data Server\"\n)\n\nif __name__ == \"__main__\":\n mcp_server.run(transport=\"stdio\")\n```\n\nSave this as `example_mcp.py` in your project directory.\n\n## Configuring Claude Desktop\n\nTo use your MCP server with Claude Desktop, add it to your configuration file.\n\n**Configuration file location:**\n- **macOS:** `~/Library/Application Support/Claude/claude_desktop_config.json`\n- **Windows:** `%APPDATA%\\Claude\\claude_desktop_config.json`\n\n**Example configuration:**\n\n```json\n{\n \"mcpServers\": {\n \"flight_sm\": {\n \"command\": \"uv\",\n \"args\": [\n \"--directory\",\n \"/path/to/your/project/\",\n \"run\",\n \"example_mcp.py\"\n ]\n }\n }\n}\n```\n\nReplace `/path/to/your/project/` with the actual path to your project directory.\n\n\nThis example uses [uv](https://docs.astral.sh/uv/) to run the MCP server. You can also use `python` directly if you have BSL installed in your environment:\n\n```json\n{\n \"mcpServers\": {\n \"flight_sm\": {\n \"command\": \"python\",\n \"args\": [\"/path/to/your/project/example_mcp.py\"]\n }\n }\n}\n```\n\n\nAfter updating the configuration:\n1. Restart Claude Desktop\n2. Look for the MCP server indicator in the Claude Desktop interface\n3. You should see \"flight_sm\" listed as an available server\n\n## Available MCP Tools\n\nOnce configured, Claude will have access to these tools for interacting with your semantic models:\n\n### list_models\n\nList all available semantic model names in the MCP server.\n\n**Example usage in Claude:**\n> \"What models are available?\"\n\n**Returns:** Array of model names (e.g., `[\"flights\", \"carriers\"]`)\n\n### get_model\n\nGet detailed information about a specific model including its dimensions, measures, and descriptions.\n\n**Parameters:**\n- `model_name` (str): Name of the model to inspect\n\n**Example usage in Claude:**\n> \"Show me the details of the flights model\"\n\n**Returns:** Model schema including:\n- Model name and description\n- List of dimensions with their descriptions\n- List of measures with their descriptions\n- Available joins (if any)\n\n### get_time_range\n\nGet the available time range for time-series data in a model.\n\n**Parameters:**\n- `model_name` (str): Name of the model\n- `time_dimension` (str): Name of the time dimension\n\n**Example usage in Claude:**\n> \"What's the time range available in the flights model?\"\n\n**Returns:** Dictionary with `min_time` and `max_time` values\n\n### query_model\n\nExecute queries against a semantic model with dimensions, measures, filters, and optional chart specifications.\n\n**Parameters:**\n- `model_name` (str): Name of the model to query\n- `dimensions` (list[str]): List of dimension names to group by\n- `measures` (list[str]): List of measure names to aggregate\n- `filters` (list[str], optional): List of filter expressions (e.g., `[\"origin == 'JFK'\"]`)\n- `limit` (int, optional): Maximum number of rows to return\n- `order_by` (list[str], optional): List of columns to sort by\n- `chart_spec` (dict, optional): Vega-Lite chart specification\n\n**Example usage in Claude:**\n> \"Show me the top 10 origins by flight count\"\n> \"Create a bar chart of average distance by carrier\"\n\n**Returns:**\n- When `chart_spec` is provided: `{\"records\": [...], \"chart\": {...}}`\n- When `chart_spec` is not provided: `{\"records\": [...]}`\n\n### Example Interactions\n\nHere are some example questions you can ask Claude when the MCP server is configured:\n\n**Data Exploration:**\n- \"What models are available in the flight data server?\"\n- \"Show me all dimensions and measures in the flights model\"\n- \"What is the time range covered by the flights data?\"\n\n**Basic Queries:**\n- \"How many flights departed from JFK?\"\n- \"Show me the top 5 destinations by flight count\"\n- \"What's the average flight distance for each carrier?\"\n\n**Filtered Queries:**\n- \"Show me flights from California airports (starting with 'S')\"\n- \"What carriers have an average distance over 1000 miles?\"\n- \"List the top 10 busiest routes\"\n\n**Visualizations:**\n- \"Create a bar chart showing flights by origin airport\"\n- \"Make a line chart of flights over time\"\n- \"Show me a heatmap of routes between origins and destinations\"\n\n## Best Practices\n\n### 1. Add Descriptions to All Fields\n\nDescriptions are crucial for LLMs to understand your data model:\n\n```python\nflights = (\n to_semantic_table(flights_tbl, name=\"flights\")\n .with_dimensions(\n origin={\n \"expr\": lambda t: t.origin,\n \"description\": \"Origin airport code (3-letter IATA code)\"\n }\n )\n .with_measures(\n total_flights={\n \"expr\": lambda t: t.count(),\n \"description\": \"Total number of flights in the dataset\"\n }\n )\n)\n```\n\n### 2. Use Descriptive Model Names\n\nChoose clear, descriptive names for your models:\n\n```python\n# Good\nmcp_server = MCPSemanticModel(\n models={\"flights\": flights, \"carriers\": carriers},\n name=\"Aviation Analytics Server\"\n)\n\n# Less clear\nmcp_server = MCPSemanticModel(\n models={\"f\": flights, \"c\": carriers},\n name=\"Server\"\n)\n```\n\n### 3. Define Time Dimensions for Time-Series Queries\n\nWhen exposing models through MCP, you need to explicitly define time dimensions to enable LLMs to query time ranges and perform time-based aggregations. This is specific to MCP\u2014when using BSL's fluent API directly, you can simply use Ibis functions like `.year()` and `.month()`.\n\nTo define a time dimension, set `is_time_dimension=True` and specify the `smallest_time_grain`:\n\n```python\nfrom boring_semantic_layer import to_semantic_table\n\nflights = (\n to_semantic_table(flights_data, name=\"flights\")\n .with_dimensions(\n arr_time={\n \"expr\": lambda t: t.arr_time,\n \"description\": \"Arrival time of the flight\",\n \"is_time_dimension\": True,\n \"smallest_time_grain\": \"TIME_GRAIN_SECOND\",\n },\n origin={\n \"expr\": lambda t: t.origin,\n \"description\": \"Origin airport code\"\n },\n )\n .with_measures(\n flight_count={\n \"expr\": lambda t: t.count(),\n \"description\": \"Total number of flights\"\n }\n )\n)\n```\n\n**Available time grains:**\n- `TIME_GRAIN_SECOND` - For second-level precision\n- `TIME_GRAIN_MINUTE` - For minute-level precision\n- `TIME_GRAIN_HOUR` - For hourly data\n- `TIME_GRAIN_DAY` - For daily data\n- `TIME_GRAIN_WEEK` - For weekly data\n- `TIME_GRAIN_MONTH` - For monthly data\n- `TIME_GRAIN_QUARTER` - For quarterly data\n- `TIME_GRAIN_YEAR` - For yearly data\n\n\nIf you define multiple time dimensions in your model, the `.query()` method and MCP tools will use the first time dimension that appears in your query's dimensions list.\n\n\n**Example time-based queries:**\n\nWith time dimensions defined, you can use the `.query()` method with time ranges and grains:\n\n```python\n# Query with a specific time range\nresult = flights.query(\n dimensions=[\"origin\"],\n measures=[\"flight_count\"],\n time_range={\"start\": \"2024-01-01\", \"end\": \"2024-12-31\"}\n)\n\n# Query with time grain aggregation\nresult = flights.query(\n dimensions=[\"arr_time\"],\n measures=[\"flight_count\"],\n time_grain=\"TIME_GRAIN_MONTH\"\n)\n```\n\nLLMs can then perform similar queries through MCP:\n```\n> \"What's the time range available in the flights data?\"\n> \"Show me flights from January 2024\"\n> \"Give me monthly flight counts for the last year\"\n```\n\n### 4. Structure Your Data Logically\n\nOrganize related dimensions and measures together, and use joins to connect related models:\n\n```python\n# Flights model focuses on flight operations\nflights = (\n to_semantic_table(flights_tbl, name=\"flights\")\n .with_dimensions(origin=..., destination=..., date=...)\n .with_measures(flight_count=..., avg_delay=...)\n)\n\n# Carriers model focuses on airline information\ncarriers = (\n to_semantic_table(carriers_tbl, name=\"carriers\")\n .with_dimensions(code=..., name=..., country=...)\n .with_measures(carrier_count=...)\n)\n\n# Connect them with joins\nflights_with_carriers = flights.join_one(\n carriers,\n lambda f, c: f.carrier == c.code\n)\n```\n\n## Troubleshooting\n\n### Server Not Appearing in Claude Desktop\n\n1. Check the configuration file path is correct\n2. Verify JSON syntax in `claude_desktop_config.json`\n3. Ensure BSL is installed with MCP support: `pip install 'boring-semantic-layer[fastmcp]'`\n4. Restart Claude Desktop completely\n5. Check Claude Desktop logs for error messages\n\n### Import Errors\n\nIf you see import errors when the server starts:\n\n```bash\n# Ensure all dependencies are installed\npip install 'boring-semantic-layer[fastmcp]'\n\n# Or install specific dependencies\npip install fastmcp ibis-framework\n```\n\n### Path Issues\n\nMake sure file paths in your configuration are absolute paths, not relative:\n\n```json\n{\n \"mcpServers\": {\n \"flight_sm\": {\n \"command\": \"python\",\n \"args\": [\"/Users/username/projects/my-project/example_mcp.py\"]\n }\n }\n}\n```\n", + "markdown": "# Query Agent: MCP Server\n\nBSL includes built-in support for the [Model Context Protocol (MCP)](https://github.com/modelcontextprotocol/python-sdk), allowing you to expose your semantic models to Large Language Models like Claude.\n\n\n**Pro tip:** Use [descriptions in dimensions and measures](/building/semantic-tables#with_dimensions) to make your models more AI-friendly. Descriptions help provide context to LLMs, enabling them to understand what each field represents and when to use them.\n\n\n## Installation\n\nTo use MCP functionality, install BSL with the `fastmcp` extra:\n\n```bash\npip install 'boring-semantic-layer[fastmcp]'\n```\n\n## Setting up an MCP Server\n\nCreate an MCP server script that exposes your semantic models:\n\n```python\nimport ibis\nfrom boring_semantic_layer import to_semantic_table, MCPSemanticModel\n\n# Create synthetic flights data\nflights_data = ibis.memtable({\n \"flight_id\": list(range(1, 101)),\n \"origin\": [\"JFK\", \"LAX\", \"ORD\", \"ATL\", \"DFW\"] * 20,\n \"dest\": [\"LAX\", \"JFK\", \"DFW\", \"ORD\", \"ATL\"] * 20,\n \"carrier\": [\"AA\", \"UA\", \"DL\", \"WN\", \"B6\"] * 20,\n \"distance\": [2475, 2475, 801, 606, 732] * 20,\n})\n\n# Define your semantic table with descriptions\nflights = (\n to_semantic_table(flights_data, name=\"flights\")\n .with_dimensions(\n origin={\n \"expr\": lambda t: t.origin,\n \"description\": \"Origin airport code where the flight departed from\"\n },\n destination={\n \"expr\": lambda t: t.dest,\n \"description\": \"Destination airport code where the flight arrived\"\n },\n carrier={\n \"expr\": lambda t: t.carrier,\n \"description\": \"Airline carrier code (e.g., AA, UA, DL)\"\n },\n )\n .with_measures(\n total_flights={\n \"expr\": lambda t: t.count(),\n \"description\": \"Total number of flights\"\n },\n avg_distance={\n \"expr\": lambda t: t.distance.mean(),\n \"description\": \"Average flight distance in miles\"\n },\n )\n)\n\n# Create the MCP server\nmcp_server = MCPSemanticModel(\n models={\"flights\": flights},\n name=\"Flight Data Server\"\n)\n\nif __name__ == \"__main__\":\n mcp_server.run(transport=\"stdio\")\n```\n\nSave this as `example_mcp.py` in your project directory.\n\n## Configuring Claude Desktop\n\nTo use your MCP server with Claude Desktop, add it to your configuration file.\n\n**Configuration file location:**\n- **macOS:** `~/Library/Application Support/Claude/claude_desktop_config.json`\n- **Windows:** `%APPDATA%\\Claude\\claude_desktop_config.json`\n\n**Example configuration:**\n\n```json\n{\n \"mcpServers\": {\n \"flight_sm\": {\n \"command\": \"uv\",\n \"args\": [\n \"--directory\",\n \"/path/to/your/project/\",\n \"run\",\n \"example_mcp.py\"\n ]\n }\n }\n}\n```\n\nReplace `/path/to/your/project/` with the actual path to your project directory.\n\n\nThis example uses [uv](https://docs.astral.sh/uv/) to run the MCP server. You can also use `python` directly if you have BSL installed in your environment:\n\n```json\n{\n \"mcpServers\": {\n \"flight_sm\": {\n \"command\": \"python\",\n \"args\": [\"/path/to/your/project/example_mcp.py\"]\n }\n }\n}\n```\n\n\nAfter updating the configuration:\n1. Restart Claude Desktop\n2. Look for the MCP server indicator in the Claude Desktop interface\n3. You should see \"flight_sm\" listed as an available server\n\n## Available MCP Tools\n\nOnce configured, Claude will have access to these tools for interacting with your semantic models:\n\n### list_models\n\nList all available semantic model names in the MCP server.\n\n**Example usage in Claude:**\n> \"What models are available?\"\n\n**Returns:** Array of model names (e.g., `[\"flights\", \"carriers\"]`)\n\n### get_model\n\nGet detailed information about a specific model including its dimensions, measures, and descriptions.\n\n**Parameters:**\n- `model_name` (str): Name of the model to inspect\n\n**Example usage in Claude:**\n> \"Show me the details of the flights model\"\n\n**Returns:** Model schema including:\n- Model name and description\n- List of dimensions with their descriptions\n- List of measures with their descriptions\n- Available joins (if any)\n\n### get_time_range\n\nGet the available time range for time-series data in a model.\n\n**Parameters:**\n- `model_name` (str): Name of the model\n- `time_dimension` (str): Name of the time dimension\n\n**Example usage in Claude:**\n> \"What's the time range available in the flights model?\"\n\n**Returns:** Dictionary with `min_time` and `max_time` values\n\n### query_model\n\nExecute queries against a semantic model with dimensions, measures, filters, and optional chart specifications.\n\n**Parameters:**\n- `model_name` (str): Name of the model to query\n- `dimensions` (list[str]): List of dimension names to group by\n- `measures` (list[str]): List of measure names to aggregate\n- `filters` (list[str], optional): List of filter expressions (e.g., `[\"origin == 'JFK'\"]`)\n- `limit` (int, optional): Maximum number of rows to return\n- `order_by` (list[str], optional): List of columns to sort by\n- `chart_spec` (dict, optional): Vega-Lite chart specification\n\n**Example usage in Claude:**\n> \"Show me the top 10 origins by flight count\"\n> \"Create a bar chart of average distance by carrier\"\n\n**Returns:**\n- When `chart_spec` is provided: `{\"records\": [...], \"chart\": {...}}`\n- When `chart_spec` is not provided: `{\"records\": [...]}`\n\n### compare_periods\n\nCompare two explicit time ranges and return `{measure}_current`, `{measure}_previous`, `{measure}_delta`, and `{measure}_pct_change` columns in a single response. This is the recommended MCP pattern for period-over-period chat questions.\n\n**Parameters:**\n- `model_name` (str): Name of the model to query\n- `measures` (list[str]): Measures to compare\n- `current_time_range` (dict): Current period with `start` and `end`\n- `previous_time_range` (dict): Comparison period with `start` and `end`\n- `dimensions` (list[str], optional): Optional grouping dimensions like `carrier` or `store`\n- `filters` (list[dict], optional): Optional filters applied to both periods\n- `time_dimension` (str, optional): Explicit time dimension when a model has more than one\n\n**Example usage in Claude:**\n> \"Compare the last 10 days to the prior 10 days by carrier\"\n> \"Show revenue this month vs last month by store\"\n\n### Example Interactions\n\nHere are some example questions you can ask Claude when the MCP server is configured:\n\n**Data Exploration:**\n- \"What models are available in the flight data server?\"\n- \"Show me all dimensions and measures in the flights model\"\n- \"What is the time range covered by the flights data?\"\n\n**Basic Queries:**\n- \"How many flights departed from JFK?\"\n- \"Show me the top 5 destinations by flight count\"\n- \"What's the average flight distance for each carrier?\"\n\n**Filtered Queries:**\n- \"Show me flights from California airports (starting with 'S')\"\n- \"What carriers have an average distance over 1000 miles?\"\n- \"List the top 10 busiest routes\"\n\n**Visualizations:**\n- \"Create a bar chart showing flights by origin airport\"\n- \"Make a line chart of flights over time\"\n- \"Show me a heatmap of routes between origins and destinations\"\n\n## Best Practices\n\n### 1. Add Descriptions to All Fields\n\nDescriptions are crucial for LLMs to understand your data model:\n\n```python\nflights = (\n to_semantic_table(flights_tbl, name=\"flights\")\n .with_dimensions(\n origin={\n \"expr\": lambda t: t.origin,\n \"description\": \"Origin airport code (3-letter IATA code)\"\n }\n )\n .with_measures(\n total_flights={\n \"expr\": lambda t: t.count(),\n \"description\": \"Total number of flights in the dataset\"\n }\n )\n)\n```\n\n### 2. Use Descriptive Model Names\n\nChoose clear, descriptive names for your models:\n\n```python\n# Good\nmcp_server = MCPSemanticModel(\n models={\"flights\": flights, \"carriers\": carriers},\n name=\"Aviation Analytics Server\"\n)\n\n# Less clear\nmcp_server = MCPSemanticModel(\n models={\"f\": flights, \"c\": carriers},\n name=\"Server\"\n)\n```\n\n### 3. Define Time Dimensions for Time-Series Queries\n\nWhen exposing models through MCP, you need to explicitly define time dimensions to enable LLMs to query time ranges and perform time-based aggregations. This is specific to MCP\u2014when using BSL's fluent API directly, you can simply use Ibis functions like `.year()` and `.month()`.\n\nTo define a time dimension, set `is_time_dimension=True` and specify the `smallest_time_grain`:\n\n```python\nfrom boring_semantic_layer import to_semantic_table\n\nflights = (\n to_semantic_table(flights_data, name=\"flights\")\n .with_dimensions(\n arr_time={\n \"expr\": lambda t: t.arr_time,\n \"description\": \"Arrival time of the flight\",\n \"is_time_dimension\": True,\n \"smallest_time_grain\": \"TIME_GRAIN_SECOND\",\n },\n origin={\n \"expr\": lambda t: t.origin,\n \"description\": \"Origin airport code\"\n },\n )\n .with_measures(\n flight_count={\n \"expr\": lambda t: t.count(),\n \"description\": \"Total number of flights\"\n }\n )\n)\n```\n\n**Available time grains** (short form preferred, long form also accepted):\n- `second` / `TIME_GRAIN_SECOND` - For second-level precision\n- `minute` / `TIME_GRAIN_MINUTE` - For minute-level precision\n- `hour` / `TIME_GRAIN_HOUR` - For hourly data\n- `day` / `TIME_GRAIN_DAY` - For daily data\n- `week` / `TIME_GRAIN_WEEK` - For weekly data\n- `month` / `TIME_GRAIN_MONTH` - For monthly data\n- `quarter` / `TIME_GRAIN_QUARTER` - For quarterly data\n- `year` / `TIME_GRAIN_YEAR` - For yearly data\n\n\nUse `time_grain` to apply one grain to all time dimensions, or `time_grains` (dict) to set different grains per dimension.\n\n\n**Example time-based queries:**\n\nWith time dimensions defined, you can use the `.query()` method with time ranges and grains:\n\n```python\n# Query with a specific time range\nresult = flights.query(\n dimensions=[\"origin\"],\n measures=[\"flight_count\"],\n time_range={\"start\": \"2024-01-01\", \"end\": \"2024-12-31\"}\n)\n\n# Query with time grain aggregation (same grain for all time dims)\nresult = flights.query(\n dimensions=[\"arr_time\"],\n measures=[\"flight_count\"],\n time_grain=\"month\"\n)\n\n# Per-dimension grains (different grain per time dimension)\nresult = orders.query(\n dimensions=[\"order_date\", \"ship_date\"],\n measures=[\"total_sales\"],\n time_grains={\"order_date\": \"month\", \"ship_date\": \"quarter\"}\n)\n```\n\nLLMs can then perform similar queries through MCP:\n```\n> \"What's the time range available in the flights data?\"\n> \"Show me flights from January 2024\"\n> \"Give me monthly flight counts for the last year\"\n```\n\n### 4. Structure Your Data Logically\n\nOrganize related dimensions and measures together, and use joins to connect related models:\n\n```python\n# Flights model focuses on flight operations\nflights = (\n to_semantic_table(flights_tbl, name=\"flights\")\n .with_dimensions(origin=..., destination=..., date=...)\n .with_measures(flight_count=..., avg_delay=...)\n)\n\n# Carriers model focuses on airline information\ncarriers = (\n to_semantic_table(carriers_tbl, name=\"carriers\")\n .with_dimensions(code=..., name=..., country=...)\n .with_measures(carrier_count=...)\n)\n\n# Connect them with joins\nflights_with_carriers = flights.join_one(\n carriers,\n lambda f, c: f.carrier == c.code\n)\n```\n\n## Troubleshooting\n\n### Server Not Appearing in Claude Desktop\n\n1. Check the configuration file path is correct\n2. Verify JSON syntax in `claude_desktop_config.json`\n3. Ensure BSL is installed with MCP support: `pip install 'boring-semantic-layer[fastmcp]'`\n4. Restart Claude Desktop completely\n5. Check Claude Desktop logs for error messages\n\n### Import Errors\n\nIf you see import errors when the server starts:\n\n```bash\n# Ensure all dependencies are installed\npip install 'boring-semantic-layer[fastmcp]'\n\n# Or install specific dependencies\npip install fastmcp ibis-framework\n```\n\n### Path Issues\n\nMake sure file paths in your configuration are absolute paths, not relative:\n\n```json\n{\n \"mcpServers\": {\n \"flight_sm\": {\n \"command\": \"python\",\n \"args\": [\"/Users/username/projects/my-project/example_mcp.py\"]\n }\n }\n}\n```\n", "queries": {}, "files": {} } diff --git a/docs/web/public/bsl-data/query-methods.json b/docs/web/public/bsl-data/query-methods.json index cafefafd..bd4d9cbd 100644 --- a/docs/web/public/bsl-data/query-methods.json +++ b/docs/web/public/bsl-data/query-methods.json @@ -1,78 +1,14 @@ { - "markdown": "# Query Methods\n\n## Overview\n\nBSL provides a simple and consistent query API for retrieving data from your semantic tables. Queries are built by chaining methods, then executed or inspected using output methods.\n\nStart with a semantic table and chain methods together. Here's the typical query flow:\n\n```setup_table\nimport ibis\nfrom ibis import _\nfrom boring_semantic_layer import to_semantic_table\n\n# Create Ibis table\nflights_tbl = ibis.memtable({\n \"origin\": [\"NYC\", \"LAX\", \"NYC\", \"SFO\", \"LAX\", \"NYC\", \"SFO\", \"LAX\", \"NYC\"],\n \"carrier\": [\"AA\", \"UA\", \"AA\", \"UA\", \"AA\", \"UA\", \"AA\", \"UA\", \"AA\"],\n \"distance\": [2789, 2789, 2902, 2902, 347, 2789, 347, 347, 2789],\n \"duration\": [330, 330, 360, 360, 65, 330, 65, 65, 330],\n})\n\n# Create semantic table\nflights_st = (\n to_semantic_table(flights_tbl, name=\"flights\")\n .with_dimensions(\n origin=lambda t: t.origin,\n carrier=lambda t: t.carrier,\n )\n .with_measures(\n flight_count=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n avg_duration=lambda t: t.duration.mean(),\n )\n)\n```\n\n\n\n```python\nresult = (\n flights_st # Start with semantic table\n .filter(_.distance > 1000) # 1. Filter (optional)\n .group_by(\"origin\") # 2. Group by dimensions\n .aggregate(\"flight_count\", \"total_distance\") # 3. Aggregate measures\n .mutate(avg=lambda t: t.total_distance / t.flight_count) # 4. Transform (optional)\n .order_by(ibis.desc(\"flight_count\")) # 5. Sort (optional)\n .limit(10) # 6. Limit rows (optional)\n)\n```\n\nOnce you've built a query, you can inspect it or execute it:\n\n```simple_demo\n\n# Build a query\nresult = flights_st.group_by(\"origin\").aggregate(\"flight_count\")\n\n# Option 1: Execute and get data as pandas DataFrame\ndf = result.execute()\n\n# Option 2: View the generated SQL\nprint(result.sql())\n\n# Option 3: Generate a visualization (when applicable)\nchart = result.chart()\n\n# Option 4: See the semantic query plan\nprint(result)\n\nresult\n```\n\n\nThe output above includes a **Query Plan** tab showing how BSL translates this query into semantic operations. \n\nYou print the query object directly to see the plan:\n```python\nprint(result)\n```\n\nThis displays operations like `SemanticTableOp`, `SemanticGroupByOp`, and `SemanticAggregateOp`, useful for debugging and understanding query execution.\n\nLet's get now into the details of each query method.\n\n## group_by()\n\nThe `group_by()` method groups data by one or more dimensions.\n\n\n`group_by()` only accepts string dimension names that were previously defined in `with_dimensions()`. It does not support lambda functions or unbound `_` syntax.\n\n\n### Single Dimension\n\nGroup by a single dimension:\n\n```query_single_dimension\n# Group by one dimension\nresult = flights_st.group_by(\"origin\").aggregate(\"flight_count\")\n```\n\n\n\n### Multiple Dimensions\n\nGroup by multiple dimensions to create detailed breakdowns:\n\n```query_multiple_dimensions\n# Group by multiple dimensions\nresult = flights_st.group_by(\"origin\", \"carrier\").aggregate(\"flight_count\")\n```\n\n\n\n### No Grouping\n\nCalculate overall statistics across all rows using `group_by()` with no arguments:\n\n```query_no_grouping\n# Aggregate entire dataset without grouping\nresult = flights_st.group_by().aggregate(\"flight_count\", \"total_distance\", \"avg_duration\")\n```\n\n\n\n## aggregate()\n\nThe `aggregate()` method calculates measures after grouping. You can reference pre-defined measures or compute new ones on-the-fly.\n\n\n**CRITICAL**: `aggregate()` takes **measure names as strings**, not expressions or lambdas directly. Use measure names from `get_model()` output.\n```python\n# \u2705 CORRECT - measure names as strings\nmodel.group_by(\"category\").aggregate(\"flight_count\", \"total_revenue\")\n\n# \u274c WRONG - no standalone lambdas in aggregate\nmodel.aggregate(total=lambda t: t.sum()) # ERROR!\n```\n\n\n### Pre-defined Measures\n\nReference measures by their string names:\n\n```query_predefined_measures\n# Use measures defined in with_measures()\nresult = flights_st.group_by(\"origin\").aggregate(\"flight_count\", \"avg_duration\")\n```\n\n\n\n### On-the-Fly Transformations\n\nAdd computed measures directly in `aggregate()` without modifying the semantic table:\n\n```query_onthefly_measures\n# Mix predefined and computed measures\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\n \"flight_count\", # Pre-defined measure\n \"avg_duration\", # Pre-defined measure\n total_miles=lambda t: t.distance.sum(), # Computed on-the-fly\n max_distance=lambda t: t.flight_count + 2 # You can reference other measures as well\n )\n)\n```\n\n\n\n\nOn-the-fly measures let you add context-specific calculations without modifying your semantic table definition. This keeps your base model clean while enabling flexible queries.\n\n\n### Referencing Table Columns\n\nYou can reference **any column from the underlying table** in `aggregate()`, not just pre-defined measures. This is useful when you need one-off calculations without cluttering your semantic table definition.\n\n```query_table_columns\n# Reference table columns directly in aggregate()\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\n \"flight_count\", # Pre-defined measure\n total_distance=lambda t: t.distance.sum(), # Table column 'distance'\n avg_duration=lambda t: t.duration.mean(), # Table column 'duration'\n distance_in_km=lambda t: (t.distance * 1.60934).sum() # Transform then aggregate\n )\n)\n```\n\n\n\n**Key points:**\n- Table columns **must be aggregated** (e.g., `.sum()`, `.mean()`, `.max()`, `.count()`)\n- You can transform columns before aggregating (e.g., `(t.distance * 1.60934).sum()`)\n- This works for any column in the underlying table, even if not defined as a dimension or measure\n- Use this for ad-hoc calculations without modifying your semantic table\n\n\nTable columns cannot be used without an aggregation function. For example, `lambda t: t.distance` will fail. You must use `lambda t: t.distance.sum()` or another aggregation.\n\n\n## filter() / order_by() / limit() \n\nCombine `filter()`, `order_by()`, and `limit()` to refine your query results.\n\n```query_filter_order_limit\nfrom ibis import _\n\n# Filter data, sort, and limit results\nresult = (\n flights_st\n .filter(lambda t: t.origin.isin([\"NYC\", \"LAX\"])) # Filter origins\n .filter(_.distance > 500) # Filter distance using _ syntax\n .group_by(\"origin\")\n .aggregate(\"flight_count\", \"avg_duration\") # Aggregate both measures\n .order_by(ibis.desc(\"flight_count\")) # Sort by flight_count descending\n .limit(5) # Top 5 results\n)\n```\n\n\n\n**Key points:**\n- **`filter()`**: Use lambda or `_` syntax to apply conditions before aggregation\n- **`order_by()`**: Use `ibis.desc()` for descending order, or column name for ascending\n- **`limit()`**: Restrict the number of rows returned\n\n### Critical Filter Patterns\n\n**Multiple conditions** - use `ibis.and_()` or `ibis.or_()`:\n\n```python\n# Multiple conditions with AND\nmodel.filter(lambda t: ibis.and_(t.amount > 1000, t.year >= 2023))\n\n# Multiple conditions with OR\nmodel.filter(lambda t: ibis.or_(t.status == \"active\", t.status == \"pending\"))\n```\n\n**IN operator** - MUST use `.isin()` method:\n\n```python\n# \u2705 CORRECT - use .isin() method\nmodel.filter(lambda t: t.region.isin([\"US\", \"EU\", \"APAC\"]))\n\n# \u274c WRONG - Python's 'in' does NOT work!\nmodel.filter(lambda t: t.region in [\"US\", \"EU\"]) # ERROR: truth value of Ibis expression is not defined\n```\n\n**Lambda column names** - use column names directly, never prefix with model name:\n\n```python\n# \u2705 CORRECT - use column name directly\nmodel.filter(lambda t: t.carrier == \"AA\")\n\n# \u274c WRONG - do NOT prefix with model name\nmodel.filter(lambda t: t.model.carrier == \"AA\") # ERROR!\n```\n\n**Joined columns** - use exact prefixed name from `get_model()`:\n\n```python\n# If get_model() shows \"customers.country\", use it exactly:\nmodel.filter(lambda t: t.customers.country == \"US\")\n\n# \u274c WRONG - don't call methods on ID columns\nmodel.filter(lambda t: t.customer_id.country()) # ERROR: no such method!\n```\n\n## nest()\n\nThe `nest` parameter in `aggregate()` creates nested data structures (arrays of structs) in your query results. This is useful for API responses, hierarchical visualizations, and preserving relationships in aggregated data.\n\nUse `nest` to collect rows as structured arrays within each group:\n\n```query_basic_nest\nfrom ibis import _\n\n# Nest flight details within each origin\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\n \"flight_count\",\n \"total_distance\",\n # Create nested array of flight details\n nest={\"flights\": lambda t: t.group_by([\"carrier\", \"distance\"])}\n )\n)\n```\n\n\n\n**How it works:**\n- The `nest` parameter accepts a dictionary: `{\"column_name\": lambda t: ...}`\n- The lambda specifies which columns to collect using `.group_by()` or `.select()`\n- Results in an array of structs column named `flights`\n\nYou can also use `.select()` to specify which columns to nest:\n\n```query_nest_select\n# Nest specific columns\nresult = (\n flights_st\n .group_by(\"carrier\")\n .aggregate(\n \"flight_count\",\n nest={\"routes\": lambda t: t.select(\"origin\", \"distance\", \"duration\")}\n )\n)\n```\n\n\n\nAfter nesting, you can re-group which automatically unnests, then access the nested fields.\n\n**Step 1: Create nested data**\n\nFirst, create the nested structure. Notice the `flights` column contains arrays of structs:\n\n```query_nest_step1\nfrom ibis import _\n\n# Create nested data structure\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\n \"flight_count\",\n nest={\"flights\": lambda t: t.group_by([\"carrier\", \"distance\"])}\n )\n)\n```\n\n\n\n**Step 2: Re-group to unnest and access fields**\n\nNow re-group on the same dimension, which automatically unnests the array, allowing you to access the nested fields:\n\n```query_nest_step2\nfrom ibis import _\n\n# Re-grouping automatically unnests the 'flights' array\nresult = (\n result\n .group_by(\"origin\")\n .aggregate(\n total_flights=lambda t: t.flight_count.sum(),\n # Access unnested fields from the flights array\n unique_carriers=lambda t: t.flights.carrier.nunique(),\n avg_distance=lambda t: t.flights.distance.mean()\n )\n)\n```\n\n\n\n**Use cases for nesting:**\n- **API responses**: Create JSON-compatible hierarchical structures\n- **Hierarchical data**: Preserve parent-child relationships in results\n- **Data export**: Generate nested documents for external systems\n- **Drill-down analysis**: Keep detailed records available in aggregated views\n\n\nFor more complex nesting patterns and multi-level hierarchies, see [Nested Subtotals](/advanced/nested-subtotals).\n\n\n## mutate()\n\nThe `mutate()` method transforms aggregated results by adding new computed columns. This is different from on-the-fly measures in `aggregate()` \u2014 `mutate()` works on already-aggregated data.\n\n\n**Key difference:** `.aggregate()` computes from raw data, while `.mutate()` transforms already-aggregated results.\n\n\n```query_mutate\nfrom ibis import _\n\n# Add post-aggregation calculations\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\"flight_count\", \"total_distance\")\n .mutate(\n avg_distance_per_flight=lambda t: t.total_distance / t.flight_count,\n flight_category=lambda t: xo.case()\n .when(t.flight_count >= 3, \"high\")\n .when(t.flight_count >= 2, \"medium\")\n .else_(\"low\")\n .end()\n )\n)\n```\n\n\n\n**Use cases for `mutate()`:**\n- Calculate ratios from aggregated measures (e.g., `total / count`)\n- Create categories based on aggregated values\n- Add labels or formatting to results\n- Transform aggregated columns using the full power of Ibis\n\nFor more transformations, see [Ibis Table API reference](https://ibis-project.org/reference/expression-tables.html#ibis.expr.types.relations.Table.mutate).\n\n## Window Functions with .over()\n\nWindow functions perform calculations across ordered rows, enabling operations like running totals, moving averages, and ranking. Unlike regular aggregations that reduce many rows to one, window functions preserve row count while adding computed values.\n\n\n**Important:** Window functions can only be applied **after aggregation**, typically within a `.mutate()` call. They cannot be defined directly in measures.\n\n\n**Common window functions:**\n- **`lag()` / `lead()`**: Access previous/next row values for period-over-period comparisons\n- **`cumsum()`**: Calculate running totals\n- **`.over(window)`**: Apply functions over sliding windows (e.g., moving averages)\n- **`rank()` / `row_number()`**: Assign ranks or sequential numbers to rows\n\nHere's a simple example:\n\n```query_window_example\nfrom ibis import _\n\n# First aggregate to daily level\ndaily_flights = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\"flight_count\", \"total_distance\")\n .order_by(\"origin\")\n)\n\n# Then apply window function for cumulative distance\nwindow_spec = xo.window(order_by=\"origin\")\n\nresult = daily_flights.mutate(\n cumulative_distance=_.total_distance.cumsum(),\n flight_rank=lambda t: xo.rank().over(xo.window(order_by=xo.desc(t.flight_count)))\n).limit(10)\n```\n\n\n\n**Key points:**\n- Window functions are applied **after** `.aggregate()` using `.mutate()`\n- Use `.order_by()` to establish row order for window operations\n- Combine with `xo.window()` for advanced sliding window calculations\n\nFor comprehensive examples including lag/lead, moving averages, and ranking, see [Window Functions](/advanced/windowing).\n\n## as_table()\n\nAfter filtering or aggregating data, you may want to perform additional semantic operations. However, intermediate results don't always preserve the semantic table's dimensions and measures.\n\nThe Problem: Lost Semantic Information\n\nWhen you aggregate data, the result loses semantic metadata. The aggregated result is a `SemanticAggregate` expression, which doesn't have `.dimensions` or `.measures` attributes:\n\n```query_as_table_problem\nfrom ibis import _\n\n# Aggregate the data - this returns a SemanticAggregate\nagg_result = flights_st.group_by(\"origin\").aggregate(\"flight_count\", \"total_distance\")\n\n# Show the type/class of the result\nresult_type = type(agg_result).__name__\n\n# Try to access .dimensions - this will raise an AttributeError\ntry:\n dimensions = agg_result.dimensions\n result = f\"Type: {result_type}\\nDimensions: {dimensions}\"\nexcept AttributeError as e:\n result = f\"Type: {result_type}\\nError: {str(e)}\"\n\nresult\n```\n\n\n\n\nAfter aggregation, you can no longer access the original semantic table's dimensions and measures metadata.\n\nThe Solution: Use as_table()\n\nThe `as_table()` method converts results back into a `SemanticModel`. However, note that for aggregations, the metadata is intentionally cleared (since columns are now materialized):\n\n```query_as_table_after_aggregate\nfrom ibis import _\n\n# Aggregate the data\nagg_result = flights_st.group_by(\"origin\").aggregate(\"flight_count\", \"total_distance\")\n\n# Convert to SemanticModel using as_table()\nagg_table = agg_result.as_table()\n\n# Now .dimensions and .measures attributes exist, but they're empty (metadata was cleared)\nresult = f\"Type: {type(agg_table).__name__}\\nDimensions: {agg_table.dimensions}\\nMeasures: {agg_table.measures}\"\n```\n\n\n\nWhen are metadata preserved ?\n\nFor operations like `filter()`, `order_by()`, and `limit()`, `as_table()` **preserves** the original semantic metadata:\n\n```query_as_table_filter_preserved\nfrom ibis import _\n\n# Filter the data\nfiltered = flights_st.filter(_.distance > 2000)\n\n# Convert back to SemanticModel - metadata is preserved!\nfiltered_table = filtered.as_table()\n\n# Dimensions and measures are still available (preserved from original semantic table)\nresult = f\"Type: {type(filtered_table).__name__}\\nDimensions: {filtered_table.dimensions}\\nMeasures: {filtered_table.measures}\"\n```\n\n\n\nNotice how the dimensions and measures are preserved, unlike the aggregation case above where they were empty.\n\n**Key points:**\n- **Operations that preserve metadata**: `filter()`, `order_by()`, `limit()`, `unnest()` \u2014 calling `as_table()` restores full semantic capabilities with original dimensions/measures\n- **Operations that clear metadata**: `aggregate()`, `mutate()` \u2014 calling `as_table()` returns a `SemanticModel` with empty dimensions/measures (columns are materialized)\n- Use `as_table()` when you need to continue semantic operations on intermediate results\n\n## Next Steps\n\n- Learn about [Building Semantic Tables](/building/semantic-tables) to define dimensions and measures\n- Explore [Composing Models](/building/compose) for multi-table queries\n- Try [Advanced Patterns](/advanced/percentage-total) for complex analytics\n", + "markdown": "# Query Methods\n\n## Overview\n\nBSL provides a simple and consistent query API for retrieving data from your semantic tables. Queries are built by chaining methods, then executed or inspected using output methods.\n\nStart with a semantic table and chain methods together. Here's the typical query flow:\n\n```setup_table\nimport ibis\nfrom ibis import _\nfrom boring_semantic_layer import to_semantic_table\n\n# Create Ibis table\nflights_tbl = ibis.memtable({\n \"origin\": [\"NYC\", \"LAX\", \"NYC\", \"SFO\", \"LAX\", \"NYC\", \"SFO\", \"LAX\", \"NYC\"],\n \"carrier\": [\"AA\", \"UA\", \"AA\", \"UA\", \"AA\", \"UA\", \"AA\", \"UA\", \"AA\"],\n \"distance\": [2789, 2789, 2902, 2902, 347, 2789, 347, 347, 2789],\n \"duration\": [330, 330, 360, 360, 65, 330, 65, 65, 330],\n})\n\n# Create semantic table\nflights_st = (\n to_semantic_table(flights_tbl, name=\"flights\")\n .with_dimensions(\n origin=lambda t: t.origin,\n carrier=lambda t: t.carrier,\n )\n .with_measures(\n flight_count=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n avg_duration=lambda t: t.duration.mean(),\n )\n)\n```\n\n\n\n```python\nresult = (\n flights_st # Start with semantic table\n .filter(_.distance > 1000) # 1. Filter (optional)\n .group_by(\"origin\") # 2. Group by dimensions\n .aggregate(\"flight_count\", \"total_distance\") # 3. Aggregate measures\n .mutate(avg=lambda t: t.total_distance / t.flight_count) # 4. Transform (optional)\n .order_by(ibis.desc(\"flight_count\")) # 5. Sort (optional)\n .limit(10) # 6. Limit rows (optional)\n)\n```\n\nOnce you've built a query, you can inspect it or execute it:\n\n```simple_demo\n\n# Build a query\nresult = flights_st.group_by(\"origin\").aggregate(\"flight_count\")\n\n# Option 1: Execute and get data as pandas DataFrame\ndf = result.execute()\n\n# Option 2: View the generated SQL\nprint(result.sql())\n\n# Option 3: Generate a visualization (when applicable)\nchart = result.chart()\n\n# Option 4: See the semantic query plan\nprint(result)\n\nresult\n```\n\n\nThe output above includes a **Query Plan** tab showing how BSL translates this query into semantic operations. \n\nYou print the query object directly to see the plan:\n```python\nprint(result)\n```\n\nThis displays operations like `SemanticTableOp`, `SemanticGroupByOp`, and `SemanticAggregateOp`, useful for debugging and understanding query execution.\n\nLet's get now into the details of each query method.\n\n## group_by()\n\nThe `group_by()` method groups data by one or more dimensions.\n\n\n`group_by()` only accepts string dimension names that were previously defined in `with_dimensions()`. It does not support lambda functions or unbound `_` syntax.\n\n\n### Single Dimension\n\nGroup by a single dimension:\n\n```query_single_dimension\n# Group by one dimension\nresult = flights_st.group_by(\"origin\").aggregate(\"flight_count\")\n```\n\n\n\n### Multiple Dimensions\n\nGroup by multiple dimensions to create detailed breakdowns:\n\n```query_multiple_dimensions\n# Group by multiple dimensions\nresult = flights_st.group_by(\"origin\", \"carrier\").aggregate(\"flight_count\")\n```\n\n\n\n### No Grouping\n\nCalculate overall statistics across all rows using `group_by()` with no arguments:\n\n```query_no_grouping\n# Aggregate entire dataset without grouping\nresult = flights_st.group_by().aggregate(\"flight_count\", \"total_distance\", \"avg_duration\")\n```\n\n\n\n## aggregate()\n\nThe `aggregate()` method calculates measures after grouping. You can reference pre-defined measures or compute new ones on-the-fly.\n\n\n**CRITICAL**: `aggregate()` takes **measure names as strings**, not expressions or lambdas directly. Use measure names from `get_model()` output.\n```python\n# \u2705 CORRECT - measure names as strings\nmodel.group_by(\"category\").aggregate(\"flight_count\", \"total_revenue\")\n\n# \u274c WRONG - no standalone lambdas in aggregate\nmodel.aggregate(total=lambda t: t.sum()) # ERROR!\n```\n\n\n### Pre-defined Measures\n\nReference measures by their string names:\n\n```query_predefined_measures\n# Use measures defined in with_measures()\nresult = flights_st.group_by(\"origin\").aggregate(\"flight_count\", \"avg_duration\")\n```\n\n\n\n### On-the-Fly Transformations\n\nAdd computed measures directly in `aggregate()` without modifying the semantic table:\n\n```query_onthefly_measures\n# Mix predefined and computed measures\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\n \"flight_count\", # Pre-defined measure\n \"avg_duration\", # Pre-defined measure\n total_miles=lambda t: t.distance.sum(), # Computed on-the-fly\n max_distance=lambda t: t.flight_count + 2 # You can reference other measures as well\n )\n)\n```\n\n\n\n\nOn-the-fly measures let you add context-specific calculations without modifying your semantic table definition. This keeps your base model clean while enabling flexible queries.\n\n\n### Referencing Table Columns\n\nYou can reference **any column from the underlying table** in `aggregate()`, not just pre-defined measures. This is useful when you need one-off calculations without cluttering your semantic table definition.\n\n```query_table_columns\n# Reference table columns directly in aggregate()\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\n \"flight_count\", # Pre-defined measure\n total_distance=lambda t: t.distance.sum(), # Table column 'distance'\n avg_duration=lambda t: t.duration.mean(), # Table column 'duration'\n distance_in_km=lambda t: (t.distance * 1.60934).sum() # Transform then aggregate\n )\n)\n```\n\n\n\n**Key points:**\n- Table columns **must be aggregated** (e.g., `.sum()`, `.mean()`, `.max()`, `.count()`)\n- You can transform columns before aggregating (e.g., `(t.distance * 1.60934).sum()`)\n- This works for any column in the underlying table, even if not defined as a dimension or measure\n- Use this for ad-hoc calculations without modifying your semantic table\n\n\nTable columns cannot be used without an aggregation function. For example, `lambda t: t.distance` will fail. You must use `lambda t: t.distance.sum()` or another aggregation.\n\n\n## filter() / order_by() / limit() \n\nCombine `filter()`, `order_by()`, and `limit()` to refine your query results.\n\n```query_filter_order_limit\nfrom ibis import _\n\n# Filter data, sort, and limit results\nresult = (\n flights_st\n .filter(lambda t: t.origin.isin([\"NYC\", \"LAX\"])) # Filter origins\n .filter(_.distance > 500) # Filter distance using _ syntax\n .group_by(\"origin\")\n .aggregate(\"flight_count\", \"avg_duration\") # Aggregate both measures\n .order_by(ibis.desc(\"flight_count\")) # Sort by flight_count descending\n .limit(5) # Top 5 results\n)\n```\n\n\n\n**Key points:**\n- **`filter()`**: Use lambda or `_` syntax to apply conditions before aggregation\n- **`order_by()`**: Use `ibis.desc()` for descending order, or column name for ascending\n- **`limit()`**: Restrict the number of rows returned\n\n### Critical Filter Patterns\n\n**Multiple conditions** - use `ibis.and_()` or `ibis.or_()`:\n\n```python\n# Multiple conditions with AND\nmodel.filter(lambda t: ibis.and_(t.amount > 1000, t.year >= 2023))\n\n# Multiple conditions with OR\nmodel.filter(lambda t: ibis.or_(t.status == \"active\", t.status == \"pending\"))\n```\n\n**IN operator** - MUST use `.isin()` method:\n\n```python\n# \u2705 CORRECT - use .isin() method\nmodel.filter(lambda t: t.region.isin([\"US\", \"EU\", \"APAC\"]))\n\n# \u274c WRONG - Python's 'in' does NOT work!\nmodel.filter(lambda t: t.region in [\"US\", \"EU\"]) # ERROR: truth value of Ibis expression is not defined\n```\n\n**Lambda column names** - use column names directly, never prefix with model name:\n\n```python\n# \u2705 CORRECT - use column name directly\nmodel.filter(lambda t: t.carrier == \"AA\")\n\n# \u274c WRONG - do NOT prefix with model name\nmodel.filter(lambda t: t.model.carrier == \"AA\") # ERROR!\n```\n\n**Joined columns** - use exact prefixed name from `get_model()`:\n\n```python\n# If get_model() shows \"customers.country\", use it exactly:\nmodel.filter(lambda t: t.customers.country == \"US\")\n\n# \u274c WRONG - don't call methods on ID columns\nmodel.filter(lambda t: t.customer_id.country()) # ERROR: no such method!\n```\n\n## nest()\n\nThe `nest` parameter in `aggregate()` creates nested data structures (arrays of structs) in your query results. This is useful for API responses, hierarchical visualizations, and preserving relationships in aggregated data.\n\nUse `nest` to collect rows as structured arrays within each group:\n\n```query_basic_nest\nfrom ibis import _\n\n# Nest flight details within each origin\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\n \"flight_count\",\n \"total_distance\",\n # Create nested array of flight details\n nest={\"flights\": lambda t: t.group_by([\"carrier\", \"distance\"])}\n )\n)\n```\n\n\n\n**How it works:**\n- The `nest` parameter accepts a dictionary: `{\"column_name\": lambda t: ...}`\n- A bare `.group_by()` in the lambda specifies which row fields to collect\n- Results in an array of structs column named `flights`\n\nTo nest selected row-level fields, use the bare `group_by(...)` form. In a\n`nest` lambda this selects the struct fields to collect; it does not collapse\nduplicate source rows:\n\n```query_nest_select\n# Nest specific row-level fields\nresult = (\n flights_st\n .group_by(\"carrier\")\n .aggregate(\n \"flight_count\",\n nest={\"routes\": lambda t: t.group_by(\"origin\", \"distance\", \"duration\")}\n )\n)\n```\n\n\n\nAfter nesting, you can re-group which automatically unnests, then access the nested fields.\n\n**Step 1: Create nested data**\n\nFirst, create the nested structure. Notice the `flights` column contains arrays of structs:\n\n```query_nest_step1\nfrom ibis import _\n\n# Create nested data structure\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\n \"flight_count\",\n nest={\"flights\": lambda t: t.group_by([\"carrier\", \"distance\"])}\n )\n)\n```\n\n\n\n**Step 2: Re-group to unnest and access fields**\n\nNow re-group on the same dimension, which automatically unnests the array, allowing you to access the nested fields:\n\n```query_nest_step2\nfrom ibis import _\n\n# Re-grouping automatically unnests the 'flights' array\nresult = (\n result\n .group_by(\"origin\")\n .aggregate(\n total_flights=lambda t: t.flight_count.sum(),\n # Access unnested fields from the flights array\n unique_carriers=lambda t: t.flights.carrier.nunique(),\n avg_distance=lambda t: t.flights.distance.mean()\n )\n)\n```\n\n\n\n**Use cases for nesting:**\n- **API responses**: Create JSON-compatible hierarchical structures\n- **Hierarchical data**: Preserve parent-child relationships in results\n- **Data export**: Generate nested documents for external systems\n- **Drill-down analysis**: Keep detailed records available in aggregated views\n\n\nFor more complex nesting patterns and multi-level hierarchies, see [Nested Subtotals](/advanced/nested-subtotals).\n\n\n## mutate()\n\nThe `mutate()` method transforms aggregated results by adding new computed columns. This is different from on-the-fly measures in `aggregate()` \u2014 `mutate()` works on already-aggregated data.\n\n\n**Key difference:** `.aggregate()` computes from raw data, while `.mutate()` transforms already-aggregated results.\n\n\n```query_mutate\nfrom ibis import _\n\n# Add post-aggregation calculations\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\"flight_count\", \"total_distance\")\n .mutate(\n avg_distance_per_flight=lambda t: t.total_distance / t.flight_count,\n flight_category=lambda t: xo.case()\n .when(t.flight_count >= 3, \"high\")\n .when(t.flight_count >= 2, \"medium\")\n .else_(\"low\")\n .end()\n )\n)\n```\n\n\n\n**Use cases for `mutate()`:**\n- Calculate ratios from aggregated measures (e.g., `total / count`)\n- Create categories based on aggregated values\n- Add labels or formatting to results\n- Transform aggregated columns using the full power of Ibis\n\nFor more transformations, see [Ibis Table API reference](https://ibis-project.org/reference/expression-tables.html#ibis.expr.types.relations.Table.mutate).\n\n## Window Functions with .over()\n\nWindow functions perform calculations across ordered rows, enabling operations like running totals, moving averages, and ranking. Unlike regular aggregations that reduce many rows to one, window functions preserve row count while adding computed values.\n\n\n**Important:** Window functions can only be applied **after aggregation**, typically within a `.mutate()` call. They cannot be defined directly in measures.\n\n\n**Common window functions:**\n- **`lag()` / `lead()`**: Access previous/next row values for period-over-period comparisons\n- **`cumsum()`**: Calculate running totals\n- **`.over(window)`**: Apply functions over sliding windows (e.g., moving averages)\n- **`rank()` / `row_number()`**: Assign ranks or sequential numbers to rows\n\nHere's a simple example:\n\n```query_window_example\nfrom ibis import _\n\n# First aggregate to origin level\ndaily_flights = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\"flight_count\", \"total_distance\")\n)\n\n# Then apply window functions directly on the aggregate \u2014 the window\n# carries its own ordering, and the result stays a semantic query\nresult = daily_flights.mutate(\n cumulative_distance=lambda t: t.total_distance.sum().over(\n rows=(None, 0), order_by=\"origin\"\n ),\n flight_rank=lambda t: xo.rank().over(xo.window(order_by=xo.desc(t.flight_count)))\n).order_by(\"origin\").limit(10)\n```\n\n\n\n**Key points:**\n- Window functions are applied via `.mutate()` **directly on the aggregate** (before `.order_by()`/`.limit()` \u2014 after those the result is a plain table and `.mutate()` raises)\n- Give each window its own ordering via the keyword form of `.over()` (e.g. `rows=(None, 0), order_by=\"origin\"`)\n- For row math over a *filtered* result, drop to ibis explicitly with `.to_untagged().mutate(...)`\n\nFor comprehensive examples including lag/lead, moving averages, and ranking, see [Window Functions](/advanced/windowing).\n\n## as_table()\n\nAfter filtering or aggregating data, you may want to perform additional semantic operations. However, intermediate results don't always preserve the semantic table's dimensions and measures.\n\nThe Problem: Lost Semantic Information\n\nWhen you aggregate data, the result loses semantic metadata. The aggregated result is a `SemanticAggregate` expression, which doesn't have `.dimensions` or `.measures` attributes:\n\n```query_as_table_problem\nfrom ibis import _\n\n# Aggregate the data - this returns a SemanticAggregate\nagg_result = flights_st.group_by(\"origin\").aggregate(\"flight_count\", \"total_distance\")\n\n# Show the type/class of the result\nresult_type = type(agg_result).__name__\n\n# Try to access .dimensions - this will raise an AttributeError\ntry:\n dimensions = agg_result.dimensions\n result = f\"Type: {result_type}\\nDimensions: {dimensions}\"\nexcept AttributeError as e:\n result = f\"Type: {result_type}\\nError: {str(e)}\"\n\nresult\n```\n\n\n\n\nAfter aggregation, you can no longer access the original semantic table's dimensions and measures metadata.\n\nThe Solution: Use as_table()\n\nThe `as_table()` method converts results back into a `SemanticModel`. However, note that for aggregations, the metadata is intentionally cleared (since columns are now materialized):\n\n```query_as_table_after_aggregate\nfrom ibis import _\n\n# Aggregate the data\nagg_result = flights_st.group_by(\"origin\").aggregate(\"flight_count\", \"total_distance\")\n\n# Convert to SemanticModel using as_table()\nagg_table = agg_result.as_table()\n\n# Now .dimensions and .measures attributes exist, but they're empty (metadata was cleared)\nresult = f\"Type: {type(agg_table).__name__}\\nDimensions: {agg_table.dimensions}\\nMeasures: {agg_table.measures}\"\n```\n\n\n\nWhen are metadata preserved ?\n\nFor operations like `filter()`, `order_by()`, and `limit()`, `as_table()` **preserves** the original semantic metadata:\n\n```query_as_table_filter_preserved\nfrom ibis import _\n\n# Filter the data\nfiltered = flights_st.filter(_.distance > 2000)\n\n# Convert back to SemanticModel - metadata is preserved!\nfiltered_table = filtered.as_table()\n\n# Dimensions and measures are still available (preserved from original semantic table)\nresult = f\"Type: {type(filtered_table).__name__}\\nDimensions: {filtered_table.dimensions}\\nMeasures: {filtered_table.measures}\"\n```\n\n\n\nNotice how the dimensions and measures are preserved, unlike the aggregation case above where they were empty.\n\n**Key points:**\n- **Operations that preserve metadata**: `filter()`, `order_by()`, `limit()`, `unnest()` \u2014 calling `as_table()` restores full semantic capabilities with original dimensions/measures\n- **Operations that clear metadata**: `aggregate()`, `mutate()` \u2014 calling `as_table()` returns a `SemanticModel` with empty dimensions/measures (columns are materialized)\n- Use `as_table()` when you need to continue semantic operations on intermediate results\n\n## Next Steps\n\n- Learn about [Building Semantic Tables](/building/semantic-tables) to define dimensions and measures\n- Explore [Composing Models](/building/compose) for multi-table queries\n- Try [Advanced Patterns](/advanced/percentage-total) for complex analytics\n", "queries": { "setup_table": { - "code": "import ibis\nfrom ibis import _\nfrom boring_semantic_layer import to_semantic_table\n\n# Create Ibis table\nflights_tbl = ibis.memtable({\n \"origin\": [\"NYC\", \"LAX\", \"NYC\", \"SFO\", \"LAX\", \"NYC\", \"SFO\", \"LAX\", \"NYC\"],\n \"carrier\": [\"AA\", \"UA\", \"AA\", \"UA\", \"AA\", \"UA\", \"AA\", \"UA\", \"AA\"],\n \"distance\": [2789, 2789, 2902, 2902, 347, 2789, 347, 347, 2789],\n \"duration\": [330, 330, 360, 360, 65, 330, 65, 65, 330],\n})\n\n# Create semantic table\nflights_st = (\n to_semantic_table(flights_tbl, name=\"flights\")\n .with_dimensions(\n origin=lambda t: t.origin,\n carrier=lambda t: t.carrier,\n )\n .with_measures(\n flight_count=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n avg_duration=lambda t: t.duration.mean(),\n )\n)", - "sql": "SELECT\n *\nFROM \"ibis_pandas_memtable_onqfscphkbhtzli4oya7wmodva\"", - "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]", - "table": { - "columns": [ - "origin", - "carrier", - "distance", - "duration" - ], - "data": [ - [ - "NYC", - "AA", - 2789, - 330 - ], - [ - "LAX", - "UA", - 2789, - 330 - ], - [ - "NYC", - "AA", - 2902, - 360 - ], - [ - "SFO", - "UA", - 2902, - 360 - ], - [ - "LAX", - "AA", - 347, - 65 - ], - [ - "NYC", - "UA", - 2789, - 330 - ], - [ - "SFO", - "AA", - 347, - 65 - ], - [ - "LAX", - "UA", - 347, - 65 - ], - [ - "NYC", - "AA", - 2789, - 330 - ] - ] - } + "semantic_table": true, + "name": "flights", + "info": "Semantic table definition stored in context" }, "simple_demo": { "code": "# Build a query\nresult = flights_st.group_by(\"origin\").aggregate(\"flight_count\")\n\n# Option 1: Execute and get data as pandas DataFrame\ndf = result.execute()\n\n# Option 2: View the generated SQL\nprint(result.sql())\n\n# Option 3: Generate a visualization (when applicable)\nchart = result.chart()\n\n# Option 4: See the semantic query plan\nprint(result)\n\nresult", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_onqfscphkbhtzli4oya7wmodva\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_mhv3jujakzf5lpb247ugufiy4u\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy(origin)\n-> Aggregate(flight_count)", "table": { "columns": [ @@ -84,20 +20,77 @@ "NYC", 4 ], - [ - "SFO", - 2 - ], [ "LAX", 3 + ], + [ + "SFO", + 2 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-23a07ac3675351225fc742d6f2d8816b" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "flight_count", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "y": { + "field": "flight_count", + "type": "quantitative" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-23a07ac3675351225fc742d6f2d8816b": [ + { + "origin": "SFO", + "flight_count": 2 + }, + { + "origin": "NYC", + "flight_count": 4 + }, + { + "origin": "LAX", + "flight_count": 3 + } + ] + } + } } }, "query_single_dimension": { "code": "# Group by one dimension\nresult = flights_st.group_by(\"origin\").aggregate(\"flight_count\")", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_onqfscphkbhtzli4oya7wmodva\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_mhv3jujakzf5lpb247ugufiy4u\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy(origin)\n-> Aggregate(flight_count)", "table": { "columns": [ @@ -109,20 +102,77 @@ "SFO", 2 ], - [ - "LAX", - 3 - ], [ "NYC", 4 + ], + [ + "LAX", + 3 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-cf0d0ca0a252dccfe3241e9d50e45757" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "flight_count", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "y": { + "field": "flight_count", + "type": "quantitative" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-cf0d0ca0a252dccfe3241e9d50e45757": [ + { + "origin": "NYC", + "flight_count": 4 + }, + { + "origin": "LAX", + "flight_count": 3 + }, + { + "origin": "SFO", + "flight_count": 2 + } + ] + } + } } }, "query_multiple_dimensions": { "code": "# Group by multiple dimensions\nresult = flights_st.group_by(\"origin\", \"carrier\").aggregate(\"flight_count\")", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n \"t1\".\"carrier\",\n COUNT(*) AS \"flight_count\"\n FROM (\n SELECT\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\",\n \"t1\".\"carrier\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_onqfscphkbhtzli4oya7wmodva\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\",\n \"t1\".\"carrier\"\n) AS \"t2\"", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n \"t1\".\"carrier\",\n COUNT(*) AS \"flight_count\"\n FROM (\n SELECT\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\",\n \"t1\".\"carrier\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_mhv3jujakzf5lpb247ugufiy4u\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\",\n \"t1\".\"carrier\"\n) AS \"t2\"", "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy(origin, carrier)\n-> Aggregate(flight_count)", "table": { "columns": [ @@ -142,9 +192,9 @@ 1 ], [ - "SFO", + "LAX", "UA", - 1 + 2 ], [ "NYC", @@ -153,20 +203,104 @@ ], [ "SFO", - "AA", + "UA", 1 ], [ - "LAX", - "UA", - 2 + "SFO", + "AA", + 1 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-69b9987d937eda85c396796168e1b940" + }, + "mark": { + "type": "rect" + }, + "encoding": { + "color": { + "field": "flight_count", + "type": "quantitative" + }, + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "carrier", + "type": "nominal" + }, + { + "field": "flight_count", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "y": { + "field": "carrier", + "sort": null, + "type": "ordinal" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-69b9987d937eda85c396796168e1b940": [ + { + "origin": "LAX", + "carrier": "UA", + "flight_count": 2 + }, + { + "origin": "NYC", + "carrier": "UA", + "flight_count": 1 + }, + { + "origin": "SFO", + "carrier": "AA", + "flight_count": 1 + }, + { + "origin": "SFO", + "carrier": "UA", + "flight_count": 1 + }, + { + "origin": "NYC", + "carrier": "AA", + "flight_count": 3 + }, + { + "origin": "LAX", + "carrier": "AA", + "flight_count": 1 + } + ] + } + } } }, "query_no_grouping": { "code": "# Aggregate entire dataset without grouping\nresult = flights_st.group_by().aggregate(\"flight_count\", \"total_distance\", \"avg_duration\")", - "sql": "SELECT\n COUNT(*) AS \"flight_count\",\n SUM(\"t0\".\"distance\") AS \"total_distance\",\n AVG(\"t0\".\"duration\") AS \"avg_duration\"\nFROM \"ibis_pandas_memtable_onqfscphkbhtzli4oya7wmodva\" AS \"t0\"", + "sql": "SELECT\n *\nFROM (\n SELECT\n COUNT(*) AS \"flight_count\",\n SUM(\"t0\".\"distance\") AS \"total_distance\",\n AVG(\"t0\".\"duration\") AS \"avg_duration\"\n FROM \"ibis_pandas_memtable_mhv3jujakzf5lpb247ugufiy4u\" AS \"t0\"\n) AS \"t1\"", "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy()\n-> Aggregate(flight_count, total_distance, avg_duration)", "table": { "columns": [ @@ -181,11 +315,45 @@ 248.33333333333334 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-8a83363d11adffb1b866841c56e09dfc" + }, + "mark": { + "type": "text" + }, + "encoding": { + "text": { + "value": "Complex query - consider custom visualization" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-8a83363d11adffb1b866841c56e09dfc": [ + { + "flight_count": 9, + "total_distance": 18001, + "avg_duration": 248.33333333333334 + } + ] + } + } } }, "query_predefined_measures": { "code": "# Use measures defined in with_measures()\nresult = flights_st.group_by(\"origin\").aggregate(\"flight_count\", \"avg_duration\")", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n AVG(\"t1\".\"duration\") AS \"avg_duration\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_onqfscphkbhtzli4oya7wmodva\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n AVG(\"t1\".\"duration\") AS \"avg_duration\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_mhv3jujakzf5lpb247ugufiy4u\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy(origin)\n-> Aggregate(flight_count, avg_duration)", "table": { "columns": [ @@ -194,27 +362,110 @@ "avg_duration" ], "data": [ - [ - "LAX", - 3, - 153.33333333333334 - ], [ "NYC", 4, 337.5 ], + [ + "LAX", + 3, + 153.33333333333334 + ], [ "SFO", 2, 212.5 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-963d122a5c8d37e68233e80a126d265e" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "avg_duration" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-963d122a5c8d37e68233e80a126d265e": [ + { + "origin": "NYC", + "flight_count": 4, + "avg_duration": 337.5 + }, + { + "origin": "LAX", + "flight_count": 3, + "avg_duration": 153.33333333333334 + }, + { + "origin": "SFO", + "flight_count": 2, + "avg_duration": 212.5 + } + ] + } + } } }, "query_onthefly_measures": { "code": "# Mix predefined and computed measures\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\n \"flight_count\", # Pre-defined measure\n \"avg_duration\", # Pre-defined measure\n total_miles=lambda t: t.distance.sum(), # Computed on-the-fly\n max_distance=lambda t: t.flight_count + 2 # You can reference other measures as well\n )\n)", - "sql": "SELECT\n \"t2\".\"origin\",\n \"t2\".\"flight_count\",\n \"t2\".\"avg_duration\",\n \"t2\".\"total_miles\",\n \"t2\".\"flight_count\" + 2 AS \"max_distance\"\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n AVG(\"t1\".\"duration\") AS \"avg_duration\",\n SUM(\"t1\".\"distance\") AS \"total_miles\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_onqfscphkbhtzli4oya7wmodva\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", + "sql": "SELECT\n \"t2\".\"origin\",\n \"t2\".\"flight_count\",\n \"t2\".\"avg_duration\",\n \"t2\".\"total_miles\",\n \"t2\".\"flight_count\" + 2 AS \"max_distance\"\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n AVG(\"t1\".\"duration\") AS \"avg_duration\",\n SUM(\"t1\".\"distance\") AS \"total_miles\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_mhv3jujakzf5lpb247ugufiy4u\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy(origin)\n-> Aggregate(flight_count, avg_duration, total_miles, max_distance)", "table": { "columns": [ @@ -232,26 +483,117 @@ 3249, 4 ], - [ - "LAX", - 3, - 153.33333333333334, - 3483, - 5 - ], [ "NYC", 4, 337.5, 11269, 6 + ], + [ + "LAX", + 3, + 153.33333333333334, + 3483, + 5 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-22866678e8acf83c7eb8555595e76d9b" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "avg_duration", + "total_miles", + "max_distance" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-22866678e8acf83c7eb8555595e76d9b": [ + { + "origin": "NYC", + "flight_count": 4, + "avg_duration": 337.5, + "total_miles": 11269, + "max_distance": 6 + }, + { + "origin": "LAX", + "flight_count": 3, + "avg_duration": 153.33333333333334, + "total_miles": 3483, + "max_distance": 5 + }, + { + "origin": "SFO", + "flight_count": 2, + "avg_duration": 212.5, + "total_miles": 3249, + "max_distance": 4 + } + ] + } + } } }, "query_table_columns": { "code": "# Reference table columns directly in aggregate()\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\n \"flight_count\", # Pre-defined measure\n total_distance=lambda t: t.distance.sum(), # Table column 'distance'\n avg_duration=lambda t: t.duration.mean(), # Table column 'duration'\n distance_in_km=lambda t: (t.distance * 1.60934).sum() # Transform then aggregate\n )\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n SUM(\"t1\".\"distance\") AS \"total_distance\",\n AVG(\"t1\".\"duration\") AS \"avg_duration\",\n SUM(\"t1\".\"distance\" * 1.60934) AS \"distance_in_km\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_onqfscphkbhtzli4oya7wmodva\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n SUM(\"t1\".\"distance\") AS \"total_distance\",\n AVG(\"t1\".\"duration\") AS \"avg_duration\",\n SUM(\"t1\".\"distance\" * 1.60934) AS \"distance_in_km\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_mhv3jujakzf5lpb247ugufiy4u\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy(origin)\n-> Aggregate(flight_count, total_distance, avg_duration, distance_in_km)", "table": { "columns": [ @@ -263,11 +605,11 @@ ], "data": [ [ - "SFO", - 2, - 3249, - 212.5, - 5228.7456600000005 + "NYC", + 4, + 11269, + 337.5, + 18135.65246 ], [ "LAX", @@ -277,18 +619,109 @@ 5605.331220000001 ], [ - "NYC", - 4, - 11269, - 337.5, - 18135.65246 + "SFO", + 2, + 3249, + 212.5, + 5228.7456600000005 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-a038a006136bf110ddcf8ce95db67e8b" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "total_distance", + "avg_duration", + "distance_in_km" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-a038a006136bf110ddcf8ce95db67e8b": [ + { + "origin": "SFO", + "flight_count": 2, + "total_distance": 3249, + "avg_duration": 212.5, + "distance_in_km": 5228.7456600000005 + }, + { + "origin": "NYC", + "flight_count": 4, + "total_distance": 11269, + "avg_duration": 337.5, + "distance_in_km": 18135.65246 + }, + { + "origin": "LAX", + "flight_count": 3, + "total_distance": 3483, + "avg_duration": 153.33333333333334, + "distance_in_km": 5605.331220000001 + } + ] + } + } } }, "query_filter_order_limit": { "code": "from ibis import _\n\n# Filter data, sort, and limit results\nresult = (\n flights_st\n .filter(lambda t: t.origin.isin([\"NYC\", \"LAX\"])) # Filter origins\n .filter(_.distance > 500) # Filter distance using _ syntax\n .group_by(\"origin\")\n .aggregate(\"flight_count\", \"avg_duration\") # Aggregate both measures\n .order_by(ibis.desc(\"flight_count\")) # Sort by flight_count descending\n .limit(5) # Top 5 results\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n AVG(\"t1\".\"duration\") AS \"avg_duration\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_onqfscphkbhtzli4oya7wmodva\" AS \"t0\"\n WHERE\n \"t0\".\"origin\" IN ('NYC', 'LAX') AND \"t0\".\"distance\" > 500\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"\nORDER BY\n \"t2\".\"flight_count\" DESC NULLS LAST\nLIMIT 5", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n AVG(\"t1\".\"duration\") AS \"avg_duration\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_mhv3jujakzf5lpb247ugufiy4u\" AS \"t0\"\n WHERE\n \"t0\".\"origin\" IN ('NYC', 'LAX') AND \"t0\".\"distance\" > 500\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"\nORDER BY\n \"t2\".\"flight_count\" DESC NULLS LAST\nLIMIT 5", "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> Filter(\u03bb )\n-> Filter(\u03bb )\n-> GroupBy(origin)\n-> Aggregate(flight_count, avg_duration)\n-> OrderBy(_CallableWrapper(_fn=_['flight_count'].desc(nulls_first=False)))\n-> Limit(5)", "table": { "columns": [ @@ -308,11 +741,89 @@ 330.0 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-d715b987fbd69b5c84ba5625639181af" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "avg_duration" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-d715b987fbd69b5c84ba5625639181af": [ + { + "origin": "NYC", + "flight_count": 4, + "avg_duration": 337.5 + }, + { + "origin": "LAX", + "flight_count": 1, + "avg_duration": 330.0 + } + ] + } + } } }, "query_basic_nest": { "code": "from ibis import _\n\n# Nest flight details within each origin\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\n \"flight_count\",\n \"total_distance\",\n # Create nested array of flight details\n nest={\"flights\": lambda t: t.group_by([\"carrier\", \"distance\"])}\n )\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n SUM(\"t1\".\"distance\") AS \"total_distance\",\n ARRAY_AGG(NAMED_STRUCT('carrier', \"t1\".\"carrier\", 'distance', \"t1\".\"distance\")) FILTER(WHERE\n NAMED_STRUCT('carrier', \"t1\".\"carrier\", 'distance', \"t1\".\"distance\") IS NOT NULL) AS \"flights\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_onqfscphkbhtzli4oya7wmodva\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n SUM(\"t1\".\"distance\") AS \"total_distance\",\n ARRAY_AGG(NAMED_STRUCT('carrier', \"t1\".\"carrier\", 'distance', \"t1\".\"distance\")) FILTER(WHERE\n NAMED_STRUCT('carrier', \"t1\".\"carrier\", 'distance', \"t1\".\"distance\") IS NOT NULL) AS \"flights\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_mhv3jujakzf5lpb247ugufiy4u\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy(origin)\n-> Aggregate(flight_count, total_distance, flights)", "table": { "columns": [ @@ -322,6 +833,29 @@ "flights" ], "data": [ + [ + "NYC", + 4, + 11269, + [ + { + "carrier": "AA", + "distance": 2789 + }, + { + "carrier": "AA", + "distance": 2902 + }, + { + "carrier": "UA", + "distance": 2789 + }, + { + "carrier": "AA", + "distance": 2789 + } + ] + ], [ "LAX", 3, @@ -355,36 +889,139 @@ "distance": 347 } ] - ], - [ - "NYC", - 4, - 11269, - [ + ] + ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-84104b475a7c24825f52e7213f8ab180" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ { - "carrier": "AA", - "distance": 2789 + "field": "origin", + "type": "nominal" }, { - "carrier": "AA", - "distance": 2902 + "field": "measure", + "type": "nominal" }, { - "carrier": "UA", - "distance": 2789 + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "total_distance", + "flights" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-84104b475a7c24825f52e7213f8ab180": [ + { + "origin": "SFO", + "flight_count": 2, + "total_distance": 3249, + "flights": [ + { + "carrier": "UA", + "distance": 2902 + }, + { + "carrier": "AA", + "distance": 347 + } + ] }, { - "carrier": "AA", - "distance": 2789 + "origin": "NYC", + "flight_count": 4, + "total_distance": 11269, + "flights": [ + { + "carrier": "AA", + "distance": 2789 + }, + { + "carrier": "AA", + "distance": 2902 + }, + { + "carrier": "UA", + "distance": 2789 + }, + { + "carrier": "AA", + "distance": 2789 + } + ] + }, + { + "origin": "LAX", + "flight_count": 3, + "total_distance": 3483, + "flights": [ + { + "carrier": "UA", + "distance": 2789 + }, + { + "carrier": "AA", + "distance": 347 + }, + { + "carrier": "UA", + "distance": 347 + } + ] } ] - ] - ] + } + } } }, "query_nest_select": { - "code": "# Nest specific columns\nresult = (\n flights_st\n .group_by(\"carrier\")\n .aggregate(\n \"flight_count\",\n nest={\"routes\": lambda t: t.select(\"origin\", \"distance\", \"duration\")}\n )\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"carrier\",\n COUNT(*) AS \"flight_count\",\n ARRAY_AGG(\n NAMED_STRUCT('origin', \"t1\".\"origin\", 'distance', \"t1\".\"distance\", 'duration', \"t1\".\"duration\")\n ) FILTER(WHERE\n NAMED_STRUCT('origin', \"t1\".\"origin\", 'distance', \"t1\".\"distance\", 'duration', \"t1\".\"duration\") IS NOT NULL) AS \"routes\"\n FROM (\n SELECT\n \"t1\".\"origin\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"carrier\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_onqfscphkbhtzli4oya7wmodva\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"carrier\"\n) AS \"t2\"", + "code": "# Nest specific row-level fields\nresult = (\n flights_st\n .group_by(\"carrier\")\n .aggregate(\n \"flight_count\",\n nest={\"routes\": lambda t: t.group_by(\"origin\", \"distance\", \"duration\")}\n )\n)", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"carrier\",\n COUNT(*) AS \"flight_count\",\n ARRAY_AGG(\n NAMED_STRUCT('origin', \"t1\".\"origin\", 'distance', \"t1\".\"distance\", 'duration', \"t1\".\"duration\")\n ) FILTER(WHERE\n NAMED_STRUCT('origin', \"t1\".\"origin\", 'distance', \"t1\".\"distance\", 'duration', \"t1\".\"duration\") IS NOT NULL) AS \"routes\"\n FROM (\n SELECT\n \"t1\".\"origin\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"carrier\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_mhv3jujakzf5lpb247ugufiy4u\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"carrier\"\n) AS \"t2\"", "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy(carrier)\n-> Aggregate(flight_count, routes)", "table": { "columns": [ @@ -451,11 +1088,136 @@ ] ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-12823090065ba35b16e2d569e32088c2" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "carrier", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "carrier", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "routes" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-12823090065ba35b16e2d569e32088c2": [ + { + "carrier": "AA", + "flight_count": 5, + "routes": [ + { + "origin": "NYC", + "distance": 2789, + "duration": 330 + }, + { + "origin": "NYC", + "distance": 2902, + "duration": 360 + }, + { + "origin": "LAX", + "distance": 347, + "duration": 65 + }, + { + "origin": "SFO", + "distance": 347, + "duration": 65 + }, + { + "origin": "NYC", + "distance": 2789, + "duration": 330 + } + ] + }, + { + "carrier": "UA", + "flight_count": 4, + "routes": [ + { + "origin": "LAX", + "distance": 2789, + "duration": 330 + }, + { + "origin": "SFO", + "distance": 2902, + "duration": 360 + }, + { + "origin": "NYC", + "distance": 2789, + "duration": 330 + }, + { + "origin": "LAX", + "distance": 347, + "duration": 65 + } + ] + } + ] + } + } } }, "query_nest_step1": { "code": "from ibis import _\n\n# Create nested data structure\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\n \"flight_count\",\n nest={\"flights\": lambda t: t.group_by([\"carrier\", \"distance\"])}\n )\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n ARRAY_AGG(NAMED_STRUCT('carrier', \"t1\".\"carrier\", 'distance', \"t1\".\"distance\")) FILTER(WHERE\n NAMED_STRUCT('carrier', \"t1\".\"carrier\", 'distance', \"t1\".\"distance\") IS NOT NULL) AS \"flights\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_onqfscphkbhtzli4oya7wmodva\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n ARRAY_AGG(NAMED_STRUCT('carrier', \"t1\".\"carrier\", 'distance', \"t1\".\"distance\")) FILTER(WHERE\n NAMED_STRUCT('carrier', \"t1\".\"carrier\", 'distance', \"t1\".\"distance\") IS NOT NULL) AS \"flights\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_mhv3jujakzf5lpb247ugufiy4u\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy(origin)\n-> Aggregate(flight_count, flights)", "table": { "columns": [ @@ -479,51 +1241,173 @@ ] ], [ - "LAX", - 3, + "NYC", + 4, [ { - "carrier": "UA", + "carrier": "AA", "distance": 2789 }, { "carrier": "AA", - "distance": 347 + "distance": 2902 }, { "carrier": "UA", - "distance": 347 + "distance": 2789 + }, + { + "carrier": "AA", + "distance": 2789 } ] ], [ - "NYC", - 4, + "LAX", + 3, [ { - "carrier": "AA", + "carrier": "UA", "distance": 2789 }, { "carrier": "AA", - "distance": 2902 + "distance": 347 }, { "carrier": "UA", - "distance": 2789 - }, - { - "carrier": "AA", - "distance": 2789 + "distance": 347 } ] ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-f258e273c9236aaf2de2eeef0c1aa8d4" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "flights" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-f258e273c9236aaf2de2eeef0c1aa8d4": [ + { + "origin": "NYC", + "flight_count": 4, + "flights": [ + { + "carrier": "AA", + "distance": 2789 + }, + { + "carrier": "AA", + "distance": 2902 + }, + { + "carrier": "UA", + "distance": 2789 + }, + { + "carrier": "AA", + "distance": 2789 + } + ] + }, + { + "origin": "LAX", + "flight_count": 3, + "flights": [ + { + "carrier": "UA", + "distance": 2789 + }, + { + "carrier": "AA", + "distance": 347 + }, + { + "carrier": "UA", + "distance": 347 + } + ] + }, + { + "origin": "SFO", + "flight_count": 2, + "flights": [ + { + "carrier": "UA", + "distance": 2902 + }, + { + "carrier": "AA", + "distance": 347 + } + ] + } + ] + } + } } }, "query_nest_step2": { "code": "from ibis import _\n\n# Re-grouping automatically unnests the 'flights' array\nresult = (\n result\n .group_by(\"origin\")\n .aggregate(\n total_flights=lambda t: t.flight_count.sum(),\n # Access unnested fields from the flights array\n unique_carriers=lambda t: t.flights.carrier.nunique(),\n avg_distance=lambda t: t.flights.distance.mean()\n )\n)", - "sql": "WITH \"t3\" AS (\n SELECT\n *\n FROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n ARRAY_AGG(NAMED_STRUCT('carrier', \"t1\".\"carrier\", 'distance', \"t1\".\"distance\")) FILTER(WHERE\n NAMED_STRUCT('carrier', \"t1\".\"carrier\", 'distance', \"t1\".\"distance\") IS NOT NULL) AS \"flights\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_onqfscphkbhtzli4oya7wmodva\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n ) AS \"t2\"\n)\nSELECT\n *\nFROM (\n SELECT\n \"t7\".\"origin\",\n \"t7\".\"total_flights\",\n \"t9\".\"unique_carriers\",\n \"t9\".\"avg_distance\"\n FROM (\n SELECT\n \"t4\".\"origin\",\n SUM(\"t4\".\"flight_count\") AS \"total_flights\"\n FROM (\n SELECT\n \"t4\".\"flight_count\",\n \"t4\".\"flights\",\n \"t4\".\"origin\"\n FROM \"t3\" AS \"t4\"\n ) AS t4\n GROUP BY\n \"t4\".\"origin\"\n ) AS \"t7\"\n LEFT OUTER JOIN (\n SELECT\n \"t5\".\"origin\",\n COUNT(DISTINCT \"t5\".\"flights\"['carrier']) AS \"unique_carriers\",\n AVG(\"t5\".\"flights\"['distance']) AS \"avg_distance\"\n FROM (\n SELECT\n \"t5\".\"flight_count\",\n \"t5\".\"flights\",\n \"t5\".\"origin\"\n FROM (\n SELECT\n \"t4\".\"origin\",\n \"t4\".\"flight_count\",\n UNNEST(\"t4\".\"flights\") AS \"flights\"\n FROM \"t3\" AS \"t4\"\n ) AS \"t5\"\n ) AS t5\n GROUP BY\n \"t5\".\"origin\"\n ) AS \"t9\"\n ON \"t7\".\"origin\" = \"t9\".\"origin\"\n) AS \"t10\"", + "sql": "WITH \"t3\" AS (\n SELECT\n *\n FROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n ARRAY_AGG(NAMED_STRUCT('carrier', \"t1\".\"carrier\", 'distance', \"t1\".\"distance\")) FILTER(WHERE\n NAMED_STRUCT('carrier', \"t1\".\"carrier\", 'distance', \"t1\".\"distance\") IS NOT NULL) AS \"flights\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_mhv3jujakzf5lpb247ugufiy4u\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n ) AS \"t2\"\n)\nSELECT\n *\nFROM (\n SELECT\n \"t9\".\"origin\",\n \"t9\".\"total_flights\",\n \"t14\".\"unique_carriers\",\n \"t14\".\"avg_distance\"\n FROM (\n SELECT\n \"t4\".\"origin\",\n SUM(\"t4\".\"flight_count\") AS \"total_flights\"\n FROM (\n SELECT\n \"t4\".\"flight_count\",\n \"t4\".\"flights\",\n \"t4\".\"origin\"\n FROM \"t3\" AS \"t4\"\n ) AS t4\n GROUP BY\n \"t4\".\"origin\"\n ) AS \"t9\"\n LEFT OUTER JOIN (\n SELECT\n \"t12\".\"origin\",\n COALESCE(\"t12\".\"unique_carriers\", 0) AS \"unique_carriers\",\n \"t12\".\"avg_distance\"\n FROM (\n SELECT\n \"t7\".\"origin\",\n \"t11\".\"unique_carriers\",\n \"t11\".\"avg_distance\"\n FROM (\n SELECT\n \"t4\".\"origin\"\n FROM (\n SELECT\n \"t4\".\"flight_count\",\n \"t4\".\"flights\",\n \"t4\".\"origin\"\n FROM \"t3\" AS \"t4\"\n ) AS t4\n GROUP BY\n \"t4\".\"origin\"\n ) AS \"t7\"\n LEFT OUTER JOIN (\n SELECT\n \"t6\".\"origin\",\n COUNT(DISTINCT \"t6\".\"flights\"['carrier']) AS \"unique_carriers\",\n AVG(\"t6\".\"flights\"['distance']) AS \"avg_distance\"\n FROM (\n SELECT\n \"t6\".\"flight_count\",\n \"t6\".\"flights\",\n \"t6\".\"origin\"\n FROM (\n SELECT\n \"t4\".\"origin\",\n \"t4\".\"flight_count\",\n UNNEST(\"t4\".\"flights\") AS \"flights\"\n FROM \"t3\" AS \"t4\"\n ) AS \"t6\"\n ) AS t6\n GROUP BY\n \"t6\".\"origin\"\n ) AS \"t11\"\n ON (\n \"t7\".\"origin\" = \"t11\".\"origin\"\n )\n OR (\n (\n \"t7\".\"origin\" IS NULL\n ) AND (\n \"t11\".\"origin\" IS NULL\n )\n )\n ) AS \"t12\"\n ) AS \"t14\"\n ON (\n \"t9\".\"origin\" = \"t14\".\"origin\"\n )\n OR (\n (\n \"t9\".\"origin\" IS NULL\n ) AND (\n \"t14\".\"origin\" IS NULL\n )\n )\n) AS \"t15\"", "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy(origin)\n-> Aggregate(flight_count, flights)\n-> GroupBy(origin)\n-> Aggregate(total_flights, unique_carriers, avg_distance)", "table": { "columns": [ @@ -539,25 +1423,112 @@ 2, 2817.25 ], - [ - "SFO", - 2, - 2, - 1624.5 - ], [ "LAX", 3, 2, 1161.0 + ], + [ + "SFO", + 2, + 2, + 1624.5 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-d782e4c945aacacb2b473b1f0b3c5334" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "total_flights", + "unique_carriers", + "avg_distance" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-d782e4c945aacacb2b473b1f0b3c5334": [ + { + "origin": "NYC", + "total_flights": 4, + "unique_carriers": 2, + "avg_distance": 2817.25 + }, + { + "origin": "LAX", + "total_flights": 3, + "unique_carriers": 2, + "avg_distance": 1161.0 + }, + { + "origin": "SFO", + "total_flights": 2, + "unique_carriers": 2, + "avg_distance": 1624.5 + } + ] + } + } } }, "query_mutate": { "code": "from ibis import _\n\n# Add post-aggregation calculations\nresult = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\"flight_count\", \"total_distance\")\n .mutate(\n avg_distance_per_flight=lambda t: t.total_distance / t.flight_count,\n flight_category=lambda t: xo.case()\n .when(t.flight_count >= 3, \"high\")\n .when(t.flight_count >= 2, \"medium\")\n .else_(\"low\")\n .end()\n )\n)", - "sql": "SELECT\n \"t2\".\"origin\",\n \"t2\".\"flight_count\",\n \"t2\".\"total_distance\",\n CAST(\"t2\".\"total_distance\" AS DOUBLE PRECISION) / \"t2\".\"flight_count\" AS \"avg_distance_per_flight\",\n CASE\n WHEN \"t2\".\"flight_count\" >= 3\n THEN 'high'\n WHEN \"t2\".\"flight_count\" >= 2\n THEN 'medium'\n ELSE 'low'\n END AS \"flight_category\"\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n SUM(\"t1\".\"distance\") AS \"total_distance\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_onqfscphkbhtzli4oya7wmodva\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", - "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy(origin)\n-> Aggregate(flight_count, total_distance)\n-> Mutate(avg_distance_per_flight, flight_category)", + "sql": "SELECT\n \"t2\".\"origin\",\n \"t2\".\"flight_count\",\n \"t2\".\"total_distance\",\n CAST(\"t2\".\"total_distance\" AS DOUBLE PRECISION) / \"t2\".\"flight_count\" AS \"avg_distance_per_flight\",\n CASE\n WHEN \"t2\".\"flight_count\" >= 3\n THEN 'high'\n WHEN \"t2\".\"flight_count\" >= 2\n THEN 'medium'\n ELSE 'low'\n END AS \"flight_category\"\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n SUM(\"t1\".\"distance\") AS \"total_distance\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_mhv3jujakzf5lpb247ugufiy4u\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", + "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy(origin)\n-> Aggregate(flight_count, total_distance, avg_distance_per_flight, flight_category)", "table": { "columns": [ "origin", @@ -574,27 +1545,118 @@ 1624.0, "medium" ], - [ - "LAX", - 3, - 3483, - 1161.0, - "high" - ], [ "NYC", 4, 11269, 2817.0, "high" + ], + [ + "LAX", + 3, + 3483, + 1161.0, + "high" ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-3e2a5b4f2f349a90ce31ccde6a9c338d" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "total_distance", + "avg_distance_per_flight", + "flight_category" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-3e2a5b4f2f349a90ce31ccde6a9c338d": [ + { + "origin": "NYC", + "flight_count": 4, + "total_distance": 11269, + "avg_distance_per_flight": 2817.0, + "flight_category": "high" + }, + { + "origin": "LAX", + "flight_count": 3, + "total_distance": 3483, + "avg_distance_per_flight": 1161.0, + "flight_category": "high" + }, + { + "origin": "SFO", + "flight_count": 2, + "total_distance": 3249, + "avg_distance_per_flight": 1624.0, + "flight_category": "medium" + } + ] + } + } } }, "query_window_example": { - "code": "from ibis import _\n\n# First aggregate to daily level\ndaily_flights = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\"flight_count\", \"total_distance\")\n .order_by(\"origin\")\n)\n\n# Then apply window function for cumulative distance\nwindow_spec = xo.window(order_by=\"origin\")\n\nresult = daily_flights.mutate(\n cumulative_distance=_.total_distance.cumsum(),\n flight_rank=lambda t: xo.rank().over(xo.window(order_by=xo.desc(t.flight_count)))\n).limit(10)", - "sql": "SELECT\n \"t4\".\"origin\",\n \"t4\".\"flight_count\",\n \"t4\".\"total_distance\",\n \"t4\".\"cumulative_distance\",\n RANK() OVER (\n ORDER BY \"t4\".\"flight_count\" DESC NULLS LAST\n ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING\n ) - 1 AS \"flight_rank\"\nFROM (\n SELECT\n \"t3\".\"origin\",\n \"t3\".\"flight_count\",\n \"t3\".\"total_distance\",\n SUM(\"t3\".\"total_distance\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"cumulative_distance\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n SUM(\"t1\".\"distance\") AS \"total_distance\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_onqfscphkbhtzli4oya7wmodva\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n ) AS \"t2\"\n ORDER BY\n \"t2\".\"origin\" ASC\n ) AS \"t3\"\n) AS \"t4\"\nLIMIT 10", - "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy(origin)\n-> Aggregate(flight_count, total_distance)\n-> OrderBy(origin)\n-> Mutate(cumulative_distance, flight_rank)\n-> Limit(10)", + "code": "from ibis import _\n\n# First aggregate to origin level\ndaily_flights = (\n flights_st\n .group_by(\"origin\")\n .aggregate(\"flight_count\", \"total_distance\")\n)\n\n# Then apply window functions directly on the aggregate \u2014 the window\n# carries its own ordering, and the result stays a semantic query\nresult = daily_flights.mutate(\n cumulative_distance=lambda t: t.total_distance.sum().over(\n rows=(None, 0), order_by=\"origin\"\n ),\n flight_rank=lambda t: xo.rank().over(xo.window(order_by=xo.desc(t.flight_count)))\n).order_by(\"origin\").limit(10)", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t3\".\"origin\",\n \"t3\".\"flight_count\",\n \"t3\".\"total_distance\",\n \"t3\".\"cumulative_distance\",\n RANK() OVER (ORDER BY \"t3\".\"flight_count\" DESC NULLS LAST ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) - 1 AS \"flight_rank\"\n FROM (\n SELECT\n \"t2\".\"origin\",\n \"t2\".\"flight_count\",\n \"t2\".\"total_distance\",\n SUM(\"t2\".\"total_distance\") OVER (ORDER BY \"t2\".\"origin\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"cumulative_distance\"\n FROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n SUM(\"t1\".\"distance\") AS \"total_distance\"\n FROM (\n SELECT\n \"t1\".\"carrier\",\n \"t1\".\"distance\",\n \"t1\".\"duration\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_mhv3jujakzf5lpb247ugufiy4u\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n ) AS \"t2\"\n ) AS \"t3\"\n) AS \"t4\"\nORDER BY\n \"t4\".\"origin\" ASC\nLIMIT 10", + "plan": "SemanticTable: flights\n origin [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_duration [measure]\n-> GroupBy(origin)\n-> Aggregate(flight_count, total_distance, cumulative_distance, flight_rank)\n-> OrderBy(origin)\n-> Limit(10)", "table": { "columns": [ "origin", @@ -604,28 +1666,119 @@ "flight_rank" ], "data": [ - [ - "NYC", - 4, - 11269, - 18001, - 0 - ], [ "LAX", 3, 3483, - 6732, + 3483, 1 ], + [ + "NYC", + 4, + 11269, + 14752, + 0 + ], [ "SFO", 2, 3249, - 3249, + 18001, 2 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-3111db44e71ede1670a63c69729c5df6" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "total_distance", + "cumulative_distance", + "flight_rank" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-3111db44e71ede1670a63c69729c5df6": [ + { + "origin": "LAX", + "flight_count": 3, + "total_distance": 3483, + "cumulative_distance": 3483, + "flight_rank": 1 + }, + { + "origin": "NYC", + "flight_count": 4, + "total_distance": 11269, + "cumulative_distance": 14752, + "flight_rank": 0 + }, + { + "origin": "SFO", + "flight_count": 2, + "total_distance": 3249, + "cumulative_distance": 18001, + "flight_rank": 2 + } + ] + } + } } }, "query_as_table_problem": { diff --git a/docs/web/public/bsl-data/semantic-table.json b/docs/web/public/bsl-data/semantic-table.json index 139a665f..65eea0f7 100644 --- a/docs/web/public/bsl-data/semantic-table.json +++ b/docs/web/public/bsl-data/semantic-table.json @@ -1,5 +1,5 @@ { - "markdown": "# Building a Semantic Table\n\nDefine your data model with dimensions and measures using Ibis expressions.\n\n## Overview\n\nA Semantic Table is the core building block of BSL. It transforms a raw Ibis table into a reusable, self-documenting data model by defining:\n- **Dimensions**: Attributes to group by (e.g., origin, carrier, year)\n- **Measures**: Aggregations and calculations (e.g., flight count, total distance)\n\n## to_semantic_table()\n\n```setup_flights\nimport ibis\nfrom boring_semantic_layer import to_semantic_table\n\n# 1. Start with an Ibis table\ncon = ibis.duckdb.connect(\":memory:\")\nflights_data = ibis.memtable({\n \"origin\": [\"JFK\", \"LAX\", \"SFO\"],\n \"dest\": [\"LAX\", \"SFO\", \"JFK\"],\n \"carrier\": [\"AA\", \"UA\", \"DL\"],\n \"year\": [2023, 2023, 2024],\n \"distance\": [2475, 337, 382],\n \"dep_delay\": [10, 5, 0]\n})\nflights_tbl = con.create_table(\"flights\", flights_data)\n\n# 2. Convert to a Semantic Table\nflights_st = to_semantic_table(flights_tbl, name=\"flights\")\n```\n\n## with_dimensions()\n\nDimensions define the attributes you can group by in your queries. They represent the categorical or descriptive aspects of your data that you want to analyze.\n\nYou can define dimensions using lambda expressions, unbound syntax (`_.`), or the `Dimension` class with descriptions:\n\n```dimensions_demo\nfrom ibis import _\nfrom boring_semantic_layer import Dimension\n\nflights_st = flights_st.with_dimensions(\n # Lambda expressions - simple and explicit\n origin=lambda t: t.origin,\n\n # Unbound syntax - cleaner and more concise\n destination=_.dest,\n year=_.year,\n\n # Dimension - self-documenting and AI-friendly\n carrier=Dimension(\n expr=lambda t: t.carrier,\n description=\"Airline carrier code\"\n )\n)\n\nflights_st.dimensions\n```\n\n\n## with_measures()\n\nMeasures define the aggregations and calculations you can query. They represent the quantitative aspects of your data that you want to analyze (counts, sums, averages, etc.).\n\nYou can define measures using lambda expressions, reference other measures for composition, or use the `Measure` class with descriptions:\n\n```measures_demo\nfrom boring_semantic_layer import Measure\n\nflights_st = flights_st.with_measures(\n # Lambda expressions - simple and concise\n total_flights=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n max_delay=lambda t: t.dep_delay.max(),\n\n # Reference other measures for composition\n avg_distance_per_flight=lambda t: t.total_distance / t.total_flights,\n\n # Measure - self-documenting and AI-friendly\n avg_distance=Measure(\n expr=lambda t: t.distance.mean(),\n description=\"Average flight distance in miles\"\n )\n)\n\nflights_st.measures\n```\n\n\n\n### all()\n\nThe `all()` function references the entire dataset within measure definitions, enabling percent-of-total and comparison calculations.\n\n**Example:** Calculate market share as a percentage\n\n```measure_all_demo\nflights_with_pct = flights_st.with_measures(\n flight_count=lambda t: t.count(),\n market_share=lambda t: t.flight_count / t.all(t.flight_count) * 100 # Percent of total\n )\n\n# Query by carrier\nresult = (\n flights_with_pct\n .group_by(\"carrier\")\n .aggregate(\"flight_count\", \"market_share\")\n)\n```\n\n\n\n\n`t.all(ref)` is available on the table parameter `t` in measure definitions. It\nevaluates the supplied measure or reduction over the entire dataset regardless\nof grouping, making it useful for percentages and comparisons with the total.\n\n\nFor more examples, see the [Percent of Total pattern](/advanced/percentage-total).\n\n## graph\n\nThe `graph` property provides a dependency graph showing how dimensions and measures relate to each other. This is useful for:\n- **Understanding dependencies**: See what columns or fields each dimension/measure depends on\n- **Impact analysis**: Find what breaks when changing a field\n- **Documentation**: Generate visual representations of your data model\n- **Validation**: Ensure your model doesn't have circular dependencies\n\n```graph_demo\n# Build a semantic table with dependencies\nflights_with_deps = flights_st.with_dimensions(\n origin=lambda t: t.origin,\n destination=lambda t: t.dest,\n).with_measures(\n flight_count=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n avg_distance_per_flight=lambda t: t.total_distance / t.flight_count\n)\n\n# Access the dependency graph\ngraph = flights_with_deps.get_graph()\ngraph\n```\n\n\n### Understanding the Graph Structure\n\nThe graph is a dictionary where:\n- **Keys**: Dimension or measure names\n- **Values**: Metadata containing:\n - `deps`: Dependencies mapped to their types (`'column'`, `'dimension'`, or `'measure'`)\n - `type`: The field type (`'dimension'`, `'measure'`, or `'calc_measure'`)\n\n```graph_structure\n# Access the graph - it's a dict-like object\ngraph = flights_with_deps.get_graph()\ngraph\n```\n\n\n```python\n# Find what a specific field depends on\nflights_with_deps.get_graph()['avg_distance_per_flight']['deps']\n# Output: {'total_distance': 'measure', 'flight_count': 'measure'}\n```\n\n### Graph Traversal\n\nUse `graph_predecessors()` and `graph_successors()` to navigate dependencies:\n\n```graph_traversal\nfrom boring_semantic_layer import graph_predecessors, graph_successors\n\ngraph = flights_with_deps.get_graph()\n\n# What does this field depend on? (predecessors)\ngraph_predecessors(graph, 'avg_distance_per_flight')\n# {'total_distance', 'flight_count'}\n\n# What depends on this field? (successors)\ngraph_successors(graph, 'total_distance')\n# {'avg_distance_per_flight'}\n```\n\n\n### Working with the Dependency Graph\n\nThe dependency graph is a dict-like object where each key is a field name and the value is a dict with `\"type\"` (dimension/measure/calc_measure/column) and `\"deps\"` (dependencies with their types):\n\n```python\n# Access the graph directly as a dict\ngraph = flights_with_deps.get_graph()\n\n# Iterate over fields and their dependencies\nfor field, info in graph.items():\n print(f\"{field} ({info['type']}): depends on {info['deps']}\")\n```\n\n## join_one() / join_many() / join_cross()\n\nJoin semantic tables together to query across relationships. Joins allow you to combine data from multiple semantic tables and access dimensions and measures across all joined tables.\n\n**What Makes Semantic Joins Different?**\n\nSemantic joins explicitly capture the **relationship type** between tables, rather than just specifying SQL join mechanics:\n\n**SQL Joins:**\n```python\n# Specifies HOW to join, but not the analytical relationship\nflights_tbl.left_join(carriers_tbl, flights_tbl.carrier == carriers_tbl.code)\n```\n\n**Semantic Joins:**\n```python\n# One carrier row can match many flight rows\ncarriers.join_many(flights_st, lambda c, f: c.code == f.carrier)\n```\n\n**What You Get:**\n- **Explicit relationships**: `join_many()` documents that this is a one-to-many relationship\n- **Table hierarchy information**: The method name describes how tables relate to each other\n- **Richer metadata**: Makes the data model structure explicit for documentation and tooling\n\n\nAfter joining, dimensions and measures are prefixed with table names (e.g., `flights.origin`, `carriers.name`) to avoid naming conflicts.\n\n\n\n**Give every source in a composed model a unique name.** BSL uses model names as\nsource aliases for dimensions, measures, and grain metadata, and rejects a join\ntree containing duplicate names. If you join the same underlying table more than\nonce (for example, pickup and dropoff locations), create distinct table references\nand assign explicit aliases:\n\n```python\n# Create distinct references when joining same table twice\npickup_locs = to_semantic_table(locs_tbl.view(), \"pickup_locs\")\ndropoff_locs = to_semantic_table(locs_tbl.view(), \"dropoff_locs\")\n```\n\nThe distinct names prevent ambiguous semantic prefixes; `.view()` prevents Ibis\nfrom treating both roles as the same relation.\n\n\n\n**Source-aware aggregation requires equality-key joins.** When BSL aggregates\nmeasures at their source grain before joining, each non-cross join predicate must\nbe a direct field equality or a conjunction of direct field equalities. String,\nDeferred, and compound equality-key shorthands are supported. Predicates using\ninequality, `OR`, casts, or transformed expressions are rejected because reducing\nthem to join-key bridges could change the matched row set. Aggregate the models\nfirst or restate the relationship with plain equality keys; use `join_cross()` for\na Cartesian product.\n\n\nLet's get some additional data:\n\n```setup_carriers\nimport ibis\nfrom boring_semantic_layer import to_semantic_table\n\ncon = ibis.duckdb.connect(\":memory:\")\n\n# Create carriers data\ncarriers_data = ibis.memtable({\n \"code\": [\"AA\", \"UA\", \"DL\"],\n \"name\": [\"American Airlines\", \"United Airlines\", \"Delta Air Lines\"]\n})\ncarriers_tbl = con.create_table(\"carriers\", carriers_data)\n```\n\n\nAnd create a carriers semantic table:\n\n```carriers_st\ncarriers = (\n to_semantic_table(carriers_tbl, name=\"carriers\")\n .with_dimensions(\n code=lambda t: t.code,\n name=lambda t: t.name\n )\n .with_measures(\n carrier_count=lambda t: t.count()\n )\n)\n```\n\n### join_many() - One-to-Many Relationships\n\nUse `join_many()` when one row in the left table can match multiple rows in the right table (LEFT JOIN).\n\n```join_demo\n# One carrier row can match many flight rows\ncarriers_with_flights = carriers.join_many(\n flights_st,\n lambda c, f: c.code == f.carrier\n)\n\n# Inspect available dimensions and measures\ncarriers_with_flights.dimensions\n```\n\n\nAfter joining, all dimensions and measures from both tables are available. Each is prefixed with its table name to avoid conflicts:\n\n\n### join_one() - At-Most-One Right Match\n\nUse `join_one()` when each row on the left can match at most one row on the\nright. Like all non-cross semantic joins, it uses a LEFT JOIN so unmatched left\nrows remain visible to measures.\n\n```python\n# Many flights \u2192 one carrier (each flight matches at most one carrier)\nflights_with_carrier = flights_st.join_one(\n carriers,\n lambda f, c: f.carrier == c.code\n)\n```\n\n\n**Join predicates resolve physical columns.** A string or Deferred shorthand\nnames the same underlying column on both sides. If the columns have different\nnames, use a two-argument lambda instead.\n\n**Example:**\n```python\n# If users table has column 'id' but dimension 'customer_id':\nusers = to_semantic_table(users_tbl, \"users\").with_dimensions(\n customer_id=lambda t: t.id # Dimension renamed\n)\n\n# Compare the underlying columns explicitly:\norders.join_one(users, on=lambda order, user: order.customer_id == user.id)\n```\n\n\n### join_cross() - Cross Join\n\nUse `join_cross()` to create every possible combination of rows from both tables (CARTESIAN PRODUCT).\n\n```python\n# Every flight \u00d7 every carrier combination\nall_combinations = flights_st.join_cross(carriers)\n```\n\n### Requiring a Match\n\n`join_one()` and `join_many()` only support `how=\"left\"`. When a query should\nrequire a match, make the row removal explicit with a filter on a non-nullable\nfield from the right table:\n\n```python\nflights_matched = flights_st.join_one(\n carriers,\n lambda f, c: f.carrier == c.code,\n).filter(lambda t: t[\"carriers.name\"].notnull())\n```\n\nUse `join_cross()` for Cartesian products.\n\n## Next Steps\n\n- Learn about [Composing Models](/building/compose)\n- Explore [YAML Configuration](/building/yaml)\n- Start [Querying Semantic Tables](/querying/methods)\n", + "markdown": "# Building a Semantic Table\n\nDefine your data model with dimensions and measures using Ibis expressions.\n\n## Overview\n\nA Semantic Table is the core building block of BSL. It transforms a raw Ibis table into a reusable, self-documenting data model by defining:\n- **Dimensions**: Attributes to group by (e.g., origin, carrier, year)\n- **Measures**: Aggregations and calculations (e.g., flight count, total distance)\n\n## to_semantic_table()\n\n```setup_flights\nimport ibis\nfrom boring_semantic_layer import to_semantic_table\n\n# 1. Start with an Ibis table\ncon = ibis.duckdb.connect(\":memory:\")\nflights_data = ibis.memtable({\n \"origin\": [\"JFK\", \"LAX\", \"SFO\"],\n \"dest\": [\"LAX\", \"SFO\", \"JFK\"],\n \"carrier\": [\"AA\", \"UA\", \"DL\"],\n \"year\": [2023, 2023, 2024],\n \"distance\": [2475, 337, 382],\n \"dep_delay\": [10, 5, 0]\n})\nflights_tbl = con.create_table(\"flights\", flights_data)\n\n# 2. Convert to a Semantic Table\nflights_st = to_semantic_table(flights_tbl, name=\"flights\")\n```\n\n## with_dimensions()\n\nDimensions define the attributes you can group by in your queries. They represent the categorical or descriptive aspects of your data that you want to analyze.\n\nYou can define dimensions using lambda expressions, unbound syntax (`_.`), or the `Dimension` class with descriptions:\n\n```dimensions_demo\nfrom ibis import _\nfrom boring_semantic_layer import Dimension\n\nflights_st = flights_st.with_dimensions(\n # Lambda expressions - simple and explicit\n origin=lambda t: t.origin,\n\n # Unbound syntax - cleaner and more concise\n destination=_.dest,\n year=_.year,\n\n # Dimension - self-documenting and AI-friendly\n carrier=Dimension(\n expr=lambda t: t.carrier,\n description=\"Airline carrier code\"\n )\n)\n\nflights_st.dimensions\n```\n\n\n## with_measures()\n\nMeasures define the aggregations and calculations you can query. They represent the quantitative aspects of your data that you want to analyze (counts, sums, averages, etc.).\n\nYou can define measures using lambda expressions, reference other measures for composition, or use the `Measure` class with descriptions:\n\n```measures_demo\nfrom boring_semantic_layer import Measure\n\nflights_st = flights_st.with_measures(\n # Lambda expressions - simple and concise\n total_flights=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n max_delay=lambda t: t.dep_delay.max(),\n\n # Reference other measures for composition\n avg_distance_per_flight=lambda t: t.total_distance / t.total_flights,\n\n # Measure - self-documenting and AI-friendly\n avg_distance=Measure(\n expr=lambda t: t.distance.mean(),\n description=\"Average flight distance in miles\"\n )\n)\n\nflights_st.measures\n```\n\n\n\n### all()\n\nThe `all()` function references the entire dataset within measure definitions, enabling percent-of-total and comparison calculations.\n\n**Example:** Calculate market share as a percentage\n\n```measure_all_demo\nflights_with_pct = flights_st.with_measures(\n flight_count=lambda t: t.count(),\n market_share=lambda t: t.flight_count / t.all(t.flight_count) * 100 # Percent of total\n )\n\n# Query by carrier\nresult = (\n flights_with_pct\n .group_by(\"carrier\")\n .aggregate(\"flight_count\", \"market_share\")\n)\n```\n\n\n\n\n`t.all(ref)` is available on the table parameter `t` in measure definitions. It\nevaluates the supplied measure or reduction over the entire dataset regardless\nof grouping, making it useful for percentages and comparisons with the total.\n\n\nFor more examples, see the [Percent of Total pattern](/advanced/percentage-total).\n\n## graph\n\nThe `graph` property provides a dependency graph showing how dimensions and measures relate to each other. This is useful for:\n- **Understanding dependencies**: See what columns or fields each dimension/measure depends on\n- **Impact analysis**: Find what breaks when changing a field\n- **Documentation**: Generate visual representations of your data model\n- **Validation**: Ensure your model doesn't have circular dependencies\n\n```graph_demo\n# Build a semantic table with dependencies\nflights_with_deps = flights_st.with_dimensions(\n origin=lambda t: t.origin,\n destination=lambda t: t.dest,\n).with_measures(\n flight_count=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n avg_distance_per_flight=lambda t: t.total_distance / t.flight_count\n)\n\n# Access the dependency graph\ngraph = flights_with_deps.get_graph()\ngraph\n```\n\n\n### Understanding the Graph Structure\n\nThe graph is a dictionary where:\n- **Keys**: Dimension or measure names\n- **Values**: Metadata containing:\n - `deps`: Dependencies mapped to their types (`'column'`, `'dimension'`, or `'measure'`)\n - `type`: The field type (`'dimension'`, `'measure'`, or `'calc_measure'`)\n\n```graph_structure\n# Access the graph - it's a dict-like object\ngraph = flights_with_deps.get_graph()\ngraph\n```\n\n\n```python\n# Find what a specific field depends on\nflights_with_deps.get_graph()['avg_distance_per_flight']['deps']\n# Output: {'total_distance': 'measure', 'flight_count': 'measure'}\n```\n\n### Graph Traversal\n\nUse `graph_predecessors()` and `graph_successors()` to navigate dependencies:\n\n```graph_traversal\nfrom boring_semantic_layer.graph_utils import graph_predecessors, graph_successors\n\ngraph = flights_with_deps.get_graph()\n\n# What does this field depend on? (predecessors)\ngraph_predecessors(graph, 'avg_distance_per_flight')\n# {'total_distance', 'flight_count'}\n\n# What depends on this field? (successors)\ngraph_successors(graph, 'total_distance')\n# {'avg_distance_per_flight'}\n```\n\n\n### Working with the Dependency Graph\n\nThe dependency graph is a dict-like object where each key is a field name and the value is a dict with `\"type\"` (dimension/measure/calc_measure/column) and `\"deps\"` (dependencies with their types):\n\n```python\n# Access the graph directly as a dict\ngraph = flights_with_deps.get_graph()\n\n# Iterate over fields and their dependencies\nfor field, info in graph.items():\n print(f\"{field} ({info['type']}): depends on {info['deps']}\")\n```\n\n## join_one() / join_many() / join_cross()\n\nJoin semantic tables together to query across relationships. Joins allow you to combine data from multiple semantic tables and access dimensions and measures across all joined tables.\n\n**What Makes Semantic Joins Different?**\n\nSemantic joins explicitly capture the **relationship type** between tables, rather than just specifying SQL join mechanics:\n\n**SQL Joins:**\n```python\n# Specifies HOW to join, but not the analytical relationship\nflights_tbl.left_join(carriers_tbl, flights_tbl.carrier == carriers_tbl.code)\n```\n\n**Semantic Joins:**\n```python\n# One carrier row can match many flight rows\ncarriers.join_many(flights_st, lambda c, f: c.code == f.carrier)\n```\n\n**What You Get:**\n- **Explicit relationships**: `join_many()` documents that this is a one-to-many relationship\n- **Table hierarchy information**: The method name describes how tables relate to each other\n- **Richer metadata**: Makes the data model structure explicit for documentation and tooling\n\n\nAfter joining, dimensions and measures are prefixed with table names (e.g., `flights.origin`, `carriers.name`) to avoid naming conflicts.\n\n\n\n**Give every source in a composed model a unique name.** BSL uses model names as\nsource aliases for dimensions, measures, and grain metadata, and rejects a join\ntree containing duplicate names. If you join the same underlying table more than\nonce (for example, pickup and dropoff locations), create distinct table references\nand assign explicit aliases:\n\n```python\n# Create distinct references when joining same table twice\npickup_locs = to_semantic_table(locs_tbl.view(), \"pickup_locs\")\ndropoff_locs = to_semantic_table(locs_tbl.view(), \"dropoff_locs\")\n```\n\nThe distinct names prevent ambiguous semantic prefixes; `.view()` prevents Ibis\nfrom treating both roles as the same relation.\n\n\n\n**Source-aware aggregation requires equality-key joins.** When BSL aggregates\nmeasures at their source grain before joining, each non-cross join predicate must\nbe a direct field equality or a conjunction of direct field equalities. String,\nDeferred, and compound equality-key shorthands are supported. Predicates using\ninequality, `OR`, casts, or transformed expressions are rejected because reducing\nthem to join-key bridges could change the matched row set. Aggregate the models\nfirst or restate the relationship with plain equality keys; use `join_cross()` for\na Cartesian product.\n\n\nLet's get some additional data:\n\n```setup_carriers\nimport ibis\nfrom boring_semantic_layer import to_semantic_table\n\ncon = ibis.duckdb.connect(\":memory:\")\n\n# Create carriers data\ncarriers_data = ibis.memtable({\n \"code\": [\"AA\", \"UA\", \"DL\"],\n \"name\": [\"American Airlines\", \"United Airlines\", \"Delta Air Lines\"]\n})\ncarriers_tbl = con.create_table(\"carriers\", carriers_data)\n```\n\n\nAnd create a carriers semantic table:\n\n```carriers_st\ncarriers = (\n to_semantic_table(carriers_tbl, name=\"carriers\")\n .with_dimensions(\n code=lambda t: t.code,\n name=lambda t: t.name\n )\n .with_measures(\n carrier_count=lambda t: t.count()\n )\n)\n```\n\n### join_many() - One-to-Many Relationships\n\nUse `join_many()` when one row in the left table can match multiple rows in the right table (LEFT JOIN).\n\n```join_demo\n# One carrier row can match many flight rows\ncarriers_with_flights = carriers.join_many(\n flights_st,\n lambda c, f: c.code == f.carrier\n)\n\n# Inspect available dimensions and measures\ncarriers_with_flights.dimensions\n```\n\n\nAfter joining, all dimensions and measures from both tables are available. Each is prefixed with its table name to avoid conflicts:\n\n\n### join_one() - At-Most-One Right Match\n\nUse `join_one()` when each row on the left can match at most one row on the\nright. Like all non-cross semantic joins, it uses a LEFT JOIN so unmatched left\nrows remain visible to measures.\n\n```python\n# Many flights \u2192 one carrier (each flight matches at most one carrier)\nflights_with_carrier = flights_st.join_one(\n carriers,\n lambda f, c: f.carrier == c.code\n)\n```\n\n\n**Join predicates resolve physical columns.** A string or Deferred shorthand\nnames the same underlying column on both sides. If the columns have different\nnames, use a two-argument lambda instead.\n\n**Example:**\n```python\n# If users table has column 'id' but dimension 'customer_id':\nusers = to_semantic_table(users_tbl, \"users\").with_dimensions(\n customer_id=lambda t: t.id # Dimension renamed\n)\n\n# Compare the underlying columns explicitly:\norders.join_one(users, on=lambda order, user: order.customer_id == user.id)\n```\n\n\n### join_cross() - Cross Join\n\nUse `join_cross()` to create every possible combination of rows from both tables (CARTESIAN PRODUCT).\n\n```python\n# Every flight \u00d7 every carrier combination\nall_combinations = flights_st.join_cross(carriers)\n```\n\n### Requiring a Match\n\n`join_one()` and `join_many()` are always LEFT joins (there is no `how=`\nparameter). When a query should\nrequire a match, make the row removal explicit with a filter on a non-nullable\nfield from the right table:\n\n```python\nflights_matched = flights_st.join_one(\n carriers,\n lambda f, c: f.carrier == c.code,\n).filter(lambda t: t[\"carriers.name\"].notnull())\n```\n\nUse `join_cross()` for Cartesian products.\n\n## Next Steps\n\n- Learn about [Composing Models](/building/compose)\n- Explore [YAML Configuration](/building/yaml)\n- Start [Querying Semantic Tables](/querying/methods)\n", "queries": { "dimensions_demo": { "output": "('origin', 'destination', 'year', 'carrier')" @@ -9,7 +9,7 @@ }, "measure_all_demo": { "code": "flights_with_pct = flights_st.with_measures(\n flight_count=lambda t: t.count(),\n market_share=lambda t: t.flight_count / t.all(t.flight_count) * 100 # Percent of total\n )\n\n# Query by carrier\nresult = (\n flights_with_pct\n .group_by(\"carrier\")\n .aggregate(\"flight_count\", \"market_share\")\n)", - "sql": "WITH \"t1\" AS (\n SELECT\n *\n FROM \"memory\".\"main\".\"flights\" AS \"t0\"\n)\nSELECT\n \"t7\".\"carrier\",\n \"t7\".\"flight_count\",\n (\n CAST(\"t7\".\"flight_count\" AS DOUBLE) / CAST(\"t7\".\"flight_count_right\" AS DOUBLE)\n ) * 100 AS \"market_share\"\nFROM (\n SELECT\n \"t5\".\"carrier\",\n \"t5\".\"flight_count\",\n \"t6\".\"flight_count\" AS \"flight_count_right\"\n FROM (\n SELECT\n \"t2\".\"carrier\",\n COUNT(*) AS \"flight_count\"\n FROM \"t1\" AS \"t2\"\n GROUP BY\n 1\n ) AS \"t5\"\n CROSS JOIN (\n SELECT\n COUNT(*) AS \"flight_count\"\n FROM \"t1\" AS \"t2\"\n ) AS \"t6\"\n) AS \"t7\"", + "sql": "SELECT\n \"t3\".\"carrier\",\n \"t3\".\"flight_count\",\n (\n \"t3\".\"flight_count\" / \"t3\".\"__bsl_totals__flight_count\"\n ) * 100 AS \"market_share\"\nFROM (\n SELECT\n \"t2\".\"carrier\",\n COUNT(*) AS \"flight_count\",\n ANY_VALUE(\"t2\".\"__bsl_totals__flight_count\") AS \"__bsl_totals__flight_count\"\n FROM (\n SELECT\n \"t1\".\"origin\",\n \"t1\".\"dest\",\n \"t1\".\"carrier\",\n \"t1\".\"year\",\n \"t1\".\"distance\",\n \"t1\".\"dep_delay\",\n CAST(COUNT(*) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE) AS \"__bsl_totals__flight_count\"\n FROM (\n SELECT\n *\n FROM \"memory\".\"main\".\"flights\" AS \"t0\"\n ) AS \"t1\"\n ) AS \"t2\"\n GROUP BY\n 1\n) AS \"t3\"", "plan": "SemanticTable: flights\n origin [dim]\n destination [dim]\n year [dim]\n carrier [dim]\n total_flights [measure]\n total_distance [measure]\n max_delay [measure]\n avg_distance [measure]\n flight_count [measure]\n avg_distance_per_flight [calc]\n market_share [calc]\n-> GroupBy(carrier)\n-> Aggregate(flight_count, market_share)", "table": { "columns": [ @@ -24,21 +24,104 @@ 33.33333333333333 ], [ - "UA", + "DL", 1, 33.33333333333333 ], [ - "DL", + "UA", 1, 33.33333333333333 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-957c443b2b52e9d52db460acc818b08e" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "carrier", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "carrier", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "market_share" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-957c443b2b52e9d52db460acc818b08e": [ + { + "carrier": "AA", + "flight_count": 1, + "market_share": 33.33333333333333 + }, + { + "carrier": "DL", + "flight_count": 1, + "market_share": 33.33333333333333 + }, + { + "carrier": "UA", + "flight_count": 1, + "market_share": 33.33333333333333 + } + ] + } + } } }, "graph_demo": { "code": "# Build a semantic table with dependencies\nflights_with_deps = flights_st.with_dimensions(\n origin=lambda t: t.origin,\n destination=lambda t: t.dest,\n).with_measures(\n flight_count=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n avg_distance_per_flight=lambda t: t.total_distance / t.flight_count\n)\n\n# Access the dependency graph\ngraph = flights_with_deps.get_graph()\ngraph", - "sql": "WITH \"t1\" AS (\n SELECT\n *\n FROM \"memory\".\"main\".\"flights\" AS \"t0\"\n)\nSELECT\n \"t7\".\"carrier\",\n \"t7\".\"flight_count\",\n (\n CAST(\"t7\".\"flight_count\" AS DOUBLE) / CAST(\"t7\".\"flight_count_right\" AS DOUBLE)\n ) * 100 AS \"market_share\"\nFROM (\n SELECT\n \"t5\".\"carrier\",\n \"t5\".\"flight_count\",\n \"t6\".\"flight_count\" AS \"flight_count_right\"\n FROM (\n SELECT\n \"t2\".\"carrier\",\n COUNT(*) AS \"flight_count\"\n FROM \"t1\" AS \"t2\"\n GROUP BY\n 1\n ) AS \"t5\"\n CROSS JOIN (\n SELECT\n COUNT(*) AS \"flight_count\"\n FROM \"t1\" AS \"t2\"\n ) AS \"t6\"\n) AS \"t7\"", + "sql": "SELECT\n \"t3\".\"carrier\",\n \"t3\".\"flight_count\",\n (\n \"t3\".\"flight_count\" / \"t3\".\"__bsl_totals__flight_count\"\n ) * 100 AS \"market_share\"\nFROM (\n SELECT\n \"t2\".\"carrier\",\n COUNT(*) AS \"flight_count\",\n ANY_VALUE(\"t2\".\"__bsl_totals__flight_count\") AS \"__bsl_totals__flight_count\"\n FROM (\n SELECT\n \"t1\".\"origin\",\n \"t1\".\"dest\",\n \"t1\".\"carrier\",\n \"t1\".\"year\",\n \"t1\".\"distance\",\n \"t1\".\"dep_delay\",\n CAST(COUNT(*) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE) AS \"__bsl_totals__flight_count\"\n FROM (\n SELECT\n *\n FROM \"memory\".\"main\".\"flights\" AS \"t0\"\n ) AS \"t1\"\n ) AS \"t2\"\n GROUP BY\n 1\n) AS \"t3\"", "plan": "SemanticTable: flights\n origin [dim]\n destination [dim]\n year [dim]\n carrier [dim]\n total_flights [measure]\n total_distance [measure]\n max_delay [measure]\n avg_distance [measure]\n flight_count [measure]\n avg_distance_per_flight [calc]\n market_share [calc]\n-> GroupBy(carrier)\n-> Aggregate(flight_count, market_share)", "table": { "columns": [ @@ -48,26 +131,109 @@ ], "data": [ [ - "UA", + "AA", 1, 33.33333333333333 ], [ - "AA", + "DL", 1, 33.33333333333333 ], [ - "DL", + "UA", 1, 33.33333333333333 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-444a8c980d5c591c007caa61dccef4bc" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "carrier", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "carrier", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "market_share" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-444a8c980d5c591c007caa61dccef4bc": [ + { + "carrier": "AA", + "flight_count": 1, + "market_share": 33.33333333333333 + }, + { + "carrier": "UA", + "flight_count": 1, + "market_share": 33.33333333333333 + }, + { + "carrier": "DL", + "flight_count": 1, + "market_share": 33.33333333333333 + } + ] + } + } } }, "graph_structure": { "code": "# Access the graph - it's a dict-like object\ngraph = flights_with_deps.get_graph()\ngraph", - "sql": "WITH \"t1\" AS (\n SELECT\n *\n FROM \"memory\".\"main\".\"flights\" AS \"t0\"\n)\nSELECT\n \"t7\".\"carrier\",\n \"t7\".\"flight_count\",\n (\n CAST(\"t7\".\"flight_count\" AS DOUBLE) / CAST(\"t7\".\"flight_count_right\" AS DOUBLE)\n ) * 100 AS \"market_share\"\nFROM (\n SELECT\n \"t5\".\"carrier\",\n \"t5\".\"flight_count\",\n \"t6\".\"flight_count\" AS \"flight_count_right\"\n FROM (\n SELECT\n \"t2\".\"carrier\",\n COUNT(*) AS \"flight_count\"\n FROM \"t1\" AS \"t2\"\n GROUP BY\n 1\n ) AS \"t5\"\n CROSS JOIN (\n SELECT\n COUNT(*) AS \"flight_count\"\n FROM \"t1\" AS \"t2\"\n ) AS \"t6\"\n) AS \"t7\"", + "sql": "SELECT\n \"t3\".\"carrier\",\n \"t3\".\"flight_count\",\n (\n \"t3\".\"flight_count\" / \"t3\".\"__bsl_totals__flight_count\"\n ) * 100 AS \"market_share\"\nFROM (\n SELECT\n \"t2\".\"carrier\",\n COUNT(*) AS \"flight_count\",\n ANY_VALUE(\"t2\".\"__bsl_totals__flight_count\") AS \"__bsl_totals__flight_count\"\n FROM (\n SELECT\n \"t1\".\"origin\",\n \"t1\".\"dest\",\n \"t1\".\"carrier\",\n \"t1\".\"year\",\n \"t1\".\"distance\",\n \"t1\".\"dep_delay\",\n CAST(COUNT(*) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE) AS \"__bsl_totals__flight_count\"\n FROM (\n SELECT\n *\n FROM \"memory\".\"main\".\"flights\" AS \"t0\"\n ) AS \"t1\"\n ) AS \"t2\"\n GROUP BY\n 1\n) AS \"t3\"", "plan": "SemanticTable: flights\n origin [dim]\n destination [dim]\n year [dim]\n carrier [dim]\n total_flights [measure]\n total_distance [measure]\n max_delay [measure]\n avg_distance [measure]\n flight_count [measure]\n avg_distance_per_flight [calc]\n market_share [calc]\n-> GroupBy(carrier)\n-> Aggregate(flight_count, market_share)", "table": { "columns": [ @@ -77,12 +243,12 @@ ], "data": [ [ - "DL", + "AA", 1, 33.33333333333333 ], [ - "AA", + "DL", 1, 33.33333333333333 ], @@ -92,11 +258,94 @@ 33.33333333333333 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-444a8c980d5c591c007caa61dccef4bc" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "carrier", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "carrier", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "market_share" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-444a8c980d5c591c007caa61dccef4bc": [ + { + "carrier": "AA", + "flight_count": 1, + "market_share": 33.33333333333333 + }, + { + "carrier": "UA", + "flight_count": 1, + "market_share": 33.33333333333333 + }, + { + "carrier": "DL", + "flight_count": 1, + "market_share": 33.33333333333333 + } + ] + } + } } }, "graph_traversal": { - "code": "from boring_semantic_layer import graph_predecessors, graph_successors\n\ngraph = flights_with_deps.get_graph()\n\n# What does this field depend on? (predecessors)\ngraph_predecessors(graph, 'avg_distance_per_flight')\n# {'total_distance', 'flight_count'}\n\n# What depends on this field? (successors)\ngraph_successors(graph, 'total_distance')\n# {'avg_distance_per_flight'}", - "sql": "WITH \"t1\" AS (\n SELECT\n *\n FROM \"memory\".\"main\".\"flights\" AS \"t0\"\n)\nSELECT\n \"t7\".\"carrier\",\n \"t7\".\"flight_count\",\n (\n CAST(\"t7\".\"flight_count\" AS DOUBLE) / CAST(\"t7\".\"flight_count_right\" AS DOUBLE)\n ) * 100 AS \"market_share\"\nFROM (\n SELECT\n \"t5\".\"carrier\",\n \"t5\".\"flight_count\",\n \"t6\".\"flight_count\" AS \"flight_count_right\"\n FROM (\n SELECT\n \"t2\".\"carrier\",\n COUNT(*) AS \"flight_count\"\n FROM \"t1\" AS \"t2\"\n GROUP BY\n 1\n ) AS \"t5\"\n CROSS JOIN (\n SELECT\n COUNT(*) AS \"flight_count\"\n FROM \"t1\" AS \"t2\"\n ) AS \"t6\"\n) AS \"t7\"", + "code": "from boring_semantic_layer.graph_utils import graph_predecessors, graph_successors\n\ngraph = flights_with_deps.get_graph()\n\n# What does this field depend on? (predecessors)\ngraph_predecessors(graph, 'avg_distance_per_flight')\n# {'total_distance', 'flight_count'}\n\n# What depends on this field? (successors)\ngraph_successors(graph, 'total_distance')\n# {'avg_distance_per_flight'}", + "sql": "SELECT\n \"t3\".\"carrier\",\n \"t3\".\"flight_count\",\n (\n \"t3\".\"flight_count\" / \"t3\".\"__bsl_totals__flight_count\"\n ) * 100 AS \"market_share\"\nFROM (\n SELECT\n \"t2\".\"carrier\",\n COUNT(*) AS \"flight_count\",\n ANY_VALUE(\"t2\".\"__bsl_totals__flight_count\") AS \"__bsl_totals__flight_count\"\n FROM (\n SELECT\n \"t1\".\"origin\",\n \"t1\".\"dest\",\n \"t1\".\"carrier\",\n \"t1\".\"year\",\n \"t1\".\"distance\",\n \"t1\".\"dep_delay\",\n CAST(COUNT(*) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE) AS \"__bsl_totals__flight_count\"\n FROM (\n SELECT\n *\n FROM \"memory\".\"main\".\"flights\" AS \"t0\"\n ) AS \"t1\"\n ) AS \"t2\"\n GROUP BY\n 1\n) AS \"t3\"", "plan": "SemanticTable: flights\n origin [dim]\n destination [dim]\n year [dim]\n carrier [dim]\n total_flights [measure]\n total_distance [measure]\n max_delay [measure]\n avg_distance [measure]\n flight_count [measure]\n avg_distance_per_flight [calc]\n market_share [calc]\n-> GroupBy(carrier)\n-> Aggregate(flight_count, market_share)", "table": { "columns": [ @@ -111,21 +360,104 @@ 33.33333333333333 ], [ - "DL", + "UA", 1, 33.33333333333333 ], [ - "UA", + "DL", 1, 33.33333333333333 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-444a8c980d5c591c007caa61dccef4bc" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "carrier", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "carrier", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "market_share" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-444a8c980d5c591c007caa61dccef4bc": [ + { + "carrier": "AA", + "flight_count": 1, + "market_share": 33.33333333333333 + }, + { + "carrier": "UA", + "flight_count": 1, + "market_share": 33.33333333333333 + }, + { + "carrier": "DL", + "flight_count": 1, + "market_share": 33.33333333333333 + } + ] + } + } } }, "setup_carriers": { "code": "import ibis\nfrom boring_semantic_layer import to_semantic_table\n\ncon = ibis.duckdb.connect(\":memory:\")\n\n# Create carriers data\ncarriers_data = ibis.memtable({\n \"code\": [\"AA\", \"UA\", \"DL\"],\n \"name\": [\"American Airlines\", \"United Airlines\", \"Delta Air Lines\"]\n})\ncarriers_tbl = con.create_table(\"carriers\", carriers_data)", - "sql": "WITH \"t1\" AS (\n SELECT\n *\n FROM \"memory\".\"main\".\"flights\" AS \"t0\"\n)\nSELECT\n \"t7\".\"carrier\",\n \"t7\".\"flight_count\",\n (\n CAST(\"t7\".\"flight_count\" AS DOUBLE) / CAST(\"t7\".\"flight_count_right\" AS DOUBLE)\n ) * 100 AS \"market_share\"\nFROM (\n SELECT\n \"t5\".\"carrier\",\n \"t5\".\"flight_count\",\n \"t6\".\"flight_count\" AS \"flight_count_right\"\n FROM (\n SELECT\n \"t2\".\"carrier\",\n COUNT(*) AS \"flight_count\"\n FROM \"t1\" AS \"t2\"\n GROUP BY\n 1\n ) AS \"t5\"\n CROSS JOIN (\n SELECT\n COUNT(*) AS \"flight_count\"\n FROM \"t1\" AS \"t2\"\n ) AS \"t6\"\n) AS \"t7\"", + "sql": "SELECT\n \"t3\".\"carrier\",\n \"t3\".\"flight_count\",\n (\n \"t3\".\"flight_count\" / \"t3\".\"__bsl_totals__flight_count\"\n ) * 100 AS \"market_share\"\nFROM (\n SELECT\n \"t2\".\"carrier\",\n COUNT(*) AS \"flight_count\",\n ANY_VALUE(\"t2\".\"__bsl_totals__flight_count\") AS \"__bsl_totals__flight_count\"\n FROM (\n SELECT\n \"t1\".\"origin\",\n \"t1\".\"dest\",\n \"t1\".\"carrier\",\n \"t1\".\"year\",\n \"t1\".\"distance\",\n \"t1\".\"dep_delay\",\n CAST(COUNT(*) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE) AS \"__bsl_totals__flight_count\"\n FROM (\n SELECT\n *\n FROM \"memory\".\"main\".\"flights\" AS \"t0\"\n ) AS \"t1\"\n ) AS \"t2\"\n GROUP BY\n 1\n) AS \"t3\"", "plan": "SemanticTable: flights\n origin [dim]\n destination [dim]\n year [dim]\n carrier [dim]\n total_flights [measure]\n total_distance [measure]\n max_delay [measure]\n avg_distance [measure]\n flight_count [measure]\n avg_distance_per_flight [calc]\n market_share [calc]\n-> GroupBy(carrier)\n-> Aggregate(flight_count, market_share)", "table": { "columns": [ @@ -150,11 +482,94 @@ 33.33333333333333 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-444a8c980d5c591c007caa61dccef4bc" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "carrier", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "carrier", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "market_share" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-444a8c980d5c591c007caa61dccef4bc": [ + { + "carrier": "AA", + "flight_count": 1, + "market_share": 33.33333333333333 + }, + { + "carrier": "UA", + "flight_count": 1, + "market_share": 33.33333333333333 + }, + { + "carrier": "DL", + "flight_count": 1, + "market_share": 33.33333333333333 + } + ] + } + } } }, "join_demo": { "code": "# One carrier row can match many flight rows\ncarriers_with_flights = carriers.join_many(\n flights_st,\n lambda c, f: c.code == f.carrier\n)\n\n# Inspect available dimensions and measures\ncarriers_with_flights.dimensions", - "sql": "WITH \"t1\" AS (\n SELECT\n *\n FROM \"memory\".\"main\".\"flights\" AS \"t0\"\n)\nSELECT\n \"t7\".\"carrier\",\n \"t7\".\"flight_count\",\n (\n CAST(\"t7\".\"flight_count\" AS DOUBLE) / CAST(\"t7\".\"flight_count_right\" AS DOUBLE)\n ) * 100 AS \"market_share\"\nFROM (\n SELECT\n \"t5\".\"carrier\",\n \"t5\".\"flight_count\",\n \"t6\".\"flight_count\" AS \"flight_count_right\"\n FROM (\n SELECT\n \"t2\".\"carrier\",\n COUNT(*) AS \"flight_count\"\n FROM \"t1\" AS \"t2\"\n GROUP BY\n 1\n ) AS \"t5\"\n CROSS JOIN (\n SELECT\n COUNT(*) AS \"flight_count\"\n FROM \"t1\" AS \"t2\"\n ) AS \"t6\"\n) AS \"t7\"", + "sql": "SELECT\n \"t3\".\"carrier\",\n \"t3\".\"flight_count\",\n (\n \"t3\".\"flight_count\" / \"t3\".\"__bsl_totals__flight_count\"\n ) * 100 AS \"market_share\"\nFROM (\n SELECT\n \"t2\".\"carrier\",\n COUNT(*) AS \"flight_count\",\n ANY_VALUE(\"t2\".\"__bsl_totals__flight_count\") AS \"__bsl_totals__flight_count\"\n FROM (\n SELECT\n \"t1\".\"origin\",\n \"t1\".\"dest\",\n \"t1\".\"carrier\",\n \"t1\".\"year\",\n \"t1\".\"distance\",\n \"t1\".\"dep_delay\",\n CAST(COUNT(*) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE) AS \"__bsl_totals__flight_count\"\n FROM (\n SELECT\n *\n FROM \"memory\".\"main\".\"flights\" AS \"t0\"\n ) AS \"t1\"\n ) AS \"t2\"\n GROUP BY\n 1\n) AS \"t3\"", "plan": "SemanticTable: flights\n origin [dim]\n destination [dim]\n year [dim]\n carrier [dim]\n total_flights [measure]\n total_distance [measure]\n max_delay [measure]\n avg_distance [measure]\n flight_count [measure]\n avg_distance_per_flight [calc]\n market_share [calc]\n-> GroupBy(carrier)\n-> Aggregate(flight_count, market_share)", "table": { "columns": [ @@ -164,21 +579,104 @@ ], "data": [ [ - "DL", + "AA", 1, 33.33333333333333 ], [ - "AA", + "UA", 1, 33.33333333333333 ], [ - "UA", + "DL", 1, 33.33333333333333 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-02b3c9107cbad36a10c64f9097bccef8" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "carrier", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "carrier", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "market_share" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-02b3c9107cbad36a10c64f9097bccef8": [ + { + "carrier": "UA", + "flight_count": 1, + "market_share": 33.33333333333333 + }, + { + "carrier": "DL", + "flight_count": 1, + "market_share": 33.33333333333333 + }, + { + "carrier": "AA", + "flight_count": 1, + "market_share": 33.33333333333333 + } + ] + } + } } } }, diff --git a/docs/web/public/bsl-data/sessionized.json b/docs/web/public/bsl-data/sessionized.json index 3c3124d6..e822f8cb 100644 --- a/docs/web/public/bsl-data/sessionized.json +++ b/docs/web/public/bsl-data/sessionized.json @@ -90,8 +90,8 @@ }, "query_session_boundaries": { "code": "from ibis import _\n\nresult = (\n activity_st\n .group_by(\"user_id\", \"minute_offset\", \"page_url\", \"action\")\n .aggregate()\n .mutate(\n # Calculate time since previous event for same user\n prev_minute=lambda t: t.minute_offset.lag().over(\n group_by=\"user_id\",\n order_by=t.minute_offset\n ),\n # Calculate minutes since last event\n minutes_since_last=lambda t: t.minute_offset - t.prev_minute,\n # Mark session start (>30 min gap or first event)\n is_session_start=lambda t: (t.minutes_since_last > 30) | t.prev_minute.isnull()\n )\n .order_by(_.user_id, _.minute_offset)\n)", - "sql": "SELECT\n \"t4\".\"user_id\",\n \"t4\".\"minute_offset\",\n \"t4\".\"page_url\",\n \"t4\".\"action\",\n \"t4\".\"prev_minute\",\n \"t4\".\"minute_offset\" - \"t4\".\"prev_minute\" AS \"minutes_since_last\",\n (\n (\n \"t4\".\"minute_offset\" - \"t4\".\"prev_minute\"\n ) > 30\n )\n OR (\n \"t4\".\"prev_minute\" IS NULL\n ) AS \"is_session_start\"\nFROM (\n SELECT\n \"t3\".\"user_id\",\n \"t3\".\"minute_offset\",\n \"t3\".\"page_url\",\n \"t3\".\"action\",\n LAG(\"t3\".\"minute_offset\") OVER (\n PARTITION BY \"t3\".\"user_id\"\n ORDER BY \"t3\".\"minute_offset\" ASC\n ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING\n ) AS \"prev_minute\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"page_url\",\n \"t1\".\"action\"\n FROM (\n SELECT\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"page_url\",\n \"t1\".\"action\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_wmd3uzq7rzcljgvxljaofcdisi\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"page_url\",\n \"t1\".\"action\"\n ) AS \"t2\"\n ) AS \"t3\"\n) AS \"t4\"\nORDER BY\n \"t4\".\"user_id\" ASC,\n \"t4\".\"minute_offset\" ASC", - "plan": "SemanticTable: activity\n user_id [dim]\n minute_offset [dim]\n page_url [dim]\n action [dim]\n event_count [measure]\n unique_users [measure]\n-> GroupBy(user_id, minute_offset, page_url, action)\n-> Aggregate()\n-> Mutate(prev_minute, minutes_since_last, is_session_start)\n-> OrderBy(_CallableWrapper(_fn=_.user_id), _CallableWrapper(_fn=_.minute_offset))", + "sql": "SELECT\n \"t3\".\"user_id\",\n \"t3\".\"minute_offset\",\n \"t3\".\"page_url\",\n \"t3\".\"action\",\n \"t3\".\"prev_minute\",\n \"t3\".\"minute_offset\" - \"t3\".\"prev_minute\" AS \"minutes_since_last\",\n (\n (\n \"t3\".\"minute_offset\" - \"t3\".\"prev_minute\"\n ) > 30\n )\n OR (\n \"t3\".\"prev_minute\" IS NULL\n ) AS \"is_session_start\"\nFROM (\n SELECT\n \"t2\".\"user_id\",\n \"t2\".\"minute_offset\",\n \"t2\".\"page_url\",\n \"t2\".\"action\",\n LAG(\"t2\".\"minute_offset\") OVER (PARTITION BY \"t2\".\"user_id\" ORDER BY \"t2\".\"minute_offset\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"prev_minute\"\n FROM (\n SELECT\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"page_url\",\n \"t1\".\"action\"\n FROM (\n SELECT\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"page_url\",\n \"t1\".\"action\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_qhh3qiikzrcjpen5sb5shcf3xi\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"page_url\",\n \"t1\".\"action\"\n ) AS \"t2\"\n) AS \"t3\"\nORDER BY\n \"t3\".\"user_id\" ASC,\n \"t3\".\"minute_offset\" ASC", + "plan": "SemanticTable: activity\n user_id [dim]\n minute_offset [dim]\n page_url [dim]\n action [dim]\n event_count [measure]\n unique_users [measure]\n-> GroupBy(user_id, minute_offset, page_url, action)\n-> Aggregate(prev_minute, minutes_since_last, is_session_start)\n-> OrderBy(_CallableWrapper(_fn=_.user_id), _CallableWrapper(_fn=_.minute_offset))", "table": { "columns": [ "user_id", @@ -212,12 +212,149 @@ false ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-0e1714e04bd51403c86fef9e1144b305" + }, + "mark": { + "type": "text" + }, + "encoding": { + "text": { + "value": "Complex query - consider custom visualization" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-0e1714e04bd51403c86fef9e1144b305": [ + { + "user_id": "user1", + "minute_offset": 0, + "page_url": "/home", + "action": "view", + "prev_minute": null, + "minutes_since_last": null, + "is_session_start": true + }, + { + "user_id": "user1", + "minute_offset": 5, + "page_url": "/products", + "action": "view", + "prev_minute": 0.0, + "minutes_since_last": 5.0, + "is_session_start": false + }, + { + "user_id": "user1", + "minute_offset": 10, + "page_url": "/cart", + "action": "view", + "prev_minute": 5.0, + "minutes_since_last": 5.0, + "is_session_start": false + }, + { + "user_id": "user1", + "minute_offset": 45, + "page_url": "/checkout", + "action": "purchase", + "prev_minute": 10.0, + "minutes_since_last": 35.0, + "is_session_start": true + }, + { + "user_id": "user2", + "minute_offset": 2, + "page_url": "/home", + "action": "view", + "prev_minute": null, + "minutes_since_last": null, + "is_session_start": true + }, + { + "user_id": "user2", + "minute_offset": 40, + "page_url": "/products", + "action": "view", + "prev_minute": 2.0, + "minutes_since_last": 38.0, + "is_session_start": true + }, + { + "user_id": "user2", + "minute_offset": 42, + "page_url": "/cart", + "action": "view", + "prev_minute": 40.0, + "minutes_since_last": 2.0, + "is_session_start": false + }, + { + "user_id": "user3", + "minute_offset": 1, + "page_url": "/home", + "action": "view", + "prev_minute": null, + "minutes_since_last": null, + "is_session_start": true + }, + { + "user_id": "user3", + "minute_offset": 3, + "page_url": "/about", + "action": "view", + "prev_minute": 1.0, + "minutes_since_last": 2.0, + "is_session_start": false + }, + { + "user_id": "user3", + "minute_offset": 7, + "page_url": "/products", + "action": "view", + "prev_minute": 3.0, + "minutes_since_last": 4.0, + "is_session_start": false + }, + { + "user_id": "user3", + "minute_offset": 50, + "page_url": "/home", + "action": "view", + "prev_minute": 7.0, + "minutes_since_last": 43.0, + "is_session_start": true + }, + { + "user_id": "user3", + "minute_offset": 52, + "page_url": "/contact", + "action": "view", + "prev_minute": 50.0, + "minutes_since_last": 2.0, + "is_session_start": false + } + ] + } + } } }, "query_with_session_ids": { "code": "from ibis import _\n\nresult = (\n activity_st\n .group_by(\"user_id\", \"minute_offset\", \"page_url\", \"action\")\n .aggregate()\n .mutate(\n prev_minute=lambda t: t.minute_offset.lag().over(\n group_by=\"user_id\",\n order_by=t.minute_offset\n ),\n minutes_since_last=lambda t: t.minute_offset - t.prev_minute,\n is_session_start=lambda t: (t.minutes_since_last > 30) | t.prev_minute.isnull(),\n # Cumulative sum of session starts gives session ID\n session_id=lambda t: t.is_session_start.cast(\"int32\").sum().over(\n group_by=\"user_id\",\n order_by=t.minute_offset,\n rows=(None, 0) # Cumulative sum\n )\n )\n .order_by(_.user_id, _.minute_offset)\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t5\".\"user_id\",\n \"t5\".\"minute_offset\",\n \"t5\".\"page_url\",\n \"t5\".\"action\",\n \"t5\".\"prev_minute\",\n \"t5\".\"minutes_since_last\",\n \"t5\".\"is_session_start\",\n SUM(CAST(\"t5\".\"is_session_start\" AS INT)) OVER (\n PARTITION BY \"t5\".\"user_id\"\n ORDER BY \"t5\".\"minute_offset\" ASC\n ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW\n ) AS \"session_id\"\n FROM (\n SELECT\n \"t4\".\"user_id\",\n \"t4\".\"minute_offset\",\n \"t4\".\"page_url\",\n \"t4\".\"action\",\n \"t4\".\"prev_minute\",\n \"t4\".\"minute_offset\" - \"t4\".\"prev_minute\" AS \"minutes_since_last\",\n (\n (\n \"t4\".\"minute_offset\" - \"t4\".\"prev_minute\"\n ) > 30\n )\n OR (\n \"t4\".\"prev_minute\" IS NULL\n ) AS \"is_session_start\"\n FROM (\n SELECT\n \"t3\".\"user_id\",\n \"t3\".\"minute_offset\",\n \"t3\".\"page_url\",\n \"t3\".\"action\",\n LAG(\"t3\".\"minute_offset\") OVER (\n PARTITION BY \"t3\".\"user_id\"\n ORDER BY \"t3\".\"minute_offset\" ASC\n ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING\n ) AS \"prev_minute\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"page_url\",\n \"t1\".\"action\"\n FROM (\n SELECT\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"page_url\",\n \"t1\".\"action\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_wmd3uzq7rzcljgvxljaofcdisi\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"page_url\",\n \"t1\".\"action\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS \"t4\"\n ) AS \"t5\"\n) AS \"t6\"\nORDER BY\n \"t6\".\"user_id\" ASC,\n \"t6\".\"minute_offset\" ASC", - "plan": "SemanticTable: activity\n user_id [dim]\n minute_offset [dim]\n page_url [dim]\n action [dim]\n event_count [measure]\n unique_users [measure]\n-> GroupBy(user_id, minute_offset, page_url, action)\n-> Aggregate()\n-> Mutate(prev_minute, minutes_since_last, is_session_start, session_id)\n-> OrderBy(_CallableWrapper(_fn=_.user_id), _CallableWrapper(_fn=_.minute_offset))", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t4\".\"user_id\",\n \"t4\".\"minute_offset\",\n \"t4\".\"page_url\",\n \"t4\".\"action\",\n \"t4\".\"prev_minute\",\n \"t4\".\"minutes_since_last\",\n \"t4\".\"is_session_start\",\n SUM(CAST(\"t4\".\"is_session_start\" AS INT)) OVER (PARTITION BY \"t4\".\"user_id\" ORDER BY \"t4\".\"minute_offset\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"session_id\"\n FROM (\n SELECT\n \"t3\".\"user_id\",\n \"t3\".\"minute_offset\",\n \"t3\".\"page_url\",\n \"t3\".\"action\",\n \"t3\".\"prev_minute\",\n \"t3\".\"minute_offset\" - \"t3\".\"prev_minute\" AS \"minutes_since_last\",\n (\n (\n \"t3\".\"minute_offset\" - \"t3\".\"prev_minute\"\n ) > 30\n )\n OR (\n \"t3\".\"prev_minute\" IS NULL\n ) AS \"is_session_start\"\n FROM (\n SELECT\n \"t2\".\"user_id\",\n \"t2\".\"minute_offset\",\n \"t2\".\"page_url\",\n \"t2\".\"action\",\n LAG(\"t2\".\"minute_offset\") OVER (PARTITION BY \"t2\".\"user_id\" ORDER BY \"t2\".\"minute_offset\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"prev_minute\"\n FROM (\n SELECT\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"page_url\",\n \"t1\".\"action\"\n FROM (\n SELECT\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"page_url\",\n \"t1\".\"action\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_qhh3qiikzrcjpen5sb5shcf3xi\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"page_url\",\n \"t1\".\"action\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS \"t4\"\n) AS \"t5\"\nORDER BY\n \"t5\".\"user_id\" ASC,\n \"t5\".\"minute_offset\" ASC", + "plan": "SemanticTable: activity\n user_id [dim]\n minute_offset [dim]\n page_url [dim]\n action [dim]\n event_count [measure]\n unique_users [measure]\n-> GroupBy(user_id, minute_offset, page_url, action)\n-> Aggregate(prev_minute, minutes_since_last, is_session_start, session_id)\n-> OrderBy(_CallableWrapper(_fn=_.user_id), _CallableWrapper(_fn=_.minute_offset))", "table": { "columns": [ "user_id", @@ -351,12 +488,161 @@ 2 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-90b550172ff2549ec2dee72cb37f4b7c" + }, + "mark": { + "type": "text" + }, + "encoding": { + "text": { + "value": "Complex query - consider custom visualization" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-90b550172ff2549ec2dee72cb37f4b7c": [ + { + "user_id": "user1", + "minute_offset": 0, + "page_url": "/home", + "action": "view", + "prev_minute": null, + "minutes_since_last": null, + "is_session_start": true, + "session_id": 1 + }, + { + "user_id": "user1", + "minute_offset": 5, + "page_url": "/products", + "action": "view", + "prev_minute": 0.0, + "minutes_since_last": 5.0, + "is_session_start": false, + "session_id": 1 + }, + { + "user_id": "user1", + "minute_offset": 10, + "page_url": "/cart", + "action": "view", + "prev_minute": 5.0, + "minutes_since_last": 5.0, + "is_session_start": false, + "session_id": 1 + }, + { + "user_id": "user1", + "minute_offset": 45, + "page_url": "/checkout", + "action": "purchase", + "prev_minute": 10.0, + "minutes_since_last": 35.0, + "is_session_start": true, + "session_id": 2 + }, + { + "user_id": "user2", + "minute_offset": 2, + "page_url": "/home", + "action": "view", + "prev_minute": null, + "minutes_since_last": null, + "is_session_start": true, + "session_id": 1 + }, + { + "user_id": "user2", + "minute_offset": 40, + "page_url": "/products", + "action": "view", + "prev_minute": 2.0, + "minutes_since_last": 38.0, + "is_session_start": true, + "session_id": 2 + }, + { + "user_id": "user2", + "minute_offset": 42, + "page_url": "/cart", + "action": "view", + "prev_minute": 40.0, + "minutes_since_last": 2.0, + "is_session_start": false, + "session_id": 2 + }, + { + "user_id": "user3", + "minute_offset": 1, + "page_url": "/home", + "action": "view", + "prev_minute": null, + "minutes_since_last": null, + "is_session_start": true, + "session_id": 1 + }, + { + "user_id": "user3", + "minute_offset": 3, + "page_url": "/about", + "action": "view", + "prev_minute": 1.0, + "minutes_since_last": 2.0, + "is_session_start": false, + "session_id": 1 + }, + { + "user_id": "user3", + "minute_offset": 7, + "page_url": "/products", + "action": "view", + "prev_minute": 3.0, + "minutes_since_last": 4.0, + "is_session_start": false, + "session_id": 1 + }, + { + "user_id": "user3", + "minute_offset": 50, + "page_url": "/home", + "action": "view", + "prev_minute": 7.0, + "minutes_since_last": 43.0, + "is_session_start": true, + "session_id": 2 + }, + { + "user_id": "user3", + "minute_offset": 52, + "page_url": "/contact", + "action": "view", + "prev_minute": 50.0, + "minutes_since_last": 2.0, + "is_session_start": false, + "session_id": 2 + } + ] + } + } } }, "query_session_metrics": { "code": "from ibis import _\n\nresult = (\n activity_st\n .group_by(\"user_id\", \"minute_offset\", \"action\")\n .aggregate()\n .mutate(\n prev_minute=lambda t: t.minute_offset.lag().over(\n group_by=\"user_id\",\n order_by=t.minute_offset\n ),\n minutes_since_last=lambda t: t.minute_offset - t.prev_minute,\n is_session_start=lambda t: (t.minutes_since_last > 30) | t.prev_minute.isnull(),\n session_id=lambda t: t.is_session_start.cast(\"int32\").sum().over(\n group_by=\"user_id\",\n order_by=t.minute_offset,\n rows=(None, 0)\n )\n )\n .group_by(\"user_id\", \"session_id\")\n .aggregate(\n events_in_session=lambda t: t.count(),\n session_start_min=lambda t: t.minute_offset.min(),\n session_end_min=lambda t: t.minute_offset.max(),\n has_purchase=lambda t: (t.action == \"purchase\").any()\n )\n .mutate(\n session_duration_min=lambda t: (t.session_end_min - t.session_start_min)\n )\n .order_by(_.user_id, _.session_id)\n)", - "sql": "SELECT\n \"t8\".\"user_id\",\n \"t8\".\"session_id\",\n \"t8\".\"events_in_session\",\n \"t8\".\"session_start_min\",\n \"t8\".\"session_end_min\",\n \"t8\".\"has_purchase\",\n \"t8\".\"session_end_min\" - \"t8\".\"session_start_min\" AS \"session_duration_min\"\nFROM (\n SELECT\n \"t7\".\"user_id\",\n \"t7\".\"session_id\",\n COUNT(*) AS \"events_in_session\",\n MIN(\"t7\".\"minute_offset\") AS \"session_start_min\",\n MAX(\"t7\".\"minute_offset\") AS \"session_end_min\",\n BOOL_OR(\"t7\".\"action\" = 'purchase') AS \"has_purchase\"\n FROM (\n SELECT\n \"t7\".\"minute_offset\",\n \"t7\".\"action\",\n \"t7\".\"prev_minute\",\n \"t7\".\"minutes_since_last\",\n \"t7\".\"is_session_start\",\n \"t7\".\"user_id\",\n \"t7\".\"session_id\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t5\".\"user_id\",\n \"t5\".\"minute_offset\",\n \"t5\".\"action\",\n \"t5\".\"prev_minute\",\n \"t5\".\"minutes_since_last\",\n \"t5\".\"is_session_start\",\n SUM(CAST(\"t5\".\"is_session_start\" AS INT)) OVER (\n PARTITION BY \"t5\".\"user_id\"\n ORDER BY \"t5\".\"minute_offset\" ASC\n ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW\n ) AS \"session_id\"\n FROM (\n SELECT\n \"t4\".\"user_id\",\n \"t4\".\"minute_offset\",\n \"t4\".\"action\",\n \"t4\".\"prev_minute\",\n \"t4\".\"minute_offset\" - \"t4\".\"prev_minute\" AS \"minutes_since_last\",\n (\n (\n \"t4\".\"minute_offset\" - \"t4\".\"prev_minute\"\n ) > 30\n )\n OR (\n \"t4\".\"prev_minute\" IS NULL\n ) AS \"is_session_start\"\n FROM (\n SELECT\n \"t3\".\"user_id\",\n \"t3\".\"minute_offset\",\n \"t3\".\"action\",\n LAG(\"t3\".\"minute_offset\") OVER (\n PARTITION BY \"t3\".\"user_id\"\n ORDER BY \"t3\".\"minute_offset\" ASC\n ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING\n ) AS \"prev_minute\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"action\"\n FROM (\n SELECT\n \"t1\".\"page_url\",\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"action\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_wmd3uzq7rzcljgvxljaofcdisi\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"action\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS \"t4\"\n ) AS \"t5\"\n ) AS \"t6\"\n ) AS \"t7\"\n ) AS t7\n GROUP BY\n \"t7\".\"user_id\",\n \"t7\".\"session_id\"\n) AS \"t8\"\nORDER BY\n \"t8\".\"user_id\" ASC,\n \"t8\".\"session_id\" ASC", - "plan": "SemanticTable: activity\n user_id [dim]\n minute_offset [dim]\n page_url [dim]\n action [dim]\n event_count [measure]\n unique_users [measure]\n-> GroupBy(user_id, minute_offset, action)\n-> Aggregate()\n-> Mutate(prev_minute, minutes_since_last, is_session_start, session_id)\n-> GroupBy(user_id, session_id)\n-> Aggregate(events_in_session, session_start_min, session_end_min, has_purchase)\n-> Mutate(session_duration_min)\n-> OrderBy(_CallableWrapper(_fn=_.user_id), _CallableWrapper(_fn=_.session_id))", + "sql": "SELECT\n \"t7\".\"user_id\",\n \"t7\".\"session_id\",\n \"t7\".\"events_in_session\",\n \"t7\".\"session_start_min\",\n \"t7\".\"session_end_min\",\n \"t7\".\"has_purchase\",\n \"t7\".\"session_end_min\" - \"t7\".\"session_start_min\" AS \"session_duration_min\"\nFROM (\n SELECT\n \"t6\".\"user_id\",\n \"t6\".\"session_id\",\n COUNT(*) AS \"events_in_session\",\n MIN(\"t6\".\"minute_offset\") AS \"session_start_min\",\n MAX(\"t6\".\"minute_offset\") AS \"session_end_min\",\n BOOL_OR(\"t6\".\"action\" = 'purchase') AS \"has_purchase\"\n FROM (\n SELECT\n \"t6\".\"minute_offset\",\n \"t6\".\"action\",\n \"t6\".\"prev_minute\",\n \"t6\".\"minutes_since_last\",\n \"t6\".\"is_session_start\",\n \"t6\".\"user_id\",\n \"t6\".\"session_id\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t4\".\"user_id\",\n \"t4\".\"minute_offset\",\n \"t4\".\"action\",\n \"t4\".\"prev_minute\",\n \"t4\".\"minutes_since_last\",\n \"t4\".\"is_session_start\",\n SUM(CAST(\"t4\".\"is_session_start\" AS INT)) OVER (PARTITION BY \"t4\".\"user_id\" ORDER BY \"t4\".\"minute_offset\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"session_id\"\n FROM (\n SELECT\n \"t3\".\"user_id\",\n \"t3\".\"minute_offset\",\n \"t3\".\"action\",\n \"t3\".\"prev_minute\",\n \"t3\".\"minute_offset\" - \"t3\".\"prev_minute\" AS \"minutes_since_last\",\n (\n (\n \"t3\".\"minute_offset\" - \"t3\".\"prev_minute\"\n ) > 30\n )\n OR (\n \"t3\".\"prev_minute\" IS NULL\n ) AS \"is_session_start\"\n FROM (\n SELECT\n \"t2\".\"user_id\",\n \"t2\".\"minute_offset\",\n \"t2\".\"action\",\n LAG(\"t2\".\"minute_offset\") OVER (PARTITION BY \"t2\".\"user_id\" ORDER BY \"t2\".\"minute_offset\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"prev_minute\"\n FROM (\n SELECT\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"action\"\n FROM (\n SELECT\n \"t1\".\"page_url\",\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"action\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_qhh3qiikzrcjpen5sb5shcf3xi\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"action\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS \"t4\"\n ) AS \"t5\"\n ) AS \"t6\"\n ) AS t6\n GROUP BY\n \"t6\".\"user_id\",\n \"t6\".\"session_id\"\n) AS \"t7\"\nORDER BY\n \"t7\".\"user_id\" ASC,\n \"t7\".\"session_id\" ASC", + "plan": "SemanticTable: activity\n user_id [dim]\n minute_offset [dim]\n page_url [dim]\n action [dim]\n event_count [measure]\n unique_users [measure]\n-> GroupBy(user_id, minute_offset, action)\n-> Aggregate(prev_minute, minutes_since_last, is_session_start, session_id)\n-> GroupBy(user_id, session_id)\n-> Aggregate(events_in_session, session_start_min, session_end_min, has_purchase, session_duration_min)\n-> OrderBy(_CallableWrapper(_fn=_.user_id), _CallableWrapper(_fn=_.session_id))", "table": { "columns": [ "user_id", @@ -423,12 +709,95 @@ 2 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-07d4f06d2c07efd1ca130dd2fb6400bd" + }, + "mark": { + "type": "text" + }, + "encoding": { + "text": { + "value": "Complex query - consider custom visualization" + } + }, + "height": 400, + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-07d4f06d2c07efd1ca130dd2fb6400bd": [ + { + "user_id": "user1", + "session_id": 1, + "events_in_session": 3, + "session_start_min": 0, + "session_end_min": 10, + "has_purchase": false, + "session_duration_min": 10 + }, + { + "user_id": "user1", + "session_id": 2, + "events_in_session": 1, + "session_start_min": 45, + "session_end_min": 45, + "has_purchase": true, + "session_duration_min": 0 + }, + { + "user_id": "user2", + "session_id": 1, + "events_in_session": 1, + "session_start_min": 2, + "session_end_min": 2, + "has_purchase": false, + "session_duration_min": 0 + }, + { + "user_id": "user2", + "session_id": 2, + "events_in_session": 2, + "session_start_min": 40, + "session_end_min": 42, + "has_purchase": false, + "session_duration_min": 2 + }, + { + "user_id": "user3", + "session_id": 1, + "events_in_session": 3, + "session_start_min": 1, + "session_end_min": 7, + "has_purchase": false, + "session_duration_min": 6 + }, + { + "user_id": "user3", + "session_id": 2, + "events_in_session": 2, + "session_start_min": 50, + "session_end_min": 52, + "has_purchase": false, + "session_duration_min": 2 + } + ] + } + } } }, "query_user_summary": { "code": "from ibis import _\n\nresult = (\n activity_st\n .group_by(\"user_id\", \"minute_offset\", \"action\")\n .aggregate()\n .mutate(\n prev_minute=lambda t: t.minute_offset.lag().over(\n group_by=\"user_id\",\n order_by=t.minute_offset\n ),\n minutes_since_last=lambda t: t.minute_offset - t.prev_minute,\n is_session_start=lambda t: (t.minutes_since_last > 30) | t.prev_minute.isnull(),\n session_id=lambda t: t.is_session_start.cast(\"int32\").sum().over(\n group_by=\"user_id\",\n order_by=t.minute_offset,\n rows=(None, 0)\n )\n )\n .group_by(\"user_id\", \"session_id\")\n .aggregate(\n events_in_session=lambda t: t.count(),\n has_purchase=lambda t: (t.action == \"purchase\").any()\n )\n .group_by(\"user_id\")\n .aggregate(\n total_sessions=lambda t: t.count(),\n total_events=lambda t: t.events_in_session.sum(),\n sessions_with_purchase=lambda t: t.has_purchase.cast(\"int32\").sum(),\n avg_events_per_session=lambda t: t.events_in_session.mean().round(2)\n )\n .mutate(\n conversion_rate=lambda t: (t.sessions_with_purchase / t.total_sessions * 100).round(2)\n )\n .order_by(_.total_events.desc())\n)", - "sql": "SELECT\n \"t10\".\"user_id\",\n \"t10\".\"total_sessions\",\n \"t10\".\"total_events\",\n \"t10\".\"sessions_with_purchase\",\n \"t10\".\"avg_events_per_session\",\n ROUND(\n (\n CAST(\"t10\".\"sessions_with_purchase\" AS DOUBLE PRECISION) / \"t10\".\"total_sessions\"\n ) * 100,\n 2\n ) AS \"conversion_rate\"\nFROM (\n SELECT\n \"t9\".\"user_id\",\n COUNT(*) AS \"total_sessions\",\n SUM(\"t9\".\"events_in_session\") AS \"total_events\",\n SUM(CAST(\"t9\".\"has_purchase\" AS INT)) AS \"sessions_with_purchase\",\n ROUND(CAST(AVG(\"t9\".\"events_in_session\") AS DECIMAL), 2) AS \"avg_events_per_session\"\n FROM (\n SELECT\n \"t9\".\"session_id\",\n \"t9\".\"events_in_session\",\n \"t9\".\"has_purchase\",\n \"t9\".\"user_id\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t7\".\"user_id\",\n \"t7\".\"session_id\",\n COUNT(*) AS \"events_in_session\",\n BOOL_OR(\"t7\".\"action\" = 'purchase') AS \"has_purchase\"\n FROM (\n SELECT\n \"t7\".\"minute_offset\",\n \"t7\".\"action\",\n \"t7\".\"prev_minute\",\n \"t7\".\"minutes_since_last\",\n \"t7\".\"is_session_start\",\n \"t7\".\"user_id\",\n \"t7\".\"session_id\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t5\".\"user_id\",\n \"t5\".\"minute_offset\",\n \"t5\".\"action\",\n \"t5\".\"prev_minute\",\n \"t5\".\"minutes_since_last\",\n \"t5\".\"is_session_start\",\n SUM(CAST(\"t5\".\"is_session_start\" AS INT)) OVER (\n PARTITION BY \"t5\".\"user_id\"\n ORDER BY \"t5\".\"minute_offset\" ASC\n ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW\n ) AS \"session_id\"\n FROM (\n SELECT\n \"t4\".\"user_id\",\n \"t4\".\"minute_offset\",\n \"t4\".\"action\",\n \"t4\".\"prev_minute\",\n \"t4\".\"minute_offset\" - \"t4\".\"prev_minute\" AS \"minutes_since_last\",\n (\n (\n \"t4\".\"minute_offset\" - \"t4\".\"prev_minute\"\n ) > 30\n )\n OR (\n \"t4\".\"prev_minute\" IS NULL\n ) AS \"is_session_start\"\n FROM (\n SELECT\n \"t3\".\"user_id\",\n \"t3\".\"minute_offset\",\n \"t3\".\"action\",\n LAG(\"t3\".\"minute_offset\") OVER (\n PARTITION BY \"t3\".\"user_id\"\n ORDER BY \"t3\".\"minute_offset\" ASC\n ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING\n ) AS \"prev_minute\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"action\"\n FROM (\n SELECT\n \"t1\".\"page_url\",\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"action\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_wmd3uzq7rzcljgvxljaofcdisi\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"action\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS \"t4\"\n ) AS \"t5\"\n ) AS \"t6\"\n ) AS \"t7\"\n ) AS t7\n GROUP BY\n \"t7\".\"user_id\",\n \"t7\".\"session_id\"\n ) AS \"t8\"\n ) AS \"t9\"\n ) AS t9\n GROUP BY\n \"t9\".\"user_id\"\n) AS \"t10\"\nORDER BY\n \"t10\".\"total_events\" DESC NULLS LAST", - "plan": "SemanticTable: activity\n user_id [dim]\n minute_offset [dim]\n page_url [dim]\n action [dim]\n event_count [measure]\n unique_users [measure]\n-> GroupBy(user_id, minute_offset, action)\n-> Aggregate()\n-> Mutate(prev_minute, minutes_since_last, is_session_start, session_id)\n-> GroupBy(user_id, session_id)\n-> Aggregate(events_in_session, has_purchase)\n-> GroupBy(user_id)\n-> Aggregate(total_sessions, total_events, sessions_with_purchase, avg_events_per_session)\n-> Mutate(conversion_rate)\n-> OrderBy(_CallableWrapper(_fn=_.total_events.desc()))", + "sql": "SELECT\n \"t9\".\"user_id\",\n \"t9\".\"total_sessions\",\n \"t9\".\"total_events\",\n \"t9\".\"sessions_with_purchase\",\n \"t9\".\"avg_events_per_session\",\n ROUND(\n (\n CAST(\"t9\".\"sessions_with_purchase\" AS DOUBLE PRECISION) / \"t9\".\"total_sessions\"\n ) * 100,\n 2\n ) AS \"conversion_rate\"\nFROM (\n SELECT\n \"t8\".\"user_id\",\n COUNT(*) AS \"total_sessions\",\n SUM(\"t8\".\"events_in_session\") AS \"total_events\",\n SUM(CAST(\"t8\".\"has_purchase\" AS INT)) AS \"sessions_with_purchase\",\n ROUND(AVG(\"t8\".\"events_in_session\"), 2) AS \"avg_events_per_session\"\n FROM (\n SELECT\n \"t8\".\"session_id\",\n \"t8\".\"events_in_session\",\n \"t8\".\"has_purchase\",\n \"t8\".\"user_id\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t6\".\"user_id\",\n \"t6\".\"session_id\",\n COUNT(*) AS \"events_in_session\",\n BOOL_OR(\"t6\".\"action\" = 'purchase') AS \"has_purchase\"\n FROM (\n SELECT\n \"t6\".\"minute_offset\",\n \"t6\".\"action\",\n \"t6\".\"prev_minute\",\n \"t6\".\"minutes_since_last\",\n \"t6\".\"is_session_start\",\n \"t6\".\"user_id\",\n \"t6\".\"session_id\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t4\".\"user_id\",\n \"t4\".\"minute_offset\",\n \"t4\".\"action\",\n \"t4\".\"prev_minute\",\n \"t4\".\"minutes_since_last\",\n \"t4\".\"is_session_start\",\n SUM(CAST(\"t4\".\"is_session_start\" AS INT)) OVER (PARTITION BY \"t4\".\"user_id\" ORDER BY \"t4\".\"minute_offset\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"session_id\"\n FROM (\n SELECT\n \"t3\".\"user_id\",\n \"t3\".\"minute_offset\",\n \"t3\".\"action\",\n \"t3\".\"prev_minute\",\n \"t3\".\"minute_offset\" - \"t3\".\"prev_minute\" AS \"minutes_since_last\",\n (\n (\n \"t3\".\"minute_offset\" - \"t3\".\"prev_minute\"\n ) > 30\n )\n OR (\n \"t3\".\"prev_minute\" IS NULL\n ) AS \"is_session_start\"\n FROM (\n SELECT\n \"t2\".\"user_id\",\n \"t2\".\"minute_offset\",\n \"t2\".\"action\",\n LAG(\"t2\".\"minute_offset\") OVER (PARTITION BY \"t2\".\"user_id\" ORDER BY \"t2\".\"minute_offset\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"prev_minute\"\n FROM (\n SELECT\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"action\"\n FROM (\n SELECT\n \"t1\".\"page_url\",\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"action\"\n FROM (\n SELECT\n *\n FROM \"ibis_pandas_memtable_qhh3qiikzrcjpen5sb5shcf3xi\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"user_id\",\n \"t1\".\"minute_offset\",\n \"t1\".\"action\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS \"t4\"\n ) AS \"t5\"\n ) AS \"t6\"\n ) AS t6\n GROUP BY\n \"t6\".\"user_id\",\n \"t6\".\"session_id\"\n ) AS \"t7\"\n ) AS \"t8\"\n ) AS t8\n GROUP BY\n \"t8\".\"user_id\"\n) AS \"t9\"\nORDER BY\n \"t9\".\"total_events\" DESC NULLS LAST", + "plan": "SemanticTable: activity\n user_id [dim]\n minute_offset [dim]\n page_url [dim]\n action [dim]\n event_count [measure]\n unique_users [measure]\n-> GroupBy(user_id, minute_offset, action)\n-> Aggregate(prev_minute, minutes_since_last, is_session_start, session_id)\n-> GroupBy(user_id, session_id)\n-> Aggregate(events_in_session, has_purchase)\n-> GroupBy(user_id)\n-> Aggregate(total_sessions, total_events, sessions_with_purchase, avg_events_per_session, conversion_rate)\n-> OrderBy(_CallableWrapper(_fn=_.total_events.desc()))", "table": { "columns": [ "user_id", @@ -464,6 +833,101 @@ 0.0 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-e9f9ebd327ccbd11d15df0c271a567fd" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "user_id", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "user_id", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "total_sessions", + "total_events", + "sessions_with_purchase", + "avg_events_per_session", + "conversion_rate" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-e9f9ebd327ccbd11d15df0c271a567fd": [ + { + "user_id": "user3", + "total_sessions": 2, + "total_events": 5, + "sessions_with_purchase": 0, + "avg_events_per_session": 2.5, + "conversion_rate": 0.0 + }, + { + "user_id": "user1", + "total_sessions": 2, + "total_events": 4, + "sessions_with_purchase": 1, + "avg_events_per_session": 2.0, + "conversion_rate": 0.0 + }, + { + "user_id": "user2", + "total_sessions": 2, + "total_events": 3, + "sessions_with_purchase": 0, + "avg_events_per_session": 1.5, + "conversion_rate": 0.0 + } + ] + } + } } } }, diff --git a/docs/web/public/bsl-data/windowing.json b/docs/web/public/bsl-data/windowing.json index cc560539..87d8382b 100644 --- a/docs/web/public/bsl-data/windowing.json +++ b/docs/web/public/bsl-data/windowing.json @@ -1,5 +1,5 @@ { - "markdown": "# Window Functions\n\nPerform calculations across ordered rows using window functions like running totals, moving averages, rank, lag/lead, and more. Window functions operate on query results after aggregation, enabling powerful comparative and analytical operations.\n\n## Overview\n\nWindow functions allow you to:\n\n- **Compare rows**: Calculate differences between current and previous rows (lag/lead)\n- **Running calculations**: Compute cumulative sums and running averages\n- **Ranking**: Assign ranks, row numbers, and percentiles\n- **Moving windows**: Calculate metrics over sliding time windows\n\n\nWindow functions in BSL are applied using Ibis window operations on aggregated results. They execute logically after the aggregation stage.\n\n\n## Setup\n\nCreate a synthetic sales dataset with daily revenue data:\n\n```setup_data\nimport ibis\nfrom ibis import _\nfrom datetime import datetime, timedelta\nimport random\n\n# Create daily sales data spanning 90 days\nstart_date = datetime(2024, 1, 1)\ndates = [start_date + timedelta(days=i) for i in range(90)]\n\n# Generate synthetic revenue with upward trend and weekly patterns\nrandom.seed(42)\n\nrevenue_values = []\nfor i, date in enumerate(dates):\n # Base trend: increasing over time\n base = 1000 + (i * 10)\n\n # Weekly pattern: weekends have higher sales\n weekday_multiplier = 1.3 if date.weekday() >= 5 else 1.0\n\n # Random variation\n noise = random.uniform(-100, 100)\n\n revenue = base * weekday_multiplier + noise\n revenue_values.append(round(revenue, 2))\n\n# Create table\nsales_data = ibis.memtable({\n \"sale_date\": dates,\n \"revenue\": revenue_values,\n \"product_category\": [\"Electronics\" if i % 3 == 0 else \"Clothing\" if i % 3 == 1 else \"Home\" for i in range(90)],\n})\n```\n\n\n\n```setup_st\nfrom boring_semantic_layer import to_semantic_table\n\n# Create semantic table with measures\nsales_st = to_semantic_table(\n sales_data,\n name=\"daily_sales\"\n).with_measures(\n total_revenue=lambda t: t.revenue.sum(),\n avg_revenue=lambda t: t.revenue.mean(),\n sale_count=lambda t: t.count(),\n)\n```\n\n\n\n## Lag and Lead: Comparing to Previous/Next Rows\n\nCalculate period-over-period changes by comparing current values to previous rows:\n\n```query_lag_lead\nfrom ibis import _\n\n# Aggregate daily revenue\ndaily_revenue = (\n sales_st\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n .order_by(\"sale_date\")\n)\n\n# Add window functions for lag/lead\nresult = daily_revenue.mutate(\n prev_day_revenue=_.total_revenue.lag(),\n next_day_revenue=_.total_revenue.lead(),\n day_over_day_change=_.total_revenue - _.total_revenue.lag(),\n pct_change=((_.total_revenue - _.total_revenue.lag()) / _.total_revenue.lag() * 100).round(2)\n).limit(10)\n```\n\n\n\n\n`lag()` accesses the previous row's value, while `lead()` accesses the next row's value. The first row's lag and last row's lead will be null.\n\n\n## Running Totals: Cumulative Calculations\n\nCompute running sums to track cumulative metrics over time:\n\n```query_running_total\nfrom ibis import _\n\n# Daily revenue with cumulative total\ndaily_revenue = (\n sales_st\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n .order_by(\"sale_date\")\n)\n\n# Calculate cumulative sum and running average\nwindow_unbounded = xo.window(rows=(None, 0), order_by=\"sale_date\")\n\nresult = daily_revenue.mutate(\n cumulative_revenue=_.total_revenue.cumsum(),\n days_count=lambda t: t.count().over(window_unbounded),\n avg_daily_so_far=lambda t: (t.cumulative_revenue / t.days_count).round(2)\n).limit(10)\n```\n\n\n\n## Moving Averages: Sliding Window Calculations\n\nCalculate metrics over a rolling window of rows:\n\n```query_moving_average\nfrom ibis import _\n\n# Daily revenue\ndaily_revenue = (\n sales_st\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n .order_by(\"sale_date\")\n)\n\n# 7-day moving average\nwindow_7d = xo.window(rows=(-6, 0), order_by=\"sale_date\")\n\nresult = daily_revenue.mutate(\n ma_7day=_.total_revenue.mean().over(window_7d).round(2),\n ma_7day_sum=_.total_revenue.sum().over(window_7d).round(2),\n).limit(10)\n```\n\n\n\n\nThe window specification `rows=(-6, 0)` means \"6 rows before the current row through the current row\" (7 total rows). The moving average smooths out daily volatility.\n\n\n## Ranking: Assign Positions\n\nRank rows based on values:\n\n```query_ranking\nfrom ibis import _\n\n# Aggregate by product category\ncategory_revenue = (\n sales_st\n .group_by(\"product_category\")\n .aggregate(\"total_revenue\", \"sale_count\")\n .order_by(_.total_revenue.desc())\n)\n\n# Add rank columns\nresult = category_revenue.mutate(\n rank=lambda t: xo.rank().over(xo.window(order_by=xo.desc(t.total_revenue))),\n dense_rank=lambda t: xo.dense_rank().over(xo.window(order_by=xo.desc(t.total_revenue))),\n row_number=lambda t: xo.row_number().over(xo.window(order_by=xo.desc(t.total_revenue))),\n)\n```\n\n\n\n\n`row_number()` assigns unique sequential numbers, `rank()` assigns the same rank to ties (skipping next ranks), and `dense_rank()` assigns the same rank to ties without gaps.\n\n\n## Week-over-Week Comparison\n\nCompare metrics across weekly periods:\n\n```query_week_over_week\nfrom ibis import _\n\n# Aggregate by week\nweekly_revenue = (\n sales_st\n .mutate(week_start=_.sale_date.truncate(\"W\"))\n .group_by(\"week_start\")\n .aggregate(\"total_revenue\")\n .order_by(\"week_start\")\n)\n\n# Calculate week-over-week changes\nresult = weekly_revenue.mutate(\n prev_week_revenue=_.total_revenue.lag(),\n wow_change=_.total_revenue - _.total_revenue.lag(),\n wow_pct_change=((_.total_revenue - _.total_revenue.lag()) / _.total_revenue.lag() * 100).round(2)\n).limit(10)\n```\n\n\n\n## Percent of Running Total\n\nCalculate each row's contribution to the cumulative total:\n\n```query_pct_running\nfrom ibis import _\n\n# Top 10 days by revenue\ntop_days = (\n sales_st\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n .order_by(_.total_revenue.desc())\n .limit(10)\n)\n\n# Calculate cumulative percentage\nresult = top_days.mutate(\n cumulative_revenue=_.total_revenue.cumsum(),\n total_top10=_.total_revenue.sum(),\n pct_of_top10=(_.total_revenue.cumsum() / _.total_revenue.sum() * 100).round(2)\n)\n```\n\n\n\n## Moving Window with Filters\n\nCombine window functions with filtering for focused analysis:\n\n```query_window_filter\nfrom ibis import _\n\n# Focus on weekends only\nweekend_revenue = (\n sales_st\n .mutate(is_weekend=_.sale_date.day_of_week.index().isin([5, 6]))\n .filter(_.is_weekend)\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n .order_by(\"sale_date\")\n)\n\n# 3-weekend moving average\nwindow_3 = xo.window(rows=(-2, 0), order_by=\"sale_date\")\n\nresult = weekend_revenue.mutate(\n ma_3weekend=_.total_revenue.mean().over(window_3).round(2),\n prev_weekend=_.total_revenue.lag(),\n weekend_change=_.total_revenue - _.total_revenue.lag()\n).limit(10)\n```\n\n\n\n## Key Takeaways\n\n- **Window functions operate after aggregation**: They work on query results, not raw data\n- **Order matters**: Most window functions require `order_by()` for meaningful results\n- **Flexible windows**: Define windows by rows (`rows=(n, m)`) or ranges\n- **Common patterns**:\n - `lag()/lead()` for period-over-period comparisons\n - `cumsum()` for running totals\n - `.over(window)` for moving averages\n - `rank()`, `row_number()` for ranking\n- **Combine with filters**: Focus window calculations on specific subsets\n\n## Next Steps\n\n- Explore [Percentage of Total](/advanced/percentage-total) for ratio calculations\n- Learn about [Nested Subtotals](/advanced/nested-subtotals) for hierarchical aggregations and complex data structures\n", + "markdown": "# Window Functions\n\nPerform calculations across ordered rows using window functions like running totals, moving averages, rank, lag/lead, and more. Window functions operate on query results after aggregation, enabling powerful comparative and analytical operations.\n\n## Overview\n\nWindow functions allow you to:\n\n- **Compare rows**: Calculate differences between current and previous rows (lag/lead)\n- **Running calculations**: Compute cumulative sums and running averages\n- **Ranking**: Assign ranks, row numbers, and percentiles\n- **Moving windows**: Calculate metrics over sliding time windows\n\n\nWindow functions in BSL are applied using Ibis window operations on aggregated results. They execute logically after the aggregation stage.\n\n\n## Setup\n\nCreate a synthetic sales dataset with daily revenue data:\n\n```setup_data\nimport ibis\nfrom ibis import _\nfrom datetime import datetime, timedelta\nimport random\n\n# Create daily sales data spanning 90 days\nstart_date = datetime(2024, 1, 1)\ndates = [start_date + timedelta(days=i) for i in range(90)]\n\n# Generate synthetic revenue with upward trend and weekly patterns\nrandom.seed(42)\n\nrevenue_values = []\nfor i, date in enumerate(dates):\n # Base trend: increasing over time\n base = 1000 + (i * 10)\n\n # Weekly pattern: weekends have higher sales\n weekday_multiplier = 1.3 if date.weekday() >= 5 else 1.0\n\n # Random variation\n noise = random.uniform(-100, 100)\n\n revenue = base * weekday_multiplier + noise\n revenue_values.append(round(revenue, 2))\n\n# Create table\nsales_data = ibis.memtable({\n \"sale_date\": dates,\n \"revenue\": revenue_values,\n \"product_category\": [\"Electronics\" if i % 3 == 0 else \"Clothing\" if i % 3 == 1 else \"Home\" for i in range(90)],\n})\n```\n\n\n\n```setup_st\nfrom boring_semantic_layer import to_semantic_table\n\n# Create semantic table with measures\nsales_st = to_semantic_table(\n sales_data,\n name=\"daily_sales\"\n).with_measures(\n total_revenue=lambda t: t.revenue.sum(),\n avg_revenue=lambda t: t.revenue.mean(),\n sale_count=lambda t: t.count(),\n)\n```\n\n\n\n## Lag and Lead: Comparing to Previous/Next Rows\n\nCalculate period-over-period changes by comparing current values to previous rows:\n\n```query_lag_lead\nfrom ibis import _\n\n# Aggregate daily revenue\ndaily_revenue = (\n sales_st\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n)\n\n# Add window functions for lag/lead \u2014 applied directly on the aggregate,\n# each window carrying its own ordering\nresult = daily_revenue.mutate(\n prev_day_revenue=lambda t: t.total_revenue.lag().over(order_by=\"sale_date\"),\n next_day_revenue=lambda t: t.total_revenue.lead().over(order_by=\"sale_date\"),\n day_over_day_change=lambda t: (\n t.total_revenue - t.total_revenue.lag().over(order_by=\"sale_date\")\n ),\n pct_change=lambda t: (\n (t.total_revenue - t.total_revenue.lag().over(order_by=\"sale_date\"))\n / t.total_revenue.lag().over(order_by=\"sale_date\") * 100\n ).round(2),\n).order_by(\"sale_date\").limit(10)\n```\n\n\n\n\n`lag()` accesses the previous row's value, while `lead()` accesses the next row's value. The first row's lag and last row's lead will be null.\n\n\n## Running Totals: Cumulative Calculations\n\nCompute running sums to track cumulative metrics over time:\n\n```query_running_total\nfrom ibis import _\n\n# Daily revenue with cumulative total\ndaily_revenue = (\n sales_st\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n)\n\n# Calculate cumulative sum and running average\nwindow_unbounded = xo.window(rows=(None, 0), order_by=\"sale_date\")\n\nresult = daily_revenue.mutate(\n cumulative_revenue=lambda t: t.total_revenue.sum().over(window_unbounded),\n avg_daily_so_far=lambda t: t.total_revenue.mean().over(window_unbounded).round(2),\n).order_by(\"sale_date\").limit(10)\n```\n\n\n\n## Moving Averages: Sliding Window Calculations\n\nCalculate metrics over a rolling window of rows:\n\n```query_moving_average\nfrom ibis import _\n\n# Daily revenue\ndaily_revenue = (\n sales_st\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n)\n\n# 7-day moving average\nwindow_7d = xo.window(rows=(-6, 0), order_by=\"sale_date\")\n\nresult = daily_revenue.mutate(\n ma_7day=lambda t: t.total_revenue.mean().over(window_7d).round(2),\n ma_7day_sum=lambda t: t.total_revenue.sum().over(window_7d).round(2),\n).order_by(\"sale_date\").limit(10)\n```\n\n\n\n\nThe window specification `rows=(-6, 0)` means \"6 rows before the current row through the current row\" (7 total rows). The moving average smooths out daily volatility.\n\n\n## Ranking: Assign Positions\n\nRank rows based on values:\n\n```query_ranking\nfrom ibis import _\n\n# Aggregate by product category\ncategory_revenue = (\n sales_st\n .group_by(\"product_category\")\n .aggregate(\"total_revenue\", \"sale_count\")\n)\n\n# Add rank columns (each window carries its own ordering)\nresult = category_revenue.mutate(\n rank=lambda t: xo.rank().over(xo.window(order_by=xo.desc(t.total_revenue))),\n dense_rank=lambda t: xo.dense_rank().over(xo.window(order_by=xo.desc(t.total_revenue))),\n row_number=lambda t: xo.row_number().over(xo.window(order_by=xo.desc(t.total_revenue))),\n).order_by(_.total_revenue.desc())\n```\n\n\n\n\n`row_number()` assigns unique sequential numbers, `rank()` assigns the same rank to ties (skipping next ranks), and `dense_rank()` assigns the same rank to ties without gaps.\n\n\n## Week-over-Week Comparison\n\nCompare metrics across weekly periods:\n\n```query_week_over_week\nfrom ibis import _\n\n# Aggregate by week\nweekly_revenue = (\n sales_st\n .mutate(week_start=_.sale_date.truncate(\"W\"))\n .group_by(\"week_start\")\n .aggregate(\"total_revenue\")\n)\n\n# Calculate week-over-week changes\nresult = weekly_revenue.mutate(\n prev_week_revenue=lambda t: t.total_revenue.lag().over(order_by=\"week_start\"),\n wow_change=lambda t: t.total_revenue - t.total_revenue.lag().over(order_by=\"week_start\"),\n wow_pct_change=lambda t: (\n (t.total_revenue - t.total_revenue.lag().over(order_by=\"week_start\"))\n / t.total_revenue.lag().over(order_by=\"week_start\") * 100\n ).round(2),\n).order_by(\"week_start\").limit(10)\n```\n\n\n\n## Percent of Running Total\n\nCalculate each row's contribution to the cumulative total:\n\n```query_pct_running\nfrom ibis import _\n\n# Top 10 days by revenue\ntop_days = (\n sales_st\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n .order_by(_.total_revenue.desc())\n .limit(10)\n)\n\n# A limited query result is a plain table \u2014 row math over it drops to\n# ibis explicitly via .to_untagged()\nresult = top_days.to_untagged().mutate(\n cumulative_revenue=lambda t: t.total_revenue.cumsum(),\n total_top10=lambda t: t.total_revenue.sum(),\n pct_of_top10=lambda t: (t.total_revenue.cumsum() / t.total_revenue.sum() * 100).round(2),\n)\n```\n\n\n\n## Moving Window with Filters\n\nCombine window functions with filtering for focused analysis:\n\n```query_window_filter\nfrom ibis import _\n\n# Focus on weekends only\nweekend_revenue = (\n sales_st\n .mutate(is_weekend=_.sale_date.day_of_week.index().isin([5, 6]))\n .filter(_.is_weekend)\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n)\n\n# 3-weekend moving average\nwindow_3 = xo.window(rows=(-2, 0), order_by=\"sale_date\")\n\nresult = weekend_revenue.mutate(\n ma_3weekend=lambda t: t.total_revenue.mean().over(window_3).round(2),\n prev_weekend=lambda t: t.total_revenue.lag().over(order_by=\"sale_date\"),\n weekend_change=lambda t: t.total_revenue - t.total_revenue.lag().over(order_by=\"sale_date\"),\n).order_by(\"sale_date\").limit(10)\n```\n\n\n\n## Key Takeaways\n\n- **Window functions go on the aggregate**: apply `.mutate()` directly on the `aggregate()` result, before `.order_by()`/`.limit()` (after those the result is a plain table and `.mutate()` raises \u2014 use `.to_untagged().mutate(...)` there)\n- **Each window carries its own ordering**: pass `order_by=` inside the window (e.g. `.over(rows=(-6, 0), order_by=\"sale_date\")` or `.lag().over(order_by=\"sale_date\")`)\n- **Flexible windows**: Define windows by rows (`rows=(n, m)`) or ranges\n- **Common patterns**:\n - `lag()/lead()` for period-over-period comparisons\n - `.sum().over(rows=(None, 0), order_by=...)` for running totals\n - `.mean().over(window)` for moving averages\n - `rank()`, `row_number()` for ranking\n- **Combine with filters**: Focus window calculations on specific subsets\n\n## Next Steps\n\n- Explore [Percentage of Total](/advanced/percentage-total) for ratio calculations\n- Learn about [Nested Subtotals](/advanced/nested-subtotals) for hierarchical aggregations and complex data structures\n", "queries": { "setup_data": { "code": "import ibis\nfrom ibis import _\nfrom datetime import datetime, timedelta\nimport random\n\n# Create daily sales data spanning 90 days\nstart_date = datetime(2024, 1, 1)\ndates = [start_date + timedelta(days=i) for i in range(90)]\n\n# Generate synthetic revenue with upward trend and weekly patterns\nrandom.seed(42)\n\nrevenue_values = []\nfor i, date in enumerate(dates):\n # Base trend: increasing over time\n base = 1000 + (i * 10)\n\n # Weekly pattern: weekends have higher sales\n weekday_multiplier = 1.3 if date.weekday() >= 5 else 1.0\n\n # Random variation\n noise = random.uniform(-100, 100)\n\n revenue = base * weekday_multiplier + noise\n revenue_values.append(round(revenue, 2))\n\n# Create table\nsales_data = ibis.memtable({\n \"sale_date\": dates,\n \"revenue\": revenue_values,\n \"product_category\": [\"Electronics\" if i % 3 == 0 else \"Clothing\" if i % 3 == 1 else \"Home\" for i in range(90)],\n})", @@ -466,473 +466,14 @@ } }, "setup_st": { - "code": "from boring_semantic_layer import to_semantic_table\n\n# Create semantic table with measures\nsales_st = to_semantic_table(\n sales_data,\n name=\"daily_sales\"\n).with_measures(\n total_revenue=lambda t: t.revenue.sum(),\n avg_revenue=lambda t: t.revenue.mean(),\n sale_count=lambda t: t.count(),\n)", - "sql": "SELECT\n *\nFROM \"ibis_pandas_memtable_fbuywylsizfijdmqwyxy7yjnfm\"", - "plan": "SemanticTable: daily_sales\n total_revenue [measure]\n avg_revenue [measure]\n sale_count [measure]", - "table": { - "columns": [ - "sale_date", - "revenue", - "product_category" - ], - "data": [ - [ - "2024-01-01", - 1027.89, - "Electronics" - ], - [ - "2024-01-02", - 915.0, - "Clothing" - ], - [ - "2024-01-03", - 975.01, - "Home" - ], - [ - "2024-01-04", - 974.64, - "Electronics" - ], - [ - "2024-01-05", - 1087.29, - "Clothing" - ], - [ - "2024-01-06", - 1400.34, - "Home" - ], - [ - "2024-01-07", - 1456.44, - "Electronics" - ], - [ - "2024-01-08", - 987.39, - "Clothing" - ], - [ - "2024-01-09", - 1064.38, - "Home" - ], - [ - "2024-01-10", - 995.96, - "Electronics" - ], - [ - "2024-01-11", - 1043.73, - "Clothing" - ], - [ - "2024-01-12", - 1111.07, - "Home" - ], - [ - "2024-01-13", - 1361.31, - "Electronics" - ], - [ - "2024-01-14", - 1408.77, - "Clothing" - ], - [ - "2024-01-15", - 1169.98, - "Home" - ], - [ - "2024-01-16", - 1158.99, - "Electronics" - ], - [ - "2024-01-17", - 1104.09, - "Clothing" - ], - [ - "2024-01-18", - 1187.85, - "Home" - ], - [ - "2024-01-19", - 1241.89, - "Electronics" - ], - [ - "2024-01-20", - 1448.3, - "Clothing" - ], - [ - "2024-01-21", - 1621.16, - "Home" - ], - [ - "2024-01-22", - 1249.63, - "Electronics" - ], - [ - "2024-01-23", - 1188.05, - "Clothing" - ], - [ - "2024-01-24", - 1161.1, - "Home" - ], - [ - "2024-01-25", - 1331.44, - "Electronics" - ], - [ - "2024-01-26", - 1217.32, - "Clothing" - ], - [ - "2024-01-27", - 1556.55, - "Home" - ], - [ - "2024-01-28", - 1570.34, - "Electronics" - ], - [ - "2024-01-29", - 1349.5, - "Clothing" - ], - [ - "2024-01-30", - 1310.75, - "Home" - ], - [ - "2024-01-31", - 1361.43, - "Electronics" - ], - [ - "2024-02-01", - 1355.95, - "Clothing" - ], - [ - "2024-02-02", - 1327.25, - "Home" - ], - [ - "2024-02-03", - 1823.62, - "Electronics" - ], - [ - "2024-02-04", - 1717.71, - "Clothing" - ], - [ - "2024-02-05", - 1360.41, - "Home" - ], - [ - "2024-02-06", - 1425.88, - "Electronics" - ], - [ - "2024-02-07", - 1393.7, - "Clothing" - ], - [ - "2024-02-08", - 1452.34, - "Home" - ], - [ - "2024-02-09", - 1405.47, - "Electronics" - ], - [ - "2024-02-10", - 1860.91, - "Clothing" - ], - [ - "2024-02-11", - 1742.16, - "Home" - ], - [ - "2024-02-12", - 1365.58, - "Electronics" - ], - [ - "2024-02-13", - 1387.88, - "Clothing" - ], - [ - "2024-02-14", - 1355.96, - "Home" - ], - [ - "2024-02-15", - 1396.56, - "Electronics" - ], - [ - "2024-02-16", - 1380.2, - "Clothing" - ], - [ - "2024-02-17", - 1866.59, - "Home" - ], - [ - "2024-02-18", - 1951.14, - "Electronics" - ], - [ - "2024-02-19", - 1462.97, - "Clothing" - ], - [ - "2024-02-20", - 1474.04, - "Home" - ], - [ - "2024-02-21", - 1451.9, - "Electronics" - ], - [ - "2024-02-22", - 1473.4, - "Clothing" - ], - [ - "2024-02-23", - 1617.33, - "Home" - ], - [ - "2024-02-24", - 2031.61, - "Electronics" - ], - [ - "2024-02-25", - 2036.83, - "Clothing" - ], - [ - "2024-02-26", - 1494.23, - "Home" - ], - [ - "2024-02-27", - 1615.83, - "Electronics" - ], - [ - "2024-02-28", - 1512.68, - "Clothing" - ], - [ - "2024-02-29", - 1565.89, - "Home" - ], - [ - "2024-03-01", - 1697.9, - "Electronics" - ], - [ - "2024-03-02", - 2121.0, - "Clothing" - ], - [ - "2024-03-03", - 2117.39, - "Home" - ], - [ - "2024-03-04", - 1666.92, - "Electronics" - ], - [ - "2024-03-05", - 1708.57, - "Clothing" - ], - [ - "2024-03-06", - 1705.2, - "Home" - ], - [ - "2024-03-07", - 1605.81, - "Electronics" - ], - [ - "2024-03-08", - 1576.42, - "Clothing" - ], - [ - "2024-03-09", - 2147.09, - "Home" - ], - [ - "2024-03-10", - 2150.55, - "Electronics" - ], - [ - "2024-03-11", - 1642.2, - "Clothing" - ], - [ - "2024-03-12", - 1798.58, - "Home" - ], - [ - "2024-03-13", - 1795.27, - "Electronics" - ], - [ - "2024-03-14", - 1692.94, - "Clothing" - ], - [ - "2024-03-15", - 1771.09, - "Home" - ], - [ - "2024-03-16", - 2254.13, - "Electronics" - ], - [ - "2024-03-17", - 2370.91, - "Clothing" - ], - [ - "2024-03-18", - 1761.77, - "Home" - ], - [ - "2024-03-19", - 1732.98, - "Electronics" - ], - [ - "2024-03-20", - 1739.33, - "Clothing" - ], - [ - "2024-03-21", - 1812.27, - "Home" - ], - [ - "2024-03-22", - 1762.55, - "Electronics" - ], - [ - "2024-03-23", - 2382.92, - "Clothing" - ], - [ - "2024-03-24", - 2458.56, - "Home" - ], - [ - "2024-03-25", - 1819.88, - "Electronics" - ], - [ - "2024-03-26", - 1793.86, - "Clothing" - ], - [ - "2024-03-27", - 1959.51, - "Home" - ], - [ - "2024-03-28", - 1871.91, - "Electronics" - ], - [ - "2024-03-29", - 1798.18, - "Clothing" - ], - [ - "2024-03-30", - 2366.42, - "Home" - ] - ] - } + "semantic_table": true, + "name": "daily_sales", + "info": "Semantic table definition stored in context" }, "query_lag_lead": { - "code": "from ibis import _\n\n# Aggregate daily revenue\ndaily_revenue = (\n sales_st\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n .order_by(\"sale_date\")\n)\n\n# Add window functions for lag/lead\nresult = daily_revenue.mutate(\n prev_day_revenue=_.total_revenue.lag(),\n next_day_revenue=_.total_revenue.lead(),\n day_over_day_change=_.total_revenue - _.total_revenue.lag(),\n pct_change=((_.total_revenue - _.total_revenue.lag()) / _.total_revenue.lag() * 100).round(2)\n).limit(10)", - "sql": "SELECT\n \"t5\".\"sale_date\",\n \"t5\".\"total_revenue\",\n \"t5\".\"prev_day_revenue\",\n \"t5\".\"next_day_revenue\",\n \"t5\".\"day_over_day_change\",\n ROUND(\n (\n CAST((\n \"t5\".\"total_revenue\" - LAG(\"t5\".\"total_revenue\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)\n ) AS DOUBLE PRECISION) / LAG(\"t5\".\"total_revenue\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)\n ) * 100,\n 2\n ) AS \"pct_change\"\nFROM (\n SELECT\n \"t4\".\"sale_date\",\n \"t4\".\"total_revenue\",\n \"t4\".\"prev_day_revenue\",\n \"t4\".\"next_day_revenue\",\n \"t4\".\"total_revenue\" - LAG(\"t4\".\"total_revenue\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"day_over_day_change\"\n FROM (\n SELECT\n \"t3\".\"sale_date\",\n \"t3\".\"total_revenue\",\n \"t3\".\"prev_day_revenue\",\n LEAD(\"t3\".\"total_revenue\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"next_day_revenue\"\n FROM (\n SELECT\n \"t2\".\"sale_date\",\n \"t2\".\"total_revenue\",\n LAG(\"t2\".\"total_revenue\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"prev_day_revenue\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t0\".\"sale_date\",\n SUM(\"t0\".\"revenue\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t0\".\"revenue\",\n \"t0\".\"product_category\",\n \"t0\".\"sale_date\"\n FROM \"ibis_pandas_memtable_fbuywylsizfijdmqwyxy7yjnfm\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"sale_date\"\n ) AS \"t1\"\n ORDER BY\n \"t1\".\"sale_date\" ASC\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS \"t4\"\n) AS \"t5\"\nLIMIT 10", - "plan": "SemanticTable: daily_sales\n total_revenue [measure]\n avg_revenue [measure]\n sale_count [measure]\n-> GroupBy(sale_date)\n-> Aggregate(total_revenue)\n-> OrderBy(sale_date)\n-> Mutate(prev_day_revenue, next_day_revenue, day_over_day_change, pct_change)\n-> Limit(10)", + "code": "from ibis import _\n\n# Aggregate daily revenue\ndaily_revenue = (\n sales_st\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n)\n\n# Add window functions for lag/lead \u2014 applied directly on the aggregate,\n# each window carrying its own ordering\nresult = daily_revenue.mutate(\n prev_day_revenue=lambda t: t.total_revenue.lag().over(order_by=\"sale_date\"),\n next_day_revenue=lambda t: t.total_revenue.lead().over(order_by=\"sale_date\"),\n day_over_day_change=lambda t: (\n t.total_revenue - t.total_revenue.lag().over(order_by=\"sale_date\")\n ),\n pct_change=lambda t: (\n (t.total_revenue - t.total_revenue.lag().over(order_by=\"sale_date\"))\n / t.total_revenue.lag().over(order_by=\"sale_date\") * 100\n ).round(2),\n).order_by(\"sale_date\").limit(10)", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t4\".\"sale_date\",\n \"t4\".\"total_revenue\",\n \"t4\".\"prev_day_revenue\",\n \"t4\".\"next_day_revenue\",\n \"t4\".\"day_over_day_change\",\n ROUND(\n (\n CAST((\n \"t4\".\"total_revenue\" - LAG(\"t4\".\"total_revenue\") OVER (ORDER BY \"t4\".\"sale_date\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)\n ) AS DOUBLE PRECISION) / LAG(\"t4\".\"total_revenue\") OVER (ORDER BY \"t4\".\"sale_date\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)\n ) * 100,\n 2\n ) AS \"pct_change\"\n FROM (\n SELECT\n \"t3\".\"sale_date\",\n \"t3\".\"total_revenue\",\n \"t3\".\"prev_day_revenue\",\n \"t3\".\"next_day_revenue\",\n \"t3\".\"total_revenue\" - LAG(\"t3\".\"total_revenue\") OVER (ORDER BY \"t3\".\"sale_date\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"day_over_day_change\"\n FROM (\n SELECT\n \"t2\".\"sale_date\",\n \"t2\".\"total_revenue\",\n \"t2\".\"prev_day_revenue\",\n LEAD(\"t2\".\"total_revenue\") OVER (ORDER BY \"t2\".\"sale_date\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"next_day_revenue\"\n FROM (\n SELECT\n \"t1\".\"sale_date\",\n \"t1\".\"total_revenue\",\n LAG(\"t1\".\"total_revenue\") OVER (ORDER BY \"t1\".\"sale_date\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"prev_day_revenue\"\n FROM (\n SELECT\n \"t0\".\"sale_date\",\n SUM(\"t0\".\"revenue\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t0\".\"revenue\",\n \"t0\".\"product_category\",\n \"t0\".\"sale_date\"\n FROM \"ibis_pandas_memtable_kklap3vyhrbrnpjpijezj5ae4e\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"sale_date\"\n ) AS \"t1\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS \"t4\"\n) AS \"t5\"\nORDER BY\n \"t5\".\"sale_date\" ASC\nLIMIT 10", + "plan": "SemanticTable: daily_sales\n total_revenue [measure]\n avg_revenue [measure]\n sale_count [measure]\n-> GroupBy(sale_date)\n-> Aggregate(total_revenue, prev_day_revenue, next_day_revenue, day_over_day_change, pct_change)\n-> OrderBy(sale_date)\n-> Limit(10)", "table": { "columns": [ "sale_date", @@ -944,178 +485,447 @@ ], "data": [ [ - "2024-01-19", - 1241.89, + "2024-01-01", + 1027.89, null, - 2254.13, + 915.0, null, null ], [ - "2024-03-16", - 2254.13, - 1241.89, - 974.64, - 1012.24, - 81.51 + "2024-01-02", + 915.0, + 1027.89, + 975.01, + -112.8900000000001, + -10.98 ], [ - "2024-01-04", + "2024-01-03", + 975.01, + 915.0, 974.64, - 2254.13, - 1111.07, - -1279.4900000000002, - -56.76 + 60.00999999999999, + 6.56 ], [ - "2024-01-12", - 1111.07, + "2024-01-04", 974.64, - 1365.58, - 136.42999999999995, - 14.0 + 975.01, + 1087.29, + -0.37000000000000455, + -0.04 ], [ - "2024-02-12", - 1365.58, - 1111.07, - 1451.9, - 254.51, - 22.91 + "2024-01-05", + 1087.29, + 974.64, + 1400.34, + 112.64999999999998, + 11.56 ], [ - "2024-02-21", - 1451.9, - 1365.58, - 1473.4, - 86.32000000000016, - 6.32 + "2024-01-06", + 1400.34, + 1087.29, + 1456.44, + 313.04999999999995, + 28.79 ], [ - "2024-02-22", - 1473.4, - 1451.9, - 1666.92, - 21.5, - 1.48 + "2024-01-07", + 1456.44, + 1400.34, + 987.39, + 56.100000000000136, + 4.01 ], [ - "2024-03-04", - 1666.92, - 1473.4, - 1771.09, - 193.51999999999998, - 13.13 + "2024-01-08", + 987.39, + 1456.44, + 1064.38, + -469.05000000000007, + -32.21 ], [ - "2024-03-29", - 1798.18, - 2458.56, + "2024-01-09", + 1064.38, 987.39, - -660.3799999999999, - 7.87 + 995.96, + 76.99000000000012, + 7.8 ], [ - "2024-01-08", - 987.39, - 1798.18, - 1570.34, - -810.7900000000001, - -45.09 + "2024-01-10", + 995.96, + 1064.38, + 1043.73, + -68.42000000000007, + -6.43 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-b619f08e03be30f67dc36f4dbc7ad04c" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "sale_date", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "sale_date", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "total_revenue", + "prev_day_revenue", + "next_day_revenue", + "day_over_day_change", + "pct_change" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-b619f08e03be30f67dc36f4dbc7ad04c": [ + { + "sale_date": "2024-01-01T00:00:00", + "total_revenue": 1027.89, + "prev_day_revenue": null, + "next_day_revenue": 915.0, + "day_over_day_change": null, + "pct_change": null + }, + { + "sale_date": "2024-01-02T00:00:00", + "total_revenue": 915.0, + "prev_day_revenue": 1027.89, + "next_day_revenue": 975.01, + "day_over_day_change": -112.8900000000001, + "pct_change": -10.98 + }, + { + "sale_date": "2024-01-03T00:00:00", + "total_revenue": 975.01, + "prev_day_revenue": 915.0, + "next_day_revenue": 974.64, + "day_over_day_change": 60.00999999999999, + "pct_change": 6.56 + }, + { + "sale_date": "2024-01-04T00:00:00", + "total_revenue": 974.64, + "prev_day_revenue": 975.01, + "next_day_revenue": 1087.29, + "day_over_day_change": -0.37000000000000455, + "pct_change": -0.04 + }, + { + "sale_date": "2024-01-05T00:00:00", + "total_revenue": 1087.29, + "prev_day_revenue": 974.64, + "next_day_revenue": 1400.34, + "day_over_day_change": 112.64999999999998, + "pct_change": 11.56 + }, + { + "sale_date": "2024-01-06T00:00:00", + "total_revenue": 1400.34, + "prev_day_revenue": 1087.29, + "next_day_revenue": 1456.44, + "day_over_day_change": 313.04999999999995, + "pct_change": 28.79 + }, + { + "sale_date": "2024-01-07T00:00:00", + "total_revenue": 1456.44, + "prev_day_revenue": 1400.34, + "next_day_revenue": 987.39, + "day_over_day_change": 56.100000000000136, + "pct_change": 4.01 + }, + { + "sale_date": "2024-01-08T00:00:00", + "total_revenue": 987.39, + "prev_day_revenue": 1456.44, + "next_day_revenue": 1064.38, + "day_over_day_change": -469.05000000000007, + "pct_change": -32.21 + }, + { + "sale_date": "2024-01-09T00:00:00", + "total_revenue": 1064.38, + "prev_day_revenue": 987.39, + "next_day_revenue": 995.96, + "day_over_day_change": 76.99000000000012, + "pct_change": 7.8 + }, + { + "sale_date": "2024-01-10T00:00:00", + "total_revenue": 995.96, + "prev_day_revenue": 1064.38, + "next_day_revenue": 1043.73, + "day_over_day_change": -68.42000000000007, + "pct_change": -6.43 + } + ] + } + } } }, "query_running_total": { - "code": "from ibis import _\n\n# Daily revenue with cumulative total\ndaily_revenue = (\n sales_st\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n .order_by(\"sale_date\")\n)\n\n# Calculate cumulative sum and running average\nwindow_unbounded = xo.window(rows=(None, 0), order_by=\"sale_date\")\n\nresult = daily_revenue.mutate(\n cumulative_revenue=_.total_revenue.cumsum(),\n days_count=lambda t: t.count().over(window_unbounded),\n avg_daily_so_far=lambda t: (t.cumulative_revenue / t.days_count).round(2)\n).limit(10)", - "sql": "SELECT\n \"t4\".\"sale_date\",\n \"t4\".\"total_revenue\",\n \"t4\".\"cumulative_revenue\",\n \"t4\".\"days_count\",\n ROUND(CAST(\"t4\".\"cumulative_revenue\" AS DOUBLE PRECISION) / \"t4\".\"days_count\", 2) AS \"avg_daily_so_far\"\nFROM (\n SELECT\n \"t3\".\"sale_date\",\n \"t3\".\"total_revenue\",\n \"t3\".\"cumulative_revenue\",\n COUNT(*) OVER (ORDER BY \"t3\".\"sale_date\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"days_count\"\n FROM (\n SELECT\n \"t2\".\"sale_date\",\n \"t2\".\"total_revenue\",\n SUM(\"t2\".\"total_revenue\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"cumulative_revenue\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t0\".\"sale_date\",\n SUM(\"t0\".\"revenue\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t0\".\"revenue\",\n \"t0\".\"product_category\",\n \"t0\".\"sale_date\"\n FROM \"ibis_pandas_memtable_fbuywylsizfijdmqwyxy7yjnfm\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"sale_date\"\n ) AS \"t1\"\n ORDER BY\n \"t1\".\"sale_date\" ASC\n ) AS \"t2\"\n ) AS \"t3\"\n) AS \"t4\"\nLIMIT 10", - "plan": "SemanticTable: daily_sales\n total_revenue [measure]\n avg_revenue [measure]\n sale_count [measure]\n-> GroupBy(sale_date)\n-> Aggregate(total_revenue)\n-> OrderBy(sale_date)\n-> Mutate(cumulative_revenue, days_count, avg_daily_so_far)\n-> Limit(10)", + "code": "from ibis import _\n\n# Daily revenue with cumulative total\ndaily_revenue = (\n sales_st\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n)\n\n# Calculate cumulative sum and running average\nwindow_unbounded = xo.window(rows=(None, 0), order_by=\"sale_date\")\n\nresult = daily_revenue.mutate(\n cumulative_revenue=lambda t: t.total_revenue.sum().over(window_unbounded),\n avg_daily_so_far=lambda t: t.total_revenue.mean().over(window_unbounded).round(2),\n).order_by(\"sale_date\").limit(10)", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t2\".\"sale_date\",\n \"t2\".\"total_revenue\",\n \"t2\".\"cumulative_revenue\",\n ROUND(\n AVG(\"t2\".\"total_revenue\") OVER (ORDER BY \"t2\".\"sale_date\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW),\n 2\n ) AS \"avg_daily_so_far\"\n FROM (\n SELECT\n \"t1\".\"sale_date\",\n \"t1\".\"total_revenue\",\n SUM(\"t1\".\"total_revenue\") OVER (ORDER BY \"t1\".\"sale_date\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"cumulative_revenue\"\n FROM (\n SELECT\n \"t0\".\"sale_date\",\n SUM(\"t0\".\"revenue\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t0\".\"revenue\",\n \"t0\".\"product_category\",\n \"t0\".\"sale_date\"\n FROM \"ibis_pandas_memtable_kklap3vyhrbrnpjpijezj5ae4e\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"sale_date\"\n ) AS \"t1\"\n ) AS \"t2\"\n) AS \"t3\"\nORDER BY\n \"t3\".\"sale_date\" ASC\nLIMIT 10", + "plan": "SemanticTable: daily_sales\n total_revenue [measure]\n avg_revenue [measure]\n sale_count [measure]\n-> GroupBy(sale_date)\n-> Aggregate(total_revenue, cumulative_revenue, avg_daily_so_far)\n-> OrderBy(sale_date)\n-> Limit(10)", "table": { "columns": [ "sale_date", "total_revenue", "cumulative_revenue", - "days_count", "avg_daily_so_far" ], "data": [ [ "2024-01-01", 1027.89, - 99331.94000000002, - 1, - 99331.94 + 1027.89, + 1027.89 ], [ "2024-01-02", 915.0, - 16429.45, - 2, - 8214.73 + 1942.89, + 971.45 ], [ "2024-01-03", 975.01, - 13653.54, - 3, - 4551.18 + 2917.9, + 972.63 ], [ "2024-01-04", 974.64, - 118171.70999999999, - 4, - 29542.93 + 3892.54, + 973.14 ], [ "2024-01-05", 1087.29, - 100419.23000000001, - 5, - 20083.85 + 4979.83, + 995.97 ], [ "2024-01-06", 1400.34, - 36737.88, - 6, - 6122.98 + 6380.17, + 1063.36 ], [ "2024-01-07", 1456.44, - 32697.350000000002, - 7, - 4671.05 + 7836.610000000001, + 1119.52 ], [ "2024-01-08", 987.39, - 53532.499999999985, - 8, - 6691.56 + 8824.0, + 1103.0 ], [ "2024-01-09", 1064.38, - 87383.66, - 9, - 9709.3 + 9888.380000000001, + 1098.71 ], [ "2024-01-10", 995.96, - 128007.62999999999, - 10, - 12800.76 + 10884.34, + 1088.43 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-a13b8debb6fa0ff7c1805936f2f8aad0" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "sale_date", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "sale_date", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "total_revenue", + "cumulative_revenue", + "avg_daily_so_far" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-a13b8debb6fa0ff7c1805936f2f8aad0": [ + { + "sale_date": "2024-01-01T00:00:00", + "total_revenue": 1027.89, + "cumulative_revenue": 1027.89, + "avg_daily_so_far": 1027.89 + }, + { + "sale_date": "2024-01-02T00:00:00", + "total_revenue": 915.0, + "cumulative_revenue": 1942.89, + "avg_daily_so_far": 971.45 + }, + { + "sale_date": "2024-01-03T00:00:00", + "total_revenue": 975.01, + "cumulative_revenue": 2917.9, + "avg_daily_so_far": 972.63 + }, + { + "sale_date": "2024-01-04T00:00:00", + "total_revenue": 974.64, + "cumulative_revenue": 3892.54, + "avg_daily_so_far": 973.14 + }, + { + "sale_date": "2024-01-05T00:00:00", + "total_revenue": 1087.29, + "cumulative_revenue": 4979.83, + "avg_daily_so_far": 995.97 + }, + { + "sale_date": "2024-01-06T00:00:00", + "total_revenue": 1400.34, + "cumulative_revenue": 6380.17, + "avg_daily_so_far": 1063.36 + }, + { + "sale_date": "2024-01-07T00:00:00", + "total_revenue": 1456.44, + "cumulative_revenue": 7836.610000000001, + "avg_daily_so_far": 1119.52 + }, + { + "sale_date": "2024-01-08T00:00:00", + "total_revenue": 987.39, + "cumulative_revenue": 8824.0, + "avg_daily_so_far": 1103.0 + }, + { + "sale_date": "2024-01-09T00:00:00", + "total_revenue": 1064.38, + "cumulative_revenue": 9888.380000000001, + "avg_daily_so_far": 1098.71 + }, + { + "sale_date": "2024-01-10T00:00:00", + "total_revenue": 995.96, + "cumulative_revenue": 10884.34, + "avg_daily_so_far": 1088.43 + } + ] + } + } } }, "query_moving_average": { - "code": "from ibis import _\n\n# Daily revenue\ndaily_revenue = (\n sales_st\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n .order_by(\"sale_date\")\n)\n\n# 7-day moving average\nwindow_7d = xo.window(rows=(-6, 0), order_by=\"sale_date\")\n\nresult = daily_revenue.mutate(\n ma_7day=_.total_revenue.mean().over(window_7d).round(2),\n ma_7day_sum=_.total_revenue.sum().over(window_7d).round(2),\n).limit(10)", - "sql": "SELECT\n \"t3\".\"sale_date\",\n \"t3\".\"total_revenue\",\n \"t3\".\"ma_7day\",\n ROUND(\n SUM(\"t3\".\"total_revenue\") OVER (ORDER BY \"t3\".\"sale_date\" ASC ROWS BETWEEN 6 preceding AND CURRENT ROW),\n 2\n ) AS \"ma_7day_sum\"\nFROM (\n SELECT\n \"t2\".\"sale_date\",\n \"t2\".\"total_revenue\",\n ROUND(\n CAST(AVG(\"t2\".\"total_revenue\") OVER (ORDER BY \"t2\".\"sale_date\" ASC ROWS BETWEEN 6 preceding AND CURRENT ROW) AS DECIMAL),\n 2\n ) AS \"ma_7day\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t0\".\"sale_date\",\n SUM(\"t0\".\"revenue\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t0\".\"revenue\",\n \"t0\".\"product_category\",\n \"t0\".\"sale_date\"\n FROM \"ibis_pandas_memtable_fbuywylsizfijdmqwyxy7yjnfm\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"sale_date\"\n ) AS \"t1\"\n ORDER BY\n \"t1\".\"sale_date\" ASC\n ) AS \"t2\"\n) AS \"t3\"\nLIMIT 10", - "plan": "SemanticTable: daily_sales\n total_revenue [measure]\n avg_revenue [measure]\n sale_count [measure]\n-> GroupBy(sale_date)\n-> Aggregate(total_revenue)\n-> OrderBy(sale_date)\n-> Mutate(ma_7day, ma_7day_sum)\n-> Limit(10)", + "code": "from ibis import _\n\n# Daily revenue\ndaily_revenue = (\n sales_st\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n)\n\n# 7-day moving average\nwindow_7d = xo.window(rows=(-6, 0), order_by=\"sale_date\")\n\nresult = daily_revenue.mutate(\n ma_7day=lambda t: t.total_revenue.mean().over(window_7d).round(2),\n ma_7day_sum=lambda t: t.total_revenue.sum().over(window_7d).round(2),\n).order_by(\"sale_date\").limit(10)", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t2\".\"sale_date\",\n \"t2\".\"total_revenue\",\n \"t2\".\"ma_7day\",\n ROUND(\n SUM(\"t2\".\"total_revenue\") OVER (ORDER BY \"t2\".\"sale_date\" ASC ROWS BETWEEN 6 preceding AND CURRENT ROW),\n 2\n ) AS \"ma_7day_sum\"\n FROM (\n SELECT\n \"t1\".\"sale_date\",\n \"t1\".\"total_revenue\",\n ROUND(\n AVG(\"t1\".\"total_revenue\") OVER (ORDER BY \"t1\".\"sale_date\" ASC ROWS BETWEEN 6 preceding AND CURRENT ROW),\n 2\n ) AS \"ma_7day\"\n FROM (\n SELECT\n \"t0\".\"sale_date\",\n SUM(\"t0\".\"revenue\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t0\".\"revenue\",\n \"t0\".\"product_category\",\n \"t0\".\"sale_date\"\n FROM \"ibis_pandas_memtable_kklap3vyhrbrnpjpijezj5ae4e\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"sale_date\"\n ) AS \"t1\"\n ) AS \"t2\"\n) AS \"t3\"\nORDER BY\n \"t3\".\"sale_date\" ASC\nLIMIT 10", + "plan": "SemanticTable: daily_sales\n total_revenue [measure]\n avg_revenue [measure]\n sale_count [measure]\n-> GroupBy(sale_date)\n-> Aggregate(total_revenue, ma_7day, ma_7day_sum)\n-> OrderBy(sale_date)\n-> Limit(10)", "table": { "columns": [ "sale_date", @@ -1185,12 +995,141 @@ 7966.44 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-3d114133558af670c07d20a7e60639fa" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "sale_date", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "sale_date", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "total_revenue", + "ma_7day", + "ma_7day_sum" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-3d114133558af670c07d20a7e60639fa": [ + { + "sale_date": "2024-01-01T00:00:00", + "total_revenue": 1027.89, + "ma_7day": 1027.89, + "ma_7day_sum": 1027.89 + }, + { + "sale_date": "2024-01-02T00:00:00", + "total_revenue": 915.0, + "ma_7day": 971.45, + "ma_7day_sum": 1942.89 + }, + { + "sale_date": "2024-01-03T00:00:00", + "total_revenue": 975.01, + "ma_7day": 972.63, + "ma_7day_sum": 2917.9 + }, + { + "sale_date": "2024-01-04T00:00:00", + "total_revenue": 974.64, + "ma_7day": 973.14, + "ma_7day_sum": 3892.54 + }, + { + "sale_date": "2024-01-05T00:00:00", + "total_revenue": 1087.29, + "ma_7day": 995.97, + "ma_7day_sum": 4979.83 + }, + { + "sale_date": "2024-01-06T00:00:00", + "total_revenue": 1400.34, + "ma_7day": 1063.36, + "ma_7day_sum": 6380.17 + }, + { + "sale_date": "2024-01-07T00:00:00", + "total_revenue": 1456.44, + "ma_7day": 1119.52, + "ma_7day_sum": 7836.61 + }, + { + "sale_date": "2024-01-08T00:00:00", + "total_revenue": 987.39, + "ma_7day": 1113.73, + "ma_7day_sum": 7796.11 + }, + { + "sale_date": "2024-01-09T00:00:00", + "total_revenue": 1064.38, + "ma_7day": 1135.07, + "ma_7day_sum": 7945.49 + }, + { + "sale_date": "2024-01-10T00:00:00", + "total_revenue": 995.96, + "ma_7day": 1138.06, + "ma_7day_sum": 7966.44 + } + ] + } + } } }, "query_ranking": { - "code": "from ibis import _\n\n# Aggregate by product category\ncategory_revenue = (\n sales_st\n .group_by(\"product_category\")\n .aggregate(\"total_revenue\", \"sale_count\")\n .order_by(_.total_revenue.desc())\n)\n\n# Add rank columns\nresult = category_revenue.mutate(\n rank=lambda t: xo.rank().over(xo.window(order_by=xo.desc(t.total_revenue))),\n dense_rank=lambda t: xo.dense_rank().over(xo.window(order_by=xo.desc(t.total_revenue))),\n row_number=lambda t: xo.row_number().over(xo.window(order_by=xo.desc(t.total_revenue))),\n)", - "sql": "SELECT\n \"t4\".\"product_category\",\n \"t4\".\"total_revenue\",\n \"t4\".\"sale_count\",\n \"t4\".\"rank\",\n \"t4\".\"dense_rank\",\n ROW_NUMBER() OVER (\n ORDER BY \"t4\".\"total_revenue\" DESC NULLS LAST\n ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING\n ) - 1 AS \"row_number\"\nFROM (\n SELECT\n \"t3\".\"product_category\",\n \"t3\".\"total_revenue\",\n \"t3\".\"sale_count\",\n \"t3\".\"rank\",\n DENSE_RANK() OVER (\n ORDER BY \"t3\".\"total_revenue\" DESC NULLS LAST\n ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING\n ) - 1 AS \"dense_rank\"\n FROM (\n SELECT\n \"t2\".\"product_category\",\n \"t2\".\"total_revenue\",\n \"t2\".\"sale_count\",\n RANK() OVER (\n ORDER BY \"t2\".\"total_revenue\" DESC NULLS LAST\n ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING\n ) - 1 AS \"rank\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t0\".\"product_category\",\n SUM(\"t0\".\"revenue\") AS \"total_revenue\",\n COUNT(*) AS \"sale_count\"\n FROM (\n SELECT\n \"t0\".\"sale_date\",\n \"t0\".\"revenue\",\n \"t0\".\"product_category\"\n FROM \"ibis_pandas_memtable_fbuywylsizfijdmqwyxy7yjnfm\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"product_category\"\n ) AS \"t1\"\n ORDER BY\n \"t1\".\"total_revenue\" DESC NULLS LAST\n ) AS \"t2\"\n ) AS \"t3\"\n) AS \"t4\"", - "plan": "SemanticTable: daily_sales\n total_revenue [measure]\n avg_revenue [measure]\n sale_count [measure]\n-> GroupBy(product_category)\n-> Aggregate(total_revenue, sale_count)\n-> OrderBy(_CallableWrapper(_fn=_.total_revenue.desc()))\n-> Mutate(rank, dense_rank, row_number)", + "code": "from ibis import _\n\n# Aggregate by product category\ncategory_revenue = (\n sales_st\n .group_by(\"product_category\")\n .aggregate(\"total_revenue\", \"sale_count\")\n)\n\n# Add rank columns (each window carries its own ordering)\nresult = category_revenue.mutate(\n rank=lambda t: xo.rank().over(xo.window(order_by=xo.desc(t.total_revenue))),\n dense_rank=lambda t: xo.dense_rank().over(xo.window(order_by=xo.desc(t.total_revenue))),\n row_number=lambda t: xo.row_number().over(xo.window(order_by=xo.desc(t.total_revenue))),\n).order_by(_.total_revenue.desc())", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t3\".\"product_category\",\n \"t3\".\"total_revenue\",\n \"t3\".\"sale_count\",\n \"t3\".\"rank\",\n \"t3\".\"dense_rank\",\n ROW_NUMBER() OVER (ORDER BY \"t3\".\"total_revenue\" DESC NULLS LAST ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) - 1 AS \"row_number\"\n FROM (\n SELECT\n \"t2\".\"product_category\",\n \"t2\".\"total_revenue\",\n \"t2\".\"sale_count\",\n \"t2\".\"rank\",\n DENSE_RANK() OVER (ORDER BY \"t2\".\"total_revenue\" DESC NULLS LAST ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) - 1 AS \"dense_rank\"\n FROM (\n SELECT\n \"t1\".\"product_category\",\n \"t1\".\"total_revenue\",\n \"t1\".\"sale_count\",\n RANK() OVER (ORDER BY \"t1\".\"total_revenue\" DESC NULLS LAST ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) - 1 AS \"rank\"\n FROM (\n SELECT\n \"t0\".\"product_category\",\n SUM(\"t0\".\"revenue\") AS \"total_revenue\",\n COUNT(*) AS \"sale_count\"\n FROM (\n SELECT\n \"t0\".\"sale_date\",\n \"t0\".\"revenue\",\n \"t0\".\"product_category\"\n FROM \"ibis_pandas_memtable_kklap3vyhrbrnpjpijezj5ae4e\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"product_category\"\n ) AS \"t1\"\n ) AS \"t2\"\n ) AS \"t3\"\n) AS \"t4\"\nORDER BY\n \"t4\".\"total_revenue\" DESC NULLS LAST", + "plan": "SemanticTable: daily_sales\n total_revenue [measure]\n avg_revenue [measure]\n sale_count [measure]\n-> GroupBy(product_category)\n-> Aggregate(total_revenue, sale_count, rank, dense_rank, row_number)\n-> OrderBy(_CallableWrapper(_fn=_.total_revenue.desc()))", "table": { "columns": [ "product_category", @@ -1226,12 +1165,107 @@ 2 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-a4c4adb65d2cd90d069cb567d81f4f29" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "product_category", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "product_category", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "total_revenue", + "sale_count", + "rank", + "dense_rank", + "row_number" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-a4c4adb65d2cd90d069cb567d81f4f29": [ + { + "product_category": "Home", + "total_revenue": 47712.26999999998, + "sale_count": 30, + "rank": 0, + "dense_rank": 0, + "row_number": 0 + }, + { + "product_category": "Electronics", + "total_revenue": 46555.450000000004, + "sale_count": 30, + "rank": 1, + "dense_rank": 1, + "row_number": 1 + }, + { + "product_category": "Clothing", + "total_revenue": 46158.00000000001, + "sale_count": 30, + "rank": 2, + "dense_rank": 2, + "row_number": 2 + } + ] + } + } } }, "query_week_over_week": { - "code": "from ibis import _\n\n# Aggregate by week\nweekly_revenue = (\n sales_st\n .mutate(week_start=_.sale_date.truncate(\"W\"))\n .group_by(\"week_start\")\n .aggregate(\"total_revenue\")\n .order_by(\"week_start\")\n)\n\n# Calculate week-over-week changes\nresult = weekly_revenue.mutate(\n prev_week_revenue=_.total_revenue.lag(),\n wow_change=_.total_revenue - _.total_revenue.lag(),\n wow_pct_change=((_.total_revenue - _.total_revenue.lag()) / _.total_revenue.lag() * 100).round(2)\n).limit(10)", - "sql": "SELECT\n \"t5\".\"week_start\",\n \"t5\".\"total_revenue\",\n \"t5\".\"prev_week_revenue\",\n \"t5\".\"wow_change\",\n ROUND(\n (\n CAST((\n \"t5\".\"total_revenue\" - LAG(\"t5\".\"total_revenue\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)\n ) AS DOUBLE PRECISION) / LAG(\"t5\".\"total_revenue\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)\n ) * 100,\n 2\n ) AS \"wow_pct_change\"\nFROM (\n SELECT\n \"t4\".\"week_start\",\n \"t4\".\"total_revenue\",\n \"t4\".\"prev_week_revenue\",\n \"t4\".\"total_revenue\" - LAG(\"t4\".\"total_revenue\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"wow_change\"\n FROM (\n SELECT\n \"t3\".\"week_start\",\n \"t3\".\"total_revenue\",\n LAG(\"t3\".\"total_revenue\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"prev_week_revenue\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t1\".\"week_start\",\n SUM(\"t1\".\"revenue\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t1\".\"sale_date\",\n \"t1\".\"revenue\",\n \"t1\".\"product_category\",\n \"t1\".\"week_start\"\n FROM (\n SELECT\n \"t0\".\"sale_date\",\n \"t0\".\"revenue\",\n \"t0\".\"product_category\",\n DATE_TRUNC('WEEK', \"t0\".\"sale_date\") AS \"week_start\"\n FROM \"ibis_pandas_memtable_fbuywylsizfijdmqwyxy7yjnfm\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"week_start\"\n ) AS \"t2\"\n ORDER BY\n \"t2\".\"week_start\" ASC\n ) AS \"t3\"\n ) AS \"t4\"\n) AS \"t5\"\nLIMIT 10", - "plan": "SemanticTable: daily_sales\n total_revenue [measure]\n avg_revenue [measure]\n sale_count [measure]\n-> Mutate(week_start)\n-> GroupBy(week_start)\n-> Aggregate(total_revenue)\n-> OrderBy(week_start)\n-> Mutate(prev_week_revenue, wow_change, wow_pct_change)\n-> Limit(10)", + "code": "from ibis import _\n\n# Aggregate by week\nweekly_revenue = (\n sales_st\n .mutate(week_start=_.sale_date.truncate(\"W\"))\n .group_by(\"week_start\")\n .aggregate(\"total_revenue\")\n)\n\n# Calculate week-over-week changes\nresult = weekly_revenue.mutate(\n prev_week_revenue=lambda t: t.total_revenue.lag().over(order_by=\"week_start\"),\n wow_change=lambda t: t.total_revenue - t.total_revenue.lag().over(order_by=\"week_start\"),\n wow_pct_change=lambda t: (\n (t.total_revenue - t.total_revenue.lag().over(order_by=\"week_start\"))\n / t.total_revenue.lag().over(order_by=\"week_start\") * 100\n ).round(2),\n).order_by(\"week_start\").limit(10)", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t4\".\"week_start\",\n \"t4\".\"total_revenue\",\n \"t4\".\"prev_week_revenue\",\n \"t4\".\"wow_change\",\n ROUND(\n (\n CAST((\n \"t4\".\"total_revenue\" - LAG(\"t4\".\"total_revenue\") OVER (ORDER BY \"t4\".\"week_start\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)\n ) AS DOUBLE PRECISION) / LAG(\"t4\".\"total_revenue\") OVER (ORDER BY \"t4\".\"week_start\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)\n ) * 100,\n 2\n ) AS \"wow_pct_change\"\n FROM (\n SELECT\n \"t3\".\"week_start\",\n \"t3\".\"total_revenue\",\n \"t3\".\"prev_week_revenue\",\n \"t3\".\"total_revenue\" - LAG(\"t3\".\"total_revenue\") OVER (ORDER BY \"t3\".\"week_start\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"wow_change\"\n FROM (\n SELECT\n \"t2\".\"week_start\",\n \"t2\".\"total_revenue\",\n LAG(\"t2\".\"total_revenue\") OVER (ORDER BY \"t2\".\"week_start\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"prev_week_revenue\"\n FROM (\n SELECT\n \"t1\".\"week_start\",\n SUM(\"t1\".\"revenue\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t1\".\"sale_date\",\n \"t1\".\"revenue\",\n \"t1\".\"product_category\",\n \"t1\".\"week_start\"\n FROM (\n SELECT\n \"t0\".\"sale_date\",\n \"t0\".\"revenue\",\n \"t0\".\"product_category\",\n DATE_TRUNC('WEEK', \"t0\".\"sale_date\") AS \"week_start\"\n FROM \"ibis_pandas_memtable_kklap3vyhrbrnpjpijezj5ae4e\" AS \"t0\"\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"week_start\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS \"t4\"\n) AS \"t5\"\nORDER BY\n \"t5\".\"week_start\" ASC\nLIMIT 10", + "plan": "SemanticTable\n week_start [dim]\n total_revenue [measure]\n avg_revenue [measure]\n sale_count [measure]\n-> GroupBy(week_start)\n-> Aggregate(total_revenue, prev_week_revenue, wow_change, wow_pct_change)\n-> OrderBy(week_start)\n-> Limit(10)", "table": { "columns": [ "week_start", @@ -1242,82 +1276,222 @@ ], "data": [ [ - "2024-02-19", - 11548.08, + "2024-01-01", + 7836.610000000001, null, null, null ], + [ + "2024-01-08", + 7972.610000000001, + 7836.610000000001, + 136.0, + 1.74 + ], + [ + "2024-01-15", + 8932.26, + 7972.610000000001, + 959.6499999999996, + 12.04 + ], [ "2024-01-22", 9274.43, - 7972.610000000001, - 1301.8199999999997, - -19.69 + 8932.26, + 342.1700000000001, + 3.83 ], [ "2024-01-29", 10246.21, - 13325.119999999999, - -3078.91, + 9274.43, + 971.7799999999988, 10.48 ], [ - "2024-01-01", - 7836.610000000001, - 11548.08, - -3711.4699999999993, - -23.52 + "2024-02-05", + 10640.87, + 10246.21, + 394.6600000000017, + 3.85 ], [ - "2024-01-15", - 8932.26, - 7836.610000000001, - 1095.6499999999996, - 13.98 + "2024-02-12", + 10703.909999999998, + 10640.87, + 63.039999999997235, + 0.59 ], [ - "2024-01-08", - 7972.610000000001, - 8932.26, - -959.6499999999996, - -10.74 + "2024-02-19", + 11548.08, + 10703.909999999998, + 844.1700000000019, + 7.89 ], [ "2024-02-26", 12124.92, - 9274.43, - 2850.49, - 4.44 + 11548.08, + 576.8400000000001, + 5.0 ], [ - "2024-03-18", - 13650.38, + "2024-03-04", + 12560.560000000001, 12124.92, - 1525.4599999999991, - 12.58 - ], - [ - "2024-03-25", - 11609.76, - 10640.87, - 968.8899999999994, - 45.62 - ], - [ - "2024-02-05", - 10640.87, - 13650.38, - -3009.5099999999984, - -22.05 + 435.64000000000124, + 3.59 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-d1c41b4c07d5b0728e9592f009a61a2e" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "week_start", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "week_start", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "total_revenue", + "prev_week_revenue", + "wow_change", + "wow_pct_change" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-d1c41b4c07d5b0728e9592f009a61a2e": [ + { + "week_start": "2024-01-01T00:00:00", + "total_revenue": 7836.610000000001, + "prev_week_revenue": null, + "wow_change": null, + "wow_pct_change": null + }, + { + "week_start": "2024-01-08T00:00:00", + "total_revenue": 7972.610000000001, + "prev_week_revenue": 7836.610000000001, + "wow_change": 136.0, + "wow_pct_change": 1.74 + }, + { + "week_start": "2024-01-15T00:00:00", + "total_revenue": 8932.26, + "prev_week_revenue": 7972.610000000001, + "wow_change": 959.6499999999996, + "wow_pct_change": 12.04 + }, + { + "week_start": "2024-01-22T00:00:00", + "total_revenue": 9274.43, + "prev_week_revenue": 8932.26, + "wow_change": 342.1700000000001, + "wow_pct_change": 3.83 + }, + { + "week_start": "2024-01-29T00:00:00", + "total_revenue": 10246.21, + "prev_week_revenue": 9274.43, + "wow_change": 971.7799999999988, + "wow_pct_change": 10.48 + }, + { + "week_start": "2024-02-05T00:00:00", + "total_revenue": 10640.87, + "prev_week_revenue": 10246.21, + "wow_change": 394.6600000000017, + "wow_pct_change": 3.85 + }, + { + "week_start": "2024-02-12T00:00:00", + "total_revenue": 10703.909999999998, + "prev_week_revenue": 10640.87, + "wow_change": 63.039999999997235, + "wow_pct_change": 0.59 + }, + { + "week_start": "2024-02-19T00:00:00", + "total_revenue": 11548.08, + "prev_week_revenue": 10703.909999999998, + "wow_change": 844.1700000000019, + "wow_pct_change": 7.89 + }, + { + "week_start": "2024-02-26T00:00:00", + "total_revenue": 12124.92, + "prev_week_revenue": 11548.08, + "wow_change": 576.8400000000001, + "wow_pct_change": 5.0 + }, + { + "week_start": "2024-03-04T00:00:00", + "total_revenue": 12560.560000000001, + "prev_week_revenue": 12124.92, + "wow_change": 435.64000000000124, + "wow_pct_change": 3.59 + } + ] + } + } } }, "query_pct_running": { - "code": "from ibis import _\n\n# Top 10 days by revenue\ntop_days = (\n sales_st\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n .order_by(_.total_revenue.desc())\n .limit(10)\n)\n\n# Calculate cumulative percentage\nresult = top_days.mutate(\n cumulative_revenue=_.total_revenue.cumsum(),\n total_top10=_.total_revenue.sum(),\n pct_of_top10=(_.total_revenue.cumsum() / _.total_revenue.sum() * 100).round(2)\n)", - "sql": "SELECT\n \"t5\".\"sale_date\",\n \"t5\".\"total_revenue\",\n \"t5\".\"cumulative_revenue\",\n \"t5\".\"total_top10\",\n ROUND(\n (\n CAST(SUM(\"t5\".\"total_revenue\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS DOUBLE PRECISION) / SUM(\"t5\".\"total_revenue\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)\n ) * 100,\n 2\n ) AS \"pct_of_top10\"\nFROM (\n SELECT\n \"t4\".\"sale_date\",\n \"t4\".\"total_revenue\",\n \"t4\".\"cumulative_revenue\",\n SUM(\"t4\".\"total_revenue\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"total_top10\"\n FROM (\n SELECT\n \"t3\".\"sale_date\",\n \"t3\".\"total_revenue\",\n SUM(\"t3\".\"total_revenue\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"cumulative_revenue\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t0\".\"sale_date\",\n SUM(\"t0\".\"revenue\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t0\".\"revenue\",\n \"t0\".\"product_category\",\n \"t0\".\"sale_date\"\n FROM \"ibis_pandas_memtable_fbuywylsizfijdmqwyxy7yjnfm\" AS \"t0\"\n ) AS t0\n GROUP BY\n \"t0\".\"sale_date\"\n ) AS \"t1\"\n ORDER BY\n \"t1\".\"total_revenue\" DESC NULLS LAST\n LIMIT 10\n ) AS \"t3\"\n ) AS \"t4\"\n) AS \"t5\"", - "plan": "SemanticTable: daily_sales\n total_revenue [measure]\n avg_revenue [measure]\n sale_count [measure]\n-> GroupBy(sale_date)\n-> Aggregate(total_revenue)\n-> OrderBy(_CallableWrapper(_fn=_.total_revenue.desc()))\n-> Limit(10)\n-> Mutate(cumulative_revenue, total_top10, pct_of_top10)", + "code": "from ibis import _\n\n# Top 10 days by revenue\ntop_days = (\n sales_st\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n .order_by(_.total_revenue.desc())\n .limit(10)\n)\n\n# A limited query result is a plain table \u2014 row math over it drops to\n# ibis explicitly via .to_untagged()\nresult = top_days.to_untagged().mutate(\n cumulative_revenue=lambda t: t.total_revenue.cumsum(),\n total_top10=lambda t: t.total_revenue.sum(),\n pct_of_top10=lambda t: (t.total_revenue.cumsum() / t.total_revenue.sum() * 100).round(2),\n)", + "sql": "Error generating SQL: Table.sql() missing 1 required positional argument: 'query'", + "plan": "r0 := InMemoryTable\n data:\n PandasDataFrameProxy:\n sale_date revenue product_category\n 0 2024-01-01 1027.89 Electronics\n 1 2024-01-02 915.00 Clothing\n 2 2024-01-03 975.01 Home\n 3 2024-01-04 974.64 Electronics\n 4 2024-01-05 1087.29 Clothing\n .. ... ... ...\n 85 2024-03-26 1793.86 Clothing\n 86 2024-03-27 1959.51 Home\n 87 2024-03-28 1871.91 Electronics\n 88 2024-03-29 1798.18 Clothing\n 89 2024-03-30 2366.42 Home\n\n [90 rows x 3 columns]\n\nr1 := Aggregate[r0]\n groups:\n sale_date: r0.sale_date\n metrics:\n total_revenue: Sum(r0.revenue)\n\nr2 := Project[r1]\n sale_date: r1.sale_date\n total_revenue: r1.total_revenue\n\nr3 := Sort[r2]\n desc r2.total_revenue\n\nr4 := Limit[r3, n=10]\n\nProject[r4]\n sale_date: r4.sale_date\n total_revenue: r4.total_revenue\n cumulative_revenue: WindowFunction(func=Sum(r4.total_revenue), how='rows', end=WindowBoundary(0))\n total_top10: WindowFunction(func=Sum(r4.total_revenue), how='rows')\n pct_of_top10: Round(WindowFunction(func=Sum(r4.total_revenue), how='rows', end=WindowBoundary(0)) / WindowFunction(func=Sum(r4.total_revenue), how='rows') * 100, digits=2)", "table": { "columns": [ "sale_date", @@ -1401,9 +1575,9 @@ } }, "query_window_filter": { - "code": "from ibis import _\n\n# Focus on weekends only\nweekend_revenue = (\n sales_st\n .mutate(is_weekend=_.sale_date.day_of_week.index().isin([5, 6]))\n .filter(_.is_weekend)\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n .order_by(\"sale_date\")\n)\n\n# 3-weekend moving average\nwindow_3 = xo.window(rows=(-2, 0), order_by=\"sale_date\")\n\nresult = weekend_revenue.mutate(\n ma_3weekend=_.total_revenue.mean().over(window_3).round(2),\n prev_weekend=_.total_revenue.lag(),\n weekend_change=_.total_revenue - _.total_revenue.lag()\n).limit(10)", - "sql": "SELECT\n \"t5\".\"sale_date\",\n \"t5\".\"total_revenue\",\n \"t5\".\"ma_3weekend\",\n \"t5\".\"prev_weekend\",\n \"t5\".\"total_revenue\" - LAG(\"t5\".\"total_revenue\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"weekend_change\"\nFROM (\n SELECT\n \"t4\".\"sale_date\",\n \"t4\".\"total_revenue\",\n \"t4\".\"ma_3weekend\",\n LAG(\"t4\".\"total_revenue\") OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"prev_weekend\"\n FROM (\n SELECT\n \"t3\".\"sale_date\",\n \"t3\".\"total_revenue\",\n ROUND(\n CAST(AVG(\"t3\".\"total_revenue\") OVER (ORDER BY \"t3\".\"sale_date\" ASC ROWS BETWEEN 2 preceding AND CURRENT ROW) AS DECIMAL),\n 2\n ) AS \"ma_3weekend\"\n FROM (\n SELECT\n *\n FROM (\n SELECT\n \"t1\".\"sale_date\",\n SUM(\"t1\".\"revenue\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t1\".\"revenue\",\n \"t1\".\"product_category\",\n \"t1\".\"is_weekend\",\n \"t1\".\"sale_date\"\n FROM (\n SELECT\n \"t0\".\"sale_date\",\n \"t0\".\"revenue\",\n \"t0\".\"product_category\",\n (\n DATE_PART('dow', \"t0\".\"sale_date\") + 6\n ) % 7 IN (5, 6) AS \"is_weekend\"\n FROM \"ibis_pandas_memtable_fbuywylsizfijdmqwyxy7yjnfm\" AS \"t0\"\n WHERE\n (\n DATE_PART('dow', \"t0\".\"sale_date\") + 6\n ) % 7 IN (5, 6)\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"sale_date\"\n ) AS \"t2\"\n ORDER BY\n \"t2\".\"sale_date\" ASC\n ) AS \"t3\"\n ) AS \"t4\"\n) AS \"t5\"\nLIMIT 10", - "plan": "SemanticTable: daily_sales\n total_revenue [measure]\n avg_revenue [measure]\n sale_count [measure]\n-> Mutate(is_weekend)\n-> Filter(\u03bb )\n-> GroupBy(sale_date)\n-> Aggregate(total_revenue)\n-> OrderBy(sale_date)\n-> Mutate(ma_3weekend, prev_weekend, weekend_change)\n-> Limit(10)", + "code": "from ibis import _\n\n# Focus on weekends only\nweekend_revenue = (\n sales_st\n .mutate(is_weekend=_.sale_date.day_of_week.index().isin([5, 6]))\n .filter(_.is_weekend)\n .group_by(\"sale_date\")\n .aggregate(\"total_revenue\")\n)\n\n# 3-weekend moving average\nwindow_3 = xo.window(rows=(-2, 0), order_by=\"sale_date\")\n\nresult = weekend_revenue.mutate(\n ma_3weekend=lambda t: t.total_revenue.mean().over(window_3).round(2),\n prev_weekend=lambda t: t.total_revenue.lag().over(order_by=\"sale_date\"),\n weekend_change=lambda t: t.total_revenue - t.total_revenue.lag().over(order_by=\"sale_date\"),\n).order_by(\"sale_date\").limit(10)", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t4\".\"sale_date\",\n \"t4\".\"total_revenue\",\n \"t4\".\"ma_3weekend\",\n \"t4\".\"prev_weekend\",\n \"t4\".\"total_revenue\" - LAG(\"t4\".\"total_revenue\") OVER (ORDER BY \"t4\".\"sale_date\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"weekend_change\"\n FROM (\n SELECT\n \"t3\".\"sale_date\",\n \"t3\".\"total_revenue\",\n \"t3\".\"ma_3weekend\",\n LAG(\"t3\".\"total_revenue\") OVER (ORDER BY \"t3\".\"sale_date\" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"prev_weekend\"\n FROM (\n SELECT\n \"t2\".\"sale_date\",\n \"t2\".\"total_revenue\",\n ROUND(\n AVG(\"t2\".\"total_revenue\") OVER (ORDER BY \"t2\".\"sale_date\" ASC ROWS BETWEEN 2 preceding AND CURRENT ROW),\n 2\n ) AS \"ma_3weekend\"\n FROM (\n SELECT\n \"t1\".\"sale_date\",\n SUM(\"t1\".\"revenue\") AS \"total_revenue\"\n FROM (\n SELECT\n \"t1\".\"revenue\",\n \"t1\".\"product_category\",\n \"t1\".\"is_weekend\",\n \"t1\".\"sale_date\"\n FROM (\n SELECT\n \"t0\".\"sale_date\",\n \"t0\".\"revenue\",\n \"t0\".\"product_category\",\n (\n DATE_PART('dow', \"t0\".\"sale_date\") + 6\n ) % 7 IN (5, 6) AS \"is_weekend\"\n FROM \"ibis_pandas_memtable_kklap3vyhrbrnpjpijezj5ae4e\" AS \"t0\"\n WHERE\n (\n DATE_PART('dow', \"t0\".\"sale_date\") + 6\n ) % 7 IN (5, 6)\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"sale_date\"\n ) AS \"t2\"\n ) AS \"t3\"\n ) AS \"t4\"\n) AS \"t5\"\nORDER BY\n \"t5\".\"sale_date\" ASC\nLIMIT 10", + "plan": "SemanticTable\n is_weekend [dim]\n total_revenue [measure]\n avg_revenue [measure]\n sale_count [measure]\n-> Filter(\u03bb )\n-> GroupBy(sale_date)\n-> Aggregate(total_revenue, ma_3weekend, prev_weekend, weekend_change)\n-> OrderBy(sale_date)\n-> Limit(10)", "table": { "columns": [ "sale_date", @@ -1484,6 +1658,146 @@ -105.90999999999985 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-3a819d33c9eacd7e65be7b4ec0b9932b" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "sale_date", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "sale_date", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "total_revenue", + "ma_3weekend", + "prev_weekend", + "weekend_change" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-3a819d33c9eacd7e65be7b4ec0b9932b": [ + { + "sale_date": "2024-01-06T00:00:00", + "total_revenue": 1400.34, + "ma_3weekend": 1400.34, + "prev_weekend": null, + "weekend_change": null + }, + { + "sale_date": "2024-01-07T00:00:00", + "total_revenue": 1456.44, + "ma_3weekend": 1428.39, + "prev_weekend": 1400.34, + "weekend_change": 56.100000000000136 + }, + { + "sale_date": "2024-01-13T00:00:00", + "total_revenue": 1361.31, + "ma_3weekend": 1406.03, + "prev_weekend": 1456.44, + "weekend_change": -95.13000000000011 + }, + { + "sale_date": "2024-01-14T00:00:00", + "total_revenue": 1408.77, + "ma_3weekend": 1408.84, + "prev_weekend": 1361.31, + "weekend_change": 47.460000000000036 + }, + { + "sale_date": "2024-01-20T00:00:00", + "total_revenue": 1448.3, + "ma_3weekend": 1406.13, + "prev_weekend": 1408.77, + "weekend_change": 39.52999999999997 + }, + { + "sale_date": "2024-01-21T00:00:00", + "total_revenue": 1621.16, + "ma_3weekend": 1492.74, + "prev_weekend": 1448.3, + "weekend_change": 172.86000000000013 + }, + { + "sale_date": "2024-01-27T00:00:00", + "total_revenue": 1556.55, + "ma_3weekend": 1542.0, + "prev_weekend": 1621.16, + "weekend_change": -64.61000000000013 + }, + { + "sale_date": "2024-01-28T00:00:00", + "total_revenue": 1570.34, + "ma_3weekend": 1582.68, + "prev_weekend": 1556.55, + "weekend_change": 13.789999999999964 + }, + { + "sale_date": "2024-02-03T00:00:00", + "total_revenue": 1823.62, + "ma_3weekend": 1650.17, + "prev_weekend": 1570.34, + "weekend_change": 253.27999999999997 + }, + { + "sale_date": "2024-02-04T00:00:00", + "total_revenue": 1717.71, + "ma_3weekend": 1703.89, + "prev_weekend": 1823.62, + "weekend_change": -105.90999999999985 + } + ] + } + } } } }, diff --git a/docs/web/public/bsl-data/yaml-config.json b/docs/web/public/bsl-data/yaml-config.json index c7ed69b8..05b48ba2 100644 --- a/docs/web/public/bsl-data/yaml-config.json +++ b/docs/web/public/bsl-data/yaml-config.json @@ -2,33 +2,13 @@ "markdown": "# YAML Configuration\n\nDefine your semantic models using YAML for better organization and maintainability.\n\n## Why YAML?\n\nYAML configuration provides several advantages:\n- **Better organization**: Keep your model definitions separate from your code\n- **Version control**: Track changes to your data model structure\n- **Collaboration**: Non-developers can review and understand the model\n- **Reusability**: Share model definitions across different projects\n\n## Expression Syntax\n\nHere's a complete example with dimensions, measures, and joins:\n\n\n\n\nIn YAML configuration, **only unbound syntax (`_`) is accepted** for expressions. Lambda expressions are not supported in YAML files.\n\n\n## Loading YAML Models\n\n### Option 1: Using Profiles (Recommended)\n\n```yaml\n# File-level profile\nprofile: my_db\n\nflights:\n table: flights_tbl\n dimensions:\n origin: _.origin\n```\n\n```python\nfrom boring_semantic_layer import from_yaml\n\nmodels = from_yaml(\"flights_model.yml\")\n```\n\nSee [Profile documentation](/building/profile) for setup details.\n\n### Option 2: Passing Tables Manually\n\nCreate your ibis tables:\n\n```yaml_setup\nimport ibis\n\nflights_tbl = ibis.memtable({\n \"origin\": [\"JFK\", \"LAX\", \"SFO\"],\n \"dest\": [\"LAX\", \"SFO\", \"JFK\"],\n \"carrier\": [\"AA\", \"UA\", \"DL\"],\n \"year\": [2023, 2023, 2024],\n \"distance\": [2475, 337, 382]\n})\n\ncarriers_tbl = ibis.memtable({\n \"code\": [\"AA\", \"UA\", \"DL\"],\n \"name\": [\"American Airlines\", \"United Airlines\", \"Delta Air Lines\"]\n})\n```\n\nAnd pass them to the loaded YAML file defining your Semantic Tables:\n\n```load_yaml_example\nfrom boring_semantic_layer import from_yaml\n\n# Load models from YAML file with explicit tables\nmodels = from_yaml(\n \"yaml_example.yaml\",\n tables={\n \"flights_tbl\": flights_tbl,\n \"carriers_tbl\": carriers_tbl\n }\n)\n\nflights_sm = models[\"flights\"]\ncarriers_sm = models[\"carriers\"]\n\n# Inspect the loaded models\nflights_sm.dimensions, flights_sm.measures\n```\n\n \n\n### Option 3: Loading from a Dictionary (`from_config`)\n\nIf you're loading configuration through your own mechanism (e.g., Kedro catalog, external config management), you can use `from_config()` to construct semantic models directly from a Python dictionary:\n\n```python\nfrom boring_semantic_layer import from_config\n\nconfig = {\n \"flights\": {\n \"table\": \"flights_tbl\",\n \"dimensions\": {\n \"origin\": \"_.origin\",\n \"destination\": \"_.dest\",\n },\n \"measures\": {\n \"flight_count\": \"_.count()\",\n \"avg_distance\": \"_.distance.mean()\",\n },\n }\n}\n\nmodels = from_config(config, tables={\"flights_tbl\": flights_tbl})\nflights_sm = models[\"flights\"]\n```\n\nThis is useful for integrations where you don't want to write config to a file just to load it. The `from_config()` function accepts the same `profile` and `profile_path` parameters as `from_yaml()`:\n\n```python\n# With a profile\nmodels = from_config(config, profile=\"my_db\")\n\n# With profile in config\nconfig = {\n \"profile\": \"my_db\",\n \"flights\": {\n \"table\": \"flights_tbl\",\n ...\n }\n}\nmodels = from_config(config)\n```\n\n## Querying YAML Models\n\nYAML-defined models work exactly like Python-defined models. You can use the same `group_by()` and `aggregate()` methods to query your data.\n\n```query_yaml_model\n# Query the YAML-defined model\nresult = (\n flights_sm\n .group_by(\"origin\")\n .aggregate(\"flight_count\", \"avg_distance\")\n)\n```\n\n\n\n## Filters\n\nYou can apply a filter to all queries on a model by adding a `filter` field:\n\n```yaml\nflights:\n table: flights_tbl\n filter: _.year > 2020 # Applied to all queries\n dimensions:\n origin: _.origin\n measures:\n flight_count: _.count()\n```\n\nThe filter expression uses the same `_` syntax as dimensions and measures. It's applied automatically when you query the model.\n\n## Next Steps\n\n- See [Building Semantic Tables](/building/semantic-tables) for Python-based definitions\n- Learn [Query Methods](/querying/methods) for querying YAML-defined models\n- Explore [Composing Models](/building/compose) for joining YAML models\n", "queries": { "load_yaml_example": { - "code": "from boring_semantic_layer import from_yaml\n\n# Load models from YAML file with explicit tables\nmodels = from_yaml(\n \"yaml_example.yaml\",\n tables={\n \"flights_tbl\": flights_tbl,\n \"carriers_tbl\": carriers_tbl\n }\n)\n\nflights_sm = models[\"flights\"]\ncarriers_sm = models[\"carriers\"]\n\n# Inspect the loaded models\nflights_sm.dimensions, flights_sm.measures", - "sql": "SELECT\n *\nFROM \"ibis_pandas_memtable_j2mhogl5vvgy7bhqsygbvs4vpy\"", - "plan": "SemanticTable: carriers\n code [dim]\n name [dim]\n carrier_count [measure]", - "table": { - "columns": [ - "code", - "name" - ], - "data": [ - [ - "AA", - "American Airlines" - ], - [ - "UA", - "United Airlines" - ], - [ - "DL", - "Delta Air Lines" - ] - ] - } + "semantic_table": true, + "name": "carriers", + "info": "Semantic table definition stored in context" }, "query_yaml_model": { "code": "# Query the YAML-defined model\nresult = (\n flights_sm\n .group_by(\"origin\")\n .aggregate(\"flight_count\", \"avg_distance\")\n)", - "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n AVG(\"t1\".\"distance\") AS \"avg_distance\"\n FROM (\n SELECT\n \"t1\".\"dest\",\n \"t1\".\"carrier\",\n \"t1\".\"year\",\n \"t1\".\"distance\",\n \"t1\".\"destination\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n \"t0\".\"origin\",\n \"t0\".\"dest\",\n \"t0\".\"carrier\",\n \"t0\".\"year\",\n \"t0\".\"distance\",\n \"t0\".\"dest\" AS \"destination\"\n FROM \"ibis_pandas_memtable_2rhxc36mlrb2hpy5vpavuy7iye\" AS \"t0\"\n WHERE\n \"t0\".\"year\" > 2020\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", + "sql": "SELECT\n *\nFROM (\n SELECT\n \"t1\".\"origin\",\n COUNT(*) AS \"flight_count\",\n AVG(\"t1\".\"distance\") AS \"avg_distance\"\n FROM (\n SELECT\n \"t1\".\"dest\",\n \"t1\".\"carrier\",\n \"t1\".\"year\",\n \"t1\".\"distance\",\n \"t1\".\"destination\",\n \"t1\".\"origin\"\n FROM (\n SELECT\n \"t0\".\"origin\",\n \"t0\".\"dest\",\n \"t0\".\"carrier\",\n \"t0\".\"year\",\n \"t0\".\"distance\",\n \"t0\".\"dest\" AS \"destination\"\n FROM \"ibis_pandas_memtable_uxfq3saqxbglxotckjfeyb5vpa\" AS \"t0\"\n WHERE\n \"t0\".\"year\" > 2020\n ) AS \"t1\"\n ) AS t1\n GROUP BY\n \"t1\".\"origin\"\n) AS \"t2\"", "plan": "SemanticTable: flights\n origin [dim]\n destination [dim]\n year [dim]\n carrier [dim]\n flight_count [measure]\n total_distance [measure]\n avg_distance [measure]\n-> Filter(\u03bb )\n-> GroupBy(origin)\n-> Aggregate(flight_count, avg_distance)", "table": { "columns": [ @@ -38,9 +18,9 @@ ], "data": [ [ - "SFO", + "JFK", 1, - 382.0 + 2475.0 ], [ "LAX", @@ -48,11 +28,94 @@ 337.0 ], [ - "JFK", + "SFO", 1, - 2475.0 + 382.0 ] ] + }, + "chart": { + "type": "vega", + "spec": { + "config": { + "view": { + "continuousWidth": 300, + "continuousHeight": 300 + } + }, + "data": { + "name": "data-204dd5b444069061018d736b1906f1e7" + }, + "mark": { + "type": "bar" + }, + "encoding": { + "color": { + "field": "measure", + "type": "nominal" + }, + "tooltip": [ + { + "field": "origin", + "type": "nominal" + }, + { + "field": "measure", + "type": "nominal" + }, + { + "field": "value", + "type": "quantitative" + } + ], + "x": { + "field": "origin", + "sort": null, + "type": "ordinal" + }, + "xOffset": { + "field": "measure" + }, + "y": { + "field": "value", + "type": "quantitative" + } + }, + "height": 400, + "transform": [ + { + "fold": [ + "flight_count", + "avg_distance" + ], + "as": [ + "measure", + "value" + ] + } + ], + "width": 700, + "$schema": "https://vega.github.io/schema/vega-lite/v5.20.1.json", + "datasets": { + "data-204dd5b444069061018d736b1906f1e7": [ + { + "origin": "SFO", + "flight_count": 1, + "avg_distance": 382.0 + }, + { + "origin": "JFK", + "flight_count": 1, + "avg_distance": 2475.0 + }, + { + "origin": "LAX", + "flight_count": 1, + "avg_distance": 337.0 + } + ] + } + } } } }, diff --git a/examples/window_functions.py b/examples/window_functions.py index 692e7a9e..00caab48 100644 --- a/examples/window_functions.py +++ b/examples/window_functions.py @@ -32,8 +32,12 @@ def main(): .filter(lambda t: t.carrier == "WN") ) + # A filtered query result is a plain table, so row math over it drops to + # ibis explicitly via .to_untagged() (mutate on the *aggregate itself* + # stays semantic — see the percent_manual example below). result = ( - daily_stats.mutate( + daily_stats.to_untagged() + .mutate( rolling_avg=lambda t: t.flight_count.mean().over( xibis.window(order_by=t.flight_date, preceding=6, following=0), ), diff --git a/src/boring_semantic_layer/agents/help_topics.py b/src/boring_semantic_layer/agents/help_topics.py index ff98b1af..6320a35d 100644 --- a/src/boring_semantic_layer/agents/help_topics.py +++ b/src/boring_semantic_layer/agents/help_topics.py @@ -324,29 +324,30 @@ "content": """\ Window Functions — calculations across ordered rows -Windows run over the QUERY RESULT, so drop to ibis first with -.to_untagged(), then use .mutate() with the window carrying its own -ordering: +Apply via .mutate() DIRECTLY on the aggregate (before .order_by()/.limit()), +with the window carrying its own ordering — use the keyword form of .over() +(no ibis.window() object needed): ROLLING AVERAGE: - model.group_by("week").aggregate("count").to_untagged().mutate( - rolling_avg=lambda t: t["count"].mean().over( - rows=(-9, 0), order_by="week" - ) + model.group_by("week").aggregate("count").mutate( + rolling_avg=lambda t: t["count"].mean().over(rows=(-9, 0), order_by="week") ).order_by("week") -CUMULATIVE SUM: - .to_untagged().mutate(running_total=lambda t: t.revenue.cumsum()) - -RANK: - .to_untagged().mutate(rank=lambda t: t.revenue.rank()) +CUMULATIVE / RUNNING TOTAL: + .mutate(running_total=lambda t: t.revenue.sum().over( + rows=(None, 0), order_by="week" + )) LAG / LEAD: - .to_untagged().mutate( - prev_count=lambda t: t["count"].lag(1), - next_count=lambda t: t["count"].lead(1), + .mutate( + prev_count=lambda t: t["count"].lag(1).over(order_by="week"), ) +Result stays semantic — .chart() and further order_by/limit keep working. + +On a FILTERED result (.aggregate().filter(...)), the result is a plain +table: drop to ibis first with .to_untagged().mutate(...). + See: bsl docs query mutate""", }, "percent": { diff --git a/src/boring_semantic_layer/chart/md_parser/executor.py b/src/boring_semantic_layer/chart/md_parser/executor.py index 0f04f813..45a6beac 100644 --- a/src/boring_semantic_layer/chart/md_parser/executor.py +++ b/src/boring_semantic_layer/chart/md_parser/executor.py @@ -187,22 +187,26 @@ def _process_result( if is_chart_only and hasattr(result, "to_dict"): return self._extract_chart_spec(result) - # Semantic table definition - if hasattr(result, "group_by") and not hasattr(result, "execute"): - return { - "semantic_table": True, - "name": getattr(result, "name", "unknown"), - "info": "Semantic table definition stored in context", - } - # BSL query with execute method if hasattr(result, "execute"): + from boring_semantic_layer.errors import QueryError + from .converter import ResultConverter - result_data, _ = ResultConverter.convert_bsl_result( - result, code, self.context, is_chart_only - ) - return result_data + try: + result_data, _ = ResultConverter.convert_bsl_result( + result, code, self.context, is_chart_only + ) + return result_data + except QueryError: + # A definition-side semantic expression refuses to execute: + # render it as a definition instead of a query result (the + # frontend shows a "semantic table defined" box). + return { + "semantic_table": True, + "name": getattr(result, "name", None) or "unknown", + "info": "Semantic table definition stored in context", + } # Pandas-like object if hasattr(result, "to_pandas"): diff --git a/src/boring_semantic_layer/chart/tests/test_chart.py b/src/boring_semantic_layer/chart/tests/test_chart.py index 4ea40ad7..397e0299 100644 --- a/src/boring_semantic_layer/chart/tests/test_chart.py +++ b/src/boring_semantic_layer/chart/tests/test_chart.py @@ -389,12 +389,12 @@ def test_chart_with_dynamic_dimension_and_rolling_window(self, flights_model): flights_model.with_dimensions(flight_week=lambda t: t.flight_date.truncate("W")) .group_by("flight_week") .aggregate("flight_count") - .order_by("flight_week") .mutate( rolling_avg=lambda t: t.flight_count.mean().over( xibis.window(rows=(-2, 0), order_by="flight_week") ) ) + .order_by("flight_week") ) # The query should execute successfully @@ -407,15 +407,19 @@ def test_chart_with_dynamic_dimension_and_rolling_window(self, flights_model): # Just verify no exception was raised def test_chart_time_detection_after_limit_mutate(self, flights_model): - """Post-wrapper mutate preserves time metadata for chart detection.""" + """Mutate-then-wrapper chains preserve time metadata for chart detection. + + (Mutate goes on the aggregate itself — after order_by/limit the + result is a plain table and .mutate() raises.) + """ from boring_semantic_layer.chart.utils import get_chart_detection_params result = ( flights_model.group_by("flight_date") .aggregate("flight_count") + .mutate(doubled=lambda t: t.flight_count * 2) .order_by("flight_date") .limit(3) - .mutate(doubled=lambda t: t.flight_count * 2) ) df = result.execute() diff --git a/src/boring_semantic_layer/chart/tests/test_md_executor.py b/src/boring_semantic_layer/chart/tests/test_md_executor.py index 4457b340..a39d4b7f 100644 --- a/src/boring_semantic_layer/chart/tests/test_md_executor.py +++ b/src/boring_semantic_layer/chart/tests/test_md_executor.py @@ -28,19 +28,48 @@ def test_context_persistence(self): assert executor.context["y"] == 84 def test_bsl_query_execution(self): - """Test execution of BSL query with .execute().""" + """Test execution of BSL query with .execute(). + + The block must end with a query — a bare semantic model is a + definition and refuses to execute. + """ executor = QueryExecutor() code = """ import ibis from boring_semantic_layer import to_semantic_table t = ibis.memtable({"x": [1, 2, 3], "y": [10, 20, 30]}) -result = to_semantic_table(t) +result = ( + to_semantic_table(t) + .with_dimensions(x=lambda t: t.x) + .with_measures(total=lambda t: t.y.sum()) + .group_by("x") + .aggregate("total") +) """ result = executor.execute(code) assert "table" in result assert "sql" in result - assert result["table"]["columns"] == ["x", "y"] + assert result["table"]["columns"] == ["x", "total"] + + def test_bare_model_renders_as_definition(self): + """A block ending with a bare model renders a definition box, not rows.""" + executor = QueryExecutor() + code = """ +import ibis +from boring_semantic_layer import to_semantic_table + +t = ibis.memtable({"x": [1, 2, 3], "y": [10, 20, 30]}) +result = ( + to_semantic_table(t, name="points") + .with_dimensions(x=lambda t: t.x) + .with_measures(total=lambda t: t.y.sum()) +) +""" + result = executor.execute(code) + assert result.get("semantic_table") is True + assert result["name"] == "points" + assert "error" not in result def test_error_handling(self): """Test error handling for invalid code.""" diff --git a/src/boring_semantic_layer/tests/test_query.py b/src/boring_semantic_layer/tests/test_query.py index 2b007d34..0f4c544c 100644 --- a/src/boring_semantic_layer/tests/test_query.py +++ b/src/boring_semantic_layer/tests/test_query.py @@ -320,7 +320,10 @@ def test_filter_on_first_level_derived_still_works(self, flights_st): def test_chained_filters_on_derived_dims(self, flights_st): """Stacked filter().filter() both referencing derived dimensions.""" result = ( - flights_st.filter(ibis._.d_one > 500).filter(ibis._.d_two > 1000).to_untagged().execute() + flights_st.filter(ibis._.d_one > 500) + .filter(ibis._.d_two > 1000) + .to_untagged() + .execute() ) assert len(result) > 0 assert all(result["d_one"] > 500)