diff --git a/docs/md/doc/builder-agent.md b/docs/md/doc/builder-agent.md
index 039133a0..d2006c72 100644
--- a/docs/md/doc/builder-agent.md
+++ b/docs/md/doc/builder-agent.md
@@ -7,9 +7,9 @@ The Builder Agent is focused on authoring and evolving semantic tables: defining
- File: [`docs/md/skills/claude-code/bsl-model-builder/SKILL.md`](../skills/claude-code/bsl-model-builder/SKILL.md)
- Use it when you want Claude Desktop to help write new semantic tables, add time dimensions, or compose models.
- The skill includes:
- - Python DSL examples showing `SemanticTable(...)`, `.with_dimensions`, `.with_measures`, `.with_calculated_measures`, and `.join()` patterns.
+ - Python DSL examples showing `to_semantic_table(...)`, `.with_dimensions`, `.with_measures`, `join_one()`, `join_many()`, and `join_cross()` patterns.
- YAML equivalents so you can copy the same logic into declarative configs.
- - Gotchas such as "measures must aggregate" and "join keys must be defined dimensions".
+ - Gotchas such as choosing cardinality from the left side, using source-aware equality predicates, and assigning unique model aliases.
**Workflow:** Load the skill in Claude Desktop, paste the schema or YAML snippet you are editing, and ask "Generate a semantic table for flights with avg delay and join to airports". Claude will respond with both Python and YAML patterns that mirror the documentation.
diff --git a/docs/md/doc/compose.md b/docs/md/doc/compose.md
index 7791de6e..97d21f44 100644
--- a/docs/md/doc/compose.md
+++ b/docs/md/doc/compose.md
@@ -10,6 +10,17 @@ Model composition in BSL is achieved through **joins**. When you join semantic t
Each join creates a new semantic model with the combined dimensions and measures from all joined tables. This allows you to build progressively richer models.
+Choose cardinality from the perspective of the left model: use `join_one()`
+when each left row can match at most one right row, and `join_many()` when one
+left row can match multiple right rows. Every source model in a join tree must
+also have a unique `name`, because that name becomes its field prefix and source
+identity.
+
+For source-aware aggregation, non-cross joins support direct field equality
+predicates and conjunctions of direct field equalities. Inequality, `OR`, cast,
+and transformed predicates are rejected when source pre-aggregation is needed;
+aggregate each model first or restate the relationship as equality keys.
+
## Example: Two-Level Composition
Let's build a composed model step-by-step, showing available dimensions and measures at each level.
@@ -108,8 +119,8 @@ flights_st.dimensions, flights_st.measures
Join carriers to flights to add carrier information:
```level1_join
-# Join carriers to flights
-flights_with_carriers = flights_st.join_many(
+# Each flight row matches at most one carrier row
+flights_with_carriers = flights_st.join_one(
carriers_st,
lambda f, c: f.carrier_code == c.code
)
@@ -124,8 +135,8 @@ flights_with_carriers.dimensions, flights_with_carriers.measures
Add aircraft information to create a fully composed model:
```level2_join
-# Join aircraft to the composed model
-full_model = flights_with_carriers.join_many(
+# Each flight row matches at most one aircraft row
+full_model = flights_with_carriers.join_one(
aircraft_st,
lambda f, a: f.aircraft_id == a.id
)
@@ -152,9 +163,10 @@ result = (
## Key Takeaways
-- **Composition via Joins**: Use `join_many()`, `join_one()`, or `join_cross()` to compose models
+- **Composition via Joins**: Use `join_one()` for at-most-one right matches, `join_many()` for multiple right matches, and `join_cross()` for Cartesian products
- **Additive**: Each join adds dimensions and measures from the joined table
- **Table Prefixes**: Dimensions/measures are prefixed with table names (`flights.`, `carriers.`, `aircraft.`)
+- **Unique Source Names**: Give every base model in the join tree a distinct `name`
- **No Limit**: Compose as many models as needed for your analysis
- **Incremental**: Build from simple to complex, one join at a time
diff --git a/docs/md/doc/reference.md b/docs/md/doc/reference.md
index 9f6b7cd8..41bb41fb 100644
--- a/docs/md/doc/reference.md
+++ b/docs/md/doc/reference.md
@@ -144,17 +144,19 @@ flights_st = flights_st.with_measures(
### all()
```python
-st.all()
+t.all(measure_or_reduction)
```
-Reference the entire dataset within measure definitions. Primarily used for percentage-of-total calculations.
+Reference the entire dataset within a measure definition. Pass a declared
+measure reference, measure name, or Ibis reduction; `all()` does not have a
+zero-argument form. It is primarily used for percentage-of-total calculations.
**Example:**
```python
flights_st = to_semantic_table(data, "flights").with_measures(
flight_count=lambda t: t.count(),
pct_of_total=lambda t: (
- t.count() / t.all().count() * 100
+ t.flight_count / t.all(t.flight_count) * 100
)
)
```
@@ -168,9 +170,8 @@ Methods for composing semantic tables through joins.
```python
join_many(
other: SemanticTable,
- on: Callable,
- how: str = "left",
- name: str = None
+ on: Callable | str | Deferred | Sequence[str | Deferred],
+ how: str = "left"
) -> SemanticTable
```
@@ -179,32 +180,39 @@ One-to-many relationship join (LEFT JOIN). Use when the left table can match mul
| Parameter | Type | Description |
|-----------|------|-------------|
| `other` | `SemanticTable` | The semantic table to join with |
-| `on` | `Callable` | Lambda function defining the join condition |
+| `on` | `Callable \| str \| Deferred \| Sequence[str \| Deferred]` | Join predicate: a two-argument lambda, same-name column shorthand, or sequence of shorthands for a compound equijoin |
| `how` | `str` | Join type; only `'left'` is supported |
-| `name` | `str` | Optional name for the joined table reference |
**Example:**
```python
-flights_st = flights_st.join_many(
- carriers_st,
- on=lambda l, r: l.carrier == r.code,
- name="carrier_info"
+carriers_with_flights = carriers_st.join_many(
+ flights_st,
+ on=lambda carrier, flight: carrier.code == flight.carrier
)
```
+Model names are the source aliases used in prefixed fields. Every base model in
+a composed join tree must therefore have a unique `name`; assign explicit names
+with `to_semantic_table(..., name="...")` before joining. When the same physical
+table serves multiple roles, use both `.view()` and distinct model names.
+
+Source-aware aggregation supports direct field equijoins and conjunctions of
+direct field equijoins. Inequality, `OR`, cast, and transformed predicates are
+rejected when source pre-aggregation is required because they cannot be safely
+reconstructed from key columns alone.
+
### join_one()
```python
join_one(
other: SemanticTable,
- on: Callable,
- how: str = "left",
- name: str = None
+ on: Callable | str | Deferred | Sequence[str | Deferred],
+ how: str = "left"
) -> SemanticTable
```
-One-to-one relationship join (LEFT JOIN). Use when each left row can match at
-most one row in the right table. Only `how="left"` is supported.
+At-most-one-right-match relationship join (LEFT JOIN). Use when each left row
+can match at most one row in the right table. Only `how="left"` is supported.
**Example:**
```python
@@ -218,8 +226,7 @@ flights_st = flights_st.join_one(
```python
join_cross(
- other: SemanticTable,
- name: str = None
+ other: SemanticTable
) -> SemanticTable
```
diff --git a/docs/md/doc/semantic-table.md b/docs/md/doc/semantic-table.md
index 977eadb1..f9be7b25 100644
--- a/docs/md/doc/semantic-table.md
+++ b/docs/md/doc/semantic-table.md
@@ -112,7 +112,9 @@ result = (
-`t.all()` is a method available on the table parameter `t` in measure definitions. It references the entire dataset regardless of grouping, making it perfect for calculating percentages, or comparing groups to the total.
+`t.all(ref)` is available on the table parameter `t` in measure definitions. It
+evaluates the supplied measure or reduction over the entire dataset regardless
+of grouping, making it useful for percentages and comparisons with the total.
For more examples, see the [Percent of Total pattern](/advanced/percentage-total).
@@ -205,14 +207,14 @@ Semantic joins explicitly capture the **relationship type** between tables, rath
**SQL Joins:**
```python
-# Specifies HOW to join (LEFT/INNER), but not the relationship
-flights.join(carriers, condition, how="left")
+# Specifies HOW to join, but not the analytical relationship
+flights_tbl.left_join(carriers_tbl, flights_tbl.carrier == carriers_tbl.code)
```
**Semantic Joins:**
```python
-# Specifies the relationship: one carrier has many flights
-flights.join_many(carriers, lambda f, c: f.carrier == c.code)
+# One carrier row can match many flight rows
+carriers.join_many(flights_st, lambda c, f: c.code == f.carrier)
```
**What You Get:**
@@ -225,7 +227,11 @@ After joining, dimensions and measures are prefixed with table names (e.g., `fli
-**Joining the same table multiple times?** If you need to join to the same source table via different foreign keys (e.g., pickup and dropoff locations), you must use `.view()` to create distinct table references:
+**Give every source in a composed model a unique name.** BSL uses model names as
+source aliases for dimensions, measures, and grain metadata, and rejects a join
+tree containing duplicate names. If you join the same underlying table more than
+once (for example, pickup and dropoff locations), create distinct table references
+and assign explicit aliases:
```python
# Create distinct references when joining same table twice
@@ -233,7 +239,19 @@ pickup_locs = to_semantic_table(locs_tbl.view(), "pickup_locs")
dropoff_locs = to_semantic_table(locs_tbl.view(), "dropoff_locs")
```
-Without `.view()`, you'll encounter an `IbisInputError: Ambiguous field reference` error.
+The distinct names prevent ambiguous semantic prefixes; `.view()` prevents Ibis
+from treating both roles as the same relation.
+
+
+
+**Source-aware aggregation requires equality-key joins.** When BSL aggregates
+measures at their source grain before joining, each non-cross join predicate must
+be a direct field equality or a conjunction of direct field equalities. String,
+Deferred, and compound equality-key shorthands are supported. Predicates using
+inequality, `OR`, casts, or transformed expressions are rejected because reducing
+them to join-key bridges could change the matched row set. Aggregate the models
+first or restate the relationship with plain equality keys; use `join_cross()` for
+a Cartesian product.
Let's get some additional data:
@@ -273,28 +291,28 @@ carriers = (
Use `join_many()` when one row in the left table can match multiple rows in the right table (LEFT JOIN).
```join_demo
-# Join carriers to flights - one carrier has many flights
-flights_with_carriers = flights_st.join_many(
- carriers,
- lambda f, c: f.carrier == c.code
+# One carrier row can match many flight rows
+carriers_with_flights = carriers.join_many(
+ flights_st,
+ lambda c, f: c.code == f.carrier
)
# Inspect available dimensions and measures
-flights_with_carriers.dimensions
+carriers_with_flights.dimensions
```
After joining, all dimensions and measures from both tables are available. Each is prefixed with its table name to avoid conflicts:
-### join_one() - One-to-One Relationships
+### join_one() - At-Most-One Right Match
-Use `join_one()` when rows have a unique matching relationship. Like all
-non-cross semantic joins, it uses a LEFT JOIN so unmatched left rows remain
-visible to measures.
+Use `join_one()` when each row on the left can match at most one row on the
+right. Like all non-cross semantic joins, it uses a LEFT JOIN so unmatched left
+rows remain visible to measures.
```python
-# Many flights → one carrier (each flight has exactly one carrier)
+# Many flights → one carrier (each flight matches at most one carrier)
flights_with_carrier = flights_st.join_one(
carriers,
lambda f, c: f.carrier == c.code
@@ -302,25 +320,20 @@ flights_with_carrier = flights_st.join_one(
```
-**Important Limitation:** Currently, `left_on` and `right_on` must be **COLUMN names**, not dimension names.
-
-If you have a dimension that maps to a different column name, you must use the underlying column name in the join.
+**Join predicates resolve physical columns.** A string or Deferred shorthand
+names the same underlying column on both sides. If the columns have different
+names, use a two-argument lambda instead.
**Example:**
```python
# If users table has column 'id' but dimension 'customer_id':
-users = to_semantic_table(users_tbl).with_dimensions(
+users = to_semantic_table(users_tbl, "users").with_dimensions(
customer_id=lambda t: t.id # Dimension renamed
)
-# ❌ This will fail with a helpful error:
-orders.join_one(users, left_on="customer_id", right_on="customer_id")
-
-# ✓ Use the actual column name:
-orders.join_one(users, left_on="customer_id", right_on="id")
+# Compare the underlying columns explicitly:
+orders.join_one(users, on=lambda order, user: order.customer_id == user.id)
```
-
-This is a known limitation tracked in [issue #43](https://github.com/boringdata/boring-semantic-layer/issues/43). If you attempt to use a dimension name that doesn't match a column name, you'll get a helpful error message guiding you to use the correct column name.
### join_cross() - Cross Join
diff --git a/docs/md/prompts/build/system.md b/docs/md/prompts/build/system.md
index 1014ce93..56ab6b21 100644
--- a/docs/md/prompts/build/system.md
+++ b/docs/md/prompts/build/system.md
@@ -81,9 +81,10 @@ flights_st = flights_st.with_measures(
)
```
-### Percent of Total with all()
+### Percent of Total with all(ref)
-Use `t.all()` to reference the entire dataset:
+Pass a declared measure or reduction to `t.all(...)` to reference its value over
+the entire dataset. There is no zero-argument `t.all()` form:
```python
flights_st = flights_st.with_measures(
@@ -98,16 +99,16 @@ flights_st = flights_st.with_measures(
```python
# One carrier has many flights
-flights_with_carriers = flights_st.join_many(
- carriers_st,
- lambda f, c: f.carrier == c.code
+carriers_with_flights = carriers_st.join_many(
+ flights_st,
+ lambda c, f: c.code == f.carrier
)
```
-### join_one() - One-to-One (LEFT JOIN)
+### join_one() - At-Most-One Right Match (LEFT JOIN)
```python
-# Each flight has exactly one carrier
+# Each flight matches at most one carrier
flights_with_carrier = flights_st.join_one(
carriers_st,
lambda f, c: f.carrier == c.code
@@ -120,19 +121,21 @@ flights_with_carrier = flights_st.join_one(
all_combinations = flights_st.join_cross(carriers_st)
```
-### Custom Joins
+### Join Predicates and Source-Aware Aggregation
-```python
-flights_st.join(
- carriers_st,
- lambda f, c: f.carrier == c.code,
- how="left" # "inner", "left", "right", "outer", "cross"
-)
-```
+`join_one()` and `join_many()` are left joins. Their `on=` argument can be a
+column-name string, Deferred key, sequence of equality keys, or a lambda whose
+predicate is a direct field equality (or conjunction of direct field
+equalities). Source-aware aggregation rejects inequality, `OR`, cast, and
+transformed join predicates because pre-aggregating those predicates from key
+columns could change the matched rows. Aggregate each model first or restate the
+relationship as equality keys. Use `join_cross()` for Cartesian products.
**After joins**: Fields are prefixed with table names (e.g., `flights.origin`, `carriers.name`)
-**Multiple joins to same table**: Use `.view()` to create distinct references:
+**Unique aliases are required**: Every base model in a composed join tree must
+have a distinct `name`. When the same physical table plays multiple roles, use
+both `.view()` and explicit model aliases:
```python
pickup_locs = to_semantic_table(locs_tbl.view(), "pickup_locs")
dropoff_locs = to_semantic_table(locs_tbl.view(), "dropoff_locs")
@@ -193,6 +196,7 @@ flights_sm = models["flights"]
3. **Define composed measures** to avoid repetition
4. **Use YAML** for production models (version control, collaboration)
5. **Use profiles** for database connections (see Profile docs)
+6. **Choose join cardinality from the left side** and give every joined source a unique name
## Common Patterns
diff --git a/docs/md/skills/claude-code/bsl-model-builder/SKILL.md b/docs/md/skills/claude-code/bsl-model-builder/SKILL.md
index a94c7d9b..5bf246cf 100644
--- a/docs/md/skills/claude-code/bsl-model-builder/SKILL.md
+++ b/docs/md/skills/claude-code/bsl-model-builder/SKILL.md
@@ -86,9 +86,10 @@ flights_st = flights_st.with_measures(
)
```
-### Percent of Total with all()
+### Percent of Total with all(ref)
-Use `t.all()` to reference the entire dataset:
+Pass a declared measure or reduction to `t.all(...)` to reference its value over
+the entire dataset. There is no zero-argument `t.all()` form:
```python
flights_st = flights_st.with_measures(
@@ -103,16 +104,16 @@ flights_st = flights_st.with_measures(
```python
# One carrier has many flights
-flights_with_carriers = flights_st.join_many(
- carriers_st,
- lambda f, c: f.carrier == c.code
+carriers_with_flights = carriers_st.join_many(
+ flights_st,
+ lambda c, f: c.code == f.carrier
)
```
-### join_one() - One-to-One (LEFT JOIN)
+### join_one() - At-Most-One Right Match (LEFT JOIN)
```python
-# Each flight has exactly one carrier
+# Each flight matches at most one carrier
flights_with_carrier = flights_st.join_one(
carriers_st,
lambda f, c: f.carrier == c.code
@@ -125,19 +126,21 @@ flights_with_carrier = flights_st.join_one(
all_combinations = flights_st.join_cross(carriers_st)
```
-### Custom Joins
+### Join Predicates and Source-Aware Aggregation
-```python
-flights_st.join(
- carriers_st,
- lambda f, c: f.carrier == c.code,
- how="left" # "inner", "left", "right", "outer", "cross"
-)
-```
+`join_one()` and `join_many()` are left joins. Their `on=` argument can be a
+column-name string, Deferred key, sequence of equality keys, or a lambda whose
+predicate is a direct field equality (or conjunction of direct field
+equalities). Source-aware aggregation rejects inequality, `OR`, cast, and
+transformed join predicates because pre-aggregating those predicates from key
+columns could change the matched rows. Aggregate each model first or restate the
+relationship as equality keys. Use `join_cross()` for Cartesian products.
**After joins**: Fields are prefixed with table names (e.g., `flights.origin`, `carriers.name`)
-**Multiple joins to same table**: Use `.view()` to create distinct references:
+**Unique aliases are required**: Every base model in a composed join tree must
+have a distinct `name`. When the same physical table plays multiple roles, use
+both `.view()` and explicit model aliases:
```python
pickup_locs = to_semantic_table(locs_tbl.view(), "pickup_locs")
dropoff_locs = to_semantic_table(locs_tbl.view(), "dropoff_locs")
@@ -198,6 +201,7 @@ flights_sm = models["flights"]
3. **Define composed measures** to avoid repetition
4. **Use YAML** for production models (version control, collaboration)
5. **Use profiles** for database connections (see Profile docs)
+6. **Choose join cardinality from the left side** and give every joined source a unique name
## Common Patterns
diff --git a/docs/md/skills/codex/bsl-model-builder.codex b/docs/md/skills/codex/bsl-model-builder.codex
index 25a196b8..23cd1ac3 100644
--- a/docs/md/skills/codex/bsl-model-builder.codex
+++ b/docs/md/skills/codex/bsl-model-builder.codex
@@ -85,9 +85,10 @@ flights_st = flights_st.with_measures(
)
```
-### Percent of Total with all()
+### Percent of Total with all(ref)
-Use `t.all()` to reference the entire dataset:
+Pass a declared measure or reduction to `t.all(...)` to reference its value over
+the entire dataset. There is no zero-argument `t.all()` form:
```python
flights_st = flights_st.with_measures(
@@ -102,16 +103,16 @@ flights_st = flights_st.with_measures(
```python
# One carrier has many flights
-flights_with_carriers = flights_st.join_many(
- carriers_st,
- lambda f, c: f.carrier == c.code
+carriers_with_flights = carriers_st.join_many(
+ flights_st,
+ lambda c, f: c.code == f.carrier
)
```
-### join_one() - One-to-One (LEFT JOIN)
+### join_one() - At-Most-One Right Match (LEFT JOIN)
```python
-# Each flight has exactly one carrier
+# Each flight matches at most one carrier
flights_with_carrier = flights_st.join_one(
carriers_st,
lambda f, c: f.carrier == c.code
@@ -124,19 +125,21 @@ flights_with_carrier = flights_st.join_one(
all_combinations = flights_st.join_cross(carriers_st)
```
-### Custom Joins
+### Join Predicates and Source-Aware Aggregation
-```python
-flights_st.join(
- carriers_st,
- lambda f, c: f.carrier == c.code,
- how="left" # "inner", "left", "right", "outer", "cross"
-)
-```
+`join_one()` and `join_many()` are left joins. Their `on=` argument can be a
+column-name string, Deferred key, sequence of equality keys, or a lambda whose
+predicate is a direct field equality (or conjunction of direct field
+equalities). Source-aware aggregation rejects inequality, `OR`, cast, and
+transformed join predicates because pre-aggregating those predicates from key
+columns could change the matched rows. Aggregate each model first or restate the
+relationship as equality keys. Use `join_cross()` for Cartesian products.
**After joins**: Fields are prefixed with table names (e.g., `flights.origin`, `carriers.name`)
-**Multiple joins to same table**: Use `.view()` to create distinct references:
+**Unique aliases are required**: Every base model in a composed join tree must
+have a distinct `name`. When the same physical table plays multiple roles, use
+both `.view()` and explicit model aliases:
```python
pickup_locs = to_semantic_table(locs_tbl.view(), "pickup_locs")
dropoff_locs = to_semantic_table(locs_tbl.view(), "dropoff_locs")
@@ -197,6 +200,7 @@ flights_sm = models["flights"]
3. **Define composed measures** to avoid repetition
4. **Use YAML** for production models (version control, collaboration)
5. **Use profiles** for database connections (see Profile docs)
+6. **Choose join cardinality from the left side** and give every joined source a unique name
## Common Patterns
diff --git a/docs/md/skills/cursor/bsl-model-builder.mdc b/docs/md/skills/cursor/bsl-model-builder.mdc
index 9cfd4ca6..f9ab1bf2 100644
--- a/docs/md/skills/cursor/bsl-model-builder.mdc
+++ b/docs/md/skills/cursor/bsl-model-builder.mdc
@@ -87,9 +87,10 @@ flights_st = flights_st.with_measures(
)
```
-### Percent of Total with all()
+### Percent of Total with all(ref)
-Use `t.all()` to reference the entire dataset:
+Pass a declared measure or reduction to `t.all(...)` to reference its value over
+the entire dataset. There is no zero-argument `t.all()` form:
```python
flights_st = flights_st.with_measures(
@@ -104,16 +105,16 @@ flights_st = flights_st.with_measures(
```python
# One carrier has many flights
-flights_with_carriers = flights_st.join_many(
- carriers_st,
- lambda f, c: f.carrier == c.code
+carriers_with_flights = carriers_st.join_many(
+ flights_st,
+ lambda c, f: c.code == f.carrier
)
```
-### join_one() - One-to-One (LEFT JOIN)
+### join_one() - At-Most-One Right Match (LEFT JOIN)
```python
-# Each flight has exactly one carrier
+# Each flight matches at most one carrier
flights_with_carrier = flights_st.join_one(
carriers_st,
lambda f, c: f.carrier == c.code
@@ -126,19 +127,21 @@ flights_with_carrier = flights_st.join_one(
all_combinations = flights_st.join_cross(carriers_st)
```
-### Custom Joins
+### Join Predicates and Source-Aware Aggregation
-```python
-flights_st.join(
- carriers_st,
- lambda f, c: f.carrier == c.code,
- how="left" # "inner", "left", "right", "outer", "cross"
-)
-```
+`join_one()` and `join_many()` are left joins. Their `on=` argument can be a
+column-name string, Deferred key, sequence of equality keys, or a lambda whose
+predicate is a direct field equality (or conjunction of direct field
+equalities). Source-aware aggregation rejects inequality, `OR`, cast, and
+transformed join predicates because pre-aggregating those predicates from key
+columns could change the matched rows. Aggregate each model first or restate the
+relationship as equality keys. Use `join_cross()` for Cartesian products.
**After joins**: Fields are prefixed with table names (e.g., `flights.origin`, `carriers.name`)
-**Multiple joins to same table**: Use `.view()` to create distinct references:
+**Unique aliases are required**: Every base model in a composed join tree must
+have a distinct `name`. When the same physical table plays multiple roles, use
+both `.view()` and explicit model aliases:
```python
pickup_locs = to_semantic_table(locs_tbl.view(), "pickup_locs")
dropoff_locs = to_semantic_table(locs_tbl.view(), "dropoff_locs")
@@ -199,6 +202,7 @@ flights_sm = models["flights"]
3. **Define composed measures** to avoid repetition
4. **Use YAML** for production models (version control, collaboration)
5. **Use profiles** for database connections (see Profile docs)
+6. **Choose join cardinality from the left side** and give every joined source a unique name
## Common Patterns
diff --git a/docs/web/public/bsl-data/compose.json b/docs/web/public/bsl-data/compose.json
index 7ee6c8be..9745ab34 100644
--- a/docs/web/public/bsl-data/compose.json
+++ b/docs/web/public/bsl-data/compose.json
@@ -1,5 +1,5 @@
{
- "markdown": "# Composing Models\n\nBuild complex data models by combining multiple semantic tables through joins. Model composition allows you to create rich, multi-dimensional views of your data.\n\n## Composition via Joins\n\nModel composition in BSL is achieved through **joins**. When you join semantic tables, the result is a new composed model that contains **all dimensions and measures** from both tables.\n\n\nEach join creates a new semantic model with the combined dimensions and measures from all joined tables. This allows you to build progressively richer models.\n\n\n## Example: Two-Level Composition\n\nLet's build a composed model step-by-step, showing available dimensions and measures at each level.\n\n### Level 0: Base Models\n\nFirst, let's set up our base tables:\n\n```setup_ibis_tables\nimport ibis\nfrom boring_semantic_layer import to_semantic_table\n\n# Create sample data\ncon = ibis.duckdb.connect(\":memory:\")\n\n# Flights table\nflights_data = ibis.memtable({\n \"flight_id\": [1, 2, 3],\n \"carrier_code\": [\"AA\", \"UA\", \"DL\"],\n \"aircraft_id\": [101, 102, 103],\n \"distance\": [1000, 1500, 800],\n \"passengers\": [150, 180, 120]\n})\nflights_tbl = con.create_table(\"flights\", flights_data)\n\n# Carriers table\ncarriers_data = ibis.memtable({\n \"code\": [\"AA\", \"UA\", \"DL\"],\n \"name\": [\"American Airlines\", \"United Airlines\", \"Delta Air Lines\"],\n \"country\": [\"USA\", \"USA\", \"USA\"]\n})\ncarriers_tbl = con.create_table(\"carriers\", carriers_data)\n\n# Aircraft table\naircraft_data = ibis.memtable({\n \"id\": [101, 102, 103],\n \"model\": [\"Boeing 737\", \"Airbus A320\", \"Boeing 777\"],\n \"capacity\": [180, 200, 350]\n})\naircraft_tbl = con.create_table(\"aircraft\", aircraft_data)\n```\n\n\n\n```setup_semantic_models\n# Create semantic tables\nflights_st = (\n to_semantic_table(flights_tbl, name=\"flights\")\n .with_dimensions(\n flight_id=lambda t: t.flight_id,\n carrier_code=lambda t: t.carrier_code,\n aircraft_id=lambda t: t.aircraft_id\n )\n .with_measures(\n flight_count=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n total_passengers=lambda t: t.passengers.sum()\n )\n)\n\ncarriers_st = (\n to_semantic_table(carriers_tbl, name=\"carriers\")\n .with_dimensions(\n code=lambda t: t.code,\n name=lambda t: t.name,\n country=lambda t: t.country\n )\n .with_measures(\n carrier_count=lambda t: t.count()\n )\n)\n\naircraft_st = (\n to_semantic_table(aircraft_tbl, name=\"aircraft\")\n .with_dimensions(\n id=lambda t: t.id,\n model=lambda t: t.model\n )\n .with_measures(\n aircraft_count=lambda t: t.count(),\n total_capacity=lambda t: t.capacity.sum()\n )\n)\n```\n\n\n\n```level0_dimensions\nflights_st.dimensions, flights_st.measures\n```\n\n\n\n### Level 1: First Join (Flights + Carriers)\n\nJoin carriers to flights to add carrier information:\n\n```level1_join\n# Join carriers to flights\nflights_with_carriers = flights_st.join_many(\n carriers_st,\n lambda f, c: f.carrier_code == c.code\n)\n\n# Inspect dimensions - now includes both flights and carriers\nflights_with_carriers.dimensions, flights_with_carriers.measures\n```\n\n\n### Level 2: Second Join (+ Aircraft)\n\nAdd aircraft information to create a fully composed model:\n\n```level2_join\n# Join aircraft to the composed model\nfull_model = flights_with_carriers.join_many(\n aircraft_st,\n lambda f, a: f.aircraft_id == a.id\n)\n\n# Inspect dimensions - now includes flights, carriers, AND aircraft\nfull_model.dimensions, full_model.measures\n```\n\n\n## Query the Composed Model\n\nNow you can query across all joined tables:\n\n```composed_query\n# Query using dimensions and measures from all three tables\nresult = (\n full_model\n .group_by( \"aircraft.model\")\n .aggregate(\"flights.flight_count\", \"flights.total_passengers\", \"aircraft.total_capacity\")\n)\n```\n\n\n\n## Key Takeaways\n\n- **Composition via Joins**: Use `join_many()`, `join_one()`, or `join()` to compose models\n- **Additive**: Each join adds dimensions and measures from the joined table\n- **Table Prefixes**: Dimensions/measures are prefixed with table names (`flights.`, `carriers.`, `aircraft.`)\n- **No Limit**: Compose as many models as needed for your analysis\n- **Incremental**: Build from simple to complex, one join at a time\n\n## Next Steps\n\n- Learn about [YAML Configuration](/building/yaml) for declarative model composition\n- Explore [Query Methods](/querying/methods) for querying composed models\n",
+ "markdown": "# Composing Models\n\nBuild complex data models by combining multiple semantic tables through joins. Model composition allows you to create rich, multi-dimensional views of your data.\n\n## Composition via Joins\n\nModel composition in BSL is achieved through **joins**. When you join semantic tables, the result is a new composed model that contains **all dimensions and measures** from both tables.\n\n\nEach join creates a new semantic model with the combined dimensions and measures from all joined tables. This allows you to build progressively richer models.\n\n\nChoose cardinality from the perspective of the left model: use `join_one()`\nwhen each left row can match at most one right row, and `join_many()` when one\nleft row can match multiple right rows. Every source model in a join tree must\nalso have a unique `name`, because that name becomes its field prefix and source\nidentity.\n\nFor source-aware aggregation, non-cross joins support direct field equality\npredicates and conjunctions of direct field equalities. Inequality, `OR`, cast,\nand transformed predicates are rejected when source pre-aggregation is needed;\naggregate each model first or restate the relationship as equality keys.\n\n## Example: Two-Level Composition\n\nLet's build a composed model step-by-step, showing available dimensions and measures at each level.\n\n### Level 0: Base Models\n\nFirst, let's set up our base tables:\n\n```setup_ibis_tables\nimport ibis\nfrom boring_semantic_layer import to_semantic_table\n\n# Create sample data\ncon = ibis.duckdb.connect(\":memory:\")\n\n# Flights table\nflights_data = ibis.memtable({\n \"flight_id\": [1, 2, 3],\n \"carrier_code\": [\"AA\", \"UA\", \"DL\"],\n \"aircraft_id\": [101, 102, 103],\n \"distance\": [1000, 1500, 800],\n \"passengers\": [150, 180, 120]\n})\nflights_tbl = con.create_table(\"flights\", flights_data)\n\n# Carriers table\ncarriers_data = ibis.memtable({\n \"code\": [\"AA\", \"UA\", \"DL\"],\n \"name\": [\"American Airlines\", \"United Airlines\", \"Delta Air Lines\"],\n \"country\": [\"USA\", \"USA\", \"USA\"]\n})\ncarriers_tbl = con.create_table(\"carriers\", carriers_data)\n\n# Aircraft table\naircraft_data = ibis.memtable({\n \"id\": [101, 102, 103],\n \"model\": [\"Boeing 737\", \"Airbus A320\", \"Boeing 777\"],\n \"capacity\": [180, 200, 350]\n})\naircraft_tbl = con.create_table(\"aircraft\", aircraft_data)\n```\n\n\n\n```setup_semantic_models\n# Create semantic tables\nflights_st = (\n to_semantic_table(flights_tbl, name=\"flights\")\n .with_dimensions(\n flight_id=lambda t: t.flight_id,\n carrier_code=lambda t: t.carrier_code,\n aircraft_id=lambda t: t.aircraft_id\n )\n .with_measures(\n flight_count=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n total_passengers=lambda t: t.passengers.sum()\n )\n)\n\ncarriers_st = (\n to_semantic_table(carriers_tbl, name=\"carriers\")\n .with_dimensions(\n code=lambda t: t.code,\n name=lambda t: t.name,\n country=lambda t: t.country\n )\n .with_measures(\n carrier_count=lambda t: t.count()\n )\n)\n\naircraft_st = (\n to_semantic_table(aircraft_tbl, name=\"aircraft\")\n .with_dimensions(\n id=lambda t: t.id,\n model=lambda t: t.model\n )\n .with_measures(\n aircraft_count=lambda t: t.count(),\n total_capacity=lambda t: t.capacity.sum()\n )\n)\n```\n\n\n\n```level0_dimensions\nflights_st.dimensions, flights_st.measures\n```\n\n\n\n### Level 1: First Join (Flights + Carriers)\n\nJoin carriers to flights to add carrier information:\n\n```level1_join\n# Each flight row matches at most one carrier row\nflights_with_carriers = flights_st.join_one(\n carriers_st,\n lambda f, c: f.carrier_code == c.code\n)\n\n# Inspect dimensions - now includes both flights and carriers\nflights_with_carriers.dimensions, flights_with_carriers.measures\n```\n\n\n### Level 2: Second Join (+ Aircraft)\n\nAdd aircraft information to create a fully composed model:\n\n```level2_join\n# Each flight row matches at most one aircraft row\nfull_model = flights_with_carriers.join_one(\n aircraft_st,\n lambda f, a: f.aircraft_id == a.id\n)\n\n# Inspect dimensions - now includes flights, carriers, AND aircraft\nfull_model.dimensions, full_model.measures\n```\n\n\n## Query the Composed Model\n\nNow you can query across all joined tables:\n\n```composed_query\n# Query using dimensions and measures from all three tables\nresult = (\n full_model\n .group_by( \"aircraft.model\")\n .aggregate(\"flights.flight_count\", \"flights.total_passengers\", \"aircraft.total_capacity\")\n)\n```\n\n\n\n## Key Takeaways\n\n- **Composition via Joins**: Use `join_one()` for at-most-one right matches, `join_many()` for multiple right matches, and `join_cross()` for Cartesian products\n- **Additive**: Each join adds dimensions and measures from the joined table\n- **Table Prefixes**: Dimensions/measures are prefixed with table names (`flights.`, `carriers.`, `aircraft.`)\n- **Unique Source Names**: Give every base model in the join tree a distinct `name`\n- **No Limit**: Compose as many models as needed for your analysis\n- **Incremental**: Build from simple to complex, one join at a time\n\n## Next Steps\n\n- Learn about [YAML Configuration](/building/yaml) for declarative model composition\n- Explore [Query Methods](/querying/methods) for querying composed models\n",
"queries": {
"setup_ibis_tables": {
"code": "import ibis\nfrom boring_semantic_layer import to_semantic_table\n\n# Create sample data\ncon = ibis.duckdb.connect(\":memory:\")\n\n# Flights table\nflights_data = ibis.memtable({\n \"flight_id\": [1, 2, 3],\n \"carrier_code\": [\"AA\", \"UA\", \"DL\"],\n \"aircraft_id\": [101, 102, 103],\n \"distance\": [1000, 1500, 800],\n \"passengers\": [150, 180, 120]\n})\nflights_tbl = con.create_table(\"flights\", flights_data)\n\n# Carriers table\ncarriers_data = ibis.memtable({\n \"code\": [\"AA\", \"UA\", \"DL\"],\n \"name\": [\"American Airlines\", \"United Airlines\", \"Delta Air Lines\"],\n \"country\": [\"USA\", \"USA\", \"USA\"]\n})\ncarriers_tbl = con.create_table(\"carriers\", carriers_data)\n\n# Aircraft table\naircraft_data = ibis.memtable({\n \"id\": [101, 102, 103],\n \"model\": [\"Boeing 737\", \"Airbus A320\", \"Boeing 777\"],\n \"capacity\": [180, 200, 350]\n})\naircraft_tbl = con.create_table(\"aircraft\", aircraft_data)",
@@ -89,7 +89,7 @@
}
},
"level1_join": {
- "code": "# Join carriers to flights\nflights_with_carriers = flights_st.join_many(\n carriers_st,\n lambda f, c: f.carrier_code == c.code\n)\n\n# Inspect dimensions - now includes both flights and carriers\nflights_with_carriers.dimensions, flights_with_carriers.measures",
+ "code": "# Each flight row matches at most one carrier row\nflights_with_carriers = flights_st.join_one(\n carriers_st,\n lambda f, c: f.carrier_code == c.code\n)\n\n# Inspect dimensions - now includes both flights and carriers\nflights_with_carriers.dimensions, flights_with_carriers.measures",
"sql": "SELECT\n \"t2\".\"flight_id\",\n \"t2\".\"carrier_code\",\n \"t2\".\"aircraft_id\",\n \"t2\".\"distance\",\n \"t2\".\"passengers\",\n \"t3\".\"code\",\n \"t3\".\"name\",\n \"t3\".\"country\"\nFROM \"memory\".\"main\".\"flights\" AS \"t2\"\nLEFT OUTER JOIN \"memory\".\"main\".\"carriers\" AS \"t3\"\n ON \"t2\".\"carrier_code\" = \"t3\".\"code\"",
"plan": "SemanticTable: flights\n flight_id [dim]\n carrier_code [dim]\n aircraft_id [dim]\n flight_count [measure]\n total_distance [measure]\n total_passengers [measure]\n-> Join(left, right=carriers)",
"table": {
@@ -138,7 +138,7 @@
}
},
"level2_join": {
- "code": "# Join aircraft to the composed model\nfull_model = flights_with_carriers.join_many(\n aircraft_st,\n lambda f, a: f.aircraft_id == a.id\n)\n\n# Inspect dimensions - now includes flights, carriers, AND aircraft\nfull_model.dimensions, full_model.measures",
+ "code": "# Each flight row matches at most one aircraft row\nfull_model = flights_with_carriers.join_one(\n aircraft_st,\n lambda f, a: f.aircraft_id == a.id\n)\n\n# Inspect dimensions - now includes flights, carriers, AND aircraft\nfull_model.dimensions, full_model.measures",
"sql": "SELECT\n \"t3\".\"flight_id\",\n \"t3\".\"carrier_code\",\n \"t3\".\"aircraft_id\",\n \"t3\".\"distance\",\n \"t3\".\"passengers\",\n \"t4\".\"code\",\n \"t4\".\"name\",\n \"t4\".\"country\",\n \"t5\".\"id\",\n \"t5\".\"model\",\n \"t5\".\"capacity\"\nFROM \"memory\".\"main\".\"flights\" AS \"t3\"\nLEFT OUTER JOIN \"memory\".\"main\".\"carriers\" AS \"t4\"\n ON \"t3\".\"carrier_code\" = \"t4\".\"code\"\nLEFT OUTER JOIN \"memory\".\"main\".\"aircraft\" AS \"t5\"\n ON \"t3\".\"aircraft_id\" = \"t5\".\"id\"",
"plan": "SemanticTable: flights\n flight_id [dim]\n carrier_code [dim]\n aircraft_id [dim]\n flight_count [measure]\n total_distance [measure]\n total_passengers [measure]\n-> Join(left, right=carriers)\n-> Join(left, right=aircraft)",
"table": {
diff --git a/docs/web/public/bsl-data/reference.json b/docs/web/public/bsl-data/reference.json
index 4bae8ccd..72da429b 100644
--- a/docs/web/public/bsl-data/reference.json
+++ b/docs/web/public/bsl-data/reference.json
@@ -1,5 +1,5 @@
{
- "markdown": "# API Reference\n\nComplete API documentation for the Boring Semantic Layer.\n\n## Table Creation & Configuration\n\nMethods for creating and configuring semantic tables.\n\n### to_semantic_table()\n\n```python\nto_semantic_table(\n table: ibis.Table,\n name: str,\n description: str = None\n) -> SemanticTable\n```\n\nCreate a semantic table from an Ibis table. This is the primary entry point for building semantic models.\n\n| Parameter | Type | Description |\n|-----------|------|-------------|\n| `table` | `ibis.Table` | Ibis table to build the model from |\n| `name` | `str` | Unique identifier for the semantic table |\n| `description` | `str` | Optional description of the semantic table |\n\n**Example:**\n```python\nimport ibis\nfrom boring_semantic_layer import to_semantic_table\n\nflights = ibis.read_parquet(\"flights.parquet\")\nflights_st = to_semantic_table(flights, \"flights\")\n```\n\n### with_dimensions()\n\n```python\nwith_dimensions(\n **dimensions: Callable | Dimension\n) -> SemanticTable\n```\n\nDefine dimensions for grouping and analysis. Dimensions are attributes that categorize data.\n\n**Example:**\n```python\nflights_st = flights_st.with_dimensions(\n origin=lambda t: t.origin,\n dest=lambda t: t.dest,\n carrier=lambda t: t.carrier\n)\n```\n\n### with_measures()\n\n```python\nwith_measures(\n **measures: Callable | Measure\n) -> SemanticTable\n```\n\nDefine aggregations and calculations. Measures are numeric values that can be aggregated.\n\n**Example:**\n```python\nflights_st = flights_st.with_measures(\n flight_count=lambda t: t.count(),\n avg_delay=lambda t: t.arr_delay.mean(),\n total_distance=lambda t: t.distance.sum()\n)\n```\n\n### from_yaml()\n\n```python\nfrom_yaml(\n yaml_path: str,\n connection: ibis.Connection = None\n) -> dict[str, SemanticTable]\n```\n\nLoad semantic models from a YAML configuration file. Returns a dictionary of semantic tables.\n\n| Parameter | Type | Description |\n|-----------|------|-------------|\n| `yaml_path` | `str` | Path to YAML configuration file |\n| `connection` | `ibis.Connection` | Optional Ibis connection for database tables |\n\n**Example:**\n```python\nfrom boring_semantic_layer import from_yaml\n\nmodels = from_yaml(\"models.yaml\")\nflights_st = models[\"flights\"]\n```\n\n### Dimension Class\n\n```python\nDimension(\n expr: Callable,\n description: str = None\n)\n```\n\nSelf-documenting dimension with description. Use for better API documentation.\n\n**Example:**\n```python\nfrom boring_semantic_layer import Dimension\n\nflights_st = flights_st.with_dimensions(\n origin=Dimension(\n expr=lambda t: t.origin,\n description=\"Airport code where the flight departed from\"\n )\n)\n```\n\n### Measure Class\n\n```python\nMeasure(\n expr: Callable,\n description: str = None\n)\n```\n\nSelf-documenting measure with description. Use for better API documentation.\n\n**Example:**\n```python\nfrom boring_semantic_layer import Measure\n\nflights_st = flights_st.with_measures(\n avg_delay=Measure(\n expr=lambda t: t.arr_delay.mean(),\n description=\"Average arrival delay in minutes\"\n )\n)\n```\n\n### all()\n\n```python\nst.all()\n```\n\nReference the entire dataset within measure definitions. Primarily used for percentage-of-total calculations.\n\n**Example:**\n```python\nflights_st = to_semantic_table(data, \"flights\").with_measures(\n flight_count=lambda t: t.count(),\n pct_of_total=lambda t: (\n t.count() / t.all().count() * 100\n )\n)\n```\n\n## Join Methods\n\nMethods for composing semantic tables through joins.\n\n### join_many()\n\n```python\njoin_many(\n other: SemanticTable,\n on: Callable,\n name: str = None\n) -> SemanticTable\n```\n\nOne-to-many relationship join (LEFT JOIN). Use when the left table can match multiple rows in the right table.\n\n| Parameter | Type | Description |\n|-----------|------|-------------|\n| `other` | `SemanticTable` | The semantic table to join with |\n| `on` | `Callable` | Lambda function defining the join condition |\n| `name` | `str` | Optional name for the joined table reference |\n\n**Example:**\n```python\nflights_st = flights_st.join_many(\n carriers_st,\n on=lambda l, r: l.carrier == r.code,\n name=\"carrier_info\"\n)\n```\n\n### join_one()\n\n```python\njoin_one(\n other: SemanticTable,\n on: Callable,\n name: str = None\n) -> SemanticTable\n```\n\nOne-to-one relationship join (INNER JOIN). Use when each row in the left table matches exactly one row in the right table.\n\n**Example:**\n```python\nflights_st = flights_st.join_one(\n airports_st,\n on=lambda l, r: l.origin == r.code\n)\n```\n\n### join_cross()\n\n```python\njoin_cross(\n other: SemanticTable,\n name: str = None\n) -> SemanticTable\n```\n\nCross join (CARTESIAN PRODUCT). Creates all possible combinations of rows from both tables.\n\n### join()\n\n```python\njoin(\n other: SemanticTable,\n on: Callable,\n how: str = \"inner\",\n name: str = None\n) -> SemanticTable\n```\n\nCustom join with flexible join type. Supports 'inner', 'left', 'right', 'outer', and 'cross'.\n\n| Parameter | Type | Description |\n|-----------|------|-------------|\n| `other` | `SemanticTable` | The semantic table to join with |\n| `on` | `Callable` | Lambda function defining the join condition |\n| `how` | `str` | Join type: 'inner', 'left', 'right', 'outer', or 'cross' |\n| `name` | `str` | Optional name for the joined table reference |\n\n## Query Methods\n\nMethods for querying and transforming semantic tables.\n\n### group_by()\n\n```python\ngroup_by(\n *dimensions: str\n) -> QueryBuilder\n```\n\nGroup data by one or more dimension names. Returns a query builder for chaining with aggregate().\n\n**Example:**\n```python\nresult = flights_st.group_by(\"origin\", \"carrier\").aggregate(\"flight_count\")\n```\n\n### aggregate()\n\n```python\naggregate(\n *measures: str,\n **kwargs\n) -> ibis.Table\n```\n\nCalculate one or more measures. Can be used standalone or after group_by().\n\n**Examples:**\n```python\n# Without grouping\ntotal = flights_st.aggregate(\"flight_count\")\n\n# With grouping\nby_origin = flights_st.group_by(\"origin\").aggregate(\"flight_count\", \"avg_delay\")\n```\n\n### filter()\n\n```python\nfilter(\n condition: Callable\n) -> SemanticTable\n```\n\nApply conditions to filter data. Use lambda functions with Ibis expressions.\n\n**Example:**\n```python\ndelayed_flights = flights_st.filter(lambda t: t.arr_delay > 0)\n```\n\n### order_by()\n\n```python\norder_by(\n *columns: str | ibis.Expression\n) -> ibis.Table\n```\n\nSort query results. Use `ibis.desc()` for descending order.\n\n**Example:**\n```python\nresult = flights_st.group_by(\"origin\").aggregate(\"flight_count\")\nresult = result.order_by(ibis.desc(\"flight_count\"))\n```\n\n### limit()\n\n```python\nlimit(\n n: int\n) -> ibis.Table\n```\n\nRestrict the number of rows returned.\n\n**Example:**\n```python\ntop_10 = result.order_by(ibis.desc(\"flight_count\")).limit(10)\n```\n\n### mutate()\n\n```python\nmutate(\n **expressions: Callable | ibis.Expression\n) -> ibis.Table\n```\n\nAdd or transform columns in aggregated results. Useful for calculations after aggregation.\n\n**Example:**\n```python\nresult = flights_st.group_by(\"month\").aggregate(\"revenue\")\nresult = result.mutate(\n growth_rate=lambda t: (t.revenue - t.revenue.lag()) / t.revenue.lag() * 100\n)\n```\n\n### select()\n\n```python\nselect(\n *columns: str | ibis.Expression\n) -> ibis.Table\n```\n\nSelect specific columns from the result. Often used in nesting operations.\n\n**Example:**\n```python\nresult.select(\"origin\", \"flight_count\")\n```\n\n## Nesting\n\nCreate nested data structures within aggregations.\n\n### nest Parameter\n\n```python\naggregate(\n *measures,\n nest={\n \"nested_column\": lambda t: t.group_by([...]) | t.select(...)\n }\n)\n```\n\nCreate nested arrays of structs within aggregation results. Useful for hierarchical data or subtotals.\n\n**Example:**\n```python\nresult = flights_st.group_by(\"carrier\").aggregate(\n \"total_flights\",\n nest={\n \"by_month\": lambda t: t.group_by(\"month\").aggregate(\"monthly_flights\")\n }\n)\n```\n\n## Charting\n\nGenerate visualizations from query results.\n\n### chart()\n\n```python\nchart(\n result: ibis.Table,\n backend: str = \"altair\",\n spec: dict = None,\n format: str = \"interactive\"\n) -> Chart\n```\n\nCreate visualizations from query results. Supports Altair (default) and Plotly backends.\n\n| Parameter | Type | Description |\n|-----------|------|-------------|\n| `result` | `ibis.Table` | Query result table to visualize |\n| `backend` | `str` | \"altair\" or \"plotly\" |\n| `spec` | `dict` | Custom Vega-Lite specification (for Altair) |\n| `format` | `str` | \"interactive\", \"json\", \"png\", \"svg\" |\n\n**Auto-detection:**\nBSL automatically selects appropriate chart types:\n- Single dimension + measure \u2192 Bar chart\n- Time dimension + measure \u2192 Line chart\n- Two dimensions + measure \u2192 Heatmap\n\n**Example:**\n```python\nfrom boring_semantic_layer.chart import chart\n\nresult = flights_st.group_by(\"month\").aggregate(\"flight_count\")\nchart(result, backend=\"altair\")\n```\n\n## Dimensional Indexing\n\nCreate searchable catalogs of dimension values.\n\n### index()\n\n```python\nindex(\n dimensions: Callable | None = None,\n by: str = None,\n sample: int = None\n) -> ibis.Table\n```\n\nCreate a searchable catalog of unique dimension values with optional weighting and sampling.\n\n| Parameter | Type | Description |\n|-----------|------|-------------|\n| `dimensions` | `Callable` | None (all dimensions) or lambda returning list of fields |\n| `by` | `str` | Measure name for weighting results |\n| `sample` | `int` | Number of rows to sample (for large datasets) |\n\n**Examples:**\n```python\n# Index all dimensions\nflights_st.index()\n\n# Index specific dimensions\nflights_st.index(lambda t: [t.origin, t.dest])\n\n# Weight by measure\nflights_st.index(by=\"flight_count\")\n\n# Sample large dataset\nflights_st.index(sample=10000)\n```\n\n## Other\n\n### MCP Integration\n\n#### MCPSemanticModel()\n\n```python\nMCPSemanticModel(\n models: dict[str, SemanticTable] | str,\n description: str = None\n)\n```\n\nCreate an MCP server to expose semantic models to LLMs like Claude. Accepts either a dictionary of models or a path to a YAML configuration file.\n\n| Parameter | Type | Description |\n|-----------|------|-------------|\n| `models` | `dict` or `str` | Dictionary of SemanticTable objects or path to YAML config |\n| `description` | `str` | Optional description of the semantic model |\n\n**Available MCP Tools:**\n\n| Tool | Description |\n|------|-------------|\n| `list_models()` | List all available semantic model names |\n| `get_model()` | Get detailed model information (dimensions, measures, joins) |\n| `get_time_range()` | Get available time range for time-series data |\n| `query_model()` | Execute queries against semantic models |\n\n**Example:**\n```python\nfrom boring_semantic_layer import MCPSemanticModel\n\n# From dictionary\nserver = MCPSemanticModel(\n models={\"flights\": flights_st, \"airports\": airports_st},\n description=\"Flight data analysis\"\n)\n\n# From YAML\nserver = MCPSemanticModel(\"config.yaml\")\n```\n\n### YAML Configuration\n\n#### YAML Structure\n\n```yaml\nmodel_name:\n table: table_reference\n description: \"Optional description\"\n\n dimensions:\n dimension_name: expression\n # or with description\n dimension_name:\n expr: expression\n description: \"Dimension description\"\n\n measures:\n measure_name: expression\n # or with description\n measure_name:\n expr: expression\n description: \"Measure description\"\n\n joins:\n join_name:\n model: model_reference\n on: join_condition\n how: join_type # left, inner, right, outer, cross\n```\n\n#### Expression Syntax\n\n| Expression | Description |\n|------------|-------------|\n| `_` | Reference to the table |\n| `_.column` | Reference a column |\n| `_.count()` | Count aggregation |\n| `_.column.sum()` | Sum aggregation |\n| `_.column.mean()` | Average aggregation |\n| `_.column.min()` | Minimum value |\n| `_.column.max()` | Maximum value |\n\n**Example:**\n```yaml\nflights:\n table: flights_data\n description: \"Flight operations data\"\n\n dimensions:\n origin: _.origin\n dest: _.dest\n carrier:\n expr: _.carrier\n description: \"Airline carrier code\"\n\n measures:\n flight_count: _.count()\n avg_delay:\n expr: _.arr_delay.mean()\n description: \"Average arrival delay in minutes\"\n```\n\n## Next Steps\n\n- Learn about [Semantic Tables](/building/semantic-tables)\n- Explore [Query Methods](/querying/methods)\n- See [Advanced Patterns](/advanced/percentage-total)\n",
+ "markdown": "# API Reference\n\nComplete API documentation for the Boring Semantic Layer.\n\n## Table Creation & Configuration\n\nMethods for creating and configuring semantic tables.\n\n### to_semantic_table()\n\n```python\nto_semantic_table(\n table: ibis.Table,\n name: str,\n description: str = None\n) -> SemanticTable\n```\n\nCreate a semantic table from an Ibis table. This is the primary entry point for building semantic models.\n\n| Parameter | Type | Description |\n|-----------|------|-------------|\n| `table` | `ibis.Table` | Ibis table to build the model from |\n| `name` | `str` | Unique identifier for the semantic table |\n| `description` | `str` | Optional description of the semantic table |\n\n**Example:**\n```python\nimport ibis\nfrom boring_semantic_layer import to_semantic_table\n\nflights = ibis.read_parquet(\"flights.parquet\")\nflights_st = to_semantic_table(flights, \"flights\")\n```\n\n### with_dimensions()\n\n```python\nwith_dimensions(\n **dimensions: Callable | Dimension\n) -> SemanticTable\n```\n\nDefine dimensions for grouping and analysis. Dimensions are attributes that categorize data.\n\n**Example:**\n```python\nflights_st = flights_st.with_dimensions(\n origin=lambda t: t.origin,\n dest=lambda t: t.dest,\n carrier=lambda t: t.carrier\n)\n```\n\n### with_measures()\n\n```python\nwith_measures(\n **measures: Callable | Measure\n) -> SemanticTable\n```\n\nDefine aggregations and calculations. Measures are numeric values that can be aggregated.\n\n**Example:**\n```python\nflights_st = flights_st.with_measures(\n flight_count=lambda t: t.count(),\n avg_delay=lambda t: t.arr_delay.mean(),\n total_distance=lambda t: t.distance.sum()\n)\n```\n\n### from_yaml()\n\n```python\nfrom_yaml(\n yaml_path: str,\n connection: ibis.Connection = None\n) -> dict[str, SemanticTable]\n```\n\nLoad semantic models from a YAML configuration file. Returns a dictionary of semantic tables.\n\n| Parameter | Type | Description |\n|-----------|------|-------------|\n| `yaml_path` | `str` | Path to YAML configuration file |\n| `connection` | `ibis.Connection` | Optional Ibis connection for database tables |\n\n**Example:**\n```python\nfrom boring_semantic_layer import from_yaml\n\nmodels = from_yaml(\"models.yaml\")\nflights_st = models[\"flights\"]\n```\n\n### Dimension Class\n\n```python\nDimension(\n expr: Callable,\n description: str = None\n)\n```\n\nSelf-documenting dimension with description. Use for better API documentation.\n\n**Example:**\n```python\nfrom boring_semantic_layer import Dimension\n\nflights_st = flights_st.with_dimensions(\n origin=Dimension(\n expr=lambda t: t.origin,\n description=\"Airport code where the flight departed from\"\n )\n)\n```\n\n### Measure Class\n\n```python\nMeasure(\n expr: Callable,\n description: str = None\n)\n```\n\nSelf-documenting measure with description. Use for better API documentation.\n\n**Example:**\n```python\nfrom boring_semantic_layer import Measure\n\nflights_st = flights_st.with_measures(\n avg_delay=Measure(\n expr=lambda t: t.arr_delay.mean(),\n description=\"Average arrival delay in minutes\"\n )\n)\n```\n\n### all()\n\n```python\nt.all(measure_or_reduction)\n```\n\nReference the entire dataset within a measure definition. Pass a declared\nmeasure reference, measure name, or Ibis reduction; `all()` does not have a\nzero-argument form. It is primarily used for percentage-of-total calculations.\n\n**Example:**\n```python\nflights_st = to_semantic_table(data, \"flights\").with_measures(\n flight_count=lambda t: t.count(),\n pct_of_total=lambda t: (\n t.flight_count / t.all(t.flight_count) * 100\n )\n)\n```\n\n## Join Methods\n\nMethods for composing semantic tables through joins.\n\n### join_many()\n\n```python\njoin_many(\n other: SemanticTable,\n on: Callable | str | Deferred | Sequence[str | Deferred],\n how: str = \"left\"\n) -> SemanticTable\n```\n\nOne-to-many relationship join (LEFT JOIN). Use when the left table can match multiple rows in the right table.\n\n| Parameter | Type | Description |\n|-----------|------|-------------|\n| `other` | `SemanticTable` | The semantic table to join with |\n| `on` | `Callable \\| str \\| Deferred \\| Sequence[str \\| Deferred]` | Join predicate: a two-argument lambda, same-name column shorthand, or sequence of shorthands for a compound equijoin |\n| `how` | `str` | Join type; only `'left'` is supported |\n\n**Example:**\n```python\ncarriers_with_flights = carriers_st.join_many(\n flights_st,\n on=lambda carrier, flight: carrier.code == flight.carrier\n)\n```\n\nModel names are the source aliases used in prefixed fields. Every base model in\na composed join tree must therefore have a unique `name`; assign explicit names\nwith `to_semantic_table(..., name=\"...\")` before joining. When the same physical\ntable serves multiple roles, use both `.view()` and distinct model names.\n\nSource-aware aggregation supports direct field equijoins and conjunctions of\ndirect field equijoins. Inequality, `OR`, cast, and transformed predicates are\nrejected when source pre-aggregation is required because they cannot be safely\nreconstructed from key columns alone.\n\n### join_one()\n\n```python\njoin_one(\n other: SemanticTable,\n on: Callable | str | Deferred | Sequence[str | Deferred],\n how: str = \"left\"\n) -> SemanticTable\n```\n\nAt-most-one-right-match relationship join (LEFT JOIN). Use when each left row\ncan match at most one row in the right table. Only `how=\"left\"` is supported.\n\n**Example:**\n```python\nflights_st = flights_st.join_one(\n airports_st,\n on=lambda l, r: l.origin == r.code\n)\n```\n\n### join_cross()\n\n```python\njoin_cross(\n other: SemanticTable\n) -> SemanticTable\n```\n\nCross join (CARTESIAN PRODUCT). Creates all possible combinations of rows from both tables.\n\n## Query Methods\n\nMethods for querying and transforming semantic tables.\n\n### group_by()\n\n```python\ngroup_by(\n *dimensions: str\n) -> QueryBuilder\n```\n\nGroup data by one or more dimension names. Returns a query builder for chaining with aggregate().\n\n**Example:**\n```python\nresult = flights_st.group_by(\"origin\", \"carrier\").aggregate(\"flight_count\")\n```\n\n### aggregate()\n\n```python\naggregate(\n *measures: str,\n **kwargs\n) -> ibis.Table\n```\n\nCalculate one or more measures. Can be used standalone or after group_by().\n\n**Examples:**\n```python\n# Without grouping\ntotal = flights_st.aggregate(\"flight_count\")\n\n# With grouping\nby_origin = flights_st.group_by(\"origin\").aggregate(\"flight_count\", \"avg_delay\")\n```\n\n### filter()\n\n```python\nfilter(\n condition: Callable\n) -> SemanticTable\n```\n\nApply conditions to filter data. Use lambda functions with Ibis expressions.\n\n**Example:**\n```python\ndelayed_flights = flights_st.filter(lambda t: t.arr_delay > 0)\n```\n\n### order_by()\n\n```python\norder_by(\n *columns: str | ibis.Expression\n) -> ibis.Table\n```\n\nSort query results. Use `ibis.desc()` for descending order.\n\n**Example:**\n```python\nresult = flights_st.group_by(\"origin\").aggregate(\"flight_count\")\nresult = result.order_by(ibis.desc(\"flight_count\"))\n```\n\n### limit()\n\n```python\nlimit(\n n: int\n) -> ibis.Table\n```\n\nRestrict the number of rows returned.\n\n**Example:**\n```python\ntop_10 = result.order_by(ibis.desc(\"flight_count\")).limit(10)\n```\n\n### mutate()\n\n```python\nmutate(\n **expressions: Callable | ibis.Expression\n) -> ibis.Table\n```\n\nAdd or transform columns in aggregated results. Useful for calculations after aggregation.\n\n**Example:**\n```python\nresult = flights_st.group_by(\"month\").aggregate(\"revenue\")\nresult = result.mutate(\n growth_rate=lambda t: (t.revenue - t.revenue.lag()) / t.revenue.lag() * 100\n)\n```\n\n### select()\n\n```python\nselect(\n *columns: str | ibis.Expression\n) -> ibis.Table\n```\n\nSelect specific columns from the result. Often used in nesting operations.\n\n**Example:**\n```python\nresult.select(\"origin\", \"flight_count\")\n```\n\n## Nesting\n\nCreate nested data structures within aggregations.\n\n### nest Parameter\n\n```python\naggregate(\n *measures,\n nest={\n \"nested_column\": lambda t: t.group_by(...)\n }\n)\n```\n\nCreate nested arrays of structs within aggregation results. Useful for hierarchical data or subtotals.\n\n**Example:**\n```python\nresult = flights_st.group_by(\"carrier\").aggregate(\n \"total_flights\",\n nest={\n \"by_month\": lambda t: t.group_by(\"month\").aggregate(\"monthly_flights\")\n }\n)\n```\n\n## Charting\n\nGenerate visualizations from query results.\n\n### chart()\n\n```python\nchart(\n result: ibis.Table,\n backend: str = \"altair\",\n spec: dict = None,\n format: str = \"interactive\"\n) -> Chart\n```\n\nCreate visualizations from query results. Supports Altair (default) and Plotly backends.\n\n| Parameter | Type | Description |\n|-----------|------|-------------|\n| `result` | `ibis.Table` | Query result table to visualize |\n| `backend` | `str` | \"altair\" or \"plotly\" |\n| `spec` | `dict` | Custom Vega-Lite specification (for Altair) |\n| `format` | `str` | \"interactive\", \"json\", \"png\", \"svg\" |\n\n**Auto-detection:**\nBSL automatically selects appropriate chart types:\n- Single dimension + measure \u2192 Bar chart\n- Time dimension + measure \u2192 Line chart\n- Two dimensions + measure \u2192 Heatmap\n\n**Example:**\n```python\nfrom boring_semantic_layer.chart import chart\n\nresult = flights_st.group_by(\"month\").aggregate(\"flight_count\")\nchart(result, backend=\"altair\")\n```\n\n## Dimensional Indexing\n\nCreate searchable catalogs of dimension values.\n\n### index()\n\n```python\nindex(\n dimensions: Callable | None = None,\n by: str = None,\n sample: int = None\n) -> ibis.Table\n```\n\nCreate a searchable catalog of unique dimension values with optional weighting and sampling.\n\n| Parameter | Type | Description |\n|-----------|------|-------------|\n| `dimensions` | `Callable` | None (all dimensions) or lambda returning list of fields |\n| `by` | `str` | Measure name for weighting results |\n| `sample` | `int` | Number of rows to sample (for large datasets) |\n\n**Examples:**\n```python\n# Index all dimensions\nflights_st.index()\n\n# Index specific dimensions\nflights_st.index(lambda t: [t.origin, t.dest])\n\n# Weight by measure\nflights_st.index(by=\"flight_count\")\n\n# Sample large dataset\nflights_st.index(sample=10000)\n```\n\n## Other\n\n### MCP Integration\n\n#### MCPSemanticModel()\n\n```python\nMCPSemanticModel(\n models: dict[str, SemanticTable] | str,\n description: str = None\n)\n```\n\nCreate an MCP server to expose semantic models to LLMs like Claude. Accepts either a dictionary of models or a path to a YAML configuration file.\n\n| Parameter | Type | Description |\n|-----------|------|-------------|\n| `models` | `dict` or `str` | Dictionary of SemanticTable objects or path to YAML config |\n| `description` | `str` | Optional description of the semantic model |\n\n**Available MCP Tools:**\n\n| Tool | Description |\n|------|-------------|\n| `list_models()` | List all available semantic model names |\n| `get_model()` | Get detailed model information (dimensions, measures, joins) |\n| `get_time_range()` | Get available time range for time-series data |\n| `query_model()` | Execute queries against semantic models |\n\n**Example:**\n```python\nfrom boring_semantic_layer import MCPSemanticModel\n\n# From dictionary\nserver = MCPSemanticModel(\n models={\"flights\": flights_st, \"airports\": airports_st},\n description=\"Flight data analysis\"\n)\n\n# From YAML\nserver = MCPSemanticModel(\"config.yaml\")\n```\n\n### YAML Configuration\n\n#### YAML Structure\n\n```yaml\nmodel_name:\n table: table_reference\n description: \"Optional description\"\n\n dimensions:\n dimension_name: expression\n # or with description\n dimension_name:\n expr: expression\n description: \"Dimension description\"\n\n measures:\n measure_name: expression\n # or with description\n measure_name:\n expr: expression\n description: \"Measure description\"\n\n joins:\n join_name:\n model: model_reference\n on: join_condition\n how: left # Optional; non-cross semantic joins are always left joins\n```\n\n#### Expression Syntax\n\n| Expression | Description |\n|------------|-------------|\n| `_` | Reference to the table |\n| `_.column` | Reference a column |\n| `_.count()` | Count aggregation |\n| `_.column.sum()` | Sum aggregation |\n| `_.column.mean()` | Average aggregation |\n| `_.column.min()` | Minimum value |\n| `_.column.max()` | Maximum value |\n\n**Example:**\n```yaml\nflights:\n table: flights_data\n description: \"Flight operations data\"\n\n dimensions:\n origin: _.origin\n dest: _.dest\n carrier:\n expr: _.carrier\n description: \"Airline carrier code\"\n\n measures:\n flight_count: _.count()\n avg_delay:\n expr: _.arr_delay.mean()\n description: \"Average arrival delay in minutes\"\n```\n\n## Next Steps\n\n- Learn about [Semantic Tables](/building/semantic-tables)\n- Explore [Query Methods](/querying/methods)\n- See [Advanced Patterns](/advanced/percentage-total)\n",
"queries": {},
"files": {}
}
diff --git a/docs/web/public/bsl-data/semantic-table.json b/docs/web/public/bsl-data/semantic-table.json
index 1856bcee..139a665f 100644
--- a/docs/web/public/bsl-data/semantic-table.json
+++ b/docs/web/public/bsl-data/semantic-table.json
@@ -1,5 +1,5 @@
{
- "markdown": "# Building a Semantic Table\n\nDefine your data model with dimensions and measures using Ibis expressions.\n\n## Overview\n\nA Semantic Table is the core building block of BSL. It transforms a raw Ibis table into a reusable, self-documenting data model by defining:\n- **Dimensions**: Attributes to group by (e.g., origin, carrier, year)\n- **Measures**: Aggregations and calculations (e.g., flight count, total distance)\n\n## to_semantic_table()\n\n```setup_flights\nimport ibis\nfrom boring_semantic_layer import to_semantic_table\n\n# 1. Start with an Ibis table\ncon = ibis.duckdb.connect(\":memory:\")\nflights_data = ibis.memtable({\n \"origin\": [\"JFK\", \"LAX\", \"SFO\"],\n \"dest\": [\"LAX\", \"SFO\", \"JFK\"],\n \"carrier\": [\"AA\", \"UA\", \"DL\"],\n \"year\": [2023, 2023, 2024],\n \"distance\": [2475, 337, 382],\n \"dep_delay\": [10, 5, 0]\n})\nflights_tbl = con.create_table(\"flights\", flights_data)\n\n# 2. Convert to a Semantic Table\nflights_st = to_semantic_table(flights_tbl, name=\"flights\")\n```\n\n## with_dimensions()\n\nDimensions define the attributes you can group by in your queries. They represent the categorical or descriptive aspects of your data that you want to analyze.\n\nYou can define dimensions using lambda expressions, unbound syntax (`_.`), or the `Dimension` class with descriptions:\n\n```dimensions_demo\nfrom ibis import _\nfrom boring_semantic_layer import Dimension\n\nflights_st = flights_st.with_dimensions(\n # Lambda expressions - simple and explicit\n origin=lambda t: t.origin,\n\n # Unbound syntax - cleaner and more concise\n destination=_.dest,\n year=_.year,\n\n # Dimension - self-documenting and AI-friendly\n carrier=Dimension(\n expr=lambda t: t.carrier,\n description=\"Airline carrier code\"\n )\n)\n\nflights_st.dimensions\n```\n\n\n## with_measures()\n\nMeasures define the aggregations and calculations you can query. They represent the quantitative aspects of your data that you want to analyze (counts, sums, averages, etc.).\n\nYou can define measures using lambda expressions, reference other measures for composition, or use the `Measure` class with descriptions:\n\n```measures_demo\nfrom boring_semantic_layer import Measure\n\nflights_st = flights_st.with_measures(\n # Lambda expressions - simple and concise\n total_flights=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n max_delay=lambda t: t.dep_delay.max(),\n\n # Reference other measures for composition\n avg_distance_per_flight=lambda t: t.total_distance / t.total_flights,\n\n # Measure - self-documenting and AI-friendly\n avg_distance=Measure(\n expr=lambda t: t.distance.mean(),\n description=\"Average flight distance in miles\"\n )\n)\n\nflights_st.measures\n```\n\n\n\n### all()\n\nThe `all()` function references the entire dataset within measure definitions, enabling percent-of-total and comparison calculations.\n\n**Example:** Calculate market share as a percentage\n\n```measure_all_demo\nflights_with_pct = flights_st.with_measures(\n flight_count=lambda t: t.count(),\n market_share=lambda t: t.flight_count / t.all(t.flight_count) * 100 # Percent of total\n )\n\n# Query by carrier\nresult = (\n flights_with_pct\n .group_by(\"carrier\")\n .aggregate(\"flight_count\", \"market_share\")\n)\n```\n\n\n\n\n`t.all()` is a method available on the table parameter `t` in measure definitions. It references the entire dataset regardless of grouping, making it perfect for calculating percentages, or comparing groups to the total.\n\n\nFor more examples, see the [Percent of Total pattern](/advanced/percentage-total).\n\n## graph\n\nThe `graph` property provides a dependency graph showing how dimensions and measures relate to each other. This is useful for:\n- **Understanding dependencies**: See what columns or fields each dimension/measure depends on\n- **Impact analysis**: Find what breaks when changing a field\n- **Documentation**: Generate visual representations of your data model\n- **Validation**: Ensure your model doesn't have circular dependencies\n\n```graph_demo\n# Build a semantic table with dependencies\nflights_with_deps = flights_st.with_dimensions(\n origin=lambda t: t.origin,\n destination=lambda t: t.dest,\n).with_measures(\n flight_count=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n avg_distance_per_flight=lambda t: t.total_distance / t.flight_count\n)\n\n# Access the dependency graph\ngraph = flights_with_deps.get_graph()\ngraph\n```\n\n\n### Understanding the Graph Structure\n\nThe graph is a dictionary where:\n- **Keys**: Dimension or measure names\n- **Values**: Metadata containing:\n - `deps`: Dependencies mapped to their types (`'column'`, `'dimension'`, or `'measure'`)\n - `type`: The field type (`'dimension'`, `'measure'`, or `'calc_measure'`)\n\n```graph_structure\n# Access the graph - it's a dict-like object\ngraph = flights_with_deps.get_graph()\ngraph\n```\n\n\n```python\n# Find what a specific field depends on\nflights_with_deps.get_graph()['avg_distance_per_flight']['deps']\n# Output: {'total_distance': 'measure', 'flight_count': 'measure'}\n```\n\n### Graph Traversal\n\nUse `graph_predecessors()` and `graph_successors()` to navigate dependencies:\n\n```graph_traversal\nfrom boring_semantic_layer import graph_predecessors, graph_successors\n\ngraph = flights_with_deps.get_graph()\n\n# What does this field depend on? (predecessors)\ngraph_predecessors(graph, 'avg_distance_per_flight')\n# {'total_distance', 'flight_count'}\n\n# What depends on this field? (successors)\ngraph_successors(graph, 'total_distance')\n# {'avg_distance_per_flight'}\n```\n\n\n### Working with the Dependency Graph\n\nThe dependency graph is a dict-like object where each key is a field name and the value is a dict with `\"type\"` (dimension/measure/calc_measure/column) and `\"deps\"` (dependencies with their types):\n\n```python\n# Access the graph directly as a dict\ngraph = flights_with_deps.get_graph()\n\n# Iterate over fields and their dependencies\nfor field, info in graph.items():\n print(f\"{field} ({info['type']}): depends on {info['deps']}\")\n```\n\n## join_one() / join_many() / join_cross()\n\nJoin semantic tables together to query across relationships. Joins allow you to combine data from multiple semantic tables and access dimensions and measures across all joined tables.\n\n**What Makes Semantic Joins Different?**\n\nSemantic joins explicitly capture the **relationship type** between tables, rather than just specifying SQL join mechanics:\n\n**SQL Joins:**\n```python\n# Specifies HOW to join (LEFT/INNER), but not the relationship\nflights.join(carriers, condition, how=\"left\")\n```\n\n**Semantic Joins:**\n```python\n# Specifies the relationship: one carrier has many flights\nflights.join_many(carriers, lambda f, c: f.carrier == c.code)\n```\n\n**What You Get:**\n- **Explicit relationships**: `join_many()` documents that this is a one-to-many relationship\n- **Table hierarchy information**: The method name describes how tables relate to each other\n- **Richer metadata**: Makes the data model structure explicit for documentation and tooling\n\n\nAfter joining, dimensions and measures are prefixed with table names (e.g., `flights.origin`, `carriers.name`) to avoid naming conflicts.\n\n\n\n**Joining the same table multiple times?** If you need to join to the same source table via different foreign keys (e.g., pickup and dropoff locations), you must use `.view()` to create distinct table references:\n\n```python\n# Create distinct references when joining same table twice\npickup_locs = to_semantic_table(locs_tbl.view(), \"pickup_locs\")\ndropoff_locs = to_semantic_table(locs_tbl.view(), \"dropoff_locs\")\n```\n\nWithout `.view()`, you'll encounter an `IbisInputError: Ambiguous field reference` error. \n\n\nLet's get some additional data:\n\n```setup_carriers\nimport ibis\nfrom boring_semantic_layer import to_semantic_table\n\ncon = ibis.duckdb.connect(\":memory:\")\n\n# Create carriers data\ncarriers_data = ibis.memtable({\n \"code\": [\"AA\", \"UA\", \"DL\"],\n \"name\": [\"American Airlines\", \"United Airlines\", \"Delta Air Lines\"]\n})\ncarriers_tbl = con.create_table(\"carriers\", carriers_data)\n```\n\n\nAnd create a carriers semantic table:\n\n```carriers_st\ncarriers = (\n to_semantic_table(carriers_tbl, name=\"carriers\")\n .with_dimensions(\n code=lambda t: t.code,\n name=lambda t: t.name\n )\n .with_measures(\n carrier_count=lambda t: t.count()\n )\n)\n```\n\n### join_many() - One-to-Many Relationships\n\nUse `join_many()` when one row in the left table can match multiple rows in the right table (LEFT JOIN).\n\n```join_demo\n# Join carriers to flights - one carrier has many flights\nflights_with_carriers = flights_st.join_many(\n carriers,\n lambda f, c: f.carrier == c.code\n)\n\n# Inspect available dimensions and measures\nflights_with_carriers.dimensions\n```\n\n\nAfter joining, all dimensions and measures from both tables are available. Each is prefixed with its table name to avoid conflicts:\n\n\n### join_one() - One-to-One Relationships\n\nUse `join_one()` when rows have a unique matching relationship (INNER JOIN).\n\n```python\n# Many flights \u2192 one carrier (each flight has exactly one carrier)\nflights_with_carrier = flights_st.join_one(\n carriers,\n lambda f, c: f.carrier == c.code\n)\n```\n\n\n**Important Limitation:** Currently, `left_on` and `right_on` must be **COLUMN names**, not dimension names.\n\nIf you have a dimension that maps to a different column name, you must use the underlying column name in the join.\n\n**Example:**\n```python\n# If users table has column 'id' but dimension 'customer_id':\nusers = to_semantic_table(users_tbl).with_dimensions(\n customer_id=lambda t: t.id # Dimension renamed\n)\n\n# \u274c This will fail with a helpful error:\norders.join_one(users, left_on=\"customer_id\", right_on=\"customer_id\")\n\n# \u2713 Use the actual column name:\norders.join_one(users, left_on=\"customer_id\", right_on=\"id\")\n```\n\nThis is a known limitation tracked in [issue #43](https://github.com/boringdata/boring-semantic-layer/issues/43). If you attempt to use a dimension name that doesn't match a column name, you'll get a helpful error message guiding you to use the correct column name.\n\n\n### join_cross() - Cross Join\n\nUse `join_cross()` to create every possible combination of rows from both tables (CARTESIAN PRODUCT).\n\n```python\n# Every flight \u00d7 every carrier combination\nall_combinations = flights_st.join_cross(carriers)\n```\n\n### join() - Custom Join Conditions\n\nUse `join()` for complex join conditions or specific SQL join types.\n\n```python\n# LEFT JOIN with custom condition\nflights_with_carriers = flights_st.join(\n carriers,\n lambda f, c: f.carrier == c.code,\n how=\"left\"\n)\n\n# INNER JOIN\nflights_matched = flights_st.join(\n carriers,\n lambda f, c: f.carrier == c.code,\n how=\"inner\"\n)\n\n# Complex conditions\ndate_range_join = flights_st.join(\n promotions,\n lambda f, p: (f.date >= p.start_date) & (f.date <= p.end_date),\n how=\"left\"\n)\n```\n\n**Supported join types:** `\"inner\"`, `\"left\"`, `\"right\"`, `\"outer\"`, `\"cross\"`\n\n## Next Steps\n\n- Learn about [Composing Models](/building/compose)\n- Explore [YAML Configuration](/building/yaml)\n- Start [Querying Semantic Tables](/querying/methods)\n",
+ "markdown": "# Building a Semantic Table\n\nDefine your data model with dimensions and measures using Ibis expressions.\n\n## Overview\n\nA Semantic Table is the core building block of BSL. It transforms a raw Ibis table into a reusable, self-documenting data model by defining:\n- **Dimensions**: Attributes to group by (e.g., origin, carrier, year)\n- **Measures**: Aggregations and calculations (e.g., flight count, total distance)\n\n## to_semantic_table()\n\n```setup_flights\nimport ibis\nfrom boring_semantic_layer import to_semantic_table\n\n# 1. Start with an Ibis table\ncon = ibis.duckdb.connect(\":memory:\")\nflights_data = ibis.memtable({\n \"origin\": [\"JFK\", \"LAX\", \"SFO\"],\n \"dest\": [\"LAX\", \"SFO\", \"JFK\"],\n \"carrier\": [\"AA\", \"UA\", \"DL\"],\n \"year\": [2023, 2023, 2024],\n \"distance\": [2475, 337, 382],\n \"dep_delay\": [10, 5, 0]\n})\nflights_tbl = con.create_table(\"flights\", flights_data)\n\n# 2. Convert to a Semantic Table\nflights_st = to_semantic_table(flights_tbl, name=\"flights\")\n```\n\n## with_dimensions()\n\nDimensions define the attributes you can group by in your queries. They represent the categorical or descriptive aspects of your data that you want to analyze.\n\nYou can define dimensions using lambda expressions, unbound syntax (`_.`), or the `Dimension` class with descriptions:\n\n```dimensions_demo\nfrom ibis import _\nfrom boring_semantic_layer import Dimension\n\nflights_st = flights_st.with_dimensions(\n # Lambda expressions - simple and explicit\n origin=lambda t: t.origin,\n\n # Unbound syntax - cleaner and more concise\n destination=_.dest,\n year=_.year,\n\n # Dimension - self-documenting and AI-friendly\n carrier=Dimension(\n expr=lambda t: t.carrier,\n description=\"Airline carrier code\"\n )\n)\n\nflights_st.dimensions\n```\n\n\n## with_measures()\n\nMeasures define the aggregations and calculations you can query. They represent the quantitative aspects of your data that you want to analyze (counts, sums, averages, etc.).\n\nYou can define measures using lambda expressions, reference other measures for composition, or use the `Measure` class with descriptions:\n\n```measures_demo\nfrom boring_semantic_layer import Measure\n\nflights_st = flights_st.with_measures(\n # Lambda expressions - simple and concise\n total_flights=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n max_delay=lambda t: t.dep_delay.max(),\n\n # Reference other measures for composition\n avg_distance_per_flight=lambda t: t.total_distance / t.total_flights,\n\n # Measure - self-documenting and AI-friendly\n avg_distance=Measure(\n expr=lambda t: t.distance.mean(),\n description=\"Average flight distance in miles\"\n )\n)\n\nflights_st.measures\n```\n\n\n\n### all()\n\nThe `all()` function references the entire dataset within measure definitions, enabling percent-of-total and comparison calculations.\n\n**Example:** Calculate market share as a percentage\n\n```measure_all_demo\nflights_with_pct = flights_st.with_measures(\n flight_count=lambda t: t.count(),\n market_share=lambda t: t.flight_count / t.all(t.flight_count) * 100 # Percent of total\n )\n\n# Query by carrier\nresult = (\n flights_with_pct\n .group_by(\"carrier\")\n .aggregate(\"flight_count\", \"market_share\")\n)\n```\n\n\n\n\n`t.all(ref)` is available on the table parameter `t` in measure definitions. It\nevaluates the supplied measure or reduction over the entire dataset regardless\nof grouping, making it useful for percentages and comparisons with the total.\n\n\nFor more examples, see the [Percent of Total pattern](/advanced/percentage-total).\n\n## graph\n\nThe `graph` property provides a dependency graph showing how dimensions and measures relate to each other. This is useful for:\n- **Understanding dependencies**: See what columns or fields each dimension/measure depends on\n- **Impact analysis**: Find what breaks when changing a field\n- **Documentation**: Generate visual representations of your data model\n- **Validation**: Ensure your model doesn't have circular dependencies\n\n```graph_demo\n# Build a semantic table with dependencies\nflights_with_deps = flights_st.with_dimensions(\n origin=lambda t: t.origin,\n destination=lambda t: t.dest,\n).with_measures(\n flight_count=lambda t: t.count(),\n total_distance=lambda t: t.distance.sum(),\n avg_distance_per_flight=lambda t: t.total_distance / t.flight_count\n)\n\n# Access the dependency graph\ngraph = flights_with_deps.get_graph()\ngraph\n```\n\n\n### Understanding the Graph Structure\n\nThe graph is a dictionary where:\n- **Keys**: Dimension or measure names\n- **Values**: Metadata containing:\n - `deps`: Dependencies mapped to their types (`'column'`, `'dimension'`, or `'measure'`)\n - `type`: The field type (`'dimension'`, `'measure'`, or `'calc_measure'`)\n\n```graph_structure\n# Access the graph - it's a dict-like object\ngraph = flights_with_deps.get_graph()\ngraph\n```\n\n\n```python\n# Find what a specific field depends on\nflights_with_deps.get_graph()['avg_distance_per_flight']['deps']\n# Output: {'total_distance': 'measure', 'flight_count': 'measure'}\n```\n\n### Graph Traversal\n\nUse `graph_predecessors()` and `graph_successors()` to navigate dependencies:\n\n```graph_traversal\nfrom boring_semantic_layer import graph_predecessors, graph_successors\n\ngraph = flights_with_deps.get_graph()\n\n# What does this field depend on? (predecessors)\ngraph_predecessors(graph, 'avg_distance_per_flight')\n# {'total_distance', 'flight_count'}\n\n# What depends on this field? (successors)\ngraph_successors(graph, 'total_distance')\n# {'avg_distance_per_flight'}\n```\n\n\n### Working with the Dependency Graph\n\nThe dependency graph is a dict-like object where each key is a field name and the value is a dict with `\"type\"` (dimension/measure/calc_measure/column) and `\"deps\"` (dependencies with their types):\n\n```python\n# Access the graph directly as a dict\ngraph = flights_with_deps.get_graph()\n\n# Iterate over fields and their dependencies\nfor field, info in graph.items():\n print(f\"{field} ({info['type']}): depends on {info['deps']}\")\n```\n\n## join_one() / join_many() / join_cross()\n\nJoin semantic tables together to query across relationships. Joins allow you to combine data from multiple semantic tables and access dimensions and measures across all joined tables.\n\n**What Makes Semantic Joins Different?**\n\nSemantic joins explicitly capture the **relationship type** between tables, rather than just specifying SQL join mechanics:\n\n**SQL Joins:**\n```python\n# Specifies HOW to join, but not the analytical relationship\nflights_tbl.left_join(carriers_tbl, flights_tbl.carrier == carriers_tbl.code)\n```\n\n**Semantic Joins:**\n```python\n# One carrier row can match many flight rows\ncarriers.join_many(flights_st, lambda c, f: c.code == f.carrier)\n```\n\n**What You Get:**\n- **Explicit relationships**: `join_many()` documents that this is a one-to-many relationship\n- **Table hierarchy information**: The method name describes how tables relate to each other\n- **Richer metadata**: Makes the data model structure explicit for documentation and tooling\n\n\nAfter joining, dimensions and measures are prefixed with table names (e.g., `flights.origin`, `carriers.name`) to avoid naming conflicts.\n\n\n\n**Give every source in a composed model a unique name.** BSL uses model names as\nsource aliases for dimensions, measures, and grain metadata, and rejects a join\ntree containing duplicate names. If you join the same underlying table more than\nonce (for example, pickup and dropoff locations), create distinct table references\nand assign explicit aliases:\n\n```python\n# Create distinct references when joining same table twice\npickup_locs = to_semantic_table(locs_tbl.view(), \"pickup_locs\")\ndropoff_locs = to_semantic_table(locs_tbl.view(), \"dropoff_locs\")\n```\n\nThe distinct names prevent ambiguous semantic prefixes; `.view()` prevents Ibis\nfrom treating both roles as the same relation.\n\n\n\n**Source-aware aggregation requires equality-key joins.** When BSL aggregates\nmeasures at their source grain before joining, each non-cross join predicate must\nbe a direct field equality or a conjunction of direct field equalities. String,\nDeferred, and compound equality-key shorthands are supported. Predicates using\ninequality, `OR`, casts, or transformed expressions are rejected because reducing\nthem to join-key bridges could change the matched row set. Aggregate the models\nfirst or restate the relationship with plain equality keys; use `join_cross()` for\na Cartesian product.\n\n\nLet's get some additional data:\n\n```setup_carriers\nimport ibis\nfrom boring_semantic_layer import to_semantic_table\n\ncon = ibis.duckdb.connect(\":memory:\")\n\n# Create carriers data\ncarriers_data = ibis.memtable({\n \"code\": [\"AA\", \"UA\", \"DL\"],\n \"name\": [\"American Airlines\", \"United Airlines\", \"Delta Air Lines\"]\n})\ncarriers_tbl = con.create_table(\"carriers\", carriers_data)\n```\n\n\nAnd create a carriers semantic table:\n\n```carriers_st\ncarriers = (\n to_semantic_table(carriers_tbl, name=\"carriers\")\n .with_dimensions(\n code=lambda t: t.code,\n name=lambda t: t.name\n )\n .with_measures(\n carrier_count=lambda t: t.count()\n )\n)\n```\n\n### join_many() - One-to-Many Relationships\n\nUse `join_many()` when one row in the left table can match multiple rows in the right table (LEFT JOIN).\n\n```join_demo\n# One carrier row can match many flight rows\ncarriers_with_flights = carriers.join_many(\n flights_st,\n lambda c, f: c.code == f.carrier\n)\n\n# Inspect available dimensions and measures\ncarriers_with_flights.dimensions\n```\n\n\nAfter joining, all dimensions and measures from both tables are available. Each is prefixed with its table name to avoid conflicts:\n\n\n### join_one() - At-Most-One Right Match\n\nUse `join_one()` when each row on the left can match at most one row on the\nright. Like all non-cross semantic joins, it uses a LEFT JOIN so unmatched left\nrows remain visible to measures.\n\n```python\n# Many flights \u2192 one carrier (each flight matches at most one carrier)\nflights_with_carrier = flights_st.join_one(\n carriers,\n lambda f, c: f.carrier == c.code\n)\n```\n\n\n**Join predicates resolve physical columns.** A string or Deferred shorthand\nnames the same underlying column on both sides. If the columns have different\nnames, use a two-argument lambda instead.\n\n**Example:**\n```python\n# If users table has column 'id' but dimension 'customer_id':\nusers = to_semantic_table(users_tbl, \"users\").with_dimensions(\n customer_id=lambda t: t.id # Dimension renamed\n)\n\n# Compare the underlying columns explicitly:\norders.join_one(users, on=lambda order, user: order.customer_id == user.id)\n```\n\n\n### join_cross() - Cross Join\n\nUse `join_cross()` to create every possible combination of rows from both tables (CARTESIAN PRODUCT).\n\n```python\n# Every flight \u00d7 every carrier combination\nall_combinations = flights_st.join_cross(carriers)\n```\n\n### Requiring a Match\n\n`join_one()` and `join_many()` only support `how=\"left\"`. When a query should\nrequire a match, make the row removal explicit with a filter on a non-nullable\nfield from the right table:\n\n```python\nflights_matched = flights_st.join_one(\n carriers,\n lambda f, c: f.carrier == c.code,\n).filter(lambda t: t[\"carriers.name\"].notnull())\n```\n\nUse `join_cross()` for Cartesian products.\n\n## Next Steps\n\n- Learn about [Composing Models](/building/compose)\n- Explore [YAML Configuration](/building/yaml)\n- Start [Querying Semantic Tables](/querying/methods)\n",
"queries": {
"dimensions_demo": {
"output": "('origin', 'destination', 'year', 'carrier')"
@@ -153,7 +153,7 @@
}
},
"join_demo": {
- "code": "# Join carriers to flights - one carrier has many flights\nflights_with_carriers = flights_st.join_many(\n carriers,\n lambda f, c: f.carrier == c.code\n)\n\n# Inspect available dimensions and measures\nflights_with_carriers.dimensions",
+ "code": "# One carrier row can match many flight rows\ncarriers_with_flights = carriers.join_many(\n flights_st,\n lambda c, f: c.code == f.carrier\n)\n\n# Inspect available dimensions and measures\ncarriers_with_flights.dimensions",
"sql": "WITH \"t1\" AS (\n SELECT\n *\n FROM \"memory\".\"main\".\"flights\" AS \"t0\"\n)\nSELECT\n \"t7\".\"carrier\",\n \"t7\".\"flight_count\",\n (\n CAST(\"t7\".\"flight_count\" AS DOUBLE) / CAST(\"t7\".\"flight_count_right\" AS DOUBLE)\n ) * 100 AS \"market_share\"\nFROM (\n SELECT\n \"t5\".\"carrier\",\n \"t5\".\"flight_count\",\n \"t6\".\"flight_count\" AS \"flight_count_right\"\n FROM (\n SELECT\n \"t2\".\"carrier\",\n COUNT(*) AS \"flight_count\"\n FROM \"t1\" AS \"t2\"\n GROUP BY\n 1\n ) AS \"t5\"\n CROSS JOIN (\n SELECT\n COUNT(*) AS \"flight_count\"\n FROM \"t1\" AS \"t2\"\n ) AS \"t6\"\n) AS \"t7\"",
"plan": "SemanticTable: flights\n origin [dim]\n destination [dim]\n year [dim]\n carrier [dim]\n total_flights [measure]\n total_distance [measure]\n max_delay [measure]\n avg_distance [measure]\n flight_count [measure]\n avg_distance_per_flight [calc]\n market_share [calc]\n-> GroupBy(carrier)\n-> Aggregate(flight_count, market_share)",
"table": {
diff --git a/examples/graph_example.py b/examples/graph_example.py
index 5eab6c92..efee6342 100644
--- a/examples/graph_example.py
+++ b/examples/graph_example.py
@@ -5,31 +5,9 @@
from pathlib import Path
from pprint import pprint
-import ibis
-
from boring_semantic_layer import from_yaml
from boring_semantic_layer.graph_utils import graph_to_dict
-# Create sample data tables
-carriers_tbl = ibis.memtable(
- {
- "code": ["AA", "DL", "UA"],
- "name": ["American Airlines", "Delta Air Lines", "United Airlines"],
- "nickname": ["American", "Delta", "United"],
- }
-)
-
-flights_tbl = ibis.memtable(
- {
- "carrier": ["AA", "DL", "UA", "AA"],
- "origin": ["JFK", "LAX", "ORD", "LAX"],
- "destination": ["LAX", "JFK", "LAX", "ORD"],
- "distance": [2475, 2475, 1745, 1745],
- "dep_delay": [10, -5, 15, 0],
- "arr_time": ["2024-01-01", "2024-01-01", "2024-01-02", "2024-01-02"],
- }
-)
-
# Load semantic models from YAML
yaml_path = Path(__file__).parent / "flights.yml"
profile_file = Path(__file__).parent / "profiles.yml"
@@ -38,13 +16,13 @@
carriers = models["carriers"]
flights = models["flights"]
-# Get the graph for flights model
-print("=== flights.get_graph() ===\n")
-pprint(dict(flights.get_graph()))
+# Get the graph for a standalone model
+print("=== carriers.get_graph() ===\n")
+pprint(dict(carriers.get_graph()))
-# Get graph for joined model
-print("\n\n=== Joined graph (flights with carriers) ===\n")
-joined = flights.join_one(carriers, lambda f, c: f.carrier == c.code)
+# ``from_yaml`` has already applied the joins declared on ``flights``.
+print("\n\n=== Joined graph (flights with YAML-defined joins) ===\n")
+joined = flights
pprint(dict(joined.get_graph()))
print("\n\n=== Graph export to JSON format ===\n")
diff --git a/src/boring_semantic_layer/convert.py b/src/boring_semantic_layer/convert.py
index d9383760..fac8756f 100644
--- a/src/boring_semantic_layer/convert.py
+++ b/src/boring_semantic_layer/convert.py
@@ -266,17 +266,38 @@ def _convert_semantic_filter(node: SemanticFilterOp, catalog, *args):
Resolves dimension references in the filter predicate and applies
the filter to the base table.
"""
- from boring_semantic_layer.ops import SemanticAggregateOp, _get_merged_fields
+ from boring_semantic_layer.ops import (
+ SemanticAggregateOp,
+ _augment_dimensions_with_raw_columns,
+ _exact_filter_fields,
+ _get_merged_fields,
+ _unwrap,
+ _validate_qualified_filter_fields,
+ )
all_roots = _find_all_root_models(node.source)
base_tbl = convert(node.source, catalog=catalog)
+ pred_fn = _unwrap(node.predicate)
+ exact_fields = _exact_filter_fields(pred_fn)
dim_map = (
{}
if isinstance(node.source, SemanticAggregateOp)
- else _get_merged_fields(all_roots, "dimensions")
+ else _get_merged_fields(
+ all_roots,
+ "dimensions",
+ source=node.source,
+ )
)
- pred = node.predicate(_Resolver(base_tbl, dim_map))
+ if not isinstance(node.source, SemanticAggregateOp) and exact_fields:
+ dim_map = _augment_dimensions_with_raw_columns(
+ dim_map,
+ exact_fields,
+ all_roots,
+ node.source,
+ )
+ _validate_qualified_filter_fields(exact_fields, dim_map, all_roots)
+ pred = pred_fn(_Resolver(base_tbl, dim_map))
return base_tbl.filter(pred)
diff --git a/src/boring_semantic_layer/expr.py b/src/boring_semantic_layer/expr.py
index 38a04a58..f28b675f 100644
--- a/src/boring_semantic_layer/expr.py
+++ b/src/boring_semantic_layer/expr.py
@@ -10,12 +10,14 @@
from ibis.expr.types.groupby import GroupedTable as IbisGroupedTable
from ibis.expr.types.relations import Table as IbisTable
from returns.result import Success, safe
+
from ._xorq import (
Column as XorqColumn,
+)
+from ._xorq import (
GroupedTable,
Table,
)
-
from .chart import chart as create_chart
from .measure_scope import MeasureScope
from .ops import (
@@ -33,12 +35,15 @@
SemanticTableOp,
SemanticUnnestOp,
_classify_measure,
+ _exact_filter_fields,
+ _extract_columns_from_callable,
+ _extract_join_key_columns,
_find_all_root_models,
_get_merged_fields,
_is_deferred,
- _unwrap,
_normalize_join_predicate,
_normalize_to_name,
+ _unwrap,
make_bare_ref_lambda,
)
from .query import compare_periods as build_compare_periods
@@ -547,26 +552,109 @@ def _build_post_aggregate_model(source_op, ibis_table: ir.Table) -> SemanticMode
def _get_entity_dims(op) -> frozenset[str]:
- """Return the set of dimension names marked ``is_entity=True`` on an op."""
- from .ops import SemanticTableOp
+ """Return the logical entity grain carried by an operation.
+
+ Pass-through operations (most importantly filters) retain their source
+ grain. A ``join_one`` also retains the left grain, while a ``join_many``
+ can add entity dimensions from its many side. Deriving the grain from the
+ operation tree avoids treating the prefixed copies exposed by a joined
+ model as independent entities merely because metadata has been merged.
+ """
+ if isinstance(op, SemanticFilterOp):
+ return _get_entity_dims(op.source)
+
+ # Aggregation establishes a new result grain. SemanticAggregateOp does
+ # not currently expose entity metadata for that output, so inheriting the
+ # source entities would falsely claim (for example) that a month-level
+ # result still has its source's day-level grain.
+ if isinstance(op, SemanticAggregateOp):
+ return frozenset()
+
+ if isinstance(op, SemanticJoinOp):
+ left_entities = _get_entity_dims(op.left)
+ if op.cardinality == "one":
+ return left_entities or _get_entity_dims(op.right)
+ return left_entities | _get_entity_dims(op.right)
if isinstance(op, SemanticTableOp):
+ source_join = getattr(op, "_source_join", None)
+ if source_join is not None:
+ # A materialized join wrapper carries all inherited dimensions as
+ # prefixed copies plus any dimensions declared directly on the
+ # wrapper. Preserve only the unprefixed local entity declarations;
+ # inherited prefixed copies are already represented by the source
+ # join's logical grain.
+ local_entities = {
+ name
+ for name, dim in op.get_dimensions().items()
+ if "." not in name and getattr(dim, "is_entity", False)
+ }
+ return _get_entity_dims(source_join) | frozenset(local_entities)
return frozenset(
- name for name, dim in op.get_dimensions().items() if getattr(dim, "is_entity", False)
+ # Joined field maps use ``model.field`` names. Leaf model maps do
+ # not, so normalizing the prefix lets same-grain fact models still
+ # compare by their declared semantic entity names.
+ name.rsplit(".", 1)[-1]
+ for name, dim in op.get_dimensions().items()
+ if getattr(dim, "is_entity", False)
)
- return frozenset()
+
+ source = getattr(op, "source", None)
+ return _get_entity_dims(source) if source is not None else frozenset()
def _has_measure_fields(op) -> bool:
"""Return whether an op defines base or calculated measures."""
- from .ops import SemanticTableOp
-
+ get_measures = getattr(op, "get_measures", None)
+ get_calculated = getattr(op, "get_calculated_measures", None)
+ return bool(get_measures and get_measures()) or bool(get_calculated and get_calculated())
+
+
+def _get_entity_source_columns(op) -> frozenset[str]:
+ """Return physical columns that define an operation's logical entity grain."""
+ if isinstance(op, SemanticFilterOp):
+ return _get_entity_source_columns(op.source)
+ if isinstance(op, SemanticAggregateOp):
+ return frozenset()
+ if isinstance(op, SemanticJoinOp):
+ left_columns = _get_entity_source_columns(op.left)
+ if op.cardinality == "one":
+ return left_columns or _get_entity_source_columns(op.right)
+ return left_columns | _get_entity_source_columns(op.right)
if isinstance(op, SemanticTableOp):
- return bool(op.get_measures()) or bool(op.get_calculated_measures())
- return False
+ source_join = getattr(op, "_source_join", None)
+ if source_join is not None:
+ inherited = set(_get_entity_source_columns(source_join))
+ table = op.to_untagged()
+ for name, dim in op.get_dimensions().items():
+ if "." in name or not getattr(dim, "is_entity", False):
+ continue
+ extraction = _extract_columns_from_callable(
+ lambda t, entity_dim=dim: entity_dim(t), table
+ )
+ if extraction.is_success() and extraction.columns:
+ inherited.update(extraction.columns)
+ else:
+ inherited.add(name)
+ return frozenset(inherited)
+ table = op.to_untagged()
+ columns: set[str] = set()
+ for name, dim in op.get_dimensions().items():
+ if not getattr(dim, "is_entity", False):
+ continue
+ extraction = _extract_columns_from_callable(
+ lambda t, entity_dim=dim: entity_dim(t), table
+ )
+ if extraction.is_success() and extraction.columns:
+ columns.update(extraction.columns)
+ else:
+ columns.add(name.rsplit(".", 1)[-1])
+ return frozenset(columns)
+ source = getattr(op, "source", None)
+ return _get_entity_source_columns(source) if source is not None else frozenset()
-def _detect_grain_cardinality(left_op, right_op) -> str:
+def _detect_grain_cardinality(left_op, right_op, on=None) -> str:
"""Compare entity dimensions to detect grain mismatch.
If both sides declare ``is_entity`` dimensions and the sets differ,
@@ -579,10 +667,32 @@ def _detect_grain_cardinality(left_op, right_op) -> str:
left_entities = _get_entity_dims(left_op)
right_entities = _get_entity_dims(right_op)
+ join_misses_entity_grain = False
+ if on is not None and left_entities and left_entities == right_entities:
+ try:
+ normalized_on = _normalize_join_predicate(on)
+ left_table = left_op.to_untagged()
+ right_table = right_op.to_untagged()
+ join_columns = _extract_join_key_columns(
+ normalized_on, left_table, right_table
+ )
+ if join_columns.is_success():
+ left_entity_columns = _get_entity_source_columns(left_op)
+ right_entity_columns = _get_entity_source_columns(right_op)
+ join_misses_entity_grain = not (
+ left_entity_columns <= join_columns.left_columns
+ and right_entity_columns <= join_columns.right_columns
+ )
+ except Exception:
+ # Inconclusive predicate analysis keeps the explicit join_one
+ # contract; aggregation-time validation still fails closed where
+ # source-aware preaggregation depends on predicate shape.
+ join_misses_entity_grain = False
+
if (
left_entities
and right_entities
- and left_entities != right_entities
+ and (left_entities != right_entities or join_misses_entity_grain)
and _has_measure_fields(left_op)
and _has_measure_fields(right_op)
):
@@ -590,7 +700,8 @@ def _detect_grain_cardinality(left_op, right_op) -> str:
right_name = getattr(right_op, "name", None) or "right"
warnings.warn(
f"Grain mismatch detected: {left_name} entity dims {sorted(left_entities)} "
- f"!= {right_name} entity dims {sorted(right_entities)}. "
+ f"and {right_name} entity dims {sorted(right_entities)} are not both "
+ "covered by the join keys. "
f"Upgrading join_one to join_many for automatic pre-aggregation.",
stacklevel=2,
)
@@ -598,6 +709,225 @@ def _detect_grain_cardinality(left_op, right_op) -> str:
return "one"
+def _join_one_with_detected_grain(
+ left_op,
+ other,
+ on: Callable[[Any, Any], ir.BooleanValue] | str | Deferred | Sequence[str | Deferred],
+ how: str,
+) -> SemanticJoin:
+ """Construct ``join_one`` consistently for every semantic wrapper."""
+ other_op = other.op() if isinstance(other, SemanticTable) else other
+ cardinality = _detect_grain_cardinality(left_op, other_op, on)
+ return SemanticJoin(
+ left=left_op,
+ right=other_op,
+ on=on,
+ how=how,
+ cardinality=cardinality,
+ )
+
+
+def _find_join_provenance(op):
+ """Find the join represented by an op or a materialized model wrapper."""
+ if isinstance(op, SemanticJoinOp):
+ return op
+ if isinstance(op, SemanticTableOp):
+ return getattr(op, "_source_join", None)
+ source = getattr(op, "source", None)
+ return _find_join_provenance(source) if source is not None else None
+
+
+def _replace_metadata_preserving_filters(
+ op,
+ *,
+ dimensions: Mapping[str, Dimension | Callable | dict],
+ measures: Mapping[str, Measure | Callable],
+ calc_measures: Mapping[str, Any],
+) -> SemanticTable:
+ """Replace metadata without collapsing filters into a flat table.
+
+ Fanout-safe aggregation discovers both joins and the predicates between a
+ join and an aggregate from the semantic operation tree. Materializing a
+ ``SemanticFilterOp`` into a new leaf model erases those predicates. Peel
+ the contiguous filter chain, update the underlying model metadata, and
+ rebuild the same chain so the planner can still route each predicate to
+ its owning source before pre-aggregation.
+ """
+ predicates: list[
+ tuple[
+ Callable,
+ Mapping[str, Dimension],
+ frozenset[str],
+ bool,
+ Callable | None,
+ ]
+ ] = []
+ source = op
+ while isinstance(source, SemanticFilterOp):
+ predicate_source = source.source
+ get_dimensions = getattr(predicate_source, "get_dimensions", None)
+ predicate_dims = dict(get_dimensions()) if get_dimensions else {}
+ predicate = _unwrap(source.predicate)
+ try:
+ deferred_resolution = bool(
+ object.__getattribute__(
+ predicate, "__bsl_deferred_resolution__"
+ )
+ )
+ except (AttributeError, TypeError):
+ deferred_resolution = False
+ try:
+ serialization_predicate = object.__getattribute__(
+ predicate, "__bsl_serialization_predicate__"
+ )
+ except (AttributeError, TypeError):
+ serialization_predicate = None
+ predicates.append(
+ (
+ predicate,
+ predicate_dims,
+ _exact_filter_fields(predicate),
+ deferred_resolution,
+ serialization_predicate,
+ )
+ )
+ source = source.source
+
+ source_join = _find_join_provenance(source)
+ if isinstance(source, SemanticTableOp):
+ table = source.table
+ name = source.name
+ description = source.description
+ else:
+ table = source.to_untagged()
+ name = getattr(source, "name", None)
+ description = getattr(source, "description", None)
+
+ rebuilt: SemanticTable = SemanticModel(
+ table=table,
+ dimensions=dimensions,
+ measures=measures,
+ calc_measures=calc_measures,
+ name=name,
+ description=description,
+ _source_join=source_join,
+ )
+ for (
+ predicate,
+ predicate_dims,
+ exact_filter_fields,
+ deferred_resolution,
+ prior_serialization_predicate,
+ ) in reversed(predicates):
+ # Bind each predicate to the dimension definitions visible when the
+ # filter was authored. A later metadata overlay may intentionally
+ # replace a same-named dimension (for example raw timestamp -> month
+ # bucket); letting the earlier row filter see that replacement changes
+ # filter-before-transform semantics and drops partially covered buckets.
+ def bound_predicate(
+ t,
+ pred=predicate,
+ dims=predicate_dims,
+ fields=exact_filter_fields,
+ ):
+ from .convert import _Resolver
+
+ # Filter lowering normally passes a resolver carrying the *new*
+ # metadata. Unwrap it to the physical relation before applying the
+ # dimensions captured above, otherwise an old identity dimension
+ # still delegates through the new same-named month bucket.
+ try:
+ physical_table = object.__getattribute__(t, "_t")
+ except (AttributeError, TypeError):
+ physical_table = t
+ scope_dims = dict(dims)
+ try:
+ runtime_dims = object.__getattribute__(t, "_dims")
+ except (AttributeError, TypeError):
+ runtime_dims = {}
+ # Exact JSON fields that were undeclared when the filter was
+ # authored are synthesized for the current execution scope. A
+ # joined-table resolver maps them to collision-safe physical
+ # aliases, while a source-local resolver maps them to raw columns.
+ # Only fill missing entries: declared dimensions captured at the
+ # filter boundary must retain their original transformation.
+ for field in fields:
+ if field not in scope_dims and field in runtime_dims:
+ scope_dims[field] = runtime_dims[field]
+ scope = _Resolver(physical_table, scope_dims)
+ return pred.resolve(scope) if _is_deferred(pred) else pred(scope)
+
+ def serialization_predicate(
+ t,
+ pred=predicate,
+ dims=predicate_dims,
+ fields=exact_filter_fields,
+ ):
+ """Inline the filter against its author-time dimension scope."""
+ from .convert import _Resolver
+
+ symbolic_physical = t._t
+
+ class _SymbolicPhysicalTable:
+ """Expose raw fields while reserving exact semantic lookups."""
+
+ def __getattr__(self, name):
+ if name in fields:
+ raise AttributeError(name)
+ return getattr(symbolic_physical, name)
+
+ def __getitem__(self, name):
+ return symbolic_physical[name]
+
+ @property
+ def columns(self):
+ return symbolic_physical.columns
+
+ physical_scope = _SymbolicPhysicalTable()
+ serialization_dims = {
+ name: (
+ lambda _table, _dim=dimension, _physical=symbolic_physical: (
+ _dim.resolve(_physical)
+ if _is_deferred(_dim)
+ else _dim(_physical)
+ )
+ )
+ for name, dimension in dims.items()
+ }
+ for field in fields:
+ if field not in serialization_dims:
+ # Keep undeclared exact fields in semantic resolver scope;
+ # their sidecar metadata synthesizes the appropriate
+ # joined/raw mapping after reconstruction.
+ serialization_dims[field] = (
+ lambda _table, _field=field, _scope=t: _scope[_field]
+ )
+ scope = _Resolver(physical_scope, serialization_dims)
+ return pred.resolve(scope) if _is_deferred(pred) else pred(scope)
+
+ # JSON filters need their exact AST field spellings after this
+ # metadata-preserving rebind. Without them, a valid undeclared
+ # qualified raw field (for example ``items.status``) cannot be
+ # synthesized in the joined namespace and the filter becomes
+ # unresolvable. Preserve the separate deferred marker for ordinary
+ # string filters as well; it controls dimension-table shortcuts.
+ if exact_filter_fields:
+ bound_predicate.__bsl_filter_fields__ = exact_filter_fields
+ if deferred_resolution:
+ bound_predicate.__bsl_deferred_resolution__ = True
+ # The generic serializer cannot safely infer the semantics of this
+ # closure from its captured resolver state. Give it an explicit
+ # symbolic source that inlines the dimensions visible when the filter
+ # was authored. This retains filter-before-overlay behavior without
+ # serializing Dimension objects themselves.
+ bound_predicate.__bsl_serialization_predicate__ = (
+ prior_serialization_predicate or serialization_predicate
+ )
+
+ rebuilt = SemanticFilter(source=rebuilt.op(), predicate=bound_predicate)
+ return rebuilt
+
+
class SemanticModel(SemanticTable):
def __init__(
self,
@@ -759,9 +1089,7 @@ def join_one(
>>> orders.join_one(customers, on=_.customer_id)
>>> orders.join_one(customers, on=lambda o, c: o.customer_id == c.customer_id)
"""
- other_op = other.op() if isinstance(other, SemanticModel) else other
- cardinality = _detect_grain_cardinality(self.op(), other_op)
- return SemanticJoin(left=self.op(), right=other_op, on=on, how=how, cardinality=cardinality)
+ return _join_one_with_detected_grain(self.op(), other, on, how)
def join_many(
self,
@@ -1133,13 +1461,7 @@ def join_one(
how: str = "left",
) -> SemanticJoin:
"""Join with one-to-one relationship semantics."""
- return SemanticJoin(
- left=self.op(),
- right=other.op() if isinstance(other, SemanticModel) else other,
- on=on,
- how=how,
- cardinality="one",
- )
+ return _join_one_with_detected_grain(self.op(), other, on, how)
def join_many(
self,
@@ -1199,6 +1521,28 @@ def values(self):
def schema(self):
return self.op().schema
+ def _metadata_source(self):
+ source = self.op().source
+ while isinstance(source, SemanticFilterOp):
+ source = source.source
+ return source
+
+ @property
+ def name(self):
+ return getattr(self._metadata_source(), "name", None)
+
+ @property
+ def description(self):
+ return getattr(self._metadata_source(), "description", None)
+
+ @property
+ def table(self):
+ return self.op().to_untagged()
+
+ @property
+ def json_definition(self):
+ return getattr(self._metadata_source(), "json_definition", {})
+
def get_dimensions(self):
return self.op().get_dimensions()
@@ -1218,26 +1562,82 @@ def measures(self):
"""Return measure names as a tuple."""
return tuple(self.get_measures().keys()) + tuple(self.get_calculated_measures().keys())
+ @property
+ def calc_measures(self):
+ return dict(self.get_calculated_measures())
+
+ def query(
+ self,
+ dimensions: Sequence[str] | None = None,
+ measures: Sequence[str] | None = None,
+ filters: list | None = None,
+ order_by: Sequence[tuple[str, str]] | None = None,
+ limit: int | None = None,
+ time_grain: str | None = None,
+ time_grains: dict[str, str] | None = None,
+ time_range: dict[str, str] | None = None,
+ having: list | None = None,
+ ):
+ return build_query(
+ semantic_table=self,
+ dimensions=dimensions,
+ measures=measures,
+ filters=filters,
+ order_by=order_by,
+ limit=limit,
+ time_grain=time_grain,
+ time_grains=time_grains,
+ time_range=time_range,
+ having=having,
+ )
+
+ def compare_periods(
+ self,
+ dimensions: Sequence[str] | None = None,
+ measures: Sequence[str] | None = None,
+ current_time_range: dict[str, str] | None = None,
+ previous_time_range: dict[str, str] | None = None,
+ filters: list | None = None,
+ time_dimension: str | None = None,
+ time_grain: str | None = None,
+ time_grains: dict[str, str] | None = None,
+ order_by: Sequence[tuple[str, str]] | None = None,
+ limit: int | None = None,
+ ):
+ return build_compare_periods(
+ semantic_table=self,
+ dimensions=dimensions,
+ measures=measures,
+ current_time_range=current_time_range,
+ previous_time_range=previous_time_range,
+ filters=filters,
+ time_dimension=time_dimension,
+ time_grain=time_grain,
+ time_grains=time_grains,
+ order_by=order_by,
+ limit=limit,
+ )
+
def as_table(self) -> SemanticModel:
all_roots = _find_all_root_models(self.op().source)
return _build_semantic_model_from_roots(self.op().to_untagged(), all_roots)
- def with_dimensions(self, **dims) -> SemanticModel:
- """Add or update dimensions after filtering."""
+ def with_dimensions(self, **dims) -> SemanticTable:
+ """Add or update dimensions while retaining filter/join lineage."""
all_roots = _find_all_root_models(self.op().source)
existing_dims = _get_merged_fields(all_roots, "dimensions") if all_roots else {}
existing_meas = _get_merged_fields(all_roots, "measures") if all_roots else {}
existing_calc = _get_merged_fields(all_roots, "calc_measures") if all_roots else {}
- return SemanticModel(
- table=self.op().to_untagged(),
+ return _replace_metadata_preserving_filters(
+ self.op(),
dimensions={**existing_dims, **dims},
measures=existing_meas,
calc_measures=existing_calc,
)
- def with_measures(self, **meas) -> SemanticModel:
- """Add or update measures after filtering."""
+ def with_measures(self, **meas) -> SemanticTable:
+ """Add or update measures while retaining filter/join lineage."""
all_roots = _find_all_root_models(self.op().source)
existing_dims = _get_merged_fields(all_roots, "dimensions") if all_roots else {}
existing_meas = _get_merged_fields(all_roots, "measures") if all_roots else {}
@@ -1255,8 +1655,8 @@ def with_measures(self, **meas) -> SemanticModel:
kind, value = _classify_measure(fn_or_expr, scope, name)
_store_classified_measure(name, kind, value, new_base_meas, new_calc_meas)
- return SemanticModel(
- table=self.op().to_untagged(),
+ return _replace_metadata_preserving_filters(
+ self.op(),
dimensions=existing_dims,
measures=new_base_meas,
calc_measures=new_calc_meas,
@@ -1269,13 +1669,7 @@ def join_one(
how: str = "left",
) -> SemanticJoin:
"""Join with one-to-one relationship semantics."""
- return SemanticJoin(
- left=self.op(),
- right=other.op() if isinstance(other, SemanticModel) else other,
- on=on,
- how=how,
- cardinality="one",
- )
+ return _join_one_with_detected_grain(self.op(), other, on, how)
def join_many(
self,
@@ -1653,13 +2047,7 @@ def join_one(
how: str = "left",
) -> SemanticJoin:
"""Join with one-to-one relationship semantics."""
- return SemanticJoin(
- left=self.op(),
- right=other.op(),
- on=on,
- how=how,
- cardinality="one",
- )
+ return _join_one_with_detected_grain(self.op(), other, on, how)
def join_many(
self,
diff --git a/src/boring_semantic_layer/nested_compile.py b/src/boring_semantic_layer/nested_compile.py
index 7ea4c859..5b3fb0d8 100644
--- a/src/boring_semantic_layer/nested_compile.py
+++ b/src/boring_semantic_layer/nested_compile.py
@@ -17,7 +17,7 @@
from typing import Any
import ibis
-from toolz import curry, pipe
+from toolz import curry
from .join_utils import null_safe_equal
@@ -54,14 +54,50 @@ def _unwrap_table_proxy(obj):
return obj
-@curry
-def _extract_nested_array(prev_col: str, array_col: str, table):
+def _allocate_nested_array_name(table, idx: int) -> str:
+ """Allocate a private column name without overwriting user data."""
+ occupied = frozenset(table.columns)
+ preferred = f"__bsl_nested_array_{idx}"
+ candidate = preferred
+ suffix = 2
+ while candidate in occupied:
+ candidate = f"{preferred}_{suffix}"
+ suffix += 1
+ return candidate
+
+
+def _extract_nested_array(
+ prev_col: str,
+ array_col: str,
+ materialized_col: str,
+ table,
+ *,
+ requested_path: tuple[str, ...],
+ parent_path: tuple[str, ...],
+):
+ """Materialize ``prev_col[array_col]`` under a private column name."""
+ path_label = ".".join(requested_path)
+ parent_label = ".".join(parent_path)
if prev_col not in table.columns:
- return table
+ raise ValueError(
+ f"Cannot traverse nested array path {path_label!r}: "
+ f"parent {parent_label!r} is unavailable after unnesting."
+ )
prev_struct = table[prev_col]
- if not hasattr(prev_struct, array_col):
- return table
- return table.mutate(**{array_col: getattr(prev_struct, array_col)})
+ fields = getattr(prev_struct.type(), "fields", {})
+ if array_col not in fields:
+ raise ValueError(
+ f"Cannot traverse nested array path {path_label!r}: "
+ f"child {array_col!r} does not exist under {parent_label!r}."
+ )
+ child_type = fields[array_col]
+ if not str(child_type).startswith("array"):
+ raise ValueError(
+ f"Cannot traverse nested array path {path_label!r}: "
+ f"child {array_col!r} under {parent_label!r} is not an array "
+ f"(found {child_type})."
+ )
+ return table.mutate(**{materialized_col: prev_struct[array_col]})
@curry
@@ -71,18 +107,53 @@ def _do_unnest_array(array_col: str, table):
def unnest_nested_arrays(base_tbl, array_path: tuple[str, ...]):
"""Apply unnest steps for each level of a nested array path."""
- sorted_path = tuple(sorted(array_path))
-
- def unnest_step(table, indexed_col):
- idx, array_col = indexed_col
+ # ``array_path`` describes a traversal, not an unordered collection. In
+ # particular, for ``events.products`` we must unnest ``events`` before the
+ # nested ``products`` array can be extracted from the resulting struct.
+ # Sorting the names made the result depend on their spelling and could
+ # silently turn a nested count into a count of the base rows.
+ table = base_tbl
+ parent_col: str | None = None
+ for idx, array_col in enumerate(tuple(array_path)):
if idx == 0:
- return _do_unnest_array(array_col, table)
- prev_col = sorted_path[idx - 1]
- if array_col in table.columns:
- return _do_unnest_array(array_col, table)
- return pipe(table, _extract_nested_array(prev_col, array_col), _do_unnest_array(array_col))
-
- return reduce(unnest_step, enumerate(sorted_path), base_tbl)
+ path_label = ".".join(array_path)
+ if array_col not in table.columns:
+ raise ValueError(
+ f"Cannot traverse nested array path {path_label!r}: "
+ f"root {array_col!r} does not exist."
+ )
+ root_type = table[array_col].type()
+ if not str(root_type).startswith("array"):
+ raise ValueError(
+ f"Cannot traverse nested array path {path_label!r}: "
+ f"root {array_col!r} is not an array (found {root_type})."
+ )
+ table = _do_unnest_array(array_col, table)
+ parent_col = array_col
+ continue
+
+ # A later name is a child of the struct produced by the previous
+ # unnest step. It must never fall back to an unrelated top-level
+ # sibling that happens to have the same name.
+ if parent_col is None:
+ raise ValueError(
+ f"Cannot traverse nested array path {'.'.join(array_path)!r}: "
+ f"parent {'.'.join(array_path[:idx])!r} is unavailable after "
+ "unnesting."
+ )
+ materialized_col = _allocate_nested_array_name(table, idx)
+ table = _extract_nested_array(
+ parent_col,
+ array_col,
+ materialized_col,
+ table,
+ requested_path=tuple(array_path),
+ parent_path=tuple(array_path[:idx]),
+ )
+ table = _do_unnest_array(materialized_col, table)
+ parent_col = materialized_col
+
+ return table
@curry
@@ -145,9 +216,35 @@ def build_nested_level_table(
measures: dict[str, tuple[Any, Any]],
):
"""Aggregate nested-array measures at the unnested grain."""
+ by_cols = tuple(by_cols)
level_aggs = build_level_aggregations(base_tbl, array_path, measures)
unnested_tbl = unnest_nested_arrays(base_tbl, array_path)
- return _make_grouped_table(level_aggs, by_cols, unnested_tbl)
+ level_table = _make_grouped_table(level_aggs, by_cols, unnested_tbl)
+
+ if not by_cols:
+ # A global aggregate already produces one row for an empty input.
+ return level_table
+
+ # UNNEST has inner semantics: a base row whose array is NULL or empty
+ # contributes no unnested row. If every row in a group has an empty
+ # array, aggregating only the unnested relation drops the group entirely.
+ # Build the group domain from the base relation and attach the nested
+ # aggregate to it so nested-only queries retain the same group domain as
+ # the semantic model.
+ group_spine = _make_grouped_table({}, by_cols, base_tbl)
+ result = join_tables(by_cols, [group_spine, level_table])
+
+ # COUNT and COUNT DISTINCT have an empty-set identity of zero. After the
+ # left join, their missing level aggregate is NULL, so restore that
+ # identity. Other aggregates intentionally remain NULL for an empty set.
+ count_like = {
+ name
+ for name, (_agg_fn, marker) in measures.items()
+ if marker.operation in {"count", "nunique"}
+ }
+ if count_like:
+ result = result.mutate(**{name: result[name].fill_null(0) for name in count_like})
+ return result
def join_tables(by_cols: Iterable[str], tables: list) -> Any:
@@ -157,6 +254,9 @@ def join_tables(by_cols: Iterable[str], tables: list) -> Any:
if len(tables) == 1:
return tables[0]
+ # Materialize once because callers may provide a generator and this value
+ # is traversed repeatedly while joining multiple nesting levels.
+ by_cols = tuple(by_cols)
by_cols_set = set(by_cols)
def join_step(left, right):
diff --git a/src/boring_semantic_layer/ops.py b/src/boring_semantic_layer/ops.py
index 448d07d2..f32f0873 100644
--- a/src/boring_semantic_layer/ops.py
+++ b/src/boring_semantic_layer/ops.py
@@ -88,6 +88,63 @@ def _reductions_for_expr(expr):
_BSL_JOIN_KEY_TMP_PREFIX = "__bsl_jk_"
+def _allocate_temporary_join_names(
+ conflicting: Iterable[str],
+ left_columns: Sequence[str],
+ right_columns: Sequence[str],
+) -> dict[str, str]:
+ """Allocate collision-free internal names for left predicate columns."""
+ occupied = set(left_columns) | set(right_columns)
+ result: dict[str, str] = {}
+ for name in left_columns:
+ if name not in conflicting:
+ continue
+ preferred = f"{_BSL_JOIN_KEY_TMP_PREFIX}{name}"
+ candidate = preferred
+ suffix = 2
+ while candidate in occupied:
+ candidate = f"{preferred}_{suffix}"
+ suffix += 1
+ result[name] = candidate
+ occupied.add(candidate)
+ return result
+
+
+def _allocate_right_collision_names(
+ conflicting: Iterable[str],
+ left_columns: Sequence[str],
+ right_columns: Sequence[str],
+ depth: int,
+ *,
+ reserved: Iterable[str] = (),
+) -> dict[str, str]:
+ """Allocate the executable names of colliding right-side columns.
+
+ The usual names are ``x_right``, ``x_right2``, ... according to join
+ depth. A source table is allowed to contain those spellings itself, so
+ advance the numeric suffix until the name is unique across every column
+ that will remain in the result.
+ """
+ conflicting = frozenset(conflicting)
+ occupied = (
+ set(left_columns)
+ | (set(right_columns) - conflicting)
+ | set(reserved)
+ )
+ result: dict[str, str] = {}
+ for name in right_columns:
+ if name not in conflicting:
+ continue
+ candidate = f"{name}_right" if depth <= 1 else f"{name}_right{depth}"
+ suffix = max(2, depth + 1)
+ while candidate in occupied:
+ candidate = f"{name}_right{suffix}"
+ suffix += 1
+ result[name] = candidate
+ occupied.add(candidate)
+ return result
+
+
class _RenamedResolver:
"""Resolver that maps original column names to temporary names.
@@ -120,6 +177,10 @@ def __getattr__(self, name):
mapped = self._name_map.get(name, name)
return getattr(self._table, mapped)
+ def __getitem__(self, name):
+ mapped = self._name_map.get(name, name)
+ return self._table[mapped]
+
def _is_deferred(expr) -> bool:
# Duck-type check: works for both ibis and xorq Deferred objects
@@ -354,6 +415,23 @@ def _unwrap(wrapped: Any) -> Any:
return wrapped.unwrap if isinstance(wrapped, _CallableWrapper) else wrapped
+def _exact_filter_fields(predicate: Any) -> frozenset[str]:
+ """Return exact JSON-AST field spellings attached by ``query.Filter``."""
+ fn = _unwrap(predicate)
+ # Ibis/XORQ Deferred implements ``__getattr__`` by constructing another
+ # Deferred expression. A normal ``getattr(..., default)`` therefore does
+ # not return the default for ordinary deferred/callable filters and trying
+ # to iterate that synthetic expression raises. Only metadata explicitly
+ # attached to the generated JSON-filter callable counts here.
+ try:
+ fields = object.__getattribute__(fn, "__bsl_filter_fields__")
+ except (AttributeError, TypeError):
+ return frozenset()
+ if not isinstance(fields, (set, frozenset, tuple, list)):
+ return frozenset()
+ return frozenset(fields)
+
+
def _collect_chain(op: Relation) -> list[Relation]:
"""Walk op.source (or op.left for joins) back to root, return list from root to current."""
chain = [op]
@@ -550,11 +628,16 @@ def _get_field_dict(root: Any, field_type: str) -> dict:
return dict(getattr(root, method_name)())
-def _get_merged_fields(all_roots: list, field_type: str) -> dict:
+def _get_merged_fields(
+ all_roots: list,
+ field_type: str,
+ source: Relation | None = None,
+) -> dict:
return (
_merge_fields_with_prefixing(
all_roots,
lambda r: _get_field_dict(r, field_type),
+ source=source,
)
if len(all_roots) > 1
else _get_field_dict(all_roots[0], field_type)
@@ -582,14 +665,35 @@ def _augment_dimensions_with_raw_columns(
Declared dimensions always win over synthesized ones.
"""
+ expanded_roots: list[Any] = []
+
+ def expand(root):
+ source_join = getattr(root, "_source_join", None)
+ if source_join is None:
+ expanded_roots.append(root)
+ return
+ for child in _find_all_root_models(source_join):
+ expand(child)
+
+ for root in all_roots:
+ expand(root)
+
requested: dict[str, dict[str, Dimension]] = {}
for key in keys:
if key in merged_dimensions or "." not in key:
continue
prefix, col = key.split(".", 1)
- for root in all_roots:
+ for root in expanded_roots:
if root.name != prefix:
continue
+ declared = _get_field_dict(root, "dimensions")
+ if col in declared:
+ # A single standalone model stores semantic dimensions under
+ # bare names. Its accepted ``model.dimension`` convenience
+ # spelling must preserve the declared expression rather than
+ # synthesizing an identity over a same-named raw column.
+ requested.setdefault(prefix, {})[col] = declared[col]
+ break
cols = getattr(getattr(root, "table", None), "columns", ())
if col in cols:
requested.setdefault(prefix, {})[col] = Dimension(
@@ -599,13 +703,56 @@ def _augment_dimensions_with_raw_columns(
if not requested:
return dict(merged_dimensions)
synthesized = _merge_fields_with_prefixing(
- all_roots,
+ expanded_roots,
lambda r: requested.get(r.name, {}),
source=source,
)
return {**dict(synthesized), **dict(merged_dimensions)}
+def _validate_qualified_filter_fields(
+ fields: Iterable[str],
+ dimensions: Mapping[str, Any],
+ all_roots: Sequence[Any],
+) -> None:
+ """Fail closed when an exact JSON field names no semantic source."""
+ expanded_roots: list[Any] = []
+
+ def expand(root):
+ source_join = getattr(root, "_source_join", None)
+ if source_join is None:
+ expanded_roots.append(root)
+ return
+ for child in _find_all_root_models(source_join):
+ expand(child)
+
+ for root in all_roots:
+ expand(root)
+
+ roots_by_name = {
+ root.name: root for root in expanded_roots if getattr(root, "name", None)
+ }
+ for field_name in fields:
+ if "." not in field_name or field_name in dimensions:
+ continue
+ prefix, raw_name = field_name.split(".", 1)
+ root = roots_by_name.get(prefix)
+ if root is None:
+ available = ", ".join(sorted(roots_by_name)) or "none"
+ raise KeyError(
+ f"Unknown semantic model prefix {prefix!r} in filter field "
+ f"{field_name!r}. Available model prefixes: {available}."
+ )
+ raw_columns = frozenset(
+ getattr(getattr(root, "table", None), "columns", ())
+ )
+ if raw_name not in raw_columns:
+ raise KeyError(
+ f"Unknown field {raw_name!r} on semantic model {prefix!r} "
+ f"in filter field {field_name!r}."
+ )
+
+
def _reject_unresolvable_group_keys(
keys: Iterable[str],
merged_dimensions: Mapping[str, Any],
@@ -1077,12 +1224,20 @@ def _classify_measure(
def _build_json_definition(
dims_dict: dict,
meas_dict: dict,
+ calc_meas_dict: Mapping[str, CalcMeasure] | None = None,
name: str | None = None,
description: str | None = None,
) -> dict:
+ calc_meas_dict = dict(calc_meas_dict or {})
result = {
"dimensions": {n: spec.to_json() for n, spec in dims_dict.items()},
- "measures": {n: spec.to_json() for n, spec in meas_dict.items()},
+ "measures": {
+ **{n: spec.to_json() for n, spec in meas_dict.items()},
+ **{n: spec.to_json() for n, spec in calc_meas_dict.items()},
+ },
+ "calculated_measures": {
+ n: spec.to_json() for n, spec in calc_meas_dict.items()
+ },
"entity_dimensions": {n: spec.to_json() for n, spec in dims_dict.items() if spec.is_entity},
"event_timestamp": {
n: spec.to_json() for n, spec in dims_dict.items() if spec.is_event_timestamp
@@ -1409,6 +1564,7 @@ def json_definition(self) -> Mapping[str, Any]:
return _build_json_definition(
self.get_dimensions(),
self.get_measures(),
+ self.get_calculated_measures(),
self.name,
self.description,
)
@@ -1502,11 +1658,21 @@ def to_untagged(self):
all_roots = _find_all_root_models(self.source)
base_tbl = _to_untagged(self.source)
+ pred_fn = _unwrap(self.predicate)
+ exact_fields = _exact_filter_fields(pred_fn)
dim_map = (
{}
if isinstance(self.source, SemanticAggregateOp)
- else _get_merged_fields(all_roots, "dimensions")
+ else _get_merged_fields(all_roots, "dimensions", source=self.source)
)
+ if not isinstance(self.source, SemanticAggregateOp) and exact_fields:
+ dim_map = _augment_dimensions_with_raw_columns(
+ dim_map,
+ exact_fields,
+ all_roots,
+ self.source,
+ )
+ _validate_qualified_filter_fields(exact_fields, dim_map, all_roots)
# Enrich table with derived dimensions so multi-level deps
# (e.g. d_two -> d_one -> distance) resolve correctly in filters.
@@ -1521,7 +1687,6 @@ def to_untagged(self):
except (TypeError, KeyError, AttributeError):
pass
- pred_fn = _unwrap(self.predicate)
resolver = _Resolver(enriched, dim_map)
pred = _resolve_expr(pred_fn, resolver)
return enriched.filter(pred)
@@ -1653,7 +1818,9 @@ def to_untagged(self):
if not all_roots:
return tbl.select([getattr(tbl, f) for f in self.fields])
- merged_dimensions = _get_merged_fields(all_roots, "dimensions")
+ merged_dimensions = _get_merged_fields(
+ all_roots, "dimensions", source=self.source
+ )
merged_measures = _get_merged_fields(all_roots, "measures")
dims, meas, raw_fields = _classify_fields(self.fields, merged_dimensions, merged_measures)
@@ -2417,19 +2584,45 @@ def _collect_join_tree_info(join_op: SemanticJoinOp) -> _JoinTreeInfo:
table_cardinalities: dict[str, str] = {}
table_ops: dict[str, SemanticTableOp] = {}
- def walk(node, is_right_of_many=False):
+ def walk(node, inherited_cardinality: str | None = None):
if isinstance(node, SemanticJoinOp):
- this_is_many = node.cardinality in ("many", "cross")
- walk(node.left, is_right_of_many=is_right_of_many)
- walk(node.right, is_right_of_many=is_right_of_many or this_is_many)
+ walk(node.left, inherited_cardinality=inherited_cardinality)
+ if inherited_cardinality == "many" or node.cardinality == "many":
+ right_cardinality = "many"
+ elif inherited_cardinality == "cross" or node.cardinality == "cross":
+ right_cardinality = "cross"
+ else:
+ right_cardinality = "one"
+ walk(node.right, inherited_cardinality=right_cardinality)
elif isinstance(node, SemanticTableOp):
+ source_join = getattr(node, "_source_join", None)
+ if source_join is not None:
+ walk(source_join, inherited_cardinality=inherited_cardinality)
+ return
name = node.name
if name:
table_ops[name] = node
- if is_right_of_many:
- table_cardinalities[name] = "many"
+ if inherited_cardinality in ("many", "cross"):
+ table_cardinalities[name] = inherited_cardinality
elif name not in table_cardinalities:
table_cardinalities[name] = "one"
+ else:
+ source = getattr(node, "source", None)
+ if source is None:
+ return
+ roots = _find_all_root_models(node)
+ if len(roots) == 1 and roots[0].name:
+ # A pass-through wrapper used as a join operand (notably a
+ # pre-join filter) is the executable source for this leg.
+ # Retain it instead of silently replacing it with the raw leaf.
+ name = roots[0].name
+ table_ops[name] = node
+ if inherited_cardinality in ("many", "cross"):
+ table_cardinalities[name] = inherited_cardinality
+ elif name not in table_cardinalities:
+ table_cardinalities[name] = "one"
+ else:
+ walk(source, inherited_cardinality=inherited_cardinality)
walk(join_op)
@@ -2437,6 +2630,11 @@ def walk(node, is_right_of_many=False):
def find_leftmost(node):
if isinstance(node, SemanticJoinOp):
return find_leftmost(node.left)
+ if isinstance(node, SemanticTableOp) and node._source_join is not None:
+ return find_leftmost(node._source_join)
+ source = getattr(node, "source", None)
+ if source is not None:
+ return find_leftmost(source)
return getattr(node, "name", None)
root_name = find_leftmost(join_op)
@@ -2456,6 +2654,82 @@ def find_leftmost(node):
)
+def _validate_preaggregation_join_predicates(join_op: SemanticJoinOp) -> None:
+ """Require source-preaggregated joins to be plain field equijoins.
+
+ The preaggregation planner uses join-key bridges to reconnect source-grain
+ aggregates. Those bridges preserve conjunctions of direct field equality
+ pairs; they cannot reproduce inequalities, OR predicates, casts, or other
+ transformed expressions. Accepting those shapes and retaining only the
+ accessed column names produces a different row set from the actual join, so
+ fail closed until the planner carries the complete predicate expression.
+ """
+ from .convert import _Resolver
+
+ def _is_field(node) -> bool:
+ return type(node).__name__ == "Field"
+
+ def _valid(node, left_rel, right_rel) -> bool:
+ node_name = type(node).__name__
+ if node_name == "And":
+ return _valid(node.left, left_rel, right_rel) and _valid(
+ node.right, left_rel, right_rel
+ )
+ if node_name != "Equals":
+ return False
+ left_arg, right_arg = node.left, node.right
+ if not (_is_field(left_arg) and _is_field(right_arg)):
+ return False
+ left_arg_rel = getattr(left_arg, "rel", None)
+ right_arg_rel = getattr(right_arg, "rel", None)
+ return (left_arg_rel is left_rel and right_arg_rel is right_rel) or (
+ left_arg_rel is right_rel and right_arg_rel is left_rel
+ )
+
+ def _walk(node) -> None:
+ if not isinstance(node, SemanticJoinOp):
+ return
+ _walk(node.left)
+ _walk(node.right)
+ if node.cardinality == "cross":
+ return
+ if node.on is None:
+ raise ValueError(
+ "Source-aware aggregation requires an explicit equijoin predicate; "
+ "use join_cross() for a Cartesian product."
+ )
+
+ left_tbl = (
+ node.left.to_untagged(parent_requirements=None)
+ if isinstance(node.left, SemanticJoinOp)
+ else _to_untagged(node.left)
+ )
+ right_tbl = (
+ node.right.to_untagged(parent_requirements=None)
+ if isinstance(node.right, SemanticJoinOp)
+ else _to_untagged(node.right)
+ )
+ try:
+ predicate = node.on(_Resolver(left_tbl), _Resolver(right_tbl))
+ _reject_bool_resolution(predicate, node.on)
+ predicate_op = predicate.op()
+ except Exception as exc:
+ raise ValueError(
+ "Could not validate join predicate for source-aware aggregation. "
+ "Use a conjunction of direct field equalities such as "
+ "`lambda left, right: left.id == right.id`."
+ ) from exc
+ if not _valid(predicate_op, _to_op(left_tbl), _to_op(right_tbl)):
+ raise ValueError(
+ "Source-aware aggregation supports only conjunctions of direct "
+ "field equijoins. Inequality, OR, cast, and transformed join "
+ "predicates cannot be preaggregated soundly; aggregate each "
+ "model first or restate the relationship as plain equality keys."
+ )
+
+ _walk(join_op)
+
+
@frozen
class _DeferrableJoin:
"""A join_one that can be deferred until after aggregation."""
@@ -2467,7 +2741,12 @@ class _DeferrableJoin:
deferred_dims: tuple # Prefixed dimension names to add post-agg
-def _table_filter_resolver(raw_tbl, table_op, table_name):
+def _table_filter_resolver(
+ raw_tbl,
+ table_op,
+ table_name,
+ requested_fields: Iterable[str] = (),
+):
"""Build a filter resolver scoped to one source table.
Exposes the table's declared dimensions under both their bare and
@@ -2481,6 +2760,19 @@ def _table_filter_resolver(raw_tbl, table_op, table_name):
if table_name:
for dname, dim in list(dims.items()):
dims[f"{table_name}.{dname}"] = dim
+ raw_columns = frozenset(raw_tbl.columns)
+ for field_name in requested_fields:
+ if "." not in field_name:
+ continue
+ prefix, raw_name = field_name.split(".", 1)
+ if (
+ prefix == table_name
+ and raw_name in raw_columns
+ and field_name not in dims
+ ):
+ dims[field_name] = Dimension(
+ expr=lambda t, _name=raw_name: t[_name]
+ )
return _Resolver(raw_tbl, dims)
@@ -2643,9 +2935,14 @@ def _filter_references_table(table_name, table_op):
if not filters:
return False
raw_tbl = _to_untagged(table_op)
- resolver = _table_filter_resolver(raw_tbl, table_op, table_name)
for pred in filters:
pred_fn = _unwrap(pred)
+ resolver = _table_filter_resolver(
+ raw_tbl,
+ table_op,
+ table_name,
+ _exact_filter_fields(pred_fn),
+ )
try:
# If the predicate resolves against this table's columns or
# dimensions (bare or table-prefixed), it references the
@@ -2856,6 +3153,25 @@ def _is_count_distinct_expr(expr):
)().value_or(False)
+def _is_count_expr(expr):
+ """Check if an expression is COUNT(column) or COUNT(*)."""
+ try:
+ reductions = _reductions_for_expr(expr)
+ return isinstance(expr.op(), (reductions.Count, reductions.CountStar))
+ except Exception:
+ return False
+
+
+def _fill_missing_count_identities(table, measure_names):
+ """Restore COUNT's empty-set identity after a dimension-bridge join."""
+ replacements = {
+ name: table[name].fill_null(0)
+ for name in measure_names
+ if name in table.columns
+ }
+ return table.mutate(**replacements) if replacements else table
+
+
def _reagg_op_for_expr(expr):
"""Return the re-aggregation operation name for an ibis expression.
@@ -2895,16 +3211,117 @@ def _build_reagg(col_ref, op_name):
return getattr(col_ref, op_name)()
-def _exact_grain_preagg(raw_tbl, tbl, group_by_cols, join_keys, exact_measures):
+def _is_direct_physical_field(expr, table, column_name: str) -> bool:
+ """Return whether *expr* is exactly ``table[column_name]``.
+
+ Expression names are not lineage: transformations such as ``upper()``
+ commonly retain their input field's name. Preaggregation may reuse a raw
+ column only for a direct Field op bound to the current raw relation.
+ """
+ try:
+ op = _to_op(expr)
+ return (
+ type(op).__name__ == "Field"
+ and getattr(op, "name", None) == column_name
+ and getattr(op, "rel", None) is _to_op(table)
+ )
+ except Exception:
+ return False
+
+
+def _allocate_local_group_alias(
+ group_key: str,
+ occupied: Iterable[str],
+) -> str:
+ """Allocate a deterministic raw-table alias for a derived group key."""
+ safe_key = re.sub(r"[^0-9A-Za-z_]", "_", group_key).strip("_") or "key"
+ preferred = f"__bsl_gb_{safe_key}"
+ occupied = frozenset(occupied)
+ candidate = preferred
+ suffix = 2
+ while candidate in occupied:
+ candidate = f"{preferred}_{suffix}"
+ suffix += 1
+ return candidate
+
+
+def _compile_evaluated_measure_table(
+ base_tbl,
+ by_cols: Iterable[str],
+ evaluated_measures: Mapping[str, Any],
+):
+ """Aggregate already-evaluated regular and nested measures together.
+
+ ``NestedAccessMarker`` is intentionally not an ibis expression. Most
+ aggregation paths classify it before calling ``Table.aggregate``; the
+ source-aware join pre-aggregation path also needs that dispatch because
+ nested arrays must be unnested on their owning raw relation, never on the
+ flattened (and potentially fanned-out) join.
+ """
+ by_cols = tuple(by_cols)
+ nested_specs = {
+ name: value
+ for name, value in evaluated_measures.items()
+ if isinstance(value, NestedAccessMarker)
+ }
+ regular_exprs = {
+ name: value
+ for name, value in evaluated_measures.items()
+ if not isinstance(value, NestedAccessMarker)
+ }
+
+ if nested_specs:
+ # `_compile_aggregation_with_nested` accepts callables for regular
+ # measures. These expressions are already bound to `base_tbl`, so a
+ # constant-returning wrapper preserves that exact relation binding.
+ regular_specs = {
+ name: (lambda _table, expr=expr: expr)
+ for name, expr in regular_exprs.items()
+ }
+ return _compile_aggregation_with_nested(
+ base_tbl,
+ list(by_cols),
+ regular_specs,
+ nested_specs,
+ )
+
+ if by_cols:
+ return base_tbl.group_by([base_tbl[c] for c in by_cols]).aggregate(
+ **regular_exprs
+ )
+ return base_tbl.aggregate(**regular_exprs)
+
+
+def _compile_exact_measure_table(
+ base_tbl,
+ by_cols: Iterable[str],
+ exact_measures: Mapping[str, Callable],
+):
+ """Evaluate exact-grain measure specs and compile nested markers safely."""
+ evaluated = {name: fn(base_tbl) for name, fn in exact_measures.items()}
+ return _compile_evaluated_measure_table(base_tbl, by_cols, evaluated)
+
+
+def _exact_grain_preagg(
+ raw_tbl,
+ tbl,
+ group_by_cols,
+ join_keys,
+ exact_measures,
+ joined_key_names: Mapping[str, str] | None = None,
+ local_group_keys: Mapping[str, str] | None = None,
+):
"""Aggregate non-decomposable measures at the exact target grain.
Median, stddev, variance and compound expressions (``sum()/count()``)
cannot be re-aggregated from a finer pre-aggregate. Build a
(group keys -> join keys) bridge from the joined table and aggregate
- the raw rows directly per group: each raw row participates once per
- group-key value it maps to, matching the join-participation semantics
- of the decomposable path. Raises instead of degrading — the previous
- behavior summed per-key values silently.
+ the raw rows directly per group. Source-local group keys are also part
+ of the bridge predicate: a join key can participate in multiple local
+ dimension values, and joining on the key alone would leak raw rows from
+ sibling groups into COUNT DISTINCT, median, and other exact reductions.
+ Raises instead of degrading — the previous behavior summed per-key
+ values silently.
"""
names = ", ".join(sorted(exact_measures))
if tbl is None:
@@ -2918,24 +3335,151 @@ def _exact_grain_preagg(raw_tbl, tbl, group_by_cols, join_keys, exact_measures):
f"Cannot compute non-decomposable measure(s) {names}: group "
f"key(s) {missing} are not materialized on the joined table."
)
- shared_jk = [k for k in join_keys if k in tbl.columns]
+ joined_key_names = dict(joined_key_names or {})
+ local_group_keys = {
+ joined_name: raw_name
+ for joined_name, raw_name in dict(local_group_keys or {}).items()
+ if joined_name in group_by_cols and raw_name in raw_tbl.columns
+ }
+ shared_jk = [
+ (raw_name, joined_key_names.get(raw_name, raw_name))
+ for raw_name in join_keys
+ if raw_name in raw_tbl.columns
+ and joined_key_names.get(raw_name, raw_name) in tbl.columns
+ ]
if not shared_jk:
raise ValueError(
f"Cannot compute non-decomposable measure(s) {names}: no join "
"keys shared with the joined table to bridge the target grain."
)
- # Temp names so bridge group columns can never shadow raw columns the
- # measure expressions reference
- tmp = {c: f"__exact_gb_{i}" for i, c in enumerate(group_by_cols)}
+ # Allocate bridge-only names outside every user/executable namespace so
+ # group columns cannot shadow raw columns referenced by measure
+ # expressions (users may legitimately own ``__exact_gb_0`` already).
+ occupied = set(raw_tbl.columns) | set(tbl.columns) | set(exact_measures)
+ tmp: dict[str, str] = {}
+ for i, column in enumerate(group_by_cols):
+ preferred = f"__exact_gb_{i}"
+ candidate = preferred
+ suffix = 2
+ while candidate in occupied:
+ candidate = f"{preferred}_{suffix}"
+ suffix += 1
+ tmp[column] = candidate
+ occupied.add(candidate)
+
+ def allocate_exact_name(preferred: str) -> str:
+ candidate = preferred
+ suffix = 2
+ while candidate in occupied:
+ candidate = f"{preferred}_{suffix}"
+ suffix += 1
+ occupied.add(candidate)
+ return candidate
+
+ presence_name = allocate_exact_name("__exact_present")
+ empty_names = {
+ measure: allocate_exact_name(f"__exact_empty_{index}")
+ for index, measure in enumerate(exact_measures)
+ }
bridge = tbl.select(
- [tbl[c].name(tmp[c]) for c in group_by_cols] + [tbl[k] for k in shared_jk]
+ [tbl[c].name(tmp[c]) for c in group_by_cols]
+ + [tbl[joined].name(raw) for raw, joined in shared_jk]
).distinct()
- preds = [null_safe_equal(bridge[k], raw_tbl[k]) for k in shared_jk]
+ preds = [null_safe_equal(bridge[raw], raw_tbl[raw]) for raw, _ in shared_jk]
+ preds.extend(
+ null_safe_equal(bridge[tmp[joined_name]], raw_tbl[raw_name])
+ for joined_name, raw_name in local_group_keys.items()
+ )
joined = bridge.inner_join(raw_tbl, preds)
- aggs = {m: fn(joined) for m, fn in exact_measures.items()}
- pt = joined.group_by([joined[t] for t in tmp.values()]).aggregate(**aggs)
+ pt = _compile_exact_measure_table(joined, tmp.values(), exact_measures)
+ from .nested_compile import get_ibis_module
+
+ pt = pt.mutate(**{presence_name: get_ibis_module(pt).literal(True)})
# ibis rename convention: {new_name: old_name}
- return pt.rename({orig: tmp_name for orig, tmp_name in tmp.items()})
+ pt = pt.rename({orig: tmp_name for orig, tmp_name in tmp.items()})
+
+ # Exact source aggregation has no row for an unmatched LEFT JOIN group.
+ # Preserve the joined query's full group domain and evaluate every measure
+ # once on an actual empty source. Compound reductions can have non-NULL
+ # empty-set results (count()+1 -> 1, sum().fill_null(0) -> 0), so blanket
+ # NULL filling is unsound. A separate presence marker distinguishes an
+ # absent aggregate row from a matched row whose measure itself is NULL.
+ empty_tbl = raw_tbl.limit(0)
+ empty_values = _compile_exact_measure_table(
+ empty_tbl, (), exact_measures
+ ).rename(
+ {
+ empty_names[measure]: measure
+ for measure in exact_measures
+ }
+ )
+ group_spine = tbl.select([tbl[c] for c in group_by_cols]).distinct()
+ spine_preds = [
+ null_safe_equal(group_spine[c], pt[c]) for c in group_by_cols
+ ]
+ attached = group_spine.left_join(pt, spine_preds).cross_join(empty_values)
+ missing_aggregate = pt[presence_name].isnull()
+ return attached.select(
+ [group_spine]
+ + [
+ missing_aggregate.ifelse(
+ empty_values[empty_names[measure]], pt[measure]
+ ).name(measure)
+ for measure in exact_measures
+ ]
+ )
+
+
+def _source_join_key_pairs(
+ table_name: str,
+ join_keys: Iterable[str],
+ raw_columns: Iterable[str],
+ joined_columns: Iterable[str],
+ join_column_lineage: Mapping[str, Mapping[str, str]],
+) -> tuple[tuple[str, str], ...]:
+ """Return ``(raw key, executable joined alias)`` pairs for one source.
+
+ Join-key metadata is intentionally expressed in each source table's raw
+ namespace. Once joins are flattened, a key such as a right-side ``id``
+ may be materialized as ``id_right`` (or a collision-safe later suffix).
+ Keeping the two names explicit prevents bridges from accidentally binding
+ the raw right key to an unrelated same-named column from the left table.
+ """
+ raw_columns = frozenset(raw_columns)
+ joined_columns = frozenset(joined_columns)
+ source_names = join_column_lineage.get(table_name, {})
+ return tuple(
+ (raw_name, source_names.get(raw_name, raw_name))
+ for raw_name in sorted(join_keys)
+ if raw_name in raw_columns
+ and source_names.get(raw_name, raw_name) in joined_columns
+ )
+
+
+def _rename_preagg_grain_to_joined_aliases(
+ table,
+ grain: Iterable[str],
+ source_names: Mapping[str, str],
+):
+ """Put raw source-grain columns in the joined table's namespace."""
+ grain = tuple(grain)
+ renames = {
+ source_names[name]: name
+ for name in grain
+ if name in table.columns and source_names.get(name, name) != name
+ }
+ if not renames:
+ return table
+
+ untouched = set(table.columns) - set(renames.values())
+ collisions = sorted(set(renames) & untouched)
+ if collisions:
+ raise ValueError(
+ "Cannot attach a source pre-aggregate to the joined table because "
+ "its executable join-key alias collides with an aggregate column: "
+ f"{collisions}. Rename the aggregate field."
+ )
+ return table.rename(renames)
def _partition_agg_specs_by_source(
@@ -2963,6 +3507,201 @@ def _partition_agg_specs_by_source(
return partitioned
+def _infer_join_wrapper_measure_owner(
+ measure: Measure,
+ join_tree_info: _JoinTreeInfo,
+) -> str | None:
+ """Infer one owning leaf for a base reduction declared after a join.
+
+ Field-bearing reductions such as ``lambda t: t.amount.sum()`` should keep
+ the grain of the one source that actually owns ``amount``. Relation-only
+ reductions such as ``t.count()`` intentionally describe joined-row grain
+ and return ``None``. If the same field expression resolves on multiple
+ roots, choosing one would be a silent namespace guess, so reject it.
+ """
+ owners: list[str] = []
+ original = measure.original_expr
+ if _is_deferred(original):
+ def probe(t, expr=original):
+ return expr.resolve(t)
+ elif callable(original):
+ probe = original
+ else:
+ def probe(_t, value=original):
+ return value
+ for table_name, table_op in join_tree_info.table_ops.items():
+ try:
+ raw_tbl = _to_untagged(table_op)
+ extraction = _extract_columns_from_callable(probe, raw_tbl)
+ except Exception:
+ continue
+ if not extraction.is_success():
+ continue
+ if not extraction.columns:
+ continue
+ owners.append(table_name)
+
+ if len(owners) == 1:
+ return owners[0]
+ if len(owners) > 1:
+ raise ValueError(
+ "A base measure declared after a join references columns that "
+ f"resolve on multiple semantic models ({', '.join(sorted(owners))}). "
+ "Define the measure on its owning model before joining, or use "
+ "qualified calculated-measure references."
+ )
+ return None
+
+
+def _join_wrapper_local_dimensions(
+ roots: Iterable[SemanticTableOp],
+) -> dict[str, Dimension]:
+ """Return unprefixed dimensions declared on materialized join wrappers.
+
+ A ``SemanticJoin.with_dimensions(...)`` result is represented by a
+ ``SemanticTableOp`` whose physical table is the flattened join and whose
+ ``_source_join`` retains semantic provenance. Its inherited leaf
+ dimensions are prefixed; unprefixed entries therefore belong to the
+ wrapper namespace and must be interpreted against that flattened table,
+ not independently against every leaf that happens to share a column name.
+ """
+ result: dict[str, Dimension] = {}
+ for root in roots:
+ if not isinstance(root, SemanticTableOp) or root._source_join is None:
+ continue
+ result.update(
+ {
+ name: dimension
+ for name, dimension in root.get_dimensions().items()
+ if "." not in name and isinstance(dimension, Dimension)
+ }
+ )
+ return result
+
+
+class _JoinWrapperDimensionPrefix:
+ """Resolve one qualified prefix through a wrapper dimension scope."""
+
+ __slots__ = ("_resolver", "_prefix")
+
+ def __init__(self, resolver, prefix: str):
+ object.__setattr__(self, "_resolver", resolver)
+ object.__setattr__(self, "_prefix", prefix)
+
+ def __getattr__(self, name: str):
+ return self._resolver._resolve_dimension(f"{self._prefix}.{name}")
+
+
+class _JoinWrapperDimensionResolver:
+ """Resolve wrapper expressions with physical-self and semantic siblings.
+
+ A wrapper definition such as ``status=lambda t: t.status.upper()`` reads
+ the physical ``status`` column, while a later definition such as
+ ``label=lambda t: t.status + '!'`` reads the semantic sibling. Tracking
+ the currently resolving names distinguishes those two cases and keeps the
+ same dependency behavior as dimension materialization.
+ """
+
+ __slots__ = ("_dims", "_resolving", "_table")
+
+ def __init__(self, table, dimensions: Mapping[str, Any], resolving=()):
+ object.__setattr__(self, "_table", table)
+ object.__setattr__(self, "_dims", dimensions)
+ object.__setattr__(self, "_resolving", frozenset(resolving))
+
+ def _resolve_dimension(self, name: str):
+ if name not in self._dims:
+ raise AttributeError(f"No dimension {name!r} exists in wrapper scope")
+ if name in self._resolving:
+ return self._table[name]
+ dimension = self._dims[name]
+ nested = _JoinWrapperDimensionResolver(
+ self._table,
+ self._dims,
+ (*self._resolving, name),
+ )
+ if isinstance(dimension, Dimension):
+ return dimension(nested, _dims=dict(self._dims))
+ return _resolve_expr(dimension, nested)
+
+ def __getattr__(self, name: str):
+ if name in self._dims and name not in self._resolving:
+ return self._resolve_dimension(name)
+ prefix = f"{name}."
+ if any(key.startswith(prefix) for key in self._dims):
+ return _JoinWrapperDimensionPrefix(self, name)
+ return getattr(self._table, name)
+
+ def __getitem__(self, name: str):
+ if name in self._dims and name not in self._resolving:
+ return self._resolve_dimension(name)
+ return self._table[name]
+
+ @property
+ def columns(self):
+ return self._table.columns
+
+
+def _infer_join_wrapper_dimension_owners(
+ dimensions: Mapping[str, Dimension],
+ joined_table,
+ merged_dimensions: Mapping[str, Any],
+ join_column_lineage: Mapping[str, Mapping[str, str]],
+) -> tuple[dict[str, str | None], dict[str, str]]:
+ """Bind wrapper dimensions to their unique source in joined namespace.
+
+ The returned owner is a leaf model name, or ``None`` for a constant
+ dimension. Expressions that fail resolution, access a non-lineage
+ column, or combine multiple leaves are returned in the error mapping so
+ source preaggregation can fail closed rather than guess a namespace.
+ """
+ executable_owners: dict[str, set[str]] = {}
+ for table_name, columns in join_column_lineage.items():
+ for executable_name in columns.values():
+ executable_owners.setdefault(executable_name, set()).add(table_name)
+
+ owners: dict[str, str | None] = {}
+ errors: dict[str, str] = {}
+ dimensions_for_scope = dict(merged_dimensions)
+ for name, dimension in dimensions.items():
+
+ def resolve_on_join(t, dim=dimension):
+ scope = _JoinWrapperDimensionResolver(
+ t, dimensions_for_scope, resolving=(name,)
+ )
+ return dim(scope, _dims=dimensions_for_scope)
+
+ extraction = _extract_columns_from_callable(resolve_on_join, joined_table)
+ if not extraction.is_success():
+ errors[name] = "its expression does not resolve on the flattened join"
+ continue
+ if not extraction.columns:
+ owners[name] = None
+ continue
+
+ expression_owners: set[str] = set()
+ unknown_columns: list[str] = []
+ for column in extraction.columns:
+ candidates = executable_owners.get(column)
+ if not candidates:
+ unknown_columns.append(column)
+ else:
+ expression_owners.update(candidates)
+ if unknown_columns:
+ errors[name] = (
+ "its expression accesses joined column(s) without leaf lineage: "
+ f"{sorted(unknown_columns)}"
+ )
+ elif len(expression_owners) != 1:
+ errors[name] = (
+ "its expression spans multiple semantic models: "
+ f"{sorted(expression_owners)}"
+ )
+ else:
+ owners[name] = next(iter(expression_owners))
+ return owners, errors
+
+
class SemanticAggregateOp(Relation):
source: Relation
keys: tuple[str, ...]
@@ -3029,7 +3768,9 @@ def required_columns(self) -> dict[str, set[str]]:
Dict mapping table names to sets of required column names.
"""
all_roots = _find_all_root_models(self.source)
- merged_dimensions = _get_merged_fields(all_roots, "dimensions")
+ merged_dimensions = _get_merged_fields(
+ all_roots, "dimensions", source=self.source
+ )
base_tbl = (
self.source.to_expr() if hasattr(self.source, "to_expr") else _to_untagged(self.source)
@@ -3169,11 +3910,34 @@ def collect_filters_to_join(node):
exc_info=True,
)
- # Pre-aggregation path: when join_many is present, aggregate each
- # source table's measures at its own grain before joining.
- if join_op is not None and not is_post_agg:
+ # Source-aware aggregation path: aggregate every joined model's base
+ # measures against their owning leaf relation, then use the joined
+ # table only as a dimension/participation bridge. This is required
+ # for ``join_one`` too: evaluating a right-side measure on the
+ # flattened LEFT JOIN can bind colliding fields to the left relation,
+ # and relation reductions such as ``t.count()`` count unmatched left
+ # rows as if a right row existed.
+ if join_op is not None and not is_post_agg and self.aggs:
join_tree_info = _collect_join_tree_info(join_op)
- if join_tree_info.has_join_many:
+ root_names = {
+ name
+ for name, cardinality in join_tree_info.table_cardinalities.items()
+ if cardinality == "root"
+ }
+ requested_from_non_root = any(
+ "." in name and name.split(".", 1)[0] not in root_names
+ for name in self.aggs
+ )
+ merged_base_for_routing = _get_merged_fields(all_roots, "measures")
+ requested_wrapper_base = any(
+ "." not in name and name in merged_base_for_routing
+ for name in self.aggs
+ )
+ if (
+ join_tree_info.has_join_many
+ or requested_from_non_root
+ or requested_wrapper_base
+ ):
return self._to_untagged_with_preagg(
all_roots,
join_op,
@@ -3213,7 +3977,9 @@ def collect_filters_to_join(node):
else:
tbl = _to_untagged(self.source)
- merged_dimensions = _get_merged_fields(all_roots, "dimensions")
+ merged_dimensions = _get_merged_fields(
+ all_roots, "dimensions", source=join_op
+ )
merged_base_measures = _get_merged_fields(all_roots, "measures")
merged_calc_measures = _get_merged_fields(all_roots, "calc_measures")
@@ -3374,29 +4140,97 @@ def _to_untagged_with_preagg(
This prevents fan-out inflation when ``join_many`` is used.
"""
- merged_dimensions = _get_merged_fields(all_roots, "dimensions")
+ root_names = {
+ name
+ for name, cardinality in join_tree_info.table_cardinalities.items()
+ if cardinality == "root"
+ }
+ predicate_sensitive = bool(filters) or any(
+ "." in name and name.split(".", 1)[0] not in root_names
+ for name in (*self.keys, *self.aggs.keys())
+ ) or any("." not in name for name in self.aggs)
+ if predicate_sensitive:
+ _validate_preaggregation_join_predicates(join_op)
+ filters = filters or []
+ filter_fns = [_unwrap(pred) for pred in filters]
+ exact_filter_fields = frozenset().union(
+ *(_exact_filter_fields(fn) for fn in filter_fns)
+ )
+ merged_dimensions = _get_merged_fields(
+ all_roots, "dimensions", source=join_op
+ )
merged_dimensions = _augment_dimensions_with_raw_columns(
- merged_dimensions, self.keys, all_roots, join_op
+ merged_dimensions,
+ (*self.keys, *exact_filter_fields),
+ all_roots,
+ join_op,
)
+ if exact_filter_fields:
+ _validate_qualified_filter_fields(
+ exact_filter_fields, merged_dimensions, all_roots
+ )
merged_base_measures = _get_merged_fields(all_roots, "measures")
merged_calc_measures = _get_merged_fields(all_roots, "calc_measures")
group_by_cols = list(self.keys)
-
- filters = filters or []
- filter_fns = [_unwrap(pred) for pred in filters]
+ join_column_lineage, _joined_column_names = _build_join_column_lineage(
+ join_op
+ )
+ wrapper_local_dimensions = {
+ name: dimension
+ for name, dimension in _join_wrapper_local_dimensions(all_roots).items()
+ if name in group_by_cols
+ }
+ wrapper_dimension_owners: dict[str, str | None] = {}
# --- 1. Try to build the full joined table (for scope / dim bridge) ---
# Pre-agg needs all tables for dimension bridges — no pruning here.
try:
- tbl = join_op.to_untagged(parent_requirements=None)
- tbl = _mutate_dimensions_with_dependencies(
- tbl,
- [k for k in self.keys if k in merged_dimensions],
- merged_dimensions,
- )
+ joined_base_tbl = join_op.to_untagged(parent_requirements=None)
except Exception:
+ joined_base_tbl = None
tbl = None # chasm / column collision – work without full join
+ if joined_base_tbl is not None:
+ if wrapper_local_dimensions:
+ wrapper_dimension_owners, wrapper_dimension_errors = (
+ _infer_join_wrapper_dimension_owners(
+ wrapper_local_dimensions,
+ joined_base_tbl,
+ merged_dimensions,
+ join_column_lineage,
+ )
+ )
+ if wrapper_dimension_errors:
+ details = "; ".join(
+ f"{name!r}: {reason}"
+ for name, reason in sorted(wrapper_dimension_errors.items())
+ )
+ raise ValueError(
+ "Cannot source-preaggregate by join-wrapper dimension(s) "
+ f"because their row-level lineage is not reproducible: {details}. "
+ "Define each dimension on one owning semantic model before "
+ "joining, or aggregate the joined rows explicitly."
+ )
+ try:
+ tbl = _mutate_dimensions_with_dependencies(
+ joined_base_tbl,
+ [k for k in self.keys if k in merged_dimensions],
+ merged_dimensions,
+ )
+ except Exception:
+ tbl = None # dimension materialization fallback
+
+ if joined_base_tbl is None and wrapper_local_dimensions:
+ raise ValueError(
+ "Cannot source-preaggregate by a join-wrapper dimension because "
+ "the flattened join is unavailable for source-lineage analysis."
+ )
+
+ if tbl is not None:
+ _reject_unresolvable_group_keys(
+ self.keys, merged_dimensions, tbl, all_roots
+ )
+
# Apply collected filters to the full joined table so that
# dimension bridges only include rows surviving the filter. A
# filter that fails to resolve here may still be pushed to its
@@ -3457,7 +4291,15 @@ def _to_untagged_with_preagg(
if raw is None:
continue
try:
- _resolve_expr(pred_fn, _table_filter_resolver(raw, top, tname))
+ _resolve_expr(
+ pred_fn,
+ _table_filter_resolver(
+ raw,
+ top,
+ tname,
+ _exact_filter_fields(pred_fn),
+ ),
+ )
owners.add(tname)
except Exception:
pass
@@ -3507,23 +4349,6 @@ def _to_untagged_with_preagg(
for leg in _flatten_and_legs(expr)
]
- # Tables reached through a ``join_many`` edge. Their raw rows only
- # count when they participate in the join, so this set drives both
- # the residual-filter-leg check and the unconditional measure-leg
- # participation restriction below.
- many_side_tables: set[str] = set()
-
- def _collect_many_sides(node):
- if isinstance(node, SemanticJoinOp):
- if node.cardinality == "many":
- for r in _find_all_root_models(node.right):
- if getattr(r, "name", None):
- many_side_tables.add(r.name)
- _collect_many_sides(node.left)
- _collect_many_sides(node.right)
-
- _collect_many_sides(join_op)
-
# --- 2. Build aggregation plan ---
if tbl is not None:
scope = MeasureScope(
@@ -3561,14 +4386,36 @@ def _collect_many_sides(node):
partition_roots = _find_all_root_models(join_op)
partitioned = _partition_agg_specs_by_source(dict(plan.agg_specs), partition_roots)
+ # Measures declared on a joined wrapper have unprefixed result names.
+ # Route field-bearing reductions back to their unique owning leaf so
+ # they do not aggregate the fanned-out join. Relation-only reductions
+ # (notably t.count()) remain explicit joined-row-grain measures.
+ wrapper_specs = partitioned.get(None, {})
+ for measure_name, measure_fn in tuple(wrapper_specs.items()):
+ measure_obj = merged_base_measures.get(measure_name)
+ if not isinstance(measure_obj, Measure):
+ continue
+ owner = _infer_join_wrapper_measure_owner(measure_obj, join_tree_info)
+ if owner is None:
+ continue
+ partitioned.setdefault(owner, {})[measure_name] = measure_fn
+ del wrapper_specs[measure_name]
+ if not wrapper_specs:
+ partitioned.pop(None, None)
+
# --- 4. Pre-aggregate each source table on its raw table ---
_preagg_results: list = []
# Track MEAN measures decomposed into SUM + COUNT for correct re-agg
_decomposed_means: dict[str, tuple[str, str]] = {}
# Track correct re-aggregation op per measure (default "sum")
_reagg_ops: dict[str, str] = {}
+ # COUNT reductions produce zero for an empty group. A source-level
+ # pre-aggregate has no row for an unmatched outer-join group, so its
+ # later dimension-bridge join must restore that identity explicitly.
+ _empty_count_measures: set[str] = set()
# Track COUNT DISTINCT measures deferred past pre-aggregation.
- # Value: (table_name, short_name, raw_tbl, measure_fn)
+ # Value: (table_name, short_name, raw_tbl, measure_fn,
+ # {joined_group_name: materialized_raw_column})
_deferred_count_distincts: dict[str, tuple] = {}
# Fan-out-safe totals sources for t.all(...): per table, the
# filtered raw table plus the original (undecomposed) measure
@@ -3593,6 +4440,7 @@ def _collect_many_sides(node):
continue
raw_tbl = _to_untagged(table_op)
+ source_key_names = join_column_lineage.get(table_name, {})
# Push filters owned by this table onto its raw table. Filters
# handled elsewhere (applied to the full joined table, or owned
@@ -3604,7 +4452,12 @@ def _collect_many_sides(node):
if filter_owners[i] == frozenset({table_name}):
pred_expr = _resolve_expr(
pred_fn,
- _table_filter_resolver(raw_tbl, table_op, table_name),
+ _table_filter_resolver(
+ raw_tbl,
+ table_op,
+ table_name,
+ _exact_filter_fields(pred_fn),
+ ),
)
raw_tbl = raw_tbl.filter(pred_expr)
continue
@@ -3624,7 +4477,7 @@ def _collect_many_sides(node):
elif table_name in leg_srcs and len(leg_srcs) > 1:
residual_cross_legs = True
- if residual_cross_legs and table_name in many_side_tables and measures:
+ if residual_cross_legs and measures:
raise ValueError(
f"A filter mixes columns of {table_name!r} with other "
"tables in a way that cannot be applied row-precisely "
@@ -3634,14 +4487,16 @@ def _collect_many_sides(node):
"calls, or restate it against a single table."
)
- # Rows of a join_many table whose join keys are NULL or match no
- # left-side row never appear in the LEFT JOIN output, so measure
- # legs must ALWAYS be restricted to join participants — not only
- # when cross-table filter routing forces a bridge. Otherwise
- # grand totals and many-side-only group-bys silently count
- # orphan rows the joined table can never produce, and the sum
- # over groups stops matching the ungrouped total.
- needs_participation = table_name in many_side_tables and bool(measures)
+ # Rows of every non-root/right table whose join keys are NULL or
+ # match no left-side row never appear in a LEFT JOIN. Restrict
+ # both join_many and join_one measure legs to actual participants;
+ # otherwise scalar right measures include orphan source rows and
+ # flattened CountStar reductions count unmatched left rows.
+ needs_participation = (
+ join_tree_info.table_cardinalities.get(table_name)
+ not in ("root", "cross")
+ and bool(measures)
+ )
# Filters not pushed here (cross-table, ambiguous, or owned
# by another table) restrict via join keys from the filtered
@@ -3649,15 +4504,26 @@ def _collect_many_sides(node):
if needs_bridge or needs_participation:
jk = join_tree_info.table_join_keys.get(table_name, set())
if tbl is not None:
- shared = sorted(jk & set(raw_tbl.columns) & set(tbl.columns))
+ shared = _source_join_key_pairs(
+ table_name,
+ jk,
+ raw_tbl.columns,
+ tbl.columns,
+ join_column_lineage,
+ )
if shared:
- key_bridge = tbl.select([tbl[c] for c in shared]).distinct()
- preds = [raw_tbl[c] == key_bridge[c] for c in shared]
+ key_bridge = tbl.select(
+ [tbl[joined].name(raw) for raw, joined in shared]
+ ).distinct()
+ preds = [
+ raw_tbl[raw] == key_bridge[raw]
+ for raw, _joined in shared
+ ]
raw_tbl = raw_tbl.inner_join(key_bridge, preds).select(raw_tbl)
elif needs_participation:
raise ValueError(
f"Measures on {table_name!r} cannot be restricted "
- "to rows that participate in its join_many: no "
+ "to rows that participate in its join: no "
"join-key column is available on both the raw "
"table and the joined table. Computing them on "
"the raw table would silently count rows the "
@@ -3694,7 +4560,7 @@ def _collect_many_sides(node):
if not participation_bridged:
raise ValueError(
f"Measures on {table_name!r} cannot be restricted "
- "to rows that participate in its join_many: the "
+ "to rows that participate in its join: the "
"full joined table is unavailable (chasm fallback) "
"and no join-key column is shared with the root "
"table. Computing them on the raw table would "
@@ -3714,7 +4580,10 @@ def _collect_many_sides(node):
_resolve_expr(
pred_fn,
_table_filter_resolver(
- owner_raw, owner_op, owner_name
+ owner_raw,
+ owner_op,
+ owner_name,
+ _exact_filter_fields(pred_fn),
),
)
)
@@ -3738,10 +4607,28 @@ def _collect_many_sides(node):
agg_exprs: dict = {}
_tot_exprs: dict = {}
_exact_measures_t: dict = {}
+ _nested_measures_t: dict = {}
for mname, _mfn in measures.items():
short = mname.split(".", 1)[1] if "." in mname else mname
- if short in table_measures:
- expr = table_measures[short](raw_tbl)
+ source_measure = (
+ table_measures.get(short)
+ if "." in mname
+ else merged_base_measures.get(mname)
+ )
+ if isinstance(source_measure, Measure):
+ expr = source_measure(raw_tbl)
+ if isinstance(expr, NestedAccessMarker):
+ # Nested arrays are a separate aggregation grain, not
+ # ibis reductions that can be inspected via `.op()`.
+ # Keep the original source-bound measure for exact
+ # target-grain compilation after group-key routing.
+ _nested_measures_t[mname] = source_measure
+ _tot_exprs[mname] = expr
+ if expr.operation in {"count", "nunique"}:
+ _empty_count_measures.add(mname)
+ continue
+ if _is_count_expr(expr):
+ _empty_count_measures.add(mname)
# Original expression on the filtered raw table: at zero
# grain this is fan-out-safe, so it powers t.all(...) totals
_tot_exprs[mname] = expr
@@ -3762,7 +4649,7 @@ def _collect_many_sides(node):
elif _is_count_distinct_expr(expr):
# COUNT DISTINCT is immune to fan-out — defer past pre-agg
_deferred_count_distincts[mname] = (
- table_name, short, raw_tbl, table_measures[short],
+ table_name, short, raw_tbl, source_measure, {},
)
else:
reagg = _reagg_op_for_expr(expr)
@@ -3770,7 +4657,7 @@ def _collect_many_sides(node):
# Non-decomposable (median, stddev, compound
# ratio): computed at the exact target grain
# after the grain decision below
- _exact_measures_t[mname] = table_measures[short]
+ _exact_measures_t[mname] = source_measure
else:
if reagg is not None:
_reagg_ops[mname] = reagg
@@ -3779,23 +4666,35 @@ def _collect_many_sides(node):
if _tot_exprs:
_totals_sources[table_name] = (raw_tbl, _tot_exprs)
- if not agg_exprs and not _exact_measures_t:
- continue
-
# --- Compute grain ---
if not group_by_cols:
+ if not agg_exprs and not _exact_measures_t and not _nested_measures_t:
+ continue
# No group-by → scalar aggregate
- pt = raw_tbl.aggregate(**agg_exprs)
- # Recompute MEAN from SUM/COUNT for scalar results
- for mname, (sc, cc) in _decomposed_means.items():
- if sc in pt.columns and cc in pt.columns:
- pt = pt.mutate(**{mname: pt[sc] / pt[cc]})
- pt = pt.drop(sc, cc)
- _preagg_results.append(pt)
+ if agg_exprs:
+ pt = raw_tbl.aggregate(**agg_exprs)
+ # Recompute MEAN from SUM/COUNT for scalar results
+ for mname, (sc, cc) in _decomposed_means.items():
+ if sc in pt.columns and cc in pt.columns:
+ pt = pt.mutate(**{mname: pt[sc] / pt[cc]})
+ pt = pt.drop(sc, cc)
+ _preagg_results.append(pt)
+ if _nested_measures_t:
+ _preagg_results.append(
+ _compile_exact_measure_table(
+ raw_tbl, (), _nested_measures_t
+ )
+ )
continue
# a) group-by dims that live on this table
_local_dims = []
+ _local_group_keys: dict[str, str] = {}
+ # Derived dimensions are materialized under private, collision-safe
+ # raw aliases. Generic preaggregates rename those aliases back to
+ # the public semantic group keys before joining the dimension
+ # bridge; exact reductions use `_local_group_keys` directly.
+ _local_group_outputs: dict[str, str] = {}
has_cross_table_gb = False
for gb_key in group_by_cols:
if "." in gb_key:
@@ -3814,28 +4713,40 @@ def _collect_many_sides(node):
raw_columns = set(raw_tbl.columns)
dim_expr = raw_tbl[short]
resolved_via_deps = True
- col_name = dim_expr.get_name()
- if (
- not resolved_via_deps
- and col_name == short
- and col_name in raw_columns
+ if not resolved_via_deps and _is_direct_physical_field(
+ dim_expr, raw_tbl, short
):
# Simple column reference — use directly
- if col_name not in _local_dims:
- _local_dims.append(col_name)
- elif (
- col_name in raw_columns
- or short not in raw_columns
- or resolved_via_deps
- ):
- # Derived dimension — materialize under the
- # requested (prefixed) name so the pre-agg
- # grain matches the group-by keys and the
- # key column survives into the result
- raw_tbl = raw_tbl.mutate(**{gb_key: dim_expr})
+ if short not in _local_dims:
+ _local_dims.append(short)
+ _local_group_keys[gb_key] = short
+ else:
+ # `get_name()` is deliberately insufficient
+ # here: `t.kind.upper()` can still be named
+ # "kind". Materialize every non-Field expression
+ # under an allocator-owned alias so neither the
+ # source schema nor user internal-prefix columns
+ # can be overwritten.
+ raw_group_name = _allocate_local_group_alias(
+ gb_key, raw_columns
+ )
+ raw_tbl = raw_tbl.mutate(
+ **{raw_group_name: dim_expr}
+ )
raw_columns = set(raw_tbl.columns)
- if gb_key not in _local_dims:
- _local_dims.append(gb_key)
+ if raw_group_name not in _local_dims:
+ _local_dims.append(raw_group_name)
+ _local_group_keys[gb_key] = raw_group_name
+ _local_group_outputs[raw_group_name] = gb_key
+ elif prefix == table_name and short in raw_columns:
+ # A qualified raw column is a valid source-local
+ # dimension even when it was not declared explicitly.
+ # Keep it at its owning table's grain; falling through
+ # to join-key grain would repeat a coarser aggregate
+ # once per dimension value during bridge re-joining.
+ if short not in _local_dims:
+ _local_dims.append(short)
+ _local_group_keys[gb_key] = short
elif prefix != table_name:
has_cross_table_gb = True
elif gb_key in merged_dimensions:
@@ -3846,9 +4757,70 @@ def _collect_many_sides(node):
# per-table grain matches the requested grouping;
# tables that lack the source columns fall back to
# join keys + the dimension bridge.
- if gb_key in raw_columns:
+ if gb_key in wrapper_local_dimensions:
+ wrapper_owner = wrapper_dimension_owners.get(gb_key)
+ if wrapper_owner != table_name:
+ # Constants and dimensions owned by another leaf
+ # are attached through the full joined dimension
+ # bridge. Never re-evaluate the wrapper definition
+ # against a same-named raw column on this source.
+ has_cross_table_gb = True
+ continue
+
+ dim_fn = wrapper_local_dimensions[gb_key]
+ joined_to_raw = {
+ joined_name: raw_name
+ for raw_name, joined_name in join_column_lineage.get(
+ table_name, {}
+ ).items()
+ }
+ owner_scope = _RenamedResolver(raw_tbl, joined_to_raw)
+ dimension_scope = _JoinWrapperDimensionResolver(
+ owner_scope,
+ dict(merged_dimensions),
+ resolving=(gb_key,),
+ )
+ try:
+ dim_expr = dim_fn(
+ dimension_scope,
+ _dims=dict(merged_dimensions),
+ )
+ except Exception as exc:
+ raise ValueError(
+ f"Join-wrapper dimension {gb_key!r} was bound to "
+ f"semantic model {table_name!r} but could not be "
+ "evaluated in that model's raw namespace."
+ ) from exc
+
+ direct_raw_name = next(
+ (
+ column
+ for column in raw_columns
+ if _is_direct_physical_field(
+ dim_expr, raw_tbl, column
+ )
+ ),
+ None,
+ )
+ if direct_raw_name is not None:
+ if direct_raw_name not in _local_dims:
+ _local_dims.append(direct_raw_name)
+ _local_group_keys[gb_key] = direct_raw_name
+ else:
+ raw_group_name = _allocate_local_group_alias(
+ gb_key, raw_columns
+ )
+ raw_tbl = raw_tbl.mutate(
+ **{raw_group_name: dim_expr}
+ )
+ raw_columns = set(raw_tbl.columns)
+ _local_dims.append(raw_group_name)
+ _local_group_keys[gb_key] = raw_group_name
+ _local_group_outputs[raw_group_name] = gb_key
+ elif gb_key in raw_columns:
if gb_key not in _local_dims:
_local_dims.append(gb_key)
+ _local_group_keys[gb_key] = gb_key
else:
dim_fn = merged_dimensions[gb_key]
try:
@@ -3860,9 +4832,29 @@ def _collect_many_sides(node):
raw_columns = set(raw_tbl.columns)
if gb_key not in _local_dims:
_local_dims.append(gb_key)
+ _local_group_keys[gb_key] = gb_key
except Exception:
has_cross_table_gb = True
+ # COUNT DISTINCT is evaluated after all source pre-aggregates are
+ # combined. Carry the final (possibly dimension-materialized) raw
+ # relation and its local group mapping into that exact-grain path.
+ for mname in measures:
+ deferred = _deferred_count_distincts.get(mname)
+ if deferred is None:
+ continue
+ src_name, short, _old_raw, source_measure, _old_local = deferred
+ _deferred_count_distincts[mname] = (
+ src_name,
+ short,
+ raw_tbl,
+ source_measure,
+ dict(_local_group_keys),
+ )
+
+ if not agg_exprs and not _exact_measures_t and not _nested_measures_t:
+ continue
+
join_keys = join_tree_info.table_join_keys.get(table_name, set())
available_jk = tuple(jk for jk in sorted(join_keys) if jk in raw_columns)
@@ -3897,22 +4889,114 @@ def _collect_many_sides(node):
}
if _exact_measures_t:
- if not has_cross_table_gb:
+ exact_needs_source_spine = (
+ join_tree_info.table_cardinalities.get(table_name)
+ not in ("root", "cross")
+ )
+ if not has_cross_table_gb and not exact_needs_source_spine:
# Local grain IS the target grain — aggregate the
- # original expressions directly, no re-agg happens
+ # original root/cross expressions directly, no re-agg
+ # happens. Non-root sources still use the exact bridge so
+ # unmatched LEFT JOIN groups receive the expression's
+ # actual empty-set value rather than blanket NULL.
for m, fn in _exact_measures_t.items():
agg_exprs[m] = fn(raw_tbl)
else:
_preagg_results.append(
_exact_grain_preagg(
- raw_tbl, tbl, group_by_cols, available_jk, _exact_measures_t
+ raw_tbl,
+ tbl,
+ group_by_cols,
+ available_jk,
+ _exact_measures_t,
+ joined_key_names=source_key_names,
+ local_group_keys=_local_group_keys,
+ )
+ )
+
+ if _nested_measures_t:
+ nested_needs_source_spine = (
+ join_tree_info.table_cardinalities.get(table_name)
+ not in ("root", "cross")
+ )
+ if not has_cross_table_gb and not nested_needs_source_spine and grain:
+ # A root/cross source grouped only by its own dimensions
+ # is already at the requested grain. Compile nested
+ # measures directly there and keep them separate from the
+ # regular preaggregate so sibling joins cannot fan out the
+ # unnested rows.
+ nested_pt = _compile_exact_measure_table(
+ raw_tbl, grain, _nested_measures_t
+ )
+ local_renames = {
+ public_name: raw_name
+ for raw_name, public_name in _local_group_outputs.items()
+ if raw_name in nested_pt.columns
+ }
+ collisions = sorted(
+ set(local_renames)
+ & (set(nested_pt.columns) - set(local_renames.values()))
+ )
+ if collisions:
+ raise ValueError(
+ "Cannot restore semantic group-key names after nested "
+ "source preaggregation because they collide with "
+ f"aggregate columns: {collisions}. Rename the aggregate field."
+ )
+ if local_renames:
+ nested_pt = nested_pt.rename(local_renames)
+ joined_grain = tuple(
+ _local_group_outputs.get(name, name) for name in grain
+ )
+ _preagg_results.append(
+ _rename_preagg_grain_to_joined_aliases(
+ nested_pt, joined_grain, source_key_names
+ )
+ )
+ else:
+ # Cross-source dimensions and non-root sources need the
+ # joined group-domain spine. This also restores the true
+ # empty-set value for unmatched LEFT JOIN groups.
+ _preagg_results.append(
+ _exact_grain_preagg(
+ raw_tbl,
+ tbl,
+ group_by_cols,
+ available_jk,
+ _nested_measures_t,
+ joined_key_names=source_key_names,
+ local_group_keys=_local_group_keys,
)
)
if agg_exprs:
if grain:
+ pt = raw_tbl.group_by(
+ [raw_tbl[c] for c in grain]
+ ).aggregate(**agg_exprs)
+ local_renames = {
+ public_name: raw_name
+ for raw_name, public_name in _local_group_outputs.items()
+ if raw_name in pt.columns
+ }
+ collisions = sorted(
+ set(local_renames) & (set(pt.columns) - set(local_renames.values()))
+ )
+ if collisions:
+ raise ValueError(
+ "Cannot restore semantic group-key names after source "
+ "preaggregation because they collide with aggregate "
+ f"columns: {collisions}. Rename the aggregate field."
+ )
+ if local_renames:
+ pt = pt.rename(local_renames)
+ joined_grain = tuple(
+ _local_group_outputs.get(name, name) for name in grain
+ )
_preagg_results.append(
- raw_tbl.group_by([raw_tbl[c] for c in grain]).aggregate(**agg_exprs)
+ _rename_preagg_grain_to_joined_aliases(
+ pt, joined_grain, source_key_names
+ )
)
else:
_preagg_results.append(raw_tbl.aggregate(**agg_exprs))
@@ -3921,6 +5005,7 @@ def _collect_many_sides(node):
preagg_results = tuple(_preagg_results)
decomposed_means = tuple(_decomposed_means.items())
reagg_ops = tuple(_reagg_ops.items())
+ empty_count_measures = tuple(_empty_count_measures)
if not preagg_results and not _deferred_count_distincts:
if tbl is not None:
@@ -3943,6 +5028,7 @@ def _collect_many_sides(node):
group_by_cols,
decomposed_means=decomposed_means,
reagg_ops=reagg_ops,
+ empty_count_measures=empty_count_measures,
)
else:
# Chasm fallback with group-by: build minimal dim bridge from raw tables
@@ -3954,48 +5040,80 @@ def _collect_many_sides(node):
merged_dimensions,
decomposed_means=decomposed_means,
reagg_ops=reagg_ops,
+ empty_count_measures=empty_count_measures,
)
# --- 5b. Compute deferred COUNT DISTINCT measures ---
- # Optimisation: compute on the raw source table when all group-by
- # columns are local (avoids scanning the fanned-out joined table).
- # Fall back to the full joined table only for cross-table group-bys.
+ # COUNT DISTINCT cannot be re-aggregated from per-key partial counts.
+ # For grouped queries, bridge the requested group domain to the
+ # measure's owning raw source and evaluate the original expression
+ # there. Re-evaluating it directly on the flattened join loses source
+ # provenance when physical column names collide (e.g. both sides have
+ # ``id``), and can count an unmatched left key as a right-side value.
if _deferred_count_distincts:
cd_parts: list = []
- for mname, (_src_tbl_name, _short, src_raw, src_fn) in (
+ join_column_lineage, _joined_columns = _build_join_column_lineage(
+ join_op
+ )
+ for mname, (
+ src_tbl_name,
+ _short,
+ src_raw,
+ src_fn,
+ local_group_keys,
+ ) in (
_deferred_count_distincts.items()
):
- src_cols = set(src_raw.columns)
- # Determine which group-by columns live on the raw source table
- local_gb = [c for c in group_by_cols if c in src_cols]
- if not group_by_cols or frozenset(local_gb) == frozenset(group_by_cols):
- # All group-by cols are local (or scalar) — compute on raw table
- cd_expr = src_fn(src_raw)
- if local_gb:
- cd_pt = src_raw.group_by(
- [src_raw[c] for c in local_gb]
- ).aggregate(**{mname: cd_expr})
- else:
- cd_pt = src_raw.aggregate(**{mname: cd_expr})
- cd_parts.append(cd_pt)
- else:
- # Cross-table group-by — need the full joined table
- if tbl is None:
- raise ValueError(
- "COUNT DISTINCT measures require the full joined table "
- "for cross-table group-by but it is unavailable (chasm "
- "fallback). Use join_one for reference tables or remove "
- "count-distinct measures from this query."
- )
- cd_expr = plan.agg_specs[mname](tbl)
- gb_available = [c for c in group_by_cols if c in tbl.columns]
- if gb_available:
- cd_pt = tbl.group_by(
- [tbl[c] for c in gb_available]
- ).aggregate(**{mname: cd_expr})
- else:
- cd_pt = tbl.aggregate(**{mname: cd_expr})
- cd_parts.append(cd_pt)
+ if not group_by_cols:
+ cd_parts.append(
+ src_raw.aggregate(**{mname: src_fn(src_raw)})
+ )
+ continue
+
+ if tbl is None:
+ raise ValueError(
+ "COUNT DISTINCT measures require the full joined table "
+ "for grouped source-aware aggregation but it is unavailable "
+ "(chasm fallback)."
+ )
+ join_keys = join_tree_info.table_join_keys.get(src_tbl_name, set())
+ source_key_names = join_column_lineage.get(src_tbl_name, {})
+ available_jk = tuple(
+ key
+ for key in sorted(join_keys)
+ if key in src_raw.columns
+ and source_key_names.get(key, key) in tbl.columns
+ )
+ if not available_jk:
+ raise ValueError(
+ f"COUNT DISTINCT measure {mname!r} cannot be attached to "
+ "the requested group grain without a shared join key."
+ )
+
+ cd_exact = _exact_grain_preagg(
+ src_raw,
+ tbl,
+ group_by_cols,
+ available_jk,
+ {mname: src_fn},
+ joined_key_names=source_key_names,
+ local_group_keys=local_group_keys,
+ )
+ # The exact source aggregate has no row for an unmatched
+ # outer-join group. Attach it to the joined group domain and
+ # restore COUNT DISTINCT's empty-set identity before calc
+ # measures are compiled.
+ group_spine = tbl.select(
+ [tbl[c] for c in group_by_cols]
+ ).distinct()
+ predicates = [
+ null_safe_equal(group_spine[c], cd_exact[c])
+ for c in group_by_cols
+ ]
+ cd_pt = group_spine.left_join(cd_exact, predicates).select(
+ [group_spine] + [cd_exact[mname]]
+ )
+ cd_parts.append(_fill_missing_count_identities(cd_pt, (mname,)))
# Merge count-distinct parts into result
for cd_pt in cd_parts:
@@ -4006,7 +5124,7 @@ def _collect_many_sides(node):
elif cd_grain:
common = [c for c in cd_grain if c in result.columns]
if common:
- preds = [result[c] == cd_pt[c] for c in common]
+ preds = [null_safe_equal(result[c], cd_pt[c]) for c in common]
result = result.left_join(cd_pt, preds).select(
[result] + [cd_pt[m] for m in cd_meas]
)
@@ -4015,6 +5133,10 @@ def _collect_many_sides(node):
else:
result = result.cross_join(cd_pt)
+ result = _fill_missing_count_identities(
+ result, _deferred_count_distincts
+ )
+
# --- 6. Apply calc_specs ---
if plan.calc_specs:
@@ -4026,7 +5148,7 @@ def _fanout_safe_totals():
under join_many.
"""
parts = [
- src.aggregate(**texprs)
+ _compile_evaluated_measure_table(src, (), texprs)
for src, texprs in _totals_sources.values()
if texprs
]
@@ -4087,7 +5209,9 @@ def _to_untagged_with_deferred_joins(
filters = filters or []
deferred_names = {d.table_name for d in deferrable}
- merged_dimensions = _get_merged_fields(all_roots, "dimensions")
+ merged_dimensions = _get_merged_fields(
+ all_roots, "dimensions", source=join_op
+ )
merged_dimensions = _augment_dimensions_with_raw_columns(
merged_dimensions, self.keys, all_roots, join_op
)
@@ -4284,7 +5408,13 @@ def strip_deferred(node):
@staticmethod
def _join_preagg_with_dim_bridge(
- preagg_results, plan, tbl, group_by_cols, decomposed_means=(), reagg_ops=()
+ preagg_results,
+ plan,
+ tbl,
+ group_by_cols,
+ decomposed_means=(),
+ reagg_ops=(),
+ empty_count_measures=(),
):
"""Join pre-aggregated tables using per-table dimension bridges.
@@ -4308,8 +5438,25 @@ def _rejoin_one(pt):
# Already has group-by columns — re-aggregate if over-grouped
if frozenset(pt_grain) != gb_set:
re_aggs = {m: _build_reagg(pt[m], reagg_map.get(m, "sum")) for m in pt_meas}
- return pt.group_by([pt[c] for c in group_by_cols]).aggregate(**re_aggs)
- return pt
+ pt = pt.group_by([pt[c] for c in group_by_cols]).aggregate(
+ **re_aggs
+ )
+
+ # A right/source-local preaggregate has no row for an
+ # unmatched LEFT JOIN group even when it is already at the
+ # requested semantic grain. Reattach it to the full joined
+ # group domain so SUM/MEDIAN remain NULL and COUNT can restore
+ # its zero identity later.
+ group_spine = tbl.select(
+ [tbl[c] for c in group_by_cols]
+ ).distinct()
+ predicates = [
+ null_safe_equal(group_spine[c], pt[c])
+ for c in group_by_cols
+ ]
+ return group_spine.left_join(pt, predicates).select(
+ [group_spine] + [pt[c] for c in pt_meas]
+ )
if not pt_grain:
return pt
@@ -4355,7 +5502,7 @@ def _rejoin_one(pt):
result = result.mutate(**{mname: result[sc] / result[cc]})
result = result.drop(sc, cc)
- return result
+ return _fill_missing_count_identities(result, empty_count_measures)
@staticmethod
def _build_minimal_dim_bridge(
@@ -4366,6 +5513,7 @@ def _build_minimal_dim_bridge(
merged_dimensions,
decomposed_means=(),
reagg_ops=(),
+ empty_count_measures=(),
):
"""Build dim bridges from raw tables when full join is unavailable.
@@ -4436,7 +5584,7 @@ def _bridge_one_preagg(pt):
result = result.mutate(**{mname: result[sc] / result[cc]})
result = result.drop(sc, cc)
- return result
+ return _fill_missing_count_identities(result, empty_count_measures)
@staticmethod
def _apply_calc_specs(result, plan, tbl, totals_builder=None):
@@ -4556,9 +5704,35 @@ def __init__(
on: Callable[[Any, Any], Any] | None = None,
cardinality: str = "one",
) -> None:
+ left = Relation.__coerce__(left)
+ right = Relation.__coerce__(right)
+
+ def _root_names(node) -> list[str]:
+ if isinstance(node, SemanticTableOp):
+ source_join = getattr(node, "_source_join", None)
+ if source_join is not None:
+ return _root_names(source_join)
+ return [node.name] if node.name else []
+ if isinstance(node, SemanticJoinOp):
+ return [*_root_names(node.left), *_root_names(node.right)]
+ source = getattr(node, "source", None)
+ return _root_names(source) if source is not None else []
+
+ root_names = [*_root_names(left), *_root_names(right)]
+ duplicate_names = sorted(
+ name for name in set(root_names) if root_names.count(name) > 1
+ )
+ if duplicate_names:
+ raise ValueError(
+ "Joined semantic models must have unique names; duplicate "
+ f"name(s): {duplicate_names}. Assign explicit aliases before "
+ "joining so dimensions, measures, and grain metadata retain "
+ "an unambiguous source."
+ )
+
super().__init__(
- left=Relation.__coerce__(left),
- right=Relation.__coerce__(right),
+ left=left,
+ right=right,
how=how,
on=on,
cardinality=cardinality,
@@ -4569,10 +5743,12 @@ def __repr__(self) -> str:
@property
def values(self) -> FrozenOrderedDict[str, Any]:
- vals: dict[str, Any] = {}
- vals.update(self.left.values)
- vals.update(self.right.values)
- return FrozenOrderedDict(vals)
+ # Derive fields from the executable join so metadata uses the same
+ # collision aliases (``x_right``, ``x_right2``, …) as query execution.
+ # A dict update of left/right semantic values overwrote colliding names
+ # and exposed a schema that contradicted ``to_untagged()``.
+ table = self.to_untagged(parent_requirements=None)
+ return FrozenOrderedDict({name: table[name].op() for name in table.columns})
@property
def schema(self):
@@ -4641,7 +5817,12 @@ def measures(self) -> tuple[str, ...]:
@property
def json_definition(self) -> Mapping[str, Any]:
- return _build_json_definition(self.get_dimensions(), self.get_measures(), None)
+ return _build_json_definition(
+ self.get_dimensions(),
+ self.get_measures(),
+ self.get_calculated_measures(),
+ None,
+ )
@property
def name(self) -> str | None:
@@ -4699,6 +5880,7 @@ def with_dimensions(self, **dims) -> SemanticTable:
measures=self.get_measures(),
calc_measures=self.get_calculated_measures(),
name=None,
+ _source_join=self,
)
def with_measures(self, **meas) -> SemanticTable:
@@ -4744,15 +5926,9 @@ def join_one(
how: str = "left",
):
"""Join with one-to-one relationship semantics (left outer join)."""
- from .expr import SemanticJoin
+ from .expr import _join_one_with_detected_grain
- return SemanticJoin(
- left=self,
- right=other.op(),
- on=on,
- how=how,
- cardinality="one",
- )
+ return _join_one_with_detected_grain(self, other, on, how)
def join_many(
self,
@@ -5273,9 +6449,6 @@ def to_untagged(self, parent_requirements: dict[str, set[str]] | None = None):
)
rname = self._rname_for_depth(depth)
- if self.on is None:
- return left_tbl.join(right_tbl, how=self.how, rname=rname)
-
# Detect column name conflicts that cause ibis/xorq to raise
# ``Ambiguous field reference`` during predicate resolution. The
# rename dance + ``_RenamedResolver`` below is a workaround for
@@ -5283,6 +6456,24 @@ def to_untagged(self, parent_requirements: dict[str, set[str]] | None = None):
# remove this branch when those tests fail.
conflicting = frozenset(left_tbl.columns) & frozenset(right_tbl.columns)
+ right_collision_names = _allocate_right_collision_names(
+ conflicting,
+ left_tbl.columns,
+ right_tbl.columns,
+ depth,
+ )
+
+ if self.on is None:
+ # Cross joins have no predicate requiring temporary left aliases,
+ # but ibis's rname template can still overwrite a real left column
+ # such as ``x_right``. Rename the colliding right columns to the
+ # already-allocated unique names before joining.
+ if right_collision_names:
+ right_tbl = right_tbl.rename(
+ {new: old for old, new in right_collision_names.items()}
+ )
+ return left_tbl.join(right_tbl, how=self.how)
+
if not conflicting:
pred = self.on(_Resolver(left_tbl), _Resolver(right_tbl))
return left_tbl.join(right_tbl, pred, how=self.how, rname=rname)
@@ -5290,13 +6481,28 @@ def to_untagged(self, parent_requirements: dict[str, set[str]] | None = None):
# Temporarily rename conflicting left columns so the predicate
# can be resolved without ambiguity.
# ibis rename convention: {new_name: old_name}
- rename_left = {f"{_BSL_JOIN_KEY_TMP_PREFIX}{c}": c for c in conflicting}
+ temporary_names = _allocate_temporary_join_names(
+ conflicting,
+ left_tbl.columns,
+ right_tbl.columns,
+ )
+ # Keep final right aliases distinct from temporary columns too. This
+ # matters for adversarial-but-valid source names containing both the
+ # public ``_right`` convention and BSL's private temporary prefix.
+ right_collision_names = _allocate_right_collision_names(
+ conflicting,
+ left_tbl.columns,
+ right_tbl.columns,
+ depth,
+ reserved=temporary_names.values(),
+ )
+ rename_left = {temporary_names[c]: c for c in conflicting}
left_safe = left_tbl.rename(rename_left)
# Resolver that transparently maps original names → temp names,
# so predicates like ``lambda f, a: f.tail_num == a.tail_num``
# still work even though left's ``tail_num`` was renamed.
- orig_to_tmp = {c: f"{_BSL_JOIN_KEY_TMP_PREFIX}{c}" for c in conflicting}
+ orig_to_tmp = dict(temporary_names)
pred = self.on(
_RenamedResolver(left_safe, orig_to_tmp),
@@ -5306,9 +6512,9 @@ def to_untagged(self, parent_requirements: dict[str, set[str]] | None = None):
# Restore final column names (ibis convention: {new: old}):
# - left temp columns → original names
- # - right conflicting columns → depth-based rname suffix
- rename_final = {c: f"{_BSL_JOIN_KEY_TMP_PREFIX}{c}" for c in conflicting} | {
- rname.replace("{name}", c): c for c in conflicting
+ # - right conflicting columns → unique depth-based suffixes
+ rename_final = {c: temporary_names[c] for c in conflicting} | {
+ new: old for old, new in right_collision_names.items()
}
return joined.rename(rename_final)
@@ -5719,7 +6925,9 @@ def to_untagged(self):
else _to_untagged(self.source)
)
- merged_dimensions = _get_merged_fields(all_roots, "dimensions")
+ merged_dimensions = _get_merged_fields(
+ all_roots, "dimensions", source=self.source
+ )
fields_to_index = _get_fields_to_index(
self.selector,
merged_dimensions,
@@ -6029,17 +7237,72 @@ def find_joins(node):
return join_keys
+def _build_join_column_lineage(
+ node: Relation,
+) -> tuple[dict[str, dict[str, str]], tuple[str, ...]]:
+ """Mirror join lowering and track every root column's executable alias."""
+ if isinstance(node, SemanticJoinOp):
+ left_lineage, left_columns = _build_join_column_lineage(node.left)
+ right_lineage, right_columns = _build_join_column_lineage(node.right)
+ conflicting = frozenset(left_columns) & frozenset(right_columns)
+ reserved = (
+ _allocate_temporary_join_names(
+ conflicting, left_columns, right_columns
+ ).values()
+ if node.on is not None
+ else ()
+ )
+ aliases = _allocate_right_collision_names(
+ conflicting,
+ left_columns,
+ right_columns,
+ node._join_depth(node),
+ reserved=reserved,
+ )
+
+ remapped_right = {
+ table_name: {
+ source_name: aliases.get(current_name, current_name)
+ for source_name, current_name in columns.items()
+ }
+ for table_name, columns in right_lineage.items()
+ }
+ output_columns = (*left_columns, *(aliases.get(c, c) for c in right_columns))
+ return ({**left_lineage, **remapped_right}, output_columns)
+
+ if isinstance(node, SemanticTableOp):
+ source_join = getattr(node, "_source_join", None)
+ if source_join is not None:
+ lineage, _columns = _build_join_column_lineage(source_join)
+ return lineage, tuple(node.table.columns)
+ columns = tuple(node.table.columns)
+ lineage = (
+ {node.name: {column: column for column in columns}}
+ if node.name
+ else {}
+ )
+ return lineage, columns
+
+ source = getattr(node, "source", None)
+ if source is not None:
+ lineage, _columns = _build_join_column_lineage(source)
+ return lineage, tuple(_to_untagged(node).columns)
+
+ return {}, tuple(_to_untagged(node).columns)
+
+
def _build_column_rename_map(
all_roots: Sequence[SemanticTable],
field_accessor: callable,
source: Relation | None = None,
-) -> dict[str, str]:
+) -> dict[str, dict[str, str]]:
"""
- Build a mapping of dimension names to their renamed column names in joined tables.
+ Build per-dimension source-column mappings for a flattened joined table.
- When Ibis joins tables with duplicate column names, it renames columns from later
- tables with '_right' suffix. However, columns used as join keys are merged and
- NOT renamed, so we exclude them from the rename map.
+ When Ibis joins tables with duplicate column names, columns from later tables
+ receive a depth-specific ``_right`` suffix. A dimension may reference more
+ than one source column, so each entry retains the complete mapping needed to
+ re-evaluate the original expression against the flattened join.
Uses graph_utils for generic traversal and the returns library for safe handling.
@@ -6049,12 +7312,34 @@ def _build_column_rename_map(
source: Optional source relation to extract join keys from
Returns:
- Dict mapping dimension names like 'airports.city' to renamed columns like 'city_right'
+ Dict mapping dimension names to ``{source_column: joined_column}`` maps.
"""
+ # Prefer exact lineage from the same alias allocator used by executable
+ # join lowering. The older flat-index/depth heuristic below remains as a
+ # fallback for callers without a SemanticJoinOp source.
+ join_lineage: dict[str, dict[str, str]] = {}
+
+ def _lineage_join(node):
+ if isinstance(node, SemanticJoinOp):
+ return node
+ if isinstance(node, SemanticTableOp):
+ wrapped = getattr(node, "_source_join", None)
+ if wrapped is not None:
+ return wrapped
+ parent = getattr(node, "source", None)
+ return _lineage_join(parent) if isinstance(parent, Relation) else None
+
+ lineage_join = _lineage_join(source)
+ if lineage_join is not None:
+ try:
+ join_lineage, _columns = _build_join_column_lineage(lineage_join)
+ except Exception:
+ logger.debug("join column-lineage analysis failed", exc_info=True)
+
# Build column index using graph_utils (returns Result)
from returns.result import Failure
- from .graph_utils import build_column_index_from_roots, extract_column_from_dimension
+ from .graph_utils import build_column_index_from_roots
column_index_result = build_column_index_from_roots(all_roots)
if isinstance(column_index_result, Failure):
@@ -6063,9 +7348,6 @@ def _build_column_rename_map(
column_index = column_index_result.value_or({})
- # Extract join key columns to exclude from renaming
- join_keys = _extract_join_key_column_names(source) if source else set()
-
# Build a map from table name → actual ibis join depth by walking the
# join tree. The flat index in all_roots does NOT equal ibis join depth
# for nested joins (e.g. aircraft → aircraft_models inside a flights
@@ -6090,21 +7372,43 @@ def _build_column_rename_map(
effective_depth = join_depth_map.get(root.name, idx)
for field_name, field_value in fields_dict.items():
- # Extract column name using graph_utils (returns Maybe)
- column_maybe = extract_column_from_dimension(field_value, root_tbl)
+ # Track every source field used by the original dimension. The
+ # previous implementation selected an arbitrary first field and
+ # replaced the whole expression with it, turning e.g. ``upper()``
+ # and multi-column dimensions into identity dimensions.
+ extraction = _extract_columns_from_callable(
+ lambda t, dim=field_value: dim(t), root_tbl
+ )
+ if not extraction.is_success():
+ continue
- # Use Maybe pattern from returns library
- column_maybe.bind_optional(
- lambda base_column: _check_and_add_rename( # noqa: B023
- rename_map=rename_map,
+ field_renames: dict[str, str] = {}
+ for base_column in extraction.columns:
+ executable_name = join_lineage.get(root.name, {}).get(base_column)
+ if executable_name is not None:
+ if executable_name != base_column:
+ field_renames[base_column] = executable_name
+ continue
+
+ candidate: dict[str, str] = {}
+ _check_and_add_rename(
+ rename_map=candidate,
base_column=base_column,
- prefixed_name=f"{root.name}.{field_name}", # noqa: B023
- table_idx=idx, # noqa: B023
+ prefixed_name=base_column,
+ table_idx=idx,
column_index=column_index,
- join_keys=join_keys,
- join_depth=effective_depth, # noqa: B023
+ # SemanticJoinOp's collision-safe join path preserves the
+ # right key as a suffixed physical column too. Mapping it
+ # is necessary to distinguish an unmatched right row from
+ # the left join key.
+ join_keys=set(),
+ join_depth=effective_depth,
)
- )
+ if base_column in candidate:
+ field_renames[base_column] = candidate[base_column]
+
+ if field_renames:
+ rename_map[f"{root.name}.{field_name}"] = field_renames
return rename_map
@@ -6150,21 +7454,25 @@ def _check_and_add_rename(
rename_map[prefixed_name] = f"{base_column}{suffix}"
-def _wrap_dimension_for_renamed_column(dimension: Dimension, renamed_column: str) -> Dimension:
+def _wrap_dimension_for_renamed_column(
+ dimension: Dimension, column_renames: Mapping[str, str]
+) -> Dimension:
"""
Wrap a dimension to access a renamed column in a joined table.
Args:
dimension: The original dimension
- renamed_column: The renamed column name (e.g., 'city_right')
+ column_renames: Source-to-joined physical column mapping.
Returns:
A new Dimension that accesses the renamed column
"""
- # Create a new callable that accesses the renamed column
+ # Re-evaluate the complete original expression against a resolver that
+ # redirects only the source fields renamed by the join. Operations and
+ # non-colliding fields continue to delegate to the joined table.
def renamed_accessor(table: ir.Table) -> ir.Value:
- return table[renamed_column]
+ return dimension(_RenamedResolver(table, column_renames))
# Return a new Dimension with the wrapped callable but same metadata
return Dimension(
@@ -6175,6 +7483,40 @@ def renamed_accessor(table: ir.Table) -> ir.Value:
is_event_timestamp=dimension.is_event_timestamp,
smallest_time_grain=dimension.smallest_time_grain,
derived_dimensions=dimension.derived_dimensions,
+ metadata=dimension.metadata,
+ )
+
+
+def _qualify_calc_measure_for_root(calc: CalcMeasure, root: SemanticTableOp) -> CalcMeasure:
+ """Bind a stored calculated measure's dependencies to its owning root.
+
+ Calculated measures are authored before composition and therefore capture
+ short dependency names. Once roots are merged those names become
+ ``.``; suffix matching is ambiguous when two roots expose
+ the same short name. Qualifying at the composition boundary preserves the
+ original lexical scope and also gives the calc compiler an explicit
+ preferred match for short references in the user's callable.
+ """
+ if not root.name:
+ return calc
+
+ local_names = set(root.get_measures()) | set(root.get_calculated_measures())
+
+ def qualify(name: str) -> str:
+ if "." in name or name not in local_names:
+ return name
+ return f"{root.name}.{name}"
+
+ qualified_dependencies = frozenset(qualify(name) for name in calc.depends_on)
+ qualified_preferred = frozenset(qualify(name) for name in calc.prefer_known)
+ qualified_preferred |= qualified_dependencies
+ return CalcMeasure(
+ expr=calc.expr,
+ description=calc.description,
+ requires_unnest=calc.requires_unnest,
+ depends_on=qualified_dependencies,
+ prefer_known=qualified_preferred,
+ metadata=calc.metadata,
)
@@ -6228,6 +7570,8 @@ def _merge_fields_with_prefixing(
field_value = _wrap_dimension_for_renamed_column(
field_value, column_rename_map[prefixed_name]
)
+ elif isinstance(field_value, CalcMeasure):
+ field_value = _qualify_calc_measure_for_root(field_value, root)
merged_fields[prefixed_name] = field_value
else:
@@ -6347,11 +7691,16 @@ def __init__(self, inner_table: ir.Table, access_handler: Callable[[str], None])
def __getattr__(self, name: str):
if name.startswith("_"):
return getattr(self._table, name)
- self._on_access(name)
+ # Relation methods such as ``count`` are not columns. Tracking
+ # them makes CountStar appear to reference a fictitious field and
+ # corrupts source-ownership inference.
+ if name in self._table.columns:
+ self._on_access(name)
return getattr(self._table, name)
def __getitem__(self, name: str):
- self._on_access(name)
+ if name in self._table.columns:
+ self._on_access(name)
return self._table[name]
return _TrackingProxy(table, on_access)
diff --git a/src/boring_semantic_layer/predicate.py b/src/boring_semantic_layer/predicate.py
index a3d3843f..d5b9f40f 100644
--- a/src/boring_semantic_layer/predicate.py
+++ b/src/boring_semantic_layer/predicate.py
@@ -18,13 +18,12 @@
from __future__ import annotations
import datetime
-from collections.abc import Callable, Iterable
+from collections.abc import Callable, Iterable, Mapping
from typing import Any, ClassVar, Literal
import ibis
from attrs import field, frozen
-
_COMPARE_OPS: dict[str, Callable[[Any, Any], Any]] = {
"eq": lambda x, y: x == y,
"ne": lambda x, y: x != y,
@@ -239,13 +238,25 @@ def _is_complete_iso_datetime(value: str) -> bool:
return False
-def _field_accessor(table, name: str, *, post_agg: bool):
+def _field_accessor(
+ table,
+ name: str,
+ *,
+ post_agg: bool,
+ strict_qualified: bool = False,
+):
"""Resolve a field name on the table.
- Pre-aggregation: use attribute access on ``ibis._`` so the predicate
- can be resolved against any table later (it is a Deferred).
- Pre-aggregation also strips a model-prefix from dotted names because
- joined tables flatten columns to the top level.
+ Pre-aggregation: resolve a qualified name exactly through bracket access.
+ Semantic resolver proxies understand names such as ``items.status`` and
+ bind them to the owning model's dimension, including any physical rename
+ caused by a join collision. Stripping the prefix first is unsound: when
+ both inputs contain ``status`` it silently binds to the left input.
+
+ The unprefixed fallback preserves the convenience spelling
+ ``standalone_model.field`` when a standalone model exposes only bare
+ semantic names. Joined/scoped resolvers resolve a declared qualified
+ dimension in the exact lookup and therefore never take that fallback.
Post-aggregation: use bracket access to preserve dotted names like
``orders.total_amount`` that survive into the aggregated table.
@@ -253,8 +264,21 @@ def _field_accessor(table, name: str, *, post_agg: bool):
if post_agg:
return table[name]
if "." in name:
- _prefix, unprefixed = name.split(".", 1)
- return getattr(table, unprefixed)
+ declared_dimensions = getattr(table, "_dims", {})
+ if isinstance(declared_dimensions, Mapping) and name in declared_dimensions:
+ # Do not mask an error raised while evaluating an explicitly
+ # qualified dimension by falling back to an unrelated bare
+ # column with the same suffix.
+ return table[name]
+ try:
+ return table[name]
+ except (AttributeError, KeyError, TypeError) as exc:
+ if strict_qualified:
+ raise KeyError(
+ f"Qualified filter field {name!r} did not resolve exactly"
+ ) from exc
+ _prefix, unprefixed = name.split(".", 1)
+ return getattr(table, unprefixed)
return getattr(table, name)
@@ -264,36 +288,76 @@ def compile( # noqa: A001
*,
post_agg: bool = False,
ibis_module=ibis,
+ strict_qualified: bool = False,
) -> Any:
"""Compile *pred* into an ibis expression against *table*.
- *table* is typically a Deferred (``ibis._``) for pre-agg or an actual
- aggregated relation for post-agg. ``ibis_module`` controls the flavor
- of literal construction (plain ibis vs xorq vendored).
+ *table* may be a Deferred (``ibis._``), a semantic resolver proxy, or an
+ actual relation. ``ibis_module`` controls the flavor of literal
+ construction (plain ibis vs xorq vendored).
"""
if isinstance(pred, And):
- compiled = [compile(c, table, post_agg=post_agg, ibis_module=ibis_module) for c in pred.children]
+ compiled = [
+ compile(
+ c,
+ table,
+ post_agg=post_agg,
+ ibis_module=ibis_module,
+ strict_qualified=strict_qualified,
+ )
+ for c in pred.children
+ ]
result = compiled[0]
for c in compiled[1:]:
result = result & c
return result
if isinstance(pred, Or):
- compiled = [compile(c, table, post_agg=post_agg, ibis_module=ibis_module) for c in pred.children]
+ compiled = [
+ compile(
+ c,
+ table,
+ post_agg=post_agg,
+ ibis_module=ibis_module,
+ strict_qualified=strict_qualified,
+ )
+ for c in pred.children
+ ]
result = compiled[0]
for c in compiled[1:]:
result = result | c
return result
if isinstance(pred, Not):
- return ~compile(pred.predicate, table, post_agg=post_agg, ibis_module=ibis_module)
+ return ~compile(
+ pred.predicate,
+ table,
+ post_agg=post_agg,
+ ibis_module=ibis_module,
+ strict_qualified=strict_qualified,
+ )
if isinstance(pred, IsNull):
- col = _field_accessor(table, pred.field, post_agg=post_agg)
+ col = _field_accessor(
+ table,
+ pred.field,
+ post_agg=post_agg,
+ strict_qualified=strict_qualified,
+ )
return col.notnull() if pred.negate else col.isnull()
if isinstance(pred, In):
- col = _field_accessor(table, pred.field, post_agg=post_agg)
+ col = _field_accessor(
+ table,
+ pred.field,
+ post_agg=post_agg,
+ strict_qualified=strict_qualified,
+ )
values = [_convert_literal(v, ibis_module) for v in pred.values]
return col.notin(values) if pred.negate else col.isin(values)
if isinstance(pred, Compare):
- col = _field_accessor(table, pred.field, post_agg=post_agg)
+ col = _field_accessor(
+ table,
+ pred.field,
+ post_agg=post_agg,
+ strict_qualified=strict_qualified,
+ )
value = _convert_literal(pred.value, ibis_module)
return _COMPARE_OPS[pred.op](col, value)
if isinstance(pred, Custom):
diff --git a/src/boring_semantic_layer/query.py b/src/boring_semantic_layer/query.py
index 023750c7..f27b2acf 100644
--- a/src/boring_semantic_layer/query.py
+++ b/src/boring_semantic_layer/query.py
@@ -230,17 +230,27 @@ def _resolve_target(t):
def _dict_filter(t):
tbl = _resolve_target(t)
ibis_module = get_ibis_module(tbl)
+ # Compile directly against the semantic resolver. In
+ # particular, ``resolver["items.status"]`` retains source
+ # ownership across a joined table whose left input also has
+ # a ``status`` column. Building a Deferred after first
+ # discarding that prefix cannot recover the ownership later.
return pred_mod.compile(
pred,
- ibis_module._,
+ tbl,
ibis_module=ibis_module,
- ).resolve(tbl)
+ strict_qualified=True,
+ )
# Deferred resolution: columns can't be statically introspected
# (see ops._dimension_only_source_table). Marked so callers can opt
# out of static-column optimizations consistently regardless of
# whether xorq is installed.
_dict_filter.__bsl_deferred_resolution__ = True
+ # Preserve the exact semantic spellings from the JSON AST. Lowering
+ # uses these to synthesize qualified raw-column dimensions before
+ # resolution; callable/string filters intentionally remain opaque.
+ _dict_filter.__bsl_filter_fields__ = frozenset(pred_mod.fields(pred))
return _dict_filter
elif isinstance(self.filter, str):
filter_str = self.filter
@@ -390,7 +400,7 @@ def _build_post_agg_predicate(filter_obj: dict) -> Any:
def _normalize_post_agg_filter(
filter_spec: Any,
- known_measures: set[str],
+ known_fields: set[str],
model_name: str | None = None,
) -> Callable:
"""Normalize a measure filter for post-aggregation (HAVING) application.
@@ -398,12 +408,12 @@ def _normalize_post_agg_filter(
Handles dict, Filter objects, and callables. For dict/Filter filters the
field names are accessed via bracket notation so dotted names from joined
models work correctly after aggregation. Field names are normalised
- against *known_measures* so that ``"model.total_sales"`` resolves to
+ against *known_fields* so that ``"model.total_sales"`` resolves to
``"total_sales"`` on standalone models but stays prefixed on joins.
"""
raw = filter_spec.filter if isinstance(filter_spec, Filter) else filter_spec
if isinstance(raw, dict):
- raw = _normalize_filter_fields(raw, known_measures, model_name)
+ raw = _normalize_filter_fields(raw, known_fields, model_name)
expr = _build_post_agg_predicate(raw)
return lambda t: expr.resolve(t)
if callable(raw):
@@ -466,6 +476,36 @@ def _split_filter(
pre_agg.append(filter_spec)
+def _validate_post_agg_filter_fields(
+ filters: Sequence[Any],
+ selected_fields: set[str],
+ known_fields: set[str],
+ model_name: str | None,
+) -> None:
+ """Reject inspectable HAVING fields absent from the aggregate output.
+
+ A semantic measure can be known to the model without being selected by
+ this query. Previously such a field passed normalization and failed much
+ later as an ibis ``AttributeError`` when the HAVING predicate was applied.
+ Dict filters are fully inspectable, so fail at query construction with the
+ actual semantic mistake. Opaque callable/string filters retain their
+ existing runtime behavior because their referenced fields cannot be
+ recovered reliably here.
+ """
+ for filter_spec in filters:
+ raw = filter_spec.filter if isinstance(filter_spec, Filter) else filter_spec
+ if not isinstance(raw, dict):
+ continue
+ normalized = _normalize_filter_fields(raw, known_fields, model_name)
+ missing = sorted(_extract_filter_fields(normalized) - selected_fields)
+ if missing:
+ raise ValueError(
+ "A post-aggregation filter references field(s) not selected "
+ f"by this query: {missing}. Add them to dimensions/measures "
+ "or remove the filter."
+ )
+
+
def _build_time_range_filters(semantic_table: Any, time_dimension: str, time_range: Mapping[str, str]) -> list[Callable]:
"""Build reusable filters for a specific time dimension and range."""
if not isinstance(time_range, dict) or "start" not in time_range or "end" not in time_range:
@@ -532,6 +572,7 @@ def compare_periods(
) -> Any:
"""Compare two time ranges and return current/previous/delta columns."""
from .api import to_semantic_table
+ from .join_utils import null_safe_equal
dimensions = list(dimensions or [])
measures = list(measures or [])
@@ -597,7 +638,13 @@ def compare_periods(
)
if dimensions:
- join_predicates = [current_tbl[dim] == previous_tbl[f"__previous_{dim}"] for dim in dimensions]
+ # SQL equality does not match NULL to NULL, but grouping semantics do:
+ # a NULL dimension member is one group in each period and therefore
+ # must form one comparison row rather than independent +/- rows.
+ join_predicates = [
+ null_safe_equal(current_tbl[dim], previous_tbl[f"__previous_{dim}"])
+ for dim in dimensions
+ ]
joined = current_tbl.join(previous_tbl, join_predicates, how="outer")
result_tbl = joined.select(
*[
@@ -736,6 +783,19 @@ def query(
order_by = _normalize_order_by(order_by, known_order_fields, expected_prefix=model_name)
filters = list(filters or []) # Copy to avoid mutating input
+ selected_fields = set(dimensions) | set(measures or [])
+ produces_aggregate = bool(dimensions or measures)
+ if produces_aggregate and order_by:
+ missing_order_fields = sorted(
+ field for field, _direction in order_by if field not in selected_fields
+ )
+ if missing_order_fields:
+ raise ValueError(
+ "order_by references field(s) not selected by this query: "
+ f"{missing_order_fields}. Add them to dimensions/measures or "
+ "remove them from order_by."
+ )
+
# Step 0: Add time_range as a filter if specified
if time_range:
time_dim_name = _find_time_dimension(result, dimensions)
@@ -760,6 +820,14 @@ def query(
for filter_spec in filters:
_split_filter(filter_spec, known_measures, model_name, pre_agg_filters, post_agg_filters)
+ known_post_agg_fields = known_dimensions | known_measures
+ _validate_post_agg_filter_fields(
+ post_agg_filters,
+ selected_fields,
+ known_post_agg_fields,
+ model_name,
+ )
+
for filter_spec in pre_agg_filters:
filter_fn = _normalize_filter(filter_spec)
result = result.filter(filter_fn)
@@ -828,7 +896,9 @@ def query(
# Step 3.5: Apply measure filters after aggregation (HAVING semantics)
for filter_spec in post_agg_filters:
- filter_fn = _normalize_post_agg_filter(filter_spec, known_measures, model_name)
+ filter_fn = _normalize_post_agg_filter(
+ filter_spec, known_post_agg_fields, model_name
+ )
result = result.filter(filter_fn)
# Step 4: Apply ordering using functional composition
diff --git a/src/boring_semantic_layer/serialization/extract.py b/src/boring_semantic_layer/serialization/extract.py
index 12f9e96e..6f501802 100644
--- a/src/boring_semantic_layer/serialization/extract.py
+++ b/src/boring_semantic_layer/serialization/extract.py
@@ -113,12 +113,28 @@ def _extract_semantic_table(op, context: BSLSerializationContext) -> dict[str, A
@_register_lazy("SemanticFilterOp")
def _extract_filter(op, context: BSLSerializationContext) -> dict[str, Any]:
+ from ..ops import _exact_filter_fields, _unwrap
from ..utils import expr_to_structured
- struct_result = expr_to_structured(op.predicate)
+ predicate = _unwrap(op.predicate)
+ try:
+ serialization_predicate = object.__getattribute__(
+ predicate, "__bsl_serialization_predicate__"
+ )
+ except (AttributeError, TypeError):
+ serialization_predicate = predicate
+ struct_result = expr_to_structured(serialization_predicate)
match struct_result:
case Success():
- return {"predicate_struct": struct_result.unwrap()}
+ metadata = {"predicate_struct": struct_result.unwrap()}
+ # JSON filters carry exact semantic field spellings as callable
+ # metadata. The generic Deferred resolver tree preserves the
+ # expression but not those attributes, so serialize them beside
+ # the predicate. They are needed to synthesize qualified raw
+ # columns and to reject unknown model prefixes after round-trip.
+ if filter_fields := _exact_filter_fields(op.predicate):
+ metadata["filter_fields"] = sorted(filter_fields)
+ return metadata
case _:
raise ValueError("SemanticFilterOp: failed to serialize predicate")
diff --git a/src/boring_semantic_layer/serialization/reconstruct.py b/src/boring_semantic_layer/serialization/reconstruct.py
index 1b74e0f8..1e8b51ee 100644
--- a/src/boring_semantic_layer/serialization/reconstruct.py
+++ b/src/boring_semantic_layer/serialization/reconstruct.py
@@ -177,6 +177,25 @@ def _reconstruct_filter(
metadata.get("predicate_struct"),
"SemanticFilterOp",
)
+ filter_fields = context.parse_field(metadata, "filter_fields")
+ if filter_fields and (
+ not isinstance(filter_fields, list | tuple)
+ or not all(isinstance(field, str) and field for field in filter_fields)
+ ):
+ raise ValueError("SemanticFilterOp has invalid filter_fields metadata")
+ if filter_fields:
+ # Structured deserialization returns a Deferred and necessarily loses
+ # attributes from the original JSON-filter callable. Restore those
+ # attributes on a small callable adapter instead of inferring field
+ # ownership from the Deferred expression: string/callable filters can
+ # contain identical syntax without carrying JSON's strict semantics.
+ deferred_predicate = predicate
+
+ def predicate(t):
+ return deferred_predicate.resolve(t)
+
+ predicate.__bsl_filter_fields__ = frozenset(filter_fields)
+ predicate.__bsl_deferred_resolution__ = True
return source.filter(predicate)
diff --git a/src/boring_semantic_layer/tests/test_flights_schemas.py b/src/boring_semantic_layer/tests/test_flights_schemas.py
index 79617f51..0dc07e3c 100644
--- a/src/boring_semantic_layer/tests/test_flights_schemas.py
+++ b/src/boring_semantic_layer/tests/test_flights_schemas.py
@@ -194,7 +194,7 @@ def test_group_by_state_top_states(self, star):
assert tx["flights.flight_count"].iloc[0] == 40085
def test_group_by_state_unmatched_rows_preserved(self, star):
- """States with airports but no origin flights should appear with NULL measures.
+ """States with airports but no origin flights should appear with zero counts.
Delaware (DE) has 42 airports but zero flights originating there.
"""
@@ -205,8 +205,8 @@ def test_group_by_state_unmatched_rows_preserved(self, star):
)
de = df[df["airports.state"] == "DE"]
assert len(de) == 1, "Delaware should appear in results"
- # flight_count for unmatched state should be NULL
- assert de["flights.flight_count"].isna().iloc[0]
+ # COUNT preserves its empty-set identity for an unmatched source group.
+ assert de["flights.flight_count"].iloc[0] == 0
def test_one_side_measure_airport_count_by_state(self, star):
"""One-side measure (airport_count) grouped by one-side dimension."""
@@ -477,7 +477,7 @@ def test_group_by_manufacturer_through_star(self, full_schema):
assert boeing["flights.flight_count"].iloc[0] == 183236
def test_unmatched_states_appear(self, full_schema):
- """States with airports but no flights should appear."""
+ """States with airports but no flights should appear with zero counts."""
df = (
full_schema.group_by("airports.state")
.aggregate("flights.flight_count")
@@ -486,7 +486,7 @@ def test_unmatched_states_appear(self, full_schema):
# Delaware has 42 airports but 0 flights
de = df[df["airports.state"] == "DE"]
assert len(de) == 1
- assert de["flights.flight_count"].isna().iloc[0]
+ assert de["flights.flight_count"].iloc[0] == 0
def test_mean_not_inflated_through_snowflake_arms(self, full_schema):
"""avg_distance by state should not inflate through snowflake arms."""
diff --git a/src/boring_semantic_layer/tests/test_join_soundness_repairs.py b/src/boring_semantic_layer/tests/test_join_soundness_repairs.py
new file mode 100644
index 00000000..2b9314b2
--- /dev/null
+++ b/src/boring_semantic_layer/tests/test_join_soundness_repairs.py
@@ -0,0 +1,997 @@
+"""Regression coverage for source, grain, and namespace preservation in joins."""
+
+import ibis
+import pandas as pd
+import pytest
+
+from boring_semantic_layer import Dimension, to_semantic_table
+
+
+@pytest.fixture
+def con():
+ return ibis.duckdb.connect(":memory:")
+
+
+def test_join_one_right_measures_stay_bound_to_right_rows(con):
+ left = con.create_table(
+ "source_left", pd.DataFrame({"id": [1, 2], "value": [10, 20]})
+ )
+ right = con.create_table(
+ "source_right", pd.DataFrame({"id": [1, 3], "value": [100, 900]})
+ )
+ left_model = to_semantic_table(left, "left").with_measures(
+ total=lambda t: t.value.sum()
+ )
+ right_model = to_semantic_table(right, "right").with_measures(
+ total=lambda t: t.value.sum(),
+ row_count=lambda t: t.count(),
+ id_count=lambda t: t.id.count(),
+ )
+
+ result = left_model.join_one(right_model, on="id").aggregate(
+ "left.total", "right.total", "right.row_count", "right.id_count"
+ ).execute()
+
+ assert result.iloc[0].to_dict() == {
+ "left.total": 30,
+ "right.total": 100,
+ "right.row_count": 1,
+ "right.id_count": 1,
+ }
+
+
+def test_grouped_right_measures_use_the_right_executable_join_key_alias(con):
+ left = con.create_table(
+ "different_key_left",
+ pd.DataFrame(
+ {
+ "lkey": [1, 2],
+ # Deliberately collides with the right join key, but is not
+ # part of this relationship.
+ "rkey": [999, 999],
+ "group": ["a", "b"],
+ }
+ ),
+ )
+ right = con.create_table(
+ "different_key_right",
+ pd.DataFrame(
+ {
+ "rkey": [1, 1, 2],
+ "value": [10, 11, 20],
+ }
+ ),
+ )
+ left_model = to_semantic_table(left, "left").with_dimensions(
+ group=lambda t: t.group
+ )
+ right_model = to_semantic_table(right, "right").with_measures(
+ total=lambda t: t.value.sum(),
+ median=lambda t: t.value.median(),
+ distinct_values=lambda t: t.value.nunique(),
+ )
+
+ joined = left_model.join_many(
+ right_model,
+ on=lambda left_row, right_row: left_row.lkey == right_row.rkey,
+ )
+ assert "rkey_right" in joined.columns
+
+ result = (
+ joined.group_by("left.group")
+ .aggregate("right.total", "right.median", "right.distinct_values")
+ .order_by("left.group")
+ .execute()
+ )
+
+ assert result["left.group"].tolist() == ["a", "b"]
+ assert result["right.total"].tolist() == [21, 20]
+ assert result["right.median"].tolist() == [10.5, 20.0]
+ assert result["right.distinct_values"].tolist() == [2, 1]
+
+ filtered = (
+ joined.filter(lambda t: t["left.group"] == "a")
+ .group_by("left.group")
+ .aggregate("right.total")
+ .execute()
+ )
+ assert filtered.to_dict("records") == [
+ {"left.group": "a", "right.total": 21}
+ ]
+
+
+def test_join_one_unmatched_right_counts_are_zero_but_sum_is_null(con):
+ left = con.create_table(
+ "count_identity_left", pd.DataFrame({"id": [1, 2]})
+ )
+ right = con.create_table(
+ "count_identity_right", pd.DataFrame({"id": [1], "value": [100]})
+ )
+ left_model = to_semantic_table(left, "left").with_dimensions(
+ id=lambda t: t.id
+ )
+ right_model = to_semantic_table(right, "right").with_measures(
+ row_count=lambda t: t.count(),
+ id_count=lambda t: t.id.count(),
+ total=lambda t: t.value.sum(),
+ doubled_count=lambda t: t.row_count * 2,
+ )
+
+ result = (
+ left_model.join_one(right_model, on="id")
+ .group_by("left.id")
+ .aggregate(
+ "right.row_count",
+ "right.id_count",
+ "right.total",
+ "right.doubled_count",
+ )
+ .execute()
+ .set_index("left.id")
+ )
+
+ assert result.loc[1, "right.row_count"] == 1
+ assert result.loc[1, "right.id_count"] == 1
+ assert result.loc[1, "right.total"] == 100
+ assert result.loc[1, "right.doubled_count"] == 2
+ assert result.loc[2, "right.row_count"] == 0
+ assert result.loc[2, "right.id_count"] == 0
+ assert pd.isna(result.loc[2, "right.total"])
+ assert result.loc[2, "right.doubled_count"] == 0
+
+
+@pytest.mark.parametrize("join_method", ["join_one", "join_many"])
+def test_exact_measures_use_their_real_empty_source_result(con, join_method):
+ left = con.create_table(
+ f"exact_empty_left_{join_method}",
+ pd.DataFrame(
+ {
+ "id": [1, 2, 3],
+ "group": ["hit", "miss", "all_null"],
+ }
+ ),
+ )
+ right = con.create_table(
+ f"exact_empty_right_{join_method}",
+ pd.DataFrame(
+ {
+ "id": [1, 3],
+ "kind": ["A", "B"],
+ "token": ["x", None],
+ "value": [10.0, None],
+ }
+ ),
+ )
+ left_model = to_semantic_table(left, "left").with_dimensions(
+ group=lambda t: t.group
+ )
+ right_model = (
+ to_semantic_table(right, "right")
+ .with_dimensions(kind=lambda t: t.kind)
+ .with_measures(
+ row_count_plus_one=lambda t: t.count() + 1,
+ zero_sum=lambda t: t.value.sum().fill_null(0),
+ ratio=lambda t: t.value.sum() / t.token.count(),
+ median=lambda t: t.value.median(),
+ )
+ )
+ joined = getattr(left_model, join_method)(right_model, on="id")
+ measures = (
+ "right.row_count_plus_one",
+ "right.zero_sum",
+ "right.ratio",
+ "right.median",
+ )
+
+ by_left = (
+ joined.group_by("left.group")
+ .aggregate(*measures)
+ .execute()
+ .set_index("left.group")
+ )
+ assert by_left.loc["hit", "right.row_count_plus_one"] == 2
+ assert by_left.loc["hit", "right.zero_sum"] == 10
+ assert by_left.loc["hit", "right.ratio"] == 10
+ assert by_left.loc["hit", "right.median"] == 10
+ assert by_left.loc["miss", "right.row_count_plus_one"] == 1
+ assert by_left.loc["miss", "right.zero_sum"] == 0
+ assert pd.isna(by_left.loc["miss", "right.ratio"])
+ assert pd.isna(by_left.loc["miss", "right.median"])
+ # A matched row whose inputs are all NULL is not an absent aggregate row.
+ assert by_left.loc["all_null", "right.row_count_plus_one"] == 2
+ assert by_left.loc["all_null", "right.zero_sum"] == 0
+ assert pd.isna(by_left.loc["all_null", "right.ratio"])
+ assert pd.isna(by_left.loc["all_null", "right.median"])
+
+ by_right = (
+ joined.group_by("right.kind")
+ .aggregate(*measures)
+ .execute()
+ )
+ unmatched = by_right[by_right["right.kind"].isna()].iloc[0]
+ assert unmatched["right.row_count_plus_one"] == 1
+ assert unmatched["right.zero_sum"] == 0
+ assert pd.isna(unmatched["right.ratio"])
+ assert pd.isna(unmatched["right.median"])
+
+
+def test_join_one_count_distinct_uses_right_collision_alias_and_zero_identity(con):
+ left = con.create_table(
+ "distinct_identity_left",
+ pd.DataFrame(
+ {
+ "join_id": [1, 2],
+ # This unrelated left field collides with the right join key,
+ # so the executable right key is ``id_right``.
+ "id": [999, 888],
+ "group": ["matched", "unmatched"],
+ }
+ ),
+ )
+ right = con.create_table(
+ "distinct_identity_right", pd.DataFrame({"id": [1], "value": [10]})
+ )
+ left_model = to_semantic_table(left, "left").with_dimensions(
+ group=lambda t: t.group
+ )
+ right_model = to_semantic_table(right, "right").with_measures(
+ distinct_ids=lambda t: t.id.nunique(),
+ doubled_distinct=lambda t: t.distinct_ids * 2,
+ )
+
+ result = (
+ left_model.join_one(
+ right_model, on=lambda left_row, right_row: left_row.join_id == right_row.id
+ )
+ .group_by("left.group")
+ .aggregate("right.distinct_ids", "right.doubled_distinct")
+ .execute()
+ .set_index("left.group")
+ )
+
+ assert result.loc["matched", "right.distinct_ids"] == 1
+ assert result.loc["unmatched", "right.distinct_ids"] == 0
+ assert result.loc["matched", "right.doubled_distinct"] == 2
+ assert result.loc["unmatched", "right.doubled_distinct"] == 0
+
+
+def test_colliding_derived_dimension_keeps_complete_expression(con):
+ left = con.create_table(
+ "derived_left",
+ pd.DataFrame({"id": [1, 2, 3], "name": ["L1", "L2", "L3"]}),
+ )
+ right = con.create_table(
+ "derived_right",
+ pd.DataFrame(
+ {
+ "id": [1, 2],
+ "name": ["alice", "bob"],
+ "suffix": ["x", "y"],
+ }
+ ),
+ )
+ left_model = to_semantic_table(left, "left").with_measures(
+ row_count=lambda t: t.count()
+ )
+ right_model = to_semantic_table(right, "right").with_dimensions(
+ label=lambda t: t.name.upper() + "-" + t.suffix.upper(),
+ right_id=lambda t: t.id,
+ )
+
+ joined = left_model.join_one(right_model, on="id")
+ labels = joined.group_by("right.label").aggregate("left.row_count").execute()
+ ids = joined.group_by("right.right_id").aggregate("left.row_count").execute()
+
+ assert set(labels["right.label"].dropna()) == {"ALICE-X", "BOB-Y"}
+ assert labels["right.label"].isna().sum() == 1
+ assert ids["right.right_id"].isna().sum() == 1
+
+
+def test_join_many_derived_dimension_is_measure_selection_invariant(con):
+ orders = con.create_table(
+ "selection_orders",
+ pd.DataFrame({"id": [1, 2], "name": ["L1", "L2"], "amount": [10, 20]}),
+ )
+ customers = con.create_table(
+ "selection_customers",
+ pd.DataFrame({"id": [1, 2], "name": ["alice", "bob"]}),
+ )
+ order_model = to_semantic_table(orders, "orders").with_measures(
+ total=lambda t: t.amount.sum()
+ )
+ customer_model = (
+ to_semantic_table(customers, "customers")
+ .with_dimensions(label=lambda t: t.name.upper())
+ .with_measures(row_count=lambda t: t.count())
+ )
+ joined = order_model.join_many(customer_model, on="id")
+
+ left_only = joined.group_by("customers.label").aggregate("orders.total").execute()
+ right_only = joined.group_by("customers.label").aggregate("customers.row_count").execute()
+ together = joined.group_by("customers.label").aggregate(
+ "orders.total", "customers.row_count"
+ ).execute()
+
+ assert set(left_only["customers.label"]) == {"ALICE", "BOB"}
+ assert set(right_only["customers.label"]) == {"ALICE", "BOB"}
+ assert set(together["customers.label"]) == {"ALICE", "BOB"}
+ assert together.set_index("customers.label").loc["ALICE"].to_dict() == {
+ "orders.total": 10,
+ "customers.row_count": 1,
+ }
+
+
+def test_non_equi_join_rejected_when_preaggregation_depends_on_it(con):
+ left = con.create_table(
+ "predicate_left", pd.DataFrame({"customer": [1, 1], "id": [1, 2]})
+ )
+ right = con.create_table(
+ "predicate_right",
+ pd.DataFrame({"customer": [1, 1], "id": [1, 2], "value": [100, 200]}),
+ )
+ left_model = to_semantic_table(left, "left")
+ right_model = to_semantic_table(right, "right").with_measures(
+ total=lambda t: t.value.sum()
+ )
+ joined = left_model.join_many(
+ right_model,
+ on=lambda left_row, right_row: (left_row.customer == right_row.customer)
+ & (left_row.id != right_row.id),
+ )
+
+ with pytest.raises(ValueError, match="direct field equijoins"):
+ joined.aggregate("right.total").execute()
+
+
+def test_duplicate_model_names_are_rejected(con):
+ left = con.create_table("duplicate_left", pd.DataFrame({"id": [1]}))
+ right = con.create_table("duplicate_right", pd.DataFrame({"id": [1]}))
+
+ with pytest.raises(ValueError, match="unique names"):
+ to_semantic_table(left, "duplicate").join_many(
+ to_semantic_table(right, "duplicate"), on="id"
+ )
+
+
+def test_join_one_orphans_stay_excluded_in_a_larger_join_tree(con):
+ root = con.create_table("orphan_root", pd.DataFrame({"id": [1, 2]}))
+ lookup = con.create_table(
+ "orphan_lookup", pd.DataFrame({"id": [1, 2, 3], "value": [10, 20, 900]})
+ )
+ children = con.create_table(
+ "orphan_children", pd.DataFrame({"id": [1, 1, 2]})
+ )
+ root_model = to_semantic_table(root, "root")
+ lookup_model = to_semantic_table(lookup, "lookup").with_measures(
+ total=lambda t: t.value.sum()
+ )
+ child_model = to_semantic_table(children, "children").with_measures(
+ row_count=lambda t: t.count()
+ )
+
+ result = (
+ root_model.join_one(lookup_model, on="id")
+ .join_many(child_model, on="id")
+ .aggregate("lookup.total", "children.row_count")
+ .execute()
+ )
+
+ assert result["lookup.total"].iloc[0] == 30
+ assert result["children.row_count"].iloc[0] == 3
+
+
+def test_post_join_field_reduction_is_routed_to_its_unique_source(con):
+ orders = con.create_table(
+ "wrapper_orders", pd.DataFrame({"id": [1, 2], "amount": [100, 200]})
+ )
+ items = con.create_table(
+ "wrapper_items", pd.DataFrame({"id": [1, 1, 2], "quantity": [1, 2, 3]})
+ )
+ joined = to_semantic_table(orders, "orders").join_many(
+ to_semantic_table(items, "items"), on="id"
+ )
+
+ result = joined.with_measures(
+ source_total=lambda t: t.amount.sum(),
+ joined_row_count=lambda t: t.count(),
+ ).aggregate("source_total", "joined_row_count").execute()
+
+ assert result["source_total"].iloc[0] == 300
+ assert result["joined_row_count"].iloc[0] == 3
+
+
+def test_count_distinct_matches_null_group_when_combined(con):
+ orders = con.create_table(
+ "null_distinct_orders",
+ pd.DataFrame(
+ {
+ "id": [1, 2, 3],
+ "group": pd.Series([None, None, "a"], dtype="string"),
+ "value": [1, 1, 2],
+ "amount": [1, 3, 5],
+ }
+ ),
+ )
+ items = con.create_table(
+ "null_distinct_items", pd.DataFrame({"id": [1, 1, 2, 3]})
+ )
+ order_model = (
+ to_semantic_table(orders, "orders")
+ .with_dimensions(group=lambda t: t.group)
+ .with_measures(
+ total=lambda t: t.amount.sum(),
+ distinct_values=lambda t: t.value.nunique(),
+ )
+ )
+ joined = order_model.join_many(to_semantic_table(items, "items"), on="id")
+
+ result = joined.group_by("orders.group").aggregate(
+ "orders.total", "orders.distinct_values"
+ ).execute()
+ null_row = result[result["orders.group"].isna()].iloc[0]
+
+ assert null_row["orders.total"] == 4
+ assert null_row["orders.distinct_values"] == 1
+
+
+def test_right_count_distinct_respects_right_dimension_with_reused_join_key(con):
+ orders_df = pd.DataFrame(
+ {
+ "order_id": [10, 11, 12],
+ # Collision-prone user columns must not affect the bridge.
+ "order_ref": [901, 902, 903],
+ "sku_right": [801, 802, 803],
+ "__bsl_jk_order_ref": [701, 702, 703],
+ "__exact_gb_0": [601, 602, 603],
+ }
+ )
+ items_df = pd.DataFrame(
+ {
+ "order_ref": [10, 10, 10, 11, 99],
+ "kind": ["P", "P", "Q", "P", "X"],
+ "sku": ["a", "a", "d", "b", "c"],
+ "sku_right": [1, 2, 3, 4, 5],
+ "__bsl_jk_order_ref": [11, 12, 13, 14, 15],
+ "__exact_gb_0": [21, 22, 23, 24, 25],
+ }
+ )
+ orders = con.create_table("dimension_distinct_orders", orders_df)
+ items = con.create_table("dimension_distinct_items", items_df)
+ item_model = (
+ to_semantic_table(items, "items")
+ .with_dimensions(kind=lambda t: t.kind)
+ .with_measures(distinct_skus=lambda t: t.sku.nunique())
+ )
+ joined = to_semantic_table(orders, "orders").join_many(
+ item_model,
+ on=lambda order, item: order.order_id == item.order_ref,
+ )
+
+ actual = (
+ joined.group_by("items.kind")
+ .aggregate("items.distinct_skus")
+ .execute()
+ .set_index("items.kind")["items.distinct_skus"]
+ )
+ expected = (
+ orders_df.merge(
+ items_df,
+ how="left",
+ left_on="order_id",
+ right_on="order_ref",
+ suffixes=("_order", "_item"),
+ )
+ .groupby("kind", dropna=False)["sku"]
+ .nunique()
+ )
+
+ actual_by_kind = {
+ "" if pd.isna(kind) else kind: value
+ for kind, value in actual.items()
+ }
+ expected_by_kind = {
+ "" if pd.isna(kind) else kind: value
+ for kind, value in expected.items()
+ }
+ assert actual_by_kind == expected_by_kind == {"": 0, "P": 2, "Q": 1}
+
+
+def test_right_median_respects_local_dimension_at_mixed_group_grain(con):
+ orders_df = pd.DataFrame(
+ {
+ "order_id": [10, 11, 12],
+ "cohort": ["A", "A", "B"],
+ "order_ref": [901, 902, 903],
+ "value_right": [801, 802, 803],
+ "__bsl_jk_order_ref": [701, 702, 703],
+ "__exact_gb_0": [601, 602, 603],
+ }
+ )
+ items_df = pd.DataFrame(
+ {
+ "order_ref": [10, 10, 10, 11, 99],
+ "kind": ["P", "P", "Q", "P", "X"],
+ "value": [5.0, 6.0, 4.0, 7.0, 9.0],
+ "value_right": [1, 2, 3, 4, 5],
+ "__bsl_jk_order_ref": [11, 12, 13, 14, 15],
+ "__exact_gb_0": [21, 22, 23, 24, 25],
+ }
+ )
+ orders = con.create_table("dimension_median_orders", orders_df)
+ items = con.create_table("dimension_median_items", items_df)
+ order_model = to_semantic_table(orders, "orders").with_dimensions(
+ cohort=lambda t: t.cohort
+ )
+ item_model = (
+ to_semantic_table(items, "items")
+ .with_dimensions(kind=lambda t: t.kind)
+ .with_measures(median_value=lambda t: t.value.median())
+ )
+ joined = order_model.join_many(
+ item_model,
+ on=lambda order, item: order.order_id == item.order_ref,
+ )
+
+ actual = (
+ joined.group_by("orders.cohort", "items.kind")
+ .aggregate("items.median_value")
+ .execute()
+ )
+ expected = (
+ orders_df.merge(
+ items_df,
+ how="left",
+ left_on="order_id",
+ right_on="order_ref",
+ suffixes=("_order", "_item"),
+ )
+ .groupby(["cohort", "kind"], dropna=False)["value"]
+ .median()
+ .reset_index(name="items.median_value")
+ .rename(columns={"cohort": "orders.cohort", "kind": "items.kind"})
+ )
+ actual["items.kind"] = actual["items.kind"].fillna("")
+ expected["items.kind"] = expected["items.kind"].fillna("")
+ actual = actual.sort_values(["orders.cohort", "items.kind"]).reset_index(drop=True)
+ expected = expected.sort_values(["orders.cohort", "items.kind"]).reset_index(drop=True)
+
+ pd.testing.assert_frame_equal(actual, expected, check_dtype=False)
+
+
+def test_join_wrapper_dimension_keeps_flattened_source_lineage(con):
+ left_df = pd.DataFrame(
+ {
+ "id": [1, 2],
+ "status": ["LEFT_A", "LEFT_B"],
+ }
+ )
+ right_df = pd.DataFrame(
+ {
+ "id": [1, 1, 2],
+ # The same physical name is deliberately bound to different values
+ # on the right. A wrapper dimension is authored against the
+ # flattened join, where the unsuffixed field belongs to the left.
+ "status": ["R_X", "R_Y", "R_Z"],
+ "value": [10, 20, 30],
+ }
+ )
+ left = con.create_table("wrapper_dimension_left", left_df)
+ right = con.create_table("wrapper_dimension_right", right_df)
+ left_model = to_semantic_table(left, "left").with_dimensions(
+ status=lambda t: t.status
+ )
+ right_model = to_semantic_table(right, "right").with_measures(
+ total=lambda t: t.value.sum(),
+ median=lambda t: t.value.median(),
+ distinct_statuses=lambda t: t.status.nunique(),
+ )
+ joined = left_model.join_many(right_model, on="id").with_dimensions(
+ bucket=lambda t: t.status
+ )
+
+ actual = (
+ joined.group_by("bucket")
+ .aggregate(
+ "right.total",
+ "right.median",
+ "right.distinct_statuses",
+ )
+ .execute()
+ .sort_values("bucket")
+ .reset_index(drop=True)
+ )
+ expected = pd.DataFrame(
+ {
+ "bucket": ["LEFT_A", "LEFT_B"],
+ "right.total": [30, 30],
+ "right.median": [15.0, 30.0],
+ "right.distinct_statuses": [2, 1],
+ }
+ )
+
+ pd.testing.assert_frame_equal(actual, expected, check_dtype=False)
+
+
+def test_cross_source_wrapper_dimension_fails_closed_for_source_aggregates(con):
+ left = con.create_table(
+ "cross_wrapper_dimension_left",
+ pd.DataFrame({"id": [1, 2], "prefix": ["A", "B"]}),
+ )
+ right = con.create_table(
+ "cross_wrapper_dimension_right",
+ pd.DataFrame(
+ {
+ "id": [1, 1, 2],
+ "status": ["X", "Y", "Z"],
+ "value": [10, 20, 30],
+ }
+ ),
+ )
+ left_model = to_semantic_table(left, "left").with_dimensions(
+ prefix=lambda t: t.prefix
+ )
+ right_model = (
+ to_semantic_table(right, "right")
+ .with_dimensions(status=lambda t: t.status)
+ .with_measures(
+ total=lambda t: t.value.sum(),
+ median=lambda t: t.value.median(),
+ distinct_statuses=lambda t: t.status.nunique(),
+ )
+ )
+ joined = left_model.join_many(right_model, on="id").with_dimensions(
+ bucket=lambda t: t.left.prefix + "_" + t.right.status
+ )
+
+ with pytest.raises(
+ ValueError,
+ match="join-wrapper dimension.*spans multiple semantic models",
+ ):
+ joined.group_by("bucket").aggregate(
+ "right.total",
+ "right.median",
+ "right.distinct_statuses",
+ ).execute()
+
+
+def test_transformed_same_name_right_dimension_preserves_every_measure_grain(con):
+ orders = con.create_table(
+ "transformed_same_name_orders",
+ pd.DataFrame(
+ {
+ "id": [1, 2, 3],
+ "cohort": ["A", "A", "B"],
+ }
+ ),
+ )
+ items = con.create_table(
+ "transformed_same_name_items",
+ pd.DataFrame(
+ {
+ "id": [1, 1, 2],
+ "kind": ["p", "q", "p"],
+ "value": [10, 100, 30],
+ # Force the private group-key allocator past its preferred
+ # spelling; user-owned internal-prefix columns are valid.
+ "__bsl_gb_items_kind": [901, 902, 903],
+ }
+ ),
+ )
+ order_model = to_semantic_table(orders, "orders").with_dimensions(
+ cohort=lambda t: t.cohort
+ )
+ item_model = (
+ to_semantic_table(items, "items")
+ # upper() retains the input expression name "kind". It must not be
+ # mistaken for the direct physical field.
+ .with_dimensions(kind=lambda t: t.kind.upper())
+ .with_measures(
+ total=lambda t: t.value.sum(),
+ row_count=lambda t: t.count(),
+ distinct_values=lambda t: t.value.nunique(),
+ median_value=lambda t: t.value.median(),
+ )
+ )
+ joined = order_model.join_many(item_model, on="id")
+ measures = (
+ "items.total",
+ "items.row_count",
+ "items.distinct_values",
+ "items.median_value",
+ )
+
+ local = (
+ joined.group_by("items.kind")
+ .aggregate(*measures)
+ .execute()
+ .set_index("items.kind")
+ )
+ assert local.loc["P"].to_dict() == {
+ "items.total": 40,
+ "items.row_count": 2,
+ "items.distinct_values": 2,
+ "items.median_value": 20.0,
+ }
+ assert local.loc["Q"].to_dict() == {
+ "items.total": 100,
+ "items.row_count": 1,
+ "items.distinct_values": 1,
+ "items.median_value": 100.0,
+ }
+ null_local = local[local.index.isna()].iloc[0]
+ assert pd.isna(null_local["items.total"])
+ assert null_local["items.row_count"] == 0
+ assert null_local["items.distinct_values"] == 0
+ assert pd.isna(null_local["items.median_value"])
+
+ mixed = (
+ joined.group_by("orders.cohort", "items.kind")
+ .aggregate(*measures)
+ .execute()
+ )
+ matched = mixed[mixed["items.kind"].notna()].set_index("items.kind")
+ assert matched.loc["P", "items.total"] == 40
+ assert matched.loc["P", "items.distinct_values"] == 2
+ assert matched.loc["P", "items.median_value"] == 20.0
+ assert matched.loc["Q", "items.total"] == 100
+ assert matched.loc["Q", "items.distinct_values"] == 1
+ assert matched.loc["Q", "items.median_value"] == 100.0
+ unmatched = mixed[mixed["items.kind"].isna()].iloc[0]
+ assert unmatched["orders.cohort"] == "B"
+ assert unmatched["items.row_count"] == 0
+ assert unmatched["items.distinct_values"] == 0
+
+
+def test_calculated_measure_dependencies_keep_root_scope(con):
+ left = con.create_table("calc_left", pd.DataFrame({"id": [1, 2]}))
+ right = con.create_table("calc_right", pd.DataFrame({"id": [1, 2]}))
+ left_model = to_semantic_table(left, "left").with_measures(
+ row_count=lambda t: t.count(), doubled=lambda t: t.row_count * 2
+ )
+ right_model = to_semantic_table(right, "right").with_measures(
+ row_count=lambda t: t.count(), doubled=lambda t: t.row_count * 2
+ )
+
+ result = left_model.join_one(right_model, on="id").aggregate(
+ "left.doubled", "right.doubled"
+ ).execute()
+
+ assert result.iloc[0].to_dict() == {"left.doubled": 4, "right.doubled": 4}
+
+
+def test_join_schema_matches_executable_collision_aliases(con):
+ left = con.create_table(
+ "schema_left", pd.DataFrame({"id": [1], "value": [1]})
+ )
+ right = con.create_table(
+ "schema_right", pd.DataFrame({"id": [1], "value": ["x"]})
+ )
+ joined = to_semantic_table(left, "left").join_one(
+ to_semantic_table(right, "right"), on="id"
+ )
+
+ assert tuple(joined.columns) == tuple(joined.table.columns)
+ assert tuple(joined.columns) == ("id", "value", "id_right", "value_right")
+ assert str(joined.schema["value"]) == "int64"
+ assert str(joined.schema["value_right"]) == "string"
+
+
+def test_preexisting_right_suffix_is_preserved_and_right_fields_stay_bound(con):
+ left = con.create_table(
+ "reserved_suffix_left",
+ pd.DataFrame(
+ {
+ "id": [1, 2],
+ "value": [1, 2],
+ "value_right": [900, 901],
+ }
+ ),
+ )
+ right = con.create_table(
+ "reserved_suffix_right",
+ pd.DataFrame({"id": [1, 2], "value": [10, 20]}),
+ )
+ left_model = to_semantic_table(left, "left")
+ right_model = (
+ to_semantic_table(right, "right")
+ .with_dimensions(doubled=lambda t: t.value * 2)
+ .with_measures(total=lambda t: t.value.sum())
+ )
+ joined = left_model.join_one(right_model, on="id")
+
+ assert tuple(joined.columns) == (
+ "id",
+ "value",
+ "value_right",
+ "id_right",
+ "value_right2",
+ )
+ raw = joined.execute().sort_values("id").reset_index(drop=True)
+ assert raw["value_right"].tolist() == [900, 901]
+ assert raw["value_right2"].tolist() == [10, 20]
+ assert str(joined.schema["value_right"]) == "int64"
+ assert str(joined.schema["value_right2"]) == "int64"
+
+ result = (
+ joined.group_by("right.doubled")
+ .aggregate("right.total")
+ .order_by("right.doubled")
+ .execute()
+ )
+ assert result["right.doubled"].tolist() == [20, 40]
+ assert result["right.total"].tolist() == [10, 20]
+
+
+def test_dynamic_aliases_are_measure_selection_invariant_across_three_legs(con):
+ root = con.create_table(
+ "dynamic_alias_root",
+ pd.DataFrame(
+ {"id": [1, 2], "value": [1, 2], "value_right": [900, 901]}
+ ),
+ )
+ middle = con.create_table(
+ "dynamic_alias_middle",
+ pd.DataFrame({"id": [1, 2], "value": [100, 200]}),
+ )
+ later = con.create_table(
+ "dynamic_alias_later",
+ pd.DataFrame({"id": [1, 2], "value": [10, 20]}),
+ )
+ middle_model = to_semantic_table(middle, "middle").with_measures(
+ total=lambda t: t.value.sum()
+ )
+ later_model = (
+ to_semantic_table(later, "later")
+ .with_dimensions(doubled=lambda t: t.value * 2)
+ .with_measures(total=lambda t: t.value.sum())
+ )
+ joined = (
+ to_semantic_table(root, "root")
+ .join_one(middle_model, on="id")
+ .join_one(later_model, on="id")
+ )
+
+ later_only = (
+ joined.group_by("later.doubled")
+ .aggregate("later.total")
+ .order_by("later.doubled")
+ .execute()
+ )
+ with_middle = (
+ joined.group_by("later.doubled")
+ .aggregate("later.total", "middle.total")
+ .order_by("later.doubled")
+ .execute()
+ )
+
+ assert later_only["later.doubled"].tolist() == [20, 40]
+ assert later_only["later.total"].tolist() == [10, 20]
+ assert with_middle["later.doubled"].tolist() == [20, 40]
+ assert with_middle["later.total"].tolist() == [10, 20]
+ assert with_middle["middle.total"].tolist() == [100, 200]
+
+
+def test_user_column_with_internal_join_prefix_does_not_change_predicate(con):
+ left = con.create_table(
+ "temporary_alias_left",
+ pd.DataFrame({"id": [1], "__bsl_jk_id": [99]}),
+ )
+ right = con.create_table(
+ "temporary_alias_right",
+ pd.DataFrame({"id": [1], "value": [2]}),
+ )
+ joined = to_semantic_table(left, "left").join_one(
+ to_semantic_table(right, "right"), on="id"
+ )
+
+ result = joined.execute()
+ assert result.iloc[0].to_dict() == {
+ "id": 1,
+ "__bsl_jk_id": 99,
+ "id_right": 1,
+ "value": 2,
+ }
+
+
+def test_cross_join_uses_the_same_collision_safe_aliases(con):
+ left = con.create_table(
+ "cross_alias_left",
+ pd.DataFrame({"value": [1], "value_right": [9]}),
+ )
+ right = con.create_table(
+ "cross_alias_right",
+ pd.DataFrame({"value": [2]}),
+ )
+ joined = to_semantic_table(left, "left").join_cross(
+ to_semantic_table(right, "right")
+ )
+
+ assert tuple(joined.columns) == ("value", "value_right", "value_right2")
+ assert joined.execute().iloc[0].to_dict() == {
+ "value": 1,
+ "value_right": 9,
+ "value_right2": 2,
+ }
+
+
+def test_same_entity_names_not_joined_on_entity_keys_upgrade_grain(con):
+ orders = con.create_table(
+ "entity_orders", pd.DataFrame({"id": [1, 2], "customer_id": [10, 10]})
+ )
+ customers = con.create_table(
+ "entity_customers", pd.DataFrame({"id": [100], "customer_id": [10]})
+ )
+ orders_model = (
+ to_semantic_table(orders, "orders")
+ .with_dimensions(id=Dimension(expr=lambda t: t.id, is_entity=True))
+ .with_measures(row_count=lambda t: t.count())
+ )
+ customers_model = (
+ to_semantic_table(customers, "customers")
+ .with_dimensions(id=Dimension(expr=lambda t: t.id, is_entity=True))
+ .with_measures(row_count=lambda t: t.count())
+ )
+
+ with pytest.warns(UserWarning, match="Grain mismatch"):
+ joined = orders_model.join_one(customers_model, on="customer_id")
+
+ assert joined.op().cardinality == "many"
+
+
+def test_json_definition_includes_calculated_measure_metadata(con):
+ table = con.create_table("json_calc", pd.DataFrame({"value": [1, 2]}))
+ model = to_semantic_table(table, "model").with_measures(
+ total={"expr": lambda t: t.value.sum(), "description": "Base"},
+ doubled={"expr": lambda t: t.total * 2, "description": "Calculated"},
+ )
+
+ definition = model.json_definition
+ assert definition["measures"]["doubled"]["description"] == "Calculated"
+ assert definition["calculated_measures"]["doubled"]["description"] == "Calculated"
+
+
+def _cross_table_filter_models(con):
+ left = con.create_table(
+ "cross_filter_root",
+ pd.DataFrame(
+ {
+ # Multiple source rows deliberately share the join key. A
+ # join-key-only bridge cannot preserve a row-level predicate.
+ "id": [1, 1],
+ "status": ["A", "B"],
+ "value": [10, 20],
+ }
+ ),
+ )
+ right = con.create_table(
+ "cross_filter_many",
+ pd.DataFrame({"id": [1], "flag": [False]}),
+ )
+ left_model = (
+ to_semantic_table(left, "left")
+ .with_dimensions(status=lambda t: t.status)
+ .with_measures(total=lambda t: t.value.sum())
+ )
+ right_model = to_semantic_table(right, "right").with_dimensions(
+ flag=lambda t: t.flag
+ )
+ return left_model.join_many(right_model, on="id")
+
+
+def test_cross_table_or_with_root_only_measure_fails_closed(con):
+ joined = _cross_table_filter_models(con)
+ query = joined.filter(
+ lambda t: (t["left.status"] == "A") | t["right.flag"]
+ ).aggregate("left.total")
+
+ with pytest.raises(ValueError, match="row-precisely"):
+ query.execute()
+
+
+def test_cross_table_and_with_root_only_measure_remains_row_precise(con):
+ joined = _cross_table_filter_models(con)
+ result = joined.filter(
+ lambda t: (t["left.status"] == "A") & ~t["right.flag"]
+ ).aggregate("left.total").execute()
+
+ assert result["left.total"].iloc[0] == 10
diff --git a/src/boring_semantic_layer/tests/test_nested_compile_soundness.py b/src/boring_semantic_layer/tests/test_nested_compile_soundness.py
new file mode 100644
index 00000000..1317562d
--- /dev/null
+++ b/src/boring_semantic_layer/tests/test_nested_compile_soundness.py
@@ -0,0 +1,431 @@
+"""Soundness regressions for automatic nested-array aggregation."""
+
+import ibis
+import pandas as pd
+import pytest
+
+from boring_semantic_layer import to_semantic_table
+from boring_semantic_layer.nested_compile import unnest_nested_arrays
+
+
+def test_nested_array_path_preserves_declared_traversal_order():
+ """Array names must not be alphabetized before they are unnested."""
+ con = ibis.duckdb.connect(":memory:")
+ table = con.create_table(
+ "nested_path_order",
+ pd.DataFrame(
+ {
+ "session_id": [1, 2],
+ # The child name sorts before the parent name. Sorting the
+ # path would try to unnest ``aproducts`` before ``zhits``.
+ "zhits": [
+ [{"aproducts": [1, 2]}, {"aproducts": [3]}],
+ [{"aproducts": [4, 5]}],
+ ],
+ }
+ ),
+ )
+ model = (
+ to_semantic_table(table, name="sessions")
+ .with_dimensions(session_id=lambda t: t.session_id)
+ .with_measures(product_count=lambda t: t.zhits.aproducts.count())
+ )
+
+ result = (
+ model.group_by("session_id").aggregate("product_count").execute().set_index("session_id")
+ )
+
+ assert result.loc[1, "product_count"] == 3
+ assert result.loc[2, "product_count"] == 2
+
+
+def test_nested_array_path_ignores_colliding_top_level_child_name():
+ """A nested child must not resolve to an unrelated top-level array."""
+ con = ibis.duckdb.connect(":memory:")
+ table = con.create_table(
+ "nested_path_collision",
+ pd.DataFrame(
+ {
+ "session_id": [1, 2],
+ # This sibling deliberately has the same name as the child of
+ # ``events``. Its lengths differ from the nested ground truth.
+ "products": [[900], [800, 801, 802]],
+ "events": [
+ [{"products": [1, 2]}, {"products": [3]}],
+ [{"products": [4, 5]}],
+ ],
+ }
+ ),
+ )
+ model = (
+ to_semantic_table(table, name="sessions")
+ .with_dimensions(session_id=lambda t: t.session_id)
+ .with_measures(product_count=lambda t: t.events.products.count())
+ )
+
+ result = (
+ model.group_by("session_id")
+ .aggregate("product_count")
+ .execute()
+ .sort_values("session_id")
+ )
+
+ assert result["product_count"].tolist() == [3, 2]
+
+
+@pytest.mark.parametrize(
+ ("measure_fn", "requested_path", "message"),
+ [
+ (
+ lambda t: t.events.missing.count(),
+ "events.missing",
+ "does not exist",
+ ),
+ (
+ lambda t: t.events.products.missing.count(),
+ "events.products.missing",
+ "does not exist",
+ ),
+ (
+ lambda t: t.events.label.count(),
+ "events.label",
+ "is not an array",
+ ),
+ ],
+ ids=["missing-child", "missing-deep-child", "non-array-child"],
+)
+def test_invalid_nested_array_child_fails_closed(
+ measure_fn, requested_path, message
+):
+ con = ibis.duckdb.connect(":memory:")
+ table = con.create_table(
+ "nested_invalid_child",
+ pd.DataFrame(
+ {
+ "session_id": [1, 2],
+ "events": [
+ [
+ {"products": [1, 2], "label": "a"},
+ {"products": [3], "label": "b"},
+ ],
+ [{"products": [4], "label": "c"}],
+ ],
+ }
+ ),
+ )
+ model = (
+ to_semantic_table(table, name="sessions")
+ .with_dimensions(session_id=lambda t: t.session_id)
+ .with_measures(invalid_count=measure_fn)
+ )
+
+ with pytest.raises(ValueError) as exc_info:
+ model.group_by("session_id").aggregate("invalid_count").execute()
+
+ error = str(exc_info.value)
+ assert requested_path in error
+ assert message in error
+
+
+@pytest.mark.parametrize(
+ ("array_path", "message"),
+ [(("missing",), "does not exist"), (("scalar",), "is not an array")],
+)
+def test_invalid_nested_array_root_fails_closed(array_path, message):
+ table = ibis.memtable({"scalar": [1, 2]})
+
+ with pytest.raises(ValueError) as exc_info:
+ unnest_nested_arrays(table, array_path)
+
+ error = str(exc_info.value)
+ assert ".".join(array_path) in error
+ assert message in error
+
+
+def test_nested_only_query_retains_groups_with_empty_or_null_arrays():
+ con = ibis.duckdb.connect(":memory:")
+ table = con.create_table(
+ "nested_empty_groups",
+ pd.DataFrame(
+ {
+ "category": ["filled", "empty", "null"],
+ "hits": [[1, 2, 2], [], None],
+ }
+ ),
+ )
+ model = (
+ to_semantic_table(table, name="sessions")
+ .with_dimensions(category=lambda t: t.category)
+ .with_measures(
+ hit_count=lambda t: t.hits.count(),
+ unique_hits=lambda t: t.hits.nunique(),
+ total_hits=lambda t: t.hits.sum(),
+ )
+ )
+
+ result = (
+ model.group_by("category")
+ .aggregate("hit_count", "unique_hits", "total_hits")
+ .execute()
+ .set_index("category")
+ )
+
+ assert set(result.index) == {"filled", "empty", "null"}
+ assert result.loc["filled", "hit_count"] == 3
+ assert result.loc["filled", "unique_hits"] == 2
+ assert result.loc["filled", "total_hits"] == 5
+ for category in ("empty", "null"):
+ assert result.loc[category, "hit_count"] == 0
+ assert result.loc[category, "unique_hits"] == 0
+ assert pd.isna(result.loc[category, "total_hits"])
+
+
+def test_empty_nested_groups_are_retained_when_regular_measures_are_selected():
+ con = ibis.duckdb.connect(":memory:")
+ table = con.create_table(
+ "nested_empty_groups_mixed",
+ pd.DataFrame(
+ {
+ "category": ["filled", "empty"],
+ "hits": [[1, 2], []],
+ }
+ ),
+ )
+ model = (
+ to_semantic_table(table, name="sessions")
+ .with_dimensions(category=lambda t: t.category)
+ .with_measures(
+ session_count=lambda t: t.count(),
+ hit_count=lambda t: t.hits.count(),
+ )
+ )
+
+ result = (
+ model.group_by("category")
+ .aggregate("session_count", "hit_count")
+ .execute()
+ .set_index("category")
+ )
+
+ assert result.loc["filled", "session_count"] == 1
+ assert result.loc["filled", "hit_count"] == 2
+ assert result.loc["empty", "session_count"] == 1
+ assert result.loc["empty", "hit_count"] == 0
+
+
+@pytest.fixture
+def joined_nested_model():
+ """A nested source plus a sibling many-side arm that would fan it out."""
+ con = ibis.duckdb.connect(":memory:")
+ users_table = con.create_table(
+ "joined_nested_users",
+ pd.DataFrame(
+ {
+ "user_id": [1, 2, 3, 4, 5],
+ "segment": ["A", "A", "B", "B", "C"],
+ }
+ ),
+ )
+ sessions_table = con.create_table(
+ "joined_nested_sessions",
+ pd.DataFrame(
+ {
+ "session_id": [10, 11, 12, 13, 14],
+ "user_id": [1, 1, 2, 3, 4],
+ "kind": ["x", "y", None, "x", "x"],
+ "hits": [[1, 2, 2], [3], [], [10, 10], None],
+ }
+ ),
+ )
+ purchases_table = con.create_table(
+ "joined_nested_purchases",
+ pd.DataFrame(
+ {
+ "purchase_id": list(range(20, 28)),
+ "user_id": [1, 1, 1, 2, 2, 3, 4, 4],
+ "session_id": [10, 10, 11, 12, 12, 13, 14, 14],
+ "spend": [10.0, 20.0, 5.0, 7.0, 8.0, 11.0, 13.0, 17.0],
+ }
+ ),
+ )
+
+ users = to_semantic_table(users_table, "users").with_dimensions(
+ segment=lambda t: t.segment
+ )
+ sessions = (
+ to_semantic_table(sessions_table, "sessions")
+ .with_dimensions(kind=lambda t: t.kind)
+ .with_measures(
+ hit_count=lambda t: t.hits.count(),
+ hit_sum=lambda t: t.hits.sum(),
+ hit_mean=lambda t: t.hits.mean(),
+ unique_hits=lambda t: t.hits.nunique(),
+ )
+ .with_measures(
+ double_hits=lambda t: t.hit_count * 2,
+ hit_share=lambda t: t.hit_count / t.all(t.hit_count) * 100,
+ )
+ )
+ purchases = to_semantic_table(
+ purchases_table, "purchases"
+ ).with_measures(spend=lambda t: t.spend.sum())
+ joined = users.join_many(sessions, on="user_id").join_many(
+ purchases, on="user_id"
+ )
+ return joined, sessions, purchases
+
+
+def test_joined_nested_measures_compile_at_exact_cross_source_grain(
+ joined_nested_model,
+):
+ joined, _sessions, _purchases = joined_nested_model
+
+ result = (
+ joined.group_by("users.segment")
+ .aggregate(
+ "sessions.hit_count",
+ "sessions.hit_sum",
+ "sessions.hit_mean",
+ "sessions.unique_hits",
+ "sessions.double_hits",
+ "sessions.hit_share",
+ "purchases.spend",
+ )
+ .execute()
+ .set_index("users.segment")
+ )
+
+ assert result.loc["A", "sessions.hit_count"] == 4
+ assert result.loc["A", "sessions.hit_sum"] == 8
+ assert result.loc["A", "sessions.hit_mean"] == pytest.approx(2.0)
+ assert result.loc["A", "sessions.unique_hits"] == 3
+ assert result.loc["A", "sessions.double_hits"] == 8
+ assert result.loc["A", "sessions.hit_share"] == pytest.approx(200 / 3)
+ assert result.loc["A", "purchases.spend"] == pytest.approx(50.0)
+
+ assert result.loc["B", "sessions.hit_count"] == 2
+ assert result.loc["B", "sessions.hit_sum"] == 20
+ assert result.loc["B", "sessions.hit_mean"] == pytest.approx(10.0)
+ assert result.loc["B", "sessions.unique_hits"] == 1
+ assert result.loc["B", "sessions.hit_share"] == pytest.approx(100 / 3)
+ assert result.loc["B", "purchases.spend"] == pytest.approx(41.0)
+
+ # User 5 has no session or purchase. The joined group domain is retained
+ # with each nested reduction's real empty-set identity.
+ assert result.loc["C", "sessions.hit_count"] == 0
+ assert result.loc["C", "sessions.unique_hits"] == 0
+ assert pd.isna(result.loc["C", "sessions.hit_sum"])
+ assert pd.isna(result.loc["C", "sessions.hit_mean"])
+ assert pd.isna(result.loc["C", "purchases.spend"])
+
+
+def test_joined_nested_scalar_and_nonroot_local_group(joined_nested_model):
+ joined, _sessions, _purchases = joined_nested_model
+
+ scalar = joined.aggregate(
+ "sessions.hit_count",
+ "sessions.hit_sum",
+ "sessions.hit_mean",
+ "sessions.unique_hits",
+ "purchases.spend",
+ ).execute().iloc[0]
+ assert scalar["sessions.hit_count"] == 6
+ assert scalar["sessions.hit_sum"] == 28
+ assert scalar["sessions.hit_mean"] == pytest.approx(28 / 6)
+ assert scalar["sessions.unique_hits"] == 4
+ assert scalar["purchases.spend"] == pytest.approx(91.0)
+
+ by_kind = (
+ joined.group_by("sessions.kind")
+ .aggregate(
+ "sessions.hit_count",
+ "sessions.hit_sum",
+ "sessions.unique_hits",
+ )
+ .execute()
+ )
+ x_row = by_kind[by_kind["sessions.kind"] == "x"].iloc[0]
+ assert x_row["sessions.hit_count"] == 5
+ assert x_row["sessions.hit_sum"] == 25
+ assert x_row["sessions.unique_hits"] == 3
+ null_row = by_kind[by_kind["sessions.kind"].isna()].iloc[0]
+ assert null_row["sessions.hit_count"] == 0
+ assert null_row["sessions.unique_hits"] == 0
+ assert pd.isna(null_row["sessions.hit_sum"])
+
+ # A join key can participate in several source-local dimension values.
+ # The exact bridge must match both the key and local group value or user
+ # 1's x/y arrays leak into each other's groups.
+ mixed = (
+ joined.group_by("users.segment", "sessions.kind")
+ .aggregate("sessions.hit_count", "sessions.unique_hits")
+ .execute()
+ )
+ a_x = mixed[
+ (mixed["users.segment"] == "A") & (mixed["sessions.kind"] == "x")
+ ].iloc[0]
+ a_y = mixed[
+ (mixed["users.segment"] == "A") & (mixed["sessions.kind"] == "y")
+ ].iloc[0]
+ assert a_x["sessions.hit_count"] == 3
+ assert a_x["sessions.unique_hits"] == 2
+ assert a_y["sessions.hit_count"] == 1
+ assert a_y["sessions.unique_hits"] == 1
+
+
+def test_root_nested_measures_are_not_inflated_by_join_many(joined_nested_model):
+ _joined, sessions, purchases = joined_nested_model
+ model = sessions.join_many(purchases, on="session_id")
+
+ result = (
+ model.group_by("sessions.kind")
+ .aggregate(
+ "sessions.hit_count",
+ "sessions.hit_mean",
+ "sessions.unique_hits",
+ "purchases.spend",
+ )
+ .execute()
+ )
+
+ x_row = result[result["sessions.kind"] == "x"].iloc[0]
+ assert x_row["sessions.hit_count"] == 5
+ assert x_row["sessions.hit_mean"] == pytest.approx(5.0)
+ assert x_row["sessions.unique_hits"] == 3
+ assert x_row["purchases.spend"] == pytest.approx(71.0)
+
+ null_row = result[result["sessions.kind"].isna()].iloc[0]
+ assert null_row["sessions.hit_count"] == 0
+ assert null_row["sessions.unique_hits"] == 0
+ assert pd.isna(null_row["sessions.hit_mean"])
+ assert null_row["purchases.spend"] == pytest.approx(15.0)
+
+
+def test_joined_nested_measure_excludes_orphan_source_rows():
+ con = ibis.duckdb.connect(":memory:")
+ users_table = con.create_table(
+ "nested_orphan_users", pd.DataFrame({"user_id": [1], "segment": ["kept"]})
+ )
+ sessions_table = con.create_table(
+ "nested_orphan_sessions",
+ pd.DataFrame(
+ {
+ "user_id": [1, 999, None],
+ "hits": [[1, 2], [100, 200, 300], [400, 500]],
+ }
+ ),
+ )
+ users = to_semantic_table(users_table, "users").with_dimensions(
+ segment=lambda t: t.segment
+ )
+ sessions = to_semantic_table(sessions_table, "sessions").with_measures(
+ hit_count=lambda t: t.hits.count(),
+ hit_sum=lambda t: t.hits.sum(),
+ )
+
+ result = users.join_many(sessions, on="user_id").aggregate(
+ "sessions.hit_count", "sessions.hit_sum"
+ ).execute().iloc[0]
+
+ assert result["sessions.hit_count"] == 2
+ assert result["sessions.hit_sum"] == 3
diff --git a/src/boring_semantic_layer/tests/test_query_soundness_repairs.py b/src/boring_semantic_layer/tests/test_query_soundness_repairs.py
new file mode 100644
index 00000000..d417f8f8
--- /dev/null
+++ b/src/boring_semantic_layer/tests/test_query_soundness_repairs.py
@@ -0,0 +1,288 @@
+"""Regression coverage for query-layer semantic soundness repairs."""
+
+from __future__ import annotations
+
+import ibis
+import pandas as pd
+import pytest
+
+from boring_semantic_layer import to_semantic_table
+from boring_semantic_layer.query import Filter
+
+
+@pytest.fixture
+def con():
+ return ibis.duckdb.connect()
+
+
+def test_qualified_dict_filter_targets_the_requested_join_source(con):
+ """A qualified JSON filter must not fall back to a colliding left column."""
+ orders_tbl = con.create_table(
+ "query_soundness_orders",
+ pd.DataFrame(
+ {
+ "order_id": [1, 2],
+ "status": ["bad", "open"],
+ }
+ ),
+ )
+ items_tbl = con.create_table(
+ "query_soundness_items",
+ pd.DataFrame(
+ {
+ "item_id": [10, 11, 12],
+ "order_id": [1, 1, 2],
+ "status": ["ok", "ok", "bad"],
+ }
+ ),
+ )
+
+ orders = to_semantic_table(orders_tbl, "orders").with_dimensions(
+ order_id=lambda t: t.order_id,
+ status=lambda t: t.status,
+ )
+ items = (
+ to_semantic_table(items_tbl, "items")
+ .with_dimensions(
+ order_id=lambda t: t.order_id,
+ status=lambda t: t.status,
+ )
+ .with_measures(item_count=lambda t: t.item_id.count())
+ )
+
+ result = (
+ orders.join_many(items, on="order_id")
+ .query(
+ measures=["items.item_count"],
+ filters=[{"field": "items.status", "operator": "=", "value": "bad"}],
+ )
+ .execute()
+ )
+
+ assert result["items.item_count"].iloc[0] == 1
+
+
+def test_qualified_dict_filter_targets_undeclared_raw_right_column(con):
+ """A valid qualified raw field must not fall back to the left collision."""
+ orders_tbl = con.create_table(
+ "query_soundness_raw_orders",
+ pd.DataFrame(
+ {
+ "order_id": [1, 2],
+ "status": ["bad", "open"],
+ }
+ ),
+ )
+ items_tbl = con.create_table(
+ "query_soundness_raw_items",
+ pd.DataFrame(
+ {
+ "order_id": [1, 1, 2],
+ "status": ["ok", "ok", "bad"],
+ "amount": [1, 2, 30],
+ }
+ ),
+ )
+ orders = to_semantic_table(orders_tbl, "orders").with_dimensions(
+ status=lambda t: t.status
+ )
+ # ``status`` intentionally remains an undeclared raw column on this side.
+ items = to_semantic_table(items_tbl, "items").with_measures(
+ total=lambda t: t.amount.sum()
+ )
+
+ result = orders.join_many(items, on="order_id").query(
+ measures=["items.total"],
+ filters=[{"field": "items.status", "operator": "=", "value": "bad"}],
+ ).execute()
+
+ assert result["items.total"].iloc[0] == 30
+
+
+@pytest.mark.parametrize("overlay", ["dimensions", "measures"])
+def test_qualified_dict_filter_metadata_survives_metadata_overlay(con, overlay):
+ """Rebinding filtered metadata must retain JSON's exact field ownership."""
+ orders_tbl = con.create_table(
+ f"query_soundness_overlay_orders_{overlay}",
+ pd.DataFrame(
+ {
+ "order_id": [1, 2],
+ "status": ["bad", "open"],
+ }
+ ),
+ )
+ items_tbl = con.create_table(
+ f"query_soundness_overlay_items_{overlay}",
+ pd.DataFrame(
+ {
+ "order_id": [1, 1, 2],
+ "status": ["ok", "ok", "bad"],
+ "amount": [1, 2, 30],
+ }
+ ),
+ )
+ orders = to_semantic_table(orders_tbl, "orders")
+ items = to_semantic_table(items_tbl, "items").with_measures(
+ total=lambda t: t.amount.sum()
+ )
+ predicate = Filter(
+ filter={"field": "items.status", "operator": "=", "value": "bad"}
+ ).to_callable()
+ filtered = orders.join_many(items, on="order_id").filter(predicate)
+
+ if overlay == "dimensions":
+ rebound = filtered.with_dimensions(bucket=lambda t: t.status)
+ else:
+ rebound = filtered.with_measures(joined_total=lambda t: t.amount.sum())
+
+ result = rebound.aggregate("items.total").execute()
+
+ assert result["items.total"].iloc[0] == 30
+
+
+def test_qualified_dict_filter_rejects_unknown_join_prefix(con):
+ orders_tbl = con.create_table(
+ "query_soundness_bad_prefix_orders",
+ pd.DataFrame(
+ {
+ "order_id": [1, 2],
+ "status": ["bad", "open"],
+ }
+ ),
+ )
+ items_tbl = con.create_table(
+ "query_soundness_bad_prefix_items",
+ pd.DataFrame(
+ {
+ "order_id": [1, 2],
+ "status": ["ok", "bad"],
+ "amount": [10, 20],
+ }
+ ),
+ )
+ orders = to_semantic_table(orders_tbl, "orders")
+ items = to_semantic_table(items_tbl, "items").with_measures(
+ total=lambda t: t.amount.sum()
+ )
+
+ with pytest.raises(KeyError, match="Unknown semantic model prefix 'bogus'"):
+ orders.join_many(items, on="order_id").query(
+ measures=["items.total"],
+ filters=[
+ {"field": "bogus.status", "operator": "=", "value": "bad"}
+ ],
+ ).execute()
+
+
+def test_standalone_model_prefixed_dict_filter_remains_supported(con):
+ """Exact qualified resolution retains the standalone prefix convenience."""
+ tbl = con.create_table(
+ "query_soundness_standalone",
+ pd.DataFrame({"status": ["open", "closed"], "amount": [10, 20]}),
+ )
+ model = (
+ to_semantic_table(tbl, "orders")
+ .with_dimensions(status=lambda t: t.status)
+ .with_measures(total=lambda t: t.amount.sum())
+ )
+
+ result = model.query(
+ measures=["total"],
+ filters=[{"field": "orders.status", "operator": "=", "value": "open"}],
+ ).execute()
+
+ assert result["total"].iloc[0] == 10
+
+
+def test_standalone_model_prefixed_raw_dict_filter_remains_supported(con):
+ """The standalone-prefix fallback also supports undeclared raw columns."""
+ tbl = con.create_table(
+ "query_soundness_standalone_raw",
+ pd.DataFrame({"status": ["open", "closed"], "amount": [10, 20]}),
+ )
+ model = to_semantic_table(tbl, "orders").with_measures(
+ total=lambda t: t.amount.sum()
+ )
+
+ result = model.query(
+ measures=["total"],
+ filters=[{"field": "orders.status", "operator": "=", "value": "open"}],
+ ).execute()
+
+ assert result["total"].iloc[0] == 10
+
+
+def test_compare_periods_matches_null_dimension_members(con):
+ """NULL is one semantic group and must match itself across periods."""
+ tbl = con.create_table(
+ "query_soundness_periods",
+ pd.DataFrame(
+ {
+ "occurred_at": pd.to_datetime(["2024-01-05", "2024-02-05", "2024-03-05"]),
+ # The out-of-range anchor gives DuckDB a concrete string type.
+ "category": [None, None, "anchor"],
+ "amount": [10, 20, 0],
+ }
+ ),
+ )
+ model = (
+ to_semantic_table(tbl, "events")
+ .with_dimensions(
+ occurred_at={
+ "expr": lambda t: t.occurred_at,
+ "is_time_dimension": True,
+ "smallest_time_grain": "day",
+ },
+ category=lambda t: t.category,
+ )
+ .with_measures(total=lambda t: t.amount.sum())
+ )
+
+ result = model.compare_periods(
+ dimensions=["category"],
+ measures=["total"],
+ current_time_range={"start": "2024-02-01", "end": "2024-02-29"},
+ previous_time_range={"start": "2024-01-01", "end": "2024-01-31"},
+ ).execute()
+
+ assert len(result) == 1
+ row = result.iloc[0]
+ assert pd.isna(row["category"])
+ assert row["total_current"] == 20
+ assert row["total_previous"] == 10
+ assert row["total_delta"] == 10
+ assert row["total_pct_change"] == pytest.approx(1.0)
+
+
+@pytest.mark.parametrize(
+ ("argument", "value", "message"),
+ [
+ (
+ "having",
+ [{"field": "total", "operator": ">", "value": 0}],
+ "post-aggregation filter",
+ ),
+ ("order_by", [("total", "desc")], "order_by"),
+ ],
+)
+def test_unselected_post_aggregation_fields_fail_early(con, argument, value, message):
+ tbl = con.create_table(
+ f"query_soundness_unselected_{argument}",
+ pd.DataFrame({"group": ["a", "b"], "amount": [10, 20]}),
+ )
+ model = (
+ to_semantic_table(tbl, "events")
+ .with_dimensions(group=lambda t: t.group)
+ .with_measures(
+ row_count=lambda t: t.count(),
+ total=lambda t: t.amount.sum(),
+ )
+ )
+
+ kwargs = {
+ "dimensions": ["group"],
+ "measures": ["row_count"],
+ argument: value,
+ }
+ with pytest.raises(ValueError, match=message):
+ model.query(**kwargs)
diff --git a/src/boring_semantic_layer/tests/test_soundness_expr_lineage.py b/src/boring_semantic_layer/tests/test_soundness_expr_lineage.py
new file mode 100644
index 00000000..edae7393
--- /dev/null
+++ b/src/boring_semantic_layer/tests/test_soundness_expr_lineage.py
@@ -0,0 +1,296 @@
+"""Regression tests for expression-layer join, filter, and grain lineage."""
+
+import warnings
+
+import ibis
+import pandas as pd
+import pytest
+
+from boring_semantic_layer import Dimension, to_semantic_table
+
+
+@pytest.fixture
+def con():
+ return ibis.duckdb.connect(":memory:")
+
+
+@pytest.fixture
+def fanout_models(con):
+ customers_tbl = con.create_table(
+ "lineage_customers",
+ pd.DataFrame(
+ {
+ "customer_id": [1, 2],
+ "region": ["east", "west"],
+ }
+ ),
+ )
+ orders_tbl = con.create_table(
+ "lineage_orders",
+ pd.DataFrame(
+ {
+ "order_id": [10, 11, 12],
+ "customer_id": [1, 1, 2],
+ "amount": [5, 7, 3],
+ }
+ ),
+ )
+
+ customers = (
+ to_semantic_table(customers_tbl, name="customers")
+ .with_dimensions(
+ customer_id=lambda t: t.customer_id,
+ region=lambda t: t.region,
+ )
+ .with_measures(customer_count=lambda t: t.count())
+ )
+ orders = (
+ to_semantic_table(orders_tbl, name="orders")
+ .with_dimensions(
+ customer_id=lambda t: t.customer_id,
+ order_id=lambda t: t.order_id,
+ amount=lambda t: t.amount,
+ )
+ .with_measures(order_total=lambda t: t.amount.sum())
+ )
+ return customers.join_many(orders, on="customer_id")
+
+
+def _filtered_fanout_result(model):
+ return (
+ model.group_by("customers.customer_id")
+ .aggregate("customers.customer_count", "orders.order_total")
+ .execute()
+ .sort_values("customers.customer_id")
+ .reset_index(drop=True)
+ )
+
+
+@pytest.mark.parametrize("metadata_method", ["with_dimensions", "mutate"])
+def test_filter_then_dimension_metadata_keeps_filter_and_fanout_protection(
+ fanout_models, metadata_method
+):
+ filtered = fanout_models.filter(lambda t: t["orders.amount"] > 5)
+ transformed = getattr(filtered, metadata_method)(region_label=lambda t: t.region.upper())
+
+ result = _filtered_fanout_result(transformed)
+
+ assert result["customers.customer_id"].tolist() == [1]
+ assert result["customers.customer_count"].tolist() == [1]
+ assert result["orders.order_total"].tolist() == [7]
+
+
+def test_filter_then_with_measures_keeps_filter_and_fanout_protection(fanout_models):
+ transformed = fanout_models.filter(lambda t: t["orders.amount"] > 5).with_measures(
+ filtered_row_count=lambda t: t.count()
+ )
+
+ result = (
+ transformed.group_by("customers.customer_id")
+ .aggregate(
+ "customers.customer_count",
+ "orders.order_total",
+ "filtered_row_count",
+ )
+ .execute()
+ )
+
+ assert result["customers.customer_id"].tolist() == [1]
+ assert result["customers.customer_count"].tolist() == [1]
+ assert result["orders.order_total"].tolist() == [7]
+ assert result["filtered_row_count"].tolist() == [1]
+
+
+def test_filter_metadata_update_retains_query_surface(fanout_models):
+ transformed = fanout_models.filter(
+ lambda t: t["orders.amount"] > 5
+ ).with_dimensions(region_label=lambda t: t.region.upper())
+
+ result = transformed.query(
+ dimensions=["customers.customer_id"],
+ measures=["customers.customer_count", "orders.order_total"],
+ ).execute()
+
+ assert result["customers.customer_id"].tolist() == [1]
+ assert result["customers.customer_count"].tolist() == [1]
+ assert result["orders.order_total"].tolist() == [7]
+
+
+def test_multiple_filters_survive_metadata_update_on_join(fanout_models):
+ transformed = (
+ fanout_models.filter(lambda t: t["orders.amount"] >= 5)
+ .filter(lambda t: t["orders.amount"] < 7)
+ .with_dimensions(region_label=lambda t: t.region.upper())
+ )
+
+ result = _filtered_fanout_result(transformed)
+
+ assert result["customers.customer_id"].tolist() == [1]
+ assert result["customers.customer_count"].tolist() == [1]
+ assert result["orders.order_total"].tolist() == [5]
+
+
+def _grain_models(con):
+ monthly_tbl = con.create_table(
+ "lineage_monthly",
+ pd.DataFrame({"year": [2024], "month": [1], "revenue": [100]}),
+ )
+ daily_tbl = con.create_table(
+ "lineage_daily",
+ pd.DataFrame(
+ {
+ "year": [2024, 2024],
+ "month": [1, 1],
+ "day": [1, 2],
+ "hours": [8, 7],
+ }
+ ),
+ )
+ calendar_tbl = con.create_table(
+ "lineage_calendar",
+ pd.DataFrame({"year": [2024], "month": [1], "label": ["Jan"]}),
+ )
+
+ monthly = (
+ to_semantic_table(monthly_tbl, name="monthly")
+ .with_dimensions(
+ year=Dimension(expr=lambda t: t.year, is_entity=True),
+ month=Dimension(expr=lambda t: t.month, is_entity=True),
+ )
+ .with_measures(revenue=lambda t: t.revenue.sum())
+ )
+ daily = (
+ to_semantic_table(daily_tbl, name="daily")
+ .with_dimensions(
+ year=Dimension(expr=lambda t: t.year, is_entity=True),
+ month=Dimension(expr=lambda t: t.month, is_entity=True),
+ day=Dimension(expr=lambda t: t.day, is_entity=True),
+ )
+ .with_measures(hours=lambda t: t.hours.sum())
+ )
+ calendar = to_semantic_table(calendar_tbl, name="calendar").with_dimensions(
+ year=Dimension(expr=lambda t: t.year, is_entity=True),
+ month=Dimension(expr=lambda t: t.month, is_entity=True),
+ label=lambda t: t.label,
+ )
+ return monthly, daily, calendar
+
+
+def test_filter_join_one_reuses_automatic_grain_detection(con):
+ monthly, daily, _ = _grain_models(con)
+
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ joined = monthly.filter(lambda t: t.year == 2024).join_one(
+ daily,
+ on=lambda m, d: (m.year == d.year) & (m.month == d.month),
+ )
+
+ assert joined.op().cardinality == "many"
+ assert any("Grain mismatch" in str(item.message) for item in caught)
+
+
+def test_join_wrapper_join_one_reuses_automatic_grain_detection(con):
+ monthly, daily, calendar = _grain_models(con)
+ enriched_monthly = monthly.join_one(
+ calendar,
+ on=lambda m, c: (m.year == c.year) & (m.month == c.month),
+ )
+
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ joined = enriched_monthly.join_one(
+ daily,
+ on=lambda m, d: (m.year == d.year) & (m.month == d.month),
+ )
+
+ assert joined.op().cardinality == "many"
+ assert any("Grain mismatch" in str(item.message) for item in caught)
+
+
+def test_join_wrapper_local_entity_dimension_participates_in_grain_detection(con):
+ orders_tbl = con.create_table(
+ "lineage_wrapper_orders",
+ pd.DataFrame(
+ {"order_id": [1, 2], "account_id": [10, 10], "amount": [5, 7]}
+ ),
+ )
+ accounts_tbl = con.create_table(
+ "lineage_wrapper_accounts",
+ pd.DataFrame({"account_id": [10], "label": ["A"]}),
+ )
+ lines_tbl = con.create_table(
+ "lineage_wrapper_lines",
+ pd.DataFrame(
+ {"line_id": [100, 101], "account_id": [10, 10], "value": [2, 3]}
+ ),
+ )
+ orders = to_semantic_table(orders_tbl, "orders").with_measures(
+ total=lambda t: t.amount.sum()
+ )
+ accounts = to_semantic_table(accounts_tbl, "accounts")
+ lines = (
+ to_semantic_table(lines_tbl, "lines")
+ .with_dimensions(
+ entity_id=Dimension(expr=lambda t: t.line_id, is_entity=True)
+ )
+ .with_measures(total=lambda t: t.value.sum())
+ )
+ enriched_orders = orders.join_one(accounts, on="account_id").with_dimensions(
+ entity_id=Dimension(expr=lambda t: t.order_id, is_entity=True)
+ )
+
+ with pytest.warns(UserWarning, match="Grain mismatch"):
+ joined = enriched_orders.join_one(lines, on="account_id")
+
+ assert joined.op().cardinality == "many"
+
+
+def test_aggregate_does_not_inherit_stale_source_entity_grain(con):
+ from boring_semantic_layer.expr import (
+ _get_entity_dims,
+ _get_entity_source_columns,
+ )
+
+ daily_tbl = con.create_table(
+ "lineage_daily_aggregate",
+ pd.DataFrame(
+ {
+ "year": [2024, 2024],
+ "month": [1, 1],
+ "day": [1, 2],
+ "hours": [8, 7],
+ }
+ ),
+ )
+ month_lookup_tbl = con.create_table(
+ "lineage_month_lookup",
+ pd.DataFrame({"year": [2024], "month": [1], "target": [20]}),
+ )
+ daily = (
+ to_semantic_table(daily_tbl, "daily")
+ .with_dimensions(
+ year=Dimension(expr=lambda t: t.year, is_entity=True),
+ month=Dimension(expr=lambda t: t.month, is_entity=True),
+ day=Dimension(expr=lambda t: t.day, is_entity=True),
+ )
+ .with_measures(hours=lambda t: t.hours.sum())
+ )
+ monthly_result = daily.group_by("year", "month").aggregate("hours")
+ month_lookup = (
+ to_semantic_table(month_lookup_tbl, "month_lookup")
+ .with_dimensions(
+ year=Dimension(expr=lambda t: t.year, is_entity=True),
+ month=Dimension(expr=lambda t: t.month, is_entity=True),
+ )
+ .with_measures(target=lambda t: t.target.sum())
+ )
+
+ assert _get_entity_dims(monthly_result.op()) == frozenset()
+ assert _get_entity_source_columns(monthly_result.op()) == frozenset()
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ joined = monthly_result.join_one(month_lookup, on=["year", "month"])
+
+ assert joined.op().cardinality == "one"
+ assert not any("Grain mismatch" in str(item.message) for item in caught)
diff --git a/src/boring_semantic_layer/tests/test_xorq_string_serialization.py b/src/boring_semantic_layer/tests/test_xorq_string_serialization.py
index 1b277a8c..6fd5a508 100644
--- a/src/boring_semantic_layer/tests/test_xorq_string_serialization.py
+++ b/src/boring_semantic_layer/tests/test_xorq_string_serialization.py
@@ -793,15 +793,22 @@ def test_tagged_roundtrip_multiple_measure_types(flights_data):
}
-def test_tagged_roundtrip_filter_predicate(flights_data):
- """Filter predicates survive to_tagged -> from_tagged."""
+@pytest.mark.parametrize(
+ "predicate",
+ [
+ pytest.param(lambda t: t.distance > 100, id="callable"),
+ pytest.param(ibis._.distance > 100, id="deferred"),
+ ],
+)
+def test_tagged_roundtrip_filter_predicate(flights_data, predicate):
+ """Ordinary callable and Deferred filters survive tagged round-trips."""
from boring_semantic_layer.serialization import from_tagged, to_tagged
flights = (
to_semantic_table(flights_data, name="flights")
.with_dimensions(origin=lambda t: t.origin)
.with_measures(total_distance=lambda t: t.distance.sum())
- .filter(lambda t: t.distance > 100)
+ .filter(predicate)
)
tagged = to_tagged(flights)
@@ -813,6 +820,149 @@ def test_tagged_roundtrip_filter_predicate(flights_data):
assert result["total_distance"].sum() == 500
+def _joined_qualified_raw_json_filter(table_suffix):
+ """Build a collision-sensitive JSON filter for tagged-round-trip tests."""
+ import pandas as pd
+
+ from boring_semantic_layer.query import Filter
+
+ con = ibis.duckdb.connect(":memory:")
+ orders_tbl = con.create_table(
+ f"orders_json_rt_{table_suffix}",
+ pd.DataFrame(
+ {
+ "order_id": [1, 2],
+ "status": ["bad", "open"],
+ }
+ ),
+ )
+ items_tbl = con.create_table(
+ f"items_json_rt_{table_suffix}",
+ pd.DataFrame(
+ {
+ "order_id": [1, 1, 2],
+ "status": ["ok", "ok", "bad"],
+ "amount": [1, 2, 30],
+ }
+ ),
+ )
+ orders = to_semantic_table(orders_tbl, "orders").with_dimensions(
+ status=lambda t: t.status
+ )
+ # ``status`` intentionally remains an undeclared raw column on the right.
+ items = to_semantic_table(items_tbl, "items").with_measures(
+ total=lambda t: t.amount.sum()
+ )
+ joined = orders.join_many(items, on="order_id")
+ predicate = Filter(
+ filter={"field": "items.status", "operator": "=", "value": "bad"}
+ ).to_callable()
+ return joined.filter(predicate)
+
+
+def test_tagged_roundtrip_preserves_json_exact_fields_for_joined_raw_column():
+ """JSON field provenance survives when no semantic dimension declares it."""
+ from boring_semantic_layer.serialization import from_tagged, to_tagged
+
+ filtered = _joined_qualified_raw_json_filter("valid")
+ before = filtered.aggregate("items.total").execute()
+
+ reconstructed = from_tagged(to_tagged(filtered))
+ restored_predicate = reconstructed.op().predicate.unwrap
+ assert object.__getattribute__(
+ restored_predicate, "__bsl_filter_fields__"
+ ) == frozenset({"items.status"})
+
+ after = reconstructed.aggregate("items.total").execute()
+ reconstructed_twice = from_tagged(to_tagged(reconstructed))
+ after_twice = reconstructed_twice.aggregate("items.total").execute()
+ assert before["items.total"].tolist() == [30]
+ assert after["items.total"].tolist() == [30]
+ assert after_twice["items.total"].tolist() == [30]
+
+
+@pytest.mark.parametrize("overlay", ["dimensions", "measures"])
+def test_tagged_json_filter_survives_metadata_overlay(overlay):
+ """Overlay rebinding retains strict fields through tagged serialization."""
+ from boring_semantic_layer.serialization import from_tagged, to_tagged
+
+ filtered = _joined_qualified_raw_json_filter(f"overlay_{overlay}")
+ if overlay == "dimensions":
+ rebound = filtered.with_dimensions(bucket=lambda t: t.status)
+ else:
+ rebound = filtered.with_measures(
+ joined_total=lambda t: t.amount.sum()
+ )
+
+ reconstructed = from_tagged(to_tagged(rebound))
+ result = reconstructed.aggregate("items.total").execute()
+
+ assert result["items.total"].tolist() == [30]
+
+
+def test_tagged_filter_keeps_author_time_dimension_before_same_name_overlay():
+ """Round-trip must not rebind a raw timestamp filter to its month bucket."""
+ import pandas as pd
+
+ from boring_semantic_layer.serialization import from_tagged, to_tagged
+
+ con = ibis.duckdb.connect(":memory:")
+ table = con.create_table(
+ "filter_same_name_dimension_roundtrip",
+ pd.DataFrame(
+ {
+ "ts": pd.to_datetime(
+ ["2025-01-05", "2025-01-31", "2025-02-10"]
+ ),
+ "value": [1, 2, 3],
+ }
+ ),
+ )
+ model = (
+ to_semantic_table(table, "events")
+ .with_dimensions(ts=lambda t: t.ts)
+ .with_measures(count=lambda t: t.count())
+ )
+ rebound = model.filter(lambda t: t.ts >= "2025-01-15").with_dimensions(
+ ts=lambda t: t.ts.truncate("M")
+ )
+
+ reconstructed = from_tagged(to_tagged(rebound))
+ result = (
+ reconstructed.group_by("ts")
+ .aggregate("count")
+ .execute()
+ .sort_values("ts")
+ .reset_index(drop=True)
+ )
+
+ assert [str(value)[:10] for value in result["ts"]] == [
+ "2025-01-01",
+ "2025-02-01",
+ ]
+ assert result["count"].tolist() == [1, 1]
+
+
+def test_tagged_json_filter_metadata_rejects_unknown_prefix_after_roundtrip():
+ """Deserialized exact-field metadata remains fail-closed if untrusted."""
+ from boring_semantic_layer.serialization import from_tagged, to_tagged
+ from boring_semantic_layer.serialization.freeze import freeze, thaw
+
+ tagged = to_tagged(_joined_qualified_raw_json_filter("bad_prefix"))
+ tag_op = tagged.op()
+ metadata = {key: thaw(value) for key, value in dict(tag_op.metadata).items()}
+ metadata.pop("tag")
+ metadata["filter_fields"] = ["bogus.status"]
+ tampered = tag_op.parent.to_expr().hashing_tag(
+ tag="bsl",
+ **{key: freeze(value) for key, value in metadata.items()},
+ )
+
+ reconstructed = from_tagged(tampered)
+ with pytest.raises(KeyError, match="Unknown semantic model prefix 'bogus'"):
+ reconstructed.aggregate("items.total").execute()
+
+
def test_tagged_roundtrip_mutate_arithmetic(flights_data):
"""Mutate with arithmetic expression survives to_tagged -> from_tagged."""
from boring_semantic_layer.serialization import from_tagged, to_tagged
diff --git a/src/boring_semantic_layer/utils.py b/src/boring_semantic_layer/utils.py
index 08e2a038..6dc2df23 100644
--- a/src/boring_semantic_layer/utils.py
+++ b/src/boring_semantic_layer/utils.py
@@ -814,24 +814,26 @@ def _is_deferred(obj) -> bool:
def expr_to_structured(fn: Callable) -> Result[tuple, Exception]:
"""Convert a callable/Deferred expression to a structured tuple representation."""
from ._xorq import Deferred as XorqDeferred
+ from .ops import _CallableWrapper
@safe
def do_convert():
from ._xorq import _
- if isinstance(fn, XorqDeferred):
- return serialize_resolver(fn._resolver)
+ expr = fn._fn if isinstance(fn, _CallableWrapper) else fn
+ if isinstance(expr, XorqDeferred):
+ return serialize_resolver(expr._resolver)
# For ibis Deferred (not xorq vendor), resolve through xorq _ to get xorq types
- if _is_deferred(fn):
- result = fn.resolve(_)
+ if _is_deferred(expr):
+ result = expr.resolve(_)
if _is_deferred(result):
return serialize_resolver(result._resolver)
- if callable(fn):
- result = fn(_)
+ if callable(expr):
+ result = expr(_)
if _is_deferred(result):
return serialize_resolver(result._resolver)
raise ValueError(f"Callable did not produce a Deferred, got {type(result)}")
- raise ValueError(f"Expected callable or Deferred, got {type(fn)}")
+ raise ValueError(f"Expected callable or Deferred, got {type(expr)}")
return do_convert()