From f793e05f4fba1b2a5de946bc35a942621323bf19 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Thu, 9 Jul 2026 22:00:23 +0200 Subject: [PATCH 01/18] chore(appkit): metric-view route skeleton + measures-only SQL (PR2 phase 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /api/analytics/metric/:key over the standard SSE envelope, SP lane only. Synchronous config-parse registration from config/queries/metric-views.json against the landed metricSourceSchema (single metricViews map; lane derived from executor). Measures-only buildMetricSql (SELECT MEASURE(m) AS m FROM [LIMIT n]) gated by MEASURE_NAME_PATTERN + assertSafeFqn — grammar gate only, no name allowlist. 503 METRIC_REGISTRY_LOAD_FAILED on malformed config, 404 on unknown key (generic public bodies; detail to telemetry). Signed-off-by: Atila Fassina --- .../appkit/src/plugins/analytics/analytics.ts | 285 ++++++++++++ .../appkit/src/plugins/analytics/metric.ts | 236 ++++++++++ .../plugins/analytics/tests/analytics.test.ts | 11 +- .../plugins/analytics/tests/metric.test.ts | 421 ++++++++++++++++++ .../appkit/src/plugins/analytics/types.ts | 46 ++ 5 files changed, 996 insertions(+), 3 deletions(-) create mode 100644 packages/appkit/src/plugins/analytics/metric.ts create mode 100644 packages/appkit/src/plugins/analytics/tests/metric.test.ts diff --git a/packages/appkit/src/plugins/analytics/analytics.ts b/packages/appkit/src/plugins/analytics/analytics.ts index 4ad05cc37..eceb4b9c0 100644 --- a/packages/appkit/src/plugins/analytics/analytics.ts +++ b/packages/appkit/src/plugins/analytics/analytics.ts @@ -29,6 +29,11 @@ import { Plugin, toPlugin } from "../../plugin"; import type { PluginManifest } from "../../registry"; import { queryDefaults } from "./defaults"; import manifest from "./manifest.json"; +import { + buildMetricSql, + loadMetricRegistry, + validateMetricRequest, +} from "./metric"; import { QueryProcessor } from "./query"; import { type ArrowCapability, @@ -41,6 +46,7 @@ import { type AnalyticsStreamMessage, type IAnalyticsConfig, type IAnalyticsQueryRequest, + type MetricRegistration, normalizeAnalyticsFormat, type WarehouseStatus, } from "./types"; @@ -116,6 +122,25 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { */ private _arrowCapability = new Map(); + /** + * Metric-view registry parsed from `config/queries/metric-views.json`, keyed + * by metric key. Loaded lazily on the first `/metric/:key` request and + * memoized (see {@link _getMetricRegistry}). Empty when no config is present + * — the metric-view path stays dormant until an app opts in. + */ + private metricRegistry: Record | null = null; + + /** + * Latched error from the most recent {@link loadMetricRegistry} attempt. + * `null` means the registry loaded cleanly (or `metric-views.json` was absent + * — also fine; metric views are opt-in). When non-null, every `/metric/:key` + * request returns 503 `METRIC_REGISTRY_LOAD_FAILED` so a broken config + * (unreadable file, invalid JSON, schema violation) surfaces as a clear + * server status rather than masquerading as a 404 for every key. The full + * reason goes to telemetry only. + */ + private metricRegistryLoadError: string | null = null; + constructor(config: IAnalyticsConfig) { super(config); this.config = config; @@ -137,6 +162,19 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { }, }); + // Metric-view route. Registered parallel to `/query`; measures a + // registered UC Metric View over the same SSE envelope. Dormant until + // `config/queries/metric-views.json` exists (no config → empty registry → + // 404, nothing executes). + this.route(router, { + name: "metric", + method: "post", + path: "/metric/:key", + handler: async (req: express.Request, res: express.Response) => { + await this._handleMetricRoute(req, res); + }, + }); + // Column-names fallback for very wide Arrow schemas whose names don't fit // in the `X-Appkit-Arrow-Columns` response header (see // `_setArrowColumnsHeader`). The client hits this with the statement id @@ -426,6 +464,253 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { ); } + /** + * Lazily load and memoize the metric registry from + * `config/queries/metric-views.json`. + * + * A malformed config latches `metricRegistryLoadError` and yields an empty + * registry so the route can return a 503 (distinguishing a broken deployment + * from an unknown key, which is a 404). Loading is deferred to the first + * `/metric/:key` request rather than the constructor so apps that never adopt + * metric views pay no parse cost and a config error can't break plugin + * construction. + */ + private _getMetricRegistry(): Record { + if (this.metricRegistry === null) { + try { + this.metricRegistry = loadMetricRegistry(); + this.metricRegistryLoadError = null; + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + logger.warn("Failed to load metric registry: %s", reason); + this.metricRegistry = {}; + this.metricRegistryLoadError = reason; + } + } + return this.metricRegistry; + } + + /** + * Handle metric-view execution requests (`POST /api/analytics/metric/:key`). + * + * Mirrors {@link _handleQueryRoute}'s JSON SSE path: the outer + * `executeStream` disables cache/retry and streams warehouse-readiness + * (`warehouse_status`) events, then the inner `execute` builds the metric SQL + * and delivers rows through {@link deliverJsonResult} as a `result` message. + * The `originalError` re-throw discipline preserves each error's structured + * `errorCode`/`clientMessage` for the SSE error payload. + * + * SP lane only in this phase — every registered metric runs as the service + * principal; OBO dispatch is wired in a later phase. + */ + async _handleMetricRoute( + req: express.Request, + res: express.Response, + ): Promise { + const { key } = req.params; + + logger.debug(req, "Executing metric: %s", key); + + const event = logger.event(req); + event?.setComponent("analytics", "executeMetric").setContext("analytics", { + metric_key: key, + plugin: this.name, + }); + + if (!key) { + res.status(400).json({ error: "metric key is required" }); + return; + } + + const registry = this._getMetricRegistry(); + + // Surface a registry-load failure on the route. Without this, a malformed + // metric-views.json would yield 404 for every key — identical to "key + // never registered" — and hide the deployment error. Full reason → + // telemetry only. + if (this.metricRegistryLoadError !== null) { + event?.setContext("analytics", { + metric_registry_load_error: this.metricRegistryLoadError, + }); + res.status(503).json({ + error: "Metric registry not available", + code: "METRIC_REGISTRY_LOAD_FAILED", + }); + return; + } + + const registration = registry[key]; + if (!registration) { + // Don't echo the user-supplied `key` back in the public response — + // confirming "metric X is not registered" lets a probe enumerate keys by + // elimination. The 404 status stays (useful for tooling); the body is + // generic and detail goes to telemetry only. + event?.setContext("analytics", { unknown_metric_key: key }); + res.status(404).json({ error: "Metric not found" }); + return; + } + + // Validate the body on the canonical error path. `validateMetricRequest` + // throws a `ValidationError` (400) whose message names only field paths, + // never raw values. + let request: ReturnType; + try { + request = validateMetricRequest(req.body ?? {}); + } catch (err) { + if (err instanceof AppKitError) { + res.status(err.statusCode).json({ error: err.message, code: err.code }); + return; + } + event?.setContext("analytics", { + unexpected_error: err instanceof Error ? err.message : String(err), + metric_key: key, + }); + logger.warn( + req, + "Unexpected throw during metric request validation for %s: %s", + key, + err instanceof Error ? err.message : String(err), + ); + res.status(400).json({ error: "Invalid request body" }); + return; + } + + // SP lane only in this phase: the default executor + shared cache key. OBO + // dispatch (asUser(req) + per-user key) arrives in a later phase. + const executor = this; + const executorKey = "sp"; + + const cacheConfig = { + ...queryDefaults.cache, + cacheKey: ["analytics:metric", key, JSON.stringify(request), executorKey], + }; + + // Cache/retry/timeout scoped to the SQL execution itself (inner `execute`) + // so the warehouse-readiness phase isn't retried and the generator value + // never leaks into the cache. + const sqlConfig: PluginExecuteConfig = { + ...queryDefaults, + cache: cacheConfig, + }; + + // Outer stream: no cache/retry — `executeStream` would otherwise wrap the + // generator factory and cache the generator object itself. Telemetry + + // trace context still apply. + const streamExecutionSettings: StreamExecutionSettings = { + default: { + cache: { enabled: false }, + retry: { enabled: false }, + }, + }; + + const startupTimeoutMs = + this.config.warehouseStartupTimeoutMs ?? + DEFAULT_WAREHOUSE_STARTUP_TIMEOUT_MS; + const autoStartWarehouse = this.config.autoStartWarehouse ?? true; + + const self = this; + + await executor.executeStream( + res, + async function* ( + signal, + ): AsyncGenerator { + const workspaceClient = getWorkspaceClient(); + const warehouseId = await getWarehouseId(); + + // Stream warehouse-readiness updates as SSE events, then run SQL. + const readinessUpdates = streamCallbacks( + (emit) => + self.SQLClient.ensureWarehouseRunning( + workspaceClient, + warehouseId, + { + signal, + timeoutMs: startupTimeoutMs, + autoStart: autoStartWarehouse, + onStatus: emit, + }, + ), + ); + for await (const update of readinessUpdates) { + yield { + type: "warehouse_status", + status: { + state: update.state as WarehouseStatus["state"], + elapsedMs: update.elapsedMs, + }, + }; + } + + // `execute()` reduces a thrown error to `{ status, message }`, + // dropping the rich fields (`errorCode`, `clientMessage`). Capture the + // original here so we can re-throw it intact — the SSE error path + // (`StreamManager`) reads `errorCode`/`clientMessage` off it. + let originalError: unknown; + const sqlResult = await executor.execute( + async (sig) => { + try { + const { statement, parameters } = buildMetricSql( + registration, + request, + ); + const processedParams = + await self.queryProcessor.processQueryParams( + statement, + Object.keys(parameters).length > 0 ? parameters : undefined, + ); + // Reuse the query route's JSON delivery: INLINE JSON_ARRAY with + // an ARROW_STREAM-inline fallback, returning plain rows in a + // `result` message — byte-identical envelope to `/query`. + return await self._executeJsonArrayPath( + executor, + statement, + processedParams, + sig, + ); + } catch (err) { + originalError = err; + throw err; + } + }, + { default: sqlConfig }, + executorKey, + ); + + if (!sqlResult.ok) { + const msg = sqlResult.message; + const lower = msg.toLowerCase(); + if ( + lower.includes("operation was aborted") || + lower.includes("the request was aborted") || + lower.includes("statement was canceled") + ) { + const err = new DOMException( + lower.includes("canceled") ? msg : "The operation was aborted.", + "AbortError", + ); + throw err; + } + // Re-throw the original error so its structured `errorCode` and + // sanitized `clientMessage` survive to the SSE error payload. Fall + // back to a generic statement failure only if it wasn't an + // AppKitError. + if (originalError instanceof AppKitError) { + throw originalError; + } + const inner = msg.startsWith("Statement failed: ") + ? msg.slice("Statement failed: ".length) + : msg; + throw ExecutionError.statementFailed(inner); + } + + yield sqlResult.data as AnalyticsStreamMessage; + }, + streamExecutionSettings, + executorKey, + ); + } + /** * JSON_ARRAY SSE path. Delegates the disposition/format fallback to * {@link deliverJsonResult} (INLINE JSON_ARRAY → on `needs-arrow-inline`, diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts new file mode 100644 index 000000000..708703846 --- /dev/null +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -0,0 +1,236 @@ +import fs from "node:fs"; +import path from "node:path"; +import type { SQLTypeMarker } from "shared"; +import { z } from "zod"; +// Canonical metric-source schema — the single source of truth for +// `metric-views.json`. Imported from the shared source directly (matching the +// type-generator's runtime, which pulls the zod-free `metric-fqn.ts` from the +// same tree) so the runtime and the generated JSON schema validate identically. +import { metricSourceSchema } from "../../../../shared/src/schemas/metric-source"; +import { ValidationError } from "../../errors"; +import { createLogger } from "../../logging/logger"; +import type { + IAnalyticsMetricRequest, + MetricLane, + MetricRegistration, +} from "./types"; + +const logger = createLogger("analytics:metric"); + +/** + * Default queries directory. Mirrors `AppManager`'s + * `path.resolve(process.cwd(), "config/queries")` so dev mode and production + * share a single source of truth for where metric config lives. + */ +const QUERIES_DIR = path.resolve(process.cwd(), "config/queries"); +const METRIC_CONFIG_FILE = "metric-views.json"; + +/** + * Three-part UC FQN matcher. The registry is parsed against the landed + * `metricSourceSchema`, which already validates `source` via the composed + * `UC_THREE_PART_FQN_PATTERN`; this is a belt-and-suspenders runtime fence for + * any code path that constructs SQL from a `source` outside that parse (the FQN + * cannot be parameterized — it is interpolated into the SQL string). + */ +const FQN_PATTERN = + /^[a-zA-Z0-9_][a-zA-Z0-9_-]*\.[a-zA-Z0-9_][a-zA-Z0-9_-]*\.[a-zA-Z0-9_][a-zA-Z0-9_-]*$/; + +/** + * Validate measure names before they are interpolated into `MEASURE()`. + * + * Measure names cannot be parameterized — they are SQL identifiers, not + * literals. This conservative identifier shape is the security boundary for + * the interpolated tokens: there is deliberately NO name allowlist, so a + * well-formed-but-unknown measure falls through to the warehouse and surfaces + * as a sanitized canonical error (parity with the raw `.sql` flow). + */ +const MEASURE_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + +/** + * Map an entry's declared `executor` to the internal execution lane: + * - `"user"` → `"obo"` (per-user cache, on-behalf-of) + * - `"app_service_principal"` (default) → `"sp"` (shared cache) + */ +function laneFromExecutor( + executor: "app_service_principal" | "user", +): MetricLane { + return executor === "user" ? "obo" : "sp"; +} + +/** + * Read and validate `config/queries/metric-views.json` into a metric registry. + * + * Synchronous by design — registration is a pure config parse with no + * warehouse round-trip, no `DESCRIBE`, and no build-time metadata bundle. The + * single `metricViews` map makes keys unique by construction, so there is no + * cross-lane duplicate-key check. + * + * Returns an empty registry when the file is absent: the metric-view path is + * additive and dormant until an app opts in by adding the config. A malformed + * file (unreadable, invalid JSON, or schema violation) throws — the caller + * latches the failure so the route can surface a 503 rather than masking a + * broken deployment as a 404 for every key. + */ +export function loadMetricRegistry( + queriesDir: string = QUERIES_DIR, +): Record { + const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); + + let raw: string; + try { + raw = fs.readFileSync(metricPath, "utf8"); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + return {}; + } + throw err; + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err) { + throw new Error( + `Failed to parse metric-views.json at ${metricPath}: ${(err as Error).message}`, + ); + } + + const result = metricSourceSchema.safeParse(parsed); + if (!result.success) { + const issues = result.error.issues + .map((i) => `${i.path.join(".")}: ${i.message}`) + .join("; "); + throw new Error(`Invalid metric-views.json at ${metricPath}: ${issues}`); + } + + const registry: Record = {}; + for (const [key, entry] of Object.entries(result.data.metricViews ?? {})) { + registry[key] = { + key, + source: entry.source, + lane: laneFromExecutor(entry.executor), + }; + } + + logger.debug( + "Loaded metric registry: %d entry(ies)", + Object.keys(registry).length, + ); + return registry; +} + +/** + * SQL identifier safety guard — the FQN ships in the SQL string (it cannot be + * parameterized) so we re-check the regex at construction time. + * + * The registry loader already enforces the FQN grammar via `metricSourceSchema`; + * this is a runtime fence for any future code path that constructs SQL outside + * of a parsed registry. + */ +function assertSafeFqn(fqn: string): void { + if (!FQN_PATTERN.test(fqn)) { + throw new Error( + `Refusing to build SQL: "${fqn}" is not a valid three-part UC FQN.`, + ); + } +} + +/** + * Structural validation schema for the metric request body. + * + * The schema is **static** — any grammar-valid measure identifier is accepted; + * it is NOT a dynamic per-key `z.enum(knownMeasures)`. Unknown-but-well-formed + * names are not rejected here; they reach the warehouse and surface as a + * sanitized canonical error. This is the measures-only shape; + * dimensions/filter/timeGrain are accepted structurally (their SQL is built in + * a later phase) but left permissive here. + */ +const metricRequestSchema = z + .object({ + measures: z + .array(z.string().min(1, "measure name cannot be empty")) + .min(1, "at least one measure is required"), + dimensions: z.array(z.string().min(1)).optional(), + filter: z.unknown().optional(), + timeGrain: z.string().min(1).optional(), + limit: z.number().int().positive().optional(), + format: z.enum(["JSON_ARRAY", "ARROW_STREAM", "JSON", "ARROW"]).optional(), + }) + .strict(); + +/** + * Validate a `POST /api/analytics/metric/:key` request body against the static + * measures-only shape. Throws {@link ValidationError} (a 400 on the canonical + * error path) with the offending field paths; the raw values stay in telemetry + * context, never the public body. + */ +export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { + const result = metricRequestSchema.safeParse(body); + if (!result.success) { + const fieldPaths = result.error.issues + .map((i) => i.path.join(".") || "(root)") + .join(", "); + throw new ValidationError( + fieldPaths.length > 0 + ? `Invalid metric request body (fields: ${fieldPaths})` + : "Invalid metric request body", + { context: { issues: result.error.issues } }, + ); + } + return result.data; +} + +/** + * Construct the measures-only metric SQL. + * + * Shape: + * + * SELECT MEASURE(m) AS m[, …] FROM [LIMIT n] + * + * Every measure is gated by {@link MEASURE_NAME_PATTERN} before it is + * interpolated (measures cannot be parameterized — they are SQL identifiers), + * and the FQN is re-checked by {@link assertSafeFqn}. No user-supplied string + * reaches the SQL string without passing a grammar gate. `dimensions`, + * `filter`, and `timeGrain` are ignored here — their SQL is built in a later + * phase. Returns `{ statement, parameters }` where `parameters` is the named + * bind-var dictionary the plugin's `query()` consumes (empty in this phase). + */ +export function buildMetricSql( + registration: MetricRegistration, + request: IAnalyticsMetricRequest, +): { + statement: string; + parameters: Record; +} { + assertSafeFqn(registration.source); + + if (request.measures.length === 0) { + throw new Error("buildMetricSql requires at least one measure."); + } + + for (const m of request.measures) { + if (!MEASURE_NAME_PATTERN.test(m)) { + throw new Error( + `Refusing to build SQL: measure "${m}" is not a valid identifier.`, + ); + } + } + + // Deterministic order so cache keys collapse semantically equivalent calls. + // Alias each measure to its plain name so result rows have keys matching the + // registered measure (`{ arr: 1234 }`) rather than the SQL-function + // serialization Databricks returns by default (`{ "measure(arr)": 1234 }`). + const measureClauses = [...request.measures] + .sort() + .map((m) => `MEASURE(${m}) AS ${m}`); + + const selectList = measureClauses.join(", "); + + const limitClause = + typeof request.limit === "number" && request.limit > 0 + ? ` LIMIT ${Math.floor(request.limit)}` + : ""; + + const statement = `SELECT ${selectList} FROM ${registration.source}${limitClause}`; + return { statement, parameters: {} }; +} diff --git a/packages/appkit/src/plugins/analytics/tests/analytics.test.ts b/packages/appkit/src/plugins/analytics/tests/analytics.test.ts index 356777922..aff246145 100644 --- a/packages/appkit/src/plugins/analytics/tests/analytics.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/analytics.test.ts @@ -90,18 +90,23 @@ describe("Analytics Plugin", () => { }); describe("injectRoutes", () => { - test("should register single POST route for queries", () => { + test("should register the query and metric POST routes", () => { const plugin = new AnalyticsPlugin(config); const { router } = createMockRouter(); plugin.injectRoutes(router); - // Only 1 POST route - asUser is determined by .obo.sql file convention - expect(router.post).toHaveBeenCalledTimes(1); + // Two POST routes: the SQL query route (asUser determined by the + // .obo.sql file convention) and the metric-view route. + expect(router.post).toHaveBeenCalledTimes(2); expect(router.post).toHaveBeenCalledWith( "/query/:query_key", expect.any(Function), ); + expect(router.post).toHaveBeenCalledWith( + "/metric/:key", + expect.any(Function), + ); }); test("/query/:query_key should return 400 when query_key is missing", async () => { diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts new file mode 100644 index 000000000..b6b529a67 --- /dev/null +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -0,0 +1,421 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { + createMockRequest, + createMockResponse, + createMockRouter, + mockServiceContext, + setupDatabricksEnv, +} from "@tools/test-helpers"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { ServiceContext } from "../../../context/service-context"; +import { AnalyticsPlugin } from "../analytics"; +import { buildMetricSql, loadMetricRegistry } from "../metric"; +import type { IAnalyticsConfig, MetricRegistration } from "../types"; + +// Mirror the analytics.test.ts CacheManager mock so the inner `execute`'s +// cache interceptor is a no-op pass-through (each request re-executes). +const { mockCacheStore, mockCacheInstance } = vi.hoisted(() => { + const store = new Map(); + const generateKey = (parts: unknown[], userKey: string): string => { + const { createHash } = require("node:crypto"); + const serialized = JSON.stringify([userKey, ...parts]); + return createHash("sha256").update(serialized).digest("hex"); + }; + const instance = { + get: vi.fn(), + set: vi.fn(), + delete: vi.fn(), + getOrExecute: vi.fn( + async (key: unknown[], fn: () => Promise, userKey: string) => { + const cacheKey = generateKey(key, userKey); + if (store.has(cacheKey)) return store.get(cacheKey); + const result = await fn(); + store.set(cacheKey, result); + return result; + }, + ), + generateKey: vi.fn((parts: unknown[], userKey: string) => + generateKey(parts, userKey), + ), + }; + return { mockCacheStore: store, mockCacheInstance: instance }; +}); + +vi.mock("../../../cache", () => ({ + CacheManager: { + getInstanceSync: vi.fn(() => mockCacheInstance), + }, +})); + +/** Feed the plugin a registry directly, bypassing the disk config parse. */ +function setRegistry( + plugin: AnalyticsPlugin, + registry: Record, +): void { + (plugin as any).metricRegistry = registry; + (plugin as any).metricRegistryLoadError = null; +} + +describe("analytics metric route (Phase 1)", () => { + let config: IAnalyticsConfig; + let serviceContextMock: Awaited>; + + beforeEach(async () => { + config = { timeout: 5000 }; + setupDatabricksEnv(); + mockCacheStore.clear(); + ServiceContext.reset(); + serviceContextMock = await mockServiceContext(); + }); + + afterEach(() => { + serviceContextMock?.restore(); + }); + + describe("injectRoutes", () => { + test("registers POST /metric/:key alongside /query", () => { + const plugin = new AnalyticsPlugin(config); + const { router } = createMockRouter(); + + plugin.injectRoutes(router); + + expect(router.post).toHaveBeenCalledWith( + "/metric/:key", + expect.any(Function), + ); + }); + }); + + // ── Grammar gate — the primary security test (replaces #341's + // fail-closed-503 allowlist test). A measure that fails MEASURE_NAME_PATTERN + // throws inside buildMetricSql BEFORE any SQL string is constructed. + describe("buildMetricSql grammar gate", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + test("throws before building SQL for an injection-shaped measure", () => { + expect(() => + buildMetricSql(registration, { + measures: ["arr; DROP TABLE users"], + }), + ).toThrow(/not a valid identifier/); + }); + + test("throws for a measure with a backtick / quote", () => { + expect(() => + buildMetricSql(registration, { measures: ["arr`"] }), + ).toThrow(/not a valid identifier/); + }); + + test("throws when no measures are supplied", () => { + expect(() => buildMetricSql(registration, { measures: [] })).toThrow( + /at least one measure/, + ); + }); + + test("throws for a non-three-part source FQN", () => { + expect(() => + buildMetricSql( + { key: "x", source: "cat.sch", lane: "sp" }, + { measures: ["arr"] }, + ), + ).toThrow(/not a valid three-part UC FQN/); + }); + }); + + // ── Measures-only SQL shape. + describe("buildMetricSql SQL shape", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + test("single measure → SELECT MEASURE(m) AS m FROM ", () => { + const { statement, parameters } = buildMetricSql(registration, { + measures: ["arr"], + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr FROM cat.sch.revenue_metrics", + ); + expect(parameters).toEqual({}); + }); + + test("multiple measures are sorted for a deterministic SELECT list", () => { + const { statement } = buildMetricSql(registration, { + measures: ["revenue", "arr"], + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr, MEASURE(revenue) AS revenue FROM cat.sch.revenue_metrics", + ); + }); + + test("positive limit appends a floored LIMIT clause", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + limit: 10.9, + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr FROM cat.sch.revenue_metrics LIMIT 10", + ); + }); + }); + + // ── Envelope parity — streams warehouse_status* then a `result` message, + // byte-identical to the /query route's JSON SSE path. + describe("_handleMetricRoute SSE envelope", () => { + test("streams warehouse_status then a result message with aliased rows", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + + const executeMock = vi.fn().mockResolvedValue({ + result: { data: [{ arr: 1234 }] }, + }); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + // The measures-only SQL reaches the warehouse. + expect(executeMock).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + statement: "SELECT MEASURE(arr) AS arr FROM cat.sch.revenue_metrics", + warehouse_id: "test-warehouse-id", + }), + expect.any(AbortSignal), + ); + + // Same SSE envelope as /query: a `result` event carrying the rows. + expect(mockRes.write).toHaveBeenCalledWith("event: result\n"); + expect(mockRes.write).toHaveBeenCalledWith( + expect.stringContaining('"data":[{"arr":1234}]'), + ); + expect(mockRes.end).toHaveBeenCalled(); + }); + + test("emits warehouse_status before result for a STARTING warehouse", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + + const executeMock = vi.fn().mockResolvedValue({ + result: { data: [{ arr: 1 }] }, + }); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + + const warehouseGet = vi + .fn() + .mockResolvedValueOnce({ state: "STARTING" }) + .mockResolvedValueOnce({ state: "RUNNING" }); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }); + mockReq.serviceWorkspaceClient.warehouses.get = warehouseGet; + mockReq.userWorkspaceClient.warehouses.get = warehouseGet; + const mockRes = createMockResponse(); + + vi.useFakeTimers(); + const handlerPromise = handler(mockReq, mockRes); + await vi.runAllTimersAsync(); + await handlerPromise; + vi.useRealTimers(); + + const eventLines = (mockRes.write as any).mock.calls + .map((call: any[]) => call[0] as string) + .filter((s: string) => s.startsWith("event: ")); + const warehouseIdx = eventLines.findIndex( + (s: string) => s === "event: warehouse_status\n", + ); + const resultIdx = eventLines.findIndex( + (s: string) => s === "event: result\n", + ); + expect(warehouseIdx).toBeGreaterThanOrEqual(0); + expect(resultIdx).toBeGreaterThanOrEqual(0); + expect(warehouseIdx).toBeLessThan(resultIdx); + }); + + test("returns 400 when the body fails structural validation", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: [] }, // empty → fails .min(1) + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(400); + }); + }); + + // ── 503-vs-404 latching + dormancy. + describe("registry latching and dormancy", () => { + test("unknown key against a valid registry → 404 (generic body)", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "nope" }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(404); + expect(mockRes.json).toHaveBeenCalledWith({ error: "Metric not found" }); + }); + + test("malformed registry → 503 METRIC_REGISTRY_LOAD_FAILED", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + // Latch a load error directly (as a malformed config would). + (plugin as any).metricRegistry = {}; + (plugin as any).metricRegistryLoadError = "Invalid metric-views.json"; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(503); + expect(mockRes.json).toHaveBeenCalledWith({ + error: "Metric registry not available", + code: "METRIC_REGISTRY_LOAD_FAILED", + }); + }); + + test("no metric-views.json present → registry empty, unknown key 404, nothing executes", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + + const executeMock = vi.fn(); + (plugin as any).SQLClient.executeStatement = executeMock; + // Registry lazily loads from cwd; no config/queries/metric-views.json in + // the test cwd → empty registry (dormant). + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(404); + expect(executeMock).not.toHaveBeenCalled(); + }); + }); +}); + +// ── loadMetricRegistry: config parse against the landed metricSourceSchema. +describe("loadMetricRegistry", () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(path.join(tmpdir(), "mv-registry-")); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + test("absent metric-views.json → empty registry (dormancy)", () => { + expect(loadMetricRegistry(dir)).toEqual({}); + }); + + test("derives lane from executor (default sp, user → obo)", () => { + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { + revenue: { source: "cat.sch.revenue_metrics" }, + customers: { source: "cat.sch.customer_metrics", executor: "user" }, + }, + }), + ); + + const registry = loadMetricRegistry(dir); + expect(registry.revenue).toEqual({ + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }); + expect(registry.customers).toEqual({ + key: "customers", + source: "cat.sch.customer_metrics", + lane: "obo", + }); + }); + + test("malformed JSON throws", () => { + writeFileSync(path.join(dir, "metric-views.json"), "{ not json"); + expect(() => loadMetricRegistry(dir)).toThrow(/Failed to parse/); + }); + + test("schema-invalid config throws", () => { + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { revenue: { source: "not-a-three-part-fqn" } }, + }), + ); + expect(() => loadMetricRegistry(dir)).toThrow(/Invalid metric-views.json/); + }); +}); diff --git a/packages/appkit/src/plugins/analytics/types.ts b/packages/appkit/src/plugins/analytics/types.ts index 95c987d14..c032e6cca 100644 --- a/packages/appkit/src/plugins/analytics/types.ts +++ b/packages/appkit/src/plugins/analytics/types.ts @@ -112,3 +112,49 @@ export interface AnalyticsQueryResponse { row_count: number; data: any[]; } + +// ──────────────────────────────────────────────────────────────────────────── +// Metric views — POST /api/analytics/metric/:key +// ──────────────────────────────────────────────────────────────────────────── + +/** + * Execution lane for a registered metric view, derived from the entry's + * `executor` in `metric-views.json`: + * - `"sp"` ← `executor: "app_service_principal"` — queried as the app + * service principal (cache shared across all users). + * - `"obo"` ← `executor: "user"` — queried on-behalf-of the requesting + * user (per-user cache). OBO dispatch is wired in a later phase. + */ +export type MetricLane = "sp" | "obo"; + +/** + * A single registered metric view, loaded from `config/queries/metric-views.json`. + * + * The registration carries only what the runtime needs to build and dispatch + * SQL: the metric `key`, the three-part UC FQN `source`, and the `lane`. There + * is intentionally NO build-time measure/dimension metadata here — the security + * boundary is the grammar gate plus parameterized values, not a name allowlist, + * so the runtime never enumerates known measures/dimensions. + */ +export interface MetricRegistration { + key: string; + source: string; + lane: MetricLane; +} + +/** + * Validated request body for `POST /api/analytics/metric/:key`. + * + * `measures` is required; `dimensions`, `filter`, and `timeGrain` are part of + * the wire shape but their SQL is not built yet (a later phase adds grouping, + * time-grain truncation, and the structured filter translator). `filter` is + * typed `unknown` until that phase pins down the recursive predicate shape. + */ +export interface IAnalyticsMetricRequest { + measures: string[]; + dimensions?: string[]; + filter?: unknown; + timeGrain?: string; + limit?: number; + format?: AnalyticsFormat; +} From fd9735627ee74f493b1f29cdfe48519248dbee6c Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Thu, 9 Jul 2026 22:41:48 +0200 Subject: [PATCH 02/18] chore(appkit): metric dimensions + structured filter engine (PR2 phase 2a) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GROUP BY ALL for dimensions; recursive 12-operator filter engine (equals/notEquals/in/notIn/gt/gte/lt/lte/contains/notContains/set/notSet) with every value bound as a :f_ named parameter — never interpolated. member/dimension gated by DIMENSION_NAME_PATTERN; AND/OR composition; depth capped at 8, enforced twice (iterative pre-Zod preCheckFilterDepth + renderer re-check). Static (registration-free) request schema: operator enum, per-operator value cardinality, group/values/limit caps. No name allowlist. timeGrain application deliberately held (parsed + grammar-gated, not yet applied) pending the grain-target decision — lands in phase 2b. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../appkit/src/plugins/analytics/metric.ts | 759 ++++++++++++++++- .../plugins/analytics/tests/metric.test.ts | 762 +++++++++++++++++- .../appkit/src/plugins/analytics/types.ts | 54 +- 3 files changed, 1543 insertions(+), 32 deletions(-) diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts index 708703846..e2b03c977 100644 --- a/packages/appkit/src/plugins/analytics/metric.ts +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; import path from "node:path"; -import type { SQLTypeMarker } from "shared"; +import { type SQLTypeMarker, sql as sqlHelpers } from "shared"; import { z } from "zod"; // Canonical metric-source schema — the single source of truth for // `metric-views.json`. Imported from the shared source directly (matching the @@ -11,7 +11,10 @@ import { ValidationError } from "../../errors"; import { createLogger } from "../../logging/logger"; import type { IAnalyticsMetricRequest, + MetricFilter, + MetricFilterOperatorName, MetricLane, + MetricPredicate, MetricRegistration, } from "./types"; @@ -46,6 +49,100 @@ const FQN_PATTERN = */ const MEASURE_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; +/** + * Dimension name pattern. Matches the identifier shape we accept for measures + * — column references cannot be parameterized in SQL, so they must be + * conservatively safe identifiers (no spaces, no quotes, no SQL operators). + * This grammar gate is the security boundary for interpolated dimension + * tokens: there is deliberately NO name allowlist, so a well-formed-but-unknown + * dimension falls through to the warehouse and surfaces as a sanitized + * canonical error. + */ +const DIMENSION_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + +/** + * Time-grain token shape. Accepted structurally so a hostile token can never + * reach the SQL string, but the grain is NOT applied to any SQL this phase — + * see the seam in {@link renderDimensionClause}. + */ +const TIME_GRAIN_PATTERN = /^[a-z][a-z_]*$/; + +/** + * The exact twelve filter operators allowed at v1. The runtime tuple is the + * server-side source of truth; the client-side type union + * `MetricFilterOperatorName` mirrors these names statically. + */ +const METRIC_FILTER_OPERATORS = [ + "equals", + "notEquals", + "in", + "notIn", + "gt", + "gte", + "lt", + "lte", + "contains", + "notContains", + "set", + "notSet", +] as const satisfies readonly MetricFilterOperatorName[]; + +/** + * Maximum AND/OR nesting depth. The PRD documents 8 as a sensible cap — + * enough for any real BI filter UI, low enough that a hostile or malformed + * payload cannot stack-overflow the recursive validator or translator. + * + * The depth count is the number of nested `{ and }` / `{ or }` wrappers + * encountered while descending — leaf predicates do not count toward depth. + */ +const METRIC_FILTER_MAX_DEPTH = 8; + +/** + * Cardinality caps on user-controlled arrays. Closes the recurring + * `unbounded-request-parameters` finding: a hostile caller could otherwise + * send `values: [...10M items...]` and exhaust the validator + the named + * bind-var binding step. The limits below are deliberately generous — higher + * than any real BI UI would emit — so legitimate traffic never trips them. + */ +const METRIC_MEASURES_MAX = 50; +const METRIC_DIMENSIONS_MAX = 20; +const METRIC_FILTER_VALUES_MAX = 1000; +const METRIC_LIMIT_MAX = 100_000; + +/** + * Maximum number of children per AND/OR group node. Without this cap a single + * flat group like `{ and: [...10M empty objects...] }` would push tens of + * millions of frames onto the iterative pre-check's stack — OOM before + * validation even gets to Zod. The Zod schema enforces the same cap so the + * rejection point is consistent regardless of which validator catches it + * first. + */ +const METRIC_FILTER_GROUP_MAX = 100; + +/** Operators that require exactly one value. */ +const SINGLE_VALUE_OPERATORS = new Set([ + "equals", + "notEquals", + "gt", + "gte", + "lt", + "lte", + "contains", + "notContains", +]); + +/** Operators that require at least one value. */ +const LIST_VALUE_OPERATORS = new Set(["in", "notIn"]); + +/** Operators that reject `values` entirely. */ +const NULL_OPERATORS = new Set(["set", "notSet"]); + +/** Operators that emit `LIKE` / `NOT LIKE` and require a string value. */ +const STRING_OPERATORS = new Set([ + "contains", + "notContains", +]); + /** * Map an entry's declared `executor` to the internal execution lane: * - `"user"` → `"obo"` (per-user cache, on-behalf-of) @@ -138,33 +235,311 @@ function assertSafeFqn(fqn: string): void { /** * Structural validation schema for the metric request body. * - * The schema is **static** — any grammar-valid measure identifier is accepted; - * it is NOT a dynamic per-key `z.enum(knownMeasures)`. Unknown-but-well-formed - * names are not rejected here; they reach the warehouse and surface as a - * sanitized canonical error. This is the measures-only shape; - * dimensions/filter/timeGrain are accepted structurally (their SQL is built in - * a later phase) but left permissive here. + * The schema is **static** — any grammar-valid measure/dimension identifier is + * accepted; it is NOT a dynamic per-key `z.enum(knownMeasures)`. Unknown-but- + * well-formed names are not rejected here; they reach the warehouse and surface + * as a sanitized canonical error. The identifier grammar gate is enforced in + * {@link buildMetricSql}/{@link renderFilter} at interpolation time; the body + * schema stays structural (item shape, cardinality caps, operator enum, + * per-operator value cardinality, and AND/OR depth). + * + * `filter` is recursive (`Predicate | { and: [...] } | { or: [...] }`), built + * with `z.lazy`. `timeGrain` is accepted as a grammar-shaped token but its SQL + * application is deferred (see {@link renderDimensionClause}). */ + +/** A leaf predicate: `{ member, operator, values? }`, no extra keys. */ +const filterPredicateSchema: z.ZodType = z + .object({ + member: z + .string() + .min(1, { message: "filter predicate 'member' cannot be empty" }), + operator: z.string().min(1, { + message: "filter predicate 'operator' cannot be empty", + }) as z.ZodType, + values: z + .array(z.union([z.string(), z.number()])) + .max(METRIC_FILTER_VALUES_MAX, { + message: `filter predicate 'values' length exceeds the maximum of ${METRIC_FILTER_VALUES_MAX}`, + }) + .optional(), + }) + .strict(); + +/** Recursive filter: a predicate leaf or an `{ and }` / `{ or }` group. */ +const filterSchema: z.ZodType = z.lazy(() => + z.union([ + filterPredicateSchema, + z + .object({ + and: z.array(filterSchema).max(METRIC_FILTER_GROUP_MAX, { + message: `filter 'and' group exceeds the maximum of ${METRIC_FILTER_GROUP_MAX} children`, + }), + }) + .strict(), + z + .object({ + or: z.array(filterSchema).max(METRIC_FILTER_GROUP_MAX, { + message: `filter 'or' group exceeds the maximum of ${METRIC_FILTER_GROUP_MAX} children`, + }), + }) + .strict(), + ]), +); + const metricRequestSchema = z .object({ measures: z .array(z.string().min(1, "measure name cannot be empty")) - .min(1, "at least one measure is required"), - dimensions: z.array(z.string().min(1)).optional(), - filter: z.unknown().optional(), - timeGrain: z.string().min(1).optional(), - limit: z.number().int().positive().optional(), + .min(1, "at least one measure is required") + .max(METRIC_MEASURES_MAX, { + message: `measures length exceeds the maximum of ${METRIC_MEASURES_MAX}`, + }), + dimensions: z + .array(z.string().min(1, "dimension name cannot be empty")) + .max(METRIC_DIMENSIONS_MAX, { + message: `dimensions length exceeds the maximum of ${METRIC_DIMENSIONS_MAX}`, + }) + .optional(), + filter: filterSchema.optional(), + // Grammar-shaped only: the token is validated for safety but its SQL is + // not built this phase (grain target TBD). + timeGrain: z + .string() + .min(1, { message: "timeGrain cannot be empty" }) + .regex(TIME_GRAIN_PATTERN, { + message: "timeGrain must match /^[a-z][a-z_]*$/", + }) + .optional(), + limit: z + .number() + .int({ message: "limit must be an integer" }) + .positive({ message: "limit must be positive" }) + .max(METRIC_LIMIT_MAX, { + message: `limit exceeds the maximum of ${METRIC_LIMIT_MAX}`, + }) + .optional(), format: z.enum(["JSON_ARRAY", "ARROW_STREAM", "JSON", "ARROW"]).optional(), }) - .strict(); + .strict() + .superRefine((value, ctx) => { + if (value.filter != null) { + validateFilterTree(value.filter, ctx, ["filter"], 0); + } + }) as z.ZodType; + +/** + * Recursive Zod-time validator for the filter tree. + * + * Pushes structured issues into the refinement context with stable paths + * (`filter.and.0.or.2.member`, etc.) so the canonical 400 error carries + * actionable diagnostics. Keeps the registry-free concerns in one descent: + * + * 1. Operator is one of the twelve. + * 2. Per-operator value cardinality (single / list / null operators). + * 3. `contains`/`notContains` carry a string value. + * 4. AND/OR nesting depth cap. + * + * There is NO member-allowlist and NO op⇄dimension-type check — the member is + * grammar-gated at SQL-build time, and dimension types are not tracked (no + * registry metadata). Returns void; issues accumulate on `ctx`. + */ +function validateFilterTree( + node: MetricFilter, + ctx: z.RefinementCtx, + path: Array, + depth: number, +): void { + if (node === null || typeof node !== "object") { + // The base schema rejects this via the union, but stay defensive. + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path, + message: "filter node must be a Predicate or { and } / { or } group", + }); + return; + } + + if ("and" in node || "or" in node) { + if (depth + 1 > METRIC_FILTER_MAX_DEPTH) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path, + message: `filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}`, + }); + return; + } + + const groupKey = "and" in node ? "and" : "or"; + const children = ( + node as { and?: ReadonlyArray } & { + or?: ReadonlyArray; + } + )[groupKey]; + + if (!Array.isArray(children)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, groupKey], + message: `filter ${groupKey} group must be an array of predicates or nested groups`, + }); + return; + } + + // Reject empty `or` groups: an empty disjunction is vacuously false, which + // silently drops the surrounding intent. Empty `and` is OK (vacuously true + // → no constraint). Forcing the caller to omit the predicate entirely is + // the only unambiguous choice. + if (groupKey === "or" && children.length === 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "or"], + message: "filter 'or' group must contain at least one predicate", + }); + return; + } + + children.forEach((child, idx) => { + validateFilterTree(child, ctx, [...path, groupKey, idx], depth + 1); + }); + return; + } + + // Leaf predicate. The base schema already enforced shape; here we layer in + // the operator vocabulary + per-operator value cardinality. + const predicate = node as MetricPredicate; + + if ( + !METRIC_FILTER_OPERATORS.includes( + predicate.operator as MetricFilterOperatorName, + ) + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "operator"], + message: `filter operator "${predicate.operator}" is not one of: ${METRIC_FILTER_OPERATORS.join(", ")}`, + }); + // No further checks meaningful when the operator is unknown. + return; + } + + const op = predicate.operator; + const values = predicate.values; + const valuesLen = values?.length ?? 0; + + if (NULL_OPERATORS.has(op)) { + if (values != null && valuesLen > 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" must not carry values`, + }); + } + } else if (SINGLE_VALUE_OPERATORS.has(op)) { + if (valuesLen !== 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" requires exactly one value (got ${valuesLen})`, + }); + } + } else if (LIST_VALUE_OPERATORS.has(op)) { + if (valuesLen < 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" requires at least one value`, + }); + } + } + + // Op⇄value-type: string operators require a string value. This is a + // registry-free structural check (not an op⇄dimension-type check) — it + // converts what would otherwise be a synchronous throw in `renderPredicate` + // (rendered as a 500) into a clean 400 at validation time. + if (STRING_OPERATORS.has(op) && valuesLen > 0) { + const v = predicate.values?.[0]; + if (typeof v !== "string") { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" requires a string value (got ${typeof v})`, + }); + } + } +} + +/** + * Iterative pre-parse depth check. Zod's union/object parsers walk the input + * recursively before our `superRefine` depth cap fires, so a deeply-nested + * `{ and: [{ and: [...] }] }` payload could stack-overflow during parse, never + * reaching the validator's depth check. This walk is iterative (explicit + * stack) and aborts as soon as `METRIC_FILTER_MAX_DEPTH` is exceeded, so a + * hostile payload of any size cannot drive the call stack. + * + * Walks BOTH `and` and `or` branches when both are present on the same node — + * Zod's `.strict()` rejects the multi-key shape downstream, but the pre-check + * has to inspect every branch Zod might recurse into (an earlier version used + * `else if` and was bypassed by `{ and: [], or: }`). + * + * Group-children breadth is also capped: a flat `{ and: [...10M items...] }` + * cannot push 10M frames onto the explicit stack here. Predicate leaves do NOT + * count toward depth — only nested `and` / `or` wrappers — matching the rule + * {@link validateFilterTree} enforces. + */ +function preCheckFilterDepth(filter: unknown): void { + if (filter == null || typeof filter !== "object") return; + const stack: Array<[unknown, number]> = [[filter, 0]]; + while (stack.length > 0) { + const popped = stack.pop(); + if (popped === undefined) continue; + const [node, depth] = popped; + if (node == null || typeof node !== "object") continue; + const obj = node as Record; + for (const groupKey of ["and", "or"] as const) { + const children = obj[groupKey]; + if (!Array.isArray(children)) continue; + if (children.length > METRIC_FILTER_GROUP_MAX) { + throw new ValidationError( + "Invalid metric request body (fields: filter)", + { + context: { + reason: `filter ${groupKey} group has ${children.length} children; the maximum is ${METRIC_FILTER_GROUP_MAX}`, + }, + }, + ); + } + if (depth + 1 > METRIC_FILTER_MAX_DEPTH) { + throw new ValidationError( + "Invalid metric request body (fields: filter)", + { + context: { + reason: `filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}`, + }, + }, + ); + } + for (const child of children) { + stack.push([child, depth + 1]); + } + } + } +} /** * Validate a `POST /api/analytics/metric/:key` request body against the static - * measures-only shape. Throws {@link ValidationError} (a 400 on the canonical + * structured shape. Throws {@link ValidationError} (a 400 on the canonical * error path) with the offending field paths; the raw values stay in telemetry * context, never the public body. + * + * The recursion depth is bounded BEFORE Zod sees the input — the schema's own + * depth check fires inside `superRefine`, which only runs after Zod's recursive + * parse has already walked the tree on the call stack. */ export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { + if (body != null && typeof body === "object") { + preCheckFilterDepth((body as { filter?: unknown }).filter); + } const result = metricRequestSchema.safeParse(body); if (!result.success) { const fieldPaths = result.error.issues @@ -181,19 +556,35 @@ export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { } /** - * Construct the measures-only metric SQL. + * Construct the metric SQL. * * Shape: * - * SELECT MEASURE(m) AS m[, …] FROM [LIMIT n] + * SELECT MEASURE(m) AS m[, …][, …] + * FROM + * [WHERE ] + * [GROUP BY ALL] + * [LIMIT n] + * + * Notes: + * - Every measure and dimension is gated by {@link MEASURE_NAME_PATTERN} / + * {@link DIMENSION_NAME_PATTERN} before it is interpolated (column + * references cannot be parameterized — they are SQL identifiers), and the + * FQN is re-checked by {@link assertSafeFqn}. There is deliberately NO name + * allowlist — the grammar gate is the security boundary. No user-supplied + * string reaches the SQL string without passing a grammar gate. + * - `GROUP BY ALL` is added when at least one dimension is requested. UC + * requires GROUP BY when MEASURE() is mixed with non-aggregated columns; + * `GROUP BY ALL` is the documented form that works without re-listing each + * dimension. + * - The `WHERE` clause is rendered from the recursive filter tree. Every value + * flows through Statement Execution's named bind-var path (`:f_`); no + * value is ever interpolated as a literal. + * - `timeGrain` currently has NO effect on the emitted SQL — see the seam in + * {@link renderDimensionClause}. * - * Every measure is gated by {@link MEASURE_NAME_PATTERN} before it is - * interpolated (measures cannot be parameterized — they are SQL identifiers), - * and the FQN is re-checked by {@link assertSafeFqn}. No user-supplied string - * reaches the SQL string without passing a grammar gate. `dimensions`, - * `filter`, and `timeGrain` are ignored here — their SQL is built in a later - * phase. Returns `{ statement, parameters }` where `parameters` is the named - * bind-var dictionary the plugin's `query()` consumes (empty in this phase). + * Returns `{ statement, parameters }` where `parameters` is the named bind-var + * dictionary the plugin's `query()` consumes. */ export function buildMetricSql( registration: MetricRegistration, @@ -216,6 +607,15 @@ export function buildMetricSql( } } + const dimensions = request.dimensions ?? []; + for (const d of dimensions) { + if (!DIMENSION_NAME_PATTERN.test(d)) { + throw new Error( + `Refusing to build SQL: dimension "${d}" is not a valid identifier.`, + ); + } + } + // Deterministic order so cache keys collapse semantically equivalent calls. // Alias each measure to its plain name so result rows have keys matching the // registered measure (`{ arr: 1234 }`) rather than the SQL-function @@ -224,13 +624,320 @@ export function buildMetricSql( .sort() .map((m) => `MEASURE(${m}) AS ${m}`); - const selectList = measureClauses.join(", "); + const dimensionClauses = [...dimensions] + .sort() + .map((d) => renderDimensionClause(d, request.timeGrain)); + + const selectList = [...measureClauses, ...dimensionClauses].join(", "); + const groupByClause = dimensions.length > 0 ? " GROUP BY ALL" : ""; const limitClause = typeof request.limit === "number" && request.limit > 0 ? ` LIMIT ${Math.floor(request.limit)}` : ""; - const statement = `SELECT ${selectList} FROM ${registration.source}${limitClause}`; - return { statement, parameters: {} }; + // Filter translation. Every value is bound through `:f_` named params; + // every column identifier is grammar-gated in `renderFilter`. Empty filter or + // no filter → no WHERE. + const parameters: Record = {}; + let whereClause = ""; + if (request.filter !== undefined) { + const fragment = renderFilter(request.filter, parameters, { + counter: 0, + depth: 0, + }); + if (fragment !== null && fragment.length > 0) { + whereClause = ` WHERE ${fragment}`; + } + } + + const statement = `SELECT ${selectList} FROM ${registration.source}${whereClause}${groupByClause}${limitClause}`; + return { statement, parameters }; +} + +/** + * Mutable counter / depth threaded through {@link renderFilter}. Fresh per + * `buildMetricSql` call, so two requests never share bind-var indexes. + */ +interface FilterRenderState { + counter: number; + depth: number; +} + +/** + * Recursively render a filter tree into a SQL fragment, pushing bind values + * into `params` keyed by `:f_` names. + * + * Returns `null` for an empty group (no WHERE clause needed). The caller's + * `buildMetricSql` only emits `WHERE` when this returns a non-null, non-empty + * fragment. Empty `and: []` collapses to null (SQL's vacuous-truth semantics + * for AND); empty `or: []` renders `1 = 0` — the validator rejects `or: []` + * before the SQL builder, but if it slips through we render vacuous-false + * rather than dropping the predicate silently (defense in depth so a future + * validator bypass cannot turn `or: []` into "match everything"). + * + * Defense-in-depth: even though the request body's filter has already been + * validated, every member name is re-checked against {@link + * DIMENSION_NAME_PATTERN} here. If validation is ever bypassed, the SQL + * constructor still refuses to interpolate an unknown-shaped identifier. + */ +function renderFilter( + node: MetricFilter, + params: Record, + state: FilterRenderState, +): string | null { + if (node === null || typeof node !== "object") { + throw new Error( + "Refusing to build SQL: filter node must be an object Predicate or { and } / { or } group.", + ); + } + + if ("and" in node || "or" in node) { + const groupKey = "and" in node ? "and" : "or"; + if (state.depth + 1 > METRIC_FILTER_MAX_DEPTH) { + throw new Error( + `Refusing to build SQL: filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}.`, + ); + } + + const children = ( + node as { and?: ReadonlyArray } & { + or?: ReadonlyArray; + } + )[groupKey]; + + if (!Array.isArray(children) || children.length === 0) { + if (groupKey === "or") { + return "1 = 0"; + } + return null; + } + + // Sort-before-hash discipline: within a group, predicate leaves are + // stable-sorted by (member, operator) before contributing to the rendered + // fragment, so semantically equivalent calls produce the same SQL string + // and (downstream) the same cache key. + const sortedChildren = sortFilterChildren(children); + + const fragments: string[] = []; + const childState: FilterRenderState = { + counter: state.counter, + depth: state.depth + 1, + }; + for (const child of sortedChildren) { + const rendered = renderFilter(child, params, childState); + if (rendered != null && rendered.length > 0) { + fragments.push(rendered); + } + } + state.counter = childState.counter; + + if (fragments.length === 0) return null; + if (fragments.length === 1) return fragments[0]; + const joiner = groupKey === "and" ? " AND " : " OR "; + return `(${fragments.join(joiner)})`; + } + + // Leaf predicate — grammar-gate the member and operator, then render. + const predicate = node as MetricPredicate; + + if (!DIMENSION_NAME_PATTERN.test(predicate.member)) { + throw new Error( + `Refusing to build SQL: filter member "${predicate.member}" is not a valid identifier.`, + ); + } + if ( + !METRIC_FILTER_OPERATORS.includes( + predicate.operator as MetricFilterOperatorName, + ) + ) { + throw new Error( + `Refusing to build SQL: unknown filter operator "${predicate.operator}".`, + ); + } + + return renderPredicate(predicate, params, state); +} + +/** + * Stable-sort filter children inside an AND/OR group by `(member, operator)`. + * + * Predicates carry both fields and sort by their pair; nested groups sort + * after predicates and stay in their original relative order (a nested group + * is opaque from the outside — we cannot collapse it to a single key). This is + * the sort-before-hash invariant applied at the SQL-fragment level so + * downstream cache keys collapse semantically equivalent calls. + */ +function sortFilterChildren( + children: ReadonlyArray, +): MetricFilter[] { + const indexed = children.map((child, idx) => { + let key: string; + let isPredicate: boolean; + if ( + child !== null && + typeof child === "object" && + !("and" in child) && + !("or" in child) + ) { + const p = child as MetricPredicate; + key = `${p.member}${p.operator}`; + isPredicate = true; + } else { + // Nested groups don't have a single (member, operator) — keep their + // original index so multiple nested groups within the same parent remain + // stable relative to each other. + key = ""; + isPredicate = false; + } + return { child, idx, key, isPredicate }; + }); + + indexed.sort((a, b) => { + if (a.isPredicate && !b.isPredicate) return -1; + if (!a.isPredicate && b.isPredicate) return 1; + if (a.isPredicate && b.isPredicate) { + if (a.key < b.key) return -1; + if (a.key > b.key) return 1; + } + return a.idx - b.idx; + }); + + return indexed.map((entry) => entry.child); +} + +/** + * Translate a single predicate into a SQL fragment. + * + * Every value flows through a freshly-allocated `:f_` named bind var. + * Nothing from the request body is ever interpolated as a literal — the + * fragment carries identifiers (grammar-gated) and operators (whitelisted), + * then references the bind name for each value. + * + * `set` and `notSet` emit `IS NULL` / `IS NOT NULL` with no bind value. `in` + * and `notIn` emit `IN (:f_0, :f_1, ...)`. `contains` and `notContains` emit + * `LIKE :f_0` / `NOT LIKE :f_0` and pre-bind the value with `%` wrapping. + */ +function renderPredicate( + predicate: MetricPredicate, + params: Record, + state: FilterRenderState, +): string { + const col = predicate.member; + const op = predicate.operator; + const values = predicate.values ?? []; + + switch (op) { + case "equals": + return `${col} = ${bindValue(values[0], params, state)}`; + case "notEquals": + return `${col} <> ${bindValue(values[0], params, state)}`; + case "gt": + return `${col} > ${bindValue(values[0], params, state)}`; + case "gte": + return `${col} >= ${bindValue(values[0], params, state)}`; + case "lt": + return `${col} < ${bindValue(values[0], params, state)}`; + case "lte": + return `${col} <= ${bindValue(values[0], params, state)}`; + case "in": { + const placeholders = values.map((v) => bindValue(v, params, state)); + return `${col} IN (${placeholders.join(", ")})`; + } + case "notIn": { + const placeholders = values.map((v) => bindValue(v, params, state)); + return `${col} NOT IN (${placeholders.join(", ")})`; + } + case "contains": { + const raw = values[0]; + if (typeof raw !== "string") { + throw new Error( + `Refusing to build SQL: filter operator "contains" requires a string value (got ${typeof raw}).`, + ); + } + return `${col} LIKE ${bindLikeValue(raw, params, state)}`; + } + case "notContains": { + const raw = values[0]; + if (typeof raw !== "string") { + throw new Error( + `Refusing to build SQL: filter operator "notContains" requires a string value (got ${typeof raw}).`, + ); + } + return `${col} NOT LIKE ${bindLikeValue(raw, params, state)}`; + } + case "set": + return `${col} IS NOT NULL`; + case "notSet": + return `${col} IS NULL`; + default: { + // Exhaustiveness — the operator union is closed; if this is reached the + // operator vocabulary widened without updating the switch. + const _exhaustive: never = op; + throw new Error( + `Refusing to build SQL: unhandled filter operator "${_exhaustive as string}".`, + ); + } + } +} + +/** + * Allocate a fresh `:f_` bind name for `value`, push the typed marker into + * `params`, and return the placeholder string. Bumps the counter. + */ +function bindValue( + value: string | number | undefined, + params: Record, + state: FilterRenderState, +): string { + if (value === undefined) { + throw new Error( + "Refusing to build SQL: filter predicate is missing a required value.", + ); + } + const name = `f_${state.counter}`; + state.counter += 1; + if (typeof value === "number") { + params[name] = sqlHelpers.number(value); + } else if (typeof value === "string") { + params[name] = sqlHelpers.string(value); + } else { + throw new Error( + `Refusing to build SQL: filter value must be a string or number (got ${typeof value}).`, + ); + } + return `:${name}`; +} + +/** + * Like {@link bindValue}, but wraps the value in `%...%` for `LIKE` / + * `NOT LIKE`. SQL wildcards in the user-supplied string remain in the value + * (matching the documented "contains" semantics) — escape-on-receive could be + * added later as an opt-in if customers request strict-substring matching. + */ +function bindLikeValue( + value: string, + params: Record, + state: FilterRenderState, +): string { + const name = `f_${state.counter}`; + state.counter += 1; + params[name] = sqlHelpers.string(`%${value}%`); + return `:${name}`; +} + +/** + * Render a single SELECT-list clause for a dimension. + * + * timeGrain application deferred — see PR2 grain-target decision; dimension + * renders bare for now. When the grain target is settled, a time-typed + * dimension will render as `date_trunc('', ) AS ` here; the + * `timeGrain` token is already accepted (grammar-shaped) upstream so the wire + * contract is stable across that change. + */ +function renderDimensionClause( + dim: string, + _timeGrain: string | undefined, +): string { + return dim; } diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index b6b529a67..8c310ec4d 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -11,7 +11,11 @@ import { import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { ServiceContext } from "../../../context/service-context"; import { AnalyticsPlugin } from "../analytics"; -import { buildMetricSql, loadMetricRegistry } from "../metric"; +import { + buildMetricSql, + loadMetricRegistry, + validateMetricRequest, +} from "../metric"; import type { IAnalyticsConfig, MetricRegistration } from "../types"; // Mirror the analytics.test.ts CacheManager mock so the inner `execute`'s @@ -166,6 +170,109 @@ describe("analytics metric route (Phase 1)", () => { }); }); + // ── Phase 2: dimensions + GROUP BY ALL. No date_trunc — timeGrain is held + // this phase (grain target TBD), so dimensions render bare. + describe("buildMetricSql dimensions + GROUP BY", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + test("measures + dimensions → SELECT MEASURE(...), FROM GROUP BY ALL", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region"], + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr, region FROM cat.sch.revenue_metrics GROUP BY ALL", + ); + }); + + test("dimensions are sorted for a deterministic SELECT list", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["segment", "region"], + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr, region, segment FROM cat.sch.revenue_metrics GROUP BY ALL", + ); + }); + + test("measures sort before dimensions in the SELECT list", () => { + const { statement } = buildMetricSql(registration, { + measures: ["revenue", "arr"], + dimensions: ["region"], + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr, MEASURE(revenue) AS revenue, region FROM cat.sch.revenue_metrics GROUP BY ALL", + ); + }); + + test("empty dimensions array → no GROUP BY (ungrouped)", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: [], + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr FROM cat.sch.revenue_metrics", + ); + }); + + test("dimensions + limit compose GROUP BY ALL then LIMIT", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region"], + limit: 100, + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr, region FROM cat.sch.revenue_metrics GROUP BY ALL LIMIT 100", + ); + }); + + test("timeGrain is parsed but not yet applied (grain target TBD) — dimension renders bare, no date_trunc", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "month", + }); + // The held-timeGrain seam: the grain does not alter the SQL. The + // dimension is emitted bare — NOT wrapped in date_trunc(). + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr, order_date FROM cat.sch.revenue_metrics GROUP BY ALL", + ); + expect(statement).not.toContain("date_trunc"); + }); + }); + + // ── Phase 2: dimension grammar gate. A dimension failing + // DIMENSION_NAME_PATTERN throws inside buildMetricSql before SQL is built. + describe("buildMetricSql dimension grammar gate", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + test("throws before building SQL for an injection-shaped dimension", () => { + expect(() => + buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region; DROP TABLE users"], + }), + ).toThrow(/not a valid identifier/); + }); + + test("throws for a dimension with a backtick / quote", () => { + expect(() => + buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region`"], + }), + ).toThrow(/not a valid identifier/); + }); + }); + // ── Envelope parity — streams warehouse_status* then a `result` message, // byte-identical to the /query route's JSON SSE path. describe("_handleMetricRoute SSE envelope", () => { @@ -419,3 +526,656 @@ describe("loadMetricRegistry", () => { expect(() => loadMetricRegistry(dir)).toThrow(/Invalid metric-views.json/); }); }); + +// ── Phase 2: the structured filter engine (translator + validator). +// Registry-free: names are grammar-gated, values are parameterized. No +// allowlist, no op⇄dimension-type check. +describe("metric — filter translator", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + // Render a filter via buildMetricSql and return the WHERE fragment + params. + function render(filter: unknown) { + const { statement, parameters } = buildMetricSql(registration, { + measures: ["arr"], + filter: filter as never, + }); + const match = statement.match(/ WHERE (.+?)( GROUP BY| LIMIT|$)/); + const where = match ? match[1] : null; + return { statement, where, parameters }; + } + + describe("operators (12 unit tests)", () => { + test("equals → ` = :f_0`", () => { + const { where, parameters } = render({ + member: "region", + operator: "equals", + values: ["EMEA"], + }); + expect(where).toBe("region = :f_0"); + expect(parameters).toEqual({ + f_0: { __sql_type: "STRING", value: "EMEA" }, + }); + }); + + test("notEquals → ` <> :f_0`", () => { + const { where, parameters } = render({ + member: "region", + operator: "notEquals", + values: ["EMEA"], + }); + expect(where).toBe("region <> :f_0"); + expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "EMEA" }); + }); + + test("in → ` IN (:f_0, :f_1, ...)`", () => { + const { where, parameters } = render({ + member: "region", + operator: "in", + values: ["EMEA", "APAC", "AMER"], + }); + expect(where).toBe("region IN (:f_0, :f_1, :f_2)"); + expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "EMEA" }); + expect(parameters.f_1).toEqual({ __sql_type: "STRING", value: "APAC" }); + expect(parameters.f_2).toEqual({ __sql_type: "STRING", value: "AMER" }); + }); + + test("notIn → ` NOT IN (:f_0, :f_1, ...)`", () => { + const { where, parameters } = render({ + member: "region", + operator: "notIn", + values: ["EMEA", "APAC"], + }); + expect(where).toBe("region NOT IN (:f_0, :f_1)"); + expect(Object.keys(parameters)).toHaveLength(2); + }); + + test("gt → ` > :f_0` (numeric value bound as INT)", () => { + const { where, parameters } = render({ + member: "deal_size", + operator: "gt", + values: [10000], + }); + expect(where).toBe("deal_size > :f_0"); + expect(parameters.f_0).toEqual({ __sql_type: "INT", value: "10000" }); + }); + + test("gte → ` >= :f_0`", () => { + const { where } = render({ + member: "deal_size", + operator: "gte", + values: [5000], + }); + expect(where).toBe("deal_size >= :f_0"); + }); + + test("lt → ` < :f_0`", () => { + const { where } = render({ + member: "deal_size", + operator: "lt", + values: [100], + }); + expect(where).toBe("deal_size < :f_0"); + }); + + test("lte → ` <= :f_0`", () => { + const { where } = render({ + member: "deal_size", + operator: "lte", + values: [50000], + }); + expect(where).toBe("deal_size <= :f_0"); + }); + + test("contains → ` LIKE :f_0` (value wrapped in %...%)", () => { + const { where, parameters } = render({ + member: "region", + operator: "contains", + values: ["MEA"], + }); + expect(where).toBe("region LIKE :f_0"); + expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "%MEA%" }); + }); + + test("notContains → ` NOT LIKE :f_0`", () => { + const { where, parameters } = render({ + member: "region", + operator: "notContains", + values: ["test"], + }); + expect(where).toBe("region NOT LIKE :f_0"); + expect(parameters.f_0).toEqual({ + __sql_type: "STRING", + value: "%test%", + }); + }); + + test("set → ` IS NOT NULL` (no bind)", () => { + const { where, parameters } = render({ + member: "region", + operator: "set", + }); + expect(where).toBe("region IS NOT NULL"); + expect(parameters).toEqual({}); + }); + + test("notSet → ` IS NULL` (no bind)", () => { + const { where, parameters } = render({ + member: "region", + operator: "notSet", + }); + expect(where).toBe("region IS NULL"); + expect(parameters).toEqual({}); + }); + }); + + describe("AND/OR composition", () => { + test("flat AND group renders predicates joined by AND", () => { + const { where, parameters } = render({ + and: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { member: "segment", operator: "equals", values: ["Enterprise"] }, + ], + }); + expect(where).toBe("(region = :f_0 AND segment = :f_1)"); + expect(parameters.f_0.value).toBe("EMEA"); + expect(parameters.f_1.value).toBe("Enterprise"); + }); + + test("flat OR group renders predicates joined by OR", () => { + const { where } = render({ + or: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { member: "region", operator: "equals", values: ["APAC"] }, + ], + }); + expect(where).toBe("(region = :f_0 OR region = :f_1)"); + }); + + test("AND-of-OR composes nested groups", () => { + const { where } = render({ + and: [ + { member: "region", operator: "in", values: ["EMEA", "APAC"] }, + { + or: [ + { member: "segment", operator: "equals", values: ["Enterprise"] }, + { member: "deal_size", operator: "gt", values: [50000] }, + ], + }, + ], + }); + expect(where).toContain("(region IN ("); + expect(where).toContain(" AND "); + expect(where).toContain("OR"); + }); + + test("OR-of-AND composes nested groups", () => { + const { where } = render({ + or: [ + { + and: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { member: "segment", operator: "equals", values: ["Enterprise"] }, + ], + }, + { member: "region", operator: "equals", values: ["APAC"] }, + ], + }); + expect(where).toMatch(/^\(.+ OR .+\)$/); + expect(where).toContain(" AND "); + }); + + test("empty `and: []` group emits no WHERE clause", () => { + const { statement, parameters } = buildMetricSql(registration, { + measures: ["arr"], + filter: { and: [] }, + }); + expect(statement).not.toContain("WHERE"); + expect(parameters).toEqual({}); + }); + + test("empty `or: []` renders 1 = 0 (defense in depth past the validator)", () => { + // The validator rejects `or: []` (see cardinality tests), but if a bypass + // ever reaches the renderer, an empty disjunction must render vacuous- + // false — never drop the predicate (which would mean "match everything"). + const { where } = render({ or: [] }); + expect(where).toBe("1 = 0"); + }); + }); + + describe("depth cap", () => { + test("rejects 9 levels of AND nesting at preCheckFilterDepth (before Zod)", () => { + let node: unknown = { + member: "region", + operator: "equals", + values: ["EMEA"], + }; + for (let i = 0; i < 9; i += 1) { + node = { and: [node] }; + } + expect(() => + validateMetricRequest({ measures: ["arr"], filter: node }), + ).toThrowError(/fields:.*filter/); + }); + + test("accepts exactly 8 levels of AND nesting (validator)", () => { + let node: unknown = { + member: "region", + operator: "equals", + values: ["EMEA"], + }; + for (let i = 0; i < 8; i += 1) { + node = { and: [node] }; + } + expect(() => + validateMetricRequest({ measures: ["arr"], filter: node }), + ).not.toThrow(); + }); + + test("renderFilter independently rejects nesting past the depth cap", () => { + // Second, independent depth enforcement: even if a payload bypasses the + // pre-Zod pre-check and Zod's superRefine, the SQL renderer re-checks the + // depth as defense in depth. Build 9-deep and call buildMetricSql + // directly (skipping validateMetricRequest). + let node: unknown = { + member: "region", + operator: "equals", + values: ["EMEA"], + }; + for (let i = 0; i < 9; i += 1) { + node = { and: [node] }; + } + expect(() => + buildMetricSql(registration, { + measures: ["arr"], + filter: node as never, + }), + ).toThrow(/nesting exceeds the maximum depth/); + }); + + test("rejects pathologically deep filter without stack-overflow (pre-parse cap)", () => { + let node: unknown = { + member: "region", + operator: "equals", + values: ["EMEA"], + }; + for (let i = 0; i < 10_000; i += 1) { + node = { and: [node] }; + } + expect(() => + validateMetricRequest({ measures: ["arr"], filter: node }), + ).toThrowError(/fields:.*filter/); + }); + + test("rejects deep `or` even when paired with empty `and` (else-if bypass guard)", () => { + let deepOr: unknown = { + member: "region", + operator: "equals", + values: ["EMEA"], + }; + for (let i = 0; i < 10_000; i += 1) { + deepOr = { or: [deepOr] }; + } + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { and: [], or: [deepOr] } as never, + }), + ).toThrowError(/fields:.*filter/); + }); + + test("rejects breadth-DoS: a single group with too many children", () => { + const wide = Array.from({ length: 1000 }, () => ({ + member: "region", + operator: "equals" as const, + values: ["EMEA"], + })); + expect(() => + validateMetricRequest({ measures: ["arr"], filter: { and: wide } }), + ).toThrowError(/fields:.*filter/); + }); + }); + + describe("parameterization safety (no values in rendered SQL)", () => { + test("string values do not appear verbatim in the SQL string", () => { + const sneaky = "EMEA' OR '1'='1"; + const { statement } = render({ + member: "region", + operator: "equals", + values: [sneaky], + }); + expect(statement).not.toContain(sneaky); + expect(statement).toContain(":f_0"); + }); + + test("numeric values do not appear verbatim in the SQL string", () => { + const { statement } = render({ + member: "deal_size", + operator: "gt", + values: [987654321], + }); + expect(statement).not.toContain("987654321"); + expect(statement).toContain(":f_0"); + }); + + test("LIKE wildcard is the only value transformation; the original string is not in SQL", () => { + const { statement } = render({ + member: "region", + operator: "contains", + values: ["dangerous%"], + }); + expect(statement).not.toContain("dangerous"); + expect(statement).toContain(":f_0"); + }); + + test("IN values are individually bound (not concatenated)", () => { + const { statement, parameters } = render({ + member: "region", + operator: "in", + values: ["A", "B", "C"], + }); + expect(statement).not.toMatch(/region IN \([^:]/); + expect(Object.keys(parameters)).toHaveLength(3); + }); + + test("a filter member failing DIMENSION_NAME_PATTERN throws before SQL is built", () => { + expect(() => + render({ + member: "region; DROP TABLE foo --", + operator: "equals", + values: ["x"], + }), + ).toThrowError(/not a valid identifier/); + }); + }); + + describe("validator — operator + cardinality (registry-free)", () => { + test("rejects an unknown operator", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { + member: "region", + operator: "startsWith" as never, + values: ["E"], + }, + }), + ).toThrowError(/fields:.*filter\.operator/); + }); + + test("rejects equals with zero values (cardinality)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "equals", values: [] }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("rejects equals with multiple values (cardinality)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "equals", values: ["A", "B"] }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("rejects in with empty values (cardinality)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "in", values: [] }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("rejects set with values (cardinality — must be absent)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "set", values: ["EMEA"] }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("accepts set with no values", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "set" }, + }), + ).not.toThrow(); + }); + + test("accepts notSet with an empty values array", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "notSet", values: [] }, + }), + ).not.toThrow(); + }); + + test("rejects `contains` with a non-string value", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { + member: "region", + operator: "contains", + values: [42 as unknown as string], + }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("rejects a filter predicate with too many values (DoS guard)", () => { + const big = Array.from({ length: 2000 }, (_, i) => `v${i}`); + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "in", values: big }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("rejects a bare-array filter (not a Predicate or { and }/{ or } group)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: [ + { member: "region", operator: "in", values: ["EMEA"] }, + ] as never, + }), + ).toThrow(); + }); + + test("rejects empty `or` group (empty disjunction is vacuously false)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { or: [] }, + }), + ).toThrowError(/fields:.*filter\.or/); + }); + + test("accepts empty `and` group (no constraint contributed)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { and: [] }, + }), + ).not.toThrow(); + }); + + test("rejects an unknown operator at depth (nested filter)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { + and: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { + member: "segment", + operator: "matches" as never, + values: ["X"], + }, + ], + }, + }), + ).toThrowError(/fields:.*filter\.and\.1\.operator/); + }); + + test("well-formed-but-unknown member is NOT rejected locally (no allowlist)", () => { + // The dropped-allowlist posture: a grammar-valid member that isn't a + // real dimension passes validation AND SQL construction — it reaches the + // warehouse, which is the authority on whether the column exists. + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "ghost_column", operator: "equals", values: ["x"] }, + }), + ).not.toThrow(); + const { where } = render({ + member: "ghost_column", + operator: "equals", + values: ["x"], + }); + expect(where).toBe("ghost_column = :f_0"); + }); + }); + + describe("timeGrain (grammar-shaped, not yet applied)", () => { + test("a grammar-valid timeGrain is accepted structurally", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "month", + }), + ).not.toThrow(); + }); + + test("a timeGrain failing TIME_GRAIN_PATTERN is rejected", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "MONTH; DROP TABLE t", + }), + ).toThrowError(/fields:.*timeGrain/); + }); + + test("timeGrain with no dimensions is accepted (held: no cross-field rule this phase)", () => { + // The #341 '400 when timeGrain set with no time-typed dimension' rule is + // deliberately NOT implemented — the grain target is TBD. The grammar + // shape is still enforced; the token simply does not alter SQL. + expect(() => + validateMetricRequest({ measures: ["arr"], timeGrain: "day" }), + ).not.toThrow(); + }); + }); + + describe("sort-before-hash (predicate ordering inside groups)", () => { + test("predicate order does not affect the rendered SQL within an AND group", () => { + const a = render({ + and: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { member: "segment", operator: "equals", values: ["Ent"] }, + ], + }); + const b = render({ + and: [ + { member: "segment", operator: "equals", values: ["Ent"] }, + { member: "region", operator: "equals", values: ["EMEA"] }, + ], + }); + expect(a.where).toBe(b.where); + }); + }); + + // The warehouse-authoritative unknown-name parity test (sanitized + // clientMessage/errorCode envelope) lands here because the Phase 1 harness + // can drive the metric route end-to-end and assert on the SSE error bytes. + describe("warehouse-authoritative unknown-name parity", () => { + let config: IAnalyticsConfig; + let serviceContextMock: Awaited>; + + beforeEach(async () => { + config = { timeout: 5000 }; + setupDatabricksEnv(); + mockCacheStore.clear(); + ServiceContext.reset(); + serviceContextMock = await mockServiceContext(); + }); + + afterEach(() => { + serviceContextMock?.restore(); + }); + + test("a well-formed-but-unknown measure reaches the warehouse and surfaces a sanitized error envelope", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + + // The warehouse rejects the unknown column. The raw text carries the + // offending name; the connector wraps it in an ExecutionError whose + // `.message` is server-log-only and whose `clientMessage` is sanitized. + const { ExecutionError } = await import("../../../errors/execution"); + const rawWarehouseText = + "[UNRESOLVED_COLUMN] A column with name `ghost_measure` cannot be resolved"; + const executeMock = vi + .fn() + .mockRejectedValue( + ExecutionError.statementFailed(rawWarehouseText, "UNRESOLVED_COLUMN"), + ); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + // grammar-valid, but not a real measure — NOT rejected locally. + body: { measures: ["ghost_measure"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + // The well-formed-unknown measure was NOT rejected locally — it reached + // the warehouse (parity with the raw `.sql` flow). + expect(executeMock).toHaveBeenCalled(); + expect(executeMock.mock.calls[0][1]).toEqual( + expect.objectContaining({ + statement: + "SELECT MEASURE(ghost_measure) AS ghost_measure FROM cat.sch.revenue_metrics", + }), + ); + + // The SSE error envelope carries a sanitized clientMessage + structured + // errorCode — the raw warehouse text (with the offending column name) + // never reaches the client payload (#329 scrubbing, inherited). + const errorWrites = (mockRes.write as any).mock.calls + .map((call: any[]) => call[0] as string) + .filter((s: string) => s.startsWith("data: ")); + const errorPayload = errorWrites.find((s: string) => + s.includes('"errorCode"'), + ); + expect(errorPayload).toBeTruthy(); + expect(errorPayload).toContain('"errorCode":"UNRESOLVED_COLUMN"'); + expect(errorPayload).toContain('"error":"Query execution failed"'); + expect(errorPayload).not.toContain("ghost_measure"); + expect(errorPayload).not.toContain("UNRESOLVED_COLUMN]"); + }); + }); +}); diff --git a/packages/appkit/src/plugins/analytics/types.ts b/packages/appkit/src/plugins/analytics/types.ts index c032e6cca..8812020ea 100644 --- a/packages/appkit/src/plugins/analytics/types.ts +++ b/packages/appkit/src/plugins/analytics/types.ts @@ -142,18 +142,62 @@ export interface MetricRegistration { lane: MetricLane; } +/** + * v1 filter operator vocabulary — exactly twelve names. The runtime tuple + * `METRIC_FILTER_OPERATORS` (next to the validator in `metric.ts`) is the + * server-side source of truth; this union mirrors it statically. + */ +export type MetricFilterOperatorName = + | "equals" + | "notEquals" + | "in" + | "notIn" + | "gt" + | "gte" + | "lt" + | "lte" + | "contains" + | "notContains" + | "set" + | "notSet"; + +/** + * A single filter predicate — the leaf node of the recursive + * {@link MetricFilter} tree. `member` is a dimension name (grammar-gated, not + * allowlisted); `values` is bound through parameterized `:f_` bind vars + * and never interpolated into the SQL string. + */ +export interface MetricPredicate { + member: string; + operator: MetricFilterOperatorName; + values?: ReadonlyArray; +} + +/** + * Recursive filter expression for the metric-view request body: a leaf + * {@link MetricPredicate} or an `{ and: [...] }` / `{ or: [...] }` group. The + * shape is intentionally non-generic server-side — per-metric narrowing (if + * any) lives client-side. + */ +export type MetricFilter = + | MetricPredicate + | { and: ReadonlyArray } + | { or: ReadonlyArray }; + /** * Validated request body for `POST /api/analytics/metric/:key`. * - * `measures` is required; `dimensions`, `filter`, and `timeGrain` are part of - * the wire shape but their SQL is not built yet (a later phase adds grouping, - * time-grain truncation, and the structured filter translator). `filter` is - * typed `unknown` until that phase pins down the recursive predicate shape. + * `measures` is required. `dimensions` drive `GROUP BY ALL`; `filter` is the + * recursive structured predicate tree translated into a parameterized `WHERE` + * clause. `timeGrain` is accepted structurally (grammar-shaped) but its SQL + * application is deferred — the runtime target of the grain is an open design + * question being resolved separately, so it currently does not alter the + * emitted SQL. */ export interface IAnalyticsMetricRequest { measures: string[]; dimensions?: string[]; - filter?: unknown; + filter?: MetricFilter; timeGrain?: string; limit?: number; format?: AnalyticsFormat; From 9d2065c0bf0391050d6ab162f2aabee58bdf8268 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Thu, 9 Jul 2026 22:55:51 +0200 Subject: [PATCH 03/18] chore(appkit): apply timeGrain via explicit timeDimension (PR2 phase 2b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves the grain-target gap: PR2 drops the registry that #341 used to infer which dimension was temporal, so the grain's target is named explicitly. New optional `timeDimension` request field; when `timeGrain` is set, that column renders as date_trunc('', ) AS (grain single-quoted literal gated by TIME_GRAIN_PATTERN; column gated by DIMENSION_NAME_PATTERN — neither can be a bind param). Other dimensions render bare. Two 400 rules in the request schema superRefine: timeGrain without timeDimension; timeDimension not in dimensions. Warehouse remains the authority on column temporality. Deviates from the PRD's "date_trunc on time-typed dimensions" wording (which presupposed the deleted registry) — the explicit-field shape is the runtime query shape PR2 settles; PR5's hook contract inherits the timeDimension arg. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../appkit/src/plugins/analytics/metric.ts | 92 +++++++++++--- .../plugins/analytics/tests/metric.test.ts | 116 +++++++++++++++--- .../appkit/src/plugins/analytics/types.ts | 14 ++- 3 files changed, 186 insertions(+), 36 deletions(-) diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts index e2b03c977..bce144cd2 100644 --- a/packages/appkit/src/plugins/analytics/metric.ts +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -61,9 +61,10 @@ const MEASURE_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; const DIMENSION_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; /** - * Time-grain token shape. Accepted structurally so a hostile token can never - * reach the SQL string, but the grain is NOT applied to any SQL this phase — - * see the seam in {@link renderDimensionClause}. + * Time-grain token shape. This grammar gate is the security boundary for the + * grain: it is interpolated as a single-quoted `date_trunc` unit literal (NOT + * a bind param) in {@link renderDimensionClause}, so the pattern is what keeps + * a hostile token out of that quoted position. */ const TIME_GRAIN_PATTERN = /^[a-z][a-z_]*$/; @@ -244,8 +245,10 @@ function assertSafeFqn(fqn: string): void { * per-operator value cardinality, and AND/OR depth). * * `filter` is recursive (`Predicate | { and: [...] } | { or: [...] }`), built - * with `z.lazy`. `timeGrain` is accepted as a grammar-shaped token but its SQL - * application is deferred (see {@link renderDimensionClause}). + * with `z.lazy`. `timeGrain` buckets `timeDimension` via `date_trunc` (see + * {@link renderDimensionClause}); the two cross-field rules (grain requires + * timeDimension; timeDimension must be one of dimensions) live in the + * `superRefine` below. */ /** A leaf predicate: `{ member, operator, values? }`, no extra keys. */ @@ -302,8 +305,10 @@ const metricRequestSchema = z }) .optional(), filter: filterSchema.optional(), - // Grammar-shaped only: the token is validated for safety but its SQL is - // not built this phase (grain target TBD). + // Grammar-shaped bucketing grain, applied to `timeDimension` via + // `date_trunc`. The token is validated for safety here; the grain literal + // is interpolated (single-quoted, never a bind param) in + // `renderDimensionClause`, so this pattern gate is the security boundary. timeGrain: z .string() .min(1, { message: "timeGrain cannot be empty" }) @@ -311,6 +316,17 @@ const metricRequestSchema = z message: "timeGrain must match /^[a-z][a-z_]*$/", }) .optional(), + // The single dimension `timeGrain` applies to via `date_trunc`. Grammar- + // gated as a SQL identifier (column reference, cannot be parameterized). + // Cross-field rules in `superRefine`: required when `timeGrain` is set, and + // must be one of `dimensions`. + timeDimension: z + .string() + .min(1, { message: "timeDimension cannot be empty" }) + .regex(DIMENSION_NAME_PATTERN, { + message: "timeDimension must match /^[a-zA-Z_][a-zA-Z0-9_]*$/", + }) + .optional(), limit: z .number() .int({ message: "limit must be an integer" }) @@ -326,6 +342,28 @@ const metricRequestSchema = z if (value.filter != null) { validateFilterTree(value.filter, ctx, ["filter"], 0); } + + // Cross-field grain rules. `timeGrain` buckets exactly one selected + // dimension via `date_trunc`, so it requires an explicit `timeDimension`, + // and that dimension must be selected (so it is in the SELECT list and + // `GROUP BY ALL`). + if (value.timeGrain != null && value.timeDimension == null) { + ctx.addIssue({ + code: "custom", + message: "timeDimension is required when timeGrain is set", + path: ["timeDimension"], + }); + } + if ( + value.timeDimension != null && + !(value.dimensions ?? []).includes(value.timeDimension) + ) { + ctx.addIssue({ + code: "custom", + message: "timeDimension must be one of dimensions", + path: ["timeDimension"], + }); + } }) as z.ZodType; /** @@ -580,8 +618,10 @@ export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { * - The `WHERE` clause is rendered from the recursive filter tree. Every value * flows through Statement Execution's named bind-var path (`:f_`); no * value is ever interpolated as a literal. - * - `timeGrain` currently has NO effect on the emitted SQL — see the seam in - * {@link renderDimensionClause}. + * - When `timeGrain` is set, the `timeDimension` column renders as + * `date_trunc('', ) AS ` (the grain is a grammar-gated + * single-quoted literal, not a bind param); other dimensions render bare — + * see {@link renderDimensionClause}. * * Returns `{ statement, parameters }` where `parameters` is the named bind-var * dictionary the plugin's `query()` consumes. @@ -626,7 +666,9 @@ export function buildMetricSql( const dimensionClauses = [...dimensions] .sort() - .map((d) => renderDimensionClause(d, request.timeGrain)); + .map((d) => + renderDimensionClause(d, request.timeGrain, request.timeDimension), + ); const selectList = [...measureClauses, ...dimensionClauses].join(", "); const groupByClause = dimensions.length > 0 ? " GROUP BY ALL" : ""; @@ -929,15 +971,33 @@ function bindLikeValue( /** * Render a single SELECT-list clause for a dimension. * - * timeGrain application deferred — see PR2 grain-target decision; dimension - * renders bare for now. When the grain target is settled, a time-typed - * dimension will render as `date_trunc('', ) AS ` here; the - * `timeGrain` token is already accepted (grammar-shaped) upstream so the wire - * contract is stable across that change. + * When `timeGrain` is set and `dim` is the request's `timeDimension`, the + * column is bucketed: `date_trunc('', ) AS `. Every other + * dimension renders bare. Both the grain literal and the column identifier are + * grammar-gated before they reach the SQL string: + * + * - The grain is single-quoted (it is a `date_trunc` unit literal, NOT a bind + * param) and `TIME_GRAIN_PATTERN` upstream is the control that keeps a + * hostile token out of that quoted position. + * - The column cannot be parameterized (it is an identifier), so it is + * re-checked against {@link DIMENSION_NAME_PATTERN} here as belt-and- + * suspenders even though the schema already gated `timeDimension`. + * + * The aliasing keeps result-row keys stable (`{ order_date: ... }`) regardless + * of whether the grain was applied. */ function renderDimensionClause( dim: string, - _timeGrain: string | undefined, + timeGrain: string | undefined, + timeDimension: string | undefined, ): string { + if (timeGrain != null && dim === timeDimension) { + if (!DIMENSION_NAME_PATTERN.test(dim)) { + throw new Error( + `Refusing to build SQL: timeDimension "${dim}" is not a valid identifier.`, + ); + } + return `date_trunc('${timeGrain}', ${dim}) AS ${dim}`; + } return dim; } diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index 8c310ec4d..50efd5340 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -170,8 +170,8 @@ describe("analytics metric route (Phase 1)", () => { }); }); - // ── Phase 2: dimensions + GROUP BY ALL. No date_trunc — timeGrain is held - // this phase (grain target TBD), so dimensions render bare. + // ── Phase 2: dimensions + GROUP BY ALL. Bare dimensions here; date_trunc + // grain application (via timeDimension) is covered in its own block below. describe("buildMetricSql dimensions + GROUP BY", () => { const registration: MetricRegistration = { key: "revenue", @@ -230,21 +230,57 @@ describe("analytics metric route (Phase 1)", () => { ); }); - test("timeGrain is parsed but not yet applied (grain target TBD) — dimension renders bare, no date_trunc", () => { + test("no timeGrain → all dimensions render bare (no date_trunc)", () => { const { statement } = buildMetricSql(registration, { measures: ["arr"], - dimensions: ["order_date"], - timeGrain: "month", + dimensions: ["order_date", "region"], }); - // The held-timeGrain seam: the grain does not alter the SQL. The - // dimension is emitted bare — NOT wrapped in date_trunc(). expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, order_date FROM cat.sch.revenue_metrics GROUP BY ALL", + "SELECT MEASURE(arr) AS arr, order_date, region FROM cat.sch.revenue_metrics GROUP BY ALL", ); expect(statement).not.toContain("date_trunc"); }); }); + // ── Phase 2a: timeGrain + timeDimension → date_trunc on the named column. + // The grain is a grammar-gated single-quoted literal; the column keeps its + // plain alias; other dimensions render bare; GROUP BY ALL is present. + describe("buildMetricSql timeGrain + timeDimension (date_trunc)", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + test("buckets only the timeDimension via date_trunc; other dimensions bare", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["order_date", "region"], + timeGrain: "month", + timeDimension: "order_date", + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr, date_trunc('month', order_date) AS order_date, region FROM cat.sch.revenue_metrics GROUP BY ALL", + ); + expect(statement).toContain( + "date_trunc('month', order_date) AS order_date", + ); + expect(statement).toContain(" GROUP BY ALL"); + }); + + test("the grain literal is threaded through (day) for the timeDimension", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "day", + timeDimension: "order_date", + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr, date_trunc('day', order_date) AS order_date FROM cat.sch.revenue_metrics GROUP BY ALL", + ); + }); + }); + // ── Phase 2: dimension grammar gate. A dimension failing // DIMENSION_NAME_PATTERN throws inside buildMetricSql before SQL is built. describe("buildMetricSql dimension grammar gate", () => { @@ -1049,13 +1085,14 @@ describe("metric — filter translator", () => { }); }); - describe("timeGrain (grammar-shaped, not yet applied)", () => { - test("a grammar-valid timeGrain is accepted structurally", () => { + describe("timeGrain + timeDimension (validator)", () => { + test("a grammar-valid timeGrain + timeDimension (in dimensions) is accepted", () => { expect(() => validateMetricRequest({ measures: ["arr"], dimensions: ["order_date"], timeGrain: "month", + timeDimension: "order_date", }), ).not.toThrow(); }); @@ -1066,17 +1103,64 @@ describe("metric — filter translator", () => { measures: ["arr"], dimensions: ["order_date"], timeGrain: "MONTH; DROP TABLE t", + timeDimension: "order_date", }), ).toThrowError(/fields:.*timeGrain/); }); - test("timeGrain with no dimensions is accepted (held: no cross-field rule this phase)", () => { - // The #341 '400 when timeGrain set with no time-typed dimension' rule is - // deliberately NOT implemented — the grain target is TBD. The grammar - // shape is still enforced; the token simply does not alter SQL. + test("a capitalized grain (Month) is rejected by the grammar gate", () => { expect(() => - validateMetricRequest({ measures: ["arr"], timeGrain: "day" }), - ).not.toThrow(); + validateMetricRequest({ + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "Month", + timeDimension: "order_date", + }), + ).toThrowError(/fields:.*timeGrain/); + }); + + test("a timeDimension failing DIMENSION_NAME_PATTERN is rejected", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "month", + timeDimension: "order_date; DROP TABLE t", + }), + ).toThrowError(/fields:.*timeDimension/); + }); + + test("timeGrain without timeDimension → 400 (grain requires a target)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "day", + }), + ).toThrowError(/fields:.*timeDimension/); + }); + + test("timeDimension not in dimensions → 400 (must be selectable + in GROUP BY ALL)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["region"], + timeGrain: "month", + timeDimension: "order_date", + }), + ).toThrowError(/fields:.*timeDimension/); + }); + + test("timeDimension not in dimensions → 400 even without timeGrain", () => { + // The dimensions-membership rule holds independent of timeGrain: a + // timeDimension that isn't selected can never be bucketed or grouped. + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["region"], + timeDimension: "order_date", + }), + ).toThrowError(/fields:.*timeDimension/); }); }); diff --git a/packages/appkit/src/plugins/analytics/types.ts b/packages/appkit/src/plugins/analytics/types.ts index 8812020ea..e91a361f8 100644 --- a/packages/appkit/src/plugins/analytics/types.ts +++ b/packages/appkit/src/plugins/analytics/types.ts @@ -189,16 +189,22 @@ export type MetricFilter = * * `measures` is required. `dimensions` drive `GROUP BY ALL`; `filter` is the * recursive structured predicate tree translated into a parameterized `WHERE` - * clause. `timeGrain` is accepted structurally (grammar-shaped) but its SQL - * application is deferred — the runtime target of the grain is an open design - * question being resolved separately, so it currently does not alter the - * emitted SQL. + * clause. `timeGrain` buckets the single dimension named by `timeDimension` + * via `date_trunc`; it requires `timeDimension`, and `timeDimension` must be + * one of `dimensions` so it is selected and in `GROUP BY ALL`. Both tokens are + * grammar-gated before they reach SQL. */ export interface IAnalyticsMetricRequest { measures: string[]; dimensions?: string[]; filter?: MetricFilter; timeGrain?: string; + /** + * The single dimension that `timeGrain` buckets via `date_trunc`. Must be + * one of `dimensions` (so it is selected and in `GROUP BY ALL`) and is + * required whenever `timeGrain` is set. Grammar-gated as a SQL identifier. + */ + timeDimension?: string; limit?: number; format?: AnalyticsFormat; } From a7ecda5fe029f1cb21f5b8fe6299df124a13388e Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Thu, 9 Jul 2026 23:19:37 +0200 Subject: [PATCH 04/18] chore(appkit): metric OBO dispatch + cache-key isolation (PR2 phase 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lane dispatch from the registration (metric-views.json executor), not a URL segment: OBO-lane metrics run on-behalf-of the requesting user via asUser(req) with a per-user cache scope; SP-lane metrics run as the app service principal with a shared scope. Executor + key computed inside a try so a missing/ whitespace OBO identity (from asUser/resolveUserId/deriveMetricExecutorKey) lands on the canonical 401 envelope, not an uncaught 500. composeMetricCacheKey builds metric:{key}:{argsHash}:{executorKey} over canonicalized args (sorted measures/dimensions, order-independent filter fingerprint via canonicalizeFilter, grain, timeDimension, limit). deriveMetricExecutorKey returns "sp" for SP and a sha256 of the trimmed user identity for OBO — the raw email/principal never enters the cache layer. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../appkit/src/plugins/analytics/analytics.ts | 57 ++- .../appkit/src/plugins/analytics/metric.ts | 137 +++++- .../plugins/analytics/tests/metric.test.ts | 407 +++++++++++++++++- 3 files changed, 592 insertions(+), 9 deletions(-) diff --git a/packages/appkit/src/plugins/analytics/analytics.ts b/packages/appkit/src/plugins/analytics/analytics.ts index eceb4b9c0..8cb125884 100644 --- a/packages/appkit/src/plugins/analytics/analytics.ts +++ b/packages/appkit/src/plugins/analytics/analytics.ts @@ -31,6 +31,8 @@ import { queryDefaults } from "./defaults"; import manifest from "./manifest.json"; import { buildMetricSql, + composeMetricCacheKey, + deriveMetricExecutorKey, loadMetricRegistry, validateMetricRequest, } from "./metric"; @@ -500,8 +502,11 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { * The `originalError` re-throw discipline preserves each error's structured * `errorCode`/`clientMessage` for the SSE error payload. * - * SP lane only in this phase — every registered metric runs as the service - * principal; OBO dispatch is wired in a later phase. + * Lane dispatch is driven by the registration: an SP-lane metric runs as the + * app service principal (shared cache); an OBO-lane metric runs + * on-behalf-of the requesting user via `asUser(req)` (per-user cache keyed by + * a hash of the user identity). The executor + key are computed inside a + * try so a missing/whitespace OBO identity lands on the canonical 401 path. */ async _handleMetricRoute( req: express.Request, @@ -575,14 +580,52 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { return; } - // SP lane only in this phase: the default executor + shared cache key. OBO - // dispatch (asUser(req) + per-user key) arrives in a later phase. - const executor = this; - const executorKey = "sp"; + // Lane dispatch. The lane comes from the registration (the entry's + // `executor` in metric-views.json), NOT a URL segment or `.obo.sql` + // filename: an OBO-lane metric runs on-behalf-of the requesting user + // (per-user cache via `asUser(req)`), an SP-lane metric as the app service + // principal (shared cache). Compute the executor + key INSIDE a try (as + // `_handleQueryRoute` does) so `asUser(req)`/`resolveUserId(req)` and + // `deriveMetricExecutorKey`'s missing-identity throw land on the canonical + // 401 envelope instead of surfacing as an uncaught 500 out of the stream. + let executor: AnalyticsPlugin; + let executorKey: string; + try { + const isObo = registration.lane === "obo"; + executor = isObo ? this.asUser(req) : this; + executorKey = deriveMetricExecutorKey({ + lane: registration.lane, + userIdentity: isObo ? this.resolveUserId(req) : undefined, + }); + } catch (err) { + if (err instanceof AppKitError) { + res.status(err.statusCode).json({ error: err.message, code: err.code }); + return; + } + throw err; + } + // Cache key. Composed over the canonicalized args (sorted measures/ + // dimensions, stable-sorted predicates, grain, timeDimension, limit) plus + // the `executorKey` — `"sp"` shares the cache across all users, a per-user + // identity hash isolates OBO callers. Delivery is JSON-only in v1 (the + // route always routes through `_executeJsonArrayPath`), so the key salts on + // a stable "JSON_ARRAY" constant rather than `request.format`: two calls + // that deliver identically must not fork the cache on an unused format + // field. const cacheConfig = { ...queryDefaults.cache, - cacheKey: ["analytics:metric", key, JSON.stringify(request), executorKey], + cacheKey: composeMetricCacheKey({ + metricKey: key, + measures: request.measures, + dimensions: request.dimensions, + timeGrain: request.timeGrain, + timeDimension: request.timeDimension, + filter: request.filter, + format: "JSON_ARRAY", + executorKey, + limit: request.limit, + }), }; // Cache/retry/timeout scoped to the SQL execution itself (inner `execute`) diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts index bce144cd2..b970fc972 100644 --- a/packages/appkit/src/plugins/analytics/metric.ts +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { type SQLTypeMarker, sql as sqlHelpers } from "shared"; @@ -7,7 +8,7 @@ import { z } from "zod"; // type-generator's runtime, which pulls the zod-free `metric-fqn.ts` from the // same tree) so the runtime and the generated JSON schema validate identically. import { metricSourceSchema } from "../../../../shared/src/schemas/metric-source"; -import { ValidationError } from "../../errors"; +import { AuthenticationError, ValidationError } from "../../errors"; import { createLogger } from "../../logging/logger"; import type { IAnalyticsMetricRequest, @@ -1001,3 +1002,137 @@ function renderDimensionClause( } return dim; } + +/** + * Compose the cache key for a metric-view request. + * + * Reserved namespace `metric` separates metric-view caches from query caches. + * The returned array is consumed by `CacheManager.generateKey`, which + * concatenates and sha256-hashes the parts — the per-concern structure keeps + * the key inspectable in tests and debug logs without giving up determinism. + * + * Sort-before-hash collapses semantically equivalent calls onto one entry: + * - `measures` / `dimensions`: lexicographic sort. + * - `filter`: predicates inside each AND/OR group are stable-sorted by + * `(member, operator)` and the group kind (`and` vs `or`) is preserved by + * {@link canonicalizeFilter}; values are included verbatim so distinct + * filter values yield distinct keys. + * + * `timeGrain` AND `timeDimension` both salt the key: each independently + * changes the emitted SQL (the grain literal and which column is bucketed), so + * two calls that differ only in one of them must not share a cache entry. + * + * `executorKey` is `"sp"` for SP-lane entries (shared cache) or a sha256 hash + * of the end user's identity for OBO-lane entries (per-user isolation) — see + * {@link deriveMetricExecutorKey}. + */ +export function composeMetricCacheKey(input: { + metricKey: string; + measures: string[]; + dimensions?: string[]; + timeGrain?: string; + timeDimension?: string; + filter?: MetricFilter; + format: string; + executorKey: string; + limit?: number; +}): string[] { + const sortedMeasures = [...input.measures].sort(); + const sortedDimensions = [...(input.dimensions ?? [])].sort(); + const filterFingerprint = + input.filter !== undefined ? canonicalizeFilter(input.filter) : "_"; + return [ + "metric", + input.metricKey, + input.format, + sortedMeasures.join(","), + sortedDimensions.join(","), + input.timeGrain ?? "_", + input.timeDimension ?? "_", + filterFingerprint, + typeof input.limit === "number" ? String(input.limit) : "_", + input.executorKey, + ]; +} + +/** + * Derive the cache executor key from a metric registration's lane and the + * caller's user identity. + * + * Returns `"sp"` for SP-lane entries (every caller shares the cache) and a + * sha256 hex digest of the user identity for OBO-lane entries (each user gets + * an isolated cache scope). + * + * The user identity is hashed — never stored verbatim — so the cache layer + * (which logs keys at debug level and persists them in any cache backend) + * never sees raw user emails or principal names. A stable, opaque token is + * exactly what we need: same user → same key (so cache hits work), different + * users → different keys (so isolation holds), and reverse lookup is + * infeasible. + * + * For OBO requests without a resolvable identity (missing or whitespace-only + * user id), throw `AuthenticationError.missingUserId()` rather than falling + * back to a shared `"anonymous"` sentinel — distinct misconfigured callers + * would otherwise collide into one cache scope and read each other's cached + * results. The route computes this inside its try/catch, so the throw lands on + * the canonical 401 envelope. + */ +export function deriveMetricExecutorKey(input: { + lane: MetricLane; + userIdentity?: string | null; +}): string { + if (input.lane === "sp") { + return "sp"; + } + // OBO lane — hash the user identity so the raw email/principal never reaches + // the cache layer. Missing/whitespace identity is a hard auth failure: the + // alternative ("anonymous" sentinel) collides every misconfigured caller + // into a single cache scope, so user A's results could leak to user B. + const identity = input.userIdentity?.trim(); + if (!identity) { + throw AuthenticationError.missingUserId(); + } + return createHash("sha256").update(identity).digest("hex"); +} + +/** + * Produce a deterministic string fingerprint of the filter tree. + * + * The fingerprint sorts predicates within each AND/OR group by + * `(member, operator)` and recursively canonicalizes nested groups. Values are + * included verbatim so cache entries differ when the filter targets different + * values (`region in [EMEA]` vs `region in [APAC]` → different keys; `equals A` + * vs `equals B` → different keys), while order-insensitive predicate lists + * collapse to the same key. + */ +function canonicalizeFilter(node: MetricFilter): string { + if (node === null || typeof node !== "object") { + return "_"; + } + + if ("and" in node || "or" in node) { + const groupKey = "and" in node ? "and" : "or"; + const children = ( + node as { and?: ReadonlyArray } & { + or?: ReadonlyArray; + } + )[groupKey]; + + if (!Array.isArray(children) || children.length === 0) { + return `${groupKey}()`; + } + + const sorted = sortFilterChildren(children); + const childFingerprints = sorted.map(canonicalizeFilter); + return `${groupKey}(${childFingerprints.join(",")})`; + } + + // Leaf predicate. Use JSON.stringify (not String) for the value segment so + // strings carrying the `|` separator cannot collide with split arrays — + // e.g. `["a", "b"]` and `["a|string:b"]` stay distinct fingerprints. + const p = node as MetricPredicate; + const valuesPart = p.values + ? p.values.map((v) => `${typeof v}:${JSON.stringify(v)}`).join("|") + : ""; + return `p(${p.member}/${p.operator}/${valuesPart})`; +} diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index 50efd5340..253be7568 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -10,13 +10,20 @@ import { } from "@tools/test-helpers"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { ServiceContext } from "../../../context/service-context"; +import { AuthenticationError } from "../../../errors"; import { AnalyticsPlugin } from "../analytics"; import { buildMetricSql, + composeMetricCacheKey, + deriveMetricExecutorKey, loadMetricRegistry, validateMetricRequest, } from "../metric"; -import type { IAnalyticsConfig, MetricRegistration } from "../types"; +import type { + IAnalyticsConfig, + MetricFilter, + MetricRegistration, +} from "../types"; // Mirror the analytics.test.ts CacheManager mock so the inner `execute`'s // cache interceptor is a no-op pass-through (each request re-executes). @@ -1263,3 +1270,401 @@ describe("metric — filter translator", () => { }); }); }); + +// ── Phase 3: cache-key composition. `composeMetricCacheKey` produces the +// array `CacheManager.generateKey` concatenates + sha256s; the invariants +// below are what make the cache both correct (semantically equal calls collapse) +// and safe (distinct args / executors never collide). +describe("composeMetricCacheKey", () => { + const base: { + metricKey: string; + measures: string[]; + format: string; + executorKey: string; + } = { + metricKey: "revenue", + measures: ["arr"], + format: "JSON_ARRAY", + executorKey: "sp", + }; + + test("measure ORDER does not affect the key (sorted before hashing)", () => { + const a = composeMetricCacheKey({ + ...base, + measures: ["arr", "revenue"], + }); + const b = composeMetricCacheKey({ + ...base, + measures: ["revenue", "arr"], + }); + expect(a).toEqual(b); + }); + + test("dimension ORDER does not affect the key (sorted before hashing)", () => { + const a = composeMetricCacheKey({ + ...base, + dimensions: ["region", "segment"], + }); + const b = composeMetricCacheKey({ + ...base, + dimensions: ["segment", "region"], + }); + expect(a).toEqual(b); + }); + + test("different measures → different keys", () => { + const a = composeMetricCacheKey({ ...base, measures: ["arr"] }); + const b = composeMetricCacheKey({ ...base, measures: ["mrr"] }); + expect(a).not.toEqual(b); + }); + + test("different dimensions → different keys", () => { + const a = composeMetricCacheKey({ ...base, dimensions: ["region"] }); + const b = composeMetricCacheKey({ ...base, dimensions: ["segment"] }); + expect(a).not.toEqual(b); + }); + + test("different timeGrain → different keys", () => { + const a = composeMetricCacheKey({ + ...base, + dimensions: ["order_date"], + timeGrain: "day", + timeDimension: "order_date", + }); + const b = composeMetricCacheKey({ + ...base, + dimensions: ["order_date"], + timeGrain: "month", + timeDimension: "order_date", + }); + expect(a).not.toEqual(b); + }); + + test("different timeDimension → different keys (new field salts the key)", () => { + const a = composeMetricCacheKey({ + ...base, + dimensions: ["order_date", "ship_date"], + timeGrain: "day", + timeDimension: "order_date", + }); + const b = composeMetricCacheKey({ + ...base, + dimensions: ["order_date", "ship_date"], + timeGrain: "day", + timeDimension: "ship_date", + }); + expect(a).not.toEqual(b); + }); + + test("different limit → different keys", () => { + const a = composeMetricCacheKey({ ...base, limit: 10 }); + const b = composeMetricCacheKey({ ...base, limit: 20 }); + expect(a).not.toEqual(b); + }); + + test("predicate ORDER inside a group does not affect the key (canonicalizeFilter)", () => { + const a = composeMetricCacheKey({ + ...base, + filter: { + and: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { member: "segment", operator: "equals", values: ["Ent"] }, + ], + } as MetricFilter, + }); + const b = composeMetricCacheKey({ + ...base, + filter: { + and: [ + { member: "segment", operator: "equals", values: ["Ent"] }, + { member: "region", operator: "equals", values: ["EMEA"] }, + ], + } as MetricFilter, + }); + expect(a).toEqual(b); + }); + + test("distinct filter VALUES → distinct keys", () => { + const a = composeMetricCacheKey({ + ...base, + filter: { + member: "region", + operator: "equals", + values: ["EMEA"], + } as MetricFilter, + }); + const b = composeMetricCacheKey({ + ...base, + filter: { + member: "region", + operator: "equals", + values: ["APAC"], + } as MetricFilter, + }); + expect(a).not.toEqual(b); + }); + + test("a filter present vs absent → different keys", () => { + const withFilter = composeMetricCacheKey({ + ...base, + filter: { + member: "region", + operator: "set", + } as MetricFilter, + }); + const withoutFilter = composeMetricCacheKey({ ...base }); + expect(withFilter).not.toEqual(withoutFilter); + }); + + test("SP vs OBO (same args) → different keys via executorKey", () => { + const sp = composeMetricCacheKey({ ...base, executorKey: "sp" }); + const obo = composeMetricCacheKey({ + ...base, + executorKey: deriveMetricExecutorKey({ + lane: "obo", + userIdentity: "alice", + }), + }); + expect(sp).not.toEqual(obo); + }); +}); + +// ── Phase 3: executor-key isolation. The key is what scopes the cache — `"sp"` +// shares it across all callers, a per-user hash isolates OBO callers. The raw +// identity must never enter the key verbatim (privacy: cache keys are logged +// and persisted). +describe("deriveMetricExecutorKey", () => { + test("SP lane → literal 'sp' (shared cache)", () => { + expect(deriveMetricExecutorKey({ lane: "sp" })).toBe("sp"); + }); + + test("SP lane ignores any supplied identity", () => { + expect(deriveMetricExecutorKey({ lane: "sp", userIdentity: "alice" })).toBe( + "sp", + ); + }); + + test("OBO lane hashes the identity — raw identity never appears in the key", () => { + const key = deriveMetricExecutorKey({ + lane: "obo", + userIdentity: "alice@example.com", + }); + expect(key).toMatch(/^[a-f0-9]{64}$/); + expect(key).not.toContain("alice"); + }); + + test("OBO same identity → stable key (cache hits work)", () => { + const a = deriveMetricExecutorKey({ lane: "obo", userIdentity: "alice" }); + const b = deriveMetricExecutorKey({ lane: "obo", userIdentity: "alice" }); + expect(a).toBe(b); + }); + + test("OBO identity is trimmed before hashing (whitespace does not fork the scope)", () => { + const padded = deriveMetricExecutorKey({ + lane: "obo", + userIdentity: " alice ", + }); + const bare = deriveMetricExecutorKey({ + lane: "obo", + userIdentity: "alice", + }); + expect(padded).toBe(bare); + }); + + test("OBO different identities → different keys (isolation holds)", () => { + const alice = deriveMetricExecutorKey({ + lane: "obo", + userIdentity: "alice", + }); + const bob = deriveMetricExecutorKey({ lane: "obo", userIdentity: "bob" }); + expect(alice).not.toBe(bob); + }); + + test("OBO empty identity → throws AuthenticationError (no shared 'anonymous' scope)", () => { + expect(() => + deriveMetricExecutorKey({ lane: "obo", userIdentity: "" }), + ).toThrow(AuthenticationError); + }); + + test("OBO whitespace-only identity → throws AuthenticationError", () => { + expect(() => + deriveMetricExecutorKey({ lane: "obo", userIdentity: " " }), + ).toThrow(AuthenticationError); + }); + + test("OBO missing (undefined/null) identity → throws AuthenticationError", () => { + expect(() => deriveMetricExecutorKey({ lane: "obo" })).toThrow( + AuthenticationError, + ); + expect(() => + deriveMetricExecutorKey({ lane: "obo", userIdentity: null }), + ).toThrow(AuthenticationError); + }); +}); + +// ── Phase 3: lane dispatch at the handler level. The lane comes from the +// registration (the entry's `executor` in metric-views.json), NOT the URL: +// OBO-lane routes through `asUser(req)`, SP-lane through the default executor. +// A missing/whitespace OBO identity must land on the canonical 401 envelope, +// never an out-of-envelope 500. +describe("metric route — lane dispatch (Phase 3)", () => { + let config: IAnalyticsConfig; + let serviceContextMock: Awaited>; + + beforeEach(async () => { + config = { timeout: 5000 }; + setupDatabricksEnv(); + mockCacheStore.clear(); + ServiceContext.reset(); + serviceContextMock = await mockServiceContext(); + }); + + afterEach(() => { + serviceContextMock?.restore(); + }); + + test("OBO-lane registration routes through asUser(req)", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "obo", + }, + }); + + const asUserSpy = vi.spyOn(plugin, "asUser"); + const executeMock = vi + .fn() + .mockResolvedValue({ result: { data: [{ arr: 1 }] } }); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + headers: { + "x-forwarded-access-token": "user-token", + "x-forwarded-user": "alice", + }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + // The OBO lane dispatched through the user-scoped executor… + expect(asUserSpy).toHaveBeenCalledWith(mockReq); + // …and the SQL still reached the warehouse and streamed a result. + expect(executeMock).toHaveBeenCalled(); + expect(mockRes.write).toHaveBeenCalledWith("event: result\n"); + }); + + test("SP-lane registration uses the default executor (asUser not called)", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + + const asUserSpy = vi.spyOn(plugin, "asUser"); + const executeMock = vi + .fn() + .mockResolvedValue({ result: { data: [{ arr: 1 }] } }); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + // Even with user headers present, an SP-lane metric must NOT impersonate. + headers: { + "x-forwarded-access-token": "user-token", + "x-forwarded-user": "alice", + }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(asUserSpy).not.toHaveBeenCalled(); + expect(executeMock).toHaveBeenCalled(); + expect(mockRes.write).toHaveBeenCalledWith("event: result\n"); + }); + + test("OBO metric with no user identity → canonical 401, no SQL executed", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "obo", + }, + }); + + const executeMock = vi.fn(); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + // No x-forwarded-* headers at all → the OBO dispatch throws an + // AuthenticationError, caught and returned as a canonical 401 envelope. + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(401); + expect(mockRes.json).toHaveBeenCalledWith( + expect.objectContaining({ code: "AUTHENTICATION_ERROR" }), + ); + // Landed on the 401 path before any SQL and without opening the SSE stream. + expect(executeMock).not.toHaveBeenCalled(); + expect(mockRes.write).not.toHaveBeenCalled(); + }); + + test("OBO metric with whitespace-only user identity → canonical 401", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "obo", + }, + }); + + const executeMock = vi.fn(); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + headers: { + "x-forwarded-access-token": "user-token", + "x-forwarded-user": " ", + }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(401); + expect(mockRes.json).toHaveBeenCalledWith( + expect.objectContaining({ code: "AUTHENTICATION_ERROR" }), + ); + expect(executeMock).not.toHaveBeenCalled(); + }); +}); From 15d9371a8e50e6d619dc2eb602dc13644b6e8d6d Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 10 Jul 2026 10:32:27 +0200 Subject: [PATCH 05/18] chore(appkit): harden metric route lookup, arg uniqueness, sort key, format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four defensive fixes to the PR2 metric-view runtime surfaced by adversarial review: - Registry lookup (B): build the registry with a null prototype and gate the route read with Object.hasOwn, so a metric key colliding with an inherited Object.prototype member (__proto__, constructor, toString, …) can no longer resolve to a truthy non-registration and bypass the unknown-key 404. - Measure/dimension uniqueness (C): reject a name that repeats within measures, within dimensions, or across both. Measures and dimensions alias to their own name in the SELECT list (MEASURE(x) AS x, x), so a duplicate collapses to one row-object key and silently drops a value during row materialization. - Filter sort key (D): delimit the (member, operator) sort key with "/" instead of a bare concatenation, so two distinct pairs cannot map to the same key and fork the cache on semantically equal filters. Matches the delimiter canonicalizeFilter already uses for its leaf fingerprint. - Format (F): reject any format other than JSON_ARRAY (legacy aliases normalized first). The metric route delivers JSON rows only at v1; an Arrow request previously returned JSON silently instead of failing loud. Adds 10 tests covering all four. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../appkit/src/plugins/analytics/analytics.ts | 10 +- .../appkit/src/plugins/analytics/metric.ts | 61 +++++++- .../plugins/analytics/tests/metric.test.ts | 146 ++++++++++++++++++ 3 files changed, 214 insertions(+), 3 deletions(-) diff --git a/packages/appkit/src/plugins/analytics/analytics.ts b/packages/appkit/src/plugins/analytics/analytics.ts index 8cb125884..29eea8106 100644 --- a/packages/appkit/src/plugins/analytics/analytics.ts +++ b/packages/appkit/src/plugins/analytics/analytics.ts @@ -544,7 +544,15 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { return; } - const registration = registry[key]; + // Own-property lookup: never resolve `key` to an inherited + // `Object.prototype` member (`__proto__`, `constructor`, `toString`, …). + // The loader already builds a null-prototype registry, but the load-error + // fallback and test-injected registries are plain objects, so gate the + // read here too — an inherited hit would otherwise bypass the 404 below + // and flow a non-registration value into execution. + const registration = Object.hasOwn(registry, key) + ? registry[key] + : undefined; if (!registration) { // Don't echo the user-supplied `key` back in the public response — // confirming "metric X is not registered" lets a probe enumerate keys by diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts index b970fc972..519d26590 100644 --- a/packages/appkit/src/plugins/analytics/metric.ts +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -18,6 +18,7 @@ import type { MetricPredicate, MetricRegistration, } from "./types"; +import { normalizeAnalyticsFormat } from "./types"; const logger = createLogger("analytics:metric"); @@ -202,7 +203,13 @@ export function loadMetricRegistry( throw new Error(`Invalid metric-views.json at ${metricPath}: ${issues}`); } - const registry: Record = {}; + // Null-prototype map so a metric key that collides with an inherited + // `Object.prototype` member (`__proto__`, `constructor`, `toString`, …) + // cannot resolve to a truthy non-registration at the `registry[key]` read + // site and slip past the unknown-key 404. Keys are still grammar-gated by + // `metricKeySchema` (identifier shape), but the null prototype removes the + // whole class of inherited-property lookups as a boundary. + const registry: Record = Object.create(null); for (const [key, entry] of Object.entries(result.data.metricViews ?? {})) { registry[key] = { key, @@ -344,6 +351,48 @@ const metricRequestSchema = z validateFilterTree(value.filter, ctx, ["filter"], 0); } + // Uniqueness. Measures and dimensions become SELECT columns aliased to + // their own name (`MEASURE(x) AS x`, `x`); the warehouse returns one row + // object keyed by column name, so a repeated name — a duplicate measure, a + // duplicate dimension, or a name appearing as BOTH a measure and a + // dimension — collapses to a single key and silently drops a value during + // row materialization. Reject the collision here rather than emit SQL that + // corrupts rows. (The grammar gate at build time is a safety boundary, not + // a uniqueness check, so this lives in validation.) + const seen = new Set(); + const collided = new Set(); + for (const name of [...value.measures, ...(value.dimensions ?? [])]) { + if (seen.has(name)) { + collided.add(name); + } + seen.add(name); + } + if (collided.size > 0) { + ctx.addIssue({ + code: "custom", + message: + "measures and dimensions must be unique across both lists (a name cannot repeat, nor appear as both a measure and a dimension)", + path: ["measures"], + }); + } + + // Delivery format. The metric route delivers JSON rows only at v1 (the + // cache salts on a fixed "JSON_ARRAY" and the route always routes through + // the JSON path). Accepting an Arrow format would silently return JSON — + // reject it loudly until Arrow parity ships. Legacy aliases normalize + // first (`JSON`→`JSON_ARRAY`, `ARROW`→`ARROW_STREAM`). + if ( + value.format != null && + normalizeAnalyticsFormat(value.format) !== "JSON_ARRAY" + ) { + ctx.addIssue({ + code: "custom", + message: + "format: only JSON_ARRAY is supported on the metric route at v1 (ARROW_STREAM is not yet implemented)", + path: ["format"], + }); + } + // Cross-field grain rules. `timeGrain` buckets exactly one selected // dimension via `date_trunc`, so it requires an explicit `timeDimension`, // and that dimension must be selected (so it is in the SELECT list and @@ -824,7 +873,15 @@ function sortFilterChildren( !("or" in child) ) { const p = child as MetricPredicate; - key = `${p.member}${p.operator}`; + // Separate the fields with "/" so the (member, operator) pair maps to + // the sort key injectively. A delimiter-free concatenation collides — + // member "ab" + op "c" and member "a" + op "bc" both → "abc" — which + // tie-breaks two distinct pairs to input order and forks the cache key + // on semantically equal filters. "/" can never appear in either field + // (members match DIMENSION_NAME_PATTERN, operators are the alpha-only + // enum), matching the leaf-fingerprint delimiter canonicalizeFilter + // already uses. + key = `${p.member}/${p.operator}`; isPredicate = true; } else { // Nested groups don't have a single (member, operator) — keep their diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index 253be7568..e339be23e 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -466,6 +466,43 @@ describe("analytics metric route (Phase 1)", () => { expect(mockRes.json).toHaveBeenCalledWith({ error: "Metric not found" }); }); + test.each(["__proto__", "constructor", "toString", "hasOwnProperty"])( + "inherited Object.prototype key %j → 404, no execution (own-property lookup)", + async (dangerousKey) => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + // A real (own) entry so the registry is populated but does NOT contain + // the dangerous key. Without the own-property guard, `registry[key]` + // would resolve the inherited prototype member and slip past the 404. + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + + const executeMock = vi.fn(); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: dangerousKey }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(404); + expect(mockRes.json).toHaveBeenCalledWith({ + error: "Metric not found", + }); + expect(executeMock).not.toHaveBeenCalled(); + }, + ); + test("malformed registry → 503 METRIC_REGISTRY_LOAD_FAILED", async () => { const plugin = new AnalyticsPlugin(config); const { router, getHandler } = createMockRouter(); @@ -554,6 +591,22 @@ describe("loadMetricRegistry", () => { }); }); + test("registry has a null prototype (no inherited-property lookups)", () => { + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.revenue_metrics" } }, + }), + ); + + const registry = loadMetricRegistry(dir); + expect(Object.getPrototypeOf(registry)).toBeNull(); + // A key that would resolve to a truthy inherited member on a plain object + // resolves to undefined here. + expect((registry as Record).toString).toBeUndefined(); + expect((registry as Record).__proto__).toBeUndefined(); + }); + test("malformed JSON throws", () => { writeFileSync(path.join(dir, "metric-views.json"), "{ not json"); expect(() => loadMetricRegistry(dir)).toThrow(/Failed to parse/); @@ -1092,6 +1145,73 @@ describe("metric — filter translator", () => { }); }); + describe("validator — measure/dimension uniqueness", () => { + // Measures and dimensions become SELECT columns aliased to their own name; + // a repeated name collapses to a single row-object key and silently drops + // a value during row materialization, so uniqueness is enforced up front. + test("rejects a duplicate measure", () => { + expect(() => + validateMetricRequest({ measures: ["arr", "arr"] }), + ).toThrowError(/fields:.*measures/); + }); + + test("rejects a duplicate dimension", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["region", "region"], + }), + ).toThrowError(/fields:.*measures/); + }); + + test("rejects a name that is BOTH a measure and a dimension", () => { + // The corruption case: `SELECT MEASURE(x) AS x, x ... GROUP BY ALL` + // materializes two `x` columns and the second overwrites the first. + expect(() => + validateMetricRequest({ + measures: ["revenue"], + dimensions: ["revenue"], + }), + ).toThrowError(/fields:.*measures/); + }); + + test("accepts distinct measures and dimensions (no false positive)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr", "mrr"], + dimensions: ["region", "segment"], + }), + ).not.toThrow(); + }); + }); + + describe("validator — format (JSON-only at v1)", () => { + test("accepts JSON_ARRAY", () => { + expect(() => + validateMetricRequest({ measures: ["arr"], format: "JSON_ARRAY" }), + ).not.toThrow(); + }); + + test("accepts the legacy JSON alias (normalizes to JSON_ARRAY)", () => { + expect(() => + validateMetricRequest({ measures: ["arr"], format: "JSON" }), + ).not.toThrow(); + }); + + test("rejects ARROW_STREAM (not implemented on the metric route)", () => { + // Fail loud rather than silently deliver JSON for an Arrow request. + expect(() => + validateMetricRequest({ measures: ["arr"], format: "ARROW_STREAM" }), + ).toThrowError(/fields:.*format/); + }); + + test("rejects the legacy ARROW alias (normalizes to ARROW_STREAM)", () => { + expect(() => + validateMetricRequest({ measures: ["arr"], format: "ARROW" }), + ).toThrowError(/fields:.*format/); + }); + }); + describe("timeGrain + timeDimension (validator)", () => { test("a grammar-valid timeGrain + timeDimension (in dimensions) is accepted", () => { expect(() => @@ -1384,6 +1504,32 @@ describe("composeMetricCacheKey", () => { expect(a).toEqual(b); }); + test("predicate ORDER is stable for two ops on the SAME member (sort-key delimiter)", () => { + // Same member, different operators — the sort key is `${member}/${operator}`, + // so the "/" delimiter (not a raw concatenation) is what disambiguates the + // pair and keeps the ordering — and thus the key — independent of input + // order. Two range bounds on one column is the everyday shape of this. + const a = composeMetricCacheKey({ + ...base, + filter: { + and: [ + { member: "revenue", operator: "gt", values: [100] }, + { member: "revenue", operator: "lt", values: [200] }, + ], + } as MetricFilter, + }); + const b = composeMetricCacheKey({ + ...base, + filter: { + and: [ + { member: "revenue", operator: "lt", values: [200] }, + { member: "revenue", operator: "gt", values: [100] }, + ], + } as MetricFilter, + }); + expect(a).toEqual(b); + }); + test("distinct filter VALUES → distinct keys", () => { const a = composeMetricCacheKey({ ...base, From 3ab36ebe7851e1431b3e9f52f1f4dd0b2c87957f Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 10 Jul 2026 10:48:32 +0200 Subject: [PATCH 06/18] chore(appkit): unify metric-view FQN grammar + quote at SQL interpolation The runtime, the shared schema, and the type-generator disagreed on what a metric-view `source` FQN may contain. The shared schema and typegen accept the full UC quoted-identifier grammar (hyphens, non-ASCII) and typegen backtick- quotes before interpolation; the runtime used a narrower ASCII allowlist (assertSafeFqn) AND interpolated the FQN UNQUOTED. So a documented-legal UC name like `prod-data.analytics.revenue` passed config + type generation but threw (or, if the pattern were widened, emitted invalid SQL) at request time. Converge on one grammar + one escaper: - Move `isValidFqn` (three-part UC predicate) and `quoteFqnForSql` (backtick escaper) into the shared zod-free leaf `metric-fqn.ts`, beside `UC_FQN_PATTERN`. Grammar and quoting now have a single home both the type-generator and the analytics runtime import. `describe.ts` and `config.ts` import them back; `mv-registry.test.ts` imports the escaper from the leaf. - Runtime `buildMetricSql` replaces the narrow `assertSafeFqn` regex with `quoteSafeFqn`: validate via `isValidFqn`, then interpolate the `quoteFqnForSql`-escaped FQN. Quoting is the injection boundary, so the runtime accepts exactly what UC (and the schema, and typegen) accept. - `UC_THREE_PART_FQN_PATTERN` stays in `metric-source.ts` so zod still emits a JSON-schema `pattern`; it derives from the same per-segment charset as `isValidFqn`, so the two shapes cannot diverge. Also folds in cache-key hardening (in-flight): salt the metric cache key with `source` so repointing a metric key to a different FQN cannot stale-serve, and only salt `timeDimension` when `timeGrain` is set (it has no SQL effect otherwise). `renderDimensionClause` re-gates the grain against TIME_GRAIN_PATTERN at its interpolation point. Tests: requote the ~13 emitted-FROM assertions; add regression tests that a hyphenated FQN is accepted+quoted and that a backtick-bearing segment is neutralized by doubling. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../appkit/src/plugins/analytics/analytics.ts | 1 + .../appkit/src/plugins/analytics/metric.ts | 90 +++++++++----- .../plugins/analytics/tests/metric.test.ts | 111 ++++++++++++++++-- .../src/type-generator/mv-registry/config.ts | 27 ++--- .../type-generator/mv-registry/describe.ts | 51 ++------ .../type-generator/tests/mv-registry.test.ts | 5 +- packages/shared/src/schemas/metric-fqn.ts | 77 ++++++++++++ 7 files changed, 256 insertions(+), 106 deletions(-) diff --git a/packages/appkit/src/plugins/analytics/analytics.ts b/packages/appkit/src/plugins/analytics/analytics.ts index 29eea8106..c8bf5d4e1 100644 --- a/packages/appkit/src/plugins/analytics/analytics.ts +++ b/packages/appkit/src/plugins/analytics/analytics.ts @@ -625,6 +625,7 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { ...queryDefaults.cache, cacheKey: composeMetricCacheKey({ metricKey: key, + source: registration.source, measures: request.measures, dimensions: request.dimensions, timeGrain: request.timeGrain, diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts index 519d26590..5ed31946a 100644 --- a/packages/appkit/src/plugins/analytics/metric.ts +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -7,6 +7,10 @@ import { z } from "zod"; // `metric-views.json`. Imported from the shared source directly (matching the // type-generator's runtime, which pulls the zod-free `metric-fqn.ts` from the // same tree) so the runtime and the generated JSON schema validate identically. +import { + isValidFqn, + quoteFqnForSql, +} from "../../../../shared/src/schemas/metric-fqn"; import { metricSourceSchema } from "../../../../shared/src/schemas/metric-source"; import { AuthenticationError, ValidationError } from "../../errors"; import { createLogger } from "../../logging/logger"; @@ -30,16 +34,6 @@ const logger = createLogger("analytics:metric"); const QUERIES_DIR = path.resolve(process.cwd(), "config/queries"); const METRIC_CONFIG_FILE = "metric-views.json"; -/** - * Three-part UC FQN matcher. The registry is parsed against the landed - * `metricSourceSchema`, which already validates `source` via the composed - * `UC_THREE_PART_FQN_PATTERN`; this is a belt-and-suspenders runtime fence for - * any code path that constructs SQL from a `source` outside that parse (the FQN - * cannot be parameterized — it is interpolated into the SQL string). - */ -const FQN_PATTERN = - /^[a-zA-Z0-9_][a-zA-Z0-9_-]*\.[a-zA-Z0-9_][a-zA-Z0-9_-]*\.[a-zA-Z0-9_][a-zA-Z0-9_-]*$/; - /** * Validate measure names before they are interpolated into `MEASURE()`. * @@ -226,19 +220,35 @@ export function loadMetricRegistry( } /** - * SQL identifier safety guard — the FQN ships in the SQL string (it cannot be - * parameterized) so we re-check the regex at construction time. + * Validate a metric-view FQN and return it backtick-quoted for interpolation. + * + * The FQN ships in the SQL string — it cannot be parameterized — so it passes + * two shared, zod-free gates at construction time: + * + * 1. {@link isValidFqn} — the three-part UC grammar (exactly three segments, + * each matching `UC_FQN_PATTERN`). This is the SAME predicate the + * type-generator's describe seam uses and is derived from the same + * per-segment charset as the canonical Zod schema, so config-time, + * generation-time, and runtime accept exactly the same names. + * 2. {@link quoteFqnForSql} — backtick-quotes each segment (doubling embedded + * backticks) so the FQN cannot break out of its identifier position. This + * is the injection boundary; it is why the runtime can accept the full UC + * quoted-identifier grammar (hyphens, non-ASCII) rather than the narrow + * ASCII allowlist it used before. * - * The registry loader already enforces the FQN grammar via `metricSourceSchema`; - * this is a runtime fence for any future code path that constructs SQL outside - * of a parsed registry. + * The registry loader already enforces the grammar via `metricSourceSchema`; + * this re-gate is defense-in-depth for any code path that reaches + * `buildMetricSql` without going through a parsed registry (it is exported), + * matching how measures, dimensions, and the grain are each re-gated at their + * interpolation points. */ -function assertSafeFqn(fqn: string): void { - if (!FQN_PATTERN.test(fqn)) { +function quoteSafeFqn(fqn: string): string { + if (!isValidFqn(fqn)) { throw new Error( `Refusing to build SQL: "${fqn}" is not a valid three-part UC FQN.`, ); } + return quoteFqnForSql(fqn); } /** @@ -658,9 +668,10 @@ export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { * - Every measure and dimension is gated by {@link MEASURE_NAME_PATTERN} / * {@link DIMENSION_NAME_PATTERN} before it is interpolated (column * references cannot be parameterized — they are SQL identifiers), and the - * FQN is re-checked by {@link assertSafeFqn}. There is deliberately NO name - * allowlist — the grammar gate is the security boundary. No user-supplied - * string reaches the SQL string without passing a grammar gate. + * FQN is validated and backtick-quoted by {@link quoteSafeFqn}. There is + * deliberately NO name allowlist — the grammar gate (plus quoting for the + * FQN) is the security boundary. No user-supplied string reaches the SQL + * string without passing a grammar gate. * - `GROUP BY ALL` is added when at least one dimension is requested. UC * requires GROUP BY when MEASURE() is mixed with non-aggregated columns; * `GROUP BY ALL` is the documented form that works without re-listing each @@ -683,7 +694,7 @@ export function buildMetricSql( statement: string; parameters: Record; } { - assertSafeFqn(registration.source); + const quotedSource = quoteSafeFqn(registration.source); if (request.measures.length === 0) { throw new Error("buildMetricSql requires at least one measure."); @@ -743,7 +754,7 @@ export function buildMetricSql( } } - const statement = `SELECT ${selectList} FROM ${registration.source}${whereClause}${groupByClause}${limitClause}`; + const statement = `SELECT ${selectList} FROM ${quotedSource}${whereClause}${groupByClause}${limitClause}`; return { statement, parameters }; } @@ -1035,8 +1046,12 @@ function bindLikeValue( * grammar-gated before they reach the SQL string: * * - The grain is single-quoted (it is a `date_trunc` unit literal, NOT a bind - * param) and `TIME_GRAIN_PATTERN` upstream is the control that keeps a - * hostile token out of that quoted position. + * param), so it is re-checked against {@link TIME_GRAIN_PATTERN} here before + * interpolation. The schema also gates it, but `buildMetricSql` is exported + * and may be reached on a path that bypasses `validateMetricRequest`, so the + * grammar gate is applied at the interpolation point too — matching how + * every other interpolated identifier (measures, dimensions, `timeDimension`, + * the FQN) is re-gated in the builder. * - The column cannot be parameterized (it is an identifier), so it is * re-checked against {@link DIMENSION_NAME_PATTERN} here as belt-and- * suspenders even though the schema already gated `timeDimension`. @@ -1055,6 +1070,11 @@ function renderDimensionClause( `Refusing to build SQL: timeDimension "${dim}" is not a valid identifier.`, ); } + if (!TIME_GRAIN_PATTERN.test(timeGrain)) { + throw new Error( + `Refusing to build SQL: timeGrain "${timeGrain}" is not a valid grain token.`, + ); + } return `date_trunc('${timeGrain}', ${dim}) AS ${dim}`; } return dim; @@ -1075,9 +1095,16 @@ function renderDimensionClause( * {@link canonicalizeFilter}; values are included verbatim so distinct * filter values yield distinct keys. * - * `timeGrain` AND `timeDimension` both salt the key: each independently - * changes the emitted SQL (the grain literal and which column is bucketed), so - * two calls that differ only in one of them must not share a cache entry. + * `source` (the metric view's UC FQN) salts the key so that repointing a + * metric `key` to a different FQN in `metric-views.json` cannot serve rows + * cached under the old source within the TTL — `metricKey` alone is not enough + * because the key→source binding is mutable config. + * + * `timeGrain` salts the key whenever set. `timeDimension` only salts the key + * when `timeGrain` is also set, because `renderDimensionClause` only applies + * `date_trunc('', )` when a grain is present — with no + * grain the field has no effect on the emitted SQL, so including it would fork + * the cache on an input that produced identical SQL. * * `executorKey` is `"sp"` for SP-lane entries (shared cache) or a sha256 hash * of the end user's identity for OBO-lane entries (per-user isolation) — see @@ -1085,6 +1112,7 @@ function renderDimensionClause( */ export function composeMetricCacheKey(input: { metricKey: string; + source: string; measures: string[]; dimensions?: string[]; timeGrain?: string; @@ -1098,14 +1126,20 @@ export function composeMetricCacheKey(input: { const sortedDimensions = [...(input.dimensions ?? [])].sort(); const filterFingerprint = input.filter !== undefined ? canonicalizeFilter(input.filter) : "_"; + // `timeDimension` only changes the SQL when `timeGrain` is set (see + // renderDimensionClause); keying on it otherwise would fork the cache on a + // no-op field. + const timeDimensionPart = + input.timeGrain != null ? (input.timeDimension ?? "_") : "_"; return [ "metric", input.metricKey, + input.source, input.format, sortedMeasures.join(","), sortedDimensions.join(","), input.timeGrain ?? "_", - input.timeDimension ?? "_", + timeDimensionPart, filterFingerprint, typeof input.limit === "number" ? String(input.limit) : "_", input.executorKey, diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index e339be23e..cde552852 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -137,6 +137,36 @@ describe("analytics metric route (Phase 1)", () => { ), ).toThrow(/not a valid three-part UC FQN/); }); + + test("accepts a hyphenated UC-legal FQN and backtick-quotes it", () => { + // Regression for the grammar divergence: `prod-data` is a legal UC + // object name (hyphens are allowed in quoted identifiers) and passes the + // shared schema + typegen, but the old narrow runtime pattern + // [a-zA-Z0-9_-] anchored per segment REJECTED it — a latent prod break on + // a documented-legal name. The builder now accepts it and quotes every + // segment, so it reaches the warehouse as valid SQL rather than throwing. + const { statement } = buildMetricSql( + { key: "x", source: "prod-data.analytics.revenue", lane: "sp" }, + { measures: ["arr"] }, + ); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr FROM `prod-data`.`analytics`.`revenue`", + ); + }); + + test("backtick-quoting neutralizes an injection-shaped FQN segment", () => { + // A source carrying a backtick would break out of the quoted identifier + // if interpolated raw; quoteFqnForSql doubles it. (isValidFqn rejects + // most such names first, but the quoting is the actual injection + // boundary — this asserts it, not just the grammar gate.) + const { statement } = buildMetricSql( + { key: "x", source: "cat.sch.re`v", lane: "sp" }, + { measures: ["arr"] }, + ); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`re``v`", + ); + }); }); // ── Measures-only SQL shape. @@ -152,7 +182,7 @@ describe("analytics metric route (Phase 1)", () => { measures: ["arr"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr FROM cat.sch.revenue_metrics", + "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`revenue_metrics`", ); expect(parameters).toEqual({}); }); @@ -162,7 +192,7 @@ describe("analytics metric route (Phase 1)", () => { measures: ["revenue", "arr"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, MEASURE(revenue) AS revenue FROM cat.sch.revenue_metrics", + "SELECT MEASURE(arr) AS arr, MEASURE(revenue) AS revenue FROM `cat`.`sch`.`revenue_metrics`", ); }); @@ -172,7 +202,7 @@ describe("analytics metric route (Phase 1)", () => { limit: 10.9, }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr FROM cat.sch.revenue_metrics LIMIT 10", + "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`revenue_metrics` LIMIT 10", ); }); }); @@ -192,7 +222,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: ["region"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, region FROM cat.sch.revenue_metrics GROUP BY ALL", + "SELECT MEASURE(arr) AS arr, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); }); @@ -202,7 +232,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: ["segment", "region"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, region, segment FROM cat.sch.revenue_metrics GROUP BY ALL", + "SELECT MEASURE(arr) AS arr, region, segment FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); }); @@ -212,7 +242,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: ["region"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, MEASURE(revenue) AS revenue, region FROM cat.sch.revenue_metrics GROUP BY ALL", + "SELECT MEASURE(arr) AS arr, MEASURE(revenue) AS revenue, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); }); @@ -222,7 +252,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: [], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr FROM cat.sch.revenue_metrics", + "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`revenue_metrics`", ); }); @@ -233,7 +263,7 @@ describe("analytics metric route (Phase 1)", () => { limit: 100, }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, region FROM cat.sch.revenue_metrics GROUP BY ALL LIMIT 100", + "SELECT MEASURE(arr) AS arr, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL LIMIT 100", ); }); @@ -243,7 +273,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: ["order_date", "region"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, order_date, region FROM cat.sch.revenue_metrics GROUP BY ALL", + "SELECT MEASURE(arr) AS arr, order_date, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); expect(statement).not.toContain("date_trunc"); }); @@ -267,7 +297,7 @@ describe("analytics metric route (Phase 1)", () => { timeDimension: "order_date", }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, date_trunc('month', order_date) AS order_date, region FROM cat.sch.revenue_metrics GROUP BY ALL", + "SELECT MEASURE(arr) AS arr, date_trunc('month', order_date) AS order_date, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); expect(statement).toContain( "date_trunc('month', order_date) AS order_date", @@ -283,9 +313,24 @@ describe("analytics metric route (Phase 1)", () => { timeDimension: "order_date", }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, date_trunc('day', order_date) AS order_date FROM cat.sch.revenue_metrics GROUP BY ALL", + "SELECT MEASURE(arr) AS arr, date_trunc('day', order_date) AS order_date FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); }); + + test("a grammar-invalid timeGrain throws in the builder (defense-in-depth, even if validation is bypassed)", () => { + // buildMetricSql is exported and may be reached on a path that skips + // validateMetricRequest, so the grain — interpolated into a single-quoted + // date_trunc literal — is re-gated at the interpolation point. A quote- + // breakout payload must be refused by the builder itself. + expect(() => + buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "month'); DROP TABLE t;--", + timeDimension: "order_date", + }), + ).toThrow(/not a valid grain token/); + }); }); // ── Phase 2: dimension grammar gate. A dimension failing @@ -349,7 +394,8 @@ describe("analytics metric route (Phase 1)", () => { expect(executeMock).toHaveBeenCalledWith( expect.anything(), expect.objectContaining({ - statement: "SELECT MEASURE(arr) AS arr FROM cat.sch.revenue_metrics", + statement: + "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`revenue_metrics`", warehouse_id: "test-warehouse-id", }), expect.any(AbortSignal), @@ -1369,7 +1415,7 @@ describe("metric — filter translator", () => { expect(executeMock.mock.calls[0][1]).toEqual( expect.objectContaining({ statement: - "SELECT MEASURE(ghost_measure) AS ghost_measure FROM cat.sch.revenue_metrics", + "SELECT MEASURE(ghost_measure) AS ghost_measure FROM `cat`.`sch`.`revenue_metrics`", }), ); @@ -1398,16 +1444,55 @@ describe("metric — filter translator", () => { describe("composeMetricCacheKey", () => { const base: { metricKey: string; + source: string; measures: string[]; format: string; executorKey: string; } = { metricKey: "revenue", + source: "cat.sch.revenue_metrics", measures: ["arr"], format: "JSON_ARRAY", executorKey: "sp", }; + test("same key but different source → different keys (config repoint is not stale-served)", () => { + const a = composeMetricCacheKey({ ...base, source: "cat.sch.old_view" }); + const b = composeMetricCacheKey({ ...base, source: "cat.sch.new_view" }); + expect(a).not.toEqual(b); + }); + + test("timeDimension does NOT fork the key when timeGrain is absent (no SQL effect)", () => { + // Without a grain, renderDimensionClause emits the bare column, so + // timeDimension has no effect on the SQL — two such calls must cache-hit. + const withTd = composeMetricCacheKey({ + ...base, + dimensions: ["order_date"], + timeDimension: "order_date", + }); + const withoutTd = composeMetricCacheKey({ + ...base, + dimensions: ["order_date"], + }); + expect(withTd).toEqual(withoutTd); + }); + + test("timeDimension DOES fork the key when timeGrain is set (changes the SQL)", () => { + const a = composeMetricCacheKey({ + ...base, + dimensions: ["order_date", "region"], + timeGrain: "month", + timeDimension: "order_date", + }); + const b = composeMetricCacheKey({ + ...base, + dimensions: ["order_date", "region"], + timeGrain: "month", + timeDimension: "region", + }); + expect(a).not.toEqual(b); + }); + test("measure ORDER does not affect the key (sorted before hashing)", () => { const a = composeMetricCacheKey({ ...base, diff --git a/packages/appkit/src/type-generator/mv-registry/config.ts b/packages/appkit/src/type-generator/mv-registry/config.ts index 1995f0723..e91e3b95e 100644 --- a/packages/appkit/src/type-generator/mv-registry/config.ts +++ b/packages/appkit/src/type-generator/mv-registry/config.ts @@ -94,26 +94,13 @@ function isValidMetricKey(key: string): boolean { return /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key); } -/** - * Total predicate: is `fqn` a well-formed three-part UC metric view FQN? - * - * Well-formed = exactly three non-empty, dot-separated segments, each a valid - * Unity Catalog object name per the shared {@link UC_FQN_PATTERN} (the single - * source of truth, also used by the canonical Zod schema). Used as the - * defense-in-depth re-check at the describe fetcher seam; {@link resolveMetricConfig} - * runs the same checks but with specific, staged error messages. - * - * @note Segment length ({@link MAX_FQN_SEGMENT_LENGTH}) is NOT checked here — - * an over-long but otherwise legal name is still "valid shape". The length cap - * is a separate concern enforced (with its own message) in resolveMetricConfig. - */ -export function isValidFqn(fqn: string): boolean { - const segments = fqn.split("."); - if (segments.length !== FQN_SEGMENT_COUNT) { - return false; - } - return segments.every((segment) => UC_FQN_PATTERN.test(segment)); -} +// `isValidFqn` (the three-part UC FQN predicate) now lives in the shared +// zod-free leaf alongside `UC_FQN_PATTERN` and `quoteFqnForSql`, so the +// type-generator and the analytics runtime share one grammar + one escaper. +// `resolveMetricConfig` below intentionally does NOT call it — it re-derives +// the same checks inline so it can emit specific, staged error messages +// (arity, per-segment charset, per-segment length) rather than a single +// boolean. /** * Field allowlists enforced by {@link resolveMetricConfig}. diff --git a/packages/appkit/src/type-generator/mv-registry/describe.ts b/packages/appkit/src/type-generator/mv-registry/describe.ts index db3231d67..a77caf75b 100644 --- a/packages/appkit/src/type-generator/mv-registry/describe.ts +++ b/packages/appkit/src/type-generator/mv-registry/describe.ts @@ -1,7 +1,13 @@ import type { WorkspaceClient } from "@databricks/sdk-experimental"; +// Grammar + SQL-quoting for metric-view FQNs live together in the shared, +// zod-free leaf so the type-generator and the analytics runtime validate and +// escape against one source of truth (see the module doc in metric-fqn.ts). +import { + isValidFqn, + quoteFqnForSql, +} from "../../../../shared/src/schemas/metric-fqn"; import { type DescribeFormatMemo, describeAdaptive } from "../statement-result"; import type { DatabricksStatementExecutionResponse } from "../types"; -import { isValidFqn } from "./config"; import type { DescribeFetcher, MetricColumnMetadata } from "./types"; /** @@ -284,49 +290,6 @@ function inferTimeGrains(type: string): string[] | undefined { return undefined; } -/** - * Quote a dot-separated FQN for safe interpolation into a Spark/Databricks SQL - * statement. - * - * Each dot-split segment is wrapped in backtick-quoted-identifier syntax. The - * one character that can break out of a backtick-quoted identifier is the - * backtick itself, escaped by doubling (`` ` `` → `` `` ``) — so every backtick - * inside a segment is doubled before the segment is wrapped. Control characters - * and newlines have no valid escape inside a quoted identifier, so a segment - * containing one is rejected outright. - * - * This is a pure, standalone escaper: it is intentionally independent of FQN - * naming validation ({@link isValidFqn}). Naming validation decides whether an - * FQN is an acceptable metric source; this function only guarantees that - * whatever it is handed cannot break out of the quoted identifier it produces. - * - * An ordinary identifier is unchanged apart from the wrapping backticks: - * `catalog.schema.view` → `` `catalog`.`schema`.`view` ``. - * - * @param fqn - Dot-separated identifier (e.g. `catalog.schema.view`). - * @returns The backtick-quoted, escaped identifier ready for interpolation. - * @throws If any segment contains a control character or newline. - */ -export function quoteFqnForSql(fqn: string): string { - // Reject anything that cannot be represented inside a backtick-quoted - // identifier. \p{Cc} is the Unicode "control" category, which covers C0 - // (incl. \n, \r, \t), DEL, and C1 — i.e. every control character/newline. - const CONTROL_OR_NEWLINE = /\p{Cc}/u; - return fqn - .split(".") - .map((segment) => { - if (CONTROL_OR_NEWLINE.test(segment)) { - throw new Error( - `Cannot quote FQN segment "${segment}" for SQL: it contains a control character or newline, which has no valid escape inside a backtick-quoted identifier.`, - ); - } - // Double every backtick — the only break-out from a backtick-quoted - // identifier — then wrap the whole segment in backticks. - return `\`${segment.replace(/`/g, "``")}\``; - }) - .join("."); -} - /** * Build a DescribeFetcher from a real WorkspaceClient + warehouseId. */ diff --git a/packages/appkit/src/type-generator/tests/mv-registry.test.ts b/packages/appkit/src/type-generator/tests/mv-registry.test.ts index 068861405..8a06aca68 100644 --- a/packages/appkit/src/type-generator/tests/mv-registry.test.ts +++ b/packages/appkit/src/type-generator/tests/mv-registry.test.ts @@ -3,13 +3,16 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, test } from "vitest"; +// `quoteFqnForSql` now lives in the shared zod-free leaf alongside the FQN +// grammar (moved so the analytics runtime can reuse it); the describe seam +// imports it from there. +import { quoteFqnForSql } from "../../../../shared/src/schemas/metric-fqn"; import { metricSourceSchema } from "../../../../shared/src/schemas/metric-source"; import { readMetricConfig, resolveMetricConfig } from "../mv-registry/config"; import { createWorkspaceDescribeFetcher, extractMetricColumns, parseDescribeTableExtendedJson, - quoteFqnForSql, } from "../mv-registry/describe"; import { generateMetricTypeDeclarations } from "../mv-registry/render-types"; import { syncMetrics } from "../mv-registry/sync"; diff --git a/packages/shared/src/schemas/metric-fqn.ts b/packages/shared/src/schemas/metric-fqn.ts index b958cd1a8..98fca3d57 100644 --- a/packages/shared/src/schemas/metric-fqn.ts +++ b/packages/shared/src/schemas/metric-fqn.ts @@ -81,3 +81,80 @@ export const MAX_UC_OBJECT_NAME_LENGTH = 255; */ // biome-ignore lint/suspicious/noControlCharactersInRegex: UC explicitly prohibits ASCII control characters in object names; this negated class encodes that rule. export const UC_FQN_PATTERN = /^[^\x00-\x20\x7f./]+$/; + +/** A metric view FQN is exactly three segments: catalog.schema.metric_view. */ +const FQN_SEGMENT_COUNT = 3; + +/** + * Total predicate: is `fqn` a well-formed three-part UC metric view FQN? + * + * Well-formed = exactly three non-empty, dot-separated segments, each a valid + * Unity Catalog object name per {@link UC_FQN_PATTERN}. This is the shared, + * zod-free grammar check reused by every layer that must agree on FQN shape: + * the type-generator's config resolver and describe seam, and the analytics + * runtime's SQL builder. It is the boolean sibling of the composed + * `UC_THREE_PART_FQN_PATTERN` regex in `./metric-source.ts` (which stays a + * regex so zod can emit a JSON-schema `pattern`); both derive their per-segment + * charset from {@link UC_FQN_PATTERN}, so they cannot diverge. + * + * @note Segment length ({@link MAX_UC_OBJECT_NAME_LENGTH}) is NOT checked here — + * an over-long but otherwise legal name is still "valid shape". Callers that + * care about the length cap enforce it separately with their own message. + * + * @example + * isValidFqn("main.analytics.revenue"); // true + * isValidFqn("prod-data.analytics.rev"); // true (hyphens are UC-legal) + * isValidFqn("main.analytics"); // false (only two segments) + */ +export function isValidFqn(fqn: string): boolean { + const segments = fqn.split("."); + if (segments.length !== FQN_SEGMENT_COUNT) { + return false; + } + return segments.every((segment) => UC_FQN_PATTERN.test(segment)); +} + +/** + * Quote a dot-separated FQN for safe interpolation into a Spark/Databricks SQL + * statement. + * + * Each dot-split segment is wrapped in backtick-quoted-identifier syntax. The + * one character that can break out of a backtick-quoted identifier is the + * backtick itself, escaped by doubling (`` ` `` → `` `` ``) — so every backtick + * inside a segment is doubled before the segment is wrapped. Control characters + * and newlines have no valid escape inside a quoted identifier, so a segment + * containing one is rejected outright. + * + * This is a pure, standalone escaper: it is intentionally independent of FQN + * naming validation ({@link isValidFqn}). Naming validation decides whether an + * FQN is an acceptable metric source; this function only guarantees that + * whatever it is handed cannot break out of the quoted identifier it produces. + * Grammar and quoting live together here so a metric source is validated and + * escaped against one shared source of truth. + * + * An ordinary identifier is unchanged apart from the wrapping backticks: + * `catalog.schema.view` → `` `catalog`.`schema`.`view` ``. + * + * @param fqn - Dot-separated identifier (e.g. `catalog.schema.view`). + * @returns The backtick-quoted, escaped identifier ready for interpolation. + * @throws If any segment contains a control character or newline. + */ +export function quoteFqnForSql(fqn: string): string { + // Reject anything that cannot be represented inside a backtick-quoted + // identifier. \p{Cc} is the Unicode "control" category, which covers C0 + // (incl. \n, \r, \t), DEL, and C1 — i.e. every control character/newline. + const CONTROL_OR_NEWLINE = /\p{Cc}/u; + return fqn + .split(".") + .map((segment) => { + if (CONTROL_OR_NEWLINE.test(segment)) { + throw new Error( + `Cannot quote FQN segment "${segment}" for SQL: it contains a control character or newline, which has no valid escape inside a backtick-quoted identifier.`, + ); + } + // Double every backtick — the only break-out from a backtick-quoted + // identifier — then wrap the whole segment in backticks. + return `\`${segment.replace(/`/g, "``")}\``; + }) + .join("."); +} From 00ec7e760b147e5c09e9f661499ac8c6213f8dd8 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 10 Jul 2026 11:11:32 +0200 Subject: [PATCH 07/18] chore(appkit): load metric registry lazily with an mtime-validated cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The metric registry memoized both success AND failure on the first `/metric/:key` request and never re-read: editing `metric-views.json` needed a server restart, and a transient first-request error latched a 503 forever — unlike the sibling `.sql` query path, which re-reads `config/queries/` per request. It was also a synchronous `readFileSync`, blocking the event loop for every request (metric or not) under load. Match the `.sql` path's behavior, then beat its per-request cost: - `loadMetricRegistry` is now async (`fs.promises`) and stays a pure, stateless parse. Metric views are already heavier than a plain query on the warehouse side, so the SDK layer must not add a blocking read. - New `getMetricRegistry(dir)` wraps it with a module-level cache keyed by the queries DIR (not the plugin instance): the registry is a pure function of the config file, warehouse-independent, so two plugins at one dir share one parse. Each request does a single async `stat`; the read + JSON.parse + zod validation are skipped when the file's (mtimeMs, size) signature is unchanged — steady-state cost is below the `.sql` path (a stat, not a full read). - Failures are NOT cached (cache populated only on a successful parse), so a fixed config self-heals on the next request; an edit bumps mtime so a working config hot-reloads; an absent file stays dormant (ENOENT → empty registry). - Deletes the `metricRegistry` / `metricRegistryLoadError` fields and the `_getMetricRegistry` memo; the route calls `getMetricRegistry` in a try/catch → 503 on throw. Registry loading is now exercised through real files: `AnalyticsPlugin` takes a test-only `queriesDir` config (documented `@internal`), so tests point the loader at a temp dir and drive the real stat→read→cache path instead of poking private state. Removes the `setRegistry` backdoor. Adds hot-reload, self-heal, and mtime-cache tests. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../appkit/src/plugins/analytics/analytics.ts | 79 ++-- .../appkit/src/plugins/analytics/metric.ts | 126 +++++- .../plugins/analytics/tests/metric.test.ts | 406 +++++++++++++----- .../appkit/src/plugins/analytics/types.ts | 11 + 4 files changed, 454 insertions(+), 168 deletions(-) diff --git a/packages/appkit/src/plugins/analytics/analytics.ts b/packages/appkit/src/plugins/analytics/analytics.ts index c8bf5d4e1..451373ccd 100644 --- a/packages/appkit/src/plugins/analytics/analytics.ts +++ b/packages/appkit/src/plugins/analytics/analytics.ts @@ -32,8 +32,9 @@ import manifest from "./manifest.json"; import { buildMetricSql, composeMetricCacheKey, + QUERIES_DIR as DEFAULT_QUERIES_DIR, deriveMetricExecutorKey, - loadMetricRegistry, + getMetricRegistry, validateMetricRequest, } from "./metric"; import { QueryProcessor } from "./query"; @@ -125,28 +126,18 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { private _arrowCapability = new Map(); /** - * Metric-view registry parsed from `config/queries/metric-views.json`, keyed - * by metric key. Loaded lazily on the first `/metric/:key` request and - * memoized (see {@link _getMetricRegistry}). Empty when no config is present - * — the metric-view path stays dormant until an app opts in. + * Directory the metric-view registry is read from (holds + * `metric-views.json`). Defaults to `config/queries/` under the process cwd + * — the same directory the `.sql` query path uses. Overridable ONLY via + * `config.queriesDir` for tests (see {@link IAnalyticsConfig.queriesDir}). */ - private metricRegistry: Record | null = null; - - /** - * Latched error from the most recent {@link loadMetricRegistry} attempt. - * `null` means the registry loaded cleanly (or `metric-views.json` was absent - * — also fine; metric views are opt-in). When non-null, every `/metric/:key` - * request returns 503 `METRIC_REGISTRY_LOAD_FAILED` so a broken config - * (unreadable file, invalid JSON, schema violation) surfaces as a clear - * server status rather than masquerading as a 404 for every key. The full - * reason goes to telemetry only. - */ - private metricRegistryLoadError: string | null = null; + private readonly _queriesDir: string; constructor(config: IAnalyticsConfig) { super(config); this.config = config; this.queryProcessor = new QueryProcessor(); + this._queriesDir = config.queriesDir ?? DEFAULT_QUERIES_DIR; this.SQLClient = new SQLWarehouseConnector({ timeout: config.timeout, @@ -466,32 +457,6 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { ); } - /** - * Lazily load and memoize the metric registry from - * `config/queries/metric-views.json`. - * - * A malformed config latches `metricRegistryLoadError` and yields an empty - * registry so the route can return a 503 (distinguishing a broken deployment - * from an unknown key, which is a 404). Loading is deferred to the first - * `/metric/:key` request rather than the constructor so apps that never adopt - * metric views pay no parse cost and a config error can't break plugin - * construction. - */ - private _getMetricRegistry(): Record { - if (this.metricRegistry === null) { - try { - this.metricRegistry = loadMetricRegistry(); - this.metricRegistryLoadError = null; - } catch (err) { - const reason = err instanceof Error ? err.message : String(err); - logger.warn("Failed to load metric registry: %s", reason); - this.metricRegistry = {}; - this.metricRegistryLoadError = reason; - } - } - return this.metricRegistry; - } - /** * Handle metric-view execution requests (`POST /api/analytics/metric/:key`). * @@ -527,15 +492,20 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { return; } - const registry = this._getMetricRegistry(); - - // Surface a registry-load failure on the route. Without this, a malformed - // metric-views.json would yield 404 for every key — identical to "key - // never registered" — and hide the deployment error. Full reason → - // telemetry only. - if (this.metricRegistryLoadError !== null) { + // Resolve the registry from disk (cached + mtime-revalidated by + // `getMetricRegistry`, so the steady-state cost is one `stat`). A malformed + // config throws → 503, distinguishing a broken deployment from an unknown + // key (404). The failure is NOT latched: `getMetricRegistry` only caches on + // a successful parse, so fixing `metric-views.json` heals on the next + // request. Full reason → telemetry only. + let registry: Record; + try { + registry = await getMetricRegistry(this._queriesDir); + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + logger.warn(req, "Failed to load metric registry: %s", reason); event?.setContext("analytics", { - metric_registry_load_error: this.metricRegistryLoadError, + metric_registry_load_error: reason, }); res.status(503).json({ error: "Metric registry not available", @@ -546,10 +516,9 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { // Own-property lookup: never resolve `key` to an inherited // `Object.prototype` member (`__proto__`, `constructor`, `toString`, …). - // The loader already builds a null-prototype registry, but the load-error - // fallback and test-injected registries are plain objects, so gate the - // read here too — an inherited hit would otherwise bypass the 404 below - // and flow a non-registration value into execution. + // `getMetricRegistry` returns a null-prototype map, but gate the read here + // too as defense-in-depth — an inherited hit would otherwise bypass the 404 + // below and flow a non-registration value into execution. const registration = Object.hasOwn(registry, key) ? registry[key] : undefined; diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts index 5ed31946a..a74c3cb4e 100644 --- a/packages/appkit/src/plugins/analytics/metric.ts +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import fs from "node:fs"; +import fs from "node:fs/promises"; import path from "node:path"; import { type SQLTypeMarker, sql as sqlHelpers } from "shared"; import { z } from "zod"; @@ -29,9 +29,10 @@ const logger = createLogger("analytics:metric"); /** * Default queries directory. Mirrors `AppManager`'s * `path.resolve(process.cwd(), "config/queries")` so dev mode and production - * share a single source of truth for where metric config lives. + * share a single source of truth for where metric config lives. Exported so + * `AnalyticsPlugin` can default `config.queriesDir` to the same path. */ -const QUERIES_DIR = path.resolve(process.cwd(), "config/queries"); +export const QUERIES_DIR = path.resolve(process.cwd(), "config/queries"); const METRIC_CONFIG_FILE = "metric-views.json"; /** @@ -154,28 +155,33 @@ function laneFromExecutor( /** * Read and validate `config/queries/metric-views.json` into a metric registry. * - * Synchronous by design — registration is a pure config parse with no - * warehouse round-trip, no `DESCRIBE`, and no build-time metadata bundle. The - * single `metricViews` map makes keys unique by construction, so there is no - * cross-lane duplicate-key check. + * Async and stateless — registration is a pure config parse with no warehouse + * round-trip, no `DESCRIBE`, and no build-time metadata bundle. The single + * `metricViews` map makes keys unique by construction, so there is no + * cross-lane duplicate-key check. Async I/O (rather than `readFileSync`) keeps + * the event loop free: metric views are already heavier than a plain `.sql` + * query on the warehouse side, so the SDK layer must not add a blocking read + * on top. Caching + mtime-revalidation is layered on top by + * {@link getMetricRegistry} — this function always hits disk. * * Returns an empty registry when the file is absent: the metric-view path is * additive and dormant until an app opts in by adding the config. A malformed * file (unreadable, invalid JSON, or schema violation) throws — the caller - * latches the failure so the route can surface a 503 rather than masking a - * broken deployment as a 404 for every key. + * surfaces a 503 rather than masking a broken deployment as a 404 for every + * key. The failure is NOT cached (see {@link getMetricRegistry}), so fixing the + * file heals on the next request. */ -export function loadMetricRegistry( +export async function loadMetricRegistry( queriesDir: string = QUERIES_DIR, -): Record { +): Promise> { const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); let raw: string; try { - raw = fs.readFileSync(metricPath, "utf8"); + raw = await fs.readFile(metricPath, "utf8"); } catch (err) { if ((err as NodeJS.ErrnoException).code === "ENOENT") { - return {}; + return Object.create(null); } throw err; } @@ -219,6 +225,100 @@ export function loadMetricRegistry( return registry; } +/** + * Signature of the config file the cached registry was parsed from: its + * modification time and size. Cheap to obtain (one `stat`) and sufficient to + * detect an edit — the same pair file-watching tools compare. A content hash + * would mean reading the file, which is exactly the cost the cache avoids. + */ +interface RegistryCacheSignature { + mtimeMs: number; + size: number; +} + +/** + * Module-level registry cache, keyed by the resolved queries directory. + * + * Keyed by DIR (not by plugin instance) because the registry is a pure + * function of the config file at that path — warehouse-independent, and two + * `AnalyticsPlugin` instances pointed at the same `config/queries/` MUST see + * the same registry. Instance state would parse the identical file twice and + * risk divergence; a dir-keyed module cache shares one parse. + */ +const metricRegistryCache = new Map< + string, + { + signature: RegistryCacheSignature; + registry: Record; + } +>(); + +/** + * Resolve the metric registry for `queriesDir`, re-reading + re-parsing only + * when `metric-views.json` has changed since the cached copy. + * + * Behaves like the sibling `.sql` query path (which re-reads per request) but + * cheaper: the steady-state cost is a single async `stat`, and the read + + * `JSON.parse` + zod validation are skipped when the file's `(mtimeMs, size)` + * signature is unchanged. This delivers the agreed semantics without the old + * permanent memo: + * + * - **Hot-reload** — editing a working config bumps `mtimeMs`, so the next + * request re-parses and serves the new registry with no server restart. + * - **Self-heal** — a load failure is NOT cached (we only populate the cache + * on a successful parse), so a fixed config is picked up on the next + * request instead of latching a 503 forever. + * - **Dormant** — an absent file `stat`s as `ENOENT` → empty registry; adding + * the file later is picked up on the next request. + * + * Concurrency: two simultaneous cold requests may both `stat`+read before + * either populates the cache. That is a harmless redundant read of the same + * file (the sibling `.sql` path does not single-flight either), so no lock is + * taken. + * + * @throws Propagates {@link loadMetricRegistry}'s throw on a malformed file so + * the route can surface a 503; the cache is left untouched on failure. + */ +export async function getMetricRegistry( + queriesDir: string = QUERIES_DIR, +): Promise> { + const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); + + let signature: RegistryCacheSignature | null; + try { + const stats = await fs.stat(metricPath); + signature = { mtimeMs: stats.mtimeMs, size: stats.size }; + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + // Absent file → dormant. Drop any stale cache entry (the file may have + // been deleted) and return an empty registry without touching the cache. + metricRegistryCache.delete(queriesDir); + signature = null; + } else { + throw err; + } + } + + if (signature === null) { + return Object.create(null); + } + + const cached = metricRegistryCache.get(queriesDir); + if ( + cached !== undefined && + cached.signature.mtimeMs === signature.mtimeMs && + cached.signature.size === signature.size + ) { + return cached.registry; + } + + // Cold or stale: re-read + re-parse. Cache ONLY on success so a malformed + // file never latches — the next request re-attempts and heals. + const registry = await loadMetricRegistry(queriesDir); + metricRegistryCache.set(queriesDir, { signature, registry }); + return registry; +} + /** * Validate a metric-view FQN and return it backtick-quoted for interpolation. * diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index cde552852..598bf9323 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -16,6 +16,7 @@ import { buildMetricSql, composeMetricCacheKey, deriveMetricExecutorKey, + getMetricRegistry, loadMetricRegistry, validateMetricRequest, } from "../metric"; @@ -60,13 +61,58 @@ vi.mock("../../../cache", () => ({ }, })); -/** Feed the plugin a registry directly, bypassing the disk config parse. */ -function setRegistry( - plugin: AnalyticsPlugin, - registry: Record, +// Temp dirs created by `registryDir` / `writeRegistry`, cleaned up after each +// test. Using real files (via the test-only `queriesDir` config) exercises the +// actual stat → read → cache path in `getMetricRegistry` rather than poking +// private plugin state. +const tempRegistryDirs: string[] = []; + +/** + * Write a `metric-views.json` into a fresh temp dir and return the dir, for use + * as `new AnalyticsPlugin({ ...config, queriesDir })`. Accepts the internal + * `MetricRegistration` shape (matching the old `setRegistry` helper) and maps + * each entry's `lane` back to the config's `executor` field. + */ +function registryDir(registry: Record): string { + const dir = mkdtempSync(path.join(tmpdir(), "mv-route-")); + tempRegistryDirs.push(dir); + const metricViews: Record = {}; + for (const [key, reg] of Object.entries(registry)) { + metricViews[key] = { + source: reg.source, + executor: reg.lane === "obo" ? "user" : "app_service_principal", + }; + } + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ metricViews }), + ); + return dir; +} + +/** + * Overwrite the `metric-views.json` in an existing temp dir (for hot-reload / + * self-heal tests). `raw` lets a test write deliberately malformed content. + */ +function writeRegistry( + dir: string, + content: Record | string, ): void { - (plugin as any).metricRegistry = registry; - (plugin as any).metricRegistryLoadError = null; + const body = + typeof content === "string" + ? content + : JSON.stringify({ + metricViews: Object.fromEntries( + Object.entries(content).map(([key, reg]) => [ + key, + { + source: reg.source, + executor: reg.lane === "obo" ? "user" : "app_service_principal", + }, + ]), + ), + }); + writeFileSync(path.join(dir, "metric-views.json"), body); } describe("analytics metric route (Phase 1)", () => { @@ -83,6 +129,10 @@ describe("analytics metric route (Phase 1)", () => { afterEach(() => { serviceContextMock?.restore(); + while (tempRegistryDirs.length > 0) { + const dir = tempRegistryDirs.pop(); + if (dir) rmSync(dir, { recursive: true, force: true }); + } }); describe("injectRoutes", () => { @@ -365,15 +415,17 @@ describe("analytics metric route (Phase 1)", () => { // byte-identical to the /query route's JSON SSE path. describe("_handleMetricRoute SSE envelope", () => { test("streams warehouse_status then a result message with aliased rows", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "sp", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), }); + const { router, getHandler } = createMockRouter(); const executeMock = vi.fn().mockResolvedValue({ result: { data: [{ arr: 1234 }] }, @@ -410,15 +462,17 @@ describe("analytics metric route (Phase 1)", () => { }); test("emits warehouse_status before result for a STARTING warehouse", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "sp", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), }); + const { router, getHandler } = createMockRouter(); const executeMock = vi.fn().mockResolvedValue({ result: { data: [{ arr: 1 }] }, @@ -461,15 +515,17 @@ describe("analytics metric route (Phase 1)", () => { }); test("returns 400 when the body fails structural validation", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "sp", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), }); + const { router, getHandler } = createMockRouter(); plugin.injectRoutes(router); const handler = getHandler("POST", "/metric/:key"); @@ -488,15 +544,17 @@ describe("analytics metric route (Phase 1)", () => { // ── 503-vs-404 latching + dormancy. describe("registry latching and dormancy", () => { test("unknown key against a valid registry → 404 (generic body)", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "sp", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), }); + const { router, getHandler } = createMockRouter(); plugin.injectRoutes(router); const handler = getHandler("POST", "/metric/:key"); @@ -515,18 +573,20 @@ describe("analytics metric route (Phase 1)", () => { test.each(["__proto__", "constructor", "toString", "hasOwnProperty"])( "inherited Object.prototype key %j → 404, no execution (own-property lookup)", async (dangerousKey) => { - const plugin = new AnalyticsPlugin(config); + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), + }); const { router, getHandler } = createMockRouter(); // A real (own) entry so the registry is populated but does NOT contain // the dangerous key. Without the own-property guard, `registry[key]` // would resolve the inherited prototype member and slip past the 404. - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "sp", - }, - }); const executeMock = vi.fn(); (plugin as any).SQLClient.executeStatement = executeMock; @@ -550,11 +610,13 @@ describe("analytics metric route (Phase 1)", () => { ); test("malformed registry → 503 METRIC_REGISTRY_LOAD_FAILED", async () => { - const plugin = new AnalyticsPlugin(config); + const dir = mkdtempSync(path.join(tmpdir(), "mv-route-")); + tempRegistryDirs.push(dir); + // A real malformed config on disk (invalid JSON) → the loader throws → + // the route surfaces a 503, exercising the actual load path. + writeRegistry(dir, "{ not valid json"); + const plugin = new AnalyticsPlugin({ ...config, queriesDir: dir }); const { router, getHandler } = createMockRouter(); - // Latch a load error directly (as a malformed config would). - (plugin as any).metricRegistry = {}; - (plugin as any).metricRegistryLoadError = "Invalid metric-views.json"; plugin.injectRoutes(router); const handler = getHandler("POST", "/metric/:key"); @@ -573,6 +635,104 @@ describe("analytics metric route (Phase 1)", () => { }); }); + test("self-heal: a fixed config is picked up on the next request (no restart)", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "mv-route-")); + tempRegistryDirs.push(dir); + writeRegistry(dir, "{ not valid json"); + const plugin = new AnalyticsPlugin({ ...config, queriesDir: dir }); + const { router, getHandler } = createMockRouter(); + const executeMock = vi + .fn() + .mockResolvedValue({ result: { data: [{ arr: 1 }] } }); + (plugin as any).SQLClient.executeStatement = executeMock; + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + + // First request: malformed → 503, and the failure is NOT cached. + const res1 = createMockResponse(); + await handler( + createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }), + res1, + ); + expect(res1.status).toHaveBeenCalledWith(503); + + // Fix the file. mtime changes, so the next request re-parses and serves + // it — no server restart, no latched 503. + writeRegistry(dir, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + const res2 = createMockResponse(); + await handler( + createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }), + res2, + ); + expect(res2.status).not.toHaveBeenCalledWith(503); + expect(res2.status).not.toHaveBeenCalledWith(404); + expect(executeMock).toHaveBeenCalled(); + }); + + test("hot-reload: a new key added to a working config is visible next request", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "mv-route-")); + tempRegistryDirs.push(dir); + writeRegistry(dir, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + const plugin = new AnalyticsPlugin({ ...config, queriesDir: dir }); + const { router, getHandler } = createMockRouter(); + const executeMock = vi + .fn() + .mockResolvedValue({ result: { data: [{ arr: 1 }] } }); + (plugin as any).SQLClient.executeStatement = executeMock; + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + + // `orders` is not registered yet → 404. + const res1 = createMockResponse(); + await handler( + createMockRequest({ + params: { key: "orders" }, + body: { measures: ["cnt"] }, + }), + res1, + ); + expect(res1.status).toHaveBeenCalledWith(404); + + // Add `orders` to the config. The mtime bump invalidates the cache, so + // the next request sees it without a restart. + writeRegistry(dir, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + orders: { key: "orders", source: "cat.sch.order_metrics", lane: "sp" }, + }); + const res2 = createMockResponse(); + await handler( + createMockRequest({ + params: { key: "orders" }, + body: { measures: ["cnt"] }, + }), + res2, + ); + expect(res2.status).not.toHaveBeenCalledWith(404); + expect(executeMock).toHaveBeenCalled(); + }); + test("no metric-views.json present → registry empty, unknown key 404, nothing executes", async () => { const plugin = new AnalyticsPlugin(config); const { router, getHandler } = createMockRouter(); @@ -609,11 +769,11 @@ describe("loadMetricRegistry", () => { rmSync(dir, { recursive: true, force: true }); }); - test("absent metric-views.json → empty registry (dormancy)", () => { - expect(loadMetricRegistry(dir)).toEqual({}); + test("absent metric-views.json → empty registry (dormancy)", async () => { + expect(await loadMetricRegistry(dir)).toEqual({}); }); - test("derives lane from executor (default sp, user → obo)", () => { + test("derives lane from executor (default sp, user → obo)", async () => { writeFileSync( path.join(dir, "metric-views.json"), JSON.stringify({ @@ -624,7 +784,7 @@ describe("loadMetricRegistry", () => { }), ); - const registry = loadMetricRegistry(dir); + const registry = await loadMetricRegistry(dir); expect(registry.revenue).toEqual({ key: "revenue", source: "cat.sch.revenue_metrics", @@ -637,7 +797,7 @@ describe("loadMetricRegistry", () => { }); }); - test("registry has a null prototype (no inherited-property lookups)", () => { + test("registry has a null prototype (no inherited-property lookups)", async () => { writeFileSync( path.join(dir, "metric-views.json"), JSON.stringify({ @@ -645,7 +805,7 @@ describe("loadMetricRegistry", () => { }), ); - const registry = loadMetricRegistry(dir); + const registry = await loadMetricRegistry(dir); expect(Object.getPrototypeOf(registry)).toBeNull(); // A key that would resolve to a truthy inherited member on a plain object // resolves to undefined here. @@ -653,19 +813,55 @@ describe("loadMetricRegistry", () => { expect((registry as Record).__proto__).toBeUndefined(); }); - test("malformed JSON throws", () => { + test("absent file yields a null-prototype (dormant) registry too", async () => { + // The ENOENT dormancy path must also be prototype-free — a metric key + // colliding with an inherited member can't 200 against an empty registry. + const registry = await loadMetricRegistry(dir); + expect(Object.getPrototypeOf(registry)).toBeNull(); + }); + + test("malformed JSON throws", async () => { writeFileSync(path.join(dir, "metric-views.json"), "{ not json"); - expect(() => loadMetricRegistry(dir)).toThrow(/Failed to parse/); + await expect(loadMetricRegistry(dir)).rejects.toThrow(/Failed to parse/); }); - test("schema-invalid config throws", () => { + test("schema-invalid config throws", async () => { writeFileSync( path.join(dir, "metric-views.json"), JSON.stringify({ metricViews: { revenue: { source: "not-a-three-part-fqn" } }, }), ); - expect(() => loadMetricRegistry(dir)).toThrow(/Invalid metric-views.json/); + await expect(loadMetricRegistry(dir)).rejects.toThrow( + /Invalid metric-views.json/, + ); + }); + + test("re-reads only after the file changes (mtime-validated cache)", async () => { + // getMetricRegistry caches by dir and revalidates via stat; a second call + // with no change returns the same object, and a rewrite is picked up. + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.revenue_metrics" } }, + }), + ); + const first = await getMetricRegistry(dir); + const second = await getMetricRegistry(dir); + expect(second).toBe(first); // same cached object, no re-parse + + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { + revenue: { source: "cat.sch.revenue_metrics" }, + orders: { source: "cat.sch.order_metrics" }, + }, + }), + ); + const third = await getMetricRegistry(dir); + expect(third).not.toBe(first); + expect(Object.keys(third).sort()).toEqual(["orders", "revenue"]); }); }); @@ -1375,15 +1571,17 @@ describe("metric — filter translator", () => { }); test("a well-formed-but-unknown measure reaches the warehouse and surfaces a sanitized error envelope", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "sp", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), }); + const { router, getHandler } = createMockRouter(); // The warehouse rejects the unknown column. The raw text carries the // offending name; the connector wraps it in an ExecutionError whose @@ -1755,15 +1953,17 @@ describe("metric route — lane dispatch (Phase 3)", () => { }); test("OBO-lane registration routes through asUser(req)", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "obo", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "obo", + }, + }), }); + const { router, getHandler } = createMockRouter(); const asUserSpy = vi.spyOn(plugin, "asUser"); const executeMock = vi @@ -1793,15 +1993,17 @@ describe("metric route — lane dispatch (Phase 3)", () => { }); test("SP-lane registration uses the default executor (asUser not called)", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "sp", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), }); + const { router, getHandler } = createMockRouter(); const asUserSpy = vi.spyOn(plugin, "asUser"); const executeMock = vi @@ -1830,15 +2032,17 @@ describe("metric route — lane dispatch (Phase 3)", () => { }); test("OBO metric with no user identity → canonical 401, no SQL executed", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "obo", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "obo", + }, + }), }); + const { router, getHandler } = createMockRouter(); const executeMock = vi.fn(); (plugin as any).SQLClient.executeStatement = executeMock; @@ -1865,15 +2069,17 @@ describe("metric route — lane dispatch (Phase 3)", () => { }); test("OBO metric with whitespace-only user identity → canonical 401", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "obo", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "obo", + }, + }), }); + const { router, getHandler } = createMockRouter(); const executeMock = vi.fn(); (plugin as any).SQLClient.executeStatement = executeMock; diff --git a/packages/appkit/src/plugins/analytics/types.ts b/packages/appkit/src/plugins/analytics/types.ts index e91a361f8..c01c04e23 100644 --- a/packages/appkit/src/plugins/analytics/types.ts +++ b/packages/appkit/src/plugins/analytics/types.ts @@ -23,6 +23,17 @@ export interface IAnalyticsConfig extends BasePluginConfig { * byte arrives the stream is not time-bounded. */ arrowFirstByteTimeoutMs?: number; + /** + * Directory to read the metric-view registry (`metric-views.json`) from. + * + * @internal FOR TESTS ONLY. Production always reads the fixed + * `config/queries/` path (the default), the same directory the `.sql` query + * path uses — do NOT set this to relocate config in a real app. It exists so + * tests can point the loader at a temp directory and exercise the real + * stat → read → cache path against a real file, rather than poking private + * plugin state or stubbing the loader. + */ + queriesDir?: string; } /** From deff6b86b1a571a092b94d418e68d2625197c1dc Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 10 Jul 2026 12:36:11 +0200 Subject: [PATCH 08/18] chore(appkit): quote measure/dimension identifiers + close review-round-4 gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third adversarial review round (two reviewers, deduped). Highest-priority finding: the FQN grammar drift fixed earlier was still present for measures/dimensions/filter-members — the type-generator emits DESCRIBE column names verbatim into the generated measureKeys/dimensionKeys unions, but the runtime gated them on the narrow /^[a-zA-Z_][a-zA-Z0-9_]*$/ and interpolated bare, so a UC column like `net-revenue` / `café_sales` typechecked yet 500'd at runtime (generate-but-500, same class as the FQN bug). A+C — quote column identifiers + validate early: - Add `isValidColumnName` + `quoteIdentifier` to the shared zod-free leaf. `quoteIdentifier` is the single-identifier escaper (does NOT split on `.`, so a column literally named `net.revenue` becomes one delimited identifier); `quoteFqnForSql` now maps it over dot-split segments. The column grammar is the full delimited-identifier set — reject only control/newline (what cannot be safely quoted), matching exactly what typegen can emit. - Runtime backtick-quotes measures (MEASURE(`x`) AS `x`), dimensions, the date_trunc column, and filter members. Quoting — not a narrow allowlist — is the injection boundary, so an injection-shaped name is neutralized (inert quoted column), not rejected. Row-key preservation holds: the warehouse reports the aliased column under the unquoted name, so `{ "net-revenue": … }`. - Move identifier validation into `validateMetricRequest` (via a refine on measures/dimensions/timeDimension/filter.member) so a malformed identifier returns the canonical 400 instead of failing inside the SSE execute path as a retried 500 (finding C). The builder keeps its checks as defense-in-depth. - Delete the now-unused MEASURE_NAME_PATTERN / DIMENSION_NAME_PATTERN. - Fixed a regression this introduces: the filter sort key and canonicalize fingerprint used a "/" delimiter that was safe only while members couldn't contain "/"; now that members accept the full grammar, both JSON-encode the tuple so distinct (member, operator[, values]) pairs can't collide and fork/merge cache entries. B — registry cache signature adds ctimeMs (from the same stat): a same-size, same-mtime edit (equal-length source/executor swap on a coarse-mtime FS) now invalidates, closing a stale-source/stale-lane serve. + same-size regression test. D — runtime config caps now match the type-generator: MAX_METRIC_VIEWS (200) and per-segment length (255) enforced via the shared schema's superRefine, plus a declarative maxLength (767) on `source` (the only cap `z.toJSONSchema` can serialize — regenerated metric-source.schema.json). Refinements are invisible to JSON-schema generation, so runtime + typegen stay the authoritative gates. E — export test-only `__resetMetricRegistryCache` so cache isolation between tests is intentional, not an accident of unique temp dirs. F — comment that a non-ENOENT stat error is deliberately fatal-per-request (self-heals next call). Co-authored-by: Isaac Signed-off-by: Atila Fassina --- docs/static/schemas/metric-source.schema.json | 1 + .../appkit/src/plugins/analytics/metric.ts | 221 +++++++++------ .../plugins/analytics/tests/metric.test.ts | 264 +++++++++++++----- packages/shared/src/schemas/metric-fqn.ts | 67 +++-- packages/shared/src/schemas/metric-source.ts | 55 +++- 5 files changed, 446 insertions(+), 162 deletions(-) diff --git a/docs/static/schemas/metric-source.schema.json b/docs/static/schemas/metric-source.schema.json index 1976ea7ce..e606b92da 100644 --- a/docs/static/schemas/metric-source.schema.json +++ b/docs/static/schemas/metric-source.schema.json @@ -21,6 +21,7 @@ "properties": { "source": { "type": "string", + "maxLength": 767, "pattern": "^[^\\x00-\\x20\\x7f./]+\\.[^\\x00-\\x20\\x7f./]+\\.[^\\x00-\\x20\\x7f./]+$", "description": "Three-part Unity Catalog FQN of the metric view: ..", "examples": [ diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts index a74c3cb4e..aabfdc89d 100644 --- a/packages/appkit/src/plugins/analytics/metric.ts +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -8,8 +8,10 @@ import { z } from "zod"; // type-generator's runtime, which pulls the zod-free `metric-fqn.ts` from the // same tree) so the runtime and the generated JSON schema validate identically. import { + isValidColumnName, isValidFqn, quoteFqnForSql, + quoteIdentifier, } from "../../../../shared/src/schemas/metric-fqn"; import { metricSourceSchema } from "../../../../shared/src/schemas/metric-source"; import { AuthenticationError, ValidationError } from "../../errors"; @@ -36,32 +38,20 @@ export const QUERIES_DIR = path.resolve(process.cwd(), "config/queries"); const METRIC_CONFIG_FILE = "metric-views.json"; /** - * Validate measure names before they are interpolated into `MEASURE()`. + * Measure, dimension, and filter-member names are **column identifiers**: they + * are validated by the shared {@link isValidColumnName} (rejects only control + * characters / newlines) and backtick-quoted via {@link quoteIdentifier} at + * every interpolation point. Quoting — not a narrow ASCII allowlist — is the + * injection boundary, so the runtime accepts the full delimited-identifier + * grammar the type-generator emits from DESCRIBE (hyphens, dots, non-ASCII). + * There is deliberately NO name allowlist: a well-formed-but-unknown column + * falls through to the warehouse and surfaces as a sanitized canonical error. * - * Measure names cannot be parameterized — they are SQL identifiers, not - * literals. This conservative identifier shape is the security boundary for - * the interpolated tokens: there is deliberately NO name allowlist, so a - * well-formed-but-unknown measure falls through to the warehouse and surfaces - * as a sanitized canonical error (parity with the raw `.sql` flow). - */ -const MEASURE_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; - -/** - * Dimension name pattern. Matches the identifier shape we accept for measures - * — column references cannot be parameterized in SQL, so they must be - * conservatively safe identifiers (no spaces, no quotes, no SQL operators). - * This grammar gate is the security boundary for interpolated dimension - * tokens: there is deliberately NO name allowlist, so a well-formed-but-unknown - * dimension falls through to the warehouse and surfaces as a sanitized - * canonical error. - */ -const DIMENSION_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; - -/** - * Time-grain token shape. This grammar gate is the security boundary for the - * grain: it is interpolated as a single-quoted `date_trunc` unit literal (NOT - * a bind param) in {@link renderDimensionClause}, so the pattern is what keeps - * a hostile token out of that quoted position. + * Time-grain token shape. Unlike the column identifiers above, the grain is + * interpolated as a single-quoted `date_trunc` unit LITERAL (NOT a bind param, + * NOT a delimited identifier) in {@link renderDimensionClause}, so it keeps a + * narrow keyword-shaped gate — that pattern is what keeps a hostile token out + * of the quoted-literal position. */ const TIME_GRAIN_PATTERN = /^[a-z][a-z_]*$/; @@ -227,11 +217,22 @@ export async function loadMetricRegistry( /** * Signature of the config file the cached registry was parsed from: its - * modification time and size. Cheap to obtain (one `stat`) and sufficient to - * detect an edit — the same pair file-watching tools compare. A content hash - * would mean reading the file, which is exactly the cost the cache avoids. + * change time, modification time, and size — all from one `stat`, no read. + * + * `ctimeMs` (inode change time) is the key guard against a stale serve: it + * bumps on ANY metadata or content change and, unlike `mtimeMs`, cannot be + * restored to a prior value by tooling (`utimes` sets atime/mtime but not + * ctime). So an edit that preserves both size and mtime — a same-length value + * swap (e.g. repointing `source` to an equal-length FQN, or flipping + * `executor` between two equal-length values) on a coarse-mtime filesystem — + * still changes ctime and invalidates the cache. Without it, such an edit + * could serve a stale `source`/`lane`, reintroducing the exact stale-serve + * class the cache-key `source` salt was added to prevent, one layer up. + * A content hash would be stronger still but requires reading the file, which + * is the cost this cache exists to avoid. */ interface RegistryCacheSignature { + ctimeMs: number; mtimeMs: number; size: number; } @@ -253,15 +254,28 @@ const metricRegistryCache = new Map< } >(); +/** + * Clear the module-level registry cache. + * + * @internal FOR TESTS ONLY. The cache is keyed by directory and lives for the + * process; production never needs to clear it (a changed file is picked up via + * the stat signature). Tests that reuse a directory — or assert cold-load + * behavior — call this in `beforeEach`/`afterEach` so isolation is intentional + * rather than relying on each test happening to use a unique temp dir. + */ +export function __resetMetricRegistryCache(): void { + metricRegistryCache.clear(); +} + /** * Resolve the metric registry for `queriesDir`, re-reading + re-parsing only * when `metric-views.json` has changed since the cached copy. * * Behaves like the sibling `.sql` query path (which re-reads per request) but * cheaper: the steady-state cost is a single async `stat`, and the read + - * `JSON.parse` + zod validation are skipped when the file's `(mtimeMs, size)` - * signature is unchanged. This delivers the agreed semantics without the old - * permanent memo: + * `JSON.parse` + zod validation are skipped when the file's + * `(ctimeMs, mtimeMs, size)` signature is unchanged. This delivers the agreed + * semantics without the old permanent memo: * * - **Hot-reload** — editing a working config bumps `mtimeMs`, so the next * request re-parses and serves the new registry with no server restart. @@ -287,7 +301,11 @@ export async function getMetricRegistry( let signature: RegistryCacheSignature | null; try { const stats = await fs.stat(metricPath); - signature = { mtimeMs: stats.mtimeMs, size: stats.size }; + signature = { + ctimeMs: stats.ctimeMs, + mtimeMs: stats.mtimeMs, + size: stats.size, + }; } catch (err) { if ((err as NodeJS.ErrnoException).code === "ENOENT") { // Absent file → dormant. Drop any stale cache entry (the file may have @@ -295,6 +313,11 @@ export async function getMetricRegistry( metricRegistryCache.delete(queriesDir); signature = null; } else { + // Any other stat error (EACCES / EIO / ELOOP / …) is deliberately fatal + // for THIS request → the route surfaces a 503, consistent with the + // malformed-config → 503 path. It is not latched: with the self-heal + // design a transient error clears on the next request, which is strictly + // better than the old memo that could latch-and-serve-stale. throw err; } } @@ -306,6 +329,7 @@ export async function getMetricRegistry( const cached = metricRegistryCache.get(queriesDir); if ( cached !== undefined && + cached.signature.ctimeMs === signature.ctimeMs && cached.signature.mtimeMs === signature.mtimeMs && cached.signature.size === signature.size ) { @@ -374,7 +398,11 @@ const filterPredicateSchema: z.ZodType = z .object({ member: z .string() - .min(1, { message: "filter predicate 'member' cannot be empty" }), + .min(1, { message: "filter predicate 'member' cannot be empty" }) + .refine(isValidColumnName, { + message: + "filter predicate 'member' contains a character that cannot be used in a SQL identifier (control character or newline)", + }), operator: z.string().min(1, { message: "filter predicate 'operator' cannot be empty", }) as z.ZodType, @@ -411,13 +439,29 @@ const filterSchema: z.ZodType = z.lazy(() => const metricRequestSchema = z .object({ measures: z - .array(z.string().min(1, "measure name cannot be empty")) + .array( + z + .string() + .min(1, "measure name cannot be empty") + .refine(isValidColumnName, { + message: + "measure name contains a character that cannot be used in a SQL identifier (control character or newline)", + }), + ) .min(1, "at least one measure is required") .max(METRIC_MEASURES_MAX, { message: `measures length exceeds the maximum of ${METRIC_MEASURES_MAX}`, }), dimensions: z - .array(z.string().min(1, "dimension name cannot be empty")) + .array( + z + .string() + .min(1, "dimension name cannot be empty") + .refine(isValidColumnName, { + message: + "dimension name contains a character that cannot be used in a SQL identifier (control character or newline)", + }), + ) .max(METRIC_DIMENSIONS_MAX, { message: `dimensions length exceeds the maximum of ${METRIC_DIMENSIONS_MAX}`, }) @@ -434,15 +478,16 @@ const metricRequestSchema = z message: "timeGrain must match /^[a-z][a-z_]*$/", }) .optional(), - // The single dimension `timeGrain` applies to via `date_trunc`. Grammar- - // gated as a SQL identifier (column reference, cannot be parameterized). - // Cross-field rules in `superRefine`: required when `timeGrain` is set, and - // must be one of `dimensions`. + // The single dimension `timeGrain` applies to via `date_trunc`. A column + // identifier (backtick-quoted at interpolation), so it accepts the full + // delimited-identifier grammar. Cross-field rules in `superRefine`: + // required when `timeGrain` is set, and must be one of `dimensions`. timeDimension: z .string() .min(1, { message: "timeDimension cannot be empty" }) - .regex(DIMENSION_NAME_PATTERN, { - message: "timeDimension must match /^[a-zA-Z_][a-zA-Z0-9_]*$/", + .refine(isValidColumnName, { + message: + "timeDimension contains a character that cannot be used in a SQL identifier (control character or newline)", }) .optional(), limit: z @@ -765,12 +810,12 @@ export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { * [LIMIT n] * * Notes: - * - Every measure and dimension is gated by {@link MEASURE_NAME_PATTERN} / - * {@link DIMENSION_NAME_PATTERN} before it is interpolated (column - * references cannot be parameterized — they are SQL identifiers), and the - * FQN is validated and backtick-quoted by {@link quoteSafeFqn}. There is - * deliberately NO name allowlist — the grammar gate (plus quoting for the - * FQN) is the security boundary. No user-supplied string reaches the SQL + * - Every measure and dimension is validated by {@link isValidColumnName} and + * backtick-quoted by {@link quoteIdentifier} before it is interpolated + * (column references cannot be parameterized — they are SQL identifiers), + * and the FQN is validated and quoted by {@link quoteSafeFqn}. There is + * deliberately NO name allowlist — quoting is the security boundary. No + * user-supplied string reaches the SQL * string without passing a grammar gate. * - `GROUP BY ALL` is added when at least one dimension is requested. UC * requires GROUP BY when MEASURE() is mixed with non-aggregated columns; @@ -800,8 +845,13 @@ export function buildMetricSql( throw new Error("buildMetricSql requires at least one measure."); } + // Defense-in-depth re-gate: `validateMetricRequest` already rejects any name + // `isValidColumnName` refuses, but `buildMetricSql` is exported and may be + // reached without it. `quoteIdentifier` throws on a control/newline name, so + // the quoting below is itself the boundary; the explicit check keeps the + // error message uniform with the other interpolated tokens. for (const m of request.measures) { - if (!MEASURE_NAME_PATTERN.test(m)) { + if (!isValidColumnName(m)) { throw new Error( `Refusing to build SQL: measure "${m}" is not a valid identifier.`, ); @@ -810,7 +860,7 @@ export function buildMetricSql( const dimensions = request.dimensions ?? []; for (const d of dimensions) { - if (!DIMENSION_NAME_PATTERN.test(d)) { + if (!isValidColumnName(d)) { throw new Error( `Refusing to build SQL: dimension "${d}" is not a valid identifier.`, ); @@ -818,12 +868,15 @@ export function buildMetricSql( } // Deterministic order so cache keys collapse semantically equivalent calls. - // Alias each measure to its plain name so result rows have keys matching the - // registered measure (`{ arr: 1234 }`) rather than the SQL-function - // serialization Databricks returns by default (`{ "measure(arr)": 1234 }`). + // Alias each measure to its plain name (backtick-quoted) so result rows have + // keys matching the registered measure (`{ "net-revenue": 1234 }`) rather + // than the SQL-function serialization Databricks returns by default. The + // warehouse reports the aliased column under the UNQUOTED name (backticks are + // delimiters, not part of the name), so `MEASURE(`x`) AS `x`` yields a row + // key of exactly `x` — preserved through result materialization. const measureClauses = [...request.measures] .sort() - .map((m) => `MEASURE(${m}) AS ${m}`); + .map((m) => `MEASURE(${quoteIdentifier(m)}) AS ${quoteIdentifier(m)}`); const dimensionClauses = [...dimensions] .sort() @@ -880,9 +933,9 @@ interface FilterRenderState { * validator bypass cannot turn `or: []` into "match everything"). * * Defense-in-depth: even though the request body's filter has already been - * validated, every member name is re-checked against {@link - * DIMENSION_NAME_PATTERN} here. If validation is ever bypassed, the SQL - * constructor still refuses to interpolate an unknown-shaped identifier. + * validated, every member name is re-checked against {@link isValidColumnName} + * here and backtick-quoted. If validation is ever bypassed, the SQL + * constructor still refuses to interpolate a name it cannot safely quote. */ function renderFilter( node: MetricFilter, @@ -944,7 +997,7 @@ function renderFilter( // Leaf predicate — grammar-gate the member and operator, then render. const predicate = node as MetricPredicate; - if (!DIMENSION_NAME_PATTERN.test(predicate.member)) { + if (!isValidColumnName(predicate.member)) { throw new Error( `Refusing to build SQL: filter member "${predicate.member}" is not a valid identifier.`, ); @@ -984,15 +1037,15 @@ function sortFilterChildren( !("or" in child) ) { const p = child as MetricPredicate; - // Separate the fields with "/" so the (member, operator) pair maps to - // the sort key injectively. A delimiter-free concatenation collides — - // member "ab" + op "c" and member "a" + op "bc" both → "abc" — which - // tie-breaks two distinct pairs to input order and forks the cache key - // on semantically equal filters. "/" can never appear in either field - // (members match DIMENSION_NAME_PATTERN, operators are the alpha-only - // enum), matching the leaf-fingerprint delimiter canonicalizeFilter - // already uses. - key = `${p.member}/${p.operator}`; + // JSON.stringify the (member, operator) pair so the sort key is + // injective regardless of content. A plain delimiter is unsafe now that + // members accept the full delimited-identifier grammar (a member may + // contain "/", ".", etc.): `member "a/b" + op "c"` and + // `member "a" + op "b/c"` would both collapse to "a/b/c" under any + // single-char separator, tie-breaking distinct pairs to input order. + // JSON encoding escapes any separator, so distinct pairs map to distinct + // keys — the same technique canonicalizeFilter uses for values. + key = JSON.stringify([p.member, p.operator]); isPredicate = true; } else { // Nested groups don't have a single (member, operator) — keep their @@ -1034,7 +1087,11 @@ function renderPredicate( params: Record, state: FilterRenderState, ): string { - const col = predicate.member; + // Backtick-quote the column identifier (defense-in-depth: the member was + // already validated by `isValidColumnName` in `renderFilter`). Quoting is + // what lets a delimited column name — hyphens, dots, non-ASCII — reach SQL + // safely; the bound value still flows through `:f_` params, never here. + const col = quoteIdentifier(predicate.member); const op = predicate.operator; const values = predicate.values ?? []; @@ -1153,8 +1210,8 @@ function bindLikeValue( * every other interpolated identifier (measures, dimensions, `timeDimension`, * the FQN) is re-gated in the builder. * - The column cannot be parameterized (it is an identifier), so it is - * re-checked against {@link DIMENSION_NAME_PATTERN} here as belt-and- - * suspenders even though the schema already gated `timeDimension`. + * re-checked against {@link isValidColumnName} and backtick-quoted here as + * belt-and-suspenders even though the schema already gated `timeDimension`. * * The aliasing keeps result-row keys stable (`{ order_date: ... }`) regardless * of whether the grain was applied. @@ -1165,19 +1222,23 @@ function renderDimensionClause( timeDimension: string | undefined, ): string { if (timeGrain != null && dim === timeDimension) { - if (!DIMENSION_NAME_PATTERN.test(dim)) { + if (!isValidColumnName(dim)) { throw new Error( `Refusing to build SQL: timeDimension "${dim}" is not a valid identifier.`, ); } + // The grain is interpolated as a single-quoted `date_trunc` unit literal + // (NOT a bind param), so it stays gated by the narrow TIME_GRAIN_PATTERN — + // it is a keyword, not a delimited identifier. if (!TIME_GRAIN_PATTERN.test(timeGrain)) { throw new Error( `Refusing to build SQL: timeGrain "${timeGrain}" is not a valid grain token.`, ); } - return `date_trunc('${timeGrain}', ${dim}) AS ${dim}`; + const quoted = quoteIdentifier(dim); + return `date_trunc('${timeGrain}', ${quoted}) AS ${quoted}`; } - return dim; + return quoteIdentifier(dim); } /** @@ -1318,12 +1379,14 @@ function canonicalizeFilter(node: MetricFilter): string { return `${groupKey}(${childFingerprints.join(",")})`; } - // Leaf predicate. Use JSON.stringify (not String) for the value segment so - // strings carrying the `|` separator cannot collide with split arrays — - // e.g. `["a", "b"]` and `["a|string:b"]` stay distinct fingerprints. + // Leaf predicate. JSON.stringify every structural part — member, operator, + // and each value (with its type) — so no content can be confused with a + // separator. This matters now that `member` accepts the full delimited- + // identifier grammar (it may contain "/", ".", etc.): a bare + // `p(${member}/${operator}/...)` would let `member "a", op "b"` collide with + // `member "a/b"` and fork/merge cache entries. Encoding the whole tuple as + // JSON makes the fingerprint injective regardless of member/value content. const p = node as MetricPredicate; - const valuesPart = p.values - ? p.values.map((v) => `${typeof v}:${JSON.stringify(v)}`).join("|") - : ""; - return `p(${p.member}/${p.operator}/${valuesPart})`; + const valuesPart = (p.values ?? []).map((v) => [typeof v, v]); + return `p(${JSON.stringify([p.member, p.operator, valuesPart])})`; } diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index 598bf9323..45ed380fe 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { @@ -13,6 +13,7 @@ import { ServiceContext } from "../../../context/service-context"; import { AuthenticationError } from "../../../errors"; import { AnalyticsPlugin } from "../analytics"; import { + __resetMetricRegistryCache, buildMetricSql, composeMetricCacheKey, deriveMetricExecutorKey, @@ -129,6 +130,7 @@ describe("analytics metric route (Phase 1)", () => { afterEach(() => { serviceContextMock?.restore(); + __resetMetricRegistryCache(); while (tempRegistryDirs.length > 0) { const dir = tempRegistryDirs.pop(); if (dir) rmSync(dir, { recursive: true, force: true }); @@ -149,28 +151,46 @@ describe("analytics metric route (Phase 1)", () => { }); }); - // ── Grammar gate — the primary security test (replaces #341's - // fail-closed-503 allowlist test). A measure that fails MEASURE_NAME_PATTERN - // throws inside buildMetricSql BEFORE any SQL string is constructed. - describe("buildMetricSql grammar gate", () => { + // ── Identifier safety — the primary security test. Measure/dimension names + // are backtick-quoted (not narrow-gated) at interpolation, so an injection- + // shaped name is NEUTRALIZED by quoting rather than rejected: it becomes an + // inert (if nonexistent) column the warehouse resolves away. Only a name + // that cannot be safely quoted at all — one containing a control character + // or newline — is refused. + describe("buildMetricSql identifier safety (quoting)", () => { const registration: MetricRegistration = { key: "revenue", source: "cat.sch.revenue_metrics", lane: "sp", }; - test("throws before building SQL for an injection-shaped measure", () => { - expect(() => - buildMetricSql(registration, { - measures: ["arr; DROP TABLE users"], - }), - ).toThrow(/not a valid identifier/); + test("neutralizes an injection-shaped measure by quoting (no breakout)", () => { + // `arr; DROP TABLE users` has no control chars, so it is a valid (if + // nonexistent) column name: quoted whole, the `;` and `DROP` are inert + // inside the backtick-delimited identifier — not separate statements. + const { statement } = buildMetricSql(registration, { + measures: ["arr; DROP TABLE users"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr; DROP TABLE users`) AS `arr; DROP TABLE users` FROM `cat`.`sch`.`revenue_metrics`", + ); }); - test("throws for a measure with a backtick / quote", () => { + test("neutralizes a backtick in a measure by doubling it", () => { + // The one real breakout char is the backtick; quoteIdentifier doubles it + // so it cannot close the identifier early. + const { statement } = buildMetricSql(registration, { + measures: ["arr`"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr```) AS `arr``` FROM `cat`.`sch`.`revenue_metrics`", + ); + }); + + test("throws for a measure containing a control character (cannot be quoted)", () => { expect(() => - buildMetricSql(registration, { measures: ["arr`"] }), - ).toThrow(/not a valid identifier/); + buildMetricSql(registration, { measures: ["arr\ndrop"] }), + ).toThrow(/not a valid identifier|control character/); }); test("throws when no measures are supplied", () => { @@ -200,7 +220,7 @@ describe("analytics metric route (Phase 1)", () => { { measures: ["arr"] }, ); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr FROM `prod-data`.`analytics`.`revenue`", + "SELECT MEASURE(`arr`) AS `arr` FROM `prod-data`.`analytics`.`revenue`", ); }); @@ -214,7 +234,7 @@ describe("analytics metric route (Phase 1)", () => { { measures: ["arr"] }, ); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`re``v`", + "SELECT MEASURE(`arr`) AS `arr` FROM `cat`.`sch`.`re``v`", ); }); }); @@ -227,12 +247,12 @@ describe("analytics metric route (Phase 1)", () => { lane: "sp", }; - test("single measure → SELECT MEASURE(m) AS m FROM ", () => { + test("single measure → SELECT MEASURE(`m`) AS `m` FROM ", () => { const { statement, parameters } = buildMetricSql(registration, { measures: ["arr"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`revenue_metrics`", + "SELECT MEASURE(`arr`) AS `arr` FROM `cat`.`sch`.`revenue_metrics`", ); expect(parameters).toEqual({}); }); @@ -242,7 +262,7 @@ describe("analytics metric route (Phase 1)", () => { measures: ["revenue", "arr"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, MEASURE(revenue) AS revenue FROM `cat`.`sch`.`revenue_metrics`", + "SELECT MEASURE(`arr`) AS `arr`, MEASURE(`revenue`) AS `revenue` FROM `cat`.`sch`.`revenue_metrics`", ); }); @@ -252,7 +272,7 @@ describe("analytics metric route (Phase 1)", () => { limit: 10.9, }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`revenue_metrics` LIMIT 10", + "SELECT MEASURE(`arr`) AS `arr` FROM `cat`.`sch`.`revenue_metrics` LIMIT 10", ); }); }); @@ -272,7 +292,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: ["region"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + "SELECT MEASURE(`arr`) AS `arr`, `region` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); }); @@ -282,7 +302,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: ["segment", "region"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, region, segment FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + "SELECT MEASURE(`arr`) AS `arr`, `region`, `segment` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); }); @@ -292,7 +312,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: ["region"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, MEASURE(revenue) AS revenue, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + "SELECT MEASURE(`arr`) AS `arr`, MEASURE(`revenue`) AS `revenue`, `region` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); }); @@ -302,7 +322,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: [], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`revenue_metrics`", + "SELECT MEASURE(`arr`) AS `arr` FROM `cat`.`sch`.`revenue_metrics`", ); }); @@ -313,7 +333,7 @@ describe("analytics metric route (Phase 1)", () => { limit: 100, }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL LIMIT 100", + "SELECT MEASURE(`arr`) AS `arr`, `region` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL LIMIT 100", ); }); @@ -323,7 +343,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: ["order_date", "region"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, order_date, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + "SELECT MEASURE(`arr`) AS `arr`, `order_date`, `region` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); expect(statement).not.toContain("date_trunc"); }); @@ -347,10 +367,10 @@ describe("analytics metric route (Phase 1)", () => { timeDimension: "order_date", }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, date_trunc('month', order_date) AS order_date, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + "SELECT MEASURE(`arr`) AS `arr`, date_trunc('month', `order_date`) AS `order_date`, `region` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); expect(statement).toContain( - "date_trunc('month', order_date) AS order_date", + "date_trunc('month', `order_date`) AS `order_date`", ); expect(statement).toContain(" GROUP BY ALL"); }); @@ -363,7 +383,7 @@ describe("analytics metric route (Phase 1)", () => { timeDimension: "order_date", }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, date_trunc('day', order_date) AS order_date FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + "SELECT MEASURE(`arr`) AS `arr`, date_trunc('day', `order_date`) AS `order_date` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); }); @@ -383,31 +403,43 @@ describe("analytics metric route (Phase 1)", () => { }); }); - // ── Phase 2: dimension grammar gate. A dimension failing - // DIMENSION_NAME_PATTERN throws inside buildMetricSql before SQL is built. - describe("buildMetricSql dimension grammar gate", () => { + // ── Phase 2: dimension identifier safety. A dimension is backtick-quoted at + // interpolation, so an injection-shaped name is neutralized (inert quoted + // column), and only an unquotable (control-char) name throws. + describe("buildMetricSql dimension identifier safety (quoting)", () => { const registration: MetricRegistration = { key: "revenue", source: "cat.sch.revenue_metrics", lane: "sp", }; - test("throws before building SQL for an injection-shaped dimension", () => { - expect(() => - buildMetricSql(registration, { - measures: ["arr"], - dimensions: ["region; DROP TABLE users"], - }), - ).toThrow(/not a valid identifier/); + test("neutralizes an injection-shaped dimension by quoting", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region; DROP TABLE users"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr`, `region; DROP TABLE users` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + ); + }); + + test("neutralizes a backtick in a dimension by doubling it", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region`"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr`, `region``` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + ); }); - test("throws for a dimension with a backtick / quote", () => { + test("throws for a dimension containing a control character", () => { expect(() => buildMetricSql(registration, { measures: ["arr"], - dimensions: ["region`"], + dimensions: ["region\tbad"], }), - ).toThrow(/not a valid identifier/); + ).toThrow(/not a valid identifier|control character/); }); }); @@ -447,7 +479,7 @@ describe("analytics metric route (Phase 1)", () => { expect.anything(), expect.objectContaining({ statement: - "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`revenue_metrics`", + "SELECT MEASURE(`arr`) AS `arr` FROM `cat`.`sch`.`revenue_metrics`", warehouse_id: "test-warehouse-id", }), expect.any(AbortSignal), @@ -766,6 +798,7 @@ describe("loadMetricRegistry", () => { }); afterEach(() => { + __resetMetricRegistryCache(); rmSync(dir, { recursive: true, force: true }); }); @@ -837,6 +870,53 @@ describe("loadMetricRegistry", () => { ); }); + test("rejects more than 200 metric views (runtime parity with typegen cap)", async () => { + // A config that fails type generation (MAX_METRIC_VIEWS) must not silently + // pass at runtime. z.record has no `.max` in zod 4, so this is enforced by + // the schema's superRefine. + const metricViews: Record = {}; + for (let i = 0; i < 201; i++) { + metricViews[`m_${i}`] = { source: `cat.sch.view_${i}` }; + } + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ metricViews }), + ); + await expect(loadMetricRegistry(dir)).rejects.toThrow( + /Invalid metric-views.json/, + ); + }); + + test("rejects an FQN segment longer than 255 chars (per-segment cap)", async () => { + // The per-segment length cap is not expressible as a whole-string + // maxLength; the schema's superRefine enforces it so runtime matches the + // typegen resolver. + const longSegment = "a".repeat(256); + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { revenue: { source: `cat.sch.${longSegment}` } }, + }), + ); + await expect(loadMetricRegistry(dir)).rejects.toThrow( + /Invalid metric-views.json/, + ); + }); + + test("accepts exactly 200 views and a 255-char segment (boundary)", async () => { + const metricViews: Record = {}; + for (let i = 0; i < 200; i++) { + metricViews[`m_${i}`] = { source: `cat.sch.view_${i}` }; + } + // One entry with a segment at exactly the 255 limit — must pass. + metricViews.m_0 = { source: `cat.sch.${"a".repeat(255)}` }; + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ metricViews }), + ); + await expect(loadMetricRegistry(dir)).resolves.toBeDefined(); + }); + test("re-reads only after the file changes (mtime-validated cache)", async () => { // getMetricRegistry caches by dir and revalidates via stat; a second call // with no change returns the same object, and a rewrite is picked up. @@ -863,6 +943,51 @@ describe("loadMetricRegistry", () => { expect(third).not.toBe(first); expect(Object.keys(third).sort()).toEqual(["orders", "revenue"]); }); + + test("picks up a SAME-SIZE edit (ctime invalidation, not just size)", async () => { + // The failure mode size+mtime alone would miss: rewrite the config to the + // EXACT same byte length (repoint `source` to an equal-length FQN). `size` + // is unchanged and a coarse-mtime FS might not advance `mtimeMs`, but + // `ctimeMs` bumps on any write — so the new source must be served. + const p = path.join(dir, "metric-views.json"); + writeFileSync( + p, + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.rev_aaa" } }, + }), + ); + const before = await getMetricRegistry(dir); + expect(before.revenue?.source).toBe("cat.sch.rev_aaa"); + + // Same-length replacement: "rev_aaa" → "rev_bbb" keeps the file byte-count + // identical, so `size` cannot disambiguate. + const sizeBefore = statSync(p).size; + writeFileSync( + p, + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.rev_bbb" } }, + }), + ); + expect(statSync(p).size).toBe(sizeBefore); // same size, as designed + + const after = await getMetricRegistry(dir); + expect(after.revenue?.source).toBe("cat.sch.rev_bbb"); + }); + + test("__resetMetricRegistryCache forces a cold re-read", async () => { + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.revenue_metrics" } }, + }), + ); + const first = await getMetricRegistry(dir); + __resetMetricRegistryCache(); + const second = await getMetricRegistry(dir); + // Cleared cache → fresh parse → a new object (not the memoized instance). + expect(second).not.toBe(first); + expect(Object.keys(second)).toEqual(["revenue"]); + }); }); // ── Phase 2: the structured filter engine (translator + validator). @@ -893,7 +1018,7 @@ describe("metric — filter translator", () => { operator: "equals", values: ["EMEA"], }); - expect(where).toBe("region = :f_0"); + expect(where).toBe("`region` = :f_0"); expect(parameters).toEqual({ f_0: { __sql_type: "STRING", value: "EMEA" }, }); @@ -905,7 +1030,7 @@ describe("metric — filter translator", () => { operator: "notEquals", values: ["EMEA"], }); - expect(where).toBe("region <> :f_0"); + expect(where).toBe("`region` <> :f_0"); expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "EMEA" }); }); @@ -915,7 +1040,7 @@ describe("metric — filter translator", () => { operator: "in", values: ["EMEA", "APAC", "AMER"], }); - expect(where).toBe("region IN (:f_0, :f_1, :f_2)"); + expect(where).toBe("`region` IN (:f_0, :f_1, :f_2)"); expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "EMEA" }); expect(parameters.f_1).toEqual({ __sql_type: "STRING", value: "APAC" }); expect(parameters.f_2).toEqual({ __sql_type: "STRING", value: "AMER" }); @@ -927,7 +1052,7 @@ describe("metric — filter translator", () => { operator: "notIn", values: ["EMEA", "APAC"], }); - expect(where).toBe("region NOT IN (:f_0, :f_1)"); + expect(where).toBe("`region` NOT IN (:f_0, :f_1)"); expect(Object.keys(parameters)).toHaveLength(2); }); @@ -937,7 +1062,7 @@ describe("metric — filter translator", () => { operator: "gt", values: [10000], }); - expect(where).toBe("deal_size > :f_0"); + expect(where).toBe("`deal_size` > :f_0"); expect(parameters.f_0).toEqual({ __sql_type: "INT", value: "10000" }); }); @@ -947,7 +1072,7 @@ describe("metric — filter translator", () => { operator: "gte", values: [5000], }); - expect(where).toBe("deal_size >= :f_0"); + expect(where).toBe("`deal_size` >= :f_0"); }); test("lt → ` < :f_0`", () => { @@ -956,7 +1081,7 @@ describe("metric — filter translator", () => { operator: "lt", values: [100], }); - expect(where).toBe("deal_size < :f_0"); + expect(where).toBe("`deal_size` < :f_0"); }); test("lte → ` <= :f_0`", () => { @@ -965,7 +1090,7 @@ describe("metric — filter translator", () => { operator: "lte", values: [50000], }); - expect(where).toBe("deal_size <= :f_0"); + expect(where).toBe("`deal_size` <= :f_0"); }); test("contains → ` LIKE :f_0` (value wrapped in %...%)", () => { @@ -974,7 +1099,7 @@ describe("metric — filter translator", () => { operator: "contains", values: ["MEA"], }); - expect(where).toBe("region LIKE :f_0"); + expect(where).toBe("`region` LIKE :f_0"); expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "%MEA%" }); }); @@ -984,7 +1109,7 @@ describe("metric — filter translator", () => { operator: "notContains", values: ["test"], }); - expect(where).toBe("region NOT LIKE :f_0"); + expect(where).toBe("`region` NOT LIKE :f_0"); expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "%test%", @@ -996,7 +1121,7 @@ describe("metric — filter translator", () => { member: "region", operator: "set", }); - expect(where).toBe("region IS NOT NULL"); + expect(where).toBe("`region` IS NOT NULL"); expect(parameters).toEqual({}); }); @@ -1005,7 +1130,7 @@ describe("metric — filter translator", () => { member: "region", operator: "notSet", }); - expect(where).toBe("region IS NULL"); + expect(where).toBe("`region` IS NULL"); expect(parameters).toEqual({}); }); }); @@ -1018,7 +1143,7 @@ describe("metric — filter translator", () => { { member: "segment", operator: "equals", values: ["Enterprise"] }, ], }); - expect(where).toBe("(region = :f_0 AND segment = :f_1)"); + expect(where).toBe("(`region` = :f_0 AND `segment` = :f_1)"); expect(parameters.f_0.value).toBe("EMEA"); expect(parameters.f_1.value).toBe("Enterprise"); }); @@ -1030,7 +1155,7 @@ describe("metric — filter translator", () => { { member: "region", operator: "equals", values: ["APAC"] }, ], }); - expect(where).toBe("(region = :f_0 OR region = :f_1)"); + expect(where).toBe("(`region` = :f_0 OR `region` = :f_1)"); }); test("AND-of-OR composes nested groups", () => { @@ -1045,7 +1170,7 @@ describe("metric — filter translator", () => { }, ], }); - expect(where).toContain("(region IN ("); + expect(where).toContain("(`region` IN ("); expect(where).toContain(" AND "); expect(where).toContain("OR"); }); @@ -1219,14 +1344,25 @@ describe("metric — filter translator", () => { expect(Object.keys(parameters)).toHaveLength(3); }); - test("a filter member failing DIMENSION_NAME_PATTERN throws before SQL is built", () => { + test("an injection-shaped filter member is neutralized by quoting", () => { + // No control char → a valid (if nonexistent) column name: quoted whole, + // the `;`/`DROP`/`--` are inert inside the backtick-delimited identifier. + const { where } = render({ + member: "region; DROP TABLE foo --", + operator: "equals", + values: ["x"], + }); + expect(where).toBe("`region; DROP TABLE foo --` = :f_0"); + }); + + test("a filter member with a control character throws before SQL is built", () => { expect(() => render({ - member: "region; DROP TABLE foo --", + member: "region\ndrop", operator: "equals", values: ["x"], }), - ).toThrowError(/not a valid identifier/); + ).toThrowError(/not a valid identifier|control character/); }); }); @@ -1383,7 +1519,7 @@ describe("metric — filter translator", () => { operator: "equals", values: ["x"], }); - expect(where).toBe("ghost_column = :f_0"); + expect(where).toBe("`ghost_column` = :f_0"); }); }); @@ -1407,7 +1543,7 @@ describe("metric — filter translator", () => { }); test("rejects a name that is BOTH a measure and a dimension", () => { - // The corruption case: `SELECT MEASURE(x) AS x, x ... GROUP BY ALL` + // The corruption case: `SELECT MEASURE(`x`) AS `x`, x ... GROUP BY ALL` // materializes two `x` columns and the second overwrites the first. expect(() => validateMetricRequest({ @@ -1613,7 +1749,7 @@ describe("metric — filter translator", () => { expect(executeMock.mock.calls[0][1]).toEqual( expect.objectContaining({ statement: - "SELECT MEASURE(ghost_measure) AS ghost_measure FROM `cat`.`sch`.`revenue_metrics`", + "SELECT MEASURE(`ghost_measure`) AS `ghost_measure` FROM `cat`.`sch`.`revenue_metrics`", }), ); diff --git a/packages/shared/src/schemas/metric-fqn.ts b/packages/shared/src/schemas/metric-fqn.ts index 98fca3d57..be96c1625 100644 --- a/packages/shared/src/schemas/metric-fqn.ts +++ b/packages/shared/src/schemas/metric-fqn.ts @@ -140,21 +140,54 @@ export function isValidFqn(fqn: string): boolean { * @throws If any segment contains a control character or newline. */ export function quoteFqnForSql(fqn: string): string { - // Reject anything that cannot be represented inside a backtick-quoted - // identifier. \p{Cc} is the Unicode "control" category, which covers C0 - // (incl. \n, \r, \t), DEL, and C1 — i.e. every control character/newline. - const CONTROL_OR_NEWLINE = /\p{Cc}/u; - return fqn - .split(".") - .map((segment) => { - if (CONTROL_OR_NEWLINE.test(segment)) { - throw new Error( - `Cannot quote FQN segment "${segment}" for SQL: it contains a control character or newline, which has no valid escape inside a backtick-quoted identifier.`, - ); - } - // Double every backtick — the only break-out from a backtick-quoted - // identifier — then wrap the whole segment in backticks. - return `\`${segment.replace(/`/g, "``")}\``; - }) - .join("."); + return fqn.split(".").map(quoteIdentifier).join("."); +} + +/** + * The Unicode "control" category (`\p{Cc}`): C0 (incl. `\n`, `\r`, `\t`), DEL, + * and C1 — every control character/newline. These have no valid escape inside + * a backtick-quoted identifier, so a name containing one cannot be safely + * quoted and is rejected. + */ +const CONTROL_OR_NEWLINE = /\p{Cc}/u; + +/** + * Is `name` a column/measure/dimension identifier that {@link quoteIdentifier} + * can safely escape? True for any non-empty string free of control characters + * and newlines. + * + * This is the **column-identifier** grammar — deliberately broader than the + * FQN-segment grammar ({@link UC_FQN_PATTERN}, which also forbids `.` and `/` + * because they are FQN structural characters). A metric view's measure or + * dimension is a single *delimited* column identifier: once backtick-quoted it + * may legally contain dots, slashes, spaces, hyphens, and non-ASCII — anything + * but a control character. The type-generator emits DESCRIBE column names + * verbatim into the generated `measureKeys`/`dimensionKeys` unions, so the + * runtime must accept exactly what can be safely quoted, or a generated name + * would typecheck but fail at runtime. + */ +export function isValidColumnName(name: string): boolean { + return name.length > 0 && !CONTROL_OR_NEWLINE.test(name); +} + +/** + * Quote a SINGLE identifier (one column/measure/dimension name, or one FQN + * segment) as a backtick-delimited identifier for safe SQL interpolation. + * + * Unlike {@link quoteFqnForSql}, this does NOT split on `.` — the whole input + * is one identifier, so a column literally named `net.revenue` becomes + * `` `net.revenue` `` (one identifier), not `` `net`.`revenue` `` (two). The + * backtick — the only break-out character — is doubled; control characters and + * newlines have no valid escape and are rejected. + * + * @throws If `name` contains a control character or newline. + */ +export function quoteIdentifier(name: string): string { + if (CONTROL_OR_NEWLINE.test(name)) { + throw new Error( + `Cannot quote identifier "${name}" for SQL: it contains a control character or newline, which has no valid escape inside a backtick-quoted identifier.`, + ); + } + // Double every backtick, then wrap in backticks. + return `\`${name.replace(/`/g, "``")}\``; } diff --git a/packages/shared/src/schemas/metric-source.ts b/packages/shared/src/schemas/metric-source.ts index 2e4e1dfd4..4d64458a7 100644 --- a/packages/shared/src/schemas/metric-source.ts +++ b/packages/shared/src/schemas/metric-source.ts @@ -18,7 +18,25 @@ */ import { z } from "zod"; -import { UC_FQN_PATTERN } from "./metric-fqn"; +import { MAX_UC_OBJECT_NAME_LENGTH, UC_FQN_PATTERN } from "./metric-fqn"; + +/** + * Safety cap on the number of declared metric views — a typo / DoS guard, NOT + * a Unity Catalog limit. Mirrors the type-generator's `MAX_METRIC_VIEWS` so + * runtime config validation and type generation accept exactly the same + * configs (a config that fails generation must not silently pass at runtime). + */ +const MAX_METRIC_VIEWS = 200; + +/** + * Whole-FQN length cap: three max-length UC object names plus the two dots. + * The per-segment cap ({@link MAX_UC_OBJECT_NAME_LENGTH}) is enforced in the + * `superRefine` below; this whole-string bound is the declarative half (it + * serializes to a JSON-schema `maxLength`, whereas the per-segment check — like + * the entry-count cap — cannot be expressed declaratively and lives in the + * refinement, so it is a runtime/type-generator gate only). + */ +const MAX_FQN_LENGTH = MAX_UC_OBJECT_NAME_LENGTH * 3 + 2; /** * Three-part Unity Catalog FQN matcher, composed from the single-segment @@ -62,6 +80,7 @@ export const metricEntrySchema = z source: z .string() .regex(UC_THREE_PART_FQN_PATTERN) + .max(MAX_FQN_LENGTH) .describe( "Three-part Unity Catalog FQN of the metric view: ..", ) @@ -94,7 +113,39 @@ export const metricSourceSchema = z .strict() .describe( "Schema for AppKit metric-views.json — declares Unity Catalog Metric View sources for the analytics plugin's metric-view path. Each entry under 'metricViews' binds a metric key to a UC metric view FQN and an executor ('app_service_principal' shared cache, or 'user' per-user cache). Object form (rather than bare string) at v1 enables future per-entry option growth without breaking changes.", - ); + ) + // Caps that cannot be expressed declaratively (zod 4's `z.record` has no + // `.max`, and a per-dot-segment length bound isn't a whole-string + // `maxLength`). Enforced here so runtime config validation matches the + // type-generator's `resolveMetricConfig` exactly — a config that fails type + // generation must not silently pass at runtime. These refinements are + // invisible to `z.toJSONSchema`, so the generated JSON schema carries only + // the declarative `maxLength` on `source`; runtime + type-generator remain + // the authoritative gates for the entry-count and per-segment caps. + .superRefine((value, ctx) => { + const entries = value.metricViews ? Object.entries(value.metricViews) : []; + + if (entries.length > MAX_METRIC_VIEWS) { + ctx.addIssue({ + code: "custom", + message: `too many metric views: ${entries.length} declared, exceeding the maximum of ${MAX_METRIC_VIEWS}`, + path: ["metricViews"], + }); + } + + for (const [key, entry] of entries) { + const segments = entry.source.split("."); + for (let i = 0; i < segments.length; i++) { + if (segments[i].length > MAX_UC_OBJECT_NAME_LENGTH) { + ctx.addIssue({ + code: "custom", + message: `metric source segment ${i + 1} is ${segments[i].length} characters, exceeding the per-segment maximum of ${MAX_UC_OBJECT_NAME_LENGTH}`, + path: ["metricViews", key, "source"], + }); + } + } + } + }); export type MetricKey = z.infer; export type MetricExecutor = z.infer; From 9a9d92f80f66d1091e787cfc5c529b9e3a40c073 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 10 Jul 2026 14:36:47 +0200 Subject: [PATCH 09/18] refactor(appkit): split metric runtime modules Signed-off-by: Atila Fassina --- .../shared/appkit-types/metric-views.d.ts | 19 + .../appkit/src/plugins/analytics/metric.ts | 1393 +---------------- .../appkit/src/plugins/analytics/mv/cache.ts | 71 + .../src/plugins/analytics/mv/constants.ts | 122 ++ .../src/plugins/analytics/mv/formatters.ts | 319 ++++ .../appkit/src/plugins/analytics/mv/index.ts | 9 + .../src/plugins/analytics/mv/registry.ts | 192 +++ .../src/plugins/analytics/mv/schemas.ts | 356 +++++ .../appkit/src/plugins/analytics/mv/types.ts | 25 + 9 files changed, 1114 insertions(+), 1392 deletions(-) create mode 100644 packages/appkit/src/plugins/analytics/mv/cache.ts create mode 100644 packages/appkit/src/plugins/analytics/mv/constants.ts create mode 100644 packages/appkit/src/plugins/analytics/mv/formatters.ts create mode 100644 packages/appkit/src/plugins/analytics/mv/index.ts create mode 100644 packages/appkit/src/plugins/analytics/mv/registry.ts create mode 100644 packages/appkit/src/plugins/analytics/mv/schemas.ts create mode 100644 packages/appkit/src/plugins/analytics/mv/types.ts diff --git a/apps/dev-playground/shared/appkit-types/metric-views.d.ts b/apps/dev-playground/shared/appkit-types/metric-views.d.ts index e0ada3749..1c7fb87a9 100644 --- a/apps/dev-playground/shared/appkit-types/metric-views.d.ts +++ b/apps/dev-playground/shared/appkit-types/metric-views.d.ts @@ -30,23 +30,31 @@ declare module "@databricks/appkit-ui/react" { measures: { "active_accounts": { type: "bigint"; + display_name: "Active Accounts"; + format: "#,##0"; }; "churn_rate": { type: "decimal"; + display_name: "Churn Rate"; }; "avg_ltv": { type: "double"; + display_name: "Average LTV"; + format: "$#,##0.00"; }; }; dimensions: { "segment": { type: "string"; + display_name: "Customer Segment"; }; "region": { type: "string"; + display_name: "Region"; }; "csm_email": { type: "string"; + display_name: "CSM Email"; }; }; }; @@ -80,27 +88,38 @@ declare module "@databricks/appkit-ui/react" { measures: { "mrr": { type: "double"; + display_name: "Monthly Recurring Revenue"; + format: "$#,##0.00"; }; "arr": { type: "double"; + display_name: "Annual Recurring Revenue"; + format: "$#,##0.00"; description: "Annualized contract value across all active subscriptions"; }; "new_arr": { type: "double"; + display_name: "New ARR"; + format: "$#,##0.00"; }; "churned_arr": { type: "double"; + display_name: "Churned ARR"; + format: "$#,##0.00"; }; }; dimensions: { "region": { type: "string"; + display_name: "Region"; }; "segment": { type: "string"; + display_name: "Customer Segment"; }; "created_at": { type: "timestamp_ltz"; + display_name: "Subscription Start"; time_grain: readonly ["day", "hour", "minute", "month", "quarter", "week", "year"]; }; }; diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts index aabfdc89d..3e8a6f47b 100644 --- a/packages/appkit/src/plugins/analytics/metric.ts +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -1,1392 +1 @@ -import { createHash } from "node:crypto"; -import fs from "node:fs/promises"; -import path from "node:path"; -import { type SQLTypeMarker, sql as sqlHelpers } from "shared"; -import { z } from "zod"; -// Canonical metric-source schema — the single source of truth for -// `metric-views.json`. Imported from the shared source directly (matching the -// type-generator's runtime, which pulls the zod-free `metric-fqn.ts` from the -// same tree) so the runtime and the generated JSON schema validate identically. -import { - isValidColumnName, - isValidFqn, - quoteFqnForSql, - quoteIdentifier, -} from "../../../../shared/src/schemas/metric-fqn"; -import { metricSourceSchema } from "../../../../shared/src/schemas/metric-source"; -import { AuthenticationError, ValidationError } from "../../errors"; -import { createLogger } from "../../logging/logger"; -import type { - IAnalyticsMetricRequest, - MetricFilter, - MetricFilterOperatorName, - MetricLane, - MetricPredicate, - MetricRegistration, -} from "./types"; -import { normalizeAnalyticsFormat } from "./types"; - -const logger = createLogger("analytics:metric"); - -/** - * Default queries directory. Mirrors `AppManager`'s - * `path.resolve(process.cwd(), "config/queries")` so dev mode and production - * share a single source of truth for where metric config lives. Exported so - * `AnalyticsPlugin` can default `config.queriesDir` to the same path. - */ -export const QUERIES_DIR = path.resolve(process.cwd(), "config/queries"); -const METRIC_CONFIG_FILE = "metric-views.json"; - -/** - * Measure, dimension, and filter-member names are **column identifiers**: they - * are validated by the shared {@link isValidColumnName} (rejects only control - * characters / newlines) and backtick-quoted via {@link quoteIdentifier} at - * every interpolation point. Quoting — not a narrow ASCII allowlist — is the - * injection boundary, so the runtime accepts the full delimited-identifier - * grammar the type-generator emits from DESCRIBE (hyphens, dots, non-ASCII). - * There is deliberately NO name allowlist: a well-formed-but-unknown column - * falls through to the warehouse and surfaces as a sanitized canonical error. - * - * Time-grain token shape. Unlike the column identifiers above, the grain is - * interpolated as a single-quoted `date_trunc` unit LITERAL (NOT a bind param, - * NOT a delimited identifier) in {@link renderDimensionClause}, so it keeps a - * narrow keyword-shaped gate — that pattern is what keeps a hostile token out - * of the quoted-literal position. - */ -const TIME_GRAIN_PATTERN = /^[a-z][a-z_]*$/; - -/** - * The exact twelve filter operators allowed at v1. The runtime tuple is the - * server-side source of truth; the client-side type union - * `MetricFilterOperatorName` mirrors these names statically. - */ -const METRIC_FILTER_OPERATORS = [ - "equals", - "notEquals", - "in", - "notIn", - "gt", - "gte", - "lt", - "lte", - "contains", - "notContains", - "set", - "notSet", -] as const satisfies readonly MetricFilterOperatorName[]; - -/** - * Maximum AND/OR nesting depth. The PRD documents 8 as a sensible cap — - * enough for any real BI filter UI, low enough that a hostile or malformed - * payload cannot stack-overflow the recursive validator or translator. - * - * The depth count is the number of nested `{ and }` / `{ or }` wrappers - * encountered while descending — leaf predicates do not count toward depth. - */ -const METRIC_FILTER_MAX_DEPTH = 8; - -/** - * Cardinality caps on user-controlled arrays. Closes the recurring - * `unbounded-request-parameters` finding: a hostile caller could otherwise - * send `values: [...10M items...]` and exhaust the validator + the named - * bind-var binding step. The limits below are deliberately generous — higher - * than any real BI UI would emit — so legitimate traffic never trips them. - */ -const METRIC_MEASURES_MAX = 50; -const METRIC_DIMENSIONS_MAX = 20; -const METRIC_FILTER_VALUES_MAX = 1000; -const METRIC_LIMIT_MAX = 100_000; - -/** - * Maximum number of children per AND/OR group node. Without this cap a single - * flat group like `{ and: [...10M empty objects...] }` would push tens of - * millions of frames onto the iterative pre-check's stack — OOM before - * validation even gets to Zod. The Zod schema enforces the same cap so the - * rejection point is consistent regardless of which validator catches it - * first. - */ -const METRIC_FILTER_GROUP_MAX = 100; - -/** Operators that require exactly one value. */ -const SINGLE_VALUE_OPERATORS = new Set([ - "equals", - "notEquals", - "gt", - "gte", - "lt", - "lte", - "contains", - "notContains", -]); - -/** Operators that require at least one value. */ -const LIST_VALUE_OPERATORS = new Set(["in", "notIn"]); - -/** Operators that reject `values` entirely. */ -const NULL_OPERATORS = new Set(["set", "notSet"]); - -/** Operators that emit `LIKE` / `NOT LIKE` and require a string value. */ -const STRING_OPERATORS = new Set([ - "contains", - "notContains", -]); - -/** - * Map an entry's declared `executor` to the internal execution lane: - * - `"user"` → `"obo"` (per-user cache, on-behalf-of) - * - `"app_service_principal"` (default) → `"sp"` (shared cache) - */ -function laneFromExecutor( - executor: "app_service_principal" | "user", -): MetricLane { - return executor === "user" ? "obo" : "sp"; -} - -/** - * Read and validate `config/queries/metric-views.json` into a metric registry. - * - * Async and stateless — registration is a pure config parse with no warehouse - * round-trip, no `DESCRIBE`, and no build-time metadata bundle. The single - * `metricViews` map makes keys unique by construction, so there is no - * cross-lane duplicate-key check. Async I/O (rather than `readFileSync`) keeps - * the event loop free: metric views are already heavier than a plain `.sql` - * query on the warehouse side, so the SDK layer must not add a blocking read - * on top. Caching + mtime-revalidation is layered on top by - * {@link getMetricRegistry} — this function always hits disk. - * - * Returns an empty registry when the file is absent: the metric-view path is - * additive and dormant until an app opts in by adding the config. A malformed - * file (unreadable, invalid JSON, or schema violation) throws — the caller - * surfaces a 503 rather than masking a broken deployment as a 404 for every - * key. The failure is NOT cached (see {@link getMetricRegistry}), so fixing the - * file heals on the next request. - */ -export async function loadMetricRegistry( - queriesDir: string = QUERIES_DIR, -): Promise> { - const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); - - let raw: string; - try { - raw = await fs.readFile(metricPath, "utf8"); - } catch (err) { - if ((err as NodeJS.ErrnoException).code === "ENOENT") { - return Object.create(null); - } - throw err; - } - - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (err) { - throw new Error( - `Failed to parse metric-views.json at ${metricPath}: ${(err as Error).message}`, - ); - } - - const result = metricSourceSchema.safeParse(parsed); - if (!result.success) { - const issues = result.error.issues - .map((i) => `${i.path.join(".")}: ${i.message}`) - .join("; "); - throw new Error(`Invalid metric-views.json at ${metricPath}: ${issues}`); - } - - // Null-prototype map so a metric key that collides with an inherited - // `Object.prototype` member (`__proto__`, `constructor`, `toString`, …) - // cannot resolve to a truthy non-registration at the `registry[key]` read - // site and slip past the unknown-key 404. Keys are still grammar-gated by - // `metricKeySchema` (identifier shape), but the null prototype removes the - // whole class of inherited-property lookups as a boundary. - const registry: Record = Object.create(null); - for (const [key, entry] of Object.entries(result.data.metricViews ?? {})) { - registry[key] = { - key, - source: entry.source, - lane: laneFromExecutor(entry.executor), - }; - } - - logger.debug( - "Loaded metric registry: %d entry(ies)", - Object.keys(registry).length, - ); - return registry; -} - -/** - * Signature of the config file the cached registry was parsed from: its - * change time, modification time, and size — all from one `stat`, no read. - * - * `ctimeMs` (inode change time) is the key guard against a stale serve: it - * bumps on ANY metadata or content change and, unlike `mtimeMs`, cannot be - * restored to a prior value by tooling (`utimes` sets atime/mtime but not - * ctime). So an edit that preserves both size and mtime — a same-length value - * swap (e.g. repointing `source` to an equal-length FQN, or flipping - * `executor` between two equal-length values) on a coarse-mtime filesystem — - * still changes ctime and invalidates the cache. Without it, such an edit - * could serve a stale `source`/`lane`, reintroducing the exact stale-serve - * class the cache-key `source` salt was added to prevent, one layer up. - * A content hash would be stronger still but requires reading the file, which - * is the cost this cache exists to avoid. - */ -interface RegistryCacheSignature { - ctimeMs: number; - mtimeMs: number; - size: number; -} - -/** - * Module-level registry cache, keyed by the resolved queries directory. - * - * Keyed by DIR (not by plugin instance) because the registry is a pure - * function of the config file at that path — warehouse-independent, and two - * `AnalyticsPlugin` instances pointed at the same `config/queries/` MUST see - * the same registry. Instance state would parse the identical file twice and - * risk divergence; a dir-keyed module cache shares one parse. - */ -const metricRegistryCache = new Map< - string, - { - signature: RegistryCacheSignature; - registry: Record; - } ->(); - -/** - * Clear the module-level registry cache. - * - * @internal FOR TESTS ONLY. The cache is keyed by directory and lives for the - * process; production never needs to clear it (a changed file is picked up via - * the stat signature). Tests that reuse a directory — or assert cold-load - * behavior — call this in `beforeEach`/`afterEach` so isolation is intentional - * rather than relying on each test happening to use a unique temp dir. - */ -export function __resetMetricRegistryCache(): void { - metricRegistryCache.clear(); -} - -/** - * Resolve the metric registry for `queriesDir`, re-reading + re-parsing only - * when `metric-views.json` has changed since the cached copy. - * - * Behaves like the sibling `.sql` query path (which re-reads per request) but - * cheaper: the steady-state cost is a single async `stat`, and the read + - * `JSON.parse` + zod validation are skipped when the file's - * `(ctimeMs, mtimeMs, size)` signature is unchanged. This delivers the agreed - * semantics without the old permanent memo: - * - * - **Hot-reload** — editing a working config bumps `mtimeMs`, so the next - * request re-parses and serves the new registry with no server restart. - * - **Self-heal** — a load failure is NOT cached (we only populate the cache - * on a successful parse), so a fixed config is picked up on the next - * request instead of latching a 503 forever. - * - **Dormant** — an absent file `stat`s as `ENOENT` → empty registry; adding - * the file later is picked up on the next request. - * - * Concurrency: two simultaneous cold requests may both `stat`+read before - * either populates the cache. That is a harmless redundant read of the same - * file (the sibling `.sql` path does not single-flight either), so no lock is - * taken. - * - * @throws Propagates {@link loadMetricRegistry}'s throw on a malformed file so - * the route can surface a 503; the cache is left untouched on failure. - */ -export async function getMetricRegistry( - queriesDir: string = QUERIES_DIR, -): Promise> { - const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); - - let signature: RegistryCacheSignature | null; - try { - const stats = await fs.stat(metricPath); - signature = { - ctimeMs: stats.ctimeMs, - mtimeMs: stats.mtimeMs, - size: stats.size, - }; - } catch (err) { - if ((err as NodeJS.ErrnoException).code === "ENOENT") { - // Absent file → dormant. Drop any stale cache entry (the file may have - // been deleted) and return an empty registry without touching the cache. - metricRegistryCache.delete(queriesDir); - signature = null; - } else { - // Any other stat error (EACCES / EIO / ELOOP / …) is deliberately fatal - // for THIS request → the route surfaces a 503, consistent with the - // malformed-config → 503 path. It is not latched: with the self-heal - // design a transient error clears on the next request, which is strictly - // better than the old memo that could latch-and-serve-stale. - throw err; - } - } - - if (signature === null) { - return Object.create(null); - } - - const cached = metricRegistryCache.get(queriesDir); - if ( - cached !== undefined && - cached.signature.ctimeMs === signature.ctimeMs && - cached.signature.mtimeMs === signature.mtimeMs && - cached.signature.size === signature.size - ) { - return cached.registry; - } - - // Cold or stale: re-read + re-parse. Cache ONLY on success so a malformed - // file never latches — the next request re-attempts and heals. - const registry = await loadMetricRegistry(queriesDir); - metricRegistryCache.set(queriesDir, { signature, registry }); - return registry; -} - -/** - * Validate a metric-view FQN and return it backtick-quoted for interpolation. - * - * The FQN ships in the SQL string — it cannot be parameterized — so it passes - * two shared, zod-free gates at construction time: - * - * 1. {@link isValidFqn} — the three-part UC grammar (exactly three segments, - * each matching `UC_FQN_PATTERN`). This is the SAME predicate the - * type-generator's describe seam uses and is derived from the same - * per-segment charset as the canonical Zod schema, so config-time, - * generation-time, and runtime accept exactly the same names. - * 2. {@link quoteFqnForSql} — backtick-quotes each segment (doubling embedded - * backticks) so the FQN cannot break out of its identifier position. This - * is the injection boundary; it is why the runtime can accept the full UC - * quoted-identifier grammar (hyphens, non-ASCII) rather than the narrow - * ASCII allowlist it used before. - * - * The registry loader already enforces the grammar via `metricSourceSchema`; - * this re-gate is defense-in-depth for any code path that reaches - * `buildMetricSql` without going through a parsed registry (it is exported), - * matching how measures, dimensions, and the grain are each re-gated at their - * interpolation points. - */ -function quoteSafeFqn(fqn: string): string { - if (!isValidFqn(fqn)) { - throw new Error( - `Refusing to build SQL: "${fqn}" is not a valid three-part UC FQN.`, - ); - } - return quoteFqnForSql(fqn); -} - -/** - * Structural validation schema for the metric request body. - * - * The schema is **static** — any grammar-valid measure/dimension identifier is - * accepted; it is NOT a dynamic per-key `z.enum(knownMeasures)`. Unknown-but- - * well-formed names are not rejected here; they reach the warehouse and surface - * as a sanitized canonical error. The identifier grammar gate is enforced in - * {@link buildMetricSql}/{@link renderFilter} at interpolation time; the body - * schema stays structural (item shape, cardinality caps, operator enum, - * per-operator value cardinality, and AND/OR depth). - * - * `filter` is recursive (`Predicate | { and: [...] } | { or: [...] }`), built - * with `z.lazy`. `timeGrain` buckets `timeDimension` via `date_trunc` (see - * {@link renderDimensionClause}); the two cross-field rules (grain requires - * timeDimension; timeDimension must be one of dimensions) live in the - * `superRefine` below. - */ - -/** A leaf predicate: `{ member, operator, values? }`, no extra keys. */ -const filterPredicateSchema: z.ZodType = z - .object({ - member: z - .string() - .min(1, { message: "filter predicate 'member' cannot be empty" }) - .refine(isValidColumnName, { - message: - "filter predicate 'member' contains a character that cannot be used in a SQL identifier (control character or newline)", - }), - operator: z.string().min(1, { - message: "filter predicate 'operator' cannot be empty", - }) as z.ZodType, - values: z - .array(z.union([z.string(), z.number()])) - .max(METRIC_FILTER_VALUES_MAX, { - message: `filter predicate 'values' length exceeds the maximum of ${METRIC_FILTER_VALUES_MAX}`, - }) - .optional(), - }) - .strict(); - -/** Recursive filter: a predicate leaf or an `{ and }` / `{ or }` group. */ -const filterSchema: z.ZodType = z.lazy(() => - z.union([ - filterPredicateSchema, - z - .object({ - and: z.array(filterSchema).max(METRIC_FILTER_GROUP_MAX, { - message: `filter 'and' group exceeds the maximum of ${METRIC_FILTER_GROUP_MAX} children`, - }), - }) - .strict(), - z - .object({ - or: z.array(filterSchema).max(METRIC_FILTER_GROUP_MAX, { - message: `filter 'or' group exceeds the maximum of ${METRIC_FILTER_GROUP_MAX} children`, - }), - }) - .strict(), - ]), -); - -const metricRequestSchema = z - .object({ - measures: z - .array( - z - .string() - .min(1, "measure name cannot be empty") - .refine(isValidColumnName, { - message: - "measure name contains a character that cannot be used in a SQL identifier (control character or newline)", - }), - ) - .min(1, "at least one measure is required") - .max(METRIC_MEASURES_MAX, { - message: `measures length exceeds the maximum of ${METRIC_MEASURES_MAX}`, - }), - dimensions: z - .array( - z - .string() - .min(1, "dimension name cannot be empty") - .refine(isValidColumnName, { - message: - "dimension name contains a character that cannot be used in a SQL identifier (control character or newline)", - }), - ) - .max(METRIC_DIMENSIONS_MAX, { - message: `dimensions length exceeds the maximum of ${METRIC_DIMENSIONS_MAX}`, - }) - .optional(), - filter: filterSchema.optional(), - // Grammar-shaped bucketing grain, applied to `timeDimension` via - // `date_trunc`. The token is validated for safety here; the grain literal - // is interpolated (single-quoted, never a bind param) in - // `renderDimensionClause`, so this pattern gate is the security boundary. - timeGrain: z - .string() - .min(1, { message: "timeGrain cannot be empty" }) - .regex(TIME_GRAIN_PATTERN, { - message: "timeGrain must match /^[a-z][a-z_]*$/", - }) - .optional(), - // The single dimension `timeGrain` applies to via `date_trunc`. A column - // identifier (backtick-quoted at interpolation), so it accepts the full - // delimited-identifier grammar. Cross-field rules in `superRefine`: - // required when `timeGrain` is set, and must be one of `dimensions`. - timeDimension: z - .string() - .min(1, { message: "timeDimension cannot be empty" }) - .refine(isValidColumnName, { - message: - "timeDimension contains a character that cannot be used in a SQL identifier (control character or newline)", - }) - .optional(), - limit: z - .number() - .int({ message: "limit must be an integer" }) - .positive({ message: "limit must be positive" }) - .max(METRIC_LIMIT_MAX, { - message: `limit exceeds the maximum of ${METRIC_LIMIT_MAX}`, - }) - .optional(), - format: z.enum(["JSON_ARRAY", "ARROW_STREAM", "JSON", "ARROW"]).optional(), - }) - .strict() - .superRefine((value, ctx) => { - if (value.filter != null) { - validateFilterTree(value.filter, ctx, ["filter"], 0); - } - - // Uniqueness. Measures and dimensions become SELECT columns aliased to - // their own name (`MEASURE(x) AS x`, `x`); the warehouse returns one row - // object keyed by column name, so a repeated name — a duplicate measure, a - // duplicate dimension, or a name appearing as BOTH a measure and a - // dimension — collapses to a single key and silently drops a value during - // row materialization. Reject the collision here rather than emit SQL that - // corrupts rows. (The grammar gate at build time is a safety boundary, not - // a uniqueness check, so this lives in validation.) - const seen = new Set(); - const collided = new Set(); - for (const name of [...value.measures, ...(value.dimensions ?? [])]) { - if (seen.has(name)) { - collided.add(name); - } - seen.add(name); - } - if (collided.size > 0) { - ctx.addIssue({ - code: "custom", - message: - "measures and dimensions must be unique across both lists (a name cannot repeat, nor appear as both a measure and a dimension)", - path: ["measures"], - }); - } - - // Delivery format. The metric route delivers JSON rows only at v1 (the - // cache salts on a fixed "JSON_ARRAY" and the route always routes through - // the JSON path). Accepting an Arrow format would silently return JSON — - // reject it loudly until Arrow parity ships. Legacy aliases normalize - // first (`JSON`→`JSON_ARRAY`, `ARROW`→`ARROW_STREAM`). - if ( - value.format != null && - normalizeAnalyticsFormat(value.format) !== "JSON_ARRAY" - ) { - ctx.addIssue({ - code: "custom", - message: - "format: only JSON_ARRAY is supported on the metric route at v1 (ARROW_STREAM is not yet implemented)", - path: ["format"], - }); - } - - // Cross-field grain rules. `timeGrain` buckets exactly one selected - // dimension via `date_trunc`, so it requires an explicit `timeDimension`, - // and that dimension must be selected (so it is in the SELECT list and - // `GROUP BY ALL`). - if (value.timeGrain != null && value.timeDimension == null) { - ctx.addIssue({ - code: "custom", - message: "timeDimension is required when timeGrain is set", - path: ["timeDimension"], - }); - } - if ( - value.timeDimension != null && - !(value.dimensions ?? []).includes(value.timeDimension) - ) { - ctx.addIssue({ - code: "custom", - message: "timeDimension must be one of dimensions", - path: ["timeDimension"], - }); - } - }) as z.ZodType; - -/** - * Recursive Zod-time validator for the filter tree. - * - * Pushes structured issues into the refinement context with stable paths - * (`filter.and.0.or.2.member`, etc.) so the canonical 400 error carries - * actionable diagnostics. Keeps the registry-free concerns in one descent: - * - * 1. Operator is one of the twelve. - * 2. Per-operator value cardinality (single / list / null operators). - * 3. `contains`/`notContains` carry a string value. - * 4. AND/OR nesting depth cap. - * - * There is NO member-allowlist and NO op⇄dimension-type check — the member is - * grammar-gated at SQL-build time, and dimension types are not tracked (no - * registry metadata). Returns void; issues accumulate on `ctx`. - */ -function validateFilterTree( - node: MetricFilter, - ctx: z.RefinementCtx, - path: Array, - depth: number, -): void { - if (node === null || typeof node !== "object") { - // The base schema rejects this via the union, but stay defensive. - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path, - message: "filter node must be a Predicate or { and } / { or } group", - }); - return; - } - - if ("and" in node || "or" in node) { - if (depth + 1 > METRIC_FILTER_MAX_DEPTH) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path, - message: `filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}`, - }); - return; - } - - const groupKey = "and" in node ? "and" : "or"; - const children = ( - node as { and?: ReadonlyArray } & { - or?: ReadonlyArray; - } - )[groupKey]; - - if (!Array.isArray(children)) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: [...path, groupKey], - message: `filter ${groupKey} group must be an array of predicates or nested groups`, - }); - return; - } - - // Reject empty `or` groups: an empty disjunction is vacuously false, which - // silently drops the surrounding intent. Empty `and` is OK (vacuously true - // → no constraint). Forcing the caller to omit the predicate entirely is - // the only unambiguous choice. - if (groupKey === "or" && children.length === 0) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: [...path, "or"], - message: "filter 'or' group must contain at least one predicate", - }); - return; - } - - children.forEach((child, idx) => { - validateFilterTree(child, ctx, [...path, groupKey, idx], depth + 1); - }); - return; - } - - // Leaf predicate. The base schema already enforced shape; here we layer in - // the operator vocabulary + per-operator value cardinality. - const predicate = node as MetricPredicate; - - if ( - !METRIC_FILTER_OPERATORS.includes( - predicate.operator as MetricFilterOperatorName, - ) - ) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: [...path, "operator"], - message: `filter operator "${predicate.operator}" is not one of: ${METRIC_FILTER_OPERATORS.join(", ")}`, - }); - // No further checks meaningful when the operator is unknown. - return; - } - - const op = predicate.operator; - const values = predicate.values; - const valuesLen = values?.length ?? 0; - - if (NULL_OPERATORS.has(op)) { - if (values != null && valuesLen > 0) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: [...path, "values"], - message: `filter operator "${op}" must not carry values`, - }); - } - } else if (SINGLE_VALUE_OPERATORS.has(op)) { - if (valuesLen !== 1) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: [...path, "values"], - message: `filter operator "${op}" requires exactly one value (got ${valuesLen})`, - }); - } - } else if (LIST_VALUE_OPERATORS.has(op)) { - if (valuesLen < 1) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: [...path, "values"], - message: `filter operator "${op}" requires at least one value`, - }); - } - } - - // Op⇄value-type: string operators require a string value. This is a - // registry-free structural check (not an op⇄dimension-type check) — it - // converts what would otherwise be a synchronous throw in `renderPredicate` - // (rendered as a 500) into a clean 400 at validation time. - if (STRING_OPERATORS.has(op) && valuesLen > 0) { - const v = predicate.values?.[0]; - if (typeof v !== "string") { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: [...path, "values"], - message: `filter operator "${op}" requires a string value (got ${typeof v})`, - }); - } - } -} - -/** - * Iterative pre-parse depth check. Zod's union/object parsers walk the input - * recursively before our `superRefine` depth cap fires, so a deeply-nested - * `{ and: [{ and: [...] }] }` payload could stack-overflow during parse, never - * reaching the validator's depth check. This walk is iterative (explicit - * stack) and aborts as soon as `METRIC_FILTER_MAX_DEPTH` is exceeded, so a - * hostile payload of any size cannot drive the call stack. - * - * Walks BOTH `and` and `or` branches when both are present on the same node — - * Zod's `.strict()` rejects the multi-key shape downstream, but the pre-check - * has to inspect every branch Zod might recurse into (an earlier version used - * `else if` and was bypassed by `{ and: [], or: }`). - * - * Group-children breadth is also capped: a flat `{ and: [...10M items...] }` - * cannot push 10M frames onto the explicit stack here. Predicate leaves do NOT - * count toward depth — only nested `and` / `or` wrappers — matching the rule - * {@link validateFilterTree} enforces. - */ -function preCheckFilterDepth(filter: unknown): void { - if (filter == null || typeof filter !== "object") return; - const stack: Array<[unknown, number]> = [[filter, 0]]; - while (stack.length > 0) { - const popped = stack.pop(); - if (popped === undefined) continue; - const [node, depth] = popped; - if (node == null || typeof node !== "object") continue; - const obj = node as Record; - for (const groupKey of ["and", "or"] as const) { - const children = obj[groupKey]; - if (!Array.isArray(children)) continue; - if (children.length > METRIC_FILTER_GROUP_MAX) { - throw new ValidationError( - "Invalid metric request body (fields: filter)", - { - context: { - reason: `filter ${groupKey} group has ${children.length} children; the maximum is ${METRIC_FILTER_GROUP_MAX}`, - }, - }, - ); - } - if (depth + 1 > METRIC_FILTER_MAX_DEPTH) { - throw new ValidationError( - "Invalid metric request body (fields: filter)", - { - context: { - reason: `filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}`, - }, - }, - ); - } - for (const child of children) { - stack.push([child, depth + 1]); - } - } - } -} - -/** - * Validate a `POST /api/analytics/metric/:key` request body against the static - * structured shape. Throws {@link ValidationError} (a 400 on the canonical - * error path) with the offending field paths; the raw values stay in telemetry - * context, never the public body. - * - * The recursion depth is bounded BEFORE Zod sees the input — the schema's own - * depth check fires inside `superRefine`, which only runs after Zod's recursive - * parse has already walked the tree on the call stack. - */ -export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { - if (body != null && typeof body === "object") { - preCheckFilterDepth((body as { filter?: unknown }).filter); - } - const result = metricRequestSchema.safeParse(body); - if (!result.success) { - const fieldPaths = result.error.issues - .map((i) => i.path.join(".") || "(root)") - .join(", "); - throw new ValidationError( - fieldPaths.length > 0 - ? `Invalid metric request body (fields: ${fieldPaths})` - : "Invalid metric request body", - { context: { issues: result.error.issues } }, - ); - } - return result.data; -} - -/** - * Construct the metric SQL. - * - * Shape: - * - * SELECT MEASURE(m) AS m[, …][, …] - * FROM - * [WHERE ] - * [GROUP BY ALL] - * [LIMIT n] - * - * Notes: - * - Every measure and dimension is validated by {@link isValidColumnName} and - * backtick-quoted by {@link quoteIdentifier} before it is interpolated - * (column references cannot be parameterized — they are SQL identifiers), - * and the FQN is validated and quoted by {@link quoteSafeFqn}. There is - * deliberately NO name allowlist — quoting is the security boundary. No - * user-supplied string reaches the SQL - * string without passing a grammar gate. - * - `GROUP BY ALL` is added when at least one dimension is requested. UC - * requires GROUP BY when MEASURE() is mixed with non-aggregated columns; - * `GROUP BY ALL` is the documented form that works without re-listing each - * dimension. - * - The `WHERE` clause is rendered from the recursive filter tree. Every value - * flows through Statement Execution's named bind-var path (`:f_`); no - * value is ever interpolated as a literal. - * - When `timeGrain` is set, the `timeDimension` column renders as - * `date_trunc('', ) AS ` (the grain is a grammar-gated - * single-quoted literal, not a bind param); other dimensions render bare — - * see {@link renderDimensionClause}. - * - * Returns `{ statement, parameters }` where `parameters` is the named bind-var - * dictionary the plugin's `query()` consumes. - */ -export function buildMetricSql( - registration: MetricRegistration, - request: IAnalyticsMetricRequest, -): { - statement: string; - parameters: Record; -} { - const quotedSource = quoteSafeFqn(registration.source); - - if (request.measures.length === 0) { - throw new Error("buildMetricSql requires at least one measure."); - } - - // Defense-in-depth re-gate: `validateMetricRequest` already rejects any name - // `isValidColumnName` refuses, but `buildMetricSql` is exported and may be - // reached without it. `quoteIdentifier` throws on a control/newline name, so - // the quoting below is itself the boundary; the explicit check keeps the - // error message uniform with the other interpolated tokens. - for (const m of request.measures) { - if (!isValidColumnName(m)) { - throw new Error( - `Refusing to build SQL: measure "${m}" is not a valid identifier.`, - ); - } - } - - const dimensions = request.dimensions ?? []; - for (const d of dimensions) { - if (!isValidColumnName(d)) { - throw new Error( - `Refusing to build SQL: dimension "${d}" is not a valid identifier.`, - ); - } - } - - // Deterministic order so cache keys collapse semantically equivalent calls. - // Alias each measure to its plain name (backtick-quoted) so result rows have - // keys matching the registered measure (`{ "net-revenue": 1234 }`) rather - // than the SQL-function serialization Databricks returns by default. The - // warehouse reports the aliased column under the UNQUOTED name (backticks are - // delimiters, not part of the name), so `MEASURE(`x`) AS `x`` yields a row - // key of exactly `x` — preserved through result materialization. - const measureClauses = [...request.measures] - .sort() - .map((m) => `MEASURE(${quoteIdentifier(m)}) AS ${quoteIdentifier(m)}`); - - const dimensionClauses = [...dimensions] - .sort() - .map((d) => - renderDimensionClause(d, request.timeGrain, request.timeDimension), - ); - - const selectList = [...measureClauses, ...dimensionClauses].join(", "); - const groupByClause = dimensions.length > 0 ? " GROUP BY ALL" : ""; - - const limitClause = - typeof request.limit === "number" && request.limit > 0 - ? ` LIMIT ${Math.floor(request.limit)}` - : ""; - - // Filter translation. Every value is bound through `:f_` named params; - // every column identifier is grammar-gated in `renderFilter`. Empty filter or - // no filter → no WHERE. - const parameters: Record = {}; - let whereClause = ""; - if (request.filter !== undefined) { - const fragment = renderFilter(request.filter, parameters, { - counter: 0, - depth: 0, - }); - if (fragment !== null && fragment.length > 0) { - whereClause = ` WHERE ${fragment}`; - } - } - - const statement = `SELECT ${selectList} FROM ${quotedSource}${whereClause}${groupByClause}${limitClause}`; - return { statement, parameters }; -} - -/** - * Mutable counter / depth threaded through {@link renderFilter}. Fresh per - * `buildMetricSql` call, so two requests never share bind-var indexes. - */ -interface FilterRenderState { - counter: number; - depth: number; -} - -/** - * Recursively render a filter tree into a SQL fragment, pushing bind values - * into `params` keyed by `:f_` names. - * - * Returns `null` for an empty group (no WHERE clause needed). The caller's - * `buildMetricSql` only emits `WHERE` when this returns a non-null, non-empty - * fragment. Empty `and: []` collapses to null (SQL's vacuous-truth semantics - * for AND); empty `or: []` renders `1 = 0` — the validator rejects `or: []` - * before the SQL builder, but if it slips through we render vacuous-false - * rather than dropping the predicate silently (defense in depth so a future - * validator bypass cannot turn `or: []` into "match everything"). - * - * Defense-in-depth: even though the request body's filter has already been - * validated, every member name is re-checked against {@link isValidColumnName} - * here and backtick-quoted. If validation is ever bypassed, the SQL - * constructor still refuses to interpolate a name it cannot safely quote. - */ -function renderFilter( - node: MetricFilter, - params: Record, - state: FilterRenderState, -): string | null { - if (node === null || typeof node !== "object") { - throw new Error( - "Refusing to build SQL: filter node must be an object Predicate or { and } / { or } group.", - ); - } - - if ("and" in node || "or" in node) { - const groupKey = "and" in node ? "and" : "or"; - if (state.depth + 1 > METRIC_FILTER_MAX_DEPTH) { - throw new Error( - `Refusing to build SQL: filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}.`, - ); - } - - const children = ( - node as { and?: ReadonlyArray } & { - or?: ReadonlyArray; - } - )[groupKey]; - - if (!Array.isArray(children) || children.length === 0) { - if (groupKey === "or") { - return "1 = 0"; - } - return null; - } - - // Sort-before-hash discipline: within a group, predicate leaves are - // stable-sorted by (member, operator) before contributing to the rendered - // fragment, so semantically equivalent calls produce the same SQL string - // and (downstream) the same cache key. - const sortedChildren = sortFilterChildren(children); - - const fragments: string[] = []; - const childState: FilterRenderState = { - counter: state.counter, - depth: state.depth + 1, - }; - for (const child of sortedChildren) { - const rendered = renderFilter(child, params, childState); - if (rendered != null && rendered.length > 0) { - fragments.push(rendered); - } - } - state.counter = childState.counter; - - if (fragments.length === 0) return null; - if (fragments.length === 1) return fragments[0]; - const joiner = groupKey === "and" ? " AND " : " OR "; - return `(${fragments.join(joiner)})`; - } - - // Leaf predicate — grammar-gate the member and operator, then render. - const predicate = node as MetricPredicate; - - if (!isValidColumnName(predicate.member)) { - throw new Error( - `Refusing to build SQL: filter member "${predicate.member}" is not a valid identifier.`, - ); - } - if ( - !METRIC_FILTER_OPERATORS.includes( - predicate.operator as MetricFilterOperatorName, - ) - ) { - throw new Error( - `Refusing to build SQL: unknown filter operator "${predicate.operator}".`, - ); - } - - return renderPredicate(predicate, params, state); -} - -/** - * Stable-sort filter children inside an AND/OR group by `(member, operator)`. - * - * Predicates carry both fields and sort by their pair; nested groups sort - * after predicates and stay in their original relative order (a nested group - * is opaque from the outside — we cannot collapse it to a single key). This is - * the sort-before-hash invariant applied at the SQL-fragment level so - * downstream cache keys collapse semantically equivalent calls. - */ -function sortFilterChildren( - children: ReadonlyArray, -): MetricFilter[] { - const indexed = children.map((child, idx) => { - let key: string; - let isPredicate: boolean; - if ( - child !== null && - typeof child === "object" && - !("and" in child) && - !("or" in child) - ) { - const p = child as MetricPredicate; - // JSON.stringify the (member, operator) pair so the sort key is - // injective regardless of content. A plain delimiter is unsafe now that - // members accept the full delimited-identifier grammar (a member may - // contain "/", ".", etc.): `member "a/b" + op "c"` and - // `member "a" + op "b/c"` would both collapse to "a/b/c" under any - // single-char separator, tie-breaking distinct pairs to input order. - // JSON encoding escapes any separator, so distinct pairs map to distinct - // keys — the same technique canonicalizeFilter uses for values. - key = JSON.stringify([p.member, p.operator]); - isPredicate = true; - } else { - // Nested groups don't have a single (member, operator) — keep their - // original index so multiple nested groups within the same parent remain - // stable relative to each other. - key = ""; - isPredicate = false; - } - return { child, idx, key, isPredicate }; - }); - - indexed.sort((a, b) => { - if (a.isPredicate && !b.isPredicate) return -1; - if (!a.isPredicate && b.isPredicate) return 1; - if (a.isPredicate && b.isPredicate) { - if (a.key < b.key) return -1; - if (a.key > b.key) return 1; - } - return a.idx - b.idx; - }); - - return indexed.map((entry) => entry.child); -} - -/** - * Translate a single predicate into a SQL fragment. - * - * Every value flows through a freshly-allocated `:f_` named bind var. - * Nothing from the request body is ever interpolated as a literal — the - * fragment carries identifiers (grammar-gated) and operators (whitelisted), - * then references the bind name for each value. - * - * `set` and `notSet` emit `IS NULL` / `IS NOT NULL` with no bind value. `in` - * and `notIn` emit `IN (:f_0, :f_1, ...)`. `contains` and `notContains` emit - * `LIKE :f_0` / `NOT LIKE :f_0` and pre-bind the value with `%` wrapping. - */ -function renderPredicate( - predicate: MetricPredicate, - params: Record, - state: FilterRenderState, -): string { - // Backtick-quote the column identifier (defense-in-depth: the member was - // already validated by `isValidColumnName` in `renderFilter`). Quoting is - // what lets a delimited column name — hyphens, dots, non-ASCII — reach SQL - // safely; the bound value still flows through `:f_` params, never here. - const col = quoteIdentifier(predicate.member); - const op = predicate.operator; - const values = predicate.values ?? []; - - switch (op) { - case "equals": - return `${col} = ${bindValue(values[0], params, state)}`; - case "notEquals": - return `${col} <> ${bindValue(values[0], params, state)}`; - case "gt": - return `${col} > ${bindValue(values[0], params, state)}`; - case "gte": - return `${col} >= ${bindValue(values[0], params, state)}`; - case "lt": - return `${col} < ${bindValue(values[0], params, state)}`; - case "lte": - return `${col} <= ${bindValue(values[0], params, state)}`; - case "in": { - const placeholders = values.map((v) => bindValue(v, params, state)); - return `${col} IN (${placeholders.join(", ")})`; - } - case "notIn": { - const placeholders = values.map((v) => bindValue(v, params, state)); - return `${col} NOT IN (${placeholders.join(", ")})`; - } - case "contains": { - const raw = values[0]; - if (typeof raw !== "string") { - throw new Error( - `Refusing to build SQL: filter operator "contains" requires a string value (got ${typeof raw}).`, - ); - } - return `${col} LIKE ${bindLikeValue(raw, params, state)}`; - } - case "notContains": { - const raw = values[0]; - if (typeof raw !== "string") { - throw new Error( - `Refusing to build SQL: filter operator "notContains" requires a string value (got ${typeof raw}).`, - ); - } - return `${col} NOT LIKE ${bindLikeValue(raw, params, state)}`; - } - case "set": - return `${col} IS NOT NULL`; - case "notSet": - return `${col} IS NULL`; - default: { - // Exhaustiveness — the operator union is closed; if this is reached the - // operator vocabulary widened without updating the switch. - const _exhaustive: never = op; - throw new Error( - `Refusing to build SQL: unhandled filter operator "${_exhaustive as string}".`, - ); - } - } -} - -/** - * Allocate a fresh `:f_` bind name for `value`, push the typed marker into - * `params`, and return the placeholder string. Bumps the counter. - */ -function bindValue( - value: string | number | undefined, - params: Record, - state: FilterRenderState, -): string { - if (value === undefined) { - throw new Error( - "Refusing to build SQL: filter predicate is missing a required value.", - ); - } - const name = `f_${state.counter}`; - state.counter += 1; - if (typeof value === "number") { - params[name] = sqlHelpers.number(value); - } else if (typeof value === "string") { - params[name] = sqlHelpers.string(value); - } else { - throw new Error( - `Refusing to build SQL: filter value must be a string or number (got ${typeof value}).`, - ); - } - return `:${name}`; -} - -/** - * Like {@link bindValue}, but wraps the value in `%...%` for `LIKE` / - * `NOT LIKE`. SQL wildcards in the user-supplied string remain in the value - * (matching the documented "contains" semantics) — escape-on-receive could be - * added later as an opt-in if customers request strict-substring matching. - */ -function bindLikeValue( - value: string, - params: Record, - state: FilterRenderState, -): string { - const name = `f_${state.counter}`; - state.counter += 1; - params[name] = sqlHelpers.string(`%${value}%`); - return `:${name}`; -} - -/** - * Render a single SELECT-list clause for a dimension. - * - * When `timeGrain` is set and `dim` is the request's `timeDimension`, the - * column is bucketed: `date_trunc('', ) AS `. Every other - * dimension renders bare. Both the grain literal and the column identifier are - * grammar-gated before they reach the SQL string: - * - * - The grain is single-quoted (it is a `date_trunc` unit literal, NOT a bind - * param), so it is re-checked against {@link TIME_GRAIN_PATTERN} here before - * interpolation. The schema also gates it, but `buildMetricSql` is exported - * and may be reached on a path that bypasses `validateMetricRequest`, so the - * grammar gate is applied at the interpolation point too — matching how - * every other interpolated identifier (measures, dimensions, `timeDimension`, - * the FQN) is re-gated in the builder. - * - The column cannot be parameterized (it is an identifier), so it is - * re-checked against {@link isValidColumnName} and backtick-quoted here as - * belt-and-suspenders even though the schema already gated `timeDimension`. - * - * The aliasing keeps result-row keys stable (`{ order_date: ... }`) regardless - * of whether the grain was applied. - */ -function renderDimensionClause( - dim: string, - timeGrain: string | undefined, - timeDimension: string | undefined, -): string { - if (timeGrain != null && dim === timeDimension) { - if (!isValidColumnName(dim)) { - throw new Error( - `Refusing to build SQL: timeDimension "${dim}" is not a valid identifier.`, - ); - } - // The grain is interpolated as a single-quoted `date_trunc` unit literal - // (NOT a bind param), so it stays gated by the narrow TIME_GRAIN_PATTERN — - // it is a keyword, not a delimited identifier. - if (!TIME_GRAIN_PATTERN.test(timeGrain)) { - throw new Error( - `Refusing to build SQL: timeGrain "${timeGrain}" is not a valid grain token.`, - ); - } - const quoted = quoteIdentifier(dim); - return `date_trunc('${timeGrain}', ${quoted}) AS ${quoted}`; - } - return quoteIdentifier(dim); -} - -/** - * Compose the cache key for a metric-view request. - * - * Reserved namespace `metric` separates metric-view caches from query caches. - * The returned array is consumed by `CacheManager.generateKey`, which - * concatenates and sha256-hashes the parts — the per-concern structure keeps - * the key inspectable in tests and debug logs without giving up determinism. - * - * Sort-before-hash collapses semantically equivalent calls onto one entry: - * - `measures` / `dimensions`: lexicographic sort. - * - `filter`: predicates inside each AND/OR group are stable-sorted by - * `(member, operator)` and the group kind (`and` vs `or`) is preserved by - * {@link canonicalizeFilter}; values are included verbatim so distinct - * filter values yield distinct keys. - * - * `source` (the metric view's UC FQN) salts the key so that repointing a - * metric `key` to a different FQN in `metric-views.json` cannot serve rows - * cached under the old source within the TTL — `metricKey` alone is not enough - * because the key→source binding is mutable config. - * - * `timeGrain` salts the key whenever set. `timeDimension` only salts the key - * when `timeGrain` is also set, because `renderDimensionClause` only applies - * `date_trunc('', )` when a grain is present — with no - * grain the field has no effect on the emitted SQL, so including it would fork - * the cache on an input that produced identical SQL. - * - * `executorKey` is `"sp"` for SP-lane entries (shared cache) or a sha256 hash - * of the end user's identity for OBO-lane entries (per-user isolation) — see - * {@link deriveMetricExecutorKey}. - */ -export function composeMetricCacheKey(input: { - metricKey: string; - source: string; - measures: string[]; - dimensions?: string[]; - timeGrain?: string; - timeDimension?: string; - filter?: MetricFilter; - format: string; - executorKey: string; - limit?: number; -}): string[] { - const sortedMeasures = [...input.measures].sort(); - const sortedDimensions = [...(input.dimensions ?? [])].sort(); - const filterFingerprint = - input.filter !== undefined ? canonicalizeFilter(input.filter) : "_"; - // `timeDimension` only changes the SQL when `timeGrain` is set (see - // renderDimensionClause); keying on it otherwise would fork the cache on a - // no-op field. - const timeDimensionPart = - input.timeGrain != null ? (input.timeDimension ?? "_") : "_"; - return [ - "metric", - input.metricKey, - input.source, - input.format, - sortedMeasures.join(","), - sortedDimensions.join(","), - input.timeGrain ?? "_", - timeDimensionPart, - filterFingerprint, - typeof input.limit === "number" ? String(input.limit) : "_", - input.executorKey, - ]; -} - -/** - * Derive the cache executor key from a metric registration's lane and the - * caller's user identity. - * - * Returns `"sp"` for SP-lane entries (every caller shares the cache) and a - * sha256 hex digest of the user identity for OBO-lane entries (each user gets - * an isolated cache scope). - * - * The user identity is hashed — never stored verbatim — so the cache layer - * (which logs keys at debug level and persists them in any cache backend) - * never sees raw user emails or principal names. A stable, opaque token is - * exactly what we need: same user → same key (so cache hits work), different - * users → different keys (so isolation holds), and reverse lookup is - * infeasible. - * - * For OBO requests without a resolvable identity (missing or whitespace-only - * user id), throw `AuthenticationError.missingUserId()` rather than falling - * back to a shared `"anonymous"` sentinel — distinct misconfigured callers - * would otherwise collide into one cache scope and read each other's cached - * results. The route computes this inside its try/catch, so the throw lands on - * the canonical 401 envelope. - */ -export function deriveMetricExecutorKey(input: { - lane: MetricLane; - userIdentity?: string | null; -}): string { - if (input.lane === "sp") { - return "sp"; - } - // OBO lane — hash the user identity so the raw email/principal never reaches - // the cache layer. Missing/whitespace identity is a hard auth failure: the - // alternative ("anonymous" sentinel) collides every misconfigured caller - // into a single cache scope, so user A's results could leak to user B. - const identity = input.userIdentity?.trim(); - if (!identity) { - throw AuthenticationError.missingUserId(); - } - return createHash("sha256").update(identity).digest("hex"); -} - -/** - * Produce a deterministic string fingerprint of the filter tree. - * - * The fingerprint sorts predicates within each AND/OR group by - * `(member, operator)` and recursively canonicalizes nested groups. Values are - * included verbatim so cache entries differ when the filter targets different - * values (`region in [EMEA]` vs `region in [APAC]` → different keys; `equals A` - * vs `equals B` → different keys), while order-insensitive predicate lists - * collapse to the same key. - */ -function canonicalizeFilter(node: MetricFilter): string { - if (node === null || typeof node !== "object") { - return "_"; - } - - if ("and" in node || "or" in node) { - const groupKey = "and" in node ? "and" : "or"; - const children = ( - node as { and?: ReadonlyArray } & { - or?: ReadonlyArray; - } - )[groupKey]; - - if (!Array.isArray(children) || children.length === 0) { - return `${groupKey}()`; - } - - const sorted = sortFilterChildren(children); - const childFingerprints = sorted.map(canonicalizeFilter); - return `${groupKey}(${childFingerprints.join(",")})`; - } - - // Leaf predicate. JSON.stringify every structural part — member, operator, - // and each value (with its type) — so no content can be confused with a - // separator. This matters now that `member` accepts the full delimited- - // identifier grammar (it may contain "/", ".", etc.): a bare - // `p(${member}/${operator}/...)` would let `member "a", op "b"` collide with - // `member "a/b"` and fork/merge cache entries. Encoding the whole tuple as - // JSON makes the fingerprint injective regardless of member/value content. - const p = node as MetricPredicate; - const valuesPart = (p.values ?? []).map((v) => [typeof v, v]); - return `p(${JSON.stringify([p.member, p.operator, valuesPart])})`; -} +export * from "./mv"; diff --git a/packages/appkit/src/plugins/analytics/mv/cache.ts b/packages/appkit/src/plugins/analytics/mv/cache.ts new file mode 100644 index 000000000..45de63771 --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/cache.ts @@ -0,0 +1,71 @@ +import { createHash } from "node:crypto"; +import { AuthenticationError } from "../../../errors"; +import type { MetricFilter, MetricLane, MetricPredicate } from "../types"; +import { sortFilterChildren } from "./formatters"; +import type { MetricCacheKeyInput } from "./types"; + +export function composeMetricCacheKey(input: MetricCacheKeyInput): string[] { + const sortedMeasures = [...input.measures].sort(); + const sortedDimensions = [...(input.dimensions ?? [])].sort(); + const filterFingerprint = + input.filter !== undefined ? canonicalizeFilter(input.filter) : "_"; + // `timeDimension` only changes the SQL when `timeGrain` is set (see + // renderDimensionClause); keying on it otherwise would fork the cache on a + // no-op field. + const timeDimensionPart = + input.timeGrain != null ? (input.timeDimension ?? "_") : "_"; + return [ + "metric", + input.metricKey, + input.source, + input.format, + sortedMeasures.join(","), + sortedDimensions.join(","), + input.timeGrain ?? "_", + timeDimensionPart, + filterFingerprint, + typeof input.limit === "number" ? String(input.limit) : "_", + input.executorKey, + ]; +} + +export function deriveMetricExecutorKey(input: { + lane: MetricLane; + userIdentity?: string | null; +}): string { + if (input.lane === "sp") { + return "sp"; + } + const identity = input.userIdentity?.trim(); + if (!identity) { + throw AuthenticationError.missingUserId(); + } + return createHash("sha256").update(identity).digest("hex"); +} + +function canonicalizeFilter(node: MetricFilter): string { + if (node === null || typeof node !== "object") { + return "_"; + } + + if ("and" in node || "or" in node) { + const groupKey = "and" in node ? "and" : "or"; + const children = ( + node as { and?: ReadonlyArray } & { + or?: ReadonlyArray; + } + )[groupKey]; + + if (!Array.isArray(children) || children.length === 0) { + return `${groupKey}()`; + } + + const sorted = sortFilterChildren(children); + const childFingerprints = sorted.map(canonicalizeFilter); + return `${groupKey}(${childFingerprints.join(",")})`; + } + + const p = node as MetricPredicate; + const valuesPart = (p.values ?? []).map((v) => [typeof v, v]); + return `p(${JSON.stringify([p.member, p.operator, valuesPart])})`; +} diff --git a/packages/appkit/src/plugins/analytics/mv/constants.ts b/packages/appkit/src/plugins/analytics/mv/constants.ts new file mode 100644 index 000000000..3676c80ba --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/constants.ts @@ -0,0 +1,122 @@ +import path from "node:path"; +import type { MetricFilterOperatorName, MetricLane } from "../types"; + +/** + * Default queries directory. Mirrors `AppManager`'s + * `path.resolve(process.cwd(), "config/queries")` so dev mode and production + * share a single source of truth for where metric config lives. Exported so + * `AnalyticsPlugin` can default `config.queriesDir` to the same path. + */ +export const QUERIES_DIR = path.resolve(process.cwd(), "config/queries"); +export const METRIC_CONFIG_FILE = "metric-views.json"; + +/** + * Measure, dimension, and filter-member names are **column identifiers**: they + * are validated by the shared {@link isValidColumnName} (rejects only control + * characters / newlines) and backtick-quoted via {@link quoteIdentifier} at + * every interpolation point. Quoting — not a narrow ASCII allowlist — is the + * injection boundary, so the runtime accepts the full delimited-identifier + * grammar the type-generator emits from DESCRIBE (hyphens, dots, non-ASCII). + * There is deliberately NO name allowlist: a well-formed-but-unknown column + * falls through to the warehouse and surfaces as a sanitized canonical error. + * + * Time-grain token shape. Unlike the column identifiers above, the grain is + * interpolated as a single-quoted `date_trunc` unit LITERAL (NOT a bind param, + * NOT a delimited identifier) in {@link renderDimensionClause}, so it keeps a + * narrow keyword-shaped gate — that pattern is what keeps a hostile token out + * of the quoted-literal position. + */ +export const TIME_GRAIN_PATTERN = /^[a-z][a-z_]*$/; + +/** + * The exact twelve filter operators allowed at v1. The runtime tuple is the + * server-side source of truth; the client-side type union + * `MetricFilterOperatorName` mirrors these names statically. + */ +export const METRIC_FILTER_OPERATORS = [ + "equals", + "notEquals", + "in", + "notIn", + "gt", + "gte", + "lt", + "lte", + "contains", + "notContains", + "set", + "notSet", +] as const satisfies readonly MetricFilterOperatorName[]; + +/** + * Maximum AND/OR nesting depth. The PRD documents 8 as a sensible cap — + * enough for any real BI filter UI, low enough that a hostile or malformed + * payload cannot stack-overflow the recursive validator or translator. + * + * The depth count is the number of nested `{ and }` / `{ or }` wrappers + * encountered while descending — leaf predicates do not count toward depth. + */ +export const METRIC_FILTER_MAX_DEPTH = 8; + +/** + * Cardinality caps on user-controlled arrays. Closes the recurring + * `unbounded-request-parameters` finding: a hostile caller could otherwise + * send `values: [...10M items...]` and exhaust the validator + the named + * bind-var binding step. The limits below are deliberately generous — higher + * than any real BI UI would emit — so legitimate traffic never trips them. + */ +export const METRIC_MEASURES_MAX = 50; +export const METRIC_DIMENSIONS_MAX = 20; +export const METRIC_FILTER_VALUES_MAX = 1000; +export const METRIC_LIMIT_MAX = 100_000; + +/** + * Maximum number of children per AND/OR group node. Without this cap a single + * flat group like `{ and: [...10M empty objects...] }` would push tens of + * millions of frames onto the iterative pre-check's stack — OOM before + * validation even gets to Zod. The Zod schema enforces the same cap so the + * rejection point is consistent regardless of which validator catches it + * first. + */ +export const METRIC_FILTER_GROUP_MAX = 100; + +/** Operators that require exactly one value. */ +export const SINGLE_VALUE_OPERATORS = new Set([ + "equals", + "notEquals", + "gt", + "gte", + "lt", + "lte", + "contains", + "notContains", +]); + +/** Operators that require at least one value. */ +export const LIST_VALUE_OPERATORS = new Set([ + "in", + "notIn", +]); + +/** Operators that reject `values` entirely. */ +export const NULL_OPERATORS = new Set([ + "set", + "notSet", +]); + +/** Operators that emit `LIKE` / `NOT LIKE` and require a string value. */ +export const STRING_OPERATORS = new Set([ + "contains", + "notContains", +]); + +/** + * Map an entry's declared `executor` to the internal execution lane: + * - `"user"` → `"obo"` (per-user cache, on-behalf-of) + * - `"app_service_principal"` (default) → `"sp"` (shared cache) + */ +export function laneFromExecutor( + executor: "app_service_principal" | "user", +): MetricLane { + return executor === "user" ? "obo" : "sp"; +} diff --git a/packages/appkit/src/plugins/analytics/mv/formatters.ts b/packages/appkit/src/plugins/analytics/mv/formatters.ts new file mode 100644 index 000000000..57ac74de5 --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/formatters.ts @@ -0,0 +1,319 @@ +import { type SQLTypeMarker, sql as sqlHelpers } from "shared"; +import { + isValidColumnName, + isValidFqn, + quoteFqnForSql, + quoteIdentifier, +} from "../../../../../shared/src/schemas/metric-fqn"; +import type { + IAnalyticsMetricRequest, + MetricFilter, + MetricFilterOperatorName, + MetricPredicate, + MetricRegistration, +} from "../types"; +import { + METRIC_FILTER_MAX_DEPTH, + METRIC_FILTER_OPERATORS, + TIME_GRAIN_PATTERN, +} from "./constants"; +import type { FilterRenderState } from "./types"; + +function quoteSafeFqn(fqn: string): string { + if (!isValidFqn(fqn)) { + throw new Error( + `Refusing to build SQL: "${fqn}" is not a valid three-part UC FQN.`, + ); + } + return quoteFqnForSql(fqn); +} + +export function buildMetricSql( + registration: MetricRegistration, + request: IAnalyticsMetricRequest, +): { + statement: string; + parameters: Record; +} { + const quotedSource = quoteSafeFqn(registration.source); + + if (request.measures.length === 0) { + throw new Error("buildMetricSql requires at least one measure."); + } + + for (const m of request.measures) { + if (!isValidColumnName(m)) { + throw new Error( + `Refusing to build SQL: measure "${m}" is not a valid identifier.`, + ); + } + } + + const dimensions = request.dimensions ?? []; + for (const d of dimensions) { + if (!isValidColumnName(d)) { + throw new Error( + `Refusing to build SQL: dimension "${d}" is not a valid identifier.`, + ); + } + } + + const measureClauses = [...request.measures] + .sort() + .map((m) => `MEASURE(${quoteIdentifier(m)}) AS ${quoteIdentifier(m)}`); + + const dimensionClauses = [...dimensions] + .sort() + .map((d) => + renderDimensionClause(d, request.timeGrain, request.timeDimension), + ); + + const selectList = [...measureClauses, ...dimensionClauses].join(", "); + const groupByClause = dimensions.length > 0 ? " GROUP BY ALL" : ""; + + const limitClause = + typeof request.limit === "number" && request.limit > 0 + ? ` LIMIT ${Math.floor(request.limit)}` + : ""; + + const parameters: Record = {}; + let whereClause = ""; + if (request.filter !== undefined) { + const fragment = renderFilter(request.filter, parameters, { + counter: 0, + depth: 0, + }); + if (fragment !== null && fragment.length > 0) { + whereClause = ` WHERE ${fragment}`; + } + } + + const statement = `SELECT ${selectList} FROM ${quotedSource}${whereClause}${groupByClause}${limitClause}`; + return { statement, parameters }; +} + +function renderFilter( + node: MetricFilter, + params: Record, + state: FilterRenderState, +): string | null { + if (node === null || typeof node !== "object") { + throw new Error( + "Refusing to build SQL: filter node must be an object Predicate or { and } / { or } group.", + ); + } + + if ("and" in node || "or" in node) { + const groupKey = "and" in node ? "and" : "or"; + if (state.depth + 1 > METRIC_FILTER_MAX_DEPTH) { + throw new Error( + `Refusing to build SQL: filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}.`, + ); + } + + const children = ( + node as { and?: ReadonlyArray } & { + or?: ReadonlyArray; + } + )[groupKey]; + + if (!Array.isArray(children) || children.length === 0) { + if (groupKey === "or") { + return "1 = 0"; + } + return null; + } + + const sortedChildren = sortFilterChildren(children); + + const fragments: string[] = []; + const childState: FilterRenderState = { + counter: state.counter, + depth: state.depth + 1, + }; + for (const child of sortedChildren) { + const rendered = renderFilter(child, params, childState); + if (rendered != null && rendered.length > 0) { + fragments.push(rendered); + } + } + state.counter = childState.counter; + + if (fragments.length === 0) return null; + if (fragments.length === 1) return fragments[0]; + const joiner = groupKey === "and" ? " AND " : " OR "; + return `(${fragments.join(joiner)})`; + } + + const predicate = node as MetricPredicate; + + if (!isValidColumnName(predicate.member)) { + throw new Error( + `Refusing to build SQL: filter member "${predicate.member}" is not a valid identifier.`, + ); + } + if ( + !METRIC_FILTER_OPERATORS.includes( + predicate.operator as MetricFilterOperatorName, + ) + ) { + throw new Error( + `Refusing to build SQL: unknown filter operator "${predicate.operator}".`, + ); + } + + return renderPredicate(predicate, params, state); +} + +export function sortFilterChildren( + children: ReadonlyArray, +): MetricFilter[] { + const indexed = children.map((child, idx) => { + let key: string; + let isPredicate: boolean; + if ( + child !== null && + typeof child === "object" && + !("and" in child) && + !("or" in child) + ) { + const p = child as MetricPredicate; + key = JSON.stringify([p.member, p.operator]); + isPredicate = true; + } else { + key = ""; + isPredicate = false; + } + return { child, idx, key, isPredicate }; + }); + + indexed.sort((a, b) => { + if (a.isPredicate && !b.isPredicate) return -1; + if (!a.isPredicate && b.isPredicate) return 1; + if (a.isPredicate && b.isPredicate) { + if (a.key < b.key) return -1; + if (a.key > b.key) return 1; + } + return a.idx - b.idx; + }); + + return indexed.map((entry) => entry.child); +} + +function renderPredicate( + predicate: MetricPredicate, + params: Record, + state: FilterRenderState, +): string { + const col = quoteIdentifier(predicate.member); + const op = predicate.operator; + const values = predicate.values ?? []; + + switch (op) { + case "equals": + return `${col} = ${bindValue(values[0], params, state)}`; + case "notEquals": + return `${col} <> ${bindValue(values[0], params, state)}`; + case "gt": + return `${col} > ${bindValue(values[0], params, state)}`; + case "gte": + return `${col} >= ${bindValue(values[0], params, state)}`; + case "lt": + return `${col} < ${bindValue(values[0], params, state)}`; + case "lte": + return `${col} <= ${bindValue(values[0], params, state)}`; + case "in": { + const placeholders = values.map((v) => bindValue(v, params, state)); + return `${col} IN (${placeholders.join(", ")})`; + } + case "notIn": { + const placeholders = values.map((v) => bindValue(v, params, state)); + return `${col} NOT IN (${placeholders.join(", ")})`; + } + case "contains": { + const raw = values[0]; + if (typeof raw !== "string") { + throw new Error( + `Refusing to build SQL: filter operator "contains" requires a string value (got ${typeof raw}).`, + ); + } + return `${col} LIKE ${bindLikeValue(raw, params, state)}`; + } + case "notContains": { + const raw = values[0]; + if (typeof raw !== "string") { + throw new Error( + `Refusing to build SQL: filter operator "notContains" requires a string value (got ${typeof raw}).`, + ); + } + return `${col} NOT LIKE ${bindLikeValue(raw, params, state)}`; + } + case "set": + return `${col} IS NOT NULL`; + case "notSet": + return `${col} IS NULL`; + default: { + const _exhaustive: never = op; + throw new Error( + `Refusing to build SQL: unhandled filter operator "${_exhaustive as string}".`, + ); + } + } +} + +function bindValue( + value: string | number | undefined, + params: Record, + state: FilterRenderState, +): string { + if (value === undefined) { + throw new Error( + "Refusing to build SQL: filter predicate is missing a required value.", + ); + } + const name = `f_${state.counter}`; + state.counter += 1; + if (typeof value === "number") { + params[name] = sqlHelpers.number(value); + } else if (typeof value === "string") { + params[name] = sqlHelpers.string(value); + } else { + throw new Error( + `Refusing to build SQL: filter value must be a string or number (got ${typeof value}).`, + ); + } + return `:${name}`; +} + +function bindLikeValue( + value: string, + params: Record, + state: FilterRenderState, +): string { + const name = `f_${state.counter}`; + state.counter += 1; + params[name] = sqlHelpers.string(`%${value}%`); + return `:${name}`; +} + +function renderDimensionClause( + dim: string, + timeGrain: string | undefined, + timeDimension: string | undefined, +): string { + if (timeGrain != null && dim === timeDimension) { + if (!isValidColumnName(dim)) { + throw new Error( + `Refusing to build SQL: timeDimension "${dim}" is not a valid identifier.`, + ); + } + if (!TIME_GRAIN_PATTERN.test(timeGrain)) { + throw new Error( + `Refusing to build SQL: timeGrain "${timeGrain}" is not a valid grain token.`, + ); + } + const quoted = quoteIdentifier(dim); + return `date_trunc('${timeGrain}', ${quoted}) AS ${quoted}`; + } + return quoteIdentifier(dim); +} diff --git a/packages/appkit/src/plugins/analytics/mv/index.ts b/packages/appkit/src/plugins/analytics/mv/index.ts new file mode 100644 index 000000000..de5eec8ff --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/index.ts @@ -0,0 +1,9 @@ +export { composeMetricCacheKey, deriveMetricExecutorKey } from "./cache"; +export { QUERIES_DIR } from "./constants"; +export { buildMetricSql } from "./formatters"; +export { + __resetMetricRegistryCache, + getMetricRegistry, + loadMetricRegistry, +} from "./registry"; +export { validateMetricRequest } from "./schemas"; diff --git a/packages/appkit/src/plugins/analytics/mv/registry.ts b/packages/appkit/src/plugins/analytics/mv/registry.ts new file mode 100644 index 000000000..190fd1a69 --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/registry.ts @@ -0,0 +1,192 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +// Canonical metric-source schema — the single source of truth for +// `metric-views.json`. Imported from the shared source directly (matching the +// type-generator's runtime, which pulls the zod-free `metric-fqn.ts` from the +// same tree) so the runtime and the generated JSON schema validate identically. +import { metricSourceSchema } from "../../../../../shared/src/schemas/metric-source"; +import { createLogger } from "../../../logging/logger"; +import type { MetricRegistration } from "../types"; +import { laneFromExecutor, METRIC_CONFIG_FILE, QUERIES_DIR } from "./constants"; +import type { RegistryCacheSignature } from "./types"; + +const logger = createLogger("analytics:metric"); + +/** + * Read and validate `config/queries/metric-views.json` into a metric registry. + * + * Async and stateless — registration is a pure config parse with no warehouse + * round-trip, no `DESCRIBE`, and no build-time metadata bundle. The single + * `metricViews` map makes keys unique by construction, so there is no + * cross-lane duplicate-key check. Async I/O (rather than `readFileSync`) keeps + * the event loop free: metric views are already heavier than a plain `.sql` + * query on the warehouse side, so the SDK layer must not add a blocking read + * on top. Caching + mtime-revalidation is layered on top by + * {@link getMetricRegistry} — this function always hits disk. + * + * Returns an empty registry when the file is absent: the metric-view path is + * additive and dormant until an app opts in by adding the config. A malformed + * file (unreadable, invalid JSON, or schema violation) throws — the caller + * surfaces a 503 rather than masking a broken deployment as a 404 for every + * key. The failure is NOT cached (see {@link getMetricRegistry}), so fixing the + * file heals on the next request. + */ +export async function loadMetricRegistry( + queriesDir: string = QUERIES_DIR, +): Promise> { + const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); + + let raw: string; + try { + raw = await fs.readFile(metricPath, "utf8"); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + return Object.create(null); + } + throw err; + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err) { + throw new Error( + `Failed to parse metric-views.json at ${metricPath}: ${(err as Error).message}`, + ); + } + + const result = metricSourceSchema.safeParse(parsed); + if (!result.success) { + const issues = result.error.issues + .map((i) => `${i.path.join(".")}: ${i.message}`) + .join("; "); + throw new Error(`Invalid metric-views.json at ${metricPath}: ${issues}`); + } + + // Null-prototype map so a metric key that collides with an inherited + // `Object.prototype` member (`__proto__`, `constructor`, `toString`, …) + // cannot resolve to a truthy non-registration at the `registry[key]` read + // site and slip past the unknown-key 404. Keys are still grammar-gated by + // `metricKeySchema` (identifier shape), but the null prototype removes the + // whole class of inherited-property lookups as a boundary. + const registry: Record = Object.create(null); + for (const [key, entry] of Object.entries(result.data.metricViews ?? {})) { + registry[key] = { + key, + source: entry.source, + lane: laneFromExecutor(entry.executor), + }; + } + + logger.debug( + "Loaded metric registry: %d entry(ies)", + Object.keys(registry).length, + ); + return registry; +} + +/** + * Module-level registry cache, keyed by the resolved queries directory. + * + * Keyed by DIR (not by plugin instance) because the registry is a pure + * function of the config file at that path — warehouse-independent, and two + * `AnalyticsPlugin` instances pointed at the same `config/queries/` MUST see + * the same registry. Instance state would parse the identical file twice and + * risk divergence; a dir-keyed module cache shares one parse. + */ +const metricRegistryCache = new Map< + string, + { + signature: RegistryCacheSignature; + registry: Record; + } +>(); + +/** + * Clear the module-level registry cache. + * + * @internal FOR TESTS ONLY. The cache is keyed by directory and lives for the + * process; production never needs to clear it (a changed file is picked up via + * the stat signature). Tests that reuse a directory — or assert cold-load + * behavior — call this in `beforeEach`/`afterEach` so isolation is intentional + * rather than relying on each test happening to use a unique temp dir. + */ +export function __resetMetricRegistryCache(): void { + metricRegistryCache.clear(); +} + +/** + * Resolve the metric registry for `queriesDir`, re-reading + re-parsing only + * when `metric-views.json` has changed since the cached copy. + * + * Behaves like the sibling `.sql` query path (which re-reads per request) but + * cheaper: the steady-state cost is a single async `stat`, and the read + + * `JSON.parse` + zod validation are skipped when the file's + * `(ctimeMs, mtimeMs, size)` signature is unchanged. This delivers the agreed + * semantics without the old permanent memo: + * + * - **Hot-reload** — editing a working config bumps `mtimeMs`, so the next + * request re-parses and serves the new registry with no server restart. + * - **Self-heal** — a load failure is NOT cached (we only populate the cache + * on a successful parse), so a fixed config is picked up on the next + * request instead of latching a 503 forever. + * - **Dormant** — an absent file `stat`s as `ENOENT` → empty registry; adding + * the file later is picked up on the next request. + * + * Concurrency: two simultaneous cold requests may both `stat`+read before + * either populates the cache. That is a harmless redundant read of the same + * file (the sibling `.sql` path does not single-flight either), so no lock is + * taken. + * + * @throws Propagates {@link loadMetricRegistry}'s throw on a malformed file so + * the route can surface a 503; the cache is left untouched on failure. + */ +export async function getMetricRegistry( + queriesDir: string = QUERIES_DIR, +): Promise> { + const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); + + let signature: RegistryCacheSignature | null; + try { + const stats = await fs.stat(metricPath); + signature = { + ctimeMs: stats.ctimeMs, + mtimeMs: stats.mtimeMs, + size: stats.size, + }; + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + // Absent file → dormant. Drop any stale cache entry (the file may have + // been deleted) and return an empty registry without touching the cache. + metricRegistryCache.delete(queriesDir); + signature = null; + } else { + // Any other stat error (EACCES / EIO / ELOOP / …) is deliberately fatal + // for THIS request → the route surfaces a 503, consistent with the + // malformed-config → 503 path. It is not latched: with the self-heal + // design a transient error clears on the next request, which is strictly + // better than the old memo that could latch-and-serve-stale. + throw err; + } + } + + if (signature === null) { + return Object.create(null); + } + + const cached = metricRegistryCache.get(queriesDir); + if ( + cached !== undefined && + cached.signature.ctimeMs === signature.ctimeMs && + cached.signature.mtimeMs === signature.mtimeMs && + cached.signature.size === signature.size + ) { + return cached.registry; + } + + // Cold or stale: re-read + re-parse. Cache ONLY on success so a malformed + // file never latches — the next request re-attempts and heals. + const registry = await loadMetricRegistry(queriesDir); + metricRegistryCache.set(queriesDir, { signature, registry }); + return registry; +} diff --git a/packages/appkit/src/plugins/analytics/mv/schemas.ts b/packages/appkit/src/plugins/analytics/mv/schemas.ts new file mode 100644 index 000000000..0f00dcad5 --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/schemas.ts @@ -0,0 +1,356 @@ +import { z } from "zod"; +import { isValidColumnName } from "../../../../../shared/src/schemas/metric-fqn"; +import { ValidationError } from "../../../errors"; +import type { + IAnalyticsMetricRequest, + MetricFilter, + MetricFilterOperatorName, + MetricPredicate, +} from "../types"; +import { normalizeAnalyticsFormat } from "../types"; +import { + LIST_VALUE_OPERATORS, + METRIC_DIMENSIONS_MAX, + METRIC_FILTER_GROUP_MAX, + METRIC_FILTER_MAX_DEPTH, + METRIC_FILTER_OPERATORS, + METRIC_FILTER_VALUES_MAX, + METRIC_LIMIT_MAX, + METRIC_MEASURES_MAX, + NULL_OPERATORS, + SINGLE_VALUE_OPERATORS, + STRING_OPERATORS, + TIME_GRAIN_PATTERN, +} from "./constants"; + +/** A leaf predicate: `{ member, operator, values? }`, no extra keys. */ +const filterPredicateSchema: z.ZodType = z + .object({ + member: z + .string() + .min(1, { message: "filter predicate 'member' cannot be empty" }) + .refine(isValidColumnName, { + message: + "filter predicate 'member' contains a character that cannot be used in a SQL identifier (control character or newline)", + }), + operator: z.string().min(1, { + message: "filter predicate 'operator' cannot be empty", + }) as z.ZodType, + values: z + .array(z.union([z.string(), z.number()])) + .max(METRIC_FILTER_VALUES_MAX, { + message: `filter predicate 'values' length exceeds the maximum of ${METRIC_FILTER_VALUES_MAX}`, + }) + .optional(), + }) + .strict(); + +/** Recursive filter: a predicate leaf or an `{ and }` / `{ or }` group. */ +const filterSchema: z.ZodType = z.lazy(() => + z.union([ + filterPredicateSchema, + z + .object({ + and: z.array(filterSchema).max(METRIC_FILTER_GROUP_MAX, { + message: `filter 'and' group exceeds the maximum of ${METRIC_FILTER_GROUP_MAX} children`, + }), + }) + .strict(), + z + .object({ + or: z.array(filterSchema).max(METRIC_FILTER_GROUP_MAX, { + message: `filter 'or' group exceeds the maximum of ${METRIC_FILTER_GROUP_MAX} children`, + }), + }) + .strict(), + ]), +); + +const metricRequestSchema = z + .object({ + measures: z + .array( + z + .string() + .min(1, "measure name cannot be empty") + .refine(isValidColumnName, { + message: + "measure name contains a character that cannot be used in a SQL identifier (control character or newline)", + }), + ) + .min(1, "at least one measure is required") + .max(METRIC_MEASURES_MAX, { + message: `measures length exceeds the maximum of ${METRIC_MEASURES_MAX}`, + }), + dimensions: z + .array( + z + .string() + .min(1, "dimension name cannot be empty") + .refine(isValidColumnName, { + message: + "dimension name contains a character that cannot be used in a SQL identifier (control character or newline)", + }), + ) + .max(METRIC_DIMENSIONS_MAX, { + message: `dimensions length exceeds the maximum of ${METRIC_DIMENSIONS_MAX}`, + }) + .optional(), + filter: filterSchema.optional(), + // Grammar-shaped bucketing grain, applied to `timeDimension` via + // `date_trunc`. The token is validated for safety here; the grain literal + // is interpolated (single-quoted, never a bind param) in + // `renderDimensionClause`, so this pattern gate is the security boundary. + timeGrain: z + .string() + .min(1, { message: "timeGrain cannot be empty" }) + .regex(TIME_GRAIN_PATTERN, { + message: "timeGrain must match /^[a-z][a-z_]*$/", + }) + .optional(), + // The single dimension `timeGrain` applies to via `date_trunc`. A column + // identifier (backtick-quoted at interpolation), so it accepts the full + // delimited-identifier grammar. Cross-field rules in `superRefine`: + // required when `timeGrain` is set, and must be one of `dimensions`. + timeDimension: z + .string() + .min(1, { message: "timeDimension cannot be empty" }) + .refine(isValidColumnName, { + message: + "timeDimension contains a character that cannot be used in a SQL identifier (control character or newline)", + }) + .optional(), + limit: z + .number() + .int({ message: "limit must be an integer" }) + .positive({ message: "limit must be positive" }) + .max(METRIC_LIMIT_MAX, { + message: `limit exceeds the maximum of ${METRIC_LIMIT_MAX}`, + }) + .optional(), + format: z.enum(["JSON_ARRAY", "ARROW_STREAM", "JSON", "ARROW"]).optional(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.filter != null) { + validateFilterTree(value.filter, ctx, ["filter"], 0); + } + + const seen = new Set(); + const collided = new Set(); + for (const name of [...value.measures, ...(value.dimensions ?? [])]) { + if (seen.has(name)) { + collided.add(name); + } + seen.add(name); + } + if (collided.size > 0) { + ctx.addIssue({ + code: "custom", + message: + "measures and dimensions must be unique across both lists (a name cannot repeat, nor appear as both a measure and a dimension)", + path: ["measures"], + }); + } + + if ( + value.format != null && + normalizeAnalyticsFormat(value.format) !== "JSON_ARRAY" + ) { + ctx.addIssue({ + code: "custom", + message: + "format: only JSON_ARRAY is supported on the metric route at v1 (ARROW_STREAM is not yet implemented)", + path: ["format"], + }); + } + + if (value.timeGrain != null && value.timeDimension == null) { + ctx.addIssue({ + code: "custom", + message: "timeDimension is required when timeGrain is set", + path: ["timeDimension"], + }); + } + if ( + value.timeDimension != null && + !(value.dimensions ?? []).includes(value.timeDimension) + ) { + ctx.addIssue({ + code: "custom", + message: "timeDimension must be one of dimensions", + path: ["timeDimension"], + }); + } + }) as z.ZodType; + +function validateFilterTree( + node: MetricFilter, + ctx: z.RefinementCtx, + path: Array, + depth: number, +): void { + if (node === null || typeof node !== "object") { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path, + message: "filter node must be a Predicate or { and } / { or } group", + }); + return; + } + + if ("and" in node || "or" in node) { + if (depth + 1 > METRIC_FILTER_MAX_DEPTH) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path, + message: `filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}`, + }); + return; + } + + const groupKey = "and" in node ? "and" : "or"; + const children = ( + node as { and?: ReadonlyArray } & { + or?: ReadonlyArray; + } + )[groupKey]; + + if (!Array.isArray(children)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, groupKey], + message: `filter ${groupKey} group must be an array of predicates or nested groups`, + }); + return; + } + + if (groupKey === "or" && children.length === 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "or"], + message: "filter 'or' group must contain at least one predicate", + }); + return; + } + + children.forEach((child, idx) => { + validateFilterTree(child, ctx, [...path, groupKey, idx], depth + 1); + }); + return; + } + + const predicate = node as MetricPredicate; + + if ( + !METRIC_FILTER_OPERATORS.includes( + predicate.operator as MetricFilterOperatorName, + ) + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "operator"], + message: `filter operator "${predicate.operator}" is not one of: ${METRIC_FILTER_OPERATORS.join(", ")}`, + }); + return; + } + + const op = predicate.operator; + const values = predicate.values; + const valuesLen = values?.length ?? 0; + + if (NULL_OPERATORS.has(op)) { + if (values != null && valuesLen > 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" must not carry values`, + }); + } + } else if (SINGLE_VALUE_OPERATORS.has(op)) { + if (valuesLen !== 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" requires exactly one value (got ${valuesLen})`, + }); + } + } else if (LIST_VALUE_OPERATORS.has(op)) { + if (valuesLen < 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" requires at least one value`, + }); + } + } + + if (STRING_OPERATORS.has(op) && valuesLen > 0) { + const v = predicate.values?.[0]; + if (typeof v !== "string") { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" requires a string value (got ${typeof v})`, + }); + } + } +} + +function preCheckFilterDepth(filter: unknown): void { + if (filter == null || typeof filter !== "object") return; + const stack: Array<[unknown, number]> = [[filter, 0]]; + while (stack.length > 0) { + const popped = stack.pop(); + if (popped === undefined) continue; + const [node, depth] = popped; + if (node == null || typeof node !== "object") continue; + const obj = node as Record; + for (const groupKey of ["and", "or"] as const) { + const children = obj[groupKey]; + if (!Array.isArray(children)) continue; + if (children.length > METRIC_FILTER_GROUP_MAX) { + throw new ValidationError( + "Invalid metric request body (fields: filter)", + { + context: { + reason: `filter ${groupKey} group has ${children.length} children; the maximum is ${METRIC_FILTER_GROUP_MAX}`, + }, + }, + ); + } + if (depth + 1 > METRIC_FILTER_MAX_DEPTH) { + throw new ValidationError( + "Invalid metric request body (fields: filter)", + { + context: { + reason: `filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}`, + }, + }, + ); + } + for (const child of children) { + stack.push([child, depth + 1]); + } + } + } +} + +export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { + if (body != null && typeof body === "object") { + preCheckFilterDepth((body as { filter?: unknown }).filter); + } + const result = metricRequestSchema.safeParse(body); + if (!result.success) { + const fieldPaths = result.error.issues + .map((i) => i.path.join(".") || "(root)") + .join(", "); + throw new ValidationError( + fieldPaths.length > 0 + ? `Invalid metric request body (fields: ${fieldPaths})` + : "Invalid metric request body", + { context: { issues: result.error.issues } }, + ); + } + return result.data; +} diff --git a/packages/appkit/src/plugins/analytics/mv/types.ts b/packages/appkit/src/plugins/analytics/mv/types.ts new file mode 100644 index 000000000..be33cf7d6 --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/types.ts @@ -0,0 +1,25 @@ +import type { MetricFilter } from "../types"; + +export interface RegistryCacheSignature { + ctimeMs: number; + mtimeMs: number; + size: number; +} + +export interface FilterRenderState { + counter: number; + depth: number; +} + +export interface MetricCacheKeyInput { + metricKey: string; + source: string; + measures: string[]; + dimensions?: string[]; + timeGrain?: string; + timeDimension?: string; + filter?: MetricFilter; + format: string; + executorKey: string; + limit?: number; +} From 9a3bc06fb274f4ce05264739736f965c0c6815b1 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Mon, 20 Jul 2026 11:25:45 +0200 Subject: [PATCH 10/18] refactor(appkit): derive metric operator sets from shared bases Remove the duplicated operator lists in mv/constants.ts: SINGLE_VALUE_OPERATORS spreads STRING_OPERATORS, and METRIC_FILTER_OPERATORS derives from the union of SINGLE_VALUE / LIST_VALUE / NULL sets rather than re-listing all twelve names. One source of truth per operator category; the twelve-operator tuple can no longer drift from the per-category sets. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../src/plugins/analytics/mv/constants.ts | 54 ++++++++----------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/packages/appkit/src/plugins/analytics/mv/constants.ts b/packages/appkit/src/plugins/analytics/mv/constants.ts index 3676c80ba..6eda28750 100644 --- a/packages/appkit/src/plugins/analytics/mv/constants.ts +++ b/packages/appkit/src/plugins/analytics/mv/constants.ts @@ -28,26 +28,6 @@ export const METRIC_CONFIG_FILE = "metric-views.json"; */ export const TIME_GRAIN_PATTERN = /^[a-z][a-z_]*$/; -/** - * The exact twelve filter operators allowed at v1. The runtime tuple is the - * server-side source of truth; the client-side type union - * `MetricFilterOperatorName` mirrors these names statically. - */ -export const METRIC_FILTER_OPERATORS = [ - "equals", - "notEquals", - "in", - "notIn", - "gt", - "gte", - "lt", - "lte", - "contains", - "notContains", - "set", - "notSet", -] as const satisfies readonly MetricFilterOperatorName[]; - /** * Maximum AND/OR nesting depth. The PRD documents 8 as a sensible cap — * enough for any real BI filter UI, low enough that a hostile or malformed @@ -80,18 +60,6 @@ export const METRIC_LIMIT_MAX = 100_000; */ export const METRIC_FILTER_GROUP_MAX = 100; -/** Operators that require exactly one value. */ -export const SINGLE_VALUE_OPERATORS = new Set([ - "equals", - "notEquals", - "gt", - "gte", - "lt", - "lte", - "contains", - "notContains", -]); - /** Operators that require at least one value. */ export const LIST_VALUE_OPERATORS = new Set([ "in", @@ -110,6 +78,17 @@ export const STRING_OPERATORS = new Set([ "notContains", ]); +/** Operators that require exactly one value. */ +export const SINGLE_VALUE_OPERATORS = new Set([ + "equals", + "notEquals", + "gt", + "gte", + "lt", + "lte", + ...STRING_OPERATORS, +]); + /** * Map an entry's declared `executor` to the internal execution lane: * - `"user"` → `"obo"` (per-user cache, on-behalf-of) @@ -120,3 +99,14 @@ export function laneFromExecutor( ): MetricLane { return executor === "user" ? "obo" : "sp"; } + +/** + * The exact twelve filter operators allowed at v1. The runtime tuple is the + * server-side source of truth; the client-side type union + * `MetricFilterOperatorName` mirrors these names statically. + */ +export const METRIC_FILTER_OPERATORS = [ + ...SINGLE_VALUE_OPERATORS, + ...LIST_VALUE_OPERATORS, + ...NULL_OPERATORS, +] as const satisfies readonly MetricFilterOperatorName[]; From 509c9040df59373fac899c34e5b1d710b3e435d0 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Mon, 20 Jul 2026 13:24:34 +0200 Subject: [PATCH 11/18] chore: adjust comments --- .../appkit/src/plugins/analytics/analytics.ts | 53 +++-------- .../appkit/src/plugins/analytics/mv/cache.ts | 4 +- .../src/plugins/analytics/mv/constants.ts | 33 ++----- .../src/plugins/analytics/mv/registry.ts | 36 ++------ packages/shared/src/schemas/metric-fqn.ts | 90 ++----------------- 5 files changed, 41 insertions(+), 175 deletions(-) diff --git a/packages/appkit/src/plugins/analytics/analytics.ts b/packages/appkit/src/plugins/analytics/analytics.ts index 451373ccd..ca9358e97 100644 --- a/packages/appkit/src/plugins/analytics/analytics.ts +++ b/packages/appkit/src/plugins/analytics/analytics.ts @@ -127,9 +127,8 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { /** * Directory the metric-view registry is read from (holds - * `metric-views.json`). Defaults to `config/queries/` under the process cwd - * — the same directory the `.sql` query path uses. Overridable ONLY via - * `config.queriesDir` for tests (see {@link IAnalyticsConfig.queriesDir}). + * `metric-views.json`). + * @default `config/queries/` (under the process cwd) */ private readonly _queriesDir: string; @@ -155,10 +154,8 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { }, }); - // Metric-view route. Registered parallel to `/query`; measures a - // registered UC Metric View over the same SSE envelope. Dormant until - // `config/queries/metric-views.json` exists (no config → empty registry → - // 404, nothing executes). + // Metric-view route. Registered parallel to `/query` + // measures a registered UC Metric View over the same SSE envelope. this.route(router, { name: "metric", method: "post", @@ -169,9 +166,8 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { }); // Column-names fallback for very wide Arrow schemas whose names don't fit - // in the `X-Appkit-Arrow-Columns` response header (see - // `_setArrowColumnsHeader`). The client hits this with the statement id - // from `X-Appkit-Arrow-Columns-Ref`. + // in the `X-Appkit-Arrow-Columns` response header. + // The client hits this with the statement id from `X-Appkit-Arrow-Columns-Ref`. this.route(router, { name: "arrow-columns", method: "get", @@ -470,8 +466,7 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { * Lane dispatch is driven by the registration: an SP-lane metric runs as the * app service principal (shared cache); an OBO-lane metric runs * on-behalf-of the requesting user via `asUser(req)` (per-user cache keyed by - * a hash of the user identity). The executor + key are computed inside a - * try so a missing/whitespace OBO identity lands on the canonical 401 path. + * a hash of the user identity). */ async _handleMetricRoute( req: express.Request, @@ -492,12 +487,7 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { return; } - // Resolve the registry from disk (cached + mtime-revalidated by - // `getMetricRegistry`, so the steady-state cost is one `stat`). A malformed - // config throws → 503, distinguishing a broken deployment from an unknown - // key (404). The failure is NOT latched: `getMetricRegistry` only caches on - // a successful parse, so fixing `metric-views.json` heals on the next - // request. Full reason → telemetry only. + // Resolve the registry from disk (cached + mtime-revalidated by `getMetricRegistry`). let registry: Record; try { registry = await getMetricRegistry(this._queriesDir); @@ -514,27 +504,19 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { return; } - // Own-property lookup: never resolve `key` to an inherited - // `Object.prototype` member (`__proto__`, `constructor`, `toString`, …). - // `getMetricRegistry` returns a null-prototype map, but gate the read here - // too as defense-in-depth — an inherited hit would otherwise bypass the 404 - // below and flow a non-registration value into execution. + // Own-property lookup: never resolve `key` to an inherited `Object.prototype` member. const registration = Object.hasOwn(registry, key) ? registry[key] : undefined; if (!registration) { - // Don't echo the user-supplied `key` back in the public response — - // confirming "metric X is not registered" lets a probe enumerate keys by - // elimination. The 404 status stays (useful for tooling); the body is - // generic and detail goes to telemetry only. + // Don't echo the user-supplied `key` back in the public response. event?.setContext("analytics", { unknown_metric_key: key }); res.status(404).json({ error: "Metric not found" }); return; } - // Validate the body on the canonical error path. `validateMetricRequest` - // throws a `ValidationError` (400) whose message names only field paths, - // never raw values. + // Validate the body on the canonical error path. + // `validateMetricRequest` throws a `ValidationError` (400) whose message names only field paths, never raw values. let request: ReturnType; try { request = validateMetricRequest(req.body ?? {}); @@ -561,10 +543,7 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { // `executor` in metric-views.json), NOT a URL segment or `.obo.sql` // filename: an OBO-lane metric runs on-behalf-of the requesting user // (per-user cache via `asUser(req)`), an SP-lane metric as the app service - // principal (shared cache). Compute the executor + key INSIDE a try (as - // `_handleQueryRoute` does) so `asUser(req)`/`resolveUserId(req)` and - // `deriveMetricExecutorKey`'s missing-identity throw land on the canonical - // 401 envelope instead of surfacing as an uncaught 500 out of the stream. + // principal (shared cache). let executor: AnalyticsPlugin; let executorKey: string; try { @@ -585,11 +564,7 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { // Cache key. Composed over the canonicalized args (sorted measures/ // dimensions, stable-sorted predicates, grain, timeDimension, limit) plus // the `executorKey` — `"sp"` shares the cache across all users, a per-user - // identity hash isolates OBO callers. Delivery is JSON-only in v1 (the - // route always routes through `_executeJsonArrayPath`), so the key salts on - // a stable "JSON_ARRAY" constant rather than `request.format`: two calls - // that deliver identically must not fork the cache on an unused format - // field. + // identity hash isolates OBO callers. const cacheConfig = { ...queryDefaults.cache, cacheKey: composeMetricCacheKey({ diff --git a/packages/appkit/src/plugins/analytics/mv/cache.ts b/packages/appkit/src/plugins/analytics/mv/cache.ts index 45de63771..d95f8cc25 100644 --- a/packages/appkit/src/plugins/analytics/mv/cache.ts +++ b/packages/appkit/src/plugins/analytics/mv/cache.ts @@ -9,9 +9,7 @@ export function composeMetricCacheKey(input: MetricCacheKeyInput): string[] { const sortedDimensions = [...(input.dimensions ?? [])].sort(); const filterFingerprint = input.filter !== undefined ? canonicalizeFilter(input.filter) : "_"; - // `timeDimension` only changes the SQL when `timeGrain` is set (see - // renderDimensionClause); keying on it otherwise would fork the cache on a - // no-op field. + // `timeDimension` only changes the SQL when `timeGrain` is set (see renderDimensionClause) const timeDimensionPart = input.timeGrain != null ? (input.timeDimension ?? "_") : "_"; return [ diff --git a/packages/appkit/src/plugins/analytics/mv/constants.ts b/packages/appkit/src/plugins/analytics/mv/constants.ts index 6eda28750..f159a891c 100644 --- a/packages/appkit/src/plugins/analytics/mv/constants.ts +++ b/packages/appkit/src/plugins/analytics/mv/constants.ts @@ -14,36 +14,25 @@ export const METRIC_CONFIG_FILE = "metric-views.json"; * Measure, dimension, and filter-member names are **column identifiers**: they * are validated by the shared {@link isValidColumnName} (rejects only control * characters / newlines) and backtick-quoted via {@link quoteIdentifier} at - * every interpolation point. Quoting — not a narrow ASCII allowlist — is the - * injection boundary, so the runtime accepts the full delimited-identifier - * grammar the type-generator emits from DESCRIBE (hyphens, dots, non-ASCII). - * There is deliberately NO name allowlist: a well-formed-but-unknown column - * falls through to the warehouse and surfaces as a sanitized canonical error. + * every interpolation point. * * Time-grain token shape. Unlike the column identifiers above, the grain is - * interpolated as a single-quoted `date_trunc` unit LITERAL (NOT a bind param, - * NOT a delimited identifier) in {@link renderDimensionClause}, so it keeps a - * narrow keyword-shaped gate — that pattern is what keeps a hostile token out + * interpolated as a single-quoted `date_trunc` unit LITERAL in {@link renderDimensionClause}, + * so it keeps a narrow keyword-shaped gate — that pattern is what keeps a hostile token out * of the quoted-literal position. */ export const TIME_GRAIN_PATTERN = /^[a-z][a-z_]*$/; /** - * Maximum AND/OR nesting depth. The PRD documents 8 as a sensible cap — - * enough for any real BI filter UI, low enough that a hostile or malformed - * payload cannot stack-overflow the recursive validator or translator. - * * The depth count is the number of nested `{ and }` / `{ or }` wrappers - * encountered while descending — leaf predicates do not count toward depth. + * encountered while descending; leaf predicates do not count toward depth. + * Prevents stack-overflowing the recursive validator or translator. */ export const METRIC_FILTER_MAX_DEPTH = 8; /** - * Cardinality caps on user-controlled arrays. Closes the recurring - * `unbounded-request-parameters` finding: a hostile caller could otherwise - * send `values: [...10M items...]` and exhaust the validator + the named - * bind-var binding step. The limits below are deliberately generous — higher - * than any real BI UI would emit — so legitimate traffic never trips them. + * Cardinality caps on user-controlled arrays. + * Prevents stack-overflowing the recursive validator or translator. */ export const METRIC_MEASURES_MAX = 50; export const METRIC_DIMENSIONS_MAX = 20; @@ -51,12 +40,8 @@ export const METRIC_FILTER_VALUES_MAX = 1000; export const METRIC_LIMIT_MAX = 100_000; /** - * Maximum number of children per AND/OR group node. Without this cap a single - * flat group like `{ and: [...10M empty objects...] }` would push tens of - * millions of frames onto the iterative pre-check's stack — OOM before - * validation even gets to Zod. The Zod schema enforces the same cap so the - * rejection point is consistent regardless of which validator catches it - * first. + * Maximum number of children per AND/OR group node. + * Prevents stack-overflowing the recursive validator or translator. */ export const METRIC_FILTER_GROUP_MAX = 100; diff --git a/packages/appkit/src/plugins/analytics/mv/registry.ts b/packages/appkit/src/plugins/analytics/mv/registry.ts index 190fd1a69..66a4d032d 100644 --- a/packages/appkit/src/plugins/analytics/mv/registry.ts +++ b/packages/appkit/src/plugins/analytics/mv/registry.ts @@ -10,26 +10,16 @@ import type { MetricRegistration } from "../types"; import { laneFromExecutor, METRIC_CONFIG_FILE, QUERIES_DIR } from "./constants"; import type { RegistryCacheSignature } from "./types"; -const logger = createLogger("analytics:metric"); +const logger = createLogger("analytics:metric-views"); /** * Read and validate `config/queries/metric-views.json` into a metric registry. * * Async and stateless — registration is a pure config parse with no warehouse - * round-trip, no `DESCRIBE`, and no build-time metadata bundle. The single - * `metricViews` map makes keys unique by construction, so there is no - * cross-lane duplicate-key check. Async I/O (rather than `readFileSync`) keeps - * the event loop free: metric views are already heavier than a plain `.sql` - * query on the warehouse side, so the SDK layer must not add a blocking read - * on top. Caching + mtime-revalidation is layered on top by - * {@link getMetricRegistry} — this function always hits disk. + * round-trip, no `DESCRIBE`, and no build-time metadata bundle. * - * Returns an empty registry when the file is absent: the metric-view path is - * additive and dormant until an app opts in by adding the config. A malformed - * file (unreadable, invalid JSON, or schema violation) throws — the caller - * surfaces a 503 rather than masking a broken deployment as a 404 for every - * key. The failure is NOT cached (see {@link getMetricRegistry}), so fixing the - * file heals on the next request. + * Absent file -> empty registry. + * Malformed file -> 503. */ export async function loadMetricRegistry( queriesDir: string = QUERIES_DIR, @@ -66,9 +56,9 @@ export async function loadMetricRegistry( // Null-prototype map so a metric key that collides with an inherited // `Object.prototype` member (`__proto__`, `constructor`, `toString`, …) // cannot resolve to a truthy non-registration at the `registry[key]` read - // site and slip past the unknown-key 404. Keys are still grammar-gated by - // `metricKeySchema` (identifier shape), but the null prototype removes the - // whole class of inherited-property lookups as a boundary. + // site and slip past the unknown-key 404. + // Keys are still grammar-gated by `metricKeySchema` (identifier shape), + // but the null prototype removes the whole class of inherited-property lookups as a boundary. const registry: Record = Object.create(null); for (const [key, entry] of Object.entries(result.data.metricViews ?? {})) { registry[key] = { @@ -87,12 +77,6 @@ export async function loadMetricRegistry( /** * Module-level registry cache, keyed by the resolved queries directory. - * - * Keyed by DIR (not by plugin instance) because the registry is a pure - * function of the config file at that path — warehouse-independent, and two - * `AnalyticsPlugin` instances pointed at the same `config/queries/` MUST see - * the same registry. Instance state would parse the identical file twice and - * risk divergence; a dir-keyed module cache shares one parse. */ const metricRegistryCache = new Map< string, @@ -164,8 +148,7 @@ export async function getMetricRegistry( // Any other stat error (EACCES / EIO / ELOOP / …) is deliberately fatal // for THIS request → the route surfaces a 503, consistent with the // malformed-config → 503 path. It is not latched: with the self-heal - // design a transient error clears on the next request, which is strictly - // better than the old memo that could latch-and-serve-stale. + // design a transient error clears on the next request. throw err; } } @@ -184,8 +167,7 @@ export async function getMetricRegistry( return cached.registry; } - // Cold or stale: re-read + re-parse. Cache ONLY on success so a malformed - // file never latches — the next request re-attempts and heals. + // Cold or stale: re-read + re-parse. Cache ONLY on success. const registry = await loadMetricRegistry(queriesDir); metricRegistryCache.set(queriesDir, { signature, registry }); return registry; diff --git a/packages/shared/src/schemas/metric-fqn.ts b/packages/shared/src/schemas/metric-fqn.ts index be96c1625..2a9a8647a 100644 --- a/packages/shared/src/schemas/metric-fqn.ts +++ b/packages/shared/src/schemas/metric-fqn.ts @@ -1,9 +1,8 @@ /** - * Unity Catalog object-name grammar - the single source of truth for metric - * view FQN naming validation. + * Unity Catalog object-name grammar + * the single source of truth for metric view FQN (Fully Qualified Name). * - * This module is deliberately **zod-free**. A metric view's `source` FQN is - * validated in two places that must agree: + * A metric view's `source` FQN is validated in two places that must agree: * * 1. The canonical Zod schema (`./metric-source.ts`), which composes the * three-part FQN regex from {@link UC_FQN_PATTERN} for IDE/CI and the @@ -12,18 +11,8 @@ * which imports {@link UC_FQN_PATTERN} as a plain value to validate each * dot-split segment. * - * The type-generator's runtime path must NOT pull the shared Zod schema package - * in (locked dependency-graph ruling - see the comment in - * `packages/appkit/src/type-generator/cache.ts`). Keeping the pattern in this - * zod-free module lets the runtime import the regex without dragging zod into - * its bundle, while still single-sourcing the grammar. * - * -- UC delimited (quoted) object-name rules ------------------------------ - * The metric view FQN is always backtick-quoted before interpolation into SQL - * (see `quoteFqnForSql` in the type-generator), so the **delimited identifier** - * grammar is the one that applies - not the narrower unquoted-identifier rule. - * - * Per the Databricks SQL names reference, a Unity Catalog object name: + * A Unity Catalog object name: * - cannot exceed 255 characters ({@link MAX_UC_OBJECT_NAME_LENGTH}); and * - cannot contain any of these characters: * - period (`.`) @@ -34,43 +23,14 @@ * * Every other character is permitted in a quoted name, including non-ASCII * letters (the docs demonstrate Chinese/Russian/Portuguese names) and hyphens. - * This is intentionally broader than the old hand-rolled allowlist - * (`[a-zA-Z0-9_-]`), which was flagged in PR #433 review (pkosiec: "more - * restrictive than UC"): the goal is to accept what UC accepts as a quoted - * name and reject only what UC rejects. - * - * Verified against the Databricks docs on 2026-06-19: - * https://docs.databricks.com/aws/en/sql/language-manual/sql-ref-names - * (the link cited in the PR #433 review). If the published rules change, - * re-confirm against that page. * - * @note The period is excluded here because it is the FQN segment separator - - * a name containing a literal dot cannot be expressed in the dotted `source` - * string at all. The dotted-source arity (exactly three segments) and the - * 255-char-per-segment cap are enforced structurally by the callers; this - * pattern only encodes the per-segment allowed character set. + * @note the regex is the per-segment character set; structure (3 parts) and length are intentionally left to the callers. */ -/** - * Maximum length, in characters, of a single Unity Catalog object name - * (catalog, schema, or metric view). UC rejects names longer than this. - */ export const MAX_UC_OBJECT_NAME_LENGTH = 255; /** - * Matches a single, non-empty Unity Catalog object name as it may appear in a - * backtick-quoted (delimited) identifier - one segment of a metric view FQN. - * - * Accepts any non-empty run of characters EXCEPT the UC-prohibited set: - * period, space, forward slash, ASCII control characters (U+0000-U+001F), and - * DELETE (U+007F). Length is NOT bounded here - callers enforce - * {@link MAX_UC_OBJECT_NAME_LENGTH} separately so they can emit a precise - * "segment too long" message distinct from a charset violation. - * - * The negated character class encodes the prohibited set as one contiguous - * range plus singletons: U+0000-U+0020 (every ASCII control character plus the - * space, which sits at U+0020 immediately after the control range), U+007F - * (DELETE), `.` (period - also the FQN segment separator), and `/` (slash). + * Matches a single, non-empty Unity Catalog object name as it may appear in a backtick-quoted (delimited) identifier. * * @example * UC_FQN_PATTERN.test("revenue_metrics"); // true @@ -87,20 +47,7 @@ const FQN_SEGMENT_COUNT = 3; /** * Total predicate: is `fqn` a well-formed three-part UC metric view FQN? - * - * Well-formed = exactly three non-empty, dot-separated segments, each a valid - * Unity Catalog object name per {@link UC_FQN_PATTERN}. This is the shared, - * zod-free grammar check reused by every layer that must agree on FQN shape: - * the type-generator's config resolver and describe seam, and the analytics - * runtime's SQL builder. It is the boolean sibling of the composed - * `UC_THREE_PART_FQN_PATTERN` regex in `./metric-source.ts` (which stays a - * regex so zod can emit a JSON-schema `pattern`); both derive their per-segment - * charset from {@link UC_FQN_PATTERN}, so they cannot diverge. - * - * @note Segment length ({@link MAX_UC_OBJECT_NAME_LENGTH}) is NOT checked here — - * an over-long but otherwise legal name is still "valid shape". Callers that - * care about the length cap enforce it separately with their own message. - * + * Well-formed = exactly three non-empty, dot-separated segments, each a valid Unity Catalog object name per {@link UC_FQN_PATTERN}. * @example * isValidFqn("main.analytics.revenue"); // true * isValidFqn("prod-data.analytics.rev"); // true (hyphens are UC-legal) @@ -151,35 +98,14 @@ export function quoteFqnForSql(fqn: string): string { */ const CONTROL_OR_NEWLINE = /\p{Cc}/u; -/** - * Is `name` a column/measure/dimension identifier that {@link quoteIdentifier} - * can safely escape? True for any non-empty string free of control characters - * and newlines. - * - * This is the **column-identifier** grammar — deliberately broader than the - * FQN-segment grammar ({@link UC_FQN_PATTERN}, which also forbids `.` and `/` - * because they are FQN structural characters). A metric view's measure or - * dimension is a single *delimited* column identifier: once backtick-quoted it - * may legally contain dots, slashes, spaces, hyphens, and non-ASCII — anything - * but a control character. The type-generator emits DESCRIBE column names - * verbatim into the generated `measureKeys`/`dimensionKeys` unions, so the - * runtime must accept exactly what can be safely quoted, or a generated name - * would typecheck but fail at runtime. - */ export function isValidColumnName(name: string): boolean { return name.length > 0 && !CONTROL_OR_NEWLINE.test(name); } /** - * Quote a SINGLE identifier (one column/measure/dimension name, or one FQN + * Quote a single identifier (one column/measure/dimension name, or one FQN * segment) as a backtick-delimited identifier for safe SQL interpolation. * - * Unlike {@link quoteFqnForSql}, this does NOT split on `.` — the whole input - * is one identifier, so a column literally named `net.revenue` becomes - * `` `net.revenue` `` (one identifier), not `` `net`.`revenue` `` (two). The - * backtick — the only break-out character — is doubled; control characters and - * newlines have no valid escape and are rejected. - * * @throws If `name` contains a control character or newline. */ export function quoteIdentifier(name: string): string { From 3932e53c76188b2f483ed8326226fe709b0afe91 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Tue, 21 Jul 2026 09:49:01 +0200 Subject: [PATCH 12/18] docs: metric-view runtime endpoint (#484) * docs(appkit): document the metric-view runtime endpoint Signed-off-by: Atila Fassina --- docs/docs/plugins/analytics.md | 211 +++++++++++++++++++++++++++------ 1 file changed, 178 insertions(+), 33 deletions(-) diff --git a/docs/docs/plugins/analytics.md b/docs/docs/plugins/analytics.md index 1ed102107..3696a86b3 100644 --- a/docs/docs/plugins/analytics.md +++ b/docs/docs/plugins/analytics.md @@ -7,6 +7,7 @@ sidebar_position: 3 Enables SQL query execution against Databricks SQL Warehouses. **Key features:** + - File-based SQL queries with automatic type generation - Parameterized queries with type-safe [SQL helpers](../api/appkit/Variable.sql.md) - JSON and Arrow format support @@ -35,7 +36,6 @@ await createApp({ The execution context is determined by the SQL file name, not by the hook call. - ## SQL parameters Use `:paramName` placeholders and optionally annotate parameter types using SQL comments: @@ -56,6 +56,7 @@ Annotate with `INT`, or use `sql.number()` (auto-infers `INT` for values in at the call site. **Supported `-- @param` types** (case-insensitive): + - `STRING`, `BOOLEAN`, `DATE`, `TIMESTAMP`, `BINARY` - `INT`, `BIGINT`, `TINYINT`, `SMALLINT` — bind via `sql.int()` / `sql.bigint()` - `FLOAT`, `DOUBLE` — bind via `sql.float()` / `sql.double()` @@ -103,12 +104,149 @@ The analytics plugin exposes these endpoints (mounted under `/api/analytics`): - `POST /api/analytics/query/:query_key` - `GET /api/analytics/arrow-result/:jobId` +- `POST /api/analytics/metric/:key` — measure a Unity Catalog Metric View (see [Metric views](#metric-views)) ## Format options - `format: "JSON"` (default) returns JSON rows - `format: "ARROW"` returns an Arrow "statement_id" payload over SSE, then the client fetches binary Arrow from `/api/analytics/arrow-result/:jobId` +## Metric views + +`POST /api/analytics/metric/:key` measures a [Unity Catalog Metric View](https://docs.databricks.com/en/metric-views/index.html) that you declared in `config/queries/metric-views.json`. Instead of writing SQL, the caller sends a structured request — which measures to aggregate, which dimensions to group by, and an optional filter — and the plugin builds and runs the `SELECT MEASURE(...) ... GROUP BY ALL` for you against the view. + +The route is **dormant until `metric-views.json` exists**: with no config file, every metric key returns `404`. Declaring the file (and generating types) is covered in [Metric-view types](../development/type-generation.md#metric-view-types); this section documents the runtime endpoint that config activates. + +### Request body + +``` +POST /api/analytics/metric/:key +Content-Type: application/json + +{ + "measures": ["arr", "revenue"], + "dimensions": ["region", "order_date"], + "timeGrain": "month", + "timeDimension": "order_date", + "filter": { "member": "region", "operator": "in", "values": ["EMEA", "APAC"] }, + "limit": 100 +} +``` + +`:key` is a metric key from `metric-views.json`. The body fields: + +| Field | Type | Required | Description | +| --------------- | ---------- | -------- | ------------------------------------------------------------------------------------------------------------ | +| `measures` | `string[]` | yes | Measures to aggregate. At least 1, at most 50. Each becomes `MEASURE() AS `. | +| `dimensions` | `string[]` | no | Dimensions to group by (max 20). Selected verbatim and grouped via `GROUP BY ALL`. | +| `filter` | object | no | Structured predicate tree translated into a parameterized `WHERE` clause (see [Filters](#filters)). | +| `timeGrain` | `string` | no | Bucket a time dimension via `date_trunc('', …)` — e.g. `day`, `month`. Requires `timeDimension`. | +| `timeDimension` | `string` | no | The single dimension `timeGrain` buckets. Must be one of `dimensions`. Required whenever `timeGrain` is set. | +| `limit` | `number` | no | Positive integer row cap (max 100000). | +| `format` | `string` | no | `JSON_ARRAY` (default). `JSON` is accepted as a deprecated alias for it; Arrow formats (`ARROW`, `ARROW_STREAM`) are rejected on this route. | + +Measures and dimensions must be unique across both lists — a name cannot repeat, nor appear as both a measure and a dimension. + +### How the request becomes SQL + +Given a view registered as `catalog.schema.revenue_metrics`, the request above produces (measures and dimensions are sorted for a deterministic SELECT list): + +```sql +SELECT MEASURE(`arr`) AS `arr`, MEASURE(`revenue`) AS `revenue`, + date_trunc('month', `order_date`) AS `order_date`, `region` +FROM `catalog`.`schema`.`revenue_metrics` +WHERE `region` IN (:f_0, :f_1) +GROUP BY ALL +LIMIT 100 +``` + +The metric view's FQN and every measure/dimension identifier are backtick-quoted; filter values are bound as parameters (`:f_0`, `:f_1`, …), never interpolated into the SQL string. + +### Filters + +`filter` is a recursive tree. A leaf is a single predicate: + +```json +{ "member": "region", "operator": "equals", "values": ["EMEA"] } +``` + +Predicates combine with `and` / `or` groups, which can nest: + +```json +{ + "and": [ + { "member": "region", "operator": "in", "values": ["EMEA", "APAC"] }, + { + "or": [ + { "member": "segment", "operator": "equals", "values": ["Enterprise"] }, + { "member": "deal_size", "operator": "gt", "values": [50000] } + ] + } + ] +} +``` + +The operator vocabulary: + +| Operator | SQL | Values | +| --------------------------- | ----------------------- | ------------------ | +| `equals` | `=` | exactly one | +| `notEquals` | `<>` | exactly one | +| `in` | `IN (…)` | one or more | +| `notIn` | `NOT IN (…)` | one or more | +| `gt` / `gte` / `lt` / `lte` | `>` / `>=` / `<` / `<=` | exactly one | +| `contains` | `LIKE :param` | exactly one string | +| `notContains` | `NOT LIKE :param` | exactly one string | +| `set` | `IS NOT NULL` | none | +| `notSet` | `IS NULL` | none | + +For `contains` / `notContains`, the `%…%` wildcards are applied to the *bound parameter value* (`%value%`), not written into the SQL text — so the value is never interpolated, consistent with every other operator. + +An empty `or` group (`{ "or": [] }`) is rejected with `400`; an empty `and` group (`{ "and": [] }`) is accepted and contributes no `WHERE` clause (a no-op). Keep this in mind when building filter trees programmatically. + +Filters are bounded to keep hostile input from exhausting the server: nesting depth ≤ 8, ≤ 100 children per `and` / `or` group, and ≤ 1000 values per predicate. A request that exceeds a cap is rejected with `400`. + +### Executors (cache scope) + +Each entry in `metric-views.json` names the executor the query runs as, which also sets the cache scope. This is fixed by config, not the request: + +| `executor` | Runs as | Cache | +| --------------------------------- | ---------------------------------- | ----------------------- | +| `app_service_principal` (default) | The app service principal | Shared across all users | +| `user` | The requesting user (on-behalf-of) | Per user | + +This mirrors the `.sql` vs `.obo.sql` distinction for [file-based queries](#execution-context). + +### Response + +The response is the same SSE stream as `POST /api/analytics/query/:query_key`. If the SQL warehouse is cold it first emits `warehouse_status` events (see [Warehouse readiness](#warehouse-readiness)), then a single `result` event with the rows as objects: + +```json +{ + "type": "result", + "data": [ + { + "region": "EMEA", + "order_date": "2025-01-01", + "arr": 1200000, + "revenue": 340000 + } + ] +} +``` + +On failure it emits an `error` event instead. + +### Errors and behavior + +| Status | Body | When | +| ------ | ------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | +| `404` | `{ "error": "Metric not found" }` | `:key` is not declared in `metric-views.json` (also the response for every key when the file is absent). | +| `400` | `{ "error": "Invalid metric request body (fields: …)", "code": … }` | The request body fails validation. The message names only the offending field paths, never the submitted values. | +| `503` | `{ "error": "Metric registry not available", "code": "METRIC_REGISTRY_LOAD_FAILED" }` | `metric-views.json` is present but malformed or unreadable. | + +Editing `metric-views.json` is picked up on the next request — no server restart is needed. A previously malformed file that you fix likewise starts working on the next request. + ## Frontend usage ### useAnalyticsQuery @@ -118,15 +256,19 @@ React hook that subscribes to an analytics query over SSE and returns its latest ```ts import { useAnalyticsQuery } from "@databricks/appkit-ui/react"; -const { data, loading, error } = useAnalyticsQuery(queryKey, parameters, options); +const { data, loading, error } = useAnalyticsQuery( + queryKey, + parameters, + options, +); ``` **Return type:** ```ts { - data: T | null; // query result (typed array for JSON, TypedArrowTable for ARROW) - loading: boolean; // true while the query is executing + data: T | null; // query result (typed array for JSON, TypedArrowTable for ARROW) + loading: boolean; // true while the query is executing error: string | null; // error message, or null on success warehouseStatus: WarehouseStatus | null; // see "Warehouse readiness" below } @@ -134,11 +276,11 @@ const { data, loading, error } = useAnalyticsQuery(queryKey, parameters, options **Options:** -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `format` | `"JSON" \| "ARROW"` | `"JSON"` | Response format | -| `maxParametersSize` | `number` | `102400` | Max serialized parameters size in bytes | -| `autoStart` | `boolean` | `true` | Start query on mount | +| Option | Type | Default | Description | +| ------------------- | ------------------- | -------- | --------------------------------------- | +| `format` | `"JSON" \| "ARROW"` | `"JSON"` | Response format | +| `maxParametersSize` | `number` | `102400` | Max serialized parameters size in bytes | +| `autoStart` | `boolean` | `true` | Start query on mount | ### Warehouse readiness @@ -154,8 +296,10 @@ This means a cold start no longer freezes the UI on a stalled spinner. Render th import { useAnalyticsQuery } from "@databricks/appkit-ui/react"; function SpendTable() { - const { data, loading, error, warehouseStatus } = - useAnalyticsQuery("spend_summary", params); + const { data, loading, error, warehouseStatus } = useAnalyticsQuery( + "spend_summary", + params, + ); if (warehouseStatus && warehouseStatus.state !== "RUNNING") { return
Warehouse is {warehouseStatus.state.toLowerCase()}…
; @@ -197,10 +341,7 @@ export function AppShell({ children }) { If you already render your own `` for unrelated app toasts, drop the indicator and call `useResourceStatusToaster()` instead so resource-status toasts share that single Toaster: ```tsx -import { - useResourceStatusToaster, - Toaster, -} from "@databricks/appkit-ui/react"; +import { useResourceStatusToaster, Toaster } from "@databricks/appkit-ui/react"; function App() { useResourceStatusToaster(); @@ -219,7 +360,8 @@ For a fully custom toast body, pass `render` (rendered through `toast.custom`): (
- {agg.worst?.kind} {agg.worst?.state.toLowerCase()} ({agg.activeCount} waiting) + {agg.worst?.kind} {agg.worst?.state.toLowerCase()} ({agg.activeCount}{" "} + waiting)
)} /> @@ -232,8 +374,7 @@ To override copy for a specific kind without rewriting the whole UI, pass `rende renderers={{ warehouse: { title: () => "Spinning up your data", - description: (_s, agg) => - `${agg.affectedLabels.length} chart(s) waiting`, + description: (_s, agg) => `${agg.affectedLabels.length} chart(s) waiting`, }, }} /> @@ -263,11 +404,9 @@ import { useEffect, useId } from "react"; function useLakebaseReadiness() { const id = useId(); - const { publish, unpublish } = useResourceStatusPublisher( - id, - "lakebase", - { kindHint: "lakebase" }, - ); + const { publish, unpublish } = useResourceStatusPublisher(id, "lakebase", { + kindHint: "lakebase", + }); useEffect(() => { publish({ @@ -283,10 +422,10 @@ function useLakebaseReadiness() { **Server config (in `analytics({...})`):** -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `warehouseStartupTimeoutMs` | `number` | `300000` (5 min) | Maximum time to wait for the warehouse to reach `RUNNING` before failing the request | -| `autoStartWarehouse` | `boolean` | `true` | When `true`, a `STOPPED` warehouse is auto-started on the first request. Set to `false` for cost-controlled deployments where billable warehouse starts must not be triggered by user requests; in that case `STOPPED` surfaces as a `ConfigurationError` | +| Option | Type | Default | Description | +| --------------------------- | --------- | ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `warehouseStartupTimeoutMs` | `number` | `300000` (5 min) | Maximum time to wait for the warehouse to reach `RUNNING` before failing the request | +| `autoStartWarehouse` | `boolean` | `true` | When `true`, a `STOPPED` warehouse is auto-started on the first request. Set to `false` for cost-controlled deployments where billable warehouse starts must not be triggered by user requests; in that case `STOPPED` surfaces as a `ConfigurationError` | **Example with loading/error/empty handling:** @@ -296,21 +435,27 @@ import { sql } from "@databricks/appkit-ui/js"; import { Skeleton } from "@databricks/appkit-ui"; function SpendTable() { - const params = useMemo(() => ({ - startDate: sql.date("2025-01-01"), - endDate: sql.date("2025-12-31"), - }), []); + const params = useMemo( + () => ({ + startDate: sql.date("2025-01-01"), + endDate: sql.date("2025-12-31"), + }), + [], + ); const { data, loading, error } = useAnalyticsQuery("spend_summary", params); if (loading) return ; if (error) return
{error}
; - if (!data?.length) return
No results
; + if (!data?.length) + return
No results
; return (
    {data.map((row) => ( -
  • {row.name}: ${row.cost_usd}
    • +
  • + {row.name}: ${row.cost_usd} +
    • ))}
); From af32812aef35b59eb0c8f8096cbe832294528cc5 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Tue, 21 Jul 2026 16:32:40 +0200 Subject: [PATCH 13/18] fix(analytics): reject empty and-group in metric filter validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An empty `and` group contributes no constraint and renders to no WHERE clause — identical SQL to omitting `filter` entirely — but it canonicalizes to a distinct cache key (`and()` vs `_`), needlessly splitting the cache across semantically identical requests. Reject empty groups of either kind so request shape maps one-to-one to cache key. Addresses GitHub Copilot review finding on PR #474. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- docs/docs/plugins/analytics.md | 2 +- packages/appkit/src/plugins/analytics/mv/schemas.ts | 12 +++++++++--- .../src/plugins/analytics/tests/metric.test.ts | 13 ++++++++++--- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/docs/docs/plugins/analytics.md b/docs/docs/plugins/analytics.md index 3696a86b3..0458fedff 100644 --- a/docs/docs/plugins/analytics.md +++ b/docs/docs/plugins/analytics.md @@ -202,7 +202,7 @@ The operator vocabulary: For `contains` / `notContains`, the `%…%` wildcards are applied to the *bound parameter value* (`%value%`), not written into the SQL text — so the value is never interpolated, consistent with every other operator. -An empty `or` group (`{ "or": [] }`) is rejected with `400`; an empty `and` group (`{ "and": [] }`) is accepted and contributes no `WHERE` clause (a no-op). Keep this in mind when building filter trees programmatically. +Empty groups of either kind (`{ "or": [] }`, `{ "and": [] }`) are rejected with `400` — every `and` / `or` group must contain at least one predicate. To send no filter, omit the `filter` field entirely rather than passing an empty group. Filters are bounded to keep hostile input from exhausting the server: nesting depth ≤ 8, ≤ 100 children per `and` / `or` group, and ≤ 1000 values per predicate. A request that exceeds a cap is rejected with `400`. diff --git a/packages/appkit/src/plugins/analytics/mv/schemas.ts b/packages/appkit/src/plugins/analytics/mv/schemas.ts index 0f00dcad5..92fbed0f0 100644 --- a/packages/appkit/src/plugins/analytics/mv/schemas.ts +++ b/packages/appkit/src/plugins/analytics/mv/schemas.ts @@ -225,11 +225,17 @@ function validateFilterTree( return; } - if (groupKey === "or" && children.length === 0) { + if (children.length === 0) { + // Reject empty groups of either kind. An empty `or` is vacuously false; + // an empty `and` contributes no constraint and renders to no WHERE + // clause — identical SQL to omitting `filter` entirely, but it would + // canonicalize to a distinct cache key (`and()` vs `_`), needlessly + // splitting the cache across semantically identical requests. Requiring + // at least one child keeps request shape ↔ cache key one-to-one. ctx.addIssue({ code: z.ZodIssueCode.custom, - path: [...path, "or"], - message: "filter 'or' group must contain at least one predicate", + path: [...path, groupKey], + message: `filter '${groupKey}' group must contain at least one predicate`, }); return; } diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index 45ed380fe..fc190f848 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -1191,7 +1191,10 @@ describe("metric — filter translator", () => { expect(where).toContain(" AND "); }); - test("empty `and: []` group emits no WHERE clause", () => { + test("empty `and: []` group emits no WHERE clause (defense in depth past the validator)", () => { + // The validator rejects `and: []` (see cardinality tests), but if a + // bypass ever reaches the renderer, an empty conjunction must drop to no + // constraint — never emit a malformed WHERE. const { statement, parameters } = buildMetricSql(registration, { measures: ["arr"], filter: { and: [] }, @@ -1477,13 +1480,17 @@ describe("metric — filter translator", () => { ).toThrowError(/fields:.*filter\.or/); }); - test("accepts empty `and` group (no constraint contributed)", () => { + test("rejects empty `and` group (renders to no WHERE, would split the cache)", () => { + // An empty `and` contributes no constraint and renders to no WHERE + // clause — identical SQL to omitting `filter` — but canonicalizes to a + // distinct cache key (`and()` vs `_`). Reject it so request shape and + // cache key stay one-to-one. expect(() => validateMetricRequest({ measures: ["arr"], filter: { and: [] }, }), - ).not.toThrow(); + ).toThrowError(/fields:.*filter\.and/); }); test("rejects an unknown operator at depth (nested filter)", () => { From 5f650399910795a88331df7e9e12f5af3223606f Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Tue, 21 Jul 2026 17:06:18 +0200 Subject: [PATCH 14/18] refactor(analytics): route metric-view config reads through AppManager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The metric-view runtime read config/queries/metric-views.json via a bespoke fs loader that ran parallel to AppManager, the gateway the analytics plugin already uses for its sibling .sql query files. Centralize the read through AppManager so both file types share one dev-aware read path and one source of truth for the queries directory. - AppManager gains readConfigFile(fileName, req?, devFileReader?): a narrow, domain-agnostic single-file read that applies the existing traversal guard and switches dev-tunnel vs direct-fs like getAppQuery. Returns null only for a genuine not-found and throws on any other error, so callers keep the dormant (404) vs unreadable/malformed (503, self-heals) distinction. Adds isDevRequest(req) and a read-only queriesDir getter; the dir is now overridable only via a test-only constructor arg. - The metric registry reads through AppManager but keeps ownership of freshness (fs.stat) and parse+cache. Production behavior is unchanged (stat-signature cache preserved). A ?dev request bypasses stat+cache and re-reads every request, so a developer's local metric-views.json edits now hot-reload over the dev-remote tunnel the same way .sql edits do — the first dev-remote support for metric views. - Delete the misleading test-only IAnalyticsConfig.queriesDir field, the plugin's _queriesDir, and the duplicated mv/constants.ts QUERIES_DIR; the route now reads through the plugin's shared this.app. Trim an orphaned FQN-check comment in the typegen config reader. Resolves the PR #474 review: the AppManager-centralization thread (five comments) and the orphaned-comment nit; the queriesDir-JSDoc finding is resolved by deleting the field. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- packages/appkit/src/app/index.ts | 113 +++++++- .../src/app/tests/read-config-file.test.ts | 170 ++++++++++++ .../appkit/src/plugins/analytics/analytics.ts | 15 +- .../src/plugins/analytics/mv/constants.ts | 8 - .../appkit/src/plugins/analytics/mv/index.ts | 1 - .../src/plugins/analytics/mv/registry.ts | 90 ++++--- .../plugins/analytics/tests/metric.test.ts | 244 +++++++++++++----- .../appkit/src/plugins/analytics/types.ts | 11 - .../src/type-generator/mv-registry/config.ts | 10 +- 9 files changed, 527 insertions(+), 135 deletions(-) create mode 100644 packages/appkit/src/app/tests/read-config-file.test.ts diff --git a/packages/appkit/src/app/index.ts b/packages/appkit/src/app/index.ts index 1b8428f1e..01ac06c83 100644 --- a/packages/appkit/src/app/index.ts +++ b/packages/appkit/src/app/index.ts @@ -28,7 +28,44 @@ interface FileSystemAdapter { } export class AppManager { - private readonly queriesDir = path.resolve(process.cwd(), "config/queries"); + private readonly _queriesDir: string; + + /** + * @param queriesDir - Absolute path to the `config/queries` directory this + * manager reads from. Defaults to `/config/queries`. + * + * @internal The override exists FOR TESTS ONLY, so a spec can point + * AppManager at a temp directory instead of `process.cwd()`. Production and + * every runtime call site use `new AppManager()` with no argument; nothing + * passes this path outside tests. + */ + constructor( + queriesDir: string = path.resolve(process.cwd(), "config/queries"), + ) { + this._queriesDir = queriesDir; + } + + /** + * Absolute path to the `config/queries` directory this manager reads from. + * + * Read-only accessor over the private field so a caller (e.g. the metric + * registry) can `fs.stat` the directory or build a cache key from it without + * reaching into internals or re-deriving `process.cwd()/config/queries`. + */ + get queriesDir(): string { + return this._queriesDir; + } + + /** + * Whether `req` is a dev-tunnel (`?dev`) request. + * + * Public so a caller can pick its dev-vs-production branch with the exact + * sniff {@link createFsAdapter} uses, instead of duplicating + * `req?.query?.dev !== undefined`. + */ + isDevRequest(req?: RequestLike): boolean { + return req?.query?.dev !== undefined; + } /** * Validates that a file path is within the queries directory @@ -53,7 +90,7 @@ export class AppManager { req?: RequestLike, devFileReader?: DevFileReader, ): FileSystemAdapter { - const isDevMode = req?.query?.dev !== undefined; + const isDevMode = this.isDevRequest(req); if (isDevMode && devFileReader && req) { // Dev mode: use WebSocket tunnel to read from local filesystem @@ -155,6 +192,76 @@ export class AppManager { return null; } } + + /** + * Read a single config file from the queries directory, dev-tunnel-aware. + * + * A narrow, domain-agnostic seam: it knows nothing about JSON, Zod, or metric + * views and returns the raw file contents as a string. Uses the same + * dev-vs-production detection as {@link getAppQuery} ({@link isDevRequest} → + * dev tunnel via `devFileReader`; else direct `fs`) and applies the same + * {@link validatePath} traversal guard. + * + * The not-found-vs-error distinction is load-bearing for callers that treat + * "absent" as dormant (return `null`) but any other failure as fatal (503): + * + * - Returns `null` for a genuine not-found — `ENOENT` in production; the dev + * tunnel's not-found signal in dev (best-effort message match, since the + * tunnel surfaces a plain `Error` with no `errno` code). + * - Returns `null` for a rejected path-traversal attempt (never reads). + * - **Throws on any other error** (EACCES / EIO / tunnel / parse-of-listing / + * …). These are deliberately NOT swallowed to `null`. + * + * @param fileName - File name (or relative path) within the queries directory. + * @param req - Optional request object to detect dev mode. + * @param devFileReader - Optional DevFileReader to read via the WebSocket tunnel. + * @returns The raw file contents, or `null` when the file is absent / the path + * is rejected. + */ + async readConfigFile( + fileName: string, + req?: RequestLike, + devFileReader?: DevFileReader, + ): Promise { + // Traversal guard: refuse to read outside the queries directory. + const resolvedPath = this.validatePath(fileName); + if (!resolvedPath) { + return null; + } + + const fsAdapter = this.createFsAdapter(req, devFileReader); + + try { + return await fsAdapter.readFile(resolvedPath); + } catch (error) { + if (this.isNotFoundError(error, req)) { + return null; + } + // Any other error (permission / IO / tunnel) is fatal: propagate so the + // caller can distinguish it from a dormant (absent) config. + throw error; + } + } + + /** + * Classify a read failure as a genuine not-found (→ `null`) vs. any other + * error (→ rethrow). + * + * Production reads reject with a Node `ErrnoException` whose `code` is + * `"ENOENT"`. The dev tunnel surfaces a plain `Error` with no `errno` code, + * so in dev we fall back to a best-effort message match on the not-found + * signal — acceptable per the config-read contract. + */ + private isNotFoundError(error: unknown, req?: RequestLike): boolean { + if ((error as NodeJS.ErrnoException)?.code === "ENOENT") { + return true; + } + if (this.isDevRequest(req)) { + const message = (error as Error)?.message ?? ""; + return /ENOENT|no such file/i.test(message); + } + return false; + } } -export type { DevFileReader }; +export type { DevFileReader, RequestLike }; diff --git a/packages/appkit/src/app/tests/read-config-file.test.ts b/packages/appkit/src/app/tests/read-config-file.test.ts new file mode 100644 index 000000000..9f68e00a9 --- /dev/null +++ b/packages/appkit/src/app/tests/read-config-file.test.ts @@ -0,0 +1,170 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import type { DevFileReader } from "../index"; +import { AppManager } from "../index"; + +// NOTE: unlike the sibling `app.test.ts`, this spec deliberately does NOT mock +// `node:fs/promises` — `readConfigFile` is exercised against real temp files so +// the not-found-vs-throw distinction is driven by genuine errno codes. + +describe("AppManager.readConfigFile", () => { + let tmpDir: string; + let appManager: AppManager; + + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "appmgr-config-")); + appManager = new AppManager(tmpDir); + }); + + afterEach(async () => { + vi.restoreAllMocks(); + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + describe("dir override + getter", () => { + test("queriesDir getter exposes the overridden directory", () => { + expect(appManager.queriesDir).toBe(tmpDir); + }); + + test("defaults to /config/queries when no override is given", () => { + const defaultManager = new AppManager(); + expect(defaultManager.queriesDir).toBe( + path.resolve(process.cwd(), "config/queries"), + ); + }); + }); + + describe("isDevRequest predicate", () => { + test("returns true when ?dev is present", () => { + expect(appManager.isDevRequest({ query: { dev: "" }, headers: {} })).toBe( + true, + ); + }); + + test("returns false without ?dev and for undefined req", () => { + expect(appManager.isDevRequest({ query: {}, headers: {} })).toBe(false); + expect(appManager.isDevRequest()).toBe(false); + }); + }); + + describe("production mode (direct fs)", () => { + test("returns file contents for an existing file", async () => { + await fs.writeFile( + path.join(tmpDir, "metric-views.json"), + '{"hello":"world"}', + "utf8", + ); + + const result = await appManager.readConfigFile("metric-views.json"); + expect(result).toBe('{"hello":"world"}'); + }); + + test("returns null for a genuine not-found (ENOENT)", async () => { + const result = await appManager.readConfigFile("does-not-exist.json"); + expect(result).toBeNull(); + }); + + test("throws (does NOT return null) for a non-ENOENT error", async () => { + // Reading a directory as a file rejects with EISDIR — a real, non-ENOENT + // error that must propagate rather than be swallowed to null. + await fs.mkdir(path.join(tmpDir, "a-directory")); + + await expect(appManager.readConfigFile("a-directory")).rejects.toThrow(); + }); + + test("throws for a simulated EACCES (permission) error", async () => { + const err = Object.assign(new Error("permission denied"), { + code: "EACCES", + }); + vi.spyOn(fs, "readFile").mockRejectedValueOnce(err); + + await expect( + appManager.readConfigFile("metric-views.json"), + ).rejects.toThrow("permission denied"); + }); + + test("reads via direct fs (not the dev reader) without ?dev", async () => { + await fs.writeFile( + path.join(tmpDir, "metric-views.json"), + "prod-contents", + "utf8", + ); + const devFileReader: DevFileReader = { + readdir: vi.fn(), + readFile: vi.fn(), + }; + + const result = await appManager.readConfigFile( + "metric-views.json", + { query: {}, headers: {} }, + devFileReader, + ); + + expect(result).toBe("prod-contents"); + expect(devFileReader.readFile).not.toHaveBeenCalled(); + }); + }); + + describe("path traversal protection", () => { + test("returns null and does not read outside the queries dir", async () => { + const readSpy = vi.spyOn(fs, "readFile"); + const result = await appManager.readConfigFile("../../etc/passwd"); + + expect(result).toBeNull(); + expect(readSpy).not.toHaveBeenCalled(); + }); + }); + + describe("dev mode (WebSocket tunnel)", () => { + const devReq = { query: { dev: "true" }, headers: {} }; + + test("reads via devFileReader in dev mode", async () => { + const devFileReader: DevFileReader = { + readdir: vi.fn(), + readFile: vi.fn().mockResolvedValue("dev-contents"), + }; + + const result = await appManager.readConfigFile( + "metric-views.json", + devReq, + devFileReader, + ); + + expect(result).toBe("dev-contents"); + expect(devFileReader.readFile).toHaveBeenCalledWith( + expect.stringContaining("metric-views.json"), + devReq, + ); + }); + + test("returns null for the dev tunnel's not-found signal (best-effort)", async () => { + const devFileReader: DevFileReader = { + readdir: vi.fn(), + readFile: vi + .fn() + .mockRejectedValue(new Error("ENOENT: no such file or directory")), + }; + + const result = await appManager.readConfigFile( + "metric-views.json", + devReq, + devFileReader, + ); + + expect(result).toBeNull(); + }); + + test("throws for a non-not-found dev tunnel error", async () => { + const devFileReader: DevFileReader = { + readdir: vi.fn(), + readFile: vi.fn().mockRejectedValue(new Error("tunnel disconnected")), + }; + + await expect( + appManager.readConfigFile("metric-views.json", devReq, devFileReader), + ).rejects.toThrow("tunnel disconnected"); + }); + }); +}); diff --git a/packages/appkit/src/plugins/analytics/analytics.ts b/packages/appkit/src/plugins/analytics/analytics.ts index ca9358e97..7f96d8949 100644 --- a/packages/appkit/src/plugins/analytics/analytics.ts +++ b/packages/appkit/src/plugins/analytics/analytics.ts @@ -32,7 +32,6 @@ import manifest from "./manifest.json"; import { buildMetricSql, composeMetricCacheKey, - QUERIES_DIR as DEFAULT_QUERIES_DIR, deriveMetricExecutorKey, getMetricRegistry, validateMetricRequest, @@ -125,18 +124,10 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { */ private _arrowCapability = new Map(); - /** - * Directory the metric-view registry is read from (holds - * `metric-views.json`). - * @default `config/queries/` (under the process cwd) - */ - private readonly _queriesDir: string; - constructor(config: IAnalyticsConfig) { super(config); this.config = config; this.queryProcessor = new QueryProcessor(); - this._queriesDir = config.queriesDir ?? DEFAULT_QUERIES_DIR; this.SQLClient = new SQLWarehouseConnector({ timeout: config.timeout, @@ -488,9 +479,13 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { } // Resolve the registry from disk (cached + mtime-revalidated by `getMetricRegistry`). + // Reads through the plugin's shared `this.app` (the base `Plugin`'s + // `AppManager`, rooted at `config/queries/` under the process cwd), so this + // is dev-tunnel aware and shares the module-level mtime cache keyed by + // `app.queriesDir`. let registry: Record; try { - registry = await getMetricRegistry(this._queriesDir); + registry = await getMetricRegistry(this.app, req, this.devFileReader); } catch (err) { const reason = err instanceof Error ? err.message : String(err); logger.warn(req, "Failed to load metric registry: %s", reason); diff --git a/packages/appkit/src/plugins/analytics/mv/constants.ts b/packages/appkit/src/plugins/analytics/mv/constants.ts index f159a891c..abd210db7 100644 --- a/packages/appkit/src/plugins/analytics/mv/constants.ts +++ b/packages/appkit/src/plugins/analytics/mv/constants.ts @@ -1,13 +1,5 @@ -import path from "node:path"; import type { MetricFilterOperatorName, MetricLane } from "../types"; -/** - * Default queries directory. Mirrors `AppManager`'s - * `path.resolve(process.cwd(), "config/queries")` so dev mode and production - * share a single source of truth for where metric config lives. Exported so - * `AnalyticsPlugin` can default `config.queriesDir` to the same path. - */ -export const QUERIES_DIR = path.resolve(process.cwd(), "config/queries"); export const METRIC_CONFIG_FILE = "metric-views.json"; /** diff --git a/packages/appkit/src/plugins/analytics/mv/index.ts b/packages/appkit/src/plugins/analytics/mv/index.ts index de5eec8ff..ea1d9da18 100644 --- a/packages/appkit/src/plugins/analytics/mv/index.ts +++ b/packages/appkit/src/plugins/analytics/mv/index.ts @@ -1,5 +1,4 @@ export { composeMetricCacheKey, deriveMetricExecutorKey } from "./cache"; -export { QUERIES_DIR } from "./constants"; export { buildMetricSql } from "./formatters"; export { __resetMetricRegistryCache, diff --git a/packages/appkit/src/plugins/analytics/mv/registry.ts b/packages/appkit/src/plugins/analytics/mv/registry.ts index 66a4d032d..b1caf119b 100644 --- a/packages/appkit/src/plugins/analytics/mv/registry.ts +++ b/packages/appkit/src/plugins/analytics/mv/registry.ts @@ -5,9 +5,10 @@ import path from "node:path"; // type-generator's runtime, which pulls the zod-free `metric-fqn.ts` from the // same tree) so the runtime and the generated JSON schema validate identically. import { metricSourceSchema } from "../../../../../shared/src/schemas/metric-source"; +import type { AppManager, DevFileReader, RequestLike } from "../../../app"; import { createLogger } from "../../../logging/logger"; import type { MetricRegistration } from "../types"; -import { laneFromExecutor, METRIC_CONFIG_FILE, QUERIES_DIR } from "./constants"; +import { laneFromExecutor, METRIC_CONFIG_FILE } from "./constants"; import type { RegistryCacheSignature } from "./types"; const logger = createLogger("analytics:metric-views"); @@ -18,22 +19,32 @@ const logger = createLogger("analytics:metric-views"); * Async and stateless — registration is a pure config parse with no warehouse * round-trip, no `DESCRIBE`, and no build-time metadata bundle. * - * Absent file -> empty registry. - * Malformed file -> 503. + * The file is read **through {@link AppManager.readConfigFile}** rather than + * `node:fs` directly, so this path is dev-tunnel-aware (a `?dev` request reads + * the developer's local file over the WebSocket tunnel) and inherits the + * traversal guard. In production that's a plain `fs.readFile` under the hood, + * so the semantics below are unchanged. + * + * Absent file (or a rejected traversal path) -> empty registry (`null` from + * `readConfigFile`). + * Malformed file -> 503 (throws). + * + * @param app - The {@link AppManager} that resolves + reads the config file. + * @param req - Optional request object, used to detect dev mode. + * @param devFileReader - Optional dev tunnel reader. */ export async function loadMetricRegistry( - queriesDir: string = QUERIES_DIR, + app: AppManager, + req?: RequestLike, + devFileReader?: DevFileReader, ): Promise> { - const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); + const metricPath = path.join(app.queriesDir, METRIC_CONFIG_FILE); - let raw: string; - try { - raw = await fs.readFile(metricPath, "utf8"); - } catch (err) { - if ((err as NodeJS.ErrnoException).code === "ENOENT") { - return Object.create(null); - } - throw err; + const raw = await app.readConfigFile(METRIC_CONFIG_FILE, req, devFileReader); + if (raw === null) { + // Absent file (ENOENT in prod / dev-tunnel not-found) or a rejected + // traversal path → dormant. Same as the old ENOENT branch. + return Object.create(null); } let parsed: unknown; @@ -100,34 +111,55 @@ export function __resetMetricRegistryCache(): void { } /** - * Resolve the metric registry for `queriesDir`, re-reading + re-parsing only - * when `metric-views.json` has changed since the cached copy. + * Resolve the metric registry for `app`, re-reading + re-parsing only when + * `metric-views.json` has changed since the cached copy. + * + * Two branches, mirroring the sibling `.sql` query path: * - * Behaves like the sibling `.sql` query path (which re-reads per request) but - * cheaper: the steady-state cost is a single async `stat`, and the read + - * `JSON.parse` + zod validation are skipped when the file's - * `(ctimeMs, mtimeMs, size)` signature is unchanged. This delivers the agreed - * semantics without the old permanent memo: + * - **Dev** ({@link AppManager.isDevRequest} true): the file lives on the + * developer's machine and is served over the WebSocket tunnel, which exposes + * no `stat`. The whole point of `?dev` is to reflect the developer's local + * edits immediately, so this branch does NOT `stat`, does NOT cache, and + * simply re-reads via {@link loadMetricRegistry} every request — exactly like + * the `.sql` tunnel path. + * - **Production** (not a dev request): behavior identical to the pre-refactor + * loader. The steady-state cost is a single async `stat`, and the read + + * `JSON.parse` + zod validation are skipped when the file's + * `(ctimeMs, mtimeMs, size)` signature is unchanged. This delivers the agreed + * semantics without a permanent memo: * - * - **Hot-reload** — editing a working config bumps `mtimeMs`, so the next - * request re-parses and serves the new registry with no server restart. - * - **Self-heal** — a load failure is NOT cached (we only populate the cache - * on a successful parse), so a fixed config is picked up on the next - * request instead of latching a 503 forever. - * - **Dormant** — an absent file `stat`s as `ENOENT` → empty registry; adding - * the file later is picked up on the next request. + * - **Hot-reload** — editing a working config bumps `mtimeMs`, so the next + * request re-parses and serves the new registry with no server restart. + * - **Self-heal** — a load failure is NOT cached (we only populate the + * cache on a successful parse), so a fixed config is picked up on the + * next request instead of latching a 503 forever. + * - **Dormant** — an absent file `stat`s as `ENOENT` → empty registry; + * adding the file later is picked up on the next request. * * Concurrency: two simultaneous cold requests may both `stat`+read before * either populates the cache. That is a harmless redundant read of the same * file (the sibling `.sql` path does not single-flight either), so no lock is * taken. * + * @param app - The {@link AppManager} that resolves + reads the config file. + * The cache is keyed by `app.queriesDir`. + * @param req - Optional request object, used to pick the dev-vs-production branch. + * @param devFileReader - Optional dev tunnel reader (dev branch only). * @throws Propagates {@link loadMetricRegistry}'s throw on a malformed file so * the route can surface a 503; the cache is left untouched on failure. */ export async function getMetricRegistry( - queriesDir: string = QUERIES_DIR, + app: AppManager, + req?: RequestLike, + devFileReader?: DevFileReader, ): Promise> { + // Dev branch: never stat (the tunnel has none), never cache — re-read every + // request so the developer's local edits are reflected immediately. + if (app.isDevRequest(req)) { + return loadMetricRegistry(app, req, devFileReader); + } + + const queriesDir = app.queriesDir; const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); let signature: RegistryCacheSignature | null; @@ -168,7 +200,7 @@ export async function getMetricRegistry( } // Cold or stale: re-read + re-parse. Cache ONLY on success. - const registry = await loadMetricRegistry(queriesDir); + const registry = await loadMetricRegistry(app, req, devFileReader); metricRegistryCache.set(queriesDir, { signature, registry }); return registry; } diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index fc190f848..fe5226029 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -1,4 +1,11 @@ -import { mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { @@ -9,6 +16,7 @@ import { setupDatabricksEnv, } from "@tools/test-helpers"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { AppManager, type DevFileReader } from "../../../app"; import { ServiceContext } from "../../../context/service-context"; import { AuthenticationError } from "../../../errors"; import { AnalyticsPlugin } from "../analytics"; @@ -63,16 +71,33 @@ vi.mock("../../../cache", () => ({ })); // Temp dirs created by `registryDir` / `writeRegistry`, cleaned up after each -// test. Using real files (via the test-only `queriesDir` config) exercises the -// actual stat → read → cache path in `getMetricRegistry` rather than poking -// private plugin state. +// test. Using real files (pointing the plugin's `AppManager` at the dir, see +// `pluginForDir`) exercises the actual stat → read → cache path in +// `getMetricRegistry` rather than poking private plugin state. const tempRegistryDirs: string[] = []; +/** + * Construct an `AnalyticsPlugin` whose metric-registry gateway is pointed at + * `dir`. + * + * The plugin reads the registry through the base `Plugin`'s shared `this.app` + * (an `AppManager` rooted at `config/queries/` under the cwd). There is no + * config field to relocate that directory, so a test points the plugin at a + * fixture dir by overriding the `AppManager` with one constructed over `dir`. + * `app` is `protected` on the base `Plugin`, hence the deliberate test-only + * cast — the single seam every route-handler test threads through. + */ +function pluginForDir(config: IAnalyticsConfig, dir: string): AnalyticsPlugin { + const plugin = new AnalyticsPlugin(config); + (plugin as any).app = new AppManager(dir); + return plugin; +} + /** * Write a `metric-views.json` into a fresh temp dir and return the dir, for use - * as `new AnalyticsPlugin({ ...config, queriesDir })`. Accepts the internal - * `MetricRegistration` shape (matching the old `setRegistry` helper) and maps - * each entry's `lane` back to the config's `executor` field. + * with `pluginForDir(config, dir)`. Accepts the internal `MetricRegistration` + * shape (matching the old `setRegistry` helper) and maps each entry's `lane` + * back to the config's `executor` field. */ function registryDir(registry: Record): string { const dir = mkdtempSync(path.join(tmpdir(), "mv-route-")); @@ -447,16 +472,16 @@ describe("analytics metric route (Phase 1)", () => { // byte-identical to the /query route's JSON SSE path. describe("_handleMetricRoute SSE envelope", () => { test("streams warehouse_status then a result message with aliased rows", async () => { - const plugin = new AnalyticsPlugin({ - ...config, - queriesDir: registryDir({ + const plugin = pluginForDir( + config, + registryDir({ revenue: { key: "revenue", source: "cat.sch.revenue_metrics", lane: "sp", }, }), - }); + ); const { router, getHandler } = createMockRouter(); const executeMock = vi.fn().mockResolvedValue({ @@ -494,16 +519,16 @@ describe("analytics metric route (Phase 1)", () => { }); test("emits warehouse_status before result for a STARTING warehouse", async () => { - const plugin = new AnalyticsPlugin({ - ...config, - queriesDir: registryDir({ + const plugin = pluginForDir( + config, + registryDir({ revenue: { key: "revenue", source: "cat.sch.revenue_metrics", lane: "sp", }, }), - }); + ); const { router, getHandler } = createMockRouter(); const executeMock = vi.fn().mockResolvedValue({ @@ -547,16 +572,16 @@ describe("analytics metric route (Phase 1)", () => { }); test("returns 400 when the body fails structural validation", async () => { - const plugin = new AnalyticsPlugin({ - ...config, - queriesDir: registryDir({ + const plugin = pluginForDir( + config, + registryDir({ revenue: { key: "revenue", source: "cat.sch.revenue_metrics", lane: "sp", }, }), - }); + ); const { router, getHandler } = createMockRouter(); plugin.injectRoutes(router); @@ -576,16 +601,16 @@ describe("analytics metric route (Phase 1)", () => { // ── 503-vs-404 latching + dormancy. describe("registry latching and dormancy", () => { test("unknown key against a valid registry → 404 (generic body)", async () => { - const plugin = new AnalyticsPlugin({ - ...config, - queriesDir: registryDir({ + const plugin = pluginForDir( + config, + registryDir({ revenue: { key: "revenue", source: "cat.sch.revenue_metrics", lane: "sp", }, }), - }); + ); const { router, getHandler } = createMockRouter(); plugin.injectRoutes(router); @@ -605,16 +630,16 @@ describe("analytics metric route (Phase 1)", () => { test.each(["__proto__", "constructor", "toString", "hasOwnProperty"])( "inherited Object.prototype key %j → 404, no execution (own-property lookup)", async (dangerousKey) => { - const plugin = new AnalyticsPlugin({ - ...config, - queriesDir: registryDir({ + const plugin = pluginForDir( + config, + registryDir({ revenue: { key: "revenue", source: "cat.sch.revenue_metrics", lane: "sp", }, }), - }); + ); const { router, getHandler } = createMockRouter(); // A real (own) entry so the registry is populated but does NOT contain // the dangerous key. Without the own-property guard, `registry[key]` @@ -647,7 +672,7 @@ describe("analytics metric route (Phase 1)", () => { // A real malformed config on disk (invalid JSON) → the loader throws → // the route surfaces a 503, exercising the actual load path. writeRegistry(dir, "{ not valid json"); - const plugin = new AnalyticsPlugin({ ...config, queriesDir: dir }); + const plugin = pluginForDir(config, dir); const { router, getHandler } = createMockRouter(); plugin.injectRoutes(router); @@ -671,7 +696,7 @@ describe("analytics metric route (Phase 1)", () => { const dir = mkdtempSync(path.join(tmpdir(), "mv-route-")); tempRegistryDirs.push(dir); writeRegistry(dir, "{ not valid json"); - const plugin = new AnalyticsPlugin({ ...config, queriesDir: dir }); + const plugin = pluginForDir(config, dir); const { router, getHandler } = createMockRouter(); const executeMock = vi .fn() @@ -723,7 +748,7 @@ describe("analytics metric route (Phase 1)", () => { lane: "sp", }, }); - const plugin = new AnalyticsPlugin({ ...config, queriesDir: dir }); + const plugin = pluginForDir(config, dir); const { router, getHandler } = createMockRouter(); const executeMock = vi .fn() @@ -790,11 +815,18 @@ describe("analytics metric route (Phase 1)", () => { }); // ── loadMetricRegistry: config parse against the landed metricSourceSchema. +// The loaders now read the config file THROUGH an `AppManager` (Phase 2), so +// each test points an `AppManager` at its temp dir instead of passing a bare +// directory string. The module cache is still keyed by `app.queriesDir`, so the +// mtime/ctime/size revalidation and `__resetMetricRegistryCache` behavior are +// unchanged. describe("loadMetricRegistry", () => { let dir: string; + let app: AppManager; beforeEach(() => { dir = mkdtempSync(path.join(tmpdir(), "mv-registry-")); + app = new AppManager(dir); }); afterEach(() => { @@ -803,7 +835,7 @@ describe("loadMetricRegistry", () => { }); test("absent metric-views.json → empty registry (dormancy)", async () => { - expect(await loadMetricRegistry(dir)).toEqual({}); + expect(await loadMetricRegistry(app)).toEqual({}); }); test("derives lane from executor (default sp, user → obo)", async () => { @@ -817,7 +849,7 @@ describe("loadMetricRegistry", () => { }), ); - const registry = await loadMetricRegistry(dir); + const registry = await loadMetricRegistry(app); expect(registry.revenue).toEqual({ key: "revenue", source: "cat.sch.revenue_metrics", @@ -838,7 +870,7 @@ describe("loadMetricRegistry", () => { }), ); - const registry = await loadMetricRegistry(dir); + const registry = await loadMetricRegistry(app); expect(Object.getPrototypeOf(registry)).toBeNull(); // A key that would resolve to a truthy inherited member on a plain object // resolves to undefined here. @@ -849,13 +881,13 @@ describe("loadMetricRegistry", () => { test("absent file yields a null-prototype (dormant) registry too", async () => { // The ENOENT dormancy path must also be prototype-free — a metric key // colliding with an inherited member can't 200 against an empty registry. - const registry = await loadMetricRegistry(dir); + const registry = await loadMetricRegistry(app); expect(Object.getPrototypeOf(registry)).toBeNull(); }); test("malformed JSON throws", async () => { writeFileSync(path.join(dir, "metric-views.json"), "{ not json"); - await expect(loadMetricRegistry(dir)).rejects.toThrow(/Failed to parse/); + await expect(loadMetricRegistry(app)).rejects.toThrow(/Failed to parse/); }); test("schema-invalid config throws", async () => { @@ -865,7 +897,7 @@ describe("loadMetricRegistry", () => { metricViews: { revenue: { source: "not-a-three-part-fqn" } }, }), ); - await expect(loadMetricRegistry(dir)).rejects.toThrow( + await expect(loadMetricRegistry(app)).rejects.toThrow( /Invalid metric-views.json/, ); }); @@ -882,7 +914,7 @@ describe("loadMetricRegistry", () => { path.join(dir, "metric-views.json"), JSON.stringify({ metricViews }), ); - await expect(loadMetricRegistry(dir)).rejects.toThrow( + await expect(loadMetricRegistry(app)).rejects.toThrow( /Invalid metric-views.json/, ); }); @@ -898,7 +930,7 @@ describe("loadMetricRegistry", () => { metricViews: { revenue: { source: `cat.sch.${longSegment}` } }, }), ); - await expect(loadMetricRegistry(dir)).rejects.toThrow( + await expect(loadMetricRegistry(app)).rejects.toThrow( /Invalid metric-views.json/, ); }); @@ -914,7 +946,7 @@ describe("loadMetricRegistry", () => { path.join(dir, "metric-views.json"), JSON.stringify({ metricViews }), ); - await expect(loadMetricRegistry(dir)).resolves.toBeDefined(); + await expect(loadMetricRegistry(app)).resolves.toBeDefined(); }); test("re-reads only after the file changes (mtime-validated cache)", async () => { @@ -926,8 +958,8 @@ describe("loadMetricRegistry", () => { metricViews: { revenue: { source: "cat.sch.revenue_metrics" } }, }), ); - const first = await getMetricRegistry(dir); - const second = await getMetricRegistry(dir); + const first = await getMetricRegistry(app); + const second = await getMetricRegistry(app); expect(second).toBe(first); // same cached object, no re-parse writeFileSync( @@ -939,7 +971,7 @@ describe("loadMetricRegistry", () => { }, }), ); - const third = await getMetricRegistry(dir); + const third = await getMetricRegistry(app); expect(third).not.toBe(first); expect(Object.keys(third).sort()).toEqual(["orders", "revenue"]); }); @@ -956,7 +988,7 @@ describe("loadMetricRegistry", () => { metricViews: { revenue: { source: "cat.sch.rev_aaa" } }, }), ); - const before = await getMetricRegistry(dir); + const before = await getMetricRegistry(app); expect(before.revenue?.source).toBe("cat.sch.rev_aaa"); // Same-length replacement: "rev_aaa" → "rev_bbb" keeps the file byte-count @@ -970,7 +1002,7 @@ describe("loadMetricRegistry", () => { ); expect(statSync(p).size).toBe(sizeBefore); // same size, as designed - const after = await getMetricRegistry(dir); + const after = await getMetricRegistry(app); expect(after.revenue?.source).toBe("cat.sch.rev_bbb"); }); @@ -981,13 +1013,93 @@ describe("loadMetricRegistry", () => { metricViews: { revenue: { source: "cat.sch.revenue_metrics" } }, }), ); - const first = await getMetricRegistry(dir); + const first = await getMetricRegistry(app); __resetMetricRegistryCache(); - const second = await getMetricRegistry(dir); + const second = await getMetricRegistry(app); // Cleared cache → fresh parse → a new object (not the memoized instance). expect(second).not.toBe(first); expect(Object.keys(second)).toEqual(["revenue"]); }); + + // ── Phase 2: dev-tunnel branch. A `?dev` request must NOT stat and must NOT + // cache — it re-reads through the (stubbed) dev tunnel every call so the + // developer's local edits are reflected immediately, mirroring the `.sql` + // tunnel path. + describe("dev-remote branch (uncached, re-read every request)", () => { + // Minimal DevFileReader stub: serves the CURRENT on-disk temp file over the + // "tunnel". Reading real bytes keeps the not-found / parse semantics honest + // while letting us assert the dev branch bypasses the stat cache. + function makeDevReader(): DevFileReader { + return { + readFile: async (relativePath: string) => + readFileSync(path.join(dir, path.basename(relativePath)), "utf8"), + readdir: async () => readdirSync(dir), + }; + } + + test("dev request reflects edits on the next call without an mtime/ctime change (no cache)", async () => { + const devReq = { query: { dev: "1" }, headers: {} }; + const devReader = makeDevReader(); + const p = path.join(dir, "metric-views.json"); + + writeFileSync( + p, + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.rev_aaa" } }, + }), + ); + const first = await getMetricRegistry(app, devReq, devReader); + expect(first.revenue?.source).toBe("cat.sch.rev_aaa"); + + // Same-length repoint: `size` is identical. If the dev branch consulted + // the stat cache, this edit could be missed; because it re-reads every + // request, the NEW contents must be served. + const sizeBefore = statSync(p).size; + writeFileSync( + p, + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.rev_bbb" } }, + }), + ); + expect(statSync(p).size).toBe(sizeBefore); + + const second = await getMetricRegistry(app, devReq, devReader); + expect(second.revenue?.source).toBe("cat.sch.rev_bbb"); + // A fresh parse each time → never the same object instance. + expect(second).not.toBe(first); + }); + + test("back-to-back dev reads of an UNCHANGED file still return fresh objects (no memo)", async () => { + // Contrast with the prod path, where two reads of an unchanged file + // return the SAME cached instance (see the mtime-cache test above). In + // dev there is no cache, so even with the file untouched each call + // re-reads + re-parses and hands back a distinct object. + const devReq = { query: { dev: "1" }, headers: {} }; + const devReader = makeDevReader(); + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.revenue_metrics" } }, + }), + ); + + const first = await getMetricRegistry(app, devReq, devReader); + const second = await getMetricRegistry(app, devReq, devReader); + // Uncached → distinct instances even though nothing changed. + expect(second).not.toBe(first); + expect(second).toEqual(first); + + // And the dev reads left NO entry in the module cache: a following prod + // read is therefore cold (parses fresh) and only its SECOND call hits + // the cache — the classic prod signature. If a dev read had populated + // the cache, prodFirst would already be that cached object. + const prodFirst = await getMetricRegistry(app); + expect(prodFirst).not.toBe(first); + expect(prodFirst).not.toBe(second); + const prodSecond = await getMetricRegistry(app); + expect(prodSecond).toBe(prodFirst); + }); + }); }); // ── Phase 2: the structured filter engine (translator + validator). @@ -1714,16 +1826,16 @@ describe("metric — filter translator", () => { }); test("a well-formed-but-unknown measure reaches the warehouse and surfaces a sanitized error envelope", async () => { - const plugin = new AnalyticsPlugin({ - ...config, - queriesDir: registryDir({ + const plugin = pluginForDir( + config, + registryDir({ revenue: { key: "revenue", source: "cat.sch.revenue_metrics", lane: "sp", }, }), - }); + ); const { router, getHandler } = createMockRouter(); // The warehouse rejects the unknown column. The raw text carries the @@ -2096,16 +2208,16 @@ describe("metric route — lane dispatch (Phase 3)", () => { }); test("OBO-lane registration routes through asUser(req)", async () => { - const plugin = new AnalyticsPlugin({ - ...config, - queriesDir: registryDir({ + const plugin = pluginForDir( + config, + registryDir({ revenue: { key: "revenue", source: "cat.sch.revenue_metrics", lane: "obo", }, }), - }); + ); const { router, getHandler } = createMockRouter(); const asUserSpy = vi.spyOn(plugin, "asUser"); @@ -2136,16 +2248,16 @@ describe("metric route — lane dispatch (Phase 3)", () => { }); test("SP-lane registration uses the default executor (asUser not called)", async () => { - const plugin = new AnalyticsPlugin({ - ...config, - queriesDir: registryDir({ + const plugin = pluginForDir( + config, + registryDir({ revenue: { key: "revenue", source: "cat.sch.revenue_metrics", lane: "sp", }, }), - }); + ); const { router, getHandler } = createMockRouter(); const asUserSpy = vi.spyOn(plugin, "asUser"); @@ -2175,16 +2287,16 @@ describe("metric route — lane dispatch (Phase 3)", () => { }); test("OBO metric with no user identity → canonical 401, no SQL executed", async () => { - const plugin = new AnalyticsPlugin({ - ...config, - queriesDir: registryDir({ + const plugin = pluginForDir( + config, + registryDir({ revenue: { key: "revenue", source: "cat.sch.revenue_metrics", lane: "obo", }, }), - }); + ); const { router, getHandler } = createMockRouter(); const executeMock = vi.fn(); @@ -2212,16 +2324,16 @@ describe("metric route — lane dispatch (Phase 3)", () => { }); test("OBO metric with whitespace-only user identity → canonical 401", async () => { - const plugin = new AnalyticsPlugin({ - ...config, - queriesDir: registryDir({ + const plugin = pluginForDir( + config, + registryDir({ revenue: { key: "revenue", source: "cat.sch.revenue_metrics", lane: "obo", }, }), - }); + ); const { router, getHandler } = createMockRouter(); const executeMock = vi.fn(); diff --git a/packages/appkit/src/plugins/analytics/types.ts b/packages/appkit/src/plugins/analytics/types.ts index c01c04e23..e91a361f8 100644 --- a/packages/appkit/src/plugins/analytics/types.ts +++ b/packages/appkit/src/plugins/analytics/types.ts @@ -23,17 +23,6 @@ export interface IAnalyticsConfig extends BasePluginConfig { * byte arrives the stream is not time-bounded. */ arrowFirstByteTimeoutMs?: number; - /** - * Directory to read the metric-view registry (`metric-views.json`) from. - * - * @internal FOR TESTS ONLY. Production always reads the fixed - * `config/queries/` path (the default), the same directory the `.sql` query - * path uses — do NOT set this to relocate config in a real app. It exists so - * tests can point the loader at a temp directory and exercise the real - * stat → read → cache path against a real file, rather than poking private - * plugin state or stubbing the loader. - */ - queriesDir?: string; } /** diff --git a/packages/appkit/src/type-generator/mv-registry/config.ts b/packages/appkit/src/type-generator/mv-registry/config.ts index e91e3b95e..6d4926470 100644 --- a/packages/appkit/src/type-generator/mv-registry/config.ts +++ b/packages/appkit/src/type-generator/mv-registry/config.ts @@ -94,13 +94,9 @@ function isValidMetricKey(key: string): boolean { return /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key); } -// `isValidFqn` (the three-part UC FQN predicate) now lives in the shared -// zod-free leaf alongside `UC_FQN_PATTERN` and `quoteFqnForSql`, so the -// type-generator and the analytics runtime share one grammar + one escaper. -// `resolveMetricConfig` below intentionally does NOT call it — it re-derives -// the same checks inline so it can emit specific, staged error messages -// (arity, per-segment charset, per-segment length) rather than a single -// boolean. +// `resolveMetricConfig` re-derives the three-part UC FQN checks inline (rather +// than calling a shared predicate) so it can emit specific, staged error +// messages: arity, per-segment charset, per-segment length. /** * Field allowlists enforced by {@link resolveMetricConfig}. From dce640c6c4d62fa6bea8fe40dbbed85e3c6ee9de Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Tue, 21 Jul 2026 17:28:54 +0200 Subject: [PATCH 15/18] fix(analytics): json-encode measures/dimensions in metric cache key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `composeMetricCacheKey` joined the sorted measures/dimensions lists with a raw `.join(",")`. A comma is a legal identifier character — `isValidColumnName` rejects only control characters and newlines — so `["a,b"]` and `["a","b"]` produced the same key element while generating different SQL, which could serve wrong cached rows. Encode each list with `JSON.stringify` instead, matching the collision-safe encoding `canonicalizeFilter` already uses for predicate members, so the cache key stays one-to-one with the generated SQL. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../appkit/src/plugins/analytics/mv/cache.ts | 10 ++++++++-- .../src/plugins/analytics/tests/metric.test.ts | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/packages/appkit/src/plugins/analytics/mv/cache.ts b/packages/appkit/src/plugins/analytics/mv/cache.ts index d95f8cc25..0eb2ae8c5 100644 --- a/packages/appkit/src/plugins/analytics/mv/cache.ts +++ b/packages/appkit/src/plugins/analytics/mv/cache.ts @@ -17,8 +17,14 @@ export function composeMetricCacheKey(input: MetricCacheKeyInput): string[] { input.metricKey, input.source, input.format, - sortedMeasures.join(","), - sortedDimensions.join(","), + // JSON-encode (not raw `.join(",")`): a comma is a legal identifier + // character (`isValidColumnName` rejects only control chars / newlines), so + // joining on `,` would collapse `["a,b"]` and `["a","b"]` to the same key + // element despite rendering different SQL. JSON quoting keeps the key + // one-to-one with the generated SQL — the same encoding `canonicalizeFilter` + // uses for predicate members below. + JSON.stringify(sortedMeasures), + JSON.stringify(sortedDimensions), input.timeGrain ?? "_", timeDimensionPart, filterFingerprint, diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index fe5226029..069de0973 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -1982,6 +1982,23 @@ describe("composeMetricCacheKey", () => { expect(a).not.toEqual(b); }); + // A comma is a legal identifier character (`isValidColumnName` rejects only + // control chars / newlines), so a raw `.join(",")` would collapse a single + // comma-containing name and a two-name list to the same key element while the + // generated SQL differs — serving wrong cached rows. JSON encoding keeps the + // key one-to-one with the SQL. + test("measures with a comma in a name do NOT collide with a two-measure list", () => { + const a = composeMetricCacheKey({ ...base, measures: ["a,b"] }); + const b = composeMetricCacheKey({ ...base, measures: ["a", "b"] }); + expect(a).not.toEqual(b); + }); + + test("dimensions with a comma in a name do NOT collide with a two-dimension list", () => { + const a = composeMetricCacheKey({ ...base, dimensions: ["a,b"] }); + const b = composeMetricCacheKey({ ...base, dimensions: ["a", "b"] }); + expect(a).not.toEqual(b); + }); + test("different timeGrain → different keys", () => { const a = composeMetricCacheKey({ ...base, From 57054ce5fea0260be25392f9ebe023a07e62d1c8 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Tue, 21 Jul 2026 17:51:26 +0200 Subject: [PATCH 16/18] fix(analytics): render empty AND filter group as vacuous-true MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `renderFilter` emitted `1 = 0` (vacuous-false) for an empty OR but returned `null` for an empty AND, which the parent group drops. Dropping is only correct at the top level: nested in an OR, an empty AND is identity-true, so `TRUE OR P` must match all rows — but the drop collapsed it to `P` and under-returned. Emit `1 = 1` for empty AND, parallel to the empty-OR sentinel, so the group renders its Boolean identity element and is correct in any position. Unreachable through the route today (the validator rejects empty groups with 400 before the renderer runs), but `renderFilter`'s empty-group handling exists as the defense-in-depth fallback for a bypass, so it must be semantically correct too. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../src/plugins/analytics/mv/formatters.ts | 13 +++++--- .../plugins/analytics/tests/metric.test.ts | 31 ++++++++++++++----- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/packages/appkit/src/plugins/analytics/mv/formatters.ts b/packages/appkit/src/plugins/analytics/mv/formatters.ts index 57ac74de5..376e01055 100644 --- a/packages/appkit/src/plugins/analytics/mv/formatters.ts +++ b/packages/appkit/src/plugins/analytics/mv/formatters.ts @@ -118,10 +118,15 @@ function renderFilter( )[groupKey]; if (!Array.isArray(children) || children.length === 0) { - if (groupKey === "or") { - return "1 = 0"; - } - return null; + // Empty group → the Boolean identity element for the operator, so the + // result is correct in ANY position (including nested inside the other + // operator). An empty OR is vacuously false (`1 = 0`); an empty AND is + // vacuously true (`1 = 1`). Returning `null` for empty-AND (dropped by the + // parent) would only be correct at the top level — nested in an OR it + // would silently vanish and turn `TRUE OR P` into `P`, under-returning + // rows. (The validator rejects empty groups outright; this is the + // defense-in-depth fallback and must be semantically correct too.) + return groupKey === "or" ? "1 = 0" : "1 = 1"; } const sortedChildren = sortFilterChildren(children); diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index 069de0973..a3bc14222 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -1303,15 +1303,15 @@ describe("metric — filter translator", () => { expect(where).toContain(" AND "); }); - test("empty `and: []` group emits no WHERE clause (defense in depth past the validator)", () => { + test("empty `and: []` renders 1 = 1 (defense in depth past the validator)", () => { // The validator rejects `and: []` (see cardinality tests), but if a - // bypass ever reaches the renderer, an empty conjunction must drop to no - // constraint — never emit a malformed WHERE. - const { statement, parameters } = buildMetricSql(registration, { - measures: ["arr"], - filter: { and: [] }, - }); - expect(statement).not.toContain("WHERE"); + // bypass ever reaches the renderer, an empty conjunction must render the + // AND identity element (vacuous-true) so it is correct in any position — + // returning `null` (dropped) would only be right at the top level, not + // nested inside an OR (see the OR-of-empty-AND test below). No bind + // parameters are emitted for the tautology. + const { where, parameters } = render({ and: [] }); + expect(where).toBe("1 = 1"); expect(parameters).toEqual({}); }); @@ -1322,6 +1322,21 @@ describe("metric — filter translator", () => { const { where } = render({ or: [] }); expect(where).toBe("1 = 0"); }); + + test("empty `and` nested in `or` stays vacuous-true (does not under-return)", () => { + // `TRUE OR P` is all rows. If empty-AND returned `null` (dropped by the + // parent OR), this would collapse to just `P` and under-return. The + // identity element `1 = 1` keeps the disjunction matching everything. + // `sortFilterChildren` orders the predicate before the empty-`and` group, + // so the tautology lands on the right; the disjunction still matches all rows. + const { where } = render({ + or: [ + { and: [] }, + { member: "region", operator: "equals", values: ["EMEA"] }, + ], + }); + expect(where).toBe("(`region` = :f_0 OR 1 = 1)"); + }); }); describe("depth cap", () => { From afe41e576babab63d78dedb559c7ec884fb732c7 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Tue, 21 Jul 2026 18:24:36 +0200 Subject: [PATCH 17/18] chore: adjust comments --- packages/appkit/src/app/index.ts | 48 +------------------------------- 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/packages/appkit/src/app/index.ts b/packages/appkit/src/app/index.ts index 01ac06c83..e0574c5c2 100644 --- a/packages/appkit/src/app/index.ts +++ b/packages/appkit/src/app/index.ts @@ -30,38 +30,18 @@ interface FileSystemAdapter { export class AppManager { private readonly _queriesDir: string; - /** - * @param queriesDir - Absolute path to the `config/queries` directory this - * manager reads from. Defaults to `/config/queries`. - * - * @internal The override exists FOR TESTS ONLY, so a spec can point - * AppManager at a temp directory instead of `process.cwd()`. Production and - * every runtime call site use `new AppManager()` with no argument; nothing - * passes this path outside tests. - */ constructor( queriesDir: string = path.resolve(process.cwd(), "config/queries"), ) { this._queriesDir = queriesDir; } - /** - * Absolute path to the `config/queries` directory this manager reads from. - * - * Read-only accessor over the private field so a caller (e.g. the metric - * registry) can `fs.stat` the directory or build a cache key from it without - * reaching into internals or re-deriving `process.cwd()/config/queries`. - */ get queriesDir(): string { return this._queriesDir; } /** * Whether `req` is a dev-tunnel (`?dev`) request. - * - * Public so a caller can pick its dev-vs-production branch with the exact - * sniff {@link createFsAdapter} uses, instead of duplicating - * `req?.query?.dev !== undefined`. */ isDevRequest(req?: RequestLike): boolean { return req?.query?.dev !== undefined; @@ -196,27 +176,10 @@ export class AppManager { /** * Read a single config file from the queries directory, dev-tunnel-aware. * - * A narrow, domain-agnostic seam: it knows nothing about JSON, Zod, or metric - * views and returns the raw file contents as a string. Uses the same - * dev-vs-production detection as {@link getAppQuery} ({@link isDevRequest} → - * dev tunnel via `devFileReader`; else direct `fs`) and applies the same - * {@link validatePath} traversal guard. - * - * The not-found-vs-error distinction is load-bearing for callers that treat - * "absent" as dormant (return `null`) but any other failure as fatal (503): - * - * - Returns `null` for a genuine not-found — `ENOENT` in production; the dev - * tunnel's not-found signal in dev (best-effort message match, since the - * tunnel surfaces a plain `Error` with no `errno` code). - * - Returns `null` for a rejected path-traversal attempt (never reads). - * - **Throws on any other error** (EACCES / EIO / tunnel / parse-of-listing / - * …). These are deliberately NOT swallowed to `null`. - * * @param fileName - File name (or relative path) within the queries directory. * @param req - Optional request object to detect dev mode. * @param devFileReader - Optional DevFileReader to read via the WebSocket tunnel. - * @returns The raw file contents, or `null` when the file is absent / the path - * is rejected. + * @returns The raw file contents, or `null` when the file is absent / the path is rejected. */ async readConfigFile( fileName: string, @@ -243,15 +206,6 @@ export class AppManager { } } - /** - * Classify a read failure as a genuine not-found (→ `null`) vs. any other - * error (→ rethrow). - * - * Production reads reject with a Node `ErrnoException` whose `code` is - * `"ENOENT"`. The dev tunnel surfaces a plain `Error` with no `errno` code, - * so in dev we fall back to a best-effort message match on the not-found - * signal — acceptable per the config-read contract. - */ private isNotFoundError(error: unknown, req?: RequestLike): boolean { if ((error as NodeJS.ErrnoException)?.code === "ENOENT") { return true; From 889d5099c601496eaa717f490e86d527338ff7b0 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Tue, 21 Jul 2026 19:57:16 +0200 Subject: [PATCH 18/18] refactor(analytics): read metric-views.json per request, drop mtime cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the module-level mtime-validated registry memo (getMetricRegistry + metricRegistryCache Map + __resetMetricRegistryCache) and have the metric route call loadMetricRegistry directly, reading + parsing the config once per request — matching the sibling `.sql` query path, which already re-reads config every request with no cache. Per-request re-read still delivers hot-reload / self-heal / dormant semantics; the parse cost on a <=200-entry config is negligible next to the warehouse round-trip. Deleting the cache removes its duplicate fs.stat ENOENT check, so absent-file classification now flows through a single owner (readConfigFile -> isNotFoundError) inside AppManager. isDevRequest, whose only external caller was the deleted cache, is demoted to private; isNotFoundError (incl. its dev message-match branch) is kept intact so dev and prod agree an absent config is dormant. Drops the now-unused RegistryCacheSignature type and the cache-mechanism tests (mtime/ctime revalidation, dev-remote uncached suite); keeps the self-heal / hot-reload route tests as the regression guarantee for per-request re-read. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- packages/appkit/src/app/index.ts | 6 +- .../src/app/tests/read-config-file.test.ts | 13 -- .../appkit/src/plugins/analytics/analytics.ts | 14 +- .../appkit/src/plugins/analytics/mv/index.ts | 6 +- .../src/plugins/analytics/mv/registry.ts | 124 +----------- .../appkit/src/plugins/analytics/mv/types.ts | 6 - .../plugins/analytics/tests/metric.test.ts | 188 ++---------------- 7 files changed, 25 insertions(+), 332 deletions(-) diff --git a/packages/appkit/src/app/index.ts b/packages/appkit/src/app/index.ts index e0574c5c2..ec2260536 100644 --- a/packages/appkit/src/app/index.ts +++ b/packages/appkit/src/app/index.ts @@ -41,9 +41,11 @@ export class AppManager { } /** - * Whether `req` is a dev-tunnel (`?dev`) request. + * Whether `req` is a dev-tunnel (`?dev`) request. Internal `?dev` predicate + * shared by {@link createFsAdapter} (tunnel-vs-`fs` branch) and + * {@link isNotFoundError} (dev's message-match not-found classification). */ - isDevRequest(req?: RequestLike): boolean { + private isDevRequest(req?: RequestLike): boolean { return req?.query?.dev !== undefined; } diff --git a/packages/appkit/src/app/tests/read-config-file.test.ts b/packages/appkit/src/app/tests/read-config-file.test.ts index 9f68e00a9..0a44e192c 100644 --- a/packages/appkit/src/app/tests/read-config-file.test.ts +++ b/packages/appkit/src/app/tests/read-config-file.test.ts @@ -36,19 +36,6 @@ describe("AppManager.readConfigFile", () => { }); }); - describe("isDevRequest predicate", () => { - test("returns true when ?dev is present", () => { - expect(appManager.isDevRequest({ query: { dev: "" }, headers: {} })).toBe( - true, - ); - }); - - test("returns false without ?dev and for undefined req", () => { - expect(appManager.isDevRequest({ query: {}, headers: {} })).toBe(false); - expect(appManager.isDevRequest()).toBe(false); - }); - }); - describe("production mode (direct fs)", () => { test("returns file contents for an existing file", async () => { await fs.writeFile( diff --git a/packages/appkit/src/plugins/analytics/analytics.ts b/packages/appkit/src/plugins/analytics/analytics.ts index 7f96d8949..7515fd140 100644 --- a/packages/appkit/src/plugins/analytics/analytics.ts +++ b/packages/appkit/src/plugins/analytics/analytics.ts @@ -33,7 +33,7 @@ import { buildMetricSql, composeMetricCacheKey, deriveMetricExecutorKey, - getMetricRegistry, + loadMetricRegistry, validateMetricRequest, } from "./metric"; import { QueryProcessor } from "./query"; @@ -478,14 +478,14 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { return; } - // Resolve the registry from disk (cached + mtime-revalidated by `getMetricRegistry`). - // Reads through the plugin's shared `this.app` (the base `Plugin`'s - // `AppManager`, rooted at `config/queries/` under the process cwd), so this - // is dev-tunnel aware and shares the module-level mtime cache keyed by - // `app.queriesDir`. + // Resolve the registry from disk: read + parse `metric-views.json` once per + // request (no memoization). Reads through the plugin's shared `this.app` + // (the base `Plugin`'s `AppManager`, rooted at `config/queries/` under the + // process cwd), so this is dev-tunnel aware and inherits the traversal + // guard — same as the sibling `.sql` query path. let registry: Record; try { - registry = await getMetricRegistry(this.app, req, this.devFileReader); + registry = await loadMetricRegistry(this.app, req, this.devFileReader); } catch (err) { const reason = err instanceof Error ? err.message : String(err); logger.warn(req, "Failed to load metric registry: %s", reason); diff --git a/packages/appkit/src/plugins/analytics/mv/index.ts b/packages/appkit/src/plugins/analytics/mv/index.ts index ea1d9da18..17c97793e 100644 --- a/packages/appkit/src/plugins/analytics/mv/index.ts +++ b/packages/appkit/src/plugins/analytics/mv/index.ts @@ -1,8 +1,4 @@ export { composeMetricCacheKey, deriveMetricExecutorKey } from "./cache"; export { buildMetricSql } from "./formatters"; -export { - __resetMetricRegistryCache, - getMetricRegistry, - loadMetricRegistry, -} from "./registry"; +export { loadMetricRegistry } from "./registry"; export { validateMetricRequest } from "./schemas"; diff --git a/packages/appkit/src/plugins/analytics/mv/registry.ts b/packages/appkit/src/plugins/analytics/mv/registry.ts index b1caf119b..1b44dfc8c 100644 --- a/packages/appkit/src/plugins/analytics/mv/registry.ts +++ b/packages/appkit/src/plugins/analytics/mv/registry.ts @@ -1,4 +1,3 @@ -import fs from "node:fs/promises"; import path from "node:path"; // Canonical metric-source schema — the single source of truth for // `metric-views.json`. Imported from the shared source directly (matching the @@ -9,7 +8,6 @@ import type { AppManager, DevFileReader, RequestLike } from "../../../app"; import { createLogger } from "../../../logging/logger"; import type { MetricRegistration } from "../types"; import { laneFromExecutor, METRIC_CONFIG_FILE } from "./constants"; -import type { RegistryCacheSignature } from "./types"; const logger = createLogger("analytics:metric-views"); @@ -25,8 +23,7 @@ const logger = createLogger("analytics:metric-views"); * traversal guard. In production that's a plain `fs.readFile` under the hood, * so the semantics below are unchanged. * - * Absent file (or a rejected traversal path) -> empty registry (`null` from - * `readConfigFile`). + * Absent file -> empty registry (`null` from `readConfigFile`). * Malformed file -> 503 (throws). * * @param app - The {@link AppManager} that resolves + reads the config file. @@ -85,122 +82,3 @@ export async function loadMetricRegistry( ); return registry; } - -/** - * Module-level registry cache, keyed by the resolved queries directory. - */ -const metricRegistryCache = new Map< - string, - { - signature: RegistryCacheSignature; - registry: Record; - } ->(); - -/** - * Clear the module-level registry cache. - * - * @internal FOR TESTS ONLY. The cache is keyed by directory and lives for the - * process; production never needs to clear it (a changed file is picked up via - * the stat signature). Tests that reuse a directory — or assert cold-load - * behavior — call this in `beforeEach`/`afterEach` so isolation is intentional - * rather than relying on each test happening to use a unique temp dir. - */ -export function __resetMetricRegistryCache(): void { - metricRegistryCache.clear(); -} - -/** - * Resolve the metric registry for `app`, re-reading + re-parsing only when - * `metric-views.json` has changed since the cached copy. - * - * Two branches, mirroring the sibling `.sql` query path: - * - * - **Dev** ({@link AppManager.isDevRequest} true): the file lives on the - * developer's machine and is served over the WebSocket tunnel, which exposes - * no `stat`. The whole point of `?dev` is to reflect the developer's local - * edits immediately, so this branch does NOT `stat`, does NOT cache, and - * simply re-reads via {@link loadMetricRegistry} every request — exactly like - * the `.sql` tunnel path. - * - **Production** (not a dev request): behavior identical to the pre-refactor - * loader. The steady-state cost is a single async `stat`, and the read + - * `JSON.parse` + zod validation are skipped when the file's - * `(ctimeMs, mtimeMs, size)` signature is unchanged. This delivers the agreed - * semantics without a permanent memo: - * - * - **Hot-reload** — editing a working config bumps `mtimeMs`, so the next - * request re-parses and serves the new registry with no server restart. - * - **Self-heal** — a load failure is NOT cached (we only populate the - * cache on a successful parse), so a fixed config is picked up on the - * next request instead of latching a 503 forever. - * - **Dormant** — an absent file `stat`s as `ENOENT` → empty registry; - * adding the file later is picked up on the next request. - * - * Concurrency: two simultaneous cold requests may both `stat`+read before - * either populates the cache. That is a harmless redundant read of the same - * file (the sibling `.sql` path does not single-flight either), so no lock is - * taken. - * - * @param app - The {@link AppManager} that resolves + reads the config file. - * The cache is keyed by `app.queriesDir`. - * @param req - Optional request object, used to pick the dev-vs-production branch. - * @param devFileReader - Optional dev tunnel reader (dev branch only). - * @throws Propagates {@link loadMetricRegistry}'s throw on a malformed file so - * the route can surface a 503; the cache is left untouched on failure. - */ -export async function getMetricRegistry( - app: AppManager, - req?: RequestLike, - devFileReader?: DevFileReader, -): Promise> { - // Dev branch: never stat (the tunnel has none), never cache — re-read every - // request so the developer's local edits are reflected immediately. - if (app.isDevRequest(req)) { - return loadMetricRegistry(app, req, devFileReader); - } - - const queriesDir = app.queriesDir; - const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); - - let signature: RegistryCacheSignature | null; - try { - const stats = await fs.stat(metricPath); - signature = { - ctimeMs: stats.ctimeMs, - mtimeMs: stats.mtimeMs, - size: stats.size, - }; - } catch (err) { - if ((err as NodeJS.ErrnoException).code === "ENOENT") { - // Absent file → dormant. Drop any stale cache entry (the file may have - // been deleted) and return an empty registry without touching the cache. - metricRegistryCache.delete(queriesDir); - signature = null; - } else { - // Any other stat error (EACCES / EIO / ELOOP / …) is deliberately fatal - // for THIS request → the route surfaces a 503, consistent with the - // malformed-config → 503 path. It is not latched: with the self-heal - // design a transient error clears on the next request. - throw err; - } - } - - if (signature === null) { - return Object.create(null); - } - - const cached = metricRegistryCache.get(queriesDir); - if ( - cached !== undefined && - cached.signature.ctimeMs === signature.ctimeMs && - cached.signature.mtimeMs === signature.mtimeMs && - cached.signature.size === signature.size - ) { - return cached.registry; - } - - // Cold or stale: re-read + re-parse. Cache ONLY on success. - const registry = await loadMetricRegistry(app, req, devFileReader); - metricRegistryCache.set(queriesDir, { signature, registry }); - return registry; -} diff --git a/packages/appkit/src/plugins/analytics/mv/types.ts b/packages/appkit/src/plugins/analytics/mv/types.ts index be33cf7d6..791435d17 100644 --- a/packages/appkit/src/plugins/analytics/mv/types.ts +++ b/packages/appkit/src/plugins/analytics/mv/types.ts @@ -1,11 +1,5 @@ import type { MetricFilter } from "../types"; -export interface RegistryCacheSignature { - ctimeMs: number; - mtimeMs: number; - size: number; -} - export interface FilterRenderState { counter: number; depth: number; diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index a3bc14222..41b78079c 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -1,11 +1,4 @@ -import { - mkdtempSync, - readdirSync, - readFileSync, - rmSync, - statSync, - writeFileSync, -} from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { @@ -16,16 +9,14 @@ import { setupDatabricksEnv, } from "@tools/test-helpers"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -import { AppManager, type DevFileReader } from "../../../app"; +import { AppManager } from "../../../app"; import { ServiceContext } from "../../../context/service-context"; import { AuthenticationError } from "../../../errors"; import { AnalyticsPlugin } from "../analytics"; import { - __resetMetricRegistryCache, buildMetricSql, composeMetricCacheKey, deriveMetricExecutorKey, - getMetricRegistry, loadMetricRegistry, validateMetricRequest, } from "../metric"; @@ -72,8 +63,8 @@ vi.mock("../../../cache", () => ({ // Temp dirs created by `registryDir` / `writeRegistry`, cleaned up after each // test. Using real files (pointing the plugin's `AppManager` at the dir, see -// `pluginForDir`) exercises the actual stat → read → cache path in -// `getMetricRegistry` rather than poking private plugin state. +// `pluginForDir`) exercises the actual read → parse path in +// `loadMetricRegistry` rather than poking private plugin state. const tempRegistryDirs: string[] = []; /** @@ -155,7 +146,6 @@ describe("analytics metric route (Phase 1)", () => { afterEach(() => { serviceContextMock?.restore(); - __resetMetricRegistryCache(); while (tempRegistryDirs.length > 0) { const dir = tempRegistryDirs.pop(); if (dir) rmSync(dir, { recursive: true, force: true }); @@ -716,8 +706,8 @@ describe("analytics metric route (Phase 1)", () => { ); expect(res1.status).toHaveBeenCalledWith(503); - // Fix the file. mtime changes, so the next request re-parses and serves - // it — no server restart, no latched 503. + // Fix the file. Each request re-reads + re-parses the config, so the + // next request serves it — no server restart, no latched 503. writeRegistry(dir, { revenue: { key: "revenue", @@ -768,8 +758,8 @@ describe("analytics metric route (Phase 1)", () => { ); expect(res1.status).toHaveBeenCalledWith(404); - // Add `orders` to the config. The mtime bump invalidates the cache, so - // the next request sees it without a restart. + // Add `orders` to the config. Each request re-reads the config, so the + // next request sees it without a restart. writeRegistry(dir, { revenue: { key: "revenue", @@ -815,11 +805,10 @@ describe("analytics metric route (Phase 1)", () => { }); // ── loadMetricRegistry: config parse against the landed metricSourceSchema. -// The loaders now read the config file THROUGH an `AppManager` (Phase 2), so -// each test points an `AppManager` at its temp dir instead of passing a bare -// directory string. The module cache is still keyed by `app.queriesDir`, so the -// mtime/ctime/size revalidation and `__resetMetricRegistryCache` behavior are -// unchanged. +// The loader reads the config file THROUGH an `AppManager` (Phase 2), so each +// test points an `AppManager` at its temp dir instead of passing a bare +// directory string. The loader is stateless — it reads + parses on every call +// (no memoization), so there is no cache to reset between tests. describe("loadMetricRegistry", () => { let dir: string; let app: AppManager; @@ -830,7 +819,6 @@ describe("loadMetricRegistry", () => { }); afterEach(() => { - __resetMetricRegistryCache(); rmSync(dir, { recursive: true, force: true }); }); @@ -948,158 +936,6 @@ describe("loadMetricRegistry", () => { ); await expect(loadMetricRegistry(app)).resolves.toBeDefined(); }); - - test("re-reads only after the file changes (mtime-validated cache)", async () => { - // getMetricRegistry caches by dir and revalidates via stat; a second call - // with no change returns the same object, and a rewrite is picked up. - writeFileSync( - path.join(dir, "metric-views.json"), - JSON.stringify({ - metricViews: { revenue: { source: "cat.sch.revenue_metrics" } }, - }), - ); - const first = await getMetricRegistry(app); - const second = await getMetricRegistry(app); - expect(second).toBe(first); // same cached object, no re-parse - - writeFileSync( - path.join(dir, "metric-views.json"), - JSON.stringify({ - metricViews: { - revenue: { source: "cat.sch.revenue_metrics" }, - orders: { source: "cat.sch.order_metrics" }, - }, - }), - ); - const third = await getMetricRegistry(app); - expect(third).not.toBe(first); - expect(Object.keys(third).sort()).toEqual(["orders", "revenue"]); - }); - - test("picks up a SAME-SIZE edit (ctime invalidation, not just size)", async () => { - // The failure mode size+mtime alone would miss: rewrite the config to the - // EXACT same byte length (repoint `source` to an equal-length FQN). `size` - // is unchanged and a coarse-mtime FS might not advance `mtimeMs`, but - // `ctimeMs` bumps on any write — so the new source must be served. - const p = path.join(dir, "metric-views.json"); - writeFileSync( - p, - JSON.stringify({ - metricViews: { revenue: { source: "cat.sch.rev_aaa" } }, - }), - ); - const before = await getMetricRegistry(app); - expect(before.revenue?.source).toBe("cat.sch.rev_aaa"); - - // Same-length replacement: "rev_aaa" → "rev_bbb" keeps the file byte-count - // identical, so `size` cannot disambiguate. - const sizeBefore = statSync(p).size; - writeFileSync( - p, - JSON.stringify({ - metricViews: { revenue: { source: "cat.sch.rev_bbb" } }, - }), - ); - expect(statSync(p).size).toBe(sizeBefore); // same size, as designed - - const after = await getMetricRegistry(app); - expect(after.revenue?.source).toBe("cat.sch.rev_bbb"); - }); - - test("__resetMetricRegistryCache forces a cold re-read", async () => { - writeFileSync( - path.join(dir, "metric-views.json"), - JSON.stringify({ - metricViews: { revenue: { source: "cat.sch.revenue_metrics" } }, - }), - ); - const first = await getMetricRegistry(app); - __resetMetricRegistryCache(); - const second = await getMetricRegistry(app); - // Cleared cache → fresh parse → a new object (not the memoized instance). - expect(second).not.toBe(first); - expect(Object.keys(second)).toEqual(["revenue"]); - }); - - // ── Phase 2: dev-tunnel branch. A `?dev` request must NOT stat and must NOT - // cache — it re-reads through the (stubbed) dev tunnel every call so the - // developer's local edits are reflected immediately, mirroring the `.sql` - // tunnel path. - describe("dev-remote branch (uncached, re-read every request)", () => { - // Minimal DevFileReader stub: serves the CURRENT on-disk temp file over the - // "tunnel". Reading real bytes keeps the not-found / parse semantics honest - // while letting us assert the dev branch bypasses the stat cache. - function makeDevReader(): DevFileReader { - return { - readFile: async (relativePath: string) => - readFileSync(path.join(dir, path.basename(relativePath)), "utf8"), - readdir: async () => readdirSync(dir), - }; - } - - test("dev request reflects edits on the next call without an mtime/ctime change (no cache)", async () => { - const devReq = { query: { dev: "1" }, headers: {} }; - const devReader = makeDevReader(); - const p = path.join(dir, "metric-views.json"); - - writeFileSync( - p, - JSON.stringify({ - metricViews: { revenue: { source: "cat.sch.rev_aaa" } }, - }), - ); - const first = await getMetricRegistry(app, devReq, devReader); - expect(first.revenue?.source).toBe("cat.sch.rev_aaa"); - - // Same-length repoint: `size` is identical. If the dev branch consulted - // the stat cache, this edit could be missed; because it re-reads every - // request, the NEW contents must be served. - const sizeBefore = statSync(p).size; - writeFileSync( - p, - JSON.stringify({ - metricViews: { revenue: { source: "cat.sch.rev_bbb" } }, - }), - ); - expect(statSync(p).size).toBe(sizeBefore); - - const second = await getMetricRegistry(app, devReq, devReader); - expect(second.revenue?.source).toBe("cat.sch.rev_bbb"); - // A fresh parse each time → never the same object instance. - expect(second).not.toBe(first); - }); - - test("back-to-back dev reads of an UNCHANGED file still return fresh objects (no memo)", async () => { - // Contrast with the prod path, where two reads of an unchanged file - // return the SAME cached instance (see the mtime-cache test above). In - // dev there is no cache, so even with the file untouched each call - // re-reads + re-parses and hands back a distinct object. - const devReq = { query: { dev: "1" }, headers: {} }; - const devReader = makeDevReader(); - writeFileSync( - path.join(dir, "metric-views.json"), - JSON.stringify({ - metricViews: { revenue: { source: "cat.sch.revenue_metrics" } }, - }), - ); - - const first = await getMetricRegistry(app, devReq, devReader); - const second = await getMetricRegistry(app, devReq, devReader); - // Uncached → distinct instances even though nothing changed. - expect(second).not.toBe(first); - expect(second).toEqual(first); - - // And the dev reads left NO entry in the module cache: a following prod - // read is therefore cold (parses fresh) and only its SECOND call hits - // the cache — the classic prod signature. If a dev read had populated - // the cache, prodFirst would already be that cached object. - const prodFirst = await getMetricRegistry(app); - expect(prodFirst).not.toBe(first); - expect(prodFirst).not.toBe(second); - const prodSecond = await getMetricRegistry(app); - expect(prodSecond).toBe(prodFirst); - }); - }); }); // ── Phase 2: the structured filter engine (translator + validator).