Implement repeat directive

README.md

@@ -40,6 +40,7 @@ Additional steps if you would like to run scripts, unit tests or edit the projec
 | `--macros` | | | Experimental - Path to file(s) or directory(ies) containing additional SQL macros. Prefix with `@` to reference files in the templates directory. This argument may be repeated. See [details below](#macros---macros-parameter).| 
 | `--param` | | | `name=value` pair of user defined variables to be used when generating SQL with a [custom template](#custom-templates). This argument may be repeated. | 
 | `--var` | | | `name=value` pair of FHIRPath variables for use in ViewDefinition expressions (referenced as `%name`). This argument may be repeated. | 
+| `--repeat-depth` | | `5` | Maximum nesting depth for `repeat` traversal. Nodes deeper than this limit are silently dropped. Increase this value if your data has deeply nested recursive structures (e.g. `--repeat-depth 10`). |
 | `--verbose` | | false | Print debugging information to the console when running FlatQuack. |
 
 #### Modes (--mode parameter)

src/cli.js

@@ -29,6 +29,7 @@ Options:
       --macros <path>           Custom macro file or directory (can be repeated)
       --var <name=value>        Values for FHIRPath constants in ViewDefinition (can be repeated)
       --param <name=value>      Template parameters (can be used repeated)
+      --repeat-depth <n>        Maximum nesting depth for repeat traversal (default: 5)
       --verbose                 Enable verbose output
       --help                    Show this help message
       --version                 Show version information
@@ -167,6 +168,7 @@ const args = parseArgs({
 		"mode": {type: "string", short: "m", default: "preview"},
 		"param": {type: "string", multiple: true},
 		"var": {type: "string", multiple: true},
+		"repeat-depth": {type: "string"},
 		"help": {type: "boolean"},
 		"version": {type: "boolean"}
 	}
@@ -213,7 +215,8 @@ for (const file of glob.scanSync(args.values["view-path"],{onlyFiles:true})) {
 	const outputPath = path.join(path.dirname(inputPath), basename + ".sql");
 
 	const view = JSON.parse(fs.readFileSync(inputPath));
-	const query = templateToQuery(view, schema, template, params, args.values["verbose"], undefined, customMacros, vars);
+	const repeatDepth = args.values["repeat-depth"] != null ? parseInt(args.values["repeat-depth"], 10) : null;
+	const query = templateToQuery(view, schema, template, params, args.values["verbose"], undefined, customMacros, vars, repeatDepth);
 	const formattedQuery = formatSQL(query);
 
 	if (args.values["mode"] == "build") {

src/query-builder.js

@@ -3,12 +3,17 @@ import {astToSql, pathsToSchema, tablesToSql} from "./ddb-sql-builder.js"
 import {parseVd, extractPathsFromAst} from "./view-parser.js";
 import macros from "../templates/duck-macros.js";
 
-export function buildQuery(vd, schema, filterByResourceType, verbose, vars) {
+export function buildQuery(vd, schema, filterByResourceType, verbose, vars, repeatDepth) {
 	const parsedVd = parseVd(vd);
 	if (verbose) console.log(parsedVd.path)
 
 	const fpAst = fhirpathToAst(parsedVd.path, vd.resource, schema, vars);
-	const fpSql = astToSql(fpAst).sql;
+
+	const repeatEntryFields = collectRepeatEntryFields(vd);
+	const options = {_jsonScopeFields: new Set(repeatEntryFields), _lateralDefs: []};
+	if (repeatDepth != null) options._nestedRepeatDepth = repeatDepth;
+	const fpSql = astToSql(fpAst, false, {}, options).sql;
+	const lateralDefs = options._lateralDefs;
 
 	const whereAsts = (vd.where||[]).map(w => w.path)
 		.concat([filterByResourceType ? `resourceType = '${vd.resource}'` : null])
@@ -23,21 +28,52 @@ export function buildQuery(vd, schema, filterByResourceType, verbose, vars) {
 	}).join(" and ");
 
 	const schemaPaths = extractPathsFromAst({asts: [fpAst].concat(whereAsts)});
+
+	// For each repeat block, retype the entry path's first segment to JSON[]
+	// in the source schema so read_json_auto preserves the full nested data
+	// for JSON traversal in the recursive CTE.
+	for (const field of repeatEntryFields) {
+		const entryNode = schemaPaths.find(n => n.value === field);
+		if (entryNode) {
+			entryNode.fhirType = 'JSON';
+			entryNode.children = [];
+		}
+	}
+
 	const schemaSql = pathsToSchema(schemaPaths)
 	const outputSql = tablesToSql(parsedVd.tables);
-	return {pathSql: fpSql, schemaSql, outputSql, whereSql}
+	return {pathSql: fpSql, schemaSql, outputSql, whereSql, lateralDefs}
+}
+
+function collectRepeatEntryFields(vd) {
+	// Walk the VD tree, return the set of top-level field names that are
+	// entry paths for any repeat block (the first segment of repeat[0]).
+	const fields = new Set();
+	function walk(node) {
+		if (!node || typeof node !== 'object') return;
+		if (node.repeat && Array.isArray(node.repeat) && node.repeat[0]) {
+			fields.add(node.repeat[0].split('.')[0]);
+		}
+		if (node.select) node.select.forEach(walk);
+		if (node.unionAll) node.unionAll.forEach(walk);
+	}
+	walk(vd);
+	return Array.from(fields);
 }
 
 //TODO: consider replacing this with a full template language
-export function templateToQuery(vd, schema, template, args=[], verbose, filterByResourceType, customMacros=null, vars=null) {
-	//Setting filterByResourceType to btrue can only be used if the schema for the
+export function templateToQuery(vd, schema, template, args=[], verbose, filterByResourceType, customMacros=null, vars=null, repeatDepth=null) {
+	//Setting filterByResourceType to true can only be used if the schema for the
 	//elements being use is compatible between all of the resources being read
 	//(e.g., element with the same names have the same structure). This is used
 	//in some of the tests that mix resource types.
-	
-	const queryParts = buildQuery(vd, schema, filterByResourceType, verbose, vars);
+
+	const queryParts = buildQuery(vd, schema, filterByResourceType, verbose, vars, repeatDepth);
 	const whereSql = queryParts.whereSql ? "WHERE " + queryParts.whereSql : "";
 	const schemaSql = queryParts.schemaSql ? `, columns=${queryParts.schemaSql}` : "";
+	const lateralColsSql = queryParts.lateralDefs && queryParts.lateralDefs.length > 0
+		? queryParts.lateralDefs.map(d => `${d.sql} AS ${d.name}`).join(',\n\t\t') + ',\n\t\t'
+		: '';
 
 	// Concatenate base macros with custom macros
 	const allMacros = customMacros ? macros + '\n' + customMacros : macros;
@@ -46,7 +82,7 @@ export function templateToQuery(vd, schema, template, args=[], verbose, filterBy
 		["fq_input_dir", process.cwd()],
 		["fq_output_dir", process.cwd()],
 		["fq_where_filter", whereSql],
-		["fq_sql_transform_expression", queryParts.pathSql],
+		["fq_sql_transform_expression", lateralColsSql + queryParts.pathSql],
 		["fq_sql_input_schema", schemaSql],
 		["fq_sql_flattening_cols", queryParts.outputSql.fieldSql],
 		["fq_sql_flattening_tables", queryParts.outputSql.joinSql],
@@ -61,4 +97,4 @@ export function templateToQuery(vd, schema, template, args=[], verbose, filterBy
 	})
 
 	return template;
-}
+}

src/view-parser.js

@@ -20,7 +20,18 @@ export function validateVd(vd) {
 			if (!node.select && !node.column && !node.unionAll)
 				throw new Error("forEach and forEachOrNull elements must be used together with a column, select or unionAll element");
 		}
-
+
+		if (node.repeat) {
+			if (!Array.isArray(node.repeat) || !node.repeat.every(r => typeof r === "string"))
+				throw new Error("repeat must be an array of strings");
+			if (node.repeat.length === 0)
+				throw new Error("repeat must contain at least one path");
+			if (node.forEach || node.forEachOrNull)
+				throw new Error("repeat cannot be used with forEach or forEachOrNull");
+			if (!node.select && !node.column && !node.unionAll)
+				throw new Error("repeat must be used with a column, select or unionAll element");
+		}
+
 		//collection must be boolean
 		if (node.select) {
 			if (!Array.isArray(node.select))
@@ -88,6 +99,15 @@ export function parseVd(vd, skipValidation) {
 	}
 
 	function parseNode(node, isRoot, inUnion, parentTable) {
+		if (node.repeat) {
+			const repeatTable = !inUnion ? addTable("each", parentTable, false) : parentTable;
+			const rest = parseNode({...node, repeat: undefined}, false, false, repeatTable);
+			const firstPath = node.repeat[0];
+			const pathArgs = node.repeat.map(p => `'${p}'`).join(", ");
+			const path = `${firstPath}._repeat(${pathArgs}, ${rest})`;
+			return !inUnion ? `_col_collection('${repeatTable}', ${path})` : path;
+		}
+
 		if (node.forEach || node.forEachOrNull) {
 			const eachTable = !inUnion ? addTable(node.forEach ? "each" : "nullEach", parentTable, !!node.forEachOrNull) : parentTable;
 			if (inUnion && node.forEachOrNull) updateTable(eachTable, true);

templates/csv.sql

@@ -2,7 +2,7 @@
 
 COPY (
 	WITH transformed AS (
-		SELECT {{fq_sql_transform_expression}} AS result 
+		SELECT {{fq_sql_transform_expression}} AS result
 		FROM read_json_auto(
 			'{{fq_input_dir}}/**/*{{fq_vd_resource}}*.ndjson'
 			{{fq_sql_input_schema}}

templates/dbt_model.sql

@@ -1,5 +1,5 @@
 WITH transformed AS (
-	SELECT {{fq_sql_transform_expression}} AS result 
+	SELECT {{fq_sql_transform_expression}} AS result
 	FROM {{ source('fhir_db', '{{fq_vd_resource}}') }}
 	{{fq_where_filter}}
 )

templates/explore.sql

@@ -1,7 +1,7 @@
 {{fq_sql_macros}}
 
 WITH transformed AS (
-	SELECT {{fq_sql_transform_expression}} AS result 
+	SELECT {{fq_sql_transform_expression}} AS result
 	FROM read_json_auto(
 		'{{fq_input_dir}}/**/*{{fq_vd_resource}}*.ndjson'
 		{{fq_sql_input_schema}}

templates/ndjson.sql

@@ -2,7 +2,7 @@
 
 COPY (
 	WITH transformed AS (
-		SELECT {{fq_sql_transform_expression}} AS result 
+		SELECT {{fq_sql_transform_expression}} AS result
 		FROM read_json_auto(
 			'{{fq_input_dir}}/**/*{{fq_vd_resource}}*.ndjson'
 			{{fq_sql_input_schema}}

templates/parquet.sql

@@ -2,7 +2,7 @@
 
 COPY (
 	WITH transformed AS (
-		SELECT {{fq_sql_transform_expression}} AS result 
+		SELECT {{fq_sql_transform_expression}} AS result
 		FROM read_json_auto(
 			'{{fq_input_dir}}/**/*{{fq_vd_resource}}*.ndjson'
 			{{fq_sql_input_schema}}