diff --git a/.github/scripts/check_sql_syntax.py b/.github/scripts/check_sql_syntax.py new file mode 100644 index 00000000..6dfe2c9c --- /dev/null +++ b/.github/scripts/check_sql_syntax.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Validate that every BigQuery concept SQL file parses with sqlglot. + +Usage: ``python .github/scripts/check_sql_syntax.py [ROOT ...]`` +(defaults to ``mimic-iv/concepts``). Exits non-zero if any file fails to parse. +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import sqlglot +from sqlglot.errors import ParseError + + +def main(argv: list[str]) -> int: + roots = [Path(p) for p in argv[1:]] or [Path("mimic-iv/concepts")] + files = sorted({f for root in roots for f in root.rglob("*.sql")}) + if not files: + print(f"::error::No .sql files found under: {', '.join(map(str, roots))}") + return 1 + + failures = 0 + for f in files: + try: + sqlglot.parse_one(f.read_text(), read="bigquery") + except ParseError as exc: + failures += 1 + # Keep the GitHub annotation to a single line so it anchors to the + # file; the full multi-line message follows in the log. + first_line = str(exc).splitlines()[0] + print(f"::error file={f}::sqlglot failed to parse: {first_line}") + print(str(exc)) + else: + print(f"OK {f}") + + print() + if failures: + print(f"{failures} of {len(files)} concept file(s) failed to parse.") + return 1 + print(f"All {len(files)} concept file(s) parsed successfully.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/.github/workflows/bigquery_dry_run.yml b/.github/workflows/bigquery_dry_run.yml index a270bb4a..b3d7f7a1 100644 --- a/.github/workflows/bigquery_dry_run.yml +++ b/.github/workflows/bigquery_dry_run.yml @@ -1,16 +1,22 @@ -name: BigQuery syntax check (dry run) +name: BigQuery dry run -# Verify syntax of bq through `bq query --dry_run` -# Note: dry-run resolves inter-concept references against mimiciv_derived dataset, -# so a brand-new concept that depends on another brand-new concept (not yet published) -# will report the dependency as missing until the first is released. +# Semantic validation of the BigQuery concept sources via `bq query --dry_run`, +# which resolves table/column references against the live mimiciv_derived +# dataset. This requires cloud credentials (OIDC), so it is only run on main. +# +# Note: dry-run resolves inter-concept references against mimiciv_derived, so a +# brand-new concept that depends on another brand-new concept (not yet +# published) will report the dependency as missing until the first is released. on: - pull_request_review: - types: [submitted] + push: + branches: + - main + paths: + - 'mimic-iv/concepts/**' + - '.github/workflows/bigquery_dry_run.yml' jobs: dry-run: - if: github.event.review.state == 'approved' runs-on: ubuntu-latest permissions: contents: 'read' diff --git a/.github/workflows/sql_syntax_check.yml b/.github/workflows/sql_syntax_check.yml new file mode 100644 index 00000000..fb9783ea --- /dev/null +++ b/.github/workflows/sql_syntax_check.yml @@ -0,0 +1,27 @@ +name: BigQuery SQL syntax check via sqlglot + +on: + pull_request: + paths: + - 'mimic-iv/concepts/**' + - 'src/mimic_utils/**' + - '.github/scripts/check_sql_syntax.py' + - '.github/workflows/sql_syntax_check.yml' + +permissions: + contents: read + +jobs: + syntax-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install locked dependencies + run: pip install -r requirements-lock.txt + - name: Install package + run: pip install -e . --no-deps + - name: Parse concept SQL with sqlglot + run: python .github/scripts/check_sql_syntax.py mimic-iv/concepts diff --git a/mimic-iv/concepts_duckdb/duckdb.sql b/mimic-iv/concepts_duckdb/duckdb.sql index 621cde34..b4cb96b0 100644 --- a/mimic-iv/concepts_duckdb/duckdb.sql +++ b/mimic-iv/concepts_duckdb/duckdb.sql @@ -62,6 +62,8 @@ .read medication/acei.sql .print 'medication/antibiotic.sql' .read medication/antibiotic.sql +.print 'medication/arb.sql' +.read medication/arb.sql .print 'medication/dobutamine.sql' .read medication/dobutamine.sql .print 'medication/dopamine.sql' @@ -82,6 +84,8 @@ .read medication/vasopressin.sql -- treatment +.print 'treatment/code_status.sql' +.read treatment/code_status.sql .print 'treatment/crrt.sql' .read treatment/crrt.sql .print 'treatment/invasive_line.sql'