From e6cedee37199a1327630fdff07b442831eeade86 Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Mon, 18 May 2026 17:25:30 -0400 Subject: [PATCH 1/3] =?UTF-8?q?Fix=20lint=20violations=20in=20dim=5Fdbt=5F?= =?UTF-8?q?=5Fcurrent=5Fmodels=20and=20stg=5Fdbt=5F=5F*=20Surfaced=20by=20?= =?UTF-8?q?the=20new=20Tier=202=20lint=20job=20=E2=80=94=20the=20prior=20C?= =?UTF-8?q?I=20was=20either=20tolerating=20these=20or=20never=20ran=20sqlf?= =?UTF-8?q?luff=20cleanly=20against=20the=20models.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/dim_dbt__current_models.sql | 5 +++-- models/staging/stg_dbt__model_executions.sql | 2 +- models/staging/stg_dbt__models.sql | 2 +- models/staging/stg_dbt__seed_executions.sql | 2 +- models/staging/stg_dbt__seeds.sql | 2 +- models/staging/stg_dbt__snapshot_executions.sql | 2 +- models/staging/stg_dbt__sources.sql | 2 +- 7 files changed, 9 insertions(+), 8 deletions(-) diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql index 27273cbb..5454c35d 100644 --- a/models/dim_dbt__current_models.sql +++ b/models/dim_dbt__current_models.sql @@ -1,10 +1,12 @@ with base as (select * from {{ ref("stg_dbt__models") }}), + model_executions as (select * from {{ ref("stg_dbt__model_executions") }}), + latest_models as ( {# Retrieves the models present in the most recent run #} - select * from base where run_started_at = (select max(run_started_at) from base) + select * from base where run_started_at = (select max(b.run_started_at) from base as b) ), @@ -132,4 +134,3 @@ with select * from final - diff --git a/models/staging/stg_dbt__model_executions.sql b/models/staging/stg_dbt__model_executions.sql index ad2c4f50..c025ac35 100644 --- a/models/staging/stg_dbt__model_executions.sql +++ b/models/staging/stg_dbt__model_executions.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("model_executions") }}), + enhanced as ( select @@ -30,4 +31,3 @@ with select * from enhanced - diff --git a/models/staging/stg_dbt__models.sql b/models/staging/stg_dbt__models.sql index d8184183..61c619f7 100644 --- a/models/staging/stg_dbt__models.sql +++ b/models/staging/stg_dbt__models.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("models") }}), + enhanced as ( select @@ -29,4 +30,3 @@ with select * from enhanced - diff --git a/models/staging/stg_dbt__seed_executions.sql b/models/staging/stg_dbt__seed_executions.sql index 90f52870..2fcb154c 100644 --- a/models/staging/stg_dbt__seed_executions.sql +++ b/models/staging/stg_dbt__seed_executions.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("seed_executions") }}), + enhanced as ( select @@ -29,4 +30,3 @@ with select * from enhanced - diff --git a/models/staging/stg_dbt__seeds.sql b/models/staging/stg_dbt__seeds.sql index 0e5dcb1d..de2052c5 100644 --- a/models/staging/stg_dbt__seeds.sql +++ b/models/staging/stg_dbt__seeds.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("seeds") }}), + enhanced as ( select @@ -26,4 +27,3 @@ with select * from enhanced - diff --git a/models/staging/stg_dbt__snapshot_executions.sql b/models/staging/stg_dbt__snapshot_executions.sql index 8794b99b..25cc25e5 100644 --- a/models/staging/stg_dbt__snapshot_executions.sql +++ b/models/staging/stg_dbt__snapshot_executions.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("snapshot_executions") }}), + enhanced as ( select @@ -29,4 +30,3 @@ with select * from enhanced - diff --git a/models/staging/stg_dbt__sources.sql b/models/staging/stg_dbt__sources.sql index 1bd8cef1..508d869a 100644 --- a/models/staging/stg_dbt__sources.sql +++ b/models/staging/stg_dbt__sources.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("sources") }}), + enhanced as ( select @@ -26,4 +27,3 @@ with select * from enhanced - From a47bd7dafa558bfd6f9aace81b3dcc3451e2accb Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Mon, 18 May 2026 17:30:03 -0400 Subject: [PATCH 2/3] Fix compose-up.sh to handle short-lived init containers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Tier 2 SQL Server lane was failing on CI even though the sqlserver-configurator container ran init.sql successfully and exited 0. Root cause: `docker compose up --wait` treats "not running" as a failure regardless of exit code, so a container that runs to completion makes --wait return non-zero. This worked on developer laptops because the configurator was still mid-execution when --wait polled — a timing race that masked the bug. CI runners are fast enough to lose the race. Fix: split the service list in scripts/ci/compose-up.sh into long-running services (healthchecked, brought up with --wait) and init containers (brought up plain, then waited on with `docker compose wait`, a separate command that blocks on exit and surfaces the exit code). Convention used: anything matching `*-configurator` is treated as an init container. Easy to extend if other init sidecars are added later. Also retracted a wrong assertion in compose.yml — the comment on sqlserver-configurator claimed --wait treats exit-0 as success. It does not. Replaced with an accurate description of the constraint and a pointer to how compose-up.sh handles it. Verified locally: ./scripts/ci/test.sh sqlserver # PASS=49 ERROR=0 --- compose.yml | 10 ++++--- scripts/ci/compose-up.sh | 56 +++++++++++++++++++++++++++++++++------- 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/compose.yml b/compose.yml index 1db84317..9ce45f72 100644 --- a/compose.yml +++ b/compose.yml @@ -96,9 +96,13 @@ services: /opt/mssql-tools18/bin/sqlcmd -S sqlserver -U sa -P ${MSSQL_SA_PASSWORD:-123Administrator} -No -d master -i docker-entrypoint-initdb.d/init.sql && echo "SQL Server bootstrap complete."; ' - # The configurator exits after init.sql completes; that exit is success. - # `docker compose up --wait` treats a service as healthy when it has no - # healthcheck and exits 0, so this satisfies the wait without a healthcheck. + # This is a run-to-completion init container: it runs init.sql, prints + # the success message, and exits. Crucially, it is NOT compatible with + # `docker compose up --wait` — `--wait` treats "not running" as a + # failure regardless of exit code, so a container that exits cleanly + # still makes `--wait` return non-zero. `scripts/ci/compose-up.sh` + # handles that by bringing init containers up separately and using + # `docker compose wait` (different command) to block on their exit. volumes: postgres_data: diff --git a/scripts/ci/compose-up.sh b/scripts/ci/compose-up.sh index 87664496..098d6b64 100755 --- a/scripts/ci/compose-up.sh +++ b/scripts/ci/compose-up.sh @@ -1,14 +1,26 @@ #!/usr/bin/env bash -# Bring up local data warehouse containers and wait for them to be healthy. +# Bring up local data warehouse containers and wait for them to be ready. # # Usage: # scripts/ci/compose-up.sh # all services # scripts/ci/compose-up.sh postgres # one service (plus its dependencies) # scripts/ci/compose-up.sh trino sqlserver # -# Uses `docker compose up --wait`, which exits 0 only when every requested -# service is healthy (or has run-to-completion successfully, in the case of -# the sqlserver-configurator init job). +# "Ready" means two different things depending on the service type: +# +# - Long-running services (postgres, trino, sqlserver): brought up with +# `docker compose up --wait`, which blocks until each service's +# healthcheck reports healthy. +# +# - Short-lived init containers (sqlserver-configurator): brought up +# separately, then waited on with `docker compose wait`, which blocks +# until the container exits and surfaces its exit code. We can't use +# `--wait` for these because `--wait` treats "not running" as a failure +# regardless of exit code — so a container that runs init.sql, exits 0, +# and is therefore "not running" makes `--wait` return non-zero. That's +# what was breaking the SQL Server lane on CI runners (the configurator +# just happens to exit faster on Linux runners than on a dev laptop, so +# the local race was masking the bug). set -euo pipefail source "$(dirname "${BASH_SOURCE[0]}")/_lib.sh" @@ -17,8 +29,9 @@ cd "${repo_root}" require_cmd docker -# Map the warehouse short-name a user might pass to its compose service set. -# `sqlserver` requires the configurator sidecar to run init.sql. +# Map warehouse short-names to the compose services they require. +# `sqlserver` needs both `sqlserver` itself and the `sqlserver-configurator` +# sidecar that runs `init.sql`. services=() if (( $# == 0 )); then services=(postgres trino sqlserver sqlserver-configurator) @@ -38,7 +51,32 @@ else done fi -banner "Starting services: ${services[*]}" -docker compose up -d --wait "${services[@]}" +# Split: short-lived init containers vs. long-running healthchecked services. +# Convention: anything matching `*-configurator` is treated as init. +long_running=() +init=() +for svc in "${services[@]}"; do + if [[ "${svc}" == *-configurator ]]; then + init+=("${svc}") + else + long_running+=("${svc}") + fi +done -log "all requested services are healthy" +if (( ${#long_running[@]} > 0 )); then + banner "Starting long-running services: ${long_running[*]}" + docker compose up -d --wait "${long_running[@]}" + log "long-running services are healthy" +fi + +if (( ${#init[@]} > 0 )); then + banner "Starting init containers: ${init[*]}" + docker compose up -d "${init[@]}" + # `docker compose wait` blocks until each named container exits and + # returns the highest exit code observed. Failure here means an init + # script (e.g. SQL Server `init.sql`) errored. + if ! docker compose wait "${init[@]}"; then + die "init container(s) failed: ${init[*]} — check 'docker compose logs ${init[*]}'" + fi + log "init containers completed successfully" +fi From 586fa25767938d9a149d52589808ac931e08ed36 Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Mon, 18 May 2026 17:34:07 -0400 Subject: [PATCH 3/3] Mitigate Node 20 deprecation warning in CI workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitHub Actions is migrating its default JavaScript runtime from Node 20 to Node 24. The currently SHA-pinned action versions we use (actions/checkout@v4, astral-sh/setup-uv@v3, google-github-actions/auth@v2) are Node 20-based and would fail after the June 2, 2026 default flip. Set `ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true` at workflow scope in all four CI workflows (pr.yml, main.yml, release.yml, cut-release-candidate.yml). This keeps the existing pins working through the June flip and until we have time to regression-test the workflows against Node 24-compatible action versions. ⚠ Hard deadline: 2026-09-16. On that date GitHub removes Node 20 from the runner entirely. The env var stops working then. Before that date, bump each action SHA to a Node 24-compatible release, verify all four workflows pass, then remove the env block from each file. Tracked in specs/ci-rework/README.md §12.5 alongside the existing operational debt items. Dependabot (already configured) should surface compatible bumps automatically once the upstream actions tag Node 24 releases. --- .github/workflows/cut-release-candidate.yml | 9 +++++++++ .github/workflows/main.yml | 8 ++++++++ .github/workflows/pr.yml | 9 +++++++++ .github/workflows/release.yml | 8 ++++++++ specs/ci-rework/README.md | 21 +++++++++++++++++++++ 5 files changed, 55 insertions(+) diff --git a/.github/workflows/cut-release-candidate.yml b/.github/workflows/cut-release-candidate.yml index 74fb83b6..713fe9b2 100644 --- a/.github/workflows/cut-release-candidate.yml +++ b/.github/workflows/cut-release-candidate.yml @@ -41,6 +41,15 @@ concurrency: group: cut-release-candidate cancel-in-progress: false +env: + # Temporary: keep Node 20 as the actions runtime so the currently + # SHA-pinned action versions (which are Node 20-based) keep working + # after the June 2, 2026 default flip to Node 24. HARD DEADLINE: + # September 16, 2026, when Node 20 is removed from the runner entirely. + # Remove this once the pinned actions are bumped to Node 24-compatible + # versions. Tracked in specs/ci-rework/README.md §12.5. + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true" + jobs: cut: runs-on: ubuntu-latest diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index db233aba..c947b86e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,6 +41,14 @@ concurrency: cancel-in-progress: false env: + # Temporary: keep Node 20 as the actions runtime so the currently + # SHA-pinned action versions (which are Node 20-based) keep working + # after the June 2, 2026 default flip to Node 24. HARD DEADLINE: + # September 16, 2026, when Node 20 is removed from the runner entirely. + # Remove this once the pinned actions are bumped to Node 24-compatible + # versions. Tracked in specs/ci-rework/README.md §12.5. + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true" + # Non-secret env vars consumed by the dbt invocations test model. These # are the same values used pre-rework and are checked in to profiles.yml # via env_var() — they need to be present, not secret. diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 962835a3..46c84d7f 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -39,6 +39,15 @@ concurrency: group: tier1-${{ github.event.pull_request.number }} cancel-in-progress: true +env: + # Temporary: keep Node 20 as the actions runtime so the currently + # SHA-pinned action versions (which are Node 20-based) keep working + # after the June 2, 2026 default flip to Node 24. HARD DEADLINE: + # September 16, 2026, when Node 20 is removed from the runner entirely. + # Remove this once the pinned actions are bumped to Node 24-compatible + # versions. Tracked in specs/ci-rework/README.md §12.5. + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true" + jobs: integration-local: name: integration (${{ matrix.warehouse }}) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 222e9e18..fc31d4bd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,6 +51,14 @@ concurrency: cancel-in-progress: false env: + # Temporary: keep Node 20 as the actions runtime so the currently + # SHA-pinned action versions (which are Node 20-based) keep working + # after the June 2, 2026 default flip to Node 24. HARD DEADLINE: + # September 16, 2026, when Node 20 is removed from the runner entirely. + # Remove this once the pinned actions are bumped to Node 24-compatible + # versions. Tracked in specs/ci-rework/README.md §12.5. + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true" + # Non-secret env vars consumed by the invocations test model (see # integration_test_project/profiles.yml). DBT_CLOUD_PROJECT_ID: 123 diff --git a/specs/ci-rework/README.md b/specs/ci-rework/README.md index 157d8612..71f48f93 100644 --- a/specs/ci-rework/README.md +++ b/specs/ci-rework/README.md @@ -865,6 +865,27 @@ or dbt Slack #package-ecosystem channel. ### 12.5. Hardening debt — operational +- **Node 20 → Node 24 action migration. ⚠ Has a hard deadline.** + All four workflow files currently set + `ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true"` at workflow scope + as a temporary mitigation. The SHA-pinned action versions we use + (`actions/checkout@34e1148...` v4, `astral-sh/setup-uv@caf0cab...` + v3, `google-github-actions/auth@c200f36...` v2) are Node 20-based. + GitHub Actions timeline: + + - **2026-06-02**: Node 24 becomes the default runtime. Without + the env var, our pinned actions would fail. The env var keeps + Node 20 working. + - **2026-09-16**: Node 20 is **removed entirely** from the + runner. The env var stops working on this date. Workflows + will fail if action SHAs haven't been bumped. + + Before **2026-09-16**, bump each action's SHA pin to a Node + 24-compatible release, regression-test all four workflows + (Tier 1/2/3 + the cutter), then remove the + `ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION` env block from every + workflow file. Dependabot (already configured) should surface the + bumps automatically once Node 24-compatible versions are tagged. - **Pre-commit hooks for SQLFluff.** Catches lint issues before they reach CI. Optional, contributor convenience. Constraint inherited from §12.2: lint requires the dbt-snowflake adapter init, which