diff --git a/.github/workflows/cut-release-candidate.yml b/.github/workflows/cut-release-candidate.yml index 74fb83b6..713fe9b2 100644 --- a/.github/workflows/cut-release-candidate.yml +++ b/.github/workflows/cut-release-candidate.yml @@ -41,6 +41,15 @@ concurrency: group: cut-release-candidate cancel-in-progress: false +env: + # Temporary: keep Node 20 as the actions runtime so the currently + # SHA-pinned action versions (which are Node 20-based) keep working + # after the June 2, 2026 default flip to Node 24. HARD DEADLINE: + # September 16, 2026, when Node 20 is removed from the runner entirely. + # Remove this once the pinned actions are bumped to Node 24-compatible + # versions. Tracked in specs/ci-rework/README.md §12.5. + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true" + jobs: cut: runs-on: ubuntu-latest diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index db233aba..c947b86e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,6 +41,14 @@ concurrency: cancel-in-progress: false env: + # Temporary: keep Node 20 as the actions runtime so the currently + # SHA-pinned action versions (which are Node 20-based) keep working + # after the June 2, 2026 default flip to Node 24. HARD DEADLINE: + # September 16, 2026, when Node 20 is removed from the runner entirely. + # Remove this once the pinned actions are bumped to Node 24-compatible + # versions. Tracked in specs/ci-rework/README.md §12.5. + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true" + # Non-secret env vars consumed by the dbt invocations test model. These # are the same values used pre-rework and are checked in to profiles.yml # via env_var() — they need to be present, not secret. diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 962835a3..46c84d7f 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -39,6 +39,15 @@ concurrency: group: tier1-${{ github.event.pull_request.number }} cancel-in-progress: true +env: + # Temporary: keep Node 20 as the actions runtime so the currently + # SHA-pinned action versions (which are Node 20-based) keep working + # after the June 2, 2026 default flip to Node 24. HARD DEADLINE: + # September 16, 2026, when Node 20 is removed from the runner entirely. + # Remove this once the pinned actions are bumped to Node 24-compatible + # versions. Tracked in specs/ci-rework/README.md §12.5. + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true" + jobs: integration-local: name: integration (${{ matrix.warehouse }}) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 222e9e18..fc31d4bd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,6 +51,14 @@ concurrency: cancel-in-progress: false env: + # Temporary: keep Node 20 as the actions runtime so the currently + # SHA-pinned action versions (which are Node 20-based) keep working + # after the June 2, 2026 default flip to Node 24. HARD DEADLINE: + # September 16, 2026, when Node 20 is removed from the runner entirely. + # Remove this once the pinned actions are bumped to Node 24-compatible + # versions. Tracked in specs/ci-rework/README.md §12.5. + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true" + # Non-secret env vars consumed by the invocations test model (see # integration_test_project/profiles.yml). DBT_CLOUD_PROJECT_ID: 123 diff --git a/compose.yml b/compose.yml index 1db84317..9ce45f72 100644 --- a/compose.yml +++ b/compose.yml @@ -96,9 +96,13 @@ services: /opt/mssql-tools18/bin/sqlcmd -S sqlserver -U sa -P ${MSSQL_SA_PASSWORD:-123Administrator} -No -d master -i docker-entrypoint-initdb.d/init.sql && echo "SQL Server bootstrap complete."; ' - # The configurator exits after init.sql completes; that exit is success. - # `docker compose up --wait` treats a service as healthy when it has no - # healthcheck and exits 0, so this satisfies the wait without a healthcheck. + # This is a run-to-completion init container: it runs init.sql, prints + # the success message, and exits. Crucially, it is NOT compatible with + # `docker compose up --wait` — `--wait` treats "not running" as a + # failure regardless of exit code, so a container that exits cleanly + # still makes `--wait` return non-zero. `scripts/ci/compose-up.sh` + # handles that by bringing init containers up separately and using + # `docker compose wait` (different command) to block on their exit. volumes: postgres_data: diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql index 27273cbb..5454c35d 100644 --- a/models/dim_dbt__current_models.sql +++ b/models/dim_dbt__current_models.sql @@ -1,10 +1,12 @@ with base as (select * from {{ ref("stg_dbt__models") }}), + model_executions as (select * from {{ ref("stg_dbt__model_executions") }}), + latest_models as ( {# Retrieves the models present in the most recent run #} - select * from base where run_started_at = (select max(run_started_at) from base) + select * from base where run_started_at = (select max(b.run_started_at) from base as b) ), @@ -132,4 +134,3 @@ with select * from final - diff --git a/models/staging/stg_dbt__model_executions.sql b/models/staging/stg_dbt__model_executions.sql index ad2c4f50..c025ac35 100644 --- a/models/staging/stg_dbt__model_executions.sql +++ b/models/staging/stg_dbt__model_executions.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("model_executions") }}), + enhanced as ( select @@ -30,4 +31,3 @@ with select * from enhanced - diff --git a/models/staging/stg_dbt__models.sql b/models/staging/stg_dbt__models.sql index d8184183..61c619f7 100644 --- a/models/staging/stg_dbt__models.sql +++ b/models/staging/stg_dbt__models.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("models") }}), + enhanced as ( select @@ -29,4 +30,3 @@ with select * from enhanced - diff --git a/models/staging/stg_dbt__seed_executions.sql b/models/staging/stg_dbt__seed_executions.sql index 90f52870..2fcb154c 100644 --- a/models/staging/stg_dbt__seed_executions.sql +++ b/models/staging/stg_dbt__seed_executions.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("seed_executions") }}), + enhanced as ( select @@ -29,4 +30,3 @@ with select * from enhanced - diff --git a/models/staging/stg_dbt__seeds.sql b/models/staging/stg_dbt__seeds.sql index 0e5dcb1d..de2052c5 100644 --- a/models/staging/stg_dbt__seeds.sql +++ b/models/staging/stg_dbt__seeds.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("seeds") }}), + enhanced as ( select @@ -26,4 +27,3 @@ with select * from enhanced - diff --git a/models/staging/stg_dbt__snapshot_executions.sql b/models/staging/stg_dbt__snapshot_executions.sql index 8794b99b..25cc25e5 100644 --- a/models/staging/stg_dbt__snapshot_executions.sql +++ b/models/staging/stg_dbt__snapshot_executions.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("snapshot_executions") }}), + enhanced as ( select @@ -29,4 +30,3 @@ with select * from enhanced - diff --git a/models/staging/stg_dbt__sources.sql b/models/staging/stg_dbt__sources.sql index 1bd8cef1..508d869a 100644 --- a/models/staging/stg_dbt__sources.sql +++ b/models/staging/stg_dbt__sources.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("sources") }}), + enhanced as ( select @@ -26,4 +27,3 @@ with select * from enhanced - diff --git a/scripts/ci/compose-up.sh b/scripts/ci/compose-up.sh index 87664496..098d6b64 100755 --- a/scripts/ci/compose-up.sh +++ b/scripts/ci/compose-up.sh @@ -1,14 +1,26 @@ #!/usr/bin/env bash -# Bring up local data warehouse containers and wait for them to be healthy. +# Bring up local data warehouse containers and wait for them to be ready. # # Usage: # scripts/ci/compose-up.sh # all services # scripts/ci/compose-up.sh postgres # one service (plus its dependencies) # scripts/ci/compose-up.sh trino sqlserver # -# Uses `docker compose up --wait`, which exits 0 only when every requested -# service is healthy (or has run-to-completion successfully, in the case of -# the sqlserver-configurator init job). +# "Ready" means two different things depending on the service type: +# +# - Long-running services (postgres, trino, sqlserver): brought up with +# `docker compose up --wait`, which blocks until each service's +# healthcheck reports healthy. +# +# - Short-lived init containers (sqlserver-configurator): brought up +# separately, then waited on with `docker compose wait`, which blocks +# until the container exits and surfaces its exit code. We can't use +# `--wait` for these because `--wait` treats "not running" as a failure +# regardless of exit code — so a container that runs init.sql, exits 0, +# and is therefore "not running" makes `--wait` return non-zero. That's +# what was breaking the SQL Server lane on CI runners (the configurator +# just happens to exit faster on Linux runners than on a dev laptop, so +# the local race was masking the bug). set -euo pipefail source "$(dirname "${BASH_SOURCE[0]}")/_lib.sh" @@ -17,8 +29,9 @@ cd "${repo_root}" require_cmd docker -# Map the warehouse short-name a user might pass to its compose service set. -# `sqlserver` requires the configurator sidecar to run init.sql. +# Map warehouse short-names to the compose services they require. +# `sqlserver` needs both `sqlserver` itself and the `sqlserver-configurator` +# sidecar that runs `init.sql`. services=() if (( $# == 0 )); then services=(postgres trino sqlserver sqlserver-configurator) @@ -38,7 +51,32 @@ else done fi -banner "Starting services: ${services[*]}" -docker compose up -d --wait "${services[@]}" +# Split: short-lived init containers vs. long-running healthchecked services. +# Convention: anything matching `*-configurator` is treated as init. +long_running=() +init=() +for svc in "${services[@]}"; do + if [[ "${svc}" == *-configurator ]]; then + init+=("${svc}") + else + long_running+=("${svc}") + fi +done -log "all requested services are healthy" +if (( ${#long_running[@]} > 0 )); then + banner "Starting long-running services: ${long_running[*]}" + docker compose up -d --wait "${long_running[@]}" + log "long-running services are healthy" +fi + +if (( ${#init[@]} > 0 )); then + banner "Starting init containers: ${init[*]}" + docker compose up -d "${init[@]}" + # `docker compose wait` blocks until each named container exits and + # returns the highest exit code observed. Failure here means an init + # script (e.g. SQL Server `init.sql`) errored. + if ! docker compose wait "${init[@]}"; then + die "init container(s) failed: ${init[*]} — check 'docker compose logs ${init[*]}'" + fi + log "init containers completed successfully" +fi diff --git a/specs/ci-rework/README.md b/specs/ci-rework/README.md index 157d8612..71f48f93 100644 --- a/specs/ci-rework/README.md +++ b/specs/ci-rework/README.md @@ -865,6 +865,27 @@ or dbt Slack #package-ecosystem channel. ### 12.5. Hardening debt — operational +- **Node 20 → Node 24 action migration. ⚠ Has a hard deadline.** + All four workflow files currently set + `ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true"` at workflow scope + as a temporary mitigation. The SHA-pinned action versions we use + (`actions/checkout@34e1148...` v4, `astral-sh/setup-uv@caf0cab...` + v3, `google-github-actions/auth@c200f36...` v2) are Node 20-based. + GitHub Actions timeline: + + - **2026-06-02**: Node 24 becomes the default runtime. Without + the env var, our pinned actions would fail. The env var keeps + Node 20 working. + - **2026-09-16**: Node 20 is **removed entirely** from the + runner. The env var stops working on this date. Workflows + will fail if action SHAs haven't been bumped. + + Before **2026-09-16**, bump each action's SHA pin to a Node + 24-compatible release, regression-test all four workflows + (Tier 1/2/3 + the cutter), then remove the + `ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION` env block from every + workflow file. Dependabot (already configured) should surface the + bumps automatically once Node 24-compatible versions are tagged. - **Pre-commit hooks for SQLFluff.** Catches lint issues before they reach CI. Optional, contributor convenience. Constraint inherited from §12.2: lint requires the dbt-snowflake adapter init, which