Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# CI / build plumbing — changes here can subvert the security model.
.github/ @mtcarlone
scripts/ci/ @mtcarlone
compose.yml @mtcarlone
init-scripts/ @mtcarlone

# Test runtime config — versions, deps, lock files.
tox.ini @mtcarlone
pyproject.toml @mtcarlone
uv.lock @mtcarlone

# Package metadata — affects what consumers install.
dbt_project.yml @mtcarlone
packages.yml @mtcarlone
package-lock.yml @mtcarlone

# Spec docs — change history matters for future maintainers.
specs/ @mtcarlone
34 changes: 34 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Dependabot configuration.
#
# Why this exists: every action in `.github/workflows/*.yml` is pinned to a
# commit SHA (not a tag) for supply-chain safety. That removes auto-updates,
# so Dependabot fills the gap by opening grouped PRs that re-resolve those
# SHAs from the action's latest stable tag. Each PR includes the action's
# changelog, so the maintainer can review what changed before merging.
#
# See specs/ci-rework/README.md §9 (shared hardening) for the SHA-pin rationale.

version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
day: "monday"
# Group all action updates into a single PR per week. Avoids three
# tiny PRs that all need the same review attention.
groups:
github-actions:
patterns: ["*"]
commit-message:
prefix: "ci"
include: "scope"
labels:
- "dependencies"
- "github-actions"

# NOTE: Python dependency updates (uv.lock / pyproject.toml) are
# intentionally NOT managed by Dependabot. dbt and adapter versions are
# part of the test surface area — bumps deserve a deliberate `tox.ini`
# change and matrix update, not an automated PR. Revisit if this gets
# cumbersome in practice.
70 changes: 70 additions & 0 deletions .github/workflows/cut-release-candidate.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
# Cut a release-candidate branch via `workflow_dispatch`.
#
# Auto-bumps the version in dbt_project.yml + README.md, creates a
# `release-candidate/X.Y.Z` branch, commits, and pushes. The push then
# triggers Tier 3 (release.yml) for the full warehouse × dbt-version
# matrix.
#
# Auto-bump, but NOT auto-merge. The maintainer reviews Tier 3 results
# and drives the tag (and any merge-back to main) manually. See
# docs/dev-workflow.md Stage 3.
#
# This workflow is a thin shim over scripts/release/cut-candidate.py.
# The same script can be run locally:
# ./scripts/release/cut-candidate.py --patch # or --minor / --major
# ./scripts/release/cut-candidate.py --version 2.11.0

name: Cut release-candidate

on:
workflow_dispatch:
inputs:
bump:
description: "Version bump (ignored if explicit_version is set)"
required: true
type: choice
default: "patch"
options:
- patch
- minor
- major
explicit_version:
description: "(optional) Explicit X.Y.Z; overrides bump"
required: false
type: string

permissions:
contents: write # needed to push the new branch

concurrency:
# Prevent two cutters running simultaneously and racing on the version.
group: cut-release-candidate
cancel-in-progress: false

jobs:
cut:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Checkout main
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with:
ref: main
# Need full history so `git checkout -b` and `git push` work.
fetch-depth: 0

- name: Configure git identity
run: |
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"

- name: Cut release-candidate branch
env:
BUMP: ${{ inputs.bump }}
EXPLICIT: ${{ inputs.explicit_version }}
run: |
if [[ -n "${EXPLICIT}" ]]; then
./scripts/release/cut-candidate.py --version "${EXPLICIT}"
else
./scripts/release/cut-candidate.py "--${BUMP}"
fi
163 changes: 163 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
# Tier 2 — Post-merge integration.
#
# Runs on every push to `main` (after a maintainer-reviewed PR merge), and on
# manual `workflow_dispatch`. This is the first workflow that touches
# repository secrets — Snowflake credentials and GCP Workload Identity. The
# trust boundary that protects those secrets is **branch protection on
# `main`**, which requires PR + approving review. By the time this workflow's
# event fires, a human has signed off on the code.
#
# See specs/ci-rework/README.md §5 (CI tier design) and §3 (threat model).
#
# Scope:
# - Everything Tier 1 runs (lint deferred from Tier 1 lands here)
# - Plus Snowflake + BigQuery on the latest supported dbt version
# - Full version matrix runs in Tier 3 (release.yml), not here
#
# Secret minimization:
# - The `integration-local` matrix declares no secrets at all — those jobs
# don't need them. Even though this workflow file has access to secrets,
# each job only sees the env vars it explicitly opts into.
# - Snowflake creds are scoped to `lint` and `integration-snowflake`.
# - GCP WIF only happens in `integration-bigquery`, gated on
# `id-token: write` at the job level.

name: Tier 2 — Post-merge integration

on:
push:
branches:
- main
workflow_dispatch:

permissions:
contents: read

# On `main`, don't cancel in-progress runs — each merge represents a
# distinct state that deserves its own full validation. Group still applies
# to prevent two runs racing for the same ref.
concurrency:
group: tier2-${{ github.ref }}
cancel-in-progress: false

env:
# Non-secret env vars consumed by the dbt invocations test model. These
# are the same values used pre-rework and are checked in to profiles.yml
# via env_var() — they need to be present, not secret.
DBT_CLOUD_PROJECT_ID: 123
DBT_CLOUD_JOB_ID: ABC
DBT_CLOUD_RUN_REASON: "String with 'quotes' !"
TEST_ENV_VAR_1: TEST_VALUE
TEST_ENV_VAR_NUMBER: 3
TEST_ENV_VAR_EMPTY: ""
TEST_ENV_VAR_WITH_QUOTE: "Triggered via Apache Airflow by task 'trigger_dbt_cloud_job_run' in the airtable_ingest DAG."
DBT_ENV_CUSTOM_ENV_FAVOURITE_DBT_PACKAGE: dbt_artifacts

jobs:
# -------- Lint -----------------------------------------------------------
# Lives here, not in Tier 1, because the package's models call adapter
# methods at compile time. Removing that requirement would let lint move
# to Tier 1 — tracked in specs/ci-rework/README.md §12.2.
lint:
name: lint (sqlfluff)
runs-on: ubuntu-latest
timeout-minutes: 15
env:
DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
- name: Set up uv
uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39
with:
enable-cache: true
- name: Install Python dependencies
run: ./scripts/ci/setup.sh
- name: Run sqlfluff lint
run: ./scripts/ci/lint.sh

# -------- Local-DWH integration -----------------------------------------
# Same matrix as Tier 1, run again post-merge on the merged code. Catches
# merge-resolution issues that PR-tier validation couldn't see.
integration-local:
name: integration (${{ matrix.warehouse }})
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
warehouse: [postgres, trino, sqlserver]
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
- name: Install Microsoft ODBC Driver 18 (sqlserver only)
if: matrix.warehouse == 'sqlserver'
run: |
sudo apt-get update
sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18
- name: Set up uv
uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39
with:
enable-cache: true
- name: Install Python dependencies
run: ./scripts/ci/setup.sh
- name: Run integration tests against ${{ matrix.warehouse }}
run: ./scripts/ci/test.sh ${{ matrix.warehouse }}

# -------- Snowflake (cloud) ---------------------------------------------
integration-snowflake:
name: integration (snowflake)
runs-on: ubuntu-latest
timeout-minutes: 30
env:
DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
- name: Set up uv
uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39
with:
enable-cache: true
- name: Install Python dependencies
run: ./scripts/ci/setup.sh
- name: Run integration tests against snowflake
run: ./scripts/ci/test.sh snowflake

# -------- BigQuery (cloud, WIF auth) ------------------------------------
integration-bigquery:
name: integration (bigquery)
runs-on: ubuntu-latest
timeout-minutes: 30
# WIF requires an OIDC token from the runner. Scoped to this job only —
# no other job in this workflow gets id-token: write.
permissions:
contents: read
id-token: write
env:
DBT_ENV_SECRET_GCP_PROJECT: ${{ secrets.GCP_PROJECT }}
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
- name: Authenticate to GCP via Workload Identity Federation
uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed
with:
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
- name: Set up uv
uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39
with:
enable-cache: true
- name: Install Python dependencies
run: ./scripts/ci/setup.sh
- name: Run integration tests against bigquery
run: ./scripts/ci/test.sh bigquery
58 changes: 0 additions & 58 deletions .github/workflows/main_lint_package.yml

This file was deleted.

Loading
Loading