Skip to content

Commit c1ddd69

Browse files
vikrantpuppalavikrantpuppala
committed
Merge remote-tracking branch 'origin/main' into vp/security-scan
2 parents 7a66f7f + 815b341 commit c1ddd69

31 files changed

Lines changed: 5004 additions & 537 deletions

‎.github/actions/setup-jfrog/action.yml‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,15 @@
11
name: Setup JFrog OIDC
2-
description: Obtain a JFrog access token via GitHub OIDC and configure pip to use JFrog PyPI proxy
2+
description: Obtain a JFrog access token via GitHub OIDC and configure pip / cargo to use JFrog package proxies
3+
4+
inputs:
5+
configure-cargo:
6+
description: |
7+
Write ~/.cargo/config.toml + credentials.toml pointing at the
8+
Databricks JFrog Cargo proxy. Required for any job that runs
9+
`cargo` on `databricks-protected-runner-group`, where direct
10+
access to index.crates.io is blocked. Off by default because
11+
most jobs in this repo are Python-only.
12+
default: "false"
313

414
runs:
515
using: composite
@@ -30,3 +40,34 @@ runs:
3040
set -euo pipefail
3141
echo "PIP_INDEX_URL=https://gha-service-account:${JFROG_ACCESS_TOKEN}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple" >> "$GITHUB_ENV"
3242
echo "pip configured to use JFrog registry"
43+
44+
- name: Configure Cargo
45+
if: inputs.configure-cargo == 'true'
46+
shell: bash
47+
# databricks-protected-runner-group blocks direct egress to
48+
# index.crates.io, so cargo must route through JFrog's
49+
# db-cargo-remote proxy. Mirrors the recipe used in
50+
# databricks-odbc's setup-jfrog action.
51+
#
52+
# Note: JFrog's Cargo proxy quarantines crates released within
53+
# the last 7 days. If a fresh dependency version isn't yet
54+
# mirrored, the build will fail until JFrog ingests it — bump
55+
# Cargo.lock to an older version or wait it out.
56+
run: |
57+
set -euo pipefail
58+
mkdir -p ~/.cargo
59+
cat > ~/.cargo/config.toml << 'EOF'
60+
[source.crates-io]
61+
replace-with = "jfrog"
62+
[source.jfrog]
63+
registry = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/"
64+
[registries.jfrog]
65+
index = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/"
66+
credential-provider = ["cargo:token"]
67+
EOF
68+
cat > ~/.cargo/credentials.toml << EOF
69+
[registries.jfrog]
70+
token = "Bearer ${JFROG_ACCESS_TOKEN}"
71+
EOF
72+
echo "CARGO_REGISTRIES_JFROG_TOKEN=Bearer ${JFROG_ACCESS_TOKEN}" >> "$GITHUB_ENV"
73+
echo "Cargo configured to use JFrog registry"

‎.github/actions/setup-poetry/action.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,21 @@ inputs:
1717
description: Extra suffix for the cache key to avoid collisions across job variants
1818
required: false
1919
default: ""
20+
configure-cargo:
21+
description: |
22+
Forwarded to setup-jfrog. Set to "true" for jobs that also need
23+
Cargo configured against the JFrog crates proxy (e.g. anything
24+
that builds a Rust extension via maturin).
25+
required: false
26+
default: "false"
2027

2128
runs:
2229
using: composite
2330
steps:
2431
- name: Setup JFrog
2532
uses: ./.github/actions/setup-jfrog
33+
with:
34+
configure-cargo: ${{ inputs.configure-cargo }}
2635

2736
- name: Set up python ${{ inputs.python-version }}
2837
id: setup-python

‎.github/workflows/code-coverage.yml‎

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,36 @@ permissions:
55
id-token: write
66

77
on:
8-
push:
9-
branches:
10-
- main
118
pull_request:
9+
merge_group:
1210
workflow_dispatch:
1311

12+
# `pull_request` gives the PR author fast feedback as they iterate.
13+
# `merge_group` runs the same suite against the queue's transient
14+
# branch (current main + the queued PR diff, freshly merged) and is
15+
# the run that actually protects main — by the time `push:main` fires,
16+
# the merge has already happened and the coverage check has no power
17+
# to block. Hence we deliberately don't subscribe to `push:main`.
18+
#
19+
# Concurrency groups:
20+
# - pull_request: per-ref + cancel-in-progress. A force-push or fast
21+
# follow-up commit on a PR cancels the previous run instead of
22+
# racing it against shared warehouse state (Delta tables, UC Volume
23+
# files, telemetry endpoints, etc.).
24+
# - merge_group: serialised globally with a fixed group name. The
25+
# warehouse can't tolerate two parallel queue entries hammering
26+
# telemetry / retry paths simultaneously — we have observed flaky
27+
# retry-test failures (extra `/telemetry-ext` retries inflating
28+
# mock.call_count) under that load. Running queue entries one at a
29+
# time costs queue throughput (one entry at a time, ~17 min each)
30+
# but keeps signal trustworthy. cancel-in-progress is off so each
31+
# entry gets a complete run.
32+
# - workflow_dispatch: shares the merge_group group; manual triggers
33+
# are rare enough that serialising them with the queue is fine.
34+
concurrency:
35+
group: ${{ github.event_name == 'pull_request' && format('e2e-pr-{0}', github.ref) || 'e2e-mq-serial' }}
36+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
37+
1438
jobs:
1539
test-with-coverage:
1640
runs-on:
@@ -39,8 +63,21 @@ jobs:
3963
install-args: "--all-extras"
4064
- name: Run all tests with coverage
4165
continue-on-error: false
66+
# This job installs --all-extras, so the REAL databricks-sql-kernel
67+
# wheel is present. The unit suite fakes databricks_sql_kernel in
68+
# sys.modules, which would shadow the real wheel in this shared
69+
# session — so the kernel-backed suites that need the real wheel
70+
# are excluded here and covered by the dedicated kernel-e2e.yml
71+
# (isolated session, real wheel, live warehouse):
72+
# - --ignore the kernel e2e files (they assert the real wheel and
73+
# now FAIL LOUDLY rather than silently skip if shadowed), and
74+
# - -m "not realkernel" deselects the no-network real-wheel
75+
# routing test for the same reason.
4276
run: |
4377
poetry run pytest tests/unit tests/e2e \
78+
--ignore=tests/e2e/test_kernel_backend.py \
79+
--ignore=tests/e2e/test_kernel_tls.py \
80+
-m "not realkernel" \
4481
-n 4 \
4582
--dist=loadgroup \
4683
--cov=src \
@@ -50,6 +87,11 @@ jobs:
5087
- name: Check for coverage override
5188
id: override
5289
env:
90+
# PR_BODY is empty on `merge_group` (no pull_request payload).
91+
# That's intentional — coverage overrides are an author-time
92+
# escape hatch, not a queue-time bypass, so the queue run
93+
# always enforces the threshold regardless of the PR's
94+
# SKIP_COVERAGE_CHECK marker.
5395
PR_BODY: ${{ github.event.pull_request.body }}
5496
run: |
5597
OVERRIDE_COMMENT=$(echo "$PR_BODY" | grep -E "SKIP_COVERAGE_CHECK\s*=" || echo "")

‎.github/workflows/code-quality-checks.yml‎

Lines changed: 57 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ jobs:
4848
echo "=== Dependency Version: ${{ matrix.dependency-version }} ==="
4949
poetry run pip list
5050
- name: Run tests
51-
run: poetry run python -m pytest tests/unit
51+
run: poetry run python -m pytest tests/unit -m "not realkernel"
5252

5353
run-unit-tests-with-arrow:
5454
runs-on:
@@ -77,7 +77,11 @@ jobs:
7777
uses: ./.github/actions/setup-poetry
7878
with:
7979
python-version: ${{ matrix.python-version }}
80-
install-args: "--all-extras"
80+
# Install ONLY the pyarrow extra (not --all-extras) so this
81+
# tier isolates the "pyarrow present, kernel absent"
82+
# configuration. --all-extras would also pull the kernel wheel,
83+
# making this job redundant with "Unit Tests + Kernel".
84+
install-args: "--extras pyarrow"
8185
cache-suffix: "pyarrow-${{ matrix.dependency-version }}-"
8286
- name: Install Python tools for custom versions
8387
if: matrix.dependency-version != 'default'
@@ -96,7 +100,57 @@ jobs:
96100
echo "=== Dependency Version: ${{ matrix.dependency-version }} with PyArrow ==="
97101
poetry run pip list
98102
- name: Run tests
99-
run: poetry run python -m pytest tests/unit
103+
run: poetry run python -m pytest tests/unit -m "not realkernel"
104+
105+
run-unit-tests-with-kernel:
106+
runs-on:
107+
group: databricks-protected-runner-group
108+
labels: linux-ubuntu-latest
109+
strategy:
110+
matrix:
111+
# Kernel wheel is cp310-abi3 (Requires-Python >=3.10), so this
112+
# matrix omits 3.9 — the [kernel] extra is a no-op there.
113+
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
114+
115+
name: "Unit Tests + Kernel (Python ${{ matrix.python-version }})"
116+
117+
steps:
118+
- name: Check out repository
119+
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
120+
- name: Install Kerberos system dependencies
121+
run: |
122+
sudo apt-get update
123+
sudo apt-get install -y libkrb5-dev
124+
- name: Setup Poetry
125+
uses: ./.github/actions/setup-poetry
126+
with:
127+
python-version: ${{ matrix.python-version }}
128+
# Install the kernel extra (pulls the published
129+
# databricks-sql-kernel wheel, which transitively brings
130+
# pyarrow). Explicit --extras kernel rather than --all-extras
131+
# so this tier targets the kernel configuration specifically.
132+
install-args: "--extras kernel"
133+
cache-suffix: "kernel-"
134+
- name: Show installed versions
135+
run: |
136+
echo "=== with databricks-sql-kernel ==="
137+
poetry run pip list
138+
- name: Assert the real kernel wheel is installed (not a stub)
139+
run: |
140+
poetry run python -c "import databricks_sql_kernel as k; assert k.__file__, 'kernel wheel missing __file__ — not the real wheel'; print('real kernel wheel:', k.__file__)"
141+
- name: Unit tests (kernel wheel present, realkernel deselected)
142+
# The bulk of tests/unit fakes databricks_sql_kernel in
143+
# sys.modules, so the real-wheel routing test is deselected here
144+
# and run on its own below (a shared session would shadow the
145+
# real wheel — both real-wheel tests fail loudly if that happens).
146+
run: poetry run python -m pytest tests/unit -m "not realkernel"
147+
- name: Drive use_kernel=True through the REAL wheel (routing)
148+
# Separate invocation, explicit file path: never collects the
149+
# fake-module test file, so sys.modules stays unpolluted. This is
150+
# the no-network proof that sql.connect(use_kernel=True) actually
151+
# instantiates the real KernelDatabricksClient (not a stub, not a
152+
# Thrift fallback). Fails loudly if the real wheel is shadowed.
153+
run: poetry run python -m pytest tests/unit/test_session.py -m realkernel -v
100154

101155
check-linting:
102156
runs-on:

0 commit comments

Comments
 (0)