diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 8eff32c6..9aa7c90b 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -63,6 +63,17 @@ jobs: run: pip install -e cli/ shell: bash + - name: Sign in to the box access gateway + # The box is fronted by an authenticating gateway. The CLI stores the + # session in ~/.lager_gateway_auth and silently refreshes it for the + # rest of the run, so one login covers the whole job. + env: + LAGER_AUTH_URL: ${{ vars.LAGER_AUTH_URL }} + LAGER_CI_EMAIL: ${{ secrets.LAGER_CI_EMAIL }} + LAGER_CI_PASSWORD: ${{ secrets.LAGER_CI_PASSWORD }} + run: lager login "$LAGER_AUTH_URL" --email "$LAGER_CI_EMAIL" --password "$LAGER_CI_PASSWORD" + shell: bash + - name: Register lager box run: | lager boxes add \ @@ -82,7 +93,10 @@ jobs: shell: bash - name: Verify box connectivity - run: lager hello --box "$LAGER_BOX" + # First authenticated contact after a fresh login can fail exactly once + # while the gateway records the box<->auth-server link; the retry is + # then expected to pass. + run: lager hello --box "$LAGER_BOX" || lager hello --box "$LAGER_BOX" shell: bash - name: Verify update probe (non-fatal) diff --git a/.github/workflows/update-regression.yml b/.github/workflows/update-regression.yml index 1c0400d4..c979e30b 100644 --- a/.github/workflows/update-regression.yml +++ b/.github/workflows/update-regression.yml @@ -1,16 +1,29 @@ # Copyright 2024-2026 Lager Data # SPDX-License-Identifier: Apache-2.0 # -# Weekly "clean rebuild" regression: force a full `lager update` on the shared -# self-hosted bench and confirm the box comes back healthy with a smoke test. -# Distinct from integration-tests.yml (which gates current code on hardware) — -# this verifies that a from-scratch box rebuild still works. +# Weekly box-lifecycle regression on the shared self-hosted bench. Exercises +# every lifecycle path a user actually runs, in an order where each step sets +# up the next and the box ends the run freshly installed on current main: +# +# dry run (--check) -> downgrade to the previous release tag -> common-path +# `lager update` (N-1 -> main, the path users hit most) -> no-op update +# (asserts the "already at version" fast path, incl. its container-liveness +# gate) -> forced clean rebuild -> hardware smoke -> uninstall -> install +# from scratch -> final hardware smoke. +# +# Uninstall scope is deliberate and load-bearing: +# - NEVER `--all`: it removes this runner's key from the box's +# authorized_keys, and CI has no TTY for install's password fallback — +# the bench would be stranded until someone re-keys it by hand. +# - ALWAYS `--keep-config`: /etc/lager holds the saved bench nets AND +# control_plane.json (the box's auth-gateway enrollment). Wiping either +# breaks the rest of the run and the box's gateway auth. # # Shares integration-tests.yml's concurrency group so it never runs at the same # time as the hardware suites against the one physical bench. Schedule + manual # dispatch only (public repo + self-hosted runner => no pull_request trigger). -name: Update Regression +name: Box Lifecycle on: schedule: @@ -23,10 +36,10 @@ concurrency: cancel-in-progress: false jobs: - update-regression: - name: Full lager update on the bench + box-lifecycle: + name: Box lifecycle on the bench runs-on: [self-hosted, lager-bench] - timeout-minutes: 30 + timeout-minutes: 90 env: LAGER_BOX: ${{ vars.LAGER_BOX || 'MASTER' }} @@ -44,6 +57,17 @@ jobs: run: pip install -e cli/ shell: bash + - name: Sign in to the box access gateway + # The box is fronted by an authenticating gateway. The CLI stores the + # session in ~/.lager_gateway_auth and silently refreshes it for the + # rest of the run, so one login covers the whole job. + env: + LAGER_AUTH_URL: ${{ vars.LAGER_AUTH_URL }} + LAGER_CI_EMAIL: ${{ secrets.LAGER_CI_EMAIL }} + LAGER_CI_PASSWORD: ${{ secrets.LAGER_CI_PASSWORD }} + run: lager login "$LAGER_AUTH_URL" --email "$LAGER_CI_EMAIL" --password "$LAGER_CI_PASSWORD" + shell: bash + - name: Register lager box run: | lager boxes add \ @@ -53,18 +77,193 @@ jobs: --yes shell: bash - - name: Run full update (force rebuild) - run: lager update --force --box "$LAGER_BOX" + - name: Set up lager SSH key + run: | + if [ ! -f ~/.ssh/lager_box ]; then + ssh-keygen -t ed25519 -f ~/.ssh/lager_box -N "" -C "lager-box-access" + fi + grep -qF "$(cat ~/.ssh/lager_box.pub)" ~/.ssh/authorized_keys 2>/dev/null || \ + cat ~/.ssh/lager_box.pub >> ~/.ssh/authorized_keys + # install/uninstall probe SSH with the DEFAULT identity (plain + # `ssh user@ip`), unlike update which passes the lager key + # explicitly. Point default ssh at the key for this host so the + # reinstall phase authenticates non-interactively. + touch ~/.ssh/config && chmod 600 ~/.ssh/config + if ! grep -q "^# lager lifecycle CI box identity" ~/.ssh/config; then + { + echo "# lager lifecycle CI box identity" + echo "Host ${{ secrets.LAGER_BOX_IP }}" + echo " IdentityFile ~/.ssh/lager_box" + } >> ~/.ssh/config + fi + shell: bash + + - name: Verify box connectivity + # First authenticated contact after a fresh login can fail exactly once + # while the gateway records the box<->auth-server link; the retry is + # then expected to pass. + run: lager hello --box "$LAGER_BOX" || lager hello --box "$LAGER_BOX" + shell: bash + + - name: Determine previous release tag + # The downgrade target: second-newest v* tag, i.e. what a slightly + # stale fielded box would be running. Floored at v0.32.0 — a current + # CLI cannot drive a pre-migration box (hardware-service :9000 + + # gateway auth landed in v0.32.0), so older targets would test an + # unsupported skew, not a real upgrade path. + run: | + PREV_TAG=$(git ls-remote --tags --refs origin 'v*' \ + | awk -F/ '{print $NF}' | sort -V | tail -2 | head -1) + PREV_TAG=$(printf '%s\n' "$PREV_TAG" v0.32.0 | sort -V | tail -1) + echo "Previous release tag: $PREV_TAG" + echo "PREV_TAG=$PREV_TAG" >> "$GITHUB_ENV" + shell: bash + + - name: Update dry run (--check) + # rc 0 = up to date, rc 1 = update available; both fine. --check must + # not mutate the box; the phases below prove the real paths. + run: | + rc=0; lager update --check --box "$LAGER_BOX" || rc=$? + [ "$rc" -le 1 ] || exit "$rc" + shell: bash + + - name: Downgrade to previous release (version pin + rollback path) + run: lager update --box "$LAGER_BOX" --version "$PREV_TAG" --yes --verbose + shell: bash + + - name: Verify box healthy on previous release + run: lager hello --box "$LAGER_BOX" shell: bash - - name: Verify box recovered + - name: Common-path update (previous release -> main) + # The update invocation users actually run. If a release breaks + # upgrading from the previous release, THIS step goes red. + run: lager update --box "$LAGER_BOX" --yes --verbose + shell: bash + + - name: Verify box healthy on main run: lager hello --box "$LAGER_BOX" shell: bash - - name: Post-update hardware smoke test + - name: No-op update hits the fast path + # Re-running with the box already current must take the "already at + # version" early-exit (which also verifies the container-liveness and + # version-file reconciliation logic behind it). + run: | + set -o pipefail + lager update --box "$LAGER_BOX" --yes | tee "$RUNNER_TEMP/noop-update.log" + grep -q "already at version" "$RUNNER_TEMP/noop-update.log" + shell: bash + + - name: Forced clean rebuild (--force) + run: lager update --force --box "$LAGER_BOX" --yes --verbose + shell: bash + + - name: Verify box healthy after rebuild + run: lager hello --box "$LAGER_BOX" + shell: bash + + - name: Hardware smoke after update phases + # Separate smoke per phase so a red run attributes cleanly to + # update-vs-install. + env: + SUPPLY_NET: ${{ vars.RIGOL_CH1_NET || 'supply2' }} + CHANNEL_MAX_VOLTAGE: ${{ vars.RIGOL_CH1_MAX_VOLTAGE || '60.0' }} + CHANNEL_MAX_CURRENT: ${{ vars.RIGOL_CH1_MAX_CURRENT || '1.0' }} + run: lager python test/api/power/test_supply_Rigol_DP821.py --box "$LAGER_BOX" + shell: bash + + - name: Uninstall box software (scoped — see header) + run: lager uninstall --box "$LAGER_BOX" --yes --keep-config + shell: bash + + - name: Install box software from scratch + # By --ip: uninstall --yes removes the local box entry as part of its + # cleanup, so --box would no longer resolve here. + run: | + lager install \ + --ip "${{ secrets.LAGER_BOX_IP }}" \ + --user lagerdata \ + --version main \ + --yes + shell: bash + + - name: Re-register lager box + # install --yes deliberately skips the add-box prompt. + run: | + lager boxes add \ + --name "$LAGER_BOX" \ + --ip "${{ secrets.LAGER_BOX_IP }}" \ + --user lagerdata \ + --yes + shell: bash + + - name: Release the install auto-lock (workaround) + # install's lock release can be lost when the deploy restarts the + # box's docker daemon (the single-shot release POST races the + # gateway coming back up), leaving the box locked against its own + # job for the full lock TTL. Force-release our own leaked lock + # before verifying. Product follow-up tracked: the release should + # retry, and a same-job unlock should match the CI holder identity + # without --force. + run: lager boxes unlock --box "$LAGER_BOX" --force 2>/dev/null || true + shell: bash + + - name: Verify box healthy after reinstall + # Fresh containers and the gateway need a settling window after the + # deploy's docker-daemon restart; poll for up to two minutes. + run: | + for _ in $(seq 1 12); do + lager hello --box "$LAGER_BOX" && exit 0 + sleep 10 + done + echo "Box did not become healthy within 120s of reinstall" + exit 1 + shell: bash + + - name: Hardware smoke after reinstall env: SUPPLY_NET: ${{ vars.RIGOL_CH1_NET || 'supply2' }} CHANNEL_MAX_VOLTAGE: ${{ vars.RIGOL_CH1_MAX_VOLTAGE || '60.0' }} CHANNEL_MAX_CURRENT: ${{ vars.RIGOL_CH1_MAX_CURRENT || '1.0' }} run: lager python test/api/power/test_supply_Rigol_DP821.py --box "$LAGER_BOX" shell: bash + + - name: Recovery — leave the bench usable on failure + # The uninstall->install window is the one place a failure leaves the + # box with nothing. One best-effort reinstall converts "transient + # failure strands hardware CI for a week" into "red run, healthy + # bench". The run itself stays red. + # + # cancelled() is included deliberately: a run cancelled mid-install + # leaves the box stripped, and plain failure() does not fire on + # cancellation — the one observed bench stranding happened exactly + # this way. Prefer letting this workflow fail over cancelling it + # once the uninstall step has started. + if: failure() || cancelled() + run: | + lager install \ + --ip "${{ secrets.LAGER_BOX_IP }}" \ + --user lagerdata \ + --version main \ + --yes || true + lager boxes add \ + --name "$LAGER_BOX" \ + --ip "${{ secrets.LAGER_BOX_IP }}" \ + --user lagerdata \ + --yes || true + lager boxes unlock --box "$LAGER_BOX" --force 2>/dev/null || true + lager hello --box "$LAGER_BOX" || true + shell: bash + + - name: Best-effort lock cleanup + # install/`lager python` auto-lock the box; if a step died in a way + # that bypassed the CLI's release hook, free the lock rather than + # waiting out the server TTL. --force because the lock identity + # embeds the acquiring PID — a plain unlock from this (different) + # process is refused as a different holder, observed as a 403 in + # the box log. This is the end of the job on a CI-owned bench; + # there is no other holder to stomp. + if: always() + run: lager boxes unlock --box "$LAGER_BOX" --force 2>/dev/null || true + shell: bash