diff --git a/.github/review-tooling/review_dependabot_major.py b/.github/review-tooling/review_dependabot_major.py index 318e32a..5b2497b 100644 --- a/.github/review-tooling/review_dependabot_major.py +++ b/.github/review-tooling/review_dependabot_major.py @@ -16,10 +16,16 @@ before the merge lands, so a bump that breaks CI never merges regardless of this verdict. -Inference runs on **GitHub Models** (https://models.github.ai/inference) using the -workflow's built-in GITHUB_TOKEN + `models: read` permission -- no external API -key. The OpenAI SDK is only the client library; it targets the GitHub Models -endpoint, not OpenAI. Model defaults to `openai/gpt-5` (override via MODEL_ID). +Inference runs on **GitHub Models** (https://models.github.ai/inference). The +OpenAI SDK is only the client library; it targets the GitHub Models endpoint, not +OpenAI. Model defaults to `openai/gpt-4.1` (override via MODEL_ID). + +Auth: MODEL_TOKEN if set, else the workflow's built-in GITHUB_TOKEN (`models: +read`). The built-in token only works where the org is entitled to Models +inference. The realrate org is not: its Actions token reads the model catalogue +(HTTP 200) but every inference call returns a bare 403 with an empty body, +regardless of model or tier. Hence MODEL_TOKEN, a PAT whose account does have +inference. Background and evidence: realrate/.github#14. Contract: ALWAYS writes a valid verdict JSON and exits 0. On any error (missing token, API failure, unparseable response) it emits verdict="human" so the @@ -34,11 +40,18 @@ import re import sys -MODEL = os.getenv("MODEL_ID", "openai/gpt-5") +# gpt-4.1 is the strongest model that actually works here: the gpt-5 entries are +# reasoning-tier and reject this script's `max_tokens` (BadRequestError), so the +# old gpt-5 default failed for anyone not overriding MODEL_ID. +MODEL = os.getenv("MODEL_ID", "openai/gpt-4.1") # GitHub Models inference endpoint (OpenAI-compatible). Override for tests. BASE_URL = os.getenv("MODEL_ENDPOINT", "https://models.github.ai/inference") -# The built-in Actions token authenticates GitHub Models (needs `models: read`). -TOKEN = os.getenv("GITHUB_TOKEN") or os.getenv("MODEL_TOKEN") +# Inference token. MODEL_TOKEN wins on purpose: GITHUB_TOKEN is always set in +# Actions, so checking it first would silently ignore an explicitly supplied PAT. +# The built-in token only reaches Models where the org is entitled to inference +# (realrate is not -- it reads the catalogue but 403s every inference call, see +# realrate/.github#14), so MODEL_TOKEN is the working path there. +TOKEN = os.getenv("MODEL_TOKEN") or os.getenv("GITHUB_TOKEN") SYSTEM_PROMPT = ( "You classify whether a MAJOR version bump of a software dependency is safe " @@ -87,6 +100,24 @@ def _fallback(reason): return {"verdict": "human", "risk": "unknown", "reason": reason} +def _explain(exc): + """Turn an SDK exception into something a maintainer can act on. + + GitHub Models answers a token without inference entitlement with a bare 403 + and an empty body, so the exception name alone ("PermissionDeniedError") sent + people hunting through workflow permissions for hours. Name the likely cause. + """ + name = type(exc).__name__ + if getattr(exc, "status_code", None) == 403 or "PermissionDenied" in name: + source = "MODEL_TOKEN" if os.getenv("MODEL_TOKEN") else "GITHUB_TOKEN" + return ( + f"403 from GitHub Models via {source} -- the account behind that token " + "has no inference entitlement (catalogue reads succeed, inference is " + "refused). Not a workflow-permission problem. See realrate/.github#14" + ) + return name + + def _extract_json(text): """Parse a JSON object from a model response, tolerating ```json fences/prose.""" text = (text or "").strip() @@ -116,7 +147,7 @@ def _call_model(client, user, structured): def get_verdict(dep_names, title, body, ecosystem=""): """Ask GitHub Models for a validated verdict dict. Never raises.""" if not TOKEN: - return _fallback("GITHUB_TOKEN not set (need `models: read`); manual review.") + return _fallback("no inference token (MODEL_TOKEN/GITHUB_TOKEN); manual review.") try: from openai import OpenAI except Exception as exc: # noqa: BLE001 - import guard @@ -142,7 +173,7 @@ def get_verdict(dep_names, title, body, ecosystem=""): if data: break except Exception as exc: # noqa: BLE001 - any API/parse failure - last = type(exc).__name__ + last = _explain(exc) if not data: return _fallback(f"AI verdict unavailable ({locals().get('last', 'no JSON')}); manual review.") diff --git a/.github/review-tooling/selftest-fixtures/breaking/body.md b/.github/review-tooling/selftest-fixtures/breaking/body.md new file mode 100644 index 0000000..a15d020 --- /dev/null +++ b/.github/review-tooling/selftest-fixtures/breaking/body.md @@ -0,0 +1,34 @@ +Bumps [numpy](https://github.com/numpy/numpy) from 1.26.4 to 2.5.0. + +## Release notes + +### NumPy 2.5.0 Release Notes + +This release drops support for Python 3.11; Python >=3.12 is now required. + +**Breaking changes** + +* Python 3.11 is no longer supported. Building or installing on 3.11 will fail. +* `numpy.distutils` has been removed. Projects still importing it must migrate + to `meson-python` or `setuptools`. +* Several long-deprecated aliases have now expired and been removed, including + `np.float_`, `np.unicode_` and `np.NaN`. Use `np.float64`, `np.str_` and + `np.nan` instead. +* The C ABI has changed. Extension modules and packages compiled against the + numpy 1.x ABI must be rebuilt against 2.x, or they will fail to import. +* `copy=False` in `np.array` now raises instead of silently copying. + +**Improvements** + +* Faster sorting for small integer types. +* Improved error messages for shape mismatches. + +--- +updated-dependencies: +- dependency-name: numpy + dependency-version: 2.5.0 + dependency-type: direct:production + update-type: version-update:semver-major +... + +Signed-off-by: dependabot[bot] diff --git a/.github/review-tooling/selftest-fixtures/breaking/title.txt b/.github/review-tooling/selftest-fixtures/breaking/title.txt new file mode 100644 index 0000000..86f26e7 --- /dev/null +++ b/.github/review-tooling/selftest-fixtures/breaking/title.txt @@ -0,0 +1 @@ +Bump numpy from 1.26.4 to 2.5.0 diff --git a/.github/review-tooling/selftest-fixtures/breaking/verdict.json b/.github/review-tooling/selftest-fixtures/breaking/verdict.json new file mode 100644 index 0000000..ad69e53 --- /dev/null +++ b/.github/review-tooling/selftest-fixtures/breaking/verdict.json @@ -0,0 +1 @@ +{"verdict": "human", "risk": "unknown", "reason": "AI verdict unavailable (AuthenticationError); manual review."} \ No newline at end of file diff --git a/.github/review-tooling/selftest-fixtures/safe/body.md b/.github/review-tooling/selftest-fixtures/safe/body.md new file mode 100644 index 0000000..8f3f9eb --- /dev/null +++ b/.github/review-tooling/selftest-fixtures/safe/body.md @@ -0,0 +1,27 @@ +Bumps [some-action](https://github.com/example/some-action) from 4 to 5. + +## Release notes + +### v5 + +This release contains no changes to the action's inputs, outputs or behaviour. + +* Bump the Node runtime used internally from node20 to node24. Consumers do not + need to change anything; the action's `with:` inputs are unchanged. +* Update internal dependencies. +* Documentation: clarify the caching example in the README. +* Add support for the linux-arm64 runner image. +* Internal refactor of the logging helper. No user-visible effect. + +No inputs were removed or renamed. No outputs were removed or renamed. There are +no deprecations in this release. + +--- +updated-dependencies: +- dependency-name: example/some-action + dependency-version: '5' + dependency-type: direct:production + update-type: version-update:semver-major +... + +Signed-off-by: dependabot[bot] diff --git a/.github/review-tooling/selftest-fixtures/safe/title.txt b/.github/review-tooling/selftest-fixtures/safe/title.txt new file mode 100644 index 0000000..16a0e0e --- /dev/null +++ b/.github/review-tooling/selftest-fixtures/safe/title.txt @@ -0,0 +1 @@ +Bump some-action from 4 to 5 diff --git a/.github/review-tooling/selftest-fixtures/safe/verdict.json b/.github/review-tooling/selftest-fixtures/safe/verdict.json new file mode 100644 index 0000000..ad69e53 --- /dev/null +++ b/.github/review-tooling/selftest-fixtures/safe/verdict.json @@ -0,0 +1 @@ +{"verdict": "human", "risk": "unknown", "reason": "AI verdict unavailable (AuthenticationError); manual review."} \ No newline at end of file diff --git a/.github/workflows/dependabot-automerge.yaml b/.github/workflows/dependabot-automerge.yaml index 2a67f11..b00a8a6 100644 --- a/.github/workflows/dependabot-automerge.yaml +++ b/.github/workflows/dependabot-automerge.yaml @@ -127,9 +127,14 @@ jobs: steps.metadata.outputs.update-type == 'version-update:semver-major' && github.event.action == 'opened' env: - # Inference runs on GitHub Models via the built-in token (`models: read`); - # no external API key. GITHUB_TOKEN also authenticates the `gh` calls below. + # Authenticates the `gh` calls below, and is the inference fallback. GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Inference token, preferred over GITHUB_TOKEN by the script. The org's + # built-in token can read the Models catalogue but 403s every inference + # call (realrate/.github#14), so a PAT whose account has Models access + # does the inference. Unset -> script falls back to GITHUB_TOKEN -> 403 + # -> verdict=human, i.e. exactly today's safe behaviour, never a merge. + MODEL_TOKEN: ${{ secrets.MODELS_PAT }} # gpt-4.1 is the strongest model actually available for GitHub Models # inference (gpt-5* are cataloged but return `unavailable_model`). MODEL_ID: openai/gpt-4.1 diff --git a/.github/workflows/models-selftest.yaml b/.github/workflows/models-selftest.yaml new file mode 100644 index 0000000..8a6a73c --- /dev/null +++ b/.github/workflows/models-selftest.yaml @@ -0,0 +1,188 @@ +name: models-selftest + +# Proves the AI risk gate's inference path actually works, on demand, without +# waiting for a real major Dependabot bump to come along. +# +# gh workflow run models-selftest.yaml --repo realrate/.github +# (or: Actions -> models-selftest -> Run workflow) +# +# Use it to answer "did the MODELS_PAT secret fix the 403?" (realrate/.github#14) +# and, on every change to the gate's script, to catch a regression early. +# +# Deliberately the INVERSE of the gate's error posture. The gate fails SAFE: any +# error becomes verdict=human, because a broken AI must never merge anything. +# That is also why a broken AI stayed invisible for weeks. This job fails LOUD, +# so breakage shows up as a red run instead of silently routing every major bump +# to a human forever. + +on: + workflow_dispatch: + inputs: + model_id: + description: "Model to test (must exist in the GitHub Models catalogue)" + default: openai/gpt-4.1 + type: string + # Regression cover: re-run whenever the gate's inference path or this test changes. + pull_request: + paths: + - .github/review-tooling/review_dependabot_major.py + - .github/workflows/models-selftest.yaml + +permissions: + contents: read + models: read # lets the built-in token reach GitHub Models, for the comparison probe + +jobs: + selftest: + runs-on: ubuntu-latest + env: + MODEL_ID: ${{ inputs.model_id || 'openai/gpt-4.1' }} + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + + # Absent secret must not paint PRs red -- it is the expected state until an + # admin adds it. Report and stop cleanly instead. + - name: Check whether MODELS_PAT is configured + id: cfg + env: + # MODELS_PAT_TEST lets a new token be trialled without touching the + # production MODELS_PAT secret; falls back to MODELS_PAT when unset. + MODELS_PAT: ${{ secrets.MODELS_PAT_TEST || secrets.MODELS_PAT }} + run: | + if [ -z "$MODELS_PAT" ]; then + echo "configured=false" >> "$GITHUB_OUTPUT" + { + echo "### models-selftest: skipped" + echo + echo "\`MODELS_PAT\` is not configured, so there is nothing to test yet." + echo "The gate falls back to the built-in token, gets a 403, and routes" + echo "every major bump to a human -- today's expected behaviour." + echo + echo "To enable: add an **org** secret \`MODELS_PAT\` (fine-grained PAT," + echo "\`Models: read\` only, visibility **all repositories**), then re-run" + echo "this workflow. See realrate/.github#14." + } >> "$GITHUB_STEP_SUMMARY" + echo "::notice::MODELS_PAT not configured -- skipping (this is not a failure)" + else + echo "configured=true" >> "$GITHUB_OUTPUT" + fi + + # Side-by-side proof of which credential can actually do inference. This is + # the exact comparison that took a full session to establish by hand. + - name: Probe inference with each credential + if: steps.cfg.outputs.configured == 'true' + env: + BUILTIN: ${{ secrets.GITHUB_TOKEN }} + PAT: ${{ secrets.MODELS_PAT_TEST || secrets.MODELS_PAT }} + run: | + probe() { # $1=label $2=token + code=$(curl -s -o /tmp/b.json -w '%{http_code}' \ + https://models.github.ai/inference/chat/completions \ + -H "Authorization: Bearer $2" -H "Content-Type: application/json" \ + -d "{\"model\":\"$MODEL_ID\",\"messages\":[{\"role\":\"user\",\"content\":\"say OK\"}],\"max_tokens\":5}") + echo "$code" + } + builtin_code=$(probe "GITHUB_TOKEN" "$BUILTIN") + pat_code=$(probe "MODELS_PAT" "$PAT") + echo "GITHUB_TOKEN -> $builtin_code | MODELS_PAT -> $pat_code" + { + echo "### models-selftest" + echo + echo "Model: \`$MODEL_ID\`" + echo + echo "| credential | inference |" + echo "|---|---|" + echo "| built-in \`GITHUB_TOKEN\` | $builtin_code |" + echo "| \`MODELS_PAT\` | $pat_code |" + } >> "$GITHUB_STEP_SUMMARY" + + if [ "$pat_code" != "200" ]; then + echo "::error::MODELS_PAT cannot run inference (HTTP $pat_code). The PAT needs 'Models: read' and its account needs Models access." + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "❌ **MODELS_PAT failed inference (HTTP $pat_code)** -- the gate cannot produce verdicts." >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + if [ "$builtin_code" = "200" ]; then + echo "::notice::The built-in token now works too -- org entitlement looks fixed, so MODELS_PAT may no longer be needed (see realrate/.github#14)." + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "ℹ️ The built-in token also returned 200. If that holds, the org is entitled to inference and \`MODELS_PAT\` can be retired." >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Set up uv + if: steps.cfg.outputs.configured == 'true' + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + + # End-to-end through the REAL script, on fixtures with known-correct answers. + - name: Run the real verdict script on fixture bumps + if: steps.cfg.outputs.configured == 'true' + env: + MODEL_TOKEN: ${{ secrets.MODELS_PAT_TEST || secrets.MODELS_PAT }} + run: | + fail=0 + { + echo "" + echo "| fixture | expected | verdict | risk | reason |" + echo "|---|---|---|---|---|" + } >> "$GITHUB_STEP_SUMMARY" + + for spec in "breaking:human:numpy:pip" "safe:auto:example/some-action:github_actions"; do + name="${spec%%:*}"; rest="${spec#*:}" + expect="${rest%%:*}"; rest="${rest#*:}" + deps="${rest%%:*}"; eco="${rest#*:}" + dir=".github/review-tooling/selftest-fixtures/$name" + + # Run inline, NOT in a subshell: a failure must be able to fail this job. + uv run --script .github/review-tooling/review_dependabot_major.py \ + --title-file "$dir/title.txt" --body-file "$dir/body.md" \ + --dep-names "$deps" --ecosystem "$eco" --out "$dir/verdict.json" || true + + if [ ! -s "$dir/verdict.json" ]; then + echo "::error::$name: the script produced no verdict file at all" + echo "| $name | $expect | ❌ none | - | script produced no verdict |" >> "$GITHUB_STEP_SUMMARY" + fail=1 + continue + fi + + verdict=$(jq -r '.verdict // ""' "$dir/verdict.json") + risk=$(jq -r '.risk // ""' "$dir/verdict.json") + reason=$(jq -r '.reason // ""' "$dir/verdict.json") + + # HARD requirement: a well-formed verdict is what proves inference ran. + # The script fails safe, so it emits verdict=human on an API error too -- + # hence checking the reason for our own fallback wording, otherwise a + # total outage would masquerade as a correct "human" verdict here. + case "$verdict" in + auto|human) ;; + *) echo "::error::$name: invalid verdict '$verdict'"; fail=1; continue ;; + esac + # _fallback() is the only thing that emits risk="unknown" AND a reason + # ending "manual review." Matching both keeps a genuine model verdict + # that happens to mention manual review from tripping this. + if [ "$risk" = "unknown" ] && [ "${reason%manual review.}" != "$reason" ]; then + echo "::error::$name: verdict came from the error fallback, not the model: $reason" + echo "| $name | $expect | ❌ fallback | $risk | ${reason:0:130} |" >> "$GITHUB_STEP_SUMMARY" + fail=1 + continue + fi + + # SOFT check: the model is non-deterministic, so a differing judgement is + # reported, not fatal. Only a malformed/fallback verdict fails this job. + mark="✅" + if [ "$verdict" != "$expect" ]; then + mark="⚠️" + echo "::warning::$name: expected verdict=$expect but got $verdict -- judgement drift, not an outage. Reason: $reason" + fi + echo "| $name | $expect | $mark $verdict | $risk | ${reason:0:130} |" >> "$GITHUB_STEP_SUMMARY" + echo "$name -> verdict=$verdict risk=$risk" + done + + if [ "$fail" != "0" ]; then + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "❌ **The gate could not produce a valid verdict.**" >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + { + echo "" + echo "✅ **Inference works and the gate produces valid verdicts.**" + echo "Major bumps will now get an AI verdict (still gated on green CI)." + } >> "$GITHUB_STEP_SUMMARY"