Use a PAT for Models inference so the major-bump AI gate actually runs

.github/review-tooling/review_dependabot_major.py

@@ -16,10 +16,16 @@
 before the merge lands, so a bump that breaks CI never merges regardless of this
 verdict.
 
-Inference runs on **GitHub Models** (https://models.github.ai/inference) using the
-workflow's built-in GITHUB_TOKEN + `models: read` permission -- no external API
-key. The OpenAI SDK is only the client library; it targets the GitHub Models
-endpoint, not OpenAI. Model defaults to `openai/gpt-5` (override via MODEL_ID).
+Inference runs on **GitHub Models** (https://models.github.ai/inference). The
+OpenAI SDK is only the client library; it targets the GitHub Models endpoint, not
+OpenAI. Model defaults to `openai/gpt-4.1` (override via MODEL_ID).
+
+Auth: MODEL_TOKEN if set, else the workflow's built-in GITHUB_TOKEN (`models:
+read`). The built-in token only works where the org is entitled to Models
+inference. The realrate org is not: its Actions token reads the model catalogue
+(HTTP 200) but every inference call returns a bare 403 with an empty body,
+regardless of model or tier. Hence MODEL_TOKEN, a PAT whose account does have
+inference. Background and evidence: realrate/.github#14.
 
 Contract: ALWAYS writes a valid verdict JSON and exits 0. On any error
 (missing token, API failure, unparseable response) it emits verdict="human" so the
@@ -34,11 +40,18 @@
 import re
 import sys
 
-MODEL = os.getenv("MODEL_ID", "openai/gpt-5")
+# gpt-4.1 is the strongest model that actually works here: the gpt-5 entries are
+# reasoning-tier and reject this script's `max_tokens` (BadRequestError), so the
+# old gpt-5 default failed for anyone not overriding MODEL_ID.
+MODEL = os.getenv("MODEL_ID", "openai/gpt-4.1")
 # GitHub Models inference endpoint (OpenAI-compatible). Override for tests.
 BASE_URL = os.getenv("MODEL_ENDPOINT", "https://models.github.ai/inference")
-# The built-in Actions token authenticates GitHub Models (needs `models: read`).
-TOKEN = os.getenv("GITHUB_TOKEN") or os.getenv("MODEL_TOKEN")
+# Inference token. MODEL_TOKEN wins on purpose: GITHUB_TOKEN is always set in
+# Actions, so checking it first would silently ignore an explicitly supplied PAT.
+# The built-in token only reaches Models where the org is entitled to inference
+# (realrate is not -- it reads the catalogue but 403s every inference call, see
+# realrate/.github#14), so MODEL_TOKEN is the working path there.
+TOKEN = os.getenv("MODEL_TOKEN") or os.getenv("GITHUB_TOKEN")
 
 SYSTEM_PROMPT = (
     "You classify whether a MAJOR version bump of a software dependency is safe "
@@ -87,6 +100,24 @@ def _fallback(reason):
     return {"verdict": "human", "risk": "unknown", "reason": reason}
 
 
+def _explain(exc):
+    """Turn an SDK exception into something a maintainer can act on.
+
+    GitHub Models answers a token without inference entitlement with a bare 403
+    and an empty body, so the exception name alone ("PermissionDeniedError") sent
+    people hunting through workflow permissions for hours. Name the likely cause.
+    """
+    name = type(exc).__name__
+    if getattr(exc, "status_code", None) == 403 or "PermissionDenied" in name:
+        source = "MODEL_TOKEN" if os.getenv("MODEL_TOKEN") else "GITHUB_TOKEN"
+        return (
+            f"403 from GitHub Models via {source} -- the account behind that token "
+            "has no inference entitlement (catalogue reads succeed, inference is "
+            "refused). Not a workflow-permission problem. See realrate/.github#14"
+        )
+    return name
+
+
 def _extract_json(text):
     """Parse a JSON object from a model response, tolerating ```json fences/prose."""
     text = (text or "").strip()
@@ -116,7 +147,7 @@ def _call_model(client, user, structured):
 def get_verdict(dep_names, title, body, ecosystem=""):
     """Ask GitHub Models for a validated verdict dict. Never raises."""
     if not TOKEN:
-        return _fallback("GITHUB_TOKEN not set (need `models: read`); manual review.")
+        return _fallback("no inference token (MODEL_TOKEN/GITHUB_TOKEN); manual review.")
     try:
         from openai import OpenAI
     except Exception as exc:  # noqa: BLE001 - import guard
@@ -142,7 +173,7 @@ def get_verdict(dep_names, title, body, ecosystem=""):
             if data:
                 break
         except Exception as exc:  # noqa: BLE001 - any API/parse failure
-            last = type(exc).__name__
+            last = _explain(exc)
     if not data:
         return _fallback(f"AI verdict unavailable ({locals().get('last', 'no JSON')}); manual review.")
 

.github/review-tooling/selftest-fixtures/breaking/body.md

@@ -0,0 +1,34 @@
+Bumps [numpy](https://github.com/numpy/numpy) from 1.26.4 to 2.5.0.
+
+## Release notes
+
+### NumPy 2.5.0 Release Notes
+
+This release drops support for Python 3.11; Python >=3.12 is now required.
+
+**Breaking changes**
+
+* Python 3.11 is no longer supported. Building or installing on 3.11 will fail.
+* `numpy.distutils` has been removed. Projects still importing it must migrate
+  to `meson-python` or `setuptools`.
+* Several long-deprecated aliases have now expired and been removed, including
+  `np.float_`, `np.unicode_` and `np.NaN`. Use `np.float64`, `np.str_` and
+  `np.nan` instead.
+* The C ABI has changed. Extension modules and packages compiled against the
+  numpy 1.x ABI must be rebuilt against 2.x, or they will fail to import.
+* `copy=False` in `np.array` now raises instead of silently copying.
+
+**Improvements**
+
+* Faster sorting for small integer types.
+* Improved error messages for shape mismatches.
+
+---
+updated-dependencies:
+- dependency-name: numpy
+  dependency-version: 2.5.0
+  dependency-type: direct:production
+  update-type: version-update:semver-major
+...
+
+Signed-off-by: dependabot[bot] <support@github.com>

.github/review-tooling/selftest-fixtures/breaking/title.txt

@@ -0,0 +1 @@
+Bump numpy from 1.26.4 to 2.5.0

.github/review-tooling/selftest-fixtures/breaking/verdict.json

@@ -0,0 +1 @@
+{"verdict": "human", "risk": "unknown", "reason": "AI verdict unavailable (AuthenticationError); manual review."}

.github/review-tooling/selftest-fixtures/safe/body.md

@@ -0,0 +1,27 @@
+Bumps [some-action](https://github.com/example/some-action) from 4 to 5.
+
+## Release notes
+
+### v5
+
+This release contains no changes to the action's inputs, outputs or behaviour.
+
+* Bump the Node runtime used internally from node20 to node24. Consumers do not
+  need to change anything; the action's `with:` inputs are unchanged.
+* Update internal dependencies.
+* Documentation: clarify the caching example in the README.
+* Add support for the linux-arm64 runner image.
+* Internal refactor of the logging helper. No user-visible effect.
+
+No inputs were removed or renamed. No outputs were removed or renamed. There are
+no deprecations in this release.
+
+---
+updated-dependencies:
+- dependency-name: example/some-action
+  dependency-version: '5'
+  dependency-type: direct:production
+  update-type: version-update:semver-major
+...
+
+Signed-off-by: dependabot[bot] <support@github.com>

.github/review-tooling/selftest-fixtures/safe/title.txt

@@ -0,0 +1 @@
+Bump some-action from 4 to 5

.github/review-tooling/selftest-fixtures/safe/verdict.json

@@ -0,0 +1 @@
+{"verdict": "human", "risk": "unknown", "reason": "AI verdict unavailable (AuthenticationError); manual review."}

.github/workflows/dependabot-automerge.yaml

@@ -127,9 +127,14 @@ jobs:
           steps.metadata.outputs.update-type == 'version-update:semver-major' &&
           github.event.action == 'opened'
         env:
-          # Inference runs on GitHub Models via the built-in token (`models: read`);
-          # no external API key. GITHUB_TOKEN also authenticates the `gh` calls below.
+          # Authenticates the `gh` calls below, and is the inference fallback.
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # Inference token, preferred over GITHUB_TOKEN by the script. The org's
+          # built-in token can read the Models catalogue but 403s every inference
+          # call (realrate/.github#14), so a PAT whose account has Models access
+          # does the inference. Unset -> script falls back to GITHUB_TOKEN -> 403
+          # -> verdict=human, i.e. exactly today's safe behaviour, never a merge.
+          MODEL_TOKEN: ${{ secrets.MODELS_PAT }}
           # gpt-4.1 is the strongest model actually available for GitHub Models
           # inference (gpt-5* are cataloged but return `unavailable_model`).
           MODEL_ID: openai/gpt-4.1

.github/workflows/models-selftest.yaml

@@ -0,0 +1,188 @@
+name: models-selftest
+
+# Proves the AI risk gate's inference path actually works, on demand, without
+# waiting for a real major Dependabot bump to come along.
+#
+#   gh workflow run models-selftest.yaml --repo realrate/.github
+#   (or: Actions -> models-selftest -> Run workflow)
+#
+# Use it to answer "did the MODELS_PAT secret fix the 403?" (realrate/.github#14)
+# and, on every change to the gate's script, to catch a regression early.
+#
+# Deliberately the INVERSE of the gate's error posture. The gate fails SAFE: any
+# error becomes verdict=human, because a broken AI must never merge anything.
+# That is also why a broken AI stayed invisible for weeks. This job fails LOUD,
+# so breakage shows up as a red run instead of silently routing every major bump
+# to a human forever.
+
+on:
+  workflow_dispatch:
+    inputs:
+      model_id:
+        description: "Model to test (must exist in the GitHub Models catalogue)"
+        default: openai/gpt-4.1
+        type: string
+  # Regression cover: re-run whenever the gate's inference path or this test changes.
+  pull_request:
+    paths:
+      - .github/review-tooling/review_dependabot_major.py
+      - .github/workflows/models-selftest.yaml
+
+permissions:
+  contents: read
+  models: read # lets the built-in token reach GitHub Models, for the comparison probe
+
+jobs:
+  selftest:
+    runs-on: ubuntu-latest
+    env:
+      MODEL_ID: ${{ inputs.model_id || 'openai/gpt-4.1' }}
+    steps:
+      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
+
+      # Absent secret must not paint PRs red -- it is the expected state until an
+      # admin adds it. Report and stop cleanly instead.
+      - name: Check whether MODELS_PAT is configured
+        id: cfg
+        env:
+          # MODELS_PAT_TEST lets a new token be trialled without touching the
+          # production MODELS_PAT secret; falls back to MODELS_PAT when unset.
+          MODELS_PAT: ${{ secrets.MODELS_PAT_TEST || secrets.MODELS_PAT }}
+        run: |
+          if [ -z "$MODELS_PAT" ]; then
+            echo "configured=false" >> "$GITHUB_OUTPUT"
+            {
+              echo "### models-selftest: skipped"
+              echo
+              echo "\`MODELS_PAT\` is not configured, so there is nothing to test yet."
+              echo "The gate falls back to the built-in token, gets a 403, and routes"
+              echo "every major bump to a human -- today's expected behaviour."
+              echo
+              echo "To enable: add an **org** secret \`MODELS_PAT\` (fine-grained PAT,"
+              echo "\`Models: read\` only, visibility **all repositories**), then re-run"
+              echo "this workflow. See realrate/.github#14."
+            } >> "$GITHUB_STEP_SUMMARY"
+            echo "::notice::MODELS_PAT not configured -- skipping (this is not a failure)"
+          else
+            echo "configured=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      # Side-by-side proof of which credential can actually do inference. This is
+      # the exact comparison that took a full session to establish by hand.
+      - name: Probe inference with each credential
+        if: steps.cfg.outputs.configured == 'true'
+        env:
+          BUILTIN: ${{ secrets.GITHUB_TOKEN }}
+          PAT: ${{ secrets.MODELS_PAT_TEST || secrets.MODELS_PAT }}
+        run: |
+          probe() { # $1=label $2=token
+            code=$(curl -s -o /tmp/b.json -w '%{http_code}' \
+              https://models.github.ai/inference/chat/completions \
+              -H "Authorization: Bearer $2" -H "Content-Type: application/json" \
+              -d "{\"model\":\"$MODEL_ID\",\"messages\":[{\"role\":\"user\",\"content\":\"say OK\"}],\"max_tokens\":5}")
+            echo "$code"
+          }
+          builtin_code=$(probe "GITHUB_TOKEN" "$BUILTIN")
+          pat_code=$(probe "MODELS_PAT" "$PAT")
+          echo "GITHUB_TOKEN -> $builtin_code | MODELS_PAT -> $pat_code"
+          {
+            echo "### models-selftest"
+            echo
+            echo "Model: \`$MODEL_ID\`"
+            echo
+            echo "| credential | inference |"
+            echo "|---|---|"
+            echo "| built-in \`GITHUB_TOKEN\` | $builtin_code |"
+            echo "| \`MODELS_PAT\` | $pat_code |"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          if [ "$pat_code" != "200" ]; then
+            echo "::error::MODELS_PAT cannot run inference (HTTP $pat_code). The PAT needs 'Models: read' and its account needs Models access."
+            echo "" >> "$GITHUB_STEP_SUMMARY"
+            echo "❌ **MODELS_PAT failed inference (HTTP $pat_code)** -- the gate cannot produce verdicts." >> "$GITHUB_STEP_SUMMARY"
+            exit 1
+          fi
+          if [ "$builtin_code" = "200" ]; then
+            echo "::notice::The built-in token now works too -- org entitlement looks fixed, so MODELS_PAT may no longer be needed (see realrate/.github#14)."
+            echo "" >> "$GITHUB_STEP_SUMMARY"
+            echo "ℹ️ The built-in token also returned 200. If that holds, the org is entitled to inference and \`MODELS_PAT\` can be retired." >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Set up uv
+        if: steps.cfg.outputs.configured == 'true'
+        uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
+
+      # End-to-end through the REAL script, on fixtures with known-correct answers.
+      - name: Run the real verdict script on fixture bumps
+        if: steps.cfg.outputs.configured == 'true'
+        env:
+          MODEL_TOKEN: ${{ secrets.MODELS_PAT_TEST || secrets.MODELS_PAT }}
+        run: |
+          fail=0
+          {
+            echo ""
+            echo "| fixture | expected | verdict | risk | reason |"
+            echo "|---|---|---|---|---|"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          for spec in "breaking:human:numpy:pip" "safe:auto:example/some-action:github_actions"; do
+            name="${spec%%:*}"; rest="${spec#*:}"
+            expect="${rest%%:*}"; rest="${rest#*:}"
+            deps="${rest%%:*}"; eco="${rest#*:}"
+            dir=".github/review-tooling/selftest-fixtures/$name"
+
+            # Run inline, NOT in a subshell: a failure must be able to fail this job.
+            uv run --script .github/review-tooling/review_dependabot_major.py \
+              --title-file "$dir/title.txt" --body-file "$dir/body.md" \
+              --dep-names "$deps" --ecosystem "$eco" --out "$dir/verdict.json" || true
+
+            if [ ! -s "$dir/verdict.json" ]; then
+              echo "::error::$name: the script produced no verdict file at all"
+              echo "| $name | $expect | ❌ none | - | script produced no verdict |" >> "$GITHUB_STEP_SUMMARY"
+              fail=1
+              continue
+            fi
+
+            verdict=$(jq -r '.verdict // ""' "$dir/verdict.json")
+            risk=$(jq -r '.risk // ""' "$dir/verdict.json")
+            reason=$(jq -r '.reason // ""' "$dir/verdict.json")
+
+            # HARD requirement: a well-formed verdict is what proves inference ran.
+            # The script fails safe, so it emits verdict=human on an API error too --
+            # hence checking the reason for our own fallback wording, otherwise a
+            # total outage would masquerade as a correct "human" verdict here.
+            case "$verdict" in
+              auto|human) ;;
+              *) echo "::error::$name: invalid verdict '$verdict'"; fail=1; continue ;;
+            esac
+            # _fallback() is the only thing that emits risk="unknown" AND a reason
+            # ending "manual review." Matching both keeps a genuine model verdict
+            # that happens to mention manual review from tripping this.
+            if [ "$risk" = "unknown" ] && [ "${reason%manual review.}" != "$reason" ]; then
+              echo "::error::$name: verdict came from the error fallback, not the model: $reason"
+              echo "| $name | $expect | ❌ fallback | $risk | ${reason:0:130} |" >> "$GITHUB_STEP_SUMMARY"
+              fail=1
+              continue
+            fi
+
+            # SOFT check: the model is non-deterministic, so a differing judgement is
+            # reported, not fatal. Only a malformed/fallback verdict fails this job.
+            mark="✅"
+            if [ "$verdict" != "$expect" ]; then
+              mark="⚠️"
+              echo "::warning::$name: expected verdict=$expect but got $verdict -- judgement drift, not an outage. Reason: $reason"
+            fi
+            echo "| $name | $expect | $mark $verdict | $risk | ${reason:0:130} |" >> "$GITHUB_STEP_SUMMARY"
+            echo "$name -> verdict=$verdict risk=$risk"
+          done
+
+          if [ "$fail" != "0" ]; then
+            echo "" >> "$GITHUB_STEP_SUMMARY"
+            echo "❌ **The gate could not produce a valid verdict.**" >> "$GITHUB_STEP_SUMMARY"
+            exit 1
+          fi
+          {
+            echo ""
+            echo "✅ **Inference works and the gate produces valid verdicts.**"
+            echo "Major bumps will now get an AI verdict (still gated on green CI)."
+          } >> "$GITHUB_STEP_SUMMARY"