diff --git a/.github/scripts/enforce-pr-target.test.cjs b/.github/scripts/enforce-pr-target.test.cjs new file mode 100644 index 000000000..018f8b3c3 --- /dev/null +++ b/.github/scripts/enforce-pr-target.test.cjs @@ -0,0 +1,46 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); + +describe("enforce-pr-target workflow", () => { + const workflowPath = path.join(__dirname, "../workflows/enforce-pr-target.yml"); + const workflow = fs.readFileSync(workflowPath, "utf8"); + + it("uses pull_request_target without checking out PR head code", () => { + assert.match(workflow, /pull_request_target:/); + assert.doesNotMatch( + workflow, + /actions\/checkout@/, + "wrong-branch enforcer must not check out untrusted PR code", + ); + }); + + it("grants contents:write so draft GraphQL mutations work with GITHUB_TOKEN", () => { + // convertPullRequestToDraft / markPullRequestReadyForReview fail with + // "Resource not accessible by integration" when contents stays unset/read + // (seen on #626). Assert the real permissions block, not comment text + // that also mentions these scopes. + const permissionsBlock = workflow.match(/^permissions:\n((?:[ \t]+.+\n)+)/m); + assert.ok(permissionsBlock, "workflow must declare a top-level permissions block"); + const lines = permissionsBlock[1] + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .sort(); + assert.deepEqual(lines, ["contents: write", "pull-requests: write"]); + }); + + it("fails the required check on a wrong base even if draft conversion fails", () => { + assert.match(workflow, /core\.setFailed\(/); + assert.match(workflow, /draftConversionFailed/); + assert.match(workflow, /Could not convert pull request to draft/); + }); + + it("soft-fails ready-for-review restoration the same way", () => { + assert.match(workflow, /readyConversionFailed/); + assert.match(workflow, /Could not mark pull request ready for review/); + }); +}); diff --git a/.github/scripts/issue-quality.cjs b/.github/scripts/issue-quality.cjs index b973b3a92..681cdc55b 100644 --- a/.github/scripts/issue-quality.cjs +++ b/.github/scripts/issue-quality.cjs @@ -25,19 +25,29 @@ function unwrapSingleEnclosingFence(text) { return match[2]; } -function isPlaceholderOnlyValue(raw) { - if (typeof raw !== "string") return false; +/** + * Shared strip/trim/unwrap used by placeholder and unusable-stand-in matchers. + * Returns null when the value is absent after normalisation. + */ +function normalizeRawSectionValue(raw) { + if (typeof raw !== "string") return null; let value = raw.replace(//g, "").trim(); - if (!value) return false; + if (!value) return null; - // A lone fenced block whose entire body is a placeholder is still placeholder - // text (e.g. ```text\nN/A\n```), not a real example. + // A lone fenced block whose entire body is a stand-in is still a stand-in + // (e.g. ```text\nN/A\n```), not a real example. const unwrapped = unwrapSingleEnclosingFence(value); if (unwrapped !== null) { value = unwrapped.trim(); - if (!value) return false; + if (!value) return null; } + return value; +} + +function isPlaceholderOnlyValue(raw) { + const value = normalizeRawSectionValue(raw); + if (value === null) return false; return PLACEHOLDER_ONLY_RE.test(value); } @@ -398,6 +408,20 @@ function isPlaceholder(text) { return isPlaceholderOnlyValue(text); } +/** + * True when Version is an "I don't know" stand-in rather than an install id. + * Kept separate from PLACEHOLDER_ONLY_RE so legacy N/A / No response soft-pass + * behaviour is unchanged. + */ +const UNUSABLE_VERSION_RE = + /^[\s_*~`]*(?:unknown|unkown|uknown|don'?t\s+know|do\s+not\s+know|idk|dunno|not\s+sure|unsure|\?+|모름|잘\s*모름|모르겠(?:습니다|음)?|不明|わからない|分からない|不知道|不清楚|keine\s+ahnung|wei[sß]{1,2}\s+nicht)[\s_*~`]*[.!?]*$/i; + +function isUnusableVersion(raw) { + const value = normalizeRawSectionValue(raw); + if (value === null) return false; + return UNUSABLE_VERSION_RE.test(value); +} + const CJK_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/gu; @@ -433,6 +457,16 @@ function isTooTerseFeatureSection(text) { return true; } +/** + * Bug Reproduction needs steps or concrete signals. A title-like phrase with + * no commands, paths, digits, or product keywords is not actionable. + */ +function isTooTerseBugReproduction(text) { + if (isEmpty(text) || isPlaceholder(text)) return false; + if (hasConcreteDetail(text)) return false; + return countWords(text) < 12; +} + /** * Check if raw section text is a placeholder-only variant without relying on * clean() first. Used to distinguish intentionally blank optional fields @@ -628,6 +662,8 @@ function validateIssue(issue) { const repro = extractSection(body, "Reproduction"); const version = extractSection(body, "Version"); const os = extractSection(body, "Operating system") ?? extractSection(body, "OS"); + // New Bug report template always includes Client or integration. + const isNewBugForm = extractSection(body, "Client or integration") !== null; if (isEmpty(summary) && isEmpty(repro)) { // Soft-pass substantial non-English / freeform structured reports once @@ -655,17 +691,56 @@ function validateIssue(issue) { if (isEmpty(repro)) { reasons.push("Reproduction is empty."); guidance.push("List the exact steps to reproduce the problem."); + } else if (!softPass && isTooTerseBugReproduction(repro)) { + reasons.push("Reproduction is too vague to act on."); + guidance.push("List exact steps, commands, and the observed failure — not only a short phrase."); } } - // Required environment fields removed after submission. - // Only fire when the headings exist in the body (new form). Legacy bug - // reports never had Version or OS fields, so null means absent, not removed. - // Skip when the raw value is a "No response" placeholder -- the old form had - // both fields as optional, so legacy issues legitimately contain those headings - // with the GitHub placeholder. Only close when the field was actively cleared. - if (!softPass && version !== null && os !== null && isEmpty(version) && isEmpty(os) && - !isRawPlaceholder(version) && !isRawPlaceholder(os)) { + // Version "Unknown" / "모름" / "idk" is never actionable, on any form. + if (!softPass && version !== null && isUnusableVersion(version)) { + reasons.push("Version is missing or unknown."); + guidance.push("Report the installed `@bitkyc08/opencodex` version (for example `2.7.42`) or a commit SHA from `ocx --version`."); + } else if ( + !softPass && + isNewBugForm && + version !== null && + (isEmpty(version) || isRawPlaceholder(version)) + ) { + // New form requires Version. Legacy N/A / No response soft-pass stays + // only for bodies without Client or integration. + reasons.push("Version is missing."); + guidance.push("Add your OpenCodex version so we can reproduce the environment."); + } + + if (!softPass && isNewBugForm && os !== null && isUnusableVersion(os)) { + reasons.push("Operating system is missing or unknown."); + guidance.push("Add your OS name and version (for example Windows 11 24H2)."); + } else if ( + !softPass && + isNewBugForm && + os !== null && + (isEmpty(os) || isRawPlaceholder(os)) + ) { + reasons.push("Operating system is missing."); + guidance.push("Add your OS name and version (for example Windows 11 24H2)."); + } + + // Required environment fields removed after submission on bodies that are + // not the new form (no Client or integration). Legacy reports never had + // Version or OS fields, so null means absent, not removed. Skip when the + // raw value is a "No response" placeholder — the old form had both fields + // as optional. Only close when the field was actively cleared. + if ( + !softPass && + !isNewBugForm && + version !== null && + os !== null && + isEmpty(version) && + isEmpty(os) && + !isRawPlaceholder(version) && + !isRawPlaceholder(os) + ) { reasons.push("Version and Operating system are both missing."); guidance.push("Add your OpenCodex version and OS so we can reproduce the environment."); } @@ -873,6 +948,7 @@ module.exports = { isPlaceholderOnlyValue, isPlaceholder, isRawPlaceholder, + isUnusableVersion, countWords, hasConcreteDetail, labelForKind, diff --git a/.github/scripts/issue-quality.test.cjs b/.github/scripts/issue-quality.test.cjs index 6dbe61e3c..9c8f9bb86 100644 --- a/.github/scripts/issue-quality.test.cjs +++ b/.github/scripts/issue-quality.test.cjs @@ -16,6 +16,7 @@ const { isPlaceholderOnlyValue, isPlaceholder, isRawPlaceholder, + isUnusableVersion, countWords, hasConcreteDetail, rejectsWorkflowDispatchPullRequest, @@ -693,6 +694,162 @@ describe("validateIssue - bug", () => { assert.equal(result.valid, false); assert.ok(result.reasons.some((r) => r.includes("Version"))); }); + + it("rejects unknown / don't-know Version values (#624)", () => { + const versions = [ + "Unknown", + "Uknown", + "unkown", + "Don't know", + "dont know", + "idk", + "모름", + "잘 모름", + "?", + "???", + ]; + for (const version of versions) { + const body = [ + "### Client or integration", + "Codex CLI", + "### Area", + "CLI", + "### Summary", + "The OpenCodex proxy keeps dropping the Codex CLI connection mid-request.", + "### Reproduction", + "1. ocx start --port 10100", + "2. Send any Codex CLI request through the proxy", + "3. Observe the connection drop", + "### Version", + version, + "### Operating system", + "Windows 11", + ].join("\n"); + const result = validateIssue({ + title: "Unexpected interruption continues to occur", + body, + labels: ["bug"], + }); + assert.equal(result.kind, "bug"); + assert.equal( + result.valid, + false, + `Expected unusable Version "${version}" to be invalid, got: ${result.reasons.join("; ")}`, + ); + assert.ok( + result.reasons.some((r) => /Version/i.test(r) && /unknown|missing/i.test(r)), + `Expected Version unknown/missing reason for "${version}", got: ${result.reasons.join("; ")}`, + ); + } + }); + + it("rejects issue #624-style low-effort new-form bug", () => { + const body = [ + "### Client or integration", + "Codex CLI", + "### Area", + "CLI", + "### Summary", + "CLI로 확인해봤는데 오픈코덱스 프록시가 중간에 자꾸 연결이 끊어져서 그런거라고 합니다.", + "", + "수정 바랍니다.", + "### Reproduction", + "예기치않게중단됨", + "### Version", + "모름", + "### Operating system", + "윈11", + "### Provider and model", + "_No response_", + "### Logs or error output", + "```shell", + "", + "```", + ].join("\n"); + const result = validateIssue({ + title: "Unexpected interruption continues to occur", + body, + labels: ["bug"], + }); + assert.equal(result.kind, "bug"); + assert.equal(result.valid, false); + assert.ok(result.reasons.some((r) => /Version/i.test(r))); + assert.ok(result.reasons.some((r) => /Reproduction/i.test(r) && /vague|empty/i.test(r))); + }); + + it("rejects a new-form bug with a usable Version but placeholder OS", () => { + const body = [ + "### Client or integration", + "Codex CLI", + "### Area", + "CLI", + "### Summary", + "Proxy returns 502 when streaming is enabled on Windows.", + "### Reproduction", + "1. ocx start", + "2. Send a streaming /v1/responses request", + "### Version", + "2.7.42", + "### Operating system", + "No response", + ].join("\n"); + const result = validateIssue({ title: "Streaming 502", body, labels: ["bug"] }); + assert.equal(result.kind, "bug"); + assert.equal(result.valid, false); + assert.ok(result.reasons.some((r) => /Operating system/i.test(r))); + }); + + it("rejects a new-form bug whose Reproduction is only a vague phrase", () => { + const body = [ + "### Client or integration", + "Codex CLI", + "### Area", + "CLI", + "### Summary", + "The OpenCodex proxy keeps dropping the Codex CLI connection mid-request.", + "### Reproduction", + "Unexpected interruption", + "### Version", + "2.7.42", + "### Operating system", + "Windows 11", + ].join("\n"); + const result = validateIssue({ + title: "Unexpected interruption continues to occur", + body, + labels: ["bug"], + }); + assert.equal(result.kind, "bug"); + assert.equal(result.valid, false); + assert.ok(result.reasons.some((r) => /Reproduction/i.test(r) && /vague/i.test(r))); + }); + + it("rejects unknown Operating system stand-ins on the new bug form", () => { + const body = [ + "### Client or integration", + "Codex CLI", + "### Area", + "CLI", + "### Summary", + "The OpenCodex proxy keeps dropping the Codex CLI connection mid-request.", + "### Reproduction", + "1. ocx start --port 10100", + "2. Send any Codex CLI request through the proxy", + "3. Observe the connection drop", + "### Version", + "2.7.42", + "### Operating system", + "Unknown", + ].join("\n"); + const result = validateIssue({ + title: "Unexpected interruption continues to occur", + body, + labels: ["bug"], + }); + assert.equal(result.kind, "bug"); + assert.equal(result.valid, false); + assert.ok(result.reasons.some((r) => /Operating system/i.test(r))); + }); }); // --------------------------------------------------------------------------- @@ -839,6 +996,16 @@ describe("normalisation", () => { assert.equal(clean("Not available!"), ""); }); + it("detects unusable Version stand-ins without treating them as generic placeholders", () => { + for (const value of ["Unknown", "Uknown", "모름", "idk", "don't know"]) { + assert.equal(isUnusableVersion(value), true, value); + assert.equal(isPlaceholderOnlyValue(value), false, value); + } + for (const value of ["2.7.42", "N/A", "No response", "main@abc1234"]) { + assert.equal(isUnusableVersion(value), false, value); + } + }); + it("does not treat sentences containing placeholder phrases as empty", () => { assert.equal(clean("This is N/A for voice mode today."), "This is N/A for voice mode today."); assert.equal(clean("Not applicable to Claude Code."), "Not applicable to Claude Code."); diff --git a/.github/workflows/enforce-pr-target.yml b/.github/workflows/enforce-pr-target.yml index da76509d9..6e8a52af6 100644 --- a/.github/workflows/enforce-pr-target.yml +++ b/.github/workflows/enforce-pr-target.yml @@ -8,7 +8,13 @@ on: - edited - ready_for_review +# pull-requests:write covers title/comment updates. +# contents:write is required for convertPullRequestToDraft / +# markPullRequestReadyForReview GraphQL mutations with GITHUB_TOKEN +# (otherwise: "Resource not accessible by integration"). This workflow +# never checks out PR head code. permissions: + contents: write pull-requests: write concurrency: @@ -58,6 +64,7 @@ jobs: comment.user?.login === "github-actions[bot]" && comment.body?.includes(COMMENT_MARKER) ); + let botCommentId = botComment?.id ?? null; function parseState(body) { const match = body?.match(STATE_PATTERN); @@ -90,23 +97,24 @@ jobs: } async function upsertComment(body) { - if (botComment) { + if (botCommentId) { await github.rest.issues.updateComment({ owner, repo, - comment_id: botComment.id, + comment_id: botCommentId, body }); return; } - await github.rest.issues.createComment({ + const created = await github.rest.issues.createComment({ owner, repo, issue_number: pull_number, body }); + botCommentId = created.data.id; } async function convertToDraft() { @@ -158,10 +166,12 @@ jobs: // Wrong branch: // - add the title prefix - // - convert ready PRs to draft + // - convert ready PRs to draft (best-effort) // - post or update the explanation + // - fail the required check so merge stays blocked even if draft fails // - remember which changes were made by this workflow if (wrongBase) { + const willPrefixTitle = !pr.title.startsWith(TITLE_PREFIX); const state = storedState?.active ? { ...storedState } : { @@ -171,20 +181,15 @@ jobs: titlePrefixedByBot: false }; - if (!pr.draft) { - state.autoDraftedByBot = true; - } - - if (!pr.title.startsWith(TITLE_PREFIX)) { + // Claim title ownership before mutating so a crash after the + // title write still records what the bot owns. Do not claim a + // new draft here — only after convertToDraft succeeds. + if (willPrefixTitle) { state.titlePrefixedByBot = true; } - const draftExplanation = state.autoDraftedByBot - ? "This pull request is being kept as a draft automatically. Once the target branch is corrected, it will be marked ready for review again." - : "This pull request was already a draft. Its draft status will be preserved after the target branch is corrected."; + let draftConversionFailed = false; - // Store the restoration state before changing the PR, allowing - // a rerun to recover if an API request fails partway through. await upsertComment( [ COMMENT_MARKER, @@ -194,15 +199,11 @@ jobs: "", `This pull request currently targets ${inlineCode(pr.base.ref)}, but pull requests must target one of ${ALLOWED_BASES.map(inlineCode).join(" or ")}.`, "", - `Its title has been prefixed with ${inlineCode(TITLE_PREFIX.trim())}.`, - "", - `@${pr.user.login} Please retarget this PR to ${inlineCode(DEFAULT_BASE)}. Most contributions go to ${inlineCode(DEFAULT_BASE)} first; use ${inlineCode("dev2-go")} only for scoped Go native-port work. \`main\` receives only release promotions. See our [Contributing guide](https://lidge-jun.github.io/opencodex/contributing/) for details. Thanks! 🙏`, - "", - draftExplanation + "Recording ownership state before applying title/draft changes…" ].join("\n") ); - if (!pr.title.startsWith(TITLE_PREFIX)) { + if (willPrefixTitle) { await github.rest.pulls.update({ owner, repo, @@ -212,9 +213,58 @@ jobs: } if (!pr.draft) { - await convertToDraft(); + try { + await convertToDraft(); + state.autoDraftedByBot = true; + // Checkpoint after a successful draft so ownership survives + // if the final explanation write fails. + await upsertComment( + [ + COMMENT_MARKER, + stateMarker(state), + "", + "⚠️ **Wrong target branch**", + "", + `This pull request currently targets ${inlineCode(pr.base.ref)}, but pull requests must target one of ${ALLOWED_BASES.map(inlineCode).join(" or ")}.`, + "", + "Draft conversion succeeded; finalising explanation…" + ].join("\n") + ); + } catch (error) { + draftConversionFailed = true; + state.autoDraftedByBot = false; + core.warning( + `Could not convert pull request to draft: ${error.message}` + ); + } } + const draftExplanation = draftConversionFailed + ? "Automatic draft conversion failed (token cannot change draft status). Please convert this pull request to a draft manually, or retarget it to the correct base. The required `enforce-target` check will keep failing until the base is corrected." + : state.autoDraftedByBot + ? "This pull request is being kept as a draft automatically. Once the target branch is corrected, it will be marked ready for review again." + : "This pull request was already a draft. Its draft status will be preserved after the target branch is corrected."; + + await upsertComment( + [ + COMMENT_MARKER, + stateMarker(state), + "", + "⚠️ **Wrong target branch**", + "", + `This pull request currently targets ${inlineCode(pr.base.ref)}, but pull requests must target one of ${ALLOWED_BASES.map(inlineCode).join(" or ")}.`, + "", + `Its title has been prefixed with ${inlineCode(TITLE_PREFIX.trim())}.`, + "", + `@${pr.user.login} Please retarget this PR to ${inlineCode(DEFAULT_BASE)}. Most contributions go to ${inlineCode(DEFAULT_BASE)} first; use ${inlineCode("dev2-go")} only for scoped Go native-port work. \`main\` receives only release promotions. See our [Contributing guide](https://lidge-jun.github.io/opencodex/contributing/) for details. Thanks! 🙏`, + "", + draftExplanation + ].join("\n") + ); + + core.setFailed( + `Pull request targets ${pr.base.ref}; allowed bases are ${ALLOWED_BASES.join(", ")}.` + ); return; } @@ -243,27 +293,46 @@ jobs: // Mark the PR ready only if this workflow originally converted it // to draft. Intentionally drafted PRs remain drafts. + let readyConversionFailed = false; if ( storedState.autoDraftedByBot && pr.draft ) { - await markReadyForReview(); + try { + await markReadyForReview(); + } catch (error) { + readyConversionFailed = true; + core.warning( + `Could not mark pull request ready for review: ${error.message}` + ); + } } - const completedState = { - version: 1, - active: false, - autoDraftedByBot: false, - titlePrefixedByBot: false - }; + const completedState = readyConversionFailed + ? { + // Keep ownership active so a later edited/reopened/ + // ready_for_review event retries markReadyForReview. + version: 1, + active: true, + autoDraftedByBot: true, + titlePrefixedByBot: false + } + : { + version: 1, + active: false, + autoDraftedByBot: false, + titlePrefixedByBot: false + }; const titleResult = storedState.titlePrefixedByBot ? `The ${inlineCode(TITLE_PREFIX.trim())} title prefix has been removed.` : "The title was left unchanged."; - const draftResult = storedState.autoDraftedByBot - ? "The pull request has been marked ready for review again." - : "Its existing draft status has been preserved."; + const draftResult = readyConversionFailed + ? "Automatic ready-for-review conversion failed; please mark the pull request ready manually if it is still a draft." + : storedState.autoDraftedByBot + ? "The pull request has been marked ready for review again." + : "Its existing draft status has been preserved."; await upsertComment( [ diff --git a/.github/workflows/issue-quality-tests.yml b/.github/workflows/issue-quality-tests.yml index 76d7e97ad..c6c72b66a 100644 --- a/.github/workflows/issue-quality-tests.yml +++ b/.github/workflows/issue-quality-tests.yml @@ -8,6 +8,7 @@ on: - ".github/scripts/issue-quality.test.cjs" - ".github/scripts/pr-labeler.cjs" - ".github/scripts/pr-labeler.test.cjs" + - ".github/scripts/enforce-pr-target.test.cjs" - ".github/scripts/issue-translation.cjs" - ".github/scripts/issue-translation.test.cjs" - ".github/scripts/issue-triage.cjs" @@ -15,6 +16,7 @@ on: - ".github/scripts/parse-issue-translation-response.cjs" - ".github/scripts/parse-issue-translation-response.test.cjs" - ".github/workflows/enforce-issue-quality.yml" + - ".github/workflows/enforce-pr-target.yml" - ".github/workflows/pr-labeler.yml" - ".github/workflows/issue-triage.yml" - ".github/workflows/issue-quality-tests.yml" @@ -25,6 +27,7 @@ on: - ".github/scripts/issue-quality.test.cjs" - ".github/scripts/pr-labeler.cjs" - ".github/scripts/pr-labeler.test.cjs" + - ".github/scripts/enforce-pr-target.test.cjs" - ".github/scripts/issue-translation.cjs" - ".github/scripts/issue-translation.test.cjs" - ".github/scripts/issue-triage.cjs" @@ -32,6 +35,7 @@ on: - ".github/scripts/parse-issue-translation-response.cjs" - ".github/scripts/parse-issue-translation-response.test.cjs" - ".github/workflows/enforce-issue-quality.yml" + - ".github/workflows/enforce-pr-target.yml" - ".github/workflows/pr-labeler.yml" - ".github/workflows/issue-triage.yml" - ".github/workflows/issue-quality-tests.yml" @@ -53,6 +57,7 @@ jobs: run: | node --test .github/scripts/issue-quality.test.cjs node --test .github/scripts/pr-labeler.test.cjs + node --test .github/scripts/enforce-pr-target.test.cjs node --test .github/scripts/issue-translation.test.cjs node --test .github/scripts/issue-triage.test.cjs node --test .github/scripts/parse-issue-translation-response.test.cjs diff --git a/tests/ci-workflows.test.ts b/tests/ci-workflows.test.ts index ec95bc200..644f29c42 100644 --- a/tests/ci-workflows.test.ts +++ b/tests/ci-workflows.test.ts @@ -5,8 +5,17 @@ import { callsTo, methodsOf, runEnforcePrTarget, + type HarnessResult, } from "./helpers/enforce-pr-target-harness"; +/** Final enforcer comment body after pending/draft checkpoints. */ +function lastEnforcerCommentBody(result: HarnessResult): string { + const updates = callsTo(result, "issues.updateComment") as Array<{ body: string }>; + if (updates.length > 0) return updates[updates.length - 1]!.body; + const creates = callsTo(result, "issues.createComment") as Array<{ body: string }>; + return creates[creates.length - 1]!.body; +} + const root = new URL("../", import.meta.url); const doctorGuiIfChangedScript = fileURLToPath(new URL("../scripts/doctor-gui-if-changed.ts", import.meta.url)); @@ -466,10 +475,15 @@ describe("GitHub Actions hardening", () => { // single assertion. expect(Object.keys(workflow.on?.pull_request_target ?? {})).toEqual(["types"]); - // Exactly one permission scope. Asserting that `pull-requests: write` is - // present says nothing about what was added beside it, and a `write-all` - // scalar is not an object at all. - expect(workflow.permissions).toEqual({ "pull-requests": "write" }); + // Exactly the scopes this gate needs. `pull-requests: write` covers title + // and comment updates. `contents: write` is required for the draft GraphQL + // mutations with GITHUB_TOKEN (#626: "Resource not accessible by integration" + // when contents was unset). Asserting the whole object pins both presence + // and the absence of anything broader (write-all, contents alone, …). + expect(workflow.permissions).toEqual({ + contents: "write", + "pull-requests": "write", + }); // One run per PR, so two rapid events cannot race on the title/draft state, // and no `cancel-in-progress` — cancelling the in-flight run mid-mutation is @@ -738,9 +752,11 @@ describe("GitHub Actions hardening", () => { "issues.createComment", "pulls.update", "graphql", + "issues.updateComment", + "issues.updateComment", ]); - const [comment] = callsTo(result, "issues.createComment") as [{ body: string }]; - expect(comment.body).toContain(`\`${ref}\``); + expect(lastEnforcerCommentBody(result)).toContain(`\`${ref}\``); + expect(result.warnings.some((w) => w.startsWith("setFailed:"))).toBe(true); } }); @@ -787,14 +803,16 @@ describe("GitHub Actions hardening", () => { "issues.updateComment", "pulls.update", "graphql", + "issues.updateComment", + "issues.updateComment", ]); expect(callsTo(result, "pulls.update")).toEqual([ { owner: "lidge-jun", repo: "opencodex", pull_number: 42, title: "[WRONG BRANCH] Port the runtime entry" }, ]); - const [comment] = callsTo(result, "issues.updateComment") as [{ body: string }]; - expect(comment.body).toContain('"active":true'); - expect(comment.body).toContain('"autoDraftedByBot":true'); - expect(comment.body).toContain('"titlePrefixedByBot":true'); + expect(lastEnforcerCommentBody(result)).toContain('"active":true'); + expect(lastEnforcerCommentBody(result)).toContain('"autoDraftedByBot":true'); + expect(lastEnforcerCommentBody(result)).toContain('"titlePrefixedByBot":true'); + expect(result.warnings.some((w) => w.startsWith("setFailed:"))).toBe(true); }); test("the wrong-target explanation lists every allowed base and names the default", async () => { @@ -805,11 +823,11 @@ describe("GitHub Actions hardening", () => { const result = await run({ pr: { base: { ref: "main" }, title: "Add a thing", draft: false }, }); - const [comment] = callsTo(result, "issues.createComment") as [{ body: string }]; + const commentBody = lastEnforcerCommentBody(result); - expect(comment.body).toContain("must target one of `dev` or `dev2-go`"); - expect(comment.body).toContain("Please retarget this PR to `dev`"); - expect(comment.body).toContain("only for scoped Go native-port work"); + expect(commentBody).toContain("must target one of `dev` or `dev2-go`"); + expect(commentBody).toContain("Please retarget this PR to `dev`"); + expect(commentBody).toContain("only for scoped Go native-port work"); }); test("a PR targeting main is prefixed, drafted, and explained — and nothing else", async () => { @@ -817,14 +835,17 @@ describe("GitHub Actions hardening", () => { pr: { base: { ref: "main" }, title: "Add a thing", draft: false }, }); - // The comment lands before the mutations, so a rerun after a failed - // mutation can still find the restoration state. + // Pending ownership comment first, then title/draft, then checkpoints so + // a mid-run crash still records ownership and autoDraftedByBot only after + // convertToDraft succeeds (#626 / Codex review on #631). expect(methodsOf(result)).toEqual([ "pulls.get", "issues.listComments", "issues.createComment", "pulls.update", "graphql", + "issues.updateComment", + "issues.updateComment", ]); // The title update carries the title and nothing else. `base`, `state` @@ -834,17 +855,22 @@ describe("GitHub Actions hardening", () => { { owner: "lidge-jun", repo: "opencodex", pull_number: 42, title: "[WRONG BRANCH] Add a thing" }, ]); - // The comment addresses this PR, by its own number. - const [comment] = callsTo(result, "issues.createComment") as [{ issue_number: number; body: string }]; - expect(comment.issue_number).toBe(42); - expect(comment.body).toContain(MARKER); - expect(comment.body).toContain("@contributor"); + // The first comment create addresses this PR, by its own number. + const [created] = callsTo(result, "issues.createComment") as [{ issue_number: number; body: string }]; + expect(created.issue_number).toBe(42); + expect(created.body).toContain(MARKER); + const commentBody = lastEnforcerCommentBody(result); + expect(commentBody).toContain("@contributor"); + expect(commentBody).toContain('"autoDraftedByBot":true'); // The only GraphQL mutation is the draft conversion — not a retarget. const [draft] = callsTo(result, "graphql") as [{ query: string; variables: unknown }]; expect(draft.query).toContain("convertPullRequestToDraft"); expect(draft.query).not.toContain("updatePullRequest"); expect(draft.variables).toEqual({ pullRequestId: "PR_kwDOnode42" }); + + // Wrong-base runs must fail the required check even when mutations succeed. + expect(result.warnings.some((w) => w.startsWith("setFailed:"))).toBe(true); }); test("a PR that was already a draft is not un-drafted afterwards", async () => { @@ -852,16 +878,18 @@ describe("GitHub Actions hardening", () => { pr: { base: { ref: "main" }, draft: true }, }); - // No draft conversion: it is already a draft. And the state it records - // says so, which is what stops the restore path from marking it ready. + // No draft conversion: it is already a draft. Pending ownership first, + // then title prefix, then final explanation. State records that the bot + // did not draft — which stops restore from marking it ready. expect(methodsOf(wrong)).toEqual([ "pulls.get", "issues.listComments", "issues.createComment", "pulls.update", + "issues.updateComment", ]); - const [comment] = callsTo(wrong, "issues.createComment") as [{ body: string }]; - expect(comment.body).toContain('"autoDraftedByBot":false'); + expect(lastEnforcerCommentBody(wrong)).toContain('"autoDraftedByBot":false'); + expect(wrong.warnings.some((w) => w.startsWith("setFailed:"))).toBe(true); // Now retarget it correctly, feeding that state back in. const restored = await run({ @@ -925,8 +953,13 @@ describe("GitHub Actions hardening", () => { comments: [botComment({ version: 1, active: true, autoDraftedByBot: true, titlePrefixedByBot: true })], }); - // The comment is refreshed; the title and draft state are already right. - expect(methodsOf(result)).toEqual(["pulls.get", "issues.listComments", "issues.updateComment"]); + // The comment is refreshed (pending + final); title and draft are already right. + expect(methodsOf(result)).toEqual([ + "pulls.get", + "issues.listComments", + "issues.updateComment", + "issues.updateComment", + ]); }); test("the verdict comes from the fetched PR, not the stale event payload", async () => { @@ -948,6 +981,8 @@ describe("GitHub Actions hardening", () => { "issues.createComment", "pulls.update", "graphql", + "issues.updateComment", + "issues.updateComment", ]); expect(callsTo(wentWrong, "pulls.update")).toEqual([ { owner: "lidge-jun", repo: "opencodex", pull_number: 42, title: "[WRONG BRANCH] Add a thing" }, @@ -972,9 +1007,8 @@ describe("GitHub Actions hardening", () => { pr: { base: { ref: "main" }, title: "Add a thing", draft: false }, eventPayload: { base: { ref: "dev" }, title: "Add a thing", draft: false }, }); - const [posted] = callsTo(wrongTarget, "issues.createComment") as [{ body: string }]; - expect(posted.body).toContain("currently targets `main`"); - expect(posted.body).not.toContain("currently targets `dev`"); + expect(lastEnforcerCommentBody(wrongTarget)).toContain("currently targets `main`"); + expect(lastEnforcerCommentBody(wrongTarget)).not.toContain("currently targets `dev`"); // The corrected-path sentence: the event still carries the old wrong // base, the live PR is on dev2-go. Naming the event's base here tells the @@ -1043,6 +1077,8 @@ describe("GitHub Actions hardening", () => { "issues.updateComment", "pulls.update", "graphql", + "issues.updateComment", + "issues.updateComment", ]); expect(result.warnings.join(" ")).toContain("Could not parse stored workflow state"); }); @@ -1069,6 +1105,7 @@ describe("GitHub Actions hardening", () => { "pulls.get", "issues.listComments", "issues.updateComment", + "issues.updateComment", ]); // Corrected: nothing to undo, but the state must still be cleared or the @@ -1137,20 +1174,20 @@ describe("GitHub Actions hardening", () => { pr: { base: { ref: "main" }, title: "Add a thing", draft: false, user: { login: "someone-else" } }, }); - const [comment] = callsTo(result, "issues.createComment") as [{ body: string }]; + const commentBody = lastEnforcerCommentBody(result); // Addressed to the PR author, so GitHub actually notifies them. - expect(comment.body).toContain("@someone-else"); + expect(commentBody).toContain("@someone-else"); // Names every branch involved, so the instruction is actionable without // context: where the PR is now, where it should go, and the one other // base that is legitimate but not the default. - expect(comment.body).toContain("`main`"); - expect(comment.body).toContain("`dev`"); - expect(comment.body).toContain("`dev2-go`"); + expect(commentBody).toContain("`main`"); + expect(commentBody).toContain("`dev`"); + expect(commentBody).toContain("`dev2-go`"); // Points at the documentation rather than assuming the reader knows. - expect(comment.body).toContain("https://lidge-jun.github.io/opencodex/contributing/"); + expect(commentBody).toContain("https://lidge-jun.github.io/opencodex/contributing/"); // And carries the state the next run needs. - expect(comment.body).toContain(MARKER); - expect(comment.body).toContain('"version":1'); + expect(commentBody).toContain(MARKER); + expect(commentBody).toContain('"version":1'); }); test("comment listing asks for full pages, so the bot's own comment is found", async () => { @@ -1230,10 +1267,12 @@ describe("GitHub Actions hardening", () => { "issues.updateComment", "pulls.update", "graphql", + "issues.updateComment", + "issues.updateComment", ]); - const [refreshed] = callsTo(wrong, "issues.updateComment") as [{ body: string }]; - expect(refreshed.body).toContain(`"version":${version}`); - expect(refreshed.body).toContain('"active":true'); + expect(lastEnforcerCommentBody(wrong)).toContain(`"version":${version}`); + expect(lastEnforcerCommentBody(wrong)).toContain('"active":true'); + expect(wrong.warnings.some((w) => w.startsWith("setFailed:"))).toBe(true); } }); @@ -1280,19 +1319,21 @@ describe("GitHub Actions hardening", () => { expect(cleared.body).toContain('"active":false'); }); - test("the state comment is written before the PR is touched", async () => { - // Order is the recovery story. The comment records what the workflow is - // about to change; if a mutation fails afterwards, a rerun reads that - // record and finishes the job. Write the PR first and a failure in - // between leaves a renamed, drafted PR with no record that the bot did - // it — permanently stuck. The scenario above asserts the full call list, - // but this states the invariant on its own so a reordering says why. + test("ownership comment is checkpointed before mutations and finalized after", async () => { + // Pending ownership is written before title/draft so a crash mid-mutation + // still records what the bot owns. autoDraftedByBot is only true in the + // final body after convertToDraft succeeds (#626 / #631). const result = await run({ pr: { base: { ref: "main" }, draft: false } }); const methods = methodsOf(result); - const comment = methods.indexOf("issues.createComment"); - expect(comment).toBeGreaterThan(-1); - expect(comment).toBeLessThan(methods.indexOf("pulls.update")); - expect(comment).toBeLessThan(methods.indexOf("graphql")); + const pending = methods.indexOf("issues.createComment"); + const title = methods.indexOf("pulls.update"); + const draft = methods.indexOf("graphql"); + const finalUpdate = methods.lastIndexOf("issues.updateComment"); + expect(pending).toBeGreaterThan(-1); + expect(pending).toBeLessThan(title); + expect(title).toBeLessThan(draft); + expect(draft).toBeLessThan(finalUpdate); + expect(lastEnforcerCommentBody(result)).toContain('"autoDraftedByBot":true'); }); test("a title that is exactly the prefix is still enforced", async () => { @@ -1305,18 +1346,18 @@ describe("GitHub Actions hardening", () => { pr: { base: { ref: "main" }, title: "[WRONG BRANCH] ", draft: false }, }); - // Already prefixed, so no title write — but the state and the draft - // conversion still have to happen. + // Already prefixed, so no title write — but pending/draft/final still run. expect(callsTo(result, "pulls.update")).toEqual([]); expect(methodsOf(result)).toEqual([ "pulls.get", "issues.listComments", "issues.createComment", "graphql", + "issues.updateComment", + "issues.updateComment", ]); - const [comment] = callsTo(result, "issues.createComment") as [{ body: string }]; - expect(comment.body).toContain('"titlePrefixedByBot":false'); - expect(comment.body).toContain('"autoDraftedByBot":true'); + expect(lastEnforcerCommentBody(result)).toContain('"titlePrefixedByBot":false'); + expect(lastEnforcerCommentBody(result)).toContain('"autoDraftedByBot":true'); }); test("an empty title is enforced rather than skipped", async () => { @@ -1335,6 +1376,7 @@ describe("GitHub Actions hardening", () => { "issues.listComments", "issues.createComment", "pulls.update", + "issues.updateComment", ]); }); @@ -1357,10 +1399,10 @@ describe("GitHub Actions hardening", () => { "pulls.get", "issues.listComments", "issues.createComment", + "issues.updateComment", ]); - const [comment] = callsTo(result, "issues.createComment") as [{ body: string }]; - expect(comment.body).toContain('"titlePrefixedByBot":false'); - expect(comment.body).toContain('"active":true'); + expect(lastEnforcerCommentBody(result)).toContain('"titlePrefixedByBot":false'); + expect(lastEnforcerCommentBody(result)).toContain('"active":true'); }); test("a title that already carries the prefix twice is still enforced", async () => { @@ -1379,12 +1421,13 @@ describe("GitHub Actions hardening", () => { "issues.listComments", "issues.createComment", "graphql", + "issues.updateComment", + "issues.updateComment", ]); // Already prefixed by the `startsWith` test, so no third prefix is added. expect(callsTo(result, "pulls.update")).toEqual([]); - const [comment] = callsTo(result, "issues.createComment") as [{ body: string }]; - expect(comment.body).toContain('"active":true'); - expect(comment.body).toContain('"autoDraftedByBot":true'); + expect(lastEnforcerCommentBody(result)).toContain('"active":true'); + expect(lastEnforcerCommentBody(result)).toContain('"autoDraftedByBot":true'); }); test("with two bot comments, the workflow reads and writes the first", async () => { @@ -1564,27 +1607,37 @@ describe("GitHub Actions hardening", () => { expect(probe.bun).toBe("undefined"); }); - test("an API failure is never swallowed, whatever status it carries", async () => { - // Octokit rejects with a `RequestError` that has a `.status`, and an - // audit round swallowed exactly one code: - // - // catch (error) { if (error.status === 404) return; throw error; } - // - // A green workflow, an un-drafted PR. Drive the failure at several - // statuses so a code-specific catch cannot hide in the gap. + test("a draft GraphQL failure is soft-failed with accurate state and a hard check failure", async () => { + // Observed on PR #626: convertPullRequestToDraft failed with + // "Resource not accessible by integration", the job crashed before + // setFailed, and the PR stayed ready. Soft-catch the draft mutation, + // record autoDraftedByBot:false, explain the fallback, and still fail + // the required check so merge stays blocked. const { script } = await readEnforcePrTarget(); for (const status of [403, 404, 422, 500]) { - await expect( - runEnforcePrTarget(script, { - pr: { base: { ref: "main" }, draft: false }, - failOn: ["graphql"], - failStatus: status, - }), - ).rejects.toThrow(/simulated failure: graphql/); + const result = await runEnforcePrTarget(script, { + pr: { base: { ref: "main" }, draft: false }, + failOn: ["graphql"], + failStatus: status, + }); + expect(methodsOf(result)).toEqual([ + "pulls.get", + "issues.listComments", + "issues.createComment", + "pulls.update", + "graphql", + "issues.updateComment", + ]); + const commentBody = lastEnforcerCommentBody(result); + expect(commentBody).toContain('"autoDraftedByBot":false'); + expect(commentBody).toContain("Automatic draft conversion failed"); + expect(result.warnings.some((w) => w.includes("Could not convert pull request to draft"))).toBe(true); + expect(result.warnings.some((w) => w.startsWith("setFailed:"))).toBe(true); } - // Same for the title update, which fails before the draft conversion. + // Title update failures still propagate — without the prefix the gate + // has no durable signal on the PR itself. for (const status of [403, 404, 422]) { await expect( runEnforcePrTarget(script, { @@ -1596,40 +1649,44 @@ describe("GitHub Actions hardening", () => { } }); - test("a failed draft conversion propagates, and the state is already stored", async () => { - // Observed on PR #527 (devlog .../050_live_evidence.md): the GraphQL - // mutation failed, the workflow ended in failure, and the PR stayed - // ready — but the comment already claimed `autoDraftedByBot: true`. - // - // The ordering is deliberate: writing the state first is what makes a - // rerun recoverable. This pins both halves of it, including the part - // that is wrong — the recorded state disagrees with reality until the - // rerun. The redesign in 040 owns fixing that; this test documents it so - // the fix is visible when it lands. + test("a failed draft conversion does not claim autoDraftedByBot", async () => { const { script } = await readEnforcePrTarget(); - const observed: string[] = []; - - await expect( - runEnforcePrTarget(script, { - pr: { base: { ref: "main" }, draft: false }, - failOn: ["graphql"], - }).catch((error: unknown) => { - observed.push(String(error)); - throw error; - }), - ).rejects.toThrow(/simulated failure: graphql/); - - // The failure is not swallowed. An audit round wrapped the whole script - // in `try { … } catch {}`, which turns every API failure into a silent - // no-op and a green check; this assertion is what makes that visible. - expect(observed).toHaveLength(1); - - // …and the state comment went out before the mutation that failed. - const upTo = await runEnforcePrTarget(script, { + const result = await runEnforcePrTarget(script, { pr: { base: { ref: "main" }, draft: false }, - failOn: ["pulls.update"], - }).catch(() => null); - expect(upTo).toBeNull(); + failOn: ["graphql"], + }); + const commentBody = lastEnforcerCommentBody(result); + expect(commentBody).toContain('"autoDraftedByBot":false'); + expect(commentBody).toContain('"titlePrefixedByBot":true'); + expect(result.warnings.some((w) => w.startsWith("setFailed:"))).toBe(true); + }); + + test("a failed ready-for-review conversion keeps ownership active for retry", async () => { + const { script } = await readEnforcePrTarget(); + const result = await runEnforcePrTarget(script, { + pr: { base: { ref: "dev" }, draft: true, title: "[WRONG BRANCH] Add a thing" }, + comments: [ + { + id: 7, + user: { login: "github-actions[bot]" }, + body: [ + "", + ``, + ].join("\n"), + }, + ], + failOn: ["graphql"], + }); + const commentBody = lastEnforcerCommentBody(result); + expect(commentBody).toContain('"active":true'); + expect(commentBody).toContain('"autoDraftedByBot":true'); + expect(commentBody).toContain("Automatic ready-for-review conversion failed"); + expect(result.warnings.some((w) => w.includes("Could not mark pull request ready for review"))).toBe(true); }); }); @@ -1646,6 +1703,7 @@ describe("GitHub Actions hardening", () => { expect(script).toMatch(/storedState\.titlePrefixedByBot/); expect(script).toMatch(/await\s+convertToDraft\(\)/); expect(script).toMatch(/await\s+markReadyForReview\(\)/); + expect(script).toMatch(/core\.setFailed\(/); // Tie each helper to its GraphQL body. Asserting that the call and the // mutation name both appear somewhere leaves a gap: declaring an empty @@ -1673,25 +1731,20 @@ describe("GitHub Actions hardening", () => { expect(script).toMatch(/\n\s*if \(!storedState\?\.active\) \{\n/); expect(script).toMatch(/\n\s*if \(wrongBase\) \{\n/); - // Observed on PR #527 (devlog .../050_live_evidence.md): the state is written - // BEFORE the mutation and is not reconciled when the mutation fails, so a - // failed convertToDraft still records autoDraftedByBot: true. This documents - // that ordering rather than endorsing it — the redesign in 040 owns fixing - // it, and this assertion should change with it. - // - // Scope the comparison to the wrong-base branch: upsertComment is called from - // both branches, so comparing across the whole script would measure whichever - // call happens to come first in the text. + // Pending ownership is written before mutations; convertToDraft runs next; + // a later upsertComment records autoDraftedByBot only after success (#631). const branchStart = script.indexOf("if (wrongBase) {"); expect(branchStart).toBeGreaterThan(-1); const branch = script.slice(branchStart); - const stateWriteIndex = branch.indexOf("await upsertComment("); + const pendingWriteIndex = branch.indexOf("await upsertComment("); const draftCallIndex = branch.indexOf("await convertToDraft()"); - expect(stateWriteIndex).toBeGreaterThan(-1); - expect(draftCallIndex).toBeGreaterThan(stateWriteIndex); + const afterDraftWriteIndex = branch.indexOf("await upsertComment(", draftCallIndex); + expect(pendingWriteIndex).toBeGreaterThan(-1); + expect(draftCallIndex).toBeGreaterThan(-1); + expect(pendingWriteIndex).toBeLessThan(draftCallIndex); + expect(afterDraftWriteIndex).toBeGreaterThan(draftCallIndex); }); - test("docs deployment is pinned, bounded, and scoped to Pages", async () => { const workflow = await readText(".github/workflows/deploy-docs.yml");