From 620091d5c25a199724b73db135d526cc1585687a Mon Sep 17 00:00:00 2001 From: aidan garske Date: Tue, 14 Jul 2026 12:35:57 -0700 Subject: [PATCH 1/3] Fix PR OSP concurrency collision and add AI triage PR comments --- .github/scripts/osp-triage.py | 114 +++++++++++++++++-- .github/workflows/lint-workflows.yml | 79 +++++++++++++ .github/workflows/nightly-multi-compiler.yml | 4 +- .github/workflows/osp-report.yml | 73 +++++++++++- .github/workflows/perf-regression.yml | 4 +- 5 files changed, 253 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/lint-workflows.yml diff --git a/.github/scripts/osp-triage.py b/.github/scripts/osp-triage.py index d0097f70..2c1475a5 100644 --- a/.github/scripts/osp-triage.py +++ b/.github/scripts/osp-triage.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Triage a nightly-osp run and post one clean Slack health report. +"""Triage an OSP run: Slack health report for the nightly, PR comment for ci:* runs. Classification is by RETRY OUTCOME, not guesswork: every job that did not succeed is retried once. Cleared on retry = flake; failed twice = a real @@ -16,6 +16,7 @@ ANTHROPIC_API_KEY optional; without it survivors report without notes CLAUDE_MODEL optional; default claude-sonnet-4-6 AUTO_RETRY "true" to rerun non-passing jobs once before reporting + PR_NUMBER optional; report to this PR as a comment instead of Slack DRY_RUN "true" to print the payload instead of posting ANALYZE_ATTEMPT optional; report a specific past attempt as-is """ @@ -36,8 +37,14 @@ ANTHROPIC_KEY = os.environ.get("ANTHROPIC_API_KEY", "").strip() CLAUDE_MODEL = os.environ.get("CLAUDE_MODEL", "claude-sonnet-4-6") AUTO_RETRY = os.environ.get("AUTO_RETRY", "false").lower() == "true" -DRY_RUN = os.environ.get("DRY_RUN", "false").lower() == "true" or not SLACK_WEBHOOK +PR_NUMBER = os.environ.get("PR_NUMBER", "").strip() +DRY_RUN = os.environ.get("DRY_RUN", "false").lower() == "true" or not ( + PR_NUMBER or SLACK_WEBHOOK) FORCE_ATTEMPT = os.environ.get("ANALYZE_ATTEMPT") +PR_MARKER = "" +COMMENT_LIMIT = 65536 +SUITE_DESC = ("wolfProvider OSP CI suite, run on a pull request via a ci:* label" + if PR_NUMBER else "wolfProvider nightly OSP CI suite") LOG_TAIL_LINES = 300 AI_LOG_CHARS = 4000 @@ -176,6 +183,11 @@ def prefix_of(name): return name.split(" / ")[0].strip() +def is_suite(job): + """A called-workflow job; the bare `select` orchestrator is not one.""" + return " / " in job.get("name", "") + + def suite_of(prefix): return WAVE_SUFFIX.sub("", prefix) @@ -200,7 +212,7 @@ def ai_triage(failures): f"failing steps: {', '.join(f['steps']) or 'n/a'}\n" f"log tail:\n{f['log'][-AI_LOG_CHARS:]}\n") prompt = ( - "You are triaging failures in the wolfProvider nightly OSP CI suite. " + f"You are triaging failures in the {SUITE_DESC}. " "Each job below failed twice (an auto-retry did not clear it). Infra " "setup failures are already filtered out.\n" "Judge each job ONLY from its own log; do not speculate about outages " @@ -283,6 +295,70 @@ def render_text(blocks): return "\n".join(out) +def mrkdwn_to_gfm(text): + text = re.sub(r"<(https?://[^|>]+)\|([^>]+)>", r"[\2](\1)", text) + text = re.sub(r"<(https?://[^>]+)>", r"\1", text) + # Anchored on non-space: log text like "rm -f *.o *.a" is not bold. + return re.sub(r"(? COMMENT_LIMIT: + body = body[:COMMENT_LIMIT - 40].rstrip() + "\n\n_(report truncated)_" + if DRY_RUN: + print("=== DRY RUN (no PR comment) ===\n") + print(body) + return + mine = find_prior_comment() + if mine: + # PATCH sets an absolute body, so retrying is safe. + gh(f"/repos/{REPO}/issues/comments/{mine['id']}", + method="PATCH", data={"body": body}) + else: + # POST is not idempotent: a 5xx retry would duplicate the comment. + gh(f"/repos/{REPO}/issues/{PR_NUMBER}/comments", + method="POST", data={"body": body}, retry=False) + + def section(text): return {"type": "section", "text": {"type": "mrkdwn", "text": text}} @@ -372,6 +448,14 @@ def main(): return jobs = merged_jobs(attempt) + + # Any label starts a PR OSP run; with no ci:* label only `select` runs, and "healthy 1/1" reads as OSP-green. + if PR_NUMBER: + jobs = [j for j in jobs if is_suite(j)] + if not jobs: + print("no OSP suite jobs in this run - nothing to report") + return + failed = [j for j in jobs if j.get("conclusion") == "failure"] n_success = sum(1 for j in jobs if j.get("conclusion") == "success") n_cancelled = sum(1 for j in jobs if j.get("conclusion") == "cancelled") @@ -380,7 +464,8 @@ def main(): recovered = [] if attempt >= 2: prev = {prefix_of(j["name"]) for j in all_jobs(RUN_ID, attempt=1) - if j.get("conclusion") in ("failure", "cancelled")} + if j.get("conclusion") in ("failure", "cancelled") + and (not PR_NUMBER or is_suite(j))} recovered = sorted(prev - {prefix_of(j["name"]) for j in failed}) # Group failed jobs per suite. Infra-setup failures are obvious flakes and @@ -439,8 +524,8 @@ def main(): date_str = (run.get("created_at") or "")[:10] or datetime.date.today().isoformat() - # Trend vs prior nightlies (best-effort). - hist = history() + # Trend vs prior nightlies (best-effort). Meaningless on a PR branch. + hist = [] if PR_NUMBER else history() trend = "" if hist: spark = sparkline(list(reversed(hist)) + [n_success / total_pf if total_pf else 1]) @@ -453,16 +538,20 @@ def main(): {"type": "mrkdwn", "text": f"*Status:*\n{status}"}, {"type": "mrkdwn", "text": f"*Pass rate:*\n{n_success} / {total_pf} jobs ({pct}%)"}, - {"type": "mrkdwn", "text": f"*Trend:*\n{trend or 'building history'}"}, ] + if not PR_NUMBER: + fields.append({"type": "mrkdwn", + "text": f"*Trend:*\n{trend or 'building history'}"}) breakdown = (f"\U0001F7E2 {n_success} passed" + (f" ({len(recovered)} recovered)" if recovered else "") + f" \U0001F7E1 {flake_jobs} flaked" f" \U0001F534 {real_jobs} real" f" ⚪ {n_cancelled} cancelled") + title = ("wolfProvider OSP — PR checks" if PR_NUMBER + else "wolfProvider Nightly OSP") blocks = [ {"type": "header", "text": {"type": "plain_text", - "text": "wolfProvider Nightly OSP", "emoji": True}}, + "text": title, "emoji": True}}, {"type": "section", "fields": fields}, section(breakdown), ] @@ -495,9 +584,14 @@ def main(): for chunk in chunk_lines(flake_lines): blocks.append(section(chunk)) + scope = "" if PR_NUMBER else "wolfSSL v5.9.1 (Wave 1) + v5.8.4 (Wave 2) · " blocks.append({"type": "context", "elements": [{"type": "mrkdwn", - "text": f"wolfSSL v5.9.1 (Wave 1) + v5.8.4 (Wave 2) · " - f"attempt {attempt} · {total_all} jobs · <{run_url()}|View run>"}]}) + "text": f"{scope}attempt {attempt} · {total_all} jobs · " + f"<{run_url()}|View run>"}]}) + + if PR_NUMBER: + post_pr_comment(blocks) + return fallback = f"wolfProvider Nightly OSP: {status} ({n_success}/{total_pf} passed)" post_slack(color, fallback, blocks) diff --git a/.github/workflows/lint-workflows.yml b/.github/workflows/lint-workflows.yml new file mode 100644 index 00000000..f295fb10 --- /dev/null +++ b/.github/workflows/lint-workflows.yml @@ -0,0 +1,79 @@ +name: Workflow lint + +# START OF COMMON SECTION +on: + push: + branches: [ 'master', 'main', 'release/**' ] + pull_request: + branches: [ '**' ] + types: [opened, synchronize, reopened, ready_for_review] + paths: + - '.github/workflows/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +# END OF COMMON SECTION + +permissions: + contents: read + +jobs: + reusable-concurrency: + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + name: Reusable workflows must not self-cancel + runs-on: ubuntu-22.04 + timeout-minutes: 5 + steps: + - name: Checkout wolfProvider + uses: actions/checkout@v4 + + - name: Check concurrency in reusable workflows + run: | + python3 - <<'EOF' + import glob, sys, yaml + + paths = sorted(glob.glob('.github/workflows/*.yml') + + glob.glob('.github/workflows/*.yaml')) + bad = [] + for path in paths: + try: + with open(path) as f: + wf = yaml.safe_load(f) + except yaml.YAMLError as e: + print(f'{path}: unparseable: {e}') + sys.exit(1) + if not isinstance(wf, dict): + continue + # YAML 1.1 parses a bare `on:` key as the boolean True. + triggers = wf.get('on', wf.get(True)) or {} + # `on:` is legal as a mapping, a list, or a bare string. + if isinstance(triggers, str): + names = {triggers} + elif isinstance(triggers, list): + names = set(triggers) + elif isinstance(triggers, dict): + names = set(triggers) + else: + names = set() + if 'workflow_call' not in names: + continue + conc = wf.get('concurrency') or {} + if not isinstance(conc, dict): + continue + # Anything not literally false counts: "true"/${{ }} still cancel. + cip = conc.get('cancel-in-progress', False) + if cip is not False and str(cip).strip().lower() != 'false': + bad.append(f"{path}: cancel-in-progress={cip!r} " + f"group={conc.get('group')!r}") + + if bad: + print('Reusable workflow sets cancel-in-progress to something other') + print('than false. github.workflow resolves to the CALLER, so sibling') + print('calls in one run share a group and cancel each other. Set') + print('cancel-in-progress: false, or drop the concurrency block.') + for b in bad: + print(' ' + b) + sys.exit(1) + print('ok: no reusable workflow sets cancel-in-progress') + EOF diff --git a/.github/workflows/nightly-multi-compiler.yml b/.github/workflows/nightly-multi-compiler.yml index afcc3024..0552f1ef 100644 --- a/.github/workflows/nightly-multi-compiler.yml +++ b/.github/workflows/nightly-multi-compiler.yml @@ -12,9 +12,7 @@ on: workflow_call: {} workflow_dispatch: {} -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# No concurrency group: github.workflow is the caller's here, so any group collides with sibling calls. jobs: build_wolfprovider: diff --git a/.github/workflows/osp-report.yml b/.github/workflows/osp-report.yml index c10a7435..77f491c2 100644 --- a/.github/workflows/osp-report.yml +++ b/.github/workflows/osp-report.yml @@ -2,14 +2,19 @@ name: OSP Triage Report on: workflow_run: - workflows: ["Nightly OSP Suite"] + workflows: ["Nightly OSP Suite", "PR OSP (label-selected)"] types: [completed] workflow_dispatch: inputs: run_id: - description: "nightly-osp run ID to analyze" + description: "run ID to analyze (nightly-osp, or a PR OSP run)" required: true type: string + pr_number: + description: "comment on this PR instead of Slack (blank = nightly Slack report)" + required: false + type: string + default: "" auto_retry: description: "rerun failed jobs once before reporting" type: boolean @@ -18,25 +23,83 @@ on: permissions: actions: write contents: read + pull-requests: write jobs: report: runs-on: ubuntu-latest - # Skip untrusted fork-PR runs; only analyze canonical-repo runs. + # A fork controls workflow_run's NAME, so the nightly path (which carries SLACK_WEBHOOK_URL) must also match head_repository. if: > github.event_name == 'workflow_dispatch' || - github.event.workflow_run.head_repository.full_name == github.repository + (github.event.workflow_run.name == 'PR OSP (label-selected)' && + github.event.workflow_run.conclusion != 'cancelled') || + (github.event.workflow_run.name == 'Nightly OSP Suite' && + github.event.workflow_run.head_repository.full_name == github.repository) steps: + # PR OSP only: the nightly is already gated on head_repository, and its scheduled actor is not a collaborator. + - name: Verify trigger is trusted + if: github.event.workflow_run.name == 'PR OSP (label-selected)' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + ACTOR: ${{ github.event.workflow_run.actor.login }} + WF_PATH: ${{ github.event.workflow_run.path }} + run: | + # A fork controls both the workflow name and a new file's path, so pin the path. + if [ "$WF_PATH" != ".github/workflows/pr-osp-select.yml" ]; then + echo "::error::untrusted triggering workflow: $WF_PATH" + exit 1 + fi + # Distinguish "not a collaborator" (404) from an API failure; never conflate them. + if ! body=$(gh api "/repos/$REPO/collaborators/$ACTOR/permission" 2>&1); then + case "$body" in + *"Not Found"*) echo "::error::actor $ACTOR is not a collaborator"; exit 1 ;; + *) echo "::error::could not verify $ACTOR: $body"; exit 1 ;; + esac + fi + perm=$(printf '%s' "$body" | jq -r '.permission') + # triage can add labels, so it can legitimately start a PR OSP run. + case "$perm" in + admin|write|maintain|triage) echo "actor $ACTOR ok ($perm)" ;; + *) echo "::error::actor $ACTOR cannot label ($perm)"; exit 1 ;; + esac + - uses: actions/checkout@v4 with: fetch-depth: 1 + - name: Resolve PR number + id: pr + if: github.event.workflow_run.name == 'PR OSP (label-selected)' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + OWNER: ${{ github.event.workflow_run.head_repository.owner.login }} + BRANCH: ${{ github.event.workflow_run.head_branch }} + run: | + # pull_requests[] is empty for fork PRs; -f percent-encodes, since '&' is legal in a ref and would forge the query. + if ! raw=$(gh api -X GET "/repos/$REPO/pulls" -f head="$OWNER:$BRANCH" -f state=open 2>&1); then + echo "::error::pulls lookup failed for $OWNER:$BRANCH: $raw" + exit 1 + fi + n=$(printf '%s' "$raw" | jq -r --arg o "$OWNER" --arg b "$BRANCH" \ + '[.[] | select(.head.repo.owner.login == $o and .head.ref == $b)][0].number // empty') + # Empty here is legitimate (PR merged/closed mid-run); the triage step then skips. + [ -n "$n" ] || echo "no open PR for $OWNER:$BRANCH - skipping report" + echo "number=$n" >> "$GITHUB_OUTPUT" + - name: Triage and report + # A PR OSP run with no resolved PR has nowhere to report; never fall back to Slack. + if: > + github.event.workflow_run.name != 'PR OSP (label-selected)' || + steps.pr.outputs.number != '' env: GH_REPO: ${{ github.repository }} RUN_ID: ${{ github.event_name == 'workflow_dispatch' && inputs.run_id || github.event.workflow_run.id }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + PR_NUMBER: ${{ github.event_name == 'workflow_dispatch' && inputs.pr_number || steps.pr.outputs.number }} + # Slack only for a nightly-shaped report; a PR-targeted run must never reach the channel. + SLACK_WEBHOOK_URL: ${{ (github.event_name == 'workflow_dispatch' && inputs.pr_number == '' && secrets.SLACK_WEBHOOK_URL) || (github.event_name == 'workflow_run' && github.event.workflow_run.name != 'PR OSP (label-selected)' && secrets.SLACK_WEBHOOK_URL) || '' }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} AUTO_RETRY: ${{ github.event_name == 'workflow_run' && 'true' || github.event_name == 'workflow_dispatch' && inputs.auto_retry && 'true' || 'false' }} run: python3 .github/scripts/osp-triage.py diff --git a/.github/workflows/perf-regression.yml b/.github/workflows/perf-regression.yml index d218585e..2f44c0e2 100644 --- a/.github/workflows/perf-regression.yml +++ b/.github/workflows/perf-regression.yml @@ -6,9 +6,7 @@ on: workflow_dispatch: workflow_call: -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# No concurrency group: github.workflow is the caller's here, so any group collides with sibling calls. jobs: discover_versions: From 075a957c0c9d54a55d777620e5ae9b412375fd3d Mon Sep 17 00:00:00 2001 From: aidan garske Date: Tue, 14 Jul 2026 14:04:35 -0700 Subject: [PATCH 2/3] Fix codespell typo, install PyYAML for lint, add issues:write for PR comments --- .github/workflows/lint-workflows.yml | 5 ++++- .github/workflows/osp-report.yml | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-workflows.yml b/.github/workflows/lint-workflows.yml index f295fb10..9f356256 100644 --- a/.github/workflows/lint-workflows.yml +++ b/.github/workflows/lint-workflows.yml @@ -28,6 +28,9 @@ jobs: - name: Checkout wolfProvider uses: actions/checkout@v4 + - name: Install PyYAML + run: pip install --quiet pyyaml + - name: Check concurrency in reusable workflows run: | python3 - <<'EOF' @@ -41,7 +44,7 @@ jobs: with open(path) as f: wf = yaml.safe_load(f) except yaml.YAMLError as e: - print(f'{path}: unparseable: {e}') + print(f'{path}: unparsable: {e}') sys.exit(1) if not isinstance(wf, dict): continue diff --git a/.github/workflows/osp-report.yml b/.github/workflows/osp-report.yml index 77f491c2..22b022c4 100644 --- a/.github/workflows/osp-report.yml +++ b/.github/workflows/osp-report.yml @@ -23,6 +23,8 @@ on: permissions: actions: write contents: read + # issues: write because the comment upsert goes through the issues endpoint. + issues: write pull-requests: write jobs: From 8d64f3bb638e005ae345999f6e9d86744e687f85 Mon Sep 17 00:00:00 2001 From: aidan garske Date: Tue, 14 Jul 2026 15:27:36 -0700 Subject: [PATCH 3/3] Drop PR triage comments; keep the one-shot retry for PR OSP runs --- .github/scripts/osp-triage.py | 114 +++-------------------------- .github/workflows/osp-report.yml | 75 ++----------------- .github/workflows/pr-osp-retry.yml | 55 ++++++++++++++ 3 files changed, 70 insertions(+), 174 deletions(-) create mode 100644 .github/workflows/pr-osp-retry.yml diff --git a/.github/scripts/osp-triage.py b/.github/scripts/osp-triage.py index 2c1475a5..d0097f70 100644 --- a/.github/scripts/osp-triage.py +++ b/.github/scripts/osp-triage.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Triage an OSP run: Slack health report for the nightly, PR comment for ci:* runs. +"""Triage a nightly-osp run and post one clean Slack health report. Classification is by RETRY OUTCOME, not guesswork: every job that did not succeed is retried once. Cleared on retry = flake; failed twice = a real @@ -16,7 +16,6 @@ ANTHROPIC_API_KEY optional; without it survivors report without notes CLAUDE_MODEL optional; default claude-sonnet-4-6 AUTO_RETRY "true" to rerun non-passing jobs once before reporting - PR_NUMBER optional; report to this PR as a comment instead of Slack DRY_RUN "true" to print the payload instead of posting ANALYZE_ATTEMPT optional; report a specific past attempt as-is """ @@ -37,14 +36,8 @@ ANTHROPIC_KEY = os.environ.get("ANTHROPIC_API_KEY", "").strip() CLAUDE_MODEL = os.environ.get("CLAUDE_MODEL", "claude-sonnet-4-6") AUTO_RETRY = os.environ.get("AUTO_RETRY", "false").lower() == "true" -PR_NUMBER = os.environ.get("PR_NUMBER", "").strip() -DRY_RUN = os.environ.get("DRY_RUN", "false").lower() == "true" or not ( - PR_NUMBER or SLACK_WEBHOOK) +DRY_RUN = os.environ.get("DRY_RUN", "false").lower() == "true" or not SLACK_WEBHOOK FORCE_ATTEMPT = os.environ.get("ANALYZE_ATTEMPT") -PR_MARKER = "" -COMMENT_LIMIT = 65536 -SUITE_DESC = ("wolfProvider OSP CI suite, run on a pull request via a ci:* label" - if PR_NUMBER else "wolfProvider nightly OSP CI suite") LOG_TAIL_LINES = 300 AI_LOG_CHARS = 4000 @@ -183,11 +176,6 @@ def prefix_of(name): return name.split(" / ")[0].strip() -def is_suite(job): - """A called-workflow job; the bare `select` orchestrator is not one.""" - return " / " in job.get("name", "") - - def suite_of(prefix): return WAVE_SUFFIX.sub("", prefix) @@ -212,7 +200,7 @@ def ai_triage(failures): f"failing steps: {', '.join(f['steps']) or 'n/a'}\n" f"log tail:\n{f['log'][-AI_LOG_CHARS:]}\n") prompt = ( - f"You are triaging failures in the {SUITE_DESC}. " + "You are triaging failures in the wolfProvider nightly OSP CI suite. " "Each job below failed twice (an auto-retry did not clear it). Infra " "setup failures are already filtered out.\n" "Judge each job ONLY from its own log; do not speculate about outages " @@ -295,70 +283,6 @@ def render_text(blocks): return "\n".join(out) -def mrkdwn_to_gfm(text): - text = re.sub(r"<(https?://[^|>]+)\|([^>]+)>", r"[\2](\1)", text) - text = re.sub(r"<(https?://[^>]+)>", r"\1", text) - # Anchored on non-space: log text like "rm -f *.o *.a" is not bold. - return re.sub(r"(? COMMENT_LIMIT: - body = body[:COMMENT_LIMIT - 40].rstrip() + "\n\n_(report truncated)_" - if DRY_RUN: - print("=== DRY RUN (no PR comment) ===\n") - print(body) - return - mine = find_prior_comment() - if mine: - # PATCH sets an absolute body, so retrying is safe. - gh(f"/repos/{REPO}/issues/comments/{mine['id']}", - method="PATCH", data={"body": body}) - else: - # POST is not idempotent: a 5xx retry would duplicate the comment. - gh(f"/repos/{REPO}/issues/{PR_NUMBER}/comments", - method="POST", data={"body": body}, retry=False) - - def section(text): return {"type": "section", "text": {"type": "mrkdwn", "text": text}} @@ -448,14 +372,6 @@ def main(): return jobs = merged_jobs(attempt) - - # Any label starts a PR OSP run; with no ci:* label only `select` runs, and "healthy 1/1" reads as OSP-green. - if PR_NUMBER: - jobs = [j for j in jobs if is_suite(j)] - if not jobs: - print("no OSP suite jobs in this run - nothing to report") - return - failed = [j for j in jobs if j.get("conclusion") == "failure"] n_success = sum(1 for j in jobs if j.get("conclusion") == "success") n_cancelled = sum(1 for j in jobs if j.get("conclusion") == "cancelled") @@ -464,8 +380,7 @@ def main(): recovered = [] if attempt >= 2: prev = {prefix_of(j["name"]) for j in all_jobs(RUN_ID, attempt=1) - if j.get("conclusion") in ("failure", "cancelled") - and (not PR_NUMBER or is_suite(j))} + if j.get("conclusion") in ("failure", "cancelled")} recovered = sorted(prev - {prefix_of(j["name"]) for j in failed}) # Group failed jobs per suite. Infra-setup failures are obvious flakes and @@ -524,8 +439,8 @@ def main(): date_str = (run.get("created_at") or "")[:10] or datetime.date.today().isoformat() - # Trend vs prior nightlies (best-effort). Meaningless on a PR branch. - hist = [] if PR_NUMBER else history() + # Trend vs prior nightlies (best-effort). + hist = history() trend = "" if hist: spark = sparkline(list(reversed(hist)) + [n_success / total_pf if total_pf else 1]) @@ -538,20 +453,16 @@ def main(): {"type": "mrkdwn", "text": f"*Status:*\n{status}"}, {"type": "mrkdwn", "text": f"*Pass rate:*\n{n_success} / {total_pf} jobs ({pct}%)"}, + {"type": "mrkdwn", "text": f"*Trend:*\n{trend or 'building history'}"}, ] - if not PR_NUMBER: - fields.append({"type": "mrkdwn", - "text": f"*Trend:*\n{trend or 'building history'}"}) breakdown = (f"\U0001F7E2 {n_success} passed" + (f" ({len(recovered)} recovered)" if recovered else "") + f" \U0001F7E1 {flake_jobs} flaked" f" \U0001F534 {real_jobs} real" f" ⚪ {n_cancelled} cancelled") - title = ("wolfProvider OSP — PR checks" if PR_NUMBER - else "wolfProvider Nightly OSP") blocks = [ {"type": "header", "text": {"type": "plain_text", - "text": title, "emoji": True}}, + "text": "wolfProvider Nightly OSP", "emoji": True}}, {"type": "section", "fields": fields}, section(breakdown), ] @@ -584,14 +495,9 @@ def main(): for chunk in chunk_lines(flake_lines): blocks.append(section(chunk)) - scope = "" if PR_NUMBER else "wolfSSL v5.9.1 (Wave 1) + v5.8.4 (Wave 2) · " blocks.append({"type": "context", "elements": [{"type": "mrkdwn", - "text": f"{scope}attempt {attempt} · {total_all} jobs · " - f"<{run_url()}|View run>"}]}) - - if PR_NUMBER: - post_pr_comment(blocks) - return + "text": f"wolfSSL v5.9.1 (Wave 1) + v5.8.4 (Wave 2) · " + f"attempt {attempt} · {total_all} jobs · <{run_url()}|View run>"}]}) fallback = f"wolfProvider Nightly OSP: {status} ({n_success}/{total_pf} passed)" post_slack(color, fallback, blocks) diff --git a/.github/workflows/osp-report.yml b/.github/workflows/osp-report.yml index 22b022c4..c10a7435 100644 --- a/.github/workflows/osp-report.yml +++ b/.github/workflows/osp-report.yml @@ -2,19 +2,14 @@ name: OSP Triage Report on: workflow_run: - workflows: ["Nightly OSP Suite", "PR OSP (label-selected)"] + workflows: ["Nightly OSP Suite"] types: [completed] workflow_dispatch: inputs: run_id: - description: "run ID to analyze (nightly-osp, or a PR OSP run)" + description: "nightly-osp run ID to analyze" required: true type: string - pr_number: - description: "comment on this PR instead of Slack (blank = nightly Slack report)" - required: false - type: string - default: "" auto_retry: description: "rerun failed jobs once before reporting" type: boolean @@ -23,85 +18,25 @@ on: permissions: actions: write contents: read - # issues: write because the comment upsert goes through the issues endpoint. - issues: write - pull-requests: write jobs: report: runs-on: ubuntu-latest - # A fork controls workflow_run's NAME, so the nightly path (which carries SLACK_WEBHOOK_URL) must also match head_repository. + # Skip untrusted fork-PR runs; only analyze canonical-repo runs. if: > github.event_name == 'workflow_dispatch' || - (github.event.workflow_run.name == 'PR OSP (label-selected)' && - github.event.workflow_run.conclusion != 'cancelled') || - (github.event.workflow_run.name == 'Nightly OSP Suite' && - github.event.workflow_run.head_repository.full_name == github.repository) + github.event.workflow_run.head_repository.full_name == github.repository steps: - # PR OSP only: the nightly is already gated on head_repository, and its scheduled actor is not a collaborator. - - name: Verify trigger is trusted - if: github.event.workflow_run.name == 'PR OSP (label-selected)' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - ACTOR: ${{ github.event.workflow_run.actor.login }} - WF_PATH: ${{ github.event.workflow_run.path }} - run: | - # A fork controls both the workflow name and a new file's path, so pin the path. - if [ "$WF_PATH" != ".github/workflows/pr-osp-select.yml" ]; then - echo "::error::untrusted triggering workflow: $WF_PATH" - exit 1 - fi - # Distinguish "not a collaborator" (404) from an API failure; never conflate them. - if ! body=$(gh api "/repos/$REPO/collaborators/$ACTOR/permission" 2>&1); then - case "$body" in - *"Not Found"*) echo "::error::actor $ACTOR is not a collaborator"; exit 1 ;; - *) echo "::error::could not verify $ACTOR: $body"; exit 1 ;; - esac - fi - perm=$(printf '%s' "$body" | jq -r '.permission') - # triage can add labels, so it can legitimately start a PR OSP run. - case "$perm" in - admin|write|maintain|triage) echo "actor $ACTOR ok ($perm)" ;; - *) echo "::error::actor $ACTOR cannot label ($perm)"; exit 1 ;; - esac - - uses: actions/checkout@v4 with: fetch-depth: 1 - - name: Resolve PR number - id: pr - if: github.event.workflow_run.name == 'PR OSP (label-selected)' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - OWNER: ${{ github.event.workflow_run.head_repository.owner.login }} - BRANCH: ${{ github.event.workflow_run.head_branch }} - run: | - # pull_requests[] is empty for fork PRs; -f percent-encodes, since '&' is legal in a ref and would forge the query. - if ! raw=$(gh api -X GET "/repos/$REPO/pulls" -f head="$OWNER:$BRANCH" -f state=open 2>&1); then - echo "::error::pulls lookup failed for $OWNER:$BRANCH: $raw" - exit 1 - fi - n=$(printf '%s' "$raw" | jq -r --arg o "$OWNER" --arg b "$BRANCH" \ - '[.[] | select(.head.repo.owner.login == $o and .head.ref == $b)][0].number // empty') - # Empty here is legitimate (PR merged/closed mid-run); the triage step then skips. - [ -n "$n" ] || echo "no open PR for $OWNER:$BRANCH - skipping report" - echo "number=$n" >> "$GITHUB_OUTPUT" - - name: Triage and report - # A PR OSP run with no resolved PR has nowhere to report; never fall back to Slack. - if: > - github.event.workflow_run.name != 'PR OSP (label-selected)' || - steps.pr.outputs.number != '' env: GH_REPO: ${{ github.repository }} RUN_ID: ${{ github.event_name == 'workflow_dispatch' && inputs.run_id || github.event.workflow_run.id }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ github.event_name == 'workflow_dispatch' && inputs.pr_number || steps.pr.outputs.number }} - # Slack only for a nightly-shaped report; a PR-targeted run must never reach the channel. - SLACK_WEBHOOK_URL: ${{ (github.event_name == 'workflow_dispatch' && inputs.pr_number == '' && secrets.SLACK_WEBHOOK_URL) || (github.event_name == 'workflow_run' && github.event.workflow_run.name != 'PR OSP (label-selected)' && secrets.SLACK_WEBHOOK_URL) || '' }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} AUTO_RETRY: ${{ github.event_name == 'workflow_run' && 'true' || github.event_name == 'workflow_dispatch' && inputs.auto_retry && 'true' || 'false' }} run: python3 .github/scripts/osp-triage.py diff --git a/.github/workflows/pr-osp-retry.yml b/.github/workflows/pr-osp-retry.yml new file mode 100644 index 00000000..08ba84f4 --- /dev/null +++ b/.github/workflows/pr-osp-retry.yml @@ -0,0 +1,55 @@ +name: PR OSP retry + +# No concurrency group: workflow_run reports github.ref as the default branch, so any group self-cancels. + +on: + workflow_run: + workflows: ["PR OSP (label-selected)"] + types: [completed] + +permissions: + actions: write + +jobs: + retry: + name: Rerun failed jobs once + runs-on: ubuntu-latest + timeout-minutes: 5 + # 'failure' skips superseded runs (those land as 'cancelled'); attempt 1 bounds this to one retry. + if: > + github.event.workflow_run.conclusion == 'failure' && + github.event.workflow_run.run_attempt == 1 + steps: + - name: Verify trigger is trusted + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + ACTOR: ${{ github.event.workflow_run.actor.login }} + WF_PATH: ${{ github.event.workflow_run.path }} + run: | + # A fork controls both the workflow name and a new file's path, so pin the path. + if [ "$WF_PATH" != ".github/workflows/pr-osp-select.yml" ]; then + echo "::error::untrusted triggering workflow: $WF_PATH" + exit 1 + fi + # Distinguish "not a collaborator" (404) from an API failure; never conflate them. + if ! body=$(gh api "/repos/$REPO/collaborators/$ACTOR/permission" 2>&1); then + case "$body" in + *"Not Found"*) echo "::error::actor $ACTOR is not a collaborator"; exit 1 ;; + *) echo "::error::could not verify $ACTOR: $body"; exit 1 ;; + esac + fi + perm=$(printf '%s' "$body" | jq -r '.permission') + # triage can add labels, so it can legitimately start a PR OSP run. + case "$perm" in + admin|write|maintain|triage) echo "actor $ACTOR ok ($perm)" ;; + *) echo "::error::actor $ACTOR cannot label ($perm)"; exit 1 ;; + esac + + - name: Rerun failed jobs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_ID: ${{ github.event.workflow_run.id }} + run: | + gh api -X POST \ + "/repos/${{ github.repository }}/actions/runs/$RUN_ID/rerun-failed-jobs"