diff --git a/Makefile.toml b/Makefile.toml index ef771cb..42242e9 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -60,12 +60,7 @@ dependencies = [ [tasks.check-site-content] workspace = false script = [ - "python3 scripts/github/validate_change_bundle.py artifacts/github/bundles", - "python3 scripts/github/validate_upstream_review.py artifacts/github/review-queue", - "python3 scripts/github/validate_upstream_review.py artifacts/github/reviews", - "python3 scripts/github/test_social_post_contract.py", - "python3 scripts/github/validate_social_post.py artifacts/social/x", - "python3 scripts/github/validate_signal_entry.py site/src/content/signals", + "cargo run -p decodex --bin decodex -- radar validate", ] [tasks.check-site-types] diff --git a/README.md b/README.md index 4a12bdf..44184ae 100644 --- a/README.md +++ b/README.md @@ -61,10 +61,10 @@ runtime. - `apps/decodex-app/` owns the native macOS app that manages Decodex Codex accounts through the bundled Rust app helper. - `site/` owns the Astro static site and checked-in public content. -- `apps/decodex/src/radar.rs` owns Rust Radar queue, release-delta, and validation - commands. -- `scripts/github/` owns deterministic GitHub bundle, render, validation, backfill, - ledger import, and analysis-support scripts. +- `apps/decodex/src/radar.rs` owns Rust Radar queue, release-delta, bundle, render, + validation, backfill, and ledger commands. +- `scripts/github/` owns the automation-only Codex AI analysis helper and shared + schema support for that helper. - `artifacts/github/` owns checked-in review queues, upstream reviews, GitHub bundles, impact records, and editorial analysis drafts. - `artifacts/archive/` owns checked-in recovery manifests for cold Radar batches stored @@ -79,8 +79,8 @@ runtime. Runtime authority stays in `apps/decodex/src/`, the registered project contracts under `~/.codex/decodex/projects//`, and the governing specs under `docs/spec/`. -Public site authority stays in `site/`, `scripts/github/`, `artifacts/github/`, and -the site/content specs. +Public site authority stays in `site/`, `apps/decodex/src/radar.rs`, +`artifacts/github/`, and the site/content specs. Historical Radar trace is local by default. `decodex radar refresh-upstream-queue` writes `.decodex/radar.sqlite3` and refreshes `upstream_review_queue/v1` so every @@ -215,7 +215,8 @@ Codex automation reviews source evidence: - `docs/spec/upstream-impact.md` records how upstream Codex changes are classified for public signals and Control Plane follow-up work. - `decodex radar render-signal` renders reviewed analysis drafts into site content. -- `scripts/github/validate_signal_entry.py` validates the published signal collection. +- `decodex radar validate` validates the published signal collection and checked Radar + artifact contracts. - `decodex radar refresh-upstream-queue`, `decodex radar refresh-release-delta`, `decodex radar bundle validate`, `decodex radar ledger ...`, `decodex radar render-signal`, `decodex radar backfill-release-range`, and `decodex radar diff --git a/apps/decodex/src/cli.rs b/apps/decodex/src/cli.rs index 66fbaae..4ae46f0 100644 --- a/apps/decodex/src/cli.rs +++ b/apps/decodex/src/cli.rs @@ -1028,7 +1028,7 @@ struct RadarBackfillReleaseRangeCommand { /// Compare pair limit passed through only by --refresh-release-delta-first. #[arg(long)] refresh_pair_limit: Option, - /// Python executable for non-ported helper boundaries. + /// Python executable for the Codex AI analysis helper boundary. #[arg(long, default_value = "python3")] python_bin: String, } diff --git a/apps/decodex/src/radar.rs b/apps/decodex/src/radar.rs index 68f4f10..0c53a0a 100644 --- a/apps/decodex/src/radar.rs +++ b/apps/decodex/src/radar.rs @@ -82,7 +82,6 @@ const UPSTREAM_SUBJECT_KINDS: &[&str] = &["commit", "pr"]; const GENERIC_COMMIT_TITLES: &[&str] = &["update", "fix", "fix.", "fix tests", "fix tests.", "merge fixes", "flaky syntax"]; const CONFIG_FEATURE_CATALOG_PATH: &str = "site/src/generated/codex-config-features.json"; -const BUILD_RELEASE_DELTA_SCRIPT: &str = "scripts/github/build_release_delta.py"; const RUN_CODEX_ANALYSIS_SCRIPT: &str = "scripts/github/run_codex_analysis.py"; const HIGH_VALUE_SURFACES: &[&str] = &[ "app_server_protocol", @@ -415,7 +414,7 @@ pub(crate) struct RadarBackfillReleaseRangeRequest { pub(crate) refresh_preview_limit: Option, /// Compare-pair limit passed through only when refreshing first. pub(crate) refresh_pair_limit: Option, - /// Python executable used for non-ported helper boundaries and the AI boundary. + /// Python executable used for the Codex AI analysis helper boundary. pub(crate) python_bin: String, } @@ -1252,7 +1251,7 @@ pub(crate) fn backfill_release_range( } validate(&RadarValidateRequest { paths: vec![resolve_against(&root, &request.signals_dir)] })?; - run_build_release_delta(&root, request, &request.release_delta, false)?; + run_refresh_release_delta(request, &request.release_delta, false)?; Ok(report) } @@ -2814,7 +2813,7 @@ fn prepare_release_delta_path( let release_delta = temp_root.join("release-delta.json"); - run_build_release_delta(root, request, &release_delta, true)?; + run_refresh_release_delta(request, &release_delta, true)?; Ok(PreparedReleaseDelta { path: release_delta, cleanup_dir: Some(temp_root) }) } @@ -2864,40 +2863,34 @@ fn run_codex_analysis( run_helper(command, RUN_CODEX_ANALYSIS_SCRIPT) } -fn run_build_release_delta( - root: &Path, +fn run_refresh_release_delta( request: &RadarBackfillReleaseRangeRequest, out: &Path, include_refresh_limits: bool, ) -> crate::prelude::Result<()> { - let mut command = helper_command(root, request, BUILD_RELEASE_DELTA_SCRIPT); - - command.args([ - "--repo", - request.repo.as_str(), - "--signals-dir", - &path_arg(root, &request.signals_dir), - "--out", - &path_arg(root, out), - ]); - - if let Some(token_env) = &request.token_env { - command.args(["--token-env", token_env]); - } + let mut refresh_request = RadarRefreshReleaseDeltaRequest { + repo: request.repo.clone(), + signals_dir: request.signals_dir.clone(), + out: out.to_path_buf(), + token_env: request.token_env.clone(), + ..RadarRefreshReleaseDeltaRequest::default() + }; if include_refresh_limits { - push_optional_limit(&mut command, "--stable-limit", request.refresh_stable_limit); - push_optional_limit(&mut command, "--preview-limit", request.refresh_preview_limit); - push_optional_limit(&mut command, "--pair-limit", request.refresh_pair_limit); + if let Some(limit) = request.refresh_stable_limit { + refresh_request.stable_limit = limit; + } + if let Some(limit) = request.refresh_preview_limit { + refresh_request.preview_limit = limit; + } + if let Some(limit) = request.refresh_pair_limit { + refresh_request.pair_limit = limit; + } } - run_helper(command, BUILD_RELEASE_DELTA_SCRIPT) -} + refresh_release_delta(&refresh_request)?; -fn push_optional_limit(command: &mut Command, flag: &str, value: Option) { - if let Some(value) = value { - command.args([flag, &value.to_string()]); - } + Ok(()) } fn helper_command( diff --git a/dev/skills/github-signal/SKILL.md b/dev/skills/github-signal/SKILL.md index 4d1a18f..6fc946a 100644 --- a/dev/skills/github-signal/SKILL.md +++ b/dev/skills/github-signal/SKILL.md @@ -170,7 +170,7 @@ decodex radar render-signal \ Validate the published output: ```bash -python3 scripts/github/validate_signal_entry.py site/src/content/signals +decodex radar validate site/src/content/signals npm run build --prefix site npm run check --prefix site ``` diff --git a/dev/skills/x-post-publisher/SKILL.md b/dev/skills/x-post-publisher/SKILL.md index 0997e6c..ea1cebe 100644 --- a/dev/skills/x-post-publisher/SKILL.md +++ b/dev/skills/x-post-publisher/SKILL.md @@ -131,5 +131,5 @@ Write `artifacts/social/x/posts//.json` with: Run: ```bash -python3 scripts/github/validate_social_post.py artifacts/social/x +decodex radar validate artifacts/social/x ``` diff --git a/docs/reference/workspace-layout.md b/docs/reference/workspace-layout.md index a9fe0a6..750397c 100644 --- a/docs/reference/workspace-layout.md +++ b/docs/reference/workspace-layout.md @@ -19,7 +19,7 @@ should not be treated as repository source. | `apps/decodex/` | Rust package that builds the `decodex` CLI and runtime. Runtime, orchestration, tracker integration, app-server integration, operator HTTP, and local control-plane behavior live under `apps/decodex/src/`. | | `apps/decodex-app/` | SwiftPM macOS app for local Decodex Codex account-pool management. It talks to the bundled `decodex-app-helper`, which links the Rust account service directly, and does not own runtime scheduling or operator dashboard state. | | `site/` | Astro static site for the public Decodex signal surface. It renders checked-in content and generated JSON from `site/src/content/`; it is not backed by a live Decodex daemon. | -| `scripts/github/` | Deterministic GitHub collection, normalization, render, validation, and sync scripts for public signal content. | +| `scripts/github/` | Automation-only Codex AI analysis helper and shared schema support for that helper. Deterministic Radar commands live in the Rust CLI. | | `scripts/config/` | Repository automation scripts for config-derived artifacts. | | `artifacts/github/` | Checked-in GitHub change bundles and editorial analysis drafts used by the public signal pipeline. | | `artifacts/archive/` | Checked-in manifests for cold Radar archive batches stored as GitHub Release assets. | @@ -76,7 +76,7 @@ Those runtime and operator surfaces stay in `apps/decodex/` and `docs/spec/`. ## GitHub signal tooling -`scripts/github/` owns deterministic content scripts. `decodex radar +`apps/decodex/src/radar.rs` owns deterministic Radar commands. `decodex radar refresh-upstream-queue` is the continuous Radar entrypoint: it scans recent upstream commits, resolves them back to PRs when possible, records local ledger state, and writes an `upstream_review_queue/v1` artifact for Codex automation. It does not run @@ -154,8 +154,8 @@ tracker routing, and policy. - Runtime authority stays in `apps/decodex/src/`, the registered project contract under `~/.codex/decodex/projects//`, and the governing specs under `docs/spec/`. -- Public site authority stays in `site/`, `scripts/github/`, `artifacts/github/`, and - the site/content specs. +- Public site authority stays in `site/`, `apps/decodex/src/radar.rs`, + `artifacts/github/`, and the site/content specs. - Reusable agent-facing Decodex usage instructions live under `plugins/decodex/`. - `docs/runbook/`, `docs/reference/`, and `docs/decisions/` must not override runtime or workflow authority. diff --git a/docs/runbook/local-github-signal-workflow.md b/docs/runbook/local-github-signal-workflow.md index 4d32557..db9cbcf 100644 --- a/docs/runbook/local-github-signal-workflow.md +++ b/docs/runbook/local-github-signal-workflow.md @@ -89,7 +89,7 @@ decodex radar render-signal \ Validate the published signal entries and the site collection: ```bash -python3 scripts/github/validate_signal_entry.py site/src/content/signals +decodex radar validate site/src/content/signals npm run build --prefix site npm run check --prefix site cargo make decodex-checks @@ -118,7 +118,7 @@ decodex radar backfill-release-range \ Use release-range backfill to fill gaps in the accumulated commit/PR analysis before a release or prerelease summary. It should supplement continuous commit tracking, not replace it. Execute mode is still a Codex automation or local operator path: Rust -selects the release-window gaps and sequences deterministic helper boundaries, while +selects the release-window gaps and sequences deterministic Radar commands, while `scripts/github/run_codex_analysis.py` remains the read-only Codex AI helper that creates validated `analysis_draft` artifacts. diff --git a/docs/runbook/social-publishing-workflow.md b/docs/runbook/social-publishing-workflow.md index 2515290..9e759ee 100644 --- a/docs/runbook/social-publishing-workflow.md +++ b/docs/runbook/social-publishing-workflow.md @@ -94,7 +94,7 @@ for technical claims. - Run: ```bash -python3 scripts/github/validate_social_post.py artifacts/social/x +decodex radar validate artifacts/social/x ``` ## Mode Guidance diff --git a/docs/spec/github-change-bundle.md b/docs/spec/github-change-bundle.md index 440d437..94b8c26 100644 --- a/docs/spec/github-change-bundle.md +++ b/docs/spec/github-change-bundle.md @@ -5,7 +5,7 @@ Purpose: Define the normalized GitHub input bundle that feeds Decodex signal ana Status: normative Read this when: -- You are writing GitHub collection or normalization scripts. +- You are changing `decodex radar bundle build` or bundle normalization behavior. - You are deciding what data Codex should read before drafting a signal. - You are validating whether a bundle contains enough context for GitHub-first analysis. @@ -25,8 +25,8 @@ decodex radar bundle build --repo openai/codex --pr 15222 --out artifacts/github decodex radar bundle validate artifacts/github/bundles/openai-codex-pr-15222.json ``` -The legacy Python scripts remain shared migration contracts until the final cleanup -issue removes them. +The Rust `decodex radar bundle ...` surface is the single active deterministic bundle +command path. Defines: - The canonical `github_change_bundle/v1` shape. diff --git a/docs/spec/radar-ledger.md b/docs/spec/radar-ledger.md index cc0d9f3..1330e54 100644 --- a/docs/spec/radar-ledger.md +++ b/docs/spec/radar-ledger.md @@ -6,10 +6,8 @@ traceable without putting every raw or low-value artifact into Git. Status: normative Read this when: -- You are changing `decodex radar refresh-upstream-queue` or - `scripts/github/sync_upstream_radar.py`. +- You are changing `decodex radar refresh-upstream-queue`. - You are changing `decodex radar ledger ...`. -- You are changing `scripts/github/radar_ledger.py`. - You are importing existing GitHub bundles, analysis drafts, or signal entries into historical Radar state. - You need to decide what belongs in local history instead of checked-in public @@ -42,10 +40,9 @@ but it is the preferred place for high-frequency trace and skip history. ## Schema The schema is created by `decodex radar refresh-upstream-queue` and -`decodex radar ledger bootstrap`. The legacy `scripts/github/radar_ledger.py` -entrypoint remains available during migration, but the Rust -`decodex radar ledger ...` surface owns the command path for new ledger bootstrap, -ingest, ingest-existing, artifact-link, and summary operations. +`decodex radar ledger bootstrap`. The Rust `decodex radar ledger ...` surface owns the +command path for ledger bootstrap, ingest, ingest-existing, artifact-link, and summary +operations. Required tables: diff --git a/scripts/README.md b/scripts/README.md index faea930..529f0c7 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,11 +1,16 @@ # Scripts Root -This directory contains executable repository automation. +This directory contains executable repository automation helpers. -- `scripts/github/` owns deterministic GitHub upstream review queue, release-delta, - render, sync, and validation scripts. +- `scripts/github/` owns the automation-only Codex AI analysis helper and shared + schema support used by that helper. - `scripts/config/` owns config-derived artifact synchronization scripts. Checked-in data produced or consumed by scripts belongs outside this directory. GitHub review queues, upstream reviews, bundles, impact records, and analysis drafts live under `artifacts/github/`. + +Durable deterministic Radar workflows are owned by the Rust CLI. Use +`decodex radar ...` for queue refresh, release-delta refresh, bundle build/validation, +ledger maintenance, signal rendering, release-window backfill, and Radar artifact +validation. diff --git a/scripts/github/README.md b/scripts/github/README.md index cf61159..ac69672 100644 --- a/scripts/github/README.md +++ b/scripts/github/README.md @@ -1,20 +1,20 @@ -# GitHub Scripts +# GitHub Script Helpers -This directory owns deterministic GitHub-first Decodex scripts. +This directory owns the remaining Python helper boundary for GitHub-backed Radar +analysis. Durable deterministic Radar workflows live in the Rust CLI. -Current scripts: +Current helper: -- `build_change_bundle.py` -- `build_release_delta.py` -- `backfill_release_range.py` -- `radar_ledger.py` -- `run_codex_analysis.py` -- `sync_upstream_radar.py` -- `validate_change_bundle.py` -- `validate_upstream_review.py` -- `validate_social_post.py` -- `render_signal_entry.py` -- `validate_signal_entry.py` +- `run_codex_analysis.py` invokes Codex in a read-only session and writes a validated + `analysis_draft` artifact. It is the explicit AI boundary and is not a GitHub + Actions entrypoint. + +Shared support: + +- `contracts.py` supports the AI helper validation path. +- `analysis_draft.schema.json` is the Codex output schema for that helper. +- The remaining schema JSON files are checked contract references. They do not define + the operator command path. Rust CLI entrypoints: @@ -24,29 +24,24 @@ Rust CLI entrypoints: `artifacts/github/review-queue/openai-codex-latest.json`. - `decodex radar refresh-release-delta` refreshes `site/src/content/release-deltas/openai-codex-latest.json`. -- `decodex radar bundle build` replaces deterministic `build_change_bundle.py` bundle - generation for PR-first and commit-only inputs. -- `decodex radar bundle validate` replaces deterministic `validate_change_bundle.py` - bundle validation. -- `decodex radar ledger ...` replaces `radar_ledger.py` bootstrap, ingest, - ingest-existing, artifact-link, and summary operations. +- `decodex radar bundle build` builds deterministic bundles for PR-first and + commit-only inputs. +- `decodex radar bundle validate` validates deterministic bundles. +- `decodex radar ledger ...` owns bootstrap, ingest, ingest-existing, artifact-link, + and summary operations. - `decodex radar render-signal` renders `signal_entry/v1` from a validated `github_change_bundle/v1` plus Codex-owned `analysis_draft`. - `decodex radar backfill-release-range` selects release-window signal gaps and can sequence the remaining helper boundaries for local or Codex automation backfills. -The Python scripts remain compatibility and non-ported helper boundaries until cleanup -issues remove them. In particular, `run_codex_analysis.py` is the explicit Codex AI -boundary for creating `analysis_draft`; it is not a GitHub Actions entrypoint. - Current checked contracts: -- `analysis_draft.schema.json` -- `upstream_review_queue/v1` is validated by `contracts.py` -- `upstream_review.schema.json` -- `release_delta/v1` is validated by `contracts.py` -- `upstream_impact.schema.json` -- `social_post.schema.json` +- `analysis_draft.schema.json` is the Codex AI helper output schema. +- `upstream_review_queue/v1` is validated by `decodex radar validate`. +- `upstream_review.schema.json` is validated by `decodex radar validate`. +- `release_delta/v1` is validated by `decodex radar validate`. +- `upstream_impact.schema.json` is validated by `decodex radar validate`. +- `social_post.schema.json` is validated by `decodex radar validate`. Contract ownership: @@ -69,7 +64,7 @@ decodex radar render-signal \ --analysis artifacts/github/analysis/openai-codex-pr-22414.analysis.json \ --out site/src/content/signals/openai-codex-pr-22414.json -python3 scripts/github/validate_signal_entry.py \ +decodex radar validate \ site/src/content/signals/openai-codex-pr-22414.json ``` @@ -117,10 +112,10 @@ decodex radar backfill-release-range \ --max-prs 3 ``` -These scripts stay deterministic on purpose. GitHub Actions may refresh upstream -queues, release deltas, and validation. Codex automation owns AI review of queued -subjects and may then promote source-backed conclusions into `upstream_impact/v1`, -`analysis_draft`, `decodex radar render-signal` output, or `social_post/v1`. +GitHub Actions may refresh upstream queues, release deltas, and validation through +`decodex radar ...`. Codex automation owns AI review of queued subjects and may then +promote source-backed conclusions into `upstream_impact/v1`, `analysis_draft`, +`decodex radar render-signal` output, or `social_post/v1`. Repo-local skills under `dev/skills/` are reasoning instructions for the Codex analysis step and for manual Radar/Publisher work. They do not introduce extra diff --git a/scripts/github/backfill_release_range.py b/scripts/github/backfill_release_range.py deleted file mode 100644 index 4d28252..0000000 --- a/scripts/github/backfill_release_range.py +++ /dev/null @@ -1,276 +0,0 @@ -#!/usr/bin/env python3 -"""Backfill unpublished GitHub signals for a selected release compare range.""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import subprocess -import sys -import tempfile -from pathlib import Path -from typing import Any - -SCRIPT_HOME = Path(__file__).resolve().parent -if str(SCRIPT_HOME) not in sys.path: - sys.path.insert(0, str(SCRIPT_HOME)) - -from build_change_bundle import build_pr_bundle, github_request, routed_token_env # noqa: E402 -from contracts import dump_json, load_json, validate_release_delta, validate_signal # noqa: E402 - -PR_URL_RE = re.compile(r"/pull/(\d+)$") - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--repo", default="openai/codex", help="GitHub repository in owner/name format.") - parser.add_argument("--release-delta", default="site/src/content/release-deltas/openai-codex-latest.json") - parser.add_argument( - "--stable-tag", - help="Stable tag name to backfill from. Defaults to the top-level stable release.", - ) - parser.add_argument("--preview-tag", help="Preview tag name to backfill to. Defaults to the top-level prerelease.") - parser.add_argument("--signals-dir", default="site/src/content/signals") - parser.add_argument("--bundles-dir", default="artifacts/github/bundles") - parser.add_argument("--analysis-dir", default="artifacts/github/analysis") - parser.add_argument("--token-env", help="Environment variable containing a GitHub token.") - parser.add_argument("--codex-bin", default="codex", help="Codex executable to invoke.") - parser.add_argument("--model", help="Optional Codex model override.") - parser.add_argument("--max-prs", type=int, help="Optional limit for debugging or partial runs.") - parser.add_argument("--dry-run", action="store_true", help="Print target PRs without generating new content.") - parser.add_argument( - "--refresh-release-delta-first", - action="store_true", - help="Refresh the release-delta artifact before selecting the prerelease compare range.", - ) - parser.add_argument( - "--refresh-stable-limit", - type=int, - help="Stable release limit used only by --refresh-release-delta-first.", - ) - parser.add_argument( - "--refresh-preview-limit", - type=int, - help="Prerelease limit used only by --refresh-release-delta-first.", - ) - parser.add_argument( - "--refresh-pair-limit", - type=int, - help="Compare pair limit used only by --refresh-release-delta-first.", - ) - return parser.parse_args() - - -def repo_root() -> Path: - return SCRIPT_HOME.parents[1] - - -def run_script(script: str, *extra: str) -> None: - cmd = [sys.executable, str(SCRIPT_HOME / script), *extra] - completed = subprocess.run(cmd, check=False, text=True, capture_output=True, cwd=repo_root()) - if completed.returncode != 0: - stderr = completed.stderr.strip() - stdout = completed.stdout.strip() - details = stderr or stdout or "unknown error" - raise SystemExit(f"{script} failed: {details}") - - -def published_pr_numbers(signals_dir: Path) -> set[int]: - published: set[int] = set() - for path in sorted(signals_dir.glob("*.json")): - payload = load_json(path) - validation = validate_signal(payload) - if not validation.ok: - raise SystemExit(f"Signal validation failed for {path}:\n- " + "\n- ".join(validation.errors)) - pr_url = payload.get("source_refs", {}).get("pr_url") - if not isinstance(pr_url, str): - continue - match = PR_URL_RE.search(pr_url) - if match: - published.add(int(match.group(1))) - return published - - -def load_selected_comparison( - path: Path, - stable_tag: str | None, - preview_tag: str | None, -) -> tuple[dict[str, Any], str, str]: - payload = load_json(path) - validation = validate_release_delta(payload) - if not validation.ok: - raise SystemExit("Release-delta validation failed:\n- " + "\n- ".join(validation.errors)) - - target_stable = stable_tag or payload["stable_release"]["tag_name"] - target_preview = preview_tag or payload["prerelease"]["tag_name"] - for item in payload.get("comparisons", []): - if item["stable_tag_name"] == target_stable and item["prerelease_tag_name"] == target_preview: - return item, target_stable, target_preview - raise SystemExit(f"No comparison found for {target_stable} -> {target_preview}") - - -def pr_lookup(repo: str, pr_number: int, token: str | None) -> dict[str, Any]: - payload, _headers = github_request(f"https://api.github.com/repos/{repo}/pulls/{pr_number}", token) - - if not isinstance(payload, dict): - raise SystemExit(f"Expected pull request object from GitHub for PR #{pr_number}") - - return payload - - -def signal_paths(pr_number: int, args: argparse.Namespace) -> tuple[Path, Path, Path]: - stem = f"openai-codex-pr-{pr_number}" - return ( - Path(args.bundles_dir) / f"{stem}.json", - Path(args.analysis_dir) / f"{stem}.analysis.json", - Path(args.signals_dir) / f"{stem}.json", - ) - - -def refresh_release_delta(args: argparse.Namespace) -> None: - command = [ - "build_release_delta.py", - "--repo", - args.repo, - "--signals-dir", - args.signals_dir, - "--out", - args.release_delta, - ] - if args.token_env: - command.extend(["--token-env", args.token_env]) - if args.refresh_stable_limit is not None: - command.extend(["--stable-limit", str(args.refresh_stable_limit)]) - if args.refresh_preview_limit is not None: - command.extend(["--preview-limit", str(args.refresh_preview_limit)]) - if args.refresh_pair_limit is not None: - command.extend(["--pair-limit", str(args.refresh_pair_limit)]) - run_script(*command) - - -def prepare_release_delta_path(args: argparse.Namespace, root: Path) -> tuple[Path, tempfile.TemporaryDirectory[str] | None]: - if not args.refresh_release_delta_first: - return (root / args.release_delta).resolve(), None - - tmpdir = tempfile.TemporaryDirectory(prefix="decodex-prerelease-delta-") - temp_release_delta = Path(tmpdir.name) / "release-delta.json" - refresh_args = argparse.Namespace(**{**vars(args), "release_delta": str(temp_release_delta)}) - refresh_release_delta(refresh_args) - return temp_release_delta.resolve(), tmpdir - - -def main() -> None: - args = parse_args() - root = repo_root() - release_delta_path, tmpdir = prepare_release_delta_path(args, root) - try: - comparison, stable_tag, preview_tag = load_selected_comparison( - release_delta_path, - args.stable_tag, - args.preview_tag, - ) - token_env = args.token_env or routed_token_env() or "GITHUB_TOKEN" - token = os.environ.get(token_env) - - signals_dir = (root / args.signals_dir).resolve() - published = published_pr_numbers(signals_dir) - target_prs = [ - int(number) - for number in comparison["compare"].get("pr_numbers", []) - if int(number) not in published - ] - if args.max_prs is not None: - target_prs = target_prs[: args.max_prs] - - pr_details: list[dict[str, Any]] = [] - for pr_number in target_prs: - details = pr_lookup(args.repo, pr_number, token) - pr_details.append( - { - "number": pr_number, - "title": details.get("title") or f"PR #{pr_number}", - "merged_at": details.get("merged_at") or "", - "url": details.get("html_url") or "", - } - ) - pr_details.sort(key=lambda item: item["merged_at"]) - - if args.dry_run: - print( - json.dumps( - { - "stable_tag": stable_tag, - "preview_tag": preview_tag, - "target_pr_count": len(pr_details), - "target_prs": pr_details, - }, - indent=2, - sort_keys=True, - ) - ) - return - - created = 0 - for pr in pr_details: - bundle_path, analysis_path, signal_path = signal_paths(pr["number"], args) - bundle = build_pr_bundle( - args.repo, - pr["number"], - token, - [f"Backfilled from release compare range {stable_tag}...{preview_tag}"], - ) - dump_json(root / bundle_path, bundle) - - run_script( - "run_codex_analysis.py", - "--bundle", - str(root / bundle_path), - "--out", - str(root / analysis_path), - "--repo-root", - str(root), - "--codex-bin", - args.codex_bin, - *(["--model", args.model] if args.model else []), - ) - run_script( - "render_signal_entry.py", - "--bundle", - str(root / bundle_path), - "--analysis", - str(root / analysis_path), - "--out", - str(root / signal_path), - ) - created += 1 - - run_script("validate_signal_entry.py", str(root / args.signals_dir)) - run_script( - "build_release_delta.py", - "--repo", - args.repo, - "--signals-dir", - args.signals_dir, - "--out", - args.release_delta, - *(["--token-env", args.token_env] if args.token_env else []), - ) - print( - json.dumps( - { - "stable_tag": stable_tag, - "preview_tag": preview_tag, - "created": created, - }, - sort_keys=True, - ) - ) - finally: - if tmpdir is not None: - tmpdir.cleanup() - - -if __name__ == "__main__": - main() diff --git a/scripts/github/build_change_bundle.py b/scripts/github/build_change_bundle.py deleted file mode 100644 index 80f5ec2..0000000 --- a/scripts/github/build_change_bundle.py +++ /dev/null @@ -1,308 +0,0 @@ -#!/usr/bin/env python3 -"""Build a PR-first or commit-only GitHub change bundle for Decodex.""" - -from __future__ import annotations - -import argparse -import json -import os -import socket -import ssl -import subprocess -import sys -import time -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path -from typing import Any - -SCRIPT_HOME = Path(__file__).resolve().parent -if str(SCRIPT_HOME) not in sys.path: - sys.path.insert(0, str(SCRIPT_HOME)) - -from contracts import ( # noqa: E402 - BUNDLE_SCHEMA, - collect_flags, - collect_issue_refs, - dump_json, - first_line, - truncate_patch, - validate_bundle, -) - -RETRYABLE_HTTP_STATUS_CODES = {429, 500, 502, 503, 504} -GITHUB_REQUEST_ATTEMPTS = 4 -GITHUB_REQUEST_BACKOFF_SECONDS = 1.0 -GITHUB_REQUEST_TIMEOUT_SECONDS = 30.0 - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--repo", required=True, help="GitHub repository in owner/name format.") - parser.add_argument("--pr", type=int, help="Pull request number to fetch.") - parser.add_argument("--commit", help="Commit SHA to fetch when PR is unavailable.") - parser.add_argument("--force-commit-only", action="store_true", help="Skip PR lookup for commit input.") - parser.add_argument("--token-env", help="Environment variable name holding a GitHub token.") - parser.add_argument("--out", required=True, help="Path to write the bundle JSON.") - parser.add_argument("--note", action="append", default=[], help="Additional note strings to store in the bundle.") - args = parser.parse_args() - if not args.pr and not args.commit: - parser.error("one of --pr or --commit is required") - return args - - -def routed_token_env() -> str | None: - try: - identity = ( - subprocess.run( - ["git", "config", "--get", "codex.github-identity"], - check=True, - capture_output=True, - text=True, - ) - .stdout.strip() - ) - except subprocess.CalledProcessError: - return None - return {"x": "GITHUB_PAT_X", "y": "GITHUB_PAT_Y"}.get(identity, "GITHUB_TOKEN") - - -def is_retryable_github_error(exc: urllib.error.HTTPError | urllib.error.URLError) -> bool: - if isinstance(exc, urllib.error.HTTPError): - return exc.code in RETRYABLE_HTTP_STATUS_CODES - - reason = exc.reason - return isinstance(reason, (ConnectionResetError, TimeoutError, socket.timeout, ssl.SSLError)) - - -def github_request(url: str, token: str | None) -> tuple[Any, dict[str, str]]: - request = urllib.request.Request( - url, - headers={ - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {token}" if token else "", - "User-Agent": "decodex-github-bundle-builder", - }, - ) - if not token: - request.headers.pop("Authorization") - - for attempt in range(1, GITHUB_REQUEST_ATTEMPTS + 1): - try: - with urllib.request.urlopen(request, timeout=GITHUB_REQUEST_TIMEOUT_SECONDS) as response: - return json.load(response), dict(response.headers) - except urllib.error.HTTPError as exc: - details = exc.read().decode("utf-8", errors="replace") - if not is_retryable_github_error(exc) or attempt == GITHUB_REQUEST_ATTEMPTS: - raise SystemExit(f"GitHub API request failed for {url}: {exc.code} {details}") from exc - except urllib.error.URLError as exc: - if not is_retryable_github_error(exc) or attempt == GITHUB_REQUEST_ATTEMPTS: - raise SystemExit(f"GitHub API request failed for {url}: {exc.reason}") from exc - time.sleep(GITHUB_REQUEST_BACKOFF_SECONDS * attempt) - - raise SystemExit(f"GitHub API request failed for {url}: exhausted retry loop") - - -def github_paginated(url: str, token: str | None) -> list[Any]: - items: list[Any] = [] - next_url: str | None = url - while next_url: - payload, headers = github_request(next_url, token) - if not isinstance(payload, list): - raise SystemExit(f"Expected list payload from {next_url}") - items.extend(payload) - next_url = parse_next_link(headers.get("Link")) - return items - - -def parse_next_link(header: str | None) -> str | None: - if not header: - return None - for part in header.split(","): - section = part.strip().split(";") - if len(section) < 2: - continue - url_part, *meta = section - if any(item.strip() == 'rel="next"' for item in meta): - return url_part.strip()[1:-1] - return None - - -def repo_default_branch(repo: str, token: str | None) -> str: - payload, _ = github_request(f"https://api.github.com/repos/{repo}", token) - default_branch = payload.get("default_branch") - if not isinstance(default_branch, str) or not default_branch: - raise SystemExit(f"Unable to resolve default branch for {repo}") - return default_branch - - -def build_pr_bundle(repo: str, pr_number: int, token: str | None, notes: list[str]) -> dict[str, Any]: - pr, _ = github_request(f"https://api.github.com/repos/{repo}/pulls/{pr_number}", token) - commits = github_paginated( - f"https://api.github.com/repos/{repo}/pulls/{pr_number}/commits?per_page=100", token - ) - files = github_paginated( - f"https://api.github.com/repos/{repo}/pulls/{pr_number}/files?per_page=100", token - ) - default_branch = repo_default_branch(repo, token) - - commit_items = [ - { - "sha": item["sha"], - "message": first_line(item["commit"]["message"]), - "url": item["html_url"], - "author": (item.get("author") or {}).get("login") - or (item["commit"].get("author") or {}).get("name"), - "committed_at": (item["commit"].get("author") or {}).get("date"), - } - for item in commits - ] - - file_items = [ - { - "path": item["filename"], - "status": item["status"], - "additions": item["additions"], - "deletions": item["deletions"], - "patch_excerpt": truncate_patch(item.get("patch")), - } - for item in files - ] - - docs_refs = [ - item["filename"] - for item in files - if item["filename"].startswith("docs/") or item["filename"].endswith("README.md") - ] - examples_refs = [ - item["filename"] - for item in files - if "example" in item["filename"].lower() or "examples/" in item["filename"] - ] - all_patch_text = "\n".join(item.get("patch", "") for item in files) - all_commit_text = "\n".join(item["commit"]["message"] for item in commits) - linked_issues = collect_issue_refs(pr.get("body", ""), all_commit_text) - extracted_flags = collect_flags(pr.get("body", ""), all_commit_text, all_patch_text) - - bundle = { - "schema": BUNDLE_SCHEMA, - "repo": repo, - "analysis_mode": "pr_first", - "default_branch": default_branch, - "primary_pr": { - "number": pr["number"], - "title": pr["title"], - "body": pr.get("body") or "", - "state": "merged" if pr.get("merged_at") else pr["state"], - "merged_at": pr.get("merged_at"), - "labels": [label["name"] for label in pr.get("labels", [])], - "url": pr["html_url"], - }, - "commits": commit_items, - "files": file_items, - "linked_issues": linked_issues, - "extracted_flags": extracted_flags, - "docs_refs": docs_refs, - "examples_refs": examples_refs, - "notes": [ - "Built from GitHub pull-request, commits, files, and repo endpoints.", - *notes, - ], - } - result = validate_bundle(bundle) - if not result.ok: - raise SystemExit("Bundle validation failed:\n- " + "\n- ".join(result.errors)) - return bundle - - -def build_commit_bundle(repo: str, commit_sha: str, token: str | None, notes: list[str]) -> dict[str, Any]: - commit, _ = github_request(f"https://api.github.com/repos/{repo}/commits/{commit_sha}", token) - default_branch = repo_default_branch(repo, token) - files = commit.get("files") or [] - bundle = { - "schema": BUNDLE_SCHEMA, - "repo": repo, - "analysis_mode": "commit_only", - "default_branch": default_branch, - "commits": [ - { - "sha": commit["sha"], - "message": first_line(commit["commit"]["message"]), - "url": commit["html_url"], - "author": (commit.get("author") or {}).get("login") - or (commit["commit"].get("author") or {}).get("name"), - "committed_at": (commit["commit"].get("author") or {}).get("date"), - } - ], - "files": [ - { - "path": item["filename"], - "status": item["status"], - "additions": item["additions"], - "deletions": item["deletions"], - "patch_excerpt": truncate_patch(item.get("patch")), - } - for item in files - ], - "linked_issues": collect_issue_refs(commit["commit"]["message"]), - "extracted_flags": collect_flags( - commit["commit"]["message"], "\n".join(item.get("patch", "") for item in files) - ), - "docs_refs": [ - item["filename"] - for item in files - if item["filename"].startswith("docs/") or item["filename"].endswith("README.md") - ], - "examples_refs": [ - item["filename"] - for item in files - if "example" in item["filename"].lower() or "examples/" in item["filename"] - ], - "notes": [ - "Built from GitHub commit endpoint without PR context.", - *notes, - ], - } - result = validate_bundle(bundle) - if not result.ok: - raise SystemExit("Bundle validation failed:\n- " + "\n- ".join(result.errors)) - return bundle - - -def maybe_promote_commit_to_pr(repo: str, commit_sha: str, token: str | None) -> int | None: - url = f"https://api.github.com/repos/{repo}/commits/{commit_sha}/pulls" - try: - pulls = github_paginated(url, token) - except SystemExit: - return None - if not pulls: - return None - first = pulls[0] - if not isinstance(first, dict) or "number" not in first: - return None - return int(first["number"]) - - -def main() -> None: - args = parse_args() - token_env = args.token_env or routed_token_env() or "GITHUB_TOKEN" - token = os.environ.get(token_env) - - if args.pr is not None: - bundle = build_pr_bundle(args.repo, args.pr, token, args.note) - else: - assert args.commit - promoted_pr = None if args.force_commit_only else maybe_promote_commit_to_pr(args.repo, args.commit, token) - bundle = ( - build_pr_bundle(args.repo, promoted_pr, token, args.note) - if promoted_pr is not None - else build_commit_bundle(args.repo, args.commit, token, args.note) - ) - - dump_json(args.out, bundle) - print(args.out) - - -if __name__ == "__main__": - main() diff --git a/scripts/github/build_release_delta.py b/scripts/github/build_release_delta.py deleted file mode 100644 index 7c3d196..0000000 --- a/scripts/github/build_release_delta.py +++ /dev/null @@ -1,442 +0,0 @@ -#!/usr/bin/env python3 -"""Build the latest stable-vs-prerelease release-delta artifact for Decodex.""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import socket -import ssl -import subprocess -import sys -import time -import urllib.error -import urllib.request -from pathlib import Path -from typing import Any - -SCRIPT_HOME = Path(__file__).resolve().parent -if str(SCRIPT_HOME) not in sys.path: - sys.path.insert(0, str(SCRIPT_HOME)) - -from contracts import ( # noqa: E402 - RELEASE_DELTA_SCHEMA, - dump_json, - load_json, - utc_now_iso, - validate_release_delta, - validate_signal, -) - -COMMIT_URL_RE = re.compile(r"/commit/([0-9a-f]{7,40})$") -PR_URL_RE = re.compile(r"/pull/(\d+)$") -PR_IN_MESSAGE_RE = re.compile(r"\(#(\d+)\)") -RETRYABLE_HTTP_STATUS_CODES = {429, 500, 502, 503, 504} -GITHUB_REQUEST_ATTEMPTS = 4 -GITHUB_REQUEST_BACKOFF_SECONDS = 1.0 -GITHUB_REQUEST_TIMEOUT_SECONDS = 30.0 -DEFAULT_STABLE_LIMIT = 0 -DEFAULT_PREVIEW_LIMIT = 0 -DEFAULT_PAIR_LIMIT = 24 - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--repo", required=True, help="GitHub repository in owner/name format.") - parser.add_argument("--signals-dir", required=True, help="Directory containing published signal-entry JSON files.") - parser.add_argument("--out", required=True, help="Path to write the release-delta JSON artifact.") - parser.add_argument("--tag-prefix", default="rust-v", help="Release tag prefix to scope the tracked channel.") - parser.add_argument("--token-env", help="Environment variable name holding a GitHub token.") - parser.add_argument( - "--stable-limit", - type=int, - default=DEFAULT_STABLE_LIMIT, - help="Maximum number of recent stable releases to include. Use 0 for all releases at or above the floor.", - ) - parser.add_argument( - "--preview-limit", - type=int, - default=DEFAULT_PREVIEW_LIMIT, - help="Maximum number of recent prereleases to include. Use 0 for all supported prereleases.", - ) - parser.add_argument( - "--pair-limit", - type=int, - default=DEFAULT_PAIR_LIMIT, - help="Maximum number of signal-bearing stable->preview compare entries. Use 0 for all valid pairs.", - ) - parser.add_argument( - "--min-stable-tag", - default="rust-v0.116.0", - help="Minimum stable tag to include in the comparator option set.", - ) - return parser.parse_args() - - -def routed_token_env() -> str | None: - try: - identity = ( - subprocess.run( - ["git", "config", "--get", "codex.github-identity"], - check=True, - capture_output=True, - text=True, - ) - .stdout.strip() - ) - except subprocess.CalledProcessError: - return None - return {"x": "GITHUB_PAT_X", "y": "GITHUB_PAT_Y"}.get(identity, "GITHUB_TOKEN") - - -def is_retryable_github_error(exc: urllib.error.HTTPError | urllib.error.URLError) -> bool: - if isinstance(exc, urllib.error.HTTPError): - return exc.code in RETRYABLE_HTTP_STATUS_CODES - - reason = exc.reason - return isinstance(reason, (ConnectionResetError, TimeoutError, socket.timeout, ssl.SSLError)) - - -def github_request(url: str, token: str | None) -> Any: - request = urllib.request.Request( - url, - headers={ - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {token}" if token else "", - "User-Agent": "decodex-release-delta-builder", - }, - ) - if not token: - request.headers.pop("Authorization") - - for attempt in range(1, GITHUB_REQUEST_ATTEMPTS + 1): - try: - with urllib.request.urlopen(request, timeout=GITHUB_REQUEST_TIMEOUT_SECONDS) as response: - return json.load(response) - except urllib.error.HTTPError as exc: - details = exc.read().decode("utf-8", errors="replace") - if not is_retryable_github_error(exc) or attempt == GITHUB_REQUEST_ATTEMPTS: - raise SystemExit(f"GitHub API request failed for {url}: {exc.code} {details}") from exc - except urllib.error.URLError as exc: - if not is_retryable_github_error(exc) or attempt == GITHUB_REQUEST_ATTEMPTS: - raise SystemExit(f"GitHub API request failed for {url}: {exc.reason}") from exc - time.sleep(GITHUB_REQUEST_BACKOFF_SECONDS * attempt) - - raise SystemExit(f"GitHub API request failed for {url}: exhausted retry loop") - - -def github_releases(repo: str, token: str | None) -> list[dict[str, Any]]: - releases: list[dict[str, Any]] = [] - - for page in range(1, 6): - payload = github_request(f"https://api.github.com/repos/{repo}/releases?per_page=100&page={page}", token) - if not isinstance(payload, list): - raise SystemExit("Expected releases list payload from GitHub API") - releases.extend(payload) - if len(payload) < 100: - break - - return releases - - -def select_release_pair(releases: list[dict[str, Any]], tag_prefix: str) -> tuple[dict[str, Any], dict[str, Any]]: - relevant = [ - release - for release in releases - if not release.get("draft") and isinstance(release.get("tag_name"), str) and release["tag_name"].startswith(tag_prefix) - ] - if not relevant: - raise SystemExit(f"No releases found for tag prefix {tag_prefix!r}") - - stable = next((release for release in relevant if not release.get("prerelease")), None) - prerelease = next((release for release in relevant if release.get("prerelease")), None) - if stable is None: - raise SystemExit(f"No stable release found for tag prefix {tag_prefix!r}") - if prerelease is None: - raise SystemExit(f"No prerelease found for tag prefix {tag_prefix!r}") - return stable, prerelease - - -def relevant_releases(releases: list[dict[str, Any]], tag_prefix: str) -> list[dict[str, Any]]: - return [ - release - for release in releases - if not release.get("draft") and isinstance(release.get("tag_name"), str) and release["tag_name"].startswith(tag_prefix) - ] - - -def stable_version_key(tag_name: str, tag_prefix: str) -> tuple[int, ...]: - raw = tag_name.removeprefix(tag_prefix) - parts = raw.split(".") - key: list[int] = [] - for part in parts: - digits = "".join(ch for ch in part if ch.isdigit()) - key.append(int(digits or "0")) - return tuple(key) - - -def select_release_options( - releases: list[dict[str, Any]], - tag_prefix: str, - stable_limit: int, - preview_limit: int, - min_stable_tag: str, -) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - relevant = relevant_releases(releases, tag_prefix) - min_stable_key = stable_version_key(min_stable_tag, tag_prefix) - stable = [ - release - for release in relevant - if not release.get("prerelease") - and stable_version_key(release["tag_name"], tag_prefix) >= min_stable_key - ] - preview = [release for release in relevant if release.get("prerelease")] - if stable_limit > 0: - stable = stable[:stable_limit] - if preview_limit > 0: - preview = preview[:preview_limit] - if not stable: - raise SystemExit( - f"No stable releases found for tag prefix {tag_prefix!r} at or above {min_stable_tag!r}" - ) - if not preview: - raise SystemExit(f"No prereleases found for tag prefix {tag_prefix!r}") - return stable, preview - - -def compact_release(release: dict[str, Any]) -> dict[str, Any]: - return { - "tag_name": release["tag_name"], - "name": release.get("name") or release["tag_name"], - "prerelease": bool(release.get("prerelease")), - "published_at": release["published_at"], - "url": release["html_url"], - } - - -def release_sort_key(release: dict[str, Any]) -> str: - published_at = release.get("published_at") - return published_at if isinstance(published_at, str) else "" - - -def compare_candidates( - stable_releases: list[dict[str, Any]], - preview_releases: list[dict[str, Any]], -) -> list[tuple[dict[str, Any], dict[str, Any]]]: - candidates: list[tuple[dict[str, Any], dict[str, Any]]] = [] - for stable in stable_releases: - stable_key = release_sort_key(stable) - for preview in preview_releases: - preview_key = release_sort_key(preview) - if preview_key <= stable_key: - continue - candidates.append((stable, preview)) - candidates.sort( - key=lambda pair: ( - release_sort_key(pair[1]), - release_sort_key(pair[0]), - ), - reverse=True, - ) - return candidates - - -def extract_signal_commit_shas(signal: dict[str, Any]) -> set[str]: - shas: set[str] = set() - for url in signal.get("source_refs", {}).get("commit_urls", []): - match = COMMIT_URL_RE.search(url) - if match: - shas.add(match.group(1)) - return shas - - -def extract_signal_pr_number(signal: dict[str, Any]) -> int | None: - pr_url = signal.get("source_refs", {}).get("pr_url") - if not isinstance(pr_url, str): - return None - match = PR_URL_RE.search(pr_url) - if not match: - return None - return int(match.group(1)) - - -def load_signals(signals_dir: str | Path, repo: str) -> list[dict[str, Any]]: - entries: list[dict[str, Any]] = [] - for path in sorted(Path(signals_dir).glob("*.json")): - if path.name == "README.md": - continue - payload = load_json(path) - result = validate_signal(payload) - if not result.ok: - raise SystemExit(f"Signal validation failed for {path}:\n- " + "\n- ".join(result.errors)) - if payload.get("source_refs", {}).get("repo") == repo: - entries.append(payload) - return entries - - -def previous_signal_pair_keys(path: Path) -> list[tuple[str, str]]: - if not path.exists(): - return [] - - try: - previous = load_json(path) - except (OSError, json.JSONDecodeError): - return [] - - keys: list[tuple[str, str]] = [] - seen: set[tuple[str, str]] = set() - for comparison in previous.get("comparisons", []): - if not comparison.get("tracked_signal_slugs"): - continue - key = (comparison.get("stable_tag_name"), comparison.get("prerelease_tag_name")) - if not all(isinstance(value, str) and value for value in key): - continue - if key in seen: - continue - seen.add(key) - keys.append(key) - return keys - - -def unique_release_pairs( - pairs: list[tuple[dict[str, Any], dict[str, Any]]], -) -> list[tuple[dict[str, Any], dict[str, Any]]]: - unique: list[tuple[dict[str, Any], dict[str, Any]]] = [] - seen: set[tuple[str, str]] = set() - - for stable, preview in pairs: - key = (stable["tag_name"], preview["tag_name"]) - if key in seen: - continue - seen.add(key) - unique.append((stable, preview)) - - return unique - - -def main() -> None: - args = parse_args() - token_env = args.token_env or routed_token_env() or "GITHUB_TOKEN" - token = os.environ.get(token_env) - - releases = github_releases(args.repo, token) - stable_release, prerelease = select_release_pair(releases, args.tag_prefix) - stable_releases, preview_releases = select_release_options( - releases, - args.tag_prefix, - args.stable_limit, - args.preview_limit, - args.min_stable_tag, - ) - release_pairs = compare_candidates(stable_releases, preview_releases) - default_pair = (stable_release, prerelease) - releases_by_tag = {release["tag_name"]: release for release in [*stable_releases, *preview_releases]} - previous_pairs = [ - (releases_by_tag[stable_tag], releases_by_tag[preview_tag]) - for stable_tag, preview_tag in previous_signal_pair_keys(Path(args.out)) - if stable_tag in releases_by_tag and preview_tag in releases_by_tag - ] - if previous_pairs: - release_pairs = unique_release_pairs([default_pair, *previous_pairs]) - else: - release_pairs = unique_release_pairs([default_pair, *release_pairs]) - if args.pair_limit > 0: - release_pairs = release_pairs[: args.pair_limit] - - signal_entries = load_signals(args.signals_dir, args.repo) - comparison_entries: list[dict[str, Any]] = [] - default_tracked_signal_slugs: list[str] = [] - default_compare_payload: dict[str, Any] | None = None - - for stable_candidate, preview_candidate in release_pairs: - is_default_pair = ( - stable_candidate["tag_name"] == stable_release["tag_name"] - and preview_candidate["tag_name"] == prerelease["tag_name"] - ) - compare = github_request( - f"https://api.github.com/repos/{args.repo}/compare/{stable_candidate['tag_name']}...{preview_candidate['tag_name']}", - token, - ) - commits = compare.get("commits") - if not isinstance(commits, list): - raise SystemExit("Expected compare.commits from GitHub API") - compare_commit_shas = [commit["sha"] for commit in commits if isinstance(commit.get("sha"), str)] - compare_commit_set = set(compare_commit_shas) - compare_pr_numbers = sorted( - { - int(match.group(1)) - for commit in commits - for match in PR_IN_MESSAGE_RE.finditer((commit.get("commit") or {}).get("message", "")) - } - ) - compare_pr_number_set = set(compare_pr_numbers) - - tracked_signal_slugs: list[str] = [] - for signal in sorted(signal_entries, key=lambda item: item["published_at"], reverse=True): - signal_shas = extract_signal_commit_shas(signal) - signal_pr_number = extract_signal_pr_number(signal) - if signal_shas.intersection(compare_commit_set) or ( - signal_pr_number is not None and signal_pr_number in compare_pr_number_set - ): - tracked_signal_slugs.append(signal["slug"]) - - compare_payload = { - "status": compare["status"], - "ahead_by": compare["ahead_by"], - "total_commits": compare["total_commits"], - "url": compare["html_url"], - "commit_shas": compare_commit_shas, - "pr_numbers": compare_pr_numbers, - } - comparison_entries.append( - { - "stable_tag_name": stable_candidate["tag_name"], - "prerelease_tag_name": preview_candidate["tag_name"], - "compare": compare_payload, - "tracked_signal_slugs": tracked_signal_slugs, - } - ) - - if is_default_pair: - default_compare_payload = compare_payload - default_tracked_signal_slugs = tracked_signal_slugs - - if args.pair_limit > 0 and len(comparison_entries) >= args.pair_limit and default_compare_payload is not None: - break - - if default_compare_payload is None: - raise SystemExit("Default stable/prerelease pair was not included in comparison entries") - - allowed_stable_tags = {entry["stable_tag_name"] for entry in comparison_entries} - allowed_preview_tags = {entry["prerelease_tag_name"] for entry in comparison_entries} - stable_releases = [release for release in stable_releases if release["tag_name"] in allowed_stable_tags] - preview_releases = [release for release in preview_releases if release["tag_name"] in allowed_preview_tags] - - payload = { - "schema": RELEASE_DELTA_SCHEMA, - "repo": args.repo, - "tag_prefix": args.tag_prefix, - "generated_at": utc_now_iso(), - "stable_release": compact_release(stable_release), - "prerelease": compact_release(prerelease), - "compare": default_compare_payload, - "release_options": { - "stable": [compact_release(release) for release in stable_releases], - "preview": [compact_release(release) for release in preview_releases], - }, - "comparisons": comparison_entries, - "tracked_signal_slugs": default_tracked_signal_slugs, - } - - validation = validate_release_delta(payload) - if not validation.ok: - raise SystemExit("Release-delta validation failed:\n- " + "\n- ".join(validation.errors)) - - dump_json(args.out, payload) - print(args.out) - - -if __name__ == "__main__": - main() diff --git a/scripts/github/radar_ledger.py b/scripts/github/radar_ledger.py deleted file mode 100644 index 376af63..0000000 --- a/scripts/github/radar_ledger.py +++ /dev/null @@ -1,613 +0,0 @@ -#!/usr/bin/env python3 -"""Maintain the local Decodex Radar SQLite ledger.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import re -import sqlite3 -import sys -from pathlib import Path -from typing import Any - -SCRIPT_HOME = Path(__file__).resolve().parent -if str(SCRIPT_HOME) not in sys.path: - sys.path.insert(0, str(SCRIPT_HOME)) - -from contracts import load_json, utc_now_iso, validate_bundle, validate_signal # noqa: E402 - -SCHEMA_VERSION = 2 -DEFAULT_LEDGER_PATH = ".decodex/radar.sqlite3" -COMMIT_URL_RE = re.compile(r"/commit/([0-9a-f]{7,40})$") -PR_URL_RE = re.compile(r"/pull/(\d+)$") -SUBJECT_KINDS = {"commit", "pr"} -REVIEW_STATUSES = { - "seen", - "skipped", - "watch", - "signal", - "control_plane", - "social", - "deprecated", - "archived", -} -CONFIDENCE_VALUES = {"confirmed", "likely", "weak"} -ARTIFACT_KINDS = { - "bundle", - "analysis", - "signal", - "upstream_impact", - "social_post", - "release_delta", - "archive_manifest", - "ledger_export", -} - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--db", default=DEFAULT_LEDGER_PATH, help="SQLite ledger path.") - subcommands = parser.add_subparsers(dest="command", required=True) - - subcommands.add_parser("init", help="Initialize the ledger schema.") - - ingest = subcommands.add_parser("ingest", help="Ingest one bundle and optional derived artifacts.") - ingest.add_argument("--bundle", required=True, help="Path to a github_change_bundle/v1 JSON file.") - ingest.add_argument("--analysis", help="Optional analysis draft path.") - ingest.add_argument("--signal", help="Optional rendered signal_entry/v1 path.") - - ingest_existing = subcommands.add_parser( - "ingest-existing", - help="Ingest existing checked-in bundles, analyses, and signals.", - ) - ingest_existing.add_argument("--bundles-dir", default="artifacts/github/bundles") - ingest_existing.add_argument("--analysis-dir", default="artifacts/github/analysis") - ingest_existing.add_argument("--signals-dir", default="site/src/content/signals") - - summary = subcommands.add_parser("summary", help="Print ledger counts.") - summary.add_argument("--json", action="store_true", help="Emit machine-readable JSON.") - - return parser.parse_args() - - -def connect(path: str | Path) -> sqlite3.Connection: - db_path = Path(path) - db_path.parent.mkdir(parents=True, exist_ok=True) - connection = sqlite3.connect(db_path) - connection.row_factory = sqlite3.Row - initialize(connection) - return connection - - -def initialize(connection: sqlite3.Connection) -> None: - connection.executescript( - """ - PRAGMA foreign_keys = ON; - - CREATE TABLE IF NOT EXISTS metadata ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS upstream_commit ( - repo TEXT NOT NULL, - sha TEXT NOT NULL, - title TEXT NOT NULL, - url TEXT NOT NULL, - committed_at TEXT, - pr_number INTEGER, - first_seen_at TEXT NOT NULL, - last_seen_at TEXT NOT NULL, - PRIMARY KEY (repo, sha) - ); - - CREATE TABLE IF NOT EXISTS radar_review ( - repo TEXT NOT NULL, - subject_kind TEXT NOT NULL CHECK (subject_kind IN ('commit', 'pr')), - subject_id TEXT NOT NULL, - status TEXT NOT NULL CHECK ( - status IN ( - 'seen', - 'skipped', - 'watch', - 'signal', - 'control_plane', - 'social', - 'deprecated', - 'archived' - ) - ), - reason TEXT NOT NULL DEFAULT '', - confidence TEXT CHECK (confidence IN ('confirmed', 'likely', 'weak')), - reviewed_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - PRIMARY KEY (repo, subject_kind, subject_id) - ); - - CREATE TABLE IF NOT EXISTS artifact_link ( - repo TEXT NOT NULL, - subject_kind TEXT NOT NULL CHECK (subject_kind IN ('commit', 'pr')), - subject_id TEXT NOT NULL, - artifact_kind TEXT NOT NULL CHECK ( - artifact_kind IN ( - 'bundle', - 'analysis', - 'signal', - 'upstream_impact', - 'social_post', - 'release_delta', - 'archive_manifest', - 'ledger_export' - ) - ), - path TEXT NOT NULL, - sha256 TEXT NOT NULL, - size_bytes INTEGER NOT NULL, - created_at TEXT NOT NULL, - PRIMARY KEY (repo, subject_kind, subject_id, artifact_kind, path) - ); - - CREATE TABLE IF NOT EXISTS source_cache ( - url TEXT PRIMARY KEY, - etag TEXT, - body_sha256 TEXT NOT NULL, - fetched_at TEXT NOT NULL, - cache_path TEXT - ); - - CREATE INDEX IF NOT EXISTS idx_upstream_commit_pr - ON upstream_commit (repo, pr_number); - - CREATE INDEX IF NOT EXISTS idx_radar_review_status - ON radar_review (status, reviewed_at); - """ - ) - migrate_artifact_link_social_kind(connection) - connection.execute( - """ - INSERT INTO metadata (key, value) - VALUES ('schema_version', ?) - ON CONFLICT(key) DO UPDATE SET value = excluded.value - """, - (str(SCHEMA_VERSION),), - ) - connection.commit() - - -def migrate_artifact_link_social_kind(connection: sqlite3.Connection) -> None: - row = connection.execute( - """ - SELECT sql - FROM sqlite_master - WHERE type = 'table' AND name = 'artifact_link' - """ - ).fetchone() - if not row or "social_draft" not in row["sql"]: - return - - connection.executescript( - """ - ALTER TABLE artifact_link RENAME TO artifact_link_old; - - CREATE TABLE artifact_link ( - repo TEXT NOT NULL, - subject_kind TEXT NOT NULL CHECK (subject_kind IN ('commit', 'pr')), - subject_id TEXT NOT NULL, - artifact_kind TEXT NOT NULL CHECK ( - artifact_kind IN ( - 'bundle', - 'analysis', - 'signal', - 'upstream_impact', - 'social_post', - 'release_delta', - 'archive_manifest', - 'ledger_export' - ) - ), - path TEXT NOT NULL, - sha256 TEXT NOT NULL, - size_bytes INTEGER NOT NULL, - created_at TEXT NOT NULL, - PRIMARY KEY (repo, subject_kind, subject_id, artifact_kind, path) - ); - - INSERT OR REPLACE INTO artifact_link ( - repo, - subject_kind, - subject_id, - artifact_kind, - path, - sha256, - size_bytes, - created_at - ) - SELECT - repo, - subject_kind, - subject_id, - CASE artifact_kind - WHEN 'social_draft' THEN 'social_post' - ELSE artifact_kind - END, - path, - sha256, - size_bytes, - created_at - FROM artifact_link_old; - - DROP TABLE artifact_link_old; - """ - ) - - -def path_for_storage(path: str | Path) -> str: - resolved = Path(path).resolve() - cwd = Path.cwd().resolve() - try: - return str(resolved.relative_to(cwd)) - except ValueError: - return str(resolved) - - -def file_digest(path: str | Path) -> tuple[str, int]: - payload = Path(path).read_bytes() - return hashlib.sha256(payload).hexdigest(), len(payload) - - -def require_member(value: str, allowed: set[str], label: str) -> None: - if value not in allowed: - raise ValueError(f"{label} must be one of {sorted(allowed)}") - - -def record_commit( - connection: sqlite3.Connection, - *, - repo: str, - sha: str, - title: str, - url: str, - committed_at: str | None = None, - pr_number: int | None = None, - seen_at: str | None = None, -) -> None: - timestamp = seen_at or utc_now_iso() - connection.execute( - """ - INSERT INTO upstream_commit ( - repo, - sha, - title, - url, - committed_at, - pr_number, - first_seen_at, - last_seen_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(repo, sha) DO UPDATE SET - title = excluded.title, - url = excluded.url, - committed_at = COALESCE(excluded.committed_at, upstream_commit.committed_at), - pr_number = COALESCE(excluded.pr_number, upstream_commit.pr_number), - last_seen_at = excluded.last_seen_at - """, - (repo, sha, title, url, committed_at, pr_number, timestamp, timestamp), - ) - - -def record_review( - connection: sqlite3.Connection, - *, - repo: str, - subject_kind: str, - subject_id: str, - status: str, - reason: str, - confidence: str | None = None, - reviewed_at: str | None = None, -) -> None: - require_member(subject_kind, SUBJECT_KINDS, "subject_kind") - require_member(status, REVIEW_STATUSES, "status") - if confidence is not None: - require_member(confidence, CONFIDENCE_VALUES, "confidence") - timestamp = reviewed_at or utc_now_iso() - connection.execute( - """ - INSERT INTO radar_review ( - repo, - subject_kind, - subject_id, - status, - reason, - confidence, - reviewed_at, - updated_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(repo, subject_kind, subject_id) DO UPDATE SET - status = excluded.status, - reason = excluded.reason, - confidence = excluded.confidence, - reviewed_at = excluded.reviewed_at, - updated_at = excluded.updated_at - """, - (repo, subject_kind, subject_id, status, reason, confidence, timestamp, timestamp), - ) - - -def record_artifact( - connection: sqlite3.Connection, - *, - repo: str, - subject_kind: str, - subject_id: str, - artifact_kind: str, - path: str | Path, - created_at: str | None = None, -) -> None: - require_member(subject_kind, SUBJECT_KINDS, "subject_kind") - require_member(artifact_kind, ARTIFACT_KINDS, "artifact_kind") - digest, size_bytes = file_digest(path) - connection.execute( - """ - INSERT INTO artifact_link ( - repo, - subject_kind, - subject_id, - artifact_kind, - path, - sha256, - size_bytes, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(repo, subject_kind, subject_id, artifact_kind, path) DO UPDATE SET - sha256 = excluded.sha256, - size_bytes = excluded.size_bytes, - created_at = excluded.created_at - """, - ( - repo, - subject_kind, - subject_id, - artifact_kind, - path_for_storage(path), - digest, - size_bytes, - created_at or utc_now_iso(), - ), - ) - - -def subject_for_bundle(bundle: dict[str, Any]) -> tuple[str, str, str]: - repo = bundle["repo"] - primary_pr = bundle.get("primary_pr") - if isinstance(primary_pr, dict) and isinstance(primary_pr.get("number"), int): - return repo, "pr", str(primary_pr["number"]) - first_commit = bundle["commits"][0] - return repo, "commit", first_commit["sha"] - - -def record_bundle( - connection: sqlite3.Connection, - bundle: dict[str, Any], - bundle_path: str | Path, - *, - status: str = "watch", - reason: str = "Imported from GitHub change bundle.", -) -> tuple[str, str, str]: - validation = validate_bundle(bundle) - if not validation.ok: - raise ValueError("Bundle validation failed:\n- " + "\n- ".join(validation.errors)) - - repo, subject_kind, subject_id = subject_for_bundle(bundle) - primary_pr = bundle.get("primary_pr") - pr_number = primary_pr.get("number") if isinstance(primary_pr, dict) else None - for commit in bundle["commits"]: - record_commit( - connection, - repo=repo, - sha=commit["sha"], - title=commit["message"], - url=commit["url"], - committed_at=commit.get("committed_at"), - pr_number=pr_number if isinstance(pr_number, int) else None, - ) - record_review( - connection, - repo=repo, - subject_kind=subject_kind, - subject_id=subject_id, - status=status, - reason=reason, - confidence="confirmed" if status == "signal" else None, - ) - record_artifact( - connection, - repo=repo, - subject_kind=subject_kind, - subject_id=subject_id, - artifact_kind="bundle", - path=bundle_path, - ) - return repo, subject_kind, subject_id - - -def subject_refs_for_signal(signal: dict[str, Any]) -> list[tuple[str, str, str]]: - refs = signal.get("source_refs", {}) - repo = refs.get("repo") - if not isinstance(repo, str): - return [] - subjects: list[tuple[str, str, str]] = [] - pr_url = refs.get("pr_url") - if isinstance(pr_url, str): - match = PR_URL_RE.search(pr_url) - if match: - subjects.append((repo, "pr", match.group(1))) - for url in refs.get("commit_urls", []): - if not isinstance(url, str): - continue - match = COMMIT_URL_RE.search(url) - if match: - subjects.append((repo, "commit", match.group(1))) - return subjects - - -def record_signal_artifact(connection: sqlite3.Connection, signal_path: str | Path) -> list[tuple[str, str, str]]: - signal = load_json(signal_path) - validation = validate_signal(signal) - if not validation.ok: - raise ValueError(f"Signal validation failed for {signal_path}:\n- " + "\n- ".join(validation.errors)) - - subjects = subject_refs_for_signal(signal) - for repo, subject_kind, subject_id in subjects: - record_review( - connection, - repo=repo, - subject_kind=subject_kind, - subject_id=subject_id, - status="signal", - reason=f"Published signal_entry/v1: {signal['slug']}", - confidence=signal["confidence"], - ) - record_artifact( - connection, - repo=repo, - subject_kind=subject_kind, - subject_id=subject_id, - artifact_kind="signal", - path=signal_path, - ) - return subjects - - -def ingest_artifact_set( - connection: sqlite3.Connection, - *, - bundle_path: str | Path, - analysis_path: str | Path | None = None, - signal_path: str | Path | None = None, -) -> None: - bundle = load_json(bundle_path) - signal_exists = signal_path is not None and Path(signal_path).exists() - repo, subject_kind, subject_id = record_bundle( - connection, - bundle, - bundle_path, - status="signal" if signal_exists else "watch", - reason="Imported from generated Radar artifacts.", - ) - if analysis_path is not None and Path(analysis_path).exists(): - record_artifact( - connection, - repo=repo, - subject_kind=subject_kind, - subject_id=subject_id, - artifact_kind="analysis", - path=analysis_path, - ) - if signal_exists: - signal_subjects = record_signal_artifact(connection, signal_path) - if (repo, subject_kind, subject_id) not in signal_subjects: - record_artifact( - connection, - repo=repo, - subject_kind=subject_kind, - subject_id=subject_id, - artifact_kind="signal", - path=signal_path, - ) - - -def ingest_existing( - connection: sqlite3.Connection, - *, - bundles_dir: str | Path, - analysis_dir: str | Path, - signals_dir: str | Path, -) -> dict[str, int]: - bundles_path = Path(bundles_dir) - analysis_path = Path(analysis_dir) - signals_path = Path(signals_dir) - ingested = 0 - for bundle_path in sorted(bundles_path.glob("*.json")): - stem = bundle_path.stem - candidate_analysis = analysis_path / f"{stem}.analysis.json" - candidate_signal = signals_path / f"{stem}.json" - ingest_artifact_set( - connection, - bundle_path=bundle_path, - analysis_path=candidate_analysis if candidate_analysis.exists() else None, - signal_path=candidate_signal if candidate_signal.exists() else None, - ) - ingested += 1 - - linked_signal_paths = {signals_path / f"{path.stem}.json" for path in bundles_path.glob("*.json")} - for signal_path in sorted(signals_path.glob("*.json")): - if signal_path in linked_signal_paths: - continue - record_signal_artifact(connection, signal_path) - - connection.commit() - return {**summary_counts(connection), "bundles_ingested": ingested} - - -def summary_counts(connection: sqlite3.Connection) -> dict[str, int]: - tables = { - "upstream_commits": "upstream_commit", - "radar_reviews": "radar_review", - "artifact_links": "artifact_link", - "source_cache_entries": "source_cache", - } - result: dict[str, int] = {} - for key, table in tables.items(): - row = connection.execute(f"SELECT COUNT(*) AS count FROM {table}").fetchone() - result[key] = int(row["count"]) - return result - - -def print_summary(connection: sqlite3.Connection, *, as_json: bool) -> None: - payload = summary_counts(connection) - if as_json: - print(json.dumps(payload, indent=2, sort_keys=True)) - return - for key, value in payload.items(): - print(f"{key}\t{value}") - - -def main() -> None: - args = parse_args() - connection = connect(args.db) - try: - if args.command == "init": - print(args.db) - elif args.command == "ingest": - ingest_artifact_set( - connection, - bundle_path=args.bundle, - analysis_path=args.analysis, - signal_path=args.signal, - ) - connection.commit() - print_summary(connection, as_json=True) - elif args.command == "ingest-existing": - payload = ingest_existing( - connection, - bundles_dir=args.bundles_dir, - analysis_dir=args.analysis_dir, - signals_dir=args.signals_dir, - ) - print(json.dumps(payload, indent=2, sort_keys=True)) - elif args.command == "summary": - print_summary(connection, as_json=args.json) - else: - raise SystemExit(f"unknown command: {args.command}") - finally: - connection.close() - - -if __name__ == "__main__": - main() diff --git a/scripts/github/render_signal_entry.py b/scripts/github/render_signal_entry.py deleted file mode 100644 index 28fe5ee..0000000 --- a/scripts/github/render_signal_entry.py +++ /dev/null @@ -1,234 +0,0 @@ -#!/usr/bin/env python3 -"""Render a final Decodex signal entry from a GitHub bundle plus local analysis.""" - -from __future__ import annotations - -import argparse -import sys -import json -import re -from pathlib import Path -from typing import Any - -SCRIPT_HOME = Path(__file__).resolve().parent -REPO_HOME = SCRIPT_HOME.parent.parent -CONFIG_FEATURE_CATALOG_PATH = REPO_HOME / "site/src/generated/codex-config-features.json" -if str(SCRIPT_HOME) not in sys.path: - sys.path.insert(0, str(SCRIPT_HOME)) - -from contracts import ( # noqa: E402 - GENERIC_COMMIT_TITLES, - SIGNAL_SCHEMA, - dump_json, - first_line, - load_json, - slugify, - utc_now_iso, - validate_analysis_draft, - validate_bundle, - validate_signal, -) - -ENABLE_FEATURE_RE = re.compile(r"^--enable\s+([a-z0-9_]+)$", re.IGNORECASE) -FEATURE_PATH_RE = re.compile(r"^(?:features\.)?([a-z0-9_]+)(?:\s*=\s*true)?$", re.IGNORECASE) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--bundle", required=True, help="Path to github_change_bundle/v1 JSON.") - parser.add_argument("--analysis", required=True, help="Path to local editorial analysis JSON.") - parser.add_argument("--out", required=True, help="Path to write the rendered signal entry.") - parser.add_argument("--published-at", help="Override publication timestamp.") - return parser.parse_args() - - -def short_sha(value: str) -> str: - return value[:7] - - -def normalized_config_flags(raw_flags: Any) -> list[str]: - if not isinstance(raw_flags, list): - return [] - - known_features = load_known_feature_names() - normalized: list[str] = [] - seen: set[str] = set() - for flag in raw_flags: - if not isinstance(flag, str): - continue - value = flag.strip() - if not value or value in seen: - continue - - feature_name = normalize_feature_flag(value, known_features) - if feature_name: - value = f"features.{feature_name} = true" - - actionable = ( - value.startswith("--") - or "=" in value - or value.endswith(".json") - or value.endswith(".toml") - ) - if not actionable: - continue - - seen.add(value) - normalized.append(value) - - return normalized - - -def load_known_feature_names() -> set[str]: - if not CONFIG_FEATURE_CATALOG_PATH.exists(): - return set() - try: - payload = json.loads(CONFIG_FEATURE_CATALOG_PATH.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return set() - - items = payload.get("features") - if not isinstance(items, list): - return set() - - names: set[str] = set() - for item in items: - if not isinstance(item, dict): - continue - name = item.get("name") - if isinstance(name, str) and name: - names.add(name) - return names - - -def normalize_feature_flag(value: str, known_features: set[str]) -> str | None: - enable_match = ENABLE_FEATURE_RE.fullmatch(value) - if enable_match: - candidate = enable_match.group(1).lower() - return candidate if candidate in known_features else None - - feature_match = FEATURE_PATH_RE.fullmatch(value) - if feature_match: - candidate = feature_match.group(1).lower() - return candidate if candidate in known_features else None - - return None - - -def rendered_source_items(bundle: dict[str, Any]) -> list[dict[str, str]]: - items: list[dict[str, str]] = [] - primary_pr = bundle.get("primary_pr") - if isinstance(primary_pr, dict) and primary_pr.get("url") and primary_pr.get("title"): - meta = primary_pr.get("number") - item: dict[str, str] = { - "kind": "pull_request", - "title": first_line(primary_pr["title"]), - "url": primary_pr["url"], - } - if isinstance(meta, int): - item["meta"] = f"#{meta}" - items.append(item) - - fallback_items: list[dict[str, str]] = [] - picked_items: list[dict[str, str]] = [] - seen_titles: set[str] = set() - - for commit in bundle["commits"]: - title = first_line(commit.get("message", "")) - if not title or title in seen_titles: - continue - seen_titles.add(title) - entry = { - "kind": "commit", - "title": title, - "url": commit["url"], - "meta": short_sha(commit["sha"]), - } - if title.startswith("Merge branch "): - continue - fallback_items.append(entry) - if title.lower() in GENERIC_COMMIT_TITLES: - continue - picked_items.append(entry) - - items.extend(picked_items or fallback_items) - return items - - -def rendered_source_refs(bundle: dict[str, Any]) -> dict[str, Any]: - refs: dict[str, Any] = { - "repo": bundle["repo"], - "commit_urls": [commit["url"] for commit in bundle["commits"]], - "items": rendered_source_items(bundle), - } - primary_pr = bundle.get("primary_pr") - if isinstance(primary_pr, dict) and primary_pr.get("url"): - refs["pr_url"] = primary_pr["url"] - return refs - - -def pick_published_at(bundle: dict[str, Any], analysis: dict[str, Any], override: str | None) -> str: - if override: - return override - if isinstance(analysis.get("published_at"), str) and analysis["published_at"]: - return analysis["published_at"] - primary_pr = bundle.get("primary_pr") - if isinstance(primary_pr, dict) and primary_pr.get("merged_at"): - return primary_pr["merged_at"] - first_commit = bundle["commits"][0] - return first_commit.get("committed_at") or utc_now_iso() - - -def main() -> None: - args = parse_args() - bundle = load_json(args.bundle) - analysis = load_json(args.analysis) - - bundle_result = validate_bundle(bundle) - if not bundle_result.ok: - raise SystemExit("Bundle validation failed:\n- " + "\n- ".join(bundle_result.errors)) - - analysis_result = validate_analysis_draft(analysis) - if not analysis_result.ok: - raise SystemExit("Analysis draft validation failed:\n- " + "\n- ".join(analysis_result.errors)) - - config_flags = analysis.get("config_flags") - if config_flags is None: - config_flags = bundle.get("extracted_flags", []) - config_flags = normalized_config_flags(config_flags) - - signal = { - "schema": SIGNAL_SCHEMA, - "slug": analysis.get("slug") or slugify(analysis["title"]), - "lane": "github", - "kind": analysis["kind"], - "title": analysis["title"], - "published_at": pick_published_at(bundle, analysis, args.published_at), - "summary": analysis["summary"], - "why_it_matters": analysis["why_it_matters"], - "confidence": analysis["confidence"], - "impact": analysis["impact"], - "config_flags": config_flags, - "proof_points": analysis["proof_points"], - "source_refs": rendered_source_refs(bundle), - } - - if analysis.get("how_to_try"): - signal["how_to_try"] = analysis["how_to_try"] - if analysis.get("expected_effect"): - signal["expected_effect"] = analysis["expected_effect"] - if analysis.get("caveats"): - signal["caveats"] = analysis["caveats"] - if analysis.get("watch_state"): - signal["watch_state"] = analysis["watch_state"] - - validation = validate_signal(signal) - if not validation.ok: - raise SystemExit("Signal validation failed:\n- " + "\n- ".join(validation.errors)) - - dump_json(args.out, signal) - print(args.out) - - -if __name__ == "__main__": - main() diff --git a/scripts/github/sync_upstream_radar.py b/scripts/github/sync_upstream_radar.py deleted file mode 100644 index c1f1c09..0000000 --- a/scripts/github/sync_upstream_radar.py +++ /dev/null @@ -1,416 +0,0 @@ -#!/usr/bin/env python3 -"""Sync the deterministic Codex upstream review queue for Decodex Radar.""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import urllib.parse -from pathlib import Path -from typing import Any - -from build_change_bundle import ( - build_commit_bundle, - build_pr_bundle, - github_request, - maybe_promote_commit_to_pr, - repo_default_branch, - routed_token_env, -) -from contracts import ( - dump_json, - load_json, - utc_now_iso, - validate_signal, - validate_upstream_review_queue, -) -from radar_ledger import DEFAULT_LEDGER_PATH, connect as connect_ledger, record_commit, record_review - -SCRIPT_HOME = Path(__file__).resolve().parent -COMMIT_URL_RE = re.compile(r"/commit/([0-9a-f]{7,40})$") -PR_URL_RE = re.compile(r"/pull/(\d+)$") - -SURFACE_RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( - ("app_server_protocol", ("app-server", "app_server", "protocol", "jsonrpc", "json-rpc")), - ("mcp_plugins", ("mcp", "plugin", "tool-search", "tool_search")), - ("browser_chrome", ("browser", "chrome", "webview")), - ("sandbox_permissions", ("sandbox", "permission", "approval", "policy", "denylist", "allowlist")), - ("config_hooks", ("config", "hook", "settings", "toml")), - ("auth_accounts", ("auth", "account", "login", "token")), - ("model_provider", ("model", "provider", "rate-limit", "ratelimit", "quota")), - ("cli_tui", ("cli", "tui", "terminal", "chatwidget")), - ("release_packaging", ("release", "appcast", "sparkle", "version", "install", "package")), - ("docs_examples", ("docs/", "readme", "example")), - ("tests_ci", ("test", "tests", ".github", "ci", "fixture")), -) - -ATTENTION_RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( - ("new_feature", ("feat", "feature", "add ", "adds ", "support", "enable", "implement", "introduce")), - ("deprecated_removed", ("deprecat", "remove", "removed", "delete", "disable", "no longer")), - ("protocol_change", ("protocol", "schema", "api", "json-rpc", "jsonrpc", "notification", "request", "response")), - ("breaking_change", ("breaking", "break ", "rename", "migration", "incompat", "no longer")), - ("security_policy", ("sandbox", "permission", "approval", "full access", "network", "denylist", "allowlist")), - ("rate_limit", ("rate limit", "ratelimit", "quota", "usage limit", "message cap")), - ("auth_account", ("auth", "account", "login", "token")), - ("release_packaging", ("release", "appcast", "sparkle", "beta", "version")), -) - -HIGH_VALUE_SURFACES = { - "app_server_protocol", - "mcp_plugins", - "browser_chrome", - "sandbox_permissions", - "config_hooks", - "auth_accounts", - "model_provider", -} -HIGH_VALUE_FLAGS = { - "deprecated_removed", - "protocol_change", - "breaking_change", - "security_policy", - "rate_limit", - "auth_account", -} - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--repo", default="openai/codex", help="GitHub repository in owner/name format.") - parser.add_argument("--search-limit", type=int, default=40, help="How many recent upstream commits to inspect.") - parser.add_argument("--signals-dir", default="site/src/content/signals", help="Published signal directory.") - parser.add_argument( - "--queue-out", - default="artifacts/github/review-queue/openai-codex-latest.json", - help="Path to write the deterministic upstream_review_queue/v1 artifact.", - ) - parser.add_argument("--token-env", help="Environment variable containing a GitHub token.") - parser.add_argument( - "--ledger", - default=DEFAULT_LEDGER_PATH, - help="Local SQLite Radar ledger path. Defaults to .decodex/radar.sqlite3.", - ) - parser.add_argument("--no-ledger", action="store_true", help="Disable local Radar ledger writes.") - parser.add_argument("--dry-run", action="store_true", help="Print the queue without writing queue-out.") - return parser.parse_args() - - -def repo_root() -> Path: - return SCRIPT_HOME.parents[1] - - -def published_subjects(signals_dir: Path) -> tuple[set[int], set[str]]: - published_prs: set[int] = set() - published_shas: set[str] = set() - for path in sorted(signals_dir.glob("*.json")): - payload = load_json(path) - validation = validate_signal(payload) - if not validation.ok: - raise SystemExit(f"Signal validation failed for {path}:\n- " + "\n- ".join(validation.errors)) - pr_url = payload.get("source_refs", {}).get("pr_url") - if isinstance(pr_url, str): - match = PR_URL_RE.search(pr_url) - if match: - published_prs.add(int(match.group(1))) - for url in payload.get("source_refs", {}).get("commit_urls", []): - if not isinstance(url, str): - continue - match = COMMIT_URL_RE.search(url) - if match: - published_shas.add(match.group(1)) - return published_prs, published_shas - - -def recent_commits(repo: str, token: str | None, search_limit: int) -> tuple[str, list[dict[str, Any]]]: - default_branch = repo_default_branch(repo, token) - payload, _ = github_request( - f"https://api.github.com/repos/{repo}/commits?sha={urllib.parse.quote(default_branch)}&per_page={search_limit}", - token, - ) - if not isinstance(payload, list): - raise SystemExit("Expected commits list payload from GitHub API") - results: list[dict[str, Any]] = [] - for item in payload: - sha = item.get("sha") - commit = item.get("commit") - url = item.get("html_url") - if not isinstance(sha, str) or not isinstance(commit, dict) or not isinstance(url, str): - continue - message = commit.get("message") - if not isinstance(message, str) or not message: - continue - results.append( - { - "sha": sha, - "title": message.strip().splitlines()[0], - "url": url, - "committed_at": (commit.get("committer") or {}).get("date"), - } - ) - return default_branch, results - - -def text_blob(bundle: dict[str, Any]) -> str: - parts: list[str] = [] - primary_pr = bundle.get("primary_pr") - if isinstance(primary_pr, dict): - parts.extend([str(primary_pr.get("title") or ""), str(primary_pr.get("body") or "")]) - for commit in bundle.get("commits", []): - if isinstance(commit, dict): - parts.append(str(commit.get("message") or "")) - for item in bundle.get("files", []): - if isinstance(item, dict): - parts.extend([str(item.get("path") or ""), str(item.get("patch_excerpt") or "")]) - return "\n".join(parts).lower() - - -def detect_surface_hints(bundle: dict[str, Any]) -> list[str]: - paths = [str(item.get("path") or "").lower() for item in bundle.get("files", []) if isinstance(item, dict)] - haystack = "\n".join(paths) - hints = { - surface - for surface, terms in SURFACE_RULES - if any(term in haystack for term in terms) - } - if not hints: - hints.add("internal_churn") - return sorted(hints) - - -def detect_attention_flags(bundle: dict[str, Any]) -> list[str]: - haystack = text_blob(bundle) - return sorted( - flag - for flag, terms in ATTENTION_RULES - if any(term in haystack for term in terms) - ) - - -def priority_for(surface_hints: list[str], attention_flags: list[str]) -> str: - surfaces = set(surface_hints) - flags = set(attention_flags) - if (flags & {"breaking_change", "deprecated_removed"}) and (surfaces & HIGH_VALUE_SURFACES): - return "critical" - if (surfaces & HIGH_VALUE_SURFACES) and (flags & HIGH_VALUE_FLAGS): - return "high" - if surfaces & HIGH_VALUE_SURFACES: - return "high" - if flags & {"new_feature", "protocol_change", "release_packaging"}: - return "normal" - return "low" - - -def review_reason(surface_hints: list[str], attention_flags: list[str]) -> str: - if "internal_churn" in surface_hints and not attention_flags: - return "Needs AI review because every recent upstream commit is tracked, but deterministic hints found only internal churn." - if attention_flags: - return "Needs AI review for " + ", ".join(attention_flags) + "." - return "Needs AI review for surface hints: " + ", ".join(surface_hints) + "." - - -def subject_from_bundle( - *, - bundle: dict[str, Any], - subject_kind: str, - subject_id: str, - seed_commit: dict[str, Any], -) -> dict[str, Any]: - primary_pr = bundle.get("primary_pr") - commits = [item for item in bundle.get("commits", []) if isinstance(item, dict)] - files = [item for item in bundle.get("files", []) if isinstance(item, dict)] - commit_shas = [str(item["sha"]) for item in commits if isinstance(item.get("sha"), str)] - surface_hints = detect_surface_hints(bundle) - attention_flags = detect_attention_flags(bundle) - title = seed_commit["title"] - url = seed_commit["url"] - source_state = "commit_only" - subject: dict[str, Any] = { - "subject_kind": subject_kind, - "subject_id": subject_id, - "title": title, - "url": url, - "source_state": source_state, - "commit_shas": commit_shas or [seed_commit["sha"]], - "committed_at": seed_commit.get("committed_at"), - "changed_file_count": len(files), - "sample_paths": [str(item.get("path")) for item in files[:12] if item.get("path")], - "surface_hints": surface_hints, - "attention_flags": attention_flags, - "review_priority": priority_for(surface_hints, attention_flags), - "review_reason": review_reason(surface_hints, attention_flags), - "next_step": "ai_review_required", - } - if isinstance(primary_pr, dict): - title = str(primary_pr.get("title") or title) - url = str(primary_pr.get("url") or url) - subject.update( - { - "title": title, - "url": url, - "source_state": str(primary_pr.get("state") or "pr_first"), - "pr_number": primary_pr.get("number"), - "pr_url": primary_pr.get("url"), - } - ) - return subject - - -def sort_subjects(subjects: list[dict[str, Any]]) -> list[dict[str, Any]]: - priority_rank = {"critical": 0, "high": 1, "normal": 2, "low": 3} - return sorted( - subjects, - key=lambda item: ( - priority_rank.get(str(item.get("review_priority")), 9), - str(item.get("committed_at") or ""), - str(item.get("subject_kind") or ""), - str(item.get("subject_id") or ""), - ), - ) - - -def build_review_queue(args: argparse.Namespace) -> tuple[dict[str, Any], dict[str, int]]: - token_env = args.token_env or routed_token_env() or "GITHUB_TOKEN" - token = os.environ.get(token_env) - root = repo_root() - default_branch, commits = recent_commits(args.repo, token, args.search_limit) - published_prs, published_shas = published_subjects((root / args.signals_dir).resolve()) - ledger_path = None if args.no_ledger else Path(args.ledger) - if ledger_path is not None and not ledger_path.is_absolute(): - ledger_path = root / ledger_path - ledger = connect_ledger(ledger_path) if ledger_path is not None else None - subjects: dict[tuple[str, str], dict[str, Any]] = {} - published_seen = 0 - try: - for commit in commits: - pr_number = maybe_promote_commit_to_pr(args.repo, commit["sha"], token) - subject_kind = "pr" if pr_number is not None else "commit" - subject_id = str(pr_number) if pr_number is not None else commit["sha"] - if ledger is not None: - record_commit( - ledger, - repo=args.repo, - sha=commit["sha"], - title=commit["title"], - url=commit["url"], - committed_at=commit.get("committed_at"), - pr_number=pr_number, - ) - if commit["sha"] in published_shas or (pr_number is not None and pr_number in published_prs): - published_seen += 1 - if ledger is not None: - record_review( - ledger, - repo=args.repo, - subject_kind=subject_kind, - subject_id=subject_id, - status="signal", - reason="Already present in published signal collection.", - confidence="confirmed", - ) - continue - key = (subject_kind, subject_id) - if key in subjects: - current = subjects[key] - if commit["sha"] not in current["commit_shas"]: - current["commit_shas"].append(commit["sha"]) - continue - bundle = ( - build_pr_bundle(args.repo, pr_number, token, ["Built in-memory for deterministic Radar queue hints."]) - if isinstance(pr_number, int) - else build_commit_bundle(args.repo, commit["sha"], token, ["Built in-memory for deterministic Radar queue hints."]) - ) - subjects[key] = subject_from_bundle( - bundle=bundle, - subject_kind=subject_kind, - subject_id=subject_id, - seed_commit=commit, - ) - if ledger is not None: - record_review( - ledger, - repo=args.repo, - subject_kind=subject_kind, - subject_id=subject_id, - status="watch", - reason="Queued for AI upstream review by deterministic Radar sync.", - confidence="likely", - ) - if ledger is not None: - ledger.commit() - finally: - if ledger is not None: - ledger.close() - - ordered_subjects = sort_subjects(list(subjects.values())) - queue = { - "schema": "upstream_review_queue/v1", - "repo": args.repo, - "generated_at": utc_now_iso(), - "source": { - "default_branch": default_branch, - "search_limit": args.search_limit, - "signals_dir": args.signals_dir, - }, - "subjects": ordered_subjects, - "counts": { - "recent_commits_scanned": len(commits), - "published_subjects_seen": published_seen, - "subjects_queued": len(ordered_subjects), - "critical": sum(1 for item in ordered_subjects if item["review_priority"] == "critical"), - "high": sum(1 for item in ordered_subjects if item["review_priority"] == "high"), - "normal": sum(1 for item in ordered_subjects if item["review_priority"] == "normal"), - "low": sum(1 for item in ordered_subjects if item["review_priority"] == "low"), - }, - } - return queue, {"ledger_enabled": 0 if args.no_ledger else 1} - - -def material_queue(value: dict[str, Any]) -> dict[str, Any]: - normalized = json.loads(json.dumps(value, sort_keys=True)) - if isinstance(normalized, dict): - normalized["generated_at"] = "" - return normalized - - -def write_queue_if_changed(path: Path, queue: dict[str, Any]) -> bool: - if path.exists(): - try: - existing = load_json(path) - except json.JSONDecodeError: - existing = None - if isinstance(existing, dict) and material_queue(existing) == material_queue(queue): - return False - dump_json(path, queue) - return True - - -def main() -> None: - args = parse_args() - root = repo_root() - queue, extra_counts = build_review_queue(args) - validation = validate_upstream_review_queue(queue) - if not validation.ok: - raise SystemExit("Upstream review queue validation failed:\n- " + "\n- ".join(validation.errors)) - if args.dry_run: - print(json.dumps(queue, indent=2, sort_keys=True)) - return - out = root / args.queue_out - changed = write_queue_if_changed(out, queue) - print( - json.dumps( - { - "repo": queue["repo"], - **queue["counts"], - **extra_counts, - "changed": changed, - "queue_out": str(out), - }, - sort_keys=True, - ) - ) - - -if __name__ == "__main__": - main() diff --git a/scripts/github/test_backfill_release_range.py b/scripts/github/test_backfill_release_range.py deleted file mode 100644 index 7e606cb..0000000 --- a/scripts/github/test_backfill_release_range.py +++ /dev/null @@ -1,103 +0,0 @@ -from __future__ import annotations - -import importlib.util -import json -import tempfile -import unittest -from pathlib import Path - -MODULE_PATH = Path(__file__).resolve().with_name("backfill_release_range.py") -MODULE_SPEC = importlib.util.spec_from_file_location("backfill_release_range", MODULE_PATH) -if MODULE_SPEC is None or MODULE_SPEC.loader is None: - raise RuntimeError(f"Unable to load {MODULE_PATH}") -backfill_release_range = importlib.util.module_from_spec(MODULE_SPEC) -MODULE_SPEC.loader.exec_module(backfill_release_range) - - -def release(tag_name: str, prerelease: bool, published_at: str) -> dict[str, object]: - return { - "tag_name": tag_name, - "name": tag_name, - "prerelease": prerelease, - "published_at": published_at, - "url": f"https://github.com/openai/codex/releases/tag/{tag_name}", - } - - -def compare(stable_tag: str, preview_tag: str, pr_numbers: list[int]) -> dict[str, object]: - return { - "stable_tag_name": stable_tag, - "prerelease_tag_name": preview_tag, - "compare": { - "status": "ahead", - "ahead_by": len(pr_numbers), - "total_commits": len(pr_numbers), - "url": f"https://github.com/openai/codex/compare/{stable_tag}...{preview_tag}", - "commit_shas": [f"deadbeef{number}" for number in pr_numbers], - "pr_numbers": pr_numbers, - }, - "tracked_signal_slugs": [], - } - - -class LoadSelectedComparisonTests(unittest.TestCase): - def write_release_delta(self, path: Path) -> None: - payload = { - "schema": "release_delta/v1", - "repo": "openai/codex", - "tag_prefix": "rust-v", - "generated_at": "2026-05-13T00:00:00Z", - "stable_release": release("rust-v0.130.0", False, "2026-05-01T00:00:00Z"), - "prerelease": release("rust-v0.131.0-alpha.9", True, "2026-05-12T00:00:00Z"), - "compare": compare("rust-v0.130.0", "rust-v0.131.0-alpha.9", [22404])["compare"], - "release_options": { - "stable": [ - release("rust-v0.130.0", False, "2026-05-01T00:00:00Z"), - release("rust-v0.129.0", False, "2026-04-20T00:00:00Z"), - ], - "preview": [ - release("rust-v0.131.0-alpha.9", True, "2026-05-12T00:00:00Z"), - release("rust-v0.131.0-alpha.8", True, "2026-05-11T00:00:00Z"), - ], - }, - "comparisons": [ - compare("rust-v0.130.0", "rust-v0.131.0-alpha.9", [22404]), - compare("rust-v0.129.0", "rust-v0.131.0-alpha.8", [22397]), - ], - "tracked_signal_slugs": [], - } - path.write_text(json.dumps(payload), encoding="utf-8") - - def test_defaults_to_top_level_stable_and_prerelease(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - path = Path(tmpdir) / "release-delta.json" - self.write_release_delta(path) - - comparison, stable_tag, preview_tag = backfill_release_range.load_selected_comparison( - path, - None, - None, - ) - - self.assertEqual(stable_tag, "rust-v0.130.0") - self.assertEqual(preview_tag, "rust-v0.131.0-alpha.9") - self.assertEqual(comparison["compare"]["pr_numbers"], [22404]) - - def test_can_select_explicit_stable_and_prerelease(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - path = Path(tmpdir) / "release-delta.json" - self.write_release_delta(path) - - comparison, stable_tag, preview_tag = backfill_release_range.load_selected_comparison( - path, - "rust-v0.129.0", - "rust-v0.131.0-alpha.8", - ) - - self.assertEqual(stable_tag, "rust-v0.129.0") - self.assertEqual(preview_tag, "rust-v0.131.0-alpha.8") - self.assertEqual(comparison["compare"]["pr_numbers"], [22397]) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/github/test_build_release_delta.py b/scripts/github/test_build_release_delta.py deleted file mode 100644 index 2fd1eba..0000000 --- a/scripts/github/test_build_release_delta.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import annotations - -import importlib.util -import io -import ssl -import unittest -import urllib.error -from pathlib import Path -from unittest import mock - -MODULE_PATH = Path(__file__).resolve().with_name("build_release_delta.py") -MODULE_SPEC = importlib.util.spec_from_file_location("build_release_delta", MODULE_PATH) -if MODULE_SPEC is None or MODULE_SPEC.loader is None: - raise RuntimeError(f"Unable to load {MODULE_PATH}") -build_release_delta = importlib.util.module_from_spec(MODULE_SPEC) -MODULE_SPEC.loader.exec_module(build_release_delta) - - -class FakeResponse(io.StringIO): - def __init__(self, body: str): - super().__init__(body) - self.headers: dict[str, str] = {} - - def __enter__(self) -> "FakeResponse": - return self - - def __exit__(self, exc_type, exc, tb) -> bool: - self.close() - return False - - -class GithubRequestTests(unittest.TestCase): - def test_retries_transient_ssl_eof(self) -> None: - attempts: list[object] = [ - urllib.error.URLError(ssl.SSLEOFError("EOF occurred in violation of protocol")), - FakeResponse('{"ok": true}'), - ] - - def fake_urlopen(_request, timeout=None): - self.assertEqual(timeout, build_release_delta.GITHUB_REQUEST_TIMEOUT_SECONDS) - result = attempts.pop(0) - if isinstance(result, Exception): - raise result - return result - - with ( - mock.patch.object( - build_release_delta.urllib.request, - "urlopen", - side_effect=fake_urlopen, - ) as urlopen_mock, - mock.patch.object(build_release_delta.time, "sleep") as sleep_mock, - ): - payload = build_release_delta.github_request("https://api.github.com/repos/openai/codex/releases", "token") - - self.assertEqual(payload, {"ok": True}) - self.assertEqual(urlopen_mock.call_count, 2) - sleep_mock.assert_called_once_with(build_release_delta.GITHUB_REQUEST_BACKOFF_SECONDS) - - def test_non_retryable_url_error_still_fails(self) -> None: - with ( - mock.patch.object( - build_release_delta.urllib.request, - "urlopen", - side_effect=urllib.error.URLError("name resolution failed"), - ), - mock.patch.object(build_release_delta.time, "sleep") as sleep_mock, - ): - with self.assertRaises(SystemExit) as exc_info: - build_release_delta.github_request("https://api.github.com/repos/openai/codex/releases", "token") - - self.assertIn("name resolution failed", str(exc_info.exception)) - sleep_mock.assert_not_called() - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/github/test_radar_ledger.py b/scripts/github/test_radar_ledger.py deleted file mode 100644 index ed204c5..0000000 --- a/scripts/github/test_radar_ledger.py +++ /dev/null @@ -1,115 +0,0 @@ -from __future__ import annotations - -import importlib.util -import json -import tempfile -import unittest -from pathlib import Path - -MODULE_PATH = Path(__file__).resolve().with_name("radar_ledger.py") -MODULE_SPEC = importlib.util.spec_from_file_location("radar_ledger", MODULE_PATH) -if MODULE_SPEC is None or MODULE_SPEC.loader is None: - raise RuntimeError(f"Unable to load {MODULE_PATH}") -radar_ledger = importlib.util.module_from_spec(MODULE_SPEC) -MODULE_SPEC.loader.exec_module(radar_ledger) - - -class RadarLedgerTests(unittest.TestCase): - def write_json(self, path: Path, payload: dict[str, object]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload), encoding="utf-8") - - def test_ingests_existing_bundle_analysis_and_signal(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - bundle_path = root / "artifacts/github/bundles/openai-codex-pr-123.json" - analysis_path = root / "artifacts/github/analysis/openai-codex-pr-123.analysis.json" - signal_path = root / "site/src/content/signals/openai-codex-pr-123.json" - self.write_json( - bundle_path, - { - "schema": "github_change_bundle/v1", - "repo": "openai/codex", - "analysis_mode": "pr_first", - "default_branch": "main", - "primary_pr": { - "number": 123, - "title": "Add useful behavior", - "body": "", - "state": "merged", - "labels": [], - "url": "https://github.com/openai/codex/pull/123", - }, - "commits": [ - { - "sha": "abc1234", - "message": "Add useful behavior", - "url": "https://github.com/openai/codex/commit/abc1234", - "committed_at": "2026-05-13T00:00:00Z", - } - ], - "files": [ - { - "path": "codex-rs/core/src/lib.rs", - "status": "modified", - "additions": 1, - "deletions": 0, - } - ], - }, - ) - self.write_json( - analysis_path, - { - "kind": "capability", - "title": "Useful behavior", - "summary": "Adds behavior.", - "why_it_matters": "It helps users.", - "confidence": "confirmed", - "impact": "medium", - "proof_points": ["PR exists."], - }, - ) - self.write_json( - signal_path, - { - "schema": "signal_entry/v1", - "slug": "useful-behavior", - "lane": "github", - "kind": "capability", - "title": "Useful behavior", - "published_at": "2026-05-13T00:00:00Z", - "summary": "Adds behavior.", - "why_it_matters": "It helps users.", - "confidence": "confirmed", - "impact": "medium", - "config_flags": [], - "caveats": [], - "proof_points": ["PR exists."], - "source_refs": { - "repo": "openai/codex", - "pr_url": "https://github.com/openai/codex/pull/123", - "commit_urls": ["https://github.com/openai/codex/commit/abc1234"], - }, - }, - ) - - connection = radar_ledger.connect(root / "radar.sqlite3") - try: - payload = radar_ledger.ingest_existing( - connection, - bundles_dir=root / "artifacts/github/bundles", - analysis_dir=root / "artifacts/github/analysis", - signals_dir=root / "site/src/content/signals", - ) - finally: - connection.close() - - self.assertEqual(payload["upstream_commits"], 1) - self.assertEqual(payload["radar_reviews"], 2) - self.assertEqual(payload["artifact_links"], 4) - self.assertEqual(payload["bundles_ingested"], 1) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/github/test_social_post_contract.py b/scripts/github/test_social_post_contract.py deleted file mode 100644 index 3afa1e9..0000000 --- a/scripts/github/test_social_post_contract.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -"""Exercise the Decodex social_post/v1 validator.""" - -from __future__ import annotations - -from contracts import validate_social_post - - -def base_record() -> dict[str, object]: - return { - "schema": "social_post/v1", - "slug": "openai-codex-pr-22414", - "channel": "x", - "target_account": "decodexspace", - "controller_account": "hackink", - "mode": "operator_impact", - "status": "published", - "audience": "Codex operators", - "text": ["Remote Codex can now use Unix socket endpoints. Source: https://github.com/openai/codex/pull/22414"], - "source_refs": {"urls": ["https://github.com/openai/codex/pull/22414"]}, - "evidence_notes": ["PR #22414 changes the remote app-server endpoint handling."], - "claims": [ - { - "text": "Remote Codex can use Unix socket endpoints.", - "evidence": "https://github.com/openai/codex/pull/22414", - "confidence": "confirmed", - } - ], - "decision": { - "worthiness": "publish", - "priority": "high", - "idempotency_key": "x:decodexspace:operator_impact:openai-codex-pr-22414", - "reason": "High-value Control Plane transport implication.", - "daily_limit": 8, - "daily_count_before": 2, - "daily_count_after": 3, - "day": "2026-06-02", - "timezone": "Asia/Shanghai", - }, - "publication": { - "posted_at": "2026-06-02T03:00:00Z", - "published_urls": ["https://x.com/decodexspace/status/1"], - "publisher": "chrome", - "account_verified": True, - "made_with_ai": True, - "image_template": "decodex_signal_card", - }, - "media_refs": ["artifacts/social/x/images/openai-codex-pr-22414.png"], - } - - -def assert_valid(record: dict[str, object]) -> None: - validation = validate_social_post(record) - assert validation.ok, validation.errors - - -def assert_invalid(record: dict[str, object], expected: str) -> None: - validation = validate_social_post(record) - assert not validation.ok - assert any(expected in error for error in validation.errors), validation.errors - - -def main() -> None: - published = base_record() - assert_valid(published) - - blocked = base_record() - blocked["status"] = "blocked" - blocked.pop("publication") - blocked["decision"] = { - **blocked["decision"], # type: ignore[arg-type] - "worthiness": "block", - "daily_count_before": 8, - "daily_count_after": 8, - } - blocked["block"] = { - "reason": "daily_cap_exceeded", - "operator_notice": "Candidate blocked because @decodexspace already posted 8 times today.", - } - assert_valid(blocked) - - over_cap_published = base_record() - over_cap_published["decision"] = { - **over_cap_published["decision"], # type: ignore[arg-type] - "daily_limit": 9, - } - assert_invalid(over_cap_published, "daily_limit must be 8") - - print("OK") - - -if __name__ == "__main__": - main() diff --git a/scripts/github/validate_change_bundle.py b/scripts/github/validate_change_bundle.py deleted file mode 100644 index 3960c64..0000000 --- a/scripts/github/validate_change_bundle.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 -"""Validate one or more GitHub change bundle JSON files.""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path - -SCRIPT_HOME = Path(__file__).resolve().parent -if str(SCRIPT_HOME) not in sys.path: - sys.path.insert(0, str(SCRIPT_HOME)) - -from contracts import load_json, validate_bundle # noqa: E402 - - -def iter_json_files(paths: list[str]) -> list[Path]: - files: list[Path] = [] - for raw in paths: - path = Path(raw) - if path.is_dir(): - files.extend(sorted(path.glob("*.json"))) - else: - files.append(path) - return files - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("paths", nargs="+", help="Bundle JSON files or directories.") - args = parser.parse_args() - - errors: list[str] = [] - for path in iter_json_files(args.paths): - result = validate_bundle(load_json(path)) - if result.ok: - continue - for error in result.errors: - errors.append(f"{path}: {error}") - - if errors: - raise SystemExit("Bundle validation failed:\n- " + "\n- ".join(errors)) - - print("OK") - - -if __name__ == "__main__": - main() diff --git a/scripts/github/validate_signal_entry.py b/scripts/github/validate_signal_entry.py deleted file mode 100644 index afaac0c..0000000 --- a/scripts/github/validate_signal_entry.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 -"""Validate one or more rendered Decodex signal-entry JSON files.""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path - -SCRIPT_HOME = Path(__file__).resolve().parent -if str(SCRIPT_HOME) not in sys.path: - sys.path.insert(0, str(SCRIPT_HOME)) - -from contracts import load_json, validate_signal # noqa: E402 - - -def iter_json_files(paths: list[str]) -> list[Path]: - files: list[Path] = [] - for raw in paths: - path = Path(raw) - if path.is_dir(): - files.extend(sorted(path.glob("*.json"))) - else: - files.append(path) - return files - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("paths", nargs="+", help="Signal JSON files or directories.") - args = parser.parse_args() - - errors: list[str] = [] - seen_slugs: dict[str, Path] = {} - for path in iter_json_files(args.paths): - payload = load_json(path) - result = validate_signal(payload) - if not result.ok: - for error in result.errors: - errors.append(f"{path}: {error}") - slug = payload.get("slug") - if isinstance(slug, str): - if slug in seen_slugs: - errors.append(f"{path}: duplicate slug {slug!r} also used by {seen_slugs[slug]}") - else: - seen_slugs[slug] = path - - if errors: - raise SystemExit("Signal validation failed:\n- " + "\n- ".join(errors)) - - print("OK") - - -if __name__ == "__main__": - main() diff --git a/scripts/github/validate_social_post.py b/scripts/github/validate_social_post.py deleted file mode 100644 index 096cf36..0000000 --- a/scripts/github/validate_social_post.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -"""Validate Decodex social post publication-record JSON files.""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path - -SCRIPT_HOME = Path(__file__).resolve().parent -if str(SCRIPT_HOME) not in sys.path: - sys.path.insert(0, str(SCRIPT_HOME)) - -from contracts import SOCIAL_POST_SCHEMA, load_json, validate_social_post # noqa: E402 - - -def iter_json_files(paths: list[str]) -> list[Path]: - files: list[Path] = [] - for raw in paths: - path = Path(raw) - if path.is_dir(): - files.extend(sorted(path.rglob("*.json"))) - else: - files.append(path) - return files - - -def validate_payload(path: Path) -> list[str]: - payload = load_json(path) - if payload.get("schema") != SOCIAL_POST_SCHEMA: - return [f"{path}: schema must be {SOCIAL_POST_SCHEMA}"] - validation = validate_social_post(payload) - return [f"{path}: {error}" for error in validation.errors] - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("paths", nargs="+", help="Social post JSON files or directories.") - args = parser.parse_args() - - errors: list[str] = [] - for path in iter_json_files(args.paths): - errors.extend(validate_payload(path)) - - if errors: - raise SystemExit("Social post validation failed:\n- " + "\n- ".join(errors)) - - print("OK") - - -if __name__ == "__main__": - main() diff --git a/scripts/github/validate_upstream_review.py b/scripts/github/validate_upstream_review.py deleted file mode 100644 index b276680..0000000 --- a/scripts/github/validate_upstream_review.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -"""Validate upstream review queue and review JSON files.""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path - -SCRIPT_HOME = Path(__file__).resolve().parent -if str(SCRIPT_HOME) not in sys.path: - sys.path.insert(0, str(SCRIPT_HOME)) - -from contracts import ( # noqa: E402 - UPSTREAM_REVIEW_QUEUE_SCHEMA, - UPSTREAM_REVIEW_SCHEMA, - load_json, - validate_upstream_review, - validate_upstream_review_queue, -) - - -def iter_json_files(paths: list[str]) -> list[Path]: - files: list[Path] = [] - for raw in paths: - path = Path(raw) - if path.is_dir(): - files.extend(sorted(path.glob("*.json"))) - else: - files.append(path) - return files - - -def validate_payload(path: Path) -> list[str]: - payload = load_json(path) - schema = payload.get("schema") - if schema == UPSTREAM_REVIEW_QUEUE_SCHEMA: - validation = validate_upstream_review_queue(payload) - elif schema == UPSTREAM_REVIEW_SCHEMA: - validation = validate_upstream_review(payload) - else: - return [f"{path}: schema must be {UPSTREAM_REVIEW_QUEUE_SCHEMA} or {UPSTREAM_REVIEW_SCHEMA}"] - return [f"{path}: {error}" for error in validation.errors] - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("paths", nargs="+", help="Review queue JSON files or directories.") - args = parser.parse_args() - - errors: list[str] = [] - for path in iter_json_files(args.paths): - errors.extend(validate_payload(path)) - - if errors: - raise SystemExit("Upstream review validation failed:\n- " + "\n- ".join(errors)) - - print("OK") - - -if __name__ == "__main__": - main() diff --git a/site/src/content/release-deltas/README.md b/site/src/content/release-deltas/README.md index 359442d..2d0d585 100644 --- a/site/src/content/release-deltas/README.md +++ b/site/src/content/release-deltas/README.md @@ -10,7 +10,7 @@ The current artifact is a bounded comparator payload: Build the latest `openai/codex` artifact with: ```bash -python3 scripts/github/build_release_delta.py \ +cargo run -p decodex --bin decodex -- radar refresh-release-delta \ --repo openai/codex \ --signals-dir site/src/content/signals \ --out site/src/content/release-deltas/openai-codex-latest.json