diff --git a/.github/workflows/release-chocolatey-channel.yml b/.github/workflows/release-chocolatey-channel.yml index 844196eb8..487b87465 100644 --- a/.github/workflows/release-chocolatey-channel.yml +++ b/.github/workflows/release-chocolatey-channel.yml @@ -11,6 +11,9 @@ on: expected_source_sha: {required: true, type: string} release_id: {required: true, type: string} release_ref: {required: true, type: string} + secrets: + CHOCOLATEY_API_KEY: + required: false permissions: {} diff --git a/.github/workflows/release-linux-repository-build.yml b/.github/workflows/release-linux-repository-build.yml index 0a7fd93da..3330845fe 100644 --- a/.github/workflows/release-linux-repository-build.yml +++ b/.github/workflows/release-linux-repository-build.yml @@ -9,11 +9,32 @@ on: expected_source_sha: {required: true, type: string} release_id: {required: true, type: string} release_ref: {required: true, type: string} + secrets: + RMUX_APT_GPG_PRIVATE_KEY: + required: false + RMUX_APT_GPG_KEY: + required: false + RMUX_RPM_GPG_PRIVATE_KEY: + required: false + RMUX_RPM_GPG_KEY: + required: false + RMUX_RPM_REPO_GPG_PRIVATE_KEY: + required: false + RMUX_RPM_REPO_GPG_KEY: + required: false outputs: repository_artifact_id: value: ${{ jobs.build.outputs.repository_artifact_id }} repository_artifact_digest: value: ${{ jobs.build.outputs.repository_artifact_digest }} + workflow_dispatch: + inputs: + receipt_run_id: {required: true, type: string} + payload_artifact_id: {required: true, type: string} + payload_artifact_digest: {required: true, type: string} + expected_source_sha: {required: true, type: string} + release_id: {required: true, type: string} + release_ref: {required: true, type: string} permissions: {} diff --git a/.github/workflows/release-linux-repository-publish.yml b/.github/workflows/release-linux-repository-publish.yml index ff6826b01..bf2d88d3b 100644 --- a/.github/workflows/release-linux-repository-publish.yml +++ b/.github/workflows/release-linux-repository-publish.yml @@ -13,6 +13,9 @@ on: expected_source_sha: {required: true, type: string} release_id: {required: true, type: string} release_ref: {required: true, type: string} + secrets: + RMUX_DOWNSTREAM_APP_PRIVATE_KEY: + required: false permissions: {} diff --git a/.github/workflows/release-owned-repository-channel.yml b/.github/workflows/release-owned-repository-channel.yml index b87dfb425..c0195d901 100644 --- a/.github/workflows/release-owned-repository-channel.yml +++ b/.github/workflows/release-owned-repository-channel.yml @@ -12,6 +12,9 @@ on: expected_source_sha: {required: true, type: string} release_id: {required: true, type: string} release_ref: {required: true, type: string} + secrets: + RMUX_DOWNSTREAM_APP_PRIVATE_KEY: + required: false permissions: {} diff --git a/.github/workflows/release-snap-channel.yml b/.github/workflows/release-snap-channel.yml index 41de58142..58c556712 100644 --- a/.github/workflows/release-snap-channel.yml +++ b/.github/workflows/release-snap-channel.yml @@ -11,6 +11,9 @@ on: expected_source_sha: {required: true, type: string} release_id: {required: true, type: string} release_ref: {required: true, type: string} + secrets: + SNAPCRAFT_STORE_CREDENTIALS: + required: false permissions: {} diff --git a/scripts/release/downstream_workflow_contract.py b/scripts/release/downstream_workflow_contract.py index 0b1a62ba7..b8f90887e 100644 --- a/scripts/release/downstream_workflow_contract.py +++ b/scripts/release/downstream_workflow_contract.py @@ -64,13 +64,29 @@ def _validate_reusable_workflow(path: Path, *, require_repository_guard: bool) - text = path.read_text(encoding="utf-8") if "on:\n workflow_call:" not in text or "permissions: {}" not in text: raise ValueError(f"{path.name} must remain reusable and default-deny") + allows_signing_recovery = path.name == "release-linux-repository-build.yml" for forbidden in ( "\n push:", - "\n workflow_dispatch:", "larger-runner", ): if forbidden in text: raise ValueError(f"{path.name} contains forbidden value {forbidden}") + if allows_signing_recovery: + dispatch = text.split("\n workflow_dispatch:\n", 1) + if len(dispatch) != 2 or "\npermissions: {}\n" not in dispatch[1]: + raise ValueError("Linux repository signer lost its recovery trigger") + for name in ( + "expected_source_sha", + "payload_artifact_digest", + "payload_artifact_id", + "receipt_run_id", + "release_id", + "release_ref", + ): + if f" {name}:" not in dispatch[1]: + raise ValueError(f"Linux repository recovery lost input {name}") + elif "\n workflow_dispatch:" in text: + raise ValueError(f"{path.name} gained a mutation-capable dispatch trigger") if "runs-on: self-hosted" in text or "\n - self-hosted" in text: raise ValueError(f"{path.name} gained a self-hosted runner label") if require_repository_guard and ( @@ -80,6 +96,35 @@ def _validate_reusable_workflow(path: Path, *, require_repository_guard: bool) - raise ValueError(f"{path.name} does not reject external callers") +def _validate_worker_secret_declarations(root: Path) -> None: + workflows = root / ".github/workflows" + expected = { + "release-chocolatey-channel.yml": ("CHOCOLATEY_API_KEY",), + "release-linux-repository-build.yml": ( + "RMUX_APT_GPG_PRIVATE_KEY", + "RMUX_APT_GPG_KEY", + "RMUX_RPM_GPG_PRIVATE_KEY", + "RMUX_RPM_GPG_KEY", + "RMUX_RPM_REPO_GPG_PRIVATE_KEY", + "RMUX_RPM_REPO_GPG_KEY", + ), + "release-linux-repository-publish.yml": ("RMUX_DOWNSTREAM_APP_PRIVATE_KEY",), + "release-owned-repository-channel.yml": ("RMUX_DOWNSTREAM_APP_PRIVATE_KEY",), + "release-snap-channel.yml": ("SNAPCRAFT_STORE_CREDENTIALS",), + } + for name, secret_names in expected.items(): + text = (workflows / name).read_text(encoding="utf-8") + call = text.split("\n workflow_dispatch:", 1)[0] + if "\n secrets:\n" not in call: + raise ValueError(f"{name} does not declare protected environment secrets") + for secret_name in secret_names: + declaration = f" {secret_name}:\n required: false" + if call.count(declaration) != 1: + raise ValueError(f"{name} lost secret declaration {secret_name}") + if text.count(f"secrets.{secret_name}") < 1: + raise ValueError(f"{name} no longer consumes secret {secret_name}") + + def _calls_workflow(line: str, workflow: str) -> bool: normalized = line.strip().removeprefix("- ").strip() if not normalized.startswith("uses:"): @@ -138,8 +183,7 @@ def _validate_receipt_origin(main: str) -> None: "${{ runner.temp }}/rmux-downstream/publication-receipt-predicate.json" ) nested_predicate = ( - "${{ runner.temp }}/rmux-downstream/receipt/" - "publication-receipt-predicate.json" + "${{ runner.temp }}/rmux-downstream/receipt/publication-receipt-predicate.json" ) if main.count(flat_predicate) != 1 or nested_predicate in main: raise ValueError("downstream plan artifact must flatten its receipt predicate") @@ -419,13 +463,12 @@ def validate_downstream_workflows(root: Path) -> None: ) for path in _worker_paths(root): _validate_reusable_workflow(path, require_repository_guard=False) + _validate_worker_secret_declarations(root) _validate_callers(root, paths) main = paths[0].read_text(encoding="utf-8") _validate_receipt_origin(main) _validate_channel_orchestration(main) - _validate_payload_prepare( - root / ".github/workflows/release-downstream-prepare.yml" - ) + _validate_payload_prepare(root / ".github/workflows/release-downstream-prepare.yml") _validate_channel_result_action(_channel_result_action_path(root)) for path in paths[1:]: _validate_retry(path) diff --git a/scripts/release/publish-crate-set.py b/scripts/release/publish-crate-set.py index 94a6efd98..097ac1390 100755 --- a/scripts/release/publish-crate-set.py +++ b/scripts/release/publish-crate-set.py @@ -5,6 +5,7 @@ import argparse import hashlib +import json import os import re import subprocess @@ -50,7 +51,38 @@ def registry_url(name: str, version: str, *, download: bool = False) -> str: return f"https://crates.io/api/v1/crates/{name}/{version}{suffix}" +def registry_version_exists(name: str, version: str) -> bool: + request = urllib.request.Request( + registry_url(name, version), + headers={"User-Agent": "rmux-release-writer/1 (security@rmux.io)"}, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + data = response.read(2 * 1024 * 1024 + 1) + except urllib.error.HTTPError as error: + if error.code == 404: + return False + raise ValueError(f"crates.io lookup failed with HTTP {error.code}") from error + if len(data) > 2 * 1024 * 1024: + raise ValueError("crates.io metadata exceeds the release size limit") + try: + metadata = json.loads(data) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError("crates.io metadata is invalid") from error + observed = metadata.get("version") if isinstance(metadata, dict) else None + if ( + not isinstance(observed, dict) + or observed.get("crate") != name + or observed.get("num") != version + or observed.get("dl_path") != f"/api/v1/crates/{name}/{version}/download" + ): + raise ValueError("crates.io metadata identity differs") + return True + + def registry_bytes(name: str, version: str) -> bytes | None: + if not registry_version_exists(name, version): + return None request = urllib.request.Request( registry_url(name, version, download=True), headers={"User-Agent": "rmux-release-writer/1 (security@rmux.io)"}, @@ -59,9 +91,7 @@ def registry_bytes(name: str, version: str) -> bytes | None: with urllib.request.urlopen(request, timeout=30) as response: data = response.read(16 * 1024 * 1024 + 1) except urllib.error.HTTPError as error: - if error.code == 404: - return None - raise ValueError(f"crates.io lookup failed with HTTP {error.code}") from error + raise ValueError(f"crates.io download failed with HTTP {error.code}") from error if len(data) > 16 * 1024 * 1024: raise ValueError("crates.io package exceeds the release size limit") return data @@ -86,6 +116,27 @@ def run_cargo(args: list[str], *, token: str, target_dir: Path) -> None: raise ValueError(f"cargo {' '.join(args[:2])} failed: {detail}") +def generated_package_paths(target_dir: Path, filename: str) -> tuple[Path, Path]: + package_dir = target_dir / "package" + return package_dir / filename, package_dir / "tmp-crate" / filename + + +def remove_generated_package(target_dir: Path, filename: str) -> None: + for path in generated_package_paths(target_dir, filename): + path.unlink(missing_ok=True) + + +def generated_package(target_dir: Path, filename: str) -> Path: + files = [ + path + for path in generated_package_paths(target_dir, filename) + if path.is_file() and not path.is_symlink() + ] + if len(files) != 1: + raise ValueError(f"Cargo generated an unexpected package set: {filename}") + return files[0] + + def wait_for_exact(name: str, version: str, expected: Path) -> None: expected_hash = file_hash(expected) for attempt in range(19): @@ -125,21 +176,25 @@ def execute(args: argparse.Namespace) -> None: if hashlib.sha256(existing).hexdigest() != package["sha256"]: raise ValueError(f"existing crates.io bytes differ: {name}") continue - generated = cargo_target / "package" / package["file"] - generated.unlink(missing_ok=True) + remove_generated_package(cargo_target, package["file"]) run_cargo( ["publish", "--dry-run", "--locked", "--package", name], token=token, target_dir=cargo_target, ) - if not generated.is_file() or file_hash(generated) != file_hash(canonical): + if file_hash(generated_package(cargo_target, package["file"])) != file_hash( + canonical + ): raise ValueError(f"Cargo package bytes differ from the candidate: {name}") + remove_generated_package(cargo_target, package["file"]) run_cargo( ["publish", "--locked", "--package", name], token=token, target_dir=cargo_target, ) - if not generated.is_file() or file_hash(generated) != file_hash(canonical): + if file_hash(generated_package(cargo_target, package["file"])) != file_hash( + canonical + ): raise ValueError(f"published Cargo package bytes changed: {name}") mutated = True wait_for_exact(name, version, canonical) diff --git a/tests/release_downstream_workflows_spec.rs b/tests/release_downstream_workflows_spec.rs index 772b0a583..e444c3624 100644 --- a/tests/release_downstream_workflows_spec.rs +++ b/tests/release_downstream_workflows_spec.rs @@ -269,6 +269,85 @@ fn downstream_writers_keep_the_python_310_runtime_contract() { } } +#[cfg(unix)] +#[test] +fn crates_writer_distinguishes_missing_versions_from_download_denials() { + let fixture = r#" +import importlib.util +import pathlib +import sys +import tempfile +import urllib.error + +root = pathlib.Path.cwd() +scripts = root / "scripts" / "release" +sys.path.insert(0, str(scripts)) +spec = importlib.util.spec_from_file_location( + "publish_crate_set", scripts / "publish-crate-set.py" +) +module = importlib.util.module_from_spec(spec) +assert spec.loader is not None +spec.loader.exec_module(module) + +calls = [] +class Response: + def __init__(self, body): + self.body = body + def __enter__(self): + return self + def __exit__(self, *_args): + return False + def read(self, _limit): + return self.body + +def missing(request, timeout): + calls.append((request.full_url, timeout)) + raise urllib.error.HTTPError(request.full_url, 404, "missing", {}, None) + +module.urllib.request.urlopen = missing +assert module.registry_bytes("rmux", "0.9.1") is None +assert calls == [("https://crates.io/api/v1/crates/rmux/0.9.1", 30)] + +def forbidden(request, timeout): + raise urllib.error.HTTPError(request.full_url, 403, "forbidden", {}, None) + +module.urllib.request.urlopen = forbidden +try: + module.registry_bytes("rmux", "0.9.1") +except ValueError as error: + assert "lookup failed with HTTP 403" in str(error) +else: + raise AssertionError("metadata access denial was treated as a missing version") + +with tempfile.TemporaryDirectory() as directory: + target = pathlib.Path(directory) + filename = "rmux-types-0.9.1.crate" + direct, temporary = module.generated_package_paths(target, filename) + temporary.parent.mkdir(parents=True) + temporary.write_bytes(b"canonical") + assert module.generated_package(target, filename) == temporary + direct.write_bytes(b"duplicate") + try: + module.generated_package(target, filename) + except ValueError as error: + assert "unexpected package set" in str(error) + else: + raise AssertionError("ambiguous Cargo package output was accepted") + module.remove_generated_package(target, filename) + assert not direct.exists() and not temporary.exists() +"#; + let output = Command::new("python3") + .args(["-c", fixture]) + .current_dir(repo_root()) + .output() + .expect("run crates.io metadata regression fixture"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} + #[test] fn downstream_json_writer_uses_canonical_lf_bytes_on_every_platform() { let nonce = SystemTime::now()