From 3f27bebbc77d32d49733333eabc055b5e9ae3f8c Mon Sep 17 00:00:00 2001 From: Michael Roy Date: Wed, 5 Aug 2026 17:14:27 -0700 Subject: [PATCH] feat(release): distribute rocm-cli as a Python wheel Add a second distribution channel so `pip install rocm-cli` installs the prebuilt `rocm` and `rocmd` binaries, alongside the existing install.sh and install.ps1 bundles. scripts/build_wheel.py packages binaries that release CI has already built and signed, so the wheel payload stays byte-identical to the copy inside the signed archive for the same tag. The binaries ship in the wheel's `.data/scripts` directory as real executables rather than console-script shims, because `std::env::current_exe()` has to resolve to the real binary: the CLI re-execs itself to launch managed engine services and locates `rocmd` as a sibling file. Wheels land in `dist/wheels/`, a subdirectory, so the exact-asset gate in release_readiness.py keeps guarding the GitHub release asset set unchanged. Only `manylinux_2_17_x86_64` and `win_amd64` wheels are produced and no source distribution is published, so pip refuses to install on an unsupported platform instead of attempting a build. The released Linux binaries already satisfy manylinux_2_17: they link only allowlisted shared objects and need at most GLIBC_2.16. Git tags are mapped onto PEP 440 exhaustively-or-error and cross-checked against the workspace version, and both smoke tests assert that the installed files are native executables reporting the expected version. `rocm uninstall` previously deleted every rocm-named file beside the running executable. Inside a pip-created environment that deletes files pip owns and leaves the wheel RECORD dangling while pip still reports the package installed. It now detects a Python-managed layout and skips binary removal, pointing at the matching package manager instead. Probe failures resolve conservatively: a tree that exists but cannot be read is treated as managed rather than deleted. Publication is gated on the repository variable ROCM_CLI_PUBLISH_PYPI and stays dormant until the PyPI project and its Trusted Publisher exist. Until then CI builds, smoke-tests, and retains the wheels as artifacts without uploading. Signed-off-by: Michael Roy --- .github/workflows/release.yml | 251 +++++++++++++- README.md | 25 ++ apps/rocm/src/main.rs | 393 +++++++++++++++++++++ docs/release-trust.md | 68 ++++ docs/testing.md | 11 + scripts/build_wheel.py | 631 ++++++++++++++++++++++++++++++++++ 6 files changed, 1378 insertions(+), 1 deletion(-) create mode 100755 scripts/build_wheel.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9202a2dd..e876c348 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,6 +8,14 @@ on: tag: description: "Release tag (e.g. v0.1.0)" required: true + pypi_repository: + description: "PyPI index for the publish job (defaults to a TestPyPI rehearsal)" + required: false + default: testpypi + type: choice + options: + - testpypi + - pypi permissions: contents: write @@ -144,6 +152,62 @@ jobs: --asset "${DIST}.tar.gz" \ --asset rocm-cli-linux-amd64.tar.gz + - name: Build Python wheel + run: | + # Reuse the exact target/release binaries that were just packaged and + # signed — the wheel payload must stay byte-identical to the binaries + # inside the signed archive, so nothing is rebuilt here. + python scripts/build_wheel.py \ + --bin-dir target/release \ + --platform linux-amd64 \ + --tag "${{ steps.version.outputs.value }}" \ + --out-dir dist/wheels + + - name: Smoke test the Python wheel + run: | + shopt -s nullglob + wheels=(dist/wheels/*.whl) + if [ "${#wheels[@]}" -ne 1 ]; then + echo "expected exactly one linux wheel, found ${#wheels[@]}" >&2 + exit 1 + fi + venv="${RUNNER_TEMP}/wheel-smoke" + rm -rf "${venv}" + python -m venv "${venv}" + "${venv}/bin/python" -m pip install --no-index --disable-pip-version-check "${wheels[0]}" + # The wheel ships the real ELF binaries in its .data/scripts + # directory rather than console-script shims, so these must be the + # native executables themselves and must report their own version. + for binary in rocm rocmd; do + magic="$(head -c 4 "${venv}/bin/${binary}" | od -An -tx1 | tr -d ' \n')" + if [ "${magic}" != "7f454c46" ]; then + echo "${binary} is not an ELF executable (magic ${magic}); a shim would break current_exe()" >&2 + exit 1 + fi + done + # Assert the installed binaries are the version the wheel claims, so + # a stale target/release tree (for example from a restored build + # cache) cannot ship binaries that disagree with the release tag. + wheel_version="$(basename "${wheels[0]}")" + wheel_version="${wheel_version#rocm_cli-}" + wheel_version="${wheel_version%%-py3-none-*}" + release="$(printf '%s' "${wheel_version}" | sed -E 's/^([0-9]+\.[0-9]+\.[0-9]+).*/\1/')" + for binary in rocm rocmd; do + reported="$("${venv}/bin/${binary}" --version)" + if [ "${reported}" != "${binary} ${release}" ]; then + echo "${binary} reports '${reported}', expected '${binary} ${release}' from wheel ${wheel_version}" >&2 + exit 1 + fi + done + rm -rf "${venv}" + + - name: Upload Python wheel + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wheel-linux-amd64 + path: dist/wheels/ + if-no-files-found: error + - name: Create release env: GH_TOKEN: ${{ github.token }} @@ -222,9 +286,87 @@ jobs: GH_TOKEN: ${{ github.token }} run: | $version = "${{ needs.release.outputs.version }}" - $assets = Get-ChildItem dist -File -Include *.zip,*.sha256,*.sig -Recurse | ForEach-Object { $_.FullName } + # `dist\*` plus -Include, never -Recurse: with -Recurse this glob also + # matches the wheel checksum sidecars under dist\wheels, which are not + # GitHub release assets. Without a path wildcard, -Include silently + # matches nothing at all. + $assets = Get-ChildItem dist\* -File -Include *.zip,*.sha256,*.sig | ForEach-Object { $_.FullName } gh release upload $version @assets --clobber + # The wheel steps run after the release upload so the release asset set + # is produced from a tree that has no wheels in it yet. The upload glob + # above is non-recursive, so this ordering is defence in depth rather + # than the only thing keeping the asset set correct. + - name: Build Python wheel + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + # Reuse the exact target\release binaries that were just packaged and + # signed — the wheel payload must stay byte-identical to the binaries + # inside the signed archive, so nothing is rebuilt here. + python .\scripts\build_wheel.py ` + --bin-dir target\release ` + --platform windows-amd64 ` + --tag "${{ needs.release.outputs.version }}" ` + --out-dir dist\wheels + + - name: Smoke test the Python wheel + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + function Invoke-Checked { + param([string]$Exe, [string[]]$Arguments) + & $Exe @Arguments + if ($LASTEXITCODE -ne 0) { + throw "$Exe $($Arguments -join ' ') failed with exit code $LASTEXITCODE" + } + } + $wheels = @(Get-ChildItem dist\wheels -File -Filter *.whl) + if ($wheels.Count -ne 1) { + throw "expected exactly one windows wheel, found $($wheels.Count)" + } + $venv = Join-Path $env:RUNNER_TEMP "wheel-smoke" + if (Test-Path -LiteralPath $venv) { + Remove-Item -Recurse -Force -LiteralPath $venv + } + Invoke-Checked python @("-m", "venv", $venv) + Invoke-Checked "$venv\Scripts\python.exe" @("-m", "pip", "install", "--no-index", "--disable-pip-version-check", $wheels[0].FullName) + # The wheel ships the real PE binaries in its .data/scripts directory + # rather than console-script shims, so these must be the native + # executables themselves and must report their own version. + foreach ($binary in @("rocm.exe", "rocmd.exe")) { + $path = Join-Path "$venv\Scripts" $binary + $magic = [System.IO.File]::ReadAllBytes($path)[0..1] + if ($magic[0] -ne 0x4D -or $magic[1] -ne 0x5A) { + throw "$binary is not a PE executable; a shim would break current_exe()" + } + } + # Assert the installed binaries are the version the wheel claims, so + # a stale target\release tree cannot ship binaries that disagree with + # the release tag. + $wheelVersion = $wheels[0].Name -replace '^rocm_cli-', '' -replace '-py3-none-.*$', '' + if ($wheelVersion -notmatch '^(?\d+\.\d+\.\d+)') { + throw "cannot read a release segment out of wheel version '$wheelVersion'" + } + $release = $Matches['release'] + foreach ($binary in @("rocm", "rocmd")) { + $reported = (& "$venv\Scripts\$binary.exe" "--version") -join "" + if ($LASTEXITCODE -ne 0) { + throw "$binary --version failed with exit code $LASTEXITCODE" + } + if ($reported.Trim() -ne "$binary $release") { + throw "$binary reports '$reported', expected '$binary $release' from wheel $wheelVersion" + } + } + Remove-Item -Recurse -Force -LiteralPath $venv + + - name: Upload Python wheel + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wheel-windows-amd64 + path: dist/wheels/ + if-no-files-found: error + publish-release: name: Publish release runs-on: ubuntu-latest @@ -240,3 +382,110 @@ jobs: VERSION="${{ needs.release.outputs.version }}" RELEASE_ID="$(gh release view "${VERSION}" --json databaseId -q .databaseId)" gh api --method PATCH "repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}" -f draft=false + + publish-pypi: + name: Publish wheels to PyPI + runs-on: ubuntu-latest + needs: + - release + - windows-release + # Sequenced after publish-release so wheels never reach an index before + # the GitHub release they correspond to is out of draft. + - publish-release + # PyPI publication gate — dormant until the owner sets the repository + # variable ROCM_CLI_PUBLISH_PYPI to "1" or "true", mirroring the + # ROCM_CLI_REQUIRE_PRODUCTION_TRUST gate in the env block above. It stays + # unset until the project owns the PyPI `rocm-cli` project and has + # registered this workflow as its Trusted Publisher. The gate is an + # allowlist rather than a not-falsy test so that any other value — + # including "FALSE", "no", or "off" — leaves the job dormant. While it is + # dormant this job is skipped and nothing reaches any index; the wheels are + # still built, smoke-tested, and retained as workflow artifacts, and the + # GitHub release asset set is unchanged. + if: vars.ROCM_CLI_PUBLISH_PYPI == '1' || vars.ROCM_CLI_PUBLISH_PYPI == 'true' + environment: pypi + permissions: + # Trusted Publishing mints a short-lived OIDC token instead of using a + # stored API token, so this job needs id-token: write and no upload + # secrets. Declaring permissions here also keeps the workflow-level + # `contents: write` from applying to a job that writes nothing back. + contents: read + id-token: write + steps: + - name: Resolve publication target + id: target + env: + # Empty on tag pushes, which always mean the real index. A manual + # dispatch defaults to testpypi so that the irreversible action — + # a real upload burns that version forever — is always opt-in. + REQUESTED: ${{ github.event.inputs.pypi_repository }} + run: | + TARGET="${REQUESTED:-pypi}" + case "${TARGET}" in + pypi|testpypi) ;; + *) + echo "unknown pypi_repository input: ${TARGET}" >&2 + exit 1 + ;; + esac + echo "value=${TARGET}" >> "$GITHUB_OUTPUT" + + - name: Download built wheels + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: wheel-* + path: dist/wheels + merge-multiple: true + + - name: Verify wheel checksums and stage the upload directory + run: | + shopt -s nullglob + wheels=(dist/wheels/*.whl) + if [ "${#wheels[@]}" -ne 2 ]; then + echo "expected 2 wheels (linux + windows), found ${#wheels[@]}" >&2 + exit 1 + fi + + # Re-verify each wheel against the sidecar written beside it on the + # builder runner. The sidecar travels in the same artifact, so this + # catches a file corrupted or truncated in artifact storage transit, + # not a compromised builder that wrote both files consistently. + for wheel in "${wheels[@]}"; do + sidecar="${wheel}.sha256" + if [ ! -f "${sidecar}" ]; then + echo "missing checksum sidecar: ${sidecar}" >&2 + exit 1 + fi + expected="$(awk 'NR == 1 { print $1 }' "${sidecar}")" + actual="$(sha256sum "${wheel}" | awk '{ print $1 }')" + if [ -z "${expected}" ] || [ "${expected}" != "${actual}" ]; then + echo "checksum mismatch for ${wheel}: expected '${expected}', got '${actual}'" >&2 + exit 1 + fi + echo "verified $(basename "${wheel}") ${actual}" + done + + # The publish action uploads every file in packages-dir, so stage the + # wheels on their own — the .sha256 sidecars are not distributions. + mkdir -p dist/pypi + cp "${wheels[@]}" dist/pypi/ + + # Exactly one of the two steps below runs. A rehearsal resolves to + # testpypi and therefore cannot reach the real index, and any other + # value already failed in "Resolve publication target". Neither step + # passes credentials (Trusted Publishing only), and both leave PEP 740 + # attestations at the action's enabled default. + - name: Publish to TestPyPI (rehearsal) + if: steps.target.outputs.value == 'testpypi' + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + packages-dir: dist/pypi + repository-url: https://test.pypi.org/legacy/ + print-hash: true + + - name: Publish to PyPI + if: steps.target.outputs.value == 'pypi' + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + packages-dir: dist/pypi + print-hash: true diff --git a/README.md b/README.md index 86311c28..d4d4f2ef 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,31 @@ irm https://raw.githubusercontent.com/ROCm/rocm-cli/main/install.ps1 | iex Drop the `ROCM_CLI_CHANNEL` line to track the default `release` channel once a stable release is published. +### Python package (x86_64 Linux and Windows) + +No wheels are published yet; once the first release ships, `rocm-cli` on PyPI +is the supported install path for Python-packaging workflows. It carries the +same prebuilt `rocm` and `rocmd` binaries as the installers above. Because +these are command-line tools rather than a library, install them into their own +isolated environment: + +```bash +pipx install rocm-cli +``` + +```bash +uv tool install rocm-cli +``` + +A plain `pip install rocm-cli` also works, but it only puts `rocm` and `rocmd` +on `PATH` while that virtual environment is active. + +Wheels are built for Linux x86_64 and Windows x86_64 only, and there is no +source distribution — the wheel ships the same binaries that are inside the +signed release archives, not a Python reimplementation. It cannot carry the +release `.sig` sidecar, so see `docs/release-trust.md` for what secures this +channel instead. + ## Build from source Building requires [Rust](https://rustup.rs/); the pinned toolchain in diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index b0a3fd13..b20008ca 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -53,6 +53,7 @@ use rocm_engine_protocol::{ }; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; +use std::borrow::Cow; use std::collections::{BTreeMap, VecDeque}; use std::ffi::OsString; use std::fmt::Write as _; @@ -14888,6 +14889,11 @@ fn build_uninstall_plan(paths: &AppPaths, options: &UninstallOptions) -> Result< "binary removal skipped because {} looks like a cargo target build; pass --force-dev-binaries to remove sibling debug/release binaries", current_exe.display() )); + } else if is_python_managed_layout(¤t_exe) { + plan.skipped.push(format!( + "binary removal skipped because {} was installed by a Python package manager; remove it with the matching command instead: `pip uninstall rocm-cli`, `pipx uninstall rocm-cli`, or `uv tool uninstall rocm-cli`", + current_exe.display() + )); } else { for path in collect_installed_binary_candidates(¤t_exe)? { if rocm_core::runtime_is_windows() && path == current_exe { @@ -15045,6 +15051,225 @@ fn is_dev_binary_layout(path: &Path) -> bool { == Some("target") } +/// Detect an executable that a Python package manager (`pip` inside a virtual +/// environment, `pipx`, or `uv tool`) installed and still tracks. +/// +/// Only paths derived from `current_exe` are probed — no environment variables +/// and no process globals — so the decision is reproducible over a temporary +/// tree in unit tests. +/// +/// The probe is deliberately conservative, because the two mistakes do not cost +/// the same. A false positive only leaves the binaries in place and tells the +/// user to uninstall through their Python package manager; if that advice is +/// wrong they simply delete the files themselves. A false negative deletes +/// files another package manager owns: the wheel `RECORD` is left dangling, +/// `pip list` keeps reporting `rocm-cli` as installed, and pip can no longer +/// clean up its own install. +/// +/// That asymmetry decides how failures resolve. An *absent* marker means "not +/// Python-managed": no `pyvenv.cfg`, no `lib` directory, and no matching +/// `RECORD` row is the ordinary shape of an installer-managed `~/.local/bin` +/// tree, which must stay removable. But a probe that cannot read something that +/// does exist resolves to "Python-managed" instead of guessing: an unreadable +/// `lib` directory, an unreadable site-packages directory, a directory listing +/// that breaks midway, and an existing `RECORD` that cannot be read as text all +/// report ownership. Nothing here panics. +/// +/// Known and accepted limitation: the `RECORD` scan is not scoped to the +/// `rocm-cli` distribution. Any `*.dist-info/RECORD` under the environment that +/// lists a file named like this executable inside a `bin`/`Scripts` component +/// claims it, so a foreign distribution shipping a same-named script suppresses +/// binary removal. The only consequence is that the binaries stay in place and +/// the user is told to remove them through Python, which is not worth the +/// fragility of resolving relative `RECORD` paths against the real environment. +fn is_python_managed_layout(current_exe: &Path) -> bool { + let Some(script_dir) = current_exe.parent() else { + return false; + }; + let Some(script_dir_name) = script_dir.file_name().and_then(|name| name.to_str()) else { + return false; + }; + // Python environments place installed scripts directly under the + // environment root, in `bin` on unix and `Scripts` on Windows. + if script_dir_name != "bin" && script_dir_name != "Scripts" { + return false; + } + let Some(env_root) = script_dir.parent() else { + return false; + }; + // `python -m venv`, `pipx`, and `uv tool` all write a real `pyvenv.cfg` at + // the environment root. + if env_root.join("pyvenv.cfg").is_file() { + return true; + } + // No `pyvenv.cfg`: a `--user` or system-site install still records the + // script in the installed distribution's `RECORD`. + let Ok(site_packages_dirs) = python_site_packages_dirs(env_root) else { + return true; + }; + site_packages_dirs.iter().any(|site_packages| { + // An unreadable tree hides whatever owns the script, so claim it. + dist_info_record_owns_script(site_packages, current_exe).unwrap_or(true) + }) +} + +/// A layout probe that could not complete: something that exists on disk could +/// not be listed or read, so ownership is unknown. Callers must resolve it to +/// "Python-managed" — see [`is_python_managed_layout`] for why an unknown +/// answer has to fail closed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ProbeFailed; + +/// Site-packages directories to probe for the wheel `RECORD` that owns an +/// installed script. Both the unix (`lib/python3.13/site-packages`) and the +/// Windows (`Lib/site-packages`) layouts are probed on every host: this is pure +/// path inspection, and a tree must be classified the same way regardless of +/// which platform is doing the inspecting. +/// +/// A missing `lib` directory yields an empty list, because that is the ordinary +/// shape of a plain `~/.local/bin` install. A `lib` directory that exists but +/// cannot be listed yields [`ProbeFailed`]: its contents could name the +/// distribution that owns the script. +fn python_site_packages_dirs(env_root: &Path) -> Result, ProbeFailed> { + let mut dirs = Vec::new(); + let windows_layout = env_root.join("Lib").join("site-packages"); + if windows_layout.is_dir() { + dirs.push(windows_layout); + } + let unix_lib = env_root.join("lib"); + let entries = match fs::read_dir(&unix_lib) { + Ok(entries) => entries, + Err(_) if unix_lib.is_dir() => return Err(ProbeFailed), + Err(_) => return Ok(dirs), + }; + for entry in entries { + let Ok(entry) = entry else { + return Err(ProbeFailed); + }; + let version_dir = entry.path(); + if !version_dir + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("python")) + { + continue; + } + let candidate = version_dir.join("site-packages"); + if candidate.is_dir() { + dirs.push(candidate); + } + } + Ok(dirs) +} + +/// Whether any installed distribution under `site_packages` lists `current_exe` +/// in its `RECORD`. +/// +/// A missing site-packages directory, a `*.dist-info` without a `RECORD`, and a +/// `RECORD` whose rows never name the script all mean "not owned". A +/// site-packages directory that exists but cannot be listed, and a `RECORD` +/// that exists but cannot be read as text, mean [`ProbeFailed`]: the row that +/// would have claimed the script may be exactly the one that could not be read. +fn dist_info_record_owns_script( + site_packages: &Path, + current_exe: &Path, +) -> Result { + let Some(script_name) = current_exe.file_name().and_then(|name| name.to_str()) else { + return Ok(false); + }; + let Some(script_dir_name) = current_exe + .parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + else { + return Ok(false); + }; + let entries = match fs::read_dir(site_packages) { + Ok(entries) => entries, + Err(_) if site_packages.is_dir() => return Err(ProbeFailed), + Err(_) => return Ok(false), + }; + for entry in entries { + let Ok(entry) = entry else { + return Err(ProbeFailed); + }; + let dist_info = entry.path(); + if !dist_info + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.ends_with(".dist-info")) + { + continue; + } + let record_path = dist_info.join("RECORD"); + let record = match fs::read_to_string(&record_path) { + Ok(record) => record, + Err(_) if record_path.is_file() => return Err(ProbeFailed), + Err(_) => continue, + }; + if record + .lines() + .any(|row| record_row_targets_script(row, script_name, script_dir_name)) + { + return Ok(true); + } + } + Ok(false) +} + +/// Whether a wheel `RECORD` row (`,,`, with `/` separators on +/// every platform) refers to the installed script. +/// +/// Wheel records reference installed scripts by a site-packages-relative path +/// such as `../../../bin/rocm`. Matching on the file name plus a `bin`/`Scripts` +/// path component is intentional: resolving those relative paths exactly would +/// mean canonicalizing through the environment's symlinks and junctions, which +/// is fragile and can fail outright while an uninstall is in progress. +fn record_row_targets_script(row: &str, script_name: &str, script_dir_name: &str) -> bool { + let Some(target) = record_row_target_path(row) else { + return false; + }; + let target = Path::new(&*target); + if target.file_name().and_then(|name| name.to_str()) != Some(script_name) { + return false; + } + target + .components() + .any(|component| component.as_os_str().to_str() == Some(script_dir_name)) +} + +/// The first (path) field of a wheel `RECORD` row, unquoted and unescaped. +/// +/// `RECORD` is RFC 4180 CSV, so a path holding a comma or a double quote is +/// written quoted with every inner quote doubled. Splitting the row on its +/// first comma would truncate such a path and silently miss the script it +/// names, so the quoted form is parsed here instead. The common unquoted row +/// stays borrowed; only a quoted row allocates. A row that opens a quote and +/// never closes it is malformed and names nothing. +fn record_row_target_path(row: &str) -> Option> { + let row = row.trim_start(); + let Some(quoted) = row.strip_prefix('"') else { + let target = row.split(',').next().unwrap_or_default().trim_end(); + return (!target.is_empty()).then_some(Cow::Borrowed(target)); + }; + let mut target = String::new(); + let mut characters = quoted.chars().peekable(); + while let Some(character) = characters.next() { + if character != '"' { + target.push(character); + } else if characters.peek() == Some(&'"') { + // A doubled quote inside the field is one literal quote. + characters.next(); + target.push('"'); + } else { + // A lone quote closes the field; the rest of the row is the hash + // and size columns. + return (!target.is_empty()).then_some(Cow::Owned(target)); + } + } + None +} + fn remove_path(path: &Path) -> Result<()> { if !path.exists() { return Ok(()); @@ -16965,6 +17190,174 @@ mod tests { assert!(is_rocm_install_entry_name("rocm-codex.exe")); } + #[test] + fn is_python_managed_layout_detects_virtualenv_bin_script() { + // `pip install rocm-cli` inside a venv drops the real binary in + // `/bin`, and the env root always carries a `pyvenv.cfg`. + let (root, _paths) = test_paths("python-managed-venv"); + let script_dir = root.join("bin"); + fs::create_dir_all(&script_dir).expect("create venv bin dir"); + fs::write(root.join("pyvenv.cfg"), "home = /usr/bin\n").expect("write pyvenv.cfg"); + let executable = script_dir.join("rocm"); + fs::write(&executable, b"elf").expect("write rocm script"); + + assert!( + is_python_managed_layout(&executable), + "a wheel-installed script under a venv bin dir is owned by pip" + ); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn is_python_managed_layout_detects_windows_scripts_directory() { + // Same environment shape as above, written the way Windows wheels + // install it: `\Scripts\rocm.exe`. + let (root, _paths) = test_paths("python-managed-scripts"); + let script_dir = root.join("Scripts"); + fs::create_dir_all(&script_dir).expect("create venv Scripts dir"); + fs::write(root.join("pyvenv.cfg"), "home = C:/Python313\n").expect("write pyvenv.cfg"); + let executable = script_dir.join("rocm.exe"); + fs::write(&executable, b"pe").expect("write rocm.exe script"); + + assert!( + is_python_managed_layout(&executable), + "the Windows venv layout uses Scripts/ and must be detected the same way" + ); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn is_python_managed_layout_ignores_plain_local_bin_install() { + // An installer-managed `~/.local/bin` tree has a `bin` parent but no + // Python environment around it, so uninstall must still remove it. + let (root, _paths) = test_paths("python-managed-local-bin"); + let script_dir = root.join(".local").join("bin"); + fs::create_dir_all(&script_dir).expect("create local bin dir"); + let executable = script_dir.join("rocm"); + fs::write(&executable, b"elf").expect("write rocm binary"); + + assert!( + !is_python_managed_layout(&executable), + "an installer-managed ~/.local/bin install keeps today's removal behavior" + ); + + // The same tree, one file different: adding the marker every venv, + // pipx, and `uv tool` environment carries must flip the answer, so the + // negative above discriminates instead of matching a constant `false`. + fs::write(root.join(".local").join("pyvenv.cfg"), "home = /usr/bin\n") + .expect("write pyvenv.cfg"); + + assert!( + is_python_managed_layout(&executable), + "the same tree with a pyvenv.cfg at the environment root is pip-managed" + ); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn is_python_managed_layout_detects_dist_info_record_without_pyvenv_cfg() { + // A `--user` or system-site install has no `pyvenv.cfg`, but the + // distribution's RECORD still lists the installed script. + let (root, _paths) = test_paths("python-managed-record"); + let script_dir = root.join("bin"); + fs::create_dir_all(&script_dir).expect("create bin dir"); + let dist_info = root + .join("lib") + .join("python3.13") + .join("site-packages") + .join("rocm_cli-0.1.0a1.dist-info"); + fs::create_dir_all(&dist_info).expect("create dist-info dir"); + fs::write( + dist_info.join("RECORD"), + "rocm_cli-0.1.0a1.dist-info/METADATA,sha256=aaa,512\n\ + ../../../bin/rocm,sha256=bbb,17000000\n\ + ../../../bin/rocmd,sha256=ccc,16000000\n", + ) + .expect("write RECORD"); + let executable = script_dir.join("rocm"); + fs::write(&executable, b"elf").expect("write rocm script"); + + assert!( + is_python_managed_layout(&executable), + "a wheel RECORD that lists the script owns it even without pyvenv.cfg" + ); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn is_python_managed_layout_ignores_dist_info_record_for_other_scripts() { + // Some other distribution owns this environment; nothing in it claims + // our executable, so the binary is not pip-managed. + let (root, _paths) = test_paths("python-managed-foreign-record"); + let script_dir = root.join("bin"); + fs::create_dir_all(&script_dir).expect("create bin dir"); + let dist_info = root + .join("lib") + .join("python3.13") + .join("site-packages") + .join("other_tool-1.2.3.dist-info"); + fs::create_dir_all(&dist_info).expect("create dist-info dir"); + fs::write( + dist_info.join("RECORD"), + "other_tool-1.2.3.dist-info/METADATA,sha256=aaa,512\n\ + ../../../bin/other-tool,sha256=bbb,4096\n", + ) + .expect("write RECORD"); + let executable = script_dir.join("rocm"); + fs::write(&executable, b"elf").expect("write rocm binary"); + + assert!( + !is_python_managed_layout(&executable), + "a RECORD that never mentions the executable must not claim it" + ); + + // The same tree, one row different: teaching that RECORD to name this + // script must flip the answer, so the negative above discriminates + // instead of matching a constant `false`. + fs::write( + dist_info.join("RECORD"), + "other_tool-1.2.3.dist-info/METADATA,sha256=aaa,512\n\ + ../../../bin/other-tool,sha256=bbb,4096\n\ + ../../../bin/rocm,sha256=ccc,17000000\n", + ) + .expect("rewrite RECORD"); + + assert!( + is_python_managed_layout(&executable), + "the same RECORD does claim the executable once it lists it" + ); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn is_python_managed_layout_parses_quoted_record_rows() { + // RECORD is RFC 4180: a path holding a comma is quoted and an inner + // quote is doubled. Splitting the row on its first comma truncates the + // path to `"../../../bin/nested` — a fragment whose file name is + // `nested`, so the script it really names is missed. + assert!( + record_row_targets_script( + "\"../../../bin/nested,dir/rocm\",sha256=aaa,17000000", + "rocm", + "bin" + ), + "a quoted path containing a comma still resolves to the script" + ); + assert!( + record_row_targets_script("\"../../../bin/ro\"\"cm\",sha256=bbb,4096", "ro\"cm", "bin"), + "a doubled inner quote unescapes to one literal quote" + ); + assert!( + !record_row_targets_script("\"../../../bin/rocm,sha256=ccc,4096", "rocm", "bin"), + "an unterminated quoted field is malformed and names nothing" + ); + } + #[test] fn hybrid_planner_normalizes_model_alias_and_structured_serve_call() { let plan = build_freeform_plan("serve qwen3.5 with vllm", &RocmCliConfig::default()); diff --git a/docs/release-trust.md b/docs/release-trust.md index 1294749b..2cea453c 100644 --- a/docs/release-trust.md +++ b/docs/release-trust.md @@ -116,6 +116,74 @@ happen before activation. Run the current host's set with `E2E_INCLUDE_LIFECYCLE=1 E2E_ONLY_LIFECYCLE=1 cargo xtask e2e` (see `docs/testing.md`). +## PyPI Wheel Channel + +`pip install rocm-cli` distributes the same binaries through a second channel +with a *different trust root*. PyPI stores only the uploaded distribution +files, so it cannot carry the detached `.sig` sidecar described above: there is +nowhere for a wheel consumer to fetch `.sig` from, and `pip` would not +check it. Do not describe the wheel channel as detached-signature verified. + +What the wheel channel does provide: + +- **Trusted Publishing (OIDC).** The `publish-pypi` job in + `.github/workflows/release.yml` authenticates with a short-lived OpenID + Connect token minted for this repository's release workflow and its `pypi` + environment, so PyPI accepts `rocm-cli` uploads only from that identity. + There is no long-lived upload token to leak or rotate. +- **PEP 740 attestations.** `pypa/gh-action-pypi-publish` publishes a signed + attestation with each wheel, binding the file digest to the workflow that + produced it. Attestations stay enabled; the publish steps do not opt out. + Note what this is and is not: `pip` does not verify attestations at install + time and offers no flag to require them, so for a plain `pip install` this is + an auditable provenance record rather than an install-time gate. Fetch it + from PyPI's integrity API, + `https://pypi.org/integrity/rocm-cli///provenance`. +- **In-pipeline verification.** Each wheel is built in the same job that + packaged and signed the release archive, from the same `target/release` + binaries, smoke-tested by installing it into a throwaway virtualenv, and + re-checked against its `.sha256` sidecar in the publish job before upload. + The sidecar is written next to the wheel on the builder runner and travels in + the same artifact, so that check catches a file corrupted or truncated in + artifact storage transit. It cannot detect a compromised builder that wrote a + consistent wheel and sidecar together; the control against that is the + attestation's binding to the workflow identity, not the checksum. + +The GitHub release assets remain the signature-bearing artifacts. The `.tar.gz` +and `.zip` bundles and their detached `.sig` files are the only place the RSA +release signature is published, and they are what the installers verify — see +[Installer Verification](#installer-verification) for the verification inputs +and commands. `cargo xtask package` copies the release binaries verbatim, so +the `rocm` and `rocmd` payload in the wheel is byte-identical to the copy +inside the signed archive for the same tag; anyone who needs the detached +signature as their trust root should install from the release with `install.sh` +or `install.ps1` and verify there. + +Publication is gated on the repository variable `ROCM_CLI_PUBLISH_PYPI`, which +must be set to `1` or `true`; every other value, including unset, leaves the +job dormant. While it is dormant, release CI still builds, smoke-tests, and +retains the wheels as workflow artifacts, but nothing is uploaded to any index. +The wheel build and its smoke test do run on every release, so a wheel +regression fails the release job even while publication is off — the uploaded +GitHub asset set is unchanged, the pipeline is not. + +### Version mapping is one-way and unrepeatable + +`scripts/build_wheel.py` maps the git tag onto a PEP 440 version: `vX.Y.Z` to +`X.Y.Z`, `-alpha.N` and `-experimental.N` to `X.Y.ZaN`, `-beta.N` to `X.Y.ZbN`, +`-rc.N` to `X.Y.ZrcN`. Any other tag shape, and any version carrying a local, +dev, post, or epoch segment, is refused rather than published. The mapped +release segment must also equal `[workspace.package] version` in the root +`Cargo.toml`, so a tag can never publish a wheel whose binaries report a +different version. + +PyPI never lets a version be re-uploaded, even after deletion. Two consequences +bind the tagging policy: a botched release must be yanked and superseded by a +new version rather than replaced, and because `-alpha.N` and `-experimental.N` +both map to `X.Y.ZaN`, only one of those two spellings may be used for a given +serial. Tagging both `v1.2.3-experimental.1` and `v1.2.3-alpha.1` makes the +second release fail at upload. + ## WSL ROCDXG Package Verification `scripts/wsl_setup_rocdxg.sh` does not bake in a production checksum for the diff --git a/docs/testing.md b/docs/testing.md index 73aa878c..29a0ff1f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -729,6 +729,7 @@ Release trust checks: cargo test -p rocm --bin rocm metadata_signature_verification_accepts_generated_key_and_rejects_tamper cargo test -p rocm-core model_recipe_index_signature_accepts_generated_key_and_rejects_tamper python scripts/release_readiness.py --self-test +python scripts/build_wheel.py --self-test ROCDXG_CHECKSUM_SELF_TEST=1 bash scripts/wsl_setup_rocdxg.sh bash scripts/setup-wsl-portable-build-deps.sh --self-test ``` @@ -740,6 +741,16 @@ release asset sets, so stale archives and orphan checksum/signature sidecars in configured production trust inputs. Normal Linux and Windows CI run this self-test before the install-lifecycle E2E scenarios. +The wheel self-test is offline and cross-platform too. It runs entirely inside +a system temporary directory, building a wheel from a synthetic workspace and +throwaway binaries, and covers the git-tag-to-PEP-440 mapping (including the +rejected tag shapes), the cross-check against the `[workspace.package]` +version, both platform tags, the exact wheel member set, `RECORD` hashes and +sizes, byte-identical rebuilds, the `.sha256` sidecar text, and the +regular-file plus executable mode bits on the packed `rocm`/`rocmd` entries — +without those bits `pip install rocm-cli` yields non-executable binaries. See +`docs/release-trust.md` for how the PyPI channel is secured. + ### Packaging `cargo xtask package [output-dir] [--target ]` builds the diff --git a/scripts/build_wheel.py b/scripts/build_wheel.py new file mode 100755 index 00000000..f251bdb4 --- /dev/null +++ b/scripts/build_wheel.py @@ -0,0 +1,631 @@ +#!/usr/bin/env python3 +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT + +"""Build the `rocm-cli` PyPI wheel from already-built rocm/rocmd binaries. + +The wheel is a thin distribution channel: it carries no Python modules and no +console-script shims. The native binaries are placed in the wheel's +`.data/scripts/` directory so pip installs them verbatim into the environment's +`bin`/`Scripts` directory, keeping `std::env::current_exe()` pointed at the real +executable (the CLI re-execs itself and locates `rocmd` as a sibling file). + +Only prebuilt binaries are packaged here; this script never invokes cargo. +""" + +from __future__ import annotations + +import argparse +import base64 +import csv +import hashlib +import io +import re +import shutil +import sys +import tempfile +import zipfile +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parent + +DISTRIBUTION = "rocm-cli" +NORMALIZED_NAME = "rocm_cli" +GENERATOR = "rocm-cli build_wheel.py" +SUMMARY = "Command-line control plane for local ROCm AI inference (rocm, rocmd)." + +PLATFORM_TAGS = { + "linux-amd64": "manylinux_2_17_x86_64.manylinux2014_x86_64", + "windows-amd64": "win_amd64", +} +BINARY_STEMS = ("rocm", "rocmd") +LICENSE_FILES = ("LICENSE.TXT", "THIRD_PARTY_NOTICES.txt") + +# Reproducible zip timestamp: the earliest value the zip format can encode. +ZIP_DATE_TIME = (1980, 1, 1, 0, 0, 0) +SCRIPT_MODE = 0o755 +DATA_MODE = 0o644 +S_IFREG = 0o100000 +CREATE_SYSTEM_UNIX = 3 + +TAG_RE = re.compile( + r"^v?(?P\d+\.\d+\.\d+)" + r"(?:-(?Palpha|beta|rc|experimental)\.(?P\d+))?$" +) +PRE_RELEASE_MARKERS = { + "alpha": "a", + "experimental": "a", + "beta": "b", + "rc": "rc", +} +# Every version this builder is allowed to emit. Deliberately narrower than +# PEP 440: local versions (`+local`) are illegal in a wheel filename, and dev, +# post, and epoch versions are rejected by PyPI or sort confusingly against the +# release tags this project actually pushes. `--version` bypasses the tag +# mapper, so it is validated against the same shape rather than trusted. +PUBLISHABLE_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?$") +WORKSPACE_VERSION_RE = re.compile(r'^version\s*=\s*"([^"]+)"\s*$') + +DESCRIPTION = """\ +# rocm-cli + +`rocm-cli` distributes the prebuilt ROCm AI command-line control plane as a +Python wheel. Installing it places two native executables on your `PATH`: + +- `rocm` — the user-facing CLI: discover hardware, install and serve models, + chat with a local endpoint, and inspect system health. +- `rocmd` — the background daemon the CLI supervises for long-running engine + services. + +The wheel contains no Python code. It is a delivery vehicle for the same native +binaries published on the GitHub release page, so `pip install rocm-cli` is +simply a convenient way to get them. + +Supported platforms: Linux x86-64 (manylinux2014 or newer) and Windows x86-64. +No other platform wheels are published. + +Source, issues, and documentation: https://github.com/ROCm/rocm-cli +""" + + +class WheelBuildError(Exception): + """The wheel could not be built.""" + + +def fail(message: str) -> None: + print(f"wheel build failed: {message}", file=sys.stderr) + raise SystemExit(1) + + +def pep440_version_from_tag(tag: str) -> str: + """Map a release git tag onto its PEP 440 version. + + `vX.Y.Z` -> `X.Y.Z`, `-alpha.N`/`-experimental.N` -> `X.Y.ZaN`, + `-beta.N` -> `X.Y.ZbN`, `-rc.N` -> `X.Y.ZrcN`. Every other shape (nightly + and staging tags included) is a hard error. + + `-alpha.N` and `-experimental.N` intentionally collapse onto the same + `aN` form, because PEP 440 has one alpha marker and both tag spellings mean + the same thing here. The consequence is that only one of the two spellings + may be used per serial: pushing both `v1.2.3-experimental.1` and + `v1.2.3-alpha.1` produces the same wheel version twice, and PyPI refuses + the second upload because a released version can never be replaced. + """ + match = TAG_RE.fullmatch(tag.strip()) + if match is None: + raise WheelBuildError(f"tag is not a publishable release tag: {tag}") + release = match.group("release") + kind = match.group("kind") + if kind is None: + return release + serial = int(match.group("serial")) + return f"{release}{PRE_RELEASE_MARKERS[kind]}{serial}" + + +def release_segment(version: str) -> str: + """Return the `X.Y.Z` release segment of a mapped PEP 440 version.""" + match = re.match(r"^(\d+\.\d+\.\d+)", version) + if match is None: + raise WheelBuildError(f"version is not a PEP 440 release version: {version}") + return match.group(1) + + +def workspace_version(repo_root: Path) -> str: + """Parse `[workspace.package] version` out of the root `Cargo.toml`.""" + manifest = repo_root / "Cargo.toml" + if not manifest.is_file(): + raise WheelBuildError(f"workspace manifest not found: {manifest}") + in_section = False + for raw_line in manifest.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if line.startswith("["): + # A section header may carry a trailing comment. Strip it before + # matching so `[workspace.package] # ...` is still recognized. + header = line.split("#", 1)[0].strip() + if not header.endswith("]"): + continue + in_section = header == "[workspace.package]" + continue + if not in_section: + continue + match = WORKSPACE_VERSION_RE.match(line) + if match is not None: + return match.group(1) + raise WheelBuildError(f"[workspace.package] version not found in {manifest}") + + +def resolve_version(repo_root: Path, *, tag: str | None, version: str | None) -> str: + """Resolve the wheel version and cross-check it against the workspace.""" + if (tag is None) == (version is None): + raise WheelBuildError("exactly one of --tag or --version is required") + resolved = pep440_version_from_tag(tag) if tag is not None else str(version) + if PUBLISHABLE_VERSION_RE.fullmatch(resolved) is None: + source = f"tag {tag}" if tag is not None else f"version {version}" + raise WheelBuildError( + f"{source} resolves to {resolved}, which is not a publishable wheel " + "version (expected X.Y.Z with an optional aN, bN, or rcN suffix; " + "local, dev, post, and epoch versions are refused)" + ) + expected = workspace_version(repo_root) + actual = release_segment(resolved) + if actual != expected: + source = f"tag {tag}" if tag is not None else f"version {version}" + raise WheelBuildError( + f"{source} maps to release segment {actual}, but the workspace version " + f"in {repo_root / 'Cargo.toml'} is {expected}" + ) + return resolved + + +def wheel_platform_tag(platform: str) -> str: + """Return the wheel platform tag for a supported release platform.""" + try: + return PLATFORM_TAGS[platform] + except KeyError: + raise WheelBuildError(f"unsupported platform: {platform}") from None + + +def binary_names(platform: str) -> tuple[str, ...]: + suffix = ".exe" if platform == "windows-amd64" else "" + return tuple(f"{stem}{suffix}" for stem in BINARY_STEMS) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_sha256(path: Path) -> Path: + digest = sha256_file(path) + sidecar = path.with_suffix(path.suffix + ".sha256") + sidecar.write_text(f"{digest} {path.name}\n", encoding="ascii") + return sidecar + + +def record_hash(data: bytes) -> str: + encoded = base64.urlsafe_b64encode(hashlib.sha256(data).digest()) + return "sha256=" + encoded.rstrip(b"=").decode("ascii") + + +def metadata_text(version: str) -> str: + lines = [ + "Metadata-Version: 2.4", + f"Name: {DISTRIBUTION}", + f"Version: {version}", + f"Summary: {SUMMARY}", + "License-Expression: MIT", + "License-File: LICENSE.TXT", + "License-File: THIRD_PARTY_NOTICES.txt", + "Requires-Python: >=3.9", + "Project-URL: Source, https://github.com/ROCm/rocm-cli", + "Classifier: Environment :: Console", + "Classifier: Intended Audience :: Developers", + "Classifier: Operating System :: POSIX :: Linux", + "Classifier: Operating System :: Microsoft :: Windows", + "Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence", + "Description-Content-Type: text/markdown", + "", + DESCRIPTION, + ] + return "\n".join(lines) + + +def wheel_text(platform_tag: str) -> str: + return ( + "Wheel-Version: 1.0\n" + f"Generator: {GENERATOR}\n" + "Root-Is-Purelib: false\n" + f"Tag: py3-none-{platform_tag}\n" + ) + + +def zip_info(name: str, mode: int) -> zipfile.ZipInfo: + info = zipfile.ZipInfo(name, date_time=ZIP_DATE_TIME) + info.compress_type = zipfile.ZIP_DEFLATED + info.create_system = CREATE_SYSTEM_UNIX + # The regular-file bit is mandatory: without S_IFREG pip installs the + # binaries without their executable bit. The low 16 bits are MS-DOS + # attributes and are left at zero, as conventional wheel writers do. + info.external_attr = (S_IFREG | mode) << 16 + return info + + +def collect_members( + *, + repo_root: Path, + bin_dir: Path, + platform: str, + version: str, +) -> list[tuple[str, bytes, int]]: + """Return `(arcname, payload, mode)` triples in stable write order.""" + data_dir = f"{NORMALIZED_NAME}-{version}.data/scripts" + dist_info = f"{NORMALIZED_NAME}-{version}.dist-info" + members: list[tuple[str, bytes, int]] = [] + + for name in binary_names(platform): + source = bin_dir / name + if not source.is_file(): + raise WheelBuildError(f"required binary not found: {source}") + members.append((f"{data_dir}/{name}", source.read_bytes(), SCRIPT_MODE)) + + members.append( + (f"{dist_info}/METADATA", metadata_text(version).encode("utf-8"), DATA_MODE) + ) + members.append( + ( + f"{dist_info}/WHEEL", + wheel_text(wheel_platform_tag(platform)).encode("utf-8"), + DATA_MODE, + ) + ) + for name in LICENSE_FILES: + source = repo_root / name + if not source.is_file(): + raise WheelBuildError(f"required license file not found: {source}") + members.append((f"{dist_info}/licenses/{name}", source.read_bytes(), DATA_MODE)) + return members + + +def record_text(members: list[tuple[str, bytes, int]], record_name: str) -> str: + buffer = io.StringIO(newline="") + writer = csv.writer(buffer, lineterminator="\n") + for arcname, payload, _mode in members: + writer.writerow([arcname, record_hash(payload), len(payload)]) + writer.writerow([record_name, "", ""]) + return buffer.getvalue() + + +def build_wheel( + *, + repo_root: Path, + bin_dir: Path, + platform: str, + version: str, + out_dir: Path, +) -> Path: + """Write the wheel (and its `.sha256` sidecar) and return the wheel path.""" + platform_tag = wheel_platform_tag(platform) + if not bin_dir.is_dir(): + raise WheelBuildError(f"binary directory not found: {bin_dir}") + + members = collect_members( + repo_root=repo_root, bin_dir=bin_dir, platform=platform, version=version + ) + record_name = f"{NORMALIZED_NAME}-{version}.dist-info/RECORD" + members.append( + (record_name, record_text(members, record_name).encode("utf-8"), DATA_MODE) + ) + + out_dir.mkdir(parents=True, exist_ok=True) + wheel_path = out_dir / f"{NORMALIZED_NAME}-{version}-py3-none-{platform_tag}.whl" + with zipfile.ZipFile(wheel_path, "w", zipfile.ZIP_DEFLATED) as package: + for arcname, payload, mode in members: + package.writestr(zip_info(arcname, mode), payload) + write_sha256(wheel_path) + return wheel_path + + +def expect_error(label: str, func) -> None: + try: + func() + except WheelBuildError: + return + raise WheelBuildError(f"{label} unexpectedly succeeded") + + +def self_test_tag_mapping() -> None: + cases = { + "v1.2.3": "1.2.3", + "1.2.3": "1.2.3", + "v1.2.3-alpha.4": "1.2.3a4", + "v1.2.3-experimental.4": "1.2.3a4", + "v1.2.3-beta.4": "1.2.3b4", + "v1.2.3-rc.4": "1.2.3rc4", + "v0.1.0-experimental.1": "0.1.0a1", + } + for tag, expected in cases.items(): + actual = pep440_version_from_tag(tag) + if actual != expected: + raise WheelBuildError(f"tag {tag} mapped to {actual}, expected {expected}") + for bad in ( + "nightly-20260804-abc1234", + "staging-v1.2.3", + "v1.2.3-experimental", + "v1.2.3-alpha", + "v1.2.3-pre.1", + "1.2", + "v1.2.3.4", + "", + ): + expect_error(f"tag {bad!r}", lambda bad=bad: pep440_version_from_tag(bad)) + + +def make_fake_repo(root: Path) -> Path: + repo = root / "repo" + (repo).mkdir(parents=True) + (repo / "Cargo.toml").write_text( + "[workspace]\n" + 'members = ["apps/rocm"]\n' + "\n" + "[workspace.dependencies]\n" + 'version = "9.9.9"\n' + "\n" + "[workspace.package]\n" + 'version = "0.1.0"\n' + 'edition = "2024"\n', + encoding="utf-8", + ) + for name in LICENSE_FILES: + (repo / name).write_text(f"{name} test content\n", encoding="utf-8") + return repo + + +def self_test_wheel(root: Path, repo: Path) -> None: + bin_dir = root / "bin" + bin_dir.mkdir() + for name in BINARY_STEMS: + (bin_dir / name).write_bytes(b"\x7fELF fake " + name.encode("ascii") + b"\n") + + version = resolve_version(repo, tag="v0.1.0-experimental.1", version=None) + if version != "0.1.0a1": + raise WheelBuildError(f"unexpected resolved version: {version}") + + first = build_wheel( + repo_root=repo, + bin_dir=bin_dir, + platform="linux-amd64", + version=version, + out_dir=root / "out-a", + ) + second = build_wheel( + repo_root=repo, + bin_dir=bin_dir, + platform="linux-amd64", + version=version, + out_dir=root / "out-b", + ) + if ( + first.name + != "rocm_cli-0.1.0a1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + ): + raise WheelBuildError(f"unexpected wheel name: {first.name}") + if first.read_bytes() != second.read_bytes(): + # Byte-identical within one interpreter, which is what CI reproduces; + # deflate output is not guaranteed identical across zlib versions. + raise WheelBuildError("consecutive builds are not byte-identical") + sidecar = first.with_suffix(first.suffix + ".sha256") + if sidecar.read_text(encoding="ascii") != f"{sha256_file(first)} {first.name}\n": + raise WheelBuildError("sha256 sidecar contents are wrong") + + dist_info = "rocm_cli-0.1.0a1.dist-info" + data_scripts = "rocm_cli-0.1.0a1.data/scripts" + expected_names = { + f"{data_scripts}/rocm", + f"{data_scripts}/rocmd", + f"{dist_info}/METADATA", + f"{dist_info}/WHEEL", + f"{dist_info}/licenses/LICENSE.TXT", + f"{dist_info}/licenses/THIRD_PARTY_NOTICES.txt", + f"{dist_info}/RECORD", + } + with zipfile.ZipFile(first) as package: + names = set(package.namelist()) + if names != expected_names: + raise WheelBuildError(f"unexpected wheel members: {sorted(names)}") + for name in (f"{data_scripts}/rocm", f"{data_scripts}/rocmd"): + info = package.getinfo(name) + attrs = info.external_attr >> 16 + if attrs & 0o7777 != SCRIPT_MODE: + raise WheelBuildError(f"{name} mode is {oct(attrs & 0o7777)}") + if attrs & 0o170000 != S_IFREG: + raise WheelBuildError(f"{name} is missing the S_IFREG bit") + if info.date_time != ZIP_DATE_TIME: + raise WheelBuildError(f"{name} has a non-fixed timestamp") + wheel_meta = package.read(f"{dist_info}/WHEEL").decode("utf-8") + if "Root-Is-Purelib: false" not in wheel_meta: + raise WheelBuildError("WHEEL is missing Root-Is-Purelib: false") + if "Tag: py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64" not in wheel_meta: + raise WheelBuildError("WHEEL is missing the expected platform tag") + metadata = package.read(f"{dist_info}/METADATA").decode("utf-8") + for required in ("Metadata-Version: 2.4", "Name: rocm-cli", "Version: 0.1.0a1"): + if required not in metadata: + raise WheelBuildError(f"METADATA is missing {required!r}") + + rows = list( + csv.reader(package.read(f"{dist_info}/RECORD").decode("utf-8").splitlines()) + ) + listed = {row[0] for row in rows} + if listed != expected_names: + raise WheelBuildError(f"RECORD lists {sorted(listed)}") + for path, digest, size in rows: + if path == f"{dist_info}/RECORD": + if digest or size: + raise WheelBuildError("RECORD's own row must have empty fields") + continue + payload = package.read(path) + if digest != record_hash(payload): + raise WheelBuildError(f"RECORD hash mismatch for {path}") + if int(size) != len(payload): + raise WheelBuildError(f"RECORD size mismatch for {path}") + + windows_bin = root / "bin-win" + windows_bin.mkdir() + expect_error( + "windows wheel with missing binaries", + lambda: build_wheel( + repo_root=repo, + bin_dir=windows_bin, + platform="windows-amd64", + version=version, + out_dir=root / "out-win", + ), + ) + + windows_ok_bin = root / "bin-win-ok" + windows_ok_bin.mkdir() + for name in BINARY_STEMS: + (windows_ok_bin / f"{name}.exe").write_bytes(b"MZ fake " + name.encode("ascii")) + windows_wheel = build_wheel( + repo_root=repo, + bin_dir=windows_ok_bin, + platform="windows-amd64", + version=version, + out_dir=root / "out-win-ok", + ) + if windows_wheel.name != "rocm_cli-0.1.0a1-py3-none-win_amd64.whl": + raise WheelBuildError(f"unexpected windows wheel name: {windows_wheel.name}") + with zipfile.ZipFile(windows_wheel) as package: + names = set(package.namelist()) + expected_windows = { + f"{data_scripts}/rocm.exe", + f"{data_scripts}/rocmd.exe", + f"{dist_info}/METADATA", + f"{dist_info}/WHEEL", + f"{dist_info}/licenses/LICENSE.TXT", + f"{dist_info}/licenses/THIRD_PARTY_NOTICES.txt", + f"{dist_info}/RECORD", + } + if names != expected_windows: + raise WheelBuildError(f"unexpected windows wheel members: {sorted(names)}") + wheel_meta = package.read(f"{dist_info}/WHEEL").decode("utf-8") + if "Tag: py3-none-win_amd64" not in wheel_meta: + raise WheelBuildError("windows WHEEL is missing the win_amd64 tag") + + +def run_self_test(root: Path) -> None: + if root.exists(): + shutil.rmtree(root) + root.mkdir(parents=True) + try: + self_test_tag_mapping() + + if wheel_platform_tag("windows-amd64") != "win_amd64": + raise WheelBuildError("windows platform tag is wrong") + if ( + wheel_platform_tag("linux-amd64") + != "manylinux_2_17_x86_64.manylinux2014_x86_64" + ): + raise WheelBuildError("linux platform tag is wrong") + expect_error( + "platform darwin-arm64", lambda: wheel_platform_tag("darwin-arm64") + ) + + repo = make_fake_repo(root) + if workspace_version(repo) != "0.1.0": + raise WheelBuildError( + f"workspace version parsed as {workspace_version(repo)}" + ) + expect_error( + "workspace version cross-check", + lambda: resolve_version(repo, tag="v9.9.9", version=None), + ) + expect_error( + "explicit version cross-check", + lambda: resolve_version(repo, tag=None, version="2.0.0"), + ) + for rejected in ("0.1.0+local", "0.1.0.dev1", "0.1.0.post1", "1!0.1.0"): + expect_error( + f"unpublishable explicit version {rejected}", + lambda value=rejected: resolve_version(repo, tag=None, version=value), + ) + if resolve_version(repo, tag=None, version="0.1.0a1") != "0.1.0a1": + raise WheelBuildError("a mapped pre-release version must be accepted") + expect_error( + "both tag and version", + lambda: resolve_version(repo, tag="v0.1.0", version="0.1.0"), + ) + expect_error( + "neither tag nor version", + lambda: resolve_version(repo, tag=None, version=None), + ) + expect_error( + "missing manifest", lambda: workspace_version(root / "does-not-exist") + ) + commented = root / "commented" + commented.mkdir() + (commented / "Cargo.toml").write_text( + '[workspace.package] # release metadata\nversion = "0.1.0"\n', + encoding="utf-8", + ) + if workspace_version(commented) != "0.1.0": + raise WheelBuildError("a commented section header must still be parsed") + + self_test_wheel(root, repo) + finally: + shutil.rmtree(root, ignore_errors=True) + print("wheel builder self-test: ok") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--bin-dir", type=Path, help="Directory holding the built rocm/rocmd binaries." + ) + parser.add_argument("--platform", choices=sorted(PLATFORM_TAGS)) + version_group = parser.add_mutually_exclusive_group() + version_group.add_argument("--tag", help="Release git tag, e.g. v0.1.0-rc.1.") + version_group.add_argument( + "--version", help="Already-mapped PEP 440 version, e.g. 0.1.0rc1." + ) + parser.add_argument("--out-dir", type=Path, default=REPO_ROOT / "dist" / "wheels") + parser.add_argument("--repo-root", type=Path, default=REPO_ROOT) + parser.add_argument( + "--self-test", action="store_true", help="Run offline wheel policy tests." + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + try: + if args.self_test: + with tempfile.TemporaryDirectory(prefix="rocm-wheel-selftest-") as temp: + run_self_test(Path(temp) / "work") + return 0 + if args.bin_dir is None: + raise WheelBuildError("--bin-dir is required") + if args.platform is None: + raise WheelBuildError("--platform is required") + repo_root = args.repo_root.resolve() + version = resolve_version(repo_root, tag=args.tag, version=args.version) + wheel = build_wheel( + repo_root=repo_root, + bin_dir=args.bin_dir.resolve(), + platform=args.platform, + version=version, + out_dir=args.out_dir.resolve(), + ) + print(f"wheel: {wheel}") + return 0 + except (WheelBuildError, OSError) as error: + fail(str(error)) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main())