diff --git a/Cargo.lock b/Cargo.lock index 3d8f702..0b1955f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -204,6 +204,12 @@ dependencies = [ "syn", ] +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -283,6 +289,12 @@ dependencies = [ "r-efi", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.5.0" @@ -480,6 +492,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -783,6 +805,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -969,6 +1000,47 @@ dependencies = [ "syn", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "tower" version = "0.5.3" @@ -1220,6 +1292,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "workbench" version = "0.1.0" @@ -1268,6 +1349,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "toml", "tracing", "tracing-subscriber", "zingo-consensus", diff --git a/zcash_local_net/CHANGELOG.md b/zcash_local_net/CHANGELOG.md index e830561..9dea7b5 100644 --- a/zcash_local_net/CHANGELOG.md +++ b/zcash_local_net/CHANGELOG.md @@ -7,8 +7,85 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Containerized artifacts.** Every managed process (the zebrad + Validator, the zainod Indexer, the zcash-devtool Wallet) can now run + from a container image via the `docker` or `podman` CLI instead of a + host binary. Launch configs gained a `source: + container::ArtifactSource` field (default: the historical + `TEST_BINARIES_DIR` / `PATH` resolution, so existing callers are + unaffected). Container mode changes *nothing else*: the container + runs in the **foreground** (`run --rm`) so its stdio streams through + the child the harness already manages, joins the **host network + namespace**, and has the harness's temp config/data dirs + bind-mounted at identical paths — readiness-indicator scanning, + launch-log endpoint discovery, port-collision retry, the front + proxies, and Indexer-convergence parsing are shared verbatim with + host processes. Host networking makes container mode Linux-first + (Docker Desktop's VM loopback is not reachable from the host). + Wallet operations in container mode each run as their own one-shot + container. Stopping a containerized process force-removes the + container by name (`zcash-local-net---`). +- **Artifact manifest** (`container::manifest::ArtifactManifest`): a + small TOML file consumers keep with their test setup describing + where each artifact comes from — an `image` (with optional + `entrypoint` and `pull` policy `never`/`if-missing`/`always`) or a + `local` path to a locally built binary (the **escape hatch**; also + available programmatically as `ArtifactSource::local_binary`). + Image references name each binary's actual published image, one per + artifact, and **must be pinned**: digest-pinned (`repo@sha256:…`) + or carrying an explicit non-`latest` tag. Untagged and `:latest` + references are rejected at load time + (`ManifestError::UnpinnedImage`), and preflight reports the digest + each reference resolved to as reproducibility evidence. + `ArtifactManifest::from_env()` loads the file named by + `ZCASH_LOCAL_NET_MANIFEST`; `launch_local_net()` launches the core + stack from it; `zebrad_config()` / `zainod_config()` / + `wallet_source()` seed individual configs for customized launches. +- **Explicit pin bumping** (`container::update`, and `zcash-local-net + update [--manifest ] [artifact…]`): the manifest doubles as + intent and lockfile. Each image artifact may declare a `track` + field — the floating reference the consumer prefers to follow + (`…:latest`, a release channel tag) — while `image` stays the pin + every run uses. Pins never move at run time; the update command is + the only thing that moves them: it resolves each tracked reference + (the `track` field, or the tag `image` was pinned from) against the + registry and rewrites `image` to the digest-pinned result + (`repo:tag@sha256:…`) in canonical formatting, reporting every + bump — a reviewable version-control diff, exactly like a lockfile + update. A `track`-only artifact is deliberately not launchable + until its first update mints the pin + (`ManifestError::UnresolvedTrack`); digest-only images without + `track` have no floating preference and are never touched. +- **Preflight pre-check** (`ArtifactManifest::preflight`): validates a + manifest against the current environment before any launch — runtime + client present and daemon reachable, images present locally (pulled + per their pull policy), host binaries resolvable and executable — + and reports every problem at once instead of as buried launch + failures. +- **`zcash-local-net` CLI** (new binary in this crate): + `zcash-local-net preflight [--manifest ]` runs the pre-check + from a shell (exit 0/1), resolving the manifest from the flag, then + `$ZCASH_LOCAL_NET_MANIFEST`, then `./zcash-local-net.toml`, then the + built-in host-process default; `zcash-local-net template` prints a + starting-point manifest. Intended use: + `zcash-local-net preflight --manifest m.toml && + ZCASH_LOCAL_NET_MANIFEST=m.toml cargo nextest run`. The `toml` + crate is a new dependency (already covered by existing cargo-vet + exemptions for its stack). + ### Changed +- **Breaking** — `ZebradConfig`, `ZainodConfig`, and + `ZcashDevtoolConfig` gained the public `source` field described + above. Callers constructing these configs with struct literals must + add `source: Default::default()` (or use `..Default::default()`); + callers using `Default` and the builder methods are unaffected. +- **Breaking** — `Zebrad::stop` no longer panics when the child was + already reaped (`InvalidInput` from `kill`), matching `Zainod::stop`; + in container mode the child is the runtime client and routinely + exits when its container is removed first. - **Breaking, read this one** — **every published port and address accessor of the core stack now returns a front proxy, not the process's own listener.** A transparent TCP relay (the *front*) diff --git a/zcash_local_net/Cargo.toml b/zcash_local_net/Cargo.toml index 01e9036..8e47cc8 100644 --- a/zcash_local_net/Cargo.toml +++ b/zcash_local_net/Cargo.toml @@ -35,6 +35,7 @@ hex = { workspace = true } tokio = { workspace = true, features = ["io-util", "net", "rt", "time", "process"] } serde = { version = "1.0", features = ["derive"] } sha2 = "0.10" +toml = "0.8" [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net", "io-util"] } diff --git a/zcash_local_net/src/bin/zcash-local-net.rs b/zcash_local_net/src/bin/zcash-local-net.rs new file mode 100644 index 0000000..c9b3359 --- /dev/null +++ b/zcash_local_net/src/bin/zcash-local-net.rs @@ -0,0 +1,246 @@ +//! `zcash-local-net` — the harness's artifact pre-check CLI. +//! +//! Test suites that consume `zcash_local_net` describe where their +//! artifacts (zebrad, zainod, zcash-devtool) come from in a manifest +//! file; this CLI validates that manifest against the current +//! environment *before* the test runner starts, so a missing image or +//! binary is one clear report instead of N launch failures: +//! +//! ```sh +//! zcash-local-net preflight --manifest ci-artifacts.toml \ +//! && ZCASH_LOCAL_NET_MANIFEST=ci-artifacts.toml cargo nextest run +//! ``` +//! +//! Argument parsing is hand-rolled: two subcommands and one flag do +//! not justify a CLI-parser dependency in the library crate. + +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use zcash_local_net::container::manifest::{ + ArtifactManifest, DEFAULT_MANIFEST_FILENAME, MANIFEST_ENV_VAR, +}; + +const USAGE: &str = "\ +zcash-local-net — artifact pre-checks for the zcash_local_net harness + +USAGE: + zcash-local-net preflight [--manifest ] + zcash-local-net update [--manifest ] [validator|indexer|wallet ...] + zcash-local-net template + zcash-local-net help + +COMMANDS: + preflight Validate the artifact manifest against this machine: + container runtime present and reachable, images + present (pulled per their pull policy), host binaries + resolvable and executable. Exits 0 when every check + passes, 1 otherwise. + update Explicitly bump the manifest's image pins: resolve each + artifact's tracked reference (its `track` field, or the + tag its `image` was pinned from) against the registry + and rewrite `image` to the digest-pinned result + (`repo:tag@sha256:…`). Pins never move at run time — + this command is the only thing that moves them. Name + artifacts to bump a subset. The file is rewritten in + canonical formatting; review the diff and commit it. + template Print an example manifest (TOML) to stdout. + help Print this message. + +MANIFEST RESOLUTION: + 1. --manifest + 2. $ZCASH_LOCAL_NET_MANIFEST + 3. ./zcash-local-net.toml, if it exists + 4. none — for `preflight`, the built-in default (every artifact + resolved as a host process via TEST_BINARIES_DIR / PATH); + `update` needs a file to rewrite and exits with an error. +"; + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + match args.first().map(String::as_str) { + Some("preflight") => preflight(&args[1..]), + Some("update") => update(&args[1..]), + Some("template") => { + print!("{}", ArtifactManifest::template_toml()); + ExitCode::SUCCESS + } + Some("help") | Some("--help") | Some("-h") => { + print!("{USAGE}"); + ExitCode::SUCCESS + } + Some(other) => { + eprintln!("unknown command {other:?}\n\n{USAGE}"); + ExitCode::from(2) + } + None => { + eprint!("{USAGE}"); + ExitCode::from(2) + } + } +} + +fn preflight(args: &[String]) -> ExitCode { + let manifest_path = match parse_manifest_flag(args) { + Ok(path) => path, + Err(message) => { + eprintln!("{message}\n\n{USAGE}"); + return ExitCode::from(2); + } + }; + + let (manifest, provenance) = match resolve_manifest(manifest_path) { + Ok(resolved) => resolved, + Err(error) => { + eprintln!("error: {error}"); + return ExitCode::FAILURE; + } + }; + println!("manifest: {provenance}"); + + // The library is async throughout; the CLI hosts a minimal + // current-thread runtime for the one call. + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("the preflight runtime should build"); + let report = runtime.block_on(manifest.preflight()); + print!("{report}"); + if report.passed() { + println!("preflight passed"); + ExitCode::SUCCESS + } else { + println!("preflight FAILED"); + ExitCode::FAILURE + } +} + +fn update(args: &[String]) -> ExitCode { + // `update` shares the --manifest flag but additionally accepts + // artifact names. + let mut manifest_flag: Option = None; + let mut artifacts: Vec<&str> = Vec::new(); + let mut rest = args.iter(); + while let Some(arg) = rest.next() { + if arg == "--manifest" { + match rest.next() { + Some(path) => manifest_flag = Some(PathBuf::from(path)), + None => { + eprintln!("--manifest requires a path\n\n{USAGE}"); + return ExitCode::from(2); + } + } + } else { + artifacts.push(arg.as_str()); + } + } + + let (path, provenance) = match resolve_manifest_path(manifest_flag) { + Some(resolved) => resolved, + None => { + eprintln!( + "error: no manifest file to update — pass --manifest, set \ + ${MANIFEST_ENV_VAR}, or create ./{DEFAULT_MANIFEST_FILENAME}" + ); + return ExitCode::FAILURE; + } + }; + println!("manifest: {provenance}"); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("the update runtime should build"); + match runtime.block_on(zcash_local_net::container::update::update_manifest_file( + &path, &artifacts, + )) { + Ok(bumps) if bumps.is_empty() => { + println!("no artifact has a floating reference to follow; nothing to update"); + ExitCode::SUCCESS + } + Ok(bumps) => { + for bump in &bumps { + if bump.changed() { + let previous = if bump.previous.is_empty() { + "" + } else { + &bump.previous + }; + println!( + "{}: {} resolved; {previous} -> {}", + bump.artifact, bump.tracked, bump.pinned + ); + } else { + println!("{}: {} already current", bump.artifact, bump.tracked); + } + } + ExitCode::SUCCESS + } + Err(error) => { + eprintln!("error: {error}"); + ExitCode::FAILURE + } + } +} + +/// Parse `[--manifest ]`, rejecting anything else. +fn parse_manifest_flag(args: &[String]) -> Result, String> { + match args { + [] => Ok(None), + [flag, path] if flag == "--manifest" => Ok(Some(PathBuf::from(path))), + [flag] if flag == "--manifest" => Err("--manifest requires a path".to_string()), + other => Err(format!("unexpected arguments: {other:?}")), + } +} + +/// The documented resolution order, for commands that need the file +/// itself rather than a loaded manifest. `None` when nothing resolves. +fn resolve_manifest_path(explicit: Option) -> Option<(PathBuf, String)> { + if let Some(path) = explicit { + let provenance = format!("{} (--manifest)", path.display()); + return Some((path, provenance)); + } + if let Some(path) = std::env::var_os(MANIFEST_ENV_VAR) { + let path = PathBuf::from(path); + let provenance = format!("{} (${MANIFEST_ENV_VAR})", path.display()); + return Some((path, provenance)); + } + let cwd_manifest = Path::new(DEFAULT_MANIFEST_FILENAME); + cwd_manifest.exists().then(|| { + ( + cwd_manifest.to_path_buf(), + format!("./{DEFAULT_MANIFEST_FILENAME} (working directory)"), + ) + }) +} + +/// Apply the documented manifest-resolution order, reporting which +/// source won so the user knows what was actually checked. +fn resolve_manifest( + explicit: Option, +) -> Result<(ArtifactManifest, String), Box> { + if let Some(path) = explicit { + let manifest = ArtifactManifest::load(&path)?; + return Ok((manifest, format!("{} (--manifest)", path.display()))); + } + if let Some(path) = std::env::var_os(MANIFEST_ENV_VAR) { + let path = PathBuf::from(path); + let manifest = ArtifactManifest::load(&path)?; + return Ok(( + manifest, + format!("{} (${MANIFEST_ENV_VAR})", path.display()), + )); + } + let cwd_manifest = Path::new(DEFAULT_MANIFEST_FILENAME); + if cwd_manifest.exists() { + let manifest = ArtifactManifest::load(cwd_manifest)?; + return Ok(( + manifest, + format!("./{DEFAULT_MANIFEST_FILENAME} (working directory)"), + )); + } + Ok(( + ArtifactManifest::default(), + "built-in default (host processes via TEST_BINARIES_DIR / PATH)".to_string(), + )) +} diff --git a/zcash_local_net/src/container.rs b/zcash_local_net/src/container.rs new file mode 100644 index 0000000..c0fd5e5 --- /dev/null +++ b/zcash_local_net/src/container.rs @@ -0,0 +1,556 @@ +//! Containerized artifacts: run the managed processes from container +//! images instead of host binaries. +//! +//! Every process this crate manages (the zebrad Validator, the zainod +//! Indexer, the zcash-devtool Wallet) is spawned through one choke +//! point: an [`ArtifactSource`] on its launch config. The default, +//! [`ArtifactSource::HostProcess`], is the classic behavior — resolve +//! the binary via `TEST_BINARIES_DIR` / `PATH` (or an explicit path, +//! the escape hatch for locally built binaries). The alternative, +//! [`ArtifactSource::Container`], runs the same binary out of a +//! container image via the `docker` or `podman` CLI. +//! +//! Container mode deliberately changes *nothing else*: the container +//! is launched in the **foreground** (`run --rm`, no `--detach`), so +//! its stdout/stderr stream through the runtime client — the child +//! process the harness already manages — and every existing mechanism +//! (readiness-indicator scanning, launch-log endpoint discovery, +//! port-collision retry, front proxies, Indexer-convergence parsing) +//! operates on containers exactly as it does on host processes. +//! Containers join the **host network namespace** (`--network host`) +//! and the harness's temporary config/data directories are +//! bind-mounted at identical paths, so the generated config files are +//! valid verbatim inside the container and all loopback wiring (the +//! front proxies, the Indexer→Validator connection) is unchanged. +//! Host networking makes this mode Linux-first: on Docker Desktop +//! (macOS/Windows) loopback listeners inside the VM are not reachable +//! from the host. +//! +//! Consumers describe which artifacts come from where with an +//! [`manifest::ArtifactManifest`] — a small TOML file that can be +//! validated ahead of a test run with [`manifest::ArtifactManifest::preflight`] +//! (also exposed by the `zcash-local-net` CLI as `zcash-local-net +//! preflight`). +//! +//! ## Lifecycle and cleanup +//! +//! A foreground `run --rm` container is removed by the runtime when +//! its process exits. Stopping a managed process in container mode +//! additionally force-removes the container by name (`rm --force`), +//! because killing the *client* process alone would leave the +//! container running. Containers are named +//! `zcash-local-net---`, so stragglers from a killed +//! harness process can be cleaned up with e.g. +//! `docker ps -aq --filter name=zcash-local-net- | xargs docker rm -f`. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::utils::executable_finder::pick_command; + +pub mod manifest; +pub mod preflight; +pub mod update; + +/// The container runtime CLI used to run [`ArtifactSource::Container`] +/// artifacts. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum ContainerRuntime { + /// The `docker` CLI. + Docker, + /// The `podman` CLI. + Podman, +} + +impl ContainerRuntime { + /// The CLI executable name for this runtime. + pub fn cli_name(&self) -> &'static str { + match self { + Self::Docker => "docker", + Self::Podman => "podman", + } + } + + /// Detect an available container runtime by probing `docker + /// --version` then `podman --version`. Returns the first CLI that + /// runs successfully, or `None` when neither is installed. Client + /// presence only — daemon reachability is a preflight check + /// ([`manifest::ArtifactManifest::preflight`]). + pub fn detect() -> Option { + [Self::Docker, Self::Podman] + .into_iter() + .find(|runtime| runtime.client_available()) + } + + /// Whether ` --version` runs and exits successfully. + pub(crate) fn client_available(&self) -> bool { + Command::new(self.cli_name()) + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()) + } + + /// Parse the manifest's `runtime` string. + pub(crate) fn from_manifest_name(name: &str) -> Option { + match name { + "docker" => Some(Self::Docker), + "podman" => Some(Self::Podman), + _ => None, + } + } +} + +impl std::fmt::Display for ContainerRuntime { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.cli_name()) + } +} + +/// When a container image is (re)fetched from its registry. +/// +/// Enforced both at preflight (an explicit `pull` step) and at launch +/// (mapped to the runtime's `run --pull=` flag, so a launch +/// can never silently pull an image the policy forbids). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum PullPolicy { + /// Never pull: the image must already be present locally. The + /// right policy for hermetic CI caches and for images built + /// locally from a source checkout. + Never, + /// Pull only when the image is not present locally (the default). + #[default] + IfMissing, + /// Always pull, refreshing a floating tag like `latest`. + Always, +} + +impl PullPolicy { + /// The value for the runtime's `run --pull=` flag. + fn as_run_flag_value(&self) -> &'static str { + match self { + Self::Never => "never", + Self::IfMissing => "missing", + Self::Always => "always", + } + } + + /// Parse the manifest's `pull` string. + pub(crate) fn from_manifest_name(name: &str) -> Option { + match name { + "never" => Some(Self::Never), + "if-missing" => Some(Self::IfMissing), + "always" => Some(Self::Always), + _ => None, + } + } +} + +/// A container image an artifact runs from. +#[derive(Clone, Debug)] +pub struct ContainerImage { + /// The runtime CLI that runs this image. + pub runtime: ContainerRuntime, + /// Image reference (`repository[:tag][@digest]`), passed verbatim + /// to the runtime — the binary's *actual published image* (e.g. + /// Zebra's official `zfnd/zebra`). Pin it: prefer a digest + /// (`repo@sha256:…`), or at least an explicit non-`latest` tag. + /// Manifest loading enforces this + /// ([`manifest::ManifestError::UnpinnedImage`]); constructing this + /// struct directly bypasses that check, so programmatic callers + /// carry the reproducibility responsibility themselves. + pub image: String, + /// Entrypoint override. `None` uses the artifact's executable name + /// (`zebrad`, `zainod`, `zcash-devtool`), which deliberately + /// bypasses image entrypoint scripts — the harness writes complete + /// config files itself and wants the bare binary. + pub entrypoint: Option, + /// When the image is (re)fetched from its registry. + pub pull: PullPolicy, +} + +/// Where an artifact (a managed binary) comes from. +/// +/// Every launch config carries one of these; the default is the +/// classic host-process resolution, so existing callers are +/// unaffected. +#[derive(Clone, Debug)] +pub enum ArtifactSource { + /// Run the binary as a host process. + /// + /// `binary: None` resolves via the `TEST_BINARIES_DIR` environment + /// variable, falling back to `PATH` — exactly the historical + /// behavior. `binary: Some(path)` runs that path directly: the + /// **escape hatch** for a Validator or Indexer built locally from + /// source (e.g. `Some("…/zebra/target/release/zebrad".into())`). + HostProcess { + /// Explicit path to the binary, or `None` to resolve via + /// `TEST_BINARIES_DIR` / `PATH`. + binary: Option, + }, + /// Run the binary from a container image (foreground, host + /// network, harness dirs bind-mounted — see the module docs). + Container(ContainerImage), +} + +impl Default for ArtifactSource { + /// The historical behavior: resolve the binary via + /// `TEST_BINARIES_DIR` / `PATH`. + fn default() -> Self { + Self::HostProcess { binary: None } + } +} + +impl ArtifactSource { + /// Convenience constructor for the local-build escape hatch. + pub fn local_binary(path: impl Into) -> Self { + Self::HostProcess { + binary: Some(path.into()), + } + } + + /// Whether this source runs from a container image. + pub fn is_container(&self) -> bool { + matches!(self, Self::Container(_)) + } + + /// Mint the uniquely-named container instance a daemon launch in + /// container mode will run as (and be force-removed by name + /// through). `None` in host-process mode, and for the one-shot + /// wallet operations, which need no name. + pub(crate) fn new_instance(&self, executable_name: &str) -> Option { + match self { + Self::HostProcess { .. } => None, + Self::Container(image) => Some(ContainerInstance { + runtime: image.runtime, + name: unique_container_name(executable_name), + }), + } + } + + /// Build the command that spawns this artifact — the single choke + /// point through which every managed process is launched. The + /// caller appends the binary's own arguments afterwards, identical + /// in both modes. + pub(crate) fn command(&self, spec: &LaunchSpec<'_>) -> Command { + match self { + Self::HostProcess { binary: None } => pick_command(spec.executable_name, false), + Self::HostProcess { binary: Some(path) } => Command::new(path), + Self::Container(image) => container_run_command(image, spec), + } + } +} + +/// How an artifact spawn should be shaped, beyond the binary's own +/// arguments. +pub(crate) struct LaunchSpec<'a> { + /// The binary's executable name — the host-mode lookup key and the + /// container-mode default entrypoint. + pub executable_name: &'static str, + /// Container name for daemon launches (minted by + /// [`ArtifactSource::new_instance`]); `None` for one-shot + /// invocations, which rely on `--rm` alone. + pub container_name: Option<&'a str>, + /// Host directories bind-mounted into the container at identical + /// paths, so harness-written config files are valid verbatim + /// inside. Ignored in host mode. + pub mounts: &'a [&'a Path], + /// Keep stdin open (`--interactive`) so the caller can pipe to it + /// (the wallet's `init` receives its mnemonic on stdin). Ignored + /// in host mode — a host `Command` pipes stdin without ceremony. + pub interactive: bool, +} + +impl<'a> LaunchSpec<'a> { + /// A spec with no mounts, no name, and no stdin — version traces + /// and other one-shot probes. + pub(crate) fn bare(executable_name: &'static str) -> Self { + Self { + executable_name, + container_name: None, + mounts: &[], + interactive: false, + } + } +} + +/// Assemble `docker|podman run --rm [--name N] --pull=P --network host +/// [--user uid:gid] [-i] --entrypoint E -v p:p… IMAGE`. +/// +/// `--network host` keeps every loopback assumption of the process +/// harness true inside the container (see the module docs); `--user` +/// runs the containerized binary as the harness's own uid/gid so it +/// can write the bind-mounted temp dirs and the files it creates there +/// are owned by the harness. +fn container_run_command(image: &ContainerImage, spec: &LaunchSpec<'_>) -> Command { + let mut command = Command::new(image.runtime.cli_name()); + command.args(["run", "--rm"]); + if let Some(name) = spec.container_name { + command.args(["--name", name]); + } + command.arg(format!("--pull={}", image.pull.as_run_flag_value())); + command.args(["--network", "host"]); + if let Some((uid, gid)) = current_uid_gid() { + command.arg("--user").arg(format!("{uid}:{gid}")); + } + if spec.interactive { + command.arg("--interactive"); + } + command + .arg("--entrypoint") + .arg(image.entrypoint.as_deref().unwrap_or(spec.executable_name)); + for mount in spec.mounts { + command + .arg("--volume") + .arg(format!("{}:{}", mount.display(), mount.display())); + } + command.arg(&image.image); + command +} + +/// The harness process's `(uid, gid)`, learned from the ownership of a +/// freshly created temp dir (no libc dependency). Cached for the +/// process lifetime. `None` on non-Unix targets or if the probe fails, +/// in which case `--user` is omitted and the image's default user +/// applies. +fn current_uid_gid() -> Option<(u32, u32)> { + static CACHED: OnceLock> = OnceLock::new(); + *CACHED.get_or_init(|| { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt as _; + let probe = tempfile::tempdir().ok()?; + let metadata = std::fs::metadata(probe.path()).ok()?; + Some((metadata.uid(), metadata.gid())) + } + #[cfg(not(unix))] + { + None + } + }) +} + +/// Monotonic counter distinguishing containers launched by one harness +/// process. +static CONTAINER_COUNTER: AtomicU64 = AtomicU64::new(0); + +/// `zcash-local-net---`: unique per launch within a +/// host, greppable for cleanup of stragglers. +fn unique_container_name(executable_name: &str) -> String { + format!( + "zcash-local-net-{executable_name}-{}-{}", + std::process::id(), + CONTAINER_COUNTER.fetch_add(1, Ordering::Relaxed), + ) +} + +/// A named container a daemon launch is running as. Held by the +/// process wrapper so `stop` can force-remove the container — killing +/// the foreground runtime *client* alone would leave the container +/// running. +#[derive(Debug)] +pub(crate) struct ContainerInstance { + runtime: ContainerRuntime, + name: String, +} + +impl ContainerInstance { + /// The container's `--name`. + pub(crate) fn name(&self) -> &str { + &self.name + } + + /// `docker|podman rm --force `, blocking and best-effort. + /// A "no such container" failure is the goal state (the container + /// already exited and `--rm` reaped it) and is only traced at + /// debug level; any other failure is surfaced as a warning — the + /// caller is tearing down and has nothing better to do with the + /// error than report it. + pub(crate) fn force_remove(&self) { + match Command::new(self.runtime.cli_name()) + .args(["rm", "--force", &self.name]) + .output() + { + Ok(output) if output.status.success() => {} + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + if stderr.to_ascii_lowercase().contains("no such container") { + tracing::debug!(container = %self.name, "container already removed"); + } else { + tracing::warn!( + container = %self.name, + %stderr, + "failed to force-remove container" + ); + } + } + Err(io_error) => { + tracing::warn!( + container = %self.name, + %io_error, + "failed to run the container runtime CLI for force-remove" + ); + } + } + } +} + +/// Trace the artifact's version and provenance, in whichever mode the +/// source runs it. The container branch is the source-aware analogue +/// of [`crate::utils::executable_finder::trace_version_and_location`]; +/// failures are traced, not fatal — the subsequent launch produces the +/// authoritative error with full context. +pub(crate) fn trace_version( + source: &ArtifactSource, + executable_name: &'static str, + version_flag: &str, +) { + match source { + ArtifactSource::HostProcess { binary: None } => { + crate::utils::executable_finder::trace_version_and_location( + executable_name, + version_flag, + ); + } + _ => { + let mut command = source.command(&LaunchSpec::bare(executable_name)); + command.arg(version_flag); + match command.output() { + Ok(version) => { + tracing::info!("$ {executable_name} {version_flag} ({source:?})\n {version:?}"); + } + Err(io_error) => tracing::warn!( + %io_error, + "could not probe {executable_name} {version_flag} via {source:?}" + ), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_image(entrypoint: Option<&str>, pull: PullPolicy) -> ContainerImage { + ContainerImage { + runtime: ContainerRuntime::Docker, + image: "example.com/zebra:v6.0.0".to_string(), + entrypoint: entrypoint.map(str::to_string), + pull, + } + } + + fn rendered_args(command: &Command) -> Vec { + command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect() + } + + /// Pins the container-run CLI contract this module emits: flag + /// set, flag order, identical-path volume syntax, entrypoint + /// default. Consumers script cleanup against the `--name` prefix + /// and the harness relies on `--rm` + foreground semantics, so + /// drift here is behavior drift. + #[test] + fn daemon_run_command_shape() { + let mounts: &[&Path] = &[Path::new("/tmp/cfg"), Path::new("/tmp/data")]; + let source = ArtifactSource::Container(test_image(None, PullPolicy::IfMissing)); + let command = source.command(&LaunchSpec { + executable_name: "zebrad", + container_name: Some("zcash-local-net-zebrad-1-0"), + mounts, + interactive: false, + }); + + assert_eq!(command.get_program(), "docker"); + let args = rendered_args(&command); + let mut expected = vec![ + "run".to_string(), + "--rm".to_string(), + "--name".to_string(), + "zcash-local-net-zebrad-1-0".to_string(), + "--pull=missing".to_string(), + "--network".to_string(), + "host".to_string(), + ]; + if let Some((uid, gid)) = current_uid_gid() { + expected.push("--user".to_string()); + expected.push(format!("{uid}:{gid}")); + } + expected.extend( + [ + "--entrypoint", + "zebrad", + "--volume", + "/tmp/cfg:/tmp/cfg", + "--volume", + "/tmp/data:/tmp/data", + "example.com/zebra:v6.0.0", + ] + .map(str::to_string), + ); + assert_eq!(args, expected); + } + + /// One-shot (wallet-op) shape: no `--name`, `--interactive` when + /// stdin will be piped, explicit entrypoint override respected, + /// and `--pull=never` enforced at run time. + #[test] + fn one_shot_interactive_run_command_shape() { + let mounts: &[&Path] = &[Path::new("/tmp/wallet")]; + let source = ArtifactSource::Container(test_image( + Some("/usr/local/bin/zcash-devtool"), + PullPolicy::Never, + )); + let command = source.command(&LaunchSpec { + executable_name: "zcash-devtool", + container_name: None, + mounts, + interactive: true, + }); + + let args = rendered_args(&command); + assert!(!args.contains(&"--name".to_string())); + assert!(args.contains(&"--interactive".to_string())); + assert!(args.contains(&"--pull=never".to_string())); + let entrypoint_index = args + .iter() + .position(|arg| arg == "--entrypoint") + .expect("entrypoint flag present"); + assert_eq!(args[entrypoint_index + 1], "/usr/local/bin/zcash-devtool"); + assert_eq!(args.last().unwrap(), "example.com/zebra:v6.0.0"); + } + + /// The escape hatch: an explicit binary path is run directly, + /// bypassing `TEST_BINARIES_DIR` / `PATH` resolution entirely. + #[test] + fn explicit_local_binary_is_run_directly() { + let source = ArtifactSource::local_binary("/home/dev/zebra/target/release/zebrad"); + let command = source.command(&LaunchSpec::bare("zebrad")); + assert_eq!( + command.get_program(), + "/home/dev/zebra/target/release/zebrad" + ); + assert_eq!(command.get_args().count(), 0); + } + + /// The default source is the historical host-process resolution. + #[test] + fn default_source_is_host_process_resolution() { + assert!(matches!( + ArtifactSource::default(), + ArtifactSource::HostProcess { binary: None } + )); + } + + #[test] + fn container_names_are_unique_and_greppable() { + let a = unique_container_name("zebrad"); + let b = unique_container_name("zebrad"); + assert_ne!(a, b); + assert!(a.starts_with("zcash-local-net-zebrad-")); + } +} diff --git a/zcash_local_net/src/container/manifest.rs b/zcash_local_net/src/container/manifest.rs new file mode 100644 index 0000000..3a3a897 --- /dev/null +++ b/zcash_local_net/src/container/manifest.rs @@ -0,0 +1,823 @@ +//! The artifact manifest: a consumer-authored description of where +//! each managed artifact (Validator, Indexer, Wallet) comes from. +//! +//! Consumers of this library check a manifest file into their test +//! setup, point the `ZCASH_LOCAL_NET_MANIFEST` environment variable +//! (or a `--manifest` CLI flag) at it, and get the same harness +//! behavior everywhere: CI runs pinned container images, while a +//! developer iterating on zebrad or zaino flips one artifact to a +//! `local` path — the escape hatch — without touching the tests. +//! +//! The format is TOML — the configuration language everything else in +//! this ecosystem already speaks (`zebrad.toml`, `zindexer.toml`, +//! `Cargo.toml`), comments included. `zcash-local-net template` +//! prints a starting point. +//! +//! ```toml +//! version = 1 +//! runtime = "docker" +//! +//! [validator] +//! image = "zfnd/zebra:v6.0.0" +//! track = "zfnd/zebra:latest" +//! +//! [indexer] +//! local = "/home/dev/zaino/target/release/zainod" +//! +//! [wallet] +//! ``` +//! +//! ## Intent vs pin: bumping references explicitly +//! +//! The manifest plays both the role of *intent* and of *lockfile* +//! (think `Cargo.toml` and `Cargo.lock` in one file): `track` records +//! the floating reference a consumer prefers to follow (`…:latest`, a +//! release-channel tag, …), while `image` records the pinned +//! reference every run actually uses. Nothing at run time ever +//! consults the registry for "what does the tag mean now" — moving +//! the pin is a deliberate act: +//! +//! ```sh +//! zcash-local-net update --manifest ci-artifacts.toml +//! ``` +//! +//! resolves each artifact's tracked reference against the registry, +//! rewrites `image` to the digest-pinned result +//! (`repo:tag@sha256:…`), and reports every bump — leaving a +//! reviewable diff in version control. Artifacts without `track` are +//! bumped by re-resolving their `image` tag; digest-only images +//! without `track` are left untouched (there is no floating +//! preference to follow). See [`crate::container::update`] for the +//! library entry point. +//! +//! Per artifact (`validator`, `indexer`, `wallet`): +//! +//! - `image`: run from this container image reference — the binary's +//! *actual published image* (e.g. Zebra's official `zfnd/zebra`), +//! one image per artifact. The reference **must be pinned**: either +//! digest-pinned (`repo@sha256:…`, the strongest form — immune to +//! tag reassignment) or carrying an explicit non-`latest` tag. +//! Untagged and `:latest` references are rejected at load time, +//! because a floating reference silently changes what a test run +//! means when the registry moves the tag. Optional companions: +//! `entrypoint` (defaults to the artifact's binary name) and `pull` +//! (`"never"` / `"if-missing"` / `"always"`, default +//! `"if-missing"`). +//! - `local`: run this host binary directly — the escape hatch for +//! locally built artifacts. Mutually exclusive with `image`. +//! - `{}` or omitted: resolve the binary via `TEST_BINARIES_DIR` / +//! `PATH`, the historical default. +//! +//! Top-level `runtime` (`"docker"` / `"podman"`) is optional; when +//! omitted and any artifact names an `image`, the runtime is +//! auto-detected at load time (docker first, then podman). + +use std::path::{Path, PathBuf}; + +use serde::Deserialize; + +use super::{ArtifactSource, ContainerImage, ContainerRuntime, PullPolicy}; +use crate::{ + LocalNet, error::LaunchError, indexer::zainod::Zainod, indexer::zainod::ZainodConfig, + validator::zebrad::Zebrad, validator::zebrad::ZebradConfig, +}; + +/// The manifest file name looked for in the working directory when no +/// explicit path is given (CLI convention). +pub const DEFAULT_MANIFEST_FILENAME: &str = "zcash-local-net.toml"; + +/// Environment variable naming the manifest file test suites should +/// load (see [`ArtifactManifest::from_env`]). +pub const MANIFEST_ENV_VAR: &str = "ZCASH_LOCAL_NET_MANIFEST"; + +/// The only manifest schema version this build understands. +const SUPPORTED_VERSION: u32 = 1; + +/// Errors loading or validating an artifact manifest. +#[derive(thiserror::Error, Debug)] +pub enum ManifestError { + /// The manifest file could not be read. + #[error("could not read manifest {path}: {io_error}")] + Unreadable { + /// Path that was being read. + path: PathBuf, + /// Underlying io::Error description. + io_error: String, + }, + /// The manifest is not valid TOML, or does not match the schema + /// (unknown fields are rejected so typos fail loudly). + #[error("manifest {path:?} did not parse: {detail}")] + Parse { + /// Path of the offending file, or `None` when parsing an + /// in-memory string. + path: Option, + /// serde error description. + detail: String, + }, + /// `version` names a schema this build does not understand. + #[error("unsupported manifest version {found} (this build supports {SUPPORTED_VERSION})")] + UnsupportedVersion { + /// The version the manifest declared. + found: u32, + }, + /// An artifact entry is self-contradictory. + #[error("manifest artifact `{artifact}`: {reason}")] + InvalidArtifact { + /// Which artifact table (`validator` / `indexer` / `wallet`). + artifact: &'static str, + /// What is wrong with it. + reason: String, + }, + /// An artifact has a `track` preference but no pinned `image` yet. + /// The manifest cannot be launched from until the pin is minted — + /// deliberately, by running the update command. + #[error( + "manifest artifact `{artifact}` tracks {track:?} but has no pinned `image` yet; \ + run `zcash-local-net update` to resolve the tracked reference and write the pin" + )] + UnresolvedTrack { + /// Which artifact table (`validator` / `indexer` / `wallet`). + artifact: &'static str, + /// The floating reference the artifact tracks. + track: String, + }, + /// An `image` reference is not pinned (no tag/digest, or the + /// floating `latest` tag). Pinning is mandatory: container + /// artifacts exist to make test runs reproducible, and a floating + /// reference silently changes meaning when the registry moves it. + #[error( + "manifest artifact `{artifact}`: image {image:?} is not pinned ({reason}); \ + pin it with a digest (`repo@sha256:…`) or an explicit non-`latest` tag" + )] + UnpinnedImage { + /// Which artifact table (`validator` / `indexer` / `wallet`). + artifact: &'static str, + /// The offending image reference. + image: String, + /// What makes the reference floating. + reason: &'static str, + }, + /// `runtime` names something other than `docker` or `podman`. + #[error("unknown container runtime {name:?} (expected \"docker\" or \"podman\")")] + UnknownRuntime { + /// The runtime string the manifest declared. + name: String, + }, + /// The manifest names container images but neither `docker` nor + /// `podman` is installed. + #[error( + "manifest names container images but no container runtime was found \ + (probed `docker --version`, then `podman --version`)" + )] + NoContainerRuntime, +} + +/// Serde-facing manifest schema. Unknown fields are rejected so a +/// typo'd key fails at load time instead of silently reverting an +/// artifact to its default source. +/// +/// Also `Serialize`, because `zcash-local-net update` rewrites the +/// manifest file in place; struct fields serialize in declaration +/// order, so this doubles as the manifest's canonical field order. +#[derive(Clone, Debug, Deserialize, serde::Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawManifest { + pub(crate) version: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) runtime: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) validator: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) indexer: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) wallet: Option, +} + +/// Serde-facing artifact entry. +#[derive(Clone, Debug, Default, Deserialize, serde::Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawArtifact { + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) image: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) track: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) local: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) entrypoint: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) pull: Option, +} + +/// A validated artifact manifest: one resolved [`ArtifactSource`] per +/// managed artifact. +/// +/// The `Default` manifest resolves everything as a host process via +/// `TEST_BINARIES_DIR` / `PATH` — byte-for-byte the crate's historical +/// behavior — so `ArtifactManifest::from_env()?.unwrap_or_default()` +/// is a safe drop-in for any test suite. +#[derive(Clone, Debug, Default)] +pub struct ArtifactManifest { + /// Where the Validator (zebrad) comes from. + pub validator: ArtifactSource, + /// Where the Indexer (zainod) comes from. + pub indexer: ArtifactSource, + /// Where the Wallet binary (zcash-devtool) comes from. + pub wallet: ArtifactSource, +} + +impl ArtifactManifest { + /// Load and validate the manifest at `path`. + pub fn load(path: &Path) -> Result { + let text = std::fs::read_to_string(path).map_err(|io_error| ManifestError::Unreadable { + path: path.to_path_buf(), + io_error: io_error.to_string(), + })?; + Self::parse(&text, Some(path)) + } + + /// Parse and validate a manifest from a TOML string. + pub fn from_toml_str(toml_text: &str) -> Result { + Self::parse(toml_text, None) + } + + /// Load the manifest named by the `ZCASH_LOCAL_NET_MANIFEST` + /// environment variable, or `Ok(None)` when the variable is not + /// set. The intended test-suite entry point: + /// + /// ```no_run + /// # async fn f() -> Result<(), Box> { + /// use zcash_local_net::container::manifest::ArtifactManifest; + /// + /// let manifest = ArtifactManifest::from_env()?.unwrap_or_default(); + /// let report = manifest.preflight().await; + /// assert!(report.passed(), "artifact preflight failed:\n{report}"); + /// let net = manifest.launch_local_net().await?; + /// # Ok(()) } + /// ``` + pub fn from_env() -> Result, ManifestError> { + match std::env::var_os(MANIFEST_ENV_VAR) { + Some(path) => Self::load(Path::new(&path)).map(Some), + None => Ok(None), + } + } + + fn parse(toml_text: &str, path: Option<&Path>) -> Result { + Self::from_raw(Self::parse_raw(toml_text, path)?) + } + + /// Parse the serde-facing schema and check the version, without + /// resolving artifacts. `zcash-local-net update` operates on this + /// representation: a manifest whose `track`-only artifacts are not + /// yet launchable must still be readable for updating. + pub(crate) fn parse_raw( + toml_text: &str, + path: Option<&Path>, + ) -> Result { + let raw: RawManifest = toml::from_str(toml_text).map_err(|error| ManifestError::Parse { + path: path.map(Path::to_path_buf), + detail: error.to_string(), + })?; + if raw.version != SUPPORTED_VERSION { + return Err(ManifestError::UnsupportedVersion { found: raw.version }); + } + Ok(raw) + } + + /// Validate the raw schema and resolve every artifact source. + pub(crate) fn from_raw(raw: RawManifest) -> Result { + let declared_runtime = raw + .runtime + .as_deref() + .map(|name| { + ContainerRuntime::from_manifest_name(name).ok_or_else(|| { + ManifestError::UnknownRuntime { + name: name.to_string(), + } + }) + }) + .transpose()?; + + // Resolve the runtime once, lazily: probing for docker/podman + // only happens when some artifact actually names an image. + let needs_runtime = [&raw.validator, &raw.indexer, &raw.wallet] + .into_iter() + .flatten() + .any(|artifact| artifact.image.is_some()); + let runtime = if needs_runtime { + Some(match declared_runtime { + Some(runtime) => runtime, + None => ContainerRuntime::detect().ok_or(ManifestError::NoContainerRuntime)?, + }) + } else { + None + }; + + Ok(Self { + validator: resolve_artifact("validator", raw.validator, runtime)?, + indexer: resolve_artifact("indexer", raw.indexer, runtime)?, + wallet: resolve_artifact("wallet", raw.wallet, runtime)?, + }) + } + + /// A `ZebradConfig` whose artifact source is this manifest's + /// `validator`; every other field keeps its default. + pub fn zebrad_config(&self) -> ZebradConfig { + ZebradConfig { + source: self.validator.clone(), + ..ZebradConfig::default() + } + } + + /// A `ZainodConfig` whose artifact source is this manifest's + /// `indexer`; every other field keeps its default. + pub fn zainod_config(&self) -> ZainodConfig { + ZainodConfig { + source: self.indexer.clone(), + ..ZainodConfig::default() + } + } + + /// The Wallet binary's artifact source, for seeding a wallet + /// config (e.g. `ZcashDevtoolConfig`'s `source` field) inside a + /// `LocalNet::launch_wallet` closure. + pub fn wallet_source(&self) -> ArtifactSource { + self.wallet.clone() + } + + /// Launch the core stack (zebrad Validator + zainod Indexer) with + /// each artifact taken from this manifest and default settings + /// otherwise. For non-default settings, seed configs via + /// [`Self::zebrad_config`] / [`Self::zainod_config`], adjust them, + /// and call [`LocalNet::launch_from_two_configs`] yourself. + /// + /// # Errors + /// Returns `LaunchError` if a process (or its container) fails to + /// launch. Running [`Self::preflight`] first turns most + /// environment problems into precise, pre-launch diagnostics. + pub async fn launch_local_net(&self) -> Result, LaunchError> { + LocalNet::launch_from_two_configs(self.zebrad_config(), self.zainod_config()).await + } + + /// An example manifest, printed by `zcash-local-net template`. + /// Kept parseable by a unit test. + pub fn template_toml() -> &'static str { + r#"# zcash_local_net artifact manifest. +# Validate with `zcash-local-net preflight`; bump image pins with +# `zcash-local-net update`. Point tests at it via ZCASH_LOCAL_NET_MANIFEST. +version = 1 + +# "docker" or "podman"; omit to auto-detect (docker first). +runtime = "docker" + +[validator] +# The binary's actual published image. Must be pinned: a digest +# (`repo@sha256:...`) or an explicit non-`latest` tag. +image = "zfnd/zebra:v6.0.0" +# Optional floating reference to follow; `zcash-local-net update` +# resolves it and rewrites `image` to the digest-pinned result. +track = "zfnd/zebra:latest" +# Optional; defaults to the artifact's binary name ("zebrad"). +entrypoint = "zebrad" +# "never" | "if-missing" (default) | "always" +pull = "if-missing" + +[indexer] +# Escape hatch: a locally built binary runs as a host process. +local = "/home/dev/zaino/target/release/zainod" + +# An empty (or omitted) table resolves the binary via +# TEST_BINARIES_DIR / PATH, the historical default. +[wallet] +"# + } +} + +/// Validate one artifact entry and resolve it to an [`ArtifactSource`]. +fn resolve_artifact( + artifact: &'static str, + raw: Option, + runtime: Option, +) -> Result { + let raw = raw.unwrap_or_default(); + if let Some(track) = &raw.track { + if raw.local.is_some() { + return Err(ManifestError::InvalidArtifact { + artifact, + reason: "`track` only applies to `image` artifacts, not `local` ones".to_string(), + }); + } + if track.contains('@') { + return Err(ManifestError::InvalidArtifact { + artifact, + reason: format!( + "`track` must be a floating reference (a tag to follow), \ + but {track:?} is digest-pinned — there is nothing to follow" + ), + }); + } + if raw.image.is_none() { + return Err(ManifestError::UnresolvedTrack { + artifact, + track: track.clone(), + }); + } + } + match (raw.image, raw.local) { + (Some(_), Some(_)) => Err(ManifestError::InvalidArtifact { + artifact, + reason: "`image` and `local` are mutually exclusive — pick one".to_string(), + }), + (None, local) => { + if raw.entrypoint.is_some() || raw.pull.is_some() { + return Err(ManifestError::InvalidArtifact { + artifact, + reason: "`entrypoint` and `pull` only apply to `image` artifacts".to_string(), + }); + } + Ok(ArtifactSource::HostProcess { binary: local }) + } + (Some(image), None) => { + if let Err(reason) = require_pinned(&image) { + return Err(ManifestError::UnpinnedImage { + artifact, + image, + reason, + }); + } + let pull = raw + .pull + .as_deref() + .map(|name| { + PullPolicy::from_manifest_name(name).ok_or_else(|| { + ManifestError::InvalidArtifact { + artifact, + reason: format!( + "unknown pull policy {name:?} \ + (expected \"never\", \"if-missing\" or \"always\")" + ), + } + }) + }) + .transpose()? + .unwrap_or_default(); + Ok(ArtifactSource::Container(ContainerImage { + // `needs_runtime` in the caller guarantees Some here. + runtime: runtime.expect("runtime resolved when any artifact names an image"), + image, + entrypoint: raw.entrypoint, + pull, + })) + } + } +} + +/// `Ok(())` when `image` is a pinned reference: digest-pinned +/// (`…@sha256:…`) or carrying an explicit tag other than `latest`. +/// `Err` names what makes it floating. +/// +/// The tag is looked for after the last `/`, so a registry port +/// (`registry:5000/zebra`) is not mistaken for a tag. +fn require_pinned(image: &str) -> Result<(), &'static str> { + if image.contains('@') { + // Digest-pinned: immutable by construction. + return Ok(()); + } + let last_component = image.rsplit('/').next().unwrap_or(image); + match last_component.split_once(':') { + None => Err("it has no tag or digest"), + Some((_, "latest")) => Err("`latest` is a floating tag"), + Some((_, "")) => Err("its tag is empty"), + Some((_, _tag)) => Ok(()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_manifest_resolves_everything_to_host_defaults() { + let manifest = ArtifactManifest::from_toml_str("version = 1\n").unwrap(); + for source in [&manifest.validator, &manifest.indexer, &manifest.wallet] { + assert!(matches!( + source, + ArtifactSource::HostProcess { binary: None } + )); + } + } + + #[test] + fn local_escape_hatch_resolves_to_explicit_binary() { + let manifest = ArtifactManifest::from_toml_str( + r#" + version = 1 + + [indexer] + local = "/builds/zainod" + "#, + ) + .unwrap(); + let ArtifactSource::HostProcess { + binary: Some(binary), + } = &manifest.indexer + else { + panic!("expected explicit host binary, got {:?}", manifest.indexer); + }; + assert_eq!(binary, Path::new("/builds/zainod")); + // The untouched artifacts stay on the historical default. + assert!(matches!( + manifest.validator, + ArtifactSource::HostProcess { binary: None } + )); + } + + #[test] + fn image_artifact_carries_entrypoint_and_pull_policy() { + let manifest = ArtifactManifest::from_toml_str( + r#" + version = 1 + runtime = "podman" + + [validator] + image = "example.com/zebra:v6.0.0" + entrypoint = "/usr/bin/zebrad" + pull = "never" + "#, + ) + .unwrap(); + let ArtifactSource::Container(image) = &manifest.validator else { + panic!("expected container source, got {:?}", manifest.validator); + }; + assert_eq!(image.runtime, ContainerRuntime::Podman); + assert_eq!(image.image, "example.com/zebra:v6.0.0"); + assert_eq!(image.entrypoint.as_deref(), Some("/usr/bin/zebrad")); + assert_eq!(image.pull, PullPolicy::Never); + } + + #[test] + fn image_and_local_are_mutually_exclusive() { + let error = ArtifactManifest::from_toml_str( + r#" + version = 1 + runtime = "docker" + + [validator] + image = "zebra" + local = "/builds/zebrad" + "#, + ) + .unwrap_err(); + assert!(matches!( + error, + ManifestError::InvalidArtifact { + artifact: "validator", + .. + } + )); + } + + #[test] + fn image_only_fields_are_rejected_on_host_artifacts() { + let error = ArtifactManifest::from_toml_str( + r#" + version = 1 + + [wallet] + local = "/builds/zcash-devtool" + pull = "always" + "#, + ) + .unwrap_err(); + assert!(matches!( + error, + ManifestError::InvalidArtifact { + artifact: "wallet", + .. + } + )); + } + + #[test] + fn track_is_rejected_on_local_artifacts() { + let error = ArtifactManifest::from_toml_str( + r#" + version = 1 + + [indexer] + local = "/builds/zainod" + track = "zainod:latest" + "#, + ) + .unwrap_err(); + assert!(matches!( + error, + ManifestError::InvalidArtifact { + artifact: "indexer", + .. + } + )); + } + + #[test] + fn digest_pinned_track_is_rejected() { + let error = ArtifactManifest::from_toml_str( + r#" + version = 1 + runtime = "docker" + + [validator] + image = "zfnd/zebra:v6.0.0" + track = "zfnd/zebra@sha256:aaaa" + "#, + ) + .unwrap_err(); + assert!(matches!( + error, + ManifestError::InvalidArtifact { + artifact: "validator", + .. + } + )); + } + + #[test] + fn track_without_image_is_not_launchable() { + let error = ArtifactManifest::from_toml_str( + r#" + version = 1 + runtime = "docker" + + [validator] + track = "zfnd/zebra:latest" + "#, + ) + .unwrap_err(); + assert!(matches!( + error, + ManifestError::UnresolvedTrack { + artifact: "validator", + .. + } + )); + } + + #[test] + fn unknown_keys_fail_loudly() { + let error = ArtifactManifest::from_toml_str( + r#" + version = 1 + + [validatr] + image = "zebra:v1" + "#, + ) + .unwrap_err(); + assert!(matches!(error, ManifestError::Parse { .. }), "{error:?}"); + } + + #[test] + fn unsupported_version_is_rejected() { + let error = ArtifactManifest::from_toml_str("version = 2\n").unwrap_err(); + assert!(matches!( + error, + ManifestError::UnsupportedVersion { found: 2 } + )); + } + + #[test] + fn unknown_runtime_is_rejected() { + let error = ArtifactManifest::from_toml_str( + r#" + version = 1 + runtime = "containerd" + + [validator] + image = "zebra:v1" + "#, + ) + .unwrap_err(); + assert!(matches!(error, ManifestError::UnknownRuntime { name } if name == "containerd")); + } + + #[test] + fn unknown_pull_policy_is_rejected() { + let error = ArtifactManifest::from_toml_str( + r#" + version = 1 + runtime = "docker" + + [validator] + image = "zebra:v1" + pull = "sometimes" + "#, + ) + .unwrap_err(); + assert!(matches!( + error, + ManifestError::InvalidArtifact { + artifact: "validator", + .. + } + )); + } + + /// Unpinned image references must be rejected at load time, with + /// the offending reference and the reason in the error. + #[test] + fn unpinned_image_references_are_rejected() { + for (image, expected_reason_fragment) in [ + ("zebra", "no tag or digest"), + ("zfnd/zebra:latest", "floating tag"), + ("zfnd/zebra:", "tag is empty"), + // A registry port is not a tag. + ("registry:5000/zebra", "no tag or digest"), + ] { + let error = ArtifactManifest::from_toml_str(&format!( + "version = 1\nruntime = \"docker\"\n\n[validator]\nimage = \"{image}\"\n", + )) + .unwrap_err(); + let ManifestError::UnpinnedImage { + artifact: "validator", + image: reported, + reason, + } = &error + else { + panic!("expected UnpinnedImage for {image:?}, got {error:?}"); + }; + assert_eq!(reported, image); + assert!( + reason.contains(expected_reason_fragment), + "{image:?}: reason {reason:?} should mention {expected_reason_fragment:?}" + ); + } + } + + /// Pinned references — explicit tags and digests, with or without + /// a ported registry — must pass. + #[test] + fn pinned_image_references_are_accepted() { + for image in [ + "zfnd/zebra:v6.0.0", + "registry:5000/zebra:v6.0.0", + "zfnd/zebra@sha256:0000000000000000000000000000000000000000000000000000000000000000", + "zfnd/zebra:v6.0.0@sha256:0000000000000000000000000000000000000000000000000000000000000000", + "zln-test-daemon:local", + ] { + let manifest = ArtifactManifest::from_toml_str(&format!( + "version = 1\nruntime = \"docker\"\n\n[validator]\nimage = \"{image}\"\n", + )) + .unwrap_or_else(|error| panic!("{image:?} should be accepted, got {error:?}")); + assert!(manifest.validator.is_container()); + } + } + + /// A manifest with no `image` artifacts must never probe for a + /// container runtime — the manifest (and `Default`) must work on + /// hosts with neither docker nor podman installed. Indirectly + /// pinned here by resolving a local-only manifest successfully + /// regardless of what's installed. + #[test] + fn local_only_manifest_needs_no_runtime() { + let manifest = ArtifactManifest::from_toml_str( + r#" + version = 1 + + [validator] + local = "/builds/zebrad" + + [indexer] + local = "/builds/zainod" + + [wallet] + local = "/builds/zcash-devtool" + "#, + ) + .unwrap(); + assert!(!manifest.validator.is_container()); + } + + /// The template must always parse — it declares `runtime` + /// explicitly, so no docker/podman probe runs and the test is + /// hermetic on hosts without either installed. + #[test] + fn template_parses() { + let manifest = ArtifactManifest::from_toml_str(ArtifactManifest::template_toml()).unwrap(); + assert!(manifest.validator.is_container()); + assert!(matches!( + &manifest.indexer, + ArtifactSource::HostProcess { binary: Some(_) } + )); + } + + #[test] + fn default_manifest_is_all_host_defaults() { + let manifest = ArtifactManifest::default(); + for source in [&manifest.validator, &manifest.indexer, &manifest.wallet] { + assert!(matches!( + source, + ArtifactSource::HostProcess { binary: None } + )); + } + } +} diff --git a/zcash_local_net/src/container/preflight.rs b/zcash_local_net/src/container/preflight.rs new file mode 100644 index 0000000..49631e7 --- /dev/null +++ b/zcash_local_net/src/container/preflight.rs @@ -0,0 +1,441 @@ +//! Preflight: validate a manifest's artifacts against the current +//! environment *before* any launch. +//! +//! A failed container launch surfaces as a `LaunchError` carrying the +//! runtime client's stderr — accurate, but late and buried inside a +//! test failure. Preflight front-loads the same checks into precise, +//! human-readable diagnostics: run it once in test-suite setup (or via +//! `zcash-local-net preflight` before `cargo nextest run`) and every +//! environment problem — missing runtime, unreachable daemon, absent +//! image, unpullable reference, missing or non-executable binary — is +//! reported per artifact, all at once. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use super::manifest::ArtifactManifest; +use super::{ArtifactSource, ContainerImage, ContainerRuntime, PullPolicy}; + +/// One preflight check's outcome. +#[derive(Clone, Debug)] +pub struct PreflightCheck { + /// What was checked, e.g. `validator: image zfnd/zebra:v6.0.0`. + pub subject: String, + /// Whether the check passed. + pub passed: bool, + /// Evidence: where the artifact resolved to, or what exactly is + /// wrong and how to fix it. + pub detail: String, +} + +/// Every check preflight ran, in execution order. +#[derive(Clone, Debug)] +pub struct PreflightReport { + checks: Vec, +} + +impl PreflightReport { + /// Whether every check passed. + pub fn passed(&self) -> bool { + self.checks.iter().all(|check| check.passed) + } + + /// The individual check outcomes. + pub fn checks(&self) -> &[PreflightCheck] { + &self.checks + } +} + +impl std::fmt::Display for PreflightReport { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for check in &self.checks { + let status = if check.passed { "ok " } else { "FAIL" }; + writeln!(f, "[{status}] {}: {}", check.subject, check.detail)?; + } + Ok(()) + } +} + +impl ArtifactManifest { + /// Check every artifact of this manifest against the current + /// environment. Never fails early: the report carries one entry + /// per check so a broken environment is diagnosed in a single + /// pass. See the module docs for the check list. + pub async fn preflight(&self) -> PreflightReport { + let artifacts: [(&str, &'static str, &ArtifactSource); 3] = [ + ("validator", "zebrad", &self.validator), + ("indexer", "zainod", &self.indexer), + ("wallet", "zcash-devtool", &self.wallet), + ]; + + let mut checks = Vec::new(); + + // Runtime checks, once per distinct runtime in use. + let runtimes: BTreeSet<&'static str> = artifacts + .iter() + .filter_map(|(_, _, source)| match source { + ArtifactSource::Container(image) => Some(image.runtime.cli_name()), + ArtifactSource::HostProcess { .. } => None, + }) + .collect(); + for cli_name in runtimes { + let runtime = ContainerRuntime::from_manifest_name(cli_name) + .expect("cli_name round-trips through from_manifest_name"); + checks.push(check_runtime(runtime).await); + } + + for (artifact, executable_name, source) in artifacts { + match source { + ArtifactSource::HostProcess { binary } => { + checks.push(check_host_binary( + artifact, + executable_name, + binary.as_deref(), + )); + } + ArtifactSource::Container(image) => { + checks.push(check_image(artifact, image).await); + } + } + } + + PreflightReport { checks } + } +} + +/// Runtime client present (`--version`) and daemon reachable (`info`). +async fn check_runtime(runtime: ContainerRuntime) -> PreflightCheck { + let subject = format!("runtime: {runtime}"); + let version = tokio::process::Command::new(runtime.cli_name()) + .arg("--version") + .output() + .await; + let version = match version { + Ok(output) if output.status.success() => { + String::from_utf8_lossy(&output.stdout).trim().to_string() + } + Ok(output) => { + return PreflightCheck { + subject, + passed: false, + detail: format!( + "`{runtime} --version` exited {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ), + }; + } + Err(io_error) => { + return PreflightCheck { + subject, + passed: false, + detail: format!( + "`{runtime}` could not be spawned ({io_error}); install it or set \ + the manifest's `runtime` to an installed one" + ), + }; + } + }; + + // `info` exercises the daemon/socket, which `--version` does not. + match tokio::process::Command::new(runtime.cli_name()) + .arg("info") + .output() + .await + { + Ok(output) if output.status.success() => PreflightCheck { + subject, + passed: true, + detail: format!("{version}; daemon reachable"), + }, + Ok(output) => PreflightCheck { + subject, + passed: false, + detail: format!( + "client found ({version}) but `{runtime} info` exited {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ), + }, + Err(io_error) => PreflightCheck { + subject, + passed: false, + detail: format!("client found ({version}) but `{runtime} info` failed: {io_error}"), + }, + } +} + +/// Image present locally, pulled per policy when not. +async fn check_image(artifact: &str, image: &ContainerImage) -> PreflightCheck { + let subject = format!("{artifact}: image {}", image.image); + let cli = image.runtime.cli_name(); + + let present = tokio::process::Command::new(cli) + .args(["image", "inspect", "--format", "{{.Id}}", &image.image]) + .output() + .await + .is_ok_and(|output| output.status.success()); + + let must_pull = match (present, image.pull) { + (true, PullPolicy::Always) => true, + (true, _) => { + return PreflightCheck { + subject, + passed: true, + detail: format!( + "image present locally ({})", + image_identity(cli, &image.image).await + ), + }; + } + (false, PullPolicy::Never) => { + return PreflightCheck { + subject, + passed: false, + detail: format!( + "image not present locally and pull policy is \"never\"; \ + build or load it first (`{cli} pull {}` / `{cli} load`)", + image.image + ), + }; + } + (false, _) => true, + }; + debug_assert!(must_pull); + + match tokio::process::Command::new(cli) + .args(["pull", &image.image]) + .output() + .await + { + Ok(output) if output.status.success() => { + let identity = image_identity(cli, &image.image).await; + PreflightCheck { + subject, + passed: true, + detail: if present { + format!("image refreshed (pull policy \"always\"; {identity})") + } else { + format!("image pulled ({identity})") + }, + } + } + Ok(output) => PreflightCheck { + subject, + passed: false, + detail: format!( + "`{cli} pull {}` exited {}: {}", + image.image, + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ), + }, + Err(io_error) => PreflightCheck { + subject, + passed: false, + detail: format!("`{cli} pull {}` failed to spawn: {io_error}", image.image), + }, + } +} + +/// The image's immutable identity, as reproducibility evidence in the +/// report: the first registry digest (`repo@sha256:…`) when the image +/// came from a registry, else the local image ID (locally built +/// images have no repo digest). Manifests must already *pin* their +/// references; this states what the pin resolved to on this machine — +/// for a tag-pinned reference, the line to compare across machines. +async fn image_identity(cli: &str, image: &str) -> String { + // `{{json .RepoDigests}}` rather than `{{index …}}`: the json + // helper exists in both docker's and podman's template engines and + // does not error on an empty list. + let repo_digest = tokio::process::Command::new(cli) + .args([ + "image", + "inspect", + "--format", + "{{json .RepoDigests}}", + image, + ]) + .output() + .await + .ok() + .filter(|output| output.status.success()) + .and_then(|output| serde_json::from_slice::>(&output.stdout).ok()) + .and_then(|digests| digests.into_iter().next()); + if let Some(digest) = repo_digest { + return digest; + } + tokio::process::Command::new(cli) + .args(["image", "inspect", "--format", "{{.Id}}", image]) + .output() + .await + .ok() + .filter(|output| output.status.success()) + .map(|output| { + format!( + "local image id {}", + String::from_utf8_lossy(&output.stdout).trim() + ) + }) + .unwrap_or_else(|| "identity unavailable".to_string()) +} + +/// Host binary resolvable and executable — explicit path, or the +/// `TEST_BINARIES_DIR` / `PATH` resolution the launch will perform. +fn check_host_binary( + artifact: &str, + executable_name: &'static str, + binary: Option<&Path>, +) -> PreflightCheck { + let subject = format!("{artifact}: host binary {executable_name}"); + match binary { + Some(path) => match executable_file_status(path) { + Ok(()) => PreflightCheck { + subject, + passed: true, + detail: format!("local build at {}", path.display()), + }, + Err(reason) => PreflightCheck { + subject, + passed: false, + detail: format!("{}: {reason}", path.display()), + }, + }, + None => match resolve_via_env(executable_name) { + Some((provenance, path)) => match executable_file_status(&path) { + Ok(()) => PreflightCheck { + subject, + passed: true, + detail: format!("resolved via {provenance} to {}", path.display()), + }, + Err(reason) => PreflightCheck { + subject, + passed: false, + detail: format!( + "resolved via {provenance} to {} but {reason}", + path.display() + ), + }, + }, + None => PreflightCheck { + subject, + passed: false, + detail: format!( + "not found in TEST_BINARIES_DIR or PATH; set TEST_BINARIES_DIR to a \ + directory containing {executable_name}, add it to PATH, or give this \ + artifact an explicit `local` path or an `image` in the manifest" + ), + }, + }, + } +} + +/// Mirror the launch-time lookup: `TEST_BINARIES_DIR` first (see +/// `crate::utils::executable_finder`), then a `PATH` walk. +fn resolve_via_env(executable_name: &str) -> Option<(&'static str, PathBuf)> { + if let Ok(directory) = std::env::var("TEST_BINARIES_DIR") { + let candidate = PathBuf::from(directory).join(executable_name); + if candidate.exists() { + return Some(("TEST_BINARIES_DIR", candidate)); + } + } + let path_var = std::env::var_os("PATH")?; + std::env::split_paths(&path_var) + .map(|dir| dir.join(executable_name)) + .find(|candidate| candidate.is_file()) + .map(|found| ("PATH", found)) +} + +/// `Ok(())` when `path` is an existing, executable regular file; +/// otherwise the human-readable reason it is not. +fn executable_file_status(path: &Path) -> Result<(), String> { + let metadata = match std::fs::metadata(path) { + Ok(metadata) => metadata, + Err(io_error) => return Err(format!("cannot stat ({io_error})")), + }; + if !metadata.is_file() { + return Err("exists but is not a regular file".to_string()); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + if metadata.permissions().mode() & 0o111 == 0 { + return Err("exists but is not executable (chmod +x?)".to_string()); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write as _; + + fn manifest_with_wallet(source: ArtifactSource) -> ArtifactManifest { + ArtifactManifest { + // Point the other two artifacts at concrete files that + // pass, so each test observes exactly one interesting + // check. `std::env::current_exe` is a real executable + // regular file by construction. + validator: ArtifactSource::local_binary(std::env::current_exe().unwrap()), + indexer: ArtifactSource::local_binary(std::env::current_exe().unwrap()), + wallet: source, + } + } + + fn wallet_check(report: &PreflightReport) -> &PreflightCheck { + report + .checks() + .iter() + .find(|check| check.subject.starts_with("wallet:")) + .expect("wallet check present") + } + + #[tokio::test] + async fn missing_explicit_binary_fails_with_the_path() { + let manifest = manifest_with_wallet(ArtifactSource::local_binary( + "/nonexistent/zcash-devtool-build", + )); + let report = manifest.preflight().await; + let check = wallet_check(&report); + assert!(!check.passed); + assert!(check.detail.contains("/nonexistent/zcash-devtool-build")); + assert!(!report.passed()); + } + + #[cfg(unix)] + #[tokio::test] + async fn non_executable_binary_fails_with_remediation() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("zcash-devtool"); + let mut file = std::fs::File::create(&path).unwrap(); + file.write_all(b"#!/bin/sh\n").unwrap(); + // Default create mode is 0o644: not executable. + let manifest = manifest_with_wallet(ArtifactSource::local_binary(&path)); + let report = manifest.preflight().await; + let check = wallet_check(&report); + assert!(!check.passed); + assert!(check.detail.contains("not executable"), "{}", check.detail); + } + + #[tokio::test] + async fn existing_executable_passes() { + let manifest = manifest_with_wallet(ArtifactSource::local_binary( + std::env::current_exe().unwrap(), + )); + let report = manifest.preflight().await; + assert!(report.passed(), "{report}"); + } + + /// The report renders one line per check with a stable pass/fail + /// tag — the CLI's output contract. + #[tokio::test] + async fn report_renders_one_line_per_check() { + let manifest = manifest_with_wallet(ArtifactSource::local_binary("/nonexistent")); + let report = manifest.preflight().await; + let rendered = report.to_string(); + assert_eq!(rendered.lines().count(), report.checks().len()); + assert!(rendered.contains("[ok ]")); + assert!(rendered.contains("[FAIL]")); + } +} diff --git a/zcash_local_net/src/container/update.rs b/zcash_local_net/src/container/update.rs new file mode 100644 index 0000000..1f5ecba --- /dev/null +++ b/zcash_local_net/src/container/update.rs @@ -0,0 +1,472 @@ +//! Explicitly bump a manifest's image pins to what their tracked +//! references currently denote. +//! +//! Run-time code never consults the registry about what a floating +//! tag means — pins only move through this module (or its CLI face, +//! `zcash-local-net update`). For each container artifact the update: +//! +//! 1. determines the **tracked reference** — the artifact's `track` +//! field when set, else its `image` with any digest stripped (i.e. +//! the tag it was pinned from). A digest-only `image` with no +//! `track` carries no floating preference and is left untouched. +//! 2. resolves that reference against the registry (`pull`, then read +//! the repo digest); +//! 3. rewrites `image` to `@` — tag preserved for +//! readability, digest authoritative — and writes the manifest +//! back in its canonical formatting. +//! +//! The result is a reviewable version-control diff per bump, exactly +//! like a lockfile update. + +use std::path::{Path, PathBuf}; + +use super::ContainerRuntime; +use super::manifest::{ArtifactManifest, ManifestError}; + +/// The outcome of updating one container artifact. +#[derive(Clone, Debug)] +pub struct ImageBump { + /// Which artifact (`validator` / `indexer` / `wallet`). + pub artifact: &'static str, + /// The floating reference that was resolved. + pub tracked: String, + /// The pinned `image` before the update. + pub previous: String, + /// The pinned `image` written by the update. + pub pinned: String, +} + +impl ImageBump { + /// Whether the update actually moved the pin. + pub fn changed(&self) -> bool { + self.previous != self.pinned + } +} + +/// Errors from [`update_manifest_file`]. +#[derive(thiserror::Error, Debug)] +pub enum UpdateError { + /// The manifest could not be read, parsed, or — after bumping — + /// no longer validated. + #[error(transparent)] + Manifest(#[from] ManifestError), + /// An `only` filter named something other than + /// `validator` / `indexer` / `wallet`. + #[error("unknown artifact {name:?} (expected \"validator\", \"indexer\" or \"wallet\")")] + UnknownArtifact { + /// The unrecognized artifact name. + name: String, + }, + /// Resolving a tracked reference against the registry failed. + #[error("artifact `{artifact}`: could not resolve {reference:?}: {detail}")] + Resolve { + /// Which artifact was being updated. + artifact: &'static str, + /// The tracked reference that failed to resolve. + reference: String, + /// What went wrong (runtime CLI stderr or spawn failure). + detail: String, + }, + /// Writing the updated manifest back failed. + #[error("could not write updated manifest {path}: {io_error}")] + Write { + /// The manifest path. + path: PathBuf, + /// Underlying io::Error description. + io_error: String, + }, +} + +/// Resolve every (selected) container artifact's tracked reference and +/// rewrite the manifest's `image` pins in place. `only` restricts the +/// update to the named artifacts (`"validator"` / `"indexer"` / +/// `"wallet"`); empty means all. Returns one [`ImageBump`] per +/// artifact that has a floating preference to follow — including +/// unchanged ones, so callers can report "already current". +/// +/// The file is rewritten in the manifest's canonical formatting +/// (stable field order, two-space indent). Host-process artifacts and +/// digest-only images without `track` are untouched and produce no +/// bump entry. +pub async fn update_manifest_file( + path: &Path, + only: &[&str], +) -> Result, UpdateError> { + update_with_resolver(path, only, resolve_via_registry).await +} + +/// [`update_manifest_file`] with the registry interaction injected — +/// the seam the unit tests use. `resolver(runtime, reference)` returns +/// the digest (`sha256:…`) the reference currently denotes. +async fn update_with_resolver( + path: &Path, + only: &[&str], + resolver: F, +) -> Result, UpdateError> +where + F: AsyncFn(ContainerRuntime, &str) -> Result, +{ + const ARTIFACTS: [&str; 3] = ["validator", "indexer", "wallet"]; + for name in only { + if !ARTIFACTS.contains(name) { + return Err(UpdateError::UnknownArtifact { + name: (*name).to_string(), + }); + } + } + let selected = |name: &str| only.is_empty() || only.contains(&name); + + let text = std::fs::read_to_string(path).map_err(|io_error| ManifestError::Unreadable { + path: path.to_path_buf(), + io_error: io_error.to_string(), + })?; + let mut raw = ArtifactManifest::parse_raw(&text, Some(path))?; + + // The update needs a concrete runtime to resolve through, even + // though the artifacts may not all be launchable yet: resolve the + // declared runtime the same way `from_raw` does. + let runtime = match raw.runtime.as_deref() { + Some(name) => ContainerRuntime::from_manifest_name(name).ok_or_else(|| { + ManifestError::UnknownRuntime { + name: name.to_string(), + } + })?, + None => ContainerRuntime::detect().ok_or(ManifestError::NoContainerRuntime)?, + }; + + let mut bumps = Vec::new(); + for (artifact, entry) in [ + ("validator", raw.validator.as_mut()), + ("indexer", raw.indexer.as_mut()), + ("wallet", raw.wallet.as_mut()), + ] { + let Some(entry) = entry else { continue }; + if !selected(artifact) { + continue; + } + let Some(tracked) = tracked_reference(entry.track.as_deref(), entry.image.as_deref()) + else { + continue; + }; + let digest = resolver(runtime, &tracked) + .await + .map_err(|detail| UpdateError::Resolve { + artifact, + reference: tracked.clone(), + detail, + })?; + let pinned = format!("{tracked}@{digest}"); + bumps.push(ImageBump { + artifact, + tracked, + previous: entry.image.clone().unwrap_or_default(), + pinned: pinned.clone(), + }); + entry.image = Some(pinned); + } + + // Self-check: the bumped manifest must be valid and launchable + // before it replaces the file. + ArtifactManifest::from_raw(raw.clone())?; + + let rendered = toml::to_string(&raw).expect("the manifest schema serializes infallibly"); + std::fs::write(path, rendered).map_err(|io_error| UpdateError::Write { + path: path.to_path_buf(), + io_error: io_error.to_string(), + })?; + + Ok(bumps) +} + +/// The floating reference an artifact follows, or `None` when it has +/// no container image or no floating preference (digest-only `image` +/// without `track`). +fn tracked_reference(track: Option<&str>, image: Option<&str>) -> Option { + if let Some(track) = track { + return Some(track.to_string()); + } + let image = image?; + // Strip the digest to recover the tag the image was pinned from. + let base = image.split('@').next().unwrap_or(image); + let last_component = base.rsplit('/').next().unwrap_or(base); + // Digest-only pin (`repo@sha256:…`): no tag, nothing to follow. + last_component.contains(':').then(|| base.to_string()) +} + +/// The live resolver: `pull` the reference (that is the whole point of +/// an update), then read the repo digest it now denotes. Prefers the +/// `RepoDigests` entry whose repository matches the reference; a +/// locally built image with no repo digest cannot be resolved and +/// errs, since a pin that no registry serves is not reproducible +/// elsewhere. +async fn resolve_via_registry( + runtime: ContainerRuntime, + reference: &str, +) -> Result { + let cli = runtime.cli_name(); + let pull = tokio::process::Command::new(cli) + .args(["pull", reference]) + .output() + .await + .map_err(|io_error| format!("`{cli} pull` failed to spawn: {io_error}"))?; + if !pull.status.success() { + return Err(format!( + "`{cli} pull {reference}` exited {}: {}", + pull.status, + String::from_utf8_lossy(&pull.stderr).trim() + )); + } + + let inspect = tokio::process::Command::new(cli) + .args([ + "image", + "inspect", + "--format", + "{{json .RepoDigests}}", + reference, + ]) + .output() + .await + .map_err(|io_error| format!("`{cli} image inspect` failed to spawn: {io_error}"))?; + if !inspect.status.success() { + return Err(format!( + "`{cli} image inspect {reference}` exited {}: {}", + inspect.status, + String::from_utf8_lossy(&inspect.stderr).trim() + )); + } + let digests: Vec = serde_json::from_slice(&inspect.stdout) + .map_err(|error| format!("unparseable RepoDigests output: {error}"))?; + let repository = reference + .split('@') + .next() + .map(|base| { + let last = base.rsplit('/').next().unwrap_or(base); + match last.split_once(':') { + Some((name, _tag)) => &base[..base.len() - last.len() + name.len()], + None => base, + } + }) + .unwrap_or(reference); + digests + .iter() + .find(|entry| entry.split('@').next() == Some(repository)) + .or_else(|| digests.first()) + .and_then(|entry| entry.split('@').nth(1)) + .map(str::to_string) + .ok_or_else(|| { + format!( + "{reference} has no repo digest (locally built image?); \ + a pin no registry serves is not reproducible elsewhere" + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tracked_reference_prefers_track_over_image() { + assert_eq!( + tracked_reference(Some("zfnd/zebra:latest"), Some("zfnd/zebra:v6.0.0")), + Some("zfnd/zebra:latest".to_string()) + ); + } + + #[test] + fn tracked_reference_recovers_the_tag_from_a_pinned_image() { + assert_eq!( + tracked_reference(None, Some("zfnd/zebra:v6.0.0@sha256:aaaa")), + Some("zfnd/zebra:v6.0.0".to_string()) + ); + assert_eq!( + tracked_reference(None, Some("registry:5000/zebra:v1")), + Some("registry:5000/zebra:v1".to_string()) + ); + } + + #[test] + fn digest_only_image_without_track_has_no_floating_preference() { + assert_eq!( + tracked_reference(None, Some("zfnd/zebra@sha256:aaaa")), + None + ); + // A registry port is not a tag. + assert_eq!( + tracked_reference(None, Some("registry:5000/zebra@sha256:aaaa")), + None + ); + assert_eq!(tracked_reference(None, None), None); + } + + const DIGEST: &str = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; + + fn write_manifest(toml_text: &str) -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("zcash-local-net.toml"); + std::fs::write(&path, toml_text).unwrap(); + (dir, path) + } + + /// Stub resolver: every reference resolves to [`DIGEST`], and the + /// references asked about are recorded through the returned log. + fn stub_resolver() -> impl AsyncFn(ContainerRuntime, &str) -> Result + use<> { + async |_runtime, _reference| Ok(DIGEST.to_string()) + } + + #[tokio::test] + async fn update_pins_tracked_and_tagged_artifacts_and_rewrites_the_file() { + let (_dir, path) = write_manifest( + r#" + version = 1 + runtime = "docker" + + [validator] + image = "zfnd/zebra:v6.0.0" + track = "zfnd/zebra:latest" + + [indexer] + image = "example.com/zainod:0.4.3" + + [wallet] + local = "/builds/zcash-devtool" + "#, + ); + let bumps = update_with_resolver(&path, &[], stub_resolver()) + .await + .unwrap(); + + // Two bumps: the tracked validator and the tag-pinned indexer; + // the host-process wallet produces none. + assert_eq!(bumps.len(), 2); + assert_eq!(bumps[0].artifact, "validator"); + assert_eq!(bumps[0].tracked, "zfnd/zebra:latest"); + assert_eq!(bumps[0].pinned, format!("zfnd/zebra:latest@{DIGEST}")); + assert!(bumps[0].changed()); + assert_eq!(bumps[1].artifact, "indexer"); + assert_eq!( + bumps[1].pinned, + format!("example.com/zainod:0.4.3@{DIGEST}") + ); + + // The rewritten file is valid, keeps `track`, and carries the + // new pins. + let text = std::fs::read_to_string(&path).unwrap(); + assert!( + text.contains(&format!(r#"image = "zfnd/zebra:latest@{DIGEST}""#)), + "{text}" + ); + assert!(text.contains(r#"track = "zfnd/zebra:latest""#), "{text}"); + assert!(text.ends_with('\n')); + let reloaded = ArtifactManifest::load(&path).unwrap(); + assert!(reloaded.validator.is_container()); + + // A second update resolving to the same digest is a no-op bump. + let again = update_with_resolver(&path, &[], stub_resolver()) + .await + .unwrap(); + assert!(again.iter().all(|bump| !bump.changed())); + } + + #[tokio::test] + async fn update_mints_the_first_pin_for_a_track_only_artifact() { + let (_dir, path) = write_manifest( + r#" + version = 1 + runtime = "docker" + + [validator] + track = "zfnd/zebra:latest" + "#, + ); + // Not launchable before the update… + assert!(matches!( + ArtifactManifest::load(&path), + Err(ManifestError::UnresolvedTrack { .. }) + )); + // …the update mints the pin… + let bumps = update_with_resolver(&path, &[], stub_resolver()) + .await + .unwrap(); + assert_eq!(bumps.len(), 1); + assert_eq!(bumps[0].previous, ""); + // …and the manifest is launchable afterwards. + assert!( + ArtifactManifest::load(&path) + .unwrap() + .validator + .is_container() + ); + } + + #[tokio::test] + async fn update_respects_the_artifact_filter() { + let (_dir, path) = write_manifest( + r#" + version = 1 + runtime = "docker" + + [validator] + image = "zfnd/zebra:v6.0.0" + + [indexer] + image = "example.com/zainod:0.4.3" + "#, + ); + let bumps = update_with_resolver(&path, &["indexer"], stub_resolver()) + .await + .unwrap(); + assert_eq!(bumps.len(), 1); + assert_eq!(bumps[0].artifact, "indexer"); + let text = std::fs::read_to_string(&path).unwrap(); + // The unselected validator pin is untouched. + assert!(text.contains(r#"image = "zfnd/zebra:v6.0.0""#), "{text}"); + } + + #[tokio::test] + async fn update_rejects_unknown_artifact_filters() { + let (_dir, path) = write_manifest("version = 1\n"); + let error = update_with_resolver(&path, &["validatr"], stub_resolver()) + .await + .unwrap_err(); + assert!(matches!(error, UpdateError::UnknownArtifact { name } if name == "validatr")); + } + + #[tokio::test] + async fn digest_only_pin_without_track_is_left_untouched() { + let manifest = format!( + "version = 1\nruntime = \"docker\"\n\n[validator]\nimage = \"zfnd/zebra@{DIGEST}\"\n", + ); + let (_dir, path) = write_manifest(&manifest); + let bumps = update_with_resolver(&path, &[], stub_resolver()) + .await + .unwrap(); + assert!(bumps.is_empty()); + let text = std::fs::read_to_string(&path).unwrap(); + assert!( + text.contains(&format!(r#"image = "zfnd/zebra@{DIGEST}""#)), + "{text}" + ); + } + + #[tokio::test] + async fn resolver_failure_surfaces_without_touching_the_file() { + let original = + "version = 1\nruntime = \"docker\"\n\n[validator]\nimage = \"zfnd/zebra:v6.0.0\"\n"; + let (_dir, path) = write_manifest(original); + let error = update_with_resolver(&path, &[], async |_runtime, _reference| { + Err("registry unreachable".to_string()) + }) + .await + .unwrap_err(); + assert!(matches!( + error, + UpdateError::Resolve { + artifact: "validator", + .. + } + )); + assert_eq!(std::fs::read_to_string(&path).unwrap(), original); + } +} diff --git a/zcash_local_net/src/indexer/zainod.rs b/zcash_local_net/src/indexer/zainod.rs index b35a659..ca69460 100644 --- a/zcash_local_net/src/indexer/zainod.rs +++ b/zcash_local_net/src/indexer/zainod.rs @@ -8,7 +8,6 @@ use zingo_consensus::NetworkKind; use crate::logs::LogsToDir; use crate::logs::LogsToStdoutAndStderr as _; -use crate::utils::executable_finder::trace_version_and_location; use crate::{ ProcessId, config, error::{IndexerSyncError, LaunchError}, @@ -16,7 +15,6 @@ use crate::{ launch, network::{self}, process::Process, - utils::executable_finder::pick_command, }; /// The stdout marker zainod prints for every block its chain index @@ -133,6 +131,11 @@ pub struct ZainodConfig { /// lifetime — the launch-time listener probe included. `None` /// (the default) is a passthrough front with no observer. pub grpc_front_observer: Option>, + /// Where the zainod binary comes from: host-process resolution + /// (the default), an explicit local build, or a container image. + /// See [`crate::container::ArtifactSource`]; typically seeded from + /// an [`crate::container::manifest::ArtifactManifest`]. + pub source: crate::container::ArtifactSource, } impl Default for ZainodConfig { @@ -143,6 +146,7 @@ impl Default for ZainodConfig { chain_cache: None, network: NetworkKind::Regtest, grpc_front_observer: None, + source: crate::container::ArtifactSource::default(), } } } @@ -160,8 +164,15 @@ impl IndexerConfig for ZainodConfig { /// This struct is used to represent and manage the Zainod process. #[derive(Debug)] pub struct Zainod { - /// Child process handle + /// Child process handle. In container mode this is the foreground + /// runtime client (`docker|podman run`), through which the + /// container's stdio streams. handle: Child, + /// The named container zainod runs as when the artifact source is + /// a container image; `None` in host-process mode. Held so `stop` + /// can force-remove the container — killing the client alone would + /// leave it running. + container: Option, /// gRPC front proxy — the canonical public endpoint of the gRPC /// listener, bound before the process started. grpc_front: crate::front::Front, @@ -291,15 +302,33 @@ impl Zainod { .unwrap(); let executable_name = "zainod"; - trace_version_and_location(executable_name, "--version"); - let mut command = pick_command(executable_name, false); + crate::container::trace_version(&config.source, executable_name, "--version"); + // In container mode, mount the config dir (and the caller's + // chain cache, when one is set) at identical paths so the + // config file just written is valid verbatim inside the + // container. The ephemeral cache dir is deliberately *not* + // mounted: its `TempDir` is deleted when this function + // returns even in host mode, and zainod (re)creates the + // configured path itself — inside the container's own + // filesystem, reaped with the container by `--rm`. + let container = config.source.new_instance(executable_name); + let mut mounts: Vec<&std::path::Path> = vec![config_dir.path()]; + if let Some(cache) = config.chain_cache.as_ref() { + mounts.push(cache.as_path()); + } + let mut command = config.source.command(&crate::container::LaunchSpec { + executable_name, + container_name: container.as_ref().map(|c| c.name()), + mounts: &mounts, + interactive: false, + }); command.args([ "start", "--config", config_file_path.to_str().expect("should be valid UTF-8"), ]); - let handle = launch::spawn_and_wait( + let spawned = launch::spawn_and_wait( ProcessId::Zainod, &mut command, &logs_dir, @@ -308,10 +337,23 @@ impl Zainod { &["Error:"], &[], ) - .await?; + .await; + let handle = match spawned { + Ok(handle) => handle, + Err(error) => { + // A failed container launch may leave the container + // running even though `launch::wait` gave up on it; + // don't leak it past the failed launch. + if let Some(container) = &container { + container.force_remove(); + } + return Err(error); + } + }; let mut zainod = Zainod { handle, + container, grpc_front, raw_grpc_listen_addr: std::net::SocketAddr::from(([127, 0, 0, 1], port)), logs_dir, @@ -397,6 +439,12 @@ impl Process for Zainod { } fn stop(&mut self) { + // Container mode: the handle is only the runtime client; + // killing it would orphan the container, so remove the + // container first (which also ends the client). + if let Some(container) = &self.container { + container.force_remove(); + } match self.handle.kill() { Ok(()) => {} // `kill` returns `InvalidInput` when the child has already diff --git a/zcash_local_net/src/lib.rs b/zcash_local_net/src/lib.rs index 05f84b9..d0163a0 100644 --- a/zcash_local_net/src/lib.rs +++ b/zcash_local_net/src/lib.rs @@ -41,8 +41,33 @@ //! //! See [`crate::LocalNet`]. //! +//! ## Containers and the artifact manifest +//! +//! Every managed process can alternatively run from a **container +//! image** (via the `docker` or `podman` CLI) instead of a host +//! binary. Which artifacts come from where is described by an +//! [artifact manifest](crate::container::manifest::ArtifactManifest) +//! — a small TOML file consumers keep with their test setup — with a +//! `local` **escape hatch** for artifacts built locally from source. +//! The manifest doubles as a pre-check: validate it against the +//! current environment with +//! [`ArtifactManifest::preflight`](crate::container::manifest::ArtifactManifest::preflight), +//! or from a shell with the bundled `zcash-local-net preflight` CLI +//! before invoking the test runner: +//! +//! ```sh +//! zcash-local-net preflight --manifest ci-artifacts.toml \ +//! && ZCASH_LOCAL_NET_MANIFEST=ci-artifacts.toml cargo nextest run +//! ``` +//! +//! See [`crate::container`] for the mechanics (foreground containers +//! on the host network, harness dirs bind-mounted at identical +//! paths — all launch/readiness/proxy machinery is shared with host +//! processes). +//! pub mod config; +pub mod container; pub mod error; pub mod front; pub mod indexer; diff --git a/zcash_local_net/src/validator/zebrad.rs b/zcash_local_net/src/validator/zebrad.rs index 97de967..5e1076d 100644 --- a/zcash_local_net/src/validator/zebrad.rs +++ b/zcash_local_net/src/validator/zebrad.rs @@ -6,7 +6,6 @@ use crate::{ launch, logs::{LogsToDir, LogsToStdoutAndStderr as _}, process::Process, - utils::executable_finder::{pick_command, trace_version_and_location}, validator::{Validator, ValidatorConfig}, }; use zingo_consensus::{ActivationHeights, MinerPool, NetworkType}; @@ -21,8 +20,10 @@ use tempfile::TempDir; /// Zebrad configuration /// -/// Use `zebrad_bin` to specify the binary location. -/// If the binary is in $PATH, `None` can be specified to run "zebrad". +/// Use `source` to say where the zebrad binary comes from: the default +/// resolves via `TEST_BINARIES_DIR` / `PATH`, and the alternatives are +/// an explicit local-build path or a container image (see +/// [`crate::container::ArtifactSource`]). /// /// Each `*_listen_port` field pins the corresponding **raw** zebrad /// listener when `Some(N)`; `None` (the default) lets zebrad bind port @@ -89,6 +90,11 @@ pub struct ZebradConfig { /// probes and the regtest launch-mine. `None` (the default) is a /// passthrough front with no observer. pub rpc_front_observer: Option>, + /// Where the zebrad binary comes from: host-process resolution + /// (the default), an explicit local build, or a container image. + /// See [`crate::container::ArtifactSource`]; typically seeded from + /// an [`crate::container::manifest::ArtifactManifest`]. + pub source: crate::container::ArtifactSource, } impl Default for ZebradConfig { @@ -113,6 +119,7 @@ impl Default for ZebradConfig { ), min_connected_peers: 0, rpc_front_observer: None, + source: crate::container::ArtifactSource::default(), } } } @@ -154,8 +161,15 @@ impl ValidatorConfig for ZebradConfig { /// This struct is used to represent and manage the Zebrad process. #[derive(Debug)] pub struct Zebrad { - /// Child process handle + /// Child process handle. In container mode this is the foreground + /// runtime client (`docker|podman run`), through which the + /// container's stdio streams. handle: Child, + /// The named container zebrad runs as when the artifact source is + /// a container image; `None` in host-process mode. Held so `stop` + /// can force-remove the container — killing the client alone would + /// leave it running. + container: Option, /// JSON-RPC front proxy — the canonical public endpoint of the /// JSON-RPC listener, bound before the process started. rpc_front: crate::front::Front, @@ -461,15 +475,29 @@ impl Zebrad { .unwrap(); let executable_name = "zebrad"; - trace_version_and_location(executable_name, "--version"); - let mut command = pick_command(executable_name, false); + crate::container::trace_version(&config.source, executable_name, "--version"); + // In container mode, mount the harness dirs at identical paths + // so the config file just written is valid verbatim inside the + // container (see `crate::container`). The chain cache is + // mounted too so cached-chain launches can read it. + let container = config.source.new_instance(executable_name); + let mut mounts: Vec<&std::path::Path> = vec![config_dir.path(), data_dir.path()]; + if let Some(cache) = config.chain_cache.as_ref() { + mounts.push(cache.as_path()); + } + let mut command = config.source.command(&crate::container::LaunchSpec { + executable_name, + container_name: container.as_ref().map(|c| c.name()), + mounts: &mounts, + interactive: false, + }); command.args([ "--config", config_file_path.to_str().expect("should be valid UTF-8"), "start", ]); - let mut handle = launch::spawn_and_wait( + let spawned = launch::spawn_and_wait( ProcessId::Zebrad, &mut command, &logs_dir, @@ -507,7 +535,20 @@ impl Zebrad { "warning: some trace filter directives would enable traces that are disabled statically", ], ) - .await?; + .await; + let mut handle = match spawned { + Ok(handle) => handle, + Err(error) => { + // A failed container launch may leave the container + // running even though `launch::wait` gave up on it + // (e.g. an error indicator matched while the process + // survives); don't leak it past the failed launch. + if let Some(container) = &container { + container.force_remove(); + } + return Err(error); + } + }; // Discover where zebrad actually bound each listener. With // unpinned (port 0) listeners the launch log is the only place @@ -519,6 +560,9 @@ impl Zebrad { // endpoint is not an exited process); don't leak it // past the failed launch. A kill error only means the // child already exited, which is the state kill wants. + if let Some(container) = &container { + container.force_remove(); + } let _ = handle.kill(); return Err(error); } @@ -547,6 +591,7 @@ impl Zebrad { let zebrad = Zebrad { handle, + container, rpc_front, raw_listen_addrs, config_dir, @@ -631,7 +676,20 @@ impl Process for Zebrad { } fn stop(&mut self) { - self.handle.kill().expect("zebrad couldn't be killed"); + // Container mode: the handle is only the runtime client; + // killing it would orphan the container, so remove the + // container first (which also ends the client). + if let Some(container) = &self.container { + container.force_remove(); + } + match self.handle.kill() { + Ok(()) => {} + // `kill` returns `InvalidInput` when the child has already + // exited and been reaped — e.g. the runtime client after + // its container was just force-removed. + Err(e) if e.kind() == std::io::ErrorKind::InvalidInput => {} + Err(e) => panic!("zebrad couldn't be killed: {e}"), + } } fn print_all(&self) { diff --git a/zcash_local_net/src/wallet/zcash_devtool.rs b/zcash_local_net/src/wallet/zcash_devtool.rs index b8d7ec7..a579a13 100644 --- a/zcash_local_net/src/wallet/zcash_devtool.rs +++ b/zcash_local_net/src/wallet/zcash_devtool.rs @@ -25,7 +25,6 @@ use crate::{ error::WalletError, indexer::Indexer, logs::LogsToDir, - utils::executable_finder::pick_command, wallet::{ AddressReceiver, GetInfo, ValidatorHeights, Wallet, WalletBalance, WalletConfig, WalletNetwork, @@ -117,6 +116,14 @@ pub struct ZcashDevtoolConfig { /// policy (3 trusted / 10 untrusted confirmations) makes nothing /// spendable on a shallow regtest chain. pub min_confirmations: std::num::NonZeroU32, + /// Where the zcash-devtool binary comes from: host-process + /// resolution (the default), an explicit local build, or a + /// container image — in which case **every wallet operation runs + /// as its own one-shot container** (the wallet has no resident + /// process to containerize). See + /// [`crate::container::ArtifactSource`]; typically seeded from an + /// [`crate::container::manifest::ArtifactManifest::wallet_source`]. + pub source: crate::container::ArtifactSource, } impl ZcashDevtoolConfig { @@ -137,6 +144,7 @@ impl ZcashDevtoolConfig { indexer_port: 0, network, min_confirmations: std::num::NonZeroU32::MIN, + source: crate::container::ArtifactSource::default(), } } @@ -158,6 +166,7 @@ impl ZcashDevtoolConfig { indexer_port: 0, network, min_confirmations: std::num::NonZeroU32::MIN, + source: crate::container::ArtifactSource::default(), } } } @@ -307,7 +316,15 @@ impl ZcashDevtool { args: &[&str], stdin_line: Option<&str>, ) -> Result { - let mut command = pick_command(EXECUTABLE_NAME, false); + // The wallet dir is bind-mounted at an identical path in + // container mode, so `-w` and the identity/heights paths under + // it are valid verbatim inside the one-shot container. + let mut command = self.config.source.command(&crate::container::LaunchSpec { + executable_name: EXECUTABLE_NAME, + container_name: None, + mounts: &[self.wallet_dir.path()], + interactive: stdin_line.is_some(), + }); command .arg("wallet") .arg("-w") @@ -372,7 +389,7 @@ impl Wallet for ZcashDevtool { type Config = ZcashDevtoolConfig; async fn launch(config: Self::Config) -> Result { - crate::utils::executable_finder::trace_version_and_location(EXECUTABLE_NAME, "--help"); + crate::container::trace_version(&config.source, EXECUTABLE_NAME, "--help"); // tempfile failures here are environment errors (no tmpfs // space/permissions), same unwrap policy as the daemon structs. diff --git a/zcash_local_net/tests/containers.rs b/zcash_local_net/tests/containers.rs new file mode 100644 index 0000000..659a9f3 --- /dev/null +++ b/zcash_local_net/tests/containers.rs @@ -0,0 +1,50 @@ +//! Live tests for the containerized artifact path. +//! +//! These launch the real stack from whatever a manifest names — +//! container images and/or local builds — so they are `#[ignore]`d by +//! default: they need a container runtime plus real zebrad/zainod +//! artifacts, which plain `cargo test` environments don't guarantee. +//! Run them explicitly once the environment provides both: +//! +//! ```sh +//! zcash-local-net preflight --manifest ci-artifacts.toml +//! ZCASH_LOCAL_NET_MANIFEST=ci-artifacts.toml \ +//! cargo test -p zcash_local_net --test containers -- --ignored +//! ``` +//! +//! The container *plumbing* itself (foreground `run` semantics, host +//! networking, identical-path mounts, readiness scanning on container +//! logs, front wiring, `rm --force` teardown) is exercised without any +//! zcash artifacts by the unit tests in `src/container.rs` and was +//! validated live against a scratch image; this file pins the missing +//! piece — the real images' behavior — wherever a manifest provides +//! them. + +use zcash_local_net::container::manifest::ArtifactManifest; + +/// The consumer flow, end to end: manifest → preflight (the pre-check) +/// → LocalNet launch → mine to Indexer convergence. Artifact sources +/// come from `ZCASH_LOCAL_NET_MANIFEST`, so one invocation covers +/// all-images, all-local-builds, or any mix (the escape hatch). +#[ignore = "requires ZCASH_LOCAL_NET_MANIFEST plus the artifacts (images or binaries) it names"] +#[tokio::test(flavor = "multi_thread")] +async fn manifest_local_net_launches_and_converges() { + tracing_subscriber::fmt().init(); + + let manifest = ArtifactManifest::from_env() + .expect("manifest should load") + .expect("set ZCASH_LOCAL_NET_MANIFEST to run this test"); + + let report = manifest.preflight().await; + assert!(report.passed(), "artifact preflight failed:\n{report}"); + + let local_net = manifest + .launch_local_net() + .await + .expect("LocalNet should launch from the manifest's artifacts"); + + local_net + .generate_blocks_converged(2) + .await + .expect("mined blocks should reach the Indexer's chain index"); +}