diff --git a/Cargo.lock b/Cargo.lock index 35b1408ab..8e3ae2951 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -129,6 +129,7 @@ dependencies = [ "clap_complete_nushell", "git2", "glob", + "humansize", "oci-spec", "regex", "reqwest", @@ -741,6 +742,15 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "humansize" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] + [[package]] name = "hyper" version = "1.9.0" @@ -1106,6 +1116,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libssh2-sys" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index a74124a43..278861127 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ clap_complete = "4.5.55" clap_complete_nushell = "4.5.8" git2 = "0.20.1" glob = "0.3.2" +humansize = "2.1.3" oci-spec = "0.9.0" reqwest = { version = "0.13.2", features = ["json"] } rstest = "0.26.1" diff --git a/rust/boil/Cargo.toml b/rust/boil/Cargo.toml index cb9e7597a..3fe22d90c 100644 --- a/rust/boil/Cargo.toml +++ b/rust/boil/Cargo.toml @@ -14,6 +14,7 @@ clap_complete.workspace = true clap_complete_nushell.workspace = true git2.workspace = true glob.workspace = true +humansize.workspace = true oci-spec.workspace = true reqwest.workspace = true regex.workspace = true diff --git a/rust/boil/src/cli/image.rs b/rust/boil/src/cli/image.rs index db208917c..768c2da9b 100644 --- a/rust/boil/src/cli/image.rs +++ b/rust/boil/src/cli/image.rs @@ -1,6 +1,12 @@ use clap::{Args, Subcommand, ValueEnum}; -use crate::{cli::Cli, core::image::ImageSelector}; +use crate::{ + cli::Cli, + core::{ + image::ImageSelector, + platform::{Architecture, TargetPlatform}, + }, +}; #[derive(Debug, Args)] pub struct ImageArguments { @@ -17,6 +23,9 @@ pub enum ImageCommand { /// /// Access tokens must be provided with the following name: `BOIL_REGISTRY_TOKEN_`. Check(ImageCheckArguments), + + /// Calculates the size of images known by boil. + Size(ImageSizeArguments), } #[derive(Debug, Args)] @@ -35,7 +44,23 @@ pub struct ImageCheckArguments { pub image: Vec, // NOTE (@Techassi): Should this maybe be renamed to vendor_version? - /// The image version being built. + /// The image version to check. + #[arg( + short, long, + value_parser = Cli::parse_image_version, + default_value_t = Cli::default_image_version(), + help_heading = "Image Options" + )] + pub image_version: String, +} + +#[derive(Debug, Args)] +pub struct ImageSizeArguments { + /// Optionally specify one or more images to check. Checks all images by default. + pub image: Vec, + + // NOTE (@Techassi): Should this maybe be renamed to vendor_version? + /// The image version to use. #[arg( short, long, value_parser = Cli::parse_image_version, @@ -43,9 +68,31 @@ pub struct ImageCheckArguments { help_heading = "Image Options" )] pub image_version: String, + + /// Target platform of the image. + #[arg( + short, long, + short_alias = 'a', alias = "architecture", + default_value_t = Self::default_architecture(), + help_heading = "Image Options" + )] + pub target_platform: TargetPlatform, + + /// Pretty print the structured output. + #[arg(long, value_enum, default_value_t = Pretty::default(), help_heading = "Output Options")] + pub pretty: Pretty, + + #[arg(short, long, value_enum, default_value_t = Format::default(), help_heading = "Output Options")] + pub format: Format, +} + +impl ImageSizeArguments { + // TODO: Auto-detect this + fn default_architecture() -> TargetPlatform { + TargetPlatform::Linux(Architecture::Amd64) + } } -// #[derive(Clone, Debug, Default, strum::Display, strum::EnumString)] #[derive(Clone, Debug, Default, ValueEnum)] pub enum Pretty { #[default] @@ -53,3 +100,10 @@ pub enum Pretty { Always, Never, } + +#[derive(Clone, Debug, Default, ValueEnum)] +pub enum Format { + #[default] + Plain, + Json, +} diff --git a/rust/boil/src/cmd/image.rs b/rust/boil/src/cmd/image.rs index 6a2b2cc66..9f60feb60 100644 --- a/rust/boil/src/cmd/image.rs +++ b/rust/boil/src/cmd/image.rs @@ -1,14 +1,18 @@ use std::{collections::BTreeMap, io::IsTerminal}; use secrecy::{ExposeSecret, SecretString}; +use serde::Serialize; use snafu::{ResultExt, Snafu}; use crate::{ - cli::{ImageCheckArguments, ImageListArguments, Pretty}, + cli::{Format, ImageCheckArguments, ImageListArguments, ImageSizeArguments, Pretty}, config::Config, core::bakefile::{self, Targets, TargetsOptions}, - models::TagList, - utils::{format_image_index_manifest_tag, format_registry_token_env_var_name}, + models::{Manifest, TagList}, + utils::{ + format_image_index_manifest_tag, format_image_manifest_tag, + format_registry_token_env_var_name, + }, }; #[derive(Debug, Snafu)] @@ -22,8 +26,8 @@ pub enum Error { #[snafu(display("failed to build request client"))] BuildClient { source: reqwest::Error }, - #[snafu(display("failed to send request"))] - SendRequest { source: reqwest::Error }, + #[snafu(display("failed to send request ({url})"))] + SendRequest { source: reqwest::Error, url: String }, #[snafu(display("failed to deserialize response"))] DeserializeResponse { source: reqwest::Error }, @@ -51,7 +55,7 @@ pub fn list_images(arguments: ImageListArguments) -> Result<(), Error> { .context(BuildTargetsSnafu)? }; - let list = targets + let list: BTreeMap> = targets .into_iter() .map(|(image_name, (image_config, _))| { let versions: Vec<_> = image_config @@ -63,7 +67,7 @@ pub fn list_images(arguments: ImageListArguments) -> Result<(), Error> { }) .collect(); - print_to_stdout(list, arguments.pretty) + serialize_to_stdout(list, arguments.pretty) } pub async fn check_images(arguments: ImageCheckArguments, config: Config) -> Result<(), Error> { @@ -109,7 +113,7 @@ pub async fn check_images(arguments: ImageCheckArguments, config: Config) -> Res "https://{registry}/v2/{registry_namespace}/{image_name}/tags/list", registry_namespace = registry_options.namespace, ); - let request = client.get(url); + let request = client.get(&url); let request = match ®istry_token { Some(registry_token) => request.bearer_auth(registry_token.expose_secret()), @@ -119,7 +123,7 @@ pub async fn check_images(arguments: ImageCheckArguments, config: Config) -> Res let tag_list: TagList = request .send() .await - .context(SendRequestSnafu)? + .context(SendRequestSnafu { url })? .json() .await .context(DeserializeResponseSnafu)?; @@ -148,14 +152,134 @@ pub async fn check_images(arguments: ImageCheckArguments, config: Config) -> Res Ok(()) } -fn print_to_stdout(list: BTreeMap>, pretty: Pretty) -> Result<(), Error> { +pub async fn calculate_size(arguments: ImageSizeArguments, config: Config) -> Result<(), Error> { + let targets = if arguments.image.is_empty() { + Targets::all(TargetsOptions { + only_entry: true, + non_recursive: true, + }) + .context(BuildTargetsSnafu)? + } else { + Targets::set( + &arguments.image, + TargetsOptions { + only_entry: true, + non_recursive: true, + }, + ) + .context(BuildTargetsSnafu)? + }; + + let mut registry_tokens = BTreeMap::new(); + let client = reqwest::ClientBuilder::new() + .build() + .context(BuildClientSnafu)?; + + #[derive(Serialize)] + struct SizeResult { + images: BTreeMap, + total: u64, + } + + let mut result = SizeResult { + images: BTreeMap::new(), + total: 0, + }; + + for (image_name, (image_config, _)) in targets { + for (registry, registry_options) in image_config.metadata.registries { + // Add tokens to a map so that we don't need construct the key and retrieve the value + // over and over again. + let registry_token = registry_tokens.entry(registry.clone()).or_insert_with(|| { + let name = format_registry_token_env_var_name(®istry); + std::env::var(name).ok().map(SecretString::from) + }); + + for (image_version, _) in image_config.versions.iter() { + let image_index_manifest_tag = format_image_index_manifest_tag( + image_version, + &config.metadata.vendor_tag_prefix, + &arguments.image_version, + ); + + let manifest_tag = format_image_manifest_tag( + &image_index_manifest_tag, + arguments.target_platform.architecture(), + // Never strip the architecture, because we need to reference the exact manifest + // to be able to calculate the sizes + false, + ); + + let url = format!( + "https://{registry}/v2/{registry_namespace}/{image_name}/manifests/{manifest_tag}", + registry_namespace = registry_options.namespace, + ); + + let request = client.get(&url); + + let request = match ®istry_token { + Some(registry_token) => request.bearer_auth(registry_token.expose_secret()), + None => request, + }; + + let manifest: Manifest = request + .send() + .await + .context(SendRequestSnafu { url })? + .json() + .await + .context(DeserializeResponseSnafu)?; + + let layer_size = manifest.layers.iter().fold(0u64, |acc, e| acc + e.size); + + let size = result.images.entry(image_name.clone()).or_default(); + *size += layer_size; + + result.total += layer_size; + } + } + } + + match arguments.format { + Format::Plain => { + if let Some(max_width) = result + .images + .iter() + .map(|i| i.0.len()) + .max_by(|lhs, rhs| lhs.cmp(rhs)) + { + // let sorted = result.images; + for (image, size) in result.images { + println!( + "{image:max_width$} {size}", + size = humansize::format_size(size, humansize::BINARY) + ); + } + + println!( + "{total_text:max_width$} {total}", + total_text = "Total", + total = humansize::format_size(result.total, humansize::BINARY) + ) + } + + Ok(()) + } + Format::Json => serialize_to_stdout(&result, arguments.pretty), + } +} + +fn serialize_to_stdout(value: T, pretty: Pretty) -> Result<(), Error> +where + T: Serialize, +{ let stdout = std::io::stdout(); match pretty { Pretty::Always | Pretty::Auto if stdout.is_terminal() => { - serde_json::to_writer_pretty(stdout, &list) + serde_json::to_writer_pretty(stdout, &value) } - _ => serde_json::to_writer(stdout, &list), + _ => serde_json::to_writer(stdout, &value), } .context(SerializeListSnafu) } diff --git a/rust/boil/src/main.rs b/rust/boil/src/main.rs index 0f76196f4..b88352307 100644 --- a/rust/boil/src/main.rs +++ b/rust/boil/src/main.rs @@ -75,6 +75,12 @@ async fn main() -> Result<(), Error> { .await .context(ImageSnafu) } + ImageCommand::Size(arguments) => { + let config = Config::from_file(&cli.config_path).context(ReadConfigSnafu)?; + cmd::image::calculate_size(arguments, config) + .await + .context(ImageSnafu) + } }, Command::Images(arguments) => cmd::image::list_images(arguments).context(ImageSnafu), Command::Completions(arguments) => { diff --git a/rust/boil/src/models/mod.rs b/rust/boil/src/models/mod.rs index 54df693bb..033bed021 100644 --- a/rust/boil/src/models/mod.rs +++ b/rust/boil/src/models/mod.rs @@ -6,3 +6,16 @@ pub struct TagList { // pub _name: String, pub tags: Vec, } + +// TODO (@Techassi): We should eventually use the complete, upstream types from oci-spec +/// A partial OCI manifest. +#[derive(Debug, Deserialize)] +pub struct Manifest { + pub layers: Vec, +} + +/// A partial OCI manifest layer. +#[derive(Debug, Deserialize)] +pub struct ManifestLayer { + pub size: u64, +}