[2/3] Move support bundle collection into a shared crate

Cargo.toml

@@ -161,6 +161,7 @@ members = [
     "sled-storage",
     "sled-storage/zfs-test-harness",
     "sp-sim",
+    "support-bundle-collection",
     "test-utils",
     "trust-quorum",
     "trust-quorum/gfss",
@@ -347,6 +348,7 @@ default-members = [
     "sled-storage",
     "sled-storage/zfs-test-harness",
     "sp-sim",
+    "support-bundle-collection",
     "trust-quorum",
     "trust-quorum/gfss",
     "trust-quorum/protocol",
@@ -823,6 +825,7 @@ strum = { version = "0.27.2", features = [ "derive" ] }
 subprocess = "0.2.9"
 subtle = "2.6.1"
 supports-color = "3.0.2"
+support-bundle-collection = { path = "support-bundle-collection" }
 support-bundle-viewer = "0.1.2"
 swrite = "0.1.0"
 sync-ptr = "0.1.4"

dev-tools/ls-apis/tests/api_dependencies.out

@@ -49,7 +49,7 @@ Management Gateway Service (client: gateway-client)
     consumed by: dpd (dendrite/dpd) via 1 path
     consumed by: lldpd (lldp/lldpd) via 1 path
     consumed by: mgd (maghemite/mgd) via 1 path
-    consumed by: omicron-nexus (omicron/nexus) via 5 paths
+    consumed by: omicron-nexus (omicron/nexus) via 6 paths
     consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path
     consumed by: wicketd (omicron/wicketd) via 3 paths
 
@@ -96,7 +96,7 @@ Repo Depot API (client: repo-depot-client)
     consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path
 
 Sled Agent (client: sled-agent-client)
-    consumed by: omicron-nexus (omicron/nexus) via 8 paths
+    consumed by: omicron-nexus (omicron/nexus) via 9 paths
 
 Wicketd (client: wicketd-client)
 

nexus/Cargo.toml

@@ -159,6 +159,7 @@ raw-cpuid = { workspace = true, features = ["std"] }
 rustls = { workspace = true }
 rustls-pemfile = { workspace = true }
 scim2-rs.workspace = true
+support-bundle-collection.workspace = true
 update-common.workspace = true
 update-engine.workspace = true
 omicron-workspace-hack.workspace = true

nexus/src/app/background/tasks/mod.rs

@@ -49,7 +49,6 @@ pub mod region_snapshot_replacement_step;
 pub mod saga_recovery;
 pub mod service_firewall_rules;
 pub mod session_cleanup;
-pub mod support_bundle;
 pub mod support_bundle_collector;
 pub mod sync_service_zone_nat;
 pub mod sync_switch_configuration;

nexus/src/app/background/tasks/support_bundle_collector.rs

@@ -6,11 +6,8 @@
 
 use crate::app::background::BackgroundTask;
 use anyhow::Context;
-use camino::Utf8DirEntry;
-use camino::Utf8Path;
 use camino_tempfile::Utf8TempDir;
 use camino_tempfile::tempdir_in;
-use camino_tempfile::tempfile_in;
 use futures::FutureExt;
 use futures::future::BoxFuture;
 use internal_dns_resolver::Resolver;
@@ -38,16 +35,14 @@ use slog_error_chain::InlineErrorChain;
 use std::io::Write;
 use std::num::NonZeroU64;
 use std::sync::Arc;
+use support_bundle_collection::BundleCollection;
+use support_bundle_collection::BundleInfo;
+use support_bundle_collection::zip::bundle_to_zipfile;
 use tokio::io::AsyncReadExt;
 use tokio::io::AsyncSeekExt;
 use tokio::io::SeekFrom;
 use tokio_util::sync::CancellationToken;
 use tufaceous_artifact::ArtifactHash;
-use zip::ZipWriter;
-use zip::write::FullFileOptions;
-
-use super::support_bundle::collection::BundleCollection;
-use super::support_bundle::collection::BundleInfo;
 
 /// We use "/var/tmp" to use Nexus' filesystem for temporary storage,
 /// rather than "/tmp", which would keep this collected data in-memory.
@@ -481,7 +476,10 @@ impl SupportBundleCollector {
         dir: Utf8TempDir,
     ) -> anyhow::Result<()> {
         // Create the zipfile as a temporary file
-        let mut zipfile = tokio::fs::File::from_std(bundle_to_zipfile(&dir)?);
+        let mut zipfile = tokio::fs::File::from_std(bundle_to_zipfile(
+            &dir,
+            camino::Utf8Path::new(TEMPDIR),
+        )?);
         let total_len = zipfile.metadata().await?.len();
 
         // Collect the hash locally before we send it over the network
@@ -635,67 +633,6 @@ async fn check_for_cancellation(
     }
 }
 
-// Takes a directory "dir", and zips the contents into a single zipfile.
-fn bundle_to_zipfile(dir: &Utf8TempDir) -> anyhow::Result<std::fs::File> {
-    let tempfile = tempfile_in(TEMPDIR)?;
-    let mut zip = ZipWriter::new(tempfile);
-
-    recursively_add_directory_to_zipfile(&mut zip, dir.path(), dir.path())?;
-
-    Ok(zip.finish()?)
-}
-
-fn recursively_add_directory_to_zipfile(
-    zip: &mut ZipWriter<std::fs::File>,
-    root_path: &Utf8Path,
-    dir_path: &Utf8Path,
-) -> anyhow::Result<()> {
-    // Readdir might return entries in a non-deterministic order.
-    // Let's sort it for the zipfile, to be nice.
-    let mut entries = dir_path
-        .read_dir_utf8()?
-        .filter_map(Result::ok)
-        .collect::<Vec<Utf8DirEntry>>();
-    entries.sort_by(|a, b| a.file_name().cmp(&b.file_name()));
-
-    for entry in &entries {
-        // Remove the "/tmp/..." prefix from the path when we're storing it in the
-        // zipfile.
-        let dst = entry.path().strip_prefix(root_path)?;
-
-        let file_type = entry.file_type()?;
-        if file_type.is_file() {
-            let src = entry.path();
-
-            let zip_time = entry
-                .path()
-                .metadata()
-                .and_then(|m| m.modified())
-                .ok()
-                .and_then(|sys_time| jiff::Zoned::try_from(sys_time).ok())
-                .and_then(|zoned| {
-                    zip::DateTime::try_from(zoned.datetime()).ok()
-                })
-                .unwrap_or_else(zip::DateTime::default);
-
-            let opts = FullFileOptions::default()
-                .last_modified_time(zip_time)
-                .compression_method(zip::CompressionMethod::Deflated)
-                .large_file(true);
-
-            zip.start_file_from_path(dst, opts)?;
-            let mut file = std::fs::File::open(&src)?;
-            std::io::copy(&mut file, zip)?;
-        }
-        if file_type.is_dir() {
-            let opts = FullFileOptions::default();
-            zip.add_directory_from_path(dst, opts)?;
-            recursively_add_directory_to_zipfile(zip, root_path, entry.path())?;
-        }
-    }
-    Ok(())
-}
-
 async fn sha2_hash(file: &mut tokio::fs::File) -> anyhow::Result<ArtifactHash> {
     let mut buf = vec![0u8; 65536];
     let mut ctx = Sha256::new();
@@ -757,7 +694,6 @@ impl BackgroundTask for SupportBundleCollector {
 mod test {
     use super::*;
 
-    use crate::app::background::tasks::support_bundle::perfetto;
     use crate::app::support_bundles::SupportBundleQueryType;
     use http_body_util::BodyExt;
     use nexus_db_model::PhysicalDisk;
@@ -788,6 +724,7 @@ mod test {
     };
     use sled_agent_types::inventory::ZpoolHealth;
     use std::num::NonZeroU64;
+    use support_bundle_collection::perfetto;
     use uuid::Uuid;
 
     type ControlPlaneTestContext =
@@ -2137,28 +2074,4 @@ mod test {
             SupportBundleCollectionStepStatus::Skipped
         );
     }
-
-    // Ensure that we can convert a temporary directory into a zipfile
-    #[test]
-    fn test_zipfile_creation() {
-        let dir = camino_tempfile::tempdir().unwrap();
-
-        std::fs::create_dir_all(dir.path().join("dir-a")).unwrap();
-        std::fs::create_dir_all(dir.path().join("dir-b")).unwrap();
-        std::fs::write(dir.path().join("dir-a").join("file-a"), "some data")
-            .unwrap();
-        std::fs::write(dir.path().join("file-b"), "more data").unwrap();
-
-        let zipfile = bundle_to_zipfile(&dir)
-            .expect("Should have been able to bundle zipfile");
-        let archive = zip::read::ZipArchive::new(zipfile).unwrap();
-
-        // We expect the order to be deterministically alphabetical
-        let mut names = archive.file_names();
-        assert_eq!(names.next(), Some("dir-a/"));
-        assert_eq!(names.next(), Some("dir-a/file-a"));
-        assert_eq!(names.next(), Some("dir-b/"));
-        assert_eq!(names.next(), Some("file-b"));
-        assert_eq!(names.next(), None);
-    }
 }

support-bundle-collection/Cargo.toml

@@ -0,0 +1,47 @@
+[package]
+name = "support-bundle-collection"
+version = "0.1.0"
+edition.workspace = true
+
+[lints]
+workspace = true
+
+[build-dependencies]
+omicron-rpaths.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+base64.workspace = true
+camino.workspace = true
+camino-tempfile.workspace = true
+chrono.workspace = true
+dropshot.workspace = true
+futures.workspace = true
+gateway-client.workspace = true
+gateway-types.workspace = true
+internal-dns-resolver.workspace = true
+internal-dns-types.workspace = true
+jiff.workspace = true
+nexus-db-model.workspace = true
+nexus-db-queries.workspace = true
+nexus-networking.workspace = true
+nexus-reconfigurator-preparation.workspace = true
+nexus-types.workspace = true
+omicron-common.workspace = true
+omicron-uuid-kinds.workspace = true
+parallel-task-set.workspace = true
+# See omicron-rpaths for more about the "pq-sys" dependency.
+pq-sys = "*"
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+sled-agent-client.workspace = true
+slog.workspace = true
+slog-error-chain.workspace = true
+tokio.workspace = true
+tokio-util.workspace = true
+tufaceous-artifact.workspace = true
+uuid.workspace = true
+zip.workspace = true
+
+omicron-workspace-hack.workspace = true

nexus/src/app/background/tasks/support_bundle/mod.rs → support-bundle-collection/build.rs

@@ -2,10 +2,9 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//! Support bundle related types and utilities
-
-mod cache;
-pub mod collection;
-pub mod perfetto;
-mod step;
-mod steps;
+// See omicron-rpaths for documentation.
+// NOTE: This file MUST be kept in sync with the other build.rs files in this
+// repository.
+fn main() {
+    omicron_rpaths::configure_default_omicron_rpaths();
+}

nexus/src/app/background/tasks/support_bundle/cache.rs → support-bundle-collection/src/cache.rs

@@ -7,7 +7,7 @@
 //! This is used to share data which may be used by multiple
 //! otherwise independent steps.
 
-use crate::app::background::tasks::support_bundle::collection::BundleCollection;
+use crate::collection::BundleCollection;
 
 use gateway_client::Client as MgsClient;
 use internal_dns_types::names::ServiceName;

nexus/src/app/background/tasks/support_bundle/collection.rs → support-bundle-collection/src/collection.rs

@@ -14,10 +14,10 @@
 //! bundle state. Those responsibilities live in the outer management
 //! layer (`support_bundle_collector.rs` for Nexus).
 
-use crate::app::background::tasks::support_bundle::cache::Cache;
-use crate::app::background::tasks::support_bundle::perfetto;
-use crate::app::background::tasks::support_bundle::step::CollectionStep;
-use crate::app::background::tasks::support_bundle::steps;
+use crate::cache::Cache;
+use crate::perfetto;
+use crate::step::CollectionStep;
+use crate::steps;
 use nexus_types::support_bundle::BundleDataSelection;
 
 use anyhow::Context;

support-bundle-collection/src/lib.rs

@@ -0,0 +1,26 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! The mechanism layer of support bundle collection.
+//!
+//! This crate provides the data-gathering primitives used to assemble a
+//! support bundle. It is consumed by both the Nexus background task
+//! (which manages bundle state in the `support_bundle` table) and by
+//! `omdb` (which collects bundles ad-hoc, including when Nexus is down).
+//!
+//! See `README.md` in this crate for a developer-oriented overview of
+//! the step framework.
+
+#[macro_use]
+extern crate slog;
+
+mod cache;
+pub mod collection;
+pub mod perfetto;
+mod step;
+mod steps;
+pub mod zip;
+
+pub use collection::BundleCollection;
+pub use collection::BundleInfo;

nexus/src/app/background/tasks/support_bundle/step.rs → support-bundle-collection/src/step.rs

@@ -4,7 +4,7 @@
 
 //! Support bundle collection step execution framework
 
-use crate::app::background::tasks::support_bundle::collection::BundleCollection;
+use crate::collection::BundleCollection;
 
 use camino::Utf8Path;
 use chrono::DateTime;