From 148c15d9f64101fe6644818024264c721080b5cd Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 10 Jun 2026 02:12:30 +0200 Subject: [PATCH 01/13] add support for gvisor as docker engine --- .github/workflows/main.yml | 22 ++++- CHANGELOG.md | 3 + src/cmd/sandbox/mod.rs | 128 +++++++++++++++++++++++---- tests/buildtest/container_cleanup.rs | 17 ++-- tests/buildtest/mod.rs | 34 +++---- tests/buildtest/runner.rs | 10 +-- tests/integration/crates_git.rs | 4 +- tests/integration/purge_caches.rs | 3 +- tests/utils/mod.rs | 17 +++- 9 files changed, 178 insertions(+), 60 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 055934f8..947c4e39 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,11 +40,16 @@ jobs: - "--no-default-features" - "--features tracing" - "--all-features" + docker_runtime: + - "" include: # Integration tests are disabled on Windows as they take *way* too # long to pull the Docker image - os: windows-latest test_flags: --skip buildtest --skip integration --skip run_binary_with_same_name_as_file + - os: ubuntu-22.04 + cargo_flags: "" + docker_runtime: runsc steps: - name: Checkout the source code uses: actions/checkout@main @@ -57,11 +62,26 @@ jobs: - name: Build rustwide run: cargo build --all ${{ matrix.cargo_flags }} + - name: Install gVisor + if: matrix.docker_runtime == 'runsc' + run: | + sudo apt-get update + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg + curl -fsSL https://gvisor.dev/archive.key | sudo gpg --dearmor -o /usr/share/keyrings/gvisor-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/gvisor-archive-keyring.gpg] https://storage.googleapis.com/gvisor/releases release main" | sudo tee /etc/apt/sources.list.d/gvisor.list > /dev/null + sudo apt-get update + sudo apt-get install -y runsc + sudo runsc install + sudo systemctl restart docker + docker run --rm --runtime=runsc hello-world + # Having swap enabled causes problems with the OOM detector, so let's # disable all swapfiles before running the build. - name: Disable swap on Linux run: sudo swapoff -a - if: matrix.os == 'ubuntu-latest' + if: startsWith(matrix.os, 'ubuntu-') - name: Test rustwide run: cargo test --all ${{ matrix.cargo_flags }} -- ${{ matrix.test_flags }} + env: + RUSTWIDE_DOCKER_RUNTIME: ${{ matrix.docker_runtime }} diff --git a/CHANGELOG.md b/CHANGELOG.md index e96fbbe5..383bb444 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +* Added `DockerRuntime`, `ParseDockerRuntimeError`, and + `SandboxBuilder::docker_runtime` for selecting a Docker runtime such as + gVisor's `runsc` for sandbox containers. ## [0.25.1] - 2026-05-22 diff --git a/src/cmd/sandbox/mod.rs b/src/cmd/sandbox/mod.rs index 751cce3d..25077269 100644 --- a/src/cmd/sandbox/mod.rs +++ b/src/cmd/sandbox/mod.rs @@ -16,6 +16,7 @@ use std::{ ops::RangeInclusive, path::{Path, PathBuf}, rc::Rc, + str, time::Duration, }; @@ -162,8 +163,63 @@ pub struct SandboxBuilder { cpu_limit: Option, cpuset_cpus: Option>, enable_networking: bool, + docker_runtime: DockerRuntime, } +/// The Docker runtime used for sandbox containers. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum DockerRuntime { + /// Let Docker use the daemon's configured default runtime. + #[default] + Default, + /// Use gVisor's `runsc` runtime. + Runsc, +} + +impl DockerRuntime { + /// name of the runtime for the `--runtime` docker arg. + fn docker_name(self) -> Option<&'static str> { + match self { + Self::Default => None, + Self::Runsc => Some("runsc"), + } + } +} + +impl fmt::Display for DockerRuntime { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::Default => "default".fmt(f), + Self::Runsc => "runsc".fmt(f), + } + } +} + +impl str::FromStr for DockerRuntime { + type Err = ParseDockerRuntimeError; + + fn from_str(value: &str) -> Result { + match value { + "" | "default" => Ok(Self::Default), + "runsc" => Ok(Self::Runsc), + _ => Err(ParseDockerRuntimeError), + } + } +} + +/// Error returned when parsing a Docker runtime name fails. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ParseDockerRuntimeError; + +impl fmt::Display for ParseDockerRuntimeError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + "unsupported Docker runtime".fmt(f) + } +} + +impl std::error::Error for ParseDockerRuntimeError {} + /// Statistics collected for a sandbox. #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct SandboxStatistics { @@ -281,6 +337,7 @@ impl SandboxBuilder { cpu_limit: None, cpuset_cpus: None, enable_networking: true, + docker_runtime: DockerRuntime::default(), } } @@ -351,6 +408,24 @@ impl SandboxBuilder { self } + /// Use a specific Docker runtime for the sandbox container. + /// + /// [`DockerRuntime::Runsc`] maps to Docker's `--runtime runsc` flag. By + /// default no runtime is passed, so Docker uses the daemon's configured + /// default runtime. + pub fn docker_runtime(mut self, runtime: DockerRuntime) -> Self { + self.docker_runtime = runtime; + self + } + + /// Use Docker's configured default runtime for the sandbox container. + /// + /// This clears any runtime selected with [`SandboxBuilder::docker_runtime`]. + pub fn default_docker_runtime(mut self) -> Self { + self.docker_runtime = DockerRuntime::Default; + self + } + /// Start a live sandbox from this configuration. /// /// The returned sandbox can be used to run one or more commands against a @@ -411,6 +486,7 @@ impl SandboxBuilder { cpu_limit = ?self.cpu_limit, cpuset_cpus = ?self.cpuset_cpus, enable_networking = self.enable_networking, + docker_runtime = ?self.docker_runtime, ) ) )] @@ -457,6 +533,11 @@ impl SandboxBuilder { args.push("--isolation=process".into()); } + if let Some(runtime) = self.docker_runtime.docker_name() { + args.push("--runtime".into()); + args.push(runtime.into()); + } + args.push(workspace.sandbox_image().name.clone()); // Use an idle command; the real command runs via `docker exec` so the container stays @@ -761,6 +842,7 @@ impl<'w> Sandbox<'w> { cpu_limit = ?self.builder.cpu_limit, cpuset_cpus = ?self.builder.cpuset_cpus, enable_networking = self.builder.enable_networking, + docker_runtime = ?self.builder.docker_runtime, capture, timeout_secs = ?timeout.map(|timeout| timeout.as_secs()), no_output_timeout_secs = ?no_output_timeout.map(|timeout| timeout.as_secs()), @@ -856,11 +938,36 @@ mod tests { const USER_AGENT: &str = "rustwide-tests (https://github.com/rust-lang/rustwide)"; + fn sandbox_builder() -> SandboxBuilder { + let builder = SandboxBuilder::new().enable_networking(false); + let Ok(runtime) = env::var("RUSTWIDE_DOCKER_RUNTIME") else { + return builder; + }; + builder.docker_runtime(runtime.parse().expect("invalid RUSTWIDE_DOCKER_RUNTIME")) + } + #[test] fn formats_cpuset_cpus() { assert_eq!(format_cpuset_cpus(&(2..=4)), "2-4"); } + #[test_case("", Ok(DockerRuntime::Default))] + #[test_case("default", Ok(DockerRuntime::Default))] + #[test_case("runsc", Ok(DockerRuntime::Runsc))] + #[test_case("runc", Err(ParseDockerRuntimeError))] + fn parses_docker_runtime_values( + value: &str, + expected: Result, + ) { + assert_eq!(value.parse(), expected); + } + + #[test_case(DockerRuntime::Default, None)] + #[test_case(DockerRuntime::Runsc, Some("runsc"))] + fn renders_docker_runtime_names(runtime: DockerRuntime, expected: Option<&str>) { + assert_eq!(runtime.docker_name(), expected); + } + const fn stats(peak: Option) -> SandboxStatistics { SandboxStatistics { memory_peak: peak } } @@ -918,11 +1025,8 @@ mod tests { let workspace = init_test_workspace("build-unit")?; let source_dir = tempdir()?; let target_dir = tempdir()?; - let mut sandbox = SandboxBuilder::new().enable_networking(false).start( - &workspace, - source_dir.path(), - target_dir.path(), - )?; + let mut sandbox = + sandbox_builder().start(&workspace, source_dir.path(), target_dir.path())?; let host_cgroup = sandbox .detect_host_cgroup() .expect("sandbox should resolve host cgroup files"); @@ -939,11 +1043,8 @@ mod tests { let workspace = init_test_workspace("build-unit")?; let source_dir = tempdir()?; let target_dir = tempdir()?; - let mut sandbox = SandboxBuilder::new().enable_networking(false).start( - &workspace, - source_dir.path(), - target_dir.path(), - )?; + let mut sandbox = + sandbox_builder().start(&workspace, source_dir.path(), target_dir.path())?; let host_cgroup = sandbox .detect_host_cgroup() .expect("sandbox should resolve host cgroup files"); @@ -978,11 +1079,8 @@ mod tests { let workspace = init_test_workspace("build-unit")?; let source_dir = tempdir()?; let target_dir = tempdir()?; - let mut sandbox = SandboxBuilder::new().enable_networking(false).start( - &workspace, - source_dir.path(), - target_dir.path(), - )?; + let mut sandbox = + sandbox_builder().start(&workspace, source_dir.path(), target_dir.path())?; let host_cgroup = sandbox .detect_host_cgroup() .expect("sandbox should resolve host cgroup files"); diff --git a/tests/buildtest/container_cleanup.rs b/tests/buildtest/container_cleanup.rs index 14267f48..a91f5443 100644 --- a/tests/buildtest/container_cleanup.rs +++ b/tests/buildtest/container_cleanup.rs @@ -1,10 +1,9 @@ -use rustwide::cmd::SandboxBuilder; use std::time::Duration; #[test] fn test_container_cleanup_on_success() { super::runner::run("hello-world", |run| { - let container_id = run.run(SandboxBuilder::new().enable_networking(false), |build| { + let container_id = run.run(crate::utils::sandbox_builder(), |build| { // Verify we are running inside a Docker container let dockerenv = build.cmd("test").args(["-f", "/.dockerenv"]).run_capture(); assert!( @@ -29,7 +28,7 @@ fn test_container_cleanup_on_success() { #[test] fn test_container_reused_across_commands() { super::runner::run("hello-world", |run| { - let container_ids = run.run(SandboxBuilder::new().enable_networking(false), |build| { + let container_ids = run.run(crate::utils::sandbox_builder(), |build| { let first = build.cmd("cat").args(["/etc/hostname"]).run_capture()?; let second = build.cmd("cat").args(["/etc/hostname"]).run_capture()?; @@ -55,7 +54,7 @@ fn test_container_reused_across_commands() { #[cfg(not(windows))] fn test_container_recreated_when_previous_dies() { super::runner::run("hello-world", |run| { - let container_ids = run.run(SandboxBuilder::new().enable_networking(false), |build| { + let container_ids = run.run(crate::utils::sandbox_builder(), |build| { // Capture the original container's short ID via /etc/hostname. let first = build.cmd("cat").args(["/etc/hostname"]).run_capture()?; let first_id = first.stdout_lines()[0].trim().to_string(); @@ -103,9 +102,7 @@ fn test_reused_container_oom_does_not_poison_later_commands() { super::runner::run("allocate", |run| { run.run( - SandboxBuilder::new() - .enable_networking(false) - .memory_limit(Some(512 * 1024 * 1024)), + crate::utils::sandbox_builder().memory_limit(Some(512 * 1024 * 1024)), |build| { let first = build.cargo().args(["run", "--", "1024"]).run(); assert!( @@ -127,7 +124,7 @@ fn test_reused_container_timeout_recreates_container() { use rustwide::cmd::CommandError; super::runner::run("hello-world", |run| { - let container_ids = run.run(SandboxBuilder::new().enable_networking(false), |build| { + let container_ids = run.run(crate::utils::sandbox_builder(), |build| { let first = build.cmd("cat").args(["/etc/hostname"]).run_capture()?; let first_id = first.stdout_lines()[0].trim().to_string(); @@ -177,7 +174,7 @@ fn test_reused_container_no_output_timeout_recreates_container() { use rustwide::cmd::CommandError; super::runner::run("hello-world", |run| { - let container_ids = run.run(SandboxBuilder::new().enable_networking(false), |build| { + let container_ids = run.run(crate::utils::sandbox_builder(), |build| { let first = build.cmd("cat").args(["/etc/hostname"]).run_capture()?; let first_id = first.stdout_lines()[0].trim().to_string(); @@ -222,7 +219,7 @@ fn test_reused_container_no_output_timeout_recreates_container() { #[test] fn test_container_cleanup_on_command_failure() { super::runner::run("hello-world", |run| { - let container_id = run.run(SandboxBuilder::new().enable_networking(false), |build| { + let container_id = run.run(crate::utils::sandbox_builder(), |build| { // Verify we are running inside a Docker container let dockerenv = build.cmd("test").args(["-f", "/.dockerenv"]).run_capture(); assert!( diff --git a/tests/buildtest/mod.rs b/tests/buildtest/mod.rs index 4ba998e7..8bdc7411 100644 --- a/tests/buildtest/mod.rs +++ b/tests/buildtest/mod.rs @@ -1,5 +1,5 @@ use log::LevelFilter; -use rustwide::cmd::{ProcessLinesActions, SandboxBuilder}; +use rustwide::cmd::ProcessLinesActions; mod container_cleanup; #[macro_use] @@ -54,7 +54,7 @@ fn buildtest_crate_name_matches_folder_name() { #[test] fn test_hello_world() { runner::run("hello-world", |run| { - run.run(SandboxBuilder::new().enable_networking(false), |build| { + run.run(crate::utils::sandbox_builder(), |build| { let storage = rustwide::logging::LogStorage::new(LevelFilter::Info); rustwide::logging::capture(&storage, || -> anyhow::Result<_> { build.cargo().args(["run"]).run()?; @@ -82,7 +82,7 @@ fn test_fetch_build_std() { let target = std::fs::read_to_string(target_file).unwrap(); runner::run("hello-world", |run| { - run.run(SandboxBuilder::new().enable_networking(false), |build| { + run.run(crate::utils::sandbox_builder(), |build| { build.fetch_build_std_dependencies(&[target.as_str()])?; let storage = rustwide::logging::LogStorage::new(LevelFilter::Info); rustwide::logging::capture(&storage, || -> anyhow::Result<_> { @@ -109,7 +109,7 @@ fn test_fetch_build_std() { #[test] fn path_based_patch() { runner::run("path-based-patch", |run| { - run.build(SandboxBuilder::new().enable_networking(false), |builder| { + run.build(crate::utils::sandbox_builder(), |builder| { builder .patch_with_path("empty-library", "./patch") .run(move |build| { @@ -135,7 +135,7 @@ fn path_based_patch() { #[test] fn test_process_lines() { runner::run("process-lines", |run| { - run.run(SandboxBuilder::new().enable_networking(false), |build| { + run.run(crate::utils::sandbox_builder(), |build| { let storage = rustwide::logging::LogStorage::new(LevelFilter::Info); let mut ex = false; let mut saw_reentrant_error = false; @@ -181,9 +181,7 @@ fn test_process_lines() { fn test_memory_peak() { runner::run("allocate", |run| { run.run( - SandboxBuilder::new() - .enable_networking(false) - .memory_limit(Some(512 * 1024 * 1024)), + crate::utils::sandbox_builder().memory_limit(Some(512 * 1024 * 1024)), |build| { build.cargo().args(["run", "--", "400"]).run_capture()?; Ok(()) @@ -210,9 +208,7 @@ fn test_sandbox_oom() { runner::run("allocate", |run| { let res = run.run( - SandboxBuilder::new() - .enable_networking(false) - .memory_limit(Some(512 * 1024 * 1024)), + crate::utils::sandbox_builder().memory_limit(Some(512 * 1024 * 1024)), |build| { build.cargo().args(["run", "--", "1024"]).run()?; Ok(()) @@ -234,9 +230,7 @@ fn test_invalid_cpuset_cpus() { runner::run("hello-world", |run| { let res = run.run( - SandboxBuilder::new() - .enable_networking(false) - .cpuset_cpus(Some(999_999..=999_999)), + crate::utils::sandbox_builder().cpuset_cpus(Some(999_999..=999_999)), |build| { build.cmd("true").run()?; Ok(()) @@ -261,9 +255,7 @@ fn test_invalid_cpuset_cpus() { fn test_cpuset_cpus_applied() { runner::run("hello-world", |run| { run.run( - SandboxBuilder::new() - .enable_networking(false) - .cpuset_cpus(Some(0..=1)), + crate::utils::sandbox_builder().cpuset_cpus(Some(0..=1)), |build| { let output = build .cmd("sh") @@ -282,7 +274,7 @@ fn test_cpuset_cpus_applied() { #[cfg(not(windows))] fn test_sandbox_current_directory_inside_source() { runner::run("hello-world", |run| { - run.run(SandboxBuilder::new().enable_networking(false), |build| { + run.run(crate::utils::sandbox_builder(), |build| { let output = build .cmd("pwd") .current_directory(build.host_source_dir().join("src")) @@ -299,7 +291,7 @@ fn test_sandbox_current_directory_inside_source() { #[should_panic(expected = "explicit workdir must be inside the sandbox source directory")] fn test_sandbox_current_directory_outside_source_panics() { runner::run("hello-world", |run| { - run.run(SandboxBuilder::new().enable_networking(false), |build| { + run.run(crate::utils::sandbox_builder(), |build| { build.cmd("pwd").current_directory("/tmp").run()?; Ok(()) })?; @@ -310,7 +302,7 @@ fn test_sandbox_current_directory_outside_source_panics() { #[test] fn test_override_files() { runner::run("cargo-config", |run| { - run.run(SandboxBuilder::new().enable_networking(false), |build| { + run.run(crate::utils::sandbox_builder(), |build| { let storage = rustwide::logging::LogStorage::new(LevelFilter::Info); rustwide::logging::capture(&storage, || -> anyhow::Result<_> { build.cargo().args(["--version"]).run()?; @@ -329,7 +321,7 @@ fn test_override_files() { #[test] fn test_cargo_workspace() { runner::run("cargo-workspace", |run| { - run.run(SandboxBuilder::new().enable_networking(false), |build| { + run.run(crate::utils::sandbox_builder(), |build| { let storage = rustwide::logging::LogStorage::new(LevelFilter::Info); rustwide::logging::capture(&storage, || -> anyhow::Result<_> { build.cargo().args(["run"]).run()?; diff --git a/tests/buildtest/runner.rs b/tests/buildtest/runner.rs index 79702f1a..666db374 100644 --- a/tests/buildtest/runner.rs +++ b/tests/buildtest/runner.rs @@ -70,10 +70,7 @@ macro_rules! test_prepare_error { #[test] fn $name() { runner::run($krate, |run| { - let res = run.run( - rustwide::cmd::SandboxBuilder::new().enable_networking(false), - |_| Ok(()), - ); + let res = run.run(crate::utils::sandbox_builder(), |_| Ok(())); match res.err().and_then(|err| err.downcast().ok()) { Some(rustwide::PrepareError::$expected) => { // Everything is OK! @@ -103,10 +100,7 @@ macro_rules! test_prepare_error_stderr { #[test] fn $name() { runner::run($krate, |run| { - let res = run.run( - rustwide::cmd::SandboxBuilder::new().enable_networking(false), - |_| Ok(()), - ); + let res = run.run(crate::utils::sandbox_builder(), |_| Ok(())); match res.err().and_then(|err| err.downcast().ok()) { Some(rustwide::PrepareError::$expected(output)) => { assert!(output.contains($expected_output), "output: {:?}", output); diff --git a/tests/integration/crates_git.rs b/tests/integration/crates_git.rs index 54252c41..8f95d0ac 100644 --- a/tests/integration/crates_git.rs +++ b/tests/integration/crates_git.rs @@ -1,5 +1,5 @@ use anyhow::{anyhow, bail}; -use rustwide::cmd::{Command, CommandError, SandboxBuilder}; +use rustwide::cmd::{Command, CommandError}; use rustwide::{Crate, PrepareError, Toolchain, Workspace}; #[test] @@ -16,7 +16,7 @@ fn test_fetch() -> anyhow::Result<()> { let cloned_commit = || -> anyhow::Result { let mut dir = workspace.build_dir("integration-crates_git-test_fetch"); dir.purge()?; - dir.build(&toolchain, &krate, SandboxBuilder::new()) + dir.build(&toolchain, &krate, crate::utils::sandbox_builder()) .run(|build| { Ok(Command::new(&workspace, "git") .args(["rev-parse", "HEAD"]) diff --git a/tests/integration/purge_caches.rs b/tests/integration/purge_caches.rs index 3547db04..89ec9681 100644 --- a/tests/integration/purge_caches.rs +++ b/tests/integration/purge_caches.rs @@ -1,4 +1,3 @@ -use rustwide::cmd::SandboxBuilder; use rustwide::{Crate, Toolchain}; use std::collections::HashMap; use std::path::{Path, PathBuf}; @@ -28,7 +27,7 @@ fn test_purge_caches() -> anyhow::Result<()> { for krate in &crates { krate.fetch(&workspace)?; - let sandbox = SandboxBuilder::new().enable_networking(false); + let sandbox = crate::utils::sandbox_builder(); let mut build_dir = workspace.build_dir("shared"); build_dir.build(&toolchain, krate, sandbox).run(|build| { build.cargo().args(["check"]).run()?; diff --git a/tests/utils/mod.rs b/tests/utils/mod.rs index ce244861..3f072932 100644 --- a/tests/utils/mod.rs +++ b/tests/utils/mod.rs @@ -1,5 +1,8 @@ use log::LevelFilter; -use rustwide::{Workspace, WorkspaceBuilder, cmd::SandboxImage}; +use rustwide::{ + Workspace, WorkspaceBuilder, + cmd::{SandboxBuilder, SandboxImage}, +}; use std::path::{Path, PathBuf}; static USER_AGENT: &str = "rustwide-tests (https://github.com/rust-lang/rustwide)"; @@ -31,6 +34,18 @@ pub(crate) fn init_named_workspace(name: &str) -> anyhow::Result { builder.init() } +#[allow(dead_code)] +pub(crate) fn sandbox_builder() -> SandboxBuilder { + configure_sandbox_builder(SandboxBuilder::new().enable_networking(false)) +} + +pub(crate) fn configure_sandbox_builder(builder: SandboxBuilder) -> SandboxBuilder { + let Ok(runtime) = std::env::var("RUSTWIDE_DOCKER_RUNTIME") else { + return builder; + }; + builder.docker_runtime(runtime.parse().expect("invalid RUSTWIDE_DOCKER_RUNTIME")) +} + fn init_logs() { let env = env_logger::Builder::new() .filter_module("rustwide", LevelFilter::Info) From 13bbfa243459dfc12268cc1125a85ee221842f69 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 10 Jun 2026 02:47:31 +0200 Subject: [PATCH 02/13] test --- src/cmd/sandbox/docker.rs | 39 +++++++++++++++++++++++---------- src/cmd/sandbox/mod.rs | 46 +++++++++++++++++++++++++++------------ 2 files changed, 60 insertions(+), 25 deletions(-) diff --git a/src/cmd/sandbox/docker.rs b/src/cmd/sandbox/docker.rs index e2e452c5..2e80bb42 100644 --- a/src/cmd/sandbox/docker.rs +++ b/src/cmd/sandbox/docker.rs @@ -1,4 +1,7 @@ -use crate::{Workspace, cmd::Command}; +use crate::{ + Workspace, + cmd::{Command, DockerRuntime}, +}; use std::{ fs, path::{Path, PathBuf}, @@ -219,10 +222,15 @@ pub(super) struct CgroupStatsReader<'w> { workspace: &'w Workspace, container_id: String, pub(super) pid: Option, + docker_runtime: DockerRuntime, } impl<'w> CgroupStatsReader<'w> { - pub(super) fn new(workspace: &'w Workspace, container_id: impl Into) -> Self { + pub(super) fn new( + workspace: &'w Workspace, + container_id: impl Into, + docker_runtime: DockerRuntime, + ) -> Self { Self { oom_kill_count: None, cgroup_version: None, @@ -230,6 +238,7 @@ impl<'w> CgroupStatsReader<'w> { workspace, container_id: container_id.into(), pid: None, + docker_runtime, } } @@ -270,18 +279,26 @@ impl<'w> CgroupStatsReader<'w> { } pub(super) fn read_memory_peak_from_container(&mut self) -> Option { - self.exec_cat_cgroup_file( - "/sys/fs/cgroup/memory.peak", - "/sys/fs/cgroup/memory/memory.max_usage_in_bytes", - ) - .and_then(parse_memory_peak) + if self.docker_runtime.supports_cgroup_files_inside_container() { + self.exec_cat_cgroup_file( + "/sys/fs/cgroup/memory.peak", + "/sys/fs/cgroup/memory/memory.max_usage_in_bytes", + ) + .and_then(parse_memory_peak) + } else { + None + } } pub(super) fn read_oom_kill_count_from_container(&mut self) -> Option { - Some(parse_oom_kill_count(self.exec_cat_cgroup_file( - "/sys/fs/cgroup/memory.events", - "/sys/fs/cgroup/memory/memory.oom_control", - )?)) + if self.docker_runtime.supports_cgroup_files_inside_container() { + Some(parse_oom_kill_count(self.exec_cat_cgroup_file( + "/sys/fs/cgroup/memory.events", + "/sys/fs/cgroup/memory/memory.oom_control", + )?)) + } else { + None + } } pub(super) fn detect_host_cgroup(&mut self) -> Option<&HostCgroup> { diff --git a/src/cmd/sandbox/mod.rs b/src/cmd/sandbox/mod.rs index 25077269..f9ac3c83 100644 --- a/src/cmd/sandbox/mod.rs +++ b/src/cmd/sandbox/mod.rs @@ -185,6 +185,13 @@ impl DockerRuntime { Self::Runsc => Some("runsc"), } } + + fn supports_cgroup_files_inside_container(&self) -> bool { + match self { + DockerRuntime::Default => true, + DockerRuntime::Runsc => false, + } + } } impl fmt::Display for DockerRuntime { @@ -418,14 +425,6 @@ impl SandboxBuilder { self } - /// Use Docker's configured default runtime for the sandbox container. - /// - /// This clears any runtime selected with [`SandboxBuilder::docker_runtime`]. - pub fn default_docker_runtime(mut self) -> Self { - self.docker_runtime = DockerRuntime::Default; - self - } - /// Start a live sandbox from this configuration. /// /// The returned sandbox can be used to run one or more commands against a @@ -555,7 +554,7 @@ impl SandboxBuilder { workspace, running: true, oom_killed: false, - cgroup: CgroupStatsReader::new(workspace, id), + cgroup: CgroupStatsReader::new(workspace, id, self.docker_runtime), }) } } @@ -1043,8 +1042,13 @@ mod tests { let workspace = init_test_workspace("build-unit")?; let source_dir = tempdir()?; let target_dir = tempdir()?; - let mut sandbox = - sandbox_builder().start(&workspace, source_dir.path(), target_dir.path())?; + + let builder = sandbox_builder(); + let supports_cgroup_files_inside_container = builder + .docker_runtime + .supports_cgroup_files_inside_container(); + + let mut sandbox = builder.start(&workspace, source_dir.path(), target_dir.path())?; let host_cgroup = sandbox .detect_host_cgroup() .expect("sandbox should resolve host cgroup files"); @@ -1052,6 +1056,13 @@ mod tests { let host_peak = host_cgroup .read_memory_peak() .expect("host-side memory peak should be readable"); + + assert!(host_peak > 0, "host-side memory peak should be nonzero"); + + if !supports_cgroup_files_inside_container { + return Ok(()); + } + let exec_peak = sandbox .container .as_mut() @@ -1060,7 +1071,6 @@ mod tests { .read_memory_peak_from_container() .expect("exec-side memory peak should be readable"); - assert!(host_peak > 0, "host-side memory peak should be nonzero"); assert!(exec_peak > 0, "exec-side memory peak should be nonzero"); let min_peak = host_peak.min(exec_peak); @@ -1079,8 +1089,16 @@ mod tests { let workspace = init_test_workspace("build-unit")?; let source_dir = tempdir()?; let target_dir = tempdir()?; - let mut sandbox = - sandbox_builder().start(&workspace, source_dir.path(), target_dir.path())?; + + let builder = sandbox_builder(); + if !builder + .docker_runtime + .supports_cgroup_files_inside_container() + { + return Ok(()); + } + + let mut sandbox = builder.start(&workspace, source_dir.path(), target_dir.path())?; let host_cgroup = sandbox .detect_host_cgroup() .expect("sandbox should resolve host cgroup files"); From b917f5482ff5c84ea8949d77b4bae3e9f9d7e7b1 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 10 Jun 2026 02:49:40 +0200 Subject: [PATCH 03/13] kk --- src/cmd/sandbox/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cmd/sandbox/mod.rs b/src/cmd/sandbox/mod.rs index f9ac3c83..371018a4 100644 --- a/src/cmd/sandbox/mod.rs +++ b/src/cmd/sandbox/mod.rs @@ -186,6 +186,9 @@ impl DockerRuntime { } } + /// to see if the used docker engine also exposes the cgroup files + /// inside the container. + /// If not, we have to rely on the host-level files. fn supports_cgroup_files_inside_container(&self) -> bool { match self { DockerRuntime::Default => true, From 9a9bae4c56c47411d52d687047ed96ade884d6c3 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 10 Jun 2026 02:51:19 +0200 Subject: [PATCH 04/13] docs: clarify docker runtime behavior --- CHANGELOG.md | 7 ++++++- src/cmd/sandbox/mod.rs | 28 +++++++++++++++++++++------- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 383bb444..b551daae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Added `DockerRuntime`, `ParseDockerRuntimeError`, and `SandboxBuilder::docker_runtime` for selecting a Docker runtime such as - gVisor's `runsc` for sandbox containers. + gVisor's `runsc` for sandbox containers. Runtime-aware sandbox statistics + now avoid in-container cgroup reads when the runtime does not expose those + files. + +* Added a CI test variant that runs the Linux sandbox tests with gVisor's + `runsc` Docker runtime. ## [0.25.1] - 2026-05-22 diff --git a/src/cmd/sandbox/mod.rs b/src/cmd/sandbox/mod.rs index 371018a4..97e070dd 100644 --- a/src/cmd/sandbox/mod.rs +++ b/src/cmd/sandbox/mod.rs @@ -167,18 +167,27 @@ pub struct SandboxBuilder { } /// The Docker runtime used for sandbox containers. +/// +/// This controls Docker's `--runtime` option on sandbox container creation. +/// [`DockerRuntime::Default`] omits the option and lets the Docker daemon use +/// its configured default runtime. #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] pub enum DockerRuntime { /// Let Docker use the daemon's configured default runtime. + /// + /// This does not pass a `--runtime` argument to Docker. #[default] Default, + /// Use gVisor's `runsc` runtime. + /// + /// This passes `--runtime runsc` to Docker. Runsc, } impl DockerRuntime { - /// name of the runtime for the `--runtime` docker arg. + /// Name of the runtime for Docker's `--runtime` argument. fn docker_name(self) -> Option<&'static str> { match self { Self::Default => None, @@ -186,9 +195,10 @@ impl DockerRuntime { } } - /// to see if the used docker engine also exposes the cgroup files - /// inside the container. - /// If not, we have to rely on the host-level files. + /// Whether the runtime exposes the host-managed cgroup files inside the + /// sandbox container. + /// + /// If not, statistics must use host-level cgroup files. fn supports_cgroup_files_inside_container(&self) -> bool { match self { DockerRuntime::Default => true, @@ -209,6 +219,10 @@ impl fmt::Display for DockerRuntime { impl str::FromStr for DockerRuntime { type Err = ParseDockerRuntimeError; + /// Parse a Docker runtime name. + /// + /// Accepts `""` and `"default"` for [`DockerRuntime::Default`], and + /// `"runsc"` for [`DockerRuntime::Runsc`]. fn from_str(value: &str) -> Result { match value { "" | "default" => Ok(Self::Default), @@ -218,7 +232,7 @@ impl str::FromStr for DockerRuntime { } } -/// Error returned when parsing a Docker runtime name fails. +/// Error returned when parsing an unsupported Docker runtime name. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ParseDockerRuntimeError; @@ -421,8 +435,8 @@ impl SandboxBuilder { /// Use a specific Docker runtime for the sandbox container. /// /// [`DockerRuntime::Runsc`] maps to Docker's `--runtime runsc` flag. By - /// default no runtime is passed, so Docker uses the daemon's configured - /// default runtime. + /// default, [`DockerRuntime::Default`] is used and no runtime is passed, so + /// Docker uses the daemon's configured default runtime. pub fn docker_runtime(mut self, runtime: DockerRuntime) -> Self { self.docker_runtime = runtime; self From 961e99a96266f914df78995f43dd6865e4fd3c90 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 10 Jun 2026 02:57:47 +0200 Subject: [PATCH 05/13] kk --- tests/buildtest/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/buildtest/mod.rs b/tests/buildtest/mod.rs index 8bdc7411..32dac6e2 100644 --- a/tests/buildtest/mod.rs +++ b/tests/buildtest/mod.rs @@ -425,7 +425,7 @@ test_prepare_error_stderr!( test_missing_deps_typo, "missing-deps-typo", MissingDependencies, - "error: no matching package found" + "error: no matching package" ); test_prepare_error_stderr!( From dc7381cef050c6cd05b33f6f0a92f96d8917d30f Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 10 Jun 2026 03:08:55 +0200 Subject: [PATCH 06/13] dbg --- tests/buildtest/container_cleanup.rs | 2 +- tests/buildtest/mod.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/buildtest/container_cleanup.rs b/tests/buildtest/container_cleanup.rs index a91f5443..03f3bb0b 100644 --- a/tests/buildtest/container_cleanup.rs +++ b/tests/buildtest/container_cleanup.rs @@ -110,7 +110,7 @@ fn test_reused_container_oom_does_not_poison_later_commands() { "expected first command to OOM, got {first:?}" ); - build.cmd("true").run()?; + dbg!(build.cmd("true").run())?; Ok(()) }, )?; diff --git a/tests/buildtest/mod.rs b/tests/buildtest/mod.rs index 32dac6e2..7b878990 100644 --- a/tests/buildtest/mod.rs +++ b/tests/buildtest/mod.rs @@ -191,7 +191,7 @@ fn test_memory_peak() { .memory_peak_bytes() .inspect(|peak| { assert!( - *peak > 400_000_000 && *peak < 450_000_000, + *peak > 400_000_000 && *peak < 500_000_000, "memory peak roughly be 400 MiB, but it is {}", peak ); @@ -262,7 +262,7 @@ fn test_cpuset_cpus_applied() { .args(["-c", "grep '^Cpus_allowed_list:' /proc/self/status"]) .run_capture()?; - assert_eq!(output.stdout_lines(), ["Cpus_allowed_list:\t0-1"]); + assert_eq!(dbg!(output.stdout_lines()), ["Cpus_allowed_list:\t0-1"]); Ok(()) }, )?; From a9304aa06c3147178b839d7b50a6c7e47699e39d Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 10 Jun 2026 03:18:59 +0200 Subject: [PATCH 07/13] more debug --- src/cmd/mod.rs | 1 + tests/buildtest/mod.rs | 10 ++++++---- tests/buildtest/runner.rs | 5 ++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs index 890bcd5f..1732f748 100644 --- a/src/cmd/mod.rs +++ b/src/cmd/mod.rs @@ -560,6 +560,7 @@ impl From for ProcessOutput { /// Output of a [`Command`](struct.Command.html) when it was executed with the /// [`run_capture`](struct.Command.html#method.run_capture) method. +#[derive(Debug)] pub struct ProcessOutput { stdout: Vec, stderr: Vec, diff --git a/tests/buildtest/mod.rs b/tests/buildtest/mod.rs index 7b878990..1fe5f9e3 100644 --- a/tests/buildtest/mod.rs +++ b/tests/buildtest/mod.rs @@ -257,10 +257,12 @@ fn test_cpuset_cpus_applied() { run.run( crate::utils::sandbox_builder().cpuset_cpus(Some(0..=1)), |build| { - let output = build - .cmd("sh") - .args(["-c", "grep '^Cpus_allowed_list:' /proc/self/status"]) - .run_capture()?; + let output = dbg!( + build + .cmd("sh") + .args(["-c", "grep '^Cpus_allowed_list:' /proc/self/status"]) + .run_capture() + )?; assert_eq!(dbg!(output.stdout_lines()), ["Cpus_allowed_list:\t0-1"]); Ok(()) diff --git a/tests/buildtest/runner.rs b/tests/buildtest/runner.rs index 666db374..4db76a1b 100644 --- a/tests/buildtest/runner.rs +++ b/tests/buildtest/runner.rs @@ -1,3 +1,4 @@ +use log::error; use rand::{RngExt as _, distr::Alphanumeric}; use rustwide::{ Build, BuildBuilder, BuildResult, Crate, Toolchain, Workspace, cmd::SandboxBuilder, @@ -6,7 +7,9 @@ use std::path::Path; pub(crate) fn run(crate_name: &str, f: impl FnOnce(&mut Runner) -> anyhow::Result<()>) { let mut runner = Runner::new(crate_name).unwrap(); - f(&mut runner).unwrap(); + if let Err(err) = f(&mut runner) { + error!("error running command: \n{:?}", err); + } } pub(crate) struct Runner { From b58bccb5467e0d86e7607abdfc193af0475e90c8 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 10 Jun 2026 03:26:47 +0200 Subject: [PATCH 08/13] nocapt --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 947c4e39..a69ddb0e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,6 +50,7 @@ jobs: - os: ubuntu-22.04 cargo_flags: "" docker_runtime: runsc + test_flags: "--nocapture" steps: - name: Checkout the source code uses: actions/checkout@main From d21ae0e3b90fd403568de02ea764443240139516 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 10 Jun 2026 03:30:02 +0200 Subject: [PATCH 09/13] kk --- tests/buildtest/runner.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/buildtest/runner.rs b/tests/buildtest/runner.rs index 4db76a1b..5be2795e 100644 --- a/tests/buildtest/runner.rs +++ b/tests/buildtest/runner.rs @@ -1,4 +1,3 @@ -use log::error; use rand::{RngExt as _, distr::Alphanumeric}; use rustwide::{ Build, BuildBuilder, BuildResult, Crate, Toolchain, Workspace, cmd::SandboxBuilder, @@ -8,7 +7,7 @@ use std::path::Path; pub(crate) fn run(crate_name: &str, f: impl FnOnce(&mut Runner) -> anyhow::Result<()>) { let mut runner = Runner::new(crate_name).unwrap(); if let Err(err) = f(&mut runner) { - error!("error running command: \n{:?}", err); + panic!("error running command: \n{:?}", err); } } From d0ebf94ec2c8fe845fbffedd62d8b3a592d2bf08 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 10 Jun 2026 03:40:21 +0200 Subject: [PATCH 10/13] test --- src/cmd/sandbox/mod.rs | 9 ++++++++- tests/buildtest/mod.rs | 34 ++++++++++++++++++++-------------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/cmd/sandbox/mod.rs b/src/cmd/sandbox/mod.rs index 97e070dd..0b695dc5 100644 --- a/src/cmd/sandbox/mod.rs +++ b/src/cmd/sandbox/mod.rs @@ -163,7 +163,7 @@ pub struct SandboxBuilder { cpu_limit: Option, cpuset_cpus: Option>, enable_networking: bool, - docker_runtime: DockerRuntime, + pub docker_runtime: DockerRuntime, } /// The Docker runtime used for sandbox containers. @@ -205,6 +205,13 @@ impl DockerRuntime { DockerRuntime::Runsc => false, } } + + pub fn exposes_self_status_inside_container(&self) -> bool { + match self { + DockerRuntime::Default => true, + DockerRuntime::Runsc => false, + } + } } impl fmt::Display for DockerRuntime { diff --git a/tests/buildtest/mod.rs b/tests/buildtest/mod.rs index 1fe5f9e3..ac0d5965 100644 --- a/tests/buildtest/mod.rs +++ b/tests/buildtest/mod.rs @@ -253,21 +253,27 @@ fn test_invalid_cpuset_cpus() { #[test] #[cfg(not(windows))] fn test_cpuset_cpus_applied() { + let builder = crate::utils::sandbox_builder().cpuset_cpus(Some(0..=1)); + + if builder + .docker_runtime + .exposes_self_status_inside_container() + { + return; + } + runner::run("hello-world", |run| { - run.run( - crate::utils::sandbox_builder().cpuset_cpus(Some(0..=1)), - |build| { - let output = dbg!( - build - .cmd("sh") - .args(["-c", "grep '^Cpus_allowed_list:' /proc/self/status"]) - .run_capture() - )?; - - assert_eq!(dbg!(output.stdout_lines()), ["Cpus_allowed_list:\t0-1"]); - Ok(()) - }, - )?; + run.run(builder, |build| { + let output = dbg!( + build + .cmd("sh") + .args(["-c", "grep '^Cpus_allowed_list:' /proc/self/status"]) + .run_capture() + )?; + + assert_eq!(dbg!(output.stdout_lines()), ["Cpus_allowed_list:\t0-1"]); + Ok(()) + })?; Ok(()) }); } From 0bf71237107dea057008d6f26fa166e221c43d14 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 10 Jun 2026 03:43:10 +0200 Subject: [PATCH 11/13] try --- src/cmd/sandbox/mod.rs | 15 ++++++++++++--- tests/buildtest/container_cleanup.rs | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/cmd/sandbox/mod.rs b/src/cmd/sandbox/mod.rs index 0b695dc5..d3a6e7cd 100644 --- a/src/cmd/sandbox/mod.rs +++ b/src/cmd/sandbox/mod.rs @@ -163,6 +163,7 @@ pub struct SandboxBuilder { cpu_limit: Option, cpuset_cpus: Option>, enable_networking: bool, + /// Docker runtime selected for this sandbox. pub docker_runtime: DockerRuntime, } @@ -206,6 +207,8 @@ impl DockerRuntime { } } + /// Whether the runtime exposes the current process status file inside the + /// sandbox container. pub fn exposes_self_status_inside_container(&self) -> bool { match self { DockerRuntime::Default => true, @@ -783,6 +786,10 @@ impl<'w> Sandbox<'w> { ) } + fn command_needs_fresh_container(res: &Result) -> bool { + Self::command_timed_out(res) || matches!(res, Err(CommandError::SandboxOOM)) + } + /// Return the statistics gathered across the sandbox lifetime so far. pub fn statistics(&self) -> SandboxStatistics { self.statistics.snapshot() @@ -908,10 +915,12 @@ impl<'w> Sandbox<'w> { // command inside the container keeps running on the container's // `sleep infinity` init. Reusing the container would let the // abandoned process race the next command (sharing files, target - // dir, CPU/memory budget). Tear the container down so the next - // command in this build gets a clean one via + // dir, CPU/memory budget). OOMs can also leave the container's + // exec path unusable even when Docker still reports it as running + // (for example with gVisor/runsc). Tear the container down so the + // next command in this build gets a clean one via // `ensure_reusable_container`. - if Self::command_timed_out(&res) + if Self::command_needs_fresh_container(&res) && let Some(mut container) = self.container.take() { container.delete()?; diff --git a/tests/buildtest/container_cleanup.rs b/tests/buildtest/container_cleanup.rs index 03f3bb0b..a91f5443 100644 --- a/tests/buildtest/container_cleanup.rs +++ b/tests/buildtest/container_cleanup.rs @@ -110,7 +110,7 @@ fn test_reused_container_oom_does_not_poison_later_commands() { "expected first command to OOM, got {first:?}" ); - dbg!(build.cmd("true").run())?; + build.cmd("true").run()?; Ok(()) }, )?; From 204a10913521aacc6156afcdc86201e9f7df76e3 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sat, 13 Jun 2026 03:19:21 +0200 Subject: [PATCH 12/13] don't test --- tests/buildtest/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/buildtest/mod.rs b/tests/buildtest/mod.rs index ac0d5965..a50f322e 100644 --- a/tests/buildtest/mod.rs +++ b/tests/buildtest/mod.rs @@ -255,7 +255,7 @@ fn test_invalid_cpuset_cpus() { fn test_cpuset_cpus_applied() { let builder = crate::utils::sandbox_builder().cpuset_cpus(Some(0..=1)); - if builder + if !builder .docker_runtime .exposes_self_status_inside_container() { From 51d38a74526f488e414740faae7b626301a4fc5a Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sat, 13 Jun 2026 03:52:43 +0200 Subject: [PATCH 13/13] no dbg --- .github/workflows/main.yml | 1 - tests/buildtest/mod.rs | 12 +++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a69ddb0e..947c4e39 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,7 +50,6 @@ jobs: - os: ubuntu-22.04 cargo_flags: "" docker_runtime: runsc - test_flags: "--nocapture" steps: - name: Checkout the source code uses: actions/checkout@main diff --git a/tests/buildtest/mod.rs b/tests/buildtest/mod.rs index a50f322e..4ce0d15b 100644 --- a/tests/buildtest/mod.rs +++ b/tests/buildtest/mod.rs @@ -264,14 +264,12 @@ fn test_cpuset_cpus_applied() { runner::run("hello-world", |run| { run.run(builder, |build| { - let output = dbg!( - build - .cmd("sh") - .args(["-c", "grep '^Cpus_allowed_list:' /proc/self/status"]) - .run_capture() - )?; + let output = build + .cmd("sh") + .args(["-c", "grep '^Cpus_allowed_list:' /proc/self/status"]) + .run_capture()?; - assert_eq!(dbg!(output.stdout_lines()), ["Cpus_allowed_list:\t0-1"]); + assert_eq!(output.stdout_lines(), ["Cpus_allowed_list:\t0-1"]); Ok(()) })?; Ok(())