From 775dff53ccbcbb01a0f5515132613e31f9d5ad0f Mon Sep 17 00:00:00 2001 From: Marcel Sery Date: Thu, 24 Jan 2019 12:47:14 +0100 Subject: [PATCH 1/4] Added support for multiple input dirs --- warp-packer/src/main.rs | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/warp-packer/src/main.rs b/warp-packer/src/main.rs index 1921695..a6b2736 100644 --- a/warp-packer/src/main.rs +++ b/warp-packer/src/main.rs @@ -84,12 +84,15 @@ fn patch_runner(arch: &str, exec_name: &str) -> io::Result> { Ok(buf) } -fn create_tgz(dir: &Path, out: &Path) -> io::Result<()> { +fn create_tgz(dirs: &Vec<&Path>, out: &Path) -> io::Result<()> { let f = fs::File::create(out)?; let gz = GzEncoder::new(f, Compression::best()); let mut tar = tar::Builder::new(gz); tar.follow_symlinks(false); - tar.append_dir_all(".", dir)?; + for dir in dirs.iter() { + println!("Compressing input directory {:?}...", dir); + tar.append_dir_all(".", dir)?; + } Ok(()) } @@ -120,6 +123,14 @@ fn create_app(runner_buf: &Vec, tgz_path: &Path, out: &Path) -> io::Result<( Ok(()) } +fn make_path(path_str: &str) -> &Path { + let path = Path::new(path_str); + if fs::metadata(path).is_err() { + bail!("Cannot access specified input directory {:?}", path); + } + return &path; +} + fn main() -> Result<(), Box> { let args = App::new(APP_NAME) .settings(&[AppSettings::ArgRequiredElseHelp, AppSettings::ColoredHelp]) @@ -138,10 +149,12 @@ fn main() -> Result<(), Box> { .short("i") .long("input_dir") .value_name("input_dir") - .help("Sets the input directory containing the application and dependencies") + .help("Sets the input directories for packing. Might provide multiple directories, but the first must contain the executed application") .display_order(2) - .takes_value(true) - .required(true)) + .takes_value(true) + .required(true) + .multiple(true) + .min_values(1)) .arg(Arg::with_name("exec") .short("e") .long("exec") @@ -165,17 +178,17 @@ fn main() -> Result<(), Box> { bail!("Unknown architecture specified: {}, supported: {:?}", arch, RUNNER_BY_ARCH.keys()); } - let input_dir = Path::new(args.value_of("input_dir").unwrap()); - if fs::metadata(input_dir).is_err() { - bail!("Cannot access specified input directory {:?}", input_dir); - } + let input_dirs: Vec<&Path> = args.values_of("input_dir") + .unwrap() + .map(make_path) + .collect(); let exec_name = args.value_of("exec").unwrap(); if exec_name.len() >= RUNNER_MAGIC.len() { bail!("Executable name is too long, please consider using a shorter name"); } - let exec_path = Path::new(input_dir).join(exec_name); + let exec_path = Path::new(input_dirs[0]).join(exec_name); match fs::metadata(&exec_path) { Err(_) => { bail!("Cannot find file {:?}", exec_path); @@ -189,10 +202,11 @@ fn main() -> Result<(), Box> { let runner_buf = patch_runner(&arch, &exec_name)?; - println!("Compressing input directory {:?}...", input_dir); let tmp_dir = TempDir::new(APP_NAME)?; let tgz_path = tmp_dir.path().join("input.tgz"); - create_tgz(&input_dir, &tgz_path)?; + + create_tgz(&input_dirs, &tgz_path)?; + let exec_name = Path::new(args.value_of("output").unwrap()); println!("Creating self-contained application binary {:?}...", exec_name); From 33406f096ad8c1d1fa1bdca2f5da81023322b642 Mon Sep 17 00:00:00 2001 From: Marcel Sery Date: Mon, 28 Jan 2019 15:30:22 +0100 Subject: [PATCH 2/4] Added support for including already packed resources --- warp-packer/src/main.rs | 86 +++++++++++++++++++++++++----------- warp-runner/src/extractor.rs | 1 - 2 files changed, 59 insertions(+), 28 deletions(-) diff --git a/warp-packer/src/main.rs b/warp-packer/src/main.rs index a6b2736..89f3f75 100644 --- a/warp-packer/src/main.rs +++ b/warp-packer/src/main.rs @@ -13,6 +13,7 @@ use flate2::write::GzEncoder; use std::collections::HashMap; use std::error::Error; use std::fs; +use std::iter; use std::fs::File; use std::io; use std::io::copy; @@ -84,15 +85,15 @@ fn patch_runner(arch: &str, exec_name: &str) -> io::Result> { Ok(buf) } -fn create_tgz(dirs: &Vec<&Path>, out: &Path) -> io::Result<()> { +fn create_tgz(dirs: &Vec<&Path>, out: &Path) -> io::Result<()> { let f = fs::File::create(out)?; - let gz = GzEncoder::new(f, Compression::best()); + let gz = GzEncoder::new(f, Compression::best()); let mut tar = tar::Builder::new(gz); tar.follow_symlinks(false); for dir in dirs.iter() { println!("Compressing input directory {:?}...", dir); - tar.append_dir_all(".", dir)?; - } + tar.append_dir_all(".", dir)?; + } Ok(()) } @@ -115,22 +116,39 @@ fn create_app_file(out: &Path) -> io::Result { .open(out) } -fn create_app(runner_buf: &Vec, tgz_path: &Path, out: &Path) -> io::Result<()> { +fn create_app(runner_buf: &Vec, tgz_paths: &Vec<&Path>, out: &Path) -> io::Result<()> { let mut outf = create_app_file(out)?; - let mut tgzf = fs::File::open(tgz_path)?; outf.write_all(runner_buf)?; - copy(&mut tgzf, &mut outf)?; + + for tgz_path in tgz_paths.iter() { + let mut tgzf = fs::File::open(tgz_path)?; + copy(&mut tgzf, &mut outf)?; + } + Ok(()) } fn make_path(path_str: &str) -> &Path { let path = Path::new(path_str); if fs::metadata(path).is_err() { - bail!("Cannot access specified input directory {:?}", path); + bail!("Cannot access specified input path {:?}", path); } return &path; } +fn check_executable_exists(exec_path: &Path){ + match fs::metadata(&exec_path) { + Err(_) => { + bail!("Cannot find file {:?}", exec_path); + } + Ok(metadata) => { + if !metadata.is_file() { + bail!("{:?} isn't a file", exec_path); + } + } + } +} + fn main() -> Result<(), Box> { let args = App::new(APP_NAME) .settings(&[AppSettings::ArgRequiredElseHelp, AppSettings::ColoredHelp]) @@ -155,20 +173,35 @@ fn main() -> Result<(), Box> { .required(true) .multiple(true) .min_values(1)) + .arg(Arg::with_name("input_tgz") + .short("t") + .long("input_tgz") + .value_name("input_tgz") + .help("Sets additional already packed tar-gzipped files to be included in package. Might provide multiple files. Can be used with --disable_exec_check param if main executable file is in packed file.") + .display_order(3) + .takes_value(true) + .required(false) + .multiple(true)) .arg(Arg::with_name("exec") .short("e") .long("exec") .value_name("exec") .help("Sets the application executable file name") - .display_order(3) + .display_order(4) .takes_value(true) .required(true)) + .arg(Arg::with_name("disable_exec_check") + .long("disable_exec_check") + .help("Disables the check for existence of executable file in target directory. Useful for cases when main executable file is in already packed tgzip file (see input_tgz param)") + .display_order(5) + .takes_value(false) + .required(false)) .arg(Arg::with_name("output") .short("o") .long("output") .value_name("output") .help("Sets the resulting self-contained application file name") - .display_order(4) + .display_order(6) .takes_value(true) .required(true)) .get_matches(); @@ -178,40 +211,39 @@ fn main() -> Result<(), Box> { bail!("Unknown architecture specified: {}, supported: {:?}", arch, RUNNER_BY_ARCH.keys()); } + let tmp_dir = TempDir::new(APP_NAME)?; + let main_tgz = tmp_dir.path().join("input.tgz"); + let main_tgz_path = main_tgz.as_path(); + let input_dirs: Vec<&Path> = args.values_of("input_dir") .unwrap() .map(make_path) .collect(); + let input_tgzs: Vec<&Path> = args.values_of("input_tgz") + .unwrap_or(clap::Values::default()) + .map(make_path) + .chain(iter::once(main_tgz_path)) + .collect(); + let exec_name = args.value_of("exec").unwrap(); if exec_name.len() >= RUNNER_MAGIC.len() { bail!("Executable name is too long, please consider using a shorter name"); } - let exec_path = Path::new(input_dirs[0]).join(exec_name); - match fs::metadata(&exec_path) { - Err(_) => { - bail!("Cannot find file {:?}", exec_path); - } - Ok(metadata) => { - if !metadata.is_file() { - bail!("{:?} isn't a file", exec_path); - } - } + let do_check_exec_existence = !args.is_present("disable_exec_check"); + if do_check_exec_existence { + let exec_path = Path::new(input_dirs[0]).join(exec_name); + check_executable_exists(&exec_path); } let runner_buf = patch_runner(&arch, &exec_name)?; - let tmp_dir = TempDir::new(APP_NAME)?; - let tgz_path = tmp_dir.path().join("input.tgz"); - - create_tgz(&input_dirs, &tgz_path)?; - + create_tgz(&input_dirs, &main_tgz_path)?; let exec_name = Path::new(args.value_of("output").unwrap()); println!("Creating self-contained application binary {:?}...", exec_name); - create_app(&runner_buf, &tgz_path, &exec_name)?; - + create_app(&runner_buf, &input_tgzs, &exec_name)?; println!("All done"); Ok(()) } diff --git a/warp-runner/src/extractor.rs b/warp-runner/src/extractor.rs index 6a71f76..5be9158 100644 --- a/warp-runner/src/extractor.rs +++ b/warp-runner/src/extractor.rs @@ -77,7 +77,6 @@ pub fn extract_to(src: &Path, dst: &Path) -> io::Result<()> { if extract_at_offset(src, offs, dst).is_ok() { trace!("tarball found at offset {} was extracted successfully", offs); found = true; - break; } } From 063507e63cdaa27b44a20435270e84cd66c10152 Mon Sep 17 00:00:00 2001 From: Marcel Sery Date: Tue, 29 Jan 2019 19:05:52 +0100 Subject: [PATCH 3/4] Added possibility to generate unique id for each build --- warp-packer/Cargo.toml | 1 + warp-packer/src/main.rs | 51 ++++++++++++++++++++++++++++++++--------- warp-runner/src/main.rs | 26 ++++++++++++++++----- 3 files changed, 61 insertions(+), 17 deletions(-) diff --git a/warp-packer/Cargo.toml b/warp-packer/Cargo.toml index 38d24d2..34907e8 100644 --- a/warp-packer/Cargo.toml +++ b/warp-packer/Cargo.toml @@ -12,6 +12,7 @@ clap = "2.32.0" dirs = "3.0.0" reqwest = "0.11.0" tempdir = "0.3.7" +rand = "0.6" flate2 = "1.0" tar = "0.4" lazy_static = "1.1.0" diff --git a/warp-packer/src/main.rs b/warp-packer/src/main.rs index 89f3f75..6acb9ae 100644 --- a/warp-packer/src/main.rs +++ b/warp-packer/src/main.rs @@ -3,6 +3,7 @@ extern crate dirs; extern crate flate2; #[macro_use] extern crate lazy_static; +extern crate rand; extern crate reqwest; extern crate tar; extern crate tempdir; @@ -10,6 +11,8 @@ extern crate tempdir; use clap::{App, AppSettings, Arg}; use flate2::Compression; use flate2::write::GzEncoder; +use rand::{thread_rng, Rng}; +use rand::distributions::Alphanumeric; use std::collections::HashMap; use std::error::Error; use std::fs; @@ -26,7 +29,8 @@ const APP_NAME: &str = env!("CARGO_PKG_NAME"); const AUTHOR: &str = env!("CARGO_PKG_AUTHORS"); const VERSION: &str = env!("CARGO_PKG_VERSION"); -const RUNNER_MAGIC: &[u8] = b"tVQhhsFFlGGD3oWV4lEPST8I8FEPP54IM0q7daes4E1y3p2U2wlJRYmWmjPYfkhZ0PlT14Ls0j8fdDkoj33f2BlRJavLj3mWGibJsGt5uLAtrCDtvxikZ8UX2mQDCrgE\0"; +const RUNNER_EXEC_MAGIC: &[u8] = b"tVQhhsFFlGGD3oWV4lEPST8I8FEPP54IM0q7daes4E1y3p2U2wlJRYmWmjPYfkhZ0PlT14Ls0j8fdDkoj33f2BlRJavLj3mWGibJsGt5uLAtrCDtvxikZ8UX2mQDCrgE\0"; +const RUNNER_UID_MAGIC: &[u8] = b"DR1PWsJsM6KxNbng9Y38\0"; const RUNNER_LINUX_X64: &[u8] = include_bytes!("../../target/x86_64-unknown-linux-gnu/release/warp-runner"); const RUNNER_LINUX_AARCH64: &[u8] = include_bytes!("../../target/aarch64-unknown-linux-gnu/release/warp-runner"); @@ -55,34 +59,38 @@ macro_rules! bail { }) } -fn patch_runner(arch: &str, exec_name: &str) -> io::Result> { +fn patch_runner(arch: &str, exec_name: &str, uid: &str) -> io::Result> { // Read runner executable in memory let runner_contents = RUNNER_BY_ARCH.get(arch).unwrap(); let mut buf = runner_contents.to_vec(); - // Set the correct target executable name into the local magic buffer - let magic_len = RUNNER_MAGIC.len(); + write_magic(&mut buf, RUNNER_UID_MAGIC, uid); + write_magic(&mut buf, RUNNER_EXEC_MAGIC, exec_name); + Ok(buf) +} + +fn write_magic(buf: &mut Vec, magic: &[u8], new_value: &str) { + // Set the correct target executable name into the local magic buffer + let magic_len = magic.len(); let mut new_magic = vec![0; magic_len]; - new_magic[..exec_name.len()].clone_from_slice(exec_name.as_bytes()); + new_magic[..new_value.len()].clone_from_slice(new_value.as_bytes()); // Find the magic buffer offset inside the runner executable let mut offs_opt = None; for (i, chunk) in buf.windows(magic_len).enumerate() { - if chunk == RUNNER_MAGIC { + if chunk == magic { offs_opt = Some(i); break; } } if offs_opt.is_none() { - return Err(io::Error::new(io::ErrorKind::Other, "no magic found inside runner")); + bail!("no magic found inside runner"); } // Replace the magic with the new one that points to the target executable let offs = offs_opt.unwrap(); buf[offs..offs + magic_len].clone_from_slice(&new_magic); - - Ok(buf) } fn create_tgz(dirs: &Vec<&Path>, out: &Path) -> io::Result<()> { @@ -149,6 +157,13 @@ fn check_executable_exists(exec_path: &Path){ } } +fn generate_uid() -> String { + return thread_rng() + .sample_iter(&Alphanumeric) + .take(RUNNER_UID_MAGIC.len() - 1) + .collect(); +} + fn main() -> Result<(), Box> { let args = App::new(APP_NAME) .settings(&[AppSettings::ArgRequiredElseHelp, AppSettings::ColoredHelp]) @@ -204,6 +219,14 @@ fn main() -> Result<(), Box> { .display_order(6) .takes_value(true) .required(true)) + .arg(Arg::with_name("unique_id") + .short("q") + .long("unique_id") + .value_name("unique_id") + .help("Generate unique id for each package build") + .display_order(7) + .takes_value(false) + .required(false)) .get_matches(); let arch = args.value_of("arch").unwrap(); @@ -227,7 +250,7 @@ fn main() -> Result<(), Box> { .collect(); let exec_name = args.value_of("exec").unwrap(); - if exec_name.len() >= RUNNER_MAGIC.len() { + if exec_name.len() >= RUNNER_EXEC_MAGIC.len() { bail!("Executable name is too long, please consider using a shorter name"); } @@ -237,7 +260,13 @@ fn main() -> Result<(), Box> { check_executable_exists(&exec_path); } - let runner_buf = patch_runner(&arch, &exec_name)?; + let mut uid:String = "".to_string(); + let do_generate_uid = args.is_present("unique_id"); + if do_generate_uid { + uid = generate_uid(); + } + + let runner_buf = patch_runner(&arch, &exec_name, &uid)?; create_tgz(&input_dirs, &main_tgz_path)?; diff --git a/warp-runner/src/main.rs b/warp-runner/src/main.rs index 0192401..131bc26 100644 --- a/warp-runner/src/main.rs +++ b/warp-runner/src/main.rs @@ -16,17 +16,26 @@ mod extractor; mod executor; static TARGET_FILE_NAME_BUF: &'static [u8] = b"tVQhhsFFlGGD3oWV4lEPST8I8FEPP54IM0q7daes4E1y3p2U2wlJRYmWmjPYfkhZ0PlT14Ls0j8fdDkoj33f2BlRJavLj3mWGibJsGt5uLAtrCDtvxikZ8UX2mQDCrgE\0"; +static TARGET_UID_BUF: &'static [u8] = b"DR1PWsJsM6KxNbng9Y38\0"; + +fn build_uid() -> &'static str { + return read_magic("TARGET_UID_BUF", &TARGET_UID_BUF) +} fn target_file_name() -> &'static str { - let nul_pos = TARGET_FILE_NAME_BUF.iter() + return read_magic("TARGET_FILE_NAME_BUF", &TARGET_FILE_NAME_BUF) +} + +fn read_magic(magic_name: &str, magic: &'static [u8]) -> &'static str { + let nul_pos = magic.iter() .position(|elem| *elem == b'\0') - .expect("TARGET_FILE_NAME_BUF has no NUL terminator"); + .expect(&format!("{} has no NUL terminator", magic_name)); - let slice = &TARGET_FILE_NAME_BUF[..(nul_pos + 1)]; + let slice = &magic[..(nul_pos + 1)]; CStr::from_bytes_with_nul(slice) - .expect("Can't convert TARGET_FILE_NAME_BUF slice to CStr") + .expect(&format!("Can't convert {} slice to CStr", magic_name)) .to_str() - .expect("Can't convert TARGET_FILE_NAME_BUF CStr to str") + .expect(&format!("Can't convert {} CStr to str", magic_name)) } fn cache_path(target: &str) -> PathBuf { @@ -48,12 +57,17 @@ fn main() -> Result<(), Box> { simple_logger::init_with_level(Level::Trace)?; } + + + let build_uid = build_uid(); let self_path = env::current_exe()?; let self_file_name = self_path.file_name().unwrap(); - let cache_path = cache_path(&self_file_name.to_string_lossy()); + let cache_folder_name = format!("{}.{}", self_file_name.to_string_lossy(), build_uid); + let cache_path = cache_path(&cache_folder_name); trace!("self_path={:?}", self_path); trace!("self_file_name={:?}", self_file_name); + trace!("build_uid={:?}", build_uid); trace!("cache_path={:?}", cache_path); let target_file_name = target_file_name(); From dcc2a8e523760de34b8fde6a1d76ecbeb974c0fc Mon Sep 17 00:00:00 2001 From: Nik V Date: Sun, 14 Nov 2021 11:56:27 +0800 Subject: [PATCH 4/4] allow specifying runner cache dir via WARP_CACHE_DIR env var --- warp-runner/src/main.rs | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/warp-runner/src/main.rs b/warp-runner/src/main.rs index 131bc26..558a227 100644 --- a/warp-runner/src/main.rs +++ b/warp-runner/src/main.rs @@ -12,22 +12,23 @@ use std::io; use std::path::*; use std::process; -mod extractor; mod executor; +mod extractor; static TARGET_FILE_NAME_BUF: &'static [u8] = b"tVQhhsFFlGGD3oWV4lEPST8I8FEPP54IM0q7daes4E1y3p2U2wlJRYmWmjPYfkhZ0PlT14Ls0j8fdDkoj33f2BlRJavLj3mWGibJsGt5uLAtrCDtvxikZ8UX2mQDCrgE\0"; static TARGET_UID_BUF: &'static [u8] = b"DR1PWsJsM6KxNbng9Y38\0"; fn build_uid() -> &'static str { - return read_magic("TARGET_UID_BUF", &TARGET_UID_BUF) + return read_magic("TARGET_UID_BUF", &TARGET_UID_BUF); } fn target_file_name() -> &'static str { - return read_magic("TARGET_FILE_NAME_BUF", &TARGET_FILE_NAME_BUF) + return read_magic("TARGET_FILE_NAME_BUF", &TARGET_FILE_NAME_BUF); } fn read_magic(magic_name: &str, magic: &'static [u8]) -> &'static str { - let nul_pos = magic.iter() + let nul_pos = magic + .iter() .position(|elem| *elem == b'\0') .expect(&format!("{} has no NUL terminator", magic_name)); @@ -39,11 +40,20 @@ fn read_magic(magic_name: &str, magic: &'static [u8]) -> &'static str { } fn cache_path(target: &str) -> PathBuf { - dirs::data_local_dir() - .expect("No data local dir found") - .join("warp") + if env::var("WARP_CACHE_DIR").is_err() { + dirs::data_local_dir() + .expect("No data local dir found") + .join("warp") + .join("packages") + .join(target) + } else { + PathBuf::from( + env::var("WARP_CACHE_DIR") + .expect("Invalid local cache path specified in WARP_CACHE_DIR"), + ) .join("packages") .join(target) + } } fn extract(exe_path: &Path, cache_path: &Path) -> io::Result<()> { @@ -57,8 +67,6 @@ fn main() -> Result<(), Box> { simple_logger::init_with_level(Level::Trace)?; } - - let build_uid = build_uid(); let self_path = env::current_exe()?; let self_file_name = self_path.file_name().unwrap();