Allow specifying cache dir for runner via WARP_CACHE_DIR env var

warp-packer/Cargo.toml

@@ -12,6 +12,7 @@ clap = "2.32.0"
 dirs = "3.0.0"
 reqwest = "0.11.0"
 tempdir = "0.3.7"
+rand = "0.6"
 flate2 = "1.0"
 tar = "0.4"
 lazy_static = "1.1.0"

warp-packer/src/main.rs

@@ -3,16 +3,20 @@ extern crate dirs;
 extern crate flate2;
 #[macro_use]
 extern crate lazy_static;
+extern crate rand;
 extern crate reqwest;
 extern crate tar;
 extern crate tempdir;
 
 use clap::{App, AppSettings, Arg};
 use flate2::Compression;
 use flate2::write::GzEncoder;
+use rand::{thread_rng, Rng};
+use rand::distributions::Alphanumeric;
 use std::collections::HashMap;
 use std::error::Error;
 use std::fs;
+use std::iter;
 use std::fs::File;
 use std::io;
 use std::io::copy;
@@ -25,7 +29,8 @@ const APP_NAME: &str = env!("CARGO_PKG_NAME");
 const AUTHOR: &str = env!("CARGO_PKG_AUTHORS");
 const VERSION: &str = env!("CARGO_PKG_VERSION");
 
-const RUNNER_MAGIC: &[u8] = b"tVQhhsFFlGGD3oWV4lEPST8I8FEPP54IM0q7daes4E1y3p2U2wlJRYmWmjPYfkhZ0PlT14Ls0j8fdDkoj33f2BlRJavLj3mWGibJsGt5uLAtrCDtvxikZ8UX2mQDCrgE\0";
+const RUNNER_EXEC_MAGIC: &[u8] = b"tVQhhsFFlGGD3oWV4lEPST8I8FEPP54IM0q7daes4E1y3p2U2wlJRYmWmjPYfkhZ0PlT14Ls0j8fdDkoj33f2BlRJavLj3mWGibJsGt5uLAtrCDtvxikZ8UX2mQDCrgE\0";
+const RUNNER_UID_MAGIC: &[u8] = b"DR1PWsJsM6KxNbng9Y38\0";
 
 const RUNNER_LINUX_X64: &[u8] = include_bytes!("../../target/x86_64-unknown-linux-gnu/release/warp-runner");
 const RUNNER_LINUX_AARCH64: &[u8] = include_bytes!("../../target/aarch64-unknown-linux-gnu/release/warp-runner");
@@ -54,42 +59,49 @@ macro_rules! bail {
     })
 }
 
-fn patch_runner(arch: &str, exec_name: &str) -> io::Result<Vec<u8>> {
+fn patch_runner(arch: &str, exec_name: &str, uid: &str) -> io::Result<Vec<u8>> {
     // Read runner executable in memory
     let runner_contents = RUNNER_BY_ARCH.get(arch).unwrap();
     let mut buf = runner_contents.to_vec();
 
-    // Set the correct target executable name into the local magic buffer
-    let magic_len = RUNNER_MAGIC.len();
+    write_magic(&mut buf, RUNNER_UID_MAGIC, uid);  
+    write_magic(&mut buf, RUNNER_EXEC_MAGIC, exec_name);  
+    Ok(buf)
+}
+
+fn write_magic(buf: &mut Vec<u8>, magic: &[u8], new_value: &str) {
+        // Set the correct target executable name into the local magic buffer
+    let magic_len = magic.len();
     let mut new_magic = vec![0; magic_len];
-    new_magic[..exec_name.len()].clone_from_slice(exec_name.as_bytes());
+    new_magic[..new_value.len()].clone_from_slice(new_value.as_bytes());
 
     // Find the magic buffer offset inside the runner executable
     let mut offs_opt = None;
     for (i, chunk) in buf.windows(magic_len).enumerate() {
-        if chunk == RUNNER_MAGIC {
+        if chunk == magic {
             offs_opt = Some(i);
             break;
         }
     }
 
     if offs_opt.is_none() {
-        return Err(io::Error::new(io::ErrorKind::Other, "no magic found inside runner"));
+        bail!("no magic found inside runner");
     }
 
     // Replace the magic with the new one that points to the target executable
     let offs = offs_opt.unwrap();
     buf[offs..offs + magic_len].clone_from_slice(&new_magic);
-
-    Ok(buf)
 }
 
-fn create_tgz(dir: &Path, out: &Path) -> io::Result<()> {
+fn create_tgz(dirs: &Vec<&Path>, out: &Path) -> io::Result<()> {  
     let f = fs::File::create(out)?;
-    let gz = GzEncoder::new(f, Compression::best());
+    let gz = GzEncoder::new(f, Compression::best());    
     let mut tar = tar::Builder::new(gz);
     tar.follow_symlinks(false);
-    tar.append_dir_all(".", dir)?;
+    for dir in dirs.iter() {
+        println!("Compressing input directory {:?}...", dir);
+        tar.append_dir_all(".", dir)?;    
+    }         
     Ok(())
 }
 
@@ -112,14 +124,46 @@ fn create_app_file(out: &Path) -> io::Result<File> {
         .open(out)
 }
 
-fn create_app(runner_buf: &Vec<u8>, tgz_path: &Path, out: &Path) -> io::Result<()> {
+fn create_app(runner_buf: &Vec<u8>, tgz_paths: &Vec<&Path>, out: &Path) -> io::Result<()> {
     let mut outf = create_app_file(out)?;
-    let mut tgzf = fs::File::open(tgz_path)?;
     outf.write_all(runner_buf)?;
-    copy(&mut tgzf, &mut outf)?;
+
+    for tgz_path in tgz_paths.iter() {
+        let mut tgzf = fs::File::open(tgz_path)?;
+        copy(&mut tgzf, &mut outf)?;    
+    }
+
     Ok(())
 }
 
+fn make_path(path_str: &str) -> &Path {
+    let path  = Path::new(path_str);    
+    if fs::metadata(path).is_err() {
+        bail!("Cannot access specified input path {:?}", path);
+    }
+    return &path;
+}
+
+fn check_executable_exists(exec_path: &Path){
+    match fs::metadata(&exec_path) {
+        Err(_) => {
+            bail!("Cannot find file {:?}", exec_path);
+        }
+        Ok(metadata) => {
+            if !metadata.is_file() {
+                bail!("{:?} isn't a file", exec_path);
+            }
+        }
+    }
+}
+
+fn generate_uid() -> String  {
+    return thread_rng()
+        .sample_iter(&Alphanumeric)
+        .take(RUNNER_UID_MAGIC.len() - 1)
+        .collect(); 
+}
+
 fn main() -> Result<(), Box<dyn Error>> {
     let args = App::new(APP_NAME)
         .settings(&[AppSettings::ArgRequiredElseHelp, AppSettings::ColoredHelp])
@@ -138,66 +182,97 @@ fn main() -> Result<(), Box<dyn Error>> {
             .short("i")
             .long("input_dir")
             .value_name("input_dir")
-            .help("Sets the input directory containing the application and dependencies")
+            .help("Sets the input directories for packing. Might provide multiple directories, but the first must contain the executed application")
             .display_order(2)
-            .takes_value(true)
-            .required(true))
+            .takes_value(true)            
+            .required(true)
+            .multiple(true)
+            .min_values(1))
+        .arg(Arg::with_name("input_tgz")
+            .short("t")
+            .long("input_tgz")
+            .value_name("input_tgz")
+            .help("Sets additional already packed tar-gzipped files to be included in package. Might provide multiple files. Can be used with --disable_exec_check param if main executable file is in packed file.")
+            .display_order(3)
+            .takes_value(true)            
+            .required(false)
+            .multiple(true))
         .arg(Arg::with_name("exec")
             .short("e")
             .long("exec")
             .value_name("exec")
             .help("Sets the application executable file name")
-            .display_order(3)
+            .display_order(4)
             .takes_value(true)
             .required(true))
+        .arg(Arg::with_name("disable_exec_check")            
+            .long("disable_exec_check")
+            .help("Disables the check for existence of executable file in target directory. Useful for cases when main executable file is in already packed tgzip file (see input_tgz param)")
+            .display_order(5)
+            .takes_value(false)
+            .required(false))
         .arg(Arg::with_name("output")
             .short("o")
             .long("output")
             .value_name("output")
             .help("Sets the resulting self-contained application file name")
-            .display_order(4)
+            .display_order(6)
             .takes_value(true)
             .required(true))
+        .arg(Arg::with_name("unique_id")
+            .short("q")
+            .long("unique_id")
+            .value_name("unique_id")
+            .help("Generate unique id for each package build")
+            .display_order(7)
+            .takes_value(false)
+            .required(false))
         .get_matches();
 
     let arch = args.value_of("arch").unwrap();
     if !RUNNER_BY_ARCH.contains_key(&arch) {
         bail!("Unknown architecture specified: {}, supported: {:?}", arch, RUNNER_BY_ARCH.keys());
     }
 
-    let input_dir = Path::new(args.value_of("input_dir").unwrap());
-    if fs::metadata(input_dir).is_err() {
-        bail!("Cannot access specified input directory {:?}", input_dir);
-    }
+    let tmp_dir = TempDir::new(APP_NAME)?;
+    let main_tgz = tmp_dir.path().join("input.tgz");
+    let main_tgz_path = main_tgz.as_path();
+
+    let input_dirs: Vec<&Path> = args.values_of("input_dir")    
+        .unwrap()
+        .map(make_path)
+        .collect();
+
+    let input_tgzs: Vec<&Path> = args.values_of("input_tgz")
+        .unwrap_or(clap::Values::default())
+        .map(make_path)        
+        .chain(iter::once(main_tgz_path))
+        .collect();
 
     let exec_name = args.value_of("exec").unwrap();
-    if exec_name.len() >= RUNNER_MAGIC.len() {
+    if exec_name.len() >= RUNNER_EXEC_MAGIC.len() {
         bail!("Executable name is too long, please consider using a shorter name");
     }
 
-    let exec_path = Path::new(input_dir).join(exec_name);
-    match fs::metadata(&exec_path) {
-        Err(_) => {
-            bail!("Cannot find file {:?}", exec_path);
-        }
-        Ok(metadata) => {
-            if !metadata.is_file() {
-                bail!("{:?} isn't a file", exec_path);
-            }
-        }
+    let do_check_exec_existence = !args.is_present("disable_exec_check");
+    if do_check_exec_existence {
+        let exec_path = Path::new(input_dirs[0]).join(exec_name);
+        check_executable_exists(&exec_path);
     }
 
-    let runner_buf = patch_runner(&arch, &exec_name)?;
+    let mut uid:String = "".to_string();
+    let do_generate_uid = args.is_present("unique_id");
+    if do_generate_uid {
+        uid = generate_uid();
+    }
 
-    println!("Compressing input directory {:?}...", input_dir);
-    let tmp_dir = TempDir::new(APP_NAME)?;
-    let tgz_path = tmp_dir.path().join("input.tgz");
-    create_tgz(&input_dir, &tgz_path)?;
+    let runner_buf = patch_runner(&arch, &exec_name, &uid)?;
+
+    create_tgz(&input_dirs, &main_tgz_path)?; 
 
     let exec_name = Path::new(args.value_of("output").unwrap());
     println!("Creating self-contained application binary {:?}...", exec_name);
-    create_app(&runner_buf, &tgz_path, &exec_name)?;
-
+    create_app(&runner_buf, &input_tgzs, &exec_name)?;
     println!("All done");
     Ok(())
 }

warp-runner/src/extractor.rs

@@ -77,7 +77,6 @@ pub fn extract_to(src: &Path, dst: &Path) -> io::Result<()> {
         if extract_at_offset(src, offs, dst).is_ok() {
             trace!("tarball found at offset {} was extracted successfully", offs);
             found = true;
-            break;
         }
     }
 

warp-runner/src/main.rs

@@ -12,29 +12,48 @@ use std::io;
 use std::path::*;
 use std::process;
 
-mod extractor;
 mod executor;
+mod extractor;
 
 static TARGET_FILE_NAME_BUF: &'static [u8] = b"tVQhhsFFlGGD3oWV4lEPST8I8FEPP54IM0q7daes4E1y3p2U2wlJRYmWmjPYfkhZ0PlT14Ls0j8fdDkoj33f2BlRJavLj3mWGibJsGt5uLAtrCDtvxikZ8UX2mQDCrgE\0";
+static TARGET_UID_BUF: &'static [u8] = b"DR1PWsJsM6KxNbng9Y38\0";
+
+fn build_uid() -> &'static str {
+    return read_magic("TARGET_UID_BUF", &TARGET_UID_BUF);
+}
 
 fn target_file_name() -> &'static str {
-    let nul_pos = TARGET_FILE_NAME_BUF.iter()
+    return read_magic("TARGET_FILE_NAME_BUF", &TARGET_FILE_NAME_BUF);
+}
+
+fn read_magic(magic_name: &str, magic: &'static [u8]) -> &'static str {
+    let nul_pos = magic
+        .iter()
         .position(|elem| *elem == b'\0')
-        .expect("TARGET_FILE_NAME_BUF has no NUL terminator");
+        .expect(&format!("{} has no NUL terminator", magic_name));
 
-    let slice = &TARGET_FILE_NAME_BUF[..(nul_pos + 1)];
+    let slice = &magic[..(nul_pos + 1)];
     CStr::from_bytes_with_nul(slice)
-        .expect("Can't convert TARGET_FILE_NAME_BUF slice to CStr")
+        .expect(&format!("Can't convert {} slice to CStr", magic_name))
         .to_str()
-        .expect("Can't convert TARGET_FILE_NAME_BUF CStr to str")
+        .expect(&format!("Can't convert {} CStr to str", magic_name))
 }
 
 fn cache_path(target: &str) -> PathBuf {
-    dirs::data_local_dir()
-        .expect("No data local dir found")
-        .join("warp")
+    if env::var("WARP_CACHE_DIR").is_err() {
+        dirs::data_local_dir()
+            .expect("No data local dir found")
+            .join("warp")
+            .join("packages")
+            .join(target)
+    } else {
+        PathBuf::from(
+            env::var("WARP_CACHE_DIR")
+                .expect("Invalid local cache path specified in WARP_CACHE_DIR"),
+        )
         .join("packages")
         .join(target)
+    }
 }
 
 fn extract(exe_path: &Path, cache_path: &Path) -> io::Result<()> {
@@ -48,12 +67,15 @@ fn main() -> Result<(), Box<dyn Error>> {
         simple_logger::init_with_level(Level::Trace)?;
     }
 
+    let build_uid = build_uid();
     let self_path = env::current_exe()?;
     let self_file_name = self_path.file_name().unwrap();
-    let cache_path = cache_path(&self_file_name.to_string_lossy());
+    let cache_folder_name = format!("{}.{}", self_file_name.to_string_lossy(), build_uid);
+    let cache_path = cache_path(&cache_folder_name);
 
     trace!("self_path={:?}", self_path);
     trace!("self_file_name={:?}", self_file_name);
+    trace!("build_uid={:?}", build_uid);
     trace!("cache_path={:?}", cache_path);
 
     let target_file_name = target_file_name();