From b3ebd03b068d0d62bcddad818a990ac34f688169 Mon Sep 17 00:00:00 2001 From: Benjamin Chastanier Date: Mon, 7 Jun 2021 13:08:55 +0200 Subject: [PATCH] feat: move out terraform folder form engine archive This CL introduces: - rename generated archive with the proper extension 'tgz' instead of 'tar.gz' - a way to exclude files / folders for generated archive - excludes terraform's folder and hcl DEV-897 --- Cargo.lock | 22 ++++++--- Cargo.toml | 5 ++- src/fs.rs | 130 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 142 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 46ab04bd..968da420 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -664,11 +664,11 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da80be589a72651dcda34d8b35bcdc9b7254ad06325611074d9cc0fbb19f60ee" +checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "crc32fast", "libc", "miniz_oxide", @@ -2051,6 +2051,7 @@ dependencies = [ "serde_json", "sysinfo", "tar", + "tempdir", "tera", "test-utilities", "timeout-readwrite", @@ -3053,16 +3054,25 @@ dependencies = [ [[package]] name = "tar" -version = "0.4.30" +version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489997b7557e9a43e192c527face4feacc78bfbe6eed67fd55c4c9e381cba290" +checksum = "7d779dc6aeff029314570f666ec83f19df7280bb36ef338442cfa8c604021b80" dependencies = [ "filetime", "libc", - "redox_syscall 0.1.57", "xattr", ] +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + [[package]] name = "tempfile" version = "3.1.0" diff --git a/Cargo.toml b/Cargo.toml index 0ee4e84b..bc4e4c72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,8 +27,8 @@ lazy_static = "1.4.0" # FIXME use https://crates.io/crates/blocking instead of runtime.rs # tar gz -flate2 = "1.0.16" # tar gz -tar = "0.4.29" +flate2 = "1.0.20" # tar gz +tar = "0.4.35" # logger tracing = "0.1" @@ -62,6 +62,7 @@ digitalocean = "0.1.1" [dev-dependencies] test-utilities = { path = "test_utilities" } +tempdir = "0.3" [features] default = [] diff --git a/src/fs.rs b/src/fs.rs index 7a1df292..f2ce21ab 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::fs; use std::fs::{create_dir_all, File}; use std::io::Error; @@ -65,16 +66,31 @@ where fn archive_workspace_directory(working_root_dir: &str, execution_id: &str) -> Result { let workspace_dir = crate::fs::root_workspace_directory(working_root_dir, execution_id); + let tgz_file_path = format!("{}/.qovery-workspace/{}.tgz", working_root_dir, execution_id); + let tgz_file = File::create(tgz_file_path.as_str())?; - let tar_gz_file_path = format!("{}/.qovery-workspace/{}.tar.gz", working_root_dir, execution_id); - - let tar_gz_file = File::create(tar_gz_file_path.as_str())?; - - let enc = GzEncoder::new(tar_gz_file, Compression::fast()); + let enc = GzEncoder::new(tgz_file, Compression::fast()); let mut tar = tar::Builder::new(enc); - tar.append_dir_all(execution_id, workspace_dir)?; - Ok(tar_gz_file_path) + let excluded_files: HashSet<&'static str> = vec![ + ".terraform.lock.hcl", + ".terraform",] + .into_iter() + .collect(); + + for entry in WalkDir::new(workspace_dir.clone()) + .into_iter() + .filter_entry(|e| !excluded_files.contains(e.file_name().to_str().unwrap())) + { + let entry = entry.unwrap(); + let entry_path = entry.path(); + if entry_path.is_file() { + tar.append_path_with_name(entry_path, entry_path.strip_prefix(workspace_dir.as_str()).unwrap()) + .unwrap(); + } + } + + Ok(tgz_file_path) } pub fn cleanup_workspace_directory(working_root_dir: &str, execution_id: &str) { @@ -97,3 +113,103 @@ pub fn create_workspace_archive(working_root_dir: &str, execution_id: &str) -> R } } } + +#[cfg(test)] +mod tests { + extern crate tempdir; + + use super::*; + use flate2::read::GzDecoder; + use std::collections::HashSet; + use std::fs::File; + use std::io::prelude::*; + use std::io::BufReader; + use tempdir::TempDir; + + #[test] + fn test_archive_workspace_directory() { + // setup: + let execution_id: &str = "123"; + let tmp_dir = TempDir::new("workspace_directory").unwrap(); + let root_dir = format!( + "{}/.qovery-workspace/{}", + tmp_dir.path().to_str().unwrap(), + execution_id + ); + let root_dir_path = Path::new(root_dir.as_str()); + + let directories_to_create = vec![ + format!("{}", root_dir), + format!("{}/.terraform", root_dir), + format!("{}/.terraform/dir-1", root_dir), + format!("{}/dir-1", root_dir), + format!("{}/dir-1/.terraform", root_dir), + format!("{}/dir-1/.terraform/dir-1", root_dir), + format!("{}/dir-2", root_dir), + format!("{}/dir-2/.terraform", root_dir), + format!("{}/dir-2/dir-1/.terraform", root_dir), + format!("{}/dir-2/dir-1/.terraform/dir-1", root_dir), + format!("{}/dir-2/.terraform/dir-1", root_dir), + ]; + directories_to_create + .iter() + .for_each(|d| fs::create_dir_all(d).unwrap()); + + let tmp_files = vec![ + (".terraform/file-1.txt", "content"), + (".terraform/dir-1/file-1.txt", "content"), + ("dir-1/.terraform/file-1.txt", "content"), + ("dir-1/.terraform/dir-1/file-1.txt", "content"), + ("dir-2/dir-1/.terraform/file-1.txt", "content"), + ("dir-2/dir-1/.terraform/dir-1/file-1.txt", "content"), + ("file-1.txt", "content"), + (".terraform.lock.hcl", "content"), + ("dir-1/.terraform.lock.hcl", "content"), + ("dir-2/dir-1/.terraform.lock.hcl", "content"), + ("dir-2/dir-1/file-2.txt", "content"), + ] + .iter() + .map(|(p, c)| { + let mut file = File::create(root_dir_path.join(p)).unwrap(); + file.write_all(c.as_bytes()).unwrap(); + + file + }) + .collect::>(); + + // execute: + let result = archive_workspace_directory(tmp_dir.path().to_str().unwrap(), execution_id); + + // verify: + assert_eq!(true, result.is_ok()); + + let expected_files_in_tar: HashSet = + vec![String::from("file-1.txt"), String::from("dir-2/dir-1/file-2.txt")] + .into_iter() + .collect(); + + let archive = File::open(result.unwrap()).unwrap(); + let archive = BufReader::new(archive); + let archive = GzDecoder::new(archive); + let mut archive = tar::Archive::new(archive); + let mut files_in_tar = HashSet::new(); + + for entry in archive.entries().unwrap() { + let encoded_entry = entry.unwrap(); + let encoded_entry_path = &encoded_entry.path().unwrap(); + files_in_tar.insert(encoded_entry_path.to_str().unwrap().to_string()); + } + + assert_eq!(expected_files_in_tar.len(), files_in_tar.len()); + for e in expected_files_in_tar.iter() { + assert_eq!(true, files_in_tar.contains(e)); + } + for e in files_in_tar.iter() { + assert_eq!(true, expected_files_in_tar.contains(e)); + } + + // clean: + tmp_files.iter().for_each(|f| drop(f)); + tmp_dir.close().unwrap(); + } +}