diff --git a/Cargo.lock b/Cargo.lock index 276b4e03..23664cab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -531,6 +531,20 @@ dependencies = [ "syn 1.0.48", ] +[[package]] +name = "err-derive" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22deed3a8124cff5fa835713fa105621e43bbdc46690c3a6b68328a012d350d4" +dependencies = [ + "proc-macro-error", + "proc-macro2 1.0.24", + "quote 1.0.7", + "rustversion", + "syn 1.0.48", + "synstructure", +] + [[package]] name = "error-chain" version = "0.12.4" @@ -1614,6 +1628,15 @@ dependencies = [ "regex", ] +[[package]] +name = "partition-identity" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec13ba9a0eec5c10a89f6ec1b6e9e2ef7d29b810d771355abbd1c43cae003ed6" +dependencies = [ + "err-derive", +] + [[package]] name = "percent-encoding" version = "1.0.1" @@ -1733,6 +1756,30 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.7", + "version_check", +] + [[package]] name = "proc-macro-hack" version = "0.5.18" @@ -1763,6 +1810,16 @@ dependencies = [ "unicode-xid 0.2.1", ] +[[package]] +name = "proc-mounts" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad7e9c8d1b8c20f16a84d61d7c4c0325a5837c1307a2491b509cd92fb4e4442" +dependencies = [ + "lazy_static", + "partition-identity", +] + [[package]] name = "publicsuffix" version = "1.5.4" @@ -1791,6 +1848,7 @@ dependencies = [ "gethostname", "git2", "itertools", + "proc-mounts", "rand 0.7.3", "reqwest 0.10.8", "retry", @@ -2351,6 +2409,12 @@ dependencies = [ "semver", ] +[[package]] +name = "rustversion" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb5d2a036dc6d2d8fd16fde3498b04306e29bd193bf306a57427019b823d5acd" + [[package]] name = "ryu" version = "1.0.5" diff --git a/Cargo.toml b/Cargo.toml index 63b8b4ff..12d26fe0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ tracing-subscriber = "0.2" # Filesystem fs2 = "0.4.3" +proc-mounts = "0.2.4" # Jinja2 tera = "1.3.1" diff --git a/src/build_platform/local_docker.rs b/src/build_platform/local_docker.rs index e2083eb0..51bc829d 100644 --- a/src/build_platform/local_docker.rs +++ b/src/build_platform/local_docker.rs @@ -364,18 +364,34 @@ impl BuildPlatform for LocalDocker { } // ensure there is enough disk space left before building a new image - let docker_path = Path::new("/var/lib/docker"); + let docker_path_string = "/var/lib/docker"; + let docker_path = Path::new(docker_path_string); if docker_path.exists() { - // only used in production on Linux OS - let docker_path_size_info = match fs2::statvfs(docker_path) { - Ok(fs_stats) => fs_stats, - Err(err) => { - return Err(self.engine_error(EngineErrorCause::Internal, format!("{:?}", err))); - } - }; + let mounted_disks = proc_mounts::MountList::new(); - check_docker_space_usage_and_clean(docker_path_size_info, self.get_docker_host_envs()); + // ensure docker_path is a mounted volume, otherwise ignore because it's not what Qovery does in production + // ex: this cause regular cleanup on CI, leading to random tests errors + match mounted_disks { + Ok(m) => match m.get_mount_by_dest(Path::new(docker_path)) { + Some(_) => { + // only used in production on Linux OS + let docker_path_size_info = match fs2::statvfs(docker_path) { + Ok(fs_stats) => fs_stats, + Err(err) => { + return Err(self.engine_error(EngineErrorCause::Internal, format!("{:?}", err))); + } + }; + + check_docker_space_usage_and_clean(docker_path_size_info, self.get_docker_host_envs()); + } + None => info!( + "ignoring docker cleanup because {} is not a mounted volume", + docker_path_string + ), + }, + Err(_) => error!("wasn't able to get info from {} volume", docker_path_string), + }; } let application_id = build.image.application_id.clone(); @@ -457,8 +473,7 @@ impl Listen for LocalDocker { } fn check_docker_space_usage_and_clean(docker_path_size_info: FsStats, envs: Vec<(&str, &str)>) { - let docker_max_disk_percentage_usage_before_purge = 50; // arbitrary percentage that should make the job anytime - + let docker_max_disk_percentage_usage_before_purge = 60; // arbitrary percentage that should make the job anytime let docker_percentage_used = docker_path_size_info.available_space() * 100 / docker_path_size_info.total_space(); if docker_percentage_used > docker_max_disk_percentage_usage_before_purge {