From 118fdcdaf63f583fee7a736c5a7f38f29c4ed622 Mon Sep 17 00:00:00 2001 From: Pierre Mavro Date: Sun, 21 Feb 2021 18:47:47 +0100 Subject: [PATCH] fix: docker prune didn't worked properly on docker --- Cargo.lock | 241 ++++++++++++++++++----------- Cargo.toml | 3 +- src/build_platform/local_docker.rs | 84 +++++----- tests/aws/aws_databases.rs | 5 +- tests/aws/aws_environment.rs | 2 +- tests/aws/aws_kubernetes.rs | 1 - tests/digitalocean/do_databases.rs | 4 +- 7 files changed, 195 insertions(+), 145 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 23664cab..c63620f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -306,7 +306,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.7.0", "libc", ] @@ -316,6 +316,12 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + [[package]] name = "cpuid-bool" version = "0.1.2" @@ -331,17 +337,38 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.2", +] + [[package]] name = "crossbeam-deque" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", + "crossbeam-epoch 0.8.2", + "crossbeam-utils 0.7.2", "maybe-uninit", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch 0.9.2", + "crossbeam-utils 0.8.2", +] + [[package]] name = "crossbeam-epoch" version = "0.8.2" @@ -350,10 +377,24 @@ checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" dependencies = [ "autocfg 1.0.1", "cfg-if 0.1.10", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "lazy_static", "maybe-uninit", - "memoffset", + "memoffset 0.5.6", + "scopeguard", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60ab4a8dba064f2fbb5aa270c28da5cf4bbd0e72dae1140a6b0353a779dbe00" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.2", + "lazy_static", + "loom 0.4.0", + "memoffset 0.6.1", "scopeguard", ] @@ -364,7 +405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" dependencies = [ "cfg-if 0.1.10", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "maybe-uninit", ] @@ -379,6 +420,18 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae8f328835f8f5a6ceb6a7842a7f2d0c03692adb5c889347235d59194731fe3" +dependencies = [ + "autocfg 1.0.1", + "cfg-if 1.0.0", + "lazy_static", + "loom 0.4.0", +] + [[package]] name = "crypto-mac" version = "0.8.0" @@ -498,6 +551,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "dtoa" version = "0.4.6" @@ -531,20 +590,6 @@ dependencies = [ "syn 1.0.48", ] -[[package]] -name = "err-derive" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22deed3a8124cff5fa835713fa105621e43bbdc46690c3a6b68328a012d350d4" -dependencies = [ - "proc-macro-error", - "proc-macro2 1.0.24", - "quote 1.0.7", - "rustversion", - "syn 1.0.48", - "synstructure", -] - [[package]] name = "error-chain" version = "0.12.4" @@ -627,16 +672,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi 0.3.9", -] - [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -1144,7 +1179,7 @@ version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22dcbf2a4a289528dbef21686354904e1c694ac642610a9bff9e7df730d9ec72" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.2", "globset", "lazy_static", "log", @@ -1244,9 +1279,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.79" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743" +checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" [[package]] name = "libgit2-sys" @@ -1325,6 +1360,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "loom" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d44c73b4636e497b4917eb21c33539efa3816741a2d3ff26c6316f1b529481a4" +dependencies = [ + "cfg-if 1.0.0", + "generator", + "scoped-tls", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -1388,6 +1434,15 @@ dependencies = [ "autocfg 1.0.1", ] +[[package]] +name = "memoffset" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +dependencies = [ + "autocfg 1.0.1", +] + [[package]] name = "mime" version = "0.3.16" @@ -1507,6 +1562,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "num-integer" version = "0.1.43" @@ -1628,15 +1692,6 @@ dependencies = [ "regex", ] -[[package]] -name = "partition-identity" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec13ba9a0eec5c10a89f6ec1b6e9e2ef7d29b810d771355abbd1c43cae003ed6" -dependencies = [ - "err-derive", -] - [[package]] name = "percent-encoding" version = "1.0.1" @@ -1756,30 +1811,6 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.48", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "version_check", -] - [[package]] name = "proc-macro-hack" version = "0.5.18" @@ -1810,16 +1841,6 @@ dependencies = [ "unicode-xid 0.2.1", ] -[[package]] -name = "proc-mounts" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad7e9c8d1b8c20f16a84d61d7c4c0325a5837c1307a2491b509cd92fb4e4442" -dependencies = [ - "lazy_static", - "partition-identity", -] - [[package]] name = "publicsuffix" version = "1.5.4" @@ -1843,12 +1864,10 @@ dependencies = [ "digitalocean", "dirs 3.0.1", "flate2", - "fs2", "futures 0.3.7", "gethostname", "git2", "itertools", - "proc-mounts", "rand 0.7.3", "reqwest 0.10.8", "retry", @@ -1863,6 +1882,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "sysinfo", "tar", "tera", "test-utilities", @@ -2067,6 +2087,31 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "rayon" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +dependencies = [ + "autocfg 1.0.1", + "crossbeam-deque 0.8.0", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque 0.8.0", + "crossbeam-utils 0.8.2", + "lazy_static", + "num_cpus", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -2372,7 +2417,7 @@ dependencies = [ "base64 0.12.3", "blake2b_simd", "constant_time_eq", - "crossbeam-utils", + "crossbeam-utils 0.7.2", ] [[package]] @@ -2409,12 +2454,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rustversion" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb5d2a036dc6d2d8fd16fde3498b04306e29bd193bf306a57427019b823d5acd" - [[package]] name = "ryu" version = "1.0.5" @@ -2460,7 +2499,7 @@ checksum = "64808902d7d99f78eaddd2b4e2509713babc3dc3c85ad6f4c447680f3c01e535" dependencies = [ "bitflags", "core-foundation", - "core-foundation-sys", + "core-foundation-sys 0.7.0", "libc", "security-framework-sys", ] @@ -2471,7 +2510,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17bf11d99252f512695eb468de5516e5cf75455521e69dfe343f3b74e4748405" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.7.0", "libc", ] @@ -2583,7 +2622,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b4921be914e16899a80adefb821f8ddb7974e3f1250223575a44ed994882127" dependencies = [ "lazy_static", - "loom", + "loom 0.3.6", ] [[package]] @@ -2751,6 +2790,22 @@ dependencies = [ "unicode-xid 0.2.1", ] +[[package]] +name = "sysinfo" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c280c91abd1aed2e36be1bc8f56fbc7a2acbb2b58fbcac9641510179cc72dd9" +dependencies = [ + "cfg-if 1.0.0", + "core-foundation-sys 0.8.2", + "doc-comment", + "libc", + "ntapi", + "once_cell", + "rayon", + "winapi 0.3.9", +] + [[package]] name = "tar" version = "0.4.30" @@ -2973,7 +3028,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures 0.1.30", ] @@ -3005,7 +3060,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures 0.1.30", "lazy_static", "log", @@ -3048,9 +3103,9 @@ version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" dependencies = [ - "crossbeam-deque", + "crossbeam-deque 0.7.3", "crossbeam-queue", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures 0.1.30", "lazy_static", "log", @@ -3065,7 +3120,7 @@ version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures 0.1.30", "slab", "tokio-executor", diff --git a/Cargo.toml b/Cargo.toml index 12d26fe0..61011210 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,8 +36,7 @@ tracing-subscriber = "0.2" # shiplift = "0.6.0" # Filesystem -fs2 = "0.4.3" -proc-mounts = "0.2.4" +sysinfo = "0.16.4" # Jinja2 tera = "1.3.1" diff --git a/src/build_platform/local_docker.rs b/src/build_platform/local_docker.rs index dc1d5f94..c9bd8c03 100644 --- a/src/build_platform/local_docker.rs +++ b/src/build_platform/local_docker.rs @@ -1,7 +1,3 @@ -use std::path::Path; - -use fs2::FsStats; - use crate::build_platform::{Build, BuildPlatform, BuildResult, Image, Kind}; use crate::error::{EngineError, EngineErrorCause, SimpleError, SimpleErrorKind}; use crate::fs::workspace_directory; @@ -11,6 +7,8 @@ use crate::models::{ }; use crate::{cmd, git}; use std::env; +use std::path::Path; +use sysinfo::{Disk, DiskExt, SystemExt}; /// https://buildpacks.io/ const BUILDPACKS_BUILDERS: [&str; 1] = [ @@ -364,41 +362,29 @@ impl BuildPlatform for LocalDocker { } } - // ensure there is enough disk space left before building a new image - let docker_path_string = "/var/lib/docker"; - let docker_path = Path::new(docker_path_string); + // ensure docker_path is a mounted volume, otherwise ignore because it's not what Qovery does in production + // ex: this cause regular cleanup on CI, leading to random tests errors + match env::var_os("CI") { + Some(_) => info!("CI environment variable found, no docker prune will be made"), + None => { + // ensure there is enough disk space left before building a new image + let docker_path_string = "/var/lib/docker"; + let docker_path = Path::new(docker_path_string); - if docker_path.exists() { - let mounted_disks = proc_mounts::MountList::new(); + // get system info + let mut system = sysinfo::System::new_all(); + system.refresh_all(); - // ensure docker_path is a mounted volume, otherwise ignore because it's not what Qovery does in production - // ex: this cause regular cleanup on CI, leading to random tests errors - match mounted_disks { - Ok(m) => match m.get_mount_by_dest(Path::new(docker_path)) { - Some(_) => { - let ci_env_var = "CI"; - match env::var_os(ci_env_var) { - Some(_) => {} - None => { - // only used in production on Linux OS - let docker_path_size_info = match fs2::statvfs(docker_path) { - Ok(fs_stats) => fs_stats, - Err(err) => { - return Err(self.engine_error(EngineErrorCause::Internal, format!("{:?}", err))); - } - }; - - check_docker_space_usage_and_clean(docker_path_size_info, self.get_docker_host_envs()); - } + for disk in system.get_disks() { + if disk.get_mount_point() == docker_path { + match check_docker_space_usage_and_clean(disk, self.get_docker_host_envs()) { + Ok(msg) => info!("{:?}", msg), + Err(e) => error!("{:?}", e.message), } - } - None => info!( - "ignoring docker cleanup because {} is not a mounted volume", - docker_path_string - ), - }, - Err(_) => error!("wasn't able to get info from {} volume", docker_path_string), - }; + break; + }; + } + } } let application_id = build.image.application_id.clone(); @@ -479,21 +465,33 @@ impl Listen for LocalDocker { } } -fn check_docker_space_usage_and_clean(docker_path_size_info: FsStats, envs: Vec<(&str, &str)>) { +fn check_docker_space_usage_and_clean( + docker_path_size_info: &Disk, + envs: Vec<(&str, &str)>, +) -> Result { let docker_max_disk_percentage_usage_before_purge = 60; // arbitrary percentage that should make the job anytime - let docker_percentage_used = docker_path_size_info.available_space() * 100 / docker_path_size_info.total_space(); + let docker_percentage_used = + docker_path_size_info.get_available_space() * 100 / docker_path_size_info.get_total_space(); if docker_percentage_used > docker_max_disk_percentage_usage_before_purge { warn!( - "Docker disk usage is higher than {}%, requesting cleaning", - docker_max_disk_percentage_usage_before_purge + "Docker disk usage ({}%) is higher than {}%, requesting cleaning", + docker_percentage_used, docker_max_disk_percentage_usage_before_purge ); - match docker_prune_images(envs) { - Err(e) => error!("error while purging docker images: {:?}", e.message), - _ => info!("docker images have been purged"), + return match docker_prune_images(envs) { + Err(e) => { + error!("error while purging docker images: {:?}", e.message); + Err(e) + } + _ => Ok("docker images have been purged".to_string()), }; }; + + Ok(format!( + "no need to purge old docker images, only {}% disk used", + docker_percentage_used + )) } fn docker_prune_images(envs: Vec<(&str, &str)>) -> Result<(), SimpleError> { diff --git a/tests/aws/aws_databases.rs b/tests/aws/aws_databases.rs index 60b4e5ef..e9ee7676 100644 --- a/tests/aws/aws_databases.rs +++ b/tests/aws/aws_databases.rs @@ -290,7 +290,7 @@ fn test_postgresql_configuration(context: Context, mut environment: Environment, let database_username = "superuser".to_string(); let database_password = generate_id(); - let is_rds = match environment.kind { + let _is_rds = match environment.kind { Kind::Production => true, Kind::Development => false, }; @@ -671,7 +671,7 @@ fn test_mysql_configuration(context: Context, mut environment: Environment, vers let database_username = "superuser".to_string(); let database_password = generate_id(); - let is_rds = match environment.kind { + let _is_rds = match environment.kind { Kind::Production => true, Kind::Development => false, }; @@ -931,7 +931,6 @@ fn redis_v5_deploy_a_working_dev_environment() { #[test] fn redis_v6_deploy_a_working_dev_environment() { let context = context(); - const TEST_NAME: &str = "redis_v6_0_dev"; let environment = test_utilities::aws::working_minimal_environment(&context); test_redis_configuration(context, environment, "6", "redis_v6_deploy_a_working_dev_environment"); } diff --git a/tests/aws/aws_environment.rs b/tests/aws/aws_environment.rs index 9fc38e06..a9cd7d41 100644 --- a/tests/aws/aws_environment.rs +++ b/tests/aws/aws_environment.rs @@ -561,7 +561,7 @@ fn deploy_ok_fail_fail_ok_environment() { // not working 2 let context_for_not_working_2 = context.clone_not_same_execution_id(); - let mut not_working_env_2 = not_working_env_1.clone(); + let not_working_env_2 = not_working_env_1.clone(); // work for delete let context_for_delete = context.clone_not_same_execution_id(); diff --git a/tests/aws/aws_kubernetes.rs b/tests/aws/aws_kubernetes.rs index 778991f2..548141f2 100644 --- a/tests/aws/aws_kubernetes.rs +++ b/tests/aws/aws_kubernetes.rs @@ -8,7 +8,6 @@ use gethostname; use test_utilities::aws::AWS_KUBERNETES_VERSION; use tracing::{span, Level}; -use qovery_engine::build_platform::GitRepository; use qovery_engine::cloud_provider::aws::kubernetes::EKS; use qovery_engine::transaction::TransactionResult; diff --git a/tests/digitalocean/do_databases.rs b/tests/digitalocean/do_databases.rs index 026dc86c..0b2407e8 100644 --- a/tests/digitalocean/do_databases.rs +++ b/tests/digitalocean/do_databases.rs @@ -1,4 +1,4 @@ -use test_utilities::utilities::{context, engine_run_test, init}; +use test_utilities::utilities::{context, engine_run_test}; use tracing::{span, Level}; use qovery_engine::models::{Action, Clone2, EnvironmentAction}; @@ -15,7 +15,7 @@ fn deploy_one_postgresql() { let context = context(); let context_for_deletion = context.clone_not_same_execution_id(); - let mut environment = test_utilities::aws::working_minimal_environment(&context); + let environment = test_utilities::aws::working_minimal_environment(&context); let mut environment_delete = environment.clone(); environment_delete.action = Action::Delete;