fix: docker prune didn't worked properly on docker

This commit is contained in:
Pierre Mavro
2021-02-21 18:47:47 +01:00
parent 8ac4f11e9b
commit 118fdcdaf6
7 changed files with 195 additions and 145 deletions

241
Cargo.lock generated
View File

@@ -306,7 +306,7 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171"
dependencies = [
"core-foundation-sys",
"core-foundation-sys 0.7.0",
"libc",
]
@@ -316,6 +316,12 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac"
[[package]]
name = "core-foundation-sys"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b"
[[package]]
name = "cpuid-bool"
version = "0.1.2"
@@ -331,17 +337,38 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-utils 0.8.2",
]
[[package]]
name = "crossbeam-deque"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
"crossbeam-epoch 0.8.2",
"crossbeam-utils 0.7.2",
"maybe-uninit",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-epoch 0.9.2",
"crossbeam-utils 0.8.2",
]
[[package]]
name = "crossbeam-epoch"
version = "0.8.2"
@@ -350,10 +377,24 @@ checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
dependencies = [
"autocfg 1.0.1",
"cfg-if 0.1.10",
"crossbeam-utils",
"crossbeam-utils 0.7.2",
"lazy_static",
"maybe-uninit",
"memoffset",
"memoffset 0.5.6",
"scopeguard",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d60ab4a8dba064f2fbb5aa270c28da5cf4bbd0e72dae1140a6b0353a779dbe00"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-utils 0.8.2",
"lazy_static",
"loom 0.4.0",
"memoffset 0.6.1",
"scopeguard",
]
@@ -364,7 +405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570"
dependencies = [
"cfg-if 0.1.10",
"crossbeam-utils",
"crossbeam-utils 0.7.2",
"maybe-uninit",
]
@@ -379,6 +420,18 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bae8f328835f8f5a6ceb6a7842a7f2d0c03692adb5c889347235d59194731fe3"
dependencies = [
"autocfg 1.0.1",
"cfg-if 1.0.0",
"lazy_static",
"loom 0.4.0",
]
[[package]]
name = "crypto-mac"
version = "0.8.0"
@@ -498,6 +551,12 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0"
[[package]]
name = "doc-comment"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
[[package]]
name = "dtoa"
version = "0.4.6"
@@ -531,20 +590,6 @@ dependencies = [
"syn 1.0.48",
]
[[package]]
name = "err-derive"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22deed3a8124cff5fa835713fa105621e43bbdc46690c3a6b68328a012d350d4"
dependencies = [
"proc-macro-error",
"proc-macro2 1.0.24",
"quote 1.0.7",
"rustversion",
"syn 1.0.48",
"synstructure",
]
[[package]]
name = "error-chain"
version = "0.12.4"
@@ -627,16 +672,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
"libc",
"winapi 0.3.9",
]
[[package]]
name = "fuchsia-cprng"
version = "0.1.1"
@@ -1144,7 +1179,7 @@ version = "0.4.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22dcbf2a4a289528dbef21686354904e1c694ac642610a9bff9e7df730d9ec72"
dependencies = [
"crossbeam-utils",
"crossbeam-utils 0.7.2",
"globset",
"lazy_static",
"log",
@@ -1244,9 +1279,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.79"
version = "0.2.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743"
checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c"
[[package]]
name = "libgit2-sys"
@@ -1325,6 +1360,17 @@ dependencies = [
"serde_json",
]
[[package]]
name = "loom"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d44c73b4636e497b4917eb21c33539efa3816741a2d3ff26c6316f1b529481a4"
dependencies = [
"cfg-if 1.0.0",
"generator",
"scoped-tls",
]
[[package]]
name = "lru-cache"
version = "0.1.2"
@@ -1388,6 +1434,15 @@ dependencies = [
"autocfg 1.0.1",
]
[[package]]
name = "memoffset"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87"
dependencies = [
"autocfg 1.0.1",
]
[[package]]
name = "mime"
version = "0.3.16"
@@ -1507,6 +1562,15 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "ntapi"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44"
dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "num-integer"
version = "0.1.43"
@@ -1628,15 +1692,6 @@ dependencies = [
"regex",
]
[[package]]
name = "partition-identity"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec13ba9a0eec5c10a89f6ec1b6e9e2ef7d29b810d771355abbd1c43cae003ed6"
dependencies = [
"err-derive",
]
[[package]]
name = "percent-encoding"
version = "1.0.1"
@@ -1756,30 +1811,6 @@ version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20"
[[package]]
name = "proc-macro-error"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
dependencies = [
"proc-macro-error-attr",
"proc-macro2 1.0.24",
"quote 1.0.7",
"syn 1.0.48",
"version_check",
]
[[package]]
name = "proc-macro-error-attr"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
dependencies = [
"proc-macro2 1.0.24",
"quote 1.0.7",
"version_check",
]
[[package]]
name = "proc-macro-hack"
version = "0.5.18"
@@ -1810,16 +1841,6 @@ dependencies = [
"unicode-xid 0.2.1",
]
[[package]]
name = "proc-mounts"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ad7e9c8d1b8c20f16a84d61d7c4c0325a5837c1307a2491b509cd92fb4e4442"
dependencies = [
"lazy_static",
"partition-identity",
]
[[package]]
name = "publicsuffix"
version = "1.5.4"
@@ -1843,12 +1864,10 @@ dependencies = [
"digitalocean",
"dirs 3.0.1",
"flate2",
"fs2",
"futures 0.3.7",
"gethostname",
"git2",
"itertools",
"proc-mounts",
"rand 0.7.3",
"reqwest 0.10.8",
"retry",
@@ -1863,6 +1882,7 @@ dependencies = [
"serde",
"serde_derive",
"serde_json",
"sysinfo",
"tar",
"tera",
"test-utilities",
@@ -2067,6 +2087,31 @@ dependencies = [
"rand_core 0.3.1",
]
[[package]]
name = "rayon"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674"
dependencies = [
"autocfg 1.0.1",
"crossbeam-deque 0.8.0",
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
dependencies = [
"crossbeam-channel",
"crossbeam-deque 0.8.0",
"crossbeam-utils 0.8.2",
"lazy_static",
"num_cpus",
]
[[package]]
name = "rdrand"
version = "0.4.0"
@@ -2372,7 +2417,7 @@ dependencies = [
"base64 0.12.3",
"blake2b_simd",
"constant_time_eq",
"crossbeam-utils",
"crossbeam-utils 0.7.2",
]
[[package]]
@@ -2409,12 +2454,6 @@ dependencies = [
"semver",
]
[[package]]
name = "rustversion"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb5d2a036dc6d2d8fd16fde3498b04306e29bd193bf306a57427019b823d5acd"
[[package]]
name = "ryu"
version = "1.0.5"
@@ -2460,7 +2499,7 @@ checksum = "64808902d7d99f78eaddd2b4e2509713babc3dc3c85ad6f4c447680f3c01e535"
dependencies = [
"bitflags",
"core-foundation",
"core-foundation-sys",
"core-foundation-sys 0.7.0",
"libc",
"security-framework-sys",
]
@@ -2471,7 +2510,7 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17bf11d99252f512695eb468de5516e5cf75455521e69dfe343f3b74e4748405"
dependencies = [
"core-foundation-sys",
"core-foundation-sys 0.7.0",
"libc",
]
@@ -2583,7 +2622,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b4921be914e16899a80adefb821f8ddb7974e3f1250223575a44ed994882127"
dependencies = [
"lazy_static",
"loom",
"loom 0.3.6",
]
[[package]]
@@ -2751,6 +2790,22 @@ dependencies = [
"unicode-xid 0.2.1",
]
[[package]]
name = "sysinfo"
version = "0.16.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c280c91abd1aed2e36be1bc8f56fbc7a2acbb2b58fbcac9641510179cc72dd9"
dependencies = [
"cfg-if 1.0.0",
"core-foundation-sys 0.8.2",
"doc-comment",
"libc",
"ntapi",
"once_cell",
"rayon",
"winapi 0.3.9",
]
[[package]]
name = "tar"
version = "0.4.30"
@@ -2973,7 +3028,7 @@ version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671"
dependencies = [
"crossbeam-utils",
"crossbeam-utils 0.7.2",
"futures 0.1.30",
]
@@ -3005,7 +3060,7 @@ version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351"
dependencies = [
"crossbeam-utils",
"crossbeam-utils 0.7.2",
"futures 0.1.30",
"lazy_static",
"log",
@@ -3048,9 +3103,9 @@ version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89"
dependencies = [
"crossbeam-deque",
"crossbeam-deque 0.7.3",
"crossbeam-queue",
"crossbeam-utils",
"crossbeam-utils 0.7.2",
"futures 0.1.30",
"lazy_static",
"log",
@@ -3065,7 +3120,7 @@ version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296"
dependencies = [
"crossbeam-utils",
"crossbeam-utils 0.7.2",
"futures 0.1.30",
"slab",
"tokio-executor",

View File

@@ -36,8 +36,7 @@ tracing-subscriber = "0.2"
# shiplift = "0.6.0"
# Filesystem
fs2 = "0.4.3"
proc-mounts = "0.2.4"
sysinfo = "0.16.4"
# Jinja2
tera = "1.3.1"

View File

@@ -1,7 +1,3 @@
use std::path::Path;
use fs2::FsStats;
use crate::build_platform::{Build, BuildPlatform, BuildResult, Image, Kind};
use crate::error::{EngineError, EngineErrorCause, SimpleError, SimpleErrorKind};
use crate::fs::workspace_directory;
@@ -11,6 +7,8 @@ use crate::models::{
};
use crate::{cmd, git};
use std::env;
use std::path::Path;
use sysinfo::{Disk, DiskExt, SystemExt};
/// https://buildpacks.io/
const BUILDPACKS_BUILDERS: [&str; 1] = [
@@ -364,41 +362,29 @@ impl BuildPlatform for LocalDocker {
}
}
// ensure there is enough disk space left before building a new image
let docker_path_string = "/var/lib/docker";
let docker_path = Path::new(docker_path_string);
// ensure docker_path is a mounted volume, otherwise ignore because it's not what Qovery does in production
// ex: this cause regular cleanup on CI, leading to random tests errors
match env::var_os("CI") {
Some(_) => info!("CI environment variable found, no docker prune will be made"),
None => {
// ensure there is enough disk space left before building a new image
let docker_path_string = "/var/lib/docker";
let docker_path = Path::new(docker_path_string);
if docker_path.exists() {
let mounted_disks = proc_mounts::MountList::new();
// get system info
let mut system = sysinfo::System::new_all();
system.refresh_all();
// ensure docker_path is a mounted volume, otherwise ignore because it's not what Qovery does in production
// ex: this cause regular cleanup on CI, leading to random tests errors
match mounted_disks {
Ok(m) => match m.get_mount_by_dest(Path::new(docker_path)) {
Some(_) => {
let ci_env_var = "CI";
match env::var_os(ci_env_var) {
Some(_) => {}
None => {
// only used in production on Linux OS
let docker_path_size_info = match fs2::statvfs(docker_path) {
Ok(fs_stats) => fs_stats,
Err(err) => {
return Err(self.engine_error(EngineErrorCause::Internal, format!("{:?}", err)));
}
};
check_docker_space_usage_and_clean(docker_path_size_info, self.get_docker_host_envs());
}
for disk in system.get_disks() {
if disk.get_mount_point() == docker_path {
match check_docker_space_usage_and_clean(disk, self.get_docker_host_envs()) {
Ok(msg) => info!("{:?}", msg),
Err(e) => error!("{:?}", e.message),
}
}
None => info!(
"ignoring docker cleanup because {} is not a mounted volume",
docker_path_string
),
},
Err(_) => error!("wasn't able to get info from {} volume", docker_path_string),
};
break;
};
}
}
}
let application_id = build.image.application_id.clone();
@@ -479,21 +465,33 @@ impl Listen for LocalDocker {
}
}
fn check_docker_space_usage_and_clean(docker_path_size_info: FsStats, envs: Vec<(&str, &str)>) {
fn check_docker_space_usage_and_clean(
docker_path_size_info: &Disk,
envs: Vec<(&str, &str)>,
) -> Result<String, SimpleError> {
let docker_max_disk_percentage_usage_before_purge = 60; // arbitrary percentage that should make the job anytime
let docker_percentage_used = docker_path_size_info.available_space() * 100 / docker_path_size_info.total_space();
let docker_percentage_used =
docker_path_size_info.get_available_space() * 100 / docker_path_size_info.get_total_space();
if docker_percentage_used > docker_max_disk_percentage_usage_before_purge {
warn!(
"Docker disk usage is higher than {}%, requesting cleaning",
docker_max_disk_percentage_usage_before_purge
"Docker disk usage ({}%) is higher than {}%, requesting cleaning",
docker_percentage_used, docker_max_disk_percentage_usage_before_purge
);
match docker_prune_images(envs) {
Err(e) => error!("error while purging docker images: {:?}", e.message),
_ => info!("docker images have been purged"),
return match docker_prune_images(envs) {
Err(e) => {
error!("error while purging docker images: {:?}", e.message);
Err(e)
}
_ => Ok("docker images have been purged".to_string()),
};
};
Ok(format!(
"no need to purge old docker images, only {}% disk used",
docker_percentage_used
))
}
fn docker_prune_images(envs: Vec<(&str, &str)>) -> Result<(), SimpleError> {

View File

@@ -290,7 +290,7 @@ fn test_postgresql_configuration(context: Context, mut environment: Environment,
let database_username = "superuser".to_string();
let database_password = generate_id();
let is_rds = match environment.kind {
let _is_rds = match environment.kind {
Kind::Production => true,
Kind::Development => false,
};
@@ -671,7 +671,7 @@ fn test_mysql_configuration(context: Context, mut environment: Environment, vers
let database_username = "superuser".to_string();
let database_password = generate_id();
let is_rds = match environment.kind {
let _is_rds = match environment.kind {
Kind::Production => true,
Kind::Development => false,
};
@@ -931,7 +931,6 @@ fn redis_v5_deploy_a_working_dev_environment() {
#[test]
fn redis_v6_deploy_a_working_dev_environment() {
let context = context();
const TEST_NAME: &str = "redis_v6_0_dev";
let environment = test_utilities::aws::working_minimal_environment(&context);
test_redis_configuration(context, environment, "6", "redis_v6_deploy_a_working_dev_environment");
}

View File

@@ -561,7 +561,7 @@ fn deploy_ok_fail_fail_ok_environment() {
// not working 2
let context_for_not_working_2 = context.clone_not_same_execution_id();
let mut not_working_env_2 = not_working_env_1.clone();
let not_working_env_2 = not_working_env_1.clone();
// work for delete
let context_for_delete = context.clone_not_same_execution_id();

View File

@@ -8,7 +8,6 @@ use gethostname;
use test_utilities::aws::AWS_KUBERNETES_VERSION;
use tracing::{span, Level};
use qovery_engine::build_platform::GitRepository;
use qovery_engine::cloud_provider::aws::kubernetes::EKS;
use qovery_engine::transaction::TransactionResult;

View File

@@ -1,4 +1,4 @@
use test_utilities::utilities::{context, engine_run_test, init};
use test_utilities::utilities::{context, engine_run_test};
use tracing::{span, Level};
use qovery_engine::models::{Action, Clone2, EnvironmentAction};
@@ -15,7 +15,7 @@ fn deploy_one_postgresql() {
let context = context();
let context_for_deletion = context.clone_not_same_execution_id();
let mut environment = test_utilities::aws::working_minimal_environment(&context);
let environment = test_utilities::aws::working_minimal_environment(&context);
let mut environment_delete = environment.clone();
environment_delete.action = Action::Delete;