diff --git a/Cargo.lock b/Cargo.lock index 3b38f581..e42eba76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2104,6 +2104,7 @@ dependencies = [ "tempfile", "tera", "test-utilities", + "thiserror", "timeout-readwrite", "tokio 1.10.0", "tracing", @@ -3257,18 +3258,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.26" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93119e4feac1cbe6c798c34d3a53ea0026b0b1de6a120deef895137c0529bfe2" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.26" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "060d69a0afe7796bf42e9e2ff91f5ee691fb15c53d38b4b62a9a53eb23164745" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" dependencies = [ "proc-macro2 1.0.28", "quote 1.0.9", diff --git a/Cargo.toml b/Cargo.toml index f3b8d6b5..765acbd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ lazy_static = "1.4.0" uuid = { version = "0.8", features = ["v4", "serde"] } url = "2.2.2" function_name = "0.2.0" +thiserror = "1.0.30" # FIXME use https://crates.io/crates/blocking instead of runtime.rs diff --git a/src/build_platform/local_docker.rs b/src/build_platform/local_docker.rs index c13b4a22..1eb121e2 100644 --- a/src/build_platform/local_docker.rs +++ b/src/build_platform/local_docker.rs @@ -5,13 +5,14 @@ use chrono::Duration; use sysinfo::{Disk, DiskExt, SystemExt}; use crate::build_platform::{Build, BuildPlatform, BuildResult, Image, Kind}; +use crate::cmd::utilities::QoveryCommand; use crate::error::{EngineError, EngineErrorCause, SimpleError, SimpleErrorKind}; use crate::fs::workspace_directory; +use crate::git; use crate::git::checkout_submodules; use crate::models::{ Context, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope, }; -use crate::{cmd, git}; const BUILD_DURATION_TIMEOUT_MIN: i64 = 30; @@ -42,14 +43,12 @@ impl LocalDocker { } fn image_does_exist(&self, image: &Image) -> Result { - Ok(matches!( - crate::cmd::utilities::exec( - "docker", - vec!["image", "inspect", image.name_with_tag().as_str()], - &self.get_docker_host_envs(), - ), - Ok(_) - )) + let mut cmd = QoveryCommand::new( + "docker", + &vec!["image", "inspect", image.name_with_tag().as_str()], + &self.get_docker_host_envs(), + ); + Ok(matches!(cmd.exec(), Ok(_))) } fn get_docker_host_envs(&self) -> Vec<(&str, &str)> { @@ -101,37 +100,34 @@ impl LocalDocker { docker_args.push(into_dir_docker_style); // docker build - let exit_status = cmd::utilities::exec_with_envs_and_output( - "docker", - docker_args, - self.get_docker_host_envs(), + let mut cmd = QoveryCommand::new("docker", &docker_args, &self.get_docker_host_envs()); + + let exit_status = cmd.exec_with_timeout( + Duration::minutes(BUILD_DURATION_TIMEOUT_MIN), |line| { - let line_string = line.unwrap(); - info!("{}", line_string.as_str()); + info!("{}", line); lh.deployment_in_progress(ProgressInfo::new( ProgressScope::Application { id: build.image.application_id.clone(), }, ProgressLevel::Info, - Some(line_string.as_str()), + Some(line), self.context.execution_id(), )); }, |line| { - let line_string = line.unwrap(); - error!("{}", line_string.as_str()); + error!("{}", line); lh.deployment_in_progress(ProgressInfo::new( ProgressScope::Application { id: build.image.application_id.clone(), }, ProgressLevel::Warn, - Some(line_string.as_str()), + Some(line), self.context.execution_id(), )); }, - Duration::minutes(BUILD_DURATION_TIMEOUT_MIN), ); match exit_status { @@ -161,7 +157,7 @@ impl LocalDocker { let args = self.context.docker_build_options(); - let mut exit_status: Result, SimpleError> = + let mut exit_status: Result<(), SimpleError> = Err(SimpleError::new(SimpleErrorKind::Other, Some("no builder names"))); for builder_name in BUILDPACKS_BUILDERS.iter() { @@ -211,38 +207,36 @@ impl LocalDocker { } // buildpacks build - exit_status = cmd::utilities::exec_with_envs_and_output( - "pack", - buildpacks_args, - self.get_docker_host_envs(), - |line| { - let line_string = line.unwrap(); - info!("{}", line_string.as_str()); + let mut cmd = QoveryCommand::new("pack", &buildpacks_args, &self.get_docker_host_envs()); + exit_status = cmd + .exec_with_timeout( + Duration::minutes(BUILD_DURATION_TIMEOUT_MIN), + |line| { + info!("{}", line); - lh.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: build.image.application_id.clone(), - }, - ProgressLevel::Info, - Some(line_string.as_str()), - self.context.execution_id(), - )); - }, - |line| { - let line_string = line.unwrap(); - error!("{}", line_string.as_str()); + lh.deployment_in_progress(ProgressInfo::new( + ProgressScope::Application { + id: build.image.application_id.clone(), + }, + ProgressLevel::Info, + Some(line), + self.context.execution_id(), + )); + }, + |line| { + error!("{}", line); - lh.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: build.image.application_id.clone(), - }, - ProgressLevel::Warn, - Some(line_string.as_str()), - self.context.execution_id(), - )); - }, - Duration::minutes(BUILD_DURATION_TIMEOUT_MIN), - ); + lh.deployment_in_progress(ProgressInfo::new( + ProgressScope::Application { + id: build.image.application_id.clone(), + }, + ProgressLevel::Warn, + Some(line), + self.context.execution_id(), + )); + }, + ) + .map_err(|err| SimpleError::new(SimpleErrorKind::Other, Some(format!("{}", err)))); if exit_status.is_ok() { // quit now if the builder successfully build the app @@ -543,22 +537,18 @@ fn docker_prune_images(envs: Vec<(&str, &str)>) -> Result<(), SimpleError> { ]; for prune in all_prunes_commands { - match cmd::utilities::exec_with_envs_and_output( - "docker", - prune.clone(), - envs.clone(), - |line| { - let line_string = line.unwrap_or_default(); - debug!("{}", line_string.as_str()); - }, - |line| { - let line_string = line.unwrap_or_default(); - debug!("{}", line_string.as_str()); - }, + let mut cmd = QoveryCommand::new("docker", &prune, &envs); + match cmd.exec_with_timeout( Duration::minutes(BUILD_DURATION_TIMEOUT_MIN), + |line| { + debug!("{}", line); + }, + |line| { + debug!("{}", line); + }, ) { Ok(_) => {} - Err(e) => error!("error while puring {}. {:?}", prune[0], e.message), + Err(e) => error!("error while puring {}. {:?}", prune[0], e), }; } diff --git a/src/cloud_provider/digitalocean/kubernetes/cidr.rs b/src/cloud_provider/digitalocean/kubernetes/cidr.rs index ca7c5626..3665be25 100644 --- a/src/cloud_provider/digitalocean/kubernetes/cidr.rs +++ b/src/cloud_provider/digitalocean/kubernetes/cidr.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; use std::borrow::Borrow; -use crate::cmd::utilities; +use crate::cmd::utilities::QoveryCommand; #[derive(Serialize, Deserialize, Debug)] pub struct DoVpc { @@ -16,16 +16,15 @@ pub struct DoVpc { pub fn get_used_cidr_on_region(token: &str) { let mut output_from_cli = String::new(); - let _ = utilities::exec_with_output( - "doctl", - vec!["vpcs", "list", "--output", "json", "-t", token], - |r_out| match r_out { - Ok(s) => output_from_cli.push_str(&s), - Err(e) => error!("DOCTL CLI does not respond correctly {}", e), - }, - |r_err| match r_err { - Ok(s) => error!("DOCTL CLI error from cmd inserted, please check vpcs list command{}", s), - Err(e) => error!("DOCTL CLI does not respond correctly {}", e), + + let mut cmd = QoveryCommand::new("doctl", &vec!["vpcs", "list", "--output", "json", "-t", token], &vec![]); + let _ = cmd.exec_with_output( + |r_out| output_from_cli.push_str(&r_out), + |r_err| { + error!( + "DOCTL CLI error from cmd inserted, please check vpcs list command{}", + r_err + ) }, ); diff --git a/src/cmd/helm.rs b/src/cmd/helm.rs index 23737518..6e8970fa 100644 --- a/src/cmd/helm.rs +++ b/src/cmd/helm.rs @@ -7,8 +7,8 @@ use crate::cloud_provider::helm::{deploy_charts_levels, ChartInfo, CommonChart}; use crate::cloud_provider::service::ServiceType; use crate::cmd::helm::HelmLockErrors::{IncorrectFormatDate, NotYetExpired, ParsingError}; use crate::cmd::kubectl::{kubectl_exec_delete_secret, kubectl_exec_get_secrets}; -use crate::cmd::structs::{Helm, HelmChart, HelmHistoryRow, Item, KubernetesList}; -use crate::cmd::utilities::exec_with_envs_and_output; +use crate::cmd::structs::{HelmChart, HelmHistoryRow, HelmListItem, Item, KubernetesList}; +use crate::cmd::utilities::QoveryCommand; use crate::error::{SimpleError, SimpleErrorKind}; use chrono::{DateTime, Duration, Utc}; use core::time; @@ -226,50 +226,40 @@ where let helm_ret = helm_exec_with_output( args, envs.clone(), - |out| match out { - Ok(line) => { - info!("{}", line); - json_output_string = line - } - Err(err) => error!("{}", &err), + |line| { + info!("{}", line); + json_output_string = line }, - |out| match out { - Ok(line) => { - if line.contains("another operation (install/upgrade/rollback) is in progress") { - error_message = format!("helm lock detected for {}, looking for cleaning lock", chart.name); - helm_error_during_deployment.message = Some(error_message.clone()); - warn!("{}. {}", &error_message, &line); - should_clean_helm_lock = true; - return; - } - - if !chart.parse_stderr_for_error { - warn!("chart {}: {}", chart.name, line); - return; - } - - // helm errors are not json formatted unfortunately - if line.contains("has been rolled back") { - error_message = format!("deployment {} has been rolled back", chart.name); - helm_error_during_deployment.message = Some(error_message.clone()); - warn!("{}. {}", &error_message, &line); - } else if line.contains("has been uninstalled") { - error_message = format!("deployment {} has been uninstalled due to failure", chart.name); - helm_error_during_deployment.message = Some(error_message.clone()); - warn!("{}. {}", &error_message, &line); - // special fix for prometheus operator - } else if line.contains("info: skipping unknown hook: \"crd-install\"") { - debug!("chart {}: {}", chart.name, line); - } else { - error_message = format!("deployment {} has failed", chart.name); - helm_error_during_deployment.message = Some(error_message.clone()); - error!("{}. {}", &error_message, &line); - } - } - Err(err) => { - error_message = format!("helm chart {} failed before deployment. {:?}", chart.name, err); + |line| { + if line.contains("another operation (install/upgrade/rollback) is in progress") { + error_message = format!("helm lock detected for {}, looking for cleaning lock", chart.name); helm_error_during_deployment.message = Some(error_message.clone()); - error!("{}", error_message); + warn!("{}. {}", &error_message, &line); + should_clean_helm_lock = true; + return; + } + + if !chart.parse_stderr_for_error { + warn!("chart {}: {}", chart.name, line); + return; + } + + // helm errors are not json formatted unfortunately + if line.contains("has been rolled back") { + error_message = format!("deployment {} has been rolled back", chart.name); + helm_error_during_deployment.message = Some(error_message.clone()); + warn!("{}. {}", &error_message, &line); + } else if line.contains("has been uninstalled") { + error_message = format!("deployment {} has been uninstalled due to failure", chart.name); + helm_error_during_deployment.message = Some(error_message.clone()); + warn!("{}. {}", &error_message, &line); + // special fix for prometheus operator + } else if line.contains("info: skipping unknown hook: \"crd-install\"") { + debug!("chart {}: {}", chart.name, line); + } else { + error_message = format!("deployment {} has failed", chart.name); + helm_error_during_deployment.message = Some(error_message.clone()); + error!("{}. {}", &error_message, &line); } }, ); @@ -473,14 +463,8 @@ where &chart.name, ], envs.clone(), - |out| match out { - Ok(line) => info!("{}", line.as_str()), - Err(err) => error!("{}", err), - }, - |out| match out { - Ok(line) => error!("{}", line.as_str()), - Err(err) => error!("{}", err), - }, + |line| info!("{}", line.as_str()), + |line| error!("{}", line.as_str()), ) } @@ -503,14 +487,8 @@ where release_name, ], envs, - |out| match out { - Ok(line) => info!("{}", line.as_str()), - Err(err) => error!("{}", err), - }, - |out| match out { - Ok(line) => error!("{}", line.as_str()), - Err(err) => error!("{}", err), - }, + |line| info!("{}", line.as_str()), + |line| error!("{}", line.as_str()), ) } @@ -537,19 +515,13 @@ where release_name, ], envs.clone(), - |out| match out { - Ok(line) => output_string = line, - Err(err) => error!("{:?}", err), - }, - |out| match out { - Ok(line) => { - if line.contains("Error: release: not found") { - info!("{}", line) - } else { - error!("{}", line) - } + |line| output_string = line, + |line| { + if line.contains("Error: release: not found") { + info!("{}", line) + } else { + error!("{}", line) } - Err(err) => error!("{:?}", err), }, ) { Ok(_) => info!("Helm history success for release name: {}", release_name), @@ -591,14 +563,8 @@ where kubernetes_config.as_ref().to_str().unwrap(), ], envs.clone(), - |out| match out { - Ok(line) => output_vec.push(line), - Err(err) => error!("{:?}", err), - }, - |out| match out { - Ok(line) => error!("{}", line), - Err(err) => error!("{:?}", err), - }, + |line| output_vec.push(line), + |line| error!("{}", line), ) { Ok(_) => info!( "Helm uninstall succeed for {} on namespace {}", @@ -643,15 +609,14 @@ where override_file, ], envs, - |out| match out { - Ok(line) => info!("{}", line.as_str()), - Err(err) => error!("{}", err), - }, - |out| match out { + |line| info!("{}", line.as_str()), + |line| { // don't crash errors if releases are not found - Ok(line) if line.contains("Error: release: not found") => info!("{}", line.as_str()), - Ok(line) => error!("{}", line.as_str()), - Err(err) => error!("{}", err), + if line.contains("Error: release: not found") { + info!("{}", line) + } else { + error!("{}", line) + } }, ) } @@ -769,21 +734,10 @@ where None => helm_args.push("-A"), } - let _ = helm_exec_with_output( - helm_args, - envs, - |out| match out { - Ok(line) => output_vec.push(line), - Err(err) => error!("{}", err), - }, - |out| match out { - Ok(line) => error!("{}", line.as_str()), - Err(err) => error!("{}", err), - }, - ); + let _ = helm_exec_with_output(helm_args, envs, |line| output_vec.push(line), |line| error!("{}", line)); let output_string: String = output_vec.join(""); - let values = serde_json::from_str::>(output_string.as_str()); + let values = serde_json::from_str::>(output_string.as_str()); let mut helms_charts: Vec = Vec::new(); match values { @@ -871,14 +825,8 @@ where .iter() .map(|x| (x.0.as_str(), x.1.as_str())) .collect(), - |out| match out { - Ok(line) => info!("{}", line), - Err(err) => error!("{}", &err), - }, - |out| match out { - Ok(line) => error!("{}", line), - Err(err) => error!("{}", err), - }, + |line| info!("{}", line), + |line| error!("{}", line), ) } @@ -887,10 +835,10 @@ pub fn helm_exec(args: Vec<&str>, envs: Vec<(&str, &str)>) -> Result<(), SimpleE args, envs, |line| { - span!(Level::INFO, "{}", "{}", line.unwrap()); + span!(Level::INFO, "{}", "{}", line); }, |line_err| { - span!(Level::INFO, "{}", "{}", line_err.unwrap()); + span!(Level::INFO, "{}", "{}", line_err); }, ) } @@ -902,26 +850,15 @@ pub fn helm_exec_with_output( stderr_output: X, ) -> Result<(), SimpleError> where - F: FnMut(Result), - X: FnMut(Result), + F: FnMut(String), + X: FnMut(String), { // Note: Helm CLI use spf13/cobra lib for the CLI; One function is mainly used to return an error if a command failed. // Helm returns an error each time a command does not succeed as they want. Which leads to handling error with status code 1 // It means that the command successfully ran, but it didn't terminate as expected - match exec_with_envs_and_output("helm", args, envs, stdout_output, stderr_output, Duration::max_value()) { - Err(err) => match err.kind { - SimpleErrorKind::Command(exit_status) => match exit_status.code() { - Some(exit_status_code) => { - if exit_status_code == 0 { - Ok(()) - } else { - Err(err) - } - } - None => Err(err), - }, - SimpleErrorKind::Other => Err(err), - }, + let mut cmd = QoveryCommand::new("helm", &args, &envs); + match cmd.exec_with_timeout(Duration::max_value(), stdout_output, stderr_output) { + Err(err) => Err(SimpleError::new(SimpleErrorKind::Other, Some(format!("{}", err)))), _ => Ok(()), } } diff --git a/src/cmd/kubectl.rs b/src/cmd/kubectl.rs index 734eff9c..00d74413 100644 --- a/src/cmd/kubectl.rs +++ b/src/cmd/kubectl.rs @@ -1,4 +1,3 @@ -use std::io::Error; use std::path::Path; use chrono::Duration; @@ -12,7 +11,7 @@ use crate::cmd::structs::{ Configmap, Daemonset, Item, KubernetesEvent, KubernetesJob, KubernetesKind, KubernetesList, KubernetesNode, KubernetesPod, KubernetesPodStatusPhase, KubernetesService, KubernetesVersion, LabelsContent, PVC, SVC, }; -use crate::cmd::utilities::exec_with_envs_and_output; +use crate::cmd::utilities::QoveryCommand; use crate::constants::KUBECONFIG; use crate::error::{SimpleError, SimpleErrorKind}; @@ -35,21 +34,16 @@ pub fn kubectl_exec_with_output( stderr_output: X, ) -> Result<(), SimpleError> where - F: FnMut(Result), - X: FnMut(Result), + F: FnMut(String), + X: FnMut(String), { - if let Err(err) = exec_with_envs_and_output( - "kubectl", - args.clone(), - envs.clone(), - stdout_output, - stderr_output, - Duration::max_value(), - ) { + let mut cmd = QoveryCommand::new("kubectl", &args, &envs); + + if let Err(err) = cmd.exec_with_timeout(Duration::max_value(), stdout_output, stderr_output) { let args_string = args.join(" "); let msg = format!("Error on command: kubectl {}. {:?}", args_string, &err); error!("{}", &msg); - return Err(err); + return Err(SimpleError::new(SimpleErrorKind::Other, Some(msg))); }; Ok(()) @@ -79,14 +73,8 @@ where "-o=custom-columns=:.status.containerStatuses..restartCount", ], _envs, - |out| match out { - Ok(line) => output_vec.push(line), - Err(err) => error!("{:?}", err), - }, - |out| match out { - Ok(line) => error!("{}", line), - Err(err) => error!("{:?}", err), - }, + |line| output_vec.push(line), + |line| error!("{}", line), )?; let output_string: String = output_vec.join(""); @@ -110,14 +98,8 @@ where let _ = kubectl_exec_with_output( vec!["get", "svc", "-n", namespace, service_name, "-o", "json"], _envs, - |out| match out { - Ok(line) => output_vec.push(line), - Err(err) => error!("{:?}", err), - }, - |out| match out { - Ok(line) => error!("{}", line), - Err(err) => error!("{:?}", err), - }, + |line| output_vec.push(line), + |line| error!("{}", line), )?; let output_string: String = output_vec.join("\n"); @@ -401,14 +383,8 @@ where let _ = kubectl_exec_with_output( vec!["create", "namespace", namespace], _envs, - |out| match out { - Ok(line) => info!("{}", line), - Err(err) => error!("{:?}", err), - }, - |out| match out { - Ok(line) => error!("{}", line), - Err(err) => error!("{:?}", err), - }, + |line| info!("{}", line), + |line| error!("{}", line), )?; } @@ -460,18 +436,7 @@ where _envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap())); _envs.extend(envs.clone()); - let _ = kubectl_exec_with_output( - command_args, - _envs, - |out| match out { - Ok(line) => info!("{}", line), - Err(err) => error!("{:?}", err), - }, - |out| match out { - Ok(line) => error!("{}", line), - Err(err) => error!("{:?}", err), - }, - )?; + let _ = kubectl_exec_with_output(command_args, _envs, |line| info!("{}", line), |line| error!("{}", line))?; Ok(()) } @@ -570,14 +535,8 @@ where let _ = kubectl_exec_with_output( vec!["delete", "namespace", namespace], _envs, - |out| match out { - Ok(line) => info!("{}", line), - Err(err) => error!("{:?}", err), - }, - |out| match out { - Ok(line) => error!("{}", line), - Err(err) => error!("{:?}", err), - }, + |line| info!("{}", line), + |line| error!("{}", line), )?; Ok(()) @@ -598,14 +557,8 @@ where let _ = kubectl_exec_with_output( vec!["delete", "crd", crd_name], _envs, - |out| match out { - Ok(line) => info!("{}", line), - Err(err) => error!("{:?}", err), - }, - |out| match out { - Ok(line) => error!("{}", line), - Err(err) => error!("{:?}", err), - }, + |line| info!("{}", line), + |line| error!("{}", line), )?; Ok(()) @@ -627,14 +580,8 @@ where let _ = kubectl_exec_with_output( vec!["-n", namespace, "delete", "secret", secret], _envs, - |out| match out { - Ok(line) => info!("{}", line), - Err(err) => error!("{:?}", err), - }, - |out| match out { - Ok(line) => error!("{}", line), - Err(err) => error!("{:?}", err), - }, + |line| info!("{}", line), + |line| error!("{}", line), )?; Ok(()) @@ -657,14 +604,8 @@ where let _ = kubectl_exec_with_output( vec!["logs", "--tail", "1000", "-n", namespace, "-l", selector], _envs, - |out| match out { - Ok(line) => output_vec.push(line), - Err(err) => error!("{:?}", err), - }, - |out| match out { - Ok(line) => error!("{}", line), - Err(err) => error!("{:?}", err), - }, + |line| output_vec.push(line), + |line| error!("{}", line), )?; Ok(output_vec) @@ -687,14 +628,8 @@ where let _ = kubectl_exec_with_output( vec!["describe", "pod", "-n", namespace, "-l", selector], _envs, - |out| match out { - Ok(line) => output_vec.push(line), - Err(err) => error!("{:?}", err), - }, - |out| match out { - Ok(line) => error!("{}", line), - Err(err) => error!("{:?}", err), - }, + |line| output_vec.push(line), + |line| error!("{}", line), )?; Ok(output_vec.join("\n")) @@ -747,14 +682,8 @@ where kubectl_exec_with_output( args, environment_variables.clone(), - |out| match out { - Ok(line) => info!("{}", line), - Err(err) => error!("{:?}", err), - }, - |out| match out { - Ok(line) => error!("{}", line), - Err(err) => error!("{:?}", err), - }, + |line| info!("{}", line), + |line| error!("{}", line), ) } @@ -853,17 +782,7 @@ where let args = vec!["get", "event", arg_namespace.as_str(), "--sort-by='.lastTimestamp'"]; let mut result_ok = String::new(); - let mut result_err = SimpleError::new(SimpleErrorKind::Other, Some(String::new())); - - match kubectl_exec_with_output( - args, - environment_variables, - |out| match out { - Ok(line) => result_ok = line, - Err(err) => result_err = SimpleError::from(err), - }, - |_| {}, - ) { + match kubectl_exec_with_output(args, environment_variables, |line| result_ok = line, |_| {}) { Ok(()) => Ok(result_ok), Err(err) => Err(err), } @@ -967,16 +886,8 @@ where &replicas_count.to_string(), ], _envs, - |out| { - if let Err(err) = out { - error!("{:?}", err) - } - }, - |out| { - if let Err(err) = out { - error!("{:?}", err) - } - }, + |_| {}, + |_| {}, ) } @@ -1022,16 +933,8 @@ where selector, ], _envs, - |out| { - if let Err(err) = out { - error!("{:?}", err) - } - }, - |out| { - if let Err(err) = out { - error!("{:?}", err) - } - }, + |_| {}, + |_| {}, )?; let condition = match replicas_count { @@ -1116,14 +1019,8 @@ where let _ = kubectl_exec_with_output( args.clone(), _envs.clone(), - |out| match out { - Ok(line) => output_vec.push(line), - Err(err) => error!("{:?}", err), - }, - |out| match out { - Ok(line) => error!("{}", line), - Err(err) => error!("{:?}", err), - }, + |line| output_vec.push(line), + |line| error!("{}", line), )?; let output_string: String = output_vec.join(""); diff --git a/src/cmd/structs.rs b/src/cmd/structs.rs index 9b5975a5..770689fa 100644 --- a/src/cmd/structs.rs +++ b/src/cmd/structs.rs @@ -250,7 +250,7 @@ pub struct ServerVersion { #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct Helm { +pub struct HelmListItem { pub name: String, pub namespace: String, pub revision: String, diff --git a/src/cmd/terraform.rs b/src/cmd/terraform.rs index f32c00af..a1daac74 100644 --- a/src/cmd/terraform.rs +++ b/src/cmd/terraform.rs @@ -2,10 +2,10 @@ use dirs::home_dir; use retry::delay::Fixed; use retry::OperationResult; -use crate::cmd::utilities::exec_with_envs_and_output; +use crate::cmd::utilities::QoveryCommand; use crate::constants::TF_PLUGIN_CACHE_DIR; +use crate::error::SimpleErrorKind::Other; use crate::error::{SimpleError, SimpleErrorKind}; -use chrono::Duration; use rand::Rng; use retry::Error::Operation; use std::{env, fs, thread, time}; @@ -209,30 +209,32 @@ pub fn terraform_exec(root_dir: &str, args: Vec<&str>) -> Result, Si let mut stdout = Vec::new(); let mut stderr = Vec::new(); - let result = exec_with_envs_and_output( - format!("{} terraform", root_dir).as_str(), - args, - vec![(TF_PLUGIN_CACHE_DIR, tf_plugin_cache_dir_value.as_str())], - |line: Result| { - let output = line.unwrap(); - stdout.push(output.clone()); - info!("{}", &output) + let mut cmd = QoveryCommand::new( + "terraform", + &args, + &vec![(TF_PLUGIN_CACHE_DIR, tf_plugin_cache_dir_value.as_str())], + ); + cmd.set_current_dir(root_dir); + + let result = cmd.exec_with_output( + |line| { + info!("{}", line); + stdout.push(line); }, - |line: Result| { - let output = line.unwrap(); - stderr.push(output.clone()); - error!("{}", &output); + |line| { + error!("{}", line); + stderr.push(line); }, - Duration::max_value(), ); stdout.extend(stderr); match result { - Ok(_) => Ok(result.unwrap()), - Err(mut e) => { - e.message = Some(stdout.join("\n")); - Err(e) + Ok(_) => Ok(stdout), + Err(e) => { + debug!("Terraform endend in error {:?}", e); + let err = SimpleError::new(Other, Some(stdout.join("\n"))); + Err(err) } } } diff --git a/src/cmd/utilities.rs b/src/cmd/utilities.rs index 532d63e2..7619f833 100644 --- a/src/cmd/utilities.rs +++ b/src/cmd/utilities.rs @@ -2,11 +2,11 @@ use std::ffi::OsStr; use std::io::{BufRead, BufReader}; use std::io::{Error, ErrorKind}; use std::path::Path; -use std::process::{Child, Command, Stdio}; +use std::process::{Command, ExitStatus, Stdio}; +use crate::cmd::utilities::CommandError::{ExecutionError, ExitStatusError, TimeoutError}; use crate::cmd::utilities::CommandOutputType::{STDERR, STDOUT}; -use crate::error::SimpleErrorKind::Other; -use crate::error::{SimpleError, SimpleErrorKind}; + use chrono::Duration; use itertools::Itertools; use std::time::Instant; @@ -17,244 +17,183 @@ enum CommandOutputType { STDERR(Result), } -fn command

(binary: P, args: Vec<&str>, envs: &[(&str, &str)], use_output: bool) -> Command -where - P: AsRef, -{ - let s_binary = binary - .as_ref() - .to_str() - .unwrap() - .split_whitespace() - .map(|x| x.to_string()) - .collect::>(); +#[derive(thiserror::Error, Debug)] +pub enum CommandError { + #[error("Error while executing command")] + ExecutionError(#[from] std::io::Error), - let (current_dir, _binary) = if s_binary.len() == 1 { - (None, s_binary.first().unwrap().clone()) - } else { - ( - Some(s_binary.first().unwrap().clone()), - s_binary.get(1).unwrap().clone(), - ) - }; + #[error("Command terminated with a non success exit status code: {0}")] + ExitStatusError(ExitStatus), - let mut cmd = Command::new(&_binary); - if use_output { - cmd.args(&args).stdout(Stdio::piped()).stderr(Stdio::piped()); - } else { - cmd.args(&args).stdout(Stdio::null()).stderr(Stdio::null()); - } - - if let Some(current_dir) = current_dir { - cmd.current_dir(current_dir); - } - - envs.iter().for_each(|(k, v)| { - cmd.env(k, v); - }); - - cmd + #[error("Command killed due to timeout: {0}")] + TimeoutError(String), } -pub fn exec

(binary: P, args: Vec<&str>, envs: &[(&str, &str)]) -> Result<(), SimpleError> -where - P: AsRef, -{ - let command_string = command_to_string(binary.as_ref(), &args, &envs); - - info!("command: {}", command_string.as_str()); - - let exit_status = match command(binary, args, envs, false).spawn().unwrap().wait() { - Ok(x) => x, - Err(err) => return Err(SimpleError::from(err)), - }; - - if exit_status.success() { - return Ok(()); - } - - Err(SimpleError::new( - SimpleErrorKind::Command(exit_status), - Some("error while executing an internal command"), - )) +pub struct QoveryCommand { + command: Command, } -fn _with_output(mut child: Child, mut stdout_output: F, mut stderr_output: X) -> Child -where - F: FnMut(Result), - X: FnMut(Result), -{ - let stdout_reader = BufReader::new(child.stdout.as_mut().unwrap()); - for line in stdout_reader.lines() { - stdout_output(line); +impl QoveryCommand { + pub fn new>(binary: P, args: &[&str], envs: &[(&str, &str)]) -> QoveryCommand { + let mut command = Command::new(binary.as_ref().as_os_str()); + command.args(args); + + envs.iter().for_each(|(k, v)| { + command.env(k, v); + }); + + QoveryCommand { command } } - let stderr_reader = BufReader::new(child.stderr.as_mut().unwrap()); - for line in stderr_reader.lines() { - stderr_output(line); + pub fn set_current_dir>(&mut self, root_dir: P) { + self.command.current_dir(root_dir); } - child -} + pub fn exec(&mut self) -> Result<(), CommandError> { + info!("command: {:?}", self.command); -pub fn exec_with_output( - binary: P, - args: Vec<&str>, - stdout_output: F, - stderr_output: X, -) -> Result<(), SimpleError> -where - P: AsRef, - F: FnMut(Result), - X: FnMut(Result), -{ - let command_string = command_to_string(binary.as_ref(), &args, &[]); - info!("command: {}", command_string.as_str()); + let mut cmd_handle = self + .command + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(ExecutionError)?; - let mut child = _with_output( - command(binary, args, &[], true).spawn().unwrap(), - stdout_output, - stderr_output, - ); + let exit_status = cmd_handle.wait().map_err(ExecutionError)?; - let exit_status = match child.wait() { - Ok(x) => x, - Err(err) => return Err(SimpleError::from(err)), - }; + if !exit_status.success() { + debug!( + "command: {:?} terminated with error exist status {:?}", + self.command, exit_status + ); + return Err(ExitStatusError(exit_status)); + } - if exit_status.success() { - return Ok(()); + Ok(()) } - Err(SimpleError::new( - SimpleErrorKind::Command(exit_status), - Some("error while executing an internal command"), - )) -} + pub fn exec_with_output( + &mut self, + stdout_output: STDOUT, + stderr_output: STDERR, + ) -> Result<(), CommandError> + where + STDOUT: FnMut(String), + STDERR: FnMut(String), + { + self.exec_with_timeout(Duration::max_value(), stdout_output, stderr_output) + } -pub fn exec_with_envs_and_output( - binary: P, - args: Vec<&str>, - envs: Vec<(&str, &str)>, - mut stdout_output: F, - mut stderr_output: X, - timeout: Duration, -) -> Result, SimpleError> -where - P: AsRef, - F: FnMut(Result), - X: FnMut(Result), -{ - assert!(timeout.num_seconds() > 0, "Timeout cannot be a 0 or negative duration"); + pub fn exec_with_timeout( + &mut self, + timeout: Duration, + mut stdout_output: STDOUT, + mut stderr_output: STDERR, + ) -> Result<(), CommandError> + where + STDOUT: FnMut(String), + STDERR: FnMut(String), + { + assert!(timeout.num_seconds() > 0, "Timeout cannot be a 0 or negative duration"); - let command_string = command_to_string(binary.as_ref(), &args, &envs); - info!( - "command with {}m timeout: {}", - timeout.num_minutes(), - command_string.as_str() - ); + info!("command: {:?}", self.command); + let mut cmd_handle = self + .command + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(ExecutionError)?; - // Start the process - // TODO(benjaminch): Make sure logging context is properly injected in thread here - let mut child_process = command(binary, args, &envs, true).spawn().unwrap(); - let process_start_time = Instant::now(); + let process_start_time = Instant::now(); - // Read stdout/stderr until timeout is reached - let reader_timeout = std::time::Duration::from_secs(10.min(timeout.num_seconds() as u64)); - let stdout_reader = BufReader::new(TimeoutReader::new(child_process.stdout.take().unwrap(), reader_timeout)) + // Read stdout/stderr until timeout is reached + let reader_timeout = std::time::Duration::from_secs(10.min(timeout.num_seconds() as u64)); + let stdout = cmd_handle.stdout.take().ok_or(ExecutionError(Error::new( + ErrorKind::BrokenPipe, + "Cannot get stdout for command", + )))?; + let stdout_reader = BufReader::new(TimeoutReader::new(stdout, reader_timeout)) + .lines() + .map(STDOUT); + + let stderr = cmd_handle.stderr.take().ok_or(ExecutionError(Error::new( + ErrorKind::BrokenPipe, + "Cannot get stderr for command", + )))?; + let stderr_reader = BufReader::new(TimeoutReader::new( + stderr, + std::time::Duration::from_secs(0), // don't block on stderr + )) .lines() - .map(STDOUT); + .map(STDERR); - let stderr_reader = BufReader::new(TimeoutReader::new( - child_process.stderr.take().unwrap(), - std::time::Duration::from_secs(0), // don't block on stderr - )) - .lines() - .map(STDERR); - let mut command_output = Vec::new(); - - for line in stdout_reader.interleave(stderr_reader) { - match line { - STDOUT(Err(ref err)) | STDERR(Err(ref err)) if err.kind() == ErrorKind::TimedOut => {} - STDOUT(line) => { - if let Ok(x) = &line { - command_output.push(x.to_string()) - } - stdout_output(line) + for line in stdout_reader.interleave(stderr_reader) { + match line { + STDOUT(Err(ref err)) | STDERR(Err(ref err)) if err.kind() == ErrorKind::TimedOut => {} + STDOUT(Ok(line)) => stdout_output(line), + STDERR(Ok(line)) => stderr_output(line), + STDOUT(Err(err)) => error!("Error on stdout of cmd {:?}: {:?}", self.command, err), + STDERR(Err(err)) => error!("Error on stderr of cmd {:?}: {:?}", self.command, err), } - STDERR(line) => { - if let Ok(x) = &line { - command_output.push(x.to_string()) - } - stderr_output(line) - } - } - if (process_start_time.elapsed().as_secs() as i64) >= timeout.num_seconds() { - break; - } - } - - // Wait for the process to exit before reaching the timeout - // If not, we just kill it - let exit_status; - loop { - match child_process.try_wait() { - Ok(Some(status)) => { - exit_status = status; + if (process_start_time.elapsed().as_secs() as i64) >= timeout.num_seconds() { break; } - Ok(None) => { - if (process_start_time.elapsed().as_secs() as i64) < timeout.num_seconds() { - std::thread::sleep(std::time::Duration::from_secs(1)); - continue; + } + + // Wait for the process to exit before reaching the timeout + // If not, we just kill it + let exit_status; + loop { + match cmd_handle.try_wait() { + Ok(Some(status)) => { + exit_status = status; + break; } + Ok(None) => { + if (process_start_time.elapsed().as_secs() as i64) < timeout.num_seconds() { + std::thread::sleep(std::time::Duration::from_secs(1)); + continue; + } - // Timeout ! - warn!( - "Killing process {} due to timeout {}m reached", - command_string, - timeout.num_minutes() - ); - let _ = child_process - .kill() //Fire - .map(|_| child_process.wait()) - .map_err(|err| error!("Cannot kill process {:?} {}", child_process, err)); + // Timeout ! + let msg = format!( + "Killing process {:?} due to timeout {}m reached", + self.command, + timeout.num_minutes() + ); + warn!("{}", msg); - return Err(SimpleError::new( - Other, - Some(format!("Image build timeout after {} seconds", timeout.num_seconds())), - )); - } - Err(err) => return Err(SimpleError::from(err)), - }; + let _ = cmd_handle + .kill() //Fire + .map(|_| cmd_handle.wait()) + .map_err(|err| error!("Cannot kill process {:?} {}", cmd_handle, err)); + + return Err(TimeoutError(msg)); + } + Err(err) => return Err(ExecutionError(err)), + }; + } + + if !exit_status.success() { + debug!( + "command: {:?} terminated with error exist status {:?}", + self.command, exit_status + ); + return Err(ExitStatusError(exit_status)); + } + + Ok(()) } - - // Process exited - if exit_status.success() { - return Ok(command_output); - } - - Err(SimpleError::new( - SimpleErrorKind::Command(exit_status), - Some("error while executing an internal command"), - )) } // return the output of "binary_name" --version pub fn run_version_command_for(binary_name: &str) -> String { let mut output_from_cmd = String::new(); - let _ = exec_with_output( - binary_name, - vec!["--version"], - |r_out| match r_out { - Ok(s) => output_from_cmd.push_str(&s), - Err(e) => error!("Error while getting stdout from {} {}", binary_name, e), - }, - |r_err| match r_err { - Ok(_) => error!("Error executing {}", binary_name), - Err(e) => error!("Error while getting stderr from {} {}", binary_name, e), - }, + let mut cmd = QoveryCommand::new(binary_name, &vec!["--version"], Default::default()); + let _ = cmd.exec_with_output( + |r_out| output_from_cmd.push_str(&r_out), + |r_err| error!("Error executing {}: {}", binary_name, r_err), ); output_from_cmd @@ -277,32 +216,58 @@ pub fn command_to_string

(binary: P, args: &[&str], envs: &[(&str, &str)]) -> where P: AsRef, { - let _envs = envs.iter().map(|(k, v)| format!("{}={}", k, v)).collect::>(); - - format!( - "{} {} {}", - _envs.join(" "), - binary.as_ref().to_str().unwrap(), - args.join(" ") - ) + let _envs = envs.iter().map(|(k, v)| format!("{}={}", k, v)).join(" "); + format!("{} {:?} {}", _envs, binary.as_ref().as_os_str(), args.join(" ")) } #[cfg(test)] mod tests { - use crate::cmd::utilities::exec_with_envs_and_output; + use crate::cmd::utilities::{does_binary_exist, run_version_command_for, CommandError, QoveryCommand}; use chrono::Duration; + #[test] + fn test_binary_exist() { + assert_eq!(does_binary_exist("sdfsdf"), false); + assert_eq!(does_binary_exist("ls"), true); + assert_eq!(does_binary_exist("/bin/sh"), true); + } + + #[test] + fn test_run_version_for_command() { + let ret = run_version_command_for("/bin/ls"); + assert_eq!(ret.is_empty(), false); + assert_eq!(ret.contains("GNU"), true) + } + + #[test] + fn test_error() { + let mut cmd = QoveryCommand::new("false", &vec![], &vec![]); + assert_eq!(cmd.exec().is_err(), true); + assert_eq!(matches!(cmd.exec(), Err(CommandError::ExitStatusError(_))), true); + } + #[test] fn test_command_with_timeout() { - let ret = exec_with_envs_and_output("sleep", vec!["120"], vec![], |_| {}, |_| {}, Duration::seconds(2)); + let mut cmd = QoveryCommand::new("sleep", &vec!["120"], &vec![]); + let ret = cmd.exec_with_timeout(Duration::seconds(2), |_| {}, |_| {}); + assert_eq!(ret.is_err(), true); - assert_eq!(ret.err().unwrap().message.unwrap().contains("timeout"), true); + match ret.err().unwrap() { + CommandError::TimeoutError(_) => {} + _ => assert_eq!(true, false), + } + + let mut cmd = QoveryCommand::new("yes", &vec![], &vec![]); + let ret = cmd.exec_with_timeout(Duration::seconds(2), |_| {}, |_| {}); - let ret = exec_with_envs_and_output("yes", vec![""], vec![], |_| {}, |_| {}, Duration::seconds(2)); assert_eq!(ret.is_err(), true); + match ret.err().unwrap() { + CommandError::TimeoutError(_) => {} + _ => assert_eq!(true, false), + } - let ret2 = exec_with_envs_and_output("sleep", vec!["1"], vec![], |_| {}, |_| {}, Duration::seconds(5)); - - assert_eq!(ret2.is_ok(), true); + let mut cmd = QoveryCommand::new("sleep", &vec!["1"], &vec![]); + let ret = cmd.exec_with_timeout(Duration::seconds(2), |_| {}, |_| {}); + assert_eq!(ret.is_ok(), true); } } diff --git a/src/container_registry/docker.rs b/src/container_registry/docker.rs index 6b7625cc..099a36b9 100644 --- a/src/container_registry/docker.rs +++ b/src/container_registry/docker.rs @@ -1,4 +1,5 @@ use crate::cmd; +use crate::cmd::utilities::QoveryCommand; use crate::container_registry::Kind; use crate::error::{SimpleError, SimpleErrorKind}; use chrono::Duration; @@ -54,16 +55,11 @@ pub fn docker_manifest_inspect( let binary = "docker"; let image_full_url = format!("{}/{}", registry_url.as_str(), &image_with_tag); let args = vec!["manifest", "inspect", image_full_url.as_str()]; + let mut raw_output: Vec = vec![]; - return match cmd::utilities::exec_with_envs_and_output( - binary, - args.clone(), - envs.clone(), - |_| {}, - |_| {}, - Duration::minutes(1), - ) { - Ok(raw_output) => { + let mut cmd = QoveryCommand::new("docker", &args, &envs); + return match cmd.exec_with_timeout(Duration::minutes(1), |line| raw_output.push(line), |_| {}) { + Ok(_) => { let joined = raw_output.join(""); match serde_json::from_str(&joined) { Ok(extracted_manifest) => Some(extracted_manifest), @@ -114,7 +110,8 @@ pub fn docker_login( registry_pass.as_str(), ]; - match cmd::utilities::exec(binary, args.clone(), &docker_envs.clone()) { + let mut cmd = QoveryCommand::new(binary, &args, &docker_envs); + match cmd.exec() { Ok(_) => Ok(()), Err(e) => { let error_message = format!( @@ -146,51 +143,41 @@ pub fn docker_tag_and_push_image( Kind::ScalewayCr => "Scaleway Registry", }; - match retry::retry(Fibonacci::from_millis(3000).take(5), || { - match cmd::utilities::exec("docker", vec!["tag", &image_with_tag, dest.as_str()], &docker_envs) { - Ok(_) => OperationResult::Ok(()), - Err(e) => { - info!("failed to tag image {}, retrying...", image_with_tag); - OperationResult::Retry(e) - } + let mut cmd = QoveryCommand::new("docker", &vec!["tag", &image_with_tag, dest.as_str()], &docker_envs); + match retry::retry(Fibonacci::from_millis(3000).take(5), || match cmd.exec() { + Ok(_) => OperationResult::Ok(()), + Err(e) => { + info!("failed to tag image {}, retrying...", image_with_tag); + OperationResult::Retry(e) } }) { Err(Operation { error, .. }) => { return Err(SimpleError::new( SimpleErrorKind::Other, - Some(format!("failed to tag image {}: {:?}", image_with_tag, error.message)), + Some(format!("failed to tag image {}: {:?}", image_with_tag, error)), )) } _ => {} } - match retry::retry( - Fibonacci::from_millis(5000).take(5), - || match cmd::utilities::exec_with_envs_and_output( - "docker", - vec!["push", dest.as_str()], - docker_envs.clone(), - |line| { - let line_string = line.unwrap_or_default(); - info!("{}", line_string.as_str()); - }, - |line| { - let line_string = line.unwrap_or_default(); - error!("{}", line_string.as_str()); - }, + let mut cmd = QoveryCommand::new("docker", &vec!["push", dest.as_str()], &docker_envs); + match retry::retry(Fibonacci::from_millis(5000).take(5), || { + match cmd.exec_with_timeout( Duration::minutes(10), + |line| info!("{}", line), + |line| error!("{}", line), ) { Ok(_) => OperationResult::Ok(()), Err(e) => { warn!( "failed to push image {} on {}, {:?} retrying...", - image_with_tag, registry_provider, e.message + image_with_tag, registry_provider, e ); OperationResult::Retry(e) } - }, - ) { - Err(Operation { error, .. }) => Err(error), + } + }) { + Err(Operation { error, .. }) => Err(SimpleError::new(SimpleErrorKind::Other, Some(error.to_string()))), Err(e) => Err(SimpleError::new( SimpleErrorKind::Other, Some(format!( diff --git a/src/container_registry/docker_hub.rs b/src/container_registry/docker_hub.rs index 3d622ec6..317acaf2 100644 --- a/src/container_registry/docker_hub.rs +++ b/src/container_registry/docker_hub.rs @@ -3,7 +3,7 @@ extern crate reqwest; use reqwest::StatusCode; use crate::build_platform::Image; -use crate::cmd; +use crate::cmd::utilities::QoveryCommand; use crate::container_registry::docker::docker_tag_and_push_image; use crate::container_registry::{ContainerRegistry, EngineError, Kind, PushResult}; use crate::error::EngineErrorCause; @@ -53,17 +53,10 @@ impl ContainerRegistry for DockerHub { fn is_valid(&self) -> Result<(), EngineError> { // check the version of docker and print it as info let mut output_from_cmd = String::new(); - let _ = cmd::utilities::exec_with_output( - "docker", - vec!["--version"], - |r_out| match r_out { - Ok(s) => output_from_cmd.push_str(&s), - Err(e) => error!("Error while getting sdtout from docker {}", e), - }, - |r_err| match r_err { - Ok(s) => error!("Error executing docker command {}", s), - Err(e) => error!("Error while getting stderr from docker {}", e), - }, + let mut cmd = QoveryCommand::new("docker", &vec!["--version"], &vec![]); + let _ = cmd.exec_with_output( + |r_out| output_from_cmd.push_str(&r_out), + |r_err| error!("Error executing docker command {}", r_err), ); info!("Using Docker: {}", output_from_cmd); @@ -113,11 +106,12 @@ impl ContainerRegistry for DockerHub { None => vec![], }; - if let Err(_) = cmd::utilities::exec( + let mut cmd = QoveryCommand::new( "docker", - vec!["login", "-u", self.login.as_str(), "-p", self.password.as_str()], + &vec!["login", "-u", self.login.as_str(), "-p", self.password.as_str()], &envs, - ) { + ); + if let Err(_) = cmd.exec() { return Err(self.engine_error( EngineErrorCause::User( "Your DockerHub account seems to be no longer valid (bad Credentials). \ diff --git a/src/container_registry/docr.rs b/src/container_registry/docr.rs index 762a5b5d..329ed9e1 100644 --- a/src/container_registry/docr.rs +++ b/src/container_registry/docr.rs @@ -4,13 +4,14 @@ use reqwest::StatusCode; use serde::{Deserialize, Serialize}; use crate::build_platform::Image; +use crate::cmd::utilities::QoveryCommand; use crate::container_registry::docker::docker_tag_and_push_image; use crate::container_registry::{ContainerRegistry, EngineError, Kind, PushResult}; use crate::error::{cast_simple_error_to_engine_error, EngineErrorCause, SimpleError, SimpleErrorKind}; use crate::models::{ Context, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope, }; -use crate::{cmd, utilities}; +use crate::utilities; use retry::delay::Fixed; use retry::Error::Operation; use retry::OperationResult; @@ -313,11 +314,12 @@ impl ContainerRegistry for DOCR { Err(_) => warn!("DOCR {} already exists", registry_name.as_str()), }; - if let Err(_) = cmd::utilities::exec( + let mut cmd = QoveryCommand::new( "doctl", - vec!["registry", "login", self.name.as_str(), "-t", self.api_key.as_str()], + &vec!["registry", "login", self.name.as_str(), "-t", self.api_key.as_str()], &vec![], - ) { + ); + if let Err(_) = cmd.exec() { return Err(self.engine_error( EngineErrorCause::User( "Your DOCR account seems to be no longer valid (bad Credentials). \ diff --git a/src/container_registry/ecr.rs b/src/container_registry/ecr.rs index ba6aac66..868fefbc 100644 --- a/src/container_registry/ecr.rs +++ b/src/container_registry/ecr.rs @@ -9,7 +9,7 @@ use rusoto_ecr::{ use rusoto_sts::{GetCallerIdentityRequest, Sts, StsClient}; use crate::build_platform::Image; -use crate::cmd; +use crate::cmd::utilities::QoveryCommand; use crate::container_registry::docker::docker_tag_and_push_image; use crate::container_registry::{ContainerRegistry, Kind, PushResult}; use crate::error::{EngineError, EngineErrorCause}; @@ -381,9 +381,9 @@ impl ContainerRegistry for ECR { } }; - if let Err(_) = cmd::utilities::exec( + let mut cmd = QoveryCommand::new( "docker", - vec![ + &vec![ "login", "-u", access_token.as_str(), @@ -392,7 +392,9 @@ impl ContainerRegistry for ECR { endpoint_url.as_str(), ], &self.docker_envs(), - ) { + ); + + if let Err(_) = cmd.exec() { return Err(self.engine_error( EngineErrorCause::User( "Your ECR account seems to be no longer valid (bad Credentials). \ diff --git a/src/models.rs b/src/models.rs index a25a33a0..2c2b3305 100644 --- a/src/models.rs +++ b/src/models.rs @@ -1340,6 +1340,7 @@ mod tests { for tc in test_cases { // execute: let result = Domain::new(tc.input.clone()); + tc.expected_wildcarded_output; // to avoid warning // verify: assert_eq!( diff --git a/src/object_storage/s3.rs b/src/object_storage/s3.rs index 1761935c..86552215 100644 --- a/src/object_storage/s3.rs +++ b/src/object_storage/s3.rs @@ -1,10 +1,12 @@ use std::fs::File; +use crate::cmd::utilities::QoveryCommand; use retry::delay::Fibonacci; use retry::{Error, OperationResult}; use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY}; -use crate::error::{cast_simple_error_to_engine_error, EngineError, EngineErrorCause}; +use crate::error::SimpleErrorKind::Other; +use crate::error::{cast_simple_error_to_engine_error, EngineError, EngineErrorCause, SimpleError}; use crate::models::{Context, StringPath}; use crate::object_storage::{Kind, ObjectStorage}; @@ -58,32 +60,36 @@ impl ObjectStorage for S3 { } fn create_bucket(&self, bucket_name: &str) -> Result<(), EngineError> { + let mut cmd = QoveryCommand::new( + "aws", + &vec!["s3api", "create-bucket", "--bucket", bucket_name], + &self.credentials_environment_variables(), + ); cast_simple_error_to_engine_error( self.engine_error_scope(), self.context().execution_id(), - crate::cmd::utilities::exec( - "aws", - vec!["s3api", "create-bucket", "--bucket", bucket_name], - &self.credentials_environment_variables(), - ), + cmd.exec() + .map_err(|err| SimpleError::new(Other, Some(format!("{}", err)))), ) } fn delete_bucket(&self, bucket_name: &str) -> Result<(), EngineError> { + let mut cmd = QoveryCommand::new( + "aws", + &vec![ + "s3", + "rb", + "--force", + "--bucket", + format!("s3://{}", bucket_name).as_str(), + ], + &self.credentials_environment_variables(), + ); cast_simple_error_to_engine_error( self.engine_error_scope(), self.context().execution_id(), - crate::cmd::utilities::exec( - "aws", - vec![ - "s3", - "rb", - "--force", - "--bucket", - format!("s3://{}", bucket_name).as_str(), - ], - &self.credentials_environment_variables(), - ), + cmd.exec() + .map_err(|err| SimpleError::new(Other, Some(format!("{}", err)))), ) } @@ -110,16 +116,18 @@ impl ObjectStorage for S3 { } // retrieve config file from object storage + let mut cmd = QoveryCommand::new( + "aws", + &vec!["s3", "cp", s3_url.as_str(), file_path.as_str()], + &self.credentials_environment_variables(), + ); let result = retry::retry(Fibonacci::from_millis(3000).take(5), || { // we choose to use the AWS CLI instead of Rusoto S3 due to reliability problems we faced. let result = cast_simple_error_to_engine_error( self.engine_error_scope(), self.context().execution_id(), - crate::cmd::utilities::exec( - "aws", - vec!["s3", "cp", s3_url.as_str(), file_path.as_str()], - &self.credentials_environment_variables(), - ), + cmd.exec() + .map_err(|err| SimpleError::new(Other, Some(format!("{}", err)))), ); match result { @@ -151,19 +159,21 @@ impl ObjectStorage for S3 { } fn put(&self, bucket_name: &str, object_key: &str, file_path: &str) -> Result<(), EngineError> { + let mut cmd = QoveryCommand::new( + "aws", + &vec![ + "s3", + "cp", + file_path, + format!("s3://{}/{}", bucket_name, object_key).as_str(), + ], + &self.credentials_environment_variables(), + ); cast_simple_error_to_engine_error( self.engine_error_scope(), self.context().execution_id(), - crate::cmd::utilities::exec( - "aws", - vec![ - "s3", - "cp", - file_path, - format!("s3://{}/{}", bucket_name, object_key).as_str(), - ], - &self.credentials_environment_variables(), - ), + cmd.exec() + .map_err(|err| SimpleError::new(Other, Some(format!("{}", err)))), ) } } diff --git a/test_utilities/Cargo.lock b/test_utilities/Cargo.lock index f8b197dc..4a49fd3e 100644 --- a/test_utilities/Cargo.lock +++ b/test_utilities/Cargo.lock @@ -2133,6 +2133,7 @@ dependencies = [ "sysinfo", "tar", "tera", + "thiserror", "timeout-readwrite", "tokio 1.10.0", "tracing", @@ -3285,18 +3286,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.23" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76cc616c6abf8c8928e2fdcc0dbfab37175edd8fb49a4641066ad1364fdab146" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.23" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" dependencies = [ "proc-macro2 1.0.27", "quote 1.0.8", diff --git a/test_utilities/src/utilities.rs b/test_utilities/src/utilities.rs index d29d2471..af829f58 100644 --- a/test_utilities/src/utilities.rs +++ b/test_utilities/src/utilities.rs @@ -48,6 +48,8 @@ use crate::digitalocean::{ use qovery_engine::cloud_provider::digitalocean::application::Region; use qovery_engine::cmd::kubectl::{kubectl_get_pvc, kubectl_get_svc}; use qovery_engine::cmd::structs::{KubernetesList, KubernetesPod, PVC, SVC}; +use qovery_engine::cmd::utilities::QoveryCommand; +use qovery_engine::error::SimpleErrorKind::Other; use qovery_engine::models::DatabaseMode::MANAGED; use qovery_engine::object_storage::spaces::{BucketDeleteStrategy, Spaces}; use qovery_engine::object_storage::ObjectStorage; @@ -761,15 +763,17 @@ fn aws_s3_get_object( // used as a failover when rusoto_s3 acts up let s3_url = format!("s3://{}/{}", bucket_name, object_key); - qovery_engine::cmd::utilities::exec( + let mut cmd = QoveryCommand::new( "aws", - vec!["s3", "cp", &s3_url, &local_path], + &vec!["s3", "cp", &s3_url, &local_path], &vec![ (AWS_ACCESS_KEY_ID, access_key_id), (AWS_SECRET_ACCESS_KEY, secret_access_key), ], - )?; + ); + cmd.exec() + .map_err(|err| SimpleError::new(Other, Some(format!("{}", err))))?; let s = fs::read_to_string(&local_path)?; Ok(s)