use std::io::{Error, Write}; use std::path::{Path, PathBuf}; use tracing::{error, info}; use crate::cloud_provider::helm::ChartInfo; use crate::cmd::command::QoveryCommand; use crate::cmd::helm::HelmCommand::{LIST, ROLLBACK, STATUS, UNINSTALL, UPGRADE}; use crate::cmd::helm::HelmError::{CannotRollback, CmdError, InvalidKubeConfig, ReleaseDoesNotExist}; use crate::cmd::structs::{HelmChart, HelmListItem}; use crate::errors::{CommandError, EngineError}; use crate::events::EventDetails; use chrono::Duration; use semver::Version; use serde_derive::Deserialize; use std::fs::File; use std::str::FromStr; const HELM_DEFAULT_TIMEOUT_IN_SECONDS: u32 = 600; const HELM_MAX_HISTORY: &str = "50"; pub enum Timeout { Default, Value(T), } impl Timeout { pub fn value(&self) -> u32 { match *self { Timeout::Default => HELM_DEFAULT_TIMEOUT_IN_SECONDS, Timeout::Value(t) => t, } } } #[derive(thiserror::Error, Debug)] pub enum HelmError { #[error("Kubernetes config file path is not valid or does not exist: {0}")] InvalidKubeConfig(PathBuf), #[error("Requested Helm release `{0}` does not exist")] ReleaseDoesNotExist(String), #[error("Requested Helm release `{0}` is under an helm lock. Ensure release is de-locked before going further")] ReleaseLocked(String), #[error("Helm release `{0}` during helm {1:?} has been rollbacked")] Rollbacked(String, HelmCommand), #[error("Helm release `{0}` cannot be rollbacked due to be at revision 1")] CannotRollback(String), #[error("Helm timed out for release `{0}` during helm {1:?}: {2}")] Timeout(String, HelmCommand, String), #[error("Helm command `{1:?}` for release {0} terminated with an error: {2:?}")] CmdError(String, HelmCommand, CommandError), } #[derive(Debug)] pub struct Helm { kubernetes_config: PathBuf, common_envs: Vec<(String, String)>, } #[derive(Debug, Clone, Copy)] pub enum HelmCommand { ROLLBACK, STATUS, UPGRADE, UNINSTALL, LIST, DIFF, } #[derive(Debug, Clone, Deserialize, Default)] pub struct ReleaseInfo { // https://github.com/helm/helm/blob/12f1bc0acdeb675a8c50a78462ed3917fb7b2e37/pkg/release/status.go status: String, } #[derive(Debug, Clone, Deserialize, Default)] pub struct ReleaseStatus { pub version: u64, pub info: ReleaseInfo, } impl ReleaseStatus { fn is_locked(&self) -> bool { self.info.status.starts_with("pending-") } } impl Helm { fn get_all_envs<'a>(&'a self, envs: &'a [(&'a str, &'a str)]) -> Vec<(&'a str, &'a str)> { let mut all_envs: Vec<(&str, &str)> = self.common_envs.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect(); all_envs.append(&mut envs.to_vec()); all_envs } pub fn new>(kubernetes_config: P, common_envs: &[(&str, &str)]) -> Result { // Check kube config file is valid let kubernetes_config = kubernetes_config.as_ref().to_path_buf(); if !kubernetes_config.exists() || !kubernetes_config.is_file() { return Err(InvalidKubeConfig(kubernetes_config)); } Ok(Helm { kubernetes_config, common_envs: common_envs .iter() .map(|(k, v)| (k.to_string(), v.to_string())) .collect(), }) } pub fn check_release_exist(&self, chart: &ChartInfo, envs: &[(&str, &str)]) -> Result { let namespace = chart.get_namespace_string(); let args = vec![ "status", &chart.name, "--kubeconfig", self.kubernetes_config.to_str().unwrap_or_default(), "--namespace", &namespace, "-o", "json", ]; let mut stdout = String::new(); let mut stderr = String::new(); match helm_exec_with_output( &args, &self.get_all_envs(envs), |line| stdout.push_str(&line), |line| stderr.push_str(&line), ) { Err(_) if stderr.contains("release: not found") => Err(ReleaseDoesNotExist(chart.name.clone())), Err(err) => { stderr.push_str(&err.message()); let error = CommandError::new(stderr, err.message_safe()); Err(CmdError(chart.name.clone(), STATUS, error)) } Ok(_) => { let status: ReleaseStatus = serde_json::from_str(&stdout).unwrap_or_default(); Ok(status) } } } pub fn rollback(&self, chart: &ChartInfo, envs: &[(&str, &str)]) -> Result<(), HelmError> { if self.check_release_exist(chart, envs)?.version <= 1 { return Err(CannotRollback(chart.name.clone())); } let timeout = format!("{}s", &chart.timeout_in_seconds); let namespace = chart.get_namespace_string(); let args = vec![ "rollback", &chart.name, "--kubeconfig", self.kubernetes_config.to_str().unwrap_or_default(), "--namespace", &namespace, "--timeout", &timeout, "--history-max", HELM_MAX_HISTORY, "--cleanup-on-fail", "--force", "--wait", ]; let mut stderr = String::new(); match helm_exec_with_output(&args, &self.get_all_envs(envs), |_| {}, |line| stderr.push_str(&line)) { Err(err) => { stderr.push_str(&err.message()); let error = CommandError::new(stderr, err.message_safe()); Err(CmdError(chart.name.clone(), ROLLBACK, error)) } Ok(_) => Ok(()), } } pub fn uninstall(&self, chart: &ChartInfo, envs: &[(&str, &str)]) -> Result<(), HelmError> { // If the release does not exist, we do not return an error match self.check_release_exist(chart, envs) { Ok(_) => {} Err(ReleaseDoesNotExist(_)) => return Ok(()), Err(err) => return Err(err), } let timeout = format!("{}s", &chart.timeout_in_seconds); let namespace = chart.get_namespace_string(); let args = vec![ "uninstall", &chart.name, "--kubeconfig", self.kubernetes_config.to_str().unwrap_or_default(), "--namespace", &namespace, "--timeout", &timeout, "--wait", ]; let mut stderr = String::new(); match helm_exec_with_output(&args, &self.get_all_envs(envs), |_| {}, |line| stderr.push_str(&line)) { Err(err) => { stderr.push_str(&err.message()); let error = CommandError::new(stderr, err.message_safe()); Err(CmdError(chart.name.clone(), UNINSTALL, error)) } Ok(_) => Ok(()), } } fn unlock_release(&self, chart: &ChartInfo, envs: &[(&str, &str)]) -> Result<(), HelmError> { match self.check_release_exist(chart, envs) { Ok(release) if release.is_locked() && release.version <= 1 => { info!("Helm lock detected. Uninstalling it as it is the first version and rollback is not possible"); self.uninstall(chart, envs)?; } Ok(release) if release.is_locked() => { info!("Helm lock detected. Forcing rollback to previous version"); self.rollback(chart, envs)?; } Ok(release) => { // Happy path nothing to do debug!("Helm release status: {:?}", release) } Err(_) => {} // Happy path nothing to do } Ok(()) } /// List deployed helm charts /// /// # Arguments /// /// * `envs` - environment variables required for kubernetes connection /// * `namespace` - list charts from a kubernetes namespace or use None to select all namespaces pub fn list_release(&self, namespace: Option<&str>, envs: &[(&str, &str)]) -> Result, HelmError> { let mut helm_args = vec![ "list", "-a", "--kubeconfig", self.kubernetes_config.to_str().unwrap_or_default(), "-o", "json", ]; match namespace { Some(ns) => helm_args.append(&mut vec!["-n", ns]), None => helm_args.push("-A"), } let mut output_string: Vec = Vec::with_capacity(20); if let Err(cmd_error) = helm_exec_with_output( &helm_args, &self.get_all_envs(envs), |line| output_string.push(line), |line| error!("{}", line), ) { return Err(HelmError::CmdError("none".to_string(), LIST, cmd_error)); } let values = serde_json::from_str::>(&output_string.join("")); let mut helms_charts: Vec = Vec::new(); match values { Ok(all_helms) => { for helm in all_helms { let raw_version = helm.chart.replace(format!("{}-", helm.name).as_str(), ""); let version = Version::from_str(raw_version.as_str()).ok(); helms_charts.push(HelmChart::new(helm.name, helm.namespace, version)) } Ok(helms_charts) } Err(e) => { let message_safe = "Error while deserializing all helms names"; Err(HelmError::CmdError( "none".to_string(), LIST, CommandError::new( format!("{}, error: {}", message_safe, e), Some(message_safe.to_string()), ), )) } } } pub fn get_chart_version( &self, chart_name: String, namespace: Option<&str>, envs: &[(&str, &str)], ) -> Result, HelmError> { let deployed_charts = self.list_release(namespace, envs)?; for chart in deployed_charts { if chart.name == chart_name { return Ok(chart.version); } } // found nothing ;'( Ok(None) } pub fn upgrade_diff(&self, chart: &ChartInfo, envs: &[(&str, &str)]) -> Result<(), HelmError> { let mut args_string: Vec = vec![ "diff".to_string(), "upgrade".to_string(), "--kubeconfig".to_string(), self.kubernetes_config.to_str().unwrap_or_default().to_string(), "--install".to_string(), "--namespace".to_string(), chart.get_namespace_string(), ]; for value in &chart.values { args_string.push("--set".to_string()); args_string.push(format!("{}={}", value.key, value.value)); } for value_file in &chart.values_files { args_string.push("-f".to_string()); args_string.push(value_file.clone()); } for value_file in &chart.yaml_files_content { let file_path = format!("{}/{}", chart.path, &value_file.filename); let file_create = || -> Result<(), Error> { let mut file = File::create(&file_path)?; file.write_all(value_file.yaml_content.as_bytes())?; Ok(()) }; // no need to validate yaml as it will be done by helm if let Err(e) = file_create() { let safe_message = format!("Error while writing yaml content to file `{}`", &file_path); let cmd_err = CommandError::new( format!("{}\nContent\n{}\nError: {}", safe_message, value_file.yaml_content, e), Some(safe_message), ); return Err(HelmError::CmdError(chart.name.clone(), HelmCommand::UPGRADE, cmd_err)); }; args_string.push("-f".to_string()); args_string.push(file_path); } // add last elements args_string.push(chart.name.clone()); args_string.push(chart.path.clone()); let mut stderr_msg = String::new(); let helm_ret = helm_exec_with_output( &args_string.iter().map(|x| x.as_str()).collect::>(), &self.get_all_envs(envs), |line| { info!("{}", line); }, |line| { stderr_msg.push_str(&line); warn!("chart {}: {}", chart.name, line); }, ); match helm_ret { // Ok is ok Ok(_) => Ok(()), Err(err) => { error!("Helm error: {:?}", err); Err(CmdError( chart.name.clone(), HelmCommand::DIFF, CommandError::new(stderr_msg.clone(), Some(stderr_msg)), )) } } } pub fn upgrade(&self, chart: &ChartInfo, envs: &[(&str, &str)]) -> Result<(), HelmError> { // Due to crash or error it is possible that the release is under an helm lock // Try to un-stuck the situation first if needed // We don't care if the rollback failed, as it is a best effort to remove the lock // and to re-launch an upgrade just after let unlock_ret = self.unlock_release(chart, envs); info!("Helm lock status: {:?}", unlock_ret); let debug = false; let timeout_string = format!("{}s", &chart.timeout_in_seconds); let mut args_string: Vec = vec![ "upgrade".to_string(), "--kubeconfig".to_string(), self.kubernetes_config.to_str().unwrap_or_default().to_string(), "--create-namespace".to_string(), "--install".to_string(), "--timeout".to_string(), timeout_string.as_str().to_string(), "--history-max".to_string(), HELM_MAX_HISTORY.to_string(), "--namespace".to_string(), chart.get_namespace_string(), ]; if debug { args_string.push("-o".to_string()); args_string.push("json".to_string()); } // warn: don't add debug or json output won't work if chart.atomic { args_string.push("--atomic".to_string()) } if chart.force_upgrade { args_string.push("--force".to_string()) } if chart.dry_run { args_string.push("--dry-run".to_string()) } if chart.wait { args_string.push("--wait".to_string()) } // overrides and files overrides for value in &chart.values { args_string.push("--set".to_string()); args_string.push(format!("{}={}", value.key, value.value)); } for value_file in &chart.values_files { args_string.push("-f".to_string()); args_string.push(value_file.clone()); } for value_file in &chart.yaml_files_content { let file_path = format!("{}/{}", chart.path, &value_file.filename); let file_create = || -> Result<(), Error> { let mut file = File::create(&file_path)?; file.write_all(value_file.yaml_content.as_bytes())?; Ok(()) }; // no need to validate yaml as it will be done by helm if let Err(e) = file_create() { let safe_message = format!("Error while writing yaml content to file `{}`", &file_path); let cmd_err = CommandError::new( format!("{}\nContent\n{}\nError: {}", safe_message, value_file.yaml_content, e), Some(safe_message), ); return Err(HelmError::CmdError(chart.name.clone(), HelmCommand::UPGRADE, cmd_err)); }; args_string.push("-f".to_string()); args_string.push(file_path); } // add last elements args_string.push(chart.name.clone()); args_string.push(chart.path.clone()); let mut error_message: Vec = vec![]; let helm_ret = helm_exec_with_output( &args_string.iter().map(|x| x.as_str()).collect::>(), &self.get_all_envs(envs), |line| { info!("{}", line); }, |line| { warn!("chart {}: {}", chart.name, line); error_message.push(line); }, ); match helm_ret { // Ok is ok Ok(_) => Ok(()), Err(err) => { error!("Helm error: {:?}", err); // Try do define/specify a bit more the message let stderr_msg: String = error_message.into_iter().collect(); let stderr_msg = format!("{}: {}", stderr_msg, err.message()); let error = if stderr_msg.contains("another operation (install/upgrade/rollback) is in progress") { HelmError::ReleaseLocked(chart.name.clone()) } else if stderr_msg.contains("has been rolled back") { HelmError::Rollbacked(chart.name.clone(), UPGRADE) } else if stderr_msg.contains("timed out waiting") { HelmError::Timeout(chart.name.clone(), UPGRADE, stderr_msg) } else { CmdError( chart.name.clone(), HelmCommand::UPGRADE, CommandError::new(stderr_msg.clone(), Some(stderr_msg)), ) }; Err(error) } } } pub fn uninstall_chart_if_breaking_version( &self, chart: &ChartInfo, envs: &[(&str, &str)], ) -> Result<(), HelmError> { // If there is a breaking version set for the current helm chart, // then we compare this breaking version with the currently installed version if any. // If current installed version is older than breaking change one, then we delete // the chart before applying it. if let Some(breaking_version) = &chart.last_breaking_version_requiring_restart { if let Some(installed_version) = self.get_chart_version(chart.name.clone(), Some(chart.get_namespace_string().as_str()), envs)? { if installed_version.le(breaking_version) { self.uninstall(&chart, envs)?; } } } Ok(()) } } fn helm_exec_with_output( args: &[&str], envs: &[(&str, &str)], stdout_output: F, stderr_output: X, ) -> Result<(), CommandError> where F: FnMut(String), X: FnMut(String), { // Note: Helm CLI use spf13/cobra lib for the CLI; One function is mainly used to return an error if a command failed. // Helm returns an error each time a command does not succeed as they want. Which leads to handling error with status code 1 // It means that the command successfully ran, but it didn't terminate as expected let mut cmd = QoveryCommand::new("helm", args, envs); match cmd.exec_with_timeout(Duration::max_value(), stdout_output, stderr_output) { Err(err) => Err(CommandError::new(format!("{:?}", err), None)), _ => Ok(()), } } pub fn to_command_error(error: HelmError) -> CommandError { CommandError::new_from_safe_message(error.to_string()) } pub fn to_engine_error(event_details: &EventDetails, error: HelmError) -> EngineError { EngineError::new_helm_error(event_details.clone(), error) } #[cfg(feature = "test-with-kube")] #[cfg(test)] mod tests { use crate::cloud_provider::helm::{ChartInfo, ChartSetValue}; use crate::cmd::command::QoveryCommand; use crate::cmd::helm::{helm_exec_with_output, Helm, HelmError}; use std::sync::{Arc, Barrier}; use std::thread; use std::time::Duration; struct HelmTestCtx { helm: Helm, chart: ChartInfo, } impl HelmTestCtx { fn cleanup(&self) { let ret = self.helm.uninstall(&self.chart, &vec![]); assert!(ret.is_ok()) } fn new(release_name: &str) -> HelmTestCtx { let chart = ChartInfo::new_from_custom_namespace( release_name.to_string(), "tests/helm/simple_nginx".to_string(), "default".to_string(), 600, vec![], false, None, ); let mut kube_config = dirs::home_dir().unwrap(); kube_config.push(".kube/config"); let helm = Helm::new(kube_config.to_str().unwrap(), &vec![]).unwrap(); let cleanup = HelmTestCtx { helm, chart }; cleanup.cleanup(); cleanup } } impl Drop for HelmTestCtx { fn drop(&mut self) { self.cleanup() } } #[test] fn check_version() { let mut output = String::new(); let _ = helm_exec_with_output(&vec!["version"], &vec![], |line| output.push_str(&line), |_line| {}); assert!(output.contains("Version:\"v3.7.2\"")); } #[test] fn test_release_exist() { let HelmTestCtx { ref helm, ref chart } = HelmTestCtx::new("test-release-exist"); let ret = helm.check_release_exist(chart, &vec![]); assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)) } #[test] fn test_list_release() { let HelmTestCtx { ref helm, ref mut chart, } = HelmTestCtx::new("test-list-release"); chart.custom_namespace = Some("hello-my-friend-this-is-a-test".to_string()); // no existing namespace should return an empty array let ret = helm.list_release(Some("tsdfsfsdf"), &vec![]); assert!(matches!(ret, Ok(vec) if vec.is_empty())); // install something let ret = helm.upgrade(&chart, &vec![]); assert!(matches!(ret, Ok(()))); // We should have at least one release in all the release let ret = helm.list_release(None, &vec![]); assert!(matches!(ret, Ok(vec) if !vec.is_empty())); // We should have at least one release in all the release let ret = helm.list_release(Some(&chart.get_namespace_string()), &vec![]); assert!(matches!(ret, Ok(vec) if vec.len() == 1)); // Install a second stuff let HelmTestCtx { ref helm, ref mut chart, } = HelmTestCtx::new("test-list-release-2"); chart.custom_namespace = Some("hello-my-friend-this-is-a-test".to_string()); let ret = helm.upgrade(&chart, &vec![]); assert!(matches!(ret, Ok(()))); let ret = helm.list_release(Some(&chart.get_namespace_string()), &vec![]); assert!(matches!(ret, Ok(vec) if vec.len() == 2)); } #[test] fn test_upgrade_diff() { let HelmTestCtx { ref helm, ref chart } = HelmTestCtx::new("test-upgrade-diff"); let ret = helm.upgrade_diff(&chart, &vec![]); assert!(matches!(ret, Ok(()))); } #[test] fn test_rollback() { let HelmTestCtx { ref helm, ref chart } = HelmTestCtx::new("test-rollback"); // check release does not exist yet let ret = helm.rollback(&chart, &vec![]); assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)); // install it let ret = helm.upgrade(&chart, &vec![]); assert!(matches!(ret, Ok(()))); // First revision cannot be rollback let ret = helm.rollback(&chart, &vec![]); assert!(matches!(ret, Err(HelmError::CannotRollback(_)))); // 2nd upgrade let ret = helm.upgrade(&chart, &vec![]); assert!(matches!(ret, Ok(()))); // Rollback should be ok now let ret = helm.rollback(&chart, &vec![]); assert!(matches!(ret, Ok(()))); } #[test] fn test_upgrade() { let HelmTestCtx { ref helm, ref chart } = HelmTestCtx::new("test-upgrade"); // check release does not exist yet let ret = helm.check_release_exist(&chart, &vec![]); assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)); // install it let ret = helm.upgrade(&chart, &vec![]); assert!(matches!(ret, Ok(()))); // check now it exists let ret = helm.check_release_exist(&chart, &vec![]); assert!(matches!(ret, Ok(_))); } #[test] fn test_upgrade_timeout() { let HelmTestCtx { ref helm, ref mut chart, } = HelmTestCtx::new("test-upgrade-timeout"); chart.timeout_in_seconds = 1; // check release does not exist yet let ret = helm.check_release_exist(&chart, &vec![]); assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)); // install it let ret = helm.upgrade(&chart, &vec![]); assert!(matches!(ret, Err(HelmError::Timeout(_, _, _)))); // Release should not exist if it fails let ret = helm.check_release_exist(&chart, &vec![]); assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)); } #[test] fn test_upgrade_with_lock_during_install() { // We want to check that we manage to install a chart even if a lock is present while it was the first installation let HelmTestCtx { ref helm, ref chart } = HelmTestCtx::new("test-upgrade-with-lock-install"); // check release does not exist yet let ret = helm.check_release_exist(&chart, &vec![]); assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)); // Spawn our task killer let barrier = Arc::new(Barrier::new(2)); std::thread::spawn({ let barrier = barrier.clone(); let chart_name = chart.name.clone(); move || { barrier.wait(); thread::sleep(Duration::from_millis(3000)); let mut cmd = QoveryCommand::new("pkill", &vec!["-9", "-f", &format!("helm.*{}", chart_name)], &vec![]); let _ = cmd.exec(); } }); // install it barrier.wait(); let ret = helm.upgrade(&chart, &vec![]); assert!(matches!(ret, Err(_))); // Release should be locked let ret = helm.check_release_exist(&chart, &vec![]); assert!(matches!(ret, Ok(release) if release.is_locked())); // New installation should work even if a lock is present let ret = helm.upgrade(&chart, &vec![]); assert!(matches!(ret, Ok(()))); // Release should not be locked anymore let ret = helm.check_release_exist(&chart, &vec![]); assert!(matches!(ret, Ok(release) if !release.is_locked())); } #[test] fn test_upgrade_with_lock_during_upgrade() { // We want to check that we manage to install a chart even if a lock is present while it not the first installation let HelmTestCtx { ref helm, ref mut chart, } = HelmTestCtx::new("test-upgrade-with-lock-upgrade"); // check release does not exist yet let ret = helm.check_release_exist(&chart, &vec![]); assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)); // First install let ret = helm.upgrade(&chart, &vec![]); assert!(matches!(ret, Ok(()))); // Spawn our task killer let barrier = Arc::new(Barrier::new(2)); std::thread::spawn({ let barrier = barrier.clone(); let chart_name = chart.name.clone(); move || { barrier.wait(); thread::sleep(Duration::from_millis(3000)); let mut cmd = QoveryCommand::new("pkill", &vec!["-9", "-f", &format!("helm.*{}", chart_name)], &vec![]); let _ = cmd.exec(); } }); chart.values = vec![ChartSetValue { key: "initialDelaySeconds".to_string(), value: "6".to_string(), }]; barrier.wait(); let ret = helm.upgrade(&chart, &vec![]); assert!(matches!(ret, Err(_))); // Release should be locked let ret = helm.check_release_exist(&chart, &vec![]); assert!(matches!(ret, Ok(release) if release.is_locked() && release.version == 2)); // New installation should work even if a lock is present let ret = helm.upgrade(&chart, &vec![]); assert!(matches!(ret, Ok(()))); // Release should not be locked anymore let ret = helm.check_release_exist(&chart, &vec![]); assert!(matches!(ret, Ok(release) if !release.is_locked() && release.version == 4)); } #[test] fn test_uninstall() { let HelmTestCtx { ref helm, ref chart } = HelmTestCtx::new("test-uninstall"); // check release does not exist yet let ret = helm.check_release_exist(&chart, &vec![]); assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)); // deleting something that does not exist should not be an issue let ret = helm.uninstall(&chart, &vec![]); assert!(matches!(ret, Ok(()))); // install it let ret = helm.upgrade(&chart, &vec![]); assert!(matches!(ret, Ok(()))); // check now it exists let ret = helm.check_release_exist(&chart, &vec![]); assert!(matches!(ret, Ok(_))); // Delete it let ret = helm.uninstall(&chart, &vec![]); assert!(matches!(ret, Ok(()))); // check release does not exist anymore let ret = helm.check_release_exist(&chart, &vec![]); assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)); } }