diff --git a/src/cloud_provider/aws/kubernetes/mod.rs b/src/cloud_provider/aws/kubernetes/mod.rs index ee084d3d..5b2c6487 100644 --- a/src/cloud_provider/aws/kubernetes/mod.rs +++ b/src/cloud_provider/aws/kubernetes/mod.rs @@ -8,11 +8,11 @@ use tera::Context as TeraContext; use crate::cloud_provider::aws::kubernetes::node::Node; use crate::cloud_provider::aws::AWS; use crate::cloud_provider::environment::Environment; -use crate::cloud_provider::kubernetes::{Kind, Kubernetes, KubernetesNode}; +use crate::cloud_provider::kubernetes::{uninstall_cert_manager, Kind, Kubernetes, KubernetesNode}; use crate::cloud_provider::models::WorkerNodeDataTemplate; use crate::cloud_provider::{kubernetes, CloudProvider}; use crate::cmd; -use crate::cmd::kubectl::{kubectl_delete_objects_in_all_namespaces, kubectl_exec_get_all_namespaces}; +use crate::cmd::kubectl::kubectl_exec_get_all_namespaces; use crate::deletion_utilities::{get_firsts_namespaces_to_delete, get_qovery_managed_namespaces}; use crate::dns_provider; use crate::dns_provider::DnsProvider; @@ -638,39 +638,21 @@ impl<'a> Kubernetes for EKS<'a> { self.context.execution_id(), )); - // https://cert-manager.io/docs/installation/uninstall/kubernetes/ // required to avoid namespace stuck on deletion - info!("Delete cert-manager related objects to prepare deletion"); - let cert_manager_objects = vec![ - "Issuers", - "ClusterIssuers", - "Certificates", - "CertificateRequests", - "Orders", - "Challenges", - ]; - for object in cert_manager_objects { - match kubectl_delete_objects_in_all_namespaces( - &kubernetes_config_file_path, - object, - self.cloud_provider().credentials_environment_variables(), - ) { - Ok(_) => {} - Err(e) => { - let error_message = format!( - "Wasn't able to delete all objects type {}, it's a blocker to then delete cert-manager namespace. {}", - object, - format!("{:?}", e.message) - ); - return Err(EngineError::new( - Internal, - self.engine_error_scope(), - self.context().execution_id(), - Some(error_message), - )); - } - }; - } + match uninstall_cert_manager( + &kubernetes_config_file_path, + self.cloud_provider().credentials_environment_variables(), + ) { + Ok(_) => {} + Err(e) => { + return Err(EngineError::new( + Internal, + self.engine_error_scope(), + self.context().execution_id(), + e.message, + )) + } + }; info!("Deleting Qovery managed Namespaces"); let qovery_namespaces = get_qovery_managed_namespaces(); diff --git a/src/cloud_provider/kubernetes.rs b/src/cloud_provider/kubernetes.rs index e18c0f78..b6e6cddc 100644 --- a/src/cloud_provider/kubernetes.rs +++ b/src/cloud_provider/kubernetes.rs @@ -9,11 +9,17 @@ use crate::cloud_provider::environment::Environment; use crate::cloud_provider::service::CheckAction; use crate::cloud_provider::{service, CloudProvider, DeploymentTarget}; use crate::cmd::kubectl; +use crate::cmd::kubectl::{kubectl_delete_objects_in_all_namespaces, kubectl_exec_count_all_objects}; use crate::dns_provider::DnsProvider; -use crate::error::{cast_simple_error_to_engine_error, EngineError, EngineErrorCause, EngineErrorScope}; +use crate::error::SimpleErrorKind::Other; +use crate::error::{cast_simple_error_to_engine_error, EngineError, EngineErrorCause, EngineErrorScope, SimpleError}; use crate::models::{Context, Listen, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope, StringPath}; use crate::object_storage::ObjectStorage; use crate::unit_conversion::{any_to_mi, cpu_string_to_float}; +use retry::delay::Fibonacci; +use retry::Error::Operation; +use retry::OperationResult; +use std::path::Path; pub trait Kubernetes: Listen { fn context(&self) -> &Context; @@ -520,3 +526,63 @@ pub fn check_kubernetes_has_enough_resources_to_deploy_environment( Ok(()) } + +pub fn uninstall_cert_manager

(kubernetes_config: P, envs: Vec<(&str, &str)>) -> Result<(), SimpleError> +where + P: AsRef, +{ + // https://cert-manager.io/docs/installation/uninstall/kubernetes/ + info!("Delete cert-manager related objects to prepare deletion"); + + let cert_manager_objects = vec![ + "Issuers", + "ClusterIssuers", + "Certificates", + "CertificateRequests", + "Orders", + "Challenges", + ]; + + for object in cert_manager_objects { + // check resource exist first + match kubectl_exec_count_all_objects(&kubernetes_config, object, envs.clone()) { + Ok(x) if x == 0 => continue, + Err(e) => { + warn!( + "encountering issues while trying to get objects kind {}: {:?}", + object, e.message + ); + continue; + } + _ => {} + } + + // delete if resource exists + let result = + retry::retry( + Fibonacci::from_millis(5000).take(3), + || match kubectl_delete_objects_in_all_namespaces(&kubernetes_config, object, envs.clone()) { + Ok(_) => OperationResult::Ok(()), + Err(e) => { + warn!("Failed to delete all {} objects, retrying...", object); + OperationResult::Retry(e) + } + }, + ); + + match result { + Ok(_) => {} + Err(Operation { error, .. }) => return Err(error), + Err(retry::Error::Internal(msg)) => { + let error_message = format!( + "Wasn't able to delete all objects type {}, it's a blocker to then delete cert-manager namespace. {}", + object, + format!("{:?}", msg) + ); + return Err(SimpleError::new(Other, Some(error_message))); + } + }; + } + + Ok(()) +} diff --git a/src/cmd/kubectl.rs b/src/cmd/kubectl.rs index a761f795..d3f6819e 100644 --- a/src/cmd/kubectl.rs +++ b/src/cmd/kubectl.rs @@ -7,8 +7,8 @@ use serde::de::DeserializeOwned; use crate::cloud_provider::digitalocean::models::svc::DOKubernetesList; use crate::cmd::structs::{ - Item, KubernetesEvent, KubernetesJob, KubernetesList, KubernetesNode, KubernetesPod, KubernetesPodStatusPhase, - KubernetesService, LabelsContent, + Item, KubernetesEvent, KubernetesJob, KubernetesKind, KubernetesList, KubernetesNode, KubernetesPod, + KubernetesPodStatusPhase, KubernetesService, LabelsContent, }; use crate::cmd::utilities::exec_with_envs_and_output; use crate::constants::KUBECONFIG; @@ -686,6 +686,24 @@ where kubectl_exec::>(vec!["get", "node", "-o", "json"], kubernetes_config, envs) } +pub fn kubectl_exec_count_all_objects

( + kubernetes_config: P, + object_kind: &str, + envs: Vec<(&str, &str)>, +) -> Result +where + P: AsRef, +{ + match kubectl_exec::>( + vec!["get", object_kind, "-A", "-o", "json"], + kubernetes_config, + envs, + ) { + Ok(o) => Ok(o.items.len()), + Err(e) => Err(e), + } +} + pub fn kubectl_exec_get_pod

( kubernetes_config: P, namespace: &str, diff --git a/src/cmd/structs.rs b/src/cmd/structs.rs index e2cffaf2..168ccd80 100644 --- a/src/cmd/structs.rs +++ b/src/cmd/structs.rs @@ -187,6 +187,12 @@ pub struct KubernetesInvolvedObject { pub name: String, } +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "camelCase")] +pub struct KubernetesKind { + pub kind: String, +} + #[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)] #[serde(rename_all = "camelCase")] pub struct Helm {