fix: cert-manager uninstall random issue

src/cloud_provider/aws/kubernetes/mod.rs

@@ -8,11 +8,11 @@ use tera::Context as TeraContext;
 use crate::cloud_provider::aws::kubernetes::node::Node;
 use crate::cloud_provider::aws::AWS;
 use crate::cloud_provider::environment::Environment;
-use crate::cloud_provider::kubernetes::{Kind, Kubernetes, KubernetesNode};
+use crate::cloud_provider::kubernetes::{uninstall_cert_manager, Kind, Kubernetes, KubernetesNode};
 use crate::cloud_provider::models::WorkerNodeDataTemplate;
 use crate::cloud_provider::{kubernetes, CloudProvider};
 use crate::cmd;
-use crate::cmd::kubectl::{kubectl_delete_objects_in_all_namespaces, kubectl_exec_get_all_namespaces};
+use crate::cmd::kubectl::kubectl_exec_get_all_namespaces;
 use crate::deletion_utilities::{get_firsts_namespaces_to_delete, get_qovery_managed_namespaces};
 use crate::dns_provider;
 use crate::dns_provider::DnsProvider;
@@ -638,39 +638,21 @@ impl<'a> Kubernetes for EKS<'a> {
             self.context.execution_id(),
         ));
 
-        // https://cert-manager.io/docs/installation/uninstall/kubernetes/
         // required to avoid namespace stuck on deletion
-        info!("Delete cert-manager related objects to prepare deletion");
-        let cert_manager_objects = vec![
-            "Issuers",
-            "ClusterIssuers",
-            "Certificates",
-            "CertificateRequests",
-            "Orders",
-            "Challenges",
-        ];
-        for object in cert_manager_objects {
-            match kubectl_delete_objects_in_all_namespaces(
-                &kubernetes_config_file_path,
-                object,
-                self.cloud_provider().credentials_environment_variables(),
-            ) {
-                Ok(_) => {}
-                Err(e) => {
-                    let error_message = format!(
-                        "Wasn't able to delete all objects type {}, it's a blocker to then delete cert-manager namespace. {}",
-                        object,
-                        format!("{:?}", e.message)
-                    );
-                    return Err(EngineError::new(
-                        Internal,
-                        self.engine_error_scope(),
-                        self.context().execution_id(),
-                        Some(error_message),
-                    ));
-                }
-            };
-        }
+        match uninstall_cert_manager(
+            &kubernetes_config_file_path,
+            self.cloud_provider().credentials_environment_variables(),
+        ) {
+            Ok(_) => {}
+            Err(e) => {
+                return Err(EngineError::new(
+                    Internal,
+                    self.engine_error_scope(),
+                    self.context().execution_id(),
+                    e.message,
+                ))
+            }
+        };
 
         info!("Deleting Qovery managed Namespaces");
         let qovery_namespaces = get_qovery_managed_namespaces();

src/cloud_provider/kubernetes.rs

@@ -9,11 +9,17 @@ use crate::cloud_provider::environment::Environment;
 use crate::cloud_provider::service::CheckAction;
 use crate::cloud_provider::{service, CloudProvider, DeploymentTarget};
 use crate::cmd::kubectl;
+use crate::cmd::kubectl::{kubectl_delete_objects_in_all_namespaces, kubectl_exec_count_all_objects};
 use crate::dns_provider::DnsProvider;
-use crate::error::{cast_simple_error_to_engine_error, EngineError, EngineErrorCause, EngineErrorScope};
+use crate::error::SimpleErrorKind::Other;
+use crate::error::{cast_simple_error_to_engine_error, EngineError, EngineErrorCause, EngineErrorScope, SimpleError};
 use crate::models::{Context, Listen, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope, StringPath};
 use crate::object_storage::ObjectStorage;
 use crate::unit_conversion::{any_to_mi, cpu_string_to_float};
+use retry::delay::Fibonacci;
+use retry::Error::Operation;
+use retry::OperationResult;
+use std::path::Path;
 
 pub trait Kubernetes: Listen {
     fn context(&self) -> &Context;
@@ -520,3 +526,63 @@ pub fn check_kubernetes_has_enough_resources_to_deploy_environment(
 
     Ok(())
 }
+
+pub fn uninstall_cert_manager<P>(kubernetes_config: P, envs: Vec<(&str, &str)>) -> Result<(), SimpleError>
+where
+    P: AsRef<Path>,
+{
+    // https://cert-manager.io/docs/installation/uninstall/kubernetes/
+    info!("Delete cert-manager related objects to prepare deletion");
+
+    let cert_manager_objects = vec![
+        "Issuers",
+        "ClusterIssuers",
+        "Certificates",
+        "CertificateRequests",
+        "Orders",
+        "Challenges",
+    ];
+
+    for object in cert_manager_objects {
+        // check resource exist first
+        match kubectl_exec_count_all_objects(&kubernetes_config, object, envs.clone()) {
+            Ok(x) if x == 0 => continue,
+            Err(e) => {
+                warn!(
+                    "encountering issues while trying to get objects kind {}: {:?}",
+                    object, e.message
+                );
+                continue;
+            }
+            _ => {}
+        }
+
+        // delete if resource exists
+        let result =
+            retry::retry(
+                Fibonacci::from_millis(5000).take(3),
+                || match kubectl_delete_objects_in_all_namespaces(&kubernetes_config, object, envs.clone()) {
+                    Ok(_) => OperationResult::Ok(()),
+                    Err(e) => {
+                        warn!("Failed to delete all {} objects, retrying...", object);
+                        OperationResult::Retry(e)
+                    }
+                },
+            );
+
+        match result {
+            Ok(_) => {}
+            Err(Operation { error, .. }) => return Err(error),
+            Err(retry::Error::Internal(msg)) => {
+                let error_message = format!(
+                    "Wasn't able to delete all objects type {}, it's a blocker to then delete cert-manager namespace. {}",
+                    object,
+                    format!("{:?}", msg)
+                );
+                return Err(SimpleError::new(Other, Some(error_message)));
+            }
+        };
+    }
+
+    Ok(())
+}

src/cmd/kubectl.rs

@@ -7,8 +7,8 @@ use serde::de::DeserializeOwned;
 
 use crate::cloud_provider::digitalocean::models::svc::DOKubernetesList;
 use crate::cmd::structs::{
-    Item, KubernetesEvent, KubernetesJob, KubernetesList, KubernetesNode, KubernetesPod, KubernetesPodStatusPhase,
-    KubernetesService, LabelsContent,
+    Item, KubernetesEvent, KubernetesJob, KubernetesKind, KubernetesList, KubernetesNode, KubernetesPod,
+    KubernetesPodStatusPhase, KubernetesService, LabelsContent,
 };
 use crate::cmd::utilities::exec_with_envs_and_output;
 use crate::constants::KUBECONFIG;
@@ -686,6 +686,24 @@ where
     kubectl_exec::<P, KubernetesList<KubernetesNode>>(vec!["get", "node", "-o", "json"], kubernetes_config, envs)
 }
 
+pub fn kubectl_exec_count_all_objects<P>(
+    kubernetes_config: P,
+    object_kind: &str,
+    envs: Vec<(&str, &str)>,
+) -> Result<usize, SimpleError>
+where
+    P: AsRef<Path>,
+{
+    match kubectl_exec::<P, KubernetesList<KubernetesKind>>(
+        vec!["get", object_kind, "-A", "-o", "json"],
+        kubernetes_config,
+        envs,
+    ) {
+        Ok(o) => Ok(o.items.len()),
+        Err(e) => Err(e),
+    }
+}
+
 pub fn kubectl_exec_get_pod<P>(
     kubernetes_config: P,
     namespace: &str,

src/cmd/structs.rs

@@ -187,6 +187,12 @@ pub struct KubernetesInvolvedObject {
     pub name: String,
 }
 
+#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
+#[serde(rename_all = "camelCase")]
+pub struct KubernetesKind {
+    pub kind: String,
+}
+
 #[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Helm {