feat: adding helm deployment support for AWS

Cargo.lock

@@ -955,9 +955,9 @@ dependencies = [
 
 [[package]]
 name = "globwalk"
-version = "0.8.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "178270263374052c40502e9f607134947de75302c1348d1a0e31db67c1691446"
+checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc"
 dependencies = [
  "bitflags",
  "ignore",
@@ -2140,6 +2140,18 @@ dependencies = [
  "rand_hc 0.2.0",
 ]
 
+[[package]]
+name = "rand"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
+dependencies = [
+ "libc",
+ "rand_chacha 0.3.0",
+ "rand_core 0.6.2",
+ "rand_hc 0.3.0",
+]
+
 [[package]]
 name = "rand_chacha"
 version = "0.1.1"
@@ -2160,6 +2172,16 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
+[[package]]
+name = "rand_chacha"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.6.2",
+]
+
 [[package]]
 name = "rand_core"
 version = "0.3.1"
@@ -2184,6 +2206,15 @@ dependencies = [
  "getrandom 0.1.15",
 ]
 
+[[package]]
+name = "rand_core"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
+dependencies = [
+ "getrandom 0.2.2",
+]
+
 [[package]]
 name = "rand_hc"
 version = "0.1.0"
@@ -2202,6 +2233,15 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
+[[package]]
+name = "rand_hc"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73"
+dependencies = [
+ "rand_core 0.6.2",
+]
+
 [[package]]
 name = "rand_isaac"
 version = "0.1.1"
@@ -3039,9 +3079,9 @@ dependencies = [
 
 [[package]]
 name = "tera"
-version = "1.5.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1381c83828bedd5ce4e59473110afa5381ffe523406d9ade4b77c9f7be70ff9a"
+checksum = "81060acb882480c8793782eb96bc86f5c83d2fc7175ad46c375c6956ef7afa62"
 dependencies = [
  "chrono",
  "chrono-tz",
@@ -3051,7 +3091,7 @@ dependencies = [
  "percent-encoding 2.1.0",
  "pest",
  "pest_derive",
- "rand 0.7.3",
+ "rand 0.8.3",
  "regex",
  "serde",
  "serde_json",

Cargo.toml

@@ -41,7 +41,8 @@ tracing-subscriber = "0.2"
 sysinfo = "0.16.4"
 
 # Jinja2
-tera = "1.3.1"
+tera = "1.10.0"
+# Json
 serde = "1.0.114"
 serde_json = "1.0.57"
 serde_derive = "1.0"

lib/aws/bootstrap/qovery-tf-config.j2.tf

@@ -0,0 +1,41 @@
+locals {
+  qovery_tf_config = <<TF_CONFIG
+{
+  "cloud_provider": "${var.cloud_provider}",
+  "region": "${var.region}",
+  "cluster_name": "${var.kubernetes_cluster_name}",
+  "cluster_id": "${var.kubernetes_cluster_id}",
+  "organization_id": "${var.organization_id}",
+  "test_cluster": "${var.test_cluster}",
+  "aws_access_key_id": "{{ aws_access_key }}",
+  "aws_secret_access_key": "{{ aws_secret_key }}",
+  "external_dns_provider": "{{ external_dns_provider }}",
+  "dns_email_report": {{ dns_email_report }},
+  "acme_server_url": "{{ acme_server_url }}",
+  "managed_dns_domains_terraform_format": "{{ managed_dns_domains_terraform_format }}",
+  "cloudflare_api_token": "{{ cloudflare_api_token }}",
+  "cloudflare_email": "{{ cloudflare_email }}",
+  "feature_flag_metrics_history": "{% if metrics_history_enabled %}true{% else %}false{% endif %}",
+  "aws_iam_eks_user_mapper_key": "${aws_iam_access_key.iam_eks_user_mapper.id}",
+  "aws_iam_eks_user_mapper_secret": "${aws_iam_access_key.iam_eks_user_mapper.secret}",
+  "aws_iam_cluster_autoscaler_key": "${aws_iam_access_key.iam_eks_cluster_autoscaler.id}",
+  "aws_iam_cluster_autoscaler_secret": "${aws_iam_access_key.iam_eks_cluster_autoscaler.secret}",
+  "managed_dns_resolvers_terraform_format": "{{ managed_dns_resolvers_terraform_format }}",
+  "feature_flag_log_history": "{% if log_history_enabled %}true{% else %}false{% endif %}",
+  "loki_storage_config_aws_s3": "s3://${urlencode(aws_iam_access_key.iam_eks_loki.id)}:${urlencode(aws_iam_access_key.iam_eks_loki.secret)}@${var.region}/${aws_s3_bucket.loki_bucket.bucket}",
+  "aws_iam_loki_storage_key": "${aws_iam_access_key.iam_eks_loki.id}",
+  "aws_iam_loki_storage_secret": "${aws_iam_access_key.iam_eks_loki.secret}",
+  "qovery_agent_version": "${data.external.get_agent_version_to_use.result.version},
+  "qovery_engine_version": "${data.external.get_agent_version_to_use.result.version},
+  "nats_host_url": "${var.qovery_nats_url}",
+  "nats_username": "${var.qovery_nats_user}",
+  "nats_password": "${var.qovery_nats_password}"
+}
+TF_CONFIG
+}
+
+resource "local_file" "qovery_tf_config" {
+  filename = "qovery-tf-config.json"
+  content = local.qovery_tf_config
+  file_permission = "0600"
+}

src/cloud_provider/aws/kubernetes/mod.rs

@@ -40,6 +40,7 @@ use crate::object_storage::s3::S3;
 use crate::object_storage::ObjectStorage;
 use crate::string::terraform_list_format;
 
+pub mod helm_charts;
 pub mod node;
 pub mod roles;
 

src/cloud_provider/helm.rs

@@ -0,0 +1,205 @@
+use crate::cloud_provider::helm::HelmAction::Deploy;
+use crate::cloud_provider::helm::HelmChartNamespaces::KubeSystem;
+use crate::cmd::helm::{helm_exec_uninstall_with_chart_info, helm_exec_upgrade_with_chart_info};
+use crate::cmd::kubectl::kubectl_exec_rollout_restart_deployment;
+use crate::error::{SimpleError, SimpleErrorKind};
+use std::path::Path;
+use std::{fs, thread};
+use thread::spawn;
+
+#[derive(Clone)]
+pub enum HelmAction {
+    Deploy,
+    Destroy,
+    Skip,
+}
+
+#[derive(Copy, Clone)]
+pub enum HelmChartNamespaces {
+    KubeSystem,
+    Prometheus,
+    Logging,
+    CertManager,
+    NginxIngress,
+    Qovery,
+}
+
+#[derive(Clone)]
+pub struct ChartSetValue {
+    pub key: String,
+    pub value: String,
+}
+
+#[derive(Clone)]
+pub struct ChartInfo {
+    pub name: String,
+    pub path: String,
+    pub namespace: HelmChartNamespaces,
+    pub action: HelmAction,
+    pub atomic: bool,
+    pub force_upgrade: bool,
+    pub timeout: String,
+    pub dry_run: bool,
+    pub wait: bool,
+    pub values: Vec<ChartSetValue>,
+    pub values_files: Vec<String>,
+}
+
+impl Default for ChartInfo {
+    fn default() -> ChartInfo {
+        ChartInfo {
+            name: "undefined".to_string(),
+            path: "undefined".to_string(),
+            namespace: KubeSystem,
+            action: Deploy,
+            atomic: true,
+            force_upgrade: false,
+            timeout: "300s".to_string(),
+            dry_run: false,
+            wait: true,
+            values: Vec::new(),
+            values_files: Vec::new(),
+        }
+    }
+}
+
+pub fn get_chart_namespace(namespace: HelmChartNamespaces) -> String {
+    match namespace {
+        HelmChartNamespaces::KubeSystem => "kube-system",
+        HelmChartNamespaces::Prometheus => "prometheus",
+        HelmChartNamespaces::Logging => "logging",
+        HelmChartNamespaces::CertManager => "cert-manager",
+        HelmChartNamespaces::NginxIngress => "nginx-ingress",
+        HelmChartNamespaces::Qovery => "qovery",
+    }
+    .to_string()
+}
+
+pub trait HelmChart {
+    fn check_prerequisites(&self) -> Result<(), SimpleError> {
+        let chart = self.get_chart_info();
+        for file in chart.values_files.iter() {
+            match fs::metadata(file) {
+                Ok(_) => {}
+                Err(e) => {
+                    return Err(SimpleError {
+                        kind: SimpleErrorKind::Other,
+                        message: Some(format!(
+                            "Can't access helm chart override file {} for chart {}. {:?}",
+                            file, chart.name, e
+                        )),
+                    })
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn get_chart_info(&self) -> &ChartInfo;
+
+    fn namespace(&self) -> String {
+        get_chart_namespace(self.get_chart_info().namespace)
+    }
+
+    fn pre_exec(&self, _kubernetes_config: &Path, _envs: &[(String, String)]) -> Result<(), SimpleError> {
+        //
+        Ok(())
+    }
+
+    fn run(&self, kubernetes_config: &Path, envs: &[(String, String)]) -> Result<(), SimpleError> {
+        self.check_prerequisites()?;
+        self.pre_exec(&kubernetes_config, &envs)?;
+        match self.exec(&kubernetes_config, &envs) {
+            Ok(_) => {}
+            Err(e) => {
+                error!("Error while deploying chart: {:?}", e.message);
+                return self.on_deploy_failure(&kubernetes_config, &envs);
+            }
+        };
+        self.post_exec(&kubernetes_config, &envs)?;
+        Ok(())
+    }
+
+    fn exec(&self, kubernetes_config: &Path, envs: &[(String, String)]) -> Result<(), SimpleError> {
+        let environment_variables = envs.iter().map(|x| (x.0.as_str(), x.1.as_str())).collect();
+        match self.get_chart_info().action {
+            HelmAction::Deploy => {
+                helm_exec_upgrade_with_chart_info(kubernetes_config, &environment_variables, self.get_chart_info())
+            }
+            HelmAction::Destroy => {
+                helm_exec_uninstall_with_chart_info(kubernetes_config, &environment_variables, self.get_chart_info())
+            }
+            HelmAction::Skip => Ok(()),
+        }
+    }
+
+    fn post_exec(&self, _kubernetes_config: &Path, _envs: &[(String, String)]) -> Result<(), SimpleError> {
+        Ok(())
+    }
+    fn on_deploy_failure(&self, _kubernetes_config: &Path, _envs: &[(String, String)]) -> Result<(), SimpleError> {
+        Ok(())
+    }
+}
+
+//todo: implement it
+#[allow(unused_must_use)]
+pub fn deploy_multiple_charts<H: 'static + HelmChart + Sync + Send>(
+    kubernetes_config: &'static Path,
+    envs: &Vec<(String, String)>,
+    charts: Vec<H>,
+) {
+    let mut handles = vec![];
+
+    for chart in charts.into_iter() {
+        let environment_variables = envs.clone();
+        let kubeconfig = kubernetes_config.clone();
+        let handle = spawn(move || chart.run(kubeconfig, &environment_variables));
+        handles.push(handle);
+    }
+
+    for handle in handles {
+        handle.join().unwrap();
+    }
+}
+
+//
+// Common charts
+//
+
+#[derive(Default)]
+pub struct CommonChart {
+    pub chart_info: ChartInfo,
+}
+
+impl CommonChart {}
+
+impl HelmChart for CommonChart {
+    fn get_chart_info(&self) -> &ChartInfo {
+        &self.chart_info
+    }
+}
+
+// CoreDNS config
+
+#[derive(Default)]
+pub struct CoreDNSConfigChart {
+    pub chart_info: ChartInfo,
+}
+
+impl HelmChart for CoreDNSConfigChart {
+    fn get_chart_info(&self) -> &ChartInfo {
+        &self.chart_info
+    }
+
+    // todo: it would be better to avoid rebooting coredns on every run
+    fn post_exec(&self, kubernetes_config: &Path, envs: &[(String, String)]) -> Result<(), SimpleError> {
+        let environment_variables = envs.iter().map(|x| (x.0.as_str(), x.1.as_str())).collect();
+
+        kubectl_exec_rollout_restart_deployment(
+            kubernetes_config,
+            &self.chart_info.name,
+            self.namespace().as_str(),
+            environment_variables,
+        )
+    }
+}

src/cloud_provider/mod.rs

@@ -11,6 +11,7 @@ pub mod aws;
 pub mod digitalocean;
 pub mod environment;
 pub mod gcp;
+pub mod helm;
 pub mod kubernetes;
 pub mod metrics;
 pub mod models;

src/cmd/helm.rs

@@ -3,6 +3,7 @@ use std::path::Path;
 
 use tracing::{error, info, span, Level};
 
+use crate::cloud_provider::helm::{get_chart_namespace, ChartInfo};
 use crate::cmd::structs::{Helm, HelmChart, HelmHistoryRow};
 use crate::cmd::utilities::exec_with_envs_and_output;
 use crate::error::{SimpleError, SimpleErrorKind};
@@ -57,6 +58,97 @@ where
         .map(|helm_history_row| helm_history_row.clone()))
 }
 
+pub fn helm_exec_upgrade_with_chart_info<P>(
+    kubernetes_config: P,
+    envs: &Vec<(&str, &str)>,
+    chart: &ChartInfo,
+) -> Result<(), SimpleError>
+where
+    P: AsRef<Path>,
+{
+    let mut args_string: Vec<String> = vec![
+        "upgrade",
+        "-o",
+        "json",
+        "--kubeconfig",
+        kubernetes_config.as_ref().to_str().unwrap(),
+        "--create-namespace",
+        "--install",
+        "--history-max",
+        "50",
+        "--namespace",
+        get_chart_namespace(chart.namespace).as_str(),
+    ]
+    .into_iter()
+    .map(|x| x.to_string())
+    .collect();
+
+    // warn: don't add debug or json output won't work
+    if chart.atomic {
+        args_string.push("--atomic".to_string())
+    }
+    if chart.force_upgrade {
+        args_string.push("--force".to_string())
+    }
+    if chart.dry_run {
+        args_string.push("--dry-run".to_string())
+    }
+    if chart.wait {
+        args_string.push("--wait".to_string())
+    }
+
+    // overrides and files overrides
+    for value in &chart.values {
+        args_string.push("--set".to_string());
+        args_string.push(format!("{}={}", value.key, value.value));
+    }
+    for value_file in &chart.values_files {
+        args_string.push("-f".to_string());
+        args_string.push(value_file.clone());
+    }
+
+    // add last elements
+    args_string.push(chart.name.to_string());
+    args_string.push(chart.path.to_string());
+
+    let args = args_string.iter().map(|x| x.as_str()).collect();
+
+    let mut json_output_string = String::new();
+    let mut error_message = String::new();
+    match helm_exec_with_output(
+        args,
+        envs.clone(),
+        |out| match out {
+            Ok(line) => json_output_string = line,
+            Err(err) => error!("{:?}", err),
+        },
+        |out| match out {
+            Ok(line) => {
+                // helm errors are not json formatted unfortunately
+                if line.contains("has been rolled back") {
+                    error_message = format!("Deployment {} has been rolled back", chart.name);
+                    warn!("{}. {}", &error_message, &line);
+                } else if line.contains("has been uninstalled") {
+                    error_message = format!("Deployment {} has been uninstalled due to failure", chart.name);
+                    warn!("{}. {}", &error_message, &line);
+                } else {
+                    error_message = format!("Deployment {} has failed", chart.name);
+                    warn!("{}. {}", &error_message, &line);
+                }
+            }
+            Err(err) => error!("{:?}", err),
+        },
+    ) {
+        Ok(_) => Ok(()),
+        Err(e) => {
+            return Err(SimpleError {
+                kind: SimpleErrorKind::Other,
+                message: Some(format!("{}. {:?}", error_message, e.message)),
+            })
+        }
+    }
+}
+
 pub fn helm_exec_upgrade<P>(
     kubernetes_config: P,
     namespace: &str,
@@ -105,6 +197,35 @@ where
     )
 }
 
+pub fn helm_exec_uninstall_with_chart_info<P>(
+    kubernetes_config: P,
+    envs: &Vec<(&str, &str)>,
+    chart: &ChartInfo,
+) -> Result<(), SimpleError>
+where
+    P: AsRef<Path>,
+{
+    helm_exec_with_output(
+        vec![
+            "uninstall",
+            "--kubeconfig",
+            kubernetes_config.as_ref().to_str().unwrap(),
+            "--namespace",
+            get_chart_namespace(chart.namespace).as_str(),
+            &chart.name,
+        ],
+        envs.clone(),
+        |out| match out {
+            Ok(line) => info!("{}", line.as_str()),
+            Err(err) => error!("{}", err),
+        },
+        |out| match out {
+            Ok(line) => error!("{}", line.as_str()),
+            Err(err) => error!("{}", err),
+        },
+    )
+}
+
 pub fn helm_exec_uninstall<P>(
     kubernetes_config: P,
     namespace: &str,

src/cmd/kubectl.rs

@@ -697,6 +697,57 @@ where
     kubectl_exec::<P, KubernetesVersion>(vec!["version", "-o", "json"], kubernetes_config, envs)
 }
 
+pub fn kubectl_exec_get_daemonset<P>(
+    kubernetes_config: P,
+    name: &str,
+    namespace: &str,
+    selectors: Option<&str>,
+    envs: Vec<(&str, &str)>,
+) -> Result<KubernetesList<KubernetesNode>, SimpleError>
+where
+    P: AsRef<Path>,
+{
+    let mut args = vec!["-n", namespace, "get", "daemonset", name];
+    match selectors {
+        Some(x) => {
+            args.push("-l");
+            args.push(x);
+        }
+        None => {}
+    };
+    args.push("-o");
+    args.push("json");
+
+    kubectl_exec::<P, KubernetesList<KubernetesNode>>(args, kubernetes_config, envs)
+}
+
+pub fn kubectl_exec_rollout_restart_deployment<P>(
+    kubernetes_config: P,
+    name: &str,
+    namespace: &str,
+    envs: Vec<(&str, &str)>,
+) -> Result<(), SimpleError>
+where
+    P: AsRef<Path>,
+{
+    let mut environment_variables: Vec<(&str, &str)> = envs;
+    environment_variables.push(("KUBECONFIG", kubernetes_config.as_ref().to_str().unwrap()));
+    let args = vec!["-n", namespace, "rollout", "restart", "deployment", name];
+
+    kubectl_exec_with_output(
+        args,
+        environment_variables.clone(),
+        |out| match out {
+            Ok(line) => info!("{}", line),
+            Err(err) => error!("{:?}", err),
+        },
+        |out| match out {
+            Ok(line) => error!("{}", line),
+            Err(err) => error!("{:?}", err),
+        },
+    )
+}
+
 pub fn kubectl_exec_get_node<P>(
     kubernetes_config: P,
     envs: Vec<(&str, &str)>,

src/cmd/structs.rs

@@ -63,6 +63,15 @@ pub struct Metadata {
     pub uid: String,
 }
 
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Daemonset {
+    pub api_version: String,
+    pub items: Vec<Item>,
+    pub kind: String,
+    pub metadata: Metadata,
+}
+
 #[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
 #[serde(rename_all = "camelCase")]
 pub struct KubernetesServiceStatus {

src/template.rs

@@ -42,6 +42,8 @@ where
                 tera::ErrorKind::CallFilter(x) => format!("call filter: {}", x),
                 tera::ErrorKind::CallTest(x) => format!("call test: {}", x),
                 tera::ErrorKind::__Nonexhaustive => "non exhaustive error".to_string(),
+                tera::ErrorKind::Io(x) => format!("io error {:?}", x),
+                tera::ErrorKind::Utf8Conversion { .. } => format!("utf-8 conversion issue"),
             };
 
             error!("{}", context.clone().into_json());

test_utilities/Cargo.lock

@@ -941,9 +941,9 @@ dependencies = [
 
 [[package]]
 name = "globwalk"
-version = "0.8.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "178270263374052c40502e9f607134947de75302c1348d1a0e31db67c1691446"
+checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc"
 dependencies = [
  "bitflags",
  "ignore",
@@ -2115,6 +2115,18 @@ dependencies = [
  "rand_hc 0.2.0",
 ]
 
+[[package]]
+name = "rand"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
+dependencies = [
+ "libc",
+ "rand_chacha 0.3.0",
+ "rand_core 0.6.2",
+ "rand_hc 0.3.0",
+]
+
 [[package]]
 name = "rand_chacha"
 version = "0.1.1"
@@ -2135,6 +2147,16 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
+[[package]]
+name = "rand_chacha"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.6.2",
+]
+
 [[package]]
 name = "rand_core"
 version = "0.3.1"
@@ -2159,6 +2181,15 @@ dependencies = [
  "getrandom 0.1.16",
 ]
 
+[[package]]
+name = "rand_core"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
+dependencies = [
+ "getrandom 0.2.2",
+]
+
 [[package]]
 name = "rand_hc"
 version = "0.1.0"
@@ -2177,6 +2208,15 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
+[[package]]
+name = "rand_hc"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73"
+dependencies = [
+ "rand_core 0.6.2",
+]
+
 [[package]]
 name = "rand_isaac"
 version = "0.1.1"
@@ -3012,9 +3052,9 @@ dependencies = [
 
 [[package]]
 name = "tera"
-version = "1.5.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1381c83828bedd5ce4e59473110afa5381ffe523406d9ade4b77c9f7be70ff9a"
+checksum = "81060acb882480c8793782eb96bc86f5c83d2fc7175ad46c375c6956ef7afa62"
 dependencies = [
  "chrono",
  "chrono-tz",
@@ -3024,7 +3064,7 @@ dependencies = [
  "percent-encoding 2.1.0",
  "pest",
  "pest_derive",
- "rand 0.7.3",
+ "rand 0.8.3",
  "regex",
  "serde",
  "serde_json",

tests/aws/aws_environment.rs

@@ -559,6 +559,7 @@ fn deploy_a_not_working_environment_and_after_working_environment() {
 
 // #[cfg(feature = "test-aws-self-hosted")]
 // #[test]
+#[allow(dead_code)] // todo: make it work
 fn deploy_ok_fail_fail_ok_environment() {
     init();
 

tests/aws/aws_kubernetes.rs

@@ -236,6 +236,7 @@ fn create_and_destroy_eks_cluster_in_us_east_2() {
 
 // only enable this test manually when we want to perform and validate upgrade process
 //#[test]
+#[allow(dead_code)]
 fn create_upgrade_and_destroy_eks_cluster_in_eu_west_3() {
     let region = "eu-west-3";
     let secrets = FuncTestsSecrets::new();