From 8f95ef227d53baad4ca3613ff877739f22820761 Mon Sep 17 00:00:00 2001 From: MacLikorne Date: Wed, 16 Feb 2022 17:58:15 +0100 Subject: [PATCH 1/7] chore: upgrade pleco to v0.10.2 (#609) --- lib/common/bootstrap/charts/pleco/Chart.yaml | 4 ++-- lib/common/bootstrap/charts/pleco/templates/deployment.yaml | 2 -- lib/common/bootstrap/charts/pleco/values.yaml | 2 +- lib/helm-freeze.yaml | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/common/bootstrap/charts/pleco/Chart.yaml b/lib/common/bootstrap/charts/pleco/Chart.yaml index 3c601188..1e585e88 100644 --- a/lib/common/bootstrap/charts/pleco/Chart.yaml +++ b/lib/common/bootstrap/charts/pleco/Chart.yaml @@ -1,9 +1,9 @@ apiVersion: v2 -appVersion: 0.10.1 +appVersion: 0.10.2 description: Automatically removes Cloud managed services and Kubernetes resources based on tags with TTL home: https://github.com/Qovery/pleco icon: https://github.com/Qovery/pleco/raw/main/assets/pleco_logo.png name: pleco type: application -version: 0.10.1 +version: 0.10.2 diff --git a/lib/common/bootstrap/charts/pleco/templates/deployment.yaml b/lib/common/bootstrap/charts/pleco/templates/deployment.yaml index 87854543..89f3b959 100644 --- a/lib/common/bootstrap/charts/pleco/templates/deployment.yaml +++ b/lib/common/bootstrap/charts/pleco/templates/deployment.yaml @@ -148,8 +148,6 @@ spec: {{ end }} {{- end }} env: - - name: "AWS_EXECUTION_ENV" - value: "pleco_{{ .Values.image.plecoImageTag }}_{{ .Values.environmentVariables.PLECO_IDENTIFIER }}" {{ range $key, $value := .Values.environmentVariables -}} - name: "{{ $key }}" valueFrom: diff --git a/lib/common/bootstrap/charts/pleco/values.yaml b/lib/common/bootstrap/charts/pleco/values.yaml index 480879c0..20664798 100644 --- a/lib/common/bootstrap/charts/pleco/values.yaml +++ b/lib/common/bootstrap/charts/pleco/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 image: repository: qoveryrd/pleco pullPolicy: IfNotPresent - plecoImageTag: "0.10.1" + plecoImageTag: "0.10.2" cloudProvider: "" diff --git a/lib/helm-freeze.yaml b/lib/helm-freeze.yaml index 35434bfc..38405056 100644 --- a/lib/helm-freeze.yaml +++ b/lib/helm-freeze.yaml @@ -70,7 +70,7 @@ charts: dest: services no_sync: true - name: pleco - version: 0.10.1 + version: 0.10.2 repo_name: pleco - name: do-k8s-token-rotate version: 0.1.3 From 55ca137b64a5e80290ceeb37546c50359cde40a7 Mon Sep 17 00:00:00 2001 From: MacLikorne Date: Thu, 17 Feb 2022 14:05:33 +0100 Subject: [PATCH 2/7] chore: upgrade pleco to v0.10.4 (#610) --- lib/common/bootstrap/charts/pleco/Chart.yaml | 4 ++-- lib/common/bootstrap/charts/pleco/values.yaml | 2 +- lib/helm-freeze.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/common/bootstrap/charts/pleco/Chart.yaml b/lib/common/bootstrap/charts/pleco/Chart.yaml index 1e585e88..224f4ebd 100644 --- a/lib/common/bootstrap/charts/pleco/Chart.yaml +++ b/lib/common/bootstrap/charts/pleco/Chart.yaml @@ -1,9 +1,9 @@ apiVersion: v2 -appVersion: 0.10.2 +appVersion: 0.10.4 description: Automatically removes Cloud managed services and Kubernetes resources based on tags with TTL home: https://github.com/Qovery/pleco icon: https://github.com/Qovery/pleco/raw/main/assets/pleco_logo.png name: pleco type: application -version: 0.10.2 +version: 0.10.4 diff --git a/lib/common/bootstrap/charts/pleco/values.yaml b/lib/common/bootstrap/charts/pleco/values.yaml index 20664798..09b4e135 100644 --- a/lib/common/bootstrap/charts/pleco/values.yaml +++ b/lib/common/bootstrap/charts/pleco/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 image: repository: qoveryrd/pleco pullPolicy: IfNotPresent - plecoImageTag: "0.10.2" + plecoImageTag: "0.10.4" cloudProvider: "" diff --git a/lib/helm-freeze.yaml b/lib/helm-freeze.yaml index 38405056..9035b942 100644 --- a/lib/helm-freeze.yaml +++ b/lib/helm-freeze.yaml @@ -70,7 +70,7 @@ charts: dest: services no_sync: true - name: pleco - version: 0.10.2 + version: 0.10.4 repo_name: pleco - name: do-k8s-token-rotate version: 0.1.3 From c17715411ee752c4c5c2d18cbab204ed13e7d648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Er=C3=A8be=20-=20Romain=20Gerard?= Date: Thu, 17 Feb 2022 17:40:23 +0100 Subject: [PATCH 3/7] Helm refacto (#607) * Refacto helm --- Cargo.toml | 3 + .../charts/ingress-nginx/values.yaml | 2 +- .../charts/kube-prometheus-stack/values.yaml | 4 +- .../aws/kubernetes/helm_charts.rs | 36 +- src/cloud_provider/aws/kubernetes/mod.rs | 95 +- src/cloud_provider/aws/router.rs | 41 +- .../digitalocean/kubernetes/helm_charts.rs | 12 +- .../digitalocean/kubernetes/mod.rs | 110 +- src/cloud_provider/digitalocean/router.rs | 37 +- src/cloud_provider/helm.rs | 134 +- .../scaleway/kubernetes/helm_charts.rs | 34 +- src/cloud_provider/scaleway/kubernetes/mod.rs | 95 +- src/cloud_provider/scaleway/router.rs | 39 +- src/cloud_provider/service.rs | 175 +- src/cloud_provider/utilities.rs | 10 +- src/cmd/helm.rs | 1591 ++++++++--------- src/errors/mod.rs | 24 + tests/helm/simple_nginx/.helmignore | 23 + tests/helm/simple_nginx/Chart.yaml | 24 + .../helm/simple_nginx/templates/_helpers.tpl | 62 + .../simple_nginx/templates/deployment.yaml | 62 + tests/helm/simple_nginx/templates/hpa.yaml | 28 + .../helm/simple_nginx/templates/ingress.yaml | 61 + .../helm/simple_nginx/templates/service.yaml | 15 + .../templates/serviceaccount.yaml | 12 + .../templates/tests/test-connection.yaml | 15 + tests/helm/simple_nginx/values.yaml | 83 + 27 files changed, 1507 insertions(+), 1320 deletions(-) create mode 100644 tests/helm/simple_nginx/.helmignore create mode 100644 tests/helm/simple_nginx/Chart.yaml create mode 100644 tests/helm/simple_nginx/templates/_helpers.tpl create mode 100644 tests/helm/simple_nginx/templates/deployment.yaml create mode 100644 tests/helm/simple_nginx/templates/hpa.yaml create mode 100644 tests/helm/simple_nginx/templates/ingress.yaml create mode 100644 tests/helm/simple_nginx/templates/service.yaml create mode 100644 tests/helm/simple_nginx/templates/serviceaccount.yaml create mode 100644 tests/helm/simple_nginx/templates/tests/test-connection.yaml create mode 100644 tests/helm/simple_nginx/values.yaml diff --git a/Cargo.toml b/Cargo.toml index b4bf09ce..2154f995 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,3 +106,6 @@ test-all-whole-enchilada = ["test-aws-whole-enchilada", "test-do-whole-enchilada test-aws-all = ["test-aws-infra", "test-aws-managed-services", "test-aws-self-hosted", "test-aws-whole-enchilada"] test-do-all = ["test-do-infra", "test-do-managed-services", "test-do-self-hosted", "test-do-whole-enchilada"] test-scw-all = ["test-scw-infra", "test-scw-managed-services", "test-scw-self-hosted", "test-scw-whole-enchilada"] + +# functionnal test with only a k8s cluster as a dependency +test-with-kube = [] diff --git a/lib/common/bootstrap/charts/ingress-nginx/values.yaml b/lib/common/bootstrap/charts/ingress-nginx/values.yaml index f5496eb6..0da4ff98 100644 --- a/lib/common/bootstrap/charts/ingress-nginx/values.yaml +++ b/lib/common/bootstrap/charts/ingress-nginx/values.yaml @@ -505,7 +505,7 @@ controller: admissionWebhooks: annotations: {} - enabled: true + enabled: false failurePolicy: Fail # timeoutSeconds: 10 port: 8443 diff --git a/lib/common/bootstrap/charts/kube-prometheus-stack/values.yaml b/lib/common/bootstrap/charts/kube-prometheus-stack/values.yaml index 0b2a607b..07483cf9 100644 --- a/lib/common/bootstrap/charts/kube-prometheus-stack/values.yaml +++ b/lib/common/bootstrap/charts/kube-prometheus-stack/values.yaml @@ -1342,7 +1342,7 @@ prometheusOperator: ## rules from making their way into prometheus and potentially preventing the container from starting admissionWebhooks: failurePolicy: Fail - enabled: true + enabled: false ## A PEM encoded CA bundle which will be used to validate the webhook's server certificate. ## If unspecified, system trust roots on the apiserver are used. caBundle: "" @@ -1377,7 +1377,7 @@ prometheusOperator: # Use certmanager to generate webhook certs certManager: - enabled: false + enabled: true # issuerRef: # name: "issuer" # kind: "ClusterIssuer" diff --git a/src/cloud_provider/aws/kubernetes/helm_charts.rs b/src/cloud_provider/aws/kubernetes/helm_charts.rs index 08d386f1..f764be31 100644 --- a/src/cloud_provider/aws/kubernetes/helm_charts.rs +++ b/src/cloud_provider/aws/kubernetes/helm_charts.rs @@ -704,7 +704,9 @@ datasources: }, ChartSetValue { key: "prometheus.servicemonitor.enabled".to_string(), - value: chart_config_prerequisites.ff_metrics_history_enabled.to_string(), + // Due to cycle, prometheus need tls certificate from cert manager, and enabling this will require + // prometheus to be already installed + value: "false".to_string(), }, ChartSetValue { key: "prometheus.servicemonitor.prometheusInstance".to_string(), @@ -730,11 +732,11 @@ datasources: // Webhooks resources limits ChartSetValue { key: "webhook.resources.limits.cpu".to_string(), - value: "20m".to_string(), + value: "200m".to_string(), }, ChartSetValue { key: "webhook.resources.requests.cpu".to_string(), - value: "20m".to_string(), + value: "50m".to_string(), }, ChartSetValue { key: "webhook.resources.limits.memory".to_string(), @@ -1156,23 +1158,25 @@ datasources: Box::new(old_prometheus_operator), ]; - let mut level_2: Vec> = vec![]; + let level_2: Vec> = vec![Box::new(cert_manager)]; - let mut level_3: Vec> = vec![ + let mut level_3: Vec> = vec![]; + + let mut level_4: Vec> = vec![ Box::new(cluster_autoscaler), Box::new(aws_iam_eks_user_mapper), Box::new(aws_calico), ]; - let mut level_4: Vec> = vec![ + let mut level_5: Vec> = vec![ Box::new(metrics_server), Box::new(aws_node_term_handler), Box::new(external_dns), ]; - let mut level_5: Vec> = vec![Box::new(nginx_ingress), Box::new(cert_manager)]; + let mut level_6: Vec> = vec![Box::new(nginx_ingress)]; - let mut level_6: Vec> = vec![ + let mut level_7: Vec> = vec![ Box::new(cert_manager_config), Box::new(qovery_agent), Box::new(shell_agent), @@ -1181,26 +1185,26 @@ datasources: // observability if chart_config_prerequisites.ff_metrics_history_enabled { - level_2.push(Box::new(kube_prometheus_stack)); - level_4.push(Box::new(prometheus_adapter)); - level_4.push(Box::new(kube_state_metrics)); + level_3.push(Box::new(kube_prometheus_stack)); + level_5.push(Box::new(prometheus_adapter)); + level_5.push(Box::new(kube_state_metrics)); } if chart_config_prerequisites.ff_log_history_enabled { - level_3.push(Box::new(promtail)); - level_4.push(Box::new(loki)); + level_4.push(Box::new(promtail)); + level_5.push(Box::new(loki)); } if chart_config_prerequisites.ff_metrics_history_enabled || chart_config_prerequisites.ff_log_history_enabled { - level_6.push(Box::new(grafana)) + level_7.push(Box::new(grafana)) }; // pleco if !chart_config_prerequisites.disable_pleco { - level_5.push(Box::new(pleco)); + level_6.push(Box::new(pleco)); } info!("charts configuration preparation finished"); - Ok(vec![level_1, level_2, level_3, level_4, level_5, level_6]) + Ok(vec![level_1, level_2, level_3, level_4, level_5, level_6, level_7]) } // AWS CNI diff --git a/src/cloud_provider/aws/kubernetes/mod.rs b/src/cloud_provider/aws/kubernetes/mod.rs index 9816b696..d4a4dd86 100644 --- a/src/cloud_provider/aws/kubernetes/mod.rs +++ b/src/cloud_provider/aws/kubernetes/mod.rs @@ -14,7 +14,7 @@ use crate::cloud_provider::aws::kubernetes::node::AwsInstancesType; use crate::cloud_provider::aws::kubernetes::roles::get_default_roles_to_create; use crate::cloud_provider::aws::regions::{AwsRegion, AwsZones}; use crate::cloud_provider::environment::Environment; -use crate::cloud_provider::helm::deploy_charts_levels; +use crate::cloud_provider::helm::{deploy_charts_levels, ChartInfo}; use crate::cloud_provider::kubernetes::{ is_kubernetes_upgrade_required, send_progress_on_long_task, uninstall_cert_manager, Kind, Kubernetes, KubernetesNodesType, KubernetesUpgradeStatus, ProviderOptions, @@ -24,11 +24,11 @@ use crate::cloud_provider::qovery::EngineLocation; use crate::cloud_provider::utilities::print_action; use crate::cloud_provider::{kubernetes, CloudProvider}; use crate::cmd; +use crate::cmd::helm::{to_engine_error, Helm}; use crate::cmd::kubectl::{ kubectl_exec_api_custom_metrics, kubectl_exec_get_all_namespaces, kubectl_exec_get_events, kubectl_exec_scale_replicas, ScalingKind, }; -use crate::cmd::structs::HelmChart; use crate::cmd::terraform::{terraform_exec, terraform_init_validate_plan_apply, terraform_init_validate_state_list}; use crate::deletion_utilities::{get_firsts_namespaces_to_delete, get_qovery_managed_namespaces}; use crate::dns_provider; @@ -1247,15 +1247,14 @@ impl<'a> EKS<'a> { ); // delete custom metrics api to avoid stale namespaces on deletion - let _ = cmd::helm::helm_uninstall_list( + let helm = Helm::new( &kubernetes_config_file_path, - vec![HelmChart { - name: "metrics-server".to_string(), - namespace: "kube-system".to_string(), - version: None, - }], - self.cloud_provider().credentials_environment_variables(), - ); + &self.cloud_provider.credentials_environment_variables(), + ) + .map_err(|e| to_engine_error(&event_details, e))?; + let chart = ChartInfo::new_from_release_name("metrics-server", "kube-system"); + helm.uninstall(&chart, &vec![]) + .map_err(|e| to_engine_error(&event_details, e))?; // required to avoid namespace stuck on deletion uninstall_cert_manager( @@ -1275,50 +1274,27 @@ impl<'a> EKS<'a> { let qovery_namespaces = get_qovery_managed_namespaces(); for qovery_namespace in qovery_namespaces.iter() { - let charts_to_delete = cmd::helm::helm_list( - &kubernetes_config_file_path, - self.cloud_provider().credentials_environment_variables(), - Some(qovery_namespace), - ); - match charts_to_delete { - Ok(charts) => { - for chart in charts { - match cmd::helm::helm_exec_uninstall( - &kubernetes_config_file_path, - &chart.namespace, - &chart.name, - self.cloud_provider().credentials_environment_variables(), - ) { - Ok(_) => self.logger().log( - LogLevel::Info, - EngineEvent::Deleting( - event_details.clone(), - EventMessage::new_from_safe(format!("Chart `{}` deleted", chart.name)), - ), - ), - Err(e) => { - let message_safe = format!("Can't delete chart `{}`", chart.name); - self.logger().log( - LogLevel::Error, - EngineEvent::Deleting( - event_details.clone(), - EventMessage::new(message_safe, Some(e.message())), - ), - ) - } - } - } - } - Err(e) => { - if !(e.message().contains("not found")) { + let charts_to_delete = helm + .list_release(Some(qovery_namespace), &vec![]) + .map_err(|e| to_engine_error(&event_details, e))?; + + for chart in charts_to_delete { + let chart_info = ChartInfo::new_from_release_name(&chart.name, &chart.namespace); + match helm.uninstall(&chart_info, &vec![]) { + Ok(_) => self.logger().log( + LogLevel::Info, + EngineEvent::Deleting( + event_details.clone(), + EventMessage::new_from_safe(format!("Chart `{}` deleted", chart.name)), + ), + ), + Err(e) => { + let message_safe = format!("Can't delete chart `{}`: {}", &chart.name, e); self.logger().log( LogLevel::Error, EngineEvent::Deleting( event_details.clone(), - EventMessage::new_from_safe(format!( - "Can't delete the namespace {}", - qovery_namespace - )), + EventMessage::new(message_safe, Some(e.to_string())), ), ) } @@ -1373,18 +1349,11 @@ impl<'a> EKS<'a> { ), ); - match cmd::helm::helm_list( - &kubernetes_config_file_path, - self.cloud_provider().credentials_environment_variables(), - None, - ) { + match helm.list_release(None, &vec![]) { Ok(helm_charts) => { for chart in helm_charts { - match cmd::helm::helm_uninstall_list( - &kubernetes_config_file_path, - vec![chart.clone()], - self.cloud_provider().credentials_environment_variables(), - ) { + let chart_info = ChartInfo::new_from_release_name(&chart.name, &chart.namespace); + match helm.uninstall(&chart_info, &vec![]) { Ok(_) => self.logger().log( LogLevel::Info, EngineEvent::Deleting( @@ -1393,12 +1362,12 @@ impl<'a> EKS<'a> { ), ), Err(e) => { - let message_safe = format!("Error deleting chart `{}` deleted", chart.name); + let message_safe = format!("Error deleting chart `{}`: {}", chart.name, e); self.logger().log( LogLevel::Error, EngineEvent::Deleting( event_details.clone(), - EventMessage::new(message_safe, e.message), + EventMessage::new(message_safe, Some(e.to_string())), ), ) } @@ -1411,7 +1380,7 @@ impl<'a> EKS<'a> { LogLevel::Error, EngineEvent::Deleting( event_details.clone(), - EventMessage::new(message_safe.to_string(), Some(e.message())), + EventMessage::new(message_safe.to_string(), Some(e.to_string())), ), ) } diff --git a/src/cloud_provider/aws/router.rs b/src/cloud_provider/aws/router.rs index 8cf44389..eccd99f4 100644 --- a/src/cloud_provider/aws/router.rs +++ b/src/cloud_provider/aws/router.rs @@ -1,5 +1,6 @@ use tera::Context as TeraContext; +use crate::cloud_provider::helm::ChartInfo; use crate::cloud_provider::models::{CustomDomain, CustomDomainDataTemplate, Route, RouteDataTemplate}; use crate::cloud_provider::service::{ default_tera_context, delete_router, deploy_stateless_service_error, send_progress_on_long_task, Action, Create, @@ -7,8 +8,9 @@ use crate::cloud_provider::service::{ }; use crate::cloud_provider::utilities::{check_cname_for, print_action, sanitize_name}; use crate::cloud_provider::DeploymentTarget; -use crate::cmd::helm::Timeout; -use crate::error::{EngineError, EngineErrorCause, EngineErrorScope}; +use crate::cmd::helm; +use crate::cmd::helm::{to_engine_error, Timeout}; +use crate::error::{EngineError, EngineErrorScope}; use crate::errors::EngineError as NewEngineError; use crate::events::{EnvironmentStep, Stage, ToTransmitter, Transmitter}; use crate::models::{Context, Listen, Listener, Listeners}; @@ -325,25 +327,26 @@ impl Create for Router { } // do exec helm upgrade and return the last deployment status - let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history( - kubernetes_config_file_path.as_str(), - environment.namespace(), - helm_release_name.as_str(), - self.selector(), - workspace_dir.as_str(), - self.start_timeout(), - kubernetes.cloud_provider().credentials_environment_variables(), - self.service_type(), + let helm = helm::Helm::new( + &kubernetes_config_file_path, + &kubernetes.cloud_provider().credentials_environment_variables(), ) - .map_err(|e| { - NewEngineError::new_helm_charts_upgrade_error(event_details.clone(), e).to_legacy_engine_error() - })?; + .map_err(|e| to_engine_error(&event_details, e).to_legacy_engine_error())?; + let chart = ChartInfo::new_from_custom_namespace( + helm_release_name, + workspace_dir.clone(), + environment.namespace().to_string(), + self.start_timeout().value() as i64, + match self.service_type() { + ServiceType::Database(_) => vec![format!("{}/q-values.yaml", &workspace_dir)], + _ => vec![], + }, + false, + self.selector(), + ); - if helm_history_row.is_none() || !helm_history_row.unwrap().is_successfully_deployed() { - return Err(self.engine_error(EngineErrorCause::Internal, "Router has failed to be deployed".into())); - } - - Ok(()) + helm.upgrade(&chart, &vec![]) + .map_err(|e| NewEngineError::new_helm_error(event_details.clone(), e).to_legacy_engine_error()) } fn on_create_check(&self) -> Result<(), EngineError> { diff --git a/src/cloud_provider/digitalocean/kubernetes/helm_charts.rs b/src/cloud_provider/digitalocean/kubernetes/helm_charts.rs index 3127c1b4..b420be08 100644 --- a/src/cloud_provider/digitalocean/kubernetes/helm_charts.rs +++ b/src/cloud_provider/digitalocean/kubernetes/helm_charts.rs @@ -544,7 +544,9 @@ datasources: }, ChartSetValue { key: "prometheus.servicemonitor.enabled".to_string(), - value: chart_config_prerequisites.ff_metrics_history_enabled.to_string(), + // Due to cycle, prometheus need tls certificate from cert manager, and enabling this will require + // prometheus to be already installed + value: "false".to_string(), }, ChartSetValue { key: "prometheus.servicemonitor.prometheusInstance".to_string(), @@ -570,11 +572,11 @@ datasources: // Webhooks resources limits ChartSetValue { key: "webhook.resources.limits.cpu".to_string(), - value: "20m".to_string(), + value: "200m".to_string(), }, ChartSetValue { key: "webhook.resources.requests.cpu".to_string(), - value: "20m".to_string(), + value: "50m".to_string(), }, ChartSetValue { key: "webhook.resources.limits.memory".to_string(), @@ -1033,13 +1035,13 @@ datasources: Box::new(old_prometheus_operator), ]; - let mut level_2: Vec> = vec![Box::new(container_registry_secret)]; + let mut level_2: Vec> = vec![Box::new(container_registry_secret), Box::new(cert_manager)]; let mut level_3: Vec> = vec![]; let mut level_4: Vec> = vec![Box::new(metrics_server), Box::new(external_dns)]; - let mut level_5: Vec> = vec![Box::new(nginx_ingress), Box::new(cert_manager)]; + let mut level_5: Vec> = vec![Box::new(nginx_ingress)]; let mut level_6: Vec> = vec![ Box::new(cert_manager_config), diff --git a/src/cloud_provider/digitalocean/kubernetes/mod.rs b/src/cloud_provider/digitalocean/kubernetes/mod.rs index 98a86757..9a3e7466 100644 --- a/src/cloud_provider/digitalocean/kubernetes/mod.rs +++ b/src/cloud_provider/digitalocean/kubernetes/mod.rs @@ -26,11 +26,10 @@ use crate::cloud_provider::models::NodeGroups; use crate::cloud_provider::qovery::EngineLocation; use crate::cloud_provider::utilities::{print_action, VersionsNumber}; use crate::cloud_provider::{kubernetes, CloudProvider}; -use crate::cmd::helm::{helm_exec_upgrade_with_chart_info, helm_upgrade_diff_with_chart_info}; +use crate::cmd::helm::{to_engine_error, Helm}; use crate::cmd::kubectl::{ do_kubectl_exec_get_loadbalancer_id, kubectl_exec_get_all_namespaces, kubectl_exec_get_events, }; -use crate::cmd::structs::HelmChart; use crate::cmd::terraform::{terraform_exec, terraform_init_validate_plan_apply, terraform_init_validate_state_list}; use crate::deletion_utilities::{get_firsts_namespaces_to_delete, get_qovery_managed_namespaces}; use crate::dns_provider::DnsProvider; @@ -816,18 +815,16 @@ impl<'a> DOKS<'a> { ..Default::default() }; - let _ = helm_upgrade_diff_with_chart_info( - &kubeconfig_path, - &credentials_environment_variables, - &load_balancer_dns_hostname, - ); - - helm_exec_upgrade_with_chart_info( + let helm = Helm::new( &kubeconfig_path, &self.cloud_provider.credentials_environment_variables(), - &load_balancer_dns_hostname, ) - .map_err(|e| EngineError::new_helm_charts_deploy_error(event_details.clone(), e)) + .map_err(|e| EngineError::new_helm_error(event_details.clone(), e))?; + + // This will ony print the diff on stdout + let _ = helm.upgrade_diff(&load_balancer_dns_hostname, &vec![]); + helm.upgrade(&load_balancer_dns_hostname, &vec![]) + .map_err(|e| EngineError::new_helm_error(event_details.clone(), e)) } fn create_error(&self) -> Result<(), EngineError> { @@ -1096,15 +1093,14 @@ impl<'a> DOKS<'a> { ); // delete custom metrics api to avoid stale namespaces on deletion - let _ = cmd::helm::helm_uninstall_list( + let helm = Helm::new( &kubernetes_config_file_path, - vec![HelmChart { - name: "metrics-server".to_string(), - namespace: "kube-system".to_string(), - version: None, - }], - self.cloud_provider().credentials_environment_variables(), - ); + &self.cloud_provider.credentials_environment_variables(), + ) + .map_err(|e| to_engine_error(&event_details, e))?; + let chart = ChartInfo::new_from_release_name("metrics-server", "kube-system"); + helm.uninstall(&chart, &vec![]) + .map_err(|e| to_engine_error(&event_details, e))?; // required to avoid namespace stuck on deletion uninstall_cert_manager( @@ -1124,50 +1120,27 @@ impl<'a> DOKS<'a> { let qovery_namespaces = get_qovery_managed_namespaces(); for qovery_namespace in qovery_namespaces.iter() { - let charts_to_delete = cmd::helm::helm_list( - &kubernetes_config_file_path, - self.cloud_provider().credentials_environment_variables(), - Some(qovery_namespace), - ); - match charts_to_delete { - Ok(charts) => { - for chart in charts { - match cmd::helm::helm_exec_uninstall( - &kubernetes_config_file_path, - &chart.namespace, - &chart.name, - self.cloud_provider().credentials_environment_variables(), - ) { - Ok(_) => self.logger().log( - LogLevel::Info, - EngineEvent::Deleting( - event_details.clone(), - EventMessage::new_from_safe(format!("Chart `{}` deleted", chart.name)), - ), - ), - Err(e) => { - let message_safe = format!("Can't delete chart `{}`", chart.name); - self.logger().log( - LogLevel::Error, - EngineEvent::Deleting( - event_details.clone(), - EventMessage::new(message_safe, Some(e.message())), - ), - ) - } - } - } - } - Err(e) => { - if !(e.message().contains("not found")) { + let charts_to_delete = helm + .list_release(Some(qovery_namespace), &vec![]) + .map_err(|e| to_engine_error(&event_details, e))?; + + for chart in charts_to_delete { + let chart_info = ChartInfo::new_from_release_name(&chart.name, &chart.namespace); + match helm.uninstall(&chart_info, &vec![]) { + Ok(_) => self.logger().log( + LogLevel::Info, + EngineEvent::Deleting( + event_details.clone(), + EventMessage::new_from_safe(format!("Chart `{}` deleted", chart.name)), + ), + ), + Err(e) => { + let message_safe = format!("Can't delete chart `{}`", chart.name); self.logger().log( LogLevel::Error, EngineEvent::Deleting( event_details.clone(), - EventMessage::new_from_safe(format!( - "Can't delete the namespace {}", - qovery_namespace - )), + EventMessage::new(message_safe, Some(e.to_string())), ), ) } @@ -1222,18 +1195,11 @@ impl<'a> DOKS<'a> { ), ); - match cmd::helm::helm_list( - &kubernetes_config_file_path, - self.cloud_provider().credentials_environment_variables(), - None, - ) { + match helm.list_release(None, &vec![]) { Ok(helm_charts) => { for chart in helm_charts { - match cmd::helm::helm_uninstall_list( - &kubernetes_config_file_path, - vec![chart.clone()], - self.cloud_provider().credentials_environment_variables(), - ) { + let chart_info = ChartInfo::new_from_release_name(&chart.name, &chart.namespace); + match helm.uninstall(&chart_info, &vec![]) { Ok(_) => self.logger().log( LogLevel::Info, EngineEvent::Deleting( @@ -1242,12 +1208,12 @@ impl<'a> DOKS<'a> { ), ), Err(e) => { - let message_safe = format!("Error deleting chart `{}` deleted", chart.name); + let message_safe = format!("Error deleting chart `{}`: {}", chart.name, e); self.logger().log( LogLevel::Error, EngineEvent::Deleting( event_details.clone(), - EventMessage::new(message_safe, e.message), + EventMessage::new(message_safe, Some(e.to_string())), ), ) } @@ -1260,7 +1226,7 @@ impl<'a> DOKS<'a> { LogLevel::Error, EngineEvent::Deleting( event_details.clone(), - EventMessage::new(message_safe.to_string(), Some(e.message())), + EventMessage::new(message_safe.to_string(), Some(e.to_string())), ), ) } diff --git a/src/cloud_provider/digitalocean/router.rs b/src/cloud_provider/digitalocean/router.rs index b695c9e3..44a8aafd 100644 --- a/src/cloud_provider/digitalocean/router.rs +++ b/src/cloud_provider/digitalocean/router.rs @@ -1,5 +1,6 @@ use tera::Context as TeraContext; +use crate::cloud_provider::helm::ChartInfo; use crate::cloud_provider::models::{CustomDomain, CustomDomainDataTemplate, Route, RouteDataTemplate}; use crate::cloud_provider::service::{ default_tera_context, delete_router, deploy_stateless_service_error, send_progress_on_long_task, Action, Create, @@ -7,6 +8,7 @@ use crate::cloud_provider::service::{ }; use crate::cloud_provider::utilities::{check_cname_for, print_action, sanitize_name}; use crate::cloud_provider::DeploymentTarget; +use crate::cmd::helm; use crate::cmd::helm::Timeout; use crate::error::{EngineError, EngineErrorCause, EngineErrorScope}; use crate::errors::EngineError as NewEngineError; @@ -345,25 +347,26 @@ impl Create for Router { } // do exec helm upgrade and return the last deployment status - let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history( - kubernetes_config_file_path.as_str(), - environment.namespace(), - helm_release_name.as_str(), - self.selector(), - workspace_dir.as_str(), - self.start_timeout(), - kubernetes.cloud_provider().credentials_environment_variables(), - self.service_type(), + let helm = helm::Helm::new( + &kubernetes_config_file_path, + &kubernetes.cloud_provider().credentials_environment_variables(), ) - .map_err(|e| { - NewEngineError::new_helm_charts_upgrade_error(event_details.clone(), e).to_legacy_engine_error() - })?; + .map_err(|e| helm::to_engine_error(&event_details, e).to_legacy_engine_error())?; + let chart = ChartInfo::new_from_custom_namespace( + helm_release_name, + workspace_dir.clone(), + environment.namespace().to_string(), + self.start_timeout().value() as i64, + match self.service_type() { + ServiceType::Database(_) => vec![format!("{}/q-values.yaml", &workspace_dir)], + _ => vec![], + }, + false, + self.selector(), + ); - if helm_history_row.is_none() || !helm_history_row.unwrap().is_successfully_deployed() { - return Err(self.engine_error(EngineErrorCause::Internal, "Router has failed to be deployed".into())); - } - - Ok(()) + helm.upgrade(&chart, &vec![]) + .map_err(|e| helm::to_engine_error(&event_details, e).to_legacy_engine_error()) } fn on_create_check(&self) -> Result<(), EngineError> { diff --git a/src/cloud_provider/helm.rs b/src/cloud_provider/helm.rs index 12877216..088431b5 100644 --- a/src/cloud_provider/helm.rs +++ b/src/cloud_provider/helm.rs @@ -1,10 +1,7 @@ use crate::cloud_provider::helm::HelmAction::Deploy; use crate::cloud_provider::helm::HelmChartNamespaces::KubeSystem; use crate::cloud_provider::qovery::{get_qovery_app_version, EngineLocation, QoveryAppName, QoveryShellAgent}; -use crate::cmd::helm::{ - helm_destroy_chart_if_breaking_changes_version_detected, helm_exec_uninstall_with_chart_info, - helm_exec_upgrade_with_chart_info, helm_upgrade_diff_with_chart_info, is_chart_deployed, -}; +use crate::cmd::helm::{to_command_error, Helm}; use crate::cmd::kubectl::{ kubectl_delete_crash_looping_pods, kubectl_exec_delete_crd, kubectl_exec_get_configmap, kubectl_exec_get_events, kubectl_exec_rollout_restart_deployment, kubectl_exec_with_output, @@ -20,7 +17,7 @@ use thread::spawn; use tracing::{span, Level}; use uuid::Uuid; -#[derive(Clone)] +#[derive(Clone, PartialEq, Eq)] pub enum HelmAction { Deploy, Destroy, @@ -108,6 +105,17 @@ impl ChartInfo { } } + pub fn new_from_release_name(name: &str, custom_namespace: &str) -> ChartInfo { + ChartInfo { + name: name.to_string(), + namespace: HelmChartNamespaces::Custom, + custom_namespace: Some(custom_namespace.to_string()), + timeout_in_seconds: 600, + atomic: true, + ..Default::default() + } + } + pub fn get_namespace_string(&self) -> String { match self.namespace { HelmChartNamespaces::Custom => self @@ -216,36 +224,22 @@ pub trait HelmChart: Send { ) -> Result, CommandError> { let environment_variables: Vec<(&str, &str)> = envs.iter().map(|x| (x.0.as_str(), x.1.as_str())).collect(); let chart_info = self.get_chart_info(); + let helm = Helm::new(kubernetes_config, &environment_variables).map_err(to_command_error)?; + match chart_info.action { HelmAction::Deploy => { - if let Err(e) = helm_destroy_chart_if_breaking_changes_version_detected( - kubernetes_config, - &environment_variables, - chart_info, - ) { + if let Err(e) = helm.uninstall_chart_if_breaking_version(chart_info, &vec![]) { warn!( "error while trying to destroy chart if breaking change is detected: {:?}", - e.message() + e.to_string() ); } - helm_exec_upgrade_with_chart_info(kubernetes_config, &environment_variables, chart_info)? + helm.upgrade(&chart_info, &vec![]).map_err(to_command_error)?; } HelmAction::Destroy => { let chart_info = self.get_chart_info(); - match is_chart_deployed( - kubernetes_config, - environment_variables.clone(), - Some(chart_info.get_namespace_string().as_str()), - chart_info.name.clone(), - ) { - Ok(deployed) => { - if deployed { - helm_exec_uninstall_with_chart_info(kubernetes_config, &environment_variables, chart_info)? - } - } - Err(e) => return Err(e), - }; + helm.uninstall(&chart_info, &vec![]).map_err(to_command_error)?; } HelmAction::Skip => {} } @@ -303,24 +297,31 @@ fn deploy_parallel_charts( handles.push(handle); } + let mut errors: Vec> = vec![]; for handle in handles { match handle.join() { Ok(helm_run_ret) => { if let Err(e) = helm_run_ret { - return Err(e); + errors.push(Err(e)); } } Err(e) => { let safe_message = "Thread panicked during parallel charts deployments."; - return Err(CommandError::new( + let error = Err(CommandError::new( format!("{}, error: {:?}", safe_message.to_string(), e), Some(safe_message.to_string()), )); + errors.push(error); } } } - Ok(()) + if errors.is_empty() { + Ok(()) + } else { + error!("Deployments of charts failed with: {:?}", errors); + errors.remove(0) + } } pub fn deploy_charts_levels( @@ -330,24 +331,24 @@ pub fn deploy_charts_levels( dry_run: bool, ) -> Result<(), CommandError> { // first show diff - for level in &charts { - for chart in level { + let envs_ref: Vec<(&str, &str)> = envs.iter().map(|(x, y)| (x.as_str(), y.as_str())).collect(); + let helm = Helm::new(&kubernetes_config, &envs_ref).map_err(to_command_error)?; + + for level in charts { + // Show diff for all chart in this state + for chart in &level { let chart_info = chart.get_chart_info(); - match chart_info.action { - // don't do diff on destroy or skip - HelmAction::Deploy => { - let _ = helm_upgrade_diff_with_chart_info(&kubernetes_config, envs, chart.get_chart_info()); - } - _ => {} + // don't do diff on destroy or skip + if chart_info.action == HelmAction::Deploy { + let _ = helm.upgrade_diff(chart_info, &vec![]); } } - } - // then apply - if dry_run { - return Ok(()); - } - for level in charts.into_iter() { + // Skip actual deployment if dry run + if dry_run { + continue; + } + if let Err(e) = deploy_parallel_charts(&kubernetes_config, &envs, level) { return Err(e); } @@ -591,47 +592,36 @@ impl HelmChart for PrometheusOperatorConfigChart { ) -> Result, CommandError> { let environment_variables: Vec<(&str, &str)> = envs.iter().map(|x| (x.0.as_str(), x.1.as_str())).collect(); let chart_info = self.get_chart_info(); + let helm = Helm::new(kubernetes_config, &environment_variables).map_err(to_command_error)?; + match chart_info.action { HelmAction::Deploy => { - if let Err(e) = helm_destroy_chart_if_breaking_changes_version_detected( - kubernetes_config, - &environment_variables, - chart_info, - ) { + if let Err(e) = helm.uninstall_chart_if_breaking_version(chart_info, &vec![]) { warn!( "error while trying to destroy chart if breaking change is detected: {}", - e.message() + e.to_string() ); } - helm_exec_upgrade_with_chart_info(kubernetes_config, &environment_variables, chart_info)? + helm.upgrade(&chart_info, &vec![]).map_err(to_command_error)?; } HelmAction::Destroy => { let chart_info = self.get_chart_info(); - match is_chart_deployed( - kubernetes_config, - environment_variables.clone(), - Some(chart_info.get_namespace_string().as_str()), - chart_info.name.clone(), - ) { - Ok(deployed) => { - if deployed { - let prometheus_crds = [ - "prometheuses.monitoring.coreos.com", - "prometheusrules.monitoring.coreos.com", - "servicemonitors.monitoring.coreos.com", - "podmonitors.monitoring.coreos.com", - "alertmanagers.monitoring.coreos.com", - "thanosrulers.monitoring.coreos.com", - ]; - helm_exec_uninstall_with_chart_info(kubernetes_config, &environment_variables, chart_info)?; - for crd in &prometheus_crds { - kubectl_exec_delete_crd(kubernetes_config, crd, environment_variables.clone())?; - } - } + if helm.check_release_exist(&chart_info, &vec![]).is_ok() { + helm.uninstall(&chart_info, &vec![]).map_err(to_command_error)?; + + let prometheus_crds = [ + "prometheuses.monitoring.coreos.com", + "prometheusrules.monitoring.coreos.com", + "servicemonitors.monitoring.coreos.com", + "podmonitors.monitoring.coreos.com", + "alertmanagers.monitoring.coreos.com", + "thanosrulers.monitoring.coreos.com", + ]; + for crd in &prometheus_crds { + let _ = kubectl_exec_delete_crd(kubernetes_config, crd, environment_variables.clone()); } - Err(e) => return Err(e), - }; + } } HelmAction::Skip => {} } diff --git a/src/cloud_provider/scaleway/kubernetes/helm_charts.rs b/src/cloud_provider/scaleway/kubernetes/helm_charts.rs index e2b02d44..1479fc37 100644 --- a/src/cloud_provider/scaleway/kubernetes/helm_charts.rs +++ b/src/cloud_provider/scaleway/kubernetes/helm_charts.rs @@ -493,7 +493,9 @@ datasources: }, ChartSetValue { key: "prometheus.servicemonitor.enabled".to_string(), - value: chart_config_prerequisites.ff_metrics_history_enabled.to_string(), + // Due to cycle, prometheus need tls certificate from cert manager, and enabling this will require + // prometheus to be already installed + value: "false".to_string(), }, ChartSetValue { key: "prometheus.servicemonitor.prometheusInstance".to_string(), @@ -519,11 +521,11 @@ datasources: // Webhooks resources limits ChartSetValue { key: "webhook.resources.limits.cpu".to_string(), - value: "20m".to_string(), + value: "200m".to_string(), }, ChartSetValue { key: "webhook.resources.requests.cpu".to_string(), - value: "20m".to_string(), + value: "50m".to_string(), }, ChartSetValue { key: "webhook.resources.limits.memory".to_string(), @@ -862,15 +864,17 @@ datasources: Box::new(old_prometheus_operator), ]; - let mut level_2: Vec> = vec![]; + let level_2: Vec> = vec![Box::new(cert_manager)]; let mut level_3: Vec> = vec![]; - let mut level_4: Vec> = vec![Box::new(external_dns)]; + let mut level_4: Vec> = vec![]; - let mut level_5: Vec> = vec![Box::new(nginx_ingress), Box::new(cert_manager)]; + let mut level_5: Vec> = vec![Box::new(external_dns)]; - let mut level_6: Vec> = vec![ + let mut level_6: Vec> = vec![Box::new(nginx_ingress)]; + + let mut level_7: Vec> = vec![ Box::new(cert_manager_config), Box::new(qovery_agent), Box::new(shell_agent), @@ -879,24 +883,24 @@ datasources: // // observability if chart_config_prerequisites.ff_metrics_history_enabled { - level_2.push(Box::new(kube_prometheus_stack)); - level_4.push(Box::new(prometheus_adapter)); - level_4.push(Box::new(kube_state_metrics)); + level_3.push(Box::new(kube_prometheus_stack)); + level_5.push(Box::new(prometheus_adapter)); + level_5.push(Box::new(kube_state_metrics)); } if chart_config_prerequisites.ff_log_history_enabled { - level_3.push(Box::new(promtail)); - level_4.push(Box::new(loki)); + level_4.push(Box::new(promtail)); + level_5.push(Box::new(loki)); } if chart_config_prerequisites.ff_metrics_history_enabled || chart_config_prerequisites.ff_log_history_enabled { - level_6.push(Box::new(grafana)) + level_7.push(Box::new(grafana)) }; // pleco if !chart_config_prerequisites.disable_pleco { - level_5.push(Box::new(pleco)); + level_6.push(Box::new(pleco)); } info!("charts configuration preparation finished"); - Ok(vec![level_1, level_2, level_3, level_4, level_5, level_6]) + Ok(vec![level_1, level_2, level_3, level_4, level_5, level_6, level_7]) } diff --git a/src/cloud_provider/scaleway/kubernetes/mod.rs b/src/cloud_provider/scaleway/kubernetes/mod.rs index 52239ce6..ea977e8c 100644 --- a/src/cloud_provider/scaleway/kubernetes/mod.rs +++ b/src/cloud_provider/scaleway/kubernetes/mod.rs @@ -3,7 +3,7 @@ pub mod node; use crate::cloud_provider::aws::regions::AwsZones; use crate::cloud_provider::environment::Environment; -use crate::cloud_provider::helm::deploy_charts_levels; +use crate::cloud_provider::helm::{deploy_charts_levels, ChartInfo}; use crate::cloud_provider::kubernetes::{ is_kubernetes_upgrade_required, send_progress_on_long_task, uninstall_cert_manager, Kind, Kubernetes, KubernetesUpgradeStatus, ProviderOptions, @@ -15,8 +15,8 @@ use crate::cloud_provider::scaleway::kubernetes::helm_charts::{scw_helm_charts, use crate::cloud_provider::scaleway::kubernetes::node::{ScwInstancesType, ScwNodeGroup}; use crate::cloud_provider::utilities::print_action; use crate::cloud_provider::{kubernetes, CloudProvider}; +use crate::cmd::helm::{to_engine_error, Helm}; use crate::cmd::kubectl::{kubectl_exec_api_custom_metrics, kubectl_exec_get_all_namespaces, kubectl_exec_get_events}; -use crate::cmd::structs::HelmChart; use crate::cmd::terraform::{terraform_exec, terraform_init_validate_plan_apply, terraform_init_validate_state_list}; use crate::deletion_utilities::{get_firsts_namespaces_to_delete, get_qovery_managed_namespaces}; use crate::dns_provider::DnsProvider; @@ -1497,15 +1497,14 @@ impl<'a> Kapsule<'a> { ); // delete custom metrics api to avoid stale namespaces on deletion - let _ = cmd::helm::helm_uninstall_list( + let helm = Helm::new( &kubernetes_config_file_path, - vec![HelmChart { - name: "metrics-server".to_string(), - namespace: "kube-system".to_string(), - version: None, - }], - self.cloud_provider().credentials_environment_variables(), - ); + &self.cloud_provider.credentials_environment_variables(), + ) + .map_err(|e| to_engine_error(&event_details, e))?; + let chart = ChartInfo::new_from_release_name("metrics-server", "kube-system"); + helm.uninstall(&chart, &vec![]) + .map_err(|e| to_engine_error(&event_details, e))?; // required to avoid namespace stuck on deletion uninstall_cert_manager( @@ -1525,50 +1524,27 @@ impl<'a> Kapsule<'a> { let qovery_namespaces = get_qovery_managed_namespaces(); for qovery_namespace in qovery_namespaces.iter() { - let charts_to_delete = cmd::helm::helm_list( - &kubernetes_config_file_path, - self.cloud_provider().credentials_environment_variables(), - Some(qovery_namespace), - ); - match charts_to_delete { - Ok(charts) => { - for chart in charts { - match cmd::helm::helm_exec_uninstall( - &kubernetes_config_file_path, - &chart.namespace, - &chart.name, - self.cloud_provider().credentials_environment_variables(), - ) { - Ok(_) => self.logger().log( - LogLevel::Info, - EngineEvent::Deleting( - event_details.clone(), - EventMessage::new_from_safe(format!("Chart `{}` deleted", chart.name)), - ), - ), - Err(e) => { - let message_safe = format!("Can't delete chart `{}`", chart.name); - self.logger().log( - LogLevel::Error, - EngineEvent::Deleting( - event_details.clone(), - EventMessage::new(message_safe, Some(e.message())), - ), - ) - } - } - } - } - Err(e) => { - if !(e.message().contains("not found")) { + let charts_to_delete = helm + .list_release(Some(qovery_namespace), &vec![]) + .map_err(|e| to_engine_error(&event_details, e))?; + + for chart in charts_to_delete { + let chart_info = ChartInfo::new_from_release_name(&chart.name, &chart.namespace); + match helm.uninstall(&chart_info, &vec![]) { + Ok(_) => self.logger().log( + LogLevel::Info, + EngineEvent::Deleting( + event_details.clone(), + EventMessage::new_from_safe(format!("Chart `{}` deleted", chart.name)), + ), + ), + Err(e) => { + let message_safe = format!("Can't delete chart `{}`", chart.name); self.logger().log( LogLevel::Error, EngineEvent::Deleting( event_details.clone(), - EventMessage::new_from_safe(format!( - "Can't delete the namespace {}", - qovery_namespace - )), + EventMessage::new(message_safe, Some(e.to_string())), ), ) } @@ -1623,18 +1599,11 @@ impl<'a> Kapsule<'a> { ), ); - match cmd::helm::helm_list( - &kubernetes_config_file_path, - self.cloud_provider().credentials_environment_variables(), - None, - ) { + match helm.list_release(None, &vec![]) { Ok(helm_charts) => { for chart in helm_charts { - match cmd::helm::helm_uninstall_list( - &kubernetes_config_file_path, - vec![chart.clone()], - self.cloud_provider().credentials_environment_variables(), - ) { + let chart_info = ChartInfo::new_from_release_name(&chart.name, &chart.namespace); + match helm.uninstall(&chart_info, &vec![]) { Ok(_) => self.logger().log( LogLevel::Info, EngineEvent::Deleting( @@ -1643,12 +1612,12 @@ impl<'a> Kapsule<'a> { ), ), Err(e) => { - let message_safe = format!("Error deleting chart `{}` deleted", chart.name); + let message_safe = format!("Error deleting chart `{}`: {}", chart.name, e); self.logger().log( LogLevel::Error, EngineEvent::Deleting( event_details.clone(), - EventMessage::new(message_safe, e.message), + EventMessage::new(message_safe, Some(e.to_string())), ), ) } @@ -1661,7 +1630,7 @@ impl<'a> Kapsule<'a> { LogLevel::Error, EngineEvent::Deleting( event_details.clone(), - EventMessage::new(message_safe.to_string(), Some(e.message())), + EventMessage::new(message_safe.to_string(), Some(e.to_string())), ), ) } diff --git a/src/cloud_provider/scaleway/router.rs b/src/cloud_provider/scaleway/router.rs index 3769b7c1..dfe3f5f7 100644 --- a/src/cloud_provider/scaleway/router.rs +++ b/src/cloud_provider/scaleway/router.rs @@ -1,5 +1,6 @@ use tera::Context as TeraContext; +use crate::cloud_provider::helm::ChartInfo; use crate::cloud_provider::models::{CustomDomain, CustomDomainDataTemplate, Route, RouteDataTemplate}; use crate::cloud_provider::service::{ default_tera_context, delete_router, deploy_stateless_service_error, send_progress_on_long_task, Action, Create, @@ -7,8 +8,9 @@ use crate::cloud_provider::service::{ }; use crate::cloud_provider::utilities::{check_cname_for, print_action, sanitize_name}; use crate::cloud_provider::DeploymentTarget; +use crate::cmd::helm; use crate::cmd::helm::Timeout; -use crate::error::{EngineError, EngineErrorCause, EngineErrorScope}; +use crate::error::{EngineError, EngineErrorScope}; use crate::errors::EngineError as NewEngineError; use crate::events::{EnvironmentStep, Stage, ToTransmitter, Transmitter}; use crate::models::{Context, Listen, Listener, Listeners}; @@ -293,25 +295,26 @@ impl Create for Router { } // do exec helm upgrade and return the last deployment status - let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history( - kubernetes_config_file_path.as_str(), - environment.namespace(), - helm_release_name.as_str(), - self.selector(), - workspace_dir.as_str(), - self.start_timeout(), - kubernetes.cloud_provider().credentials_environment_variables(), - self.service_type(), + let helm = helm::Helm::new( + &kubernetes_config_file_path, + &kubernetes.cloud_provider().credentials_environment_variables(), ) - .map_err(|e| { - NewEngineError::new_helm_charts_upgrade_error(event_details.clone(), e).to_legacy_engine_error() - })?; + .map_err(|e| helm::to_engine_error(&event_details, e).to_legacy_engine_error())?; + let chart = ChartInfo::new_from_custom_namespace( + helm_release_name, + workspace_dir.clone(), + environment.namespace().to_string(), + self.start_timeout().value() as i64, + match self.service_type() { + ServiceType::Database(_) => vec![format!("{}/q-values.yaml", &workspace_dir)], + _ => vec![], + }, + false, + self.selector(), + ); - if helm_history_row.is_none() || !helm_history_row.unwrap().is_successfully_deployed() { - return Err(self.engine_error(EngineErrorCause::Internal, "Router has failed to be deployed".into())); - } - - Ok(()) + helm.upgrade(&chart, &vec![]) + .map_err(|e| helm::to_engine_error(&event_details, e).to_legacy_engine_error()) } fn on_create_check(&self) -> Result<(), EngineError> { diff --git a/src/cloud_provider/service.rs b/src/cloud_provider/service.rs index 0384eb2f..e0606f44 100644 --- a/src/cloud_provider/service.rs +++ b/src/cloud_provider/service.rs @@ -8,9 +8,12 @@ use tera::Context as TeraContext; use crate::build_platform::Image; use crate::cloud_provider::environment::Environment; +use crate::cloud_provider::helm::ChartInfo; use crate::cloud_provider::kubernetes::Kubernetes; use crate::cloud_provider::utilities::check_domain_for; use crate::cloud_provider::DeploymentTarget; +use crate::cmd; +use crate::cmd::helm; use crate::cmd::helm::Timeout; use crate::cmd::kubectl::ScalingKind::Statefulset; use crate::cmd::kubectl::{kubectl_exec_delete_secret, kubectl_exec_scale_replicas_by_selector, ScalingKind}; @@ -365,30 +368,11 @@ pub fn deploy_user_stateless_service(target: &DeploymentTarget, service: &T) where T: Service + Helm, { - deploy_stateless_service( - target, - service, - service.engine_error( - EngineErrorCause::User( - "Your application has failed to start. \ - Ensure you can run it without issues with `qovery run` and check its logs from the web interface or the CLI with `qovery log`. \ - This issue often occurs due to ports misconfiguration. Make sure you exposed the correct port (using EXPOSE statement in Dockerfile or via Qovery configuration).", - ), - format!( - "{} {} has failed to start ⤬", - service.service_type().name(), - service.name_with_id() - ), - ), - ) + deploy_stateless_service(target, service) } /// deploy a stateless service (app, router, database...) on Kubernetes -pub fn deploy_stateless_service( - target: &DeploymentTarget, - service: &T, - thrown_error: EngineError, -) -> Result<(), EngineError> +pub fn deploy_stateless_service(target: &DeploymentTarget, service: &T) -> Result<(), EngineError> where T: Service + Helm, { @@ -441,26 +425,26 @@ where })?; // do exec helm upgrade and return the last deployment status - let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history( - kubernetes_config_file_path.as_str(), - environment.namespace(), - helm_release_name.as_str(), - service.selector(), - workspace_dir.as_str(), - service.start_timeout(), - kubernetes.cloud_provider().credentials_environment_variables(), - service.service_type(), + let helm = helm::Helm::new( + &kubernetes_config_file_path, + &kubernetes.cloud_provider().credentials_environment_variables(), ) - .map_err(|e| NewEngineError::new_helm_charts_upgrade_error(event_details.clone(), e).to_legacy_engine_error())?; + .map_err(|e| helm::to_engine_error(&event_details, e).to_legacy_engine_error())?; + let chart = ChartInfo::new_from_custom_namespace( + helm_release_name, + workspace_dir.clone(), + environment.namespace().to_string(), + 600_i64, + match service.service_type() { + ServiceType::Database(_) => vec![format!("{}/q-values.yaml", &workspace_dir)], + _ => vec![], + }, + false, + service.selector(), + ); - // check deployment status - if helm_history_row.is_none() - || !helm_history_row - .expect("Error getting helm history row") - .is_successfully_deployed() - { - return Err(thrown_error); - } + helm.upgrade(&chart, &vec![]) + .map_err(|e| helm::to_engine_error(&event_details, e).to_legacy_engine_error())?; crate::cmd::kubectl::kubectl_exec_is_pod_ready_with_retry( kubernetes_config_file_path.as_str(), @@ -482,48 +466,12 @@ where } /// do specific operations on a stateless service deployment error -pub fn deploy_stateless_service_error(target: &DeploymentTarget, service: &T) -> Result<(), EngineError> +pub fn deploy_stateless_service_error(_target: &DeploymentTarget, _service: &T) -> Result<(), EngineError> where T: Service + Helm, { - let kubernetes = target.kubernetes; - let environment = target.environment; - let helm_release_name = service.helm_release_name(); - let event_details = service.get_event_details(Stage::Environment(EnvironmentStep::Deploy)); - let kubernetes_config_file_path = match kubernetes.get_kubeconfig_file_path() { - Ok(path) => path, - Err(e) => return Err(e.to_legacy_engine_error()), - }; - - let history_rows = crate::cmd::helm::helm_exec_history( - kubernetes_config_file_path.as_str(), - environment.namespace(), - helm_release_name.as_str(), - &kubernetes.cloud_provider().credentials_environment_variables(), - ) - .map_err(|e| { - NewEngineError::new_helm_chart_history_error( - event_details.clone(), - helm_release_name.to_string(), - environment.namespace().to_string(), - e, - ) - .to_legacy_engine_error() - })?; - - if history_rows.len() == 1 { - crate::cmd::helm::helm_exec_uninstall( - kubernetes_config_file_path.as_str(), - environment.namespace(), - helm_release_name.as_str(), - kubernetes.cloud_provider().credentials_environment_variables(), - ) - .map_err(|e| { - NewEngineError::new_helm_chart_uninstall_error(event_details.clone(), helm_release_name.to_string(), e) - .to_legacy_engine_error() - })?; - } - + // Nothing to do as we sait --atomic on chart release that we do + // So helm rollback for us if a deployment fails Ok(()) } @@ -789,30 +737,26 @@ where })?; // do exec helm upgrade and return the last deployment status - let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history( - kubernetes_config_file_path.to_string(), - environment.namespace(), - service.helm_release_name().as_str(), - service.selector(), - workspace_dir.to_string(), - service.start_timeout(), - kubernetes.cloud_provider().credentials_environment_variables(), - service.service_type(), + let helm = helm::Helm::new( + &kubernetes_config_file_path, + &kubernetes.cloud_provider().credentials_environment_variables(), ) - .map_err(|e| { - NewEngineError::new_helm_charts_upgrade_error(event_details.clone(), e).to_legacy_engine_error() - })?; + .map_err(|e| helm::to_engine_error(&event_details, e).to_legacy_engine_error())?; + let chart = ChartInfo::new_from_custom_namespace( + service.helm_release_name(), + workspace_dir.clone(), + environment.namespace().to_string(), + 600_i64, + match service.service_type() { + ServiceType::Database(_) => vec![format!("{}/q-values.yaml", &workspace_dir)], + _ => vec![], + }, + false, + service.selector(), + ); - // check deployment status - if helm_history_row.is_none() || !helm_history_row.unwrap().is_successfully_deployed() { - return Err(service.engine_error( - EngineErrorCause::Internal, - format!( - "{} service fails to be deployed (before start)", - service.service_type().name() - ), - )); - } + helm.upgrade(&chart, &vec![]) + .map_err(|e| helm::to_engine_error(&event_details, e).to_legacy_engine_error())?; // check app status match crate::cmd::kubectl::kubectl_exec_is_pod_ready_with_retry( @@ -1306,34 +1250,15 @@ pub fn helm_uninstall_release( .get_kubeconfig_file_path() .map_err(|e| e.to_legacy_engine_error())?; - let history_rows = crate::cmd::helm::helm_exec_history( - kubernetes_config_file_path.as_str(), - environment.namespace(), - helm_release_name, + let helm = cmd::helm::Helm::new( + &kubernetes_config_file_path, &kubernetes.cloud_provider().credentials_environment_variables(), ) - .map_err(|e| { - NewEngineError::new_k8s_history(event_details.clone(), environment.namespace().to_string(), e) - .to_legacy_engine_error() - })?; + .map_err(|e| NewEngineError::new_helm_error(event_details.clone(), e).to_legacy_engine_error())?; - // if there is no valid history - then delete the helm chart - let first_valid_history_row = history_rows.iter().find(|x| x.is_successfully_deployed()); - - if first_valid_history_row.is_some() { - crate::cmd::helm::helm_exec_uninstall( - kubernetes_config_file_path.as_str(), - environment.namespace(), - helm_release_name, - kubernetes.cloud_provider().credentials_environment_variables(), - ) - .map_err(|e| { - NewEngineError::new_helm_chart_uninstall_error(event_details.clone(), helm_release_name.to_string(), e) - .to_legacy_engine_error() - })?; - } - - Ok(()) + let chart = ChartInfo::new_from_release_name(helm_release_name, environment.namespace()); + helm.uninstall(&chart, &vec![]) + .map_err(|e| NewEngineError::new_helm_error(event_details.clone(), e).to_legacy_engine_error()) } /// This function call (start|pause|delete)_in_progress function every 10 seconds when a diff --git a/src/cloud_provider/utilities.rs b/src/cloud_provider/utilities.rs index 1407b3c3..e01f050f 100644 --- a/src/cloud_provider/utilities.rs +++ b/src/cloud_provider/utilities.rs @@ -325,8 +325,16 @@ fn cloudflare_dns_resolver() -> Resolver { // We want to avoid cache and using host file of the host, as some provider force caching // which lead to stale response resolver_options.cache_size = 0; - resolver_options.use_hosts_file = false; + resolver_options.use_hosts_file = true; + //resolver_options.ip_strategy = LookupIpStrategy::Ipv4Only; + //let dns = IpAddr::V4(Ipv4Addr::new(192, 168, 1, 254)); + //let resolver = ResolverConfig::from_parts( + // None, + // vec![], + // NameServerConfigGroup::from_ips_clear(&vec![dns], 53, true), + //); + //Resolver::new(resolver, resolver_options).unwrap() Resolver::new(ResolverConfig::cloudflare(), resolver_options) .expect("Invalid cloudflare DNS resolver configuration") } diff --git a/src/cmd/helm.rs b/src/cmd/helm.rs index 7a59f1ce..b7a2780a 100644 --- a/src/cmd/helm.rs +++ b/src/cmd/helm.rs @@ -1,27 +1,23 @@ use std::io::{Error, Write}; -use std::path::Path; +use std::path::{Path, PathBuf}; -use tracing::{error, info, span, Level}; +use tracing::{error, info}; -use crate::cloud_provider::helm::{deploy_charts_levels, ChartInfo, CommonChart}; -use crate::cloud_provider::service::ServiceType; -use crate::cmd::helm::HelmLockErrors::{IncorrectFormatDate, NotYetExpired, ParsingError}; -use crate::cmd::kubectl::{kubectl_exec_delete_secret, kubectl_exec_get_secrets}; -use crate::cmd::structs::{HelmChart, HelmHistoryRow, HelmListItem, Secrets}; +use crate::cloud_provider::helm::ChartInfo; +use crate::cmd::helm::HelmCommand::{LIST, ROLLBACK, STATUS, UNINSTALL, UPGRADE}; +use crate::cmd::helm::HelmError::{CannotRollback, CmdError, InvalidKubeConfig, ReleaseDoesNotExist}; +use crate::cmd::structs::{HelmChart, HelmListItem}; use crate::cmd::utilities::QoveryCommand; -use crate::error::{SimpleError, SimpleErrorKind}; -use crate::errors::CommandError; -use chrono::{DateTime, Duration, Utc}; -use core::time; -use retry::delay::Fixed; -use retry::Error::Operation; -use retry::OperationResult; +use crate::errors::{CommandError, EngineError}; +use crate::events::EventDetails; +use chrono::Duration; use semver::Version; +use serde_derive::Deserialize; use std::fs::File; use std::str::FromStr; -use std::thread; const HELM_DEFAULT_TIMEOUT_IN_SECONDS: u32 = 300; +const HELM_MAX_HISTORY: &str = "50"; pub enum Timeout { Default, @@ -29,7 +25,7 @@ pub enum Timeout { } impl Timeout { - fn value(&self) -> u32 { + pub fn value(&self) -> u32 { match *self { Timeout::Default => HELM_DEFAULT_TIMEOUT_IN_SECONDS, Timeout::Value(t) => t, @@ -37,812 +33,505 @@ impl Timeout { } } -pub fn helm_exec_with_upgrade_history

( - kubernetes_config: P, - namespace: &str, - release_name: &str, - selector: Option, - chart_root_dir: P, - timeout: Timeout, - envs: Vec<(&str, &str)>, - service_type: ServiceType, -) -> Result, CommandError> -where - P: AsRef, -{ - // do exec helm upgrade - info!( - "exec helm upgrade for namespace {} and chart {}", - &namespace, - chart_root_dir.as_ref().to_str().unwrap() - ); +#[derive(thiserror::Error, Debug)] +pub enum HelmError { + #[error("Kubernetes config file path is not valid or does not exist: {0}")] + InvalidKubeConfig(PathBuf), - let path = match chart_root_dir.as_ref().to_str().is_some() { - true => chart_root_dir.as_ref().to_str().unwrap(), - false => "", - } - .to_string(); + #[error("Requested Helm release `{0}` does not exist")] + ReleaseDoesNotExist(String), - let current_chart = CommonChart { - chart_info: ChartInfo::new_from_custom_namespace( - release_name.to_string(), - path.clone(), - namespace.to_string(), - timeout.value() as i64, - match service_type { - ServiceType::Database(_) => vec![format!("{}/q-values.yaml", path)], - _ => vec![], - }, - false, - selector, - ), - }; + #[error("Requested Helm release `{0}` is under an helm lock. Ensure release is de-locked before going further")] + ReleaseLocked(String), - let environment_variables: Vec<(String, String)> = - envs.iter().map(|x| (x.0.to_string(), x.1.to_string())).collect(); + #[error("Helm release `{0}` during helm {1:?} has been rollbacked")] + Rollbacked(String, HelmCommand), - deploy_charts_levels( - kubernetes_config.as_ref(), - &environment_variables, - vec![vec![Box::new(current_chart)]], - false, - )?; + #[error("Helm release `{0}` cannot be rollbacked due to be at revision 1")] + CannotRollback(String), - // list helm history - info!( - "exec helm history for namespace {} and chart {}", - namespace, - chart_root_dir.as_ref().to_str().unwrap() - ); + #[error("Helm timed out for release `{0}` during helm {1:?}: {2}")] + Timeout(String, HelmCommand, String), - let helm_history_rows = helm_exec_history(kubernetes_config.as_ref(), namespace, release_name, &envs)?; - - // take the last deployment from helm history - or return none if there is no history - Ok(helm_history_rows - .first() - .map(|helm_history_row| helm_history_row.clone())) + #[error("Helm command `{1:?}` for release {0} terminated with an error: {2:?}")] + CmdError(String, HelmCommand, CommandError), } -pub fn helm_destroy_chart_if_breaking_changes_version_detected( - kubernetes_config: &Path, - environment_variables: &Vec<(&str, &str)>, - chart_info: &ChartInfo, -) -> Result<(), CommandError> { - // If there is a breaking version set for the current helm chart, - // then we compare this breaking version with the currently installed version if any. - // If current installed version is older than breaking change one, then we delete - // the chart before applying it. - if let Some(breaking_version) = &chart_info.last_breaking_version_requiring_restart { - let chart_namespace = chart_info.get_namespace_string(); - if let Some(installed_version) = helm_get_chart_version( +#[derive(Debug)] +pub struct Helm { + kubernetes_config: PathBuf, + common_envs: Vec<(String, String)>, +} + +#[derive(Debug, Clone, Copy)] +pub enum HelmCommand { + ROLLBACK, + STATUS, + UPGRADE, + UNINSTALL, + LIST, + DIFF, +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ReleaseInfo { + // https://github.com/helm/helm/blob/12f1bc0acdeb675a8c50a78462ed3917fb7b2e37/pkg/release/status.go + status: String, +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ReleaseStatus { + pub version: u64, + pub info: ReleaseInfo, +} + +impl ReleaseStatus { + fn is_locked(&self) -> bool { + self.info.status.starts_with("pending-") + } +} + +impl Helm { + fn get_all_envs<'a>(&'a self, envs: &'a [(&'a str, &'a str)]) -> Vec<(&'a str, &'a str)> { + let mut all_envs: Vec<(&str, &str)> = self.common_envs.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect(); + all_envs.append(&mut envs.to_vec()); + + all_envs + } + + pub fn new>(kubernetes_config: P, common_envs: &[(&str, &str)]) -> Result { + // Check kube config file is valid + let kubernetes_config = kubernetes_config.as_ref().to_path_buf(); + if !kubernetes_config.exists() || !kubernetes_config.is_file() { + return Err(InvalidKubeConfig(kubernetes_config)); + } + + Ok(Helm { kubernetes_config, - environment_variables.to_owned(), - Some(chart_namespace.as_str()), - chart_info.name.clone(), + common_envs: common_envs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + }) + } + + pub fn check_release_exist(&self, chart: &ChartInfo, envs: &[(&str, &str)]) -> Result { + let namespace = chart.get_namespace_string(); + let args = vec![ + "status", + &chart.name, + "--kubeconfig", + self.kubernetes_config.to_str().unwrap_or_default(), + "--namespace", + &namespace, + "-o", + "json", + ]; + + let mut stdout = String::new(); + let mut stderr = String::new(); + match helm_exec_with_output( + &args, + &self.get_all_envs(envs), + |line| stdout.push_str(&line), + |line| stderr.push_str(&line), ) { - if installed_version.le(breaking_version) { - return helm_exec_uninstall( - kubernetes_config, - chart_namespace.as_str(), - chart_info.name.as_str(), - environment_variables.to_owned(), - ); + Err(_) if stderr.contains("release: not found") => Err(ReleaseDoesNotExist(chart.name.clone())), + Err(err) => { + stderr.push_str(&err.message()); + let error = CommandError::new(stderr, err.message_safe()); + Err(CmdError(chart.name.clone(), STATUS, error)) + } + Ok(_) => { + let status: ReleaseStatus = serde_json::from_str(&stdout).unwrap_or_default(); + Ok(status) } } } - Ok(()) -} + pub fn rollback(&self, chart: &ChartInfo, envs: &[(&str, &str)]) -> Result<(), HelmError> { + if self.check_release_exist(chart, envs)?.version <= 1 { + return Err(CannotRollback(chart.name.clone())); + } -pub fn helm_exec_upgrade_with_chart_info

( - kubernetes_config: P, - envs: &Vec<(&str, &str)>, - chart: &ChartInfo, -) -> Result<(), CommandError> -where - P: AsRef, -{ - let debug = false; - let timeout_string = format!("{}s", &chart.timeout_in_seconds); + let timeout = format!("{}s", &chart.timeout_in_seconds); + let namespace = chart.get_namespace_string(); + let args = vec![ + "rollback", + &chart.name, + "--kubeconfig", + self.kubernetes_config.to_str().unwrap_or_default(), + "--namespace", + &namespace, + "--timeout", + &timeout, + "--history-max", + HELM_MAX_HISTORY, + "--cleanup-on-fail", + "--force", + "--wait", + ]; - let mut args_string: Vec = vec![ - "upgrade", - "--kubeconfig", - kubernetes_config.as_ref().to_str().unwrap(), - "--create-namespace", - "--install", - "--timeout", - timeout_string.as_str(), - "--history-max", - "50", - "--namespace", - chart.get_namespace_string().as_str(), - ] - .into_iter() - .map(|x| x.to_string()) - .collect(); - - if debug { - args_string.push("-o".to_string()); - args_string.push("json".to_string()); - } - // warn: don't add debug or json output won't work - if chart.atomic { - args_string.push("--atomic".to_string()) - } - if chart.force_upgrade { - args_string.push("--force".to_string()) - } - if chart.dry_run { - args_string.push("--dry-run".to_string()) - } - if chart.wait { - args_string.push("--wait".to_string()) + let mut stderr = String::new(); + match helm_exec_with_output(&args, &self.get_all_envs(envs), |_| {}, |line| stderr.push_str(&line)) { + Err(err) => { + stderr.push_str(&err.message()); + let error = CommandError::new(stderr, err.message_safe()); + Err(CmdError(chart.name.clone(), ROLLBACK, error)) + } + Ok(_) => Ok(()), + } } - // overrides and files overrides - for value in &chart.values { - args_string.push("--set".to_string()); - args_string.push(format!("{}={}", value.key, value.value)); - } - for value_file in &chart.values_files { - args_string.push("-f".to_string()); - args_string.push(value_file.clone()); - } - for value_file in &chart.yaml_files_content { - let file_path = format!("{}/{}", chart.path, &value_file.filename); - let file_create = || -> Result<(), Error> { - let mut file = File::create(&file_path)?; - file.write_all(value_file.yaml_content.as_bytes())?; - Ok(()) - }; - // no need to validate yaml as it will be done by helm - if let Err(e) = file_create() { - let safe_message = format!("Error while writing yaml content to file `{}`", &file_path); - return Err(CommandError::new( - format!( - "{}\nContent\n{}\nError: {}", - safe_message.to_string(), - value_file.yaml_content, - e - ), - Some(safe_message.to_string()), - )); - }; + pub fn uninstall(&self, chart: &ChartInfo, envs: &[(&str, &str)]) -> Result<(), HelmError> { + // If the release does not exist, we do not return an error + match self.check_release_exist(chart, envs) { + Ok(_) => {} + Err(ReleaseDoesNotExist(_)) => return Ok(()), + Err(err) => return Err(err), + } - args_string.push("-f".to_string()); - args_string.push(file_path.clone()); + let timeout = format!("{}s", &chart.timeout_in_seconds); + let namespace = chart.get_namespace_string(); + let args = vec![ + "uninstall", + &chart.name, + "--kubeconfig", + self.kubernetes_config.to_str().unwrap_or_default(), + "--namespace", + &namespace, + "--timeout", + &timeout, + "--wait", + ]; + + let mut stderr = String::new(); + match helm_exec_with_output(&args, &self.get_all_envs(envs), |_| {}, |line| stderr.push_str(&line)) { + Err(err) => { + stderr.push_str(&err.message()); + let error = CommandError::new(stderr, err.message_safe()); + Err(CmdError(chart.name.clone(), UNINSTALL, error)) + } + Ok(_) => Ok(()), + } } - // add last elements - args_string.push(chart.name.to_string()); - args_string.push(chart.path.to_string()); + fn unlock_release(&self, chart: &ChartInfo, envs: &[(&str, &str)]) -> Result<(), HelmError> { + match self.check_release_exist(chart, envs) { + Ok(release) if release.is_locked() && release.version <= 1 => { + info!("Helm lock detected. Uninstalling it as it is the first version and rollback is not possible"); + self.uninstall(chart, envs)?; + } + Ok(release) if release.is_locked() => { + info!("Helm lock detected. Forcing rollback to previous version"); + self.rollback(chart, envs)?; + } + Ok(release) => { + // Happy path nothing to do + debug!("Helm release status: {:?}", release) + } + Err(_) => {} // Happy path nothing to do + } - let mut json_output_string = String::new(); - let mut error_message = String::new(); + Ok(()) + } - let result = retry::retry(Fixed::from_millis(15000).take(3), || { - let args = args_string.iter().map(|x| x.as_str()).collect(); - let mut helm_error_during_deployment = SimpleError { - kind: SimpleErrorKind::Other, - message: None, - }; - let mut should_clean_helm_lock = false; + /// List deployed helm charts + /// + /// # Arguments + /// + /// * `envs` - environment variables required for kubernetes connection + /// * `namespace` - list charts from a kubernetes namespace or use None to select all namespaces + pub fn list_release(&self, namespace: Option<&str>, envs: &[(&str, &str)]) -> Result, HelmError> { + let mut helm_args = vec![ + "list", + "-a", + "--kubeconfig", + self.kubernetes_config.to_str().unwrap_or_default(), + "-o", + "json", + ]; + match namespace { + Some(ns) => helm_args.append(&mut vec!["-n", ns]), + None => helm_args.push("-A"), + } + let mut output_string: Vec = Vec::with_capacity(20); + if let Err(cmd_error) = helm_exec_with_output( + &helm_args, + &self.get_all_envs(envs), + |line| output_string.push(line), + |line| error!("{}", line), + ) { + return Err(HelmError::CmdError("none".to_string(), LIST, cmd_error)); + } + + let values = serde_json::from_str::>(&output_string.join("")); + let mut helms_charts: Vec = Vec::new(); + + match values { + Ok(all_helms) => { + for helm in all_helms { + let raw_version = helm.chart.replace(format!("{}-", helm.name).as_str(), ""); + let version = Version::from_str(raw_version.as_str()).ok(); + helms_charts.push(HelmChart::new(helm.name, helm.namespace, version)) + } + + Ok(helms_charts) + } + Err(e) => { + let message_safe = "Error while deserializing all helms names"; + Err(HelmError::CmdError( + "none".to_string(), + LIST, + CommandError::new( + format!("{}, error: {}", message_safe, e), + Some(message_safe.to_string()), + ), + )) + } + } + } + + pub fn get_chart_version( + &self, + chart_name: String, + namespace: Option<&str>, + envs: &[(&str, &str)], + ) -> Result, HelmError> { + let deployed_charts = self.list_release(namespace, envs)?; + for chart in deployed_charts { + if chart.name == chart_name { + return Ok(chart.version); + } + } + + // found nothing ;'( + Ok(None) + } + + pub fn upgrade_diff(&self, chart: &ChartInfo, envs: &[(&str, &str)]) -> Result<(), HelmError> { + let mut args_string: Vec = vec![ + "diff".to_string(), + "upgrade".to_string(), + "--kubeconfig".to_string(), + self.kubernetes_config.to_str().unwrap_or_default().to_string(), + "--install".to_string(), + "--namespace".to_string(), + chart.get_namespace_string(), + ]; + + for value in &chart.values { + args_string.push("--set".to_string()); + args_string.push(format!("{}={}", value.key, value.value)); + } + + for value_file in &chart.values_files { + args_string.push("-f".to_string()); + args_string.push(value_file.clone()); + } + + for value_file in &chart.yaml_files_content { + let file_path = format!("{}/{}", chart.path, &value_file.filename); + let file_create = || -> Result<(), Error> { + let mut file = File::create(&file_path)?; + file.write_all(value_file.yaml_content.as_bytes())?; + Ok(()) + }; + + // no need to validate yaml as it will be done by helm + if let Err(e) = file_create() { + let safe_message = format!("Error while writing yaml content to file `{}`", &file_path); + let cmd_err = CommandError::new( + format!("{}\nContent\n{}\nError: {}", safe_message, value_file.yaml_content, e), + Some(safe_message), + ); + return Err(HelmError::CmdError(chart.name.clone(), HelmCommand::UPGRADE, cmd_err)); + }; + + args_string.push("-f".to_string()); + args_string.push(file_path); + } + + // add last elements + args_string.push(chart.name.clone()); + args_string.push(chart.path.clone()); + + let mut stderr_msg = String::new(); let helm_ret = helm_exec_with_output( - args, - envs.clone(), + &args_string.iter().map(|x| x.as_str()).collect::>(), + &self.get_all_envs(envs), |line| { info!("{}", line); - json_output_string = line }, |line| { - if line.contains("another operation (install/upgrade/rollback) is in progress") { - error_message = format!("helm lock detected for {}, looking for cleaning lock", chart.name); - helm_error_during_deployment.message = Some(error_message.clone()); - warn!("{}. {}", &error_message, &line); - should_clean_helm_lock = true; - return; - } - - if !chart.parse_stderr_for_error { - warn!("chart {}: {}", chart.name, line); - return; - } - - // helm errors are not json formatted unfortunately - if line.contains("has been rolled back") { - error_message = format!("deployment {} has been rolled back", chart.name); - helm_error_during_deployment.message = Some(error_message.clone()); - warn!("{}. {}", &error_message, &line); - } else if line.contains("has been uninstalled") { - error_message = format!("deployment {} has been uninstalled due to failure", chart.name); - helm_error_during_deployment.message = Some(error_message.clone()); - warn!("{}. {}", &error_message, &line); - // special fix for prometheus operator - } else if line.contains("info: skipping unknown hook: \"crd-install\"") { - debug!("chart {}: {}", chart.name, line); - } else { - error_message = format!("deployment {} has failed", chart.name); - helm_error_during_deployment.message = Some(error_message.clone()); - error!("{}. {}", &error_message, &line); - } + stderr_msg.push_str(&line); + warn!("chart {}: {}", chart.name, line); }, ); - if should_clean_helm_lock { - match clean_helm_lock( - &kubernetes_config, - chart.get_namespace_string().as_str(), - &chart.name, - chart.timeout_in_seconds, - envs.clone(), - ) { - Ok(_) => info!("Helm lock detected and cleaned"), - Err(e) => warn!("Couldn't cleanup Helm lock. {:?}", e.message()), + match helm_ret { + // Ok is ok + Ok(_) => Ok(()), + Err(err) => { + error!("Helm error: {:?}", err); + Err(CmdError( + chart.name.clone(), + HelmCommand::DIFF, + CommandError::new(stderr_msg.clone(), Some(stderr_msg)), + )) } } + } + + pub fn upgrade(&self, chart: &ChartInfo, envs: &[(&str, &str)]) -> Result<(), HelmError> { + // Due to crash or error it is possible that the release is under an helm lock + // Try to un-stuck the situation first if needed + // We don't care if the rollback failed, as it is a best effort to remove the lock + // and to re-launch an upgrade just after + let unlock_ret = self.unlock_release(chart, envs); + info!("Helm lock status: {:?}", unlock_ret); + + let debug = false; + let timeout_string = format!("{}s", &chart.timeout_in_seconds); + + let mut args_string: Vec = vec![ + "upgrade".to_string(), + "--kubeconfig".to_string(), + self.kubernetes_config.to_str().unwrap_or_default().to_string(), + "--create-namespace".to_string(), + "--install".to_string(), + "--timeout".to_string(), + timeout_string.as_str().to_string(), + "--history-max".to_string(), + HELM_MAX_HISTORY.to_string(), + "--namespace".to_string(), + chart.get_namespace_string(), + ]; + + if debug { + args_string.push("-o".to_string()); + args_string.push("json".to_string()); + } + + // warn: don't add debug or json output won't work + if chart.atomic { + args_string.push("--atomic".to_string()) + } + if chart.force_upgrade { + args_string.push("--force".to_string()) + } + if chart.dry_run { + args_string.push("--dry-run".to_string()) + } + if chart.wait { + args_string.push("--wait".to_string()) + } + + // overrides and files overrides + for value in &chart.values { + args_string.push("--set".to_string()); + args_string.push(format!("{}={}", value.key, value.value)); + } + + for value_file in &chart.values_files { + args_string.push("-f".to_string()); + args_string.push(value_file.clone()); + } + for value_file in &chart.yaml_files_content { + let file_path = format!("{}/{}", chart.path, &value_file.filename); + let file_create = || -> Result<(), Error> { + let mut file = File::create(&file_path)?; + file.write_all(value_file.yaml_content.as_bytes())?; + Ok(()) + }; + + // no need to validate yaml as it will be done by helm + if let Err(e) = file_create() { + let safe_message = format!("Error while writing yaml content to file `{}`", &file_path); + let cmd_err = CommandError::new( + format!("{}\nContent\n{}\nError: {}", safe_message, value_file.yaml_content, e), + Some(safe_message), + ); + return Err(HelmError::CmdError(chart.name.clone(), HelmCommand::UPGRADE, cmd_err)); + }; + + args_string.push("-f".to_string()); + args_string.push(file_path); + } + + // add last elements + args_string.push(chart.name.clone()); + args_string.push(chart.path.clone()); + + let mut error_message: Vec = vec![]; + + let helm_ret = helm_exec_with_output( + &args_string.iter().map(|x| x.as_str()).collect::>(), + &self.get_all_envs(envs), + |line| { + info!("{}", line); + }, + |line| { + warn!("chart {}: {}", chart.name, line); + error_message.push(line); + }, + ); match helm_ret { - Ok(_) => { - if helm_error_during_deployment.message.is_some() { - OperationResult::Retry(helm_error_during_deployment) + // Ok is ok + Ok(_) => Ok(()), + Err(err) => { + error!("Helm error: {:?}", err); + + // Try do define/specify a bit more the message + let stderr_msg: String = error_message.into_iter().collect(); + let stderr_msg = format!("{}: {}", stderr_msg, err.message()); + let error = if stderr_msg.contains("another operation (install/upgrade/rollback) is in progress") { + HelmError::ReleaseLocked(chart.name.clone()) + } else if stderr_msg.contains("has been rolled back") { + HelmError::Rollbacked(chart.name.clone(), UPGRADE) + } else if stderr_msg.contains("timed out waiting") { + HelmError::Timeout(chart.name.clone(), UPGRADE, stderr_msg) } else { - OperationResult::Ok(()) - } - } - Err(e) => OperationResult::Retry(SimpleError::new(SimpleErrorKind::Other, Some(e.message()))), - } - }); - - match result { - Ok(_) => Ok(()), - Err(Operation { error, .. }) => { - return Err(CommandError::new( - error.message.unwrap_or("No error message".to_string()), - None, - )); - } - Err(retry::Error::Internal(e)) => return Err(CommandError::new(e, None)), - } -} - -pub fn clean_helm_lock

( - kubernetes_config: P, - namespace: &str, - release_name: &str, - timeout: i64, - envs: Vec<(&str, &str)>, -) -> Result<(), CommandError> -where - P: AsRef, -{ - let selector = format!("name={}", release_name); - let timeout_i64 = timeout; - - let result = retry::retry(Fixed::from_millis(3000).take(5), || { - // get secrets for this helm deployment - let result = match kubectl_exec_get_secrets(&kubernetes_config, namespace, &selector, envs.clone()) { - Ok(x) => x, - Err(e) => return OperationResult::Retry(e), - }; - - // get helm release name (secret) containing the lock and clean if possible - match helm_get_secret_lock_name(&result, timeout_i64.clone()) { - Ok(x) => return OperationResult::Ok(x), - Err(e) => match e.kind { - ParsingError => OperationResult::Retry(CommandError::new(e.message, None)), - IncorrectFormatDate => OperationResult::Retry(CommandError::new(e.message, None)), - NotYetExpired => { - if e.wait_before_release_lock.is_none() { - return OperationResult::Retry(CommandError::new_from_safe_message( - "Missing helm time to wait information, before releasing the lock".to_string(), - )); - }; - - let time_to_wait = e.wait_before_release_lock.unwrap() as u64; - // wait 2min max to avoid the customer to re-launch a job or exit - if time_to_wait < 120 { - info!("waiting {}s before retrying the deployment...", time_to_wait); - thread::sleep(time::Duration::from_secs(time_to_wait)); - } else { - return OperationResult::Err(CommandError::new(e.message, None)); - } - - // retrieve now the secret - match helm_get_secret_lock_name(&result, timeout_i64.clone()) { - Ok(x) => OperationResult::Ok(x), - Err(e) => OperationResult::Err(CommandError::new(e.message, None)), - } - } - }, - } - }); - - match result { - Err(err) => { - return match err { - retry::Error::Operation { .. } => Err(CommandError::new_from_safe_message(format!( - "internal error while trying to deploy helm chart {}", - release_name - ))), - retry::Error::Internal(err) => Err(CommandError::new_from_safe_message(err)), - } - } - Ok(x) => { - if let Err(e) = kubectl_exec_delete_secret(&kubernetes_config, namespace, x.as_str(), envs.clone()) { - return Err(e); - }; - Ok(()) - } - } -} - -pub enum HelmDeploymentErrors { - SimpleError, - HelmLockError, -} - -#[derive(Debug)] -pub enum HelmLockErrors { - ParsingError, - IncorrectFormatDate, - NotYetExpired, -} - -#[derive(Debug)] -pub struct HelmLockError { - kind: HelmLockErrors, - message: String, - wait_before_release_lock: Option, -} - -/// Get helm secret name containing the lock -pub fn helm_get_secret_lock_name(secrets_items: &Secrets, timeout: i64) -> Result { - match secrets_items.items.last() { - None => Err(HelmLockError { - kind: ParsingError, - message: "couldn't parse the list of secrets, it's certainly empty".to_string(), - wait_before_release_lock: None, - }), - Some(x) => { - let creation_time = match DateTime::parse_from_rfc3339(&x.metadata.creation_timestamp) { - Ok(x) => x, - Err(e) => { - return Err(HelmLockError { - kind: IncorrectFormatDate, - message: format!("incorrect format date input from secrets. {:?}", e), - wait_before_release_lock: None, - }) - } - }; - let now = Utc::now().timestamp(); - let max_timeout = creation_time.timestamp() + timeout; - - // not yet expired - if &now < &max_timeout { - let time_to_wait = &max_timeout - &now; - return Err(HelmLockError { - kind: NotYetExpired, - message: format!( - "helm lock has not yet expired, please wait {}s before retrying", - &time_to_wait - ), - wait_before_release_lock: Some(time_to_wait), - }); - } - - //expired - Ok(x.metadata.name.to_string()) - } - } -} - -pub fn helm_exec_uninstall_with_chart_info

( - kubernetes_config: P, - envs: &Vec<(&str, &str)>, - chart: &ChartInfo, -) -> Result<(), CommandError> -where - P: AsRef, -{ - helm_exec_with_output( - vec![ - "uninstall", - "--kubeconfig", - kubernetes_config.as_ref().to_str().unwrap(), - "--namespace", - chart.get_namespace_string().as_str(), - &chart.name, - ], - envs.clone(), - |line| info!("{}", line.as_str()), - |line| error!("{}", line.as_str()), - ) -} - -pub fn helm_exec_uninstall

( - kubernetes_config: P, - namespace: &str, - release_name: &str, - envs: Vec<(&str, &str)>, -) -> Result<(), CommandError> -where - P: AsRef, -{ - helm_exec_with_output( - vec![ - "uninstall", - "--kubeconfig", - kubernetes_config.as_ref().to_str().unwrap(), - "--namespace", - namespace, - release_name, - ], - envs, - |line| info!("{}", line.as_str()), - |line| error!("{}", line.as_str()), - ) -} - -pub fn helm_exec_history

( - kubernetes_config: P, - namespace: &str, - release_name: &str, - envs: &Vec<(&str, &str)>, -) -> Result, CommandError> -where - P: AsRef, -{ - let mut output_string = String::new(); - let _ = helm_exec_with_output( - // WARN: do not add argument --debug, otherwise JSON decoding will not work - vec![ - "history", - "--kubeconfig", - kubernetes_config.as_ref().to_str().unwrap(), - "--namespace", - namespace, - "-o", - "json", - release_name, - ], - envs.clone(), - |line| output_string = line, - |line| { - if line.contains("Error: release: not found") { - info!("{}", line) - } else { - error!("{}", line) - } - }, - ); - - // TODO better check, release not found - - let mut results = match serde_json::from_str::>(output_string.as_str()) { - Ok(x) => x, - Err(_) => vec![], - }; - - // unsort results by revision number - let _ = results.sort_by_key(|x| x.revision); - // there is no performance penalty to do it in 2 operations instead of one, but who really cares anyway - let _ = results.reverse(); - - Ok(results) -} - -pub fn helm_uninstall_list

( - kubernetes_config: P, - helm_list: Vec, - envs: Vec<(&str, &str)>, -) -> Result -where - P: AsRef, -{ - let mut output_vec: Vec = Vec::new(); - - for chart in helm_list { - match helm_exec_with_output( - vec![ - "uninstall", - "-n", - chart.namespace.as_str(), - chart.name.as_str(), - "--kubeconfig", - kubernetes_config.as_ref().to_str().unwrap(), - ], - envs.clone(), - |line| output_vec.push(line), - |line| error!("{}", line), - ) { - Ok(_) => info!( - "Helm uninstall succeed for {} on namespace {}", - chart.name, chart.namespace - ), - Err(_) => info!( - "Helm history found for release name {} on namespace {}", - chart.name, chart.namespace - ), - }; - } - - Ok(output_vec.join("\n")) -} - -pub fn helm_exec_upgrade_with_override_file

( - kubernetes_config: P, - namespace: &str, - release_name: &str, - chart_root_dir: P, - override_file: &str, - envs: Vec<(&str, &str)>, -) -> Result<(), CommandError> -where - P: AsRef, -{ - helm_exec_with_output( - vec![ - "upgrade", - "--kubeconfig", - kubernetes_config.as_ref().to_str().unwrap(), - "--create-namespace", - "--install", - "--history-max", - "50", - "--wait", - "--namespace", - namespace, - release_name, - chart_root_dir.as_ref().to_str().unwrap(), - "-f", - override_file, - ], - envs, - |line| info!("{}", line.as_str()), - |line| { - // don't crash errors if releases are not found - if line.contains("Error: release: not found") { - info!("{}", line) - } else { - error!("{}", line) - } - }, - ) -} - -pub fn helm_exec_with_upgrade_history_with_override

( - kubernetes_config: P, - namespace: &str, - release_name: &str, - chart_root_dir: P, - override_file: &str, - envs: Vec<(&str, &str)>, -) -> Result, CommandError> -where - P: AsRef, -{ - // do exec helm upgrade - info!( - "exec helm upgrade for namespace {} and chart {}", - namespace, - chart_root_dir.as_ref().to_str().unwrap() - ); - - let _ = helm_exec_upgrade_with_override_file( - kubernetes_config.as_ref(), - namespace, - release_name, - chart_root_dir.as_ref(), - override_file, - envs.clone(), - )?; - - // list helm history - info!( - "exec helm history for namespace {} and chart {}", - namespace, - chart_root_dir.as_ref().to_str().unwrap() - ); - - let helm_history_rows = helm_exec_history(kubernetes_config.as_ref(), namespace, release_name, &envs)?; - - // take the last deployment from helm history - or return none if there is no history - Ok(helm_history_rows - .first() - .map(|helm_history_row| helm_history_row.clone())) -} - -pub fn is_chart_deployed

( - kubernetes_config: P, - envs: Vec<(&str, &str)>, - namespace: Option<&str>, - chart_name: String, -) -> Result -where - P: AsRef, -{ - let deployed_charts = helm_list(kubernetes_config, envs, namespace)?; - - for chart in deployed_charts { - if chart.name == chart_name { - return Ok(true); - } - } - - Ok(false) -} - -pub fn helm_get_chart_version

( - kubernetes_config: P, - envs: Vec<(&str, &str)>, - namespace: Option<&str>, - chart_name: String, -) -> Option -where - P: AsRef, -{ - match helm_list(kubernetes_config, envs, namespace) { - Ok(deployed_charts) => { - for chart in deployed_charts { - if chart.name == chart_name { - return chart.version; - } - } - - None - } - Err(_) => None, - } -} - -/// List deployed helm charts -/// -/// # Arguments -/// -/// * `kubernetes_config` - kubernetes config path -/// * `envs` - environment variables required for kubernetes connection -/// * `namespace` - list charts from a kubernetes namespace or use None to select all namespaces -pub fn helm_list

( - kubernetes_config: P, - envs: Vec<(&str, &str)>, - namespace: Option<&str>, -) -> Result, CommandError> -where - P: AsRef, -{ - let mut output_vec: Vec = Vec::new(); - let mut helm_args = vec![ - "list", - "--kubeconfig", - kubernetes_config.as_ref().to_str().unwrap(), - "-o", - "json", - ]; - match namespace { - Some(ns) => helm_args.append(&mut vec!["-n", ns]), - None => helm_args.push("-A"), - } - - let _ = helm_exec_with_output(helm_args, envs, |line| output_vec.push(line), |line| error!("{}", line)); - - let output_string: String = output_vec.join(""); - let values = serde_json::from_str::>(output_string.as_str()); - let mut helms_charts: Vec = Vec::new(); - - match values { - Ok(all_helms) => { - for helm in all_helms { - let raw_version = helm.chart.replace(format!("{}-", helm.name).as_str(), ""); - let version = match Version::from_str(raw_version.as_str()) { - Ok(v) => Some(v), - Err(_) => None, + CmdError( + chart.name.clone(), + HelmCommand::UPGRADE, + CommandError::new(stderr_msg.clone(), Some(stderr_msg)), + ) }; - helms_charts.push(HelmChart::new(helm.name, helm.namespace, version)) + Err(error) } } - Err(e) => { - let message_safe = "Error while deserializing all helms names"; - return Err(CommandError::new( - format!("{}, error: {}", message_safe, e), - Some(message_safe.to_string()), - )); + } + + pub fn uninstall_chart_if_breaking_version( + &self, + chart: &ChartInfo, + envs: &[(&str, &str)], + ) -> Result<(), HelmError> { + // If there is a breaking version set for the current helm chart, + // then we compare this breaking version with the currently installed version if any. + // If current installed version is older than breaking change one, then we delete + // the chart before applying it. + if let Some(breaking_version) = &chart.last_breaking_version_requiring_restart { + if let Some(installed_version) = + self.get_chart_version(chart.name.clone(), Some(chart.get_namespace_string().as_str()), envs)? + { + if installed_version.le(breaking_version) { + self.uninstall(&chart, envs)?; + } + } } - } - Ok(helms_charts) + Ok(()) + } } -pub fn helm_upgrade_diff_with_chart_info

( - kubernetes_config: P, - envs: &Vec<(String, String)>, - chart: &ChartInfo, -) -> Result<(), CommandError> -where - P: AsRef, -{ - let mut environment_variables = envs.clone(); - environment_variables.push(("HELM_NAMESPACE".to_string(), chart.get_namespace_string())); - - let mut args_string: Vec = vec![ - "diff", - "upgrade", - "--no-color", - "--allow-unreleased", - "--kubeconfig", - kubernetes_config.as_ref().to_str().unwrap(), - ] - .into_iter() - .map(|x| x.to_string()) - .collect(); - - // overrides and files overrides - for value in &chart.values { - args_string.push("--set".to_string()); - args_string.push(format!("{}={}", value.key, value.value)); - } - for value_file in &chart.values_files { - args_string.push("-f".to_string()); - args_string.push(value_file.clone()); - } - for value_file in &chart.yaml_files_content { - let file_path = format!("{}/{}", chart.path, &value_file.filename); - let file_create = || -> Result<(), Error> { - let mut file = File::create(&file_path)?; - file.write_all(value_file.yaml_content.as_bytes())?; - Ok(()) - }; - // no need to validate yaml as it will be done by helm - if let Err(e) = file_create() { - let safe_message = format!("Error while writing yaml content to file `{}`", &file_path); - return Err(CommandError::new( - format!( - "{}\nContent\n{}\nError: {}", - safe_message.to_string(), - value_file.yaml_content, - e - ), - Some(safe_message.to_string()), - )); - }; - - args_string.push("-f".to_string()); - args_string.push(file_path.clone()); - } - - // add last elements - args_string.push(chart.name.to_string()); - args_string.push(chart.path.to_string()); - - helm_exec_with_output( - args_string.iter().map(|x| x.as_str()).collect(), - environment_variables - .iter() - .map(|x| (x.0.as_str(), x.1.as_str())) - .collect(), - |line| info!("{}", line), - |line| error!("{}", line), - ) -} - -pub fn helm_exec(args: Vec<&str>, envs: Vec<(&str, &str)>) -> Result<(), CommandError> { - helm_exec_with_output( - args, - envs, - |line| { - span!(Level::INFO, "{}", "{}", line); - }, - |line_err| { - span!(Level::INFO, "{}", "{}", line_err); - }, - ) -} - -pub fn helm_exec_with_output( - args: Vec<&str>, - envs: Vec<(&str, &str)>, +fn helm_exec_with_output( + args: &[&str], + envs: &[(&str, &str)], stdout_output: F, stderr_output: X, ) -> Result<(), CommandError> @@ -853,69 +542,309 @@ where // Note: Helm CLI use spf13/cobra lib for the CLI; One function is mainly used to return an error if a command failed. // Helm returns an error each time a command does not succeed as they want. Which leads to handling error with status code 1 // It means that the command successfully ran, but it didn't terminate as expected - let mut cmd = QoveryCommand::new("helm", &args, &envs); + let mut cmd = QoveryCommand::new("helm", args, envs); match cmd.exec_with_timeout(Duration::max_value(), stdout_output, stderr_output) { Err(err) => Err(CommandError::new(format!("{:?}", err), None)), _ => Ok(()), } } +pub fn to_command_error(error: HelmError) -> CommandError { + CommandError::new_from_safe_message(error.to_string()) +} + +pub fn to_engine_error(event_details: &EventDetails, error: HelmError) -> EngineError { + EngineError::new_helm_error(event_details.clone(), error) +} + +#[cfg(feature = "test-with-kube")] #[cfg(test)] mod tests { - use crate::cmd::helm::helm_get_secret_lock_name; - use crate::cmd::structs::Secrets; - use chrono::{DateTime, NaiveDateTime, Utc}; + use crate::cloud_provider::helm::{ChartInfo, ChartSetValue}; + use crate::cmd::helm::{helm_exec_with_output, Helm, HelmError}; + use crate::cmd::utilities::QoveryCommand; + use std::sync::{Arc, Barrier}; + use std::thread; + use std::time::Duration; + + struct HelmTestCtx { + helm: Helm, + chart: ChartInfo, + } + + impl HelmTestCtx { + fn cleanup(&self) { + let ret = self.helm.uninstall(&self.chart, &vec![]); + assert!(ret.is_ok()) + } + + fn new(release_name: &str) -> HelmTestCtx { + let mut chart = ChartInfo::new_from_custom_namespace( + release_name.to_string(), + "tests/helm/simple_nginx".to_string(), + "default".to_string(), + 300, + vec![], + false, + None, + ); + chart.wait = true; + chart.atomic = true; + let mut kube_config = dirs::home_dir().unwrap(); + kube_config.push(".kube/config"); + let helm = Helm::new(kube_config.to_str().unwrap(), &vec![]).unwrap(); + + let cleanup = HelmTestCtx { helm, chart }; + cleanup.cleanup(); + cleanup + } + } + + impl Drop for HelmTestCtx { + fn drop(&mut self) { + self.cleanup() + } + } #[test] - fn test_helm_lock_get_name() { - let json_content = r#" -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "data": { - "release": "coucou" - }, - "kind": "Secret", - "metadata": { - "creationTimestamp": "2021-09-02T23:20:36Z", - "labels": { - "modifiedAt": "1632324195", - "name": "cert-manager", - "owner": "helm", - "status": "superseded", - "version": "1" - }, - "name": "sh.helm.release.v1.cert-manager.v1", - "namespace": "cert-manager", - "resourceVersion": "7287406", - "uid": "173b76c4-4f48-4544-8928-64a9b8b376d5" - }, - "type": "helm.sh/release.v1" - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "", - "selfLink": "" - } -} - "#; - let mut secrets = serde_json::from_str::(json_content).unwrap(); - - // expired lock should be ok - let res = helm_get_secret_lock_name(&secrets, 300).unwrap(); - assert_eq!(res, "sh.helm.release.v1.cert-manager.v1".to_string()); - - // lock is not expired yet - let time_in_future = NaiveDateTime::from_timestamp(Utc::now().timestamp() + 30, 0); - let time_in_future_datetime_format: DateTime = DateTime::from_utc(time_in_future, Utc); - secrets.items[0].metadata.creation_timestamp = time_in_future_datetime_format.to_rfc3339(); - let res = helm_get_secret_lock_name(&secrets, 300); - assert_eq!( - res.unwrap_err().message, - "helm lock has not yet expired, please wait 330s before retrying".to_string() - ) + fn check_version() { + let mut output = String::new(); + let _ = helm_exec_with_output(&vec!["version"], &vec![], |line| output.push_str(&line), |_line| {}); + assert!(output.contains("Version:\"v3.7.2\"")); + } + + #[test] + fn test_release_exist() { + let HelmTestCtx { ref helm, ref chart } = HelmTestCtx::new("test-release-exist"); + let ret = helm.check_release_exist(chart, &vec![]); + + assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)) + } + + #[test] + fn test_list_release() { + let HelmTestCtx { + ref helm, + ref mut chart, + } = HelmTestCtx::new("test-list-release"); + chart.custom_namespace = Some("hello-my-friend-this-is-a-test".to_string()); + + // no existing namespace should return an empty array + let ret = helm.list_release(Some("tsdfsfsdf"), &vec![]); + assert!(matches!(ret, Ok(vec) if vec.is_empty())); + + // install something + let ret = helm.upgrade(&chart, &vec![]); + assert!(matches!(ret, Ok(()))); + + // We should have at least one release in all the release + let ret = helm.list_release(None, &vec![]); + assert!(matches!(ret, Ok(vec) if !vec.is_empty())); + + // We should have at least one release in all the release + let ret = helm.list_release(Some(&chart.get_namespace_string()), &vec![]); + assert!(matches!(ret, Ok(vec) if vec.len() == 1)); + + // Install a second stuff + let HelmTestCtx { + ref helm, + ref mut chart, + } = HelmTestCtx::new("test-list-release-2"); + chart.custom_namespace = Some("hello-my-friend-this-is-a-test".to_string()); + let ret = helm.upgrade(&chart, &vec![]); + assert!(matches!(ret, Ok(()))); + + let ret = helm.list_release(Some(&chart.get_namespace_string()), &vec![]); + assert!(matches!(ret, Ok(vec) if vec.len() == 2)); + } + + #[test] + fn test_upgrade_diff() { + let HelmTestCtx { ref helm, ref chart } = HelmTestCtx::new("test-upgrade-diff"); + + let ret = helm.upgrade_diff(&chart, &vec![]); + assert!(matches!(ret, Ok(()))); + } + + #[test] + fn test_rollback() { + let HelmTestCtx { ref helm, ref chart } = HelmTestCtx::new("test-rollback"); + + // check release does not exist yet + let ret = helm.rollback(&chart, &vec![]); + assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)); + + // install it + let ret = helm.upgrade(&chart, &vec![]); + assert!(matches!(ret, Ok(()))); + + // First revision cannot be rollback + let ret = helm.rollback(&chart, &vec![]); + assert!(matches!(ret, Err(HelmError::CannotRollback(_)))); + + // 2nd upgrade + let ret = helm.upgrade(&chart, &vec![]); + assert!(matches!(ret, Ok(()))); + + // Rollback should be ok now + let ret = helm.rollback(&chart, &vec![]); + assert!(matches!(ret, Ok(()))); + } + + #[test] + fn test_upgrade() { + let HelmTestCtx { ref helm, ref chart } = HelmTestCtx::new("test-upgrade"); + + // check release does not exist yet + let ret = helm.check_release_exist(&chart, &vec![]); + assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)); + + // install it + let ret = helm.upgrade(&chart, &vec![]); + assert!(matches!(ret, Ok(()))); + + // check now it exists + let ret = helm.check_release_exist(&chart, &vec![]); + assert!(matches!(ret, Ok(_))); + } + + #[test] + fn test_upgrade_timeout() { + let HelmTestCtx { + ref helm, + ref mut chart, + } = HelmTestCtx::new("test-upgrade-timeout"); + chart.timeout_in_seconds = 1; + + // check release does not exist yet + let ret = helm.check_release_exist(&chart, &vec![]); + assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)); + + // install it + let ret = helm.upgrade(&chart, &vec![]); + assert!(matches!(ret, Err(HelmError::Timeout(_, _, _)))); + + // Release should not exist if it fails + let ret = helm.check_release_exist(&chart, &vec![]); + assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)); + } + + #[test] + fn test_upgrade_with_lock_during_install() { + // We want to check that we manage to install a chart even if a lock is present while it was the first installation + let HelmTestCtx { ref helm, ref chart } = HelmTestCtx::new("test-upgrade-with-lock-install"); + + // check release does not exist yet + let ret = helm.check_release_exist(&chart, &vec![]); + assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)); + + // Spawn our task killer + let barrier = Arc::new(Barrier::new(2)); + std::thread::spawn({ + let barrier = barrier.clone(); + let chart_name = chart.name.clone(); + move || { + barrier.wait(); + thread::sleep(Duration::from_millis(3000)); + let mut cmd = QoveryCommand::new("pkill", &vec!["-9", "-f", &format!("helm.*{}", chart_name)], &vec![]); + let _ = cmd.exec(); + } + }); + + // install it + barrier.wait(); + let ret = helm.upgrade(&chart, &vec![]); + assert!(matches!(ret, Err(_))); + + // Release should be locked + let ret = helm.check_release_exist(&chart, &vec![]); + assert!(matches!(ret, Ok(release) if release.is_locked())); + + // New installation should work even if a lock is present + let ret = helm.upgrade(&chart, &vec![]); + assert!(matches!(ret, Ok(()))); + + // Release should not be locked anymore + let ret = helm.check_release_exist(&chart, &vec![]); + assert!(matches!(ret, Ok(release) if !release.is_locked())); + } + + #[test] + fn test_upgrade_with_lock_during_upgrade() { + // We want to check that we manage to install a chart even if a lock is present while it not the first installation + let HelmTestCtx { + ref helm, + ref mut chart, + } = HelmTestCtx::new("test-upgrade-with-lock-upgrade"); + + // check release does not exist yet + let ret = helm.check_release_exist(&chart, &vec![]); + assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)); + + // First install + let ret = helm.upgrade(&chart, &vec![]); + assert!(matches!(ret, Ok(()))); + + // Spawn our task killer + let barrier = Arc::new(Barrier::new(2)); + std::thread::spawn({ + let barrier = barrier.clone(); + let chart_name = chart.name.clone(); + move || { + barrier.wait(); + thread::sleep(Duration::from_millis(3000)); + let mut cmd = QoveryCommand::new("pkill", &vec!["-9", "-f", &format!("helm.*{}", chart_name)], &vec![]); + let _ = cmd.exec(); + } + }); + + chart.values = vec![ChartSetValue { + key: "initialDelaySeconds".to_string(), + value: "6".to_string(), + }]; + barrier.wait(); + let ret = helm.upgrade(&chart, &vec![]); + assert!(matches!(ret, Err(_))); + + // Release should be locked + let ret = helm.check_release_exist(&chart, &vec![]); + assert!(matches!(ret, Ok(release) if release.is_locked() && release.version == 2)); + + // New installation should work even if a lock is present + let ret = helm.upgrade(&chart, &vec![]); + assert!(matches!(ret, Ok(()))); + + // Release should not be locked anymore + let ret = helm.check_release_exist(&chart, &vec![]); + assert!(matches!(ret, Ok(release) if !release.is_locked() && release.version == 4)); + } + + #[test] + fn test_uninstall() { + let HelmTestCtx { ref helm, ref chart } = HelmTestCtx::new("test-uninstall"); + + // check release does not exist yet + let ret = helm.check_release_exist(&chart, &vec![]); + assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)); + + // deleting something that does not exist should not be an issue + let ret = helm.uninstall(&chart, &vec![]); + assert!(matches!(ret, Ok(()))); + + // install it + let ret = helm.upgrade(&chart, &vec![]); + assert!(matches!(ret, Ok(()))); + + // check now it exists + let ret = helm.check_release_exist(&chart, &vec![]); + assert!(matches!(ret, Ok(_))); + + // Delete it + let ret = helm.uninstall(&chart, &vec![]); + assert!(matches!(ret, Ok(()))); + + // check release does not exist anymore + let ret = helm.check_release_exist(&chart, &vec![]); + assert!(matches!(ret, Err(HelmError::ReleaseDoesNotExist(test)) if test == chart.name)); } } diff --git a/src/errors/mod.rs b/src/errors/mod.rs index 38da4362..915dd366 100644 --- a/src/errors/mod.rs +++ b/src/errors/mod.rs @@ -3,6 +3,7 @@ pub mod io; extern crate url; use crate::cloud_provider::utilities::VersionsNumber; +use crate::cmd::helm::HelmError; use crate::error::{EngineError as LegacyEngineError, EngineErrorCause, EngineErrorScope}; use crate::events::EventDetails; use url::Url; @@ -1503,6 +1504,29 @@ impl EngineError { ) } + /// Creates new error from an Helm error + /// + /// Arguments: + /// + /// * `event_details`: Error linked event details. + /// * `error`: Raw error message. + pub fn new_helm_error(event_details: EventDetails, error: HelmError) -> EngineError { + let cmd_error = match &error { + HelmError::CmdError(_, _, cmd_error) => Some(cmd_error.clone()), + _ => None, + }; + + EngineError::new( + event_details, + Tag::HelmChartUninstallError, + error.to_string(), + error.to_string(), + cmd_error, + None, + None, + ) + } + /// Creates new error while uninstalling Helm chart. /// /// Arguments: diff --git a/tests/helm/simple_nginx/.helmignore b/tests/helm/simple_nginx/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/tests/helm/simple_nginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/tests/helm/simple_nginx/Chart.yaml b/tests/helm/simple_nginx/Chart.yaml new file mode 100644 index 00000000..3464a394 --- /dev/null +++ b/tests/helm/simple_nginx/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: nginx +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/tests/helm/simple_nginx/templates/_helpers.tpl b/tests/helm/simple_nginx/templates/_helpers.tpl new file mode 100644 index 00000000..7423a2c9 --- /dev/null +++ b/tests/helm/simple_nginx/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "toto.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "toto.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "toto.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "toto.labels" -}} +helm.sh/chart: {{ include "toto.chart" . }} +{{ include "toto.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "toto.selectorLabels" -}} +app.kubernetes.io/name: {{ include "toto.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "toto.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "toto.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/tests/helm/simple_nginx/templates/deployment.yaml b/tests/helm/simple_nginx/templates/deployment.yaml new file mode 100644 index 00000000..259e9faa --- /dev/null +++ b/tests/helm/simple_nginx/templates/deployment.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "toto.fullname" . }} + labels: + {{- include "toto.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "toto.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "toto.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "toto.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 80 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + initialDelaySeconds: {{ .Values.initialDelaySeconds }} + httpGet: + path: / + port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/tests/helm/simple_nginx/templates/hpa.yaml b/tests/helm/simple_nginx/templates/hpa.yaml new file mode 100644 index 00000000..d7c1529d --- /dev/null +++ b/tests/helm/simple_nginx/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "toto.fullname" . }} + labels: + {{- include "toto.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "toto.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/tests/helm/simple_nginx/templates/ingress.yaml b/tests/helm/simple_nginx/templates/ingress.yaml new file mode 100644 index 00000000..c0309c25 --- /dev/null +++ b/tests/helm/simple_nginx/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "toto.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "toto.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/tests/helm/simple_nginx/templates/service.yaml b/tests/helm/simple_nginx/templates/service.yaml new file mode 100644 index 00000000..c57264e0 --- /dev/null +++ b/tests/helm/simple_nginx/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "toto.fullname" . }} + labels: + {{- include "toto.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "toto.selectorLabels" . | nindent 4 }} diff --git a/tests/helm/simple_nginx/templates/serviceaccount.yaml b/tests/helm/simple_nginx/templates/serviceaccount.yaml new file mode 100644 index 00000000..8e86f4e2 --- /dev/null +++ b/tests/helm/simple_nginx/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "toto.serviceAccountName" . }} + labels: + {{- include "toto.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/tests/helm/simple_nginx/templates/tests/test-connection.yaml b/tests/helm/simple_nginx/templates/tests/test-connection.yaml new file mode 100644 index 00000000..89ca584c --- /dev/null +++ b/tests/helm/simple_nginx/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "toto.fullname" . }}-test-connection" + labels: + {{- include "toto.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "toto.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/tests/helm/simple_nginx/values.yaml b/tests/helm/simple_nginx/values.yaml new file mode 100644 index 00000000..8ab1b5c2 --- /dev/null +++ b/tests/helm/simple_nginx/values.yaml @@ -0,0 +1,83 @@ +# Default values for toto. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 +initialDelaySeconds: 5 + +image: + repository: nginx + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} From 60bc3918e330acbb0de08962f5e79063b1deecdf Mon Sep 17 00:00:00 2001 From: Romain GERARD Date: Thu, 17 Feb 2022 17:53:00 +0100 Subject: [PATCH 4/7] Remove old un-install --- src/cloud_provider/aws/kubernetes/helm_charts.rs | 4 ++-- .../digitalocean/kubernetes/helm_charts.rs | 9 +++------ src/cloud_provider/scaleway/kubernetes/helm_charts.rs | 9 +++------ 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/cloud_provider/aws/kubernetes/helm_charts.rs b/src/cloud_provider/aws/kubernetes/helm_charts.rs index f764be31..dde79657 100644 --- a/src/cloud_provider/aws/kubernetes/helm_charts.rs +++ b/src/cloud_provider/aws/kubernetes/helm_charts.rs @@ -458,6 +458,7 @@ pub fn aws_helm_charts( }, }; + /* Example to delete an old install let old_prometheus_operator = PrometheusOperatorConfigChart { chart_info: ChartInfo { name: "prometheus-operator".to_string(), @@ -465,7 +466,7 @@ pub fn aws_helm_charts( action: HelmAction::Destroy, ..Default::default() }, - }; + };*/ let kube_prometheus_stack = PrometheusOperatorConfigChart { chart_info: ChartInfo { @@ -1155,7 +1156,6 @@ datasources: Box::new(q_storage_class), Box::new(coredns_config), Box::new(aws_vpc_cni_chart), - Box::new(old_prometheus_operator), ]; let level_2: Vec> = vec![Box::new(cert_manager)]; diff --git a/src/cloud_provider/digitalocean/kubernetes/helm_charts.rs b/src/cloud_provider/digitalocean/kubernetes/helm_charts.rs index b420be08..280b124e 100644 --- a/src/cloud_provider/digitalocean/kubernetes/helm_charts.rs +++ b/src/cloud_provider/digitalocean/kubernetes/helm_charts.rs @@ -309,6 +309,7 @@ pub fn do_helm_charts( }, }; + /* let old_prometheus_operator = PrometheusOperatorConfigChart { chart_info: ChartInfo { name: "prometheus-operator".to_string(), @@ -316,7 +317,7 @@ pub fn do_helm_charts( action: HelmAction::Destroy, ..Default::default() }, - }; + };*/ let kube_prometheus_stack = PrometheusOperatorConfigChart { chart_info: ChartInfo { @@ -1029,11 +1030,7 @@ datasources: }; // chart deployment order matters!!! - let level_1: Vec> = vec![ - Box::new(q_storage_class), - Box::new(coredns_config), - Box::new(old_prometheus_operator), - ]; + let level_1: Vec> = vec![Box::new(q_storage_class), Box::new(coredns_config)]; let mut level_2: Vec> = vec![Box::new(container_registry_secret), Box::new(cert_manager)]; diff --git a/src/cloud_provider/scaleway/kubernetes/helm_charts.rs b/src/cloud_provider/scaleway/kubernetes/helm_charts.rs index 1479fc37..5627c6bd 100644 --- a/src/cloud_provider/scaleway/kubernetes/helm_charts.rs +++ b/src/cloud_provider/scaleway/kubernetes/helm_charts.rs @@ -283,6 +283,7 @@ pub fn scw_helm_charts( }, }; + /* Example to delete an old chart let old_prometheus_operator = PrometheusOperatorConfigChart { chart_info: ChartInfo { name: "prometheus-operator".to_string(), @@ -290,7 +291,7 @@ pub fn scw_helm_charts( action: HelmAction::Destroy, ..Default::default() }, - }; + };*/ let kube_prometheus_stack = PrometheusOperatorConfigChart { chart_info: ChartInfo { @@ -858,11 +859,7 @@ datasources: }; // chart deployment order matters!!! - let level_1: Vec> = vec![ - Box::new(q_storage_class), - Box::new(coredns_config), - Box::new(old_prometheus_operator), - ]; + let level_1: Vec> = vec![Box::new(q_storage_class), Box::new(coredns_config)]; let level_2: Vec> = vec![Box::new(cert_manager)]; From 752a54e715a1f4ada869fd9f88f4a917f99ec8c4 Mon Sep 17 00:00:00 2001 From: Romain GERARD Date: Thu, 17 Feb 2022 17:56:14 +0100 Subject: [PATCH 5/7] Lint --- src/cloud_provider/aws/kubernetes/helm_charts.rs | 2 +- src/cloud_provider/digitalocean/kubernetes/helm_charts.rs | 3 +-- src/cloud_provider/scaleway/kubernetes/helm_charts.rs | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/cloud_provider/aws/kubernetes/helm_charts.rs b/src/cloud_provider/aws/kubernetes/helm_charts.rs index dde79657..78279ba8 100644 --- a/src/cloud_provider/aws/kubernetes/helm_charts.rs +++ b/src/cloud_provider/aws/kubernetes/helm_charts.rs @@ -1,7 +1,7 @@ use crate::cloud_provider::aws::kubernetes::{Options, VpcQoveryNetworkMode}; use crate::cloud_provider::helm::{ get_chart_for_shell_agent, get_engine_helm_action_from_location, ChartInfo, ChartPayload, ChartSetValue, - ChartValuesGenerated, CommonChart, CoreDNSConfigChart, HelmAction, HelmChart, HelmChartNamespaces, + ChartValuesGenerated, CommonChart, CoreDNSConfigChart, HelmChart, HelmChartNamespaces, PrometheusOperatorConfigChart, ShellAgentContext, }; use crate::cloud_provider::qovery::{get_qovery_app_version, EngineLocation, QoveryAgent, QoveryAppName, QoveryEngine}; diff --git a/src/cloud_provider/digitalocean/kubernetes/helm_charts.rs b/src/cloud_provider/digitalocean/kubernetes/helm_charts.rs index 280b124e..88c6e0f1 100644 --- a/src/cloud_provider/digitalocean/kubernetes/helm_charts.rs +++ b/src/cloud_provider/digitalocean/kubernetes/helm_charts.rs @@ -1,8 +1,7 @@ use crate::cloud_provider::digitalocean::kubernetes::DoksOptions; use crate::cloud_provider::helm::{ get_chart_for_shell_agent, get_engine_helm_action_from_location, ChartInfo, ChartSetValue, ChartValuesGenerated, - CommonChart, CoreDNSConfigChart, HelmAction, HelmChart, HelmChartNamespaces, PrometheusOperatorConfigChart, - ShellAgentContext, + CommonChart, CoreDNSConfigChart, HelmChart, HelmChartNamespaces, PrometheusOperatorConfigChart, ShellAgentContext, }; use crate::cloud_provider::qovery::{get_qovery_app_version, EngineLocation, QoveryAgent, QoveryAppName, QoveryEngine}; use crate::errors::CommandError; diff --git a/src/cloud_provider/scaleway/kubernetes/helm_charts.rs b/src/cloud_provider/scaleway/kubernetes/helm_charts.rs index 5627c6bd..98bb8f3e 100644 --- a/src/cloud_provider/scaleway/kubernetes/helm_charts.rs +++ b/src/cloud_provider/scaleway/kubernetes/helm_charts.rs @@ -1,7 +1,6 @@ use crate::cloud_provider::helm::{ get_chart_for_shell_agent, get_engine_helm_action_from_location, ChartInfo, ChartSetValue, ChartValuesGenerated, - CommonChart, CoreDNSConfigChart, HelmAction, HelmChart, HelmChartNamespaces, PrometheusOperatorConfigChart, - ShellAgentContext, + CommonChart, CoreDNSConfigChart, HelmChart, HelmChartNamespaces, PrometheusOperatorConfigChart, ShellAgentContext, }; use crate::cloud_provider::qovery::{get_qovery_app_version, EngineLocation, QoveryAgent, QoveryAppName, QoveryEngine}; use crate::cloud_provider::scaleway::application::{ScwRegion, ScwZone}; From 04ef1a1a04990aeb634dcdf58828f652b8866b95 Mon Sep 17 00:00:00 2001 From: Romain GERARD Date: Fri, 18 Feb 2022 09:09:40 +0100 Subject: [PATCH 6/7] Fix and normalize timeout for all helm charts --- src/cloud_provider/aws/router.rs | 2 +- src/cloud_provider/digitalocean/router.rs | 2 +- src/cloud_provider/helm.rs | 4 +--- src/cloud_provider/scaleway/router.rs | 2 +- src/cmd/helm.rs | 8 +++----- 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/cloud_provider/aws/router.rs b/src/cloud_provider/aws/router.rs index eccd99f4..47dfdfc7 100644 --- a/src/cloud_provider/aws/router.rs +++ b/src/cloud_provider/aws/router.rs @@ -336,7 +336,7 @@ impl Create for Router { helm_release_name, workspace_dir.clone(), environment.namespace().to_string(), - self.start_timeout().value() as i64, + 600_i64, match self.service_type() { ServiceType::Database(_) => vec![format!("{}/q-values.yaml", &workspace_dir)], _ => vec![], diff --git a/src/cloud_provider/digitalocean/router.rs b/src/cloud_provider/digitalocean/router.rs index 44a8aafd..ab48f336 100644 --- a/src/cloud_provider/digitalocean/router.rs +++ b/src/cloud_provider/digitalocean/router.rs @@ -356,7 +356,7 @@ impl Create for Router { helm_release_name, workspace_dir.clone(), environment.namespace().to_string(), - self.start_timeout().value() as i64, + 600_i64, match self.service_type() { ServiceType::Database(_) => vec![format!("{}/q-values.yaml", &workspace_dir)], _ => vec![], diff --git a/src/cloud_provider/helm.rs b/src/cloud_provider/helm.rs index 088431b5..f78fe1c1 100644 --- a/src/cloud_provider/helm.rs +++ b/src/cloud_provider/helm.rs @@ -110,8 +110,6 @@ impl ChartInfo { name: name.to_string(), namespace: HelmChartNamespaces::Custom, custom_namespace: Some(custom_namespace.to_string()), - timeout_in_seconds: 600, - atomic: true, ..Default::default() } } @@ -138,7 +136,7 @@ impl Default for ChartInfo { atomic: true, force_upgrade: false, last_breaking_version_requiring_restart: None, - timeout_in_seconds: 300, + timeout_in_seconds: 600, dry_run: false, wait: true, values: Vec::new(), diff --git a/src/cloud_provider/scaleway/router.rs b/src/cloud_provider/scaleway/router.rs index dfe3f5f7..53a60057 100644 --- a/src/cloud_provider/scaleway/router.rs +++ b/src/cloud_provider/scaleway/router.rs @@ -304,7 +304,7 @@ impl Create for Router { helm_release_name, workspace_dir.clone(), environment.namespace().to_string(), - self.start_timeout().value() as i64, + 600_i64, match self.service_type() { ServiceType::Database(_) => vec![format!("{}/q-values.yaml", &workspace_dir)], _ => vec![], diff --git a/src/cmd/helm.rs b/src/cmd/helm.rs index b7a2780a..c118d829 100644 --- a/src/cmd/helm.rs +++ b/src/cmd/helm.rs @@ -16,7 +16,7 @@ use serde_derive::Deserialize; use std::fs::File; use std::str::FromStr; -const HELM_DEFAULT_TIMEOUT_IN_SECONDS: u32 = 300; +const HELM_DEFAULT_TIMEOUT_IN_SECONDS: u32 = 600; const HELM_MAX_HISTORY: &str = "50"; pub enum Timeout { @@ -579,17 +579,15 @@ mod tests { } fn new(release_name: &str) -> HelmTestCtx { - let mut chart = ChartInfo::new_from_custom_namespace( + let chart = ChartInfo::new_from_custom_namespace( release_name.to_string(), "tests/helm/simple_nginx".to_string(), "default".to_string(), - 300, + 600, vec![], false, None, ); - chart.wait = true; - chart.atomic = true; let mut kube_config = dirs::home_dir().unwrap(); kube_config.push(".kube/config"); let helm = Helm::new(kube_config.to_str().unwrap(), &vec![]).unwrap(); From 0e94a5f61042dea261163566b16bda09de8d1c2c Mon Sep 17 00:00:00 2001 From: Romain GERARD Date: Fri, 18 Feb 2022 13:40:40 +0100 Subject: [PATCH 7/7] Use google dns resolver --- src/cloud_provider/utilities.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/cloud_provider/utilities.rs b/src/cloud_provider/utilities.rs index e01f050f..c21bc489 100644 --- a/src/cloud_provider/utilities.rs +++ b/src/cloud_provider/utilities.rs @@ -319,7 +319,7 @@ impl fmt::Display for VersionsNumber { } } -fn cloudflare_dns_resolver() -> Resolver { +fn google_dns_resolver() -> Resolver { let mut resolver_options = ResolverOpts::default(); // We want to avoid cache and using host file of the host, as some provider force caching @@ -335,8 +335,7 @@ fn cloudflare_dns_resolver() -> Resolver { //); //Resolver::new(resolver, resolver_options).unwrap() - Resolver::new(ResolverConfig::cloudflare(), resolver_options) - .expect("Invalid cloudflare DNS resolver configuration") + Resolver::new(ResolverConfig::google(), resolver_options).expect("Invalid google DNS resolver configuration") } fn get_cname_record_value(resolver: &Resolver, cname: &str) -> Option { @@ -360,7 +359,7 @@ pub fn check_cname_for( cname_to_check: &str, execution_id: &str, ) -> Result { - let resolver = cloudflare_dns_resolver(); + let resolver = google_dns_resolver(); let listener_helper = ListenersHelper::new(listeners); let send_deployment_progress = |msg: &str| { @@ -428,7 +427,7 @@ pub fn check_domain_for( execution_id: &str, context_id: &str, ) -> Result<(), EngineError> { - let resolver = cloudflare_dns_resolver(); + let resolver = google_dns_resolver(); for domain in domains_to_check { listener_helper.deployment_in_progress(ProgressInfo::new( @@ -586,7 +585,7 @@ pub fn print_action(cloud_provider_name: &str, struct_name: &str, fn_name: &str, mod tests { use crate::cloud_provider::models::CpuLimits; use crate::cloud_provider::utilities::{ - cloudflare_dns_resolver, convert_k8s_cpu_value_to_f32, get_cname_record_value, + convert_k8s_cpu_value_to_f32, get_cname_record_value, google_dns_resolver, validate_k8s_required_cpu_and_burstable, VersionsNumber, }; use crate::error::StringError; @@ -634,7 +633,7 @@ mod tests { #[test] pub fn test_cname_resolution() { - let resolver = cloudflare_dns_resolver(); + let resolver = google_dns_resolver(); let cname = get_cname_record_value(&resolver, "ci-test-no-delete.qovery.io"); assert_eq!(cname, Some(String::from("qovery.io.")));