From a749dd86b984ffb7126d3c69ac77a70e4be3667c Mon Sep 17 00:00:00 2001 From: Pierre Mavro Date: Fri, 4 Dec 2020 14:30:17 +0100 Subject: [PATCH] feat: bump new Pleco version to support kubernetes delete --- lib/aws/bootstrap/helm-pleco.j2.tf | 5 + lib/common/bootstrap/charts/pleco/Chart.yaml | 4 +- .../charts/pleco/templates/deployment.yaml | 13 +- lib/common/bootstrap/charts/pleco/values.yaml | 12 +- lib/helm-freeze.yaml | 2 +- src/cloud_provider/aws/application.rs | 29 ++++ src/cloud_provider/aws/databases/mongodb.rs | 2 +- src/cloud_provider/aws/databases/mysql.rs | 2 +- .../aws/databases/postgresql.rs | 2 +- src/cloud_provider/aws/databases/redis.rs | 2 +- src/cloud_provider/aws/databases/utilities.rs | 4 +- src/cmd/kubectl.rs | 126 ++++++++++++++++-- src/cmd/structs.rs | 5 + 13 files changed, 187 insertions(+), 21 deletions(-) diff --git a/lib/aws/bootstrap/helm-pleco.j2.tf b/lib/aws/bootstrap/helm-pleco.j2.tf index 82a9f183..1d084d46 100644 --- a/lib/aws/bootstrap/helm-pleco.j2.tf +++ b/lib/aws/bootstrap/helm-pleco.j2.tf @@ -13,6 +13,11 @@ resource "helm_release" "pleco" { value = timestamp() } + set { + name = "environmentVariables.DISABLE_DRY_RUN" + value = "true" + } + set { name = "environmentVariables.AWS_ACCESS_KEY_ID" value = "{{ aws_access_key }}" diff --git a/lib/common/bootstrap/charts/pleco/Chart.yaml b/lib/common/bootstrap/charts/pleco/Chart.yaml index 5848b844..2b353d85 100644 --- a/lib/common/bootstrap/charts/pleco/Chart.yaml +++ b/lib/common/bootstrap/charts/pleco/Chart.yaml @@ -1,9 +1,9 @@ apiVersion: v2 -appVersion: 0.2.1 +appVersion: 0.3.0 description: Automatically removes Cloud managed services and Kubernetes resources based on tags with TTL home: https://github.com/Qovery/pleco icon: https://github.com/Qovery/pleco/raw/main/assets/pleco_logo.png name: pleco type: application -version: 0.2.1 +version: 0.3.0 diff --git a/lib/common/bootstrap/charts/pleco/templates/deployment.yaml b/lib/common/bootstrap/charts/pleco/templates/deployment.yaml index 9b5eb157..73530886 100644 --- a/lib/common/bootstrap/charts/pleco/templates/deployment.yaml +++ b/lib/common/bootstrap/charts/pleco/templates/deployment.yaml @@ -38,9 +38,20 @@ spec: - {{ .Values.environmentVariables.LOG_LEVEL | default "info" }} - --check-interval - "{{ .Values.environmentVariables.CHECK_INTERVAL | default 120 }}" - {{ if .Values.environmentVariables.DRY_RUN }} + {{ if not .Values.environmentVariables.DISABLE_DRY_RUN}} - --dry-run {{ end }} + - --kube-conn + - {{ .Values.enabledFeatures.kubernetes }} + {{ if eq .Values.enabledFeatures.rds true}} + - --enable-rds + {{ end }} + {{ if eq .Values.enabledFeatures.elasticache true}} + - --enable-elasticache + {{ end }} + {{ if eq .Values.enabledFeatures.documentdb true}} + - --enable-documentdb + {{ end }} env: {{ range $key, $value := .Values.environmentVariables -}} - name: "{{ $key }}" diff --git a/lib/common/bootstrap/charts/pleco/values.yaml b/lib/common/bootstrap/charts/pleco/values.yaml index df7c88a0..d4fae419 100644 --- a/lib/common/bootstrap/charts/pleco/values.yaml +++ b/lib/common/bootstrap/charts/pleco/values.yaml @@ -3,15 +3,23 @@ replicaCount: 1 image: repository: qoveryrd/pleco pullPolicy: IfNotPresent - plecoImageTag: "v0.2.1" + plecoImageTag: "v0.3.0" environmentVariables: CHECK_INTERVAL: "120" - DRY_RUN: "true" + DISABLE_DRY_RUN: "false" LOG_LEVEL: "info" # AWS_ACCESS_KEY_ID: "" # AWS_SECRET_ACCESS_KEY: "" # AWS_DEFAULT_REGION: "" + # KUBECONFIG: "" + +enabledFeatures: + # Choose between in/out/off + kubernetes: "in" + rds: true + documentdb: true + elasticache: true imagePullSecrets: [] nameOverride: "" diff --git a/lib/helm-freeze.yaml b/lib/helm-freeze.yaml index b5829686..c9b4f605 100644 --- a/lib/helm-freeze.yaml +++ b/lib/helm-freeze.yaml @@ -55,7 +55,7 @@ charts: version: 12.0.1 dest: services - name: pleco - version: 0.2.1 + version: 0.3.0 repo_name: pleco repos: diff --git a/src/cloud_provider/aws/application.rs b/src/cloud_provider/aws/application.rs index f948fc73..ba860b9e 100644 --- a/src/cloud_provider/aws/application.rs +++ b/src/cloud_provider/aws/application.rs @@ -16,6 +16,7 @@ use crate::error::{ cast_simple_error_to_engine_error, EngineError, EngineErrorCause, }; use crate::models::Context; +use crate::cmd::structs::LabelsContent; #[derive(Clone, Eq, PartialEq, Hash)] pub struct Application { @@ -131,6 +132,13 @@ impl Application { context.insert("clone", &false); context.insert("start_timeout_in_seconds", &self.start_timeout_in_seconds); + if self.context.resource_expiration_in_seconds().is_some() { + context.insert( + "resource_expiration_in_seconds", + &self.context.resource_expiration_in_seconds(), + ) + } + context } @@ -277,6 +285,27 @@ impl Create for Application { ), )?; + // define labels to add to namespace + let namespace_labels = match self.context.resource_expiration_in_seconds() { + Some(v) => Some(vec![(LabelsContent{ + name: "ttl".to_string(), + value: format!{"{}", self.context.resource_expiration_in_seconds().unwrap()}, + })]), + None => None, + }; + + // create a namespace with labels if do not exists + let _ = cast_simple_error_to_engine_error( + self.engine_error_scope(), + self.context.execution_id(), + crate::cmd::kubectl::kubectl_exec_create_namespace( + kubernetes_config_file_path.as_str(), + environment.namespace(), + namespace_labels, + aws_credentials_envs.clone(), + ), + )?; + // do exec helm upgrade and return the last deployment status let helm_history_row = cast_simple_error_to_engine_error( self.engine_error_scope(), diff --git a/src/cloud_provider/aws/databases/mongodb.rs b/src/cloud_provider/aws/databases/mongodb.rs index b9541a73..7ff86c03 100644 --- a/src/cloud_provider/aws/databases/mongodb.rs +++ b/src/cloud_provider/aws/databases/mongodb.rs @@ -98,7 +98,7 @@ impl MongoDB { .downcast_ref::() .unwrap(); - utilities::create_namespace(&environment.namespace(), kube_config.as_str(), aws); + utilities::create_namespace_without_labels(&environment.namespace(), kube_config.as_str(), aws); } Err(e) => error!( "Failed to generate the kubernetes config file path: {:?}", diff --git a/src/cloud_provider/aws/databases/mysql.rs b/src/cloud_provider/aws/databases/mysql.rs index e2e5bebf..eaced64c 100644 --- a/src/cloud_provider/aws/databases/mysql.rs +++ b/src/cloud_provider/aws/databases/mysql.rs @@ -95,7 +95,7 @@ impl MySQL { .downcast_ref::() .unwrap(); - utilities::create_namespace(&environment.namespace(), kube_config.as_str(), aws); + utilities::create_namespace_without_labels(&environment.namespace(), kube_config.as_str(), aws); } Err(e) => error!( "Failed to generate the kubernetes config file path: {:?}", diff --git a/src/cloud_provider/aws/databases/postgresql.rs b/src/cloud_provider/aws/databases/postgresql.rs index bb784ad7..35b17af8 100644 --- a/src/cloud_provider/aws/databases/postgresql.rs +++ b/src/cloud_provider/aws/databases/postgresql.rs @@ -95,7 +95,7 @@ impl PostgreSQL { .downcast_ref::() .unwrap(); - utilities::create_namespace(&environment.namespace(), kube_config.as_str(), aws); + utilities::create_namespace_without_labels(&environment.namespace(), kube_config.as_str(), aws); } Err(e) => error!( "Failed to generate the kubernetes config file path: {:?}", diff --git a/src/cloud_provider/aws/databases/redis.rs b/src/cloud_provider/aws/databases/redis.rs index f2c54f5c..7cc957bb 100644 --- a/src/cloud_provider/aws/databases/redis.rs +++ b/src/cloud_provider/aws/databases/redis.rs @@ -102,7 +102,7 @@ impl Redis { .downcast_ref::() .unwrap(); - utilities::create_namespace(&environment.namespace(), kube_config.as_str(), aws); + utilities::create_namespace_without_labels(&environment.namespace(), kube_config.as_str(), aws); } Err(e) => error!( "Failed to generate the kubernetes config file path: {:?}", diff --git a/src/cloud_provider/aws/databases/utilities.rs b/src/cloud_provider/aws/databases/utilities.rs index a1e41ee3..40c62c20 100644 --- a/src/cloud_provider/aws/databases/utilities.rs +++ b/src/cloud_provider/aws/databases/utilities.rs @@ -29,12 +29,12 @@ pub fn get_kubernetes_config_path( ) } -pub fn create_namespace(namespace: &str, kube_config: &str, aws: &AWS) { +pub fn create_namespace_without_labels(namespace: &str, kube_config: &str, aws: &AWS) { let aws_credentials_envs = vec![ (AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()), (AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()), ]; - kubectl_exec_create_namespace(kube_config, namespace, aws_credentials_envs); + kubectl_exec_create_namespace(kube_config, namespace,None, aws_credentials_envs); } pub fn delete_terraform_tfstate_secret( diff --git a/src/cmd/kubectl.rs b/src/cmd/kubectl.rs index 9b805df4..1a608ed1 100644 --- a/src/cmd/kubectl.rs +++ b/src/cmd/kubectl.rs @@ -4,10 +4,7 @@ use std::path::Path; use retry::delay::Fibonacci; use retry::OperationResult; -use crate::cmd::structs::{ - Item, KubernetesJob, KubernetesList, KubernetesNode, KubernetesPod, KubernetesPodStatusPhase, - KubernetesService, -}; +use crate::cmd::structs::{Item, KubernetesJob, KubernetesList, KubernetesNode, KubernetesPod, KubernetesPodStatusPhase, KubernetesService, LabelsContent}; use crate::cmd::utilities::exec_with_envs_and_output; use crate::error::{SimpleError, SimpleErrorKind}; use crate::constants::KUBECONFIG; @@ -347,20 +344,131 @@ where Ok(Some(false)) } -pub fn kubectl_exec_create_namespace

( +pub fn kubectl_exec_is_namespace_present

( kubernetes_config: P, namespace: &str, envs: Vec<(&str, &str)>, -) -> Result<(), SimpleError> -where - P: AsRef, +) -> bool + where + P: AsRef, { let mut _envs = Vec::with_capacity(envs.len() + 1); _envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap())); _envs.extend(envs); + let mut output_vec: Vec = Vec::new(); + let result = kubectl_exec_with_output( + vec!["get", "namespace", namespace], + _envs, + |out| match out { + Ok(line) => output_vec.push(line), + Err(err) => error!("{:?}", err), + }, + |out| match out { + Ok(line) => error!("{}", line), + Err(err) => error!("{:?}", err), + }, + ); + + match result { + Ok(_) => true, + Err(_) => false, + } +} + +pub fn kubectl_exec_create_namespace

( + kubernetes_config: P, + namespace: &str, + labels: Option>, + envs: Vec<(&str, &str)>, +) -> Result<(), SimpleError> +where + P: AsRef, +{ + // don't create the namespace if already exists and not not return error in this case + if !kubectl_exec_is_namespace_present( + kubernetes_config.as_ref(), + namespace.clone(), + envs.clone(), + ) { + // create namespace + let mut _envs = Vec::with_capacity(envs.len() + 1); + _envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap())); + _envs.extend(envs.clone()); + + let _ = kubectl_exec_with_output( + vec!["create", "namespace", namespace], + _envs, + |out| match out { + Ok(line) => info!("{}", line), + Err(err) => error!("{:?}", err), + }, + |out| match out { + Ok(line) => error!("{}", line), + Err(err) => error!("{:?}", err), + }, + )?; + } + + // additional labels + if labels.is_some() { + match kubectl_add_labels_to_namespace( + kubernetes_config, + namespace, + labels.unwrap(), + envs, + ) { + Ok(_) => {}, + Err(e) => return Err(e), + } + }; + + Ok(()) +} + +pub fn kubectl_add_labels_to_namespace

( + kubernetes_config: P, + namespace: &str, + labels: Vec, + envs: Vec<(&str, &str)>, +) -> Result<(), SimpleError> + where + P: AsRef, +{ + if labels.iter().count() > 0 { + return Err(SimpleError::new( + SimpleErrorKind::Other, + Some("No labels were defined, can't set them"), + )); + }; + + if !kubectl_exec_is_namespace_present( + kubernetes_config.as_ref(), + namespace.clone(), + envs.clone(), + ) { + return Err(SimpleError::new( + SimpleErrorKind::Other, + Some(format!{"Can't set labels on namespace {} because it doesn't exists", namespace}), + )); + } + + let mut command_args = Vec::new(); + let mut labels_string = Vec::new(); + command_args.extend(vec!["label", "namespace", namespace, "--overwrite"]); + + for label in labels.iter() { + labels_string.push(format!{"{}={}", label.name, label.value}); + }; + let labels_str = labels_string.iter().map(|x| x.as_ref()).collect::>(); + command_args.extend(labels_str); + + let mut _envs = Vec::with_capacity(envs.len() + 1); + _envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap())); + _envs.extend(envs.clone()); + let _ = kubectl_exec_with_output( - vec!["create", "namespace", namespace], + command_args, _envs, |out| match out { Ok(line) => info!("{}", line), diff --git a/src/cmd/structs.rs b/src/cmd/structs.rs index ed51bf1b..e6cd2210 100644 --- a/src/cmd/structs.rs +++ b/src/cmd/structs.rs @@ -18,6 +18,11 @@ pub struct Labels { pub name: String, } +pub struct LabelsContent { + pub name: String, + pub value: String, +} + #[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)] #[serde(rename_all = "camelCase")] pub struct Spec {