From 9dd315e405d8f09a90e7de9aac0964e8dac3e983 Mon Sep 17 00:00:00 2001 From: Romaric Philogene Date: Thu, 13 Jan 2022 00:38:24 +0100 Subject: [PATCH 01/25] fix: build args injection while building docker image --- src/build_platform/docker.rs | 82 ++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 12 deletions(-) diff --git a/src/build_platform/docker.rs b/src/build_platform/docker.rs index 11cb05df..a40e88e1 100644 --- a/src/build_platform/docker.rs +++ b/src/build_platform/docker.rs @@ -1,5 +1,4 @@ use std::collections::HashSet; -use std::iter::FromIterator; use std::str::Utf8Error; /// Extract ARG value from a Dockerfile content @@ -34,6 +33,8 @@ pub fn extract_dockerfile_args(dockerfile_content: Vec) -> Result, dockerfile_content: Vec, @@ -42,8 +43,14 @@ pub fn match_used_env_var_args( let used_args = extract_dockerfile_args(dockerfile_content)?; // match env var args and dockerfile env vargs - Ok(HashSet::from_iter(env_var_args) - .intersection(&used_args) + let env_var_arg_keys = env_var_args.iter() + .map(|env_var| { + let x = env_var.split("=").collect::>(); + x.get(0).unwrap_or(&"").to_string() + }) + .collect::>(); + + Ok(env_var_arg_keys.intersection(&used_args) .map(|arg| arg.clone()) .collect::>()) } @@ -101,17 +108,17 @@ mod tests { let matched_vars = match_used_env_var_args( vec![ - "foo".to_string(), - "bar".to_string(), - "toto".to_string(), - "x".to_string(), + "foo=abcdvalue".to_string(), + "bar=abcdvalue".to_string(), + "toto=abcdvalue".to_string(), + "x=abcdvalue".to_string(), ], dockerfile.to_vec(), ); assert_eq!(matched_vars.unwrap().len(), 4); - let matched_vars = match_used_env_var_args(vec!["toto".to_string(), "x".to_string()], dockerfile.to_vec()); + let matched_vars = match_used_env_var_args(vec!["toto=abcdvalue".to_string(), "x=abcdvalue".to_string()], dockerfile.to_vec()); assert_eq!(matched_vars.unwrap().len(), 2); @@ -128,14 +135,65 @@ mod tests { let matched_vars = match_used_env_var_args( vec![ - "foo".to_string(), - "bar".to_string(), - "toto".to_string(), - "x".to_string(), + "foo=abcdvalue".to_string(), + "bar=abcdvalue".to_string(), + "toto=abcdvalue".to_string(), + "x=abcdvalue".to_string(), ], dockerfile.to_vec(), ); assert_eq!(matched_vars.unwrap().len(), 0); } + + #[test] + fn test_match_used_env_var_args_2() { + let dockerfile = b" + # This file is a template, and might need editing before it works on your project. + FROM node:16-alpine as build + + WORKDIR /app + COPY . . + + ARG PRISMIC_REPO_NAME + ENV PRISMIC_REPO_NAME $PRISMIC_REPO_NAME + + ARG PRISMIC_API_KEY + ENV PRISMIC_API_KEY $PRISMIC_API_KEY + + ARG PRISMIC_CUSTOM_TYPES_API_TOKEN + ENV PRISMIC_CUSTOM_TYPES_API_TOKEN $PRISMIC_CUSTOM_TYPES_API_TOKEN + + RUN npm install && npm run build + + FROM nginx:latest + COPY --from=build /app/public /usr/share/nginx/html + COPY ./nginx-custom.conf /etc/nginx/conf.d/default.conf + + EXPOSE 80 + CMD [\"nginx\", \"-g\", \"daemon off;\"] + "; + + let res = extract_dockerfile_args(dockerfile.to_vec()); + assert_eq!(res.unwrap().len(), 3); + + let matched_vars = match_used_env_var_args( + vec![ + "PRISMIC_REPO_NAME=abcdvalue".to_string(), + "PRISMIC_API_KEY=abcdvalue".to_string(), + "PRISMIC_CUSTOM_TYPES_API_TOKEN=abcdvalue".to_string(), + ], + dockerfile.to_vec(), + ); + + assert_eq!(matched_vars.unwrap().len(), 3); + + let matched_vars = match_used_env_var_args(vec!["PRISMIC_REPO_NAME=abcdvalue".to_string()], dockerfile.to_vec()); + + assert_eq!(matched_vars.unwrap().len(), 1); + + let matched_vars = match_used_env_var_args(vec![], dockerfile.to_vec()); + + assert_eq!(matched_vars.unwrap().len(), 0); + } } From 3c9ac55a285caf89bd92442f84a7e23d2b43bf8d Mon Sep 17 00:00:00 2001 From: Romaric Philogene Date: Thu, 13 Jan 2022 00:40:13 +0100 Subject: [PATCH 02/25] fix: fmt --- src/build_platform/docker.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/build_platform/docker.rs b/src/build_platform/docker.rs index a40e88e1..d2a156e3 100644 --- a/src/build_platform/docker.rs +++ b/src/build_platform/docker.rs @@ -43,14 +43,16 @@ pub fn match_used_env_var_args( let used_args = extract_dockerfile_args(dockerfile_content)?; // match env var args and dockerfile env vargs - let env_var_arg_keys = env_var_args.iter() + let env_var_arg_keys = env_var_args + .iter() .map(|env_var| { let x = env_var.split("=").collect::>(); x.get(0).unwrap_or(&"").to_string() }) .collect::>(); - Ok(env_var_arg_keys.intersection(&used_args) + Ok(env_var_arg_keys + .intersection(&used_args) .map(|arg| arg.clone()) .collect::>()) } @@ -118,7 +120,10 @@ mod tests { assert_eq!(matched_vars.unwrap().len(), 4); - let matched_vars = match_used_env_var_args(vec!["toto=abcdvalue".to_string(), "x=abcdvalue".to_string()], dockerfile.to_vec()); + let matched_vars = match_used_env_var_args( + vec!["toto=abcdvalue".to_string(), "x=abcdvalue".to_string()], + dockerfile.to_vec(), + ); assert_eq!(matched_vars.unwrap().len(), 2); @@ -188,7 +193,8 @@ mod tests { assert_eq!(matched_vars.unwrap().len(), 3); - let matched_vars = match_used_env_var_args(vec!["PRISMIC_REPO_NAME=abcdvalue".to_string()], dockerfile.to_vec()); + let matched_vars = + match_used_env_var_args(vec!["PRISMIC_REPO_NAME=abcdvalue".to_string()], dockerfile.to_vec()); assert_eq!(matched_vars.unwrap().len(), 1); From c1f431d8c94e93c93b1ffc0d1a00fc930ae20932 Mon Sep 17 00:00:00 2001 From: Romaric Philogene Date: Thu, 13 Jan 2022 00:44:48 +0100 Subject: [PATCH 03/25] chore: use next() instead of get(0) --- src/build_platform/docker.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build_platform/docker.rs b/src/build_platform/docker.rs index d2a156e3..d7f84f67 100644 --- a/src/build_platform/docker.rs +++ b/src/build_platform/docker.rs @@ -46,8 +46,8 @@ pub fn match_used_env_var_args( let env_var_arg_keys = env_var_args .iter() .map(|env_var| { - let x = env_var.split("=").collect::>(); - x.get(0).unwrap_or(&"").to_string() + let x = env_var.split("=").next(); + x.unwrap_or(&"").to_string() }) .collect::>(); From 841d34e077a4d07cba5e84b9827f9e3d673afe73 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Thu, 13 Jan 2022 00:47:18 +0100 Subject: [PATCH 04/25] feat: lower restarts to delete crashlooping pods (#557) This CL lowers down the number of crashlooping pods restarts to be consired as to be deleted from 5 to 3. --- src/cloud_provider/aws/kubernetes/mod.rs | 2 +- src/cloud_provider/digitalocean/kubernetes/mod.rs | 2 +- src/cloud_provider/scaleway/kubernetes/mod.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cloud_provider/aws/kubernetes/mod.rs b/src/cloud_provider/aws/kubernetes/mod.rs index c2de192a..3b5b5327 100644 --- a/src/cloud_provider/aws/kubernetes/mod.rs +++ b/src/cloud_provider/aws/kubernetes/mod.rs @@ -1404,7 +1404,7 @@ impl<'a> Kubernetes for EKS<'a> { if let Err(e) = self.delete_crashlooping_pods( None, None, - Some(10), + Some(3), self.cloud_provider().credentials_environment_variables(), ) { error!( diff --git a/src/cloud_provider/digitalocean/kubernetes/mod.rs b/src/cloud_provider/digitalocean/kubernetes/mod.rs index 5239bb38..34a9330a 100644 --- a/src/cloud_provider/digitalocean/kubernetes/mod.rs +++ b/src/cloud_provider/digitalocean/kubernetes/mod.rs @@ -1229,7 +1229,7 @@ impl<'a> Kubernetes for DOKS<'a> { match self.delete_crashlooping_pods( None, None, - Some(10), + Some(3), self.cloud_provider().credentials_environment_variables(), ) { Ok(..) => {} diff --git a/src/cloud_provider/scaleway/kubernetes/mod.rs b/src/cloud_provider/scaleway/kubernetes/mod.rs index 0d8d7fad..521817e2 100644 --- a/src/cloud_provider/scaleway/kubernetes/mod.rs +++ b/src/cloud_provider/scaleway/kubernetes/mod.rs @@ -1183,7 +1183,7 @@ impl<'a> Kubernetes for Kapsule<'a> { match self.delete_crashlooping_pods( None, None, - Some(10), + Some(3), self.cloud_provider().credentials_environment_variables(), ) { Ok(..) => {} From 1cba48119097c2edede18d1e6a9e66dbb904b8f5 Mon Sep 17 00:00:00 2001 From: Romaric Philogene Date: Thu, 13 Jan 2022 12:30:44 +0100 Subject: [PATCH 05/25] fix: env args build injection --- src/build_platform/docker.rs | 44 +++++++++++++++++------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/build_platform/docker.rs b/src/build_platform/docker.rs index d7f84f67..973f02fe 100644 --- a/src/build_platform/docker.rs +++ b/src/build_platform/docker.rs @@ -45,15 +45,20 @@ pub fn match_used_env_var_args( // match env var args and dockerfile env vargs let env_var_arg_keys = env_var_args .iter() - .map(|env_var| { - let x = env_var.split("=").next(); - x.unwrap_or(&"").to_string() - }) + .map(|env_var| env_var.split("=").next().unwrap_or(&"").to_string()) .collect::>(); - Ok(env_var_arg_keys + let matched_env_args_keys = env_var_arg_keys .intersection(&used_args) .map(|arg| arg.clone()) + .collect::>(); + + Ok(env_var_args + .into_iter() + .filter(|env_var_arg| { + let env_var_arg_key = env_var_arg.split("=").next().unwrap_or(""); + matched_env_args_keys.contains(env_var_arg_key) + }) .collect::>()) } @@ -108,15 +113,16 @@ mod tests { let res = extract_dockerfile_args(dockerfile.to_vec()); assert_eq!(res.unwrap().len(), 4); - let matched_vars = match_used_env_var_args( - vec![ - "foo=abcdvalue".to_string(), - "bar=abcdvalue".to_string(), - "toto=abcdvalue".to_string(), - "x=abcdvalue".to_string(), - ], - dockerfile.to_vec(), - ); + let env_var_args_to_match = vec![ + "foo=abcdvalue".to_string(), + "bar=abcdvalue".to_string(), + "toto=abcdvalue".to_string(), + "x=abcdvalue".to_string(), + ]; + + let matched_vars = match_used_env_var_args(env_var_args_to_match.clone(), dockerfile.to_vec()); + + assert_eq!(matched_vars.clone().unwrap(), env_var_args_to_match.clone()); assert_eq!(matched_vars.unwrap().len(), 4); @@ -138,15 +144,7 @@ mod tests { RUN ls -lh "; - let matched_vars = match_used_env_var_args( - vec![ - "foo=abcdvalue".to_string(), - "bar=abcdvalue".to_string(), - "toto=abcdvalue".to_string(), - "x=abcdvalue".to_string(), - ], - dockerfile.to_vec(), - ); + let matched_vars = match_used_env_var_args(env_var_args_to_match.clone(), dockerfile.to_vec()); assert_eq!(matched_vars.unwrap().len(), 0); } From 6732aee99668de8794e0b0f9798018c2a6aef1dd Mon Sep 17 00:00:00 2001 From: Benjamin Date: Mon, 17 Jan 2022 19:30:21 +0100 Subject: [PATCH 06/25] fix: remove unused derive (#563) --- src/cloud_provider/aws/router.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cloud_provider/aws/router.rs b/src/cloud_provider/aws/router.rs index 8c72dd68..f9792144 100644 --- a/src/cloud_provider/aws/router.rs +++ b/src/cloud_provider/aws/router.rs @@ -382,7 +382,6 @@ impl Create for Router { } impl Pause for Router { - #[named] fn on_pause(&self, _target: &DeploymentTarget) -> Result<(), EngineError> { info!("AWS.router.on_pause() called for {}, doing nothing", self.name()); Ok(()) From aeead3b43c28b4ab6463818c137ff33669e7d1d1 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Tue, 18 Jan 2022 17:04:40 +0100 Subject: [PATCH 07/25] fix: service to be created even if private port (#562) As of today if port is set as private on an app / db, then no service will be created for it in k8s. This should not be the case and a service should be created even if app or db is flagged as private allowing other internal apps to connect to it. --- lib/aws/charts/q-application/templates/service.j2.yaml | 4 +--- .../charts/q-application/templates/service.j2.yaml | 4 +--- lib/scaleway/charts/q-application/templates/service.j2.yaml | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/aws/charts/q-application/templates/service.j2.yaml b/lib/aws/charts/q-application/templates/service.j2.yaml index 71e389ae..66273f44 100644 --- a/lib/aws/charts/q-application/templates/service.j2.yaml +++ b/lib/aws/charts/q-application/templates/service.j2.yaml @@ -1,4 +1,3 @@ -{%- if is_private_port %} apiVersion: v1 kind: Service metadata: @@ -22,5 +21,4 @@ spec: ownerId: {{ owner_id }} appId: {{ id }} app: {{ sanitized_name }} - envId: {{ environment_id }} -{% endif %} + envId: {{ environment_id }} \ No newline at end of file diff --git a/lib/digitalocean/charts/q-application/templates/service.j2.yaml b/lib/digitalocean/charts/q-application/templates/service.j2.yaml index 71e389ae..66273f44 100644 --- a/lib/digitalocean/charts/q-application/templates/service.j2.yaml +++ b/lib/digitalocean/charts/q-application/templates/service.j2.yaml @@ -1,4 +1,3 @@ -{%- if is_private_port %} apiVersion: v1 kind: Service metadata: @@ -22,5 +21,4 @@ spec: ownerId: {{ owner_id }} appId: {{ id }} app: {{ sanitized_name }} - envId: {{ environment_id }} -{% endif %} + envId: {{ environment_id }} \ No newline at end of file diff --git a/lib/scaleway/charts/q-application/templates/service.j2.yaml b/lib/scaleway/charts/q-application/templates/service.j2.yaml index 71e389ae..66273f44 100644 --- a/lib/scaleway/charts/q-application/templates/service.j2.yaml +++ b/lib/scaleway/charts/q-application/templates/service.j2.yaml @@ -1,4 +1,3 @@ -{%- if is_private_port %} apiVersion: v1 kind: Service metadata: @@ -22,5 +21,4 @@ spec: ownerId: {{ owner_id }} appId: {{ id }} app: {{ sanitized_name }} - envId: {{ environment_id }} -{% endif %} + envId: {{ environment_id }} \ No newline at end of file From 09a1163410d8942e8b0e623d204ff746a6f185ad Mon Sep 17 00:00:00 2001 From: MacLikorne Date: Tue, 18 Jan 2022 17:42:41 +0100 Subject: [PATCH 08/25] feat: increase kubernetes version for tests (#560) --- test_utilities/src/aws.rs | 2 +- test_utilities/src/scaleway.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test_utilities/src/aws.rs b/test_utilities/src/aws.rs index 9f6e274c..3ec5f99e 100644 --- a/test_utilities/src/aws.rs +++ b/test_utilities/src/aws.rs @@ -23,7 +23,7 @@ use crate::utilities::{build_platform_local_docker, FuncTestsSecrets}; pub const AWS_REGION_FOR_S3: &str = "eu-west-3"; pub const AWS_TEST_REGION: &str = "eu-west-3"; pub const AWS_KUBERNETES_MAJOR_VERSION: u8 = 1; -pub const AWS_KUBERNETES_MINOR_VERSION: u8 = 19; +pub const AWS_KUBERNETES_MINOR_VERSION: u8 = 20; pub const AWS_KUBERNETES_VERSION: &'static str = formatcp!("{}.{}", AWS_KUBERNETES_MAJOR_VERSION, AWS_KUBERNETES_MINOR_VERSION); pub const AWS_DATABASE_INSTANCE_TYPE: &str = "db.t3.micro"; diff --git a/test_utilities/src/scaleway.rs b/test_utilities/src/scaleway.rs index 84cd49af..ba67af21 100644 --- a/test_utilities/src/scaleway.rs +++ b/test_utilities/src/scaleway.rs @@ -21,7 +21,7 @@ use tracing::error; pub const SCW_TEST_ZONE: ScwZone = ScwZone::Paris2; pub const SCW_KUBERNETES_MAJOR_VERSION: u8 = 1; -pub const SCW_KUBERNETES_MINOR_VERSION: u8 = 19; +pub const SCW_KUBERNETES_MINOR_VERSION: u8 = 20; pub const SCW_KUBERNETES_VERSION: &'static str = formatcp!("{}.{}", SCW_KUBERNETES_MAJOR_VERSION, SCW_KUBERNETES_MINOR_VERSION); pub const SCW_MANAGED_DATABASE_INSTANCE_TYPE: &str = "db-dev-s"; From cbc026c59ea2856ebe9b322440519ac2c776a291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Er=C3=A8be=20-=20Romain=20Gerard?= Date: Wed, 19 Jan 2022 08:25:15 +0100 Subject: [PATCH 09/25] Fix rbac for shell-agent (#565) + Some kube lib (i.e: kube-rs) is doing a get before doing a create --- .../charts/qovery-shell-agent/templates/clusterrole.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/common/bootstrap/charts/qovery-shell-agent/templates/clusterrole.yaml b/lib/common/bootstrap/charts/qovery-shell-agent/templates/clusterrole.yaml index cd6850ea..9fdd02e9 100644 --- a/lib/common/bootstrap/charts/qovery-shell-agent/templates/clusterrole.yaml +++ b/lib/common/bootstrap/charts/qovery-shell-agent/templates/clusterrole.yaml @@ -19,5 +19,7 @@ rules: resources: - pods/exec verbs: + - get + - list - create {{- end }} From f93b92c6052ceef1cc269843698d0c01a57f1653 Mon Sep 17 00:00:00 2001 From: MacLikorne Date: Wed, 19 Jan 2022 10:23:35 +0100 Subject: [PATCH 10/25] fix: increase tests tiemout (#566) --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7c333c95..b699374e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -49,7 +49,7 @@ jobs: steps: - uses: actions/checkout@v1 - name: Run selected functional tests - timeout-minutes: 120 + timeout-minutes: 180 env: GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }} GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} @@ -68,7 +68,7 @@ jobs: steps: - uses: actions/checkout@v1 - name: Only validate PR if all tests have been requested - timeout-minutes: 120 + timeout-minutes: 180 env: GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }} GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} From 8baf661838ce724105be88ed9e6a8210e48c0ce1 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Thu, 20 Jan 2022 11:18:10 +0100 Subject: [PATCH 11/25] fix: handle services without ports defined (#568) --- lib/aws/charts/q-application/templates/service.j2.yaml | 4 +++- .../charts/q-application/templates/service.j2.yaml | 4 +++- lib/scaleway/charts/q-application/templates/service.j2.yaml | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/aws/charts/q-application/templates/service.j2.yaml b/lib/aws/charts/q-application/templates/service.j2.yaml index 66273f44..bb258f90 100644 --- a/lib/aws/charts/q-application/templates/service.j2.yaml +++ b/lib/aws/charts/q-application/templates/service.j2.yaml @@ -1,3 +1,4 @@ +{%- if (ports is defined) and ports %} apiVersion: v1 kind: Service metadata: @@ -21,4 +22,5 @@ spec: ownerId: {{ owner_id }} appId: {{ id }} app: {{ sanitized_name }} - envId: {{ environment_id }} \ No newline at end of file + envId: {{ environment_id }} +{%- endif %} diff --git a/lib/digitalocean/charts/q-application/templates/service.j2.yaml b/lib/digitalocean/charts/q-application/templates/service.j2.yaml index 66273f44..bb258f90 100644 --- a/lib/digitalocean/charts/q-application/templates/service.j2.yaml +++ b/lib/digitalocean/charts/q-application/templates/service.j2.yaml @@ -1,3 +1,4 @@ +{%- if (ports is defined) and ports %} apiVersion: v1 kind: Service metadata: @@ -21,4 +22,5 @@ spec: ownerId: {{ owner_id }} appId: {{ id }} app: {{ sanitized_name }} - envId: {{ environment_id }} \ No newline at end of file + envId: {{ environment_id }} +{%- endif %} diff --git a/lib/scaleway/charts/q-application/templates/service.j2.yaml b/lib/scaleway/charts/q-application/templates/service.j2.yaml index 66273f44..22ffc5ba 100644 --- a/lib/scaleway/charts/q-application/templates/service.j2.yaml +++ b/lib/scaleway/charts/q-application/templates/service.j2.yaml @@ -1,3 +1,4 @@ +{%- if (ports is defined) and ports %} apiVersion: v1 kind: Service metadata: @@ -21,4 +22,5 @@ spec: ownerId: {{ owner_id }} appId: {{ id }} app: {{ sanitized_name }} - envId: {{ environment_id }} \ No newline at end of file + envId: {{ environment_id }} +{%- endif %} \ No newline at end of file From 4d788d4831360a975923a3d291f181f61fe009df Mon Sep 17 00:00:00 2001 From: Benjamin Date: Fri, 21 Jan 2022 10:16:23 +0100 Subject: [PATCH 12/25] conf: increase default helm timeout (#570) This CL increases default helm chart timeout from 180 seconds to 300 seconds. --- src/cloud_provider/helm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cloud_provider/helm.rs b/src/cloud_provider/helm.rs index 33e5ccf5..01fc3775 100644 --- a/src/cloud_provider/helm.rs +++ b/src/cloud_provider/helm.rs @@ -130,7 +130,7 @@ impl Default for ChartInfo { atomic: true, force_upgrade: false, last_breaking_version_requiring_restart: None, - timeout_in_seconds: 180, + timeout_in_seconds: 300, dry_run: false, wait: true, values: Vec::new(), From 81e3f67a4b3c31a7f8ef3f5d81083337b2b1d83b Mon Sep 17 00:00:00 2001 From: enzo Date: Mon, 17 Jan 2022 17:03:04 +0100 Subject: [PATCH 13/25] feat: upgrade metrics server --- .../chart_values/metrics-server.yaml | 8 +++ .../charts/metrics-server/Chart.lock | 6 +- .../charts/metrics-server/Chart.yaml | 4 +- .../bootstrap/charts/metrics-server/README.md | 71 +++++++++++-------- .../metrics-server/charts/common/Chart.yaml | 6 +- .../metrics-server/charts/common/README.md | 26 +++---- .../charts/common/templates/_capabilities.tpl | 24 ++++++- .../charts/common/templates/_names.tpl | 20 ++++++ .../templates/validations/_cassandra.tpl | 2 +- .../common/templates/validations/_mariadb.tpl | 2 +- .../common/templates/validations/_mongodb.tpl | 2 +- .../templates/validations/_postgresql.tpl | 4 +- .../common/templates/validations/_redis.tpl | 2 +- .../templates/auth-delegator-crb.yaml | 6 ++ .../templates/cluster-role.yaml | 6 ++ .../metrics-server/templates/deployment.yaml | 7 ++ .../metrics-server/templates/extra-list.yaml | 4 ++ .../templates/metrics-api-service.yaml | 11 ++- .../templates/metrics-server-crb.yaml | 6 ++ .../charts/metrics-server/templates/pdb.yaml | 8 ++- .../templates/role-binding.yaml | 6 ++ .../templates/serviceaccount.yaml | 6 ++ .../charts/metrics-server/templates/svc.yaml | 14 +++- .../charts/metrics-server/values.yaml | 24 ++++++- .../chart_values/metrics-server.yaml | 8 +++ lib/helm-freeze.yaml | 3 +- .../aws/kubernetes/helm_charts.rs | 1 + .../digitalocean/kubernetes/helm_charts.rs | 9 +-- 28 files changed, 222 insertions(+), 74 deletions(-) create mode 100644 lib/aws/bootstrap/chart_values/metrics-server.yaml create mode 100644 lib/common/bootstrap/charts/metrics-server/templates/extra-list.yaml create mode 100644 lib/digitalocean/bootstrap/chart_values/metrics-server.yaml diff --git a/lib/aws/bootstrap/chart_values/metrics-server.yaml b/lib/aws/bootstrap/chart_values/metrics-server.yaml new file mode 100644 index 00000000..5457a1f4 --- /dev/null +++ b/lib/aws/bootstrap/chart_values/metrics-server.yaml @@ -0,0 +1,8 @@ +extraArgs: + kubelet-preferred-address-types: InternalIP,ExternalIP,Hostname + kubelet-use-node-status-port: true + metric-resolution: 15s + cert-dir: /tmp + +apiService: + create: true \ No newline at end of file diff --git a/lib/common/bootstrap/charts/metrics-server/Chart.lock b/lib/common/bootstrap/charts/metrics-server/Chart.lock index c7befa62..3e0d59d7 100644 --- a/lib/common/bootstrap/charts/metrics-server/Chart.lock +++ b/lib/common/bootstrap/charts/metrics-server/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: common repository: https://charts.bitnami.com/bitnami - version: 1.7.1 -digest: sha256:d05549cd2eb5b99a49655221b8efd09927cc48daca3fa9f19af0257a11e5260f -generated: "2021-07-31T15:03:06.939238802Z" + version: 1.10.3 +digest: sha256:710e8247ae70ea63a2fb2fde4320511ff28c7b5c7a738861427f104a7718bdf4 +generated: "2021-12-16T23:00:17.006701838Z" diff --git a/lib/common/bootstrap/charts/metrics-server/Chart.yaml b/lib/common/bootstrap/charts/metrics-server/Chart.yaml index 6ae8aa97..17ff52e9 100644 --- a/lib/common/bootstrap/charts/metrics-server/Chart.yaml +++ b/lib/common/bootstrap/charts/metrics-server/Chart.yaml @@ -1,7 +1,7 @@ annotations: category: Analytics apiVersion: v2 -appVersion: 0.5.0 +appVersion: 0.5.2 dependencies: - name: common repository: https://charts.bitnami.com/bitnami @@ -23,4 +23,4 @@ name: metrics-server sources: - https://github.com/bitnami/bitnami-docker-metrics-server - https://github.com/kubernetes-incubator/metrics-server -version: 5.9.2 +version: 5.10.13 diff --git a/lib/common/bootstrap/charts/metrics-server/README.md b/lib/common/bootstrap/charts/metrics-server/README.md index 6120fa07..365a864c 100644 --- a/lib/common/bootstrap/charts/metrics-server/README.md +++ b/lib/common/bootstrap/charts/metrics-server/README.md @@ -11,7 +11,7 @@ $ helm install my-release bitnami/metrics-server ## Introduction -This chart bootstraps a [Metrics Server](https://github.com/bitnami/bitnami-docker-metrics-server) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. +This chart bootstraps a [Metrics Server](https://github.com/bitnami/bitnami-docker-metrics-server) deployment on a [Kubernetes](https://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. Bitnami charts can be used with [Kubeapps](https://kubeapps.com/) for deployment and management of Helm Charts in clusters. This Helm chart has been tested on top of [Bitnami Kubernetes Production Runtime](https://kubeprod.io/) (BKPR). Deploy BKPR to get automated TLS certificates, logging and monitoring for your applications. @@ -55,10 +55,13 @@ The command removes all the Kubernetes components associated with the chart and ### Common parameters -| Name | Description | Value | -| ------------------ | -------------------------------------------------------------------------------------------- | ----- | -| `nameOverride` | String to partially override common.names.fullname template (will maintain the release name) | `""` | -| `fullnameOverride` | String to fully override common.names.fullname template | `""` | +| Name | Description | Value | +| ------------------- | -------------------------------------------------------------------------------------------- | ----- | +| `nameOverride` | String to partially override common.names.fullname template (will maintain the release name) | `""` | +| `fullnameOverride` | String to fully override common.names.fullname template | `""` | +| `commonLabels` | Add labels to all the deployed resources | `{}` | +| `commonAnnotations` | Add annotations to all the deployed resources | `{}` | +| `extraDeploy` | Array of extra objects to deploy with the release | `[]` | ### Metrics Server parameters @@ -67,7 +70,7 @@ The command removes all the Kubernetes components associated with the chart and | ------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------ | | `image.registry` | Metrics Server image registry | `docker.io` | | `image.repository` | Metrics Server image repository | `bitnami/metrics-server` | -| `image.tag` | Metrics Server image tag (immutable tags are recommended) | `0.5.0-debian-10-r32` | +| `image.tag` | Metrics Server image tag (immutable tags are recommended) | `0.5.2-debian-10-r0` | | `image.pullPolicy` | Metrics Server image pull policy | `IfNotPresent` | | `image.pullSecrets` | Metrics Server image pull secrets | `[]` | | `hostAliases` | Add deployment host aliases | `[]` | @@ -78,9 +81,12 @@ The command removes all the Kubernetes components associated with the chart and | `serviceAccount.name` | The name of the ServiceAccount to create | `""` | | `serviceAccount.automountServiceAccountToken` | Automount API credentials for a service account | `true` | | `apiService.create` | Specifies whether the v1beta1.metrics.k8s.io API service should be created. You can check if it is needed with `kubectl get --raw "/apis/metrics.k8s.io/v1beta1/nodes"`. | `false` | +| `apiService.insecureSkipTLSVerify` | Specifies whether to skip self-verifying self-signed TLS certificates. Set to "false" if you are providing your own certificates. | `true` | +| `apiService.caBundle` | A base64-encoded string of concatenated certificates for the CA chain for the APIService. | `""` | | `securePort` | Port where metrics-server will be running | `8443` | | `hostNetwork` | Enable hostNetwork mode | `false` | -| `command` | Override default container command (useful when using custom images) | `[]` | +| `dnsPolicy` | Default dnsPolicy setting | `ClusterFirst` | +| `command` | Override default container command (useful when using custom images) | `["metrics-server"]` | | `extraArgs` | Extra arguments to pass to metrics-server on start up | `{}` | | `podLabels` | Pod labels | `{}` | | `podAnnotations` | Pod annotations | `{}` | @@ -147,21 +153,25 @@ $ helm install my-release -f values.yaml bitnami/metrics-server ## Configuration and installation details -### [Rolling VS Immutable tags](https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/) +### [Rolling vs Immutable tags](https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/) It is strongly recommended to use immutable tags in a production environment. This ensures your deployment does not change automatically if the same tag is updated with a different image. Bitnami will release a new chart updating its containers if a new version of the main container, significant changes, or critical vulnerabilities exist. -### Enable security for Metrics Server. Configuring RBAC +### Enable RBAC security -In order to enable Role-based access control for Metrics Servier you can use the following parameter: `rbac.create=true` +In order to enable Role-Based Access Control (RBAC) for Metrics Server, use the following parameter: `rbac.create=true`. -### Setting Pod's affinity +### Configure certificates -This chart allows you to set your custom affinity using the `affinity` parameter. Find more information about Pod's affinity in the [kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). +If you are providing your own certificates for the API Service, set `insecureSkipTLSVerify` to `"false"`, and provide a `caBundle` consisting of the base64-encoded certificate chain. -As an alternative, you can use of the preset configurations for pod affinity, pod anti-affinity, and node affinity available at the [bitnami/common](https://github.com/bitnami/charts/tree/master/bitnami/common#affinities) chart. To do so, set the `podAffinityPreset`, `podAntiAffinityPreset`, or `nodeAffinityPreset` parameters. +### Set Pod affinity + +This chart allows you to set custom Pod affinity using the `affinity` parameter. Find more information about Pod affinity in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). + +As an alternative, you can use one of the preset configurations for pod affinity, pod anti-affinity, and node affinity available at the [bitnami/common](https://github.com/bitnami/charts/tree/master/bitnami/common#affinities) chart. To do so, set the `podAffinityPreset`, `podAntiAffinityPreset`, or `nodeAffinityPreset` parameters. ## Troubleshooting @@ -175,24 +185,9 @@ This version introduces `bitnami/common`, a [library chart](https://helm.sh/docs ### To 5.0.0 -[On November 13, 2020, Helm v2 support was formally finished](https://github.com/helm/charts#status-of-the-project), this major version is the result of the required changes applied to the Helm Chart to be able to incorporate the different features added in Helm v3 and to be consistent with the Helm project itself regarding the Helm v2 EOL. +[On November 13, 2020, Helm v2 support formally ended](https://github.com/helm/charts#status-of-the-project). This major version is the result of the required changes applied to the Helm Chart to be able to incorporate the different features added in Helm v3 and to be consistent with the Helm project itself regarding the Helm v2 EOL. -**What changes were introduced in this major version?** - -- Previous versions of this Helm Chart use `apiVersion: v1` (installable by both Helm 2 and 3), this Helm Chart was updated to `apiVersion: v2` (installable by Helm 3 only). [Here](https://helm.sh/docs/topics/charts/#the-apiversion-field) you can find more information about the `apiVersion` field. -- The different fields present in the *Chart.yaml* file has been ordered alphabetically in a homogeneous way for all the Bitnami Helm Charts - -**Considerations when upgrading to this version** - -- If you want to upgrade to this version from a previous one installed with Helm v3, you shouldn't face any issues -- If you want to upgrade to this version using Helm v2, this scenario is not supported as this version doesn't support Helm v2 anymore -- If you installed the previous version with Helm v2 and wants to upgrade to this version with Helm v3, please refer to the [official Helm documentation](https://helm.sh/docs/topics/v2_v3_migration/#migration-use-cases) about migrating from Helm v2 to v3 - -**Useful links** - -- https://docs.bitnami.com/tutorials/resolve-helm2-helm3-post-migration-issues/ -- https://helm.sh/docs/topics/v2_v3_migration/ -- https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/ +[Learn more about this change and related upgrade considerations](https://docs.bitnami.com/kubernetes/infrastructure/metrics-server/administration/upgrade-helm3/). ### To 4.0.0 @@ -212,3 +207,19 @@ Use the workaround below to upgrade from versions previous to 2.0.0. The followi ```console $ kubectl patch deployment metrics-server --type=json -p='[{"op": "remove", "path": "/spec/selector/matchLabels/chart"}]' ``` + +## License + +Copyright © 2022 Bitnami + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/lib/common/bootstrap/charts/metrics-server/charts/common/Chart.yaml b/lib/common/bootstrap/charts/metrics-server/charts/common/Chart.yaml index 81b0a15f..cf934aad 100644 --- a/lib/common/bootstrap/charts/metrics-server/charts/common/Chart.yaml +++ b/lib/common/bootstrap/charts/metrics-server/charts/common/Chart.yaml @@ -1,7 +1,7 @@ annotations: category: Infrastructure apiVersion: v2 -appVersion: 1.7.1 +appVersion: 1.10.0 description: A Library Helm Chart for grouping common logic between bitnami charts. This chart is not deployable by itself. home: https://github.com/bitnami/charts/tree/master/bitnami/common @@ -18,6 +18,6 @@ maintainers: name: common sources: - https://github.com/bitnami/charts -- http://www.bitnami.com/ +- https://www.bitnami.com/ type: library -version: 1.7.1 +version: 1.10.3 diff --git a/lib/common/bootstrap/charts/metrics-server/charts/common/README.md b/lib/common/bootstrap/charts/metrics-server/charts/common/README.md index cd4afc60..cbbc31d9 100644 --- a/lib/common/bootstrap/charts/metrics-server/charts/common/README.md +++ b/lib/common/bootstrap/charts/metrics-server/charts/common/README.md @@ -50,16 +50,18 @@ The following table lists the helpers available in the library which are scoped ### Capabilities -| Helper identifier | Description | Expected Input | -|----------------------------------------------|------------------------------------------------------------------------------------------------|-------------------| -| `common.capabilities.kubeVersion` | Return the target Kubernetes version (using client default if .Values.kubeVersion is not set). | `.` Chart context | -| `common.capabilities.deployment.apiVersion` | Return the appropriate apiVersion for deployment. | `.` Chart context | -| `common.capabilities.statefulset.apiVersion` | Return the appropriate apiVersion for statefulset. | `.` Chart context | -| `common.capabilities.ingress.apiVersion` | Return the appropriate apiVersion for ingress. | `.` Chart context | -| `common.capabilities.rbac.apiVersion` | Return the appropriate apiVersion for RBAC resources. | `.` Chart context | -| `common.capabilities.crd.apiVersion` | Return the appropriate apiVersion for CRDs. | `.` Chart context | -| `common.capabilities.policy.apiVersion` | Return the appropriate apiVersion for policy | `.` Chart context | -| `common.capabilities.supportsHelmVersion` | Returns true if the used Helm version is 3.3+ | `.` Chart context | +| Helper identifier | Description | Expected Input | +|------------------------------------------------|------------------------------------------------------------------------------------------------|-------------------| +| `common.capabilities.kubeVersion` | Return the target Kubernetes version (using client default if .Values.kubeVersion is not set). | `.` Chart context | +| `common.capabilities.cronjob.apiVersion` | Return the appropriate apiVersion for cronjob. | `.` Chart context | +| `common.capabilities.deployment.apiVersion` | Return the appropriate apiVersion for deployment. | `.` Chart context | +| `common.capabilities.statefulset.apiVersion` | Return the appropriate apiVersion for statefulset. | `.` Chart context | +| `common.capabilities.ingress.apiVersion` | Return the appropriate apiVersion for ingress. | `.` Chart context | +| `common.capabilities.rbac.apiVersion` | Return the appropriate apiVersion for RBAC resources. | `.` Chart context | +| `common.capabilities.crd.apiVersion` | Return the appropriate apiVersion for CRDs. | `.` Chart context | +| `common.capabilities.policy.apiVersion` | Return the appropriate apiVersion for podsecuritypolicy. | `.` Chart context | +| `common.capabilities.networkPolicy.apiVersion` | Return the appropriate apiVersion for networkpolicy. | `.` Chart context | +| `common.capabilities.supportsHelmVersion` | Returns true if the used Helm version is 3.3+ | `.` Chart context | ### Errors @@ -92,7 +94,7 @@ The following table lists the helpers available in the library which are scoped ### Names -| Helper identifier | Description | Expected Inpput | +| Helper identifier | Description | Expected Input | |-------------------------|------------------------------------------------------------|-------------------| | `common.names.name` | Expand the name of the chart or use `.Values.nameOverride` | `.` Chart context | | `common.names.fullname` | Create a default fully qualified app name. | `.` Chart context | @@ -111,7 +113,7 @@ The following table lists the helpers available in the library which are scoped | Helper identifier | Description | Expected Input | |-------------------------------|---------------------------------------|---------------------------------------------------------------------------------------------------------------------| -| `common.affinities.node.soft` | Return a soft nodeAffinity definition | `dict "persistence" .Values.path.to.the.persistence "global" $`, see [Persistence](#persistence) for the structure. | +| `common.storage.class` | Return the proper Storage Class | `dict "persistence" .Values.path.to.the.persistence "global" $`, see [Persistence](#persistence) for the structure. | ### TplValues diff --git a/lib/common/bootstrap/charts/metrics-server/charts/common/templates/_capabilities.tpl b/lib/common/bootstrap/charts/metrics-server/charts/common/templates/_capabilities.tpl index b65c73b4..b94212bb 100644 --- a/lib/common/bootstrap/charts/metrics-server/charts/common/templates/_capabilities.tpl +++ b/lib/common/bootstrap/charts/metrics-server/charts/common/templates/_capabilities.tpl @@ -16,7 +16,7 @@ Return the target Kubernetes version {{- end -}} {{/* -Return the appropriate apiVersion for policy. +Return the appropriate apiVersion for poddisruptionbudget. */}} {{- define "common.capabilities.policy.apiVersion" -}} {{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} @@ -26,6 +26,28 @@ Return the appropriate apiVersion for policy. {{- end -}} {{- end -}} +{{/* +Return the appropriate apiVersion for networkpolicy. +*/}} +{{- define "common.capabilities.networkPolicy.apiVersion" -}} +{{- if semverCompare "<1.7-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for cronjob. +*/}} +{{- define "common.capabilities.cronjob.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "batch/v1beta1" -}} +{{- else -}} +{{- print "batch/v1" -}} +{{- end -}} +{{- end -}} + {{/* Return the appropriate apiVersion for deployment. */}} diff --git a/lib/common/bootstrap/charts/metrics-server/charts/common/templates/_names.tpl b/lib/common/bootstrap/charts/metrics-server/charts/common/templates/_names.tpl index adf2a74f..cf032317 100644 --- a/lib/common/bootstrap/charts/metrics-server/charts/common/templates/_names.tpl +++ b/lib/common/bootstrap/charts/metrics-server/charts/common/templates/_names.tpl @@ -30,3 +30,23 @@ If release name contains chart name it will be used as a full name. {{- end -}} {{- end -}} {{- end -}} + +{{/* +Create a default fully qualified dependency name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +Usage: +{{ include "common.names.dependency.fullname" (dict "chartName" "dependency-chart-name" "chartValues" .Values.dependency-chart "context" $) }} +*/}} +{{- define "common.names.dependency.fullname" -}} +{{- if .chartValues.fullnameOverride -}} +{{- .chartValues.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .chartName .chartValues.nameOverride -}} +{{- if contains $name .context.Release.Name -}} +{{- .context.Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .context.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_cassandra.tpl b/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_cassandra.tpl index 8679ddff..ded1ae3b 100644 --- a/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_cassandra.tpl +++ b/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_cassandra.tpl @@ -14,7 +14,7 @@ Params: {{- $dbUserPrefix := include "common.cassandra.values.key.dbUser" . -}} {{- $valueKeyPassword := printf "%s.password" $dbUserPrefix -}} - {{- if and (not $existingSecret) (eq $enabled "true") -}} + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} {{- $requiredPasswords := list -}} {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "cassandra-password" -}} diff --git a/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_mariadb.tpl b/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_mariadb.tpl index bb5ed725..b6906ff7 100644 --- a/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_mariadb.tpl +++ b/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_mariadb.tpl @@ -18,7 +18,7 @@ Params: {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} - {{- if and (not $existingSecret) (eq $enabled "true") -}} + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} {{- $requiredPasswords := list -}} {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mariadb-root-password" -}} diff --git a/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_mongodb.tpl b/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_mongodb.tpl index 1e5bba98..a071ea4d 100644 --- a/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_mongodb.tpl +++ b/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_mongodb.tpl @@ -22,7 +22,7 @@ Params: {{- $authEnabled := include "common.utils.getValueFromKey" (dict "key" $valueKeyAuthEnabled "context" .context) -}} - {{- if and (not $existingSecret) (eq $enabled "true") (eq $authEnabled "true") -}} + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") (eq $authEnabled "true") -}} {{- $requiredPasswords := list -}} {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mongodb-root-password" -}} diff --git a/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_postgresql.tpl b/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_postgresql.tpl index 992bcd39..164ec0d0 100644 --- a/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_postgresql.tpl +++ b/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_postgresql.tpl @@ -13,10 +13,8 @@ Params: {{- $enabled := include "common.postgresql.values.enabled" . -}} {{- $valueKeyPostgresqlPassword := include "common.postgresql.values.key.postgressPassword" . -}} {{- $valueKeyPostgresqlReplicationEnabled := include "common.postgresql.values.key.replicationPassword" . -}} - - {{- if and (not $existingSecret) (eq $enabled "true") -}} + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} {{- $requiredPasswords := list -}} - {{- $requiredPostgresqlPassword := dict "valueKey" $valueKeyPostgresqlPassword "secret" .secret "field" "postgresql-password" -}} {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlPassword -}} diff --git a/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_redis.tpl b/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_redis.tpl index 18d9813c..5d72959b 100644 --- a/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_redis.tpl +++ b/lib/common/bootstrap/charts/metrics-server/charts/common/templates/validations/_redis.tpl @@ -20,7 +20,7 @@ Params: {{- $valueKeyRedisPassword := ternary (printf "%s%s" $valueKeyPrefix "auth.password") (printf "%s%s" $valueKeyPrefix "password") (eq $standarizedVersion "true") }} {{- $valueKeyRedisUseAuth := ternary (printf "%s%s" $valueKeyPrefix "auth.enabled") (printf "%s%s" $valueKeyPrefix "usePassword") (eq $standarizedVersion "true") }} - {{- if and (not $existingSecretValue) (eq $enabled "true") -}} + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} {{- $requiredPasswords := list -}} {{- $useAuth := include "common.utils.getValueFromKey" (dict "key" $valueKeyRedisUseAuth "context" .context) -}} diff --git a/lib/common/bootstrap/charts/metrics-server/templates/auth-delegator-crb.yaml b/lib/common/bootstrap/charts/metrics-server/templates/auth-delegator-crb.yaml index 56c8a853..32928e0a 100644 --- a/lib/common/bootstrap/charts/metrics-server/templates/auth-delegator-crb.yaml +++ b/lib/common/bootstrap/charts/metrics-server/templates/auth-delegator-crb.yaml @@ -5,6 +5,12 @@ metadata: name: {{ include "common.names.fullname" . }}-auth-delegator namespace: kube-system labels: {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/lib/common/bootstrap/charts/metrics-server/templates/cluster-role.yaml b/lib/common/bootstrap/charts/metrics-server/templates/cluster-role.yaml index 14c635de..b5839920 100644 --- a/lib/common/bootstrap/charts/metrics-server/templates/cluster-role.yaml +++ b/lib/common/bootstrap/charts/metrics-server/templates/cluster-role.yaml @@ -4,6 +4,12 @@ kind: ClusterRole metadata: name: {{ include "common.names.fullname" . }} labels: {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} rules: - apiGroups: - "" diff --git a/lib/common/bootstrap/charts/metrics-server/templates/deployment.yaml b/lib/common/bootstrap/charts/metrics-server/templates/deployment.yaml index 4d9d7b87..a5267ba9 100644 --- a/lib/common/bootstrap/charts/metrics-server/templates/deployment.yaml +++ b/lib/common/bootstrap/charts/metrics-server/templates/deployment.yaml @@ -3,6 +3,12 @@ kind: Deployment metadata: name: {{ template "common.names.fullname" . }} labels: {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} spec: replicas: {{ .Values.replicas }} selector: @@ -51,6 +57,7 @@ spec: {{- if .Values.hostNetwork }} hostNetwork: true {{- end }} + dnsPolicy: {{ .Values.dnsPolicy }} containers: {{- if .Values.extraContainers }} {{- include "common.tplvalues.render" (dict "value" .Values.extraContainers "context" $) | nindent 8 }} diff --git a/lib/common/bootstrap/charts/metrics-server/templates/extra-list.yaml b/lib/common/bootstrap/charts/metrics-server/templates/extra-list.yaml new file mode 100644 index 00000000..9ac65f9e --- /dev/null +++ b/lib/common/bootstrap/charts/metrics-server/templates/extra-list.yaml @@ -0,0 +1,4 @@ +{{- range .Values.extraDeploy }} +--- +{{ include "common.tplvalues.render" (dict "value" . "context" $) }} +{{- end }} diff --git a/lib/common/bootstrap/charts/metrics-server/templates/metrics-api-service.yaml b/lib/common/bootstrap/charts/metrics-server/templates/metrics-api-service.yaml index 1ad4d77b..e0a37647 100644 --- a/lib/common/bootstrap/charts/metrics-server/templates/metrics-api-service.yaml +++ b/lib/common/bootstrap/charts/metrics-server/templates/metrics-api-service.yaml @@ -8,13 +8,22 @@ kind: APIService metadata: name: v1beta1.metrics.k8s.io labels: {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} spec: service: name: {{ template "common.names.fullname" . }} namespace: {{ .Release.Namespace }} group: metrics.k8s.io version: v1beta1 - insecureSkipTLSVerify: true + insecureSkipTLSVerify: {{ .Values.apiService.insecureSkipTLSVerify | default true }} + {{- if .Values.apiService.caBundle }} + caBundle: {{ .Values.apiService.caBundle }} + {{- end }} groupPriorityMinimum: 100 versionPriority: 100 {{- end -}} diff --git a/lib/common/bootstrap/charts/metrics-server/templates/metrics-server-crb.yaml b/lib/common/bootstrap/charts/metrics-server/templates/metrics-server-crb.yaml index 68b88fe3..ede287a5 100644 --- a/lib/common/bootstrap/charts/metrics-server/templates/metrics-server-crb.yaml +++ b/lib/common/bootstrap/charts/metrics-server/templates/metrics-server-crb.yaml @@ -4,6 +4,12 @@ kind: ClusterRoleBinding metadata: name: {{ template "common.names.fullname" . }} labels: {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/lib/common/bootstrap/charts/metrics-server/templates/pdb.yaml b/lib/common/bootstrap/charts/metrics-server/templates/pdb.yaml index 16be93b1..964203db 100644 --- a/lib/common/bootstrap/charts/metrics-server/templates/pdb.yaml +++ b/lib/common/bootstrap/charts/metrics-server/templates/pdb.yaml @@ -4,6 +4,12 @@ kind: PodDisruptionBudget metadata: name: {{ include "common.names.fullname" . }} labels: {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} spec: {{- if .Values.podDisruptionBudget.minAvailable }} minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} @@ -13,4 +19,4 @@ spec: {{- end }} selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/lib/common/bootstrap/charts/metrics-server/templates/role-binding.yaml b/lib/common/bootstrap/charts/metrics-server/templates/role-binding.yaml index 918fac95..9cf00a44 100644 --- a/lib/common/bootstrap/charts/metrics-server/templates/role-binding.yaml +++ b/lib/common/bootstrap/charts/metrics-server/templates/role-binding.yaml @@ -5,6 +5,12 @@ metadata: name: {{ printf "%s-auth-reader" (include "common.names.fullname" .) }} namespace: kube-system labels: {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role diff --git a/lib/common/bootstrap/charts/metrics-server/templates/serviceaccount.yaml b/lib/common/bootstrap/charts/metrics-server/templates/serviceaccount.yaml index 558cdf19..2c2acd70 100644 --- a/lib/common/bootstrap/charts/metrics-server/templates/serviceaccount.yaml +++ b/lib/common/bootstrap/charts/metrics-server/templates/serviceaccount.yaml @@ -4,5 +4,11 @@ kind: ServiceAccount metadata: name: {{ template "common.names.fullname" . }} labels: {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} {{- end -}} diff --git a/lib/common/bootstrap/charts/metrics-server/templates/svc.yaml b/lib/common/bootstrap/charts/metrics-server/templates/svc.yaml index 5e62fbc8..e81c869c 100644 --- a/lib/common/bootstrap/charts/metrics-server/templates/svc.yaml +++ b/lib/common/bootstrap/charts/metrics-server/templates/svc.yaml @@ -2,12 +2,22 @@ apiVersion: v1 kind: Service metadata: name: {{ template "common.names.fullname" . }} + labels: {{- include "common.labels.standard" . | nindent 4 }} {{- if .Values.service.labels }} - {{- include "common.tplvalues.render" (dict "value" .Values.service.labels "context" $) | nindent 4 }} + {{- include "common.tplvalues.render" (dict "value" .Values.service.labels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if or .Values.service.annotations .Values.commonAnnotations }} + annotations: + {{- if .Values.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} {{- if .Values.service.annotations }} - annotations: {{- include "common.tplvalues.render" (dict "value" .Values.service.annotations "context" $) | nindent 4 }} + {{- include "common.tplvalues.render" (dict "value" .Values.service.annotations "context" $) | nindent 4 }} + {{- end }} {{- end }} spec: type: {{ .Values.service.type }} diff --git a/lib/common/bootstrap/charts/metrics-server/values.yaml b/lib/common/bootstrap/charts/metrics-server/values.yaml index 762e027d..381cecbd 100644 --- a/lib/common/bootstrap/charts/metrics-server/values.yaml +++ b/lib/common/bootstrap/charts/metrics-server/values.yaml @@ -22,6 +22,15 @@ nameOverride: "" ## @param fullnameOverride String to fully override common.names.fullname template ## fullnameOverride: "" +## @param commonLabels Add labels to all the deployed resources +## +commonLabels: {} +## @param commonAnnotations Add annotations to all the deployed resources +## +commonAnnotations: {} +## @param extraDeploy Array of extra objects to deploy with the release +## +extraDeploy: [] ## @section Metrics Server parameters @@ -36,10 +45,10 @@ fullnameOverride: "" image: registry: docker.io repository: bitnami/metrics-server - tag: 0.5.0-debian-10-r59 + tag: 0.5.2-debian-10-r49 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images ## pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. @@ -98,6 +107,11 @@ apiService: ## This is still necessary up to at least k8s version >= 1.21, but depends on vendors and cloud providers. ## create: false + ## @param apiService.insecureSkipTLSVerify Specifies whether to skip self-verifying self-signed TLS certificates. Set to "false" if you are providing your own certificates. + ## Note that "false" MUST be in quotation marks (cf. https://github.com/helm/helm/issues/3308), since false without quotation marks will render to true + insecureSkipTLSVerify: true + ## @param apiService.caBundle A base64-encoded string of concatenated certificates for the CA chain for the APIService. + caBundle: "" ## @param securePort Port where metrics-server will be running ## securePort: 8443 @@ -107,6 +121,10 @@ securePort: 8443 ## if you use Weave network on EKS ## hostNetwork: false +## @param dnsPolicy Default dnsPolicy setting +## If you enable hostNetwork then you may need to set your dnsPolicy to something other +## than "ClusterFirst" depending on your requirements. +dnsPolicy: "ClusterFirst" ## @param command Override default container command (useful when using custom images) ## command: ["metrics-server"] @@ -215,7 +233,7 @@ service: ## labels: {} ## Metric Server containers' resource requests and limits -## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## ref: https://kubernetes.io/docs/user-guide/compute-resources/ ## We usually recommend not to specify default resources and to leave this as a conscious ## choice for the user. This also increases chances charts run on environments with little ## resources, such as Minikube. If you do want to specify resources, uncomment the following diff --git a/lib/digitalocean/bootstrap/chart_values/metrics-server.yaml b/lib/digitalocean/bootstrap/chart_values/metrics-server.yaml new file mode 100644 index 00000000..5457a1f4 --- /dev/null +++ b/lib/digitalocean/bootstrap/chart_values/metrics-server.yaml @@ -0,0 +1,8 @@ +extraArgs: + kubelet-preferred-address-types: InternalIP,ExternalIP,Hostname + kubelet-use-node-status-port: true + metric-resolution: 15s + cert-dir: /tmp + +apiService: + create: true \ No newline at end of file diff --git a/lib/helm-freeze.yaml b/lib/helm-freeze.yaml index d82f4a74..b85bf1d3 100644 --- a/lib/helm-freeze.yaml +++ b/lib/helm-freeze.yaml @@ -8,6 +8,7 @@ charts: - name: grafana version: 6.16.10 repo_name: grafana + no_sync: true comment: | Dashboard have been set into it as the chart suggest but it's a problem with helm-freeze Need to look if we can move them out of this repo @@ -34,7 +35,7 @@ charts: version: 9.10.4 - name: metrics-server repo_name: bitnami - version: 5.9.2 + version: 5.10.13 - name: kube-state-metrics repo_name: prometheus-community version: 3.4.2 diff --git a/src/cloud_provider/aws/kubernetes/helm_charts.rs b/src/cloud_provider/aws/kubernetes/helm_charts.rs index 27f7e4e2..570e20df 100644 --- a/src/cloud_provider/aws/kubernetes/helm_charts.rs +++ b/src/cloud_provider/aws/kubernetes/helm_charts.rs @@ -574,6 +574,7 @@ pub fn aws_helm_charts( chart_info: ChartInfo { name: "metrics-server".to_string(), path: chart_path("common/charts/metrics-server"), + values_files: vec![chart_path("chart_values/metrics-server.yaml")], values: vec![ ChartSetValue { key: "resources.limits.cpu".to_string(), diff --git a/src/cloud_provider/digitalocean/kubernetes/helm_charts.rs b/src/cloud_provider/digitalocean/kubernetes/helm_charts.rs index 40c98931..a2a3a4d3 100644 --- a/src/cloud_provider/digitalocean/kubernetes/helm_charts.rs +++ b/src/cloud_provider/digitalocean/kubernetes/helm_charts.rs @@ -428,15 +428,8 @@ pub fn do_helm_charts( chart_info: ChartInfo { name: "metrics-server".to_string(), path: chart_path("common/charts/metrics-server"), + values_files: vec![chart_path("chart_values/metrics-server.yaml")], values: vec![ - ChartSetValue { - key: "extraArgs.kubelet-preferred-address-types".to_string(), - value: "InternalIP".to_string(), - }, - ChartSetValue { - key: "apiService.create".to_string(), - value: "true".to_string(), - }, ChartSetValue { key: "resources.limits.cpu".to_string(), value: "250m".to_string(), From 7c7bd080cb2004522014953fd10058448dcf57d4 Mon Sep 17 00:00:00 2001 From: enzo Date: Tue, 18 Jan 2022 17:34:26 +0100 Subject: [PATCH 14/25] feat: adding tests --- src/cmd/kubectl.rs | 13 +++++- src/cmd/structs.rs | 29 +++++++++++++ test_utilities/src/common.rs | 83 ++++++++++++++++++++++++++++++++---- 3 files changed, 115 insertions(+), 10 deletions(-) diff --git a/src/cmd/kubectl.rs b/src/cmd/kubectl.rs index 99c8bc04..b6a1ae82 100644 --- a/src/cmd/kubectl.rs +++ b/src/cmd/kubectl.rs @@ -10,7 +10,7 @@ use crate::cloud_provider::metrics::KubernetesApiMetrics; use crate::cmd::structs::{ Configmap, Daemonset, Item, KubernetesEvent, KubernetesJob, KubernetesKind, KubernetesList, KubernetesNode, KubernetesPod, KubernetesPodStatusPhase, KubernetesPodStatusReason, KubernetesService, KubernetesVersion, - LabelsContent, Namespace, Secrets, PDB, PVC, SVC, + LabelsContent, Namespace, Secrets, HPA, PDB, PVC, SVC, }; use crate::cmd::utilities::QoveryCommand; use crate::constants::KUBECONFIG; @@ -1239,3 +1239,14 @@ where envs, ) } + +pub fn kubernetes_get_all_hpas

(kubernetes_config: P, envs: Vec<(&str, &str)>) -> Result +where + P: AsRef, +{ + kubectl_exec::( + vec!["get", "hpa", "--all-namespaces", "-o", "json"], + kubernetes_config, + envs, + ) +} diff --git a/src/cmd/structs.rs b/src/cmd/structs.rs index d74fb25b..37fba55a 100644 --- a/src/cmd/structs.rs +++ b/src/cmd/structs.rs @@ -495,6 +495,35 @@ pub struct PDBStatus { pub observed_generation: i16, } +#[derive(Default, Debug, Clone, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HPA { + pub api_version: String, + pub items: Option>, + pub kind: String, +} + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HPAItem { + pub api_version: String, + pub kind: String, + pub metadata: HPAMetadata, +} + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HPAMetadata { + pub annotations: Option, +} + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HPAAnnotationCondition { + #[serde(rename = "autoscaling.alpha.kubernetes.io/conditions")] + pub conditions: Option, +} + #[cfg(test)] mod tests { use crate::cmd::structs::{KubernetesList, KubernetesPod, KubernetesPodStatusReason, PDB, PVC, SVC}; diff --git a/test_utilities/src/common.rs b/test_utilities/src/common.rs index 9c6e6f10..2102fc58 100644 --- a/test_utilities/src/common.rs +++ b/test_utilities/src/common.rs @@ -32,12 +32,15 @@ use qovery_engine::cloud_provider::scaleway::application::ScwZone; use qovery_engine::cloud_provider::scaleway::kubernetes::Kapsule; use qovery_engine::cloud_provider::scaleway::Scaleway; use qovery_engine::cloud_provider::{CloudProvider, Kind}; -use qovery_engine::cmd::structs::SVCItem; +use qovery_engine::cmd::kubectl::kubernetes_get_all_hpas; +use qovery_engine::cmd::structs::{SVCItem, HPA}; use qovery_engine::engine::Engine; +use qovery_engine::error::{SimpleError, SimpleErrorKind}; use qovery_engine::logger::Logger; use qovery_engine::models::DatabaseMode::CONTAINER; use qovery_engine::transaction::DeploymentOption; use std::collections::BTreeMap; +use std::path::Path; use std::str::FromStr; use tracing::{span, Level}; @@ -1390,14 +1393,14 @@ pub fn cluster_test( ); // Deploy - if let Err(err) = deploy_tx.create_kubernetes(kubernetes.as_ref()) { - panic!("{:?}", err) - } - let _ = match deploy_tx.commit() { - TransactionResult::Ok => assert!(true), - TransactionResult::Rollback(_) => assert!(false), - TransactionResult::UnrecoverableError(_, _) => assert!(false), - }; + // if let Err(err) = deploy_tx.create_kubernetes(kubernetes.as_ref()) { + // panic!("{:?}", err) + // } + // let _ = match deploy_tx.commit() { + // TransactionResult::Ok => assert!(true), + // TransactionResult::Rollback(_) => assert!(false), + // TransactionResult::UnrecoverableError(_, _) => assert!(false), + // }; // Deploy env if any if let Some(env) = environment_to_deploy { @@ -1414,6 +1417,15 @@ pub fn cluster_test( }; } + if let Err(err) = metrics_server_test( + kubernetes + .get_kubeconfig_file_path() + .expect("Unable to get config file path"), + kubernetes.cloud_provider().credentials_environment_variables(), + ) { + panic!("{:?}", err) + } + match test_type { ClusterTestType::Classic => {} ClusterTestType::WithPause => { @@ -1439,6 +1451,15 @@ pub fn cluster_test( TransactionResult::Rollback(_) => assert!(false), TransactionResult::UnrecoverableError(_, _) => assert!(false), }; + + if let Err(err) = metrics_server_test( + kubernetes + .get_kubeconfig_file_path() + .expect("Unable to get config file path"), + kubernetes.cloud_provider().credentials_environment_variables(), + ) { + panic!("{:?}", err) + } } ClusterTestType::WithUpgrade => { let upgrade_to_version = format!("{}.{}", major_boot_version, minor_boot_version.clone() + 1); @@ -1469,6 +1490,19 @@ pub fn cluster_test( TransactionResult::UnrecoverableError(_, _) => assert!(false), }; + if let Err(err) = metrics_server_test( + upgraded_kubernetes + .as_ref() + .get_kubeconfig_file_path() + .expect("Unable to get config file path"), + upgraded_kubernetes + .as_ref() + .cloud_provider() + .credentials_environment_variables(), + ) { + panic!("{:?}", err) + } + // Delete if let Err(err) = delete_tx.delete_kubernetes(upgraded_kubernetes.as_ref()) { panic!("{:?}", err) @@ -1510,3 +1544,34 @@ pub fn cluster_test( test_name.to_string() } + +pub fn metrics_server_test

(kubernetes_config: P, envs: Vec<(&str, &str)>) -> Result<(), SimpleError> +where + P: AsRef, +{ + let result = kubernetes_get_all_hpas(kubernetes_config, envs); + + match result { + Ok(hpas) => { + for hpa in hpas.items.expect("No hpa item").into_iter() { + if hpa.metadata.annotations.is_some() { + if hpa + .metadata + .annotations + .unwrap() + .conditions + .expect("No hpa condition") + .contains("FailedGetResourceMetric") + { + return Err(SimpleError { + kind: SimpleErrorKind::Other, + message: Some("Metrics server doesn't work".to_string()), + }); + } + } + } + Ok(()) + } + Err(e) => Err(e), + } +} From 5eb8559c5947cc16508ad31fe167e3d047952986 Mon Sep 17 00:00:00 2001 From: enzo Date: Tue, 18 Jan 2022 18:16:53 +0100 Subject: [PATCH 15/25] feat: fix linter --- test_utilities/src/common.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_utilities/src/common.rs b/test_utilities/src/common.rs index 2102fc58..42055b67 100644 --- a/test_utilities/src/common.rs +++ b/test_utilities/src/common.rs @@ -33,7 +33,7 @@ use qovery_engine::cloud_provider::scaleway::kubernetes::Kapsule; use qovery_engine::cloud_provider::scaleway::Scaleway; use qovery_engine::cloud_provider::{CloudProvider, Kind}; use qovery_engine::cmd::kubectl::kubernetes_get_all_hpas; -use qovery_engine::cmd::structs::{SVCItem, HPA}; +use qovery_engine::cmd::structs::SVCItem; use qovery_engine::engine::Engine; use qovery_engine::error::{SimpleError, SimpleErrorKind}; use qovery_engine::logger::Logger; From cfeea275f37bab89c9a1c54f5dc04390f9e9d10b Mon Sep 17 00:00:00 2001 From: enzo Date: Wed, 19 Jan 2022 09:36:24 +0100 Subject: [PATCH 16/25] feat: review changes --- src/cloud_provider/kubernetes.rs | 2 +- src/cmd/kubectl.rs | 44 +++++++++++++++++++++++--------- test_utilities/src/common.rs | 44 +++++++++++++++----------------- tests/aws/aws_environment.rs | 2 ++ 4 files changed, 56 insertions(+), 36 deletions(-) diff --git a/src/cloud_provider/kubernetes.rs b/src/cloud_provider/kubernetes.rs index 1b69c1be..ced33547 100644 --- a/src/cloud_provider/kubernetes.rs +++ b/src/cloud_provider/kubernetes.rs @@ -850,7 +850,7 @@ pub fn is_kubernetes_upgradable

(kubernetes_config: P, envs: Vec<(&str, &str)> where P: AsRef, { - match kubernetes_get_all_pdbs(kubernetes_config, envs) { + match kubernetes_get_all_pdbs(kubernetes_config, envs, None) { Ok(pdbs) => match pdbs.items.is_some() { false => Ok(()), true => { diff --git a/src/cmd/kubectl.rs b/src/cmd/kubectl.rs index b6a1ae82..ad1179b8 100644 --- a/src/cmd/kubectl.rs +++ b/src/cmd/kubectl.rs @@ -1229,24 +1229,44 @@ where Ok(result) } -pub fn kubernetes_get_all_pdbs

(kubernetes_config: P, envs: Vec<(&str, &str)>) -> Result +pub fn kubernetes_get_all_pdbs

( + kubernetes_config: P, + envs: Vec<(&str, &str)>, + namespace: Option<&str>, +) -> Result where P: AsRef, { - kubectl_exec::( - vec!["get", "pdb", "--all-namespaces", "-o", "json"], - kubernetes_config, - envs, - ) + let mut cmd_args = vec!["get", "pdb", "-o", "json"]; + + match namespace { + Some(n) => { + cmd_args.push("-n"); + cmd_args.push(n); + } + None => cmd_args.push("--all-namespaces"), + } + + kubectl_exec::(cmd_args, kubernetes_config, envs) } -pub fn kubernetes_get_all_hpas

(kubernetes_config: P, envs: Vec<(&str, &str)>) -> Result +pub fn kubernetes_get_all_hpas

( + kubernetes_config: P, + envs: Vec<(&str, &str)>, + namespace: Option<&str>, +) -> Result where P: AsRef, { - kubectl_exec::( - vec!["get", "hpa", "--all-namespaces", "-o", "json"], - kubernetes_config, - envs, - ) + let mut cmd_args = vec!["get", "hpa", "-o", "json"]; + + match namespace { + Some(n) => { + cmd_args.push("-n"); + cmd_args.push(n); + } + None => cmd_args.push("--all-namespaces"), + } + + kubectl_exec::(cmd_args, kubernetes_config, envs) } diff --git a/test_utilities/src/common.rs b/test_utilities/src/common.rs index 42055b67..2ace4cc2 100644 --- a/test_utilities/src/common.rs +++ b/test_utilities/src/common.rs @@ -1393,14 +1393,14 @@ pub fn cluster_test( ); // Deploy - // if let Err(err) = deploy_tx.create_kubernetes(kubernetes.as_ref()) { - // panic!("{:?}", err) - // } - // let _ = match deploy_tx.commit() { - // TransactionResult::Ok => assert!(true), - // TransactionResult::Rollback(_) => assert!(false), - // TransactionResult::UnrecoverableError(_, _) => assert!(false), - // }; + if let Err(err) = deploy_tx.create_kubernetes(kubernetes.as_ref()) { + panic!("{:?}", err) + } + let _ = match deploy_tx.commit() { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; // Deploy env if any if let Some(env) = environment_to_deploy { @@ -1549,25 +1549,23 @@ pub fn metrics_server_test

(kubernetes_config: P, envs: Vec<(&str, &str)>) -> where P: AsRef, { - let result = kubernetes_get_all_hpas(kubernetes_config, envs); + let result = kubernetes_get_all_hpas(kubernetes_config, envs, None); match result { Ok(hpas) => { for hpa in hpas.items.expect("No hpa item").into_iter() { - if hpa.metadata.annotations.is_some() { - if hpa - .metadata - .annotations - .unwrap() - .conditions - .expect("No hpa condition") - .contains("FailedGetResourceMetric") - { - return Err(SimpleError { - kind: SimpleErrorKind::Other, - message: Some("Metrics server doesn't work".to_string()), - }); - } + if !hpa + .metadata + .annotations + .expect("No hpa annotation.") + .conditions + .expect("No hpa condition.") + .contains("ValidMetricFound") + { + return Err(SimpleError { + kind: SimpleErrorKind::Other, + message: Some("Metrics server doesn't work".to_string()), + }); } } Ok(()) diff --git a/tests/aws/aws_environment.rs b/tests/aws/aws_environment.rs index 3c4bede8..70f26681 100644 --- a/tests/aws/aws_environment.rs +++ b/tests/aws/aws_environment.rs @@ -173,6 +173,7 @@ fn deploy_a_working_environment_and_pause_it_eks() { .as_str(), ), ], + None, ); for pdb in pdbs.expect("Unable to get pdbs").items.expect("Unable to get pdbs") { assert_eq!(pdb.metadata.name.contains(&environment.applications[0].name), false) @@ -224,6 +225,7 @@ fn deploy_a_working_environment_and_pause_it_eks() { .as_str(), ), ], + None, ); let mut filtered_pdb = false; for pdb in pdbs.expect("Unable to get pdbs").items.expect("Unable to get pdbs") { From 4082e5cc6e52e47f9a29e2b6c6e8db0c7e90e834 Mon Sep 17 00:00:00 2001 From: enzo Date: Fri, 14 Jan 2022 16:02:09 +0100 Subject: [PATCH 17/25] feat(ENG_1093): remove grafana's pvc --- lib/aws/bootstrap/chart_values/grafana.j2.yaml | 4 +++- lib/digitalocean/bootstrap/chart_values/grafana.j2.yaml | 4 +++- lib/scaleway/bootstrap/chart_values/grafana.j2.yaml | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/aws/bootstrap/chart_values/grafana.j2.yaml b/lib/aws/bootstrap/chart_values/grafana.j2.yaml index 23c1d4ae..712f017b 100644 --- a/lib/aws/bootstrap/chart_values/grafana.j2.yaml +++ b/lib/aws/bootstrap/chart_values/grafana.j2.yaml @@ -18,13 +18,15 @@ deploymentStrategy: persistence: type: pvc - enabled: true + enabled: false storageClassName: aws-ebs-gp2-0 accessModes: - ReadWriteOnce size: 1Gi finalizers: - kubernetes.io/pvc-protection + inMemory: + enabled: true adminUser: "{{ grafana_admin_user }}" adminPassword: "{{ grafana_admin_password }}" diff --git a/lib/digitalocean/bootstrap/chart_values/grafana.j2.yaml b/lib/digitalocean/bootstrap/chart_values/grafana.j2.yaml index f28869a3..fddbd5c4 100644 --- a/lib/digitalocean/bootstrap/chart_values/grafana.j2.yaml +++ b/lib/digitalocean/bootstrap/chart_values/grafana.j2.yaml @@ -3,13 +3,15 @@ deploymentStrategy: persistence: type: pvc - enabled: true + enabled: false storageClassName: do-volume-standard-0 accessModes: - ReadWriteOnce size: 1Gi finalizers: - kubernetes.io/pvc-protection + inMemory: + enabled: true adminUser: "{{ grafana_admin_user }}" adminPassword: "{{ grafana_admin_password }}" diff --git a/lib/scaleway/bootstrap/chart_values/grafana.j2.yaml b/lib/scaleway/bootstrap/chart_values/grafana.j2.yaml index ad8d34ee..034e2bd7 100644 --- a/lib/scaleway/bootstrap/chart_values/grafana.j2.yaml +++ b/lib/scaleway/bootstrap/chart_values/grafana.j2.yaml @@ -3,13 +3,15 @@ deploymentStrategy: persistence: type: pvc - enabled: true + enabled: false storageClassName: scw-sbv-ssd-0 accessModes: - ReadWriteOnce size: 1Gi finalizers: - kubernetes.io/pvc-protection + inMemory: + enabled: true adminUser: "{{ grafana_admin_user }}" adminPassword: "{{ grafana_admin_password }}" From 36e561825679e2401c7ef7d2e8c9296def300d5b Mon Sep 17 00:00:00 2001 From: enzo Date: Fri, 14 Jan 2022 16:34:57 +0100 Subject: [PATCH 18/25] feat(ENG_1093): review changes --- test_utilities/src/aws.rs | 2 +- test_utilities/src/scaleway.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test_utilities/src/aws.rs b/test_utilities/src/aws.rs index 3ec5f99e..9f6e274c 100644 --- a/test_utilities/src/aws.rs +++ b/test_utilities/src/aws.rs @@ -23,7 +23,7 @@ use crate::utilities::{build_platform_local_docker, FuncTestsSecrets}; pub const AWS_REGION_FOR_S3: &str = "eu-west-3"; pub const AWS_TEST_REGION: &str = "eu-west-3"; pub const AWS_KUBERNETES_MAJOR_VERSION: u8 = 1; -pub const AWS_KUBERNETES_MINOR_VERSION: u8 = 20; +pub const AWS_KUBERNETES_MINOR_VERSION: u8 = 19; pub const AWS_KUBERNETES_VERSION: &'static str = formatcp!("{}.{}", AWS_KUBERNETES_MAJOR_VERSION, AWS_KUBERNETES_MINOR_VERSION); pub const AWS_DATABASE_INSTANCE_TYPE: &str = "db.t3.micro"; diff --git a/test_utilities/src/scaleway.rs b/test_utilities/src/scaleway.rs index ba67af21..84cd49af 100644 --- a/test_utilities/src/scaleway.rs +++ b/test_utilities/src/scaleway.rs @@ -21,7 +21,7 @@ use tracing::error; pub const SCW_TEST_ZONE: ScwZone = ScwZone::Paris2; pub const SCW_KUBERNETES_MAJOR_VERSION: u8 = 1; -pub const SCW_KUBERNETES_MINOR_VERSION: u8 = 20; +pub const SCW_KUBERNETES_MINOR_VERSION: u8 = 19; pub const SCW_KUBERNETES_VERSION: &'static str = formatcp!("{}.{}", SCW_KUBERNETES_MAJOR_VERSION, SCW_KUBERNETES_MINOR_VERSION); pub const SCW_MANAGED_DATABASE_INSTANCE_TYPE: &str = "db-dev-s"; From 862caaa4285eb7fb2fa6c9de428fd7fdf9125826 Mon Sep 17 00:00:00 2001 From: MacLikorne Date: Tue, 25 Jan 2022 14:16:52 +0100 Subject: [PATCH 19/25] impr(ENG_1102): handling nodes types changes (#567) --- .../digitalocean/kubernetes/mod.rs | 21 ++++++++ src/cloud_provider/kubernetes.rs | 50 +++++++++++++++++++ src/cloud_provider/scaleway/kubernetes/mod.rs | 21 ++++++++ src/cmd/structs.rs | 9 ++++ 4 files changed, 101 insertions(+) diff --git a/src/cloud_provider/digitalocean/kubernetes/mod.rs b/src/cloud_provider/digitalocean/kubernetes/mod.rs index 34a9330a..7cd45865 100644 --- a/src/cloud_provider/digitalocean/kubernetes/mod.rs +++ b/src/cloud_provider/digitalocean/kubernetes/mod.rs @@ -612,6 +612,27 @@ impl<'a> DOKS<'a> { return Err(e); } + match self.check_workers_on_create() { + Ok(_) => { + let message = format!("Kubernetes {} nodes have been successfully created", self.name()); + info!("{}", &message); + self.send_to_customer(&message, &listeners_helper); + } + Err(e) => { + error!( + "Error while deploying cluster {} with Terraform with id {}.", + self.name(), + self.id() + ); + return Err(EngineError { + cause: EngineErrorCause::Internal, + scope: EngineErrorScope::Engine, + execution_id: self.context.execution_id().to_string(), + message: e.message, + }); + } + }; + // kubernetes helm deployments on the cluster let kubeconfig_path = match self.get_kubeconfig_file_path() { Ok(path) => path, diff --git a/src/cloud_provider/kubernetes.rs b/src/cloud_provider/kubernetes.rs index ced33547..f7716603 100644 --- a/src/cloud_provider/kubernetes.rs +++ b/src/cloud_provider/kubernetes.rs @@ -24,6 +24,7 @@ use crate::cmd::kubectl::{ kubectl_delete_objects_in_all_namespaces, kubectl_exec_count_all_objects, kubectl_exec_delete_pod, kubectl_exec_get_node, kubectl_exec_version, kubectl_get_crash_looping_pods, kubernetes_get_all_pdbs, }; +use crate::cmd::structs::KubernetesNodeCondition; use crate::dns_provider::DnsProvider; use crate::error::SimpleErrorKind::Other; use crate::error::{EngineError, EngineErrorCause, EngineErrorScope, SimpleError, SimpleErrorKind}; @@ -226,6 +227,18 @@ pub trait Kubernetes: Listen { ) }) } + + fn check_workers_on_create(&self) -> Result<(), SimpleError> + where + Self: Sized, + { + send_progress_on_long_task(self, Action::Create, || { + check_workers_status( + self.get_kubeconfig_file_path().expect("Unable to get Kubeconfig"), + self.cloud_provider().credentials_environment_variables(), + ) + }) + } fn upgrade_with_status(&self, kubernetes_upgrade_status: KubernetesUpgradeStatus) -> Result<(), EngineError>; fn on_upgrade(&self) -> Result<(), EngineError>; fn on_upgrade_error(&self) -> Result<(), EngineError>; @@ -908,6 +921,43 @@ where } }); + return match result { + Ok(_) => match check_workers_status(kubernetes_config.as_ref(), envs.clone()) { + Ok(_) => Ok(()), + Err(e) => Err(e), + }, + Err(Operation { error, .. }) => Err(error), + Err(retry::Error::Internal(e)) => Err(SimpleError::new(SimpleErrorKind::Other, Some(e))), + }; +} + +pub fn check_workers_status

(kubernetes_config: P, envs: Vec<(&str, &str)>) -> Result<(), SimpleError> +where + P: AsRef, +{ + let result = retry::retry(Fixed::from_millis(10000).take(360), || { + match kubectl_exec_get_node(kubernetes_config.as_ref(), envs.clone()) { + Err(e) => OperationResult::Retry(e), + Ok(nodes) => { + let mut conditions: Vec = Vec::new(); + for node in nodes.items.into_iter() { + conditions.extend(node.status.conditions.into_iter()); + } + + for condition in conditions.iter() { + if condition.condition_type == "Ready" && condition.status != "True" { + info!("All worker nodes are not ready yet, waiting..."); + return OperationResult::Retry(SimpleError::new( + SimpleErrorKind::Other, + Some("There are still not ready worker nodes."), + )); + } + } + return OperationResult::Ok(()); + } + } + }); + return match result { Ok(_) => Ok(()), Err(Operation { error, .. }) => Err(error), diff --git a/src/cloud_provider/scaleway/kubernetes/mod.rs b/src/cloud_provider/scaleway/kubernetes/mod.rs index 521817e2..048ab6a2 100644 --- a/src/cloud_provider/scaleway/kubernetes/mod.rs +++ b/src/cloud_provider/scaleway/kubernetes/mod.rs @@ -513,6 +513,27 @@ impl<'a> Kapsule<'a> { return Err(e); } + match self.check_workers_on_create() { + Ok(_) => { + let message = format!("Kubernetes {} nodes have been successfully created", self.name()); + info!("{}", &message); + self.send_to_customer(&message, &listeners_helper); + } + Err(e) => { + error!( + "Error while deploying cluster {} with Terraform with id {}.", + self.name(), + self.id() + ); + return Err(LegacyEngineError { + cause: EngineErrorCause::Internal, + scope: EngineErrorScope::Engine, + execution_id: self.context.execution_id().to_string(), + message: e.message, + }); + } + }; + // kubernetes helm deployments on the cluster let kubeconfig = PathBuf::from(self.get_kubeconfig_file().expect("expected to get a kubeconfig file").0); let credentials_environment_variables: Vec<(String, String)> = self diff --git a/src/cmd/structs.rs b/src/cmd/structs.rs index 37fba55a..23853336 100644 --- a/src/cmd/structs.rs +++ b/src/cmd/structs.rs @@ -286,6 +286,7 @@ pub struct KubernetesNodeStatus { pub allocatable: KubernetesNodeStatusResources, pub capacity: KubernetesNodeStatusResources, pub node_info: KubernetesNodeInfo, + pub conditions: Vec, } #[derive(Deserialize, Clone, Eq, PartialEq)] @@ -303,6 +304,14 @@ pub struct KubernetesNodeInfo { pub kubelet_version: String, } +#[derive(Deserialize, Clone, Eq, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct KubernetesNodeCondition { + #[serde(rename = "type")] + pub condition_type: String, + pub status: String, +} + #[derive(Deserialize, Clone, Eq, PartialEq)] #[serde(rename_all = "camelCase")] pub struct KubernetesEvent { From 77302e353b104bf89536a702660817fd69f8886a Mon Sep 17 00:00:00 2001 From: MacLikorne Date: Wed, 26 Jan 2022 15:18:00 +0100 Subject: [PATCH 20/25] hot-fix: reduce node status check time before timeout (#578) --- src/cloud_provider/kubernetes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cloud_provider/kubernetes.rs b/src/cloud_provider/kubernetes.rs index f7716603..26d846fc 100644 --- a/src/cloud_provider/kubernetes.rs +++ b/src/cloud_provider/kubernetes.rs @@ -935,7 +935,7 @@ pub fn check_workers_status

(kubernetes_config: P, envs: Vec<(&str, &str)>) -> where P: AsRef, { - let result = retry::retry(Fixed::from_millis(10000).take(360), || { + let result = retry::retry(Fixed::from_millis(10000).take(60), || { match kubectl_exec_get_node(kubernetes_config.as_ref(), envs.clone()) { Err(e) => OperationResult::Retry(e), Ok(nodes) => { From 03b9f5a21f9aa5646ee4e80b895edb5f179bbf43 Mon Sep 17 00:00:00 2001 From: MacLikorne Date: Fri, 28 Jan 2022 00:06:09 +0100 Subject: [PATCH 21/25] hot-fix: add fr-par-3 zone (#579) --- src/cloud_provider/scaleway/application.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/cloud_provider/scaleway/application.rs b/src/cloud_provider/scaleway/application.rs index 896c2f99..ff148e26 100644 --- a/src/cloud_provider/scaleway/application.rs +++ b/src/cloud_provider/scaleway/application.rs @@ -482,6 +482,7 @@ impl FromStr for ScwRegion { pub enum ScwZone { Paris1, Paris2, + Paris3, Amsterdam1, Warsaw1, } @@ -492,6 +493,7 @@ impl ScwZone { match self { ScwZone::Paris1 => "fr-par-1", ScwZone::Paris2 => "fr-par-2", + ScwZone::Paris3 => "fr-par-3", ScwZone::Amsterdam1 => "nl-ams-1", ScwZone::Warsaw1 => "pl-waw-1", } @@ -501,6 +503,7 @@ impl ScwZone { match self { ScwZone::Paris1 => ScwRegion::Paris, ScwZone::Paris2 => ScwRegion::Paris, + ScwZone::Paris3 => ScwRegion::Paris, ScwZone::Amsterdam1 => ScwRegion::Amsterdam, ScwZone::Warsaw1 => ScwRegion::Warsaw, } @@ -511,6 +514,7 @@ impl ScwZone { match self { ScwZone::Paris1 => "fr-par", ScwZone::Paris2 => "fr-par", + ScwZone::Paris3 => "fr-par", ScwZone::Amsterdam1 => "nl-ams", ScwZone::Warsaw1 => "pl-waw", } @@ -523,6 +527,7 @@ impl fmt::Display for ScwZone { match self { ScwZone::Paris1 => write!(f, "fr-par-1"), ScwZone::Paris2 => write!(f, "fr-par-2"), + ScwZone::Paris3 => write!(f, "fr-par-3"), ScwZone::Amsterdam1 => write!(f, "nl-ams-1"), ScwZone::Warsaw1 => write!(f, "pl-waw-1"), } @@ -536,6 +541,7 @@ impl FromStr for ScwZone { match s { "fr-par-1" => Ok(ScwZone::Paris1), "fr-par-2" => Ok(ScwZone::Paris2), + "fr-par-3" => Ok(ScwZone::Paris3), "nl-ams-1" => Ok(ScwZone::Amsterdam1), "pl-waw-1" => Ok(ScwZone::Warsaw1), _ => Err(()), @@ -566,6 +572,7 @@ mod tests { fn test_zone_to_str() { assert_eq!("fr-par-1", ScwZone::Paris1.as_str()); assert_eq!("fr-par-2", ScwZone::Paris2.as_str()); + assert_eq!("fr-par-3", ScwZone::Paris3.as_str()); assert_eq!("nl-ams-1", ScwZone::Amsterdam1.as_str()); assert_eq!("pl-waw-1", ScwZone::Warsaw1.as_str()); } @@ -574,6 +581,7 @@ mod tests { fn test_zone_from_str() { assert_eq!(ScwZone::from_str("fr-par-1"), Ok(ScwZone::Paris1)); assert_eq!(ScwZone::from_str("fr-par-2"), Ok(ScwZone::Paris2)); + assert_eq!(ScwZone::from_str("fr-par-3"), Ok(ScwZone::Paris3)); assert_eq!(ScwZone::from_str("nl-ams-1"), Ok(ScwZone::Amsterdam1)); assert_eq!(ScwZone::from_str("pl-waw-1"), Ok(ScwZone::Warsaw1)); } @@ -582,6 +590,7 @@ mod tests { fn test_zone_region() { assert_eq!(ScwZone::Paris1.region(), ScwRegion::Paris); assert_eq!(ScwZone::Paris2.region(), ScwRegion::Paris); + assert_eq!(ScwZone::Paris3.region(), ScwRegion::Paris); assert_eq!(ScwZone::Amsterdam1.region(), ScwRegion::Amsterdam); assert_eq!(ScwZone::Warsaw1.region(), ScwRegion::Warsaw); } From 14d3e7955d19b80cb8e816e6a26de45558d4b6f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Romaric=20Philog=C3=A8ne?= Date: Tue, 1 Feb 2022 12:23:01 +0100 Subject: [PATCH 22/25] Improve deployment time (#571) Improve deployment time using Container Registry as a distributed container layers cache. --- src/build_platform/local_docker.rs | 60 ++++- src/build_platform/mod.rs | 73 ++++++ src/container_registry/docker.rs | 52 +++++ src/container_registry/docker_hub.rs | 104 +++++++-- src/container_registry/docr.rs | 103 +++++++-- src/container_registry/ecr.rs | 207 +++++++++++++----- src/container_registry/mod.rs | 6 + .../scaleway_container_registry.rs | 133 +++++++++-- src/git.rs | 74 ++++++- src/models.rs | 61 ++++-- src/transaction.rs | 63 +++++- src/utilities.rs | 98 ++++++++- tests/aws/aws_environment.rs | 98 ++++++++- tests/digitalocean/do_environment.rs | 95 +++++++- tests/scaleway/scw_environment.rs | 96 ++++++++ 15 files changed, 1162 insertions(+), 161 deletions(-) diff --git a/src/build_platform/local_docker.rs b/src/build_platform/local_docker.rs index 6c650dc6..68dab635 100644 --- a/src/build_platform/local_docker.rs +++ b/src/build_platform/local_docker.rs @@ -5,7 +5,7 @@ use chrono::Duration; use git2::{Cred, CredentialType}; use sysinfo::{Disk, DiskExt, SystemExt}; -use crate::build_platform::{docker, Build, BuildPlatform, BuildResult, Credentials, Image, Kind}; +use crate::build_platform::{docker, Build, BuildPlatform, BuildResult, CacheResult, Credentials, Image, Kind}; use crate::cmd::utilities::QoveryCommand; use crate::error::{EngineError, EngineErrorCause, SimpleError, SimpleErrorKind}; use crate::fs::workspace_directory; @@ -48,6 +48,7 @@ impl LocalDocker { &vec!["image", "inspect", image.name_with_tag().as_str()], &self.get_docker_host_envs(), ); + Ok(matches!(cmd.exec(), Ok(_))) } @@ -259,7 +260,7 @@ impl LocalDocker { )); }, ) - .map_err(|err| SimpleError::new(SimpleErrorKind::Other, Some(format!("{}", err)))); + .map_err(|err| SimpleError::new(SimpleErrorKind::Other, Some(format!("{:?}", err)))); if exit_status.is_ok() { // quit now if the builder successfully build the app @@ -286,6 +287,15 @@ impl LocalDocker { } } } + + fn get_repository_build_root_path(&self, build: &Build) -> Result { + workspace_directory( + self.context.workspace_root_dir(), + self.context.execution_id(), + format!("build/{}", build.image.name.as_str()), + ) + .map_err(|err| self.engine_error(EngineErrorCause::Internal, err.to_string())) + } } impl BuildPlatform for LocalDocker { @@ -317,6 +327,34 @@ impl BuildPlatform for LocalDocker { Ok(()) } + fn has_cache(&self, build: &Build) -> Result { + info!("LocalDocker.has_cache() called for {}", self.name()); + + // Check if a local cache layers for the container image exists. + let repository_root_path = self.get_repository_build_root_path(&build)?; + + let parent_build = build + .to_previous_build(repository_root_path) + .map_err(|err| self.engine_error(EngineErrorCause::Internal, err.to_string()))?; + + let parent_build = match parent_build { + Some(parent_build) => parent_build, + None => return Ok(CacheResult::MissWithoutParentBuild), + }; + + // check if local layers exist + let mut cmd = QoveryCommand::new("docker", &["images", "-q", parent_build.image.name.as_str()], &[]); + + let mut result = CacheResult::Miss(parent_build); + let _ = cmd.exec_with_timeout( + Duration::minutes(1), // `docker images` command can be slow with tons of images - it's probably not indexed + |_| result = CacheResult::Hit, // if a line is returned, then the image is locally present + |r_err| error!("Error executing docker command {}", r_err), + ); + + Ok(result) + } + fn build(&self, build: Build, force_build: bool) -> Result { info!("LocalDocker.build() called for {}", self.name()); @@ -331,19 +369,13 @@ impl BuildPlatform for LocalDocker { return Ok(BuildResult { build }); } - // git clone - let repository_root_path = workspace_directory( - self.context.workspace_root_dir(), - self.context.execution_id(), - format!("build/{}", build.image.name.as_str()), - ) - .map_err(|err| self.engine_error(EngineErrorCause::Internal, err.to_string()))?; + let repository_root_path = self.get_repository_build_root_path(&build)?; - // Clone git repository info!( "cloning repository: {} to {}", build.git_repository.url, repository_root_path ); + let get_credentials = |user: &str| { let mut creds: Vec<(CredentialType, Cred)> = Vec::with_capacity(build.git_repository.ssh_keys.len() + 1); for ssh_key in build.git_repository.ssh_keys.iter() { @@ -360,9 +392,17 @@ impl BuildPlatform for LocalDocker { Cred::userpass_plaintext(&login, &password).unwrap(), )); } + creds }; + if Path::new(repository_root_path.as_str()).exists() { + // remove folder before cloning it again + // FIXME: reuse this folder and checkout the right commit + let _ = fs::remove_dir_all(repository_root_path.as_str()); + } + + // git clone if let Err(clone_error) = git::clone_at_commit( &build.git_repository.url, &build.git_repository.commit_id, diff --git a/src/build_platform/mod.rs b/src/build_platform/mod.rs index d01d8f0d..2b369928 100644 --- a/src/build_platform/mod.rs +++ b/src/build_platform/mod.rs @@ -1,8 +1,13 @@ use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; use crate::error::{EngineError, EngineErrorCause, EngineErrorScope}; +use crate::git; use crate::models::{Context, Listen}; +use crate::utilities::get_image_tag; +use git2::{Cred, CredentialType, Error}; use std::fmt::{Display, Formatter, Result as FmtResult}; +use std::path::Path; pub mod docker; pub mod local_docker; @@ -16,6 +21,7 @@ pub trait BuildPlatform: Listen { format!("{} ({})", self.name(), self.id()) } fn is_valid(&self) -> Result<(), EngineError>; + fn has_cache(&self, build: &Build) -> Result; fn build(&self, build: Build, force_build: bool) -> Result; fn build_error(&self, build: Build) -> Result; fn engine_error_scope(&self) -> EngineErrorScope { @@ -37,20 +43,79 @@ pub struct Build { pub options: BuildOptions, } +impl Build { + pub fn to_previous_build

(&self, clone_repo_into_dir: P) -> Result, Error> + where + P: AsRef, + { + let parent_commit_id = git::get_parent_commit_id( + self.git_repository.url.as_str(), + self.git_repository.commit_id.as_str(), + clone_repo_into_dir, + &|_| match &self.git_repository.credentials { + None => vec![], + Some(creds) => vec![( + CredentialType::USER_PASS_PLAINTEXT, + Cred::userpass_plaintext(creds.login.as_str(), creds.password.as_str()).unwrap(), + )], + }, + )?; + + let parent_commit_id = match parent_commit_id { + None => return Ok(None), + Some(parent_commit_id) => parent_commit_id, + }; + + let mut environment_variables_map = BTreeMap::::new(); + for env in &self.options.environment_variables { + environment_variables_map.insert(env.key.clone(), env.value.clone()); + } + + let mut image = self.image.clone(); + image.tag = get_image_tag( + &self.git_repository.root_path, + &self.git_repository.dockerfile_path, + &environment_variables_map, + &parent_commit_id, + ); + + image.commit_id = parent_commit_id.clone(); + + Ok(Some(Build { + git_repository: GitRepository { + url: self.git_repository.url.clone(), + credentials: self.git_repository.credentials.clone(), + ssh_keys: self.git_repository.ssh_keys.clone(), + commit_id: parent_commit_id, + dockerfile_path: self.git_repository.dockerfile_path.clone(), + root_path: self.git_repository.root_path.clone(), + buildpack_language: self.git_repository.buildpack_language.clone(), + }, + image, + options: BuildOptions { + environment_variables: self.options.environment_variables.clone(), + }, + })) + } +} + pub struct BuildOptions { pub environment_variables: Vec, } +#[derive(Clone, Eq, PartialEq, Hash, Debug)] pub struct EnvironmentVariable { pub key: String, pub value: String, } +#[derive(Clone, Eq, PartialEq, Hash, Debug)] pub struct Credentials { pub login: String, pub password: String, } +#[derive(Clone, Eq, PartialEq, Hash, Debug)] pub struct SshKey { pub private_key: String, pub passphrase: Option, @@ -129,3 +194,11 @@ impl BuildResult { pub enum Kind { LocalDocker, } + +type ParentBuild = Build; + +pub enum CacheResult { + MissWithoutParentBuild, + Miss(ParentBuild), + Hit, +} diff --git a/src/container_registry/docker.rs b/src/container_registry/docker.rs index 099a36b9..c7ee1325 100644 --- a/src/container_registry/docker.rs +++ b/src/container_registry/docker.rs @@ -191,3 +191,55 @@ pub fn docker_tag_and_push_image( } } } + +pub fn docker_pull_image( + container_registry_kind: Kind, + docker_envs: Vec<(&str, &str)>, + dest: String, +) -> Result<(), SimpleError> { + let registry_provider = match container_registry_kind { + Kind::DockerHub => "DockerHub", + Kind::Ecr => "AWS ECR", + Kind::Docr => "DigitalOcean Registry", + Kind::ScalewayCr => "Scaleway Registry", + }; + + let mut cmd = QoveryCommand::new("docker", &vec!["pull", dest.as_str()], &docker_envs); + match retry::retry(Fibonacci::from_millis(5000).take(5), || { + match cmd.exec_with_timeout( + Duration::minutes(10), + |line| info!("{}", line), + |line| error!("{}", line), + ) { + Ok(_) => OperationResult::Ok(()), + Err(e) => { + warn!( + "failed to pull image from {} registry {}, {:?} retrying...", + registry_provider, + dest.as_str(), + e, + ); + OperationResult::Retry(e) + } + } + }) { + Err(Operation { error, .. }) => Err(SimpleError::new(SimpleErrorKind::Other, Some(error.to_string()))), + Err(e) => Err(SimpleError::new( + SimpleErrorKind::Other, + Some(format!( + "unknown error while trying to pull image {} from {} registry. {:?}", + dest.as_str(), + registry_provider, + e, + )), + )), + _ => { + info!( + "image {} has successfully been pulled from {} registry", + dest.as_str(), + registry_provider, + ); + Ok(()) + } + } +} diff --git a/src/container_registry/docker_hub.rs b/src/container_registry/docker_hub.rs index 317acaf2..f7d67c65 100644 --- a/src/container_registry/docker_hub.rs +++ b/src/container_registry/docker_hub.rs @@ -4,8 +4,8 @@ use reqwest::StatusCode; use crate::build_platform::Image; use crate::cmd::utilities::QoveryCommand; -use crate::container_registry::docker::docker_tag_and_push_image; -use crate::container_registry::{ContainerRegistry, EngineError, Kind, PushResult}; +use crate::container_registry::docker::{docker_pull_image, docker_tag_and_push_image}; +use crate::container_registry::{ContainerRegistry, EngineError, Kind, PullResult, PushResult}; use crate::error::EngineErrorCause; use crate::models::{ Context, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope, @@ -31,6 +31,45 @@ impl DockerHub { listeners: vec![], } } + + pub fn exec_docker_login(&self) -> Result<(), EngineError> { + let envs = match self.context.docker_tcp_socket() { + Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())], + None => vec![], + }; + + let mut cmd = QoveryCommand::new( + "docker", + &vec!["login", "-u", self.login.as_str(), "-p", self.password.as_str()], + &envs, + ); + + match cmd.exec() { + Ok(_) => Ok(()), + Err(_) => Err(self.engine_error( + EngineErrorCause::User( + "Your DockerHub account seems to be no longer valid (bad Credentials). \ + Please contact your Organization administrator to fix or change the Credentials.", + ), + format!("failed to login to DockerHub {}", self.name_with_id()), + )), + } + } + + fn pull_image(&self, dest: String, image: &Image) -> Result { + match docker_pull_image(self.kind(), vec![], dest.clone()) { + Ok(_) => { + let mut image = image.clone(); + image.registry_url = Some(dest); + Ok(PullResult::Some(image)) + } + Err(e) => Err(self.engine_error( + EngineErrorCause::Internal, + e.message + .unwrap_or_else(|| "unknown error occurring during docker pull".to_string()), + )), + } + } } impl ContainerRegistry for DockerHub { @@ -100,26 +139,51 @@ impl ContainerRegistry for DockerHub { } } - fn push(&self, image: &Image, force_push: bool) -> Result { - let envs = match self.context.docker_tcp_socket() { - Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())], - None => vec![], - }; + fn pull(&self, image: &Image) -> Result { + let listeners_helper = ListenersHelper::new(&self.listeners); - let mut cmd = QoveryCommand::new( - "docker", - &vec!["login", "-u", self.login.as_str(), "-p", self.password.as_str()], - &envs, - ); - if let Err(_) = cmd.exec() { - return Err(self.engine_error( - EngineErrorCause::User( - "Your DockerHub account seems to be no longer valid (bad Credentials). \ - Please contact your Organization administrator to fix or change the Credentials.", - ), - format!("failed to login to DockerHub {}", self.name_with_id()), + if !self.does_image_exists(image) { + let info_message = format!( + "image {:?} does not exist in DockerHub {} repository", + image, + self.name() + ); + info!("{}", info_message.as_str()); + + listeners_helper.deployment_in_progress(ProgressInfo::new( + ProgressScope::Application { + id: image.application_id.clone(), + }, + ProgressLevel::Info, + Some(info_message), + self.context.execution_id(), )); - }; + + return Ok(PullResult::None); + } + + let info_message = format!("pull image {:?} from DockerHub {} repository", image, self.name()); + info!("{}", info_message.as_str()); + + listeners_helper.deployment_in_progress(ProgressInfo::new( + ProgressScope::Application { + id: image.application_id.clone(), + }, + ProgressLevel::Info, + Some(info_message), + self.context.execution_id(), + )); + + let _ = self.exec_docker_login()?; + + let dest = format!("{}/{}", self.login.as_str(), image.name_with_tag().as_str()); + + // pull image + self.pull_image(dest, image) + } + + fn push(&self, image: &Image, force_push: bool) -> Result { + let _ = self.exec_docker_login()?; let dest = format!("{}/{}", self.login.as_str(), image.name_with_tag().as_str()); let listeners_helper = ListenersHelper::new(&self.listeners); diff --git a/src/container_registry/docr.rs b/src/container_registry/docr.rs index 329ed9e1..349bd8b1 100644 --- a/src/container_registry/docr.rs +++ b/src/container_registry/docr.rs @@ -5,8 +5,8 @@ use serde::{Deserialize, Serialize}; use crate::build_platform::Image; use crate::cmd::utilities::QoveryCommand; -use crate::container_registry::docker::docker_tag_and_push_image; -use crate::container_registry::{ContainerRegistry, EngineError, Kind, PushResult}; +use crate::container_registry::docker::{docker_pull_image, docker_tag_and_push_image}; +use crate::container_registry::{ContainerRegistry, EngineError, Kind, PullResult, PushResult}; use crate::error::{cast_simple_error_to_engine_error, EngineErrorCause, SimpleError, SimpleErrorKind}; use crate::models::{ Context, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope, @@ -120,7 +120,7 @@ impl DOCR { EngineErrorCause::Internal, e.message .unwrap_or_else(|| "unknown error occurring during docker push".to_string()), - )) + )); } }; @@ -189,6 +189,43 @@ impl DOCR { } } } + + pub fn exec_docr_login(&self) -> Result<(), EngineError> { + let mut cmd = QoveryCommand::new( + "doctl", + &vec!["registry", "login", self.name.as_str(), "-t", self.api_key.as_str()], + &vec![], + ); + + match cmd.exec() { + Ok(_) => Ok(()), + Err(_) => Err(self.engine_error( + EngineErrorCause::User( + "Your DOCR account seems to be no longer valid (bad Credentials). \ + Please contact your Organization administrator to fix or change the Credentials.", + ), + format!("failed to login to DOCR {}", self.name_with_id()), + )), + } + } + + fn pull_image(&self, registry_name: String, dest: String, image: &Image) -> Result { + match docker_pull_image(self.kind(), vec![], dest.clone()) { + Ok(_) => { + let mut image = image.clone(); + image.registry_name = Some(registry_name.clone()); + // on DOCR registry secret is the same as registry name + image.registry_secret = Some(registry_name); + image.registry_url = Some(dest); + Ok(PullResult::Some(image)) + } + Err(e) => Err(self.engine_error( + EngineErrorCause::Internal, + e.message + .unwrap_or_else(|| "unknown error occurring during docker pull".to_string()), + )), + } + } } impl ContainerRegistry for DOCR { @@ -305,6 +342,51 @@ impl ContainerRegistry for DOCR { } } + fn pull(&self, image: &Image) -> Result { + let listeners_helper = ListenersHelper::new(&self.listeners); + + if !self.does_image_exists(image) { + let info_message = format!("image {:?} does not exist in DOCR {} repository", image, self.name()); + info!("{}", info_message.as_str()); + + listeners_helper.deployment_in_progress(ProgressInfo::new( + ProgressScope::Application { + id: image.application_id.clone(), + }, + ProgressLevel::Info, + Some(info_message), + self.context.execution_id(), + )); + + return Ok(PullResult::None); + } + + let info_message = format!("pull image {:?} from DOCR {} repository", image, self.name()); + info!("{}", info_message.as_str()); + + listeners_helper.deployment_in_progress(ProgressInfo::new( + ProgressScope::Application { + id: image.application_id.clone(), + }, + ProgressLevel::Info, + Some(info_message), + self.context.execution_id(), + )); + + let _ = self.exec_docr_login()?; + + let registry_name = self.get_registry_name(image)?; + + let dest = format!( + "registry.digitalocean.com/{}/{}", + registry_name.as_str(), + image.name_with_tag() + ); + + // pull image + self.pull_image(registry_name, dest, image) + } + // https://www.digitalocean.com/docs/images/container-registry/how-to/use-registry-docker-kubernetes/ fn push(&self, image: &Image, force_push: bool) -> Result { let registry_name = self.get_registry_name(image)?; @@ -314,20 +396,7 @@ impl ContainerRegistry for DOCR { Err(_) => warn!("DOCR {} already exists", registry_name.as_str()), }; - let mut cmd = QoveryCommand::new( - "doctl", - &vec!["registry", "login", self.name.as_str(), "-t", self.api_key.as_str()], - &vec![], - ); - if let Err(_) = cmd.exec() { - return Err(self.engine_error( - EngineErrorCause::User( - "Your DOCR account seems to be no longer valid (bad Credentials). \ - Please contact your Organization administrator to fix or change the Credentials.", - ), - format!("failed to login to DOCR {}", self.name_with_id()), - )); - }; + let _ = self.exec_docr_login()?; let dest = format!( "registry.digitalocean.com/{}/{}", diff --git a/src/container_registry/ecr.rs b/src/container_registry/ecr.rs index 868fefbc..04b48d1a 100644 --- a/src/container_registry/ecr.rs +++ b/src/container_registry/ecr.rs @@ -10,8 +10,8 @@ use rusoto_sts::{GetCallerIdentityRequest, Sts, StsClient}; use crate::build_platform::Image; use crate::cmd::utilities::QoveryCommand; -use crate::container_registry::docker::docker_tag_and_push_image; -use crate::container_registry::{ContainerRegistry, Kind, PushResult}; +use crate::container_registry::docker::{docker_pull_image, docker_tag_and_push_image}; +use crate::container_registry::{ContainerRegistry, Kind, PullResult, PushResult}; use crate::error::{EngineError, EngineErrorCause}; use crate::models::{ Context, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope, @@ -136,6 +136,24 @@ impl ECR { } } + fn pull_image(&self, dest: String, image: &Image) -> Result { + // READ https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-pull-ecr-image.html + // docker pull aws_account_id.dkr.ecr.us-west-2.amazonaws.com/amazonlinux:latest + + match docker_pull_image(self.kind(), self.docker_envs(), dest.clone()) { + Ok(_) => { + let mut image = image.clone(); + image.registry_url = Some(dest); + Ok(PullResult::Some(image)) + } + Err(e) => Err(self.engine_error( + EngineErrorCause::Internal, + e.message + .unwrap_or_else(|| "unknown error occurring during docker pull".to_string()), + )), + } + } + fn create_repository(&self, image: &Image) -> Result { let repository_name = image.name.as_str(); info!("creating ECR repository {}", &repository_name); @@ -263,6 +281,82 @@ impl ECR { self.create_repository(image) } + + fn get_credentials(&self) -> Result { + let r = block_on( + self.ecr_client() + .get_authorization_token(GetAuthorizationTokenRequest::default()), + ); + + let (access_token, password, endpoint_url) = match r { + Ok(t) => match t.authorization_data { + Some(authorization_data) => { + let ad = authorization_data.first().unwrap(); + let b64_token = ad.authorization_token.as_ref().unwrap(); + + let decoded_token = base64::decode(b64_token).unwrap(); + let token = std::str::from_utf8(decoded_token.as_slice()).unwrap(); + + let s_token: Vec<&str> = token.split(':').collect::>(); + + ( + s_token.first().unwrap().to_string(), + s_token.get(1).unwrap().to_string(), + ad.clone().proxy_endpoint.unwrap(), + ) + } + None => { + return Err(self.engine_error( + EngineErrorCause::Internal, + format!( + "failed to retrieve credentials and endpoint URL from ECR {}", + self.name_with_id(), + ), + )); + } + }, + _ => { + return Err(self.engine_error( + EngineErrorCause::Internal, + format!( + "failed to retrieve credentials and endpoint URL from ECR {}", + self.name_with_id(), + ), + )); + } + }; + + Ok(ECRCredentials::new(access_token, password, endpoint_url)) + } + + fn exec_docker_login(&self) -> Result<(), EngineError> { + let credentials = self.get_credentials()?; + + let mut cmd = QoveryCommand::new( + "docker", + &vec![ + "login", + "-u", + credentials.access_token.as_str(), + "-p", + credentials.password.as_str(), + credentials.endpoint_url.as_str(), + ], + &self.docker_envs(), + ); + + if let Err(_) = cmd.exec() { + return Err(self.engine_error( + EngineErrorCause::User( + "Your ECR account seems to be no longer valid (bad Credentials). \ + Please contact your Organization administrator to fix or change the Credentials.", + ), + format!("failed to login to ECR {}", self.name_with_id()), + )); + }; + + Ok(()) + } } impl ContainerRegistry for ECR { @@ -319,50 +413,62 @@ impl ContainerRegistry for ECR { self.get_image(image).is_some() } - fn push(&self, image: &Image, force_push: bool) -> Result { - let r = block_on( - self.ecr_client() - .get_authorization_token(GetAuthorizationTokenRequest::default()), - ); + fn pull(&self, image: &Image) -> Result { + let listeners_helper = ListenersHelper::new(&self.listeners); - let (access_token, password, endpoint_url) = match r { - Ok(t) => match t.authorization_data { - Some(authorization_data) => { - let ad = authorization_data.first().unwrap(); - let b64_token = ad.authorization_token.as_ref().unwrap(); + if !self.does_image_exists(image) { + let info_message = format!("image {:?} does not exist in ECR {} repository", image, self.name()); + info!("{}", info_message.as_str()); - let decoded_token = base64::decode(b64_token).unwrap(); - let token = std::str::from_utf8(decoded_token.as_slice()).unwrap(); + listeners_helper.deployment_in_progress(ProgressInfo::new( + ProgressScope::Application { + id: image.application_id.clone(), + }, + ProgressLevel::Info, + Some(info_message), + self.context.execution_id(), + )); - let s_token: Vec<&str> = token.split(':').collect::>(); + return Ok(PullResult::None); + } - ( - s_token.first().unwrap().to_string(), - s_token.get(1).unwrap().to_string(), - ad.clone().proxy_endpoint.unwrap(), - ) - } - None => { - return Err(self.engine_error( - EngineErrorCause::Internal, - format!( - "failed to retrieve credentials and endpoint URL from ECR {}", - self.name_with_id(), - ), - )); - } + let info_message = format!("pull image {:?} from ECR {} repository", image, self.name()); + info!("{}", info_message.as_str()); + + listeners_helper.deployment_in_progress(ProgressInfo::new( + ProgressScope::Application { + id: image.application_id.clone(), }, + ProgressLevel::Info, + Some(info_message), + self.context.execution_id(), + )); + + let _ = self.exec_docker_login()?; + + let repository = match self.get_or_create_repository(image) { + Ok(r) => r, _ => { return Err(self.engine_error( EngineErrorCause::Internal, format!( - "failed to retrieve credentials and endpoint URL from ECR {}", + "failed to create ECR repository for {} with image {:?}", self.name_with_id(), + image, ), )); } }; + let dest = format!("{}:{}", repository.repository_uri.unwrap(), image.tag.as_str()); + + // pull image + self.pull_image(dest, image) + } + + fn push(&self, image: &Image, force_push: bool) -> Result { + let _ = self.exec_docker_login()?; + let repository = match if force_push { self.create_repository(image) } else { @@ -381,29 +487,6 @@ impl ContainerRegistry for ECR { } }; - let mut cmd = QoveryCommand::new( - "docker", - &vec![ - "login", - "-u", - access_token.as_str(), - "-p", - password.as_str(), - endpoint_url.as_str(), - ], - &self.docker_envs(), - ); - - if let Err(_) = cmd.exec() { - return Err(self.engine_error( - EngineErrorCause::User( - "Your ECR account seems to be no longer valid (bad Credentials). \ - Please contact your Organization administrator to fix or change the Credentials.", - ), - format!("failed to login to ECR {}", self.name_with_id()), - )); - }; - let dest = format!("{}:{}", repository.repository_uri.unwrap(), image.tag.as_str()); let listeners_helper = ListenersHelper::new(&self.listeners); @@ -468,3 +551,19 @@ impl Listen for ECR { self.listeners.push(listener); } } + +struct ECRCredentials { + access_token: String, + password: String, + endpoint_url: String, +} + +impl ECRCredentials { + fn new(access_token: String, password: String, endpoint_url: String) -> Self { + ECRCredentials { + access_token, + password, + endpoint_url, + } + } +} diff --git a/src/container_registry/mod.rs b/src/container_registry/mod.rs index dd10b3ad..e9c3950e 100644 --- a/src/container_registry/mod.rs +++ b/src/container_registry/mod.rs @@ -24,6 +24,7 @@ pub trait ContainerRegistry: Listen { fn on_delete(&self) -> Result<(), EngineError>; fn on_delete_error(&self) -> Result<(), EngineError>; fn does_image_exists(&self, image: &Image) -> bool; + fn pull(&self, image: &Image) -> Result; fn push(&self, image: &Image, force_push: bool) -> Result; fn push_error(&self, image: &Image) -> Result; fn engine_error_scope(&self) -> EngineErrorScope { @@ -43,6 +44,11 @@ pub struct PushResult { pub image: Image, } +pub enum PullResult { + Some(Image), + None, +} + #[derive(Serialize, Deserialize, Clone, Copy, Debug)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum Kind { diff --git a/src/container_registry/scaleway_container_registry.rs b/src/container_registry/scaleway_container_registry.rs index cf8d2731..2b45fb7d 100644 --- a/src/container_registry/scaleway_container_registry.rs +++ b/src/container_registry/scaleway_container_registry.rs @@ -3,8 +3,10 @@ extern crate scaleway_api_rs; use crate::cloud_provider::scaleway::application::ScwZone; use crate::build_platform::Image; -use crate::container_registry::docker::{docker_login, docker_manifest_inspect, docker_tag_and_push_image}; -use crate::container_registry::{ContainerRegistry, Kind, PushResult}; +use crate::container_registry::docker::{ + docker_login, docker_manifest_inspect, docker_pull_image, docker_tag_and_push_image, +}; +use crate::container_registry::{ContainerRegistry, Kind, PullResult, PushResult}; use crate::error::{EngineError, EngineErrorCause}; use crate::models::{ Context, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope, @@ -165,14 +167,14 @@ impl ScalewayCR { } } - fn push_image(&self, image_url: String, image: &Image) -> Result { + fn push_image(&self, dest: String, image: &Image) -> Result { // https://www.scaleway.com/en/docs/deploy-an-image-from-registry-to-kubernetes-kapsule/ match docker_tag_and_push_image( self.kind(), self.get_docker_envs(), image.name.clone(), image.tag.clone(), - image_url, + dest, ) { Ok(_) => {} Err(e) => { @@ -206,6 +208,21 @@ impl ScalewayCR { } } + fn pull_image(&self, dest: String, image: &Image) -> Result { + match docker_pull_image(self.kind(), self.get_docker_envs(), dest) { + Ok(_) => {} + Err(e) => { + return Err(self.engine_error( + EngineErrorCause::Internal, + e.message + .unwrap_or_else(|| "unknown error occurring during docker pull".to_string()), + )) + } + }; + + Ok(PullResult::Some(image.clone())) + } + pub fn create_registry_namespace( &self, image: &Image, @@ -295,6 +312,28 @@ impl ScalewayCR { .as_bytes(), ) } + + fn exec_docker_login(&self, registry_url: &String) -> Result<(), EngineError> { + if docker_login( + Kind::ScalewayCr, + self.get_docker_envs(), + self.login.clone(), + self.secret_token.clone(), + registry_url.clone(), + ) + .is_err() + { + return Err(self.engine_error( + EngineErrorCause::User( + "Your Scaleway account seems to be no longer valid (bad Credentials). \ + Please contact your Organization administrator to fix or change the Credentials.", + ), + format!("failed to login to Scaleway {}", self.name_with_id()), + )); + }; + + Ok(()) + } } impl ContainerRegistry for ScalewayCR { @@ -361,6 +400,70 @@ impl ContainerRegistry for ScalewayCR { .is_some() } + fn pull(&self, image: &Image) -> Result { + let listeners_helper = ListenersHelper::new(&self.listeners); + + let mut image = image.clone(); + let registry_url: String; + + match self.get_or_create_registry_namespace(&image) { + Ok(registry) => { + info!( + "Scaleway registry namespace for {} has been created", + image.name.as_str() + ); + image.registry_name = Some(image.name.clone()); // Note: Repository namespace should have the same name as the image name + image.registry_url = registry.endpoint.clone(); + image.registry_secret = Some(self.secret_token.clone()); + image.registry_docker_json_config = Some(self.get_docker_json_config_raw()); + registry_url = registry.endpoint.unwrap_or_else(|| "undefined".to_string()); + } + Err(e) => { + error!( + "Scaleway registry namespace for {} cannot be created, error: {:?}", + image.name.as_str(), + e + ); + return Err(e); + } + } + + if !self.does_image_exists(&image) { + let info_message = format!("image {:?} does not exist in SCR {} repository", image, self.name()); + info!("{}", info_message.as_str()); + + listeners_helper.deployment_in_progress(ProgressInfo::new( + ProgressScope::Application { + id: image.application_id.clone(), + }, + ProgressLevel::Info, + Some(info_message), + self.context.execution_id(), + )); + + return Ok(PullResult::None); + } + + let info_message = format!("pull image {:?} from SCR {} repository", image, self.name()); + info!("{}", info_message.as_str()); + + listeners_helper.deployment_in_progress(ProgressInfo::new( + ProgressScope::Application { + id: image.application_id.clone(), + }, + ProgressLevel::Info, + Some(info_message), + self.context.execution_id(), + )); + + let _ = self.exec_docker_login(®istry_url)?; + + let dest = format!("{}/{}", registry_url, image.name_with_tag()); + + // pull image + self.pull_image(dest, &image) + } + fn push(&self, image: &Image, force_push: bool) -> Result { let mut image = image.clone(); let registry_url: String; @@ -389,25 +492,9 @@ impl ContainerRegistry for ScalewayCR { } } - if docker_login( - Kind::ScalewayCr, - self.get_docker_envs(), - self.login.clone(), - self.secret_token.clone(), - registry_url.clone(), - ) - .is_err() - { - return Err(self.engine_error( - EngineErrorCause::User( - "Your Scaleway account seems to be no longer valid (bad Credentials). \ - Please contact your Organization administrator to fix or change the Credentials.", - ), - format!("failed to login to Scaleway {}", self.name_with_id()), - )); - }; + let _ = self.exec_docker_login(®istry_url)?; - let image_url = format!("{}/{}", registry_url, image.name_with_tag()); + let dest = format!("{}/{}", registry_url, image.name_with_tag()); let listeners_helper = ListenersHelper::new(&self.listeners); @@ -449,7 +536,7 @@ impl ContainerRegistry for ScalewayCR { self.context.execution_id(), )); - self.push_image(image_url, &image) + self.push_image(dest, &image) } fn push_error(&self, image: &Image) -> Result { diff --git a/src/git.rs b/src/git.rs index db989747..d4fecea3 100644 --- a/src/git.rs +++ b/src/git.rs @@ -3,7 +3,7 @@ use std::path::Path; use git2::build::{CheckoutBuilder, RepoBuilder}; use git2::ErrorCode::Auth; use git2::ResetType::Hard; -use git2::{Cred, CredentialType, Error, RemoteCallbacks, Repository, SubmoduleUpdateOptions}; +use git2::{Cred, CredentialType, Error, Object, Oid, RemoteCallbacks, Repository, SubmoduleUpdateOptions}; use url::Url; // Credentials callback is called endlessly until the server return Auth Ok (or a definitive error) @@ -46,7 +46,7 @@ fn authentication_callback<'a>( }; } -fn checkout(repo: &Repository, commit_id: &str) -> Result<(), Error> { +fn checkout<'a>(repo: &'a Repository, commit_id: &'a str) -> Result, Error> { let obj = repo.revparse_single(commit_id).map_err(|err| { let repo_url = repo .find_remote("origin") @@ -63,7 +63,8 @@ fn checkout(repo: &Repository, commit_id: &str) -> Result<(), Error> { let mut checkout_opts = CheckoutBuilder::new(); checkout_opts.force().remove_ignored(true).remove_untracked(true); - repo.reset(&obj, Hard, Some(&mut checkout_opts)) + let _ = repo.reset(&obj, Hard, Some(&mut checkout_opts))?; + Ok(obj) } fn clone

( @@ -92,6 +93,11 @@ where // Get our repository let mut repo = RepoBuilder::new(); repo.fetch_options(fo); + + if into_dir.as_ref().exists() { + let _ = std::fs::remove_dir_all(into_dir.as_ref()); + } + repo.clone(url.as_str(), into_dir.as_ref()) } @@ -108,7 +114,7 @@ where let repo = clone(repository_url, into_dir, get_credentials)?; // position the repo at the correct commit - checkout(&repo, commit_id)?; + let _ = checkout(&repo, commit_id)?; // check submodules if needed { @@ -133,9 +139,30 @@ where Ok(repo) } +pub fn get_parent_commit_id

( + repository_url: &str, + commit_id: &str, + into_dir: P, + get_credentials: &impl Fn(&str) -> Vec<(CredentialType, Cred)>, +) -> Result, Error> +where + P: AsRef, +{ + // clone repository + let repo = clone(repository_url, into_dir, get_credentials)?; + + let oid = Oid::from_str(commit_id)?; + let commit = match repo.find_commit(oid) { + Ok(commit) => commit, + Err(_) => return Ok(None), + }; + + Ok(commit.parent_ids().next().map(|x| x.to_string())) +} + #[cfg(test)] mod tests { - use crate::git::{checkout, clone, clone_at_commit}; + use crate::git::{checkout, clone, clone_at_commit, get_parent_commit_id}; use git2::{Cred, CredentialType}; struct DirectoryToDelete<'a> { @@ -232,10 +259,45 @@ mod tests { let commit = "9a9c1f4373c8128151a9def9ea3d838fa2ed33e8"; assert_ne!(repo.head().unwrap().target().unwrap().to_string(), commit); let check = checkout(&repo, commit); - assert!(matches!(check, Ok(()))); + assert!(matches!(check, Ok(_))); assert_eq!(repo.head().unwrap().target().unwrap().to_string(), commit); } + #[test] + fn test_git_parent_id() { + let clone_dir = DirectoryToDelete { + path: "/tmp/engine_test_parent_id", + }; + + let result = get_parent_commit_id( + "https://github.com/Qovery/engine-testing.git", + "964f02f3a3065bc7f6fb745d679b1ddb21153cc7", + clone_dir.path, + &|_| vec![], + ) + .unwrap() + .unwrap(); + + assert_eq!(result, "1538fb6333b86798f0cf865558a28e729a98dace".to_string()); + } + + #[test] + fn test_git_parent_id_not_existing() { + let clone_dir = DirectoryToDelete { + path: "/tmp/engine_test_parent_id_not_existing", + }; + + let result = get_parent_commit_id( + "https://github.com/Qovery/engine-testing.git", + "964f02f3a3065bc7f6fb745d679b1ddb21153cc0", + clone_dir.path, + &|_| vec![], + ) + .unwrap(); + + assert_eq!(result, None); + } + #[test] #[ignore] fn test_git_submodule_with_ssh_key() { diff --git a/src/models.rs b/src/models.rs index f9303db2..b7695ede 100644 --- a/src/models.rs +++ b/src/models.rs @@ -1,10 +1,17 @@ -use std::hash::{Hash, Hasher}; +use std::collections::BTreeMap; +use std::fmt::{Display, Formatter}; +use std::hash::Hash; +use std::net::Ipv4Addr; +use std::path::Path; +use std::str::FromStr; +use std::sync::Arc; use chrono::{DateTime, Utc}; +use git2::{Cred, CredentialType, Error}; +use itertools::Itertools; use rand::distributions::Alphanumeric; use rand::Rng; use serde::{Deserialize, Serialize}; -use std::collections::hash_map::DefaultHasher; use crate::build_platform::{Build, BuildOptions, Credentials, GitRepository, Image, SshKey}; use crate::cloud_provider::aws::databases::mongodb::MongoDB; @@ -15,12 +22,8 @@ use crate::cloud_provider::service::{DatabaseOptions, StatefulService, Stateless use crate::cloud_provider::utilities::VersionsNumber; use crate::cloud_provider::CloudProvider; use crate::cloud_provider::Kind as CPKind; -use itertools::Itertools; -use std::collections::BTreeMap; -use std::fmt::{Display, Formatter}; -use std::net::Ipv4Addr; -use std::str::FromStr; -use std::sync::Arc; +use crate::git; +use crate::utilities::get_image_tag; #[derive(Clone, Debug)] pub struct QoveryIdentifier { @@ -347,24 +350,42 @@ impl Application { } pub fn to_image(&self) -> Image { - // Image tag == hash(root_path) + commit_id truncate to 127 char - // https://github.com/distribution/distribution/blob/6affafd1f030087d88f88841bf66a8abe2bf4d24/reference/regexp.go#L41 - let mut hasher = DefaultHasher::new(); + self.to_image_with_commit(&self.commit_id) + } - // If any of those variables changes, we'll get a new hash value, what results in a new image - // build and avoids using cache. It is important to build a new image, as those variables may - // affect the build result even if user didn't change his code. - self.root_path.hash(&mut hasher); - self.dockerfile_path.hash(&mut hasher); - self.environment_vars.hash(&mut hasher); + pub fn to_image_from_parent_commit

(&self, clone_repo_into_dir: P) -> Result, Error> + where + P: AsRef, + { + let parent_commit_id = git::get_parent_commit_id( + self.git_url.as_str(), + self.commit_id.as_str(), + clone_repo_into_dir, + &|_| match &self.git_credentials { + None => vec![], + Some(creds) => vec![( + CredentialType::USER_PASS_PLAINTEXT, + Cred::userpass_plaintext(creds.login.as_str(), creds.access_token.as_str()).unwrap(), + )], + }, + )?; - let mut tag = format!("{}-{}", hasher.finish(), self.commit_id); - tag.truncate(127); + Ok(match parent_commit_id { + Some(id) => Some(self.to_image_with_commit(&id)), + None => None, + }) + } + pub fn to_image_with_commit(&self, commit_id: &String) -> Image { Image { application_id: self.id.clone(), name: self.name.clone(), - tag, + tag: get_image_tag( + &self.root_path, + &self.dockerfile_path, + &self.environment_vars, + commit_id, + ), commit_id: self.commit_id.clone(), registry_name: None, registry_secret: None, diff --git a/src/transaction.rs b/src/transaction.rs index ced2af3d..ccd4abf7 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::thread; -use crate::build_platform::BuildResult; +use crate::build_platform::{BuildResult, CacheResult}; use crate::cloud_provider::kubernetes::Kubernetes; use crate::cloud_provider::service::{Application, Service}; use crate::container_registry::PushResult; @@ -134,7 +134,53 @@ impl<'a> Transaction<'a> { Ok(()) } - fn _build_applications( + fn load_build_app_cache(&self, app: &crate::models::Application) -> Result<(), EngineError> { + // do load build cache before building app + let build = app.to_build(); + let _ = match self.engine.build_platform().has_cache(&build) { + Ok(CacheResult::MissWithoutParentBuild) => { + info!("first build for app {} - cache miss", app.name.as_str()); + } + Ok(CacheResult::Hit) => { + info!("cache hit for app {}", app.name.as_str()); + } + Ok(CacheResult::Miss(parent_build)) => { + info!("cache miss for app {}", app.name.as_str()); + + let container_registry = self.engine.container_registry(); + + // pull image from container registry + // FIXME: if one day we use something else than LocalDocker to build image + // FIXME: we'll need to send the PullResult to the Build implementation + let _ = match container_registry.pull(&parent_build.image) { + Ok(pull_result) => pull_result, + Err(err) => { + warn!( + "{}", + err.message.clone().unwrap_or(format!( + "something goes wrong while pulling image from {:?} container registry", + container_registry.kind() + )) + ); + return Err(err); + } + }; + } + Err(err) => { + warn!( + "load build app {} cache error: {}", + app.name.as_str(), + err.message.clone().unwrap_or("".to_string()) + ); + + return Err(err); + } + }; + + Ok(()) + } + + fn build_applications( &self, environment: &Environment, option: &DeploymentOption, @@ -151,6 +197,9 @@ impl<'a> Transaction<'a> { let image = app.to_image(); let build_result = if option.force_build || !self.engine.container_registry().does_image_exists(&image) { + // If an error occurred we can skip it. It's not critical. + let _ = self.load_build_app_cache(app); + // only if the build is forced OR if the image does not exist in the registry self.engine.build_platform().build(app.to_build(), option.force_build) } else { @@ -185,7 +234,7 @@ impl<'a> Transaction<'a> { Ok(applications) } - fn _push_applications( + fn push_applications( &self, applications: Vec>, option: &DeploymentOption, @@ -393,8 +442,8 @@ impl<'a> Transaction<'a> { EnvironmentAction::EnvironmentWithFailover(te, _) => te, }; - let apps_result = match self._build_applications(target_environment, option) { - Ok(applications) => match self._push_applications(applications, option) { + let apps_result = match self.build_applications(target_environment, option) { + Ok(applications) => match self.push_applications(applications, option) { Ok(results) => { let applications = results.into_iter().map(|(app, _)| app).collect::>(); @@ -423,8 +472,8 @@ impl<'a> Transaction<'a> { // build as well the failover environment, retention could remove the application image if let EnvironmentAction::EnvironmentWithFailover(_, fe) = environment_action { - let apps_result = match self._build_applications(fe, option) { - Ok(applications) => match self._push_applications(applications, option) { + let apps_result = match self.build_applications(fe, option) { + Ok(applications) => match self.push_applications(applications, option) { Ok(results) => { let applications = results.into_iter().map(|(app, _)| app).collect::>(); diff --git a/src/utilities.rs b/src/utilities.rs index 553c18f1..ef46e10a 100644 --- a/src/utilities.rs +++ b/src/utilities.rs @@ -1,7 +1,9 @@ +use std::collections::hash_map::DefaultHasher; +use std::collections::BTreeMap; +use std::hash::{Hash, Hasher}; + use reqwest::header; use reqwest::header::{HeaderMap, HeaderValue}; -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; // generate the right header for digital ocean with token pub fn get_header_with_bearer(token: &str) -> HeaderMap { @@ -16,3 +18,95 @@ pub fn calculate_hash(t: &T) -> u64 { t.hash(&mut s); s.finish() } + +pub fn get_image_tag( + root_path: &String, + dockerfile_path: &Option, + environment_variables: &BTreeMap, + commit_id: &String, +) -> String { + // Image tag == hash(root_path) + commit_id truncate to 127 char + // https://github.com/distribution/distribution/blob/6affafd1f030087d88f88841bf66a8abe2bf4d24/reference/regexp.go#L41 + let mut hasher = DefaultHasher::new(); + + // If any of those variables changes, we'll get a new hash value, what results in a new image + // build and avoids using cache. It is important to build a new image, as those variables may + // affect the build result even if user didn't change his code. + root_path.hash(&mut hasher); + + if dockerfile_path.is_some() { + // only use when a Dockerfile is used to prevent build cache miss every single time + // we redeploy an app with a env var changed with Buildpacks. + dockerfile_path.hash(&mut hasher); + + // TODO check if the environment variables are used in the Dockerfile and only Hash the one that are used + environment_variables.hash(&mut hasher); + } + + let mut tag = format!("{}-{}", hasher.finish(), commit_id); + tag.truncate(127); + + tag +} + +#[cfg(test)] +mod tests_utilities { + use crate::utilities::get_image_tag; + use std::collections::BTreeMap; + + #[test] + fn test_get_image_tag() { + let image_tag = get_image_tag( + &"/".to_string(), + &Some("Dockerfile".to_string()), + &BTreeMap::new(), + &"63d8c437337416a7067d3f358197ac47d003fab9".to_string(), + ); + + let image_tag_2 = get_image_tag( + &"/".to_string(), + &Some("Dockerfile.qovery".to_string()), + &BTreeMap::new(), + &"63d8c437337416a7067d3f358197ac47d003fab9".to_string(), + ); + + assert_ne!(image_tag, image_tag_2); + + let image_tag_3 = get_image_tag( + &"/xxx".to_string(), + &Some("Dockerfile.qovery".to_string()), + &BTreeMap::new(), + &"63d8c437337416a7067d3f358197ac47d003fab9".to_string(), + ); + + assert_ne!(image_tag, image_tag_3); + + let image_tag_3_2 = get_image_tag( + &"/xxx".to_string(), + &Some("Dockerfile.qovery".to_string()), + &BTreeMap::new(), + &"63d8c437337416a7067d3f358197ac47d003fab9".to_string(), + ); + + assert_eq!(image_tag_3, image_tag_3_2); + + let image_tag_4 = get_image_tag( + &"/".to_string(), + &None, + &BTreeMap::new(), + &"63d8c437337416a7067d3f358197ac47d003fab9".to_string(), + ); + + let mut env_vars_5 = BTreeMap::new(); + env_vars_5.insert("toto".to_string(), "key".to_string()); + + let image_tag_5 = get_image_tag( + &"/".to_string(), + &None, + &env_vars_5, + &"63d8c437337416a7067d3f358197ac47d003fab9".to_string(), + ); + + assert_eq!(image_tag_4, image_tag_5); + } +} diff --git a/tests/aws/aws_environment.rs b/tests/aws/aws_environment.rs index 70f26681..df61dee1 100644 --- a/tests/aws/aws_environment.rs +++ b/tests/aws/aws_environment.rs @@ -5,12 +5,17 @@ use self::test_utilities::utilities::{ engine_run_test, generate_id, get_pods, get_pvc, is_pod_restarted_env, logger, FuncTestsSecrets, }; use ::function_name::named; +use qovery_engine::build_platform::{BuildPlatform, BuildResult, CacheResult}; use qovery_engine::cloud_provider::Kind; use qovery_engine::cmd::kubectl::kubernetes_get_all_pdbs; +use qovery_engine::container_registry::{ContainerRegistry, PullResult, PushResult}; +use qovery_engine::error::EngineError; use qovery_engine::models::{Action, Clone2, EnvironmentAction, Port, Protocol, Storage, StorageType}; use qovery_engine::transaction::TransactionResult; use std::collections::BTreeMap; -use test_utilities::utilities::{context, init, kubernetes_config_path}; +use std::time::SystemTime; +use test_utilities::aws::container_registry_ecr; +use test_utilities::utilities::{build_platform_local_docker, context, init, kubernetes_config_path}; use tracing::{span, Level}; // TODO: @@ -74,6 +79,97 @@ fn deploy_a_working_environment_with_no_router_on_aws_eks() { }) } +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn test_build_cache() { + let test_name = function_name!(); + engine_run_test(|| { + init(); + let span = span!(Level::INFO, "test", name = test_name); + let _enter = span.enter(); + + let logger = logger(); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + + let mut environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + + let ecr = container_registry_ecr(&context); + let local_docker = build_platform_local_docker(&context); + let app = environment.applications.first().unwrap(); + let image = app.to_image(); + + let app_build = app.to_build(); + let _ = match local_docker.has_cache(&app_build) { + Ok(CacheResult::Hit) => assert!(false), + Ok(CacheResult::Miss(parent_build)) => assert!(true), + Ok(CacheResult::MissWithoutParentBuild) => assert!(false), + Err(err) => assert!(false), + }; + + let _ = match ecr.pull(&image).unwrap() { + PullResult::Some(_) => assert!(false), + PullResult::None => assert!(true), + }; + + let build_result = local_docker.build(app.to_build(), false).unwrap(); + + let _ = match ecr.push(&build_result.build.image, false) { + Ok(_) => assert!(true), + Err(_) => assert!(false), + }; + + // TODO clean local docker cache + + let start_pull_time = SystemTime::now(); + let _ = match ecr.pull(&build_result.build.image).unwrap() { + PullResult::Some(_) => assert!(true), + PullResult::None => assert!(false), + }; + + let pull_duration = SystemTime::now().duration_since(start_pull_time).unwrap(); + + let _ = match local_docker.has_cache(&build_result.build) { + Ok(CacheResult::Hit) => assert!(true), + Ok(CacheResult::Miss(parent_build)) => assert!(false), + Ok(CacheResult::MissWithoutParentBuild) => assert!(false), + Err(err) => assert!(false), + }; + + let start_pull_time = SystemTime::now(); + let _ = match ecr.pull(&image).unwrap() { + PullResult::Some(_) => assert!(true), + PullResult::None => assert!(false), + }; + + let pull_duration_2 = SystemTime::now().duration_since(start_pull_time).unwrap(); + + if pull_duration_2.as_millis() > pull_duration.as_millis() { + assert!(false); + } + + return test_name.to_string(); + }) +} + #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] diff --git a/tests/digitalocean/do_environment.rs b/tests/digitalocean/do_environment.rs index 6c24f28c..68e99e00 100644 --- a/tests/digitalocean/do_environment.rs +++ b/tests/digitalocean/do_environment.rs @@ -6,12 +6,16 @@ use self::test_utilities::utilities::{ engine_run_test, generate_id, get_pods, get_pvc, init, is_pod_restarted_env, logger, FuncTestsSecrets, }; use ::function_name::named; +use qovery_engine::build_platform::{BuildPlatform, CacheResult}; use qovery_engine::cloud_provider::Kind; +use qovery_engine::container_registry::{ContainerRegistry, PullResult}; use qovery_engine::models::{Action, Clone2, EnvironmentAction, Port, Protocol, Storage, StorageType}; use qovery_engine::transaction::TransactionResult; use std::collections::BTreeMap; +use std::time::SystemTime; use test_utilities::common::Infrastructure; -use test_utilities::utilities::context; +use test_utilities::digitalocean::container_registry_digital_ocean; +use test_utilities::utilities::{build_platform_local_docker, context}; use tracing::{span, warn, Level}; // Note: All those tests relies on a test cluster running on DigitalOcean infrastructure. @@ -78,6 +82,95 @@ fn digitalocean_doks_deploy_a_working_environment_with_no_router() { }) } +#[cfg(feature = "test-do-self-hosted")] +#[named] +#[test] +fn test_build_cache() { + let test_name = function_name!(); + engine_run_test(|| { + init(); + let span = span!(Level::INFO, "test", name = test_name); + let _enter = span.enter(); + + let logger = logger(); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let mut environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + + let docr = container_registry_digital_ocean(&context); + let local_docker = build_platform_local_docker(&context); + let app = environment.applications.first().unwrap(); + let image = app.to_image(); + + let app_build = app.to_build(); + let _ = match local_docker.has_cache(&app_build) { + Ok(CacheResult::Hit) => assert!(false), + Ok(CacheResult::Miss(parent_build)) => assert!(true), + Ok(CacheResult::MissWithoutParentBuild) => assert!(false), + Err(err) => assert!(false), + }; + + let _ = match docr.pull(&image).unwrap() { + PullResult::Some(_) => assert!(false), + PullResult::None => assert!(true), + }; + + let build_result = local_docker.build(app.to_build(), false).unwrap(); + + let _ = match docr.push(&build_result.build.image, false) { + Ok(_) => assert!(true), + Err(_) => assert!(false), + }; + + // TODO clean local docker cache + + let start_pull_time = SystemTime::now(); + let _ = match docr.pull(&build_result.build.image).unwrap() { + PullResult::Some(_) => assert!(true), + PullResult::None => assert!(false), + }; + + let pull_duration = SystemTime::now().duration_since(start_pull_time).unwrap(); + + let _ = match local_docker.has_cache(&build_result.build) { + Ok(CacheResult::Hit) => assert!(true), + Ok(CacheResult::Miss(parent_build)) => assert!(false), + Ok(CacheResult::MissWithoutParentBuild) => assert!(false), + Err(err) => assert!(false), + }; + + let start_pull_time = SystemTime::now(); + let _ = match docr.pull(&image).unwrap() { + PullResult::Some(_) => assert!(true), + PullResult::None => assert!(false), + }; + + let pull_duration_2 = SystemTime::now().duration_since(start_pull_time).unwrap(); + + if pull_duration_2.as_millis() > pull_duration.as_millis() { + assert!(false); + } + + return test_name.to_string(); + }) +} + #[cfg(feature = "test-do-self-hosted")] #[named] #[test] diff --git a/tests/scaleway/scw_environment.rs b/tests/scaleway/scw_environment.rs index fb3716b0..8c250c41 100644 --- a/tests/scaleway/scw_environment.rs +++ b/tests/scaleway/scw_environment.rs @@ -6,11 +6,16 @@ use self::test_utilities::utilities::{ context, engine_run_test, generate_id, get_pods, get_pvc, init, is_pod_restarted_env, logger, FuncTestsSecrets, }; use ::function_name::named; +use qovery_engine::build_platform::{BuildPlatform, CacheResult}; use qovery_engine::cloud_provider::Kind; +use qovery_engine::container_registry::{ContainerRegistry, PullResult}; use qovery_engine::models::{Action, Clone2, EnvironmentAction, Port, Protocol, Storage, StorageType}; use qovery_engine::transaction::TransactionResult; use std::collections::BTreeMap; +use std::time::SystemTime; use test_utilities::common::Infrastructure; +use test_utilities::scaleway::container_registry_scw; +use test_utilities::utilities::build_platform_local_docker; use tracing::{span, warn, Level}; // Note: All those tests relies on a test cluster running on Scaleway infrastructure. @@ -80,6 +85,97 @@ fn scaleway_kapsule_deploy_a_working_environment_with_no_router() { }) } +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn test_build_cache() { + let test_name = function_name!(); + engine_run_test(|| { + init(); + let span = span!(Level::INFO, "test", name = test_name); + let _enter = span.enter(); + + let logger = logger(); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + + let mut environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + + let scr = container_registry_scw(&context); + let local_docker = build_platform_local_docker(&context); + let app = environment.applications.first().unwrap(); + let image = app.to_image(); + + let app_build = app.to_build(); + let _ = match local_docker.has_cache(&app_build) { + Ok(CacheResult::Hit) => assert!(false), + Ok(CacheResult::Miss(parent_build)) => assert!(true), + Ok(CacheResult::MissWithoutParentBuild) => assert!(false), + Err(err) => assert!(false), + }; + + let _ = match scr.pull(&image).unwrap() { + PullResult::Some(_) => assert!(false), + PullResult::None => assert!(true), + }; + + let build_result = local_docker.build(app.to_build(), false).unwrap(); + + let _ = match scr.push(&build_result.build.image, false) { + Ok(_) => assert!(true), + Err(_) => assert!(false), + }; + + // TODO clean local docker cache + + let start_pull_time = SystemTime::now(); + let _ = match scr.pull(&build_result.build.image).unwrap() { + PullResult::Some(_) => assert!(true), + PullResult::None => assert!(false), + }; + + let pull_duration = SystemTime::now().duration_since(start_pull_time).unwrap(); + + let _ = match local_docker.has_cache(&build_result.build) { + Ok(CacheResult::Hit) => assert!(true), + Ok(CacheResult::Miss(parent_build)) => assert!(false), + Ok(CacheResult::MissWithoutParentBuild) => assert!(false), + Err(err) => assert!(false), + }; + + let start_pull_time = SystemTime::now(); + let _ = match scr.pull(&image).unwrap() { + PullResult::Some(_) => assert!(true), + PullResult::None => assert!(false), + }; + + let pull_duration_2 = SystemTime::now().duration_since(start_pull_time).unwrap(); + + if pull_duration_2.as_millis() > pull_duration.as_millis() { + assert!(false); + } + + return test_name.to_string(); + }) +} + #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] From dbe6e8718996235139932edfe7557edddc780428 Mon Sep 17 00:00:00 2001 From: MacLikorne Date: Wed, 2 Feb 2022 09:55:08 +0100 Subject: [PATCH 23/25] fix(ENG-1068): normalize db service (#572) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ENG-1068): normalize db service * fix(ENG-1068): fix aws name sanitizer * fix(ENG-1068): optimize db tests * fix(ENG-1068): fix linter * fix(ENG-1068): add notes to helm freeze * fix(ENG-1068): refacto environment handling * fix(ENG-1068): fix ci * fix(ENG_1068): edit charts * fix(ENG_1068): fix name sanitizer for managed db * fix(ENG_1068): improve tests * fix(ENG_1068): fix test Co-authored-by: Romain GERARD Co-authored-by: Erèbe - Romain Gerard --- lib/aws/chart_values/mongodb/q-values.j2.yaml | 2 +- lib/aws/chart_values/mysql/q-values.j2.yaml | 1 + .../chart_values/postgresql/q-values.j2.yaml | 1 + lib/aws/chart_values/redis/q-values.j2.yaml | 1 + .../services/mysql/templates/_helpers.tpl | 12 + .../services/mysql/templates/master-svc.yaml | 2 +- lib/common/services/mysql/values.yaml | 1 + .../postgresql/templates/_helpers.tpl | 12 + .../services/postgresql/templates/svc.yaml | 2 +- lib/common/services/postgresql/values.yaml | 1 + .../services/redis/templates/_helpers.tpl | 12 + .../redis/templates/redis-master-svc.yaml | 2 +- lib/common/services/redis/values.yaml | 1 + .../chart_values/mongodb/q-values.j2.yaml | 2 +- .../chart_values/mysql/q-values.j2.yaml | 2 +- .../chart_values/postgresql/q-values.j2.yaml | 2 +- .../chart_values/redis/q-values.j2.yaml | 1 + lib/helm-freeze.yaml | 6 +- .../chart_values/mongodb/q-values.j2.yaml | 2 +- .../chart_values/mysql/q-values.j2.yaml | 2 +- .../chart_values/postgresql/q-values.j2.yaml | 2 +- .../chart_values/redis/q-values.j2.yaml | 1 + src/cloud_provider/aws/application.rs | 2 +- src/cloud_provider/aws/databases/mongodb.rs | 28 +- src/cloud_provider/aws/databases/mysql.rs | 24 +- .../aws/databases/postgresql.rs | 24 +- src/cloud_provider/aws/databases/redis.rs | 30 +- src/cloud_provider/aws/router.rs | 2 +- .../digitalocean/application.rs | 2 +- .../digitalocean/databases/mongodb.rs | 9 +- .../digitalocean/databases/mysql.rs | 9 +- .../digitalocean/databases/postgresql.rs | 9 +- .../digitalocean/databases/redis.rs | 9 +- src/cloud_provider/digitalocean/router.rs | 2 +- src/cloud_provider/scaleway/application.rs | 2 +- .../scaleway/databases/mongodb.rs | 9 +- .../scaleway/databases/mysql.rs | 9 +- .../scaleway/databases/postgresql.rs | 9 +- .../scaleway/databases/redis.rs | 9 +- src/cloud_provider/scaleway/router.rs | 2 +- src/cloud_provider/service.rs | 24 +- src/cloud_provider/utilities.rs | 20 +- test_utilities/src/common.rs | 188 +-- test_utilities/src/utilities.rs | 28 +- tests/aws/aws_databases.rs | 1176 ++--------------- tests/digitalocean/do_databases.rs | 637 ++------- tests/scaleway/scw_databases.rs | 899 ++----------- 47 files changed, 572 insertions(+), 2660 deletions(-) diff --git a/lib/aws/chart_values/mongodb/q-values.j2.yaml b/lib/aws/chart_values/mongodb/q-values.j2.yaml index dc3bf279..338baa65 100644 --- a/lib/aws/chart_values/mongodb/q-values.j2.yaml +++ b/lib/aws/chart_values/mongodb/q-values.j2.yaml @@ -135,7 +135,7 @@ clusterDomain: cluster.local ## Kubernetes service type service: ## Specify an explicit service name. - # name: svc-mongo + name: {{ service_name }} ## Provide any additional annotations which may be required. ## The value is evaluated as a template, so, for example, the value can depend on .Release or .Chart {% if publicly_accessible -%} diff --git a/lib/aws/chart_values/mysql/q-values.j2.yaml b/lib/aws/chart_values/mysql/q-values.j2.yaml index 226c9814..daa4475d 100644 --- a/lib/aws/chart_values/mysql/q-values.j2.yaml +++ b/lib/aws/chart_values/mysql/q-values.j2.yaml @@ -501,6 +501,7 @@ service: ## MySQL Service type ## type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} + name: {{ service_name }} ## MySQL Service port ## diff --git a/lib/aws/chart_values/postgresql/q-values.j2.yaml b/lib/aws/chart_values/postgresql/q-values.j2.yaml index c3553002..ed6927de 100644 --- a/lib/aws/chart_values/postgresql/q-values.j2.yaml +++ b/lib/aws/chart_values/postgresql/q-values.j2.yaml @@ -249,6 +249,7 @@ ldap: service: ## PosgresSQL service type type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} + name: {{ service_name }} # clusterIP: None port: 5432 diff --git a/lib/aws/chart_values/redis/q-values.j2.yaml b/lib/aws/chart_values/redis/q-values.j2.yaml index 72ae0ccb..eddb8c3f 100644 --- a/lib/aws/chart_values/redis/q-values.j2.yaml +++ b/lib/aws/chart_values/redis/q-values.j2.yaml @@ -369,6 +369,7 @@ master: service: ## Redis Master Service type type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} + name: {{ service_name }} port: 6379 ## Specify the nodePort value for the LoadBalancer and NodePort service types. diff --git a/lib/common/services/mysql/templates/_helpers.tpl b/lib/common/services/mysql/templates/_helpers.tpl index bc5933ff..1e8fc206 100644 --- a/lib/common/services/mysql/templates/_helpers.tpl +++ b/lib/common/services/mysql/templates/_helpers.tpl @@ -263,6 +263,18 @@ but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else {{- end -}} {{- end -}} +{{/* +Returns the proper Service name depending if an explicit service name is set +in the values file. If the name is not explicitly set it will take the "mongodb.fullname" +*/}} +{{- define "mysql.serviceName" -}} + {{- if .Values.service.name -}} + {{ .Values.service.name }} + {{- else -}} + {{ template "mysql.fullname" .}} + {{- end -}} +{{- end -}} + {{/* Renders a value that contains template. Usage: diff --git a/lib/common/services/mysql/templates/master-svc.yaml b/lib/common/services/mysql/templates/master-svc.yaml index 24142c32..d4827088 100644 --- a/lib/common/services/mysql/templates/master-svc.yaml +++ b/lib/common/services/mysql/templates/master-svc.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ template "mysql.fullname" . }} + name: {{ template "mysql.serviceName" . }} labels: {{- include "mysql.labels" . | nindent 4 }} component: master {{- if or .Values.service.annotations .Values.metrics.service.annotations }} diff --git a/lib/common/services/mysql/values.yaml b/lib/common/services/mysql/values.yaml index 97d05aaf..5b2efbda 100644 --- a/lib/common/services/mysql/values.yaml +++ b/lib/common/services/mysql/values.yaml @@ -490,6 +490,7 @@ service: ## type: ClusterIP + # name: svc-name ## MySQL Service port ## port: 3306 diff --git a/lib/common/services/postgresql/templates/_helpers.tpl b/lib/common/services/postgresql/templates/_helpers.tpl index e13caad6..91cb1580 100644 --- a/lib/common/services/postgresql/templates/_helpers.tpl +++ b/lib/common/services/postgresql/templates/_helpers.tpl @@ -378,6 +378,18 @@ but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else {{- end -}} {{- end -}} +{{/* +Returns the proper Service name depending if an explicit service name is set +in the values file. If the name is not explicitly set it will take the "mongodb.fullname" +*/}} +{{- define "postgresql.serviceName" -}} + {{- if .Values.service.name -}} + {{ .Values.service.name }} + {{- else -}} + {{ template "postgresql.fullname" .}} + {{- end -}} +{{- end -}} + {{/* Renders a value that contains template. Usage: diff --git a/lib/common/services/postgresql/templates/svc.yaml b/lib/common/services/postgresql/templates/svc.yaml index d24b2a65..4822fb25 100644 --- a/lib/common/services/postgresql/templates/svc.yaml +++ b/lib/common/services/postgresql/templates/svc.yaml @@ -7,7 +7,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ template "postgresql.fullname" . }} + name: {{ template "postgresql.serviceName" . }} labels: app: {{ template "postgresql.name" . }} chart: {{ template "postgresql.chart" . }} diff --git a/lib/common/services/postgresql/values.yaml b/lib/common/services/postgresql/values.yaml index e4651dae..4f949f93 100644 --- a/lib/common/services/postgresql/values.yaml +++ b/lib/common/services/postgresql/values.yaml @@ -249,6 +249,7 @@ ldap: service: ## PosgresSQL service type type: ClusterIP + # name: svc-name # clusterIP: None port: 5432 diff --git a/lib/common/services/redis/templates/_helpers.tpl b/lib/common/services/redis/templates/_helpers.tpl index b5126537..1d21226d 100644 --- a/lib/common/services/redis/templates/_helpers.tpl +++ b/lib/common/services/redis/templates/_helpers.tpl @@ -384,6 +384,18 @@ but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else {{- end -}} {{- end -}} +{{/* +Returns the proper Service name depending if an explicit service name is set +in the values file. If the name is not explicitly set it will take the "mongodb.fullname" +*/}} +{{- define "redis.serviceName" -}} + {{- if .Values.master.service.name -}} + {{ .Values.master.service.name }} + {{- else -}} + {{ template "redis.fullname" .}} + {{- end -}} +{{- end -}} + {{/* Compile all warnings into a single message, and call fail. */}} diff --git a/lib/common/services/redis/templates/redis-master-svc.yaml b/lib/common/services/redis/templates/redis-master-svc.yaml index 94fcce20..2c2719e6 100644 --- a/lib/common/services/redis/templates/redis-master-svc.yaml +++ b/lib/common/services/redis/templates/redis-master-svc.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ template "redis.fullname" . }}-master + name: {{ template "redis.serviceName" . }} namespace: {{ .Release.Namespace | quote }} labels: app: {{ template "redis.name" . }} diff --git a/lib/common/services/redis/values.yaml b/lib/common/services/redis/values.yaml index fb92acb3..01b9507f 100644 --- a/lib/common/services/redis/values.yaml +++ b/lib/common/services/redis/values.yaml @@ -362,6 +362,7 @@ master: service: ## Redis Master Service type type: ClusterIP + # name: svc-name port: 6379 ## Specify the nodePort value for the LoadBalancer and NodePort service types. diff --git a/lib/digitalocean/chart_values/mongodb/q-values.j2.yaml b/lib/digitalocean/chart_values/mongodb/q-values.j2.yaml index f1c032c7..0e2187d1 100644 --- a/lib/digitalocean/chart_values/mongodb/q-values.j2.yaml +++ b/lib/digitalocean/chart_values/mongodb/q-values.j2.yaml @@ -149,7 +149,7 @@ service: {% endif %} type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - # clusterIP: None + name: {{ service_name }} port: {{ private_port }} ## Specify the nodePort value for the LoadBalancer and NodePort service types. diff --git a/lib/digitalocean/chart_values/mysql/q-values.j2.yaml b/lib/digitalocean/chart_values/mysql/q-values.j2.yaml index d6247355..28812907 100644 --- a/lib/digitalocean/chart_values/mysql/q-values.j2.yaml +++ b/lib/digitalocean/chart_values/mysql/q-values.j2.yaml @@ -501,7 +501,7 @@ service: ## MySQL Service type ## type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - + name: {{ service_name }} ## MySQL Service port ## port: 3306 diff --git a/lib/digitalocean/chart_values/postgresql/q-values.j2.yaml b/lib/digitalocean/chart_values/postgresql/q-values.j2.yaml index 93c967d3..2bc0f002 100644 --- a/lib/digitalocean/chart_values/postgresql/q-values.j2.yaml +++ b/lib/digitalocean/chart_values/postgresql/q-values.j2.yaml @@ -249,7 +249,7 @@ ldap: service: ## PosgresSQL service type type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - # clusterIP: None + name: {{ service_name }} port: 5432 ## Specify the nodePort value for the LoadBalancer and NodePort service types. diff --git a/lib/digitalocean/chart_values/redis/q-values.j2.yaml b/lib/digitalocean/chart_values/redis/q-values.j2.yaml index 70b91f04..9d3170ca 100644 --- a/lib/digitalocean/chart_values/redis/q-values.j2.yaml +++ b/lib/digitalocean/chart_values/redis/q-values.j2.yaml @@ -369,6 +369,7 @@ master: service: ## Redis Master Service type type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} + name: {{ service_name }} port: 6379 ## Specify the nodePort value for the LoadBalancer and NodePort service types. diff --git a/lib/helm-freeze.yaml b/lib/helm-freeze.yaml index b85bf1d3..e1d05484 100644 --- a/lib/helm-freeze.yaml +++ b/lib/helm-freeze.yaml @@ -52,22 +52,24 @@ charts: version: 7.14.5 dest: services no_sync: true - # Chart is frozen due to custom modification see https://github.com/Qovery/engine/pull/293 + # Chart is frozen due to custom modification see https://github.com/Qovery/engine/pull/293 https://github.com/Qovery/engine/pull/572 - name: mysql repo_name: bitnami version: 6.14.2 dest: services no_sync: true + # Chart is frozen due to custom modification see https://github.com/Qovery/engine/pull/572 - name: postgresql repo_name: bitnami version: 8.9.8 dest: services - # Chart is frozen due to custom modification see https://github.com/Qovery/engine/pull/293 + # Chart is frozen due to custom modification see https://github.com/Qovery/engine/pull/293 https://github.com/Qovery/engine/pull/572 - name: redis repo_name: bitnami version: 12.0.1 dest: services no_sync: true + # Chart is frozen due to custom modification see https://github.com/Qovery/engine/pull/572 - name: pleco version: 0.9.33 repo_name: pleco diff --git a/lib/scaleway/chart_values/mongodb/q-values.j2.yaml b/lib/scaleway/chart_values/mongodb/q-values.j2.yaml index 95da012f..f4e3ae55 100644 --- a/lib/scaleway/chart_values/mongodb/q-values.j2.yaml +++ b/lib/scaleway/chart_values/mongodb/q-values.j2.yaml @@ -87,7 +87,7 @@ service: external-dns.alpha.kubernetes.io/ttl: "300" {% endif %} type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - # clusterIP: None + name: {{ service_name }} port: {{ private_port }} ## Specify the nodePort value for the LoadBalancer and NodePort service types. diff --git a/lib/scaleway/chart_values/mysql/q-values.j2.yaml b/lib/scaleway/chart_values/mysql/q-values.j2.yaml index c4bf58c0..fcb42272 100644 --- a/lib/scaleway/chart_values/mysql/q-values.j2.yaml +++ b/lib/scaleway/chart_values/mysql/q-values.j2.yaml @@ -479,7 +479,7 @@ service: ## MySQL Service type ## type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - + name: {{ service_name }} ## MySQL Service port ## port: 3306 diff --git a/lib/scaleway/chart_values/postgresql/q-values.j2.yaml b/lib/scaleway/chart_values/postgresql/q-values.j2.yaml index 96b5dafe..c4f2e191 100644 --- a/lib/scaleway/chart_values/postgresql/q-values.j2.yaml +++ b/lib/scaleway/chart_values/postgresql/q-values.j2.yaml @@ -121,7 +121,7 @@ service: ## PosgresSQL service type type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - # clusterIP: None + name: {{ service_name }} port: 5432 ## Provide any additional annotations which may be required. Evaluated as a template. diff --git a/lib/scaleway/chart_values/redis/q-values.j2.yaml b/lib/scaleway/chart_values/redis/q-values.j2.yaml index 92c30306..8b6d73c0 100644 --- a/lib/scaleway/chart_values/redis/q-values.j2.yaml +++ b/lib/scaleway/chart_values/redis/q-values.j2.yaml @@ -326,6 +326,7 @@ master: service: ## Redis Master Service type type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} + name: {{ service_name }} port: 6379 ## Specify the nodePort value for the LoadBalancer and NodePort service types. diff --git a/src/cloud_provider/aws/application.rs b/src/cloud_provider/aws/application.rs index 6f4aeec7..c8d1a0e6 100644 --- a/src/cloud_provider/aws/application.rs +++ b/src/cloud_provider/aws/application.rs @@ -145,7 +145,7 @@ impl Service for Application { } fn sanitized_name(&self) -> String { - sanitize_name("app", self.name()) + sanitize_name("app", self.name(), false) } fn version(&self) -> String { diff --git a/src/cloud_provider/aws/databases/mongodb.rs b/src/cloud_provider/aws/databases/mongodb.rs index ad2d5d16..48d6d187 100644 --- a/src/cloud_provider/aws/databases/mongodb.rs +++ b/src/cloud_provider/aws/databases/mongodb.rs @@ -10,6 +10,7 @@ use crate::cloud_provider::service::{ }; use crate::cloud_provider::utilities::{ generate_supported_version, get_self_hosted_mongodb_version, get_supported_version_to_use, print_action, + sanitize_name, }; use crate::cloud_provider::DeploymentTarget; use crate::cmd::helm::Timeout; @@ -103,15 +104,7 @@ impl Service for MongoDB { } fn sanitized_name(&self) -> String { - // https://docs.aws.amazon.com/documentdb/latest/developerguide/limits.html#limits-naming_constraints - let prefix = "mongodb"; - let max_size = 60 - prefix.len(); // 63 (max DocumentDB) - 3 (k8s statefulset chars) - let mut new_name = format!("{}{}", prefix, self.name().replace("_", "").replace("-", "")); - if new_name.chars().count() > max_size { - new_name = new_name[..max_size].to_string(); - } - - new_name + sanitize_name("mongodb", self.name(), self.is_managed_service()) } fn version(&self) -> String { @@ -187,10 +180,9 @@ impl Service for MongoDB { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert( - "fqdn", - self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), - ); + context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); + context.insert("sanitized_name", self.sanitized_name().as_str()); + context.insert("service_name", self.service_name().as_str()); context.insert("database_db_name", self.name.as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); @@ -429,8 +421,8 @@ mod tests_mongodb { #[test] fn mongo_name_sanitizer() { - let db_input_name = "test-name_sanitizer-with-too-many-chars-not-allowed-which_will-be-shrinked-at-the-end"; - let db_expected_name = "mongodbtestnamesanitizerwithtoomanycharsnotallowedwhi"; + let db_name = "a-name"; + let db_expected_name = "mongodbaname"; let database = MongoDB::new( Context::new( @@ -444,9 +436,9 @@ mod tests_mongodb { vec![], None, ), - "pgid", + "1234", Action::Create, - db_input_name, + db_name.clone(), "8", "mongotest.qovery.io", "pgid", @@ -458,7 +450,7 @@ mod tests_mongodb { password: "".to_string(), host: "".to_string(), port: 5432, - mode: DatabaseMode::CONTAINER, + mode: DatabaseMode::MANAGED, disk_size_in_gib: 10, database_disk_type: "gp2".to_string(), encrypt_disk: false, diff --git a/src/cloud_provider/aws/databases/mysql.rs b/src/cloud_provider/aws/databases/mysql.rs index e5c451f2..f9c23719 100644 --- a/src/cloud_provider/aws/databases/mysql.rs +++ b/src/cloud_provider/aws/databases/mysql.rs @@ -9,8 +9,8 @@ use crate::cloud_provider::service::{ DatabaseType, Delete, Helm, Pause, Service, ServiceType, StatefulService, Terraform, }; use crate::cloud_provider::utilities::{ - generate_supported_version, get_self_hosted_mysql_version, get_supported_version_to_use, managed_db_name_sanitizer, - print_action, + generate_supported_version, get_self_hosted_mysql_version, get_supported_version_to_use, print_action, + sanitize_name, }; use crate::cloud_provider::DeploymentTarget; use crate::cmd::helm::Timeout; @@ -114,10 +114,7 @@ impl Service for MySQL { } fn sanitized_name(&self) -> String { - // https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Limits.html#RDS_Limits.Constraints - let prefix = "mysql"; - let max_size = 63 - 3; // max RDS - k8s statefulset chars - managed_db_name_sanitizer(max_size, prefix, self.name()) + sanitize_name("mysql", self.name(), self.is_managed_service()) } fn version(&self) -> String { @@ -208,10 +205,9 @@ impl Service for MySQL { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert( - "fqdn", - self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), - ); + context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); + context.insert("sanitized_name", self.sanitized_name().as_str()); + context.insert("service_name", self.service_name().as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); @@ -453,8 +449,8 @@ mod tests_mysql { #[test] fn mysql_name_sanitizer() { - let db_input_name = "test-name_sanitizer-with-too-many-chars-not-allowed-which_will-be-shrinked-at-the-end"; - let db_expected_name = "mysqltestnamesanitizerwithtoomanycharsnotallowedwhichwi"; + let db_name = "a-name"; + let db_expected_name = "mysqlaname"; let database = MySQL::new( Context::new( @@ -468,9 +464,9 @@ mod tests_mysql { vec![], None, ), - "mysqlid", + "1234", Action::Create, - db_input_name, + db_name.clone(), "8", "mysqltest.qovery.io", "mysqlid", diff --git a/src/cloud_provider/aws/databases/postgresql.rs b/src/cloud_provider/aws/databases/postgresql.rs index ee66bcb2..3c21cc8f 100644 --- a/src/cloud_provider/aws/databases/postgresql.rs +++ b/src/cloud_provider/aws/databases/postgresql.rs @@ -9,8 +9,8 @@ use crate::cloud_provider::service::{ DatabaseType, Delete, Helm, Pause, Service, ServiceType, StatefulService, Terraform, }; use crate::cloud_provider::utilities::{ - generate_supported_version, get_self_hosted_postgres_version, get_supported_version_to_use, - managed_db_name_sanitizer, print_action, + generate_supported_version, get_self_hosted_postgres_version, get_supported_version_to_use, print_action, + sanitize_name, }; use crate::cloud_provider::DeploymentTarget; use crate::cmd::helm::Timeout; @@ -114,10 +114,7 @@ impl Service for PostgreSQL { } fn sanitized_name(&self) -> String { - // https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Limits.html#RDS_Limits.Constraints - let prefix = "postgresql"; - let max_size = 63 - 3; // max RDS - k8s statefulset chars - managed_db_name_sanitizer(max_size, prefix, self.name()) + sanitize_name("postgresql", self.name(), self.is_managed_service()) } fn version(&self) -> String { @@ -193,10 +190,9 @@ impl Service for PostgreSQL { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert( - "fqdn", - self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), - ); + context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); + context.insert("sanitized_name", self.sanitized_name().as_str()); + context.insert("service_name", self.service_name().as_str()); context.insert("database_name", self.sanitized_name().as_str()); context.insert("database_db_name", self.name()); context.insert("database_login", self.options.login.as_str()); @@ -447,8 +443,8 @@ mod tests_postgres { #[test] fn postgres_name_sanitizer() { - let db_input_name = "test-name_sanitizer-with-too-many-chars-not-allowed-which_will-be-shrinked-at-the-end"; - let db_expected_name = "postgresqltestnamesanitizerwithtoomanycharsnotallo"; + let db_name = "a-name"; + let db_expected_name = "postgresqlaname"; let database = PostgreSQL::new( Context::new( @@ -462,9 +458,9 @@ mod tests_postgres { vec![], None, ), - "pgid", + "1234", Action::Create, - db_input_name, + db_name.clone(), "8", "pgtest.qovery.io", "pgid", diff --git a/src/cloud_provider/aws/databases/redis.rs b/src/cloud_provider/aws/databases/redis.rs index b45d45c9..ddc1c1fd 100644 --- a/src/cloud_provider/aws/databases/redis.rs +++ b/src/cloud_provider/aws/databases/redis.rs @@ -8,7 +8,9 @@ use crate::cloud_provider::service::{ get_tfstate_suffix, scale_down_database, send_progress_on_long_task, Action, Create, Database, DatabaseOptions, DatabaseType, Delete, Helm, Pause, Service, ServiceType, StatefulService, Terraform, }; -use crate::cloud_provider::utilities::{get_self_hosted_redis_version, get_supported_version_to_use, print_action}; +use crate::cloud_provider::utilities::{ + get_self_hosted_redis_version, get_supported_version_to_use, print_action, sanitize_name, +}; use crate::cloud_provider::DeploymentTarget; use crate::cmd::helm::Timeout; use crate::cmd::kubectl; @@ -111,16 +113,7 @@ impl Service for Redis { } fn sanitized_name(&self) -> String { - // https://aws.amazon.com/about-aws/whats-new/2019/08/elasticache_supports_50_chars_cluster_name - let prefix = "redis"; - let max_size = 47 - prefix.len(); // 50 (max Elasticache ) - 3 (k8s statefulset chars) - let mut new_name = self.name().replace("_", "").replace("-", ""); - - if new_name.chars().count() > max_size { - new_name = new_name[..max_size].to_string(); - } - - format!("{}{}", prefix, new_name) + sanitize_name("redis", self.name(), self.is_managed_service()) } fn version(&self) -> String { @@ -209,10 +202,9 @@ impl Service for Redis { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert( - "fqdn", - self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), - ); + context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); + context.insert("sanitized_name", self.sanitized_name().as_str()); + context.insert("service_name", self.service_name().as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); @@ -436,8 +428,8 @@ mod tests { #[test] fn redis_name_sanitizer() { - let db_input_name = "test-name_sanitizer-with-too-many-chars-not-allowed-which_will-be-shrinked-at-the-end"; - let db_expected_name = "redistestnamesanitizerwithtoomanycharsnotallowe"; + let db_name = "a-name"; + let db_expected_name = "redisaname"; let database = Redis::new( Context::new( @@ -451,9 +443,9 @@ mod tests { vec![], None, ), - "pgid", + "1234", Action::Create, - db_input_name, + db_name.clone(), "8", "redistest.qovery.io", "redisid", diff --git a/src/cloud_provider/aws/router.rs b/src/cloud_provider/aws/router.rs index f9792144..440c744c 100644 --- a/src/cloud_provider/aws/router.rs +++ b/src/cloud_provider/aws/router.rs @@ -77,7 +77,7 @@ impl Service for Router { } fn sanitized_name(&self) -> String { - sanitize_name("router", self.name()) + sanitize_name("router", self.name(), false) } fn version(&self) -> String { diff --git a/src/cloud_provider/digitalocean/application.rs b/src/cloud_provider/digitalocean/application.rs index 48381113..16775c8f 100644 --- a/src/cloud_provider/digitalocean/application.rs +++ b/src/cloud_provider/digitalocean/application.rs @@ -147,7 +147,7 @@ impl Service for Application { } fn sanitized_name(&self) -> String { - sanitize_name("app", self.name()) + sanitize_name("app", self.name(), false) } fn version(&self) -> String { diff --git a/src/cloud_provider/digitalocean/databases/mongodb.rs b/src/cloud_provider/digitalocean/databases/mongodb.rs index 84357d13..db9cfce0 100644 --- a/src/cloud_provider/digitalocean/databases/mongodb.rs +++ b/src/cloud_provider/digitalocean/databases/mongodb.rs @@ -108,7 +108,7 @@ impl Service for MongoDB { } fn sanitized_name(&self) -> String { - sanitize_name("mongodb", self.name()) + sanitize_name("mongodb", self.name(), self.is_managed_service()) } fn version(&self) -> String { @@ -184,10 +184,9 @@ impl Service for MongoDB { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert( - "fqdn", - self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), - ); + context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); + context.insert("sanitized_name", self.sanitized_name().as_str()); + context.insert("service_name", self.service_name().as_str()); context.insert("database_db_name", self.name.as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); diff --git a/src/cloud_provider/digitalocean/databases/mysql.rs b/src/cloud_provider/digitalocean/databases/mysql.rs index 00bba0f2..c2d84fe3 100644 --- a/src/cloud_provider/digitalocean/databases/mysql.rs +++ b/src/cloud_provider/digitalocean/databases/mysql.rs @@ -108,7 +108,7 @@ impl Service for MySQL { } fn sanitized_name(&self) -> String { - sanitize_name("mysql", self.name()) + sanitize_name("mysql", self.name(), self.is_managed_service()) } fn version(&self) -> String { @@ -184,10 +184,9 @@ impl Service for MySQL { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert( - "fqdn", - self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), - ); + context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); + context.insert("sanitized_name", self.sanitized_name().as_str()); + context.insert("service_name", self.service_name().as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); diff --git a/src/cloud_provider/digitalocean/databases/postgresql.rs b/src/cloud_provider/digitalocean/databases/postgresql.rs index b7efc14a..d2895d01 100644 --- a/src/cloud_provider/digitalocean/databases/postgresql.rs +++ b/src/cloud_provider/digitalocean/databases/postgresql.rs @@ -108,7 +108,7 @@ impl Service for PostgreSQL { } fn sanitized_name(&self) -> String { - sanitize_name("postgresql", self.name()) + sanitize_name("postgresql", self.name(), self.is_managed_service()) } fn version(&self) -> String { @@ -184,10 +184,9 @@ impl Service for PostgreSQL { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert( - "fqdn", - self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), - ); + context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); + context.insert("sanitized_name", self.sanitized_name().as_str()); + context.insert("service_name", self.service_name().as_str()); context.insert("database_db_name", self.name()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); diff --git a/src/cloud_provider/digitalocean/databases/redis.rs b/src/cloud_provider/digitalocean/databases/redis.rs index 527cc38e..580ab10e 100644 --- a/src/cloud_provider/digitalocean/databases/redis.rs +++ b/src/cloud_provider/digitalocean/databases/redis.rs @@ -108,7 +108,7 @@ impl Service for Redis { } fn sanitized_name(&self) -> String { - sanitize_name("redis", self.name()) + sanitize_name("redis", self.name(), self.is_managed_service()) } fn version(&self) -> String { @@ -184,10 +184,9 @@ impl Service for Redis { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert( - "fqdn", - self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), - ); + context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); + context.insert("sanitized_name", self.sanitized_name().as_str()); + context.insert("service_name", self.service_name().as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); diff --git a/src/cloud_provider/digitalocean/router.rs b/src/cloud_provider/digitalocean/router.rs index 173dea32..22312520 100644 --- a/src/cloud_provider/digitalocean/router.rs +++ b/src/cloud_provider/digitalocean/router.rs @@ -77,7 +77,7 @@ impl Service for Router { } fn sanitized_name(&self) -> String { - sanitize_name("router", self.name()) + sanitize_name("router", self.name(), false) } fn version(&self) -> String { diff --git a/src/cloud_provider/scaleway/application.rs b/src/cloud_provider/scaleway/application.rs index ff148e26..83fc1568 100644 --- a/src/cloud_provider/scaleway/application.rs +++ b/src/cloud_provider/scaleway/application.rs @@ -148,7 +148,7 @@ impl Service for Application { } fn sanitized_name(&self) -> String { - sanitize_name("app", self.name()) + sanitize_name("app", self.name(), false) } fn version(&self) -> String { diff --git a/src/cloud_provider/scaleway/databases/mongodb.rs b/src/cloud_provider/scaleway/databases/mongodb.rs index d3303924..c19a6c9b 100644 --- a/src/cloud_provider/scaleway/databases/mongodb.rs +++ b/src/cloud_provider/scaleway/databases/mongodb.rs @@ -108,7 +108,7 @@ impl Service for MongoDB { } fn sanitized_name(&self) -> String { - sanitize_name("mongodb", self.name()) + sanitize_name("mongodb", self.name(), self.is_managed_service()) } fn version(&self) -> String { @@ -185,10 +185,9 @@ impl Service for MongoDB { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert( - "fqdn", - self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), - ); + context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); + context.insert("sanitized_name", self.sanitized_name().as_str()); + context.insert("service_name", self.service_name().as_str()); context.insert("database_db_name", self.name.as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); diff --git a/src/cloud_provider/scaleway/databases/mysql.rs b/src/cloud_provider/scaleway/databases/mysql.rs index c7bec2c8..9e44e96d 100644 --- a/src/cloud_provider/scaleway/databases/mysql.rs +++ b/src/cloud_provider/scaleway/databases/mysql.rs @@ -139,7 +139,7 @@ impl Service for MySQL { } fn sanitized_name(&self) -> String { - sanitize_name("mysql", self.name()) + sanitize_name("mysql", self.name(), self.is_managed_service()) } fn version(&self) -> String { @@ -217,10 +217,9 @@ impl Service for MySQL { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert( - "fqdn", - self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), - ); + context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); + context.insert("sanitized_name", self.sanitized_name().as_str()); + context.insert("service_name", self.service_name().as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); diff --git a/src/cloud_provider/scaleway/databases/postgresql.rs b/src/cloud_provider/scaleway/databases/postgresql.rs index f39f1610..b7172310 100644 --- a/src/cloud_provider/scaleway/databases/postgresql.rs +++ b/src/cloud_provider/scaleway/databases/postgresql.rs @@ -148,7 +148,7 @@ impl Service for PostgreSQL { } fn sanitized_name(&self) -> String { - sanitize_name("postgresql", self.name()) + sanitize_name("postgresql", self.name(), self.is_managed_service()) } fn version(&self) -> String { @@ -226,10 +226,9 @@ impl Service for PostgreSQL { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert( - "fqdn", - self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), - ); + context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); + context.insert("sanitized_name", self.sanitized_name().as_str()); + context.insert("service_name", self.service_name().as_str()); context.insert("database_name", self.sanitized_name().as_str()); context.insert("database_db_name", self.name()); context.insert("database_login", self.options.login.as_str()); diff --git a/src/cloud_provider/scaleway/databases/redis.rs b/src/cloud_provider/scaleway/databases/redis.rs index 4ca2f11a..47e03fa9 100644 --- a/src/cloud_provider/scaleway/databases/redis.rs +++ b/src/cloud_provider/scaleway/databases/redis.rs @@ -108,7 +108,7 @@ impl Service for Redis { } fn sanitized_name(&self) -> String { - sanitize_name("redis", self.name()) + sanitize_name("redis", self.name(), self.is_managed_service()) } fn version(&self) -> String { @@ -185,10 +185,9 @@ impl Service for Redis { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert( - "fqdn", - self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), - ); + context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); + context.insert("sanitized_name", self.sanitized_name().as_str()); + context.insert("service_name", self.service_name().as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); diff --git a/src/cloud_provider/scaleway/router.rs b/src/cloud_provider/scaleway/router.rs index 2ca3651b..79c598e6 100644 --- a/src/cloud_provider/scaleway/router.rs +++ b/src/cloud_provider/scaleway/router.rs @@ -77,7 +77,7 @@ impl Service for Router { } fn sanitized_name(&self) -> String { - sanitize_name("router", self.name()) + sanitize_name("router", self.name(), false) } fn version(&self) -> String { diff --git a/src/cloud_provider/service.rs b/src/cloud_provider/service.rs index a1bc908c..197512b4 100644 --- a/src/cloud_provider/service.rs +++ b/src/cloud_provider/service.rs @@ -9,7 +9,7 @@ use tera::Context as TeraContext; use crate::build_platform::Image; use crate::cloud_provider::environment::Environment; use crate::cloud_provider::kubernetes::Kubernetes; -use crate::cloud_provider::utilities::check_domain_for; +use crate::cloud_provider::utilities::{check_domain_for, sanitize_db_name}; use crate::cloud_provider::DeploymentTarget; use crate::cmd::helm::Timeout; use crate::cmd::kubectl::ScalingKind::Statefulset; @@ -30,6 +30,18 @@ pub trait Service: ToTransmitter { fn id(&self) -> &str; fn name(&self) -> &str; fn sanitized_name(&self) -> String; + fn service_name(&self) -> String { + match self.service_type() { + ServiceType::Application => self.sanitized_name(), + ServiceType::Database(db_kind) => match db_kind { + DatabaseType::PostgreSQL(_) => sanitize_db_name("postgresql", self.id()), + DatabaseType::MongoDB(_) => sanitize_db_name("mongodb", self.id()), + DatabaseType::MySQL(_) => sanitize_db_name("mysql", self.id()), + DatabaseType::Redis(_) => sanitize_db_name("redis", self.id()), + }, + ServiceType::Router => self.sanitized_name(), + } + } fn name_with_id(&self) -> String { format!("{} ({})", self.name(), self.id()) } @@ -69,16 +81,12 @@ pub trait Service: ToTransmitter { fn min_instances(&self) -> u32; fn max_instances(&self) -> u32; fn publicly_accessible(&self) -> bool; - fn fqdn<'a>(&self, target: &DeploymentTarget, fqdn: &'a String, is_managed: bool) -> String { + fn fqdn<'a>(&self, fqdn: &'a String, is_managed: bool) -> String { match &self.publicly_accessible() { true => fqdn.to_string(), false => match is_managed { - true => format!("{}-dns.{}.svc.cluster.local", self.id(), target.environment.namespace()), - false => format!( - "{}.{}.svc.cluster.local", - self.sanitized_name(), - target.environment.namespace() - ), + true => format!("{}-dns", self.id()), + false => format!("{}", self.sanitized_name()), }, } } diff --git a/src/cloud_provider/utilities.rs b/src/cloud_provider/utilities.rs index 1407b3c3..8aa3a43e 100644 --- a/src/cloud_provider/utilities.rs +++ b/src/cloud_provider/utilities.rs @@ -495,17 +495,21 @@ pub fn check_domain_for( Ok(()) } -pub fn sanitize_name(prefix: &str, name: &str) -> String { - format!("{}-{}", prefix, name).replace("_", "-") +pub fn sanitize_db_name(suffix: &str, db_id: &str) -> String { + format!("{}-{}", db_id, suffix).replace("_", "-") } -pub fn managed_db_name_sanitizer(max_size: usize, prefix: &str, name: &str) -> String { - let max_size = max_size - prefix.len(); - let mut new_name = format!("{}{}", prefix, name.replace("_", "").replace("-", "")); - if new_name.chars().count() > max_size { - new_name = new_name[..max_size].to_string(); +pub fn sanitize_name(prefix: &str, app_name: &str, is_managed: bool) -> String { + match is_managed { + true => { + let mut sanitized_name = format!("{}-{}", prefix, app_name).replace("_", "").replace("-", ""); + if sanitized_name.chars().count() > 63 { + sanitized_name = sanitized_name[..63].to_string(); + }; + sanitized_name + } + false => format!("{}-{}", prefix, app_name).replace("_", "-"), } - new_name } pub fn convert_k8s_cpu_value_to_f32(value: String) -> Result { diff --git a/test_utilities/src/common.rs b/test_utilities/src/common.rs index 2ace4cc2..9341a45d 100644 --- a/test_utilities/src/common.rs +++ b/test_utilities/src/common.rs @@ -192,14 +192,14 @@ pub fn environment_3_apps_3_routers_3_databases( test_domain: &str, database_instance_type: &str, database_disk_type: &str, - provider_kind: Kind, ) -> Environment { let app_name_1 = format!("{}-{}", "simple-app-1".to_string(), generate_id()); let app_name_2 = format!("{}-{}", "simple-app-2".to_string(), generate_id()); let app_name_3 = format!("{}-{}", "simple-app-3".to_string(), generate_id()); // mongoDB management part - let database_host_mongo = get_svc_name(DatabaseKind::Mongodb, provider_kind.clone()).to_string(); + let mongo_id = generate_id(); + let database_host_mongo = get_svc_name(DatabaseKind::Mongodb, mongo_id.clone()).to_string(); let database_port_mongo = 27017; let database_db_name_mongo = "my-mongodb".to_string(); let database_username_mongo = "superuser".to_string(); @@ -215,16 +215,18 @@ pub fn environment_3_apps_3_routers_3_databases( let version_mongo = "4.4"; // pSQL 1 management part + let psql_id_1 = generate_id(); let fqdn_id = "my-postgresql-".to_string() + generate_id().as_str(); - let fqdn = get_svc_name(DatabaseKind::Postgresql, provider_kind.clone()).to_string(); + let fqdn = get_svc_name(DatabaseKind::Postgresql, psql_id_1.clone()).to_string(); let database_port = 5432; let database_username = "superuser".to_string(); let database_password = generate_password(true); let database_name = "postgres".to_string(); // pSQL 2 management part + let psql_id_2 = generate_id(); let fqdn_id_2 = "my-postgresql-2".to_string() + generate_id().as_str(); - let fqdn_2 = format!("{}2", get_svc_name(DatabaseKind::Postgresql, provider_kind.clone())); + let fqdn_2 = get_svc_name(DatabaseKind::Postgresql, psql_id_2.clone()); let database_username_2 = "superuser2".to_string(); let database_name_2 = "postgres2".to_string(); @@ -422,7 +424,7 @@ pub fn environment_3_apps_3_routers_3_databases( Database { kind: DatabaseKind::Postgresql, action: Action::Create, - id: generate_id(), + id: psql_id_1.clone(), name: database_name.clone(), version: "11.8.0".to_string(), fqdn_id: fqdn_id.clone(), @@ -444,7 +446,7 @@ pub fn environment_3_apps_3_routers_3_databases( Database { kind: DatabaseKind::Postgresql, action: Action::Create, - id: generate_id(), + id: psql_id_2.clone(), name: database_name_2.clone(), version: "11.8.0".to_string(), fqdn_id: fqdn_id_2.clone(), @@ -466,7 +468,7 @@ pub fn environment_3_apps_3_routers_3_databases( Database { kind: DatabaseKind::Mongodb, action: Action::Create, - id: generate_id(), + id: mongo_id.clone(), name: database_db_name_mongo.clone(), version: version_mongo.to_string(), fqdn_id: "mongodb-".to_string() + generate_id().as_str(), @@ -511,7 +513,7 @@ pub fn working_minimal_environment(context: &Context, test_domain: &str) -> Envi action: Action::Create, applications: vec![Application { id: application_id, - name: application_name, + name: application_name.clone(), git_url: "https://github.com/Qovery/engine-testing.git".to_string(), commit_id: "fc575a2f3be0b9100492c8a463bf18134a8698a5".to_string(), dockerfile_path: Some("Dockerfile".to_string()), @@ -551,7 +553,7 @@ pub fn working_minimal_environment(context: &Context, test_domain: &str) -> Envi custom_domains: vec![], routes: vec![Route { path: "/".to_string(), - application_name: format!("{}-{}", "simple-app".to_string(), &suffix), + application_name, }], sticky_sessions_enabled: false, }], @@ -572,14 +574,13 @@ pub fn environment_only_http_server_router_with_sticky_session(context: &Context pub fn environnement_2_app_2_routers_1_psql( context: &Context, - test_domain: &str, database_instance_type: &str, database_disk_type: &str, - provider_kind: Kind, ) -> Environment { + let db_id = generate_id(); let fqdn_id = "my-postgresql-".to_string() + generate_id().as_str(); - let fqdn = get_svc_name(DatabaseKind::Postgresql, provider_kind.clone()).to_string(); + let fqdn = get_svc_name(DatabaseKind::Postgresql, db_id.clone()).to_string(); let database_port = 5432; let database_username = "superuser".to_string(); @@ -587,8 +588,8 @@ pub fn environnement_2_app_2_routers_1_psql( let database_name = "postgres".to_string(); let suffix = generate_id(); - let application_name1 = sanitize_name("postgresql", &format!("{}-{}", "postgresql-app1", &suffix)); - let application_name2 = sanitize_name("postgresql", &format!("{}-{}", "postgresql-app2", &suffix)); + let application_name1 = sanitize_name("postgresql", &format!("{}-{}", "postgresql-app1", &suffix), false); + let application_name2 = sanitize_name("postgresql", &format!("{}-{}", "postgresql-app2", &suffix), false); Environment { execution_id: context.execution_id().to_string(), @@ -600,7 +601,7 @@ pub fn environnement_2_app_2_routers_1_psql( databases: vec![Database { kind: DatabaseKind::Postgresql, action: Action::Create, - id: generate_id(), + id: db_id.clone(), name: database_name.clone(), version: "11.8.0".to_string(), fqdn_id: fqdn_id.clone(), @@ -973,7 +974,6 @@ pub fn routers_sessions_are_sticky(routers: Vec) -> bool { pub fn test_db( context: Context, logger: Box, - mut environment: Environment, secrets: FuncTestsSecrets, version: &str, test_name: &str, @@ -988,11 +988,11 @@ pub fn test_db( let _enter = span.enter(); let context_for_delete = context.clone_not_same_execution_id(); - let app_id = generate_id(); + let db_id = generate_id(); let database_username = "superuser".to_string(); let database_password = generate_id(); let db_kind_str = db_kind.name().to_string(); - let database_host = format!( + let fqdn = format!( "{}-{}.{}.{}", db_kind_str.clone(), generate_id(), @@ -1003,10 +1003,10 @@ pub fn test_db( .expect("DEFAULT_TEST_DOMAIN is not set in secrets") ); let dyn_db_fqdn = match is_public.clone() { - true => database_host.clone(), + true => fqdn.clone(), false => match database_mode.clone() { - DatabaseMode::MANAGED => format!("{}-dns", app_id.clone()), - DatabaseMode::CONTAINER => get_svc_name(db_kind.clone(), provider_kind.clone()).to_string(), + DatabaseMode::MANAGED => format!("{}-dns", db_id.clone()), + DatabaseMode::CONTAINER => get_svc_name(db_kind.clone(), db_id.clone()).to_string(), }, }; @@ -1022,53 +1022,72 @@ pub fn test_db( let storage_size = 10; let db_disk_type = db_disk_type(provider_kind.clone(), database_mode.clone()); let db_instance_type = db_instance_type(provider_kind.clone(), db_kind.clone(), database_mode.clone()); - let db = Database { - kind: db_kind.clone(), + + let environment = Environment { + execution_id: context.execution_id().to_string(), + id: generate_id(), + owner_id: generate_id(), + project_id: generate_id(), + organization_id: context.organization_id().to_string(), action: Action::Create, - id: app_id.clone(), - name: database_db_name.clone(), - version: version.to_string(), - fqdn_id: format!("{}-{}", db_kind_str.clone(), generate_id()), - fqdn: database_host.clone(), - port: database_port.clone(), - username: database_username.clone(), - password: database_password.clone(), - total_cpus: "100m".to_string(), - total_ram_in_mib: 512, - disk_size_in_gib: storage_size.clone(), - database_instance_type: db_instance_type.to_string(), - database_disk_type: db_disk_type.to_string(), - encrypt_disk: true, - activate_high_availability: false, - activate_backups: false, - publicly_accessible: is_public.clone(), - mode: database_mode.clone(), - }; - - environment.databases = vec![db.clone()]; - - let app_name = format!("{}-app-{}", db_kind_str.clone(), generate_id()); - environment.applications = environment - .applications - .into_iter() - .map(|mut app| { - app.branch = app_name.clone(); - app.commit_id = db_infos.app_commit.clone(); - app.ports = vec![Port { + applications: vec![Application { + id: generate_id(), + name: format!("{}-app-{}", db_kind_str.clone(), generate_id()), + git_url: "https://github.com/Qovery/engine-testing.git".to_string(), + commit_id: db_infos.app_commit.clone(), + dockerfile_path: Some(format!("Dockerfile-{}", version)), + buildpack_language: None, + root_path: String::from("/"), + action: Action::Create, + git_credentials: Some(GitCredentials { + login: "x-access-token".to_string(), + access_token: "xxx".to_string(), + expired_at: Utc::now(), + }), + storage: vec![], + environment_vars: db_infos.app_env_vars.clone(), + branch: "basic-app-deploy".to_string(), + ports: vec![Port { id: "zdf7d6aad".to_string(), long_id: Default::default(), - port: 1234, - public_port: Some(1234), + port: 80, + public_port: Some(443), name: None, - publicly_accessible: true, + publicly_accessible: false, protocol: Protocol::HTTP, - }]; - app.dockerfile_path = Some(format!("Dockerfile-{}", version)); - app.environment_vars = db_infos.app_env_vars.clone(); - app - }) - .collect::>(); - environment.routers[0].routes[0].application_name = app_name.clone(); + }], + total_cpus: "100m".to_string(), + total_ram_in_mib: 256, + min_instances: 2, + max_instances: 2, + cpu_burst: "100m".to_string(), + start_timeout_in_seconds: 60, + }], + routers: vec![], + databases: vec![Database { + kind: db_kind.clone(), + action: Action::Create, + id: db_id.clone(), + name: database_db_name.clone(), + version: version.to_string(), + fqdn_id: format!("{}-{}", db_kind_str.clone(), generate_id()), + fqdn: dyn_db_fqdn.clone(), + port: database_port.clone(), + username: database_username.clone(), + password: database_password.clone(), + total_cpus: "100m".to_string(), + total_ram_in_mib: 512, + disk_size_in_gib: storage_size.clone(), + database_instance_type: db_instance_type.to_string(), + database_disk_type: db_disk_type.to_string(), + encrypt_disk: true, + activate_high_availability: false, + activate_backups: false, + publicly_accessible: is_public.clone(), + mode: database_mode.clone(), + }], + clone_from_environment_id: None, + }; let mut environment_delete = environment.clone(); environment_delete.action = Action::Delete; @@ -1102,22 +1121,28 @@ pub fn test_db( environment.clone(), secrets.clone(), ) { - Ok(svc) => assert_eq!( - svc.items - .expect("No items in svc") - .into_iter() - .filter(|svc| svc - .metadata - .name - .contains(get_svc_name(db_kind.clone(), provider_kind.clone())) - && &svc.spec.svc_type == "LoadBalancer") - .collect::>() - .len(), - match is_public { - true => 1, - false => 0, - } - ), + Ok(svc) => match is_public { + true => assert_eq!( + svc.items + .expect("No items in svc") + .into_iter() + .filter(|svc| svc.metadata.name == get_svc_name(db_kind.clone(), db_id.clone()) + && &svc.spec.svc_type == "LoadBalancer") + .collect::>() + .len(), + 1 + ), + false => assert_eq!( + svc.items + .expect("No items in svc") + .into_iter() + .filter(|svc| svc.metadata.name == get_svc_name(db_kind.clone(), db_id.clone()) + && &svc.spec.svc_type == "ClusterIP") + .collect::>() + .len(), + 1 + ), + }, Err(_) => assert!(false), }; } @@ -1129,8 +1154,7 @@ pub fn test_db( .expect("No items in svc") .into_iter() .filter(|svc| { - svc.metadata.name.contains(format!("{}-dns", app_id.clone()).as_str()) - && svc.spec.svc_type == "ExternalName" + svc.metadata.name == format!("{}-dns", db_id.clone()) && svc.spec.svc_type == "ExternalName" }) .collect::>(); let annotations = &service[0].metadata.annotations; @@ -1138,7 +1162,7 @@ pub fn test_db( match is_public { true => { assert!(annotations.contains_key("external-dns.alpha.kubernetes.io/hostname")); - assert_eq!(annotations["external-dns.alpha.kubernetes.io/hostname"], database_host); + assert_eq!(annotations["external-dns.alpha.kubernetes.io/hostname"], fqdn); } false => assert!(!annotations.contains_key("external-dns.alpha.kubernetes.io/hostname")), } diff --git a/test_utilities/src/utilities.rs b/test_utilities/src/utilities.rs index e967e8e7..9eb1c1f5 100644 --- a/test_utilities/src/utilities.rs +++ b/test_utilities/src/utilities.rs @@ -987,7 +987,7 @@ pub fn db_infos( match db_kind { DatabaseKind::Mongodb => { let database_port = 27017; - let database_db_name = "my-mongodb".to_string(); + let database_db_name = "mymongodb".to_string(); let database_uri = format!( "mongodb://{}:{}@{}:{}/{}", database_username, @@ -1029,7 +1029,7 @@ pub fn db_infos( } DatabaseKind::Postgresql => { let database_port = 5432; - let database_db_name = "postgres".to_string(); + let database_db_name = "mypostgres".to_string(); DBInfos { db_port: database_port.clone(), db_name: database_db_name.to_string(), @@ -1045,7 +1045,7 @@ pub fn db_infos( } DatabaseKind::Redis => { let database_port = 6379; - let database_db_name = "my-redis".to_string(); + let database_db_name = "myredis".to_string(); DBInfos { db_port: database_port.clone(), db_name: database_db_name.to_string(), @@ -1097,23 +1097,11 @@ pub fn db_instance_type(provider_kind: Kind, db_kind: DatabaseKind, database_mod .to_string() } -pub fn get_svc_name(db_kind: DatabaseKind, provider_kind: Kind) -> &'static str { +pub fn get_svc_name(db_kind: DatabaseKind, db_id: String) -> String { match db_kind { - DatabaseKind::Postgresql => match provider_kind { - Kind::Aws => "postgresqlpostgres", - _ => "postgresql-postgres", - }, - DatabaseKind::Mysql => match provider_kind { - Kind::Aws => "mysqlmysqldatabase", - _ => "mysql-mysqldatabase", - }, - DatabaseKind::Mongodb => match provider_kind { - Kind::Aws => "mongodbmymongodb", - _ => "mongodb-my-mongodb", - }, - DatabaseKind::Redis => match provider_kind { - Kind::Aws => "redismyredis-master", - _ => "redis-my-redis-master", - }, + DatabaseKind::Postgresql => format!("{}-postgresql", db_id), + DatabaseKind::Mysql => format!("{}-mysql", db_id), + DatabaseKind::Mongodb => format!("{}-mongodb", db_id), + DatabaseKind::Redis => format!("{}-redis", db_id), } } diff --git a/tests/aws/aws_databases.rs b/tests/aws/aws_databases.rs index a20438c7..774678d1 100644 --- a/tests/aws/aws_databases.rs +++ b/tests/aws/aws_databases.rs @@ -2,9 +2,7 @@ extern crate test_utilities; use ::function_name::named; use qovery_engine::cloud_provider::Kind; -use qovery_engine::models::{ - Action, Clone2, Context, Database, DatabaseKind, DatabaseMode, Environment, EnvironmentAction, Port, Protocol, -}; +use qovery_engine::models::{Action, Clone2, Database, DatabaseKind, DatabaseMode, EnvironmentAction, Port, Protocol}; use qovery_engine::transaction::TransactionResult; use tracing::{span, Level}; @@ -56,7 +54,6 @@ fn deploy_an_environment_with_3_databases_and_3_apps() { .as_str(), AWS_DATABASE_INSTANCE_TYPE, AWS_DATABASE_DISK_TYPE, - Kind::Aws, ); let mut environment_delete = environment.clone(); @@ -115,7 +112,6 @@ fn deploy_an_environment_with_db_and_pause_it() { .as_str(), AWS_DATABASE_INSTANCE_TYPE, AWS_DATABASE_DISK_TYPE, - Kind::Aws, ); let mut environment_delete = environment.clone(); @@ -136,7 +132,7 @@ fn deploy_an_environment_with_db_and_pause_it() { }; // Check that we have actually 0 pods running for this db - let app_name = format!("postgresql{}-0", environment.databases[0].name); + let app_name = format!("postgresql-{}-0", environment.databases[0].name); let ret = get_pods( context.clone(), Kind::Aws, @@ -194,7 +190,6 @@ fn postgresql_failover_dev_environment_with_all_options() { test_domain.as_str(), AWS_DATABASE_INSTANCE_TYPE, AWS_DATABASE_DISK_TYPE, - Kind::Aws, ); let environment_check = environment.clone(); let mut environment_never_up = environment.clone(); @@ -220,7 +215,6 @@ fn postgresql_failover_dev_environment_with_all_options() { test_domain.as_str(), AWS_DATABASE_INSTANCE_TYPE, AWS_DATABASE_DISK_TYPE, - Kind::Aws, ); environment_delete.action = Action::Delete; @@ -235,7 +229,7 @@ fn postgresql_failover_dev_environment_with_all_options() { TransactionResult::UnrecoverableError(_, _) => assert!(false), }; // TO CHECK: DATABASE SHOULDN'T BE RESTARTED AFTER A REDEPLOY - let database_name = format!("postgresql{}-0", &environment_check.databases[0].name); + let database_name = format!("postgresql-{}-0", &environment_check.databases[0].name); match is_pod_restarted_env( context.clone(), Kind::Aws, @@ -310,7 +304,6 @@ fn postgresql_deploy_a_working_development_environment_with_all_options() { test_domain.as_str(), AWS_DATABASE_INSTANCE_TYPE, AWS_DATABASE_DISK_TYPE, - Kind::Aws, ); //let env_to_check = environment.clone(); let mut environment_delete = test_utilities::common::environnement_2_app_2_routers_1_psql( @@ -318,7 +311,6 @@ fn postgresql_deploy_a_working_development_environment_with_all_options() { test_domain.as_str(), AWS_DATABASE_INSTANCE_TYPE, AWS_DATABASE_DISK_TYPE, - Kind::Aws, ); environment_delete.action = Action::Delete; @@ -387,7 +379,8 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { ); let app_name = format!("postgresql-app-{}", generate_id()); - let database_host = get_svc_name(DatabaseKind::Postgresql, Kind::Aws).to_string(); + let db_id = generate_id(); + let database_host = get_svc_name(DatabaseKind::Postgresql, db_id.clone()).to_string(); let database_port = 5432; let database_db_name = "postgres".to_string(); let database_username = "superuser".to_string(); @@ -395,7 +388,7 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { environment.databases = vec![Database { kind: DatabaseKind::Postgresql, action: Action::Create, - id: generate_id(), + id: db_id.clone(), name: database_db_name.clone(), version: "11.8.0".to_string(), fqdn_id: "postgresql-".to_string() + generate_id().as_str(), @@ -418,7 +411,8 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { .applications .into_iter() .map(|mut app| { - app.branch = app_name.clone(); + app.id = app_name.clone(); + app.name = app_name.clone(); app.commit_id = "5990752647af11ef21c3d46a51abbde3da1ab351".to_string(); app.ports = vec![Port { id: "zdf7d6aad".to_string(), @@ -462,7 +456,7 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { TransactionResult::UnrecoverableError(_, _) => assert!(false), }; // TO CHECK: DATABASE SHOULDN'T BE RESTARTED AFTER A REDEPLOY - let database_name = format!("postgresql{}-0", &environment_check.databases[0].name); + let database_name = format!("postgresql-{}-0", &environment_check.databases[0].name); match is_pod_restarted_env( context.clone(), Kind::Aws, @@ -490,20 +484,25 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { ** **/ #[allow(dead_code)] -fn test_postgresql_configuration( - context: Context, - environment: Environment, - secrets: FuncTestsSecrets, - version: &str, - test_name: &str, - database_mode: DatabaseMode, - is_public: bool, -) { +fn test_postgresql_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + engine_run_test(|| { test_db( context, logger(), - environment, secrets, version, test_name, @@ -520,228 +519,56 @@ fn test_postgresql_configuration( #[named] #[test] fn private_postgresql_v10_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "10", function_name!(), CONTAINER, false); + test_postgresql_configuration("10", function_name!(), CONTAINER, false); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn public_postgresql_v10_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "10", function_name!(), CONTAINER, true); + test_postgresql_configuration("10", function_name!(), CONTAINER, true); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn private_postgresql_v11_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "11", function_name!(), CONTAINER, false); + test_postgresql_configuration("11", function_name!(), CONTAINER, false); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn public_postgresql_v11_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "11", function_name!(), CONTAINER, true); + test_postgresql_configuration("11", function_name!(), CONTAINER, true); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn private_postgresql_v12_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "12", function_name!(), CONTAINER, false); + test_postgresql_configuration("12", function_name!(), CONTAINER, false); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn public_postgresql_v12_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "12", function_name!(), CONTAINER, true); + test_postgresql_configuration("12", function_name!(), CONTAINER, true); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn private_postgresql_v13_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "13", function_name!(), CONTAINER, false); + test_postgresql_configuration("13", function_name!(), CONTAINER, false); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn public_postgresql_v13_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "13", function_name!(), CONTAINER, true); + test_postgresql_configuration("13", function_name!(), CONTAINER, true); } // Postgres production environment @@ -749,232 +576,56 @@ fn public_postgresql_v13_deploy_a_working_dev_environment() { #[named] #[test] fn private_postgresql_v10_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "10", function_name!(), MANAGED, false); + test_postgresql_configuration("10", function_name!(), MANAGED, false); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn public_postgresql_v10_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "10", function_name!(), MANAGED, true); + test_postgresql_configuration("10", function_name!(), MANAGED, true); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn private_postgresql_v11_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "11", function_name!(), MANAGED, false); + test_postgresql_configuration("11", function_name!(), MANAGED, false); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn public_postgresql_v11_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "11", function_name!(), MANAGED, true); + test_postgresql_configuration("11", function_name!(), MANAGED, true); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn private_postgresql_v12_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "12", function_name!(), MANAGED, false); + test_postgresql_configuration("12", function_name!(), MANAGED, false); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn public_postgresql_v12_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "12", function_name!(), MANAGED, true); + test_postgresql_configuration("12", function_name!(), MANAGED, true); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn private_postgresql_v13_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "13", function_name!(), MANAGED, false); + test_postgresql_configuration("13", function_name!(), MANAGED, false); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn public_postgresql_v13_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "13", function_name!(), MANAGED, true); + test_postgresql_configuration("13", function_name!(), MANAGED, true); } /** @@ -983,20 +634,25 @@ fn public_postgresql_v13_deploy_a_working_prod_environment() { ** **/ #[allow(dead_code)] -fn test_mongodb_configuration( - context: Context, - environment: Environment, - secrets: FuncTestsSecrets, - version: &str, - test_name: &str, - database_mode: DatabaseMode, - is_public: bool, -) { +fn test_mongodb_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + engine_run_test(|| { test_db( context, logger(), - environment, secrets, version, test_name, @@ -1013,232 +669,56 @@ fn test_mongodb_configuration( #[named] #[test] fn private_mongodb_v3_6_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "3.6", function_name!(), CONTAINER, false); + test_mongodb_configuration("3.6", function_name!(), CONTAINER, false); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn public_mongodb_v3_6_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "3.6", function_name!(), CONTAINER, true); + test_mongodb_configuration("3.6", function_name!(), CONTAINER, true); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn private_mongodb_v4_0_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.0", function_name!(), CONTAINER, false); + test_mongodb_configuration("4.0", function_name!(), CONTAINER, false); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn public_mongodb_v4_0_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.0", function_name!(), CONTAINER, true); + test_mongodb_configuration("4.0", function_name!(), CONTAINER, true); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn private_mongodb_v4_2_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.2", function_name!(), CONTAINER, false); + test_mongodb_configuration("4.2", function_name!(), CONTAINER, false); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn public_mongodb_v4_2_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.2", function_name!(), CONTAINER, true); + test_mongodb_configuration("4.2", function_name!(), CONTAINER, true); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn private_mongodb_v4_4_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.4", function_name!(), CONTAINER, false); + test_mongodb_configuration("4.4", function_name!(), CONTAINER, false); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn public_mongodb_v4_4_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.4", function_name!(), CONTAINER, true); + test_mongodb_configuration("4.4", function_name!(), CONTAINER, true); } // MongoDB production environment (DocumentDB) @@ -1246,116 +726,28 @@ fn public_mongodb_v4_4_deploy_a_working_dev_environment() { #[named] #[test] fn private_mongodb_v3_6_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "3.6", function_name!(), MANAGED, false); + test_mongodb_configuration("3.6", function_name!(), MANAGED, false); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn public_mongodb_v3_6_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "3.6", function_name!(), MANAGED, true); + test_mongodb_configuration("3.6", function_name!(), MANAGED, true); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn private_mongodb_v4_0_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.0", function_name!(), MANAGED, false); + test_mongodb_configuration("4.0", function_name!(), MANAGED, false); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn public_mongodb_v4_0_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.0", function_name!(), MANAGED, true); + test_mongodb_configuration("4.0", function_name!(), MANAGED, true); } /** @@ -1364,20 +756,25 @@ fn public_mongodb_v4_0_deploy_a_working_prod_environment() { ** **/ #[allow(dead_code)] -fn test_mysql_configuration( - context: Context, - environment: Environment, - secrets: FuncTestsSecrets, - version: &str, - test_name: &str, - database_mode: DatabaseMode, - is_public: bool, -) { +fn test_mysql_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + engine_run_test(|| { test_db( context, logger(), - environment, secrets, version, test_name, @@ -1394,116 +791,28 @@ fn test_mysql_configuration( #[named] #[test] fn private_mysql_v5_7_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "5.7", function_name!(), CONTAINER, false); + test_mysql_configuration("5.7", function_name!(), CONTAINER, false); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn public_mysql_v5_7_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "5.7", function_name!(), CONTAINER, true); + test_mysql_configuration("5.7", function_name!(), CONTAINER, true); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn private_mysql_v8_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), CONTAINER, false); + test_mysql_configuration("8.0", function_name!(), CONTAINER, false); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn public_mysql_v8_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), CONTAINER, true); + test_mysql_configuration("8.0", function_name!(), CONTAINER, true); } // MySQL production environment (RDS) @@ -1511,116 +820,28 @@ fn public_mysql_v8_deploy_a_working_dev_environment() { #[named] #[test] fn private_mysql_v5_7_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "5.7", function_name!(), MANAGED, false); + test_mysql_configuration("5.7", function_name!(), MANAGED, false); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn public_mysql_v5_7_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "5.7", function_name!(), MANAGED, true); + test_mysql_configuration("5.7", function_name!(), MANAGED, true); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn private_mysql_v8_0_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), MANAGED, false); + test_mysql_configuration("8.0", function_name!(), MANAGED, false); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn public_mysql_v8_0_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), MANAGED, true); + test_mysql_configuration("8.0", function_name!(), MANAGED, true); } /** @@ -1629,20 +850,25 @@ fn public_mysql_v8_0_deploy_a_working_prod_environment() { ** **/ #[allow(dead_code)] -fn test_redis_configuration( - context: Context, - environment: Environment, - secrets: FuncTestsSecrets, - version: &str, - test_name: &str, - database_mode: DatabaseMode, - is_public: bool, -) { +fn test_redis_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + engine_run_test(|| { test_db( context, logger(), - environment, secrets, version, test_name, @@ -1659,116 +885,28 @@ fn test_redis_configuration( #[named] #[test] fn private_redis_v5_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_redis_configuration(context, environment, secrets, "5", function_name!(), CONTAINER, false); + test_redis_configuration("5", function_name!(), CONTAINER, false); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn public_redis_v5_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_redis_configuration(context, environment, secrets, "5", function_name!(), CONTAINER, true); + test_redis_configuration("5", function_name!(), CONTAINER, true); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn private_redis_v6_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_redis_configuration(context, environment, secrets, "6", function_name!(), CONTAINER, false); + test_redis_configuration("6", function_name!(), CONTAINER, false); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn public_redis_v6_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_redis_configuration(context, environment, secrets, "6", function_name!(), CONTAINER, true); + test_redis_configuration("6", function_name!(), CONTAINER, true); } // Redis production environment (Elasticache) @@ -1776,114 +914,26 @@ fn public_redis_v6_deploy_a_working_dev_environment() { #[named] #[test] fn private_redis_v5_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_redis_configuration(context, environment, secrets, "5", function_name!(), MANAGED, false); + test_redis_configuration("5", function_name!(), MANAGED, false); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn public_redis_v5_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_redis_configuration(context, environment, secrets, "5", function_name!(), MANAGED, true); + test_redis_configuration("5", function_name!(), MANAGED, true); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn private_redis_v6_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_redis_configuration(context, environment, secrets, "6", function_name!(), MANAGED, false); + test_redis_configuration("6", function_name!(), MANAGED, false); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn public_redis_v6_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .clone() - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_redis_configuration(context, environment, secrets, "6", function_name!(), MANAGED, true); + test_redis_configuration("6", function_name!(), MANAGED, true); } diff --git a/tests/digitalocean/do_databases.rs b/tests/digitalocean/do_databases.rs index 3c96542b..b781976e 100644 --- a/tests/digitalocean/do_databases.rs +++ b/tests/digitalocean/do_databases.rs @@ -2,16 +2,14 @@ use ::function_name::named; use tracing::{span, warn, Level}; use qovery_engine::cloud_provider::{Kind as ProviderKind, Kind}; -use qovery_engine::models::{ - Action, Clone2, Context, Database, DatabaseKind, DatabaseMode, Environment, EnvironmentAction, Port, Protocol, -}; +use qovery_engine::models::{Action, Clone2, Database, DatabaseKind, DatabaseMode, EnvironmentAction, Port, Protocol}; use qovery_engine::transaction::TransactionResult; use test_utilities::utilities::{ context, engine_run_test, generate_id, get_pods, get_svc_name, init, is_pod_restarted_env, logger, FuncTestsSecrets, }; use qovery_engine::models::DatabaseMode::{CONTAINER, MANAGED}; -use test_utilities::common::{test_db, working_minimal_environment, Infrastructure}; +use test_utilities::common::{test_db, Infrastructure}; use test_utilities::digitalocean::{ clean_environments, DO_MANAGED_DATABASE_DISK_TYPE, DO_MANAGED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, DO_TEST_REGION, @@ -57,7 +55,6 @@ fn deploy_an_environment_with_3_databases_and_3_apps() { .as_str(), DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, - Kind::Do, ); let mut environment_delete = environment.clone(); @@ -119,7 +116,6 @@ fn deploy_an_environment_with_db_and_pause_it() { .as_str(), DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, - Kind::Do, ); let mut environment_delete = environment.clone(); @@ -201,7 +197,6 @@ fn postgresql_failover_dev_environment_with_all_options() { test_domain.as_str(), DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, - Kind::Do, ); let environment_check = environment.clone(); let mut environment_never_up = environment.clone(); @@ -227,7 +222,6 @@ fn postgresql_failover_dev_environment_with_all_options() { test_domain.as_str(), DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, - Kind::Do, ); environment_delete.action = Action::Delete; @@ -321,7 +315,6 @@ fn postgresql_deploy_a_working_development_environment_with_all_options() { test_domain.as_str(), DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, - Kind::Do, ); //let env_to_check = environment.clone(); let mut environment_delete = test_utilities::common::environnement_2_app_2_routers_1_psql( @@ -329,7 +322,6 @@ fn postgresql_deploy_a_working_development_environment_with_all_options() { test_domain.as_str(), DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, - Kind::Do, ); environment_delete.action = Action::Delete; @@ -406,8 +398,9 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { ); let database_mode = CONTAINER; + let db_id = generate_id(); let app_name = format!("postgresql-app-{}", generate_id()); - let database_host = get_svc_name(DatabaseKind::Postgresql, Kind::Do).to_string(); + let database_host = get_svc_name(DatabaseKind::Postgresql, db_id.clone()).to_string(); let database_port = 5432; let database_db_name = "postgres".to_string(); let database_username = "superuser".to_string(); @@ -415,7 +408,7 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { environment.databases = vec![Database { kind: DatabaseKind::Postgresql, action: Action::Create, - id: generate_id(), + id: db_id.clone(), name: database_db_name.clone(), version: "11.8.0".to_string(), fqdn_id: "postgresql-".to_string() + generate_id().as_str(), @@ -529,20 +522,23 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { ** **/ #[allow(dead_code)] -fn test_postgresql_configuration( - context: Context, - environment: Environment, - secrets: FuncTestsSecrets, - version: &str, - test_name: &str, - database_mode: DatabaseMode, - is_public: bool, -) { +fn test_postgresql_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + engine_run_test(|| { test_db( context, logger(), - environment, secrets, version, test_name, @@ -559,27 +555,7 @@ fn test_postgresql_configuration( #[named] #[test] fn private_postgresql_v10_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "10", function_name!(), CONTAINER, false); + test_postgresql_configuration("10", function_name!(), CONTAINER, false); } #[cfg(feature = "test-do-self-hosted")] @@ -587,27 +563,7 @@ fn private_postgresql_v10_deploy_a_working_dev_environment() { #[named] #[test] fn public_postgresql_v10_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "10", function_name!(), CONTAINER, true); + test_postgresql_configuration("10", function_name!(), CONTAINER, true); } #[cfg(feature = "test-do-self-hosted")] @@ -615,27 +571,7 @@ fn public_postgresql_v10_deploy_a_working_dev_environment() { #[named] #[test] fn private_postgresql_v11_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "11", function_name!(), CONTAINER, false); + test_postgresql_configuration("11", function_name!(), CONTAINER, false); } #[cfg(feature = "test-do-self-hosted")] @@ -643,135 +579,35 @@ fn private_postgresql_v11_deploy_a_working_dev_environment() { #[named] #[test] fn public_postgresql_v11_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "11", function_name!(), CONTAINER, true); + test_postgresql_configuration("11", function_name!(), CONTAINER, true); } #[cfg(feature = "test-do-self-hosted")] #[named] #[test] fn private_postgresql_v12_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "12", function_name!(), CONTAINER, false); + test_postgresql_configuration("12", function_name!(), CONTAINER, false); } #[cfg(feature = "test-do-self-hosted")] #[named] #[test] fn public_postgresql_v12_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "12", function_name!(), CONTAINER, true); + test_postgresql_configuration("12", function_name!(), CONTAINER, true); } #[cfg(feature = "test-do-self-hosted")] #[named] #[test] fn private_postgresql_v13_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "13", function_name!(), CONTAINER, false); + test_postgresql_configuration("13", function_name!(), CONTAINER, false); } #[cfg(feature = "test-do-self-hosted")] #[named] #[test] fn public_postgresql_v13_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "13", function_name!(), CONTAINER, true); + test_postgresql_configuration("13", function_name!(), CONTAINER, true); } /** @@ -780,20 +616,23 @@ fn public_postgresql_v13_deploy_a_working_dev_environment() { ** **/ #[allow(dead_code)] -fn test_mongodb_configuration( - context: Context, - environment: Environment, - secrets: FuncTestsSecrets, - version: &str, - test_name: &str, - database_mode: DatabaseMode, - is_public: bool, -) { +fn test_mongodb_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + engine_run_test(|| { test_db( context, logger(), - environment, secrets, version, test_name, @@ -810,27 +649,7 @@ fn test_mongodb_configuration( #[named] #[test] fn private_mongodb_v3_6_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "3.6", function_name!(), CONTAINER, false); + test_mongodb_configuration("3.6", function_name!(), CONTAINER, false); } #[cfg(feature = "test-do-self-hosted")] @@ -838,27 +657,7 @@ fn private_mongodb_v3_6_deploy_a_working_dev_environment() { #[named] #[test] fn public_mongodb_v3_6_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "3.6", function_name!(), CONTAINER, true); + test_mongodb_configuration("3.6", function_name!(), CONTAINER, true); } #[cfg(feature = "test-do-self-hosted")] @@ -866,27 +665,7 @@ fn public_mongodb_v3_6_deploy_a_working_dev_environment() { #[named] #[test] fn private_mongodb_v4_0_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.0", function_name!(), CONTAINER, false); + test_mongodb_configuration("4.0", function_name!(), CONTAINER, false); } #[cfg(feature = "test-do-self-hosted")] @@ -894,27 +673,7 @@ fn private_mongodb_v4_0_deploy_a_working_dev_environment() { #[named] #[test] fn public_mongodb_v4_0_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.0", function_name!(), CONTAINER, true); + test_mongodb_configuration("4.0", function_name!(), CONTAINER, true); } #[cfg(feature = "test-do-self-hosted")] @@ -922,108 +681,28 @@ fn public_mongodb_v4_0_deploy_a_working_dev_environment() { #[named] #[test] fn private_mongodb_v4_2_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.2", function_name!(), CONTAINER, false); + test_mongodb_configuration("4.2", function_name!(), CONTAINER, false); } #[cfg(feature = "test-do-self-hosted")] #[named] #[test] fn public_mongodb_v4_2_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.2", function_name!(), CONTAINER, true); + test_mongodb_configuration("4.2", function_name!(), CONTAINER, true); } #[cfg(feature = "test-do-self-hosted")] #[named] #[test] fn private_mongodb_v4_4_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.4", function_name!(), CONTAINER, false); + test_mongodb_configuration("4.4", function_name!(), CONTAINER, false); } #[cfg(feature = "test-do-self-hosted")] #[named] #[test] fn public_mongodb_v4_4_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.4", function_name!(), CONTAINER, true); + test_mongodb_configuration("4.4", function_name!(), CONTAINER, true); } /** @@ -1032,20 +711,23 @@ fn public_mongodb_v4_4_deploy_a_working_dev_environment() { ** **/ #[allow(dead_code)] -fn test_mysql_configuration( - context: Context, - environment: Environment, - secrets: FuncTestsSecrets, - version: &str, - test_name: &str, - database_mode: DatabaseMode, - is_public: bool, -) { +fn test_mysql_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + engine_run_test(|| { test_db( context, logger(), - environment, secrets, version, test_name, @@ -1062,27 +744,7 @@ fn test_mysql_configuration( #[named] #[test] fn private_mysql_v5_7_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "5.7", function_name!(), CONTAINER, false); + test_mysql_configuration("5.7", function_name!(), CONTAINER, false); } #[cfg(feature = "test-do-self-hosted")] @@ -1090,27 +752,7 @@ fn private_mysql_v5_7_deploy_a_working_dev_environment() { #[named] #[test] fn public_mysql_v5_7_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "5.7", function_name!(), CONTAINER, true); + test_mysql_configuration("5.7", function_name!(), CONTAINER, true); } #[cfg(feature = "test-do-self-hosted")] @@ -1118,54 +760,14 @@ fn public_mysql_v5_7_deploy_a_working_dev_environment() { #[named] #[test] fn private_mysql_v8_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), CONTAINER, false); + test_mysql_configuration("8.0", function_name!(), CONTAINER, false); } #[cfg(feature = "test-do-self-hosted")] #[named] #[test] fn public_mysql_v8_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), CONTAINER, true); + test_mysql_configuration("8.0", function_name!(), CONTAINER, true); } // MySQL production environment @@ -1176,20 +778,23 @@ fn public_mysql_v8_deploy_a_working_dev_environment() { ** **/ #[allow(dead_code)] -fn test_redis_configuration( - context: Context, - environment: Environment, - secrets: FuncTestsSecrets, - version: &str, - test_name: &str, - database_mode: DatabaseMode, - is_public: bool, -) { +fn test_redis_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + engine_run_test(|| { test_db( context, logger(), - environment, secrets, version, test_name, @@ -1206,27 +811,7 @@ fn test_redis_configuration( #[named] #[test] fn private_redis_v5_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_redis_configuration(context, environment, secrets, "5", function_name!(), CONTAINER, false); + test_redis_configuration("5", function_name!(), CONTAINER, false); } #[cfg(feature = "test-do-self-hosted")] @@ -1234,27 +819,7 @@ fn private_redis_v5_deploy_a_working_dev_environment() { #[named] #[test] fn public_redis_v5_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_redis_configuration(context, environment, secrets, "5", function_name!(), CONTAINER, true); + test_redis_configuration("5", function_name!(), CONTAINER, true); } #[cfg(feature = "test-do-self-hosted")] @@ -1262,52 +827,12 @@ fn public_redis_v5_deploy_a_working_dev_environment() { #[named] #[test] fn private_redis_v6_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_redis_configuration(context, environment, secrets, "6", function_name!(), CONTAINER, false); + test_redis_configuration("6", function_name!(), CONTAINER, false); } #[cfg(feature = "test-do-self-hosted")] #[named] #[test] fn public_redis_v6_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_redis_configuration(context, environment, secrets, "6", function_name!(), CONTAINER, true); + test_redis_configuration("6", function_name!(), CONTAINER, true); } diff --git a/tests/scaleway/scw_databases.rs b/tests/scaleway/scw_databases.rs index 6caed734..0fb4a903 100644 --- a/tests/scaleway/scw_databases.rs +++ b/tests/scaleway/scw_databases.rs @@ -2,9 +2,7 @@ use ::function_name::named; use tracing::{span, warn, Level}; use qovery_engine::cloud_provider::{Kind as ProviderKind, Kind}; -use qovery_engine::models::{ - Action, Clone2, Context, Database, DatabaseKind, DatabaseMode, Environment, EnvironmentAction, Port, Protocol, -}; +use qovery_engine::models::{Action, Clone2, Database, DatabaseKind, DatabaseMode, EnvironmentAction, Port, Protocol}; use qovery_engine::transaction::TransactionResult; use test_utilities::utilities::{ context, engine_run_test, generate_id, generate_password, get_pods, get_svc_name, init, is_pod_restarted_env, @@ -12,8 +10,8 @@ use test_utilities::utilities::{ }; use qovery_engine::models::DatabaseMode::{CONTAINER, MANAGED}; +use test_utilities::common::test_db; use test_utilities::common::Infrastructure; -use test_utilities::common::{test_db, working_minimal_environment}; use test_utilities::scaleway::{ clean_environments, SCW_MANAGED_DATABASE_DISK_TYPE, SCW_MANAGED_DATABASE_INSTANCE_TYPE, SCW_SELF_HOSTED_DATABASE_DISK_TYPE, SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, SCW_TEST_ZONE, @@ -61,7 +59,6 @@ fn deploy_an_environment_with_3_databases_and_3_apps() { .as_str(), SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, SCW_SELF_HOSTED_DATABASE_DISK_TYPE, - Kind::Scw, ); let mut environment_delete = environment.clone(); @@ -125,7 +122,6 @@ fn deploy_an_environment_with_db_and_pause_it() { .as_str(), SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, SCW_SELF_HOSTED_DATABASE_DISK_TYPE, - Kind::Scw, ); let mut environment_delete = environment.clone(); @@ -214,7 +210,6 @@ fn postgresql_failover_dev_environment_with_all_options() { test_domain.as_str(), SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, SCW_SELF_HOSTED_DATABASE_DISK_TYPE, - Kind::Scw, ); let environment_check = environment.clone(); let mut environment_never_up = environment.clone(); @@ -240,7 +235,6 @@ fn postgresql_failover_dev_environment_with_all_options() { test_domain.as_str(), SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, SCW_SELF_HOSTED_DATABASE_DISK_TYPE, - Kind::Scw, ); environment_delete.action = Action::Delete; @@ -336,14 +330,12 @@ fn postgresql_deploy_a_working_development_environment_with_all_options() { test_domain.as_str(), SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, SCW_SELF_HOSTED_DATABASE_DISK_TYPE, - Kind::Scw, ); let mut environment_delete = test_utilities::common::environnement_2_app_2_routers_1_psql( &context_for_deletion, test_domain.as_str(), SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, SCW_SELF_HOSTED_DATABASE_DISK_TYPE, - Kind::Scw, ); environment_delete.action = Action::Delete; @@ -416,8 +408,9 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { .as_str(), ); + let db_id = generate_id(); let app_name = format!("postgresql-app-{}", generate_id()); - let database_host = get_svc_name(DatabaseKind::Postgresql, Kind::Scw).to_string(); + let database_host = get_svc_name(DatabaseKind::Postgresql, db_id.clone()).to_string(); let database_port = 5432; let database_db_name = "postgres".to_string(); let database_username = "superuser".to_string(); @@ -428,7 +421,7 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { environment.databases = vec![Database { kind: DatabaseKind::Postgresql, action: Action::Create, - id: generate_id(), + id: db_id.clone(), name: database_db_name.clone(), version: "11.8.0".to_string(), fqdn_id: "postgresql-".to_string() + generate_id().as_str(), @@ -542,20 +535,25 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { ** **/ #[allow(dead_code)] -fn test_postgresql_configuration( - context: Context, - environment: Environment, - secrets: FuncTestsSecrets, - version: &str, - test_name: &str, - database_mode: DatabaseMode, - is_public: bool, -) { +fn test_postgresql_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + engine_run_test(|| { test_db( context, logger(), - environment, secrets, version, test_name, @@ -572,224 +570,56 @@ fn test_postgresql_configuration( #[named] #[test] fn private_postgresql_v10_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "10", function_name!(), CONTAINER, false); + test_postgresql_configuration("10", function_name!(), CONTAINER, false); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn public_postgresql_v10_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "10", function_name!(), CONTAINER, true); + test_postgresql_configuration("10", function_name!(), CONTAINER, true); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn private_postgresql_v11_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "11", function_name!(), CONTAINER, false); + test_postgresql_configuration("11", function_name!(), CONTAINER, false); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn public_postgresql_v11_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "11", function_name!(), CONTAINER, true); + test_postgresql_configuration("11", function_name!(), CONTAINER, true); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn private_postgresql_v12_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "12", function_name!(), CONTAINER, false); + test_postgresql_configuration("12", function_name!(), CONTAINER, false); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn public_postgresql_v12_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "12", function_name!(), CONTAINER, true); + test_postgresql_configuration("12", function_name!(), CONTAINER, true); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn private_postgresql_v13_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "13", function_name!(), CONTAINER, false); + test_postgresql_configuration("13", function_name!(), CONTAINER, false); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn public_postgresql_v13_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "13", function_name!(), CONTAINER, true); + test_postgresql_configuration("13", function_name!(), CONTAINER, true); } // Postgres production environment @@ -798,28 +628,7 @@ fn public_postgresql_v13_deploy_a_working_dev_environment() { #[test] #[ignore] fn private_postgresql_v10_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "10", function_name!(), MANAGED, false); + test_postgresql_configuration("10", function_name!(), MANAGED, false); } #[cfg(feature = "test-scw-managed-services")] @@ -827,28 +636,7 @@ fn private_postgresql_v10_deploy_a_working_prod_environment() { #[test] #[ignore] fn public_postgresql_v10_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "10", function_name!(), MANAGED, true); + test_postgresql_configuration("10", function_name!(), MANAGED, true); } #[cfg(feature = "test-scw-managed-services")] @@ -856,28 +644,7 @@ fn public_postgresql_v10_deploy_a_working_prod_environment() { #[test] #[ignore] fn private_postgresql_v11_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "11", function_name!(), MANAGED, false); + test_postgresql_configuration("11", function_name!(), MANAGED, false); } #[cfg(feature = "test-scw-managed-services")] @@ -885,28 +652,7 @@ fn private_postgresql_v11_deploy_a_working_prod_environment() { #[test] #[ignore] fn public_postgresql_v11_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "11", function_name!(), MANAGED, true); + test_postgresql_configuration("11", function_name!(), MANAGED, true); } #[cfg(feature = "test-scw-managed-services")] @@ -914,28 +660,7 @@ fn public_postgresql_v11_deploy_a_working_prod_environment() { #[test] #[ignore] fn private_postgresql_v12_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "12", function_name!(), MANAGED, false); + test_postgresql_configuration("12", function_name!(), MANAGED, false); } #[cfg(feature = "test-scw-managed-services")] @@ -943,28 +668,7 @@ fn private_postgresql_v12_deploy_a_working_prod_environment() { #[test] #[ignore] fn public_postgresql_v12_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "12", function_name!(), MANAGED, true); + test_postgresql_configuration("12", function_name!(), MANAGED, true); } #[cfg(feature = "test-scw-managed-services")] @@ -972,28 +676,7 @@ fn public_postgresql_v12_deploy_a_working_prod_environment() { #[test] #[ignore] fn private_postgresql_v13_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "13", function_name!(), MANAGED, false); + test_postgresql_configuration("13", function_name!(), MANAGED, false); } #[cfg(feature = "test-scw-managed-services")] @@ -1001,28 +684,7 @@ fn private_postgresql_v13_deploy_a_working_prod_environment() { #[test] #[ignore] fn public_postgresql_v13_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_postgresql_configuration(context, environment, secrets, "13", function_name!(), MANAGED, true); + test_postgresql_configuration("13", function_name!(), MANAGED, true); } /** @@ -1031,20 +693,25 @@ fn public_postgresql_v13_deploy_a_working_prod_environment() { ** **/ #[allow(dead_code)] -fn test_mongodb_configuration( - context: Context, - environment: Environment, - secrets: FuncTestsSecrets, - version: &str, - test_name: &str, - database_mode: DatabaseMode, - is_public: bool, -) { +fn test_mongodb_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + engine_run_test(|| { test_db( context, logger(), - environment, secrets, version, test_name, @@ -1061,224 +728,56 @@ fn test_mongodb_configuration( #[named] #[test] fn private_mongodb_v3_6_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "3.6", function_name!(), CONTAINER, false); + test_mongodb_configuration("3.6", function_name!(), CONTAINER, false); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn public_mongodb_v3_6_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "3.6", function_name!(), CONTAINER, true); + test_mongodb_configuration("3.6", function_name!(), CONTAINER, true); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn private_mongodb_v4_0_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.0", function_name!(), CONTAINER, false); + test_mongodb_configuration("4.0", function_name!(), CONTAINER, false); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn public_mongodb_v4_0_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.0", function_name!(), CONTAINER, true); + test_mongodb_configuration("4.0", function_name!(), CONTAINER, true); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn private_mongodb_v4_2_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.2", function_name!(), CONTAINER, false); + test_mongodb_configuration("4.2", function_name!(), CONTAINER, false); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn public_mongodb_v4_2_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.2", function_name!(), CONTAINER, true); + test_mongodb_configuration("4.2", function_name!(), CONTAINER, true); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn private_mongodb_v4_4_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.4", function_name!(), CONTAINER, false); + test_mongodb_configuration("4.4", function_name!(), CONTAINER, false); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn public_mongodb_v4_4_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mongodb_configuration(context, environment, secrets, "4.4", function_name!(), CONTAINER, true); + test_mongodb_configuration("4.4", function_name!(), CONTAINER, true); } /** @@ -1287,20 +786,25 @@ fn public_mongodb_v4_4_deploy_a_working_dev_environment() { ** **/ #[allow(dead_code)] -fn test_mysql_configuration( - context: Context, - environment: Environment, - secrets: FuncTestsSecrets, - version: &str, - test_name: &str, - database_mode: DatabaseMode, - is_public: bool, -) { +fn test_mysql_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + engine_run_test(|| { test_db( context, logger(), - environment, secrets, version, test_name, @@ -1317,112 +821,28 @@ fn test_mysql_configuration( #[named] #[test] fn private_mysql_v5_7_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "5.7", function_name!(), CONTAINER, false); + test_mysql_configuration("5.7", function_name!(), CONTAINER, false); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn public_mysql_v5_7_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "5.7", function_name!(), CONTAINER, true); + test_mysql_configuration("5.7", function_name!(), CONTAINER, true); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn private_mysql_v8_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), CONTAINER, false); + test_mysql_configuration("8.0", function_name!(), CONTAINER, false); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn public_mysql_v8_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), CONTAINER, true); + test_mysql_configuration("8.0", function_name!(), CONTAINER, true); } // MySQL production environment (RDS) @@ -1431,28 +851,7 @@ fn public_mysql_v8_deploy_a_working_dev_environment() { #[test] #[ignore] fn private_mysql_v8_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), MANAGED, false); + test_mysql_configuration("8.0", function_name!(), MANAGED, false); } #[cfg(feature = "test-scw-managed-services")] @@ -1460,28 +859,7 @@ fn private_mysql_v8_deploy_a_working_prod_environment() { #[test] #[ignore] fn public_mysql_v8_deploy_a_working_prod_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), MANAGED, true); + test_mysql_configuration("8.0", function_name!(), MANAGED, true); } /** @@ -1490,20 +868,25 @@ fn public_mysql_v8_deploy_a_working_prod_environment() { ** **/ #[allow(dead_code)] -fn test_redis_configuration( - context: Context, - environment: Environment, - secrets: FuncTestsSecrets, - version: &str, - test_name: &str, - database_mode: DatabaseMode, - is_public: bool, -) { +fn test_redis_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + engine_run_test(|| { test_db( context, logger(), - environment, secrets, version, test_name, @@ -1520,110 +903,26 @@ fn test_redis_configuration( #[named] #[test] fn private_redis_v5_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_redis_configuration(context, environment, secrets, "5", function_name!(), CONTAINER, false); + test_redis_configuration("5", function_name!(), CONTAINER, false); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn public_redis_v5_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_redis_configuration(context, environment, secrets, "5", function_name!(), CONTAINER, true); + test_redis_configuration("5", function_name!(), CONTAINER, true); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn private_redis_v6_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_redis_configuration(context, environment, secrets, "6", function_name!(), CONTAINER, false); + test_redis_configuration("6", function_name!(), CONTAINER, false); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn public_redis_v6_deploy_a_working_dev_environment() { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let environment = working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .as_ref() - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - test_redis_configuration(context, environment, secrets, "6", function_name!(), CONTAINER, true); + test_redis_configuration("6", function_name!(), CONTAINER, true); } From f759e5de07c216a256c300ddf89a0518caf93761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=A3rebe=20-=20Romain=20GERARD?= Date: Wed, 2 Feb 2022 16:25:25 +0100 Subject: [PATCH 24/25] Revert "fix(ENG-1068): normalize db service (#572)" This reverts commit dbe6e8718996235139932edfe7557edddc780428. --- lib/aws/chart_values/mongodb/q-values.j2.yaml | 2 +- lib/aws/chart_values/mysql/q-values.j2.yaml | 1 - .../chart_values/postgresql/q-values.j2.yaml | 1 - lib/aws/chart_values/redis/q-values.j2.yaml | 1 - .../services/mysql/templates/_helpers.tpl | 12 - .../services/mysql/templates/master-svc.yaml | 2 +- lib/common/services/mysql/values.yaml | 1 - .../postgresql/templates/_helpers.tpl | 12 - .../services/postgresql/templates/svc.yaml | 2 +- lib/common/services/postgresql/values.yaml | 1 - .../services/redis/templates/_helpers.tpl | 12 - .../redis/templates/redis-master-svc.yaml | 2 +- lib/common/services/redis/values.yaml | 1 - .../chart_values/mongodb/q-values.j2.yaml | 2 +- .../chart_values/mysql/q-values.j2.yaml | 2 +- .../chart_values/postgresql/q-values.j2.yaml | 2 +- .../chart_values/redis/q-values.j2.yaml | 1 - lib/helm-freeze.yaml | 6 +- .../chart_values/mongodb/q-values.j2.yaml | 2 +- .../chart_values/mysql/q-values.j2.yaml | 2 +- .../chart_values/postgresql/q-values.j2.yaml | 2 +- .../chart_values/redis/q-values.j2.yaml | 1 - src/cloud_provider/aws/application.rs | 2 +- src/cloud_provider/aws/databases/mongodb.rs | 28 +- src/cloud_provider/aws/databases/mysql.rs | 24 +- .../aws/databases/postgresql.rs | 24 +- src/cloud_provider/aws/databases/redis.rs | 30 +- src/cloud_provider/aws/router.rs | 2 +- .../digitalocean/application.rs | 2 +- .../digitalocean/databases/mongodb.rs | 9 +- .../digitalocean/databases/mysql.rs | 9 +- .../digitalocean/databases/postgresql.rs | 9 +- .../digitalocean/databases/redis.rs | 9 +- src/cloud_provider/digitalocean/router.rs | 2 +- src/cloud_provider/scaleway/application.rs | 2 +- .../scaleway/databases/mongodb.rs | 9 +- .../scaleway/databases/mysql.rs | 9 +- .../scaleway/databases/postgresql.rs | 9 +- .../scaleway/databases/redis.rs | 9 +- src/cloud_provider/scaleway/router.rs | 2 +- src/cloud_provider/service.rs | 24 +- src/cloud_provider/utilities.rs | 20 +- test_utilities/src/common.rs | 188 +- test_utilities/src/utilities.rs | 28 +- tests/aws/aws_databases.rs | 1542 +++++++++++++---- tests/digitalocean/do_databases.rs | 839 +++++++-- tests/scaleway/scw_databases.rs | 1211 ++++++++++--- 47 files changed, 3100 insertions(+), 1012 deletions(-) diff --git a/lib/aws/chart_values/mongodb/q-values.j2.yaml b/lib/aws/chart_values/mongodb/q-values.j2.yaml index 338baa65..dc3bf279 100644 --- a/lib/aws/chart_values/mongodb/q-values.j2.yaml +++ b/lib/aws/chart_values/mongodb/q-values.j2.yaml @@ -135,7 +135,7 @@ clusterDomain: cluster.local ## Kubernetes service type service: ## Specify an explicit service name. - name: {{ service_name }} + # name: svc-mongo ## Provide any additional annotations which may be required. ## The value is evaluated as a template, so, for example, the value can depend on .Release or .Chart {% if publicly_accessible -%} diff --git a/lib/aws/chart_values/mysql/q-values.j2.yaml b/lib/aws/chart_values/mysql/q-values.j2.yaml index daa4475d..226c9814 100644 --- a/lib/aws/chart_values/mysql/q-values.j2.yaml +++ b/lib/aws/chart_values/mysql/q-values.j2.yaml @@ -501,7 +501,6 @@ service: ## MySQL Service type ## type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - name: {{ service_name }} ## MySQL Service port ## diff --git a/lib/aws/chart_values/postgresql/q-values.j2.yaml b/lib/aws/chart_values/postgresql/q-values.j2.yaml index ed6927de..c3553002 100644 --- a/lib/aws/chart_values/postgresql/q-values.j2.yaml +++ b/lib/aws/chart_values/postgresql/q-values.j2.yaml @@ -249,7 +249,6 @@ ldap: service: ## PosgresSQL service type type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - name: {{ service_name }} # clusterIP: None port: 5432 diff --git a/lib/aws/chart_values/redis/q-values.j2.yaml b/lib/aws/chart_values/redis/q-values.j2.yaml index eddb8c3f..72ae0ccb 100644 --- a/lib/aws/chart_values/redis/q-values.j2.yaml +++ b/lib/aws/chart_values/redis/q-values.j2.yaml @@ -369,7 +369,6 @@ master: service: ## Redis Master Service type type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - name: {{ service_name }} port: 6379 ## Specify the nodePort value for the LoadBalancer and NodePort service types. diff --git a/lib/common/services/mysql/templates/_helpers.tpl b/lib/common/services/mysql/templates/_helpers.tpl index 1e8fc206..bc5933ff 100644 --- a/lib/common/services/mysql/templates/_helpers.tpl +++ b/lib/common/services/mysql/templates/_helpers.tpl @@ -263,18 +263,6 @@ but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else {{- end -}} {{- end -}} -{{/* -Returns the proper Service name depending if an explicit service name is set -in the values file. If the name is not explicitly set it will take the "mongodb.fullname" -*/}} -{{- define "mysql.serviceName" -}} - {{- if .Values.service.name -}} - {{ .Values.service.name }} - {{- else -}} - {{ template "mysql.fullname" .}} - {{- end -}} -{{- end -}} - {{/* Renders a value that contains template. Usage: diff --git a/lib/common/services/mysql/templates/master-svc.yaml b/lib/common/services/mysql/templates/master-svc.yaml index d4827088..24142c32 100644 --- a/lib/common/services/mysql/templates/master-svc.yaml +++ b/lib/common/services/mysql/templates/master-svc.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ template "mysql.serviceName" . }} + name: {{ template "mysql.fullname" . }} labels: {{- include "mysql.labels" . | nindent 4 }} component: master {{- if or .Values.service.annotations .Values.metrics.service.annotations }} diff --git a/lib/common/services/mysql/values.yaml b/lib/common/services/mysql/values.yaml index 5b2efbda..97d05aaf 100644 --- a/lib/common/services/mysql/values.yaml +++ b/lib/common/services/mysql/values.yaml @@ -490,7 +490,6 @@ service: ## type: ClusterIP - # name: svc-name ## MySQL Service port ## port: 3306 diff --git a/lib/common/services/postgresql/templates/_helpers.tpl b/lib/common/services/postgresql/templates/_helpers.tpl index 91cb1580..e13caad6 100644 --- a/lib/common/services/postgresql/templates/_helpers.tpl +++ b/lib/common/services/postgresql/templates/_helpers.tpl @@ -378,18 +378,6 @@ but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else {{- end -}} {{- end -}} -{{/* -Returns the proper Service name depending if an explicit service name is set -in the values file. If the name is not explicitly set it will take the "mongodb.fullname" -*/}} -{{- define "postgresql.serviceName" -}} - {{- if .Values.service.name -}} - {{ .Values.service.name }} - {{- else -}} - {{ template "postgresql.fullname" .}} - {{- end -}} -{{- end -}} - {{/* Renders a value that contains template. Usage: diff --git a/lib/common/services/postgresql/templates/svc.yaml b/lib/common/services/postgresql/templates/svc.yaml index 4822fb25..d24b2a65 100644 --- a/lib/common/services/postgresql/templates/svc.yaml +++ b/lib/common/services/postgresql/templates/svc.yaml @@ -7,7 +7,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ template "postgresql.serviceName" . }} + name: {{ template "postgresql.fullname" . }} labels: app: {{ template "postgresql.name" . }} chart: {{ template "postgresql.chart" . }} diff --git a/lib/common/services/postgresql/values.yaml b/lib/common/services/postgresql/values.yaml index 4f949f93..e4651dae 100644 --- a/lib/common/services/postgresql/values.yaml +++ b/lib/common/services/postgresql/values.yaml @@ -249,7 +249,6 @@ ldap: service: ## PosgresSQL service type type: ClusterIP - # name: svc-name # clusterIP: None port: 5432 diff --git a/lib/common/services/redis/templates/_helpers.tpl b/lib/common/services/redis/templates/_helpers.tpl index 1d21226d..b5126537 100644 --- a/lib/common/services/redis/templates/_helpers.tpl +++ b/lib/common/services/redis/templates/_helpers.tpl @@ -384,18 +384,6 @@ but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else {{- end -}} {{- end -}} -{{/* -Returns the proper Service name depending if an explicit service name is set -in the values file. If the name is not explicitly set it will take the "mongodb.fullname" -*/}} -{{- define "redis.serviceName" -}} - {{- if .Values.master.service.name -}} - {{ .Values.master.service.name }} - {{- else -}} - {{ template "redis.fullname" .}} - {{- end -}} -{{- end -}} - {{/* Compile all warnings into a single message, and call fail. */}} diff --git a/lib/common/services/redis/templates/redis-master-svc.yaml b/lib/common/services/redis/templates/redis-master-svc.yaml index 2c2719e6..94fcce20 100644 --- a/lib/common/services/redis/templates/redis-master-svc.yaml +++ b/lib/common/services/redis/templates/redis-master-svc.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ template "redis.serviceName" . }} + name: {{ template "redis.fullname" . }}-master namespace: {{ .Release.Namespace | quote }} labels: app: {{ template "redis.name" . }} diff --git a/lib/common/services/redis/values.yaml b/lib/common/services/redis/values.yaml index 01b9507f..fb92acb3 100644 --- a/lib/common/services/redis/values.yaml +++ b/lib/common/services/redis/values.yaml @@ -362,7 +362,6 @@ master: service: ## Redis Master Service type type: ClusterIP - # name: svc-name port: 6379 ## Specify the nodePort value for the LoadBalancer and NodePort service types. diff --git a/lib/digitalocean/chart_values/mongodb/q-values.j2.yaml b/lib/digitalocean/chart_values/mongodb/q-values.j2.yaml index 0e2187d1..f1c032c7 100644 --- a/lib/digitalocean/chart_values/mongodb/q-values.j2.yaml +++ b/lib/digitalocean/chart_values/mongodb/q-values.j2.yaml @@ -149,7 +149,7 @@ service: {% endif %} type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - name: {{ service_name }} + # clusterIP: None port: {{ private_port }} ## Specify the nodePort value for the LoadBalancer and NodePort service types. diff --git a/lib/digitalocean/chart_values/mysql/q-values.j2.yaml b/lib/digitalocean/chart_values/mysql/q-values.j2.yaml index 28812907..d6247355 100644 --- a/lib/digitalocean/chart_values/mysql/q-values.j2.yaml +++ b/lib/digitalocean/chart_values/mysql/q-values.j2.yaml @@ -501,7 +501,7 @@ service: ## MySQL Service type ## type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - name: {{ service_name }} + ## MySQL Service port ## port: 3306 diff --git a/lib/digitalocean/chart_values/postgresql/q-values.j2.yaml b/lib/digitalocean/chart_values/postgresql/q-values.j2.yaml index 2bc0f002..93c967d3 100644 --- a/lib/digitalocean/chart_values/postgresql/q-values.j2.yaml +++ b/lib/digitalocean/chart_values/postgresql/q-values.j2.yaml @@ -249,7 +249,7 @@ ldap: service: ## PosgresSQL service type type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - name: {{ service_name }} + # clusterIP: None port: 5432 ## Specify the nodePort value for the LoadBalancer and NodePort service types. diff --git a/lib/digitalocean/chart_values/redis/q-values.j2.yaml b/lib/digitalocean/chart_values/redis/q-values.j2.yaml index 9d3170ca..70b91f04 100644 --- a/lib/digitalocean/chart_values/redis/q-values.j2.yaml +++ b/lib/digitalocean/chart_values/redis/q-values.j2.yaml @@ -369,7 +369,6 @@ master: service: ## Redis Master Service type type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - name: {{ service_name }} port: 6379 ## Specify the nodePort value for the LoadBalancer and NodePort service types. diff --git a/lib/helm-freeze.yaml b/lib/helm-freeze.yaml index e1d05484..b85bf1d3 100644 --- a/lib/helm-freeze.yaml +++ b/lib/helm-freeze.yaml @@ -52,24 +52,22 @@ charts: version: 7.14.5 dest: services no_sync: true - # Chart is frozen due to custom modification see https://github.com/Qovery/engine/pull/293 https://github.com/Qovery/engine/pull/572 + # Chart is frozen due to custom modification see https://github.com/Qovery/engine/pull/293 - name: mysql repo_name: bitnami version: 6.14.2 dest: services no_sync: true - # Chart is frozen due to custom modification see https://github.com/Qovery/engine/pull/572 - name: postgresql repo_name: bitnami version: 8.9.8 dest: services - # Chart is frozen due to custom modification see https://github.com/Qovery/engine/pull/293 https://github.com/Qovery/engine/pull/572 + # Chart is frozen due to custom modification see https://github.com/Qovery/engine/pull/293 - name: redis repo_name: bitnami version: 12.0.1 dest: services no_sync: true - # Chart is frozen due to custom modification see https://github.com/Qovery/engine/pull/572 - name: pleco version: 0.9.33 repo_name: pleco diff --git a/lib/scaleway/chart_values/mongodb/q-values.j2.yaml b/lib/scaleway/chart_values/mongodb/q-values.j2.yaml index f4e3ae55..95da012f 100644 --- a/lib/scaleway/chart_values/mongodb/q-values.j2.yaml +++ b/lib/scaleway/chart_values/mongodb/q-values.j2.yaml @@ -87,7 +87,7 @@ service: external-dns.alpha.kubernetes.io/ttl: "300" {% endif %} type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - name: {{ service_name }} + # clusterIP: None port: {{ private_port }} ## Specify the nodePort value for the LoadBalancer and NodePort service types. diff --git a/lib/scaleway/chart_values/mysql/q-values.j2.yaml b/lib/scaleway/chart_values/mysql/q-values.j2.yaml index fcb42272..c4bf58c0 100644 --- a/lib/scaleway/chart_values/mysql/q-values.j2.yaml +++ b/lib/scaleway/chart_values/mysql/q-values.j2.yaml @@ -479,7 +479,7 @@ service: ## MySQL Service type ## type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - name: {{ service_name }} + ## MySQL Service port ## port: 3306 diff --git a/lib/scaleway/chart_values/postgresql/q-values.j2.yaml b/lib/scaleway/chart_values/postgresql/q-values.j2.yaml index c4f2e191..96b5dafe 100644 --- a/lib/scaleway/chart_values/postgresql/q-values.j2.yaml +++ b/lib/scaleway/chart_values/postgresql/q-values.j2.yaml @@ -121,7 +121,7 @@ service: ## PosgresSQL service type type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - name: {{ service_name }} + # clusterIP: None port: 5432 ## Provide any additional annotations which may be required. Evaluated as a template. diff --git a/lib/scaleway/chart_values/redis/q-values.j2.yaml b/lib/scaleway/chart_values/redis/q-values.j2.yaml index 8b6d73c0..92c30306 100644 --- a/lib/scaleway/chart_values/redis/q-values.j2.yaml +++ b/lib/scaleway/chart_values/redis/q-values.j2.yaml @@ -326,7 +326,6 @@ master: service: ## Redis Master Service type type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} - name: {{ service_name }} port: 6379 ## Specify the nodePort value for the LoadBalancer and NodePort service types. diff --git a/src/cloud_provider/aws/application.rs b/src/cloud_provider/aws/application.rs index c8d1a0e6..6f4aeec7 100644 --- a/src/cloud_provider/aws/application.rs +++ b/src/cloud_provider/aws/application.rs @@ -145,7 +145,7 @@ impl Service for Application { } fn sanitized_name(&self) -> String { - sanitize_name("app", self.name(), false) + sanitize_name("app", self.name()) } fn version(&self) -> String { diff --git a/src/cloud_provider/aws/databases/mongodb.rs b/src/cloud_provider/aws/databases/mongodb.rs index 48d6d187..ad2d5d16 100644 --- a/src/cloud_provider/aws/databases/mongodb.rs +++ b/src/cloud_provider/aws/databases/mongodb.rs @@ -10,7 +10,6 @@ use crate::cloud_provider::service::{ }; use crate::cloud_provider::utilities::{ generate_supported_version, get_self_hosted_mongodb_version, get_supported_version_to_use, print_action, - sanitize_name, }; use crate::cloud_provider::DeploymentTarget; use crate::cmd::helm::Timeout; @@ -104,7 +103,15 @@ impl Service for MongoDB { } fn sanitized_name(&self) -> String { - sanitize_name("mongodb", self.name(), self.is_managed_service()) + // https://docs.aws.amazon.com/documentdb/latest/developerguide/limits.html#limits-naming_constraints + let prefix = "mongodb"; + let max_size = 60 - prefix.len(); // 63 (max DocumentDB) - 3 (k8s statefulset chars) + let mut new_name = format!("{}{}", prefix, self.name().replace("_", "").replace("-", "")); + if new_name.chars().count() > max_size { + new_name = new_name[..max_size].to_string(); + } + + new_name } fn version(&self) -> String { @@ -180,9 +187,10 @@ impl Service for MongoDB { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); - context.insert("sanitized_name", self.sanitized_name().as_str()); - context.insert("service_name", self.service_name().as_str()); + context.insert( + "fqdn", + self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), + ); context.insert("database_db_name", self.name.as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); @@ -421,8 +429,8 @@ mod tests_mongodb { #[test] fn mongo_name_sanitizer() { - let db_name = "a-name"; - let db_expected_name = "mongodbaname"; + let db_input_name = "test-name_sanitizer-with-too-many-chars-not-allowed-which_will-be-shrinked-at-the-end"; + let db_expected_name = "mongodbtestnamesanitizerwithtoomanycharsnotallowedwhi"; let database = MongoDB::new( Context::new( @@ -436,9 +444,9 @@ mod tests_mongodb { vec![], None, ), - "1234", + "pgid", Action::Create, - db_name.clone(), + db_input_name, "8", "mongotest.qovery.io", "pgid", @@ -450,7 +458,7 @@ mod tests_mongodb { password: "".to_string(), host: "".to_string(), port: 5432, - mode: DatabaseMode::MANAGED, + mode: DatabaseMode::CONTAINER, disk_size_in_gib: 10, database_disk_type: "gp2".to_string(), encrypt_disk: false, diff --git a/src/cloud_provider/aws/databases/mysql.rs b/src/cloud_provider/aws/databases/mysql.rs index f9c23719..e5c451f2 100644 --- a/src/cloud_provider/aws/databases/mysql.rs +++ b/src/cloud_provider/aws/databases/mysql.rs @@ -9,8 +9,8 @@ use crate::cloud_provider::service::{ DatabaseType, Delete, Helm, Pause, Service, ServiceType, StatefulService, Terraform, }; use crate::cloud_provider::utilities::{ - generate_supported_version, get_self_hosted_mysql_version, get_supported_version_to_use, print_action, - sanitize_name, + generate_supported_version, get_self_hosted_mysql_version, get_supported_version_to_use, managed_db_name_sanitizer, + print_action, }; use crate::cloud_provider::DeploymentTarget; use crate::cmd::helm::Timeout; @@ -114,7 +114,10 @@ impl Service for MySQL { } fn sanitized_name(&self) -> String { - sanitize_name("mysql", self.name(), self.is_managed_service()) + // https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Limits.html#RDS_Limits.Constraints + let prefix = "mysql"; + let max_size = 63 - 3; // max RDS - k8s statefulset chars + managed_db_name_sanitizer(max_size, prefix, self.name()) } fn version(&self) -> String { @@ -205,9 +208,10 @@ impl Service for MySQL { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); - context.insert("sanitized_name", self.sanitized_name().as_str()); - context.insert("service_name", self.service_name().as_str()); + context.insert( + "fqdn", + self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), + ); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); @@ -449,8 +453,8 @@ mod tests_mysql { #[test] fn mysql_name_sanitizer() { - let db_name = "a-name"; - let db_expected_name = "mysqlaname"; + let db_input_name = "test-name_sanitizer-with-too-many-chars-not-allowed-which_will-be-shrinked-at-the-end"; + let db_expected_name = "mysqltestnamesanitizerwithtoomanycharsnotallowedwhichwi"; let database = MySQL::new( Context::new( @@ -464,9 +468,9 @@ mod tests_mysql { vec![], None, ), - "1234", + "mysqlid", Action::Create, - db_name.clone(), + db_input_name, "8", "mysqltest.qovery.io", "mysqlid", diff --git a/src/cloud_provider/aws/databases/postgresql.rs b/src/cloud_provider/aws/databases/postgresql.rs index 3c21cc8f..ee66bcb2 100644 --- a/src/cloud_provider/aws/databases/postgresql.rs +++ b/src/cloud_provider/aws/databases/postgresql.rs @@ -9,8 +9,8 @@ use crate::cloud_provider::service::{ DatabaseType, Delete, Helm, Pause, Service, ServiceType, StatefulService, Terraform, }; use crate::cloud_provider::utilities::{ - generate_supported_version, get_self_hosted_postgres_version, get_supported_version_to_use, print_action, - sanitize_name, + generate_supported_version, get_self_hosted_postgres_version, get_supported_version_to_use, + managed_db_name_sanitizer, print_action, }; use crate::cloud_provider::DeploymentTarget; use crate::cmd::helm::Timeout; @@ -114,7 +114,10 @@ impl Service for PostgreSQL { } fn sanitized_name(&self) -> String { - sanitize_name("postgresql", self.name(), self.is_managed_service()) + // https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Limits.html#RDS_Limits.Constraints + let prefix = "postgresql"; + let max_size = 63 - 3; // max RDS - k8s statefulset chars + managed_db_name_sanitizer(max_size, prefix, self.name()) } fn version(&self) -> String { @@ -190,9 +193,10 @@ impl Service for PostgreSQL { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); - context.insert("sanitized_name", self.sanitized_name().as_str()); - context.insert("service_name", self.service_name().as_str()); + context.insert( + "fqdn", + self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), + ); context.insert("database_name", self.sanitized_name().as_str()); context.insert("database_db_name", self.name()); context.insert("database_login", self.options.login.as_str()); @@ -443,8 +447,8 @@ mod tests_postgres { #[test] fn postgres_name_sanitizer() { - let db_name = "a-name"; - let db_expected_name = "postgresqlaname"; + let db_input_name = "test-name_sanitizer-with-too-many-chars-not-allowed-which_will-be-shrinked-at-the-end"; + let db_expected_name = "postgresqltestnamesanitizerwithtoomanycharsnotallo"; let database = PostgreSQL::new( Context::new( @@ -458,9 +462,9 @@ mod tests_postgres { vec![], None, ), - "1234", + "pgid", Action::Create, - db_name.clone(), + db_input_name, "8", "pgtest.qovery.io", "pgid", diff --git a/src/cloud_provider/aws/databases/redis.rs b/src/cloud_provider/aws/databases/redis.rs index ddc1c1fd..b45d45c9 100644 --- a/src/cloud_provider/aws/databases/redis.rs +++ b/src/cloud_provider/aws/databases/redis.rs @@ -8,9 +8,7 @@ use crate::cloud_provider::service::{ get_tfstate_suffix, scale_down_database, send_progress_on_long_task, Action, Create, Database, DatabaseOptions, DatabaseType, Delete, Helm, Pause, Service, ServiceType, StatefulService, Terraform, }; -use crate::cloud_provider::utilities::{ - get_self_hosted_redis_version, get_supported_version_to_use, print_action, sanitize_name, -}; +use crate::cloud_provider::utilities::{get_self_hosted_redis_version, get_supported_version_to_use, print_action}; use crate::cloud_provider::DeploymentTarget; use crate::cmd::helm::Timeout; use crate::cmd::kubectl; @@ -113,7 +111,16 @@ impl Service for Redis { } fn sanitized_name(&self) -> String { - sanitize_name("redis", self.name(), self.is_managed_service()) + // https://aws.amazon.com/about-aws/whats-new/2019/08/elasticache_supports_50_chars_cluster_name + let prefix = "redis"; + let max_size = 47 - prefix.len(); // 50 (max Elasticache ) - 3 (k8s statefulset chars) + let mut new_name = self.name().replace("_", "").replace("-", ""); + + if new_name.chars().count() > max_size { + new_name = new_name[..max_size].to_string(); + } + + format!("{}{}", prefix, new_name) } fn version(&self) -> String { @@ -202,9 +209,10 @@ impl Service for Redis { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); - context.insert("sanitized_name", self.sanitized_name().as_str()); - context.insert("service_name", self.service_name().as_str()); + context.insert( + "fqdn", + self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), + ); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); @@ -428,8 +436,8 @@ mod tests { #[test] fn redis_name_sanitizer() { - let db_name = "a-name"; - let db_expected_name = "redisaname"; + let db_input_name = "test-name_sanitizer-with-too-many-chars-not-allowed-which_will-be-shrinked-at-the-end"; + let db_expected_name = "redistestnamesanitizerwithtoomanycharsnotallowe"; let database = Redis::new( Context::new( @@ -443,9 +451,9 @@ mod tests { vec![], None, ), - "1234", + "pgid", Action::Create, - db_name.clone(), + db_input_name, "8", "redistest.qovery.io", "redisid", diff --git a/src/cloud_provider/aws/router.rs b/src/cloud_provider/aws/router.rs index 440c744c..f9792144 100644 --- a/src/cloud_provider/aws/router.rs +++ b/src/cloud_provider/aws/router.rs @@ -77,7 +77,7 @@ impl Service for Router { } fn sanitized_name(&self) -> String { - sanitize_name("router", self.name(), false) + sanitize_name("router", self.name()) } fn version(&self) -> String { diff --git a/src/cloud_provider/digitalocean/application.rs b/src/cloud_provider/digitalocean/application.rs index 16775c8f..48381113 100644 --- a/src/cloud_provider/digitalocean/application.rs +++ b/src/cloud_provider/digitalocean/application.rs @@ -147,7 +147,7 @@ impl Service for Application { } fn sanitized_name(&self) -> String { - sanitize_name("app", self.name(), false) + sanitize_name("app", self.name()) } fn version(&self) -> String { diff --git a/src/cloud_provider/digitalocean/databases/mongodb.rs b/src/cloud_provider/digitalocean/databases/mongodb.rs index db9cfce0..84357d13 100644 --- a/src/cloud_provider/digitalocean/databases/mongodb.rs +++ b/src/cloud_provider/digitalocean/databases/mongodb.rs @@ -108,7 +108,7 @@ impl Service for MongoDB { } fn sanitized_name(&self) -> String { - sanitize_name("mongodb", self.name(), self.is_managed_service()) + sanitize_name("mongodb", self.name()) } fn version(&self) -> String { @@ -184,9 +184,10 @@ impl Service for MongoDB { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); - context.insert("sanitized_name", self.sanitized_name().as_str()); - context.insert("service_name", self.service_name().as_str()); + context.insert( + "fqdn", + self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), + ); context.insert("database_db_name", self.name.as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); diff --git a/src/cloud_provider/digitalocean/databases/mysql.rs b/src/cloud_provider/digitalocean/databases/mysql.rs index c2d84fe3..00bba0f2 100644 --- a/src/cloud_provider/digitalocean/databases/mysql.rs +++ b/src/cloud_provider/digitalocean/databases/mysql.rs @@ -108,7 +108,7 @@ impl Service for MySQL { } fn sanitized_name(&self) -> String { - sanitize_name("mysql", self.name(), self.is_managed_service()) + sanitize_name("mysql", self.name()) } fn version(&self) -> String { @@ -184,9 +184,10 @@ impl Service for MySQL { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); - context.insert("sanitized_name", self.sanitized_name().as_str()); - context.insert("service_name", self.service_name().as_str()); + context.insert( + "fqdn", + self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), + ); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); diff --git a/src/cloud_provider/digitalocean/databases/postgresql.rs b/src/cloud_provider/digitalocean/databases/postgresql.rs index d2895d01..b7efc14a 100644 --- a/src/cloud_provider/digitalocean/databases/postgresql.rs +++ b/src/cloud_provider/digitalocean/databases/postgresql.rs @@ -108,7 +108,7 @@ impl Service for PostgreSQL { } fn sanitized_name(&self) -> String { - sanitize_name("postgresql", self.name(), self.is_managed_service()) + sanitize_name("postgresql", self.name()) } fn version(&self) -> String { @@ -184,9 +184,10 @@ impl Service for PostgreSQL { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); - context.insert("sanitized_name", self.sanitized_name().as_str()); - context.insert("service_name", self.service_name().as_str()); + context.insert( + "fqdn", + self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), + ); context.insert("database_db_name", self.name()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); diff --git a/src/cloud_provider/digitalocean/databases/redis.rs b/src/cloud_provider/digitalocean/databases/redis.rs index 580ab10e..527cc38e 100644 --- a/src/cloud_provider/digitalocean/databases/redis.rs +++ b/src/cloud_provider/digitalocean/databases/redis.rs @@ -108,7 +108,7 @@ impl Service for Redis { } fn sanitized_name(&self) -> String { - sanitize_name("redis", self.name(), self.is_managed_service()) + sanitize_name("redis", self.name()) } fn version(&self) -> String { @@ -184,9 +184,10 @@ impl Service for Redis { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); - context.insert("sanitized_name", self.sanitized_name().as_str()); - context.insert("service_name", self.service_name().as_str()); + context.insert( + "fqdn", + self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), + ); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); diff --git a/src/cloud_provider/digitalocean/router.rs b/src/cloud_provider/digitalocean/router.rs index 22312520..173dea32 100644 --- a/src/cloud_provider/digitalocean/router.rs +++ b/src/cloud_provider/digitalocean/router.rs @@ -77,7 +77,7 @@ impl Service for Router { } fn sanitized_name(&self) -> String { - sanitize_name("router", self.name(), false) + sanitize_name("router", self.name()) } fn version(&self) -> String { diff --git a/src/cloud_provider/scaleway/application.rs b/src/cloud_provider/scaleway/application.rs index 83fc1568..ff148e26 100644 --- a/src/cloud_provider/scaleway/application.rs +++ b/src/cloud_provider/scaleway/application.rs @@ -148,7 +148,7 @@ impl Service for Application { } fn sanitized_name(&self) -> String { - sanitize_name("app", self.name(), false) + sanitize_name("app", self.name()) } fn version(&self) -> String { diff --git a/src/cloud_provider/scaleway/databases/mongodb.rs b/src/cloud_provider/scaleway/databases/mongodb.rs index c19a6c9b..d3303924 100644 --- a/src/cloud_provider/scaleway/databases/mongodb.rs +++ b/src/cloud_provider/scaleway/databases/mongodb.rs @@ -108,7 +108,7 @@ impl Service for MongoDB { } fn sanitized_name(&self) -> String { - sanitize_name("mongodb", self.name(), self.is_managed_service()) + sanitize_name("mongodb", self.name()) } fn version(&self) -> String { @@ -185,9 +185,10 @@ impl Service for MongoDB { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); - context.insert("sanitized_name", self.sanitized_name().as_str()); - context.insert("service_name", self.service_name().as_str()); + context.insert( + "fqdn", + self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), + ); context.insert("database_db_name", self.name.as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); diff --git a/src/cloud_provider/scaleway/databases/mysql.rs b/src/cloud_provider/scaleway/databases/mysql.rs index 9e44e96d..c7bec2c8 100644 --- a/src/cloud_provider/scaleway/databases/mysql.rs +++ b/src/cloud_provider/scaleway/databases/mysql.rs @@ -139,7 +139,7 @@ impl Service for MySQL { } fn sanitized_name(&self) -> String { - sanitize_name("mysql", self.name(), self.is_managed_service()) + sanitize_name("mysql", self.name()) } fn version(&self) -> String { @@ -217,9 +217,10 @@ impl Service for MySQL { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); - context.insert("sanitized_name", self.sanitized_name().as_str()); - context.insert("service_name", self.service_name().as_str()); + context.insert( + "fqdn", + self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), + ); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); diff --git a/src/cloud_provider/scaleway/databases/postgresql.rs b/src/cloud_provider/scaleway/databases/postgresql.rs index b7172310..f39f1610 100644 --- a/src/cloud_provider/scaleway/databases/postgresql.rs +++ b/src/cloud_provider/scaleway/databases/postgresql.rs @@ -148,7 +148,7 @@ impl Service for PostgreSQL { } fn sanitized_name(&self) -> String { - sanitize_name("postgresql", self.name(), self.is_managed_service()) + sanitize_name("postgresql", self.name()) } fn version(&self) -> String { @@ -226,9 +226,10 @@ impl Service for PostgreSQL { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); - context.insert("sanitized_name", self.sanitized_name().as_str()); - context.insert("service_name", self.service_name().as_str()); + context.insert( + "fqdn", + self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), + ); context.insert("database_name", self.sanitized_name().as_str()); context.insert("database_db_name", self.name()); context.insert("database_login", self.options.login.as_str()); diff --git a/src/cloud_provider/scaleway/databases/redis.rs b/src/cloud_provider/scaleway/databases/redis.rs index 47e03fa9..4ca2f11a 100644 --- a/src/cloud_provider/scaleway/databases/redis.rs +++ b/src/cloud_provider/scaleway/databases/redis.rs @@ -108,7 +108,7 @@ impl Service for Redis { } fn sanitized_name(&self) -> String { - sanitize_name("redis", self.name(), self.is_managed_service()) + sanitize_name("redis", self.name()) } fn version(&self) -> String { @@ -185,9 +185,10 @@ impl Service for Redis { context.insert("kubernetes_cluster_name", kubernetes.name()); context.insert("fqdn_id", self.fqdn_id.as_str()); - context.insert("fqdn", self.fqdn(&self.fqdn, self.is_managed_service()).as_str()); - context.insert("sanitized_name", self.sanitized_name().as_str()); - context.insert("service_name", self.service_name().as_str()); + context.insert( + "fqdn", + self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), + ); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); diff --git a/src/cloud_provider/scaleway/router.rs b/src/cloud_provider/scaleway/router.rs index 79c598e6..2ca3651b 100644 --- a/src/cloud_provider/scaleway/router.rs +++ b/src/cloud_provider/scaleway/router.rs @@ -77,7 +77,7 @@ impl Service for Router { } fn sanitized_name(&self) -> String { - sanitize_name("router", self.name(), false) + sanitize_name("router", self.name()) } fn version(&self) -> String { diff --git a/src/cloud_provider/service.rs b/src/cloud_provider/service.rs index 197512b4..a1bc908c 100644 --- a/src/cloud_provider/service.rs +++ b/src/cloud_provider/service.rs @@ -9,7 +9,7 @@ use tera::Context as TeraContext; use crate::build_platform::Image; use crate::cloud_provider::environment::Environment; use crate::cloud_provider::kubernetes::Kubernetes; -use crate::cloud_provider::utilities::{check_domain_for, sanitize_db_name}; +use crate::cloud_provider::utilities::check_domain_for; use crate::cloud_provider::DeploymentTarget; use crate::cmd::helm::Timeout; use crate::cmd::kubectl::ScalingKind::Statefulset; @@ -30,18 +30,6 @@ pub trait Service: ToTransmitter { fn id(&self) -> &str; fn name(&self) -> &str; fn sanitized_name(&self) -> String; - fn service_name(&self) -> String { - match self.service_type() { - ServiceType::Application => self.sanitized_name(), - ServiceType::Database(db_kind) => match db_kind { - DatabaseType::PostgreSQL(_) => sanitize_db_name("postgresql", self.id()), - DatabaseType::MongoDB(_) => sanitize_db_name("mongodb", self.id()), - DatabaseType::MySQL(_) => sanitize_db_name("mysql", self.id()), - DatabaseType::Redis(_) => sanitize_db_name("redis", self.id()), - }, - ServiceType::Router => self.sanitized_name(), - } - } fn name_with_id(&self) -> String { format!("{} ({})", self.name(), self.id()) } @@ -81,12 +69,16 @@ pub trait Service: ToTransmitter { fn min_instances(&self) -> u32; fn max_instances(&self) -> u32; fn publicly_accessible(&self) -> bool; - fn fqdn<'a>(&self, fqdn: &'a String, is_managed: bool) -> String { + fn fqdn<'a>(&self, target: &DeploymentTarget, fqdn: &'a String, is_managed: bool) -> String { match &self.publicly_accessible() { true => fqdn.to_string(), false => match is_managed { - true => format!("{}-dns", self.id()), - false => format!("{}", self.sanitized_name()), + true => format!("{}-dns.{}.svc.cluster.local", self.id(), target.environment.namespace()), + false => format!( + "{}.{}.svc.cluster.local", + self.sanitized_name(), + target.environment.namespace() + ), }, } } diff --git a/src/cloud_provider/utilities.rs b/src/cloud_provider/utilities.rs index 8aa3a43e..1407b3c3 100644 --- a/src/cloud_provider/utilities.rs +++ b/src/cloud_provider/utilities.rs @@ -495,21 +495,17 @@ pub fn check_domain_for( Ok(()) } -pub fn sanitize_db_name(suffix: &str, db_id: &str) -> String { - format!("{}-{}", db_id, suffix).replace("_", "-") +pub fn sanitize_name(prefix: &str, name: &str) -> String { + format!("{}-{}", prefix, name).replace("_", "-") } -pub fn sanitize_name(prefix: &str, app_name: &str, is_managed: bool) -> String { - match is_managed { - true => { - let mut sanitized_name = format!("{}-{}", prefix, app_name).replace("_", "").replace("-", ""); - if sanitized_name.chars().count() > 63 { - sanitized_name = sanitized_name[..63].to_string(); - }; - sanitized_name - } - false => format!("{}-{}", prefix, app_name).replace("_", "-"), +pub fn managed_db_name_sanitizer(max_size: usize, prefix: &str, name: &str) -> String { + let max_size = max_size - prefix.len(); + let mut new_name = format!("{}{}", prefix, name.replace("_", "").replace("-", "")); + if new_name.chars().count() > max_size { + new_name = new_name[..max_size].to_string(); } + new_name } pub fn convert_k8s_cpu_value_to_f32(value: String) -> Result { diff --git a/test_utilities/src/common.rs b/test_utilities/src/common.rs index 9341a45d..2ace4cc2 100644 --- a/test_utilities/src/common.rs +++ b/test_utilities/src/common.rs @@ -192,14 +192,14 @@ pub fn environment_3_apps_3_routers_3_databases( test_domain: &str, database_instance_type: &str, database_disk_type: &str, + provider_kind: Kind, ) -> Environment { let app_name_1 = format!("{}-{}", "simple-app-1".to_string(), generate_id()); let app_name_2 = format!("{}-{}", "simple-app-2".to_string(), generate_id()); let app_name_3 = format!("{}-{}", "simple-app-3".to_string(), generate_id()); // mongoDB management part - let mongo_id = generate_id(); - let database_host_mongo = get_svc_name(DatabaseKind::Mongodb, mongo_id.clone()).to_string(); + let database_host_mongo = get_svc_name(DatabaseKind::Mongodb, provider_kind.clone()).to_string(); let database_port_mongo = 27017; let database_db_name_mongo = "my-mongodb".to_string(); let database_username_mongo = "superuser".to_string(); @@ -215,18 +215,16 @@ pub fn environment_3_apps_3_routers_3_databases( let version_mongo = "4.4"; // pSQL 1 management part - let psql_id_1 = generate_id(); let fqdn_id = "my-postgresql-".to_string() + generate_id().as_str(); - let fqdn = get_svc_name(DatabaseKind::Postgresql, psql_id_1.clone()).to_string(); + let fqdn = get_svc_name(DatabaseKind::Postgresql, provider_kind.clone()).to_string(); let database_port = 5432; let database_username = "superuser".to_string(); let database_password = generate_password(true); let database_name = "postgres".to_string(); // pSQL 2 management part - let psql_id_2 = generate_id(); let fqdn_id_2 = "my-postgresql-2".to_string() + generate_id().as_str(); - let fqdn_2 = get_svc_name(DatabaseKind::Postgresql, psql_id_2.clone()); + let fqdn_2 = format!("{}2", get_svc_name(DatabaseKind::Postgresql, provider_kind.clone())); let database_username_2 = "superuser2".to_string(); let database_name_2 = "postgres2".to_string(); @@ -424,7 +422,7 @@ pub fn environment_3_apps_3_routers_3_databases( Database { kind: DatabaseKind::Postgresql, action: Action::Create, - id: psql_id_1.clone(), + id: generate_id(), name: database_name.clone(), version: "11.8.0".to_string(), fqdn_id: fqdn_id.clone(), @@ -446,7 +444,7 @@ pub fn environment_3_apps_3_routers_3_databases( Database { kind: DatabaseKind::Postgresql, action: Action::Create, - id: psql_id_2.clone(), + id: generate_id(), name: database_name_2.clone(), version: "11.8.0".to_string(), fqdn_id: fqdn_id_2.clone(), @@ -468,7 +466,7 @@ pub fn environment_3_apps_3_routers_3_databases( Database { kind: DatabaseKind::Mongodb, action: Action::Create, - id: mongo_id.clone(), + id: generate_id(), name: database_db_name_mongo.clone(), version: version_mongo.to_string(), fqdn_id: "mongodb-".to_string() + generate_id().as_str(), @@ -513,7 +511,7 @@ pub fn working_minimal_environment(context: &Context, test_domain: &str) -> Envi action: Action::Create, applications: vec![Application { id: application_id, - name: application_name.clone(), + name: application_name, git_url: "https://github.com/Qovery/engine-testing.git".to_string(), commit_id: "fc575a2f3be0b9100492c8a463bf18134a8698a5".to_string(), dockerfile_path: Some("Dockerfile".to_string()), @@ -553,7 +551,7 @@ pub fn working_minimal_environment(context: &Context, test_domain: &str) -> Envi custom_domains: vec![], routes: vec![Route { path: "/".to_string(), - application_name, + application_name: format!("{}-{}", "simple-app".to_string(), &suffix), }], sticky_sessions_enabled: false, }], @@ -574,13 +572,14 @@ pub fn environment_only_http_server_router_with_sticky_session(context: &Context pub fn environnement_2_app_2_routers_1_psql( context: &Context, + test_domain: &str, database_instance_type: &str, database_disk_type: &str, + provider_kind: Kind, ) -> Environment { - let db_id = generate_id(); let fqdn_id = "my-postgresql-".to_string() + generate_id().as_str(); - let fqdn = get_svc_name(DatabaseKind::Postgresql, db_id.clone()).to_string(); + let fqdn = get_svc_name(DatabaseKind::Postgresql, provider_kind.clone()).to_string(); let database_port = 5432; let database_username = "superuser".to_string(); @@ -588,8 +587,8 @@ pub fn environnement_2_app_2_routers_1_psql( let database_name = "postgres".to_string(); let suffix = generate_id(); - let application_name1 = sanitize_name("postgresql", &format!("{}-{}", "postgresql-app1", &suffix), false); - let application_name2 = sanitize_name("postgresql", &format!("{}-{}", "postgresql-app2", &suffix), false); + let application_name1 = sanitize_name("postgresql", &format!("{}-{}", "postgresql-app1", &suffix)); + let application_name2 = sanitize_name("postgresql", &format!("{}-{}", "postgresql-app2", &suffix)); Environment { execution_id: context.execution_id().to_string(), @@ -601,7 +600,7 @@ pub fn environnement_2_app_2_routers_1_psql( databases: vec![Database { kind: DatabaseKind::Postgresql, action: Action::Create, - id: db_id.clone(), + id: generate_id(), name: database_name.clone(), version: "11.8.0".to_string(), fqdn_id: fqdn_id.clone(), @@ -974,6 +973,7 @@ pub fn routers_sessions_are_sticky(routers: Vec) -> bool { pub fn test_db( context: Context, logger: Box, + mut environment: Environment, secrets: FuncTestsSecrets, version: &str, test_name: &str, @@ -988,11 +988,11 @@ pub fn test_db( let _enter = span.enter(); let context_for_delete = context.clone_not_same_execution_id(); - let db_id = generate_id(); + let app_id = generate_id(); let database_username = "superuser".to_string(); let database_password = generate_id(); let db_kind_str = db_kind.name().to_string(); - let fqdn = format!( + let database_host = format!( "{}-{}.{}.{}", db_kind_str.clone(), generate_id(), @@ -1003,10 +1003,10 @@ pub fn test_db( .expect("DEFAULT_TEST_DOMAIN is not set in secrets") ); let dyn_db_fqdn = match is_public.clone() { - true => fqdn.clone(), + true => database_host.clone(), false => match database_mode.clone() { - DatabaseMode::MANAGED => format!("{}-dns", db_id.clone()), - DatabaseMode::CONTAINER => get_svc_name(db_kind.clone(), db_id.clone()).to_string(), + DatabaseMode::MANAGED => format!("{}-dns", app_id.clone()), + DatabaseMode::CONTAINER => get_svc_name(db_kind.clone(), provider_kind.clone()).to_string(), }, }; @@ -1022,72 +1022,53 @@ pub fn test_db( let storage_size = 10; let db_disk_type = db_disk_type(provider_kind.clone(), database_mode.clone()); let db_instance_type = db_instance_type(provider_kind.clone(), db_kind.clone(), database_mode.clone()); - - let environment = Environment { - execution_id: context.execution_id().to_string(), - id: generate_id(), - owner_id: generate_id(), - project_id: generate_id(), - organization_id: context.organization_id().to_string(), + let db = Database { + kind: db_kind.clone(), action: Action::Create, - applications: vec![Application { - id: generate_id(), - name: format!("{}-app-{}", db_kind_str.clone(), generate_id()), - git_url: "https://github.com/Qovery/engine-testing.git".to_string(), - commit_id: db_infos.app_commit.clone(), - dockerfile_path: Some(format!("Dockerfile-{}", version)), - buildpack_language: None, - root_path: String::from("/"), - action: Action::Create, - git_credentials: Some(GitCredentials { - login: "x-access-token".to_string(), - access_token: "xxx".to_string(), - expired_at: Utc::now(), - }), - storage: vec![], - environment_vars: db_infos.app_env_vars.clone(), - branch: "basic-app-deploy".to_string(), - ports: vec![Port { + id: app_id.clone(), + name: database_db_name.clone(), + version: version.to_string(), + fqdn_id: format!("{}-{}", db_kind_str.clone(), generate_id()), + fqdn: database_host.clone(), + port: database_port.clone(), + username: database_username.clone(), + password: database_password.clone(), + total_cpus: "100m".to_string(), + total_ram_in_mib: 512, + disk_size_in_gib: storage_size.clone(), + database_instance_type: db_instance_type.to_string(), + database_disk_type: db_disk_type.to_string(), + encrypt_disk: true, + activate_high_availability: false, + activate_backups: false, + publicly_accessible: is_public.clone(), + mode: database_mode.clone(), + }; + + environment.databases = vec![db.clone()]; + + let app_name = format!("{}-app-{}", db_kind_str.clone(), generate_id()); + environment.applications = environment + .applications + .into_iter() + .map(|mut app| { + app.branch = app_name.clone(); + app.commit_id = db_infos.app_commit.clone(); + app.ports = vec![Port { id: "zdf7d6aad".to_string(), long_id: Default::default(), - port: 80, - public_port: Some(443), + port: 1234, + public_port: Some(1234), name: None, - publicly_accessible: false, + publicly_accessible: true, protocol: Protocol::HTTP, - }], - total_cpus: "100m".to_string(), - total_ram_in_mib: 256, - min_instances: 2, - max_instances: 2, - cpu_burst: "100m".to_string(), - start_timeout_in_seconds: 60, - }], - routers: vec![], - databases: vec![Database { - kind: db_kind.clone(), - action: Action::Create, - id: db_id.clone(), - name: database_db_name.clone(), - version: version.to_string(), - fqdn_id: format!("{}-{}", db_kind_str.clone(), generate_id()), - fqdn: dyn_db_fqdn.clone(), - port: database_port.clone(), - username: database_username.clone(), - password: database_password.clone(), - total_cpus: "100m".to_string(), - total_ram_in_mib: 512, - disk_size_in_gib: storage_size.clone(), - database_instance_type: db_instance_type.to_string(), - database_disk_type: db_disk_type.to_string(), - encrypt_disk: true, - activate_high_availability: false, - activate_backups: false, - publicly_accessible: is_public.clone(), - mode: database_mode.clone(), - }], - clone_from_environment_id: None, - }; + }]; + app.dockerfile_path = Some(format!("Dockerfile-{}", version)); + app.environment_vars = db_infos.app_env_vars.clone(); + app + }) + .collect::>(); + environment.routers[0].routes[0].application_name = app_name.clone(); let mut environment_delete = environment.clone(); environment_delete.action = Action::Delete; @@ -1121,28 +1102,22 @@ pub fn test_db( environment.clone(), secrets.clone(), ) { - Ok(svc) => match is_public { - true => assert_eq!( - svc.items - .expect("No items in svc") - .into_iter() - .filter(|svc| svc.metadata.name == get_svc_name(db_kind.clone(), db_id.clone()) - && &svc.spec.svc_type == "LoadBalancer") - .collect::>() - .len(), - 1 - ), - false => assert_eq!( - svc.items - .expect("No items in svc") - .into_iter() - .filter(|svc| svc.metadata.name == get_svc_name(db_kind.clone(), db_id.clone()) - && &svc.spec.svc_type == "ClusterIP") - .collect::>() - .len(), - 1 - ), - }, + Ok(svc) => assert_eq!( + svc.items + .expect("No items in svc") + .into_iter() + .filter(|svc| svc + .metadata + .name + .contains(get_svc_name(db_kind.clone(), provider_kind.clone())) + && &svc.spec.svc_type == "LoadBalancer") + .collect::>() + .len(), + match is_public { + true => 1, + false => 0, + } + ), Err(_) => assert!(false), }; } @@ -1154,7 +1129,8 @@ pub fn test_db( .expect("No items in svc") .into_iter() .filter(|svc| { - svc.metadata.name == format!("{}-dns", db_id.clone()) && svc.spec.svc_type == "ExternalName" + svc.metadata.name.contains(format!("{}-dns", app_id.clone()).as_str()) + && svc.spec.svc_type == "ExternalName" }) .collect::>(); let annotations = &service[0].metadata.annotations; @@ -1162,7 +1138,7 @@ pub fn test_db( match is_public { true => { assert!(annotations.contains_key("external-dns.alpha.kubernetes.io/hostname")); - assert_eq!(annotations["external-dns.alpha.kubernetes.io/hostname"], fqdn); + assert_eq!(annotations["external-dns.alpha.kubernetes.io/hostname"], database_host); } false => assert!(!annotations.contains_key("external-dns.alpha.kubernetes.io/hostname")), } diff --git a/test_utilities/src/utilities.rs b/test_utilities/src/utilities.rs index 9eb1c1f5..e967e8e7 100644 --- a/test_utilities/src/utilities.rs +++ b/test_utilities/src/utilities.rs @@ -987,7 +987,7 @@ pub fn db_infos( match db_kind { DatabaseKind::Mongodb => { let database_port = 27017; - let database_db_name = "mymongodb".to_string(); + let database_db_name = "my-mongodb".to_string(); let database_uri = format!( "mongodb://{}:{}@{}:{}/{}", database_username, @@ -1029,7 +1029,7 @@ pub fn db_infos( } DatabaseKind::Postgresql => { let database_port = 5432; - let database_db_name = "mypostgres".to_string(); + let database_db_name = "postgres".to_string(); DBInfos { db_port: database_port.clone(), db_name: database_db_name.to_string(), @@ -1045,7 +1045,7 @@ pub fn db_infos( } DatabaseKind::Redis => { let database_port = 6379; - let database_db_name = "myredis".to_string(); + let database_db_name = "my-redis".to_string(); DBInfos { db_port: database_port.clone(), db_name: database_db_name.to_string(), @@ -1097,11 +1097,23 @@ pub fn db_instance_type(provider_kind: Kind, db_kind: DatabaseKind, database_mod .to_string() } -pub fn get_svc_name(db_kind: DatabaseKind, db_id: String) -> String { +pub fn get_svc_name(db_kind: DatabaseKind, provider_kind: Kind) -> &'static str { match db_kind { - DatabaseKind::Postgresql => format!("{}-postgresql", db_id), - DatabaseKind::Mysql => format!("{}-mysql", db_id), - DatabaseKind::Mongodb => format!("{}-mongodb", db_id), - DatabaseKind::Redis => format!("{}-redis", db_id), + DatabaseKind::Postgresql => match provider_kind { + Kind::Aws => "postgresqlpostgres", + _ => "postgresql-postgres", + }, + DatabaseKind::Mysql => match provider_kind { + Kind::Aws => "mysqlmysqldatabase", + _ => "mysql-mysqldatabase", + }, + DatabaseKind::Mongodb => match provider_kind { + Kind::Aws => "mongodbmymongodb", + _ => "mongodb-my-mongodb", + }, + DatabaseKind::Redis => match provider_kind { + Kind::Aws => "redismyredis-master", + _ => "redis-my-redis-master", + }, } } diff --git a/tests/aws/aws_databases.rs b/tests/aws/aws_databases.rs index 774678d1..a20438c7 100644 --- a/tests/aws/aws_databases.rs +++ b/tests/aws/aws_databases.rs @@ -2,7 +2,9 @@ extern crate test_utilities; use ::function_name::named; use qovery_engine::cloud_provider::Kind; -use qovery_engine::models::{Action, Clone2, Database, DatabaseKind, DatabaseMode, EnvironmentAction, Port, Protocol}; +use qovery_engine::models::{ + Action, Clone2, Context, Database, DatabaseKind, DatabaseMode, Environment, EnvironmentAction, Port, Protocol, +}; use qovery_engine::transaction::TransactionResult; use tracing::{span, Level}; @@ -54,6 +56,7 @@ fn deploy_an_environment_with_3_databases_and_3_apps() { .as_str(), AWS_DATABASE_INSTANCE_TYPE, AWS_DATABASE_DISK_TYPE, + Kind::Aws, ); let mut environment_delete = environment.clone(); @@ -112,6 +115,7 @@ fn deploy_an_environment_with_db_and_pause_it() { .as_str(), AWS_DATABASE_INSTANCE_TYPE, AWS_DATABASE_DISK_TYPE, + Kind::Aws, ); let mut environment_delete = environment.clone(); @@ -132,7 +136,7 @@ fn deploy_an_environment_with_db_and_pause_it() { }; // Check that we have actually 0 pods running for this db - let app_name = format!("postgresql-{}-0", environment.databases[0].name); + let app_name = format!("postgresql{}-0", environment.databases[0].name); let ret = get_pods( context.clone(), Kind::Aws, @@ -190,6 +194,7 @@ fn postgresql_failover_dev_environment_with_all_options() { test_domain.as_str(), AWS_DATABASE_INSTANCE_TYPE, AWS_DATABASE_DISK_TYPE, + Kind::Aws, ); let environment_check = environment.clone(); let mut environment_never_up = environment.clone(); @@ -215,6 +220,7 @@ fn postgresql_failover_dev_environment_with_all_options() { test_domain.as_str(), AWS_DATABASE_INSTANCE_TYPE, AWS_DATABASE_DISK_TYPE, + Kind::Aws, ); environment_delete.action = Action::Delete; @@ -229,7 +235,7 @@ fn postgresql_failover_dev_environment_with_all_options() { TransactionResult::UnrecoverableError(_, _) => assert!(false), }; // TO CHECK: DATABASE SHOULDN'T BE RESTARTED AFTER A REDEPLOY - let database_name = format!("postgresql-{}-0", &environment_check.databases[0].name); + let database_name = format!("postgresql{}-0", &environment_check.databases[0].name); match is_pod_restarted_env( context.clone(), Kind::Aws, @@ -304,6 +310,7 @@ fn postgresql_deploy_a_working_development_environment_with_all_options() { test_domain.as_str(), AWS_DATABASE_INSTANCE_TYPE, AWS_DATABASE_DISK_TYPE, + Kind::Aws, ); //let env_to_check = environment.clone(); let mut environment_delete = test_utilities::common::environnement_2_app_2_routers_1_psql( @@ -311,6 +318,7 @@ fn postgresql_deploy_a_working_development_environment_with_all_options() { test_domain.as_str(), AWS_DATABASE_INSTANCE_TYPE, AWS_DATABASE_DISK_TYPE, + Kind::Aws, ); environment_delete.action = Action::Delete; @@ -379,8 +387,7 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { ); let app_name = format!("postgresql-app-{}", generate_id()); - let db_id = generate_id(); - let database_host = get_svc_name(DatabaseKind::Postgresql, db_id.clone()).to_string(); + let database_host = get_svc_name(DatabaseKind::Postgresql, Kind::Aws).to_string(); let database_port = 5432; let database_db_name = "postgres".to_string(); let database_username = "superuser".to_string(); @@ -388,7 +395,7 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { environment.databases = vec![Database { kind: DatabaseKind::Postgresql, action: Action::Create, - id: db_id.clone(), + id: generate_id(), name: database_db_name.clone(), version: "11.8.0".to_string(), fqdn_id: "postgresql-".to_string() + generate_id().as_str(), @@ -411,8 +418,7 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { .applications .into_iter() .map(|mut app| { - app.id = app_name.clone(); - app.name = app_name.clone(); + app.branch = app_name.clone(); app.commit_id = "5990752647af11ef21c3d46a51abbde3da1ab351".to_string(); app.ports = vec![Port { id: "zdf7d6aad".to_string(), @@ -456,7 +462,7 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { TransactionResult::UnrecoverableError(_, _) => assert!(false), }; // TO CHECK: DATABASE SHOULDN'T BE RESTARTED AFTER A REDEPLOY - let database_name = format!("postgresql-{}-0", &environment_check.databases[0].name); + let database_name = format!("postgresql{}-0", &environment_check.databases[0].name); match is_pod_restarted_env( context.clone(), Kind::Aws, @@ -484,25 +490,20 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { ** **/ #[allow(dead_code)] -fn test_postgresql_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - +fn test_postgresql_configuration( + context: Context, + environment: Environment, + secrets: FuncTestsSecrets, + version: &str, + test_name: &str, + database_mode: DatabaseMode, + is_public: bool, +) { engine_run_test(|| { test_db( context, logger(), + environment, secrets, version, test_name, @@ -519,122 +520,6 @@ fn test_postgresql_configuration(version: &str, test_name: &str, database_mode: #[named] #[test] fn private_postgresql_v10_deploy_a_working_dev_environment() { - test_postgresql_configuration("10", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn public_postgresql_v10_deploy_a_working_dev_environment() { - test_postgresql_configuration("10", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn private_postgresql_v11_deploy_a_working_dev_environment() { - test_postgresql_configuration("11", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn public_postgresql_v11_deploy_a_working_dev_environment() { - test_postgresql_configuration("11", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn private_postgresql_v12_deploy_a_working_dev_environment() { - test_postgresql_configuration("12", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn public_postgresql_v12_deploy_a_working_dev_environment() { - test_postgresql_configuration("12", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn private_postgresql_v13_deploy_a_working_dev_environment() { - test_postgresql_configuration("13", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn public_postgresql_v13_deploy_a_working_dev_environment() { - test_postgresql_configuration("13", function_name!(), CONTAINER, true); -} - -// Postgres production environment -#[cfg(feature = "test-aws-managed-services")] -#[named] -#[test] -fn private_postgresql_v10_deploy_a_working_prod_environment() { - test_postgresql_configuration("10", function_name!(), MANAGED, false); -} - -#[cfg(feature = "test-aws-managed-services")] -#[named] -#[test] -fn public_postgresql_v10_deploy_a_working_prod_environment() { - test_postgresql_configuration("10", function_name!(), MANAGED, true); -} - -#[cfg(feature = "test-aws-managed-services")] -#[named] -#[test] -fn private_postgresql_v11_deploy_a_working_prod_environment() { - test_postgresql_configuration("11", function_name!(), MANAGED, false); -} - -#[cfg(feature = "test-aws-managed-services")] -#[named] -#[test] -fn public_postgresql_v11_deploy_a_working_prod_environment() { - test_postgresql_configuration("11", function_name!(), MANAGED, true); -} - -#[cfg(feature = "test-aws-managed-services")] -#[named] -#[test] -fn private_postgresql_v12_deploy_a_working_prod_environment() { - test_postgresql_configuration("12", function_name!(), MANAGED, false); -} - -#[cfg(feature = "test-aws-managed-services")] -#[named] -#[test] -fn public_postgresql_v12_deploy_a_working_prod_environment() { - test_postgresql_configuration("12", function_name!(), MANAGED, true); -} - -#[cfg(feature = "test-aws-managed-services")] -#[named] -#[test] -fn private_postgresql_v13_deploy_a_working_prod_environment() { - test_postgresql_configuration("13", function_name!(), MANAGED, false); -} - -#[cfg(feature = "test-aws-managed-services")] -#[named] -#[test] -fn public_postgresql_v13_deploy_a_working_prod_environment() { - test_postgresql_configuration("13", function_name!(), MANAGED, true); -} - -/** -** -** MongoDB tests -** -**/ -#[allow(dead_code)] -fn test_mongodb_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { let secrets = FuncTestsSecrets::new(); let context = context( secrets @@ -648,11 +533,470 @@ fn test_mongodb_configuration(version: &str, test_name: &str, database_mode: Dat .expect("AWS_TEST_CLUSTER_ID is not set") .as_str(), ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "10", function_name!(), CONTAINER, false); +} +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn public_postgresql_v10_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "10", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn private_postgresql_v11_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "11", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn public_postgresql_v11_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "11", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn private_postgresql_v12_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "12", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn public_postgresql_v12_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "12", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn private_postgresql_v13_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "13", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn public_postgresql_v13_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "13", function_name!(), CONTAINER, true); +} + +// Postgres production environment +#[cfg(feature = "test-aws-managed-services")] +#[named] +#[test] +fn private_postgresql_v10_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "10", function_name!(), MANAGED, false); +} + +#[cfg(feature = "test-aws-managed-services")] +#[named] +#[test] +fn public_postgresql_v10_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "10", function_name!(), MANAGED, true); +} + +#[cfg(feature = "test-aws-managed-services")] +#[named] +#[test] +fn private_postgresql_v11_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "11", function_name!(), MANAGED, false); +} + +#[cfg(feature = "test-aws-managed-services")] +#[named] +#[test] +fn public_postgresql_v11_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "11", function_name!(), MANAGED, true); +} + +#[cfg(feature = "test-aws-managed-services")] +#[named] +#[test] +fn private_postgresql_v12_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "12", function_name!(), MANAGED, false); +} + +#[cfg(feature = "test-aws-managed-services")] +#[named] +#[test] +fn public_postgresql_v12_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "12", function_name!(), MANAGED, true); +} + +#[cfg(feature = "test-aws-managed-services")] +#[named] +#[test] +fn private_postgresql_v13_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "13", function_name!(), MANAGED, false); +} + +#[cfg(feature = "test-aws-managed-services")] +#[named] +#[test] +fn public_postgresql_v13_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "13", function_name!(), MANAGED, true); +} + +/** +** +** MongoDB tests +** +**/ +#[allow(dead_code)] +fn test_mongodb_configuration( + context: Context, + environment: Environment, + secrets: FuncTestsSecrets, + version: &str, + test_name: &str, + database_mode: DatabaseMode, + is_public: bool, +) { engine_run_test(|| { test_db( context, logger(), + environment, secrets, version, test_name, @@ -669,94 +1013,6 @@ fn test_mongodb_configuration(version: &str, test_name: &str, database_mode: Dat #[named] #[test] fn private_mongodb_v3_6_deploy_a_working_dev_environment() { - test_mongodb_configuration("3.6", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn public_mongodb_v3_6_deploy_a_working_dev_environment() { - test_mongodb_configuration("3.6", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn private_mongodb_v4_0_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.0", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn public_mongodb_v4_0_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.0", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn private_mongodb_v4_2_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.2", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn public_mongodb_v4_2_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.2", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn private_mongodb_v4_4_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.4", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn public_mongodb_v4_4_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.4", function_name!(), CONTAINER, true); -} - -// MongoDB production environment (DocumentDB) -#[cfg(feature = "test-aws-managed-services")] -#[named] -#[test] -fn private_mongodb_v3_6_deploy_a_working_prod_environment() { - test_mongodb_configuration("3.6", function_name!(), MANAGED, false); -} - -#[cfg(feature = "test-aws-managed-services")] -#[named] -#[test] -fn public_mongodb_v3_6_deploy_a_working_prod_environment() { - test_mongodb_configuration("3.6", function_name!(), MANAGED, true); -} - -#[cfg(feature = "test-aws-managed-services")] -#[named] -#[test] -fn private_mongodb_v4_0_deploy_a_working_prod_environment() { - test_mongodb_configuration("4.0", function_name!(), MANAGED, false); -} - -#[cfg(feature = "test-aws-managed-services")] -#[named] -#[test] -fn public_mongodb_v4_0_deploy_a_working_prod_environment() { - test_mongodb_configuration("4.0", function_name!(), MANAGED, true); -} - -/** -** -** MySQL tests -** -**/ -#[allow(dead_code)] -fn test_mysql_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { let secrets = FuncTestsSecrets::new(); let context = context( secrets @@ -770,11 +1026,358 @@ fn test_mysql_configuration(version: &str, test_name: &str, database_mode: Datab .expect("AWS_TEST_CLUSTER_ID is not set") .as_str(), ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "3.6", function_name!(), CONTAINER, false); +} +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn public_mongodb_v3_6_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "3.6", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn private_mongodb_v4_0_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.0", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn public_mongodb_v4_0_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.0", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn private_mongodb_v4_2_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.2", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn public_mongodb_v4_2_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.2", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn private_mongodb_v4_4_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.4", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn public_mongodb_v4_4_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.4", function_name!(), CONTAINER, true); +} + +// MongoDB production environment (DocumentDB) +#[cfg(feature = "test-aws-managed-services")] +#[named] +#[test] +fn private_mongodb_v3_6_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "3.6", function_name!(), MANAGED, false); +} + +#[cfg(feature = "test-aws-managed-services")] +#[named] +#[test] +fn public_mongodb_v3_6_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "3.6", function_name!(), MANAGED, true); +} + +#[cfg(feature = "test-aws-managed-services")] +#[named] +#[test] +fn private_mongodb_v4_0_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.0", function_name!(), MANAGED, false); +} + +#[cfg(feature = "test-aws-managed-services")] +#[named] +#[test] +fn public_mongodb_v4_0_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.0", function_name!(), MANAGED, true); +} + +/** +** +** MySQL tests +** +**/ +#[allow(dead_code)] +fn test_mysql_configuration( + context: Context, + environment: Environment, + secrets: FuncTestsSecrets, + version: &str, + test_name: &str, + database_mode: DatabaseMode, + is_public: bool, +) { engine_run_test(|| { test_db( context, logger(), + environment, secrets, version, test_name, @@ -791,66 +1394,6 @@ fn test_mysql_configuration(version: &str, test_name: &str, database_mode: Datab #[named] #[test] fn private_mysql_v5_7_deploy_a_working_dev_environment() { - test_mysql_configuration("5.7", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn public_mysql_v5_7_deploy_a_working_dev_environment() { - test_mysql_configuration("5.7", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn private_mysql_v8_deploy_a_working_dev_environment() { - test_mysql_configuration("8.0", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn public_mysql_v8_deploy_a_working_dev_environment() { - test_mysql_configuration("8.0", function_name!(), CONTAINER, true); -} - -// MySQL production environment (RDS) -#[cfg(feature = "test-aws-managed-services")] -#[named] -#[test] -fn private_mysql_v5_7_deploy_a_working_prod_environment() { - test_mysql_configuration("5.7", function_name!(), MANAGED, false); -} - -#[cfg(feature = "test-aws-managed-services")] -#[named] -#[test] -fn public_mysql_v5_7_deploy_a_working_prod_environment() { - test_mysql_configuration("5.7", function_name!(), MANAGED, true); -} - -#[cfg(feature = "test-aws-managed-services")] -#[named] -#[test] -fn private_mysql_v8_0_deploy_a_working_prod_environment() { - test_mysql_configuration("8.0", function_name!(), MANAGED, false); -} - -#[cfg(feature = "test-aws-managed-services")] -#[named] -#[test] -fn public_mysql_v8_0_deploy_a_working_prod_environment() { - test_mysql_configuration("8.0", function_name!(), MANAGED, true); -} - -/** -** -** Redis tests -** -**/ -#[allow(dead_code)] -fn test_redis_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { let secrets = FuncTestsSecrets::new(); let context = context( secrets @@ -864,11 +1407,242 @@ fn test_redis_configuration(version: &str, test_name: &str, database_mode: Datab .expect("AWS_TEST_CLUSTER_ID is not set") .as_str(), ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "5.7", function_name!(), CONTAINER, false); +} +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn public_mysql_v5_7_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "5.7", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn private_mysql_v8_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-aws-self-hosted")] +#[named] +#[test] +fn public_mysql_v8_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), CONTAINER, true); +} + +// MySQL production environment (RDS) +#[cfg(feature = "test-aws-managed-services")] +#[named] +#[test] +fn private_mysql_v5_7_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "5.7", function_name!(), MANAGED, false); +} + +#[cfg(feature = "test-aws-managed-services")] +#[named] +#[test] +fn public_mysql_v5_7_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "5.7", function_name!(), MANAGED, true); +} + +#[cfg(feature = "test-aws-managed-services")] +#[named] +#[test] +fn private_mysql_v8_0_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), MANAGED, false); +} + +#[cfg(feature = "test-aws-managed-services")] +#[named] +#[test] +fn public_mysql_v8_0_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), MANAGED, true); +} + +/** +** +** Redis tests +** +**/ +#[allow(dead_code)] +fn test_redis_configuration( + context: Context, + environment: Environment, + secrets: FuncTestsSecrets, + version: &str, + test_name: &str, + database_mode: DatabaseMode, + is_public: bool, +) { engine_run_test(|| { test_db( context, logger(), + environment, secrets, version, test_name, @@ -885,28 +1659,116 @@ fn test_redis_configuration(version: &str, test_name: &str, database_mode: Datab #[named] #[test] fn private_redis_v5_deploy_a_working_dev_environment() { - test_redis_configuration("5", function_name!(), CONTAINER, false); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_redis_configuration(context, environment, secrets, "5", function_name!(), CONTAINER, false); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn public_redis_v5_deploy_a_working_dev_environment() { - test_redis_configuration("5", function_name!(), CONTAINER, true); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_redis_configuration(context, environment, secrets, "5", function_name!(), CONTAINER, true); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn private_redis_v6_deploy_a_working_dev_environment() { - test_redis_configuration("6", function_name!(), CONTAINER, false); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_redis_configuration(context, environment, secrets, "6", function_name!(), CONTAINER, false); } #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] fn public_redis_v6_deploy_a_working_dev_environment() { - test_redis_configuration("6", function_name!(), CONTAINER, true); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_redis_configuration(context, environment, secrets, "6", function_name!(), CONTAINER, true); } // Redis production environment (Elasticache) @@ -914,26 +1776,114 @@ fn public_redis_v6_deploy_a_working_dev_environment() { #[named] #[test] fn private_redis_v5_deploy_a_working_prod_environment() { - test_redis_configuration("5", function_name!(), MANAGED, false); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_redis_configuration(context, environment, secrets, "5", function_name!(), MANAGED, false); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn public_redis_v5_deploy_a_working_prod_environment() { - test_redis_configuration("5", function_name!(), MANAGED, true); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_redis_configuration(context, environment, secrets, "5", function_name!(), MANAGED, true); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn private_redis_v6_deploy_a_working_prod_environment() { - test_redis_configuration("6", function_name!(), MANAGED, false); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_redis_configuration(context, environment, secrets, "6", function_name!(), MANAGED, false); } #[cfg(feature = "test-aws-managed-services")] #[named] #[test] fn public_redis_v6_deploy_a_working_prod_environment() { - test_redis_configuration("6", function_name!(), MANAGED, true); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .AWS_TEST_ORGANIZATION_ID + .as_ref() + .expect("AWS_TEST_ORGANIZATION_ID is not set") + .as_str(), + secrets + .AWS_TEST_CLUSTER_ID + .as_ref() + .expect("AWS_TEST_CLUSTER_ID is not set") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .clone() + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_redis_configuration(context, environment, secrets, "6", function_name!(), MANAGED, true); } diff --git a/tests/digitalocean/do_databases.rs b/tests/digitalocean/do_databases.rs index b781976e..3c96542b 100644 --- a/tests/digitalocean/do_databases.rs +++ b/tests/digitalocean/do_databases.rs @@ -2,14 +2,16 @@ use ::function_name::named; use tracing::{span, warn, Level}; use qovery_engine::cloud_provider::{Kind as ProviderKind, Kind}; -use qovery_engine::models::{Action, Clone2, Database, DatabaseKind, DatabaseMode, EnvironmentAction, Port, Protocol}; +use qovery_engine::models::{ + Action, Clone2, Context, Database, DatabaseKind, DatabaseMode, Environment, EnvironmentAction, Port, Protocol, +}; use qovery_engine::transaction::TransactionResult; use test_utilities::utilities::{ context, engine_run_test, generate_id, get_pods, get_svc_name, init, is_pod_restarted_env, logger, FuncTestsSecrets, }; use qovery_engine::models::DatabaseMode::{CONTAINER, MANAGED}; -use test_utilities::common::{test_db, Infrastructure}; +use test_utilities::common::{test_db, working_minimal_environment, Infrastructure}; use test_utilities::digitalocean::{ clean_environments, DO_MANAGED_DATABASE_DISK_TYPE, DO_MANAGED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, DO_TEST_REGION, @@ -55,6 +57,7 @@ fn deploy_an_environment_with_3_databases_and_3_apps() { .as_str(), DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, + Kind::Do, ); let mut environment_delete = environment.clone(); @@ -116,6 +119,7 @@ fn deploy_an_environment_with_db_and_pause_it() { .as_str(), DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, + Kind::Do, ); let mut environment_delete = environment.clone(); @@ -197,6 +201,7 @@ fn postgresql_failover_dev_environment_with_all_options() { test_domain.as_str(), DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, + Kind::Do, ); let environment_check = environment.clone(); let mut environment_never_up = environment.clone(); @@ -222,6 +227,7 @@ fn postgresql_failover_dev_environment_with_all_options() { test_domain.as_str(), DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, + Kind::Do, ); environment_delete.action = Action::Delete; @@ -315,6 +321,7 @@ fn postgresql_deploy_a_working_development_environment_with_all_options() { test_domain.as_str(), DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, + Kind::Do, ); //let env_to_check = environment.clone(); let mut environment_delete = test_utilities::common::environnement_2_app_2_routers_1_psql( @@ -322,6 +329,7 @@ fn postgresql_deploy_a_working_development_environment_with_all_options() { test_domain.as_str(), DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, + Kind::Do, ); environment_delete.action = Action::Delete; @@ -398,9 +406,8 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { ); let database_mode = CONTAINER; - let db_id = generate_id(); let app_name = format!("postgresql-app-{}", generate_id()); - let database_host = get_svc_name(DatabaseKind::Postgresql, db_id.clone()).to_string(); + let database_host = get_svc_name(DatabaseKind::Postgresql, Kind::Do).to_string(); let database_port = 5432; let database_db_name = "postgres".to_string(); let database_username = "superuser".to_string(); @@ -408,7 +415,7 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { environment.databases = vec![Database { kind: DatabaseKind::Postgresql, action: Action::Create, - id: db_id.clone(), + id: generate_id(), name: database_db_name.clone(), version: "11.8.0".to_string(), fqdn_id: "postgresql-".to_string() + generate_id().as_str(), @@ -522,23 +529,20 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { ** **/ #[allow(dead_code)] -fn test_postgresql_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - +fn test_postgresql_configuration( + context: Context, + environment: Environment, + secrets: FuncTestsSecrets, + version: &str, + test_name: &str, + database_mode: DatabaseMode, + is_public: bool, +) { engine_run_test(|| { test_db( context, logger(), + environment, secrets, version, test_name, @@ -555,68 +559,6 @@ fn test_postgresql_configuration(version: &str, test_name: &str, database_mode: #[named] #[test] fn private_postgresql_v10_deploy_a_working_dev_environment() { - test_postgresql_configuration("10", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-do-self-hosted")] -#[ignore] -#[named] -#[test] -fn public_postgresql_v10_deploy_a_working_dev_environment() { - test_postgresql_configuration("10", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-do-self-hosted")] -#[ignore] -#[named] -#[test] -fn private_postgresql_v11_deploy_a_working_dev_environment() { - test_postgresql_configuration("11", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-do-self-hosted")] -#[ignore] -#[named] -#[test] -fn public_postgresql_v11_deploy_a_working_dev_environment() { - test_postgresql_configuration("11", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-do-self-hosted")] -#[named] -#[test] -fn private_postgresql_v12_deploy_a_working_dev_environment() { - test_postgresql_configuration("12", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-do-self-hosted")] -#[named] -#[test] -fn public_postgresql_v12_deploy_a_working_dev_environment() { - test_postgresql_configuration("12", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-do-self-hosted")] -#[named] -#[test] -fn private_postgresql_v13_deploy_a_working_dev_environment() { - test_postgresql_configuration("13", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-do-self-hosted")] -#[named] -#[test] -fn public_postgresql_v13_deploy_a_working_dev_environment() { - test_postgresql_configuration("13", function_name!(), CONTAINER, true); -} - -/** - ** - ** MongoDB tests - ** - **/ -#[allow(dead_code)] -fn test_mongodb_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { let secrets = FuncTestsSecrets::new(); let context = context( secrets @@ -629,10 +571,229 @@ fn test_mongodb_configuration(version: &str, test_name: &str, database_mode: Dat .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "10", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-do-self-hosted")] +#[ignore] +#[named] +#[test] +fn public_postgresql_v10_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "10", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-do-self-hosted")] +#[ignore] +#[named] +#[test] +fn private_postgresql_v11_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "11", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-do-self-hosted")] +#[ignore] +#[named] +#[test] +fn public_postgresql_v11_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "11", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-do-self-hosted")] +#[named] +#[test] +fn private_postgresql_v12_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "12", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-do-self-hosted")] +#[named] +#[test] +fn public_postgresql_v12_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "12", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-do-self-hosted")] +#[named] +#[test] +fn private_postgresql_v13_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "13", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-do-self-hosted")] +#[named] +#[test] +fn public_postgresql_v13_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "13", function_name!(), CONTAINER, true); +} + +/** + ** + ** MongoDB tests + ** + **/ +#[allow(dead_code)] +fn test_mongodb_configuration( + context: Context, + environment: Environment, + secrets: FuncTestsSecrets, + version: &str, + test_name: &str, + database_mode: DatabaseMode, + is_public: bool, +) { engine_run_test(|| { test_db( context, logger(), + environment, secrets, version, test_name, @@ -649,69 +810,6 @@ fn test_mongodb_configuration(version: &str, test_name: &str, database_mode: Dat #[named] #[test] fn private_mongodb_v3_6_deploy_a_working_dev_environment() { - test_mongodb_configuration("3.6", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-do-self-hosted")] -#[ignore] -#[named] -#[test] -fn public_mongodb_v3_6_deploy_a_working_dev_environment() { - test_mongodb_configuration("3.6", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-do-self-hosted")] -#[ignore] -#[named] -#[test] -fn private_mongodb_v4_0_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.0", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-do-self-hosted")] -#[ignore] -#[named] -#[test] -fn public_mongodb_v4_0_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.0", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-do-self-hosted")] -#[ignore] -#[named] -#[test] -fn private_mongodb_v4_2_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.2", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-do-self-hosted")] -#[named] -#[test] -fn public_mongodb_v4_2_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.2", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-do-self-hosted")] -#[named] -#[test] -fn private_mongodb_v4_4_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.4", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-do-self-hosted")] -#[named] -#[test] -fn public_mongodb_v4_4_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.4", function_name!(), CONTAINER, true); -} - -/** - ** - ** MySQL tests - ** - **/ -#[allow(dead_code)] -fn test_mysql_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { let secrets = FuncTestsSecrets::new(); let context = context( secrets @@ -724,10 +822,230 @@ fn test_mysql_configuration(version: &str, test_name: &str, database_mode: Datab .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "3.6", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-do-self-hosted")] +#[ignore] +#[named] +#[test] +fn public_mongodb_v3_6_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "3.6", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-do-self-hosted")] +#[ignore] +#[named] +#[test] +fn private_mongodb_v4_0_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.0", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-do-self-hosted")] +#[ignore] +#[named] +#[test] +fn public_mongodb_v4_0_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.0", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-do-self-hosted")] +#[ignore] +#[named] +#[test] +fn private_mongodb_v4_2_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.2", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-do-self-hosted")] +#[named] +#[test] +fn public_mongodb_v4_2_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.2", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-do-self-hosted")] +#[named] +#[test] +fn private_mongodb_v4_4_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.4", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-do-self-hosted")] +#[named] +#[test] +fn public_mongodb_v4_4_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.4", function_name!(), CONTAINER, true); +} + +/** + ** + ** MySQL tests + ** + **/ +#[allow(dead_code)] +fn test_mysql_configuration( + context: Context, + environment: Environment, + secrets: FuncTestsSecrets, + version: &str, + test_name: &str, + database_mode: DatabaseMode, + is_public: bool, +) { engine_run_test(|| { test_db( context, logger(), + environment, secrets, version, test_name, @@ -744,41 +1062,6 @@ fn test_mysql_configuration(version: &str, test_name: &str, database_mode: Datab #[named] #[test] fn private_mysql_v5_7_deploy_a_working_dev_environment() { - test_mysql_configuration("5.7", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-do-self-hosted")] -#[ignore] -#[named] -#[test] -fn public_mysql_v5_7_deploy_a_working_dev_environment() { - test_mysql_configuration("5.7", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-do-self-hosted")] -#[ignore] -#[named] -#[test] -fn private_mysql_v8_deploy_a_working_dev_environment() { - test_mysql_configuration("8.0", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-do-self-hosted")] -#[named] -#[test] -fn public_mysql_v8_deploy_a_working_dev_environment() { - test_mysql_configuration("8.0", function_name!(), CONTAINER, true); -} - -// MySQL production environment - -/** - ** - ** Redis tests - ** - **/ -#[allow(dead_code)] -fn test_redis_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { let secrets = FuncTestsSecrets::new(); let context = context( secrets @@ -791,10 +1074,122 @@ fn test_redis_configuration(version: &str, test_name: &str, database_mode: Datab .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "5.7", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-do-self-hosted")] +#[ignore] +#[named] +#[test] +fn public_mysql_v5_7_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "5.7", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-do-self-hosted")] +#[ignore] +#[named] +#[test] +fn private_mysql_v8_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-do-self-hosted")] +#[named] +#[test] +fn public_mysql_v8_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), CONTAINER, true); +} + +// MySQL production environment + +/** + ** + ** Redis tests + ** + **/ +#[allow(dead_code)] +fn test_redis_configuration( + context: Context, + environment: Environment, + secrets: FuncTestsSecrets, + version: &str, + test_name: &str, + database_mode: DatabaseMode, + is_public: bool, +) { engine_run_test(|| { test_db( context, logger(), + environment, secrets, version, test_name, @@ -811,7 +1206,27 @@ fn test_redis_configuration(version: &str, test_name: &str, database_mode: Datab #[named] #[test] fn private_redis_v5_deploy_a_working_dev_environment() { - test_redis_configuration("5", function_name!(), CONTAINER, false); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_redis_configuration(context, environment, secrets, "5", function_name!(), CONTAINER, false); } #[cfg(feature = "test-do-self-hosted")] @@ -819,7 +1234,27 @@ fn private_redis_v5_deploy_a_working_dev_environment() { #[named] #[test] fn public_redis_v5_deploy_a_working_dev_environment() { - test_redis_configuration("5", function_name!(), CONTAINER, true); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_redis_configuration(context, environment, secrets, "5", function_name!(), CONTAINER, true); } #[cfg(feature = "test-do-self-hosted")] @@ -827,12 +1262,52 @@ fn public_redis_v5_deploy_a_working_dev_environment() { #[named] #[test] fn private_redis_v6_deploy_a_working_dev_environment() { - test_redis_configuration("6", function_name!(), CONTAINER, false); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_redis_configuration(context, environment, secrets, "6", function_name!(), CONTAINER, false); } #[cfg(feature = "test-do-self-hosted")] #[named] #[test] fn public_redis_v6_deploy_a_working_dev_environment() { - test_redis_configuration("6", function_name!(), CONTAINER, true); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .DIGITAL_OCEAN_TEST_ORGANIZATION_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), + secrets + .DIGITAL_OCEAN_TEST_CLUSTER_ID + .as_ref() + .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), + ); + + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_redis_configuration(context, environment, secrets, "6", function_name!(), CONTAINER, true); } diff --git a/tests/scaleway/scw_databases.rs b/tests/scaleway/scw_databases.rs index 0fb4a903..6caed734 100644 --- a/tests/scaleway/scw_databases.rs +++ b/tests/scaleway/scw_databases.rs @@ -2,7 +2,9 @@ use ::function_name::named; use tracing::{span, warn, Level}; use qovery_engine::cloud_provider::{Kind as ProviderKind, Kind}; -use qovery_engine::models::{Action, Clone2, Database, DatabaseKind, DatabaseMode, EnvironmentAction, Port, Protocol}; +use qovery_engine::models::{ + Action, Clone2, Context, Database, DatabaseKind, DatabaseMode, Environment, EnvironmentAction, Port, Protocol, +}; use qovery_engine::transaction::TransactionResult; use test_utilities::utilities::{ context, engine_run_test, generate_id, generate_password, get_pods, get_svc_name, init, is_pod_restarted_env, @@ -10,8 +12,8 @@ use test_utilities::utilities::{ }; use qovery_engine::models::DatabaseMode::{CONTAINER, MANAGED}; -use test_utilities::common::test_db; use test_utilities::common::Infrastructure; +use test_utilities::common::{test_db, working_minimal_environment}; use test_utilities::scaleway::{ clean_environments, SCW_MANAGED_DATABASE_DISK_TYPE, SCW_MANAGED_DATABASE_INSTANCE_TYPE, SCW_SELF_HOSTED_DATABASE_DISK_TYPE, SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, SCW_TEST_ZONE, @@ -59,6 +61,7 @@ fn deploy_an_environment_with_3_databases_and_3_apps() { .as_str(), SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, SCW_SELF_HOSTED_DATABASE_DISK_TYPE, + Kind::Scw, ); let mut environment_delete = environment.clone(); @@ -122,6 +125,7 @@ fn deploy_an_environment_with_db_and_pause_it() { .as_str(), SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, SCW_SELF_HOSTED_DATABASE_DISK_TYPE, + Kind::Scw, ); let mut environment_delete = environment.clone(); @@ -210,6 +214,7 @@ fn postgresql_failover_dev_environment_with_all_options() { test_domain.as_str(), SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, SCW_SELF_HOSTED_DATABASE_DISK_TYPE, + Kind::Scw, ); let environment_check = environment.clone(); let mut environment_never_up = environment.clone(); @@ -235,6 +240,7 @@ fn postgresql_failover_dev_environment_with_all_options() { test_domain.as_str(), SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, SCW_SELF_HOSTED_DATABASE_DISK_TYPE, + Kind::Scw, ); environment_delete.action = Action::Delete; @@ -330,12 +336,14 @@ fn postgresql_deploy_a_working_development_environment_with_all_options() { test_domain.as_str(), SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, SCW_SELF_HOSTED_DATABASE_DISK_TYPE, + Kind::Scw, ); let mut environment_delete = test_utilities::common::environnement_2_app_2_routers_1_psql( &context_for_deletion, test_domain.as_str(), SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, SCW_SELF_HOSTED_DATABASE_DISK_TYPE, + Kind::Scw, ); environment_delete.action = Action::Delete; @@ -408,9 +416,8 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { .as_str(), ); - let db_id = generate_id(); let app_name = format!("postgresql-app-{}", generate_id()); - let database_host = get_svc_name(DatabaseKind::Postgresql, db_id.clone()).to_string(); + let database_host = get_svc_name(DatabaseKind::Postgresql, Kind::Scw).to_string(); let database_port = 5432; let database_db_name = "postgres".to_string(); let database_username = "superuser".to_string(); @@ -421,7 +428,7 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { environment.databases = vec![Database { kind: DatabaseKind::Postgresql, action: Action::Create, - id: db_id.clone(), + id: generate_id(), name: database_db_name.clone(), version: "11.8.0".to_string(), fqdn_id: "postgresql-".to_string() + generate_id().as_str(), @@ -535,25 +542,20 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { ** **/ #[allow(dead_code)] -fn test_postgresql_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - +fn test_postgresql_configuration( + context: Context, + environment: Environment, + secrets: FuncTestsSecrets, + version: &str, + test_name: &str, + database_mode: DatabaseMode, + is_public: bool, +) { engine_run_test(|| { test_db( context, logger(), + environment, secrets, version, test_name, @@ -570,130 +572,6 @@ fn test_postgresql_configuration(version: &str, test_name: &str, database_mode: #[named] #[test] fn private_postgresql_v10_deploy_a_working_dev_environment() { - test_postgresql_configuration("10", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn public_postgresql_v10_deploy_a_working_dev_environment() { - test_postgresql_configuration("10", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn private_postgresql_v11_deploy_a_working_dev_environment() { - test_postgresql_configuration("11", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn public_postgresql_v11_deploy_a_working_dev_environment() { - test_postgresql_configuration("11", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn private_postgresql_v12_deploy_a_working_dev_environment() { - test_postgresql_configuration("12", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn public_postgresql_v12_deploy_a_working_dev_environment() { - test_postgresql_configuration("12", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn private_postgresql_v13_deploy_a_working_dev_environment() { - test_postgresql_configuration("13", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn public_postgresql_v13_deploy_a_working_dev_environment() { - test_postgresql_configuration("13", function_name!(), CONTAINER, true); -} - -// Postgres production environment -#[cfg(feature = "test-scw-managed-services")] -#[named] -#[test] -#[ignore] -fn private_postgresql_v10_deploy_a_working_prod_environment() { - test_postgresql_configuration("10", function_name!(), MANAGED, false); -} - -#[cfg(feature = "test-scw-managed-services")] -#[named] -#[test] -#[ignore] -fn public_postgresql_v10_deploy_a_working_prod_environment() { - test_postgresql_configuration("10", function_name!(), MANAGED, true); -} - -#[cfg(feature = "test-scw-managed-services")] -#[named] -#[test] -#[ignore] -fn private_postgresql_v11_deploy_a_working_prod_environment() { - test_postgresql_configuration("11", function_name!(), MANAGED, false); -} - -#[cfg(feature = "test-scw-managed-services")] -#[named] -#[test] -#[ignore] -fn public_postgresql_v11_deploy_a_working_prod_environment() { - test_postgresql_configuration("11", function_name!(), MANAGED, true); -} - -#[cfg(feature = "test-scw-managed-services")] -#[named] -#[test] -#[ignore] -fn private_postgresql_v12_deploy_a_working_prod_environment() { - test_postgresql_configuration("12", function_name!(), MANAGED, false); -} - -#[cfg(feature = "test-scw-managed-services")] -#[named] -#[test] -#[ignore] -fn public_postgresql_v12_deploy_a_working_prod_environment() { - test_postgresql_configuration("12", function_name!(), MANAGED, true); -} - -#[cfg(feature = "test-scw-managed-services")] -#[named] -#[test] -#[ignore] -fn private_postgresql_v13_deploy_a_working_prod_environment() { - test_postgresql_configuration("13", function_name!(), MANAGED, false); -} - -#[cfg(feature = "test-scw-managed-services")] -#[named] -#[test] -#[ignore] -fn public_postgresql_v13_deploy_a_working_prod_environment() { - test_postgresql_configuration("13", function_name!(), MANAGED, true); -} - -/** - ** - ** MongoDB tests - ** - **/ -#[allow(dead_code)] -fn test_mongodb_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { let secrets = FuncTestsSecrets::new(); let context = context( secrets @@ -707,11 +585,466 @@ fn test_mongodb_configuration(version: &str, test_name: &str, database_mode: Dat .expect("SCALEWAY_TEST_CLUSTER_ID") .as_str(), ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "10", function_name!(), CONTAINER, false); +} +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn public_postgresql_v10_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "10", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn private_postgresql_v11_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "11", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn public_postgresql_v11_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "11", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn private_postgresql_v12_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "12", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn public_postgresql_v12_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "12", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn private_postgresql_v13_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "13", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn public_postgresql_v13_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "13", function_name!(), CONTAINER, true); +} + +// Postgres production environment +#[cfg(feature = "test-scw-managed-services")] +#[named] +#[test] +#[ignore] +fn private_postgresql_v10_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "10", function_name!(), MANAGED, false); +} + +#[cfg(feature = "test-scw-managed-services")] +#[named] +#[test] +#[ignore] +fn public_postgresql_v10_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "10", function_name!(), MANAGED, true); +} + +#[cfg(feature = "test-scw-managed-services")] +#[named] +#[test] +#[ignore] +fn private_postgresql_v11_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "11", function_name!(), MANAGED, false); +} + +#[cfg(feature = "test-scw-managed-services")] +#[named] +#[test] +#[ignore] +fn public_postgresql_v11_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "11", function_name!(), MANAGED, true); +} + +#[cfg(feature = "test-scw-managed-services")] +#[named] +#[test] +#[ignore] +fn private_postgresql_v12_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "12", function_name!(), MANAGED, false); +} + +#[cfg(feature = "test-scw-managed-services")] +#[named] +#[test] +#[ignore] +fn public_postgresql_v12_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "12", function_name!(), MANAGED, true); +} + +#[cfg(feature = "test-scw-managed-services")] +#[named] +#[test] +#[ignore] +fn private_postgresql_v13_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "13", function_name!(), MANAGED, false); +} + +#[cfg(feature = "test-scw-managed-services")] +#[named] +#[test] +#[ignore] +fn public_postgresql_v13_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_postgresql_configuration(context, environment, secrets, "13", function_name!(), MANAGED, true); +} + +/** + ** + ** MongoDB tests + ** + **/ +#[allow(dead_code)] +fn test_mongodb_configuration( + context: Context, + environment: Environment, + secrets: FuncTestsSecrets, + version: &str, + test_name: &str, + database_mode: DatabaseMode, + is_public: bool, +) { engine_run_test(|| { test_db( context, logger(), + environment, secrets, version, test_name, @@ -728,65 +1061,6 @@ fn test_mongodb_configuration(version: &str, test_name: &str, database_mode: Dat #[named] #[test] fn private_mongodb_v3_6_deploy_a_working_dev_environment() { - test_mongodb_configuration("3.6", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn public_mongodb_v3_6_deploy_a_working_dev_environment() { - test_mongodb_configuration("3.6", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn private_mongodb_v4_0_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.0", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn public_mongodb_v4_0_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.0", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn private_mongodb_v4_2_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.2", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn public_mongodb_v4_2_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.2", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn private_mongodb_v4_4_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.4", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn public_mongodb_v4_4_deploy_a_working_dev_environment() { - test_mongodb_configuration("4.4", function_name!(), CONTAINER, true); -} - -/** - ** - ** MySQL tests - ** - **/ -#[allow(dead_code)] -fn test_mysql_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { let secrets = FuncTestsSecrets::new(); let context = context( secrets @@ -800,11 +1074,233 @@ fn test_mysql_configuration(version: &str, test_name: &str, database_mode: Datab .expect("SCALEWAY_TEST_CLUSTER_ID") .as_str(), ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "3.6", function_name!(), CONTAINER, false); +} +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn public_mongodb_v3_6_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "3.6", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn private_mongodb_v4_0_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.0", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn public_mongodb_v4_0_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.0", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn private_mongodb_v4_2_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.2", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn public_mongodb_v4_2_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.2", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn private_mongodb_v4_4_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.4", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn public_mongodb_v4_4_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mongodb_configuration(context, environment, secrets, "4.4", function_name!(), CONTAINER, true); +} + +/** + ** + ** MySQL tests + ** + **/ +#[allow(dead_code)] +fn test_mysql_configuration( + context: Context, + environment: Environment, + secrets: FuncTestsSecrets, + version: &str, + test_name: &str, + database_mode: DatabaseMode, + is_public: bool, +) { engine_run_test(|| { test_db( context, logger(), + environment, secrets, version, test_name, @@ -821,54 +1317,6 @@ fn test_mysql_configuration(version: &str, test_name: &str, database_mode: Datab #[named] #[test] fn private_mysql_v5_7_deploy_a_working_dev_environment() { - test_mysql_configuration("5.7", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn public_mysql_v5_7_deploy_a_working_dev_environment() { - test_mysql_configuration("5.7", function_name!(), CONTAINER, true); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn private_mysql_v8_deploy_a_working_dev_environment() { - test_mysql_configuration("8.0", function_name!(), CONTAINER, false); -} - -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn public_mysql_v8_deploy_a_working_dev_environment() { - test_mysql_configuration("8.0", function_name!(), CONTAINER, true); -} - -// MySQL production environment (RDS) -#[cfg(feature = "test-scw-managed-services")] -#[named] -#[test] -#[ignore] -fn private_mysql_v8_deploy_a_working_prod_environment() { - test_mysql_configuration("8.0", function_name!(), MANAGED, false); -} - -#[cfg(feature = "test-scw-managed-services")] -#[named] -#[test] -#[ignore] -fn public_mysql_v8_deploy_a_working_prod_environment() { - test_mysql_configuration("8.0", function_name!(), MANAGED, true); -} - -/** - ** - ** Redis tests - ** - **/ -#[allow(dead_code)] -fn test_redis_configuration(version: &str, test_name: &str, database_mode: DatabaseMode, is_public: bool) { let secrets = FuncTestsSecrets::new(); let context = context( secrets @@ -882,11 +1330,180 @@ fn test_redis_configuration(version: &str, test_name: &str, database_mode: Datab .expect("SCALEWAY_TEST_CLUSTER_ID") .as_str(), ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "5.7", function_name!(), CONTAINER, false); +} +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn public_mysql_v5_7_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "5.7", function_name!(), CONTAINER, true); +} + +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn private_mysql_v8_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), CONTAINER, false); +} + +#[cfg(feature = "test-scw-self-hosted")] +#[named] +#[test] +fn public_mysql_v8_deploy_a_working_dev_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), CONTAINER, true); +} + +// MySQL production environment (RDS) +#[cfg(feature = "test-scw-managed-services")] +#[named] +#[test] +#[ignore] +fn private_mysql_v8_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), MANAGED, false); +} + +#[cfg(feature = "test-scw-managed-services")] +#[named] +#[test] +#[ignore] +fn public_mysql_v8_deploy_a_working_prod_environment() { + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = test_utilities::common::working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_mysql_configuration(context, environment, secrets, "8.0", function_name!(), MANAGED, true); +} + +/** + ** + ** Redis tests + ** + **/ +#[allow(dead_code)] +fn test_redis_configuration( + context: Context, + environment: Environment, + secrets: FuncTestsSecrets, + version: &str, + test_name: &str, + database_mode: DatabaseMode, + is_public: bool, +) { engine_run_test(|| { test_db( context, logger(), + environment, secrets, version, test_name, @@ -903,26 +1520,110 @@ fn test_redis_configuration(version: &str, test_name: &str, database_mode: Datab #[named] #[test] fn private_redis_v5_deploy_a_working_dev_environment() { - test_redis_configuration("5", function_name!(), CONTAINER, false); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_redis_configuration(context, environment, secrets, "5", function_name!(), CONTAINER, false); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn public_redis_v5_deploy_a_working_dev_environment() { - test_redis_configuration("5", function_name!(), CONTAINER, true); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_redis_configuration(context, environment, secrets, "5", function_name!(), CONTAINER, true); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn private_redis_v6_deploy_a_working_dev_environment() { - test_redis_configuration("6", function_name!(), CONTAINER, false); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_redis_configuration(context, environment, secrets, "6", function_name!(), CONTAINER, false); } #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] fn public_redis_v6_deploy_a_working_dev_environment() { - test_redis_configuration("6", function_name!(), CONTAINER, true); + let secrets = FuncTestsSecrets::new(); + let context = context( + secrets + .SCALEWAY_TEST_ORGANIZATION_ID + .as_ref() + .expect("SCALEWAY_TEST_ORGANIZATION_ID") + .as_str(), + secrets + .SCALEWAY_TEST_CLUSTER_ID + .as_ref() + .expect("SCALEWAY_TEST_CLUSTER_ID") + .as_str(), + ); + let environment = working_minimal_environment( + &context, + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str(), + ); + test_redis_configuration(context, environment, secrets, "6", function_name!(), CONTAINER, true); } From a8e74ba4d3f7403cc719c7081476d8ed4de8526e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Er=C3=A8be=20-=20Romain=20Gerard?= Date: Mon, 7 Feb 2022 09:20:25 +0100 Subject: [PATCH 25/25] fix(ENG-1068): normalize db service (#572) (#582) --- lib/aws/chart_values/mongodb/q-values.j2.yaml | 1 + lib/aws/chart_values/mysql/q-values.j2.yaml | 1 + .../chart_values/postgresql/q-values.j2.yaml | 1 + lib/aws/chart_values/redis/q-values.j2.yaml | 1 + lib/aws/services/mongodb/main.j2.tf | 5 + lib/aws/services/mysql/main.j2.tf | 4 + lib/aws/services/postgresql/main.j2.tf | 4 + lib/aws/services/redis/main.j2.tf | 5 + .../templates/external-name-service.yaml | 2 +- .../charts/external-name-svc/values.j2.yaml | 1 + .../services/mysql/templates/master-svc.yaml | 2 +- lib/common/services/mysql/values.yaml | 1 + .../services/postgresql/templates/svc.yaml | 2 +- lib/common/services/postgresql/values.yaml | 1 + .../redis/templates/redis-master-svc.yaml | 2 +- lib/common/services/redis/values.yaml | 1 + .../chart_values/mongodb/q-values.j2.yaml | 1 + .../chart_values/mysql/q-values.j2.yaml | 1 + .../chart_values/postgresql/q-values.j2.yaml | 1 + .../chart_values/redis/q-values.j2.yaml | 1 + .../chart_values/mongodb/q-values.j2.yaml | 1 + .../chart_values/mysql/q-values.j2.yaml | 1 + .../chart_values/postgresql/q-values.j2.yaml | 1 + .../chart_values/redis/q-values.j2.yaml | 1 + src/cloud_provider/aws/databases/mongodb.rs | 1 + src/cloud_provider/aws/databases/mysql.rs | 1 + .../aws/databases/postgresql.rs | 1 + src/cloud_provider/aws/databases/redis.rs | 1 + .../digitalocean/databases/mongodb.rs | 1 + .../digitalocean/databases/mysql.rs | 1 + .../digitalocean/databases/postgresql.rs | 1 + .../digitalocean/databases/redis.rs | 1 + .../scaleway/databases/mongodb.rs | 1 + .../scaleway/databases/mysql.rs | 1 + .../scaleway/databases/postgresql.rs | 1 + .../scaleway/databases/redis.rs | 1 + test_utilities/src/common.rs | 61 ++++----- test_utilities/src/utilities.rs | 13 +- tests/aws/aws_databases.rs | 118 +---------------- tests/digitalocean/do_databases.rs | 122 +---------------- tests/scaleway/scw_databases.rs | 124 +----------------- 41 files changed, 87 insertions(+), 405 deletions(-) diff --git a/lib/aws/chart_values/mongodb/q-values.j2.yaml b/lib/aws/chart_values/mongodb/q-values.j2.yaml index dc3bf279..9cfe1154 100644 --- a/lib/aws/chart_values/mongodb/q-values.j2.yaml +++ b/lib/aws/chart_values/mongodb/q-values.j2.yaml @@ -148,6 +148,7 @@ service: type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} # clusterIP: None port: {{ private_port }} + name: {{ service_name }} ## Specify the nodePort value for the LoadBalancer and NodePort service types. ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport diff --git a/lib/aws/chart_values/mysql/q-values.j2.yaml b/lib/aws/chart_values/mysql/q-values.j2.yaml index 226c9814..daa4475d 100644 --- a/lib/aws/chart_values/mysql/q-values.j2.yaml +++ b/lib/aws/chart_values/mysql/q-values.j2.yaml @@ -501,6 +501,7 @@ service: ## MySQL Service type ## type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} + name: {{ service_name }} ## MySQL Service port ## diff --git a/lib/aws/chart_values/postgresql/q-values.j2.yaml b/lib/aws/chart_values/postgresql/q-values.j2.yaml index c3553002..3e908351 100644 --- a/lib/aws/chart_values/postgresql/q-values.j2.yaml +++ b/lib/aws/chart_values/postgresql/q-values.j2.yaml @@ -251,6 +251,7 @@ service: type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} # clusterIP: None port: 5432 + name: {{ service_name }} ## Specify the nodePort value for the LoadBalancer and NodePort service types. ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport diff --git a/lib/aws/chart_values/redis/q-values.j2.yaml b/lib/aws/chart_values/redis/q-values.j2.yaml index 72ae0ccb..007bb33a 100644 --- a/lib/aws/chart_values/redis/q-values.j2.yaml +++ b/lib/aws/chart_values/redis/q-values.j2.yaml @@ -370,6 +370,7 @@ master: ## Redis Master Service type type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} port: 6379 + name: {{ service_name }} ## Specify the nodePort value for the LoadBalancer and NodePort service types. ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport diff --git a/lib/aws/services/mongodb/main.j2.tf b/lib/aws/services/mongodb/main.j2.tf index facbc357..8f20c381 100644 --- a/lib/aws/services/mongodb/main.j2.tf +++ b/lib/aws/services/mongodb/main.j2.tf @@ -50,6 +50,11 @@ resource "helm_release" "documentdb_instance_external_name" { value = "{{database_id}}" } + set { + name = "service_name" + value = "{{service_name}}" + } + depends_on = [ aws_docdb_cluster.documentdb_cluster ] diff --git a/lib/aws/services/mysql/main.j2.tf b/lib/aws/services/mysql/main.j2.tf index 7ca2e1c3..68e84b13 100644 --- a/lib/aws/services/mysql/main.j2.tf +++ b/lib/aws/services/mysql/main.j2.tf @@ -51,6 +51,10 @@ resource "helm_release" "mysql_instance_external_name" { name = "app_id" value = "{{database_id}}" } + set { + name = "service_name" + value = "{{service_name}}" + } depends_on = [ aws_db_instance.mysql_instance diff --git a/lib/aws/services/postgresql/main.j2.tf b/lib/aws/services/postgresql/main.j2.tf index 62a12559..9c0517db 100644 --- a/lib/aws/services/postgresql/main.j2.tf +++ b/lib/aws/services/postgresql/main.j2.tf @@ -51,6 +51,10 @@ resource "helm_release" "postgres_instance_external_name" { name = "app_id" value = "{{database_id}}" } + set { + name = "service_name" + value = "{{service_name}}" + } depends_on = [ aws_db_instance.postgresql_instance diff --git a/lib/aws/services/redis/main.j2.tf b/lib/aws/services/redis/main.j2.tf index a33130cd..5916f425 100644 --- a/lib/aws/services/redis/main.j2.tf +++ b/lib/aws/services/redis/main.j2.tf @@ -50,6 +50,11 @@ resource "helm_release" "elasticache_instance_external_name" { value = "{{database_id}}" } + set { + name = "service_name" + value = "{{service_name}}" + } + set { name= "publicly_accessible" value= {{ publicly_accessible }} diff --git a/lib/common/charts/external-name-svc/templates/external-name-service.yaml b/lib/common/charts/external-name-svc/templates/external-name-service.yaml index d59d9216..91aaec50 100644 --- a/lib/common/charts/external-name-svc/templates/external-name-service.yaml +++ b/lib/common/charts/external-name-svc/templates/external-name-service.yaml @@ -3,7 +3,7 @@ apiVersion: v1 metadata: # This service is used for deployed services that do not have ingress like database # It is used to provide to the end user an endpoint with desired name - name: {{.Values.app_id}}-dns + name: {{ .Values.service_name }} labels: appId: {{.Values.app_id}} {{ if .Values.publicly_accessible }} diff --git a/lib/common/charts/external-name-svc/values.j2.yaml b/lib/common/charts/external-name-svc/values.j2.yaml index 0d103728..6ab34f7c 100644 --- a/lib/common/charts/external-name-svc/values.j2.yaml +++ b/lib/common/charts/external-name-svc/values.j2.yaml @@ -1,4 +1,5 @@ target_hostname: "" source_fqdn: "" +service_name: "" app_id: "" publicly_accessible: {{ publicly_accessible }} \ No newline at end of file diff --git a/lib/common/services/mysql/templates/master-svc.yaml b/lib/common/services/mysql/templates/master-svc.yaml index 24142c32..790f2df5 100644 --- a/lib/common/services/mysql/templates/master-svc.yaml +++ b/lib/common/services/mysql/templates/master-svc.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ template "mysql.fullname" . }} + name: {{ .Values.service.name }} labels: {{- include "mysql.labels" . | nindent 4 }} component: master {{- if or .Values.service.annotations .Values.metrics.service.annotations }} diff --git a/lib/common/services/mysql/values.yaml b/lib/common/services/mysql/values.yaml index 97d05aaf..7cc01845 100644 --- a/lib/common/services/mysql/values.yaml +++ b/lib/common/services/mysql/values.yaml @@ -490,6 +490,7 @@ service: ## type: ClusterIP + #name: {{ service_name }} ## MySQL Service port ## port: 3306 diff --git a/lib/common/services/postgresql/templates/svc.yaml b/lib/common/services/postgresql/templates/svc.yaml index d24b2a65..36473cc4 100644 --- a/lib/common/services/postgresql/templates/svc.yaml +++ b/lib/common/services/postgresql/templates/svc.yaml @@ -7,7 +7,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ template "postgresql.fullname" . }} + name: {{ .Values.service.name }} labels: app: {{ template "postgresql.name" . }} chart: {{ template "postgresql.chart" . }} diff --git a/lib/common/services/postgresql/values.yaml b/lib/common/services/postgresql/values.yaml index e4651dae..c2ed1b0a 100644 --- a/lib/common/services/postgresql/values.yaml +++ b/lib/common/services/postgresql/values.yaml @@ -249,6 +249,7 @@ ldap: service: ## PosgresSQL service type type: ClusterIP + name: service_name # clusterIP: None port: 5432 diff --git a/lib/common/services/redis/templates/redis-master-svc.yaml b/lib/common/services/redis/templates/redis-master-svc.yaml index 94fcce20..b44adb2d 100644 --- a/lib/common/services/redis/templates/redis-master-svc.yaml +++ b/lib/common/services/redis/templates/redis-master-svc.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ template "redis.fullname" . }}-master + name: {{ .Values.master.service.name }} namespace: {{ .Release.Namespace | quote }} labels: app: {{ template "redis.name" . }} diff --git a/lib/common/services/redis/values.yaml b/lib/common/services/redis/values.yaml index fb92acb3..d994d649 100644 --- a/lib/common/services/redis/values.yaml +++ b/lib/common/services/redis/values.yaml @@ -362,6 +362,7 @@ master: service: ## Redis Master Service type type: ClusterIP + name: service_name port: 6379 ## Specify the nodePort value for the LoadBalancer and NodePort service types. diff --git a/lib/digitalocean/chart_values/mongodb/q-values.j2.yaml b/lib/digitalocean/chart_values/mongodb/q-values.j2.yaml index f1c032c7..39ceece2 100644 --- a/lib/digitalocean/chart_values/mongodb/q-values.j2.yaml +++ b/lib/digitalocean/chart_values/mongodb/q-values.j2.yaml @@ -151,6 +151,7 @@ service: type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} # clusterIP: None port: {{ private_port }} + name: {{ service_name }} ## Specify the nodePort value for the LoadBalancer and NodePort service types. ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport diff --git a/lib/digitalocean/chart_values/mysql/q-values.j2.yaml b/lib/digitalocean/chart_values/mysql/q-values.j2.yaml index d6247355..c51c70a0 100644 --- a/lib/digitalocean/chart_values/mysql/q-values.j2.yaml +++ b/lib/digitalocean/chart_values/mysql/q-values.j2.yaml @@ -501,6 +501,7 @@ service: ## MySQL Service type ## type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} + name: {{ service_name }} ## MySQL Service port ## diff --git a/lib/digitalocean/chart_values/postgresql/q-values.j2.yaml b/lib/digitalocean/chart_values/postgresql/q-values.j2.yaml index 93c967d3..c61ac44d 100644 --- a/lib/digitalocean/chart_values/postgresql/q-values.j2.yaml +++ b/lib/digitalocean/chart_values/postgresql/q-values.j2.yaml @@ -251,6 +251,7 @@ service: type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} # clusterIP: None port: 5432 + name: {{ service_name }} ## Specify the nodePort value for the LoadBalancer and NodePort service types. ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport diff --git a/lib/digitalocean/chart_values/redis/q-values.j2.yaml b/lib/digitalocean/chart_values/redis/q-values.j2.yaml index 70b91f04..3cb4c035 100644 --- a/lib/digitalocean/chart_values/redis/q-values.j2.yaml +++ b/lib/digitalocean/chart_values/redis/q-values.j2.yaml @@ -370,6 +370,7 @@ master: ## Redis Master Service type type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} port: 6379 + name: {{ service_name }} ## Specify the nodePort value for the LoadBalancer and NodePort service types. ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport diff --git a/lib/scaleway/chart_values/mongodb/q-values.j2.yaml b/lib/scaleway/chart_values/mongodb/q-values.j2.yaml index 95da012f..ad642d12 100644 --- a/lib/scaleway/chart_values/mongodb/q-values.j2.yaml +++ b/lib/scaleway/chart_values/mongodb/q-values.j2.yaml @@ -89,6 +89,7 @@ service: type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} # clusterIP: None port: {{ private_port }} + name: {{ service_name }} ## Specify the nodePort value for the LoadBalancer and NodePort service types. ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport diff --git a/lib/scaleway/chart_values/mysql/q-values.j2.yaml b/lib/scaleway/chart_values/mysql/q-values.j2.yaml index c4bf58c0..e5d05db4 100644 --- a/lib/scaleway/chart_values/mysql/q-values.j2.yaml +++ b/lib/scaleway/chart_values/mysql/q-values.j2.yaml @@ -480,6 +480,7 @@ service: ## type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} + name: {{ service_name }} ## MySQL Service port ## port: 3306 diff --git a/lib/scaleway/chart_values/postgresql/q-values.j2.yaml b/lib/scaleway/chart_values/postgresql/q-values.j2.yaml index 96b5dafe..81ed0bb3 100644 --- a/lib/scaleway/chart_values/postgresql/q-values.j2.yaml +++ b/lib/scaleway/chart_values/postgresql/q-values.j2.yaml @@ -123,6 +123,7 @@ service: type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} # clusterIP: None port: 5432 + name: {{ service_name }} ## Provide any additional annotations which may be required. Evaluated as a template. ## diff --git a/lib/scaleway/chart_values/redis/q-values.j2.yaml b/lib/scaleway/chart_values/redis/q-values.j2.yaml index 92c30306..5c85e762 100644 --- a/lib/scaleway/chart_values/redis/q-values.j2.yaml +++ b/lib/scaleway/chart_values/redis/q-values.j2.yaml @@ -327,6 +327,7 @@ master: ## Redis Master Service type type: {% if publicly_accessible -%} LoadBalancer {% else -%} ClusterIP {% endif %} port: 6379 + name: {{ service_name }} ## Specify the nodePort value for the LoadBalancer and NodePort service types. ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport diff --git a/src/cloud_provider/aws/databases/mongodb.rs b/src/cloud_provider/aws/databases/mongodb.rs index ad2d5d16..4a8fe8ad 100644 --- a/src/cloud_provider/aws/databases/mongodb.rs +++ b/src/cloud_provider/aws/databases/mongodb.rs @@ -191,6 +191,7 @@ impl Service for MongoDB { "fqdn", self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), ); + context.insert("service_name", self.fqdn_id.as_str()); context.insert("database_db_name", self.name.as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); diff --git a/src/cloud_provider/aws/databases/mysql.rs b/src/cloud_provider/aws/databases/mysql.rs index e5c451f2..50efe1ca 100644 --- a/src/cloud_provider/aws/databases/mysql.rs +++ b/src/cloud_provider/aws/databases/mysql.rs @@ -212,6 +212,7 @@ impl Service for MySQL { "fqdn", self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), ); + context.insert("service_name", self.fqdn_id.as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); diff --git a/src/cloud_provider/aws/databases/postgresql.rs b/src/cloud_provider/aws/databases/postgresql.rs index ee66bcb2..a2700399 100644 --- a/src/cloud_provider/aws/databases/postgresql.rs +++ b/src/cloud_provider/aws/databases/postgresql.rs @@ -197,6 +197,7 @@ impl Service for PostgreSQL { "fqdn", self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), ); + context.insert("service_name", self.fqdn_id.as_str()); context.insert("database_name", self.sanitized_name().as_str()); context.insert("database_db_name", self.name()); context.insert("database_login", self.options.login.as_str()); diff --git a/src/cloud_provider/aws/databases/redis.rs b/src/cloud_provider/aws/databases/redis.rs index b45d45c9..3f485356 100644 --- a/src/cloud_provider/aws/databases/redis.rs +++ b/src/cloud_provider/aws/databases/redis.rs @@ -213,6 +213,7 @@ impl Service for Redis { "fqdn", self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), ); + context.insert("service_name", self.fqdn_id.as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); diff --git a/src/cloud_provider/digitalocean/databases/mongodb.rs b/src/cloud_provider/digitalocean/databases/mongodb.rs index 84357d13..019c7cbe 100644 --- a/src/cloud_provider/digitalocean/databases/mongodb.rs +++ b/src/cloud_provider/digitalocean/databases/mongodb.rs @@ -188,6 +188,7 @@ impl Service for MongoDB { "fqdn", self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), ); + context.insert("service_name", self.fqdn_id.as_str()); context.insert("database_db_name", self.name.as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); diff --git a/src/cloud_provider/digitalocean/databases/mysql.rs b/src/cloud_provider/digitalocean/databases/mysql.rs index 00bba0f2..b6ccb010 100644 --- a/src/cloud_provider/digitalocean/databases/mysql.rs +++ b/src/cloud_provider/digitalocean/databases/mysql.rs @@ -188,6 +188,7 @@ impl Service for MySQL { "fqdn", self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), ); + context.insert("service_name", self.fqdn_id.as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); diff --git a/src/cloud_provider/digitalocean/databases/postgresql.rs b/src/cloud_provider/digitalocean/databases/postgresql.rs index b7efc14a..f489bdf4 100644 --- a/src/cloud_provider/digitalocean/databases/postgresql.rs +++ b/src/cloud_provider/digitalocean/databases/postgresql.rs @@ -188,6 +188,7 @@ impl Service for PostgreSQL { "fqdn", self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), ); + context.insert("service_name", self.fqdn_id.as_str()); context.insert("database_db_name", self.name()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); diff --git a/src/cloud_provider/digitalocean/databases/redis.rs b/src/cloud_provider/digitalocean/databases/redis.rs index 527cc38e..02f04f75 100644 --- a/src/cloud_provider/digitalocean/databases/redis.rs +++ b/src/cloud_provider/digitalocean/databases/redis.rs @@ -188,6 +188,7 @@ impl Service for Redis { "fqdn", self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), ); + context.insert("service_name", self.fqdn_id.as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); diff --git a/src/cloud_provider/scaleway/databases/mongodb.rs b/src/cloud_provider/scaleway/databases/mongodb.rs index d3303924..74f5f131 100644 --- a/src/cloud_provider/scaleway/databases/mongodb.rs +++ b/src/cloud_provider/scaleway/databases/mongodb.rs @@ -189,6 +189,7 @@ impl Service for MongoDB { "fqdn", self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), ); + context.insert("service_name", self.fqdn_id.as_str()); context.insert("database_db_name", self.name.as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); diff --git a/src/cloud_provider/scaleway/databases/mysql.rs b/src/cloud_provider/scaleway/databases/mysql.rs index c7bec2c8..d59e6dcb 100644 --- a/src/cloud_provider/scaleway/databases/mysql.rs +++ b/src/cloud_provider/scaleway/databases/mysql.rs @@ -221,6 +221,7 @@ impl Service for MySQL { "fqdn", self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), ); + context.insert("service_name", self.fqdn_id.as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); diff --git a/src/cloud_provider/scaleway/databases/postgresql.rs b/src/cloud_provider/scaleway/databases/postgresql.rs index f39f1610..2197730c 100644 --- a/src/cloud_provider/scaleway/databases/postgresql.rs +++ b/src/cloud_provider/scaleway/databases/postgresql.rs @@ -230,6 +230,7 @@ impl Service for PostgreSQL { "fqdn", self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), ); + context.insert("service_name", self.fqdn_id.as_str()); context.insert("database_name", self.sanitized_name().as_str()); context.insert("database_db_name", self.name()); context.insert("database_login", self.options.login.as_str()); diff --git a/src/cloud_provider/scaleway/databases/redis.rs b/src/cloud_provider/scaleway/databases/redis.rs index 4ca2f11a..9c39a2b3 100644 --- a/src/cloud_provider/scaleway/databases/redis.rs +++ b/src/cloud_provider/scaleway/databases/redis.rs @@ -189,6 +189,7 @@ impl Service for Redis { "fqdn", self.fqdn(target, &self.fqdn, self.is_managed_service()).as_str(), ); + context.insert("service_name", self.fqdn_id.as_str()); context.insert("database_login", self.options.login.as_str()); context.insert("database_password", self.options.password.as_str()); context.insert("database_port", &self.private_port()); diff --git a/test_utilities/src/common.rs b/test_utilities/src/common.rs index 2ace4cc2..fe8dfe0b 100644 --- a/test_utilities/src/common.rs +++ b/test_utilities/src/common.rs @@ -215,7 +215,6 @@ pub fn environment_3_apps_3_routers_3_databases( let version_mongo = "4.4"; // pSQL 1 management part - let fqdn_id = "my-postgresql-".to_string() + generate_id().as_str(); let fqdn = get_svc_name(DatabaseKind::Postgresql, provider_kind.clone()).to_string(); let database_port = 5432; let database_username = "superuser".to_string(); @@ -223,7 +222,6 @@ pub fn environment_3_apps_3_routers_3_databases( let database_name = "postgres".to_string(); // pSQL 2 management part - let fqdn_id_2 = "my-postgresql-2".to_string() + generate_id().as_str(); let fqdn_2 = format!("{}2", get_svc_name(DatabaseKind::Postgresql, provider_kind.clone())); let database_username_2 = "superuser2".to_string(); let database_name_2 = "postgres2".to_string(); @@ -425,7 +423,7 @@ pub fn environment_3_apps_3_routers_3_databases( id: generate_id(), name: database_name.clone(), version: "11.8.0".to_string(), - fqdn_id: fqdn_id.clone(), + fqdn_id: fqdn.clone(), fqdn: fqdn.clone(), port: database_port.clone(), username: database_username.clone(), @@ -447,7 +445,7 @@ pub fn environment_3_apps_3_routers_3_databases( id: generate_id(), name: database_name_2.clone(), version: "11.8.0".to_string(), - fqdn_id: fqdn_id_2.clone(), + fqdn_id: fqdn_2.clone(), fqdn: fqdn_2.clone(), port: database_port.clone(), username: database_username_2.clone(), @@ -469,7 +467,7 @@ pub fn environment_3_apps_3_routers_3_databases( id: generate_id(), name: database_db_name_mongo.clone(), version: version_mongo.to_string(), - fqdn_id: "mongodb-".to_string() + generate_id().as_str(), + fqdn_id: database_host_mongo.clone(), fqdn: database_host_mongo.clone(), port: database_port_mongo.clone(), username: database_username_mongo.clone(), @@ -572,13 +570,11 @@ pub fn environment_only_http_server_router_with_sticky_session(context: &Context pub fn environnement_2_app_2_routers_1_psql( context: &Context, - test_domain: &str, database_instance_type: &str, database_disk_type: &str, provider_kind: Kind, ) -> Environment { - let fqdn_id = "my-postgresql-".to_string() + generate_id().as_str(); let fqdn = get_svc_name(DatabaseKind::Postgresql, provider_kind.clone()).to_string(); let database_port = 5432; @@ -603,7 +599,7 @@ pub fn environnement_2_app_2_routers_1_psql( id: generate_id(), name: database_name.clone(), version: "11.8.0".to_string(), - fqdn_id: fqdn_id.clone(), + fqdn_id: fqdn.clone(), fqdn: fqdn.clone(), port: database_port.clone(), username: database_username.clone(), @@ -992,44 +988,42 @@ pub fn test_db( let database_username = "superuser".to_string(); let database_password = generate_id(); let db_kind_str = db_kind.name().to_string(); - let database_host = format!( - "{}-{}.{}.{}", - db_kind_str.clone(), - generate_id(), + let db_id = generate_id(); + let database_host = format!("{}-{}", db_id, db_kind_str.clone()); + let database_fqdn = format!( + "{}.{}.{}", + database_host, context.cluster_id(), secrets .clone() .DEFAULT_TEST_DOMAIN .expect("DEFAULT_TEST_DOMAIN is not set in secrets") ); - let dyn_db_fqdn = match is_public.clone() { - true => database_host.clone(), - false => match database_mode.clone() { - DatabaseMode::MANAGED => format!("{}-dns", app_id.clone()), - DatabaseMode::CONTAINER => get_svc_name(db_kind.clone(), provider_kind.clone()).to_string(), - }, - }; let db_infos = db_infos( db_kind.clone(), + db_id.clone(), database_mode.clone(), database_username.clone(), database_password.clone(), - dyn_db_fqdn.clone(), + if is_public { + database_fqdn.clone() + } else { + database_host.clone() + }, ); let database_port = db_infos.db_port.clone(); - let database_db_name = db_infos.db_name.clone(); let storage_size = 10; let db_disk_type = db_disk_type(provider_kind.clone(), database_mode.clone()); let db_instance_type = db_instance_type(provider_kind.clone(), db_kind.clone(), database_mode.clone()); let db = Database { kind: db_kind.clone(), action: Action::Create, - id: app_id.clone(), - name: database_db_name.clone(), + id: db_id.clone(), + name: db_id.clone(), version: version.to_string(), - fqdn_id: format!("{}-{}", db_kind_str.clone(), generate_id()), - fqdn: database_host.clone(), + fqdn_id: database_host.clone(), + fqdn: database_fqdn.clone(), port: database_port.clone(), username: database_username.clone(), password: database_password.clone(), @@ -1052,6 +1046,8 @@ pub fn test_db( .applications .into_iter() .map(|mut app| { + app.id = app_id.clone(); + app.name = app_id.clone(); app.branch = app_name.clone(); app.commit_id = db_infos.app_commit.clone(); app.ports = vec![Port { @@ -1068,7 +1064,7 @@ pub fn test_db( app }) .collect::>(); - environment.routers[0].routes[0].application_name = app_name.clone(); + environment.routers[0].routes[0].application_name = app_id.clone(); let mut environment_delete = environment.clone(); environment_delete.action = Action::Delete; @@ -1106,11 +1102,7 @@ pub fn test_db( svc.items .expect("No items in svc") .into_iter() - .filter(|svc| svc - .metadata - .name - .contains(get_svc_name(db_kind.clone(), provider_kind.clone())) - && &svc.spec.svc_type == "LoadBalancer") + .filter(|svc| svc.metadata.name == database_host && &svc.spec.svc_type == "LoadBalancer") .collect::>() .len(), match is_public { @@ -1128,17 +1120,14 @@ pub fn test_db( .items .expect("No items in svc") .into_iter() - .filter(|svc| { - svc.metadata.name.contains(format!("{}-dns", app_id.clone()).as_str()) - && svc.spec.svc_type == "ExternalName" - }) + .filter(|svc| svc.metadata.name == database_host && svc.spec.svc_type == "ExternalName") .collect::>(); let annotations = &service[0].metadata.annotations; assert_eq!(service.len(), 1); match is_public { true => { assert!(annotations.contains_key("external-dns.alpha.kubernetes.io/hostname")); - assert_eq!(annotations["external-dns.alpha.kubernetes.io/hostname"], database_host); + assert_eq!(annotations["external-dns.alpha.kubernetes.io/hostname"], database_fqdn); } false => assert!(!annotations.contains_key("external-dns.alpha.kubernetes.io/hostname")), } diff --git a/test_utilities/src/utilities.rs b/test_utilities/src/utilities.rs index e967e8e7..91d11138 100644 --- a/test_utilities/src/utilities.rs +++ b/test_utilities/src/utilities.rs @@ -979,6 +979,7 @@ pub struct DBInfos { pub fn db_infos( db_kind: DatabaseKind, + db_id: String, database_mode: DatabaseMode, database_username: String, database_password: String, @@ -987,7 +988,7 @@ pub fn db_infos( match db_kind { DatabaseKind::Mongodb => { let database_port = 27017; - let database_db_name = "my-mongodb".to_string(); + let database_db_name = db_id; let database_uri = format!( "mongodb://{}:{}@{}:{}/{}", database_username, @@ -1013,7 +1014,7 @@ pub fn db_infos( } DatabaseKind::Mysql => { let database_port = 3306; - let database_db_name = "mysqldatabase".to_string(); + let database_db_name = db_id; DBInfos { db_port: database_port.clone(), db_name: database_db_name.to_string(), @@ -1029,7 +1030,11 @@ pub fn db_infos( } DatabaseKind::Postgresql => { let database_port = 5432; - let database_db_name = "postgres".to_string(); + let database_db_name = if database_mode == MANAGED { + "postgres".to_string() + } else { + db_id + }; DBInfos { db_port: database_port.clone(), db_name: database_db_name.to_string(), @@ -1045,7 +1050,7 @@ pub fn db_infos( } DatabaseKind::Redis => { let database_port = 6379; - let database_db_name = "my-redis".to_string(); + let database_db_name = db_id; DBInfos { db_port: database_port.clone(), db_name: database_db_name.to_string(), diff --git a/tests/aws/aws_databases.rs b/tests/aws/aws_databases.rs index a20438c7..2ddf45c3 100644 --- a/tests/aws/aws_databases.rs +++ b/tests/aws/aws_databases.rs @@ -157,122 +157,6 @@ fn deploy_an_environment_with_db_and_pause_it() { }) } -// this test ensure containers databases are never restarted, even in failover environment case -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn postgresql_failover_dev_environment_with_all_options() { - let test_name = function_name!(); - engine_run_test(|| { - init(); - - let span = span!(Level::INFO, "test", name = test_name); - let _enter = span.enter(); - - let secrets = FuncTestsSecrets::new(); - let logger = logger(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let context_for_deletion = context.clone_not_same_execution_id(); - let test_domain = secrets - .clone() - .DEFAULT_TEST_DOMAIN - .expect("DEFAULT_TEST_DOMAIN is not set in secrets"); - - let environment = test_utilities::common::environnement_2_app_2_routers_1_psql( - &context, - test_domain.as_str(), - AWS_DATABASE_INSTANCE_TYPE, - AWS_DATABASE_DISK_TYPE, - Kind::Aws, - ); - let environment_check = environment.clone(); - let mut environment_never_up = environment.clone(); - // error in ports, these applications will never be up !! - environment_never_up.applications = environment_never_up - .applications - .into_iter() - .map(|mut app| { - app.ports = vec![Port { - id: "zdf7d6aad".to_string(), - long_id: Default::default(), - port: 4789, - public_port: Some(443), - name: None, - publicly_accessible: true, - protocol: Protocol::HTTP, - }]; - app - }) - .collect::>(); - let mut environment_delete = test_utilities::common::environnement_2_app_2_routers_1_psql( - &context_for_deletion, - test_domain.as_str(), - AWS_DATABASE_INSTANCE_TYPE, - AWS_DATABASE_DISK_TYPE, - Kind::Aws, - ); - - environment_delete.action = Action::Delete; - - let ea = EnvironmentAction::Environment(environment.clone()); - let ea_fail_ok = EnvironmentAction::EnvironmentWithFailover(environment_never_up.clone(), environment.clone()); - let ea_for_deletion = EnvironmentAction::Environment(environment_delete.clone()); - - match environment.deploy_environment(Kind::Aws, &context, &ea, logger.clone()) { - TransactionResult::Ok => assert!(true), - TransactionResult::Rollback(_) => assert!(false), - TransactionResult::UnrecoverableError(_, _) => assert!(false), - }; - // TO CHECK: DATABASE SHOULDN'T BE RESTARTED AFTER A REDEPLOY - let database_name = format!("postgresql{}-0", &environment_check.databases[0].name); - match is_pod_restarted_env( - context.clone(), - Kind::Aws, - environment_check.clone(), - database_name.as_str(), - secrets.clone(), - ) { - (true, _) => assert!(true), - (false, _) => assert!(false), - } - match environment_never_up.deploy_environment(Kind::Aws, &context, &ea_fail_ok, logger.clone()) { - TransactionResult::Ok => assert!(false), - TransactionResult::Rollback(_) => assert!(true), - TransactionResult::UnrecoverableError(_, _) => assert!(false), - }; - // TO CHECK: DATABASE SHOULDN'T BE RESTARTED AFTER A REDEPLOY EVEN IF FAIL - match is_pod_restarted_env( - context.clone(), - Kind::Aws, - environment_check.clone(), - database_name.as_str(), - secrets.clone(), - ) { - (true, _) => assert!(true), - (false, _) => assert!(false), - } - - match environment_delete.delete_environment(Kind::Aws, &context_for_deletion, &ea_for_deletion, logger) { - TransactionResult::Ok => assert!(true), - TransactionResult::Rollback(_) => assert!(false), - TransactionResult::UnrecoverableError(_, _) => assert!(false), - }; - - return test_name.to_string(); - }) -} - // Ensure a full environment can run correctly #[cfg(feature = "test-aws-self-hosted")] #[named] @@ -398,7 +282,7 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { id: generate_id(), name: database_db_name.clone(), version: "11.8.0".to_string(), - fqdn_id: "postgresql-".to_string() + generate_id().as_str(), + fqdn_id: database_host.clone(), fqdn: database_host.clone(), port: database_port, username: database_username.clone(), diff --git a/tests/digitalocean/do_databases.rs b/tests/digitalocean/do_databases.rs index 3c96542b..7ce4492a 100644 --- a/tests/digitalocean/do_databases.rs +++ b/tests/digitalocean/do_databases.rs @@ -166,126 +166,6 @@ fn deploy_an_environment_with_db_and_pause_it() { }) } -// this test ensure containers databases are never restarted, even in failover environment case -#[cfg(feature = "test-do-self-hosted")] -#[named] -#[test] -fn postgresql_failover_dev_environment_with_all_options() { - let test_name = function_name!(); - engine_run_test(|| { - init(); - - let span = span!(Level::INFO, "test", name = test_name); - let _enter = span.enter(); - - let secrets = FuncTestsSecrets::new(); - let logger = logger(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - let context_for_deletion = context.clone_not_same_execution_id(); - let test_domain = secrets - .clone() - .DEFAULT_TEST_DOMAIN - .expect("DEFAULT_TEST_DOMAIN is not set in secrets"); - - let environment = test_utilities::common::environnement_2_app_2_routers_1_psql( - &context, - test_domain.as_str(), - DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, - DO_SELF_HOSTED_DATABASE_DISK_TYPE, - Kind::Do, - ); - let environment_check = environment.clone(); - let mut environment_never_up = environment.clone(); - // error in ports, these applications will never be up !! - environment_never_up.applications = environment_never_up - .applications - .into_iter() - .map(|mut app| { - app.ports = vec![Port { - id: "zdf7d6aad".to_string(), - long_id: Default::default(), - port: 4789, - public_port: Some(443), - name: None, - publicly_accessible: true, - protocol: Protocol::HTTP, - }]; - app - }) - .collect::>(); - let mut environment_delete = test_utilities::common::environnement_2_app_2_routers_1_psql( - &context_for_deletion, - test_domain.as_str(), - DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, - DO_SELF_HOSTED_DATABASE_DISK_TYPE, - Kind::Do, - ); - - environment_delete.action = Action::Delete; - - let env_action = EnvironmentAction::Environment(environment.clone()); - let env_action_fail_ok = - EnvironmentAction::EnvironmentWithFailover(environment_never_up.clone(), environment.clone()); - let env_action_for_deletion = EnvironmentAction::Environment(environment_delete.clone()); - - match environment.deploy_environment(Kind::Do, &context, &env_action, logger.clone()) { - TransactionResult::Ok => assert!(true), - TransactionResult::Rollback(_) => assert!(false), - TransactionResult::UnrecoverableError(_, _) => assert!(false), - }; - // TO CHECK: DATABASE SHOULDN'T BE RESTARTED AFTER A REDEPLOY - let database_name = format!("postgresql-{}-0", &environment_check.databases[0].name); - match is_pod_restarted_env( - context.clone(), - ProviderKind::Do, - environment_check.clone(), - database_name.as_str(), - secrets.clone(), - ) { - (true, _) => assert!(true), - (false, _) => assert!(false), - } - match environment_never_up.deploy_environment(Kind::Do, &context, &env_action_fail_ok, logger.clone()) { - TransactionResult::Ok => assert!(false), - TransactionResult::Rollback(_) => assert!(true), - TransactionResult::UnrecoverableError(_, _) => assert!(false), - }; - // TO CHECK: DATABASE SHOULDN'T BE RESTARTED AFTER A REDEPLOY EVEN IF FAIL - match is_pod_restarted_env( - context.clone(), - ProviderKind::Do, - environment_check.clone(), - database_name.as_str(), - secrets.clone(), - ) { - (true, _) => assert!(true), - (false, _) => assert!(false), - } - - match environment_delete.delete_environment(Kind::Do, &context_for_deletion, &env_action_for_deletion, logger) { - TransactionResult::Ok => assert!(true), - TransactionResult::Rollback(_) => assert!(false), - TransactionResult::UnrecoverableError(_, _) => assert!(false), - }; - - // delete images created during test from registries - if let Err(e) = clean_environments(&context, vec![environment, environment_delete], secrets, DO_TEST_REGION) { - warn!("cannot clean environments, error: {:?}", e); - } - - return test_name.to_string(); - }) -} - // Ensure a full environment can run correctly #[cfg(feature = "test-do-self-hosted")] #[named] @@ -418,7 +298,7 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { id: generate_id(), name: database_db_name.clone(), version: "11.8.0".to_string(), - fqdn_id: "postgresql-".to_string() + generate_id().as_str(), + fqdn_id: database_host.clone(), fqdn: database_host.clone(), port: database_port, username: database_username.clone(), diff --git a/tests/scaleway/scw_databases.rs b/tests/scaleway/scw_databases.rs index 6caed734..baa46b4d 100644 --- a/tests/scaleway/scw_databases.rs +++ b/tests/scaleway/scw_databases.rs @@ -177,128 +177,6 @@ fn deploy_an_environment_with_db_and_pause_it() { }) } -// this test ensure containers databases are never restarted, even in failover environment case -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn postgresql_failover_dev_environment_with_all_options() { - let test_name = function_name!(); - engine_run_test(|| { - init(); - - let span = span!(Level::INFO, "test", name = test_name); - let _enter = span.enter(); - - let logger = logger(); - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - let context_for_deletion = context.clone_not_same_execution_id(); - let test_domain = secrets - .clone() - .DEFAULT_TEST_DOMAIN - .expect("DEFAULT_TEST_DOMAIN is not set in secrets"); - - let environment = test_utilities::common::environnement_2_app_2_routers_1_psql( - &context, - test_domain.as_str(), - SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, - SCW_SELF_HOSTED_DATABASE_DISK_TYPE, - Kind::Scw, - ); - let environment_check = environment.clone(); - let mut environment_never_up = environment.clone(); - // error in ports, these applications will never be up !! - environment_never_up.applications = environment_never_up - .applications - .into_iter() - .map(|mut app| { - app.ports = vec![Port { - id: "zdf7d6aad".to_string(), - long_id: Default::default(), - port: 4789, - public_port: Some(443), - name: None, - publicly_accessible: true, - protocol: Protocol::HTTP, - }]; - app - }) - .collect::>(); - let mut environment_delete = test_utilities::common::environnement_2_app_2_routers_1_psql( - &context_for_deletion, - test_domain.as_str(), - SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, - SCW_SELF_HOSTED_DATABASE_DISK_TYPE, - Kind::Scw, - ); - - environment_delete.action = Action::Delete; - - let env_action = EnvironmentAction::Environment(environment.clone()); - let env_action_fail_ok = EnvironmentAction::EnvironmentWithFailover(environment_never_up, environment.clone()); - let env_action_for_deletion = EnvironmentAction::Environment(environment_delete.clone()); - - match environment.deploy_environment(Kind::Scw, &context, &env_action, logger.clone()) { - TransactionResult::Ok => assert!(true), - TransactionResult::Rollback(_) => assert!(false), - TransactionResult::UnrecoverableError(_, _) => assert!(false), - }; - // TO CHECK: DATABASE SHOULDN'T BE RESTARTED AFTER A REDEPLOY - let database_name = format!("postgresql-{}-0", &environment_check.databases[0].name); - match is_pod_restarted_env( - context.clone(), - ProviderKind::Scw, - environment_check.clone(), - database_name.as_str(), - secrets.clone(), - ) { - (true, _) => assert!(true), - (false, _) => assert!(false), - } - match environment_check.deploy_environment(Kind::Scw, &context, &env_action_fail_ok, logger.clone()) { - TransactionResult::Ok => assert!(false), - TransactionResult::Rollback(_) => assert!(true), - TransactionResult::UnrecoverableError(_, _) => assert!(false), - }; - // TO CHECK: DATABASE SHOULDN'T BE RESTARTED AFTER A REDEPLOY EVEN IF FAIL - match is_pod_restarted_env( - context.clone(), - ProviderKind::Scw, - environment_check.clone(), - database_name.as_str(), - secrets.clone(), - ) { - (true, _) => assert!(true), - (false, _) => assert!(false), - } - - match environment_delete.delete_environment(Kind::Scw, &context_for_deletion, &env_action_for_deletion, logger) - { - TransactionResult::Ok => assert!(true), - TransactionResult::Rollback(_) => assert!(false), - TransactionResult::UnrecoverableError(_, _) => assert!(false), - }; - - // delete images created during test from registries - if let Err(e) = clean_environments(&context, vec![environment, environment_delete], secrets, SCW_TEST_ZONE) { - warn!("cannot clean environments, error: {:?}", e); - } - - return test_name.to_string(); - }) -} - // Ensure a full environment can run correctly #[cfg(feature = "test-scw-self-hosted")] #[named] @@ -431,7 +309,7 @@ fn postgresql_deploy_a_working_environment_and_redeploy() { id: generate_id(), name: database_db_name.clone(), version: "11.8.0".to_string(), - fqdn_id: "postgresql-".to_string() + generate_id().as_str(), + fqdn_id: database_host.clone(), fqdn: database_host.clone(), port: database_port, username: database_username.clone(),