From 6167abe032e3d83e030876a47e3f57da1f082484 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Mon, 29 Nov 2021 10:33:51 +0100 Subject: [PATCH] feat: use subdomains for dns and tls (#500) Ticket: DEV-996 --- Cargo.toml | 13 +- .../chart_values/external-dns.j2.yaml | 2 +- .../chart_values/external-dns.j2.yaml | 2 +- .../chart_values/external-dns.j2.yaml | 2 +- src/cloud_provider/aws/kubernetes/mod.rs | 27 ++- .../digitalocean/kubernetes/mod.rs | 20 ++- src/cloud_provider/scaleway/kubernetes/mod.rs | 17 +- src/cloud_provider/scaleway/router.rs | 4 +- .../scaleway_container_registry.rs | 2 +- src/dns_provider/cloudflare.rs | 12 +- src/dns_provider/mod.rs | 7 +- src/models.rs | 169 ++++++++++++++++++ test_utilities/src/aws.rs | 4 +- test_utilities/src/cloudflare.rs | 15 +- test_utilities/src/common.rs | 66 +++++-- test_utilities/src/digitalocean.rs | 4 +- test_utilities/src/scaleway.rs | 4 +- tests/aws/aws_environment.rs | 2 +- tests/aws/aws_kubernetes.rs | 7 +- tests/aws/aws_whole_enchilada.rs | 50 ++++++ tests/aws/mod.rs | 1 + tests/digitalocean/do_kubernetes.rs | 6 +- ...do_utility_kubernetes_doks_test_cluster.rs | 6 +- tests/digitalocean/do_whole_enchilada.rs | 50 ++++++ tests/digitalocean/mod.rs | 1 + tests/scaleway/mod.rs | 1 + tests/scaleway/scw_kubernetes.rs | 7 +- tests/scaleway/scw_object_storage.rs | 1 - ...utility_kubernetes_kapsule_test_cluster.rs | 6 +- tests/scaleway/scw_whole_enchilada.rs | 50 ++++++ 30 files changed, 485 insertions(+), 73 deletions(-) create mode 100644 tests/aws/aws_whole_enchilada.rs create mode 100644 tests/digitalocean/do_whole_enchilada.rs create mode 100644 tests/scaleway/scw_whole_enchilada.rs diff --git a/Cargo.toml b/Cargo.toml index 8216f249..99ad0b52 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,7 @@ maplit = "1.0.2" [features] default = [] -test-all = ["test-all-self-hosted", "test-all-infra", "test-all-managed-services"] +test-all = ["test-all-self-hosted", "test-all-infra", "test-all-managed-services", "test-all-whole-enchilada"] # functionnal tests by type test-aws-self-hosted = [] @@ -94,7 +94,12 @@ test-do-infra = [] test-scw-infra = [] test-all-infra = ["test-aws-infra", "test-do-infra", "test-scw-infra"] +test-aws-whole-enchilada = [] +test-do-whole-enchilada = [] +test-scw-whole-enchilada = [] +test-all-whole-enchilada = ["test-aws-whole-enchilada", "test-do-whole-enchilada", "test-scw-whole-enchilada"] + # functionnal tests by provider -test-aws-all = ["test-aws-infra", "test-aws-managed-services", "test-aws-self-hosted"] -test-do-all = ["test-do-infra", "test-do-managed-services", "test-do-self-hosted"] -test-scw-all = ["test-scw-infra", "test-scw-managed-services", "test-scw-self-hosted"] +test-aws-all = ["test-aws-infra", "test-aws-managed-services", "test-aws-self-hosted", "test-aws-whole-enchilada"] +test-do-all = ["test-do-infra", "test-do-managed-services", "test-do-self-hosted", "test-do-whole-enchilada"] +test-scw-all = ["test-scw-infra", "test-scw-managed-services", "test-scw-self-hosted", "test-scw-whole-enchilada"] diff --git a/lib/aws/bootstrap/chart_values/external-dns.j2.yaml b/lib/aws/bootstrap/chart_values/external-dns.j2.yaml index b928b1be..231dda78 100644 --- a/lib/aws/bootstrap/chart_values/external-dns.j2.yaml +++ b/lib/aws/bootstrap/chart_values/external-dns.j2.yaml @@ -7,7 +7,7 @@ cloudflare: proxied: {{ cloudflare_proxied|default(value="false") }} {% endif %} -domainFilters: {{ managed_dns_domains_helm_format }} +domainFilters: {{ managed_dns_domains_root_helm_format }} triggerLoopOnEvent: true policy: sync txtOwnerId: "{{ kubernetes_cluster_id }}" diff --git a/lib/digitalocean/bootstrap/chart_values/external-dns.j2.yaml b/lib/digitalocean/bootstrap/chart_values/external-dns.j2.yaml index 6ca466e3..a847781c 100644 --- a/lib/digitalocean/bootstrap/chart_values/external-dns.j2.yaml +++ b/lib/digitalocean/bootstrap/chart_values/external-dns.j2.yaml @@ -7,7 +7,7 @@ cloudflare: proxied: {{ cloudflare_proxied|default(value="false") }} {% endif %} -domainFilters: {{ managed_dns_domains_helm_format }} +domainFilters: {{ managed_dns_domains_root_helm_format }} triggerLoopOnEvent: true policy: sync txtOwnerId: "{{ doks_cluster_id }}" diff --git a/lib/scaleway/bootstrap/chart_values/external-dns.j2.yaml b/lib/scaleway/bootstrap/chart_values/external-dns.j2.yaml index ead615ff..b8d22942 100644 --- a/lib/scaleway/bootstrap/chart_values/external-dns.j2.yaml +++ b/lib/scaleway/bootstrap/chart_values/external-dns.j2.yaml @@ -7,7 +7,7 @@ cloudflare: proxied: {{ cloudflare_proxied|default(value="false") }} {% endif %} -domainFilters: {{ managed_dns_domains_helm_format }} +domainFilters: {{ managed_dns_domains_root_helm_format }} triggerLoopOnEvent: true policy: sync txtOwnerId: "{{ kubernetes_cluster_id }}" diff --git a/src/cloud_provider/aws/kubernetes/mod.rs b/src/cloud_provider/aws/kubernetes/mod.rs index 30f4b21a..213aea29 100644 --- a/src/cloud_provider/aws/kubernetes/mod.rs +++ b/src/cloud_provider/aws/kubernetes/mod.rs @@ -37,7 +37,7 @@ use crate::error::EngineErrorCause::Internal; use crate::error::{ cast_simple_error_to_engine_error, EngineError, EngineErrorCause, EngineErrorScope, SimpleError, SimpleErrorKind, }; -use crate::models::{Action, Context, Features, Listen, Listener, Listeners, ListenersHelper}; +use crate::models::{Action, Context, Features, Listen, Listener, Listeners, ListenersHelper, ToHelmString}; use crate::object_storage::s3::S3; use crate::object_storage::ObjectStorage; use crate::string::terraform_list_format; @@ -287,11 +287,6 @@ impl<'a> EKS<'a> { let elasticache_cidr_subnet = self.options.elasticache_cidr_subnet.clone(); let elasticsearch_cidr_subnet = self.options.elasticsearch_cidr_subnet.clone(); - let managed_dns_list = vec![self.dns_provider.name()]; - let managed_dns_domains_helm_format = vec![format!("{}", self.dns_provider.domain())]; - let managed_dns_domains_terraform_format = terraform_list_format(vec![self.dns_provider.domain().to_string()]); - let managed_dns_resolvers_terraform_format = self.managed_dns_resolvers_terraform_format(); - // Qovery context.insert("organization_id", self.cloud_provider.organization_id()); context.insert("qovery_api_url", &qovery_api_url); @@ -325,14 +320,28 @@ impl<'a> EKS<'a> { ); // DNS configuration + let managed_dns_list = vec![self.dns_provider.name()]; + let managed_dns_domains_helm_format = vec![self.dns_provider.domain().to_string()]; + let managed_dns_domains_root_helm_format = vec![self.dns_provider.domain().root_domain().to_string()]; + let managed_dns_domains_terraform_format = terraform_list_format(vec![self.dns_provider.domain().to_string()]); + let managed_dns_domains_root_terraform_format = + terraform_list_format(vec![self.dns_provider.domain().root_domain().to_string()]); + let managed_dns_resolvers_terraform_format = self.managed_dns_resolvers_terraform_format(); + context.insert("managed_dns", &managed_dns_list); context.insert("managed_dns_domains_helm_format", &managed_dns_domains_helm_format); - + context.insert( + "managed_dns_domains_root_helm_format", + &managed_dns_domains_root_helm_format, + ); context.insert( "managed_dns_domains_terraform_format", &managed_dns_domains_terraform_format, ); - + context.insert( + "managed_dns_domains_root_terraform_format", + &managed_dns_domains_root_terraform_format, + ); context.insert( "managed_dns_resolvers_terraform_format", &managed_dns_resolvers_terraform_format, @@ -630,7 +639,7 @@ impl<'a> EKS<'a> { ff_log_history_enabled: self.context.is_feature_enabled(&Features::LogsHistory), ff_metrics_history_enabled: self.context.is_feature_enabled(&Features::MetricsHistory), managed_dns_name: self.dns_provider.domain().to_string(), - managed_dns_helm_format: self.dns_provider.domain_helm_format(), + managed_dns_helm_format: self.dns_provider.domain().to_helm_format_string(), managed_dns_resolvers_terraform_format: self.managed_dns_resolvers_terraform_format(), external_dns_provider: self.dns_provider.provider_name().to_string(), dns_email_report: self.options.tls_email_report.clone(), diff --git a/src/cloud_provider/digitalocean/kubernetes/mod.rs b/src/cloud_provider/digitalocean/kubernetes/mod.rs index 34a7ebf9..ded36295 100644 --- a/src/cloud_provider/digitalocean/kubernetes/mod.rs +++ b/src/cloud_provider/digitalocean/kubernetes/mod.rs @@ -35,7 +35,8 @@ use crate::dns_provider::DnsProvider; use crate::error::EngineErrorCause::Internal; use crate::error::{cast_simple_error_to_engine_error, EngineError, EngineErrorCause, EngineErrorScope, SimpleError}; use crate::models::{ - Action, Context, Features, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope, + Action, Context, Features, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, + ProgressScope, ToHelmString, }; use crate::object_storage::spaces::{BucketDeleteStrategy, Spaces}; use crate::object_storage::ObjectStorage; @@ -221,18 +222,29 @@ impl<'a> DOKS<'a> { // DNS let managed_dns_list = vec![self.dns_provider.name()]; - let managed_dns_domains_helm_format = vec![format!("\"{}\"", self.dns_provider.domain())]; + let managed_dns_domains_helm_format = vec![self.dns_provider.domain().to_string()]; + let managed_dns_domains_root_helm_format = vec![self.dns_provider.domain().root_domain().to_string()]; let managed_dns_domains_terraform_format = terraform_list_format(vec![self.dns_provider.domain().to_string()]); + let managed_dns_domains_root_terraform_format = + terraform_list_format(vec![self.dns_provider.domain().root_domain().to_string()]); let managed_dns_resolvers_terraform_format = self.managed_dns_resolvers_terraform_format(); context.insert("managed_dns", &managed_dns_list); context.insert("do_loadbalancer_hostname", &self.do_loadbalancer_hostname()); - context.insert("managed_dns_domain", self.dns_provider.domain()); + context.insert("managed_dns_domain", self.dns_provider.domain().to_string().as_str()); context.insert("managed_dns_domains_helm_format", &managed_dns_domains_helm_format); + context.insert( + "managed_dns_domains_root_helm_format", + &managed_dns_domains_root_helm_format, + ); context.insert( "managed_dns_domains_terraform_format", &managed_dns_domains_terraform_format, ); + context.insert( + "managed_dns_domains_root_terraform_format", + &managed_dns_domains_root_terraform_format, + ); context.insert( "managed_dns_resolvers_terraform_format", &managed_dns_resolvers_terraform_format, @@ -630,7 +642,7 @@ impl<'a> DOKS<'a> { ff_log_history_enabled: self.context.is_feature_enabled(&Features::LogsHistory), ff_metrics_history_enabled: self.context.is_feature_enabled(&Features::MetricsHistory), managed_dns_name: self.dns_provider.domain().to_string(), - managed_dns_helm_format: self.dns_provider.domain_helm_format(), + managed_dns_helm_format: self.dns_provider.domain().to_helm_format_string(), managed_dns_resolvers_terraform_format: self.managed_dns_resolvers_terraform_format(), external_dns_provider: self.dns_provider.provider_name().to_string(), dns_email_report: self.options.tls_email_report.clone(), diff --git a/src/cloud_provider/scaleway/kubernetes/mod.rs b/src/cloud_provider/scaleway/kubernetes/mod.rs index 2b77af87..f69cef69 100644 --- a/src/cloud_provider/scaleway/kubernetes/mod.rs +++ b/src/cloud_provider/scaleway/kubernetes/mod.rs @@ -23,7 +23,7 @@ use crate::error::EngineErrorCause::Internal; use crate::error::{ cast_simple_error_to_engine_error, EngineError, EngineErrorCause, EngineErrorScope, SimpleError, SimpleErrorKind, }; -use crate::models::{Action, Context, Features, Listen, Listener, Listeners, ListenersHelper}; +use crate::models::{Action, Context, Features, Listen, Listener, Listeners, ListenersHelper, ToHelmString}; use crate::object_storage::scaleway_object_storage::{BucketDeleteStrategy, ScalewayOS}; use crate::object_storage::ObjectStorage; use crate::string::terraform_list_format; @@ -201,15 +201,26 @@ impl<'a> Kapsule<'a> { // DNS let managed_dns_list = vec![self.dns_provider.name()]; let managed_dns_domains_helm_format = vec![self.dns_provider.domain().to_string()]; + let managed_dns_domains_root_helm_format = vec![self.dns_provider.domain().root_domain().to_string()]; let managed_dns_domains_terraform_format = terraform_list_format(vec![self.dns_provider.domain().to_string()]); + let managed_dns_domains_root_terraform_format = + terraform_list_format(vec![self.dns_provider.domain().root_domain().to_string()]); let managed_dns_resolvers_terraform_format = self.managed_dns_resolvers_terraform_format(); context.insert("managed_dns", &managed_dns_list); context.insert("managed_dns_domains_helm_format", &managed_dns_domains_helm_format); + context.insert( + "managed_dns_domains_root_helm_format", + &managed_dns_domains_root_helm_format, + ); context.insert( "managed_dns_domains_terraform_format", &managed_dns_domains_terraform_format, ); + context.insert( + "managed_dns_domains_root_terraform_format", + &managed_dns_domains_root_terraform_format, + ); context.insert( "managed_dns_resolvers_terraform_format", &managed_dns_resolvers_terraform_format, @@ -498,8 +509,8 @@ impl<'a> Kapsule<'a> { self.options.qovery_engine_location.clone(), self.context.is_feature_enabled(&Features::LogsHistory), self.context.is_feature_enabled(&Features::MetricsHistory), - self.dns_provider.domain().to_string(), - self.dns_provider.domain_helm_format(), + self.dns_provider.domain().root_domain().to_string(), + self.dns_provider.domain().to_helm_format_string(), self.managed_dns_resolvers_terraform_format(), self.dns_provider.provider_name().to_string(), self.options.tls_email_report.clone(), diff --git a/src/cloud_provider/scaleway/router.rs b/src/cloud_provider/scaleway/router.rs index 9e917f43..97a33bf6 100644 --- a/src/cloud_provider/scaleway/router.rs +++ b/src/cloud_provider/scaleway/router.rs @@ -158,9 +158,9 @@ impl Service for Router { .collect::>(); let router_default_domain_hash = crate::crypto::to_sha1_truncate_16(self.default_domain.as_str()); + let tls_domain = kubernetes.dns_provider().domain().wildcarded(); - let tls_domain = format!("*.{}", kubernetes.dns_provider().domain()); - context.insert("router_tls_domain", tls_domain.as_str()); + context.insert("router_tls_domain", tls_domain.to_string().as_str()); context.insert("router_default_domain", self.default_domain.as_str()); context.insert("router_default_domain_hash", router_default_domain_hash.as_str()); context.insert("custom_domains", &custom_domain_data_templates); diff --git a/src/container_registry/scaleway_container_registry.rs b/src/container_registry/scaleway_container_registry.rs index 8480afc3..65c2180c 100644 --- a/src/container_registry/scaleway_container_registry.rs +++ b/src/container_registry/scaleway_container_registry.rs @@ -303,7 +303,7 @@ impl ContainerRegistry for ScalewayCR { } fn kind(&self) -> Kind { - Kind::Docr + Kind::ScalewayCr } fn id(&self) -> &str { diff --git a/src/dns_provider/cloudflare.rs b/src/dns_provider/cloudflare.rs index f627a90e..e93ba3cf 100644 --- a/src/dns_provider/cloudflare.rs +++ b/src/dns_provider/cloudflare.rs @@ -2,13 +2,13 @@ use std::net::Ipv4Addr; use crate::dns_provider::{DnsProvider, Kind}; use crate::error::{EngineError, EngineErrorCause}; -use crate::models::Context; +use crate::models::{Context, Domain}; pub struct Cloudflare { context: Context, id: String, name: String, - domain: String, + domain: Domain, cloudflare_api_token: String, cloudflare_email: String, } @@ -18,7 +18,7 @@ impl Cloudflare { context: Context, id: &str, name: &str, - domain: &str, + domain: Domain, cloudflare_api_token: &str, cloudflare_email: &str, ) -> Self { @@ -26,7 +26,7 @@ impl Cloudflare { context, id: id.to_string(), name: name.to_string(), - domain: domain.to_string(), + domain, cloudflare_api_token: cloudflare_api_token.to_string(), cloudflare_email: cloudflare_email.to_string(), } @@ -62,8 +62,8 @@ impl DnsProvider for Cloudflare { &self.cloudflare_api_token } - fn domain(&self) -> &str { - self.domain.as_str() + fn domain(&self) -> &Domain { + &self.domain } fn resolvers(&self) -> Vec { diff --git a/src/dns_provider/mod.rs b/src/dns_provider/mod.rs index 46b300e8..859199d2 100644 --- a/src/dns_provider/mod.rs +++ b/src/dns_provider/mod.rs @@ -3,7 +3,7 @@ use std::net::Ipv4Addr; use serde::{Deserialize, Serialize}; use crate::error::{EngineError, EngineErrorCause, EngineErrorScope}; -use crate::models::Context; +use crate::models::{Context, Domain}; pub mod cloudflare; @@ -18,10 +18,7 @@ pub trait DnsProvider { } fn account(&self) -> &str; fn token(&self) -> &str; - fn domain(&self) -> &str; - fn domain_helm_format(&self) -> String { - format!("{{{}}}", self.domain()) - } + fn domain(&self) -> &Domain; fn resolvers(&self) -> Vec; fn is_valid(&self) -> Result<(), EngineError>; fn engine_error_scope(&self) -> EngineErrorScope { diff --git a/src/models.rs b/src/models.rs index 6e0ac7d0..83de2385 100644 --- a/src/models.rs +++ b/src/models.rs @@ -16,8 +16,10 @@ use crate::cloud_provider::utilities::VersionsNumber; use crate::cloud_provider::CloudProvider; use crate::cloud_provider::Kind as CPKind; use crate::git::Credentials; +use itertools::Itertools; use std::collections::BTreeMap; use std::fmt::{Display, Formatter}; +use std::net::Ipv4Addr; use std::str::FromStr; use std::sync::Arc; @@ -1165,3 +1167,170 @@ impl Metadata { /// Represent a String path instead of passing a PathBuf struct pub type StringPath = String; + +pub trait ToTerraformString { + fn to_terraform_format_string(&self) -> String; +} + +pub trait ToHelmString { + fn to_helm_format_string(&self) -> String; +} + +/// Represents a domain, just plain domain, no protocol. +/// eq. `test.com`, `sub.test.com` +#[derive(Clone)] +pub struct Domain { + raw: String, + root_domain: String, +} + +impl Domain { + pub fn new(raw: String) -> Self { + // TODO(benjaminch): This is very basic solution which doesn't take into account + // some edge cases such as: "test.co.uk" domains + let sep: &str = "."; + let items: Vec = raw.split(sep).map(|e| e.to_string()).collect(); + let items_count = raw.matches(sep).count() + 1; + let top_domain: String = match items_count > 2 { + true => items.iter().skip(items_count - 2).join("."), + false => items.iter().join("."), + }; + + Domain { + root_domain: top_domain, + raw, + } + } + + pub fn new_with_subdomain(raw: String, sub_domain: String) -> Self { + Domain::new(format!("{}.{}", sub_domain, raw)) + } + + pub fn with_sub_domain(&self, sub_domain: String) -> Domain { + Domain::new(format!("{}.{}", sub_domain, self.raw)) + } + + pub fn root_domain(&self) -> Domain { + Domain::new(self.root_domain.to_string()) + } + + pub fn wildcarded(&self) -> Domain { + if self.is_wildcarded() { + return self.clone(); + } + + match self.raw.is_empty() { + false => Domain::new_with_subdomain(self.raw.to_string(), "*".to_string()), + true => Domain::new("*".to_string()), + } + } + + fn is_wildcarded(&self) -> bool { + self.raw.starts_with("*") + } +} + +impl Display for Domain { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.raw.as_str()) + } +} + +impl ToTerraformString for Domain { + fn to_terraform_format_string(&self) -> String { + format!("{{{}}}", self.raw) + } +} + +impl ToHelmString for Domain { + fn to_helm_format_string(&self) -> String { + format!("{{{}}}", self.raw) + } +} + +impl ToTerraformString for Ipv4Addr { + fn to_terraform_format_string(&self) -> String { + format!("{{{}}}", self.to_string()) + } +} + +#[cfg(test)] +mod tests { + use crate::models::Domain; + + #[test] + fn test_domain_new() { + struct TestCase<'a> { + input: String, + expected_root_domain_output: String, + expected_wildcarded_output: String, + description: &'a str, + } + + // setup: + let test_cases: Vec = vec![ + TestCase { + input: "".to_string(), + expected_root_domain_output: "".to_string(), + expected_wildcarded_output: "*".to_string(), + description: "empty raw domain input", + }, + TestCase { + input: "*".to_string(), + expected_root_domain_output: "*".to_string(), + expected_wildcarded_output: "*".to_string(), + description: "wildcard domain input", + }, + TestCase { + input: "*.test.com".to_string(), + expected_root_domain_output: "test.com".to_string(), + expected_wildcarded_output: "*.test.com".to_string(), + description: "wildcarded domain input", + }, + TestCase { + input: "test.co.uk".to_string(), + expected_root_domain_output: "co.uk".to_string(), // TODO(benjamin) => Should be test.co.uk in the future + expected_wildcarded_output: "*.co.uk".to_string(), + description: "broken edge case domain with special tld input", + }, + TestCase { + input: "test".to_string(), + expected_root_domain_output: "test".to_string(), + expected_wildcarded_output: "*.test".to_string(), + description: "domain without tld input", + }, + TestCase { + input: "test.com".to_string(), + expected_root_domain_output: "test.com".to_string(), + expected_wildcarded_output: "*.test.com".to_string(), + description: "simple top domain input", + }, + TestCase { + input: "sub.test.com".to_string(), + expected_root_domain_output: "test.com".to_string(), + expected_wildcarded_output: "*.sub.test.com".to_string(), + description: "simple sub domain input", + }, + TestCase { + input: "yetanother.sub.test.com".to_string(), + expected_root_domain_output: "test.com".to_string(), + expected_wildcarded_output: "*.yetanother.sub.test.com".to_string(), + description: "simple sub domain input", + }, + ]; + + for tc in test_cases { + // execute: + let result = Domain::new(tc.input.clone()); + + // verify: + assert_eq!( + tc.expected_root_domain_output, + result.root_domain().to_string(), + "case {} : '{}'", + tc.description, + tc.input + ); + } + } +} diff --git a/test_utilities/src/aws.rs b/test_utilities/src/aws.rs index e48f0eb9..9627d7f5 100644 --- a/test_utilities/src/aws.rs +++ b/test_utilities/src/aws.rs @@ -13,7 +13,7 @@ use qovery_engine::models::Context; use tracing::error; use crate::cloudflare::dns_provider_cloudflare; -use crate::common::Cluster; +use crate::common::{Cluster, ClusterDomain}; use crate::utilities::{build_platform_local_docker, FuncTestsSecrets}; pub const AWS_QOVERY_ORGANIZATION_ID: &str = "u8nb94c7fwxzr2jt"; @@ -68,7 +68,7 @@ impl Cluster for AWS { // use AWS let cloud_provider = AWS::cloud_provider(context); - let dns_provider = Box::new(dns_provider_cloudflare(context)); + let dns_provider = Box::new(dns_provider_cloudflare(context, ClusterDomain::Default)); Engine::new( context.clone(), diff --git a/test_utilities/src/cloudflare.rs b/test_utilities/src/cloudflare.rs index 564896b2..16414e93 100644 --- a/test_utilities/src/cloudflare.rs +++ b/test_utilities/src/cloudflare.rs @@ -1,15 +1,20 @@ +use crate::common::ClusterDomain; use crate::utilities::FuncTestsSecrets; use qovery_engine::dns_provider::cloudflare::Cloudflare; -use qovery_engine::models::Context; +use qovery_engine::models::{Context, Domain}; -pub fn dns_provider_cloudflare(context: &Context) -> Cloudflare { +pub fn dns_provider_cloudflare(context: &Context, domain: ClusterDomain) -> Cloudflare { let secrets = FuncTestsSecrets::new(); + let domain = Domain::new(match domain { + ClusterDomain::Custom(domain) => domain, + ClusterDomain::Default => secrets.CLOUDFLARE_DOMAIN.expect("CLOUDFLARE_DOMAIN is not set"), + }); Cloudflare::new( context.clone(), "qoverytestdnsclo", "Qovery Test Cloudflare", - secrets.CLOUDFLARE_DOMAIN.unwrap().as_str(), - secrets.CLOUDFLARE_TOKEN.unwrap().as_str(), // Cloudflare name: Qovery test - secrets.CLOUDFLARE_ID.unwrap().as_str(), + domain, + secrets.CLOUDFLARE_TOKEN.expect("CLOUDFLARE_TOKEN is not set").as_str(), // Cloudflare name: Qovery test + secrets.CLOUDFLARE_ID.expect("CLOUDFLARE_ID is not set").as_str(), ) } diff --git a/test_utilities/src/common.rs b/test_utilities/src/common.rs index 3c354b9a..f37ddb8e 100644 --- a/test_utilities/src/common.rs +++ b/test_utilities/src/common.rs @@ -16,8 +16,8 @@ use crate::cloudflare::dns_provider_cloudflare; use crate::digitalocean::{DO_KUBERNETES_VERSION, DO_KUBE_TEST_CLUSTER_ID, DO_KUBE_TEST_CLUSTER_NAME}; use crate::scaleway::{SCW_KUBERNETES_VERSION, SCW_KUBE_TEST_CLUSTER_ID, SCW_KUBE_TEST_CLUSTER_NAME}; use crate::utilities::{ - context, db_disk_type, db_infos, db_instance_type, generate_cluster_id, generate_id, generate_password, get_pvc, - get_svc, get_svc_name, init, FuncTestsSecrets, + db_disk_type, db_infos, db_instance_type, generate_cluster_id, generate_id, generate_password, get_pvc, get_svc, + get_svc_name, init, FuncTestsSecrets, }; use base64; use qovery_engine::cloud_provider::aws::kubernetes::{VpcQoveryNetworkMode, EKS}; @@ -39,6 +39,11 @@ use std::collections::BTreeMap; use std::str::FromStr; use tracing::{span, Level}; +pub enum ClusterDomain { + Default, + Custom(String), +} + pub trait Cluster { fn docker_cr_engine(context: &Context) -> Engine; fn cloud_provider(context: &Context) -> Box; @@ -82,7 +87,7 @@ impl Infrastructure for Environment { let session = engine.session().unwrap(); let mut tx = session.transaction(); - let dns_provider = dns_provider_cloudflare(context); + let dns_provider = dns_provider_cloudflare(context, ClusterDomain::Default); let cp: Box; cp = match provider_kind { Kind::Aws => AWS::cloud_provider(context), @@ -119,7 +124,7 @@ impl Infrastructure for Environment { let session = engine.session().unwrap(); let mut tx = session.transaction(); - let dns_provider = dns_provider_cloudflare(context); + let dns_provider = dns_provider_cloudflare(context, ClusterDomain::Default); let cp: Box; cp = match provider_kind { Kind::Aws => AWS::cloud_provider(context), @@ -149,7 +154,7 @@ impl Infrastructure for Environment { let session = engine.session().unwrap(); let mut tx = session.transaction(); - let dns_provider = dns_provider_cloudflare(context); + let dns_provider = dns_provider_cloudflare(context, ClusterDomain::Default); let cp: Box; cp = match provider_kind { Kind::Aws => AWS::cloud_provider(context), @@ -444,6 +449,11 @@ pub fn environment_3_apps_3_routers_3_databases( pub fn working_minimal_environment(context: &Context, organization_id: &str, test_domain: &str) -> Environment { let suffix = generate_id(); + let application_id = generate_id(); + let application_name = format!("{}-{}", "simple-app".to_string(), &suffix); + let router_id = generate_id(); + let router_name = "main".to_string(); + let application_domain = format!("{}.{}", application_id, test_domain); Environment { execution_id: context.execution_id().to_string(), id: generate_id(), @@ -452,8 +462,8 @@ pub fn working_minimal_environment(context: &Context, organization_id: &str, tes organization_id: organization_id.to_string(), action: Action::Create, applications: vec![Application { - id: generate_id(), - name: format!("{}-{}", "simple-app".to_string(), &suffix), + id: application_id, + name: application_name, git_url: "https://github.com/Qovery/engine-testing.git".to_string(), commit_id: "fc575a2f3be0b9100492c8a463bf18134a8698a5".to_string(), dockerfile_path: Some("Dockerfile".to_string()), @@ -476,10 +486,10 @@ pub fn working_minimal_environment(context: &Context, organization_id: &str, tes start_timeout_in_seconds: 60, }], routers: vec![Router { - id: generate_id(), - name: "main".to_string(), + id: router_id, + name: router_name, action: Action::Create, - default_domain: format!("{}.{}", generate_id(), test_domain), + default_domain: application_domain, public_port: 443, custom_domains: vec![], routes: vec![Route { @@ -1135,19 +1145,21 @@ pub fn get_cluster_test_kubernetes<'a>( pub fn cluster_test( test_name: &str, provider_kind: Kind, + context: Context, localisation: &str, secrets: FuncTestsSecrets, test_type: ClusterTestType, major_boot_version: u8, minor_boot_version: u8, + cluster_domain: ClusterDomain, vpc_network_mode: Option, + environment_to_deploy: Option<&EnvironmentAction>, ) -> String { init(); let span = span!(Level::INFO, "test", name = test_name); let _enter = span.enter(); - let context = context(); let cluster_id = generate_cluster_id(localisation.clone()); let cluster_name = generate_cluster_id(localisation.clone()); let boot_version = format!("{}.{}", major_boot_version, minor_boot_version.clone()); @@ -1158,7 +1170,7 @@ pub fn cluster_test( Kind::Do => engine = DO::docker_cr_engine(&context), Kind::Scw => engine = Scaleway::docker_cr_engine(&context), }; - let dns_provider = dns_provider_cloudflare(&context); + let dns_provider = dns_provider_cloudflare(&context, cluster_domain); let mut deploy_tx = engine.session().unwrap().transaction(); let mut delete_tx = engine.session().unwrap().transaction(); @@ -1191,6 +1203,21 @@ pub fn cluster_test( TransactionResult::UnrecoverableError(_, _) => assert!(false), }; + // Deploy env if any + if let Some(env) = environment_to_deploy { + let mut deploy_env_tx = engine.session().unwrap().transaction(); + + // Deploy env + if let Err(err) = deploy_env_tx.deploy_environment(kubernetes.as_ref(), env) { + panic!("{:?}", err) + } + match deploy_env_tx.commit() { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + } + match test_type { ClusterTestType::Classic => {} ClusterTestType::WithPause => { @@ -1259,6 +1286,21 @@ pub fn cluster_test( } } + // Destroy env if any + if let Some(env) = environment_to_deploy { + let mut destroy_env_tx = engine.session().unwrap().transaction(); + + // Deploy env + if let Err(err) = destroy_env_tx.delete_environment(kubernetes.as_ref(), env) { + panic!("{:?}", err) + } + match destroy_env_tx.commit() { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + } + // Delete if let Err(err) = delete_tx.delete_kubernetes(kubernetes.as_ref()) { panic!("{:?}", err) diff --git a/test_utilities/src/digitalocean.rs b/test_utilities/src/digitalocean.rs index cf38c5b6..a3b17c26 100644 --- a/test_utilities/src/digitalocean.rs +++ b/test_utilities/src/digitalocean.rs @@ -11,7 +11,7 @@ use qovery_engine::error::EngineError; use qovery_engine::models::{Context, Environment}; use crate::cloudflare::dns_provider_cloudflare; -use crate::common::Cluster; +use crate::common::{Cluster, ClusterDomain}; use crate::utilities::{build_platform_local_docker, FuncTestsSecrets}; use qovery_engine::cloud_provider::digitalocean::application::Region; use qovery_engine::cloud_provider::qovery::EngineLocation; @@ -49,7 +49,7 @@ impl Cluster for DO { // use Digital Ocean let cloud_provider = DO::cloud_provider(context); - let dns_provider = Box::new(dns_provider_cloudflare(&context)); + let dns_provider = Box::new(dns_provider_cloudflare(&context, ClusterDomain::Default)); Engine::new( context.clone(), diff --git a/test_utilities/src/scaleway.rs b/test_utilities/src/scaleway.rs index 87fa101f..e11c5887 100644 --- a/test_utilities/src/scaleway.rs +++ b/test_utilities/src/scaleway.rs @@ -13,7 +13,7 @@ use qovery_engine::object_storage::scaleway_object_storage::{BucketDeleteStrateg use crate::cloudflare::dns_provider_cloudflare; use crate::utilities::{build_platform_local_docker, generate_id, FuncTestsSecrets}; -use crate::common::Cluster; +use crate::common::{Cluster, ClusterDomain}; use qovery_engine::cloud_provider::models::NodeGroups; use qovery_engine::cloud_provider::qovery::EngineLocation; use tracing::error; @@ -70,7 +70,7 @@ impl Cluster for Scaleway { // use Scaleway let cloud_provider = Scaleway::cloud_provider(context); - let dns_provider = Box::new(dns_provider_cloudflare(context)); + let dns_provider = Box::new(dns_provider_cloudflare(context, ClusterDomain::Default)); Engine::new( context.clone(), diff --git a/tests/aws/aws_environment.rs b/tests/aws/aws_environment.rs index bb709f51..955e16f4 100644 --- a/tests/aws/aws_environment.rs +++ b/tests/aws/aws_environment.rs @@ -372,8 +372,8 @@ fn deploy_a_working_environment_with_storage_on_aws_eks() { &context, AWS_QOVERY_ORGANIZATION_ID, secrets - .clone() .DEFAULT_TEST_DOMAIN + .as_ref() .expect("DEFAULT_TEST_DOMAIN is not set in secrets") .as_str(), ); diff --git a/tests/aws/aws_kubernetes.rs b/tests/aws/aws_kubernetes.rs index 62bc1ed1..4aacb244 100644 --- a/tests/aws/aws_kubernetes.rs +++ b/tests/aws/aws_kubernetes.rs @@ -1,12 +1,12 @@ extern crate test_utilities; use self::test_utilities::aws::{AWS_KUBERNETES_MAJOR_VERSION, AWS_KUBERNETES_MINOR_VERSION}; -use self::test_utilities::utilities::{engine_run_test, FuncTestsSecrets}; +use self::test_utilities::utilities::{context, engine_run_test, FuncTestsSecrets}; use ::function_name::named; use qovery_engine::cloud_provider::aws::kubernetes::VpcQoveryNetworkMode; use qovery_engine::cloud_provider::aws::kubernetes::VpcQoveryNetworkMode::{WithNatGateways, WithoutNatGateways}; use qovery_engine::cloud_provider::Kind; -use test_utilities::common::{cluster_test, ClusterTestType}; +use test_utilities::common::{cluster_test, ClusterDomain, ClusterTestType}; #[cfg(feature = "test-aws-infra")] fn create_and_destroy_eks_cluster( @@ -22,12 +22,15 @@ fn create_and_destroy_eks_cluster( cluster_test( test_name, Kind::Aws, + context(), region, secrets, test_type, major_boot_version, minor_boot_version, + ClusterDomain::Default, Option::from(vpc_network_mode), + None, ) }) } diff --git a/tests/aws/aws_whole_enchilada.rs b/tests/aws/aws_whole_enchilada.rs new file mode 100644 index 00000000..799bae15 --- /dev/null +++ b/tests/aws/aws_whole_enchilada.rs @@ -0,0 +1,50 @@ +use ::function_name::named; +use qovery_engine::cloud_provider::aws::kubernetes::VpcQoveryNetworkMode::WithNatGateways; +use qovery_engine::cloud_provider::Kind; +use qovery_engine::models::EnvironmentAction; +use test_utilities::aws::{AWS_KUBERNETES_MAJOR_VERSION, AWS_KUBERNETES_MINOR_VERSION}; +use test_utilities::common::{cluster_test, ClusterDomain, ClusterTestType}; +use test_utilities::utilities::{context, engine_run_test, generate_cluster_id, generate_id, FuncTestsSecrets}; + +#[cfg(feature = "test-aws-whole-enchilada")] +#[named] +#[test] +fn create_upgrade_and_destroy_eks_cluster_with_env_in_eu_west_3() { + let context = context(); + let secrets = FuncTestsSecrets::new(); + let region = "eu-west-3"; + let organization_id = generate_id(); + let cluster_id = generate_cluster_id(region); + let cluster_domain = format!( + "{}.{}", + cluster_id.as_str(), + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str() + ); + + let environment = test_utilities::common::working_minimal_environment( + &context, + organization_id.as_str(), + cluster_domain.as_str(), + ); + let env_action = EnvironmentAction::Environment(environment.clone()); + + engine_run_test(|| { + cluster_test( + function_name!(), + Kind::Aws, + context.clone(), + region, + secrets.clone(), + ClusterTestType::Classic, + AWS_KUBERNETES_MAJOR_VERSION, + AWS_KUBERNETES_MINOR_VERSION, + ClusterDomain::Custom(cluster_domain), + Some(WithNatGateways), + Some(&env_action), + ) + }) +} diff --git a/tests/aws/mod.rs b/tests/aws/mod.rs index 4266a38b..c4e497ac 100644 --- a/tests/aws/mod.rs +++ b/tests/aws/mod.rs @@ -1,3 +1,4 @@ mod aws_databases; mod aws_environment; mod aws_kubernetes; +mod aws_whole_enchilada; diff --git a/tests/digitalocean/do_kubernetes.rs b/tests/digitalocean/do_kubernetes.rs index c3066eff..43709f0f 100644 --- a/tests/digitalocean/do_kubernetes.rs +++ b/tests/digitalocean/do_kubernetes.rs @@ -1,7 +1,8 @@ extern crate test_utilities; +use self::test_utilities::common::ClusterDomain; use self::test_utilities::digitalocean::{DO_KUBERNETES_MAJOR_VERSION, DO_KUBERNETES_MINOR_VERSION}; -use self::test_utilities::utilities::{engine_run_test, FuncTestsSecrets}; +use self::test_utilities::utilities::{context, engine_run_test, FuncTestsSecrets}; use ::function_name::named; use qovery_engine::cloud_provider::digitalocean::application::Region; use qovery_engine::cloud_provider::Kind; @@ -20,11 +21,14 @@ fn create_and_destroy_doks_cluster( cluster_test( test_name, Kind::Do, + context(), region.as_str(), secrets, test_type, major_boot_version, minor_boot_version, + ClusterDomain::Default, + None, None, ) }) diff --git a/tests/digitalocean/do_utility_kubernetes_doks_test_cluster.rs b/tests/digitalocean/do_utility_kubernetes_doks_test_cluster.rs index e1cbb87c..72500381 100644 --- a/tests/digitalocean/do_utility_kubernetes_doks_test_cluster.rs +++ b/tests/digitalocean/do_utility_kubernetes_doks_test_cluster.rs @@ -6,7 +6,7 @@ use ::function_name::named; use qovery_engine::cloud_provider::digitalocean::DO; use tracing::{span, Level}; -use self::test_utilities::common::Cluster; +use self::test_utilities::common::{Cluster, ClusterDomain}; use qovery_engine::cloud_provider::digitalocean::kubernetes::DOKS; use qovery_engine::transaction::TransactionResult; @@ -34,7 +34,7 @@ fn create_digitalocean_kubernetes_doks_test_cluster() { let do_cluster = DO::cloud_provider(&context); let nodes = DO::kubernetes_nodes(); - let cloudflare = dns_provider_cloudflare(&context); + let cloudflare = dns_provider_cloudflare(&context, ClusterDomain::Default); let kubernetes = DOKS::new( context.clone(), @@ -91,7 +91,7 @@ fn destroy_digitalocean_kubernetes_doks_test_cluster() { let do_cluster = DO::cloud_provider(&context); let nodes = DO::kubernetes_nodes(); - let cloudflare = dns_provider_cloudflare(&context); + let cloudflare = dns_provider_cloudflare(&context, ClusterDomain::Default); let kubernetes = DOKS::new( context.clone(), diff --git a/tests/digitalocean/do_whole_enchilada.rs b/tests/digitalocean/do_whole_enchilada.rs new file mode 100644 index 00000000..e5e5581c --- /dev/null +++ b/tests/digitalocean/do_whole_enchilada.rs @@ -0,0 +1,50 @@ +use ::function_name::named; +use qovery_engine::cloud_provider::digitalocean::application::Region; +use qovery_engine::cloud_provider::Kind; +use qovery_engine::models::EnvironmentAction; +use test_utilities::common::{cluster_test, ClusterDomain, ClusterTestType}; +use test_utilities::digitalocean::{DO_KUBERNETES_MAJOR_VERSION, DO_KUBERNETES_MINOR_VERSION}; +use test_utilities::utilities::{context, engine_run_test, generate_cluster_id, generate_id, FuncTestsSecrets}; + +#[cfg(feature = "test-do-whole-enchilada")] +#[named] +#[test] +fn create_upgrade_and_destroy_doks_cluster_with_env_in_ams_3() { + let context = context(); + let region = Region::Amsterdam3; + let secrets = FuncTestsSecrets::new(); + let organization_id = generate_id(); + let cluster_id = generate_cluster_id(region.as_str()); + let cluster_domain = format!( + "{}.{}", + cluster_id.as_str(), + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str() + ); + + let environment = test_utilities::common::working_minimal_environment( + &context, + organization_id.as_str(), + cluster_domain.as_str(), + ); + let env_action = EnvironmentAction::Environment(environment.clone()); + + engine_run_test(|| { + cluster_test( + function_name!(), + Kind::Do, + context.clone(), + region.as_str(), + secrets.clone(), + ClusterTestType::Classic, + DO_KUBERNETES_MAJOR_VERSION, + DO_KUBERNETES_MINOR_VERSION, + ClusterDomain::Custom(cluster_domain), + None, + Some(&env_action), + ) + }) +} diff --git a/tests/digitalocean/mod.rs b/tests/digitalocean/mod.rs index fd81daab..a465b526 100644 --- a/tests/digitalocean/mod.rs +++ b/tests/digitalocean/mod.rs @@ -3,3 +3,4 @@ mod do_environment; mod do_kubernetes; mod do_spaces; mod do_utility_kubernetes_doks_test_cluster; +mod do_whole_enchilada; diff --git a/tests/scaleway/mod.rs b/tests/scaleway/mod.rs index a4d9c26a..292bd354 100644 --- a/tests/scaleway/mod.rs +++ b/tests/scaleway/mod.rs @@ -4,3 +4,4 @@ mod scw_environment; mod scw_kubernetes; mod scw_object_storage; mod scw_utility_kubernetes_kapsule_test_cluster; +mod scw_whole_enchilada; diff --git a/tests/scaleway/scw_kubernetes.rs b/tests/scaleway/scw_kubernetes.rs index 372134a2..79e11921 100644 --- a/tests/scaleway/scw_kubernetes.rs +++ b/tests/scaleway/scw_kubernetes.rs @@ -1,12 +1,12 @@ extern crate test_utilities; use self::test_utilities::scaleway::{SCW_KUBERNETES_MAJOR_VERSION, SCW_KUBERNETES_MINOR_VERSION}; -use self::test_utilities::utilities::{engine_run_test, FuncTestsSecrets}; +use self::test_utilities::utilities::{context, engine_run_test, FuncTestsSecrets}; use ::function_name::named; use qovery_engine::cloud_provider::aws::kubernetes::VpcQoveryNetworkMode; use qovery_engine::cloud_provider::scaleway::application::Zone; use qovery_engine::cloud_provider::Kind; -use test_utilities::common::{cluster_test, ClusterTestType}; +use test_utilities::common::{cluster_test, ClusterDomain, ClusterTestType}; #[cfg(feature = "test-scw-infra")] fn create_and_destroy_kapsule_cluster( @@ -22,12 +22,15 @@ fn create_and_destroy_kapsule_cluster( cluster_test( test_name, Kind::Scw, + context(), zone.as_str(), secrets, test_type, major_boot_version, minor_boot_version, + ClusterDomain::Default, vpc_network_mode, + None, ) }) } diff --git a/tests/scaleway/scw_object_storage.rs b/tests/scaleway/scw_object_storage.rs index 5e0c1eeb..a9d2d3b9 100644 --- a/tests/scaleway/scw_object_storage.rs +++ b/tests/scaleway/scw_object_storage.rs @@ -3,7 +3,6 @@ extern crate test_utilities; use self::test_utilities::utilities::{context, generate_id, FuncTestsSecrets}; use self::test_utilities::scaleway::{SCW_RESOURCE_TTL_IN_SECONDS, SCW_TEST_ZONE}; -use qovery_engine::cloud_provider::scaleway::application::Zone; use qovery_engine::object_storage::scaleway_object_storage::{BucketDeleteStrategy, ScalewayOS}; use qovery_engine::object_storage::ObjectStorage; use tempfile::NamedTempFile; diff --git a/tests/scaleway/scw_utility_kubernetes_kapsule_test_cluster.rs b/tests/scaleway/scw_utility_kubernetes_kapsule_test_cluster.rs index c673bf8e..67848fa5 100644 --- a/tests/scaleway/scw_utility_kubernetes_kapsule_test_cluster.rs +++ b/tests/scaleway/scw_utility_kubernetes_kapsule_test_cluster.rs @@ -5,7 +5,7 @@ use self::test_utilities::utilities::{context, engine_run_test, init, FuncTestsS use ::function_name::named; use tracing::{span, Level}; -use self::test_utilities::common::Cluster; +use self::test_utilities::common::{Cluster, ClusterDomain}; use qovery_engine::cloud_provider::scaleway::kubernetes::Kapsule; use qovery_engine::cloud_provider::scaleway::Scaleway; use qovery_engine::transaction::TransactionResult; @@ -34,7 +34,7 @@ fn create_scaleway_kubernetes_kapsule_test_cluster() { let scw_cluster = Scaleway::cloud_provider(&context); let nodes = Scaleway::kubernetes_nodes(); - let cloudflare = dns_provider_cloudflare(&context); + let cloudflare = dns_provider_cloudflare(&context, ClusterDomain::Default); let kubernetes = Kapsule::new( context.clone(), @@ -88,7 +88,7 @@ fn destroy_scaleway_kubernetes_kapsule_test_cluster() { let scw_cluster = Scaleway::cloud_provider(&context); let nodes = Scaleway::kubernetes_nodes(); - let cloudflare = dns_provider_cloudflare(&context); + let cloudflare = dns_provider_cloudflare(&context, ClusterDomain::Default); let kubernetes = Kapsule::new( context.clone(), diff --git a/tests/scaleway/scw_whole_enchilada.rs b/tests/scaleway/scw_whole_enchilada.rs new file mode 100644 index 00000000..6af202f4 --- /dev/null +++ b/tests/scaleway/scw_whole_enchilada.rs @@ -0,0 +1,50 @@ +use ::function_name::named; +use qovery_engine::cloud_provider::scaleway::application::Zone; +use qovery_engine::cloud_provider::Kind; +use qovery_engine::models::EnvironmentAction; +use test_utilities::common::{cluster_test, ClusterDomain, ClusterTestType}; +use test_utilities::scaleway::{SCW_KUBERNETES_MAJOR_VERSION, SCW_KUBERNETES_MINOR_VERSION}; +use test_utilities::utilities::{context, engine_run_test, generate_cluster_id, generate_id, FuncTestsSecrets}; + +#[cfg(feature = "test-scw-whole-enchilada")] +#[named] +#[test] +fn create_upgrade_and_destroy_kapsule_cluster_with_env_in_par_2() { + let context = context(); + let zone = Zone::Paris2; + let secrets = FuncTestsSecrets::new(); + let organization_id = generate_id(); + let cluster_id = generate_cluster_id(zone.as_str()); + let cluster_domain = format!( + "{}.{}", + cluster_id.as_str(), + secrets + .DEFAULT_TEST_DOMAIN + .as_ref() + .expect("DEFAULT_TEST_DOMAIN is not set in secrets") + .as_str() + ); + + let environment = test_utilities::common::working_minimal_environment( + &context, + organization_id.as_str(), + cluster_domain.as_str(), + ); + let env_action = EnvironmentAction::Environment(environment.clone()); + + engine_run_test(|| { + cluster_test( + function_name!(), + Kind::Scw, + context.clone(), + zone.as_str(), + secrets.clone(), + ClusterTestType::Classic, + SCW_KUBERNETES_MAJOR_VERSION, + SCW_KUBERNETES_MINOR_VERSION, + ClusterDomain::Custom(cluster_domain), + None, + Some(&env_action), + ) + }) +}