feat: splitting tests

.github/workflows/functionnal_tests.yml

@@ -46,15 +46,4 @@ jobs:
           echo "Branch name: $branch"
           export GITHUB_BRANCH=$branch
           export GITHUB_COMMIT_ID=$GITHUB_SHA
-          gh_json=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/Qovery/engine/pulls?state=open")
-          gh_pr=$(echo $gh_json | jq --compact-output '.[] | {labels, ref: .head.ref}' | grep "$branch")
-          echo "Pull request info: $(echo $gh_pr)"
-          num_labels=$(echo $gh_pr | jq '.labels | length')
-          tests_type="fast_tests"
-          if [ $num_labels -gt 0 ] ; then
-            if [ $(echo $gh_pr | grep -c 'full-tests') -eq 1 ] ; then
-              tests_type="full_tests"
-            fi
-          fi
-          echo $tests_type
-          ./helper.sh $tests_type
+          ./helper.sh autodetect

Cargo.toml

@@ -58,3 +58,24 @@ digitalocean = "0.1.1"
 
 [dev-dependencies]
 test-utilities = { path = "test_utilities" }
+
+[features]
+default = ["test-all"]
+test-all = ["test-all-self-hosted", "test-all-infra", "test-all-managed-services"]
+
+# functionnal tests by type
+test-aws-self-hosted = []
+test-do-self-hosted = []
+test-all-self-hosted = ["test-aws-self-hosted", "test-do-self-hosted"]
+
+test-aws-managed-services = []
+test-do-managed-services = []
+test-all-managed-services = ["test-aws-managed-services", "test-do-managed-services"]
+
+test-aws-infra = []
+test-do-infra = []
+test-all-infra = ["test-aws-infra", "test-do-infra"]
+
+# functionnal tests by provider
+test-do-all = ["test-do-infra", "test-do-managed-services", "test-do-self-hosted"]
+test-aws-all = ["test-aws-infra", "test-aws-infra", "test-aws-self-hosted"]

helper.sh

@@ -32,6 +32,18 @@ function release() {
   echo "Pipeline ID: $pipeline_id"
 }
 
+function gh_tags_selector_for_gitlab() {
+  gh_json=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/Qovery/engine/pulls?state=open")
+  gh_pr=$(echo $gh_json | jq --compact-output '.[] | {labels, ref: .head.ref}' | grep "$GITHUB_BRANCH")
+  num_labels=$(echo $gh_pr | jq '.labels | length')
+
+  all_labels="default"
+  if [ $num_labels -gt 0 ] ; then
+    all_labels=$(echo $gh_pr | jq -r '.labels[].url' | sed -r 's/^.+labels\/(.+)$/\1/g')
+  fi
+  echo $all_labels
+}
+
 function run_tests() {
   TESTS_TYPE=$1
   test -z $GITLAB_PROJECT_ID && variable_not_found "GITLAB_PROJECT_ID"
@@ -40,6 +52,7 @@ function run_tests() {
   test -z $GITHUB_BRANCH && variable_not_found "GITHUB_BRANCH"
   GITLAB_REF="dev"
   FORCE_CHECKOUT_CUSTOM_BRANCH='false'
+  TESTS_TO_RUN="-F \"variables[GITHUB_COMMIT_ID]=$GITHUB_COMMIT_ID\""
 
   if [ $(curl -s --header "PRIVATE-TOKEN: $GITLAB_PERSONAL_TOKEN" "https://gitlab.com/api/v4/projects/$GITLAB_PROJECT_ID/repository/branches/$GITHUB_BRANCH" | grep -c '404 Branch Not Found') -eq 0 ] ; then
     echo "Same branch name detected on gitlab, requesting to use it instead of dev branch"
@@ -124,8 +137,12 @@ full_tests)
 release)
   release
   ;;
+autodetect)
+  gh_tags_selector_for_gitlab
+  ;;
 *)
   echo "Usage:"
+  echo "$0 autodetect: autodetect tests to run based on tags"
   echo "$0 fast_tests: run fast tests"
   echo "$0 full_tests: run full tests (with cloud providers check)"
   ;;

tests/aws/aws_databases.rs

@@ -20,6 +20,7 @@ use self::test_utilities::utilities::{engine_run_test, generate_id};
 **/
 
 // to check overload between several databases and apps
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
 fn deploy_an_environment_with_3_databases_and_3_apps() {
     init();
@@ -57,6 +58,7 @@ fn deploy_an_environment_with_3_databases_and_3_apps() {
 }
 
 // this test ensure containers databases are never restarted, even in failover environment case
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
 fn postgresql_failover_dev_environment_with_all_options() {
     init();
@@ -122,8 +124,8 @@ fn postgresql_failover_dev_environment_with_all_options() {
 }
 
 // Ensure a full environment can run correctly
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
-#[ignore]
 fn postgresql_deploy_a_working_development_environment_with_all_options() {
     init();
 
@@ -167,6 +169,7 @@ fn postgresql_deploy_a_working_development_environment_with_all_options() {
 }
 
 // Ensure redeploy works as expected
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
 fn postgresql_deploy_a_working_environment_and_redeploy() {
     engine_run_test(|| {
@@ -367,8 +370,8 @@ fn test_postgresql_configuration(context: Context, mut environment: Environment,
 }
 
 // Postgres environment environment
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
-#[ignore]
 fn postgresql_v10_deploy_a_working_dev_environment() {
     let context = context();
     let environment = test_utilities::aws::working_minimal_environment(&context);
@@ -380,8 +383,8 @@ fn postgresql_v10_deploy_a_working_dev_environment() {
     );
 }
 
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
-#[ignore]
 fn postgresql_v11_deploy_a_working_dev_environment() {
     let context = context();
     let environment = test_utilities::aws::working_minimal_environment(&context);
@@ -393,6 +396,7 @@ fn postgresql_v11_deploy_a_working_dev_environment() {
     );
 }
 
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
 fn postgresql_v12_deploy_a_working_dev_environment() {
     let context = context();
@@ -406,8 +410,8 @@ fn postgresql_v12_deploy_a_working_dev_environment() {
 }
 
 // Postgres production environment
+#[cfg(feature = "test-managed-services-aws")]
 #[test]
-#[ignore]
 fn postgresql_v10_deploy_a_working_prod_environment() {
     let context = context();
     let mut environment = test_utilities::aws::working_minimal_environment(&context);
@@ -420,8 +424,8 @@ fn postgresql_v10_deploy_a_working_prod_environment() {
     );
 }
 
+#[cfg(feature = "test-managed-services-aws")]
 #[test]
-#[ignore]
 fn postgresql_v11_deploy_a_working_prod_environment() {
     let context = context();
     let mut environment = test_utilities::aws::working_minimal_environment(&context);
@@ -434,8 +438,8 @@ fn postgresql_v11_deploy_a_working_prod_environment() {
     );
 }
 
+#[cfg(feature = "test-managed-services-aws")]
 #[test]
-#[ignore]
 fn postgresql_v12_deploy_a_working_prod_environment() {
     let context = context();
     let mut environment = test_utilities::aws::working_minimal_environment(&context);
@@ -565,8 +569,8 @@ fn test_mongodb_configuration(context: Context, mut environment: Environment, ve
 }
 
 // development environment
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
-#[ignore]
 fn mongodb_v3_6_deploy_a_working_dev_environment() {
     let context = context();
     let environment = test_utilities::aws::working_minimal_environment(&context);
@@ -578,8 +582,8 @@ fn mongodb_v3_6_deploy_a_working_dev_environment() {
     );
 }
 
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
-#[ignore]
 fn mongodb_v4_0_deploy_a_working_dev_environment() {
     let context = context();
     let environment = test_utilities::aws::working_minimal_environment(&context);
@@ -591,8 +595,8 @@ fn mongodb_v4_0_deploy_a_working_dev_environment() {
     );
 }
 
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
-#[ignore]
 fn mongodb_v4_2_deploy_a_working_dev_environment() {
     let context = context();
     let environment = test_utilities::aws::working_minimal_environment(&context);
@@ -604,6 +608,7 @@ fn mongodb_v4_2_deploy_a_working_dev_environment() {
     );
 }
 
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
 fn mongodb_v4_4_deploy_a_working_dev_environment() {
     let context = context();
@@ -617,8 +622,8 @@ fn mongodb_v4_4_deploy_a_working_dev_environment() {
 }
 
 // MongoDB production environment (DocumentDB)
+#[cfg(feature = "test-managed-services-aws")]
 #[test]
-#[ignore]
 fn mongodb_v3_6_deploy_a_working_prod_environment() {
     let context = context();
     let mut environment = test_utilities::aws::working_minimal_environment(&context);
@@ -631,8 +636,8 @@ fn mongodb_v3_6_deploy_a_working_prod_environment() {
     );
 }
 
+#[cfg(feature = "test-managed-services-aws")]
 #[test]
-#[ignore]
 fn mongodb_v4_0_deploy_a_working_prod_environment() {
     let context = context();
     let mut environment = test_utilities::aws::working_minimal_environment(&context);
@@ -753,8 +758,8 @@ fn test_mysql_configuration(context: Context, mut environment: Environment, vers
 }
 
 // MySQL self-hosted environment
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
-#[ignore]
 fn mysql_v5_7_deploy_a_working_dev_environment() {
     let context = context();
     let environment = test_utilities::aws::working_minimal_environment(&context);
@@ -766,6 +771,7 @@ fn mysql_v5_7_deploy_a_working_dev_environment() {
     );
 }
 
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
 fn mysql_v8_deploy_a_working_dev_environment() {
     let context = context();
@@ -774,8 +780,8 @@ fn mysql_v8_deploy_a_working_dev_environment() {
 }
 
 // MySQL production environment (RDS)
+#[cfg(feature = "test-managed-services-aws")]
 #[test]
-#[ignore]
 fn mysql_v5_7_deploy_a_working_prod_environment() {
     let context = context();
     let mut environment = test_utilities::aws::working_minimal_environment(&context);
@@ -788,8 +794,8 @@ fn mysql_v5_7_deploy_a_working_prod_environment() {
     );
 }
 
+#[cfg(feature = "test-managed-services-aws")]
 #[test]
-#[ignore]
 fn mysql_v8_0_deploy_a_working_prod_environment() {
     let context = context();
     let mut environment = test_utilities::aws::working_minimal_environment(&context);
@@ -910,14 +916,15 @@ fn test_redis_configuration(context: Context, mut environment: Environment, vers
 }
 
 // Redis self-hosted environment
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
-#[ignore]
 fn redis_v5_deploy_a_working_dev_environment() {
     let context = context();
     let environment = test_utilities::aws::working_minimal_environment(&context);
     test_redis_configuration(context, environment, "5", "redis_v5_deploy_a_working_dev_environment");
 }
 
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
 fn redis_v6_deploy_a_working_dev_environment() {
     let context = context();
@@ -927,8 +934,8 @@ fn redis_v6_deploy_a_working_dev_environment() {
 }
 
 // Redis production environment (Elasticache)
+#[cfg(feature = "test-managed-services-aws")]
 #[test]
-#[ignore]
 fn redis_v5_deploy_a_working_prod_environment() {
     let context = context();
     let mut environment = test_utilities::aws::working_minimal_environment(&context);
@@ -936,8 +943,8 @@ fn redis_v5_deploy_a_working_prod_environment() {
     test_redis_configuration(context, environment, "5", "redis_v5_deploy_a_working_prod_environment");
 }
 
+#[cfg(feature = "test-managed-services-aws")]
 #[test]
-#[ignore]
 fn redis_v6_deploy_a_working_prod_environment() {
     let context = context();
     let mut environment = test_utilities::aws::working_minimal_environment(&context);

tests/aws/aws_environment.rs

@@ -84,6 +84,7 @@ pub fn delete_environment(context: &Context, environment_action: &EnvironmentAct
     tx.commit()
 }
 
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
 fn deploy_a_working_environment_with_no_router_on_aws_eks() {
     engine_run_test(|| {
@@ -117,6 +118,7 @@ fn deploy_a_working_environment_with_no_router_on_aws_eks() {
     })
 }
 
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
 fn deploy_a_not_working_environment_with_no_router_on_aws_eks() {
     engine_run_test(|| {
@@ -154,6 +156,7 @@ fn deploy_a_not_working_environment_with_no_router_on_aws_eks() {
     })
 }
 
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
 fn build_with_buildpacks_and_deploy_a_working_environment() {
     engine_run_test(|| {
@@ -197,6 +200,7 @@ fn build_with_buildpacks_and_deploy_a_working_environment() {
     })
 }
 
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
 fn deploy_a_working_environment_with_domain() {
     engine_run_test(|| {
@@ -275,8 +279,8 @@ fn deploy_a_working_environment_with_custom_domain() {
     })
 }
 
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
-#[ignore]
 fn deploy_a_working_environment_with_storage_on_aws_eks() {
     engine_run_test(|| {
         let span = span!(Level::INFO, "deploy_a_working_environment_with_storage_on_aws_eks");
@@ -328,6 +332,7 @@ fn deploy_a_working_environment_with_storage_on_aws_eks() {
 }
 
 // to check if app redeploy or not, it shouldn't
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
 fn redeploy_same_app_with_ebs() {
     engine_run_test(|| {
@@ -470,6 +475,7 @@ fn deploy_a_working_production_environment_with_all_options_on_aws_eks() {
     };
 }*/
 
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
 fn deploy_a_not_working_environment_and_after_working_environment() {
     engine_run_test(|| {
@@ -526,8 +532,8 @@ fn deploy_a_not_working_environment_and_after_working_environment() {
     })
 }
 
-// Todo: temporary comment this test, need to fix it asap
-//#[test]
+#[cfg(feature = "test-self-hosted-aws")]
+#[test]
 fn deploy_ok_fail_fail_ok_environment() {
     init();
 
@@ -602,6 +608,7 @@ fn deploy_ok_fail_fail_ok_environment() {
     };
 }
 
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
 fn deploy_a_non_working_environment_with_no_failover_on_aws_eks() {
     engine_run_test(|| {
@@ -635,8 +642,8 @@ fn deploy_a_non_working_environment_with_no_failover_on_aws_eks() {
     })
 }
 
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
-#[ignore]
 fn deploy_a_non_working_environment_with_a_working_failover_on_aws_eks() {
     init();
 
@@ -727,8 +734,8 @@ fn deploy_a_non_working_environment_with_a_working_failover_on_aws_eks() {
 //     };
 // }
 
+#[cfg(feature = "test-self-hosted-aws")]
 #[test]
-#[ignore]
 fn deploy_a_non_working_environment_with_a_non_working_failover_on_aws_eks() {
     init();
 

tests/aws/aws_kubernetes.rs

@@ -114,8 +114,8 @@ fn create_and_destroy_eks_cluster(region: &str, test_name: &str) {
     It is useful to keep 2 clusters deployment tests to run in // to validate there is no name collision (overlaping)
 */
 
+#[cfg(feature = "test-infra-aws")]
 #[test]
-#[ignore]
 fn create_and_destroy_eks_cluster_in_eu_west_3() {
     let region = "eu-west-3";
     create_and_destroy_eks_cluster(
@@ -124,8 +124,8 @@ fn create_and_destroy_eks_cluster_in_eu_west_3() {
     );
 }
 
+#[cfg(feature = "test-infra-aws")]
 #[test]
-#[ignore]
 fn create_and_destroy_eks_cluster_in_us_east_2() {
     let region = "us-east-2";
     create_and_destroy_eks_cluster(