Merge branch 'dev' into fix/multiple_custom_domains

2026-03-10 08:11:21 +00:00 · 2021-02-06 18:24:14 +01:00
parent 2493f4aaef ece4c2dd56
commit 692c0be009
7 changed files with 65 additions and 81 deletions
--- a/.github/workflows/functionnal_tests_dev.yml
+++ b/.github/workflows/functionnal_tests_dev.yml
@@ -1,45 +0,0 @@
-name: functionnal-tests
-
-on:
-  push:
-    branches: [ dev ]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-      - name: build engine
-        run: |
-          export PATH=$GITHUB_WORKSPACE/bin:$PATH
-          export RUSTC_WRAPPER=$GITHUB_WORKSPACE/bin/sccache
-          export SCCACHE_REDIS=${{ secrets.SCCACHE_REDIS }}
-          mkdir -p $GITHUB_WORKSPACE/bin
-          sccache_release=$(curl --silent "https://github.com/Qovery/sccache-bin/releases/latest" | sed -r 's/^.+tag\/(.+)">.+/\1/')
-          curl -sLo $GITHUB_WORKSPACE/bin/sccache https://github.com/Qovery/sccache-bin/releases/download/${sccache_release}/sccache
-          chmod 755 $GITHUB_WORKSPACE/bin/sccache
-          echo "########## SHARED CACHE STATUS ##########"
-          sccache --version
-          sccache --show-stats
-          echo "########## START BUILD ##########"
-          cargo build --release --all-features
-          sccache --show-stats
-  functionnal-tests:
-    runs-on: ubuntu-latest
-    needs: build
-    env:
-      GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
-      GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
-      GITLAB_PERSONAL_TOKEN: ${{ secrets.GITLAB_PERSONAL_TOKEN }}
-    steps:
-      - uses: actions/checkout@v1
-      - name: Run a multi-line script
-        timeout-minutes: 120
-        run: |
-          branch=$(echo ${{github.ref}} | sed -r 's/^refs\/heads\/(.+)/\1/')
-          echo "Branch name: $branch"
-          export GITHUB_BRANCH=$branch
-          ./helper.sh full_tests
--- a/.github/workflows/functionnal_tests.yml
+++ b/.github/workflows/functionnal_tests.yml
@@ -1,4 +1,4 @@
-name: functionnal-tests
+name: tests

 on:
  push:
@@ -29,13 +29,13 @@ jobs:
          echo "########## START BUILD ##########"
          cargo build --release --all-features
          sccache --show-stats
-  functionnal-tests:
+  selected-functionnal-tests:
    runs-on: ubuntu-latest
    if: github.event.pull_request.draft == false
    needs: build
    steps:
      - uses: actions/checkout@v1
-      - name: Run a multi-line script
+      - name: Run selected functional tests
        timeout-minutes: 120
        env:
          GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
@@ -47,3 +47,20 @@ jobs:
          export GITHUB_BRANCH=$branch
          export GITHUB_COMMIT_ID=$GITHUB_SHA
          ./helper.sh autodetect
+  all-tests-enabled:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v1
+      - name: Only validate PR if all tests have been requested
+        timeout-minutes: 120
+        env:
+          GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
+          GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
+          GITLAB_PERSONAL_TOKEN: ${{ secrets.GITLAB_PERSONAL_TOKEN }}
+        run: |
+          branch=$(echo ${{github.ref}} | sed -r 's/^refs\/heads\/(.+)/\1/')
+          echo "Branch name: $branch"
+          export GITHUB_BRANCH=$branch
+          export GITHUB_COMMIT_ID=$GITHUB_SHA
+          ./helper.sh check_gh_tags
--- a/README.md
+++ b/README.md
@@ -29,7 +29,7 @@

 ## ✨ Features

- **Zero infrastructure management:** Qovery Engine initialized, configure, and manage your Cloud account for you.
+- **Zero infrastructure management:** Qovery Engine initializes, configure, and manage your Cloud account for you.
 - **Multi Cloud:** Qovery Engine is built to work on AWS, GCP, Azure and any Cloud provider.
 - **On top of Kubernetes:** Qovery Engine takes advantage of the power of Kubernetes at a higher level of abstraction.
 - **Terraform and Helm:** Qovery Engine uses Terraform and Helm files to manage the infrastructure and app deployment.
--- a/helper.sh
+++ b/helper.sh
@@ -10,6 +10,7 @@ if [ "$(uname)" == "Darwin" ] ; then
  awk='gawk'
  sed='gsed'
 fi
+all_labels="test-aws-all"

 function variable_not_found() {
  echo "Required variable not found: $1"
@@ -37,15 +38,20 @@ function gh_tags_selector_for_gitlab() {
  gh_pr=$(echo $gh_json | jq --compact-output '.[] | {labels, ref: .head.ref}' | grep "$GITHUB_BRANCH")
  num_labels=$(echo $gh_pr | jq '.labels | length')

-  all_labels="default"
-  if [ $num_labels -gt 0 ] ; then
-    all_labels=$(echo $gh_pr | jq -r '.labels[].url' | sed -r 's/^.+labels\/(.+)$/\1/g')
+  all_labels=""
+  if [ "$num_labels" != "0" ] ; then
+    for i in $(echo $gh_pr | jq -r '.labels[].name' | grep 'test-') ; do
+      all_labels="$all_labels,$i"
+    done
+    all_labels=$(echo $all_labels | sed 's/^,//')
  fi
+
  echo $all_labels
 }

 function run_tests() {
  TESTS_TYPE=$1
+  echo "Requested tests: $TESTS_TYPE"
  test -z $GITLAB_PROJECT_ID && variable_not_found "GITLAB_PROJECT_ID"
  test -z $GITLAB_TOKEN && variable_not_found "GITLAB_TOKEN"
  test -z $GITLAB_PERSONAL_TOKEN && variable_not_found "GITLAB_PERSONAL_TOKEN"
@@ -60,7 +66,7 @@ function run_tests() {
  fi

  echo "Requesting Gitlab pipeline"
-  pipeline_id=$(curl -s -X POST -F "token=$GITLAB_TOKEN" -F "ref=$GITLAB_REF" -F "variables[GITHUB_COMMIT_ID]=$GITHUB_COMMIT_ID" -F "variables[GITHUB_ENGINE_BRANCH_NAME]=$GITHUB_BRANCH" -F "variables[TESTS_TYPE]=$TESTS_TYPE" -F "variables[FORCE_CHECKOUT_CUSTOM_BRANCH]=$FORCE_CHECKOUT_CUSTOM_BRANCH" https://gitlab.com/api/v4/projects/$GITLAB_PROJECT_ID/trigger/pipeline | jq --raw-output '.id')
+  pipeline_id=$(curl -s -X POST -F "token=$GITLAB_TOKEN" -F "ref=$GITLAB_REF" -F "variables[GITHUB_COMMIT_ID]=$GITHUB_COMMIT_ID" -F "variables[GITHUB_ENGINE_BRANCH_NAME]=$GITHUB_BRANCH" -F "variables[TESTS_TO_RUN]=$TESTS_TYPE" -F "variables[FORCE_CHECKOUT_CUSTOM_BRANCH]=$FORCE_CHECKOUT_CUSTOM_BRANCH" https://gitlab.com/api/v4/projects/$GITLAB_PROJECT_ID/trigger/pipeline | jq --raw-output '.id')
  if [ $(echo $pipeline_id | egrep -c '^[0-9]+$') -eq 0 ] ; then
    echo "Pipeline ID is not correct, we expected a number and got: $pipeline_id"
    exit 1
@@ -128,9 +134,6 @@ function run_tests() {
 #set -u

 case $1 in
-fast_tests)
-  run_tests fast
-  ;;
 full_tests)
  run_tests full
  ;;
@@ -138,12 +141,20 @@ release)
  release
  ;;
 autodetect)
-  gh_tags_selector_for_gitlab
+  tags=$(gh_tags_selector_for_gitlab)
+  run_tests $tags
+  ;;
+check_gh_tags)
+  if [ gh_tags_selector_for_gitlab == "$all_labels" ] ; then
+    exit 0
+  fi
+  echo "You need to enable all the tests to validate this PR"
+  exit 1
  ;;
 *)
  echo "Usage:"
  echo "$0 autodetect: autodetect tests to run based on tags"
-  echo "$0 fast_tests: run fast tests"
  echo "$0 full_tests: run full tests (with cloud providers check)"
+  echo "$0 check_gh_tags: get defined tags (only working if branch is a PR)"
  ;;
 esac
--- a/src/cloud_provider/aws/router.rs
+++ b/src/cloud_provider/aws/router.rs
@@ -11,7 +11,7 @@ use crate::cloud_provider::utilities::check_cname_for;
 use crate::cloud_provider::DeploymentTarget;
 use crate::cmd::helm::Timeout;
 use crate::error::{cast_simple_error_to_engine_error, EngineError, EngineErrorCause, EngineErrorScope};
-use crate::models::{Context, Listen, Listener, Listeners, ListenersHelper};
+use crate::models::{Context, Listen, Listener, Listeners};

 pub struct Router {
    context: Context,
@@ -399,20 +399,19 @@ impl Create for Router {
        // Wait/Check that custom domain is a CNAME targeting qovery
        for domain_to_check in self.custom_domains.iter() {
            match check_cname_for(
-                ListenersHelper::new(self.listeners()),
+                self.progress_scope(),
+                self.listeners(),
                &domain_to_check.domain,
-                self.id(),
+                self.context.execution_id(),
            ) {
                Ok(cname) if cname.trim_end_matches('.') == domain_to_check.target_domain.trim_end_matches('.') => {
                    continue
                }
                Ok(err) | Err(err) => {
-                    return Err(EngineError::new(
-                        EngineErrorCause::User("Invalid CNAME"),
-                        EngineErrorScope::Router(self.id.clone(), self.name.clone()),
-                        self.context.execution_id(),
-                        Some(err.as_str()),
-                    ))
+                    warn!(
+                        "Invalid CNAME for {}. Might not be an issue if user is using a CDN: {}",
+                        domain_to_check.domain, err
+                    );
                }
            }
        }
--- a/src/cloud_provider/digitalocean/router.rs
+++ b/src/cloud_provider/digitalocean/router.rs
@@ -15,7 +15,7 @@ use crate::cmd::helm::Timeout;
 use crate::error::{
    cast_simple_error_to_engine_error, EngineError, EngineErrorCause, EngineErrorScope, SimpleError, SimpleErrorKind,
 };
-use crate::models::{Context, Listen, Listener, Listeners, ListenersHelper};
+use crate::models::{Context, Listen, Listener, Listeners};

 pub struct Router {
    context: Context,
@@ -461,12 +461,13 @@ impl Create for Router {
        // Wait/Check that custom domain is a CNAME targeting qovery
        for domain_to_check in self.custom_domains.iter() {
            match check_cname_for(
-                ListenersHelper::new(self.listeners()),
+                self.progress_scope(),
+                self.listeners(),
                &domain_to_check.domain,
-                self.id(),
+                self.context.execution_id(),
            ) {
                Ok(cname) if cname.trim_end_matches('.') == domain_to_check.target_domain.trim_end_matches('.') => {
-                    continue
+                    continue;
                }
                Ok(err) | Err(err) => {
                    warn!(
--- a/src/cloud_provider/utilities.rs
+++ b/src/cloud_provider/utilities.rs
@@ -1,7 +1,7 @@
 use std::collections::HashMap;

 use crate::error::{EngineError, StringError};
-use crate::models::{ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope};
+use crate::models::{Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope};
 use chrono::Duration;
 use core::option::Option::{None, Some};
 use core::result::Result;
@@ -271,17 +271,17 @@ fn get_cname_record_value(resolver: &Resolver, cname: &str) -> Option<String> {
 }

 pub fn check_cname_for(
-    listener_helper: ListenersHelper,
+    scope: ProgressScope,
+    listeners: &Listeners,
    cname_to_check: &str,
    execution_id: &str,
 ) -> Result<String, String> {
    let resolver = cloudflare_dns_resolver();
+    let listener_helper = ListenersHelper::new(listeners);

    let send_deployment_progress = |msg: &str| {
        listener_helper.deployment_in_progress(ProgressInfo::new(
-            ProgressScope::Environment {
-                id: execution_id.to_string(),
-            },
+            scope.clone(),
            ProgressLevel::Info,
            Some(msg.to_string()),
            execution_id,
@@ -290,9 +290,7 @@ pub fn check_cname_for(

    let send_warn_progress = |msg: &str| {
        listener_helper.error(ProgressInfo::new(
-            ProgressScope::Environment {
-                id: execution_id.to_string(),
-            },
+            scope.clone(),
            ProgressLevel::Warn,
            Some(msg.to_string()),
            execution_id,
@@ -308,12 +306,15 @@ pub fn check_cname_for(
    );

    // Trying for 5 min to resolve CNAME
-    let fixed_iterable = Fixed::from_millis(Duration::seconds(5).num_milliseconds() as u64).take(12 * 5);
+    let fixed_iterable = Fixed::from_millis(Duration::seconds(5).num_milliseconds() as u64).take(6 * 5);
    let check_result = retry::retry(fixed_iterable, || {
        match get_cname_record_value(&resolver, cname_to_check) {
            Some(domain) => OperationResult::Ok(domain),
            None => {
-                let msg = format!("Cannot find domain under CNAME {}", cname_to_check);
+                let msg = format!(
+                    "Cannot find domain under CNAME {}. Retrying in 5 seconds...",
+                    cname_to_check
+                );
                send_deployment_progress(msg.as_str());
                OperationResult::Retry(msg)
            }