From bcd8a9eb95ef6c7cafe06a2ed56162f302ac016e Mon Sep 17 00:00:00 2001 From: Romaric Philogene Date: Sat, 6 Feb 2021 10:25:44 +0100 Subject: [PATCH 1/7] fix: Router progress status --- src/cloud_provider/aws/router.rs | 3 ++- src/cloud_provider/digitalocean/router.rs | 5 +++-- src/cloud_provider/utilities.rs | 21 +++++++++++---------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/cloud_provider/aws/router.rs b/src/cloud_provider/aws/router.rs index b0241d65..87d7c6cd 100644 --- a/src/cloud_provider/aws/router.rs +++ b/src/cloud_provider/aws/router.rs @@ -397,7 +397,8 @@ impl Create for Router { // Wait/Check that custom domain is a CNAME targeting qovery for domain_to_check in self.custom_domains.iter() { match check_cname_for( - ListenersHelper::new(self.listeners()), + self.progress_scope(), + self.listeners(), &domain_to_check.domain, self.id(), ) { diff --git a/src/cloud_provider/digitalocean/router.rs b/src/cloud_provider/digitalocean/router.rs index 1807e1ae..20ad877a 100644 --- a/src/cloud_provider/digitalocean/router.rs +++ b/src/cloud_provider/digitalocean/router.rs @@ -461,12 +461,13 @@ impl Create for Router { // Wait/Check that custom domain is a CNAME targeting qovery for domain_to_check in self.custom_domains.iter() { match check_cname_for( - ListenersHelper::new(self.listeners()), + self.progress_scope(), + self.listeners(), &domain_to_check.domain, self.id(), ) { Ok(cname) if cname.trim_end_matches('.') == domain_to_check.target_domain.trim_end_matches('.') => { - continue + continue; } Ok(err) | Err(err) => { warn!( diff --git a/src/cloud_provider/utilities.rs b/src/cloud_provider/utilities.rs index 3dc27c03..bec5b867 100644 --- a/src/cloud_provider/utilities.rs +++ b/src/cloud_provider/utilities.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use crate::error::{EngineError, StringError}; -use crate::models::{ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope}; +use crate::models::{Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope}; use chrono::Duration; use core::option::Option::{None, Some}; use core::result::Result; @@ -271,17 +271,17 @@ fn get_cname_record_value(resolver: &Resolver, cname: &str) -> Option { } pub fn check_cname_for( - listener_helper: ListenersHelper, + scope: ProgressScope, + listeners: &Listeners, cname_to_check: &str, execution_id: &str, ) -> Result { let resolver = cloudflare_dns_resolver(); + let listener_helper = ListenersHelper::new(listeners); let send_deployment_progress = |msg: &str| { listener_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Environment { - id: execution_id.to_string(), - }, + scope.clone(), ProgressLevel::Info, Some(msg.to_string()), execution_id, @@ -290,9 +290,7 @@ pub fn check_cname_for( let send_warn_progress = |msg: &str| { listener_helper.error(ProgressInfo::new( - ProgressScope::Environment { - id: execution_id.to_string(), - }, + scope.clone(), ProgressLevel::Warn, Some(msg.to_string()), execution_id, @@ -308,12 +306,15 @@ pub fn check_cname_for( ); // Trying for 5 min to resolve CNAME - let fixed_iterable = Fixed::from_millis(Duration::seconds(5).num_milliseconds() as u64).take(12 * 5); + let fixed_iterable = Fixed::from_millis(Duration::seconds(5).num_milliseconds() as u64).take(6 * 5); let check_result = retry::retry(fixed_iterable, || { match get_cname_record_value(&resolver, cname_to_check) { Some(domain) => OperationResult::Ok(domain), None => { - let msg = format!("Cannot find domain under CNAME {}", cname_to_check); + let msg = format!( + "Cannot find domain under CNAME {}. Retrying in 5 seconds...", + cname_to_check + ); send_deployment_progress(msg.as_str()); OperationResult::Retry(msg) } From 3da7b687f4741eb32c7881d2558cbdc43efd3187 Mon Sep 17 00:00:00 2001 From: Romaric Philogene Date: Sat, 6 Feb 2021 10:27:11 +0100 Subject: [PATCH 2/7] chore: remove ProgressListener imports --- src/cloud_provider/aws/router.rs | 2 +- src/cloud_provider/digitalocean/router.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cloud_provider/aws/router.rs b/src/cloud_provider/aws/router.rs index 87d7c6cd..fe54d04a 100644 --- a/src/cloud_provider/aws/router.rs +++ b/src/cloud_provider/aws/router.rs @@ -11,7 +11,7 @@ use crate::cloud_provider::utilities::check_cname_for; use crate::cloud_provider::DeploymentTarget; use crate::cmd::helm::Timeout; use crate::error::{cast_simple_error_to_engine_error, EngineError, EngineErrorCause, EngineErrorScope}; -use crate::models::{Context, Listen, Listener, Listeners, ListenersHelper}; +use crate::models::{Context, Listen, Listener, Listeners}; pub struct Router { context: Context, diff --git a/src/cloud_provider/digitalocean/router.rs b/src/cloud_provider/digitalocean/router.rs index 20ad877a..6ef01a04 100644 --- a/src/cloud_provider/digitalocean/router.rs +++ b/src/cloud_provider/digitalocean/router.rs @@ -15,7 +15,7 @@ use crate::cmd::helm::Timeout; use crate::error::{ cast_simple_error_to_engine_error, EngineError, EngineErrorCause, EngineErrorScope, SimpleError, SimpleErrorKind, }; -use crate::models::{Context, Listen, Listener, Listeners, ListenersHelper}; +use crate::models::{Context, Listen, Listener, Listeners}; pub struct Router { context: Context, From ffc2e5174df9d882af12eba61e046639efeb384d Mon Sep 17 00:00:00 2001 From: Romaric Philogene Date: Sat, 6 Feb 2021 10:44:57 +0100 Subject: [PATCH 3/7] fix: inconsistent behaviour between DO.router and AWS.router on CNAME check fix: Router.id is used instead of Router.context.execution_id to check CNAME --- src/cloud_provider/aws/router.rs | 12 +++++------- src/cloud_provider/digitalocean/router.rs | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/cloud_provider/aws/router.rs b/src/cloud_provider/aws/router.rs index fe54d04a..9ef2f444 100644 --- a/src/cloud_provider/aws/router.rs +++ b/src/cloud_provider/aws/router.rs @@ -400,18 +400,16 @@ impl Create for Router { self.progress_scope(), self.listeners(), &domain_to_check.domain, - self.id(), + self.context.execution_id(), ) { Ok(cname) if cname.trim_end_matches('.') == domain_to_check.target_domain.trim_end_matches('.') => { continue } Ok(err) | Err(err) => { - return Err(EngineError::new( - EngineErrorCause::User("Invalid CNAME"), - EngineErrorScope::Router(self.id.clone(), self.name.clone()), - self.context.execution_id(), - Some(err.as_str()), - )) + warn!( + "Invalid CNAME for {}. Might not be an issue if user is using a CDN: {}", + domain_to_check.domain, err + ); } } } diff --git a/src/cloud_provider/digitalocean/router.rs b/src/cloud_provider/digitalocean/router.rs index 6ef01a04..491c40d7 100644 --- a/src/cloud_provider/digitalocean/router.rs +++ b/src/cloud_provider/digitalocean/router.rs @@ -464,7 +464,7 @@ impl Create for Router { self.progress_scope(), self.listeners(), &domain_to_check.domain, - self.id(), + self.context.execution_id(), ) { Ok(cname) if cname.trim_end_matches('.') == domain_to_check.target_domain.trim_end_matches('.') => { continue; From b5d8b556f83a8549551218c622751c52cfe2c83f Mon Sep 17 00:00:00 2001 From: Sylvain Kerkour <6172808+skerkour@users.noreply.github.com> Date: Sat, 6 Feb 2021 14:47:51 +0000 Subject: [PATCH 4/7] README.md: typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 371d1915..e764a735 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ ## ✨ Features -- **Zero infrastructure management:** Qovery Engine initialized, configure, and manage your Cloud account for you. +- **Zero infrastructure management:** Qovery Engine initializes, configure, and manage your Cloud account for you. - **Multi Cloud:** Qovery Engine is built to work on AWS, GCP, Azure and any Cloud provider. - **On top of Kubernetes:** Qovery Engine takes advantage of the power of Kubernetes at a higher level of abstraction. - **Terraform and Helm:** Qovery Engine uses Terraform and Helm files to manage the infrastructure and app deployment. From 1fba03e51cc32c4141997cb0cc629b704475660a Mon Sep 17 00:00:00 2001 From: Pierre Mavro Date: Fri, 5 Feb 2021 18:50:17 +0100 Subject: [PATCH 5/7] feat: support tags --- helper.sh | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/helper.sh b/helper.sh index efb94cce..2e2dadf8 100755 --- a/helper.sh +++ b/helper.sh @@ -10,6 +10,7 @@ if [ "$(uname)" == "Darwin" ] ; then awk='gawk' sed='gsed' fi +all_labels="test-aws-all" function variable_not_found() { echo "Required variable not found: $1" @@ -37,15 +38,23 @@ function gh_tags_selector_for_gitlab() { gh_pr=$(echo $gh_json | jq --compact-output '.[] | {labels, ref: .head.ref}' | grep "$GITHUB_BRANCH") num_labels=$(echo $gh_pr | jq '.labels | length') - all_labels="default" + all_labels="" if [ $num_labels -gt 0 ] ; then - all_labels=$(echo $gh_pr | jq -r '.labels[].url' | sed -r 's/^.+labels\/(.+)$/\1/g') + for i in $(echo $gh_pr | jq -r '.labels[].name' | grep 'test-') ; do + all_labels="$all_labels,$i" + done + all_labels=$(echo $all_labels | sed 's/^,//') + else + # default tests if no test label are specified + all_labels="test-aws-all" fi + echo $all_labels } function run_tests() { TESTS_TYPE=$1 + echo "Requested tests: $TESTS_TYPE" test -z $GITLAB_PROJECT_ID && variable_not_found "GITLAB_PROJECT_ID" test -z $GITLAB_TOKEN && variable_not_found "GITLAB_TOKEN" test -z $GITLAB_PERSONAL_TOKEN && variable_not_found "GITLAB_PERSONAL_TOKEN" @@ -60,7 +69,7 @@ function run_tests() { fi echo "Requesting Gitlab pipeline" - pipeline_id=$(curl -s -X POST -F "token=$GITLAB_TOKEN" -F "ref=$GITLAB_REF" -F "variables[GITHUB_COMMIT_ID]=$GITHUB_COMMIT_ID" -F "variables[GITHUB_ENGINE_BRANCH_NAME]=$GITHUB_BRANCH" -F "variables[TESTS_TYPE]=$TESTS_TYPE" -F "variables[FORCE_CHECKOUT_CUSTOM_BRANCH]=$FORCE_CHECKOUT_CUSTOM_BRANCH" https://gitlab.com/api/v4/projects/$GITLAB_PROJECT_ID/trigger/pipeline | jq --raw-output '.id') + pipeline_id=$(curl -s -X POST -F "token=$GITLAB_TOKEN" -F "ref=$GITLAB_REF" -F "variables[GITHUB_COMMIT_ID]=$GITHUB_COMMIT_ID" -F "variables[GITHUB_ENGINE_BRANCH_NAME]=$GITHUB_BRANCH" -F "variables[TESTS_TO_RUN]=$TESTS_TYPE" -F "variables[FORCE_CHECKOUT_CUSTOM_BRANCH]=$FORCE_CHECKOUT_CUSTOM_BRANCH" https://gitlab.com/api/v4/projects/$GITLAB_PROJECT_ID/trigger/pipeline | jq --raw-output '.id') if [ $(echo $pipeline_id | egrep -c '^[0-9]+$') -eq 0 ] ; then echo "Pipeline ID is not correct, we expected a number and got: $pipeline_id" exit 1 @@ -128,9 +137,6 @@ function run_tests() { #set -u case $1 in -fast_tests) - run_tests fast - ;; full_tests) run_tests full ;; @@ -138,12 +144,12 @@ release) release ;; autodetect) - gh_tags_selector_for_gitlab + tags=$(gh_tags_selector_for_gitlab) + run_tests $tags ;; *) echo "Usage:" echo "$0 autodetect: autodetect tests to run based on tags" - echo "$0 fast_tests: run fast tests" echo "$0 full_tests: run full tests (with cloud providers check)" ;; esac From 42adf0b12dc587b1fbae476c9240fba449a32456 Mon Sep 17 00:00:00 2001 From: Pierre Mavro Date: Sat, 6 Feb 2021 16:23:06 +0100 Subject: [PATCH 6/7] feat: force all checks to validate a PR + remove dev tests --- .github/workflows/functionnal_tests_dev.yml | 45 ------------------- .../{functionnal_tests.yml => tests.yml} | 23 ++++++++-- helper.sh | 13 ++++-- 3 files changed, 29 insertions(+), 52 deletions(-) delete mode 100644 .github/workflows/functionnal_tests_dev.yml rename .github/workflows/{functionnal_tests.yml => tests.yml} (69%) diff --git a/.github/workflows/functionnal_tests_dev.yml b/.github/workflows/functionnal_tests_dev.yml deleted file mode 100644 index 50d47e20..00000000 --- a/.github/workflows/functionnal_tests_dev.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: functionnal-tests - -on: - push: - branches: [ dev ] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - name: build engine - run: | - export PATH=$GITHUB_WORKSPACE/bin:$PATH - export RUSTC_WRAPPER=$GITHUB_WORKSPACE/bin/sccache - export SCCACHE_REDIS=${{ secrets.SCCACHE_REDIS }} - mkdir -p $GITHUB_WORKSPACE/bin - sccache_release=$(curl --silent "https://github.com/Qovery/sccache-bin/releases/latest" | sed -r 's/^.+tag\/(.+)">.+/\1/') - curl -sLo $GITHUB_WORKSPACE/bin/sccache https://github.com/Qovery/sccache-bin/releases/download/${sccache_release}/sccache - chmod 755 $GITHUB_WORKSPACE/bin/sccache - echo "########## SHARED CACHE STATUS ##########" - sccache --version - sccache --show-stats - echo "########## START BUILD ##########" - cargo build --release --all-features - sccache --show-stats - functionnal-tests: - runs-on: ubuntu-latest - needs: build - env: - GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }} - GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} - GITLAB_PERSONAL_TOKEN: ${{ secrets.GITLAB_PERSONAL_TOKEN }} - steps: - - uses: actions/checkout@v1 - - name: Run a multi-line script - timeout-minutes: 120 - run: | - branch=$(echo ${{github.ref}} | sed -r 's/^refs\/heads\/(.+)/\1/') - echo "Branch name: $branch" - export GITHUB_BRANCH=$branch - ./helper.sh full_tests diff --git a/.github/workflows/functionnal_tests.yml b/.github/workflows/tests.yml similarity index 69% rename from .github/workflows/functionnal_tests.yml rename to .github/workflows/tests.yml index 4afc4bca..3f00cf89 100644 --- a/.github/workflows/functionnal_tests.yml +++ b/.github/workflows/tests.yml @@ -1,4 +1,4 @@ -name: functionnal-tests +name: tests on: push: @@ -29,13 +29,13 @@ jobs: echo "########## START BUILD ##########" cargo build --release --all-features sccache --show-stats - functionnal-tests: + selected-functionnal-tests: runs-on: ubuntu-latest if: github.event.pull_request.draft == false needs: build steps: - uses: actions/checkout@v1 - - name: Run a multi-line script + - name: Run selected functional tests timeout-minutes: 120 env: GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }} @@ -47,3 +47,20 @@ jobs: export GITHUB_BRANCH=$branch export GITHUB_COMMIT_ID=$GITHUB_SHA ./helper.sh autodetect + all-tests: + runs-on: ubuntu-latest + needs: build + steps: + - uses: actions/checkout@v1 + - name: Only validate PR if all tests have been requested + timeout-minutes: 120 + env: + GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }} + GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} + GITLAB_PERSONAL_TOKEN: ${{ secrets.GITLAB_PERSONAL_TOKEN }} + run: | + branch=$(echo ${{github.ref}} | sed -r 's/^refs\/heads\/(.+)/\1/') + echo "Branch name: $branch" + export GITHUB_BRANCH=$branch + export GITHUB_COMMIT_ID=$GITHUB_SHA + ./helper.sh check_gh_tags diff --git a/helper.sh b/helper.sh index 2e2dadf8..b4f69ac9 100755 --- a/helper.sh +++ b/helper.sh @@ -39,14 +39,11 @@ function gh_tags_selector_for_gitlab() { num_labels=$(echo $gh_pr | jq '.labels | length') all_labels="" - if [ $num_labels -gt 0 ] ; then + if [ "$num_labels" != "0" ] ; then for i in $(echo $gh_pr | jq -r '.labels[].name' | grep 'test-') ; do all_labels="$all_labels,$i" done all_labels=$(echo $all_labels | sed 's/^,//') - else - # default tests if no test label are specified - all_labels="test-aws-all" fi echo $all_labels @@ -147,9 +144,17 @@ autodetect) tags=$(gh_tags_selector_for_gitlab) run_tests $tags ;; +check_gh_tags) + if [ gh_tags_selector_for_gitlab == "$all_labels" ] ; then + exit 0 + fi + echo "You need to enable all the tests to validate this PR" + exit 1 + ;; *) echo "Usage:" echo "$0 autodetect: autodetect tests to run based on tags" echo "$0 full_tests: run full tests (with cloud providers check)" + echo "$0 check_gh_tags: get defined tags (only working if branch is a PR)" ;; esac From ece4c2dd56ad25b4139766521bb1845690dbea5f Mon Sep 17 00:00:00 2001 From: Pierre Mavro Date: Sat, 6 Feb 2021 16:35:50 +0100 Subject: [PATCH 7/7] refactor: renaming gh test name --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3f00cf89..b31e4fa0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,7 +47,7 @@ jobs: export GITHUB_BRANCH=$branch export GITHUB_COMMIT_ID=$GITHUB_SHA ./helper.sh autodetect - all-tests: + all-tests-enabled: runs-on: ubuntu-latest needs: build steps: