From deda89cd2eea282136dd9d82561a8d2eafab5489 Mon Sep 17 00:00:00 2001 From: Pierre Mavro Date: Fri, 7 May 2021 15:06:08 +0200 Subject: [PATCH] feat: enhance ECR reliability and policy * Enhance ECR reliability because of too many random errors (200 return code from AWS API while it's not ready yet) * Reduced to 1 day ECR policy for test purpose --- Cargo.lock | 19 +++--- src/container_registry/ecr.rs | 124 ++++++++++++++++++++++++---------- 2 files changed, 97 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8040b478..3213966d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1082,9 +1082,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.3.4" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +checksum = "4a1ce40d6fc9764887c2fdc7305c3dcc429ba11ff981c1509416afd5697e4437" [[package]] name = "httpdate" @@ -1523,7 +1523,7 @@ checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" dependencies = [ "log", "mio", - "miow 0.3.5", + "miow 0.3.7", "winapi 0.3.9", ] @@ -1552,11 +1552,10 @@ dependencies = [ [[package]] name = "miow" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07b88fb9795d4d36d62a012dfbf49a8f5cf12751f36d31a9dbe66d528e58979e" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" dependencies = [ - "socket2", "winapi 0.3.9", ] @@ -1648,9 +1647,9 @@ checksum = "37fd5004feb2ce328a52b0b3d01dbf4ffff72583493900ed15f22d4111c51693" [[package]] name = "once_cell" -version = "1.4.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" +checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" [[package]] name = "opaque-debug" @@ -3112,9 +3111,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" +checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.7", diff --git a/src/container_registry/ecr.rs b/src/container_registry/ecr.rs index c8d5a650..0c55a45d 100644 --- a/src/container_registry/ecr.rs +++ b/src/container_registry/ecr.rs @@ -3,8 +3,8 @@ use std::str::FromStr; use rusoto_core::{Client, HttpClient, Region, RusotoError}; use rusoto_credential::StaticProvider; use rusoto_ecr::{ - CreateRepositoryRequest, DescribeImagesRequest, DescribeRepositoriesRequest, Ecr, EcrClient, - GetAuthorizationTokenRequest, ImageDetail, ImageIdentifier, PutLifecyclePolicyRequest, Repository, + CreateRepositoryRequest, DescribeImagesRequest, DescribeRepositoriesError, DescribeRepositoriesRequest, Ecr, + EcrClient, GetAuthorizationTokenRequest, ImageDetail, ImageIdentifier, PutLifecyclePolicyRequest, Repository, }; use rusoto_sts::{GetCallerIdentityRequest, Sts, StsClient}; @@ -17,6 +17,10 @@ use crate::models::{ Context, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope, }; use crate::runtime::block_on; +use retry::delay::Fixed; +use retry::Error::Operation; +use retry::OperationResult; +use serde_json::json; pub struct ECR { context: Context, @@ -133,49 +137,97 @@ impl ECR { } fn create_repository(&self, image: &Image) -> Result { - info!("ECR create repository {}", image.name.as_str()); + let repository_name = image.name.as_str(); + info!("creating ECR repository {}", &repository_name); + + let mut repo_creation_counter = 0; + let container_registry_request = DescribeRepositoriesRequest { + repository_names: Some(vec![repository_name.to_string()]), + ..Default::default() + }; let crr = CreateRepositoryRequest { - repository_name: image.name.clone(), + repository_name: repository_name.to_string(), ..Default::default() }; - if let Err(err) = block_on(self.ecr_client().create_repository(crr)) { - match err { - RusotoError::Service(ref err) => error!("{:?}", err), - _ => { - let msg = format!( - "can't create ECR repository {} for {}", - image.name.as_str(), - self.name_with_id(), - ); - error!("{}: {}", msg, err); - return Err(self.engine_error(EngineErrorCause::Internal, msg)); + // ensure repository is created + // need to do all this checks and retry because of several issues encountered like: 200 API response code while repo is not created + let repo_created = retry::retry(Fixed::from_millis(5000).take(24), || { + match block_on( + self.ecr_client() + .describe_repositories(container_registry_request.clone()), + ) { + Ok(x) => { + debug!("created {:?} repository", x); + OperationResult::Ok(()) + } + Err(e) => { + match e { + RusotoError::Service(s) => match s { + DescribeRepositoriesError::RepositoryNotFound(_) => { + if repo_creation_counter != 0 { + warn!( + "repository {} was not found, {}x retrying...", + &repository_name, &repo_creation_counter + ); + } + repo_creation_counter += 1; + } + _ => warn!("{:?}", s), + }, + _ => warn!("{:?}", e), + } + + let msg = match block_on(self.ecr_client().create_repository(crr.clone())) { + Ok(_) => format!("repository {} created", &repository_name), + Err(err) => format!( + "can't create ECR repository {} for {}. {:?}", + &repository_name, + self.name_with_id(), + err + ), + }; + + OperationResult::Retry(Err(self.engine_error(EngineErrorCause::Internal, msg))) } } - } + }); + + match repo_created { + Ok(_) => info!( + "repository {} created after {} attempt(s)", + &repository_name, repo_creation_counter + ), + Err(Operation { error, .. }) => return error, + Err(retry::Error::Internal(e)) => return Err(self.engine_error(EngineErrorCause::Internal, e.to_string())), + }; + + // apply retention policy + let retention_policy_in_days = match self.context.is_test_cluster() { + true => 1, + false => 365, + }; + let lifecycle_policy_text = json!({ + "rules": [ + { + "action": { + "type": "expire" + }, + "selection": { + "countType": "sinceImagePushed", + "countUnit": "days", + "countNumber": retention_policy_in_days, + "tagStatus": "any" + }, + "description": "Images retention policy", + "rulePriority": 1 + } + ] + }); let plp = PutLifecyclePolicyRequest { repository_name: image.name.clone(), - lifecycle_policy_text: r#" - { - "rules": [ - { - "action": { - "type": "expire" - }, - "selection": { - "countType": "sinceImagePushed", - "countUnit": "days", - "countNumber": 365, - "tagStatus": "any" - }, - "description": "Remove unit test images", - "rulePriority": 1 - } - ] - } - "# - .to_string(), + lifecycle_policy_text: lifecycle_policy_text.to_string(), ..Default::default() };