From e0a1dff4b23802b31a785c3ba53cbaef6fe891bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Er=C3=A8be=20-=20Romain=20Gerard?= Date: Thu, 17 Mar 2022 10:44:56 +0100 Subject: [PATCH] build_refacto (#643) * build_refacto * build_refacto --- Cargo.lock | 8 + Cargo.toml | 1 + src/build_platform/local_docker.rs | 243 ++----- src/build_platform/mod.rs | 114 +-- src/cloud_provider/aws/application.rs | 37 +- .../digitalocean/application.rs | 37 +- src/cloud_provider/scaleway/application.rs | 38 +- src/cmd/docker.rs | 682 ++++++++++++++++++ src/cmd/mod.rs | 1 + src/container_registry/docker.rs | 462 ------------ src/container_registry/docker_hub.rs | 320 -------- src/container_registry/docr.rs | 344 +-------- src/container_registry/ecr.rs | 326 +-------- src/container_registry/mod.rs | 41 +- .../scaleway_container_registry.rs | 399 ++-------- src/errors/io.rs | 2 + src/errors/mod.rs | 21 + src/git.rs | 2 +- src/models.rs | 81 +-- src/transaction.rs | 240 +----- test_utilities/Cargo.lock | 8 + test_utilities/Cargo.toml | 1 + test_utilities/src/aws.rs | 13 - test_utilities/src/common.rs | 2 +- test_utilities/src/digitalocean.rs | 24 +- test_utilities/src/scaleway.rs | 9 +- test_utilities/src/utilities.rs | 3 +- tests/aws/aws_databases.rs | 7 +- tests/aws/aws_environment.rs | 101 +-- tests/aws/aws_kubernetes.rs | 3 +- tests/digitalocean/do_databases.rs | 11 +- tests/digitalocean/do_environment.rs | 99 +-- tests/digitalocean/do_kubernetes.rs | 4 +- ...do_utility_kubernetes_doks_test_cluster.rs | 4 +- .../multi_stage_simple/Dockerfile.buildkit | 10 + tests/docker/multi_stage_simple/hello.go | 7 + tests/scaleway/scw_container_registry.rs | 48 +- tests/scaleway/scw_databases.rs | 9 +- tests/scaleway/scw_environment.rs | 99 +-- tests/scaleway/scw_kubernetes.rs | 4 +- ...utility_kubernetes_kapsule_test_cluster.rs | 4 +- 41 files changed, 1113 insertions(+), 2756 deletions(-) create mode 100644 src/cmd/docker.rs delete mode 100644 src/container_registry/docker.rs delete mode 100644 src/container_registry/docker_hub.rs create mode 100644 tests/docker/multi_stage_simple/Dockerfile.buildkit create mode 100644 tests/docker/multi_stage_simple/hello.go diff --git a/Cargo.lock b/Cargo.lock index 7cae05a1..00a3bf2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2119,6 +2119,7 @@ dependencies = [ "tracing-test", "trust-dns-resolver", "url 2.2.2", + "urlencoding", "uuid 0.8.2", "walkdir", ] @@ -3283,6 +3284,7 @@ dependencies = [ "time 0.2.27", "tracing", "tracing-subscriber", + "url 2.2.2", "uuid 0.8.2", ] @@ -3939,6 +3941,12 @@ dependencies = [ "url 1.7.2", ] +[[package]] +name = "urlencoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b90931029ab9b034b300b797048cf23723400aa757e8a2bfb9d748102f9821" + [[package]] name = "uuid" version = "0.7.4" diff --git a/Cargo.toml b/Cargo.toml index 2154f995..abfed16b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ function_name = "0.2.0" thiserror = "1.0.30" strum = "0.23" strum_macros = "0.23" +urlencoding = "2.1.0" # FIXME use https://crates.io/crates/blocking instead of runtime.rs diff --git a/src/build_platform/local_docker.rs b/src/build_platform/local_docker.rs index 591f58ac..0079c5d0 100644 --- a/src/build_platform/local_docker.rs +++ b/src/build_platform/local_docker.rs @@ -6,10 +6,11 @@ use chrono::Duration; use git2::{Cred, CredentialType}; use sysinfo::{Disk, DiskExt, SystemExt}; -use crate::build_platform::{docker, Build, BuildPlatform, BuildResult, CacheResult, Credentials, Image, Kind}; +use crate::build_platform::{docker, Build, BuildPlatform, BuildResult, Credentials, Kind}; use crate::cmd::command; use crate::cmd::command::CommandError::Killed; use crate::cmd::command::QoveryCommand; +use crate::cmd::docker::{ContainerImage, Docker, DockerError}; use crate::errors::{CommandError, EngineError, Tag}; use crate::events::{EngineEvent, EventDetails, EventMessage, ToTransmitter, Transmitter}; use crate::fs::workspace_directory; @@ -32,6 +33,7 @@ const BUILDPACKS_BUILDERS: [&str; 1] = [ /// use Docker in local pub struct LocalDocker { context: Context, + docker: Docker, id: String, name: String, listeners: Listeners, @@ -39,31 +41,25 @@ pub struct LocalDocker { } impl LocalDocker { - pub fn new(context: Context, id: &str, name: &str, logger: Box) -> Self { - LocalDocker { + pub fn new( + context: Context, + id: &str, + name: &str, + logger: Box, + ) -> Result> { + let docker = Docker::new_with_options(true, context.docker_tcp_socket().clone())?; + Ok(LocalDocker { context, + docker, id: id.to_string(), name: name.to_string(), listeners: vec![], logger, - } - } - - fn image_does_exist(&self, image: &Image) -> Result { - let mut cmd = QoveryCommand::new( - "docker", - &vec!["image", "inspect", image.name_with_tag().as_str()], - &self.get_docker_host_envs(), - ); - - Ok(matches!(cmd.exec(), Ok(_))) + }) } fn get_docker_host_envs(&self) -> Vec<(&str, &str)> { - match self.context.docker_tcp_socket() { - Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())], - None => vec![], - } + vec![] } /// Read Dockerfile content from location path and return an array of bytes @@ -89,34 +85,20 @@ impl LocalDocker { dockerfile_complete_path: &str, into_dir_docker_style: &str, env_var_args: Vec, - use_build_cache: bool, lh: &ListenersHelper, is_task_canceled: &dyn Fn() -> bool, ) -> Result { - let mut docker_args = if !use_build_cache { - vec!["build", "--no-cache"] - } else { - vec!["build"] + let image_to_build = ContainerImage { + registry: build.image.registry_url.clone(), + name: build.image.name(), + tags: vec![build.image.tag.clone(), "latest".to_string()], }; - let args = self.context.docker_build_options(); - for v in args.iter() { - for s in v.iter() { - docker_args.push(String::as_str(s)); - } - } - - let name_with_tag = build.image.name_with_tag(); - let name_with_latest_tag = build.image.name_with_latest_tag(); - - docker_args.extend(vec![ - "-f", - dockerfile_complete_path, - "-t", - name_with_tag.as_str(), - "-t", - name_with_latest_tag.as_str(), - ]); + let image_cache = ContainerImage { + registry: build.image.registry_url.clone(), + name: build.image.name(), + tags: vec!["latest".to_string()], + }; let dockerfile_content = self.get_dockerfile_content(dockerfile_complete_path)?; let env_var_args = match docker::match_used_env_var_args(env_var_args, dockerfile_content) { @@ -133,27 +115,25 @@ impl LocalDocker { } }; - let mut docker_args = if env_var_args.is_empty() { - docker_args - } else { - let mut build_args = vec![]; + // FIXME: pass a Vec<(key, value)> instead of spliting always the string + let env_vars = env_var_args + .into_iter() + .map(|val| { + let (key, value) = val.rsplit_once('=').unwrap(); + (key.to_string(), value.to_string()) + }) + .collect::>(); - env_var_args.iter().for_each(|arg_value| { - build_args.push("--build-arg"); - build_args.push(arg_value.as_str()); - }); - - docker_args.extend(build_args); - docker_args - }; - - docker_args.push(into_dir_docker_style); - - // docker build - let mut cmd = QoveryCommand::new("docker", &docker_args, &self.get_docker_host_envs()); - - let exit_status = cmd.exec_with_abort( - Duration::minutes(BUILD_DURATION_TIMEOUT_MIN), + let exit_status = self.docker.build( + &Path::new(dockerfile_complete_path), + &Path::new(into_dir_docker_style), + &image_to_build, + &env_vars + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())) + .collect::>(), + &image_cache, + true, |line| { self.logger.log( LogLevel::Info, @@ -171,25 +151,26 @@ impl LocalDocker { }, |line| { self.logger.log( - LogLevel::Warning, - EngineEvent::Warning(self.get_event_details(), EventMessage::new_from_safe(line.to_string())), + LogLevel::Info, + EngineEvent::Info(self.get_event_details(), EventMessage::new_from_safe(line.to_string())), ); lh.deployment_in_progress(ProgressInfo::new( ProgressScope::Application { id: build.image.application_id.clone(), }, - ProgressLevel::Warn, + ProgressLevel::Info, Some(line), self.context.execution_id(), )); }, + Duration::minutes(BUILD_DURATION_TIMEOUT_MIN), is_task_canceled, ); match exit_status { Ok(_) => Ok(BuildResult { build }), - Err(Killed(_)) => Err(EngineError::new_task_cancellation_requested(self.get_event_details())), + Err(DockerError::Aborted(_)) => Err(EngineError::new_task_cancellation_requested(self.get_event_details())), Err(err) => Err(EngineError::new_docker_cannot_build_container_image( self.get_event_details(), self.name_with_id(), @@ -207,10 +188,8 @@ impl LocalDocker { lh: &ListenersHelper, is_task_canceled: &dyn Fn() -> bool, ) -> Result { - let name_with_tag = build.image.name_with_tag(); - let name_with_latest_tag = build.image.name_with_latest_tag(); - - let args = self.context.docker_build_options(); + let name_with_tag = build.image.full_image_name_with_tag(); + let name_with_latest_tag = format!("{}:latest", build.image.full_image_name()); let mut exit_status: Result<(), command::CommandError> = Err(command::CommandError::ExecutionError( Error::new(ErrorKind::InvalidData, "No builder names".to_string()), @@ -218,20 +197,13 @@ impl LocalDocker { for builder_name in BUILDPACKS_BUILDERS.iter() { let mut buildpacks_args = if !use_build_cache { - vec!["build", name_with_tag.as_str(), "--clear-cache"] + vec!["build", "--publish", name_with_tag.as_str(), "--clear-cache"] } else { - vec!["build", name_with_tag.as_str()] + vec!["build", "--publish", name_with_tag.as_str()] }; // always add 'latest' tag buildpacks_args.extend(vec!["-t", name_with_latest_tag.as_str()]); - - for v in args.iter() { - for s in v.iter() { - buildpacks_args.push(String::as_str(s)); - } - } - buildpacks_args.extend(vec!["--path", into_dir_docker_style]); let mut buildpacks_args = if env_var_args.is_empty() { @@ -414,69 +386,7 @@ impl BuildPlatform for LocalDocker { Ok(()) } - fn has_cache(&self, build: &Build) -> Result { - let event_details = self.get_event_details(); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe("LocalDocker.has_cache() called".to_string()), - ), - ); - - // Check if a local cache layers for the container image exists. - let repository_root_path = self.get_repository_build_root_path(&build)?; - - let parent_build = build.to_previous_build(repository_root_path).map_err(|err| { - EngineError::new_builder_get_build_error(self.get_event_details(), build.image.commit_id.to_string(), err) - })?; - - let parent_build = match parent_build { - Some(parent_build) => parent_build, - None => return Ok(CacheResult::MissWithoutParentBuild), - }; - - // check if local layers exist - let cmd_bin = "docker"; - let image_name = parent_build.image.name.clone(); - let cmd_args = vec!["images", "-q", &image_name]; - let mut cmd = QoveryCommand::new(cmd_bin, &cmd_args.clone(), &[]); - - let mut result = CacheResult::Miss(parent_build); - let _ = cmd.exec_with_timeout( - Duration::minutes(1), // `docker images` command can be slow with tons of images - it's probably not indexed - |_| result = CacheResult::Hit, // if a line is returned, then the image is locally present - |r_err| { - self.logger.log( - LogLevel::Error, - EngineEvent::Error( - EngineError::new_docker_cannot_list_images( - event_details.clone(), - CommandError::new_from_command_line( - "Cannot list docker images".to_string(), - cmd_bin.to_string(), - cmd_args.clone().into_iter().map(|v| v.to_string()).collect(), - vec![], - None, - Some(r_err.to_string()), - ), - ), - None, - ), - ) - }, - ); - - Ok(result) - } - - fn build( - &self, - build: Build, - force_build: bool, - is_task_canceled: &dyn Fn() -> bool, - ) -> Result { + fn build(&self, build: Build, is_task_canceled: &dyn Fn() -> bool) -> Result { let event_details = self.get_event_details(); self.logger.log( @@ -492,22 +402,6 @@ impl BuildPlatform for LocalDocker { } let listeners_helper = ListenersHelper::new(&self.listeners); - - if !force_build && self.image_does_exist(&build.image)? { - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(format!( - "Image `{}` found on repository, container build is not required", - build.image.name_with_tag() - )), - ), - ); - - return Ok(BuildResult { build }); - } - let repository_root_path = self.get_repository_build_root_path(&build)?; self.logger.log( @@ -551,6 +445,7 @@ impl BuildPlatform for LocalDocker { if is_task_canceled() { return Err(EngineError::new_task_cancellation_requested(event_details.clone())); } + if let Err(clone_error) = git::clone_at_commit( &build.git_repository.url, &build.git_repository.commit_id, @@ -669,7 +564,6 @@ impl BuildPlatform for LocalDocker { dockerfile_absolute_path.as_str(), build_context_path.as_str(), env_var_args, - !disable_build_cache, &listeners_helper, is_task_canceled, ) @@ -714,38 +608,6 @@ impl BuildPlatform for LocalDocker { result } - fn build_error(&self, build: Build) -> Result { - let event_details = self.get_event_details(); - self.logger.log( - LogLevel::Warning, - EngineEvent::Warning( - event_details.clone(), - EventMessage::new_from_safe(format!("LocalDocker.build_error() called for {}", self.name())), - ), - ); - - let listener_helper = ListenersHelper::new(&self.listeners); - - // FIXME - let message = String::from("something goes wrong (not implemented)"); - - listener_helper.error(ProgressInfo::new( - ProgressScope::Application { - id: build.image.application_id, - }, - ProgressLevel::Error, - Some(message.as_str()), - self.context.execution_id(), - )); - - let err = EngineError::new_not_implemented_error(event_details); - - self.logger.log(LogLevel::Error, EngineEvent::Error(err.clone(), None)); - - // FIXME - Err(err) - } - fn logger(&self) -> Box { self.logger.clone() } @@ -814,6 +676,7 @@ fn docker_prune_images(envs: Vec<(&str, &str)>) -> Result<(), CommandError> { vec!["image", "prune", "-a", "-f"], vec!["builder", "prune", "-a", "-f"], vec!["volume", "prune", "-f"], + vec!["buildx", "prune", "-a", "-f"], ]; let mut errored_commands = vec![]; diff --git a/src/build_platform/mod.rs b/src/build_platform/mod.rs index 5480fb37..0b9ef095 100644 --- a/src/build_platform/mod.rs +++ b/src/build_platform/mod.rs @@ -1,15 +1,11 @@ use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; -use crate::errors::{CommandError, EngineError}; +use crate::errors::EngineError; use crate::events::{EnvironmentStep, EventDetails, Stage, ToTransmitter}; -use crate::git; use crate::logger::Logger; use crate::models::{Context, Listen, QoveryIdentifier}; -use crate::utilities::get_image_tag; -use git2::{Cred, CredentialType}; use std::fmt::{Display, Formatter, Result as FmtResult}; -use std::path::Path; +use url::Url; pub mod docker; pub mod local_docker; @@ -23,14 +19,7 @@ pub trait BuildPlatform: ToTransmitter + Listen { format!("{} ({})", self.name(), self.id()) } fn is_valid(&self) -> Result<(), EngineError>; - fn has_cache(&self, build: &Build) -> Result; - fn build( - &self, - build: Build, - force_build: bool, - is_task_canceled: &dyn Fn() -> bool, - ) -> Result; - fn build_error(&self, build: Build) -> Result; + fn build(&self, build: Build, is_task_canceled: &dyn Fn() -> bool) -> Result; fn logger(&self) -> Box; fn get_event_details(&self) -> EventDetails { let context = self.context(); @@ -52,63 +41,6 @@ pub struct Build { pub options: BuildOptions, } -impl Build { - pub fn to_previous_build

(&self, clone_repo_into_dir: P) -> Result, CommandError> - where - P: AsRef, - { - let parent_commit_id = git::get_parent_commit_id( - self.git_repository.url.as_str(), - self.git_repository.commit_id.as_str(), - clone_repo_into_dir, - &|_| match &self.git_repository.credentials { - None => vec![], - Some(creds) => vec![( - CredentialType::USER_PASS_PLAINTEXT, - Cred::userpass_plaintext(creds.login.as_str(), creds.password.as_str()).unwrap(), - )], - }, - ) - .map_err(|err| CommandError::new(err.to_string(), Some("Cannot get parent commit ID.".to_string())))?; - - let parent_commit_id = match parent_commit_id { - None => return Ok(None), - Some(parent_commit_id) => parent_commit_id, - }; - - let mut environment_variables_map = BTreeMap::::new(); - for env in &self.options.environment_variables { - environment_variables_map.insert(env.key.clone(), env.value.clone()); - } - - let mut image = self.image.clone(); - image.tag = get_image_tag( - &self.git_repository.root_path, - &self.git_repository.dockerfile_path, - &environment_variables_map, - &parent_commit_id, - ); - - image.commit_id = parent_commit_id.clone(); - - Ok(Some(Build { - git_repository: GitRepository { - url: self.git_repository.url.clone(), - credentials: self.git_repository.credentials.clone(), - ssh_keys: self.git_repository.ssh_keys.clone(), - commit_id: parent_commit_id, - dockerfile_path: self.git_repository.dockerfile_path.clone(), - root_path: self.git_repository.root_path.clone(), - buildpack_language: self.git_repository.buildpack_language.clone(), - }, - image, - options: BuildOptions { - environment_variables: self.options.environment_variables.clone(), - }, - })) - } -} - pub struct BuildOptions { pub environment_variables: Vec, } @@ -149,22 +81,33 @@ pub struct Image { pub tag: String, pub commit_id: String, // registry name where the image has been pushed: Optional - pub registry_name: Option, + pub registry_name: String, // registry docker json config: Optional pub registry_docker_json_config: Option, - // registry secret to pull image: Optional - pub registry_secret: Option, // complete registry URL where the image has been pushed - pub registry_url: Option, + pub registry_url: Url, } impl Image { - pub fn name_with_tag(&self) -> String { - format!("{}:{}", self.name, self.tag) + pub fn registry_host(&self) -> &str { + self.registry_url.host_str().unwrap() } - pub fn name_with_latest_tag(&self) -> String { - format!("{}:latest", self.name) + pub fn full_image_name_with_tag(&self) -> String { + format!( + "{}/{}:{}", + self.registry_url.host_str().unwrap_or_default(), + self.name, + self.tag + ) + } + + pub fn full_image_name(&self) -> String { + format!("{}/{}", self.registry_url.host_str().unwrap_or_default(), self.name,) + } + + pub fn name(&self) -> String { + self.name.clone() } } @@ -175,10 +118,9 @@ impl Default for Image { name: "".to_string(), tag: "".to_string(), commit_id: "".to_string(), - registry_name: None, + registry_name: "".to_string(), registry_docker_json_config: None, - registry_secret: None, - registry_url: None, + registry_url: Url::parse("https://default.com").unwrap(), } } } @@ -208,11 +150,3 @@ impl BuildResult { pub enum Kind { LocalDocker, } - -type ParentBuild = Build; - -pub enum CacheResult { - MissWithoutParentBuild, - Miss(ParentBuild), - Hit, -} diff --git a/src/cloud_provider/aws/application.rs b/src/cloud_provider/aws/application.rs index 7768f164..cc11139a 100644 --- a/src/cloud_provider/aws/application.rs +++ b/src/cloud_provider/aws/application.rs @@ -15,8 +15,8 @@ use crate::cloud_provider::DeploymentTarget; use crate::cmd::helm::Timeout; use crate::cmd::kubectl::ScalingKind::{Deployment, Statefulset}; use crate::errors::EngineError; -use crate::events::{EngineEvent, EnvironmentStep, EventMessage, Stage, ToTransmitter, Transmitter}; -use crate::logger::{LogLevel, Logger}; +use crate::events::{EnvironmentStep, Stage, ToTransmitter, Transmitter}; +use crate::logger::Logger; use crate::models::{Context, Listen, Listener, Listeners, ListenersHelper, Port}; use ::function_name::named; @@ -201,26 +201,7 @@ impl Service for Application { let commit_id = self.image().commit_id.as_str(); context.insert("helm_app_version", &commit_id[..7]); - - match &self.image().registry_url { - Some(registry_url) => context.insert("image_name_with_tag", registry_url.as_str()), - None => { - let image_name_with_tag = self.image().name_with_tag(); - - self.logger().log( - LogLevel::Warning, - EngineEvent::Warning( - event_details.clone(), - EventMessage::new_from_safe(format!( - "there is no registry url, use image name with tag with the default container registry: {}", - image_name_with_tag.as_str() - )), - ), - ); - - context.insert("image_name_with_tag", image_name_with_tag.as_str()); - } - } + context.insert("image_name_with_tag", &self.image.full_image_name_with_tag()); let environment_variables = self .environment_variables @@ -233,16 +214,8 @@ impl Service for Application { context.insert("environment_variables", &environment_variables); context.insert("ports", &self.ports); - - match self.image.registry_name.as_ref() { - Some(registry_name) => { - context.insert("is_registry_secret", &true); - context.insert("registry_secret", registry_name); - } - None => { - context.insert("is_registry_secret", &false); - } - }; + context.insert("is_registry_secret", &true); + context.insert("registry_secret", self.image.registry_host()); let cpu_limits = match validate_k8s_required_cpu_and_burstable( &ListenersHelper::new(&self.listeners), diff --git a/src/cloud_provider/digitalocean/application.rs b/src/cloud_provider/digitalocean/application.rs index b7dd0d2b..bcb3454b 100644 --- a/src/cloud_provider/digitalocean/application.rs +++ b/src/cloud_provider/digitalocean/application.rs @@ -15,8 +15,8 @@ use crate::cloud_provider::DeploymentTarget; use crate::cmd::helm::Timeout; use crate::cmd::kubectl::ScalingKind::{Deployment, Statefulset}; use crate::errors::{CommandError, EngineError}; -use crate::events::{EngineEvent, EnvironmentStep, EventMessage, Stage, ToTransmitter, Transmitter}; -use crate::logger::{LogLevel, Logger}; +use crate::events::{EnvironmentStep, Stage, ToTransmitter, Transmitter}; +use crate::logger::Logger; use crate::models::{Context, Listen, Listener, Listeners, ListenersHelper, Port}; use ::function_name::named; use std::fmt; @@ -205,26 +205,7 @@ impl Service for Application { let commit_id = self.image.commit_id.as_str(); context.insert("helm_app_version", &commit_id[..7]); - - match &self.image.registry_url { - Some(registry_url) => context.insert("image_name_with_tag", registry_url.as_str()), - None => { - let image_name_with_tag = self.image.name_with_tag(); - - self.logger().log( - LogLevel::Warning, - EngineEvent::Warning( - event_details.clone(), - EventMessage::new_from_safe(format!( - "there is no registry url, use image name with tag with the default container registry: {}", - image_name_with_tag.as_str() - )), - ), - ); - - context.insert("image_name_with_tag", image_name_with_tag.as_str()); - } - } + context.insert("image_name_with_tag", &self.image.full_image_name_with_tag()); let cpu_limits = match validate_k8s_required_cpu_and_burstable( &ListenersHelper::new(&self.listeners), @@ -258,16 +239,8 @@ impl Service for Application { context.insert("environment_variables", &environment_variables); context.insert("ports", &self.ports); - - if self.image.registry_name.is_some() { - context.insert("is_registry_secret", &true); - context.insert( - "registry_secret", - &"do-container-registry-secret-for-cluster".to_string(), - ); - } else { - context.insert("is_registry_secret", &false); - }; + context.insert("is_registry_secret", &true); + context.insert("registry_secret", self.image.registry_host()); let storage = self .storage diff --git a/src/cloud_provider/scaleway/application.rs b/src/cloud_provider/scaleway/application.rs index 7405d0bf..0c9c2abe 100644 --- a/src/cloud_provider/scaleway/application.rs +++ b/src/cloud_provider/scaleway/application.rs @@ -18,8 +18,8 @@ use crate::cloud_provider::DeploymentTarget; use crate::cmd::helm::Timeout; use crate::cmd::kubectl::ScalingKind::{Deployment, Statefulset}; use crate::errors::{CommandError, EngineError}; -use crate::events::{EngineEvent, EnvironmentStep, EventMessage, Stage, ToTransmitter, Transmitter}; -use crate::logger::{LogLevel, Logger}; +use crate::events::{EnvironmentStep, Stage, ToTransmitter, Transmitter}; +use crate::logger::Logger; use crate::models::{Context, Listen, Listener, Listeners, ListenersHelper, Port}; use ::function_name::named; @@ -206,27 +206,7 @@ impl Service for Application { let commit_id = self.image().commit_id.as_str(); context.insert("helm_app_version", &commit_id[..7]); - - match &self.image().registry_url { - Some(registry_url) => context.insert( - "image_name_with_tag", - format!("{}/{}", registry_url.as_str(), self.image().name_with_tag()).as_str(), - ), - None => { - let image_name_with_tag = self.image().name_with_tag(); - self.logger().log( - LogLevel::Warning, - EngineEvent::Warning( - event_details.clone(), - EventMessage::new_from_safe(format!( - "there is no registry url, use image name with tag with the default container registry: {}", - image_name_with_tag.as_str() - )), - ), - ); - context.insert("image_name_with_tag", image_name_with_tag.as_str()); - } - } + context.insert("image_name_with_tag", &self.image.full_image_name_with_tag()); let environment_variables = self .environment_variables @@ -239,16 +219,8 @@ impl Service for Application { context.insert("environment_variables", &environment_variables); context.insert("ports", &self.ports); - - match self.image.registry_name.as_ref() { - Some(_) => { - context.insert("is_registry_secret", &true); - context.insert("registry_secret_name", &format!("registry-token-{}", &self.id)); - } - None => { - context.insert("is_registry_secret", &false); - } - }; + context.insert("is_registry_secret", &true); + context.insert("registry_secret_name", &format!("registry-token-{}", &self.id)); let cpu_limits = match validate_k8s_required_cpu_and_burstable( &ListenersHelper::new(&self.listeners), diff --git a/src/cmd/docker.rs b/src/cmd/docker.rs new file mode 100644 index 00000000..5765405b --- /dev/null +++ b/src/cmd/docker.rs @@ -0,0 +1,682 @@ +use crate::cmd::command::{CommandError, QoveryCommand}; +use crate::errors::EngineError; +use crate::events::EventDetails; +use chrono::Duration; +use std::path::Path; +use std::process::ExitStatus; +use url::Url; + +#[derive(thiserror::Error, Debug)] +pub enum DockerError { + #[error("Docker Invalid configuration: {0}")] + InvalidConfig(String), + + #[error("Docker terminated with an unknown error: {0}")] + ExecutionError(#[from] std::io::Error), + + #[error("Docker terminated with a non success exit status code: {0}")] + ExitStatusError(ExitStatus), + + #[error("Docker aborted due to user cancel request: {0}")] + Aborted(String), + + #[error("Docker command terminated due to timeout: {0}")] + Timeout(String), +} + +#[derive(Debug)] +pub struct ContainerImage { + pub registry: Url, + pub name: String, + pub tags: Vec, +} + +impl ContainerImage { + pub fn image_names(&self) -> Vec { + let host = if let Some(port) = self.registry.port() { + format!("{}:{}", self.registry.host_str().unwrap_or_default(), port) + } else { + self.registry.host_str().unwrap_or_default().to_string() + }; + + self.tags + .iter() + .map(|tag| format!("{}/{}:{}", host, &self.name, tag)) + .collect() + } + + pub fn image_name(&self) -> String { + self.image_names().remove(0) + } +} + +pub struct Docker { + use_buildkit: bool, + common_envs: Vec<(String, String)>, +} + +impl Docker { + pub fn new_with_options(enable_buildkit: bool, socket_location: Option) -> Result { + let mut docker = Docker { + use_buildkit: enable_buildkit, + common_envs: vec![( + "DOCKER_BUILDKIT".to_string(), + if enable_buildkit { + "1".to_string() + } else { + "0".to_string() + }, + )], + }; + + // Override DOCKER_HOST if we use a TCP socket + if let Some(socket_location) = socket_location { + docker + .common_envs + .push(("DOCKER_HOST".to_string(), socket_location.to_string())) + } + + // If we don't use buildkit nothing more to do + if !docker.use_buildkit { + return Ok(docker); + } + + // First check that the buildx plugin is correctly installed + let args = vec!["buildx", "version"]; + let buildx_cmd_exist = docker_exec( + &args, + &docker.get_all_envs(&vec![]), + Some(Duration::max_value()), + &|| false, + |_| {}, + |_| {}, + ); + if let Err(_) = buildx_cmd_exist { + return Err(DockerError::InvalidConfig(format!( + "Docker buildx plugin for buildkit is not correctly installed" + ))); + } + + // In order to be able to use --cache-from --cache-to for buildkit, + // we need to create our specific builder, which is not the default one (aka: the docker one) + let args = vec![ + "buildx", + "create", + "--name", + "qovery-engine", + "--driver-opt", + "network=host", + "--use", + ]; + let _ = docker_exec( + &args, + &docker.get_all_envs(&vec![]), + Some(Duration::max_value()), + &|| false, + |_| {}, + |_| {}, + ); + + Ok(docker) + } + + pub fn new(socket_location: Option) -> Result { + Self::new_with_options(true, socket_location) + } + + fn get_all_envs<'a>(&'a self, envs: &'a [(&'a str, &'a str)]) -> Vec<(&'a str, &'a str)> { + let mut all_envs: Vec<(&str, &str)> = self.common_envs.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect(); + all_envs.append(&mut envs.to_vec()); + + all_envs + } + + pub fn login(&self, registry: &Url) -> Result<(), DockerError> { + info!("Docker login {} as user {}", registry, registry.username()); + let password = urlencoding::decode(®istry.password().unwrap_or_default()) + .unwrap_or_default() + .to_string(); + let args = vec![ + "login", + registry.host_str().unwrap_or_default(), + "-u", + registry.username(), + "-p", + &password, + ]; + + docker_exec( + &args, + &self.get_all_envs(&vec![]), + None, + &|| false, + |line| info!("{}", line), + |line| warn!("{}", line), + )?; + + Ok(()) + } + + pub fn does_image_exist_locally(&self, image: &ContainerImage) -> Result { + info!("Docker check locally image exist {:?}", image); + + let ret = docker_exec( + &vec!["image", "inspect", &image.image_name()], + &self.get_all_envs(&vec![]), + None, + &|| false, + |line| info!("{}", line), + |line| warn!("{}", line), + ); + + Ok(matches!(ret, Ok(_))) + } + + // Warning: this command is slow > 10 sec + pub fn does_image_exist_remotely(&self, image: &ContainerImage) -> Result { + info!("Docker check remotely image exist {:?}", image); + + let ret = docker_exec( + &vec!["manifest", "inspect", &image.image_name()], + &self.get_all_envs(&vec![]), + None, + &|| false, + |line| info!("{}", line), + |line| warn!("{}", line), + ); + + match ret { + Ok(_) => Ok(true), + Err(DockerError::ExitStatusError(_)) => Ok(false), + Err(err) => Err(err), + } + } + + pub fn pull( + &self, + image: &ContainerImage, + stdout_output: Stdout, + stderr_output: Stderr, + timeout: Duration, + should_abort: &dyn Fn() -> bool, + ) -> Result<(), DockerError> + where + Stdout: FnMut(String), + Stderr: FnMut(String), + { + info!("Docker pull {:?}, timeout: {:?}", image, timeout); + + docker_exec( + &vec!["pull", &image.image_name()], + &self.get_all_envs(&vec![]), + Some(timeout), + should_abort, + stdout_output, + stderr_output, + ) + } + + pub fn build( + &self, + dockerfile: &Path, + context: &Path, + image_to_build: &ContainerImage, + build_args: &[(&str, &str)], + cache: &ContainerImage, + push_after_build: bool, + stdout_output: Stdout, + stderr_output: Stderr, + timeout: Duration, + should_abort: &dyn Fn() -> bool, + ) -> Result<(), DockerError> + where + Stdout: FnMut(String), + Stderr: FnMut(String), + { + // if there is no tags, nothing to build + if image_to_build.tags.is_empty() { + return Ok(()); + } + + // if it is already aborted, nothing to do + if (should_abort)() { + return Err(DockerError::Aborted("build".to_string())); + } + + // Do some checks + if !dockerfile.is_file() { + return Err(DockerError::InvalidConfig(format!( + "provided dockerfile `{:?}` is not a valid file", + dockerfile + ))); + } + + if !context.is_dir() { + return Err(DockerError::InvalidConfig(format!( + "provided docker build context `{:?}` is not a valid directory", + context + ))); + } + + if self.use_buildkit { + self.build_with_buildkit( + dockerfile, + context, + image_to_build, + build_args, + cache, + push_after_build, + stdout_output, + stderr_output, + timeout, + should_abort, + ) + } else { + self.build_with_docker( + dockerfile, + context, + image_to_build, + build_args, + cache, + push_after_build, + stdout_output, + stderr_output, + timeout, + should_abort, + ) + } + } + + fn build_with_docker( + &self, + dockerfile: &Path, + context: &Path, + image_to_build: &ContainerImage, + build_args: &[(&str, &str)], + cache: &ContainerImage, + push_after_build: bool, + stdout_output: Stdout, + stderr_output: Stderr, + timeout: Duration, + should_abort: &dyn Fn() -> bool, + ) -> Result<(), DockerError> + where + Stdout: FnMut(String), + Stderr: FnMut(String), + { + info!("Docker build {:?}", image_to_build.image_name()); + + // Best effort to pull the cache, if it does not exist that's ok too + let _ = self.pull(cache, |_| {}, |_| {}, timeout, should_abort); + + let mut args_string: Vec = vec![ + "build".to_string(), + "--network".to_string(), + "host".to_string(), + "-f".to_string(), + dockerfile.to_str().unwrap_or_default().to_string(), + ]; + + for image_name in image_to_build.image_names() { + args_string.push("--tag".to_string()); + args_string.push(image_name) + } + + for img_cache_name in cache.image_names() { + args_string.push("--tag".to_string()); + args_string.push(img_cache_name) + } + + for (k, v) in build_args { + args_string.push("--build-arg".to_string()); + args_string.push(format!("{}={}", k, v)); + } + + args_string.push(context.to_str().unwrap_or_default().to_string()); + + let _ = docker_exec( + &args_string.iter().map(|x| x.as_str()).collect::>(), + &self.get_all_envs(&vec![]), + Some(timeout), + should_abort, + stdout_output, + stderr_output, + )?; + + if push_after_build { + let _ = self.push(image_to_build, |_| {}, |_| {}, timeout, should_abort)?; + } + + Ok(()) + } + + fn build_with_buildkit( + &self, + dockerfile: &Path, + context: &Path, + image_to_build: &ContainerImage, + build_args: &[(&str, &str)], + cache: &ContainerImage, + push_after_build: bool, + stdout_output: Stdout, + stderr_output: Stderr, + timeout: Duration, + should_abort: &dyn Fn() -> bool, + ) -> Result<(), DockerError> + where + Stdout: FnMut(String), + Stderr: FnMut(String), + { + info!("Docker buildkit build {:?}", image_to_build.image_name()); + + let mut args_string: Vec = vec![ + "buildx".to_string(), + "build".to_string(), + "--progress=plain".to_string(), + "--network=host".to_string(), + if push_after_build { + "--output=type=registry".to_string() // tell buildkit to push image to registry + } else { + "--output=type=docker".to_string() // tell buildkit to load the image into docker after build + }, + "--cache-from".to_string(), + format!("type=registry,ref={}", cache.image_name()), + // Disabled for now, because private ECR does not support it ... + // https://github.com/aws/containers-roadmap/issues/876 + // "--cache-to".to_string(), + // format!("type=registry,ref={}", cache.image_name()), + "-f".to_string(), + dockerfile.to_str().unwrap_or_default().to_string(), + ]; + + for image_name in image_to_build.image_names() { + args_string.push("--tag".to_string()); + args_string.push(image_name.to_string()) + } + + for (k, v) in build_args { + args_string.push("--build-arg".to_string()); + args_string.push(format!("{}={}", k, v)); + } + + args_string.push(context.to_str().unwrap_or_default().to_string()); + + docker_exec( + &args_string.iter().map(|x| x.as_str()).collect::>(), + &self.get_all_envs(&vec![]), + Some(timeout), + should_abort, + stdout_output, + stderr_output, + ) + } + + pub fn push( + &self, + image: &ContainerImage, + stdout_output: Stdout, + stderr_output: Stderr, + timeout: Duration, + should_abort: &dyn Fn() -> bool, + ) -> Result<(), DockerError> + where + Stdout: FnMut(String), + Stderr: FnMut(String), + { + info!("Docker push {:?}, timeout: {:?}", image, timeout); + let image_names = image.image_names(); + let mut args = vec!["push"]; + args.extend(image_names.iter().map(|x| x.as_str())); + + docker_exec( + &args, + &self.get_all_envs(&vec![]), + Some(timeout), + should_abort, + stdout_output, + stderr_output, + ) + } +} + +fn docker_exec( + args: &[&str], + envs: &[(&str, &str)], + timeout: Option, + should_abort: &dyn Fn() -> bool, + stdout_output: F, + stderr_output: X, +) -> Result<(), DockerError> +where + F: FnMut(String), + X: FnMut(String), +{ + let timeout = timeout.unwrap_or_else(|| Duration::max_value()); + let mut cmd = QoveryCommand::new("docker", args, envs); + let ret = cmd.exec_with_abort(timeout, stdout_output, stderr_output, should_abort); + + match ret { + Ok(_) => Ok(()), + Err(CommandError::TimeoutError(msg)) => Err(DockerError::Timeout(msg)), + Err(CommandError::Killed(msg)) => Err(DockerError::Aborted(msg)), + Err(CommandError::ExitStatusError(err)) => Err(DockerError::ExitStatusError(err)), + Err(CommandError::ExecutionError(err)) => Err(DockerError::ExecutionError(err)), + } +} + +pub fn to_engine_error(event_details: &EventDetails, error: DockerError) -> EngineError { + EngineError::new_docker_error(event_details.clone(), error) +} + +// start a local registry to run this test +// docker run --rm -ti -p 5000:5000 --name registry registry:2 +#[cfg(feature = "test-with-docker")] +#[cfg(test)] +mod tests { + use crate::cmd::docker::{ContainerImage, Docker, DockerError}; + use chrono::Duration; + use std::path::Path; + use url::Url; + + fn private_registry_url() -> Url { + Url::parse("http://localhost:5000").unwrap() + } + + #[test] + fn test_pull() { + let docker = Docker::new(None).unwrap(); + + // Invalid image should fails + let image = ContainerImage { + registry: Url::parse("https://docker.io").unwrap(), + name: "alpine".to_string(), + tags: vec!["666".to_string()], + }; + let ret = docker.pull( + &image, + |msg| println!("{}", msg), + |msg| eprintln!("{}", msg), + Duration::max_value(), + &|| false, + ); + assert!(matches!(ret, Err(_))); + + // Valid image should be ok + let image = ContainerImage { + registry: Url::parse("https://docker.io").unwrap(), + name: "alpine".to_string(), + tags: vec!["3.15".to_string()], + }; + + let ret = docker.pull( + &image, + |msg| println!("{}", msg), + |msg| eprintln!("{}", msg), + Duration::max_value(), + &|| false, + ); + assert!(matches!(ret, Ok(_))); + + // Should timeout + let ret = docker.pull( + &image, + |msg| println!("{}", msg), + |msg| eprintln!("{}", msg), + Duration::seconds(1), + &|| false, + ); + assert!(matches!(ret, Err(DockerError::Timeout(_)))); + } + + #[test] + fn test_docker_build() { + // start a local registry to run this test + // docker run --rm -d -p 5000:5000 --name registry registry:2 + let docker = Docker::new_with_options(false, None).unwrap(); + let image_to_build = ContainerImage { + registry: private_registry_url(), + name: "erebe/alpine".to_string(), + tags: vec!["3.15".to_string()], + }; + let image_cache = ContainerImage { + registry: private_registry_url(), + name: "erebe/alpine".to_string(), + tags: vec!["cache".to_string()], + }; + + let ret = docker.build_with_docker( + Path::new("tests/docker/multi_stage_simple/Dockerfile"), + Path::new("tests/docker/multi_stage_simple/"), + &image_to_build, + &vec![], + &image_cache, + false, + |msg| println!("{}", msg), + |msg| eprintln!("{}", msg), + Duration::max_value(), + &|| false, + ); + + assert!(matches!(ret, Ok(_))); + + // It should fails with buildkit dockerfile + let ret = docker.build_with_docker( + Path::new("tests/docker/multi_stage_simple/Dockerfile.buildkit"), + Path::new("tests/docker/multi_stage_simple/"), + &image_to_build, + &vec![], + &image_cache, + false, + |msg| println!("{}", msg), + |msg| eprintln!("{}", msg), + Duration::max_value(), + &|| false, + ); + + assert!(matches!(ret, Err(_))); + } + + #[test] + fn test_buildkit_build() { + // start a local registry to run this test + // docker run --rm -d -p 5000:5000 --name registry registry:2 + let docker = Docker::new_with_options(true, None).unwrap(); + let image_to_build = ContainerImage { + registry: private_registry_url(), + name: "erebe/alpine".to_string(), + tags: vec!["3.15".to_string()], + }; + let image_cache = ContainerImage { + registry: private_registry_url(), + name: "erebe/alpine".to_string(), + tags: vec!["cache".to_string()], + }; + + // It should work + let ret = docker.build_with_buildkit( + Path::new("tests/docker/multi_stage_simple/Dockerfile"), + Path::new("tests/docker/multi_stage_simple/"), + &image_to_build, + &vec![], + &image_cache, + false, + |msg| println!("{}", msg), + |msg| eprintln!("{}", msg), + Duration::max_value(), + &|| false, + ); + + assert!(matches!(ret, Ok(_))); + + let ret = docker.build_with_buildkit( + Path::new("tests/docker/multi_stage_simple/Dockerfile.buildkit"), + Path::new("tests/docker/multi_stage_simple/"), + &image_to_build, + &vec![], + &image_cache, + false, + |msg| println!("{}", msg), + |msg| eprintln!("{}", msg), + Duration::max_value(), + &|| false, + ); + + assert!(matches!(ret, Ok(_))); + } + + #[test] + fn test_push() { + // start a local registry to run this test + // docker run --rm -d -p 5000:5000 --name registry registry:2 + let docker = Docker::new_with_options(true, None).unwrap(); + let image_to_build = ContainerImage { + registry: private_registry_url(), + name: "erebe/alpine".to_string(), + tags: vec!["3.15".to_string()], + }; + let image_cache = ContainerImage { + registry: private_registry_url(), + name: "erebe/alpine".to_string(), + tags: vec!["cache".to_string()], + }; + + // It should work + let ret = docker.build_with_buildkit( + Path::new("tests/docker/multi_stage_simple/Dockerfile"), + Path::new("tests/docker/multi_stage_simple/"), + &image_to_build, + &vec![], + &image_cache, + false, + |msg| println!("{}", msg), + |msg| eprintln!("{}", msg), + Duration::max_value(), + &|| false, + ); + assert!(matches!(ret, Ok(_))); + + let ret = docker.does_image_exist_locally(&image_to_build); + assert!(matches!(ret, Ok(true))); + + let ret = docker.does_image_exist_remotely(&image_to_build); + assert!(matches!(ret, Ok(false))); + + let ret = docker.push( + &image_to_build, + |msg| println!("{}", msg), + |msg| eprintln!("{}", msg), + Duration::max_value(), + &|| false, + ); + assert!(matches!(ret, Ok(_))); + + let ret = docker.pull( + &image_to_build, + |msg| println!("{}", msg), + |msg| eprintln!("{}", msg), + Duration::max_value(), + &|| false, + ); + assert!(matches!(ret, Ok(_))); + } +} diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs index f486a56a..153aab46 100644 --- a/src/cmd/mod.rs +++ b/src/cmd/mod.rs @@ -1,4 +1,5 @@ pub mod command; +pub mod docker; pub mod helm; pub mod kubectl; pub mod structs; diff --git a/src/container_registry/docker.rs b/src/container_registry/docker.rs deleted file mode 100644 index 4eaa688d..00000000 --- a/src/container_registry/docker.rs +++ /dev/null @@ -1,462 +0,0 @@ -use crate::build_platform::Image; -use crate::cmd; -use crate::cmd::command::QoveryCommand; -use crate::container_registry::Kind; -use crate::errors::CommandError; -use crate::events::{EngineEvent, EventDetails, EventMessage}; -use crate::logger::{LogLevel, Logger}; -use chrono::Duration; -use retry::delay::Fibonacci; -use retry::Error::Operation; -use retry::OperationResult; - -#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DockerImageManifest { - pub schema_version: i64, - pub media_type: String, - pub config: Config, - pub layers: Vec, -} - -#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Config { - pub media_type: String, - pub size: i64, - pub digest: String, -} - -#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Layer { - pub media_type: String, - pub size: i64, - pub digest: String, -} - -pub fn docker_manifest_inspect( - container_registry_kind: Kind, - docker_envs: Vec<(&str, &str)>, - image_name: String, - image_tag: String, - registry_url: String, - event_details: EventDetails, - logger: &dyn Logger, -) -> Result { - let image_with_tag = format!("{}:{}", image_name, image_tag); - let registry_provider = match container_registry_kind { - Kind::DockerHub => "DockerHub", - Kind::Ecr => "AWS ECR", - Kind::Docr => "DigitalOcean Registry", - Kind::ScalewayCr => "Scaleway Registry", - }; - - // Note: `docker manifest inspect` is still experimental for the time being: - // https://docs.docker.com/engine/reference/commandline/manifest_inspect/ - let mut envs = docker_envs.clone(); - envs.push(("DOCKER_CLI_EXPERIMENTAL", "enabled")); - - let binary = "docker"; - let image_full_url = format!("{}/{}", registry_url.as_str(), &image_with_tag); - let args = vec!["manifest", "inspect", image_full_url.as_str()]; - let mut raw_output: Vec = vec![]; - - let mut cmd = QoveryCommand::new("docker", &args, &envs); - return match cmd.exec_with_timeout(Duration::minutes(1), |line| raw_output.push(line), |_| {}) { - Ok(_) => { - let joined = raw_output.join(""); - match serde_json::from_str(&joined) { - Ok(extracted_manifest) => Ok(extracted_manifest), - Err(e) => { - let error = CommandError::new( - e.to_string(), - Some(format!( - "Error while trying to deserialize manifest image manifest for image {} in {} ({}).", - image_with_tag, registry_provider, registry_url, - )), - ); - - logger.log( - LogLevel::Warning, - EngineEvent::Warning(event_details.clone(), EventMessage::from(error.clone())), - ); - - Err(error) - } - } - } - Err(e) => { - let error = CommandError::new( - format!( - "Command `{}`: {:?}", - cmd::command::command_to_string(binary, &args, &envs), - e - ), - Some(format!( - "Error while trying to inspect image manifest for image {} in {} ({}).", - image_with_tag, registry_provider, registry_url, - )), - ); - - logger.log( - LogLevel::Warning, - EngineEvent::Warning(event_details.clone(), EventMessage::from(error.clone())), - ); - - Err(error) - } - }; -} - -pub fn docker_login( - container_registry_kind: Kind, - docker_envs: Vec<(&str, &str)>, - registry_login: String, - registry_pass: String, - registry_url: String, - event_details: EventDetails, - logger: &dyn Logger, -) -> Result<(), CommandError> { - let registry_provider = match container_registry_kind { - Kind::DockerHub => "DockerHub", - Kind::Ecr => "AWS ECR", - Kind::Docr => "DigitalOcean Registry", - Kind::ScalewayCr => "Scaleway Registry", - }; - - let binary = "docker"; - let args = vec![ - "login", - registry_url.as_str(), - "-u", - registry_login.as_str(), - "-p", - registry_pass.as_str(), - ]; - - let mut cmd = QoveryCommand::new(binary, &args, &docker_envs); - match cmd.exec() { - Ok(_) => Ok(()), - Err(e) => { - let err = CommandError::new( - format!( - "Command `{}`: {:?}", - cmd::command::command_to_string(binary, &args, &docker_envs), - e, - ), - Some(format!( - "Error while trying to login to registry {} {}.", - registry_provider, registry_url, - )), - ); - - logger.log( - LogLevel::Warning, - EngineEvent::Warning(event_details.clone(), EventMessage::from(err.clone())), - ); - - Err(err) - } - } -} - -pub fn docker_tag_and_push_image( - container_registry_kind: Kind, - docker_envs: Vec<(&str, &str)>, - image: &Image, - dest: String, - dest_latest_tag: String, - event_details: EventDetails, - logger: &dyn Logger, -) -> Result<(), CommandError> { - let image_with_tag = image.name_with_tag(); - let registry_provider = match container_registry_kind { - Kind::DockerHub => "DockerHub", - Kind::Ecr => "AWS ECR", - Kind::Docr => "DigitalOcean Registry", - Kind::ScalewayCr => "Scaleway Registry", - }; - - let binary = "docker"; - let args = vec!["tag", &image_with_tag, dest.as_str()]; - let mut cmd = QoveryCommand::new(binary, &args, &docker_envs); - match retry::retry(Fibonacci::from_millis(3000).take(5), || match cmd.exec() { - Ok(_) => OperationResult::Ok(()), - Err(e) => { - logger.log( - LogLevel::Warning, - EngineEvent::Warning( - event_details.clone(), - EventMessage::new( - format!("Failed to tag image `{}`, retrying...", image_with_tag), - Some(format!( - "Command `{}`: {:?}", - cmd::command::command_to_string(binary, &args, &docker_envs), - e - )), - ), - ), - ); - - OperationResult::Retry(e) - } - }) { - Err(Operation { error, .. }) => { - logger.log( - LogLevel::Warning, - EngineEvent::Warning( - event_details.clone(), - EventMessage::from(CommandError::new_from_legacy_command_error( - error, - Some(format!("Error while trying to tag docker image `{}`", image_with_tag)), - )), - ), - ); - } - Err(retry::Error::Internal(msg)) => { - logger.log( - LogLevel::Warning, - EngineEvent::Warning( - event_details.clone(), - EventMessage::from(CommandError::new( - msg, - Some(format!("Error while trying to tag docker image `{}`", image_with_tag)), - )), - ), - ); - } - Ok(_) => {} - } - - let mut cmd = QoveryCommand::new("docker", &vec!["push", dest.as_str()], &docker_envs); - let _ = match retry::retry(Fibonacci::from_millis(5000).take(5), || { - match cmd.exec_with_timeout( - Duration::minutes(10), - |line| { - logger.log( - LogLevel::Info, - EngineEvent::Info(event_details.clone(), EventMessage::new(line, None)), - ) - }, - |line| { - logger.log( - LogLevel::Warning, - EngineEvent::Warning(event_details.clone(), EventMessage::new(line, None)), - ) - }, - ) { - Ok(_) => OperationResult::Ok(()), - Err(e) => { - logger.log( - LogLevel::Warning, - EngineEvent::Warning( - event_details.clone(), - EventMessage::new( - format!( - "Failed to push image `{}` on `{}`, retrying ...", - image_with_tag, registry_provider - ), - Some(format!("{:?}", e)), - ), - ), - ); - - OperationResult::Retry(e) - } - } - }) { - Err(Operation { error, .. }) => Err(CommandError::new_from_legacy_command_error( - error, - Some(format!("Failed to push docker image `{}`", image_with_tag)), - )), - Err(retry::Error::Internal(msg)) => Err(CommandError::new( - msg, - Some(format!("Failed to push docker image `{}`", image_with_tag)), - )), - _ => { - logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(format!( - "Image {} has successfully been pushed on `{}`", - image_with_tag, registry_provider - )), - ), - ); - - Ok(()) - } - }; - - let image_with_latest_tag = image.name_with_latest_tag(); - let mut cmd = QoveryCommand::new( - "docker", - &vec!["tag", &image_with_latest_tag, dest_latest_tag.as_str()], - &docker_envs, - ); - match retry::retry(Fibonacci::from_millis(3000).take(5), || match cmd.exec() { - Ok(_) => OperationResult::Ok(()), - Err(e) => { - logger.log( - LogLevel::Warning, - EngineEvent::Warning( - event_details.clone(), - EventMessage::new( - format!("Failed to tag image `{}`, retrying ...", image_with_latest_tag), - Some(format!("{:?}", e)), - ), - ), - ); - OperationResult::Retry(e) - } - }) { - Err(Operation { error, .. }) => { - return Err(CommandError::new_from_legacy_command_error( - error, - Some(format!("Failed to tag docker image `{}`", image_with_tag)), - )) - } - Err(retry::Error::Internal(msg)) => { - return Err(CommandError::new( - msg, - Some(format!("Failed to tag docker image `{}`", image_with_tag)), - )) - } - _ => {} - } - - let mut cmd = QoveryCommand::new("docker", &vec!["push", dest_latest_tag.as_str()], &docker_envs); - match retry::retry(Fibonacci::from_millis(5000).take(5), || { - match cmd.exec_with_timeout( - Duration::minutes(10), - |line| { - logger.log( - LogLevel::Info, - EngineEvent::Info(event_details.clone(), EventMessage::new(line, None)), - ) - }, - |line| { - logger.log( - LogLevel::Warning, - EngineEvent::Warning(event_details.clone(), EventMessage::new(line, None)), - ) - }, - ) { - Ok(_) => OperationResult::Ok(()), - Err(e) => { - logger.log( - LogLevel::Warning, - EngineEvent::Warning( - event_details.clone(), - EventMessage::new( - format!( - "Failed to push image {} on {}, retrying...", - image_with_tag, registry_provider - ), - Some(format!("{:?}", e)), - ), - ), - ); - OperationResult::Retry(e) - } - } - }) { - Err(Operation { error, .. }) => Err(CommandError::new(error.to_string(), None)), - Err(e) => Err(CommandError::new( - format!("{:?}", e), - Some(format!( - "Unknown error while trying to push image {} to {}.", - image_with_tag, registry_provider, - )), - )), - _ => { - logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(format!("image {} has successfully been pushed", image_with_tag)), - ), - ); - Ok(()) - } - } -} - -pub fn docker_pull_image( - container_registry_kind: Kind, - docker_envs: Vec<(&str, &str)>, - dest: String, - event_details: EventDetails, - logger: &dyn Logger, -) -> Result<(), CommandError> { - let registry_provider = match container_registry_kind { - Kind::DockerHub => "DockerHub", - Kind::Ecr => "AWS ECR", - Kind::Docr => "DigitalOcean Registry", - Kind::ScalewayCr => "Scaleway Registry", - }; - - let mut cmd = QoveryCommand::new("docker", &vec!["pull", dest.as_str()], &docker_envs); - match retry::retry(Fibonacci::from_millis(5000).take(5), || { - match cmd.exec_with_timeout( - Duration::minutes(10), - |line| { - logger.log( - LogLevel::Info, - EngineEvent::Info(event_details.clone(), EventMessage::new(line, None)), - ) - }, - |line| { - logger.log( - LogLevel::Warning, - EngineEvent::Warning(event_details.clone(), EventMessage::new(line, None)), - ) - }, - ) { - Ok(_) => OperationResult::Ok(()), - Err(e) => { - logger.log( - LogLevel::Warning, - EngineEvent::Warning( - event_details.clone(), - EventMessage::new( - format!( - "failed to pull image from {} registry {}, retrying...", - registry_provider, - dest.as_str(), - ), - Some(format!("{:?}", e)), - ), - ), - ); - OperationResult::Retry(e) - } - } - }) { - Err(Operation { error, .. }) => Err(CommandError::new(error.to_string(), None)), - Err(e) => Err(CommandError::new( - format!("{:?}", e), - Some(format!( - "Unknown error while trying to pull image {} from {} registry.", - dest.as_str(), - registry_provider, - )), - )), - _ => { - logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(format!( - "Image {} has successfully been pulled from {} registry", - dest.as_str(), - registry_provider, - )), - ), - ); - Ok(()) - } - } -} diff --git a/src/container_registry/docker_hub.rs b/src/container_registry/docker_hub.rs deleted file mode 100644 index 63fb48b7..00000000 --- a/src/container_registry/docker_hub.rs +++ /dev/null @@ -1,320 +0,0 @@ -extern crate reqwest; - -use reqwest::StatusCode; -use std::borrow::Borrow; - -use crate::build_platform::Image; -use crate::cmd::command::QoveryCommand; -use crate::container_registry::docker::{docker_pull_image, docker_tag_and_push_image}; -use crate::container_registry::{ContainerRegistry, Kind, PullResult, PushResult}; -use crate::errors::{CommandError, EngineError}; -use crate::events::{EngineEvent, EventMessage, ToTransmitter, Transmitter}; -use crate::logger::{LogLevel, Logger}; -use crate::models::{ - Context, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope, -}; - -pub struct DockerHub { - context: Context, - id: String, - name: String, - login: String, - password: String, - listeners: Listeners, - logger: Box, -} - -impl DockerHub { - pub fn new(context: Context, id: &str, name: &str, login: &str, password: &str, logger: Box) -> Self { - DockerHub { - context, - id: id.to_string(), - name: name.to_string(), - login: login.to_string(), - password: password.to_string(), - listeners: vec![], - logger, - } - } - - pub fn exec_docker_login(&self) -> Result<(), EngineError> { - let event_details = self.get_event_details(); - - let envs = match self.context.docker_tcp_socket() { - Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())], - None => vec![], - }; - - let mut cmd = QoveryCommand::new( - "docker", - &vec!["login", "-u", self.login.as_str(), "-p", self.password.as_str()], - &envs, - ); - - match cmd.exec() { - Ok(_) => Ok(()), - Err(_) => Err(EngineError::new_client_invalid_cloud_provider_credentials( - event_details, - )), - } - } - - fn pull_image(&self, dest: String, image: &Image) -> Result { - let event_details = self.get_event_details(); - match docker_pull_image(self.kind(), vec![], dest.clone(), event_details.clone(), self.logger()) { - Ok(_) => { - let mut image = image.clone(); - image.registry_url = Some(dest); - Ok(PullResult::Some(image)) - } - Err(e) => Err(EngineError::new_docker_pull_image_error( - event_details, - image.name.to_string(), - dest.to_string(), - e, - )), - } - } -} - -impl ToTransmitter for DockerHub { - fn to_transmitter(&self) -> Transmitter { - Transmitter::ContainerRegistry(self.id().to_string(), self.name().to_string()) - } -} - -impl ContainerRegistry for DockerHub { - fn context(&self) -> &Context { - &self.context - } - - fn kind(&self) -> Kind { - Kind::DockerHub - } - - fn id(&self) -> &str { - self.id.as_str() - } - - fn name(&self) -> &str { - self.name.as_str() - } - - fn is_valid(&self) -> Result<(), EngineError> { - Ok(()) - } - - fn on_create(&self) -> Result<(), EngineError> { - Ok(()) - } - - fn on_create_error(&self) -> Result<(), EngineError> { - Ok(()) - } - - fn on_delete(&self) -> Result<(), EngineError> { - Ok(()) - } - - fn on_delete_error(&self) -> Result<(), EngineError> { - Ok(()) - } - - fn does_image_exists(&self, image: &Image) -> bool { - let event_details = self.get_event_details(); - use reqwest::blocking::Client; - let client = Client::new(); - let path = format!( - "https://index.docker.io/v1/repositories/{}/{}/tags", - &self.login, image.name - ); - let res = client - .get(path.as_str()) - .basic_auth(&self.login, Option::from(&self.password)) - .send(); - - // TODO (mzo) no check of existing tags as in others impl ? - match res { - Ok(out) => matches!(out.status(), StatusCode::OK), - Err(e) => { - self.logger.log( - LogLevel::Error, - EngineEvent::Error( - EngineError::new_container_registry_repository_doesnt_exist( - event_details.clone(), - image.name.to_string(), - Some(CommandError::new( - e.to_string(), - Some("Error while trying to retrieve if DockerHub repository exist.".to_string()), - )), - ), - None, - ), - ); - false - } - } - } - - fn pull(&self, image: &Image) -> Result { - let event_details = self.get_event_details(); - let listeners_helper = ListenersHelper::new(&self.listeners); - - if !self.does_image_exists(image) { - let info_message = format!( - "image {:?} does not exist in DockerHub {} repository", - image, - self.name() - ); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(info_message.to_string()), - ), - ); - - listeners_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: image.application_id.clone(), - }, - ProgressLevel::Info, - Some(info_message), - self.context.execution_id(), - )); - - return Ok(PullResult::None); - } - - let info_message = format!("pull image {:?} from DockerHub {} repository", image, self.name()); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(info_message.to_string()), - ), - ); - - listeners_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: image.application_id.clone(), - }, - ProgressLevel::Info, - Some(info_message), - self.context.execution_id(), - )); - - let _ = self.exec_docker_login()?; - - let dest = format!("{}/{}", self.login.as_str(), image.name_with_tag().as_str()); - - // pull image - self.pull_image(dest, image) - } - - fn push(&self, image: &Image, force_push: bool) -> Result { - let event_details = self.get_event_details(); - - let _ = self.exec_docker_login()?; - - let dest = format!("{}/{}", self.login.as_str(), image.name_with_tag().as_str()); - let listeners_helper = ListenersHelper::new(&self.listeners); - - if !force_push && self.does_image_exists(image) { - // check if image does exist - if yes, do not upload it again - let info_message = format!( - "image {:?} found on DockerHub {} repository, container build is not required", - image, - self.name() - ); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(info_message.to_string()), - ), - ); - - listeners_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: image.application_id.clone(), - }, - ProgressLevel::Info, - Some(info_message), - self.context.execution_id(), - )); - - let mut image = image.clone(); - image.registry_url = Some(dest); - - return Ok(PushResult { image }); - } - - let info_message = format!( - "image {:?} does not exist on DockerHub {} repository, starting image upload", - image, - self.name() - ); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(info_message.to_string()), - ), - ); - - listeners_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: image.application_id.clone(), - }, - ProgressLevel::Info, - Some(info_message), - self.context.execution_id(), - )); - - let dest_latest_tag = format!("{}/{}:latest", self.login.as_str(), image.name); - match docker_tag_and_push_image( - self.kind(), - vec![], - &image, - dest.clone(), - dest_latest_tag, - event_details.clone(), - self.logger(), - ) { - Ok(_) => { - let mut image = image.clone(); - image.registry_url = Some(dest); - Ok(PushResult { image }) - } - Err(e) => Err(EngineError::new_docker_push_image_error( - event_details.clone(), - image.name.to_string(), - dest.to_string(), - e, - )), - } - } - - fn push_error(&self, _image: &Image) -> Result { - unimplemented!() - } - - fn logger(&self) -> &dyn Logger { - self.logger.borrow() - } -} - -impl Listen for DockerHub { - fn listeners(&self) -> &Listeners { - &self.listeners - } - - fn add_listener(&mut self, listener: Listener) { - self.listeners.push(listener); - } -} diff --git a/src/container_registry/docr.rs b/src/container_registry/docr.rs index 6b71b36b..00d8060f 100644 --- a/src/container_registry/docr.rs +++ b/src/container_registry/docr.rs @@ -6,21 +6,17 @@ use std::borrow::Borrow; use crate::build_platform::Image; use crate::cmd::command::QoveryCommand; -use crate::container_registry::docker::{docker_pull_image, docker_tag_and_push_image}; -use crate::container_registry::{ContainerRegistry, EngineError, Kind, PullResult, PushResult}; +use crate::container_registry::{ContainerRegistry, ContainerRegistryInfo, EngineError, Kind}; use crate::errors::CommandError; -use crate::events::{EngineEvent, EventDetails, EventMessage, ToTransmitter, Transmitter}; +use crate::events::{EngineEvent, EventDetails, ToTransmitter, Transmitter}; use crate::logger::{LogLevel, Logger}; -use crate::models::{ - Context, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope, -}; +use crate::models::{Context, Listen, Listener, Listeners}; use crate::utilities; -use retry::delay::Fixed; -use retry::Error::Operation; -use retry::OperationResult; +use url::Url; const CR_API_PATH: &str = "https://api.digitalocean.com/v2/registry"; const CR_CLUSTER_API_PATH: &str = "https://api.digitalocean.com/v2/kubernetes/registry"; +const CR_REGISTRY_DOMAIN: &str = "registry.digitalocean.com"; // TODO : use --output json // see https://www.digitalocean.com/community/tutorials/how-to-use-doctl-the-official-digitalocean-command-line-client @@ -46,32 +42,16 @@ impl DOCR { } } - fn get_registry_name(&self, image: &Image) -> Result { + fn create_registry(&self, registry_name: &str) -> Result<(), EngineError> { let event_details = self.get_event_details(); - let registry_name = match image.registry_name.as_ref() { - // DOCR does not support upper cases - Some(registry_name) => registry_name.to_lowercase(), - None => get_current_registry_name(self.api_key.as_str(), event_details, self.logger())?, - }; - - Ok(registry_name) - } - - fn create_repository(&self, image: &Image) -> Result<(), EngineError> { - let event_details = self.get_event_details(); - - let registry_name = match image.registry_name.as_ref() { - // DOCR does not support upper cases - Some(registry_name) => registry_name.to_lowercase(), - None => self.name.clone(), - }; - + // DOCR does not support upper cases + let registry_name = registry_name.to_lowercase(); let headers = utilities::get_header_with_bearer(&self.api_key); // subscription_tier_slug: https://www.digitalocean.com/products/container-registry/ // starter and basic tiers are too limited on repository creation let repo = DoApiCreateRepository { - name: registry_name.clone(), + name: registry_name.to_string(), subscription_tier_slug: "professional".to_string(), }; @@ -133,77 +113,7 @@ impl DOCR { } } - fn push_image(&self, registry_name: String, dest: String, image: &Image) -> Result { - let event_details = self.get_event_details(); - - let dest_latest_tag = format!( - "registry.digitalocean.com/{}/{}:latest", - registry_name.as_str(), - image.name - ); - - if let Err(e) = docker_tag_and_push_image( - self.kind(), - vec![], - image, - dest.clone(), - dest_latest_tag.clone(), - event_details.clone(), - self.logger(), - ) { - return Err(EngineError::new_docker_push_image_error( - event_details, - image.name.to_string(), - dest.to_string(), - e, - )); - } - - let mut image = image.clone(); - image.registry_name = Some(registry_name.clone()); - // on DOCR registry secret is the same as registry name - image.registry_secret = Some(registry_name); - image.registry_url = Some(dest); - - let result = retry::retry(Fixed::from_millis(10000).take(12), || { - match self.does_image_exists(&image) { - true => OperationResult::Ok(&image), - false => { - self.logger.log( - LogLevel::Warning, - EngineEvent::Warning( - self.get_event_details(), - EventMessage::new_from_safe( - "Image is not yet available on DOCR, retrying in a few seconds...".to_string(), - ), - ), - ); - OperationResult::Retry(()) - } - } - }); - - let image_not_reachable = Err(EngineError::new_container_registry_image_unreachable_after_push( - event_details.clone(), - image.name.to_string(), - )); - match result { - Ok(_) => Ok(PushResult { image }), - Err(Operation { .. }) => image_not_reachable, - Err(retry::Error::Internal(_)) => image_not_reachable, - } - } - - pub fn get_image(&self, _image: &Image) -> Option<()> { - todo!() - } - - pub fn delete_image(&self, _image: &Image) -> Result<(), EngineError> { - // TODO(benjaminch): To be implemented later on, but note it must not slow down CI workflow - Ok(()) - } - - pub fn delete_repository(&self) -> Result<(), EngineError> { + pub fn delete_registry(&self) -> Result<(), EngineError> { let event_details = self.get_event_details(); let headers = utilities::get_header_with_bearer(&self.api_key); @@ -255,27 +165,6 @@ impl DOCR { )), } } - - fn pull_image(&self, registry_name: String, dest: String, image: &Image) -> Result { - let event_details = self.get_event_details(); - - match docker_pull_image(self.kind(), vec![], dest.clone(), event_details.clone(), self.logger()) { - Ok(_) => { - let mut image = image.clone(); - image.registry_name = Some(registry_name.clone()); - // on DOCR registry secret is the same as registry name - image.registry_secret = Some(registry_name); - image.registry_url = Some(dest); - Ok(PullResult::Some(image)) - } - Err(e) => Err(EngineError::new_docker_pull_image_error( - event_details, - image.name.to_string(), - dest.to_string(), - e, - )), - } - } } impl ToTransmitter for DOCR { @@ -305,38 +194,39 @@ impl ContainerRegistry for DOCR { Ok(()) } - fn on_create(&self) -> Result<(), EngineError> { + fn login(&self) -> Result { + let _ = self.exec_docr_login()?; + let registry_name = self.name.clone(); + Ok(ContainerRegistryInfo { + endpoint: Url::parse(&format!("https://{}", CR_REGISTRY_DOMAIN)).unwrap(), + registry_name: self.name.to_string(), + registry_docker_json_config: None, + get_image_name: Box::new(move |img_name| format!("{}/{}", registry_name, img_name)), + }) + } + + fn create_registry(&self) -> Result<(), EngineError> { + // Digital Ocean only allow one registry per account... + if let Err(_) = get_current_registry_name(self.api_key.as_str(), self.get_event_details(), self.logger()) { + let _ = self.create_registry(self.name())?; + } + Ok(()) } - fn on_create_error(&self) -> Result<(), EngineError> { - Ok(()) - } - - fn on_delete(&self) -> Result<(), EngineError> { - Ok(()) - } - - fn on_delete_error(&self) -> Result<(), EngineError> { + fn create_repository(&self, _repository_name: &str) -> Result<(), EngineError> { + // Nothing to do, DO only allow one registry and create repository on the flight when image are pushed Ok(()) } fn does_image_exists(&self, image: &Image) -> bool { let event_details = self.get_event_details(); - let registry_name = match self.get_registry_name(image) { - Ok(registry_name) => registry_name, - Err(err) => { - self.logger.log(LogLevel::Error, EngineEvent::Error(err, None)); - return false; - } - }; - let headers = utilities::get_header_with_bearer(self.api_key.as_str()); let url = format!( "https://api.digitalocean.com/v2/registry/{}/repositories/{}/tags", - registry_name, - image.name.as_str() + image.registry_name, + image.name() ); let res = reqwest::blocking::Client::new() @@ -353,10 +243,10 @@ impl ContainerRegistry for DOCR { EngineEvent::Error( EngineError::new_container_registry_image_doesnt_exist( event_details.clone(), - image.name.to_string(), + image.name().to_string(), Some(CommandError::new_from_safe_message(format!( "While tyring to get all tags for image: `{}`, maybe this image not exist !", - image.name.to_string() + image.name().to_string() ))), ), None, @@ -372,10 +262,10 @@ impl ContainerRegistry for DOCR { EngineEvent::Error( EngineError::new_container_registry_image_doesnt_exist( event_details.clone(), - image.name.to_string(), + image.name().to_string(), Some(CommandError::new_from_safe_message(format!( "While trying to communicate with DigitalOcean API to retrieve all tags for image `{}`.", - image.name.to_string() + image.name().to_string() ))), ), None, @@ -405,7 +295,7 @@ impl ContainerRegistry for DOCR { EngineEvent::Error( EngineError::new_container_registry_image_doesnt_exist( event_details.clone(), - image.name.to_string(), + image.name().to_string(), Some(CommandError::new( out.to_string(), Some(format!( @@ -428,10 +318,10 @@ impl ContainerRegistry for DOCR { EngineEvent::Error( EngineError::new_container_registry_image_doesnt_exist( event_details.clone(), - image.name.to_string(), + image.name().to_string(), Some(CommandError::new_from_safe_message(format!( "While retrieving tags for image `{}` Unable to get output from DigitalOcean API.", - image.name.to_string() + image.name().to_string() ))), ), None, @@ -443,164 +333,6 @@ impl ContainerRegistry for DOCR { } } - fn pull(&self, image: &Image) -> Result { - let event_details = self.get_event_details(); - let listeners_helper = ListenersHelper::new(&self.listeners); - - if !self.does_image_exists(image) { - let info_message = format!("image {:?} does not exist in DOCR {} repository", image, self.name()); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(info_message.to_string()), - ), - ); - - listeners_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: image.application_id.clone(), - }, - ProgressLevel::Info, - Some(info_message), - self.context.execution_id(), - )); - - return Ok(PullResult::None); - } - - let info_message = format!("pull image {:?} from DOCR {} repository", image, self.name()); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(info_message.to_string()), - ), - ); - - listeners_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: image.application_id.clone(), - }, - ProgressLevel::Info, - Some(info_message), - self.context.execution_id(), - )); - - let _ = self.exec_docr_login()?; - - let registry_name = self.get_registry_name(image)?; - - let dest = format!( - "registry.digitalocean.com/{}/{}", - registry_name.as_str(), - image.name_with_tag() - ); - - // pull image - self.pull_image(registry_name, dest, image) - } - - // https://www.digitalocean.com/docs/images/container-registry/how-to/use-registry-docker-kubernetes/ - fn push(&self, image: &Image, force_push: bool) -> Result { - let event_details = self.get_event_details(); - let registry_name = self.get_registry_name(image)?; - - match self.create_repository(image) { - Ok(_) => self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(format!("DOCR {} has been created", registry_name.as_str())), - ), - ), - Err(e) => self.logger.log( - LogLevel::Error, - EngineEvent::Error( - e.clone(), - Some(EventMessage::new_from_safe(format!( - "DOCR {} already exists", - registry_name.as_str() - ))), - ), - ), - }; - - let _ = self.exec_docr_login()?; - - let dest = format!( - "registry.digitalocean.com/{}/{}", - registry_name.as_str(), - image.name_with_tag() - ); - - let listeners_helper = ListenersHelper::new(&self.listeners); - - if !force_push && self.does_image_exists(image) { - // check if image does exist - if yes, do not upload it again - let info_message = format!( - "image {:?} found on DOCR {} repository, container build is not required", - image, - registry_name.as_str() - ); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(info_message.to_string()), - ), - ); - - listeners_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: image.application_id.clone(), - }, - ProgressLevel::Info, - Some(info_message), - self.context.execution_id(), - )); - - let mut image = image.clone(); - image.registry_name = Some(registry_name.clone()); - // on DOCR registry secret is the same as registry name - image.registry_secret = Some(registry_name); - image.registry_url = Some(dest); - - return Ok(PushResult { image }); - } - - let info_message = format!( - "image {:?} does not exist on DOCR {} repository, starting image upload", - image, registry_name - ); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(info_message.to_string()), - ), - ); - - listeners_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: image.application_id.clone(), - }, - ProgressLevel::Info, - Some(info_message), - self.context.execution_id(), - )); - - self.push_image(registry_name, dest, image) - } - - fn push_error(&self, image: &Image) -> Result { - Ok(PushResult { image: image.clone() }) - } - fn logger(&self) -> &dyn Logger { self.logger.borrow() } diff --git a/src/container_registry/ecr.rs b/src/container_registry/ecr.rs index a24ef38a..28b4fd4c 100644 --- a/src/container_registry/ecr.rs +++ b/src/container_registry/ecr.rs @@ -10,20 +10,18 @@ use rusoto_ecr::{ use rusoto_sts::{GetCallerIdentityRequest, Sts, StsClient}; use crate::build_platform::Image; -use crate::cmd::command::QoveryCommand; -use crate::container_registry::docker::{docker_pull_image, docker_tag_and_push_image}; -use crate::container_registry::{ContainerRegistry, Kind, PullResult, PushResult}; +use crate::cmd::docker::{to_engine_error, Docker}; +use crate::container_registry::{ContainerRegistry, ContainerRegistryInfo, Kind}; use crate::errors::{CommandError, EngineError}; use crate::events::{EngineEvent, EventMessage, ToTransmitter, Transmitter}; use crate::logger::{LogLevel, Logger}; -use crate::models::{ - Context, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope, -}; +use crate::models::{Context, Listen, Listener, Listeners}; use crate::runtime::block_on; use retry::delay::Fixed; use retry::Error::Operation; use retry::OperationResult; use serde_json::json; +use url::Url; pub struct ECR { context: Context, @@ -75,9 +73,9 @@ impl ECR { EcrClient::new_with_client(self.client(), self.region.clone()) } - fn get_repository(&self, image: &Image) -> Option { + fn get_repository(&self, repository_name: &str) -> Option { let mut drr = DescribeRepositoriesRequest::default(); - drr.repository_names = Some(vec![image.name.to_string()]); + drr.repository_names = Some(vec![repository_name.to_string()]); let r = block_on(self.ecr_client().describe_repositories(drr)); @@ -93,7 +91,7 @@ impl ECR { fn get_image(&self, image: &Image) -> Option { let mut dir = DescribeImagesRequest::default(); - dir.repository_name = image.name.to_string(); + dir.repository_name = image.name().to_string(); let mut image_identifier = ImageIdentifier::default(); image_identifier.image_tag = Some(image.tag.to_string()); @@ -111,71 +109,8 @@ impl ECR { } } - fn docker_envs(&self) -> Vec<(&str, &str)> { - match self.context.docker_tcp_socket() { - Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())], - None => vec![], - } - } - - fn push_image(&self, dest: String, dest_latest_tag: String, image: &Image) -> Result { - // READ https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html - // docker tag e9ae3c220b23 aws_account_id.dkr.ecr.region.amazonaws.com/my-web-app + fn create_repository(&self, repository_name: &str) -> Result { let event_details = self.get_event_details(); - - match docker_tag_and_push_image( - self.kind(), - self.docker_envs(), - &image, - dest.clone(), - dest_latest_tag, - event_details.clone(), - self.logger(), - ) { - Ok(_) => { - let mut image = image.clone(); - image.registry_url = Some(dest); - Ok(PushResult { image }) - } - Err(e) => Err(EngineError::new_docker_push_image_error( - event_details, - image.name.to_string(), - dest.to_string(), - e, - )), - } - } - - fn pull_image(&self, dest: String, image: &Image) -> Result { - // READ https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-pull-ecr-image.html - // docker pull aws_account_id.dkr.ecr.us-west-2.amazonaws.com/amazonlinux:latest - let event_details = self.get_event_details(); - - match docker_pull_image( - self.kind(), - self.docker_envs(), - dest.clone(), - event_details.clone(), - self.logger(), - ) { - Ok(_) => { - let mut image = image.clone(); - image.registry_url = Some(dest); - Ok(PullResult::Some(image)) - } - Err(e) => Err(EngineError::new_docker_pull_image_error( - event_details, - image.name.to_string(), - dest.to_string(), - e, - )), - } - } - - fn create_repository(&self, image: &Image) -> Result { - let event_details = self.get_event_details(); - let repository_name = image.name.as_str(); - self.logger().log( LogLevel::Info, EngineEvent::Info( @@ -314,7 +249,7 @@ impl ECR { }); let plp = PutLifecyclePolicyRequest { - repository_name: image.name.clone(), + repository_name: repository_name.to_string(), lifecycle_policy_text: lifecycle_policy_text.to_string(), ..Default::default() }; @@ -327,27 +262,27 @@ impl ECR { CommandError::new_from_safe_message(err.to_string()), ), ), - _ => Ok(self.get_repository(image).expect("cannot get repository")), + _ => Ok(self.get_repository(repository_name).expect("cannot get repository")), } } - fn get_or_create_repository(&self, image: &Image) -> Result { + fn get_or_create_repository(&self, repository_name: &str) -> Result { let event_details = self.get_event_details(); // check if the repository already exists - let repository = self.get_repository(image); + let repository = self.get_repository(repository_name); if repository.is_some() { self.logger.log( LogLevel::Info, EngineEvent::Info( event_details.clone(), - EventMessage::new_from_safe(format!("ECR repository {} already exists", image.name.as_str())), + EventMessage::new_from_safe(format!("ECR repository {} already exists", repository_name)), ), ); return Ok(repository.unwrap()); } - self.create_repository(image) + self.create_repository(repository_name) } fn get_credentials(&self) -> Result { @@ -391,32 +326,6 @@ impl ECR { Ok(ECRCredentials::new(access_token, password, endpoint_url)) } - - fn exec_docker_login(&self) -> Result<(), EngineError> { - let event_details = self.get_event_details(); - let credentials = self.get_credentials()?; - - let mut cmd = QoveryCommand::new( - "docker", - &vec![ - "login", - "-u", - credentials.access_token.as_str(), - "-p", - credentials.password.as_str(), - credentials.endpoint_url.as_str(), - ], - &self.docker_envs(), - ); - - if let Err(_) = cmd.exec() { - return Err(EngineError::new_client_invalid_cloud_provider_credentials( - event_details.clone(), - )); - }; - - Ok(()) - } } impl ToTransmitter for ECR { @@ -454,196 +363,41 @@ impl ContainerRegistry for ECR { } } - fn on_create(&self) -> Result<(), EngineError> { - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - self.get_event_details(), - EventMessage::new_from_safe("ECR.on_create() called".to_string()), - ), - ); + fn login(&self) -> Result { + let event_details = self.get_event_details(); + let credentials = self.get_credentials()?; + let docker = Docker::new(self.context.docker_tcp_socket().clone()) + .map_err(|err| to_engine_error(&event_details, err))?; + let mut registry_url = Url::parse(credentials.endpoint_url.as_str()).unwrap(); + let _ = registry_url.set_username(&credentials.access_token); + let _ = registry_url.set_password(Some(&credentials.password)); + + let _ = docker + .login(®istry_url) + .map_err(|err| to_engine_error(&event_details, err))?; + + Ok(ContainerRegistryInfo { + endpoint: registry_url, + registry_name: self.name.to_string(), + registry_docker_json_config: None, + get_image_name: Box::new(|img_name| img_name.to_string()), + }) + } + + fn create_registry(&self) -> Result<(), EngineError> { + // Nothing to do, ECR require to create only repository Ok(()) } - fn on_create_error(&self) -> Result<(), EngineError> { - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - self.get_event_details(), - EventMessage::new_from_safe("ECR.on_create_error() called".to_string()), - ), - ); - - unimplemented!() - } - - fn on_delete(&self) -> Result<(), EngineError> { - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - self.get_event_details(), - EventMessage::new_from_safe("ECR.on_delete() called".to_string()), - ), - ); - unimplemented!() - } - - fn on_delete_error(&self) -> Result<(), EngineError> { - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - self.get_event_details(), - EventMessage::new_from_safe("ECR.on_delete_error() called".to_string()), - ), - ); - unimplemented!() + fn create_repository(&self, name: &str) -> Result<(), EngineError> { + let _ = self.get_or_create_repository(name)?; + Ok(()) } fn does_image_exists(&self, image: &Image) -> bool { self.get_image(image).is_some() } - fn pull(&self, image: &Image) -> Result { - let event_details = self.get_event_details(); - let listeners_helper = ListenersHelper::new(&self.listeners); - - if !self.does_image_exists(image) { - let info_message = format!( - "image `{}` does not exist in ECR {} repository", - image.name_with_tag(), - self.name() - ); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(info_message.to_string()), - ), - ); - - listeners_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: image.application_id.clone(), - }, - ProgressLevel::Info, - Some(info_message), - self.context.execution_id(), - )); - - return Ok(PullResult::None); - } - - let info_message = format!( - "pull image `{:?}` from ECR {} repository", - image.name_with_tag(), - self.name() - ); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(info_message.to_string()), - ), - ); - - listeners_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: image.application_id.clone(), - }, - ProgressLevel::Info, - Some(info_message), - self.context.execution_id(), - )); - - let _ = self.exec_docker_login()?; - - let repository = self.get_or_create_repository(image)?; - - let dest = format!("{}:{}", repository.repository_uri.unwrap(), image.tag.as_str()); - - // pull image - self.pull_image(dest, image) - } - - fn push(&self, image: &Image, force_push: bool) -> Result { - let _ = self.exec_docker_login()?; - - let repository = if force_push { - self.create_repository(image) - } else { - self.get_or_create_repository(image) - }?; - - let repository_uri = repository.repository_uri.expect("Error getting repository URI"); - let dest = format!("{}:{}", repository_uri, image.tag.as_str()); - - let listeners_helper = ListenersHelper::new(&self.listeners); - - if !force_push && self.does_image_exists(image) { - // check if image does exist - if yes, do not upload it again - let info_message = format!( - "image {} found on ECR {} repository, container build is not required", - image.name_with_tag(), - self.name() - ); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - self.get_event_details(), - EventMessage::new_from_safe(info_message.to_string()), - ), - ); - - listeners_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: image.application_id.clone(), - }, - ProgressLevel::Info, - Some(info_message), - self.context.execution_id(), - )); - - let mut image = image.clone(); - image.registry_url = Some(dest); - - return Ok(PushResult { image }); - } - - let info_message = format!( - "image `{}` does not exist on ECR {} repository, starting image upload", - image.name_with_tag(), - self.name() - ); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - self.get_event_details(), - EventMessage::new_from_safe(info_message.to_string()), - ), - ); - - listeners_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: image.application_id.clone(), - }, - ProgressLevel::Info, - Some(info_message), - self.context.execution_id(), - )); - - let dest_latest_tag = format!("{}:latest", repository_uri); - self.push_image(dest, dest_latest_tag, image) - } - - fn push_error(&self, image: &Image) -> Result { - // TODO change this - Ok(PushResult { image: image.clone() }) - } - fn logger(&self) -> &dyn Logger { self.logger.borrow() } diff --git a/src/container_registry/mod.rs b/src/container_registry/mod.rs index f74c1dff..ddf8e56d 100644 --- a/src/container_registry/mod.rs +++ b/src/container_registry/mod.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use url::Url; use crate::build_platform::Image; use crate::errors::EngineError; @@ -6,8 +7,6 @@ use crate::events::{EnvironmentStep, EventDetails, Stage, ToTransmitter}; use crate::logger::Logger; use crate::models::{Context, Listen, QoveryIdentifier}; -pub mod docker; -pub mod docker_hub; pub mod docr; pub mod ecr; pub mod scaleway_container_registry; @@ -21,14 +20,26 @@ pub trait ContainerRegistry: Listen + ToTransmitter { format!("{} ({})", self.name(), self.id()) } fn is_valid(&self) -> Result<(), EngineError>; - fn on_create(&self) -> Result<(), EngineError>; - fn on_create_error(&self) -> Result<(), EngineError>; - fn on_delete(&self) -> Result<(), EngineError>; - fn on_delete_error(&self) -> Result<(), EngineError>; + + // Login into the registry and setup everything for it + // mainly getting creds and calling docker login behind the hood + // It is poart of the ContainerRegistry only because DigitalOcean require to call doctl + // and that we can't get credentials directly + fn login(&self) -> Result; + + // Some provider require specific action in order to allow container registry + // For now it is only digital ocean, that require 2 steps to have registries + fn create_registry(&self) -> Result<(), EngineError>; + + // Call to create a specific repository in the registry + // i.e: docker.io/erebe or docker.io/qovery + // All providers requires action for that + // The convention for us is that we create one per application + fn create_repository(&self, repository_name: &str) -> Result<(), EngineError>; + + // Check on the registry if a specific image already exist fn does_image_exists(&self, image: &Image) -> bool; - fn pull(&self, image: &Image) -> Result; - fn push(&self, image: &Image, force_push: bool) -> Result; - fn push_error(&self, image: &Image) -> Result; + fn logger(&self) -> &dyn Logger; fn get_event_details(&self) -> EventDetails { let context = self.context(); @@ -44,6 +55,17 @@ pub trait ContainerRegistry: Listen + ToTransmitter { } } +pub struct ContainerRegistryInfo { + pub endpoint: Url, // Contains username and password if necessary + pub registry_name: String, + pub registry_docker_json_config: Option, + // give it the name of your image, and it returns the full name with prefix if needed + // i.e: for DigitalOcean => registry_name/image_name + // i.e: fo scaleway => image_name/image_name + // i.e: for AWS => image_name + pub get_image_name: Box String>, +} + pub struct PushResult { pub image: Image, } @@ -56,7 +78,6 @@ pub enum PullResult { #[derive(Serialize, Deserialize, Clone, Copy, Debug)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum Kind { - DockerHub, Ecr, Docr, ScalewayCr, diff --git a/src/container_registry/scaleway_container_registry.rs b/src/container_registry/scaleway_container_registry.rs index 1f8f18f9..fcdb03ac 100644 --- a/src/container_registry/scaleway_container_registry.rs +++ b/src/container_registry/scaleway_container_registry.rs @@ -5,21 +5,15 @@ use std::borrow::Borrow; use self::scaleway_api_rs::models::scaleway_registry_v1_namespace::Status; use crate::build_platform::Image; -use crate::container_registry::docker::{ - docker_login, docker_manifest_inspect, docker_pull_image, docker_tag_and_push_image, -}; -use crate::container_registry::{ContainerRegistry, Kind, PullResult, PushResult}; +use crate::cmd::docker; +use crate::cmd::docker::Docker; +use crate::container_registry::{ContainerRegistry, ContainerRegistryInfo, Kind}; use crate::errors::{CommandError, EngineError}; use crate::events::{EngineEvent, EventMessage, ToTransmitter, Transmitter}; use crate::logger::{LogLevel, Logger}; -use crate::models::{ - Context, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope, -}; +use crate::models::{Context, Listen, Listener, Listeners}; use crate::runtime::block_on; -use retry::delay::Fibonacci; -use retry::Error::Operation; -use retry::OperationResult; -use rusoto_core::param::ToParam; +use url::Url; pub struct ScalewayCR { context: Context, @@ -29,6 +23,7 @@ pub struct ScalewayCR { login: String, secret_token: String, zone: ScwZone, + docker: Docker, listeners: Listeners, logger: Box, } @@ -43,6 +38,8 @@ impl ScalewayCR { zone: ScwZone, logger: Box, ) -> ScalewayCR { + let docker = Docker::new(context.docker_tcp_socket().clone()).unwrap(); // FIXME: remove unwrap + ScalewayCR { context, id: id.to_string(), @@ -51,6 +48,7 @@ impl ScalewayCR { login: "nologin".to_string(), secret_token: secret_token.to_string(), zone, + docker, listeners: Vec::new(), logger, } @@ -66,16 +64,9 @@ impl ScalewayCR { } } - fn get_docker_envs(&self) -> Vec<(&str, &str)> { - match self.context.docker_tcp_socket() { - Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())], - None => vec![], - } - } - pub fn get_registry_namespace( &self, - image: &Image, + namespace_name: &str, ) -> Option { // https://developers.scaleway.com/en/products/registry/api/#get-09e004 let scaleway_registry_namespaces = match block_on(scaleway_api_rs::apis::namespaces_api::list_namespaces( @@ -86,7 +77,7 @@ impl ScalewayCR { None, None, Some(self.default_project_id.as_str()), - image.registry_name.as_deref(), + Some(namespace_name), )) { Ok(res) => res.namespaces, Err(e) => { @@ -96,7 +87,7 @@ impl ScalewayCR { self.get_event_details(), EventMessage::new( "Error while interacting with Scaleway API (list_namespaces).".to_string(), - Some(format!("error: {}, image: {}", e, &image.name)), + Some(format!("error: {}, image: {}", e, namespace_name)), ), ), ); @@ -127,7 +118,7 @@ impl ScalewayCR { None, None, None, - Some(image.name.as_str()), + Some(image.name().as_str()), None, Some(self.default_project_id.as_str()), )) { @@ -139,7 +130,7 @@ impl ScalewayCR { self.get_event_details(), EventMessage::new( "Error while interacting with Scaleway API (list_namespaces).".to_string(), - Some(format!("error: {}, image: {}", e, &image.name)), + Some(format!("error: {}, image: {}", e, &image.name())), ), ), ); @@ -168,7 +159,7 @@ impl ScalewayCR { if image_to_delete.is_none() { let err = EngineError::new_container_registry_image_doesnt_exist( event_details.clone(), - image.name.to_string(), + image.name().to_string(), None, ); @@ -188,7 +179,7 @@ impl ScalewayCR { Err(e) => { let err = EngineError::new_container_registry_delete_image_error( event_details.clone(), - image.name.to_string(), + image.name().to_string(), Some(CommandError::new(e.to_string(), None)), ); @@ -199,81 +190,9 @@ impl ScalewayCR { } } - fn push_image(&self, dest: String, dest_latest_tag: String, image: &Image) -> Result { - // https://www.scaleway.com/en/docs/deploy-an-image-from-registry-to-kubernetes-kapsule/ - let event_details = self.get_event_details(); - - if let Err(e) = docker_tag_and_push_image( - self.kind(), - self.get_docker_envs(), - image, - dest.to_string(), - dest_latest_tag.to_string(), - event_details.clone(), - self.logger(), - ) { - return Err(EngineError::new_docker_push_image_error( - event_details, - image.name.to_string(), - dest.to_string(), - e, - )); - } - - let result = retry::retry(Fibonacci::from_millis(10000).take(10), || { - match self.does_image_exists(image) { - true => OperationResult::Ok(&image), - false => { - self.logger.log( - LogLevel::Warning, - EngineEvent::Warning( - self.get_event_details(), - EventMessage::new_from_safe( - "Image is not yet available on Scaleway Registry Namespace, retrying in a few seconds...".to_string(), - ), - ), - ); - OperationResult::Retry(()) - } - } - }); - - let image_not_reachable = Err(EngineError::new_container_registry_image_unreachable_after_push( - event_details.clone(), - image.name.to_string(), - )); - - match result { - Ok(_) => Ok(PushResult { image: image.clone() }), - Err(Operation { .. }) => image_not_reachable, - Err(retry::Error::Internal(_)) => image_not_reachable, - } - } - - fn pull_image(&self, dest: String, image: &Image) -> Result { - let event_details = self.get_event_details(); - - if let Err(e) = docker_pull_image( - self.kind(), - self.get_docker_envs(), - dest.to_string(), - event_details.clone(), - self.logger(), - ) { - return Err(EngineError::new_docker_pull_image_error( - event_details, - image.name.to_string(), - dest.to_string(), - e, - )); - } - - Ok(PullResult::Some(image.clone())) - } - pub fn create_registry_namespace( &self, - image: &Image, + namespace_name: &str, ) -> Result { let event_details = self.get_event_details(); @@ -282,7 +201,7 @@ impl ScalewayCR { &self.get_configuration(), self.zone.region().to_string().as_str(), scaleway_api_rs::models::inline_object_29::InlineObject29 { - name: image.name.clone(), + name: namespace_name.to_string(), description: None, project_id: Some(self.default_project_id.clone()), is_public: Some(false), @@ -293,7 +212,7 @@ impl ScalewayCR { Err(e) => { let error = EngineError::new_container_registry_namespace_creation_error( event_details.clone(), - image.name.clone(), + namespace_name.to_string(), self.name_with_id(), CommandError::new(e.to_string(), Some("Can't create SCW repository".to_string())), ); @@ -308,19 +227,15 @@ impl ScalewayCR { pub fn delete_registry_namespace( &self, - image: &Image, + namespace_name: &str, ) -> Result { // https://developers.scaleway.com/en/products/registry/api/#delete-c1ac9b let event_details = self.get_event_details(); - let registry_to_delete = self.get_registry_namespace(image); - let repository_name = match image.registry_name.as_ref() { - None => "unknown", - Some(name) => name, - }; + let registry_to_delete = self.get_registry_namespace(namespace_name); if registry_to_delete.is_none() { let error = EngineError::new_container_registry_repository_doesnt_exist( event_details.clone(), - repository_name.to_string(), + namespace_name.to_string(), None, ); @@ -341,7 +256,7 @@ impl ScalewayCR { Err(e) => { let error = EngineError::new_container_registry_delete_repository_error( event_details.clone(), - repository_name.to_string(), + namespace_name.to_string(), Some(CommandError::new(e.to_string(), None)), ); @@ -355,23 +270,25 @@ impl ScalewayCR { pub fn get_or_create_registry_namespace( &self, - image: &Image, + namespace_name: &str, ) -> Result { + info!("Get/Create repository for {}", namespace_name); + // check if the repository already exists let event_details = self.get_event_details(); - let registry_namespace = self.get_registry_namespace(&image); + let registry_namespace = self.get_registry_namespace(namespace_name); if let Some(namespace) = registry_namespace { self.logger.log( LogLevel::Info, EngineEvent::Info( event_details.clone(), - EventMessage::new_from_safe(format!("SCW repository {} already exists", image.name.as_str())), + EventMessage::new_from_safe(format!("SCW repository {} already exists", namespace_name)), ), ); return Ok(namespace); } - self.create_registry_namespace(image) + self.create_registry_namespace(namespace_name) } fn get_docker_json_config_raw(&self) -> String { @@ -384,27 +301,6 @@ impl ScalewayCR { .as_bytes(), ) } - - fn exec_docker_login(&self, registry_url: &String) -> Result<(), EngineError> { - let event_details = self.get_event_details(); - if docker_login( - Kind::ScalewayCr, - self.get_docker_envs(), - self.login.clone(), - self.secret_token.clone(), - registry_url.clone(), - event_details.clone(), - self.logger(), - ) - .is_err() - { - return Err(EngineError::new_client_invalid_cloud_provider_credentials( - event_details, - )); - }; - - Ok(()) - } } impl ToTransmitter for ScalewayCR { @@ -434,218 +330,49 @@ impl ContainerRegistry for ScalewayCR { Ok(()) } - fn on_create(&self) -> Result<(), EngineError> { + fn login(&self) -> Result { + let event_details = self.get_event_details(); + let mut registry = Url::parse(&format!("https://rg.{}.scw.cloud", self.zone.region())).unwrap(); + let _ = registry.set_username(&self.login); + let _ = registry.set_password(Some(&self.secret_token)); + + if self.docker.login(®istry).is_err() { + return Err(EngineError::new_client_invalid_cloud_provider_credentials( + event_details, + )); + } + + Ok(ContainerRegistryInfo { + endpoint: registry, + registry_name: self.name.to_string(), + registry_docker_json_config: Some(self.get_docker_json_config_raw()), + get_image_name: Box::new(move |img_name| format!("{}/{}", img_name, img_name)), + }) + } + + fn create_registry(&self) -> Result<(), EngineError> { + // Nothing to do, scaleway managed container registry per repository (aka `namespace` by the scw naming convention) Ok(()) } - fn on_create_error(&self) -> Result<(), EngineError> { - Ok(()) - } - - fn on_delete(&self) -> Result<(), EngineError> { - Ok(()) - } - - fn on_delete_error(&self) -> Result<(), EngineError> { + fn create_repository(&self, name: &str) -> Result<(), EngineError> { + let _ = self.get_or_create_registry_namespace(name)?; Ok(()) } fn does_image_exists(&self, image: &Image) -> bool { - let event_details = self.get_event_details(); - let registry_url = image - .registry_url - .as_ref() - .unwrap_or(&"undefined".to_string()) - .to_param(); - - if let Err(_) = docker_login( - Kind::ScalewayCr, - self.get_docker_envs(), - self.login.clone(), - self.secret_token.clone(), - registry_url.clone(), - event_details.clone(), - self.logger(), - ) { + let info = if let Ok(url) = self.login() { + url + } else { return false; - } + }; - docker_manifest_inspect( - Kind::ScalewayCr, - self.get_docker_envs(), - image.name.clone(), - image.tag.clone(), - registry_url, - event_details.clone(), - self.logger(), - ) - .is_ok() - } - - fn pull(&self, image: &Image) -> Result { - let event_details = self.get_event_details(); - let listeners_helper = ListenersHelper::new(&self.listeners); - - let mut image = image.clone(); - let registry_url: String; - - match self.get_or_create_registry_namespace(&image) { - Ok(registry) => { - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(format!( - "Scaleway registry namespace for {} has been created", - image.name.as_str() - )), - ), - ); - image.registry_name = Some(image.name.clone()); // Note: Repository namespace should have the same name as the image name - image.registry_url = registry.endpoint.clone(); - image.registry_secret = Some(self.secret_token.clone()); - image.registry_docker_json_config = Some(self.get_docker_json_config_raw()); - registry_url = registry.endpoint.unwrap_or_else(|| "undefined".to_string()); - } - Err(e) => { - self.logger.log(LogLevel::Error, EngineEvent::Error(e.clone(), None)); - return Err(e); - } - } - - if !self.does_image_exists(&image) { - let info_message = format!("Image {:?} does not exist in SCR {} repository", image, self.name()); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(info_message.to_string()), - ), - ); - - listeners_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: image.application_id.clone(), - }, - ProgressLevel::Info, - Some(info_message), - self.context.execution_id(), - )); - - return Ok(PullResult::None); - } - - let info_message = format!("pull image {:?} from SCR {} repository", image, self.name()); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(info_message.to_string()), - ), - ); - - listeners_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: image.application_id.clone(), - }, - ProgressLevel::Info, - Some(info_message), - self.context.execution_id(), - )); - - let _ = self.exec_docker_login(®istry_url)?; - - let dest = format!("{}/{}", registry_url, image.name_with_tag()); - - // pull image - self.pull_image(dest, &image) - } - - fn push(&self, image: &Image, force_push: bool) -> Result { - let event_details = self.get_event_details(); - let mut image = image.clone(); - let registry_url: String; - let registry_name: String; - - match self.get_or_create_registry_namespace(&image) { - Ok(registry) => { - image.registry_name = Some(image.name.clone()); // Note: Repository namespace should have the same name as the image name - image.registry_url = registry.endpoint.clone(); - image.registry_secret = Some(self.secret_token.clone()); - image.registry_docker_json_config = Some(self.get_docker_json_config_raw()); - registry_url = registry.endpoint.unwrap_or_else(|| "undefined".to_string()); - registry_name = registry.name.unwrap(); - } - Err(e) => { - self.logger.log(LogLevel::Error, EngineEvent::Error(e.clone(), None)); - return Err(e); - } - } - - let _ = self.exec_docker_login(®istry_url)?; - - let dest = format!("{}/{}", registry_url, image.name_with_tag()); - - let listeners_helper = ListenersHelper::new(&self.listeners); - - if !force_push && self.does_image_exists(&image) { - // check if image does exist - if yes, do not upload it again - let info_message = format!( - "image {} found on Scaleway {} repository, container build is not required", - image, registry_name, - ); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(info_message.to_string()), - ), - ); - - listeners_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: image.application_id.clone(), - }, - ProgressLevel::Info, - Some(info_message), - self.context.execution_id(), - )); - - return Ok(PushResult { image: image.clone() }); - } - - let info_message = format!( - "image {} does not exist on Scaleway {} repository, starting image upload", - image, - self.name() - ); - - self.logger.log( - LogLevel::Info, - EngineEvent::Info( - event_details.clone(), - EventMessage::new_from_safe(info_message.to_string()), - ), - ); - - listeners_helper.deployment_in_progress(ProgressInfo::new( - ProgressScope::Application { - id: image.application_id.clone(), - }, - ProgressLevel::Info, - Some(info_message), - self.context.execution_id(), - )); - - let dest_latest_tag = format!("{}/{}:latest", registry_url, image.name); - self.push_image(dest, dest_latest_tag, &image) - } - - fn push_error(&self, image: &Image) -> Result { - Ok(PushResult { image: image.clone() }) + let image = docker::ContainerImage { + registry: info.endpoint, + name: image.name().clone(), + tags: vec![image.tag.clone()], + }; + self.docker.does_image_exist_remotely(&image).is_ok() } fn logger(&self) -> &dyn Logger { diff --git a/src/errors/io.rs b/src/errors/io.rs index 529cc9f1..17c63858 100644 --- a/src/errors/io.rs +++ b/src/errors/io.rs @@ -96,6 +96,7 @@ pub enum Tag { BuilderBuildpackCannotBuildContainerImage, BuilderGetBuildError, BuilderCloningRepositoryError, + DockerError, DockerPushImageError, DockerPullImageError, BuilderDockerCannotListImages, @@ -206,6 +207,7 @@ impl From for Tag { errors::Tag::ContainerRegistryRepositoryDoesntExist => Tag::ContainerRegistryRepositoryDoesntExist, errors::Tag::ContainerRegistryDeleteRepositoryError => Tag::ContainerRegistryDeleteRepositoryError, errors::Tag::BuilderDockerCannotListImages => Tag::BuilderDockerCannotListImages, + errors::Tag::DockerError => Tag::DockerError, } } } diff --git a/src/errors/mod.rs b/src/errors/mod.rs index a409ae8f..b6531c75 100644 --- a/src/errors/mod.rs +++ b/src/errors/mod.rs @@ -4,6 +4,7 @@ extern crate url; use crate::cloud_provider::utilities::VersionsNumber; use crate::cmd; +use crate::cmd::docker::DockerError; use crate::cmd::helm::HelmError; use crate::error::{EngineError as LegacyEngineError, EngineErrorCause, EngineErrorScope}; use crate::events::{EventDetails, GeneralStep, Stage, Transmitter}; @@ -253,6 +254,8 @@ pub enum Tag { BuilderGetBuildError, /// BuilderCloningRepositoryError: represents an error when builder is trying to clone a git repository. BuilderCloningRepositoryError, + /// DockerError: represents an error when trying to use docker cli. + DockerError, /// DockerPushImageError: represents an error when trying to push a docker image. DockerPushImageError, /// DockerPullImageError: represents an error when trying to pull a docker image. @@ -2286,6 +2289,24 @@ impl EngineError { ) } + /// Creates new error from an Docker error + /// + /// Arguments: + /// + /// * `event_details`: Error linked event details. + /// * `error`: Raw error message. + pub fn new_docker_error(event_details: EventDetails, error: DockerError) -> EngineError { + EngineError::new( + event_details, + Tag::DockerError, + error.to_string(), + error.to_string(), + None, + None, + None, + ) + } + /// Creates new error when trying to push a Docker image. /// /// Arguments: diff --git a/src/git.rs b/src/git.rs index 4222e887..0176fc9d 100644 --- a/src/git.rs +++ b/src/git.rs @@ -313,7 +313,7 @@ mod tests { fn test_git_submodule_with_ssh_key() { // Unique Key only valid for the submodule and in read access only // https://github.com/Qovery/dumb-logger/settings/keys - let ssh_key = String::from_utf8(base64::decode("LS0tLS1CRUdJTiBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0NCmIzQmxibk56YUMxclpYa3RkakVBQUFBQUJHNXZibVVBQUFBRWJtOXVaUUFBQUFBQUFBQUJBQUFCbHdBQUFBZHpjMmd0Y24NCk5oQUFBQUF3RUFBUUFBQVlFQTFGcS95ZGF6dU84T3ZRdjVUNEdxbndOMjhZV0EzaXlqanREMFdSQXhtdDZEV3lJRlVYZ1gNClZFZ1ZVYnZyYndKNGJQa0tTbkdqd1hZRUdJYkdYa0hKUTdvWTVSMnB6b1hqUkVYTzIzZEZ2aVp4bUpOcVdEVVJqSHhjc1INCndOYWxiOFVZZVBCRVI4TEQzWWpQd0lYNXdCWm5VSjZLWTJFbXhjSlBVUnV4bUlyTjI4QndiZ3FiejJPU3NJdWg4a1ZwSngNCldheitFc3JNM282NHpHMm0wa0dxMVI1VHE0enBPRWliUk1iY1ZXTldKUzRZR29JczdsRzB0ZHZndktNRnJsWktzSUw1Y2ENCkFOQzRXTlROMm1DVVFrVGpGSDVySDlDa0ZBZjZaZ0lqYklvN0s3TTc0L1B5RVhEcStyRW5vRWdzeEkzRi9NZHMydGM2RWkNClJaY2JrUmRLVnpaUzJCMXdKNDhrOGR3Sml5VytKSWY4ejEzK2FiUXVPNGR5MWRnM2gwbEZ6dm9qaVYxTjNBRXdHcmhjZEUNClo3TXNaeThKM3JvRElZSWZCczdkbmh2T1FrME1taEpKSEpMaVlEZWZCYUk4MVdGTGlqekUxejhqMG90cExlNkt0SVhQYk8NCmV5WWdod0U2aDlhSmNrOEU3WklYMjc4MGRQMW93T2g1dC9VaE0vdjFBQUFGZ082eU9GenVzamhjQUFBQUIzTnphQzF5YzINCkVBQUFHQkFOUmF2OG5XczdqdkRyMEwrVStCcXA4RGR2R0ZnTjRzbzQ3UTlGa1FNWnJlZzFzaUJWRjRGMVJJRlZHNzYyOEMNCmVHejVDa3B4bzhGMkJCaUd4bDVCeVVPNkdPVWRxYzZGNDBSRnp0dDNSYjRtY1ppVGFsZzFFWXg4WExFY0RXcFcvRkdIancNClJFZkN3OTJJejhDRitjQVdaMUNlaW1OaEpzWENUMUVic1ppS3pkdkFjRzRLbTg5amtyQ0xvZkpGYVNjVm1zL2hMS3pONk8NCnVNeHRwdEpCcXRVZVU2dU02VGhJbTBURzNGVmpWaVV1R0JxQ0xPNVJ0TFhiNEx5akJhNVdTckNDK1hHZ0RRdUZqVXpkcGcNCmxFSkU0eFIrYXgvUXBCUUgrbVlDSTJ5S095dXpPK1B6OGhGdzZ2cXhKNkJJTE1TTnhmekhiTnJYT2hJa1dYRzVFWFNsYzINClV0Z2RjQ2VQSlBIY0NZc2x2aVNIL005ZC9tbTBManVIY3RYWU40ZEpSYzc2STRsZFRkd0JNQnE0WEhSR2V6TEdjdkNkNjYNCkF5R0NId2JPM1o0YnprSk5ESm9TU1J5UzRtQTNud1dpUE5WaFM0bzh4TmMvSTlLTGFTM3VpclNGejJ6bnNtSUljQk9vZlcNCmlYSlBCTzJTRjl1L05IVDlhTURvZWJmMUlUUDc5UUFBQUFNQkFBRUFBQUdCQUxhR1pqRkwvV0NwQWtjV0lxM25LMHZRZzQNCjBuamxQcGxKQXVKTWprOVc1RGNpNkQrSVJGTC9BK29TeUcxTit2Qk9uTnliMmhIZnNzd0dxQWRjTVEwcmtISFZ6WitWbk4NCmxVSGFxdW5UQkR4aitPSUhXN0lEczFqSWtEZWZnQngyTmh5eDR3anRBTHBhVW1ja1B1SkhTcURSV3JvQkc1c01Uc3RwWmwNCnNtb0diTmxFK0o1dE9lMnhqYVYzNzdRNVd4L0FIemd0T09RemZNL3lTZjMzTDhCS1Y0a3J4eXV3ZW95T1Q5OU9ia0ltaUUNCnpTMEQxVERuUStmSTNjdm1aL3lvcDZ0clA0a01wdWtWdC93ZUhFWU5nZkdPdHVHMndwU3oyRmpNcUcyT1NFd3ZpRXM3U0YNCmlwTGNWc2dpUzg3ckI5ZFBRejFYTGhhdW9MTDliY3BlOE9sZW50VkI5VHFaU1lqaTJoeUNtZG5id25CS2QyMGVaUlh0S3QNCnh3SUpDdkpESGwyWk9wTVVUcnIydFcwSkVFZU1QSDJWMCs4amg3aGxlQ0NLcDhmdE1pcGVuWTdvelR1M1JVTUdNcjB4eTINCmhUalVJNkVGU0ppVGlKVE9ibGVhcGVPMVE1czdHaU5ibmdZQXFhN3h3RmJuYllrODJ3ekxPbzdEUjYzODhJbzVQcEFRQUENCkFNQUtXbURSMWU5bXlncm8wZmtQUDQ3dGsxMnF5bWpkQzVtRU1SNm9TOTNMbGRaK1ptKzBxVlBxN1BSQ3JPZlpLcFJSQ1UNCmJOUkM0ZFJhUHk0ek85cEdqdzE3ZlhjUGxGQzRaQUN1anhnRzhvazdYNEdGVlZEQ2lySFRySFhWN0ozNUtPMnR5MloyR2UNCms2L0dhMUpCMlBLN0tJZFlnMWpjY3lUR0FsZTlmcjIyU21nZHVoUmt2WlZsVU9mMHp2ZDhERzlVcktYUURWTERHd1QrWlkNClp2ODhYdGduZzZneU1jZXhZaHZZY04yMUo4ay9wNmM1ZGVuUXNNL0QxN0Qyck9iNE1BQUFEQkFPcDBJWitTVWxXY0xzbjMNCmVwQk1pTVAwdm5LUTI4UUd4NDl1bW14VXdhMTI0djk5YzhtTXZ5TXJPYnFsODdjZjQwWTlqdUhsSGZKSzd0MXhNdE5qU3QNCkJWRlNjU2E5Sk56S0hKRTJaYlJma1d1ZXpScytGbytKcjU0YVppQjNvcjNFeUtaamNZY2RFTG5ROHNjNmJXd25Ic29WSHkNCmNpTThtcUhudHRqeXJPZFdJRi9CTURlYjF5WkliYlQ0aWN3Y1N2TEJOVE95dllwakg1RWNsTXdXcWlsQ2NxVVJyTmtZVXMNCnJWZkFabDZuUmE5N0FNNDd6THhBT0RZT1FzbjZhdk5RQUFBTUVBNTk2ejRYZkxrQ09MT3drUi85NS90WEYzS3p4MjFsdC8NCllBVExmRlBKbHdNaGRxN1d2VG9LZWxNV0QwNUxXYlZxYitNOGU3SWZSQlducEp0V1RxMVBCY3ltT2k1TkprSmZnWWhqdGgNCjlqT1k4WTVCWWlvcENRUUFtTWc3SHF3a0xUSUdUU25IdDN5ZGFTK21TaVFTQUhLb1VKbmp4cEdLQ3ZyVGk5eHdxTFpZT1YNClZvOHFCZ003M1c1TWUyQWI0YnpPaEt4Tm9iTFpqWkxqZDJoeHRyWENJaityRXVRa09NT1hGTmR6NkFDR0hwQ09KTGp4clUNCmk4TGNwd2c5NlpWZkhCQUFBQUNtVnlaV0psUUhOMGVYZz0NCi0tLS0tRU5EIE9QRU5TU0ggUFJJVkFURSBLRVktLS0tLQ==").unwrap()).unwrap(); + let ssh_key = String::from_utf8(base64::decode("LS0tLS1CRUdJTiBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0KYjNCbGJuTnphQzFyWlhrdGRqRUFBQUFBQkc1dmJtVUFBQUFFYm05dVpRQUFBQUFBQUFBQkFBQUFNd0FBQUF0emMyZ3RaVwpReU5UVXhPUUFBQUNBTzZlaGNrV0JrNlcwd3lTZ0FIY0dSY3JneW1IVThqRWVKRm5yQ2k1ZjZaQUFBQUpERlV0TVZ4VkxUCkZRQUFBQXR6YzJndFpXUXlOVFV4T1FBQUFDQU82ZWhja1dCazZXMHd5U2dBSGNHUmNyZ3ltSFU4akVlSkZuckNpNWY2WkEKQUFBRUQ0aGwvTmk0aGgvK3oxUm4wdWtMcm5mQ0xrN1BUWmErbVNQYk01ZS9aS0pnN3A2RnlSWUdUcGJUREpLQUFkd1pGeQp1REtZZFR5TVI0a1dlc0tMbC9wa0FBQUFDbVZ5WldKbFFITjBlWGdCQWdNPQotLS0tLUVORCBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0K").unwrap()).unwrap(); let invalid_ssh_key = String::from_utf8(base64::decode("LS0tLS1CRUdJTiBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0KYjNCbGJuTnphQzFyWlhrdGRqRUFBQUFBQ21GbGN6STFOaTFqZEhJQUFBQUdZbU55ZVhCMEFBQUFHQUFBQUJCNzZzbWIzVgp5WFB3SE12dm8zWTB5M0FBQUFFQUFBQUFFQUFBR1hBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFCZ1FDOVZHbm13cjZCClRHdWxzODhEaXRXaE5IUUoxMjV0eGxHa2EzNDNxUVB2S3dSc2VxN05SdFAzY2IxbDRMZytzdWozZ0lQYU5yM295SlBoRDIKZmIxbzF1cUFiOStkbWhwQXc4L1lCa05NZkRrdDRTWEpGZjZ3dUZwa1p4SHF3czNZUXF6cjhicVJaaHA0bXlnc2VwNFVHOApBaGxVMG5CUXFBREFhS3dBcmpLeUdBeWwwenRDYVdObm9sOVRZSmZuNEpOQW5YUDFONmMxMUVaRm5wKzJsMTVoSVdNd2NKClpCMnFFeTFSZzFVNXpuOVNSOURIVXhvN2p0ZkkrdWJWbHdnelBQaDVjZzAydVc0K0JwcFg1UGlpZ04rQlBNajc3WEJ0VTQKZzU3MmRDZHBSRjk3NjJ5SDBsY21nSkRqVnhnOTludVVGRDlwVG9nUTRrUENrdUluNmcxS3JObFdqY1R2c1hFS2JVS0xqawpkQkR2Yk1tbzZBaHJXRFhDSjZqRUN0T2Jka29XMGVjTGU4cXB3Nmh5N1NmdWppSm9QbnVsazRWenMwR2xPa3VPU0JIUmhJClhSc25NaFNiNnh2dDl6QldJcklvZDZoWnhuQ0V2SWRESzlacVBnOXJpbXc4bG8rUkFwdm1ySnRINUhsbFJiYWh4K2RUU1cKM2hCa1BlMnNDL1UvRUFBQVdBVXBEOTFIQTAzSnQyNFFSSFVXRDAvVTJGMTBzZE5WN0w4bkhMeVNibFBnSFhMc3lpSTFxOQo0NXBOUEQyNElBakNzQ08rVHREcXc3MDhlNXliUWhXUCsybkxtdGQwclEyTXh3SnZwUjlGcEV6UDFyejRYUDVUbzZDN3N1CmZpd0JPZWd6bjhQT1hGSmRvRk9Ud3E3dWhaM201NE93NHZvZkFKSHdtYWtwTGZMd2R1TnQ3S1RNQkVpT3VlM0ZXTGtCR0wKQUE1RGtoYVlpVGgyajB2YU9jUWhxZVphVEp6V2tidUcvb29DK1cwcTVXcFNZdFlxREFhWEh0bG8rZGtOMFEzZVVhcm1FTQpGcy9tdEpha3dhOVhCMVgzMndKbUpIdmN0OG4vVzA1T0N5V0U1Y2szeitRQVB3a2pGK0hKOGlOZDluVk5zckx1T010a2VQCk1aMTZreTg5WUVSZVQ1QXRJU1lRd0JQU2tsTFZKL3VaOCszK2Vyc3JrOW1aakw3ZXpISnV4ZysxUmR1T3BPeWpXMTRoTGYKblJQTDlKOXgvZWZ2MFV0L3BpR3M5NEFRcFFVZnJFdXpjL1dmejRocUtzVUxnT0VnblZBWXpuSksyWHJGeTN4aWlKVkFVUQpZcm4xak9lU1oyTWV0cjJvd05VdVM3cEhGTHZIWURRWklURmxVaFlOYUx0ejV5WU9HTCtFbEVxQm4wT1FFenNESDhROEpFCk5jWGVxUjFRTE4rTUJaMFZqQ2Q3T0ExTGpXZVVrdjNMaFJER3lPS3RjWk5OeFl5MkgwRWlmYzIvRHpLMnlpcVRQWUdMbHYKOWhZTlZZcC8xOGxhUkFOL040MlVDMjRmS0hFZ2lYVTNnL3RCZkZmbEFBWThKSE9sQUJEdXFWYjJkWHZKdXFLeUJMUElqVQo5cVl5VXNOVXhWS2M2ZWh4VU4wcVlnTmV2Z0JmMXVSZkxCY2c3SjVJVDZQQ2dSa3lNenBRakY1RkhuM0J6SVMrb3ZFSnNaCk5LNklYbDJIY3FncExTWUFkTFZlZEZOUzlkVU01blpMdlJEMjkyc0FQWm5aaU91Z3pwSWNrMllFcXpscjc2NXlUakRJdWgKR3kvdFlBQ3FIZHV4S2pMdGc0OXpjZjdNN2xESGNuVEY1MlJsazEyR2x1emZGK1dhZDF3eUFKVnNyUmtqVFZYVHhnTEV6MQo4SzF0WUtVOWoyc3grUE1Vd0JxM3lQR2lTaEgydWp6em82SUc1cnVYSTAwZXVkT2t1NVVrSHhBVnJneUI1S0M2VFRMR1BYCnhQMFN5Zk12dXJycDdvMnhsK2dkSVc0c0dudEJ2V0RHRVFSY0RxbWdLV0tuNTNsbmg5U1Urcmh2UkdhRFJueENuYkNwUEUKTE82V0lKUXVPQm54bzhWcGU0R2JLc2NmSktKSzlZV2ZIOFEvYzBncnE0ZDh5ZmRwUG1uc3hHOEpoTFVuMEhpRFEzQytaMgpzU1RPeU85TDAySUZIdDdIUEY2OWRWR3c3M0pPU1FiL05GK2g5cGRVazBScGNRdGFaTm9TMHg2a3RCQXljK0o0VUpUYTliCkdENWRaSE1KVHBvcWFZUDV0dFlnMjlBQkpUUURMa0tnbWxWRGNtK28zRTN3cTlySWFXMlhpNDQrc3RnTVJVS1J5R041d1EKM2xTWjk1QXBpWFlpRkNONUVrWitUci96TDAraVdwUHRCRzlJZmlGbmlqVlVYUnpEWHZxeGE1QTQ1YUlNWDhad2U5ckxFdAphaVRaOUI5d2tVb0tYdXlDU3plQXhMTGU2aG8wLzBDbmhSR3NoVGg1UDd6aFA4bVExRGZMYlFCRU0zOHJMWlplMExVVVhZCkZpZkFXc3BFRDk2VjBMckhxRkd0Z0dzd1NQcWRBRzBPTDBWekRUbFRucDJVWDY0SEhjUzF2MUMyQnNxbllWbkJNL3p5aUYKQXhabDB4cGRPUVVuKzV2V2VHUXZsQkhGeU0vQmtXRVhMbjc1YVNQL3JwcnlZeGdOeWx2M2NiRWNYZXoyWXdLM2UrN1NnZAoxRzFZUVVtNStqNy90Q0x5aFluL1VjRzJhTHJNc3pRY1FoWTE4Sk9IOXF6a2FacWdYckFybnE0dWluT25sbFBKaGJ3ZTVrCmgvMmdyTlVqbEsrRHYxQ2dGZUVDcm9yRHo4L3ZxZW1QNXdVWWF5bFNWWVZ3UHM1bkxDQWUrVlNobFlIOXlNb3JwanNXc3MKYlg0UlAvVGd3TmNtRnBuZ21kTXppNmtIUXhSc2pUT3VxZ3Vsb01FUVZmQ3JkNGxBeWp3eVhRaEcrd2dWMXBuempCZlR4eQpZeFBrc1VGaTg3aEVkZ1RPZ2M5MHlNamVoVGhHOGRMWGEvd0NOU0hLZ1pBbFBZbWdLd2ZvcFlBMjQxdUlxR2J0WUtqSTFSCnVHU2JqSU80dUVYbkJ5eWVZTnA3Z29iR2NVc1BGV0doY1FPV05QZnl5K1crQ0xhKzVpYkJCZEF2NStVdlZZUHFGMHhTNy8KUm1TbW9BPT0KLS0tLS1FTkQgT1BFTlNTSCBQUklWQVRFIEtFWS0tLS0t").unwrap()).unwrap(); let clone_dir = DirectoryForTests::new_with_random_suffix("/tmp/engine_test_submodule".to_string()); let get_credentials = |user: &str| { diff --git a/src/models.rs b/src/models.rs index 8a88f4bb..128634f2 100644 --- a/src/models.rs +++ b/src/models.rs @@ -2,16 +2,15 @@ use std::collections::BTreeMap; use std::fmt::{Display, Formatter}; use std::hash::Hash; use std::net::Ipv4Addr; -use std::path::Path; use std::str::FromStr; use std::sync::Arc; use chrono::{DateTime, Utc}; -use git2::{Cred, CredentialType, Error}; use itertools::Itertools; use rand::distributions::Alphanumeric; use rand::Rng; use serde::{Deserialize, Serialize}; +use url::Url; use crate::build_platform::{Build, BuildOptions, Credentials, GitRepository, Image, SshKey}; use crate::cloud_provider::aws::databases::mongodb::MongoDB; @@ -22,7 +21,7 @@ use crate::cloud_provider::service::{DatabaseOptions, StatefulService, Stateless use crate::cloud_provider::utilities::VersionsNumber; use crate::cloud_provider::CloudProvider; use crate::cloud_provider::Kind as CPKind; -use crate::git; +use crate::container_registry::ContainerRegistryInfo; use crate::logger::Logger; use crate::utilities::get_image_tag; @@ -102,17 +101,14 @@ impl Environment { pub fn to_qe_environment( &self, context: &Context, - built_applications: &Vec>, cloud_provider: &dyn CloudProvider, + container_registry: &ContainerRegistryInfo, logger: Box, ) -> crate::cloud_provider::environment::Environment { let applications = self .applications .iter() - .map(|x| match built_applications.iter().find(|y| x.id.as_str() == y.id()) { - Some(app) => x.to_stateless_service(context, app.image().clone(), cloud_provider, logger.clone()), - _ => x.to_stateless_service(context, x.to_image(), cloud_provider, logger.clone()), - }) + .map(|x| x.to_stateless_service(context, x.to_image(container_registry), cloud_provider, logger.clone())) .filter(|x| x.is_some()) .map(|x| x.unwrap()) .collect::>(); @@ -365,52 +361,24 @@ impl Application { } } - pub fn to_image(&self) -> Image { - self.to_image_with_commit(&self.commit_id) - } - - pub fn to_image_from_parent_commit

(&self, clone_repo_into_dir: P) -> Result, Error> - where - P: AsRef, - { - let parent_commit_id = git::get_parent_commit_id( - self.git_url.as_str(), - self.commit_id.as_str(), - clone_repo_into_dir, - &|_| match &self.git_credentials { - None => vec![], - Some(creds) => vec![( - CredentialType::USER_PASS_PLAINTEXT, - Cred::userpass_plaintext(creds.login.as_str(), creds.access_token.as_str()).unwrap(), - )], - }, - )?; - - Ok(match parent_commit_id { - Some(id) => Some(self.to_image_with_commit(&id)), - None => None, - }) - } - - pub fn to_image_with_commit(&self, commit_id: &String) -> Image { + pub fn to_image(&self, cr_info: &ContainerRegistryInfo) -> Image { Image { application_id: self.id.clone(), - name: self.name.clone(), + name: (cr_info.get_image_name)(&self.name), tag: get_image_tag( &self.root_path, &self.dockerfile_path, &self.environment_vars, - commit_id, + &self.commit_id, ), commit_id: self.commit_id.clone(), - registry_name: None, - registry_secret: None, - registry_url: None, - registry_docker_json_config: None, + registry_name: cr_info.registry_name.clone(), + registry_url: cr_info.endpoint.clone(), + registry_docker_json_config: cr_info.registry_docker_json_config.clone(), } } - pub fn to_build(&self) -> Build { + pub fn to_build(&self, registry_url: &ContainerRegistryInfo) -> Build { // Retrieve ssh keys from env variables const ENV_GIT_PREFIX: &str = "GIT_SSH_KEY"; let env_ssh_keys: Vec<(String, String)> = self @@ -471,7 +439,7 @@ impl Application { root_path: self.root_path.clone(), buildpack_language: self.buildpack_language.clone(), }, - image: self.to_image(), + image: self.to_image(registry_url), options: BuildOptions { environment_variables: self .environment_vars @@ -1159,7 +1127,7 @@ pub struct Context { workspace_root_dir: String, lib_root_dir: String, test_cluster: bool, - docker_host: Option, + docker_host: Option, features: Vec, metadata: Option, } @@ -1172,13 +1140,13 @@ pub enum Features { // trait used to reimplement clone without same fields // this trait is used for Context struct -pub trait Clone2 { +pub trait CloneForTest { fn clone_not_same_execution_id(&self) -> Self; } // for test we need to clone context but to change the directory workspace used // to to this we just have to suffix the execution id in tests -impl Clone2 for Context { +impl CloneForTest for Context { fn clone_not_same_execution_id(&self) -> Context { let mut new = self.clone(); let suffix = rand::thread_rng() @@ -1199,7 +1167,7 @@ impl Context { workspace_root_dir: String, lib_root_dir: String, test_cluster: bool, - docker_host: Option, + docker_host: Option, features: Vec, metadata: Option, ) -> Self { @@ -1236,8 +1204,8 @@ impl Context { self.lib_root_dir.as_str() } - pub fn docker_tcp_socket(&self) -> Option<&String> { - self.docker_host.as_ref() + pub fn docker_tcp_socket(&self) -> &Option { + &self.docker_host } pub fn metadata(&self) -> Option<&Metadata> { @@ -1276,16 +1244,6 @@ impl Context { } } - pub fn docker_build_options(&self) -> Option> { - match &self.metadata { - Some(meta) => meta - .docker_build_options - .clone() - .map(|b| b.split(' ').map(|x| x.to_string()).collect()), - _ => None, - } - } - // Qovery features pub fn is_feature_enabled(&self, name: &Features) -> bool { for feature in &self.features { @@ -1303,7 +1261,6 @@ impl Context { pub struct Metadata { pub dry_run_deploy: Option, pub resource_expiration_in_seconds: Option, - pub docker_build_options: Option, pub forced_upgrade: Option, pub disable_pleco: Option, } @@ -1312,14 +1269,12 @@ impl Metadata { pub fn new( dry_run_deploy: Option, resource_expiration_in_seconds: Option, - docker_build_options: Option, forced_upgrade: Option, disable_pleco: Option, ) -> Self { Metadata { dry_run_deploy, resource_expiration_in_seconds, - docker_build_options, forced_upgrade, disable_pleco, } diff --git a/src/transaction.rs b/src/transaction.rs index 82f3cb72..404b044f 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -1,10 +1,7 @@ -use std::collections::HashMap; use std::thread; -use crate::build_platform::BuildResult; use crate::cloud_provider::kubernetes::Kubernetes; -use crate::cloud_provider::service::{Application, Service}; -use crate::container_registry::PushResult; +use crate::cloud_provider::service::Service; use crate::engine::EngineConfig; use crate::errors::{EngineError, Tag}; use crate::events::{EngineEvent, EventMessage}; @@ -102,129 +99,48 @@ impl<'a> Transaction<'a> { Ok(()) } - fn load_build_app_cache(&self, app: &crate::models::Application) -> Result<(), EngineError> { - let container_registry = self.engine.container_registry(); - let mut image = app.to_image(); - - image.tag = String::from("latest"); - // pull image from container registry - // FIXME: if one day we use something else than LocalDocker to build image - // FIXME: we'll need to send the PullResult to the Build implementation - let _ = match container_registry.pull(&image) { - Ok(pull_result) => pull_result, - Err(err) => { - self.logger.log( - LogLevel::Error, - EngineEvent::Error( - err.clone(), - Some(EventMessage::new_from_safe( - "Something goes wrong while pulling image from container registry".to_string(), - )), - ), - ); - - return Err(err); - } - }; - - Ok(()) - } - - fn build_applications( + fn build_and_push_applications( &self, environment: &Environment, option: &DeploymentOption, - ) -> Result>, EngineError> { + ) -> Result<(), EngineError> { // do the same for applications let apps_to_build = environment .applications .iter() // build only applications that are set with Action: Create - .filter(|app| app.action == Action::Create); - - let application_and_result_tuples = apps_to_build - .map(|app| { - let image = app.to_image(); - let build_result = if option.force_build || !self.engine.container_registry().does_image_exists(&image) - { - // If an error occurred we can skip it. It's not critical. - let _ = self.load_build_app_cache(app); - - // only if the build is forced OR if the image does not exist in the registry - self.engine - .build_platform() - .build(app.to_build(), option.force_build, &self.is_transaction_aborted) - } else { - // use the cache - Ok(BuildResult::new(app.to_build())) - }; - - (app, build_result) - }) + .filter(|app| app.action == Action::Create) .collect::>(); - let mut applications: Vec> = Vec::with_capacity(application_and_result_tuples.len()); - for (application, result) in application_and_result_tuples { - // catch build error, can't do it in Fn - let build_result = match result { - Err(err) => { - error!("build error for application {}: {:?}", application.id.as_str(), err); - return Err(err); - } - Ok(build_result) => build_result, - }; - - if let Some(app) = application.to_application( - self.engine.context(), - &build_result.build.image, - self.engine.cloud_provider(), - self.logger.clone(), - ) { - applications.push(app) - } + // If nothing to build, do nothing + if apps_to_build.is_empty() { + return Ok(()); } - Ok(applications) - } + // Do setup of registry and be sure we are login to the registry + let cr_registry = self.engine.container_registry(); + let _ = cr_registry.create_registry()?; + let registry = self.engine.container_registry().login()?; - fn push_applications( - &self, - applications: Vec>, - option: &DeploymentOption, - ) -> Result, PushResult)>, EngineError> { - let application_and_push_results: Vec<_> = applications - .into_iter() - .map(|mut app| { - match self.engine.container_registry().push(app.image(), option.force_push) { - Ok(push_result) => { - // I am not a big fan of doing that but it's the most effective way - app.set_image(push_result.image.clone()); - Ok((app, push_result)) - } - Err(err) => Err(err), - } - }) - .collect(); + for app in apps_to_build.into_iter() { + let app_build = app.to_build(®istry); - let mut results: Vec<(Box, PushResult)> = vec![]; - for result in application_and_push_results.into_iter() { - match result { - Ok(tuple) => results.push(tuple), - Err(err) => { - self.logger.log( - LogLevel::Error, - EngineEvent::Error( - err.clone(), - Some(EventMessage::new_from_safe("Error pushing docker image".to_string())), - ), - ); - - return Err(err); - } + // If image already exist in the registry, skip the build + if !option.force_build && cr_registry.does_image_exists(&app_build.image) { + continue; } + + // Be sure that our repository exist before trying to pull/push images from it + let _ = self.engine.container_registry().create_repository(&app.name)?; + + // Ok now everything is setup, we can try to build the app + let _ = self + .engine + .build_platform() + .build(app_build, &self.is_transaction_aborted)?; } - Ok(results) + Ok(()) } pub fn rollback(&self) -> Result<(), RollbackError> { @@ -269,63 +185,11 @@ impl<'a> Transaction<'a> { /// This function is a wrapper to correctly revert all changes of an attempted deployment AND /// if a failover environment is provided, then rollback. - fn rollback_environment(&self, environment_action: &EnvironmentAction) -> Result<(), RollbackError> { - let qe_environment = |environment: &Environment| { - let mut _applications = Vec::with_capacity(environment.applications.len()); - for application in environment.applications.iter() { - let build = application.to_build(); - - if let Some(x) = application.to_application( - self.engine.context(), - &build.image, - self.engine.cloud_provider(), - self.logger.clone(), - ) { - _applications.push(x) - } - } - - let qe_environment = environment.to_qe_environment( - self.engine.context(), - &_applications, - self.engine.cloud_provider(), - self.logger.clone(), - ); - - qe_environment - }; - - match environment_action { - EnvironmentAction::Environment(te) => { - // revert changes but there is no failover environment - let target_qe_environment = qe_environment(te); - - let action = match te.action { - Action::Create => self - .engine - .kubernetes() - .deploy_environment_error(&target_qe_environment), - Action::Pause => self.engine.kubernetes().pause_environment_error(&target_qe_environment), - Action::Delete => self - .engine - .kubernetes() - .delete_environment_error(&target_qe_environment), - Action::Nothing => Ok(()), - }; - - let _ = match action { - Ok(_) => {} - Err(err) => return Err(RollbackError::CommitError(err)), - }; - - Err(RollbackError::NoFailoverEnvironment) - } - } + fn rollback_environment(&self, _environment_action: &EnvironmentAction) -> Result<(), RollbackError> { + Ok(()) } pub fn commit(mut self) -> TransactionResult { - let mut applications_by_environment: HashMap<&Environment, Vec>> = HashMap::new(); - for step in self.steps.clone().into_iter() { // execution loop self.executed_steps.push(step.clone()); @@ -372,7 +236,7 @@ impl<'a> Transaction<'a> { EnvironmentAction::Environment(te) => te, }; - let applications_builds = match self.build_applications(target_environment, &option) { + match self.build_and_push_applications(target_environment, &option) { Ok(apps) => apps, Err(engine_err) => { self.logger.log( @@ -392,30 +256,6 @@ impl<'a> Transaction<'a> { }; } }; - - if (self.is_transaction_aborted)() { - return TransactionResult::Canceled; - } - - let applications = match self.push_applications(applications_builds, &option) { - Ok(results) => { - let applications = results.into_iter().map(|(app, _)| app).collect::>(); - - applications - } - Err(engine_err) => { - warn!("ROLLBACK STARTED! an error occurred {:?}", engine_err); - return match self.rollback() { - Ok(_) => TransactionResult::Rollback(engine_err), - Err(err) => { - error!("ROLLBACK FAILED! fatal error: {:?}", err); - TransactionResult::UnrecoverableError(engine_err, err) - } - }; - } - }; - - applications_by_environment.insert(target_environment, applications); } Step::DeployEnvironment(environment_action) => { if (self.is_transaction_aborted)() { @@ -423,7 +263,7 @@ impl<'a> Transaction<'a> { } // deploy complete environment - match self.commit_environment(environment_action, &applications_by_environment, |qe_env| { + match self.commit_environment(environment_action, |qe_env| { self.engine.kubernetes().deploy_environment(qe_env) }) { TransactionResult::Ok => {} @@ -439,7 +279,7 @@ impl<'a> Transaction<'a> { } // pause complete environment - match self.commit_environment(environment_action, &applications_by_environment, |qe_env| { + match self.commit_environment(environment_action, |qe_env| { self.engine.kubernetes().pause_environment(qe_env) }) { TransactionResult::Ok => {} @@ -455,7 +295,7 @@ impl<'a> Transaction<'a> { } // delete complete environment - match self.commit_environment(environment_action, &applications_by_environment, |qe_env| { + match self.commit_environment(environment_action, |qe_env| { self.engine.kubernetes().delete_environment(qe_env) }) { TransactionResult::Ok => {} @@ -534,12 +374,7 @@ impl<'a> Transaction<'a> { } } - fn commit_environment( - &self, - environment_action: &EnvironmentAction, - applications_by_environment: &HashMap<&Environment, Vec>>, - action_fn: F, - ) -> TransactionResult + fn commit_environment(&self, environment_action: &EnvironmentAction, action_fn: F) -> TransactionResult where F: Fn(&crate::cloud_provider::environment::Environment) -> Result<(), EngineError>, { @@ -547,16 +382,11 @@ impl<'a> Transaction<'a> { EnvironmentAction::Environment(te) => te, }; - let empty_vec = Vec::with_capacity(0); - let built_applications = match applications_by_environment.get(target_environment) { - Some(applications) => applications, - None => &empty_vec, - }; - + let registry_info = self.engine.container_registry().login().unwrap(); let qe_environment = target_environment.to_qe_environment( self.engine.context(), - built_applications, self.engine.cloud_provider(), + ®istry_info, self.logger.clone(), ); diff --git a/test_utilities/Cargo.lock b/test_utilities/Cargo.lock index f794d3af..2b2e33bf 100644 --- a/test_utilities/Cargo.lock +++ b/test_utilities/Cargo.lock @@ -2147,6 +2147,7 @@ dependencies = [ "tracing-subscriber", "trust-dns-resolver", "url 2.2.2", + "urlencoding", "uuid 0.8.2", "walkdir", ] @@ -3321,6 +3322,7 @@ dependencies = [ "time 0.2.24", "tracing", "tracing-subscriber", + "url 2.2.2", "uuid 0.8.2", ] @@ -3957,6 +3959,12 @@ dependencies = [ "url 1.7.2", ] +[[package]] +name = "urlencoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b90931029ab9b034b300b797048cf23723400aa757e8a2bfb9d748102f9821" + [[package]] name = "uuid" version = "0.7.4" diff --git a/test_utilities/Cargo.toml b/test_utilities/Cargo.toml index f777f2d0..01e81fbd 100644 --- a/test_utilities/Cargo.toml +++ b/test_utilities/Cargo.toml @@ -28,6 +28,7 @@ hashicorp_vault = "2.0.1" maplit = "1.0.2" uuid = { version = "0.8", features = ["v4"] } const_format = "0.2.22" +url = "2.2.2" # Digital Ocean Deps digitalocean = "0.1.1" diff --git a/test_utilities/src/aws.rs b/test_utilities/src/aws.rs index 77871e97..8276d917 100644 --- a/test_utilities/src/aws.rs +++ b/test_utilities/src/aws.rs @@ -9,7 +9,6 @@ use qovery_engine::cloud_provider::models::NodeGroups; use qovery_engine::cloud_provider::qovery::EngineLocation::ClientSide; use qovery_engine::cloud_provider::Kind::Aws; use qovery_engine::cloud_provider::{CloudProvider, TerraformStateCredentials}; -use qovery_engine::container_registry::docker_hub::DockerHub; use qovery_engine::container_registry::ecr::ECR; use qovery_engine::dns_provider::DnsProvider; use qovery_engine::engine::EngineConfig; @@ -54,17 +53,6 @@ pub fn container_registry_ecr(context: &Context) -> ECR { ) } -pub fn container_registry_docker_hub(context: &Context) -> DockerHub { - DockerHub::new( - context.clone(), - "my-docker-hub-id-123", - "my-default-docker-hub", - "qoveryrd", - "3b9481fe-74e7-4d7b-bc08-e147c9fd4f24", - logger(), - ) -} - pub fn aws_default_engine_config(context: &Context, logger: Box) -> EngineConfig { AWS::docker_cr_engine( &context, @@ -75,7 +63,6 @@ pub fn aws_default_engine_config(context: &Context, logger: Box) -> None, ) } - impl Cluster for AWS { fn docker_cr_engine( context: &Context, diff --git a/test_utilities/src/common.rs b/test_utilities/src/common.rs index 6bfb7081..029ec8db 100644 --- a/test_utilities/src/common.rs +++ b/test_utilities/src/common.rs @@ -6,7 +6,7 @@ use chrono::Utc; use qovery_engine::cloud_provider::utilities::sanitize_name; use qovery_engine::dns_provider::DnsProvider; use qovery_engine::models::{ - Action, Application, Clone2, Context, Database, DatabaseKind, DatabaseMode, Environment, EnvironmentAction, + Action, Application, CloneForTest, Context, Database, DatabaseKind, DatabaseMode, Environment, EnvironmentAction, GitCredentials, Port, Protocol, Route, Router, Storage, StorageType, }; diff --git a/test_utilities/src/digitalocean.rs b/test_utilities/src/digitalocean.rs index 31d653d6..656db11e 100644 --- a/test_utilities/src/digitalocean.rs +++ b/test_utilities/src/digitalocean.rs @@ -1,5 +1,4 @@ use const_format::formatcp; -use qovery_engine::build_platform::Image; use qovery_engine::cloud_provider::aws::kubernetes::VpcQoveryNetworkMode; use qovery_engine::cloud_provider::digitalocean::kubernetes::DoksOptions; use qovery_engine::cloud_provider::digitalocean::network::vpc::VpcInitKind; @@ -37,7 +36,7 @@ pub fn container_registry_digital_ocean(context: &Context) -> DOCR { DOCR::new( context.clone(), DOCR_ID, - "default-docr-registry-qovery-do-test", + DOCR_ID, secrets.DIGITAL_OCEAN_TOKEN.unwrap().as_str(), logger(), ) @@ -163,11 +162,11 @@ impl Cluster for DO { pub fn clean_environments( context: &Context, - environments: Vec, + _environments: Vec, secrets: FuncTestsSecrets, _region: DoRegion, ) -> Result<(), EngineError> { - let do_cr = DOCR::new( + let _do_cr = DOCR::new( context.clone(), "test", "test", @@ -178,14 +177,23 @@ pub fn clean_environments( logger(), ); + // FIXME: re-enable it, or let pleco do its job ? + /* // delete images created in registry + let registry_url = do_cr.login()?; for env in environments.iter() { - for image in env.applications.iter().map(|a| a.to_image()).collect::>() { - if let Err(e) = do_cr.delete_image(&image) { - return Err(e); - } + for image in env + .applications + .iter() + .map(|a| a.to_image(®istry_url)) + .collect::>() + { + //if let Err(e) = do_cr.delete_registry(&image.name) { + // return Err(e); + //} } } + */ Ok(()) } diff --git a/test_utilities/src/scaleway.rs b/test_utilities/src/scaleway.rs index ce61d38e..c6149d54 100644 --- a/test_utilities/src/scaleway.rs +++ b/test_utilities/src/scaleway.rs @@ -18,6 +18,7 @@ use qovery_engine::cloud_provider::aws::kubernetes::VpcQoveryNetworkMode; use qovery_engine::cloud_provider::models::NodeGroups; use qovery_engine::cloud_provider::qovery::EngineLocation; use qovery_engine::cloud_provider::Kind::Scw; +use qovery_engine::container_registry::ContainerRegistry; use qovery_engine::dns_provider::DnsProvider; use qovery_engine::errors::EngineError; use qovery_engine::logger::Logger; @@ -239,8 +240,14 @@ pub fn clean_environments( ); // delete images created in registry + let registry_url = container_registry_client.login()?; for env in environments.iter() { - for image in env.applications.iter().map(|a| a.to_image()).collect::>() { + for image in env + .applications + .iter() + .map(|a| a.to_image(®istry_url)) + .collect::>() + { if let Err(e) = container_registry_client.delete_image(&image) { return Err(e); } diff --git a/test_utilities/src/utilities.rs b/test_utilities/src/utilities.rs index ca06c1d8..7c8ab263 100644 --- a/test_utilities/src/utilities.rs +++ b/test_utilities/src/utilities.rs @@ -80,7 +80,6 @@ pub fn context(organization_id: &str, cluster_id: &str) -> Context { None => Some(7200), } }, - docker_build_options: Some("--network host".to_string()), forced_upgrade: Option::from({ match env::var_os("forced_upgrade") { Some(_) => true, @@ -363,7 +362,7 @@ impl FuncTestsSecrets { } pub fn build_platform_local_docker(context: &Context, logger: Box) -> LocalDocker { - LocalDocker::new(context.clone(), "oxqlm3r99vwcmvuj", "qovery-local-docker", logger) + LocalDocker::new(context.clone(), "oxqlm3r99vwcmvuj", "qovery-local-docker", logger).unwrap() } pub fn init() -> Instant { diff --git a/tests/aws/aws_databases.rs b/tests/aws/aws_databases.rs index 43f00166..c1262911 100644 --- a/tests/aws/aws_databases.rs +++ b/tests/aws/aws_databases.rs @@ -3,19 +3,18 @@ extern crate test_utilities; use ::function_name::named; use qovery_engine::cloud_provider::Kind; use qovery_engine::models::{ - Action, Clone2, Context, Database, DatabaseKind, DatabaseMode, Environment, EnvironmentAction, Port, Protocol, + Action, CloneForTest, Database, DatabaseKind, DatabaseMode, EnvironmentAction, Port, Protocol, }; -use test_utilities::aws::{aws_default_engine_config, AWS_KUBERNETES_VERSION, AWS_TEST_REGION}; +use test_utilities::aws::aws_default_engine_config; use tracing::{span, Level}; use self::test_utilities::aws::{AWS_DATABASE_DISK_TYPE, AWS_DATABASE_INSTANCE_TYPE}; use self::test_utilities::utilities::{ context, engine_run_test, generate_id, get_pods, get_svc_name, init, is_pod_restarted_env, logger, FuncTestsSecrets, }; -use qovery_engine::cloud_provider::aws::AWS; use qovery_engine::models::DatabaseMode::{CONTAINER, MANAGED}; use qovery_engine::transaction::TransactionResult; -use test_utilities::common::{test_db, Cluster, ClusterDomain, Infrastructure}; +use test_utilities::common::{test_db, Infrastructure}; /** ** diff --git a/tests/aws/aws_environment.rs b/tests/aws/aws_environment.rs index cc767323..aaed2d71 100644 --- a/tests/aws/aws_environment.rs +++ b/tests/aws/aws_environment.rs @@ -5,16 +5,13 @@ use self::test_utilities::utilities::{ engine_run_test, generate_id, get_pods, get_pvc, is_pod_restarted_env, logger, FuncTestsSecrets, }; use ::function_name::named; -use qovery_engine::build_platform::{BuildPlatform, CacheResult}; use qovery_engine::cloud_provider::Kind; use qovery_engine::cmd::kubectl::kubernetes_get_all_pdbs; -use qovery_engine::container_registry::{ContainerRegistry, PullResult}; -use qovery_engine::models::{Action, Clone2, EnvironmentAction, Port, Protocol, Storage, StorageType}; +use qovery_engine::models::{Action, CloneForTest, EnvironmentAction, Port, Protocol, Storage, StorageType}; use qovery_engine::transaction::TransactionResult; use std::collections::BTreeMap; -use std::time::SystemTime; -use test_utilities::aws::{aws_default_engine_config, container_registry_ecr, AWS_KUBERNETES_VERSION, AWS_TEST_REGION}; -use test_utilities::utilities::{build_platform_local_docker, context, init, kubernetes_config_path}; +use test_utilities::aws::aws_default_engine_config; +use test_utilities::utilities::{context, init, kubernetes_config_path}; use tracing::{span, Level}; // TODO: @@ -75,98 +72,6 @@ fn deploy_a_working_environment_with_no_router_on_aws_eks() { }) } -#[cfg(feature = "test-aws-self-hosted")] -#[named] -#[test] -fn test_build_cache() { - let test_name = function_name!(); - engine_run_test(|| { - init(); - let span = span!(Level::INFO, "test", name = test_name); - let _enter = span.enter(); - - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .AWS_TEST_ORGANIZATION_ID - .as_ref() - .expect("AWS_TEST_ORGANIZATION_ID is not set") - .as_str(), - secrets - .AWS_TEST_CLUSTER_ID - .as_ref() - .expect("AWS_TEST_CLUSTER_ID is not set") - .as_str(), - ); - let engine_config = aws_default_engine_config(&context, logger()); - - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - - let ecr = container_registry_ecr(&context); - let local_docker = build_platform_local_docker(&context, logger()); - let app = environment.applications.first().unwrap(); - let image = app.to_image(); - - let app_build = app.to_build(); - let _ = match local_docker.has_cache(&app_build) { - Ok(CacheResult::Hit) => assert!(false), - Ok(CacheResult::Miss(_)) => assert!(true), - Ok(CacheResult::MissWithoutParentBuild) => assert!(false), - Err(_) => assert!(false), - }; - - let _ = match ecr.pull(&image).unwrap() { - PullResult::Some(_) => assert!(false), - PullResult::None => assert!(true), - }; - - let cancel_task = || false; - let build_result = local_docker.build(app.to_build(), false, &cancel_task).unwrap(); - - let _ = match ecr.push(&build_result.build.image, false) { - Ok(_) => assert!(true), - Err(_) => assert!(false), - }; - - // TODO clean local docker cache - - let start_pull_time = SystemTime::now(); - let _ = match ecr.pull(&build_result.build.image).unwrap() { - PullResult::Some(_) => assert!(true), - PullResult::None => assert!(false), - }; - - let pull_duration = SystemTime::now().duration_since(start_pull_time).unwrap(); - - let _ = match local_docker.has_cache(&build_result.build) { - Ok(CacheResult::Hit) => assert!(true), - Ok(CacheResult::Miss(_)) => assert!(false), - Ok(CacheResult::MissWithoutParentBuild) => assert!(false), - Err(_) => assert!(false), - }; - - let start_pull_time = SystemTime::now(); - let _ = match ecr.pull(&image).unwrap() { - PullResult::Some(_) => assert!(true), - PullResult::None => assert!(false), - }; - - let pull_duration_2 = SystemTime::now().duration_since(start_pull_time).unwrap(); - - if pull_duration_2.as_millis() > pull_duration.as_millis() { - assert!(false); - } - - return test_name.to_string(); - }) -} - #[cfg(feature = "test-aws-self-hosted")] #[named] #[test] diff --git a/tests/aws/aws_kubernetes.rs b/tests/aws/aws_kubernetes.rs index fbcae223..c62d6439 100644 --- a/tests/aws/aws_kubernetes.rs +++ b/tests/aws/aws_kubernetes.rs @@ -7,9 +7,8 @@ use self::test_utilities::utilities::{ use ::function_name::named; use qovery_engine::cloud_provider::aws::kubernetes::VpcQoveryNetworkMode; use qovery_engine::cloud_provider::aws::kubernetes::VpcQoveryNetworkMode::{WithNatGateways, WithoutNatGateways}; -use qovery_engine::cloud_provider::aws::regions::{AwsRegion, AwsZones}; +use qovery_engine::cloud_provider::aws::regions::AwsRegion; use qovery_engine::cloud_provider::Kind; -use std::borrow::Borrow; use std::str::FromStr; use test_utilities::common::{cluster_test, ClusterDomain, ClusterTestType}; diff --git a/tests/digitalocean/do_databases.rs b/tests/digitalocean/do_databases.rs index 873cee85..a975987f 100644 --- a/tests/digitalocean/do_databases.rs +++ b/tests/digitalocean/do_databases.rs @@ -3,7 +3,7 @@ use tracing::{span, warn, Level}; use qovery_engine::cloud_provider::{Kind as ProviderKind, Kind}; use qovery_engine::models::{ - Action, Clone2, Context, Database, DatabaseKind, DatabaseMode, Environment, EnvironmentAction, Port, Protocol, + Action, CloneForTest, Database, DatabaseKind, DatabaseMode, EnvironmentAction, Port, Protocol, }; use qovery_engine::transaction::TransactionResult; use test_utilities::utilities::{ @@ -11,11 +11,10 @@ use test_utilities::utilities::{ }; use qovery_engine::models::DatabaseMode::{CONTAINER, MANAGED}; -use test_utilities::common::{database_test_environment, test_db, working_minimal_environment, Infrastructure}; +use test_utilities::common::{database_test_environment, test_db, Infrastructure}; use test_utilities::digitalocean::{ - clean_environments, do_default_engine_config, DO_KUBERNETES_VERSION, DO_MANAGED_DATABASE_DISK_TYPE, - DO_MANAGED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, - DO_TEST_REGION, + clean_environments, do_default_engine_config, DO_MANAGED_DATABASE_DISK_TYPE, DO_MANAGED_DATABASE_INSTANCE_TYPE, + DO_SELF_HOSTED_DATABASE_DISK_TYPE, DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, DO_TEST_REGION, }; /** @@ -437,7 +436,6 @@ fn private_postgresql_v10_deploy_a_working_dev_environment() { #[ignore] #[named] #[test] -#[ignore] fn public_postgresql_v10_deploy_a_working_dev_environment() { test_postgresql_configuration("10", function_name!(), CONTAINER, true); } @@ -454,7 +452,6 @@ fn private_postgresql_v11_deploy_a_working_dev_environment() { #[ignore] #[named] #[test] -#[ignore] fn public_postgresql_v11_deploy_a_working_dev_environment() { test_postgresql_configuration("11", function_name!(), CONTAINER, true); } diff --git a/tests/digitalocean/do_environment.rs b/tests/digitalocean/do_environment.rs index 5f0e3c71..8d965c39 100644 --- a/tests/digitalocean/do_environment.rs +++ b/tests/digitalocean/do_environment.rs @@ -6,16 +6,13 @@ use self::test_utilities::utilities::{ engine_run_test, generate_id, get_pods, get_pvc, init, is_pod_restarted_env, logger, FuncTestsSecrets, }; use ::function_name::named; -use qovery_engine::build_platform::{BuildPlatform, CacheResult}; use qovery_engine::cloud_provider::Kind; -use qovery_engine::container_registry::{ContainerRegistry, PullResult}; -use qovery_engine::models::{Action, Clone2, EnvironmentAction, Port, Protocol, Storage, StorageType}; +use qovery_engine::models::{Action, CloneForTest, EnvironmentAction, Port, Protocol, Storage, StorageType}; use qovery_engine::transaction::TransactionResult; use std::collections::BTreeMap; -use std::time::SystemTime; use test_utilities::common::Infrastructure; -use test_utilities::digitalocean::{container_registry_digital_ocean, do_default_engine_config, DO_KUBERNETES_VERSION}; -use test_utilities::utilities::{build_platform_local_docker, context}; +use test_utilities::digitalocean::do_default_engine_config; +use test_utilities::utilities::context; use tracing::{span, warn, Level}; // Note: All those tests relies on a test cluster running on DigitalOcean infrastructure. @@ -78,96 +75,6 @@ fn digitalocean_doks_deploy_a_working_environment_with_no_router() { }) } -#[cfg(feature = "test-do-self-hosted")] -#[named] -#[test] -fn test_build_cache() { - let test_name = function_name!(); - engine_run_test(|| { - init(); - let span = span!(Level::INFO, "test", name = test_name); - let _enter = span.enter(); - - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .DIGITAL_OCEAN_TEST_ORGANIZATION_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"), - secrets - .DIGITAL_OCEAN_TEST_CLUSTER_ID - .as_ref() - .expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"), - ); - let engine_config = do_default_engine_config(&context, logger()); - - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - - let docr = container_registry_digital_ocean(&context); - let local_docker = build_platform_local_docker(&context, logger()); - let app = environment.applications.first().unwrap(); - let image = app.to_image(); - - let app_build = app.to_build(); - let _ = match local_docker.has_cache(&app_build) { - Ok(CacheResult::Hit) => assert!(false), - Ok(CacheResult::Miss(_)) => assert!(true), - Ok(CacheResult::MissWithoutParentBuild) => assert!(false), - Err(_) => assert!(false), - }; - - let _ = match docr.pull(&image).unwrap() { - PullResult::Some(_) => assert!(false), - PullResult::None => assert!(true), - }; - - let cancel_task = || false; - let build_result = local_docker.build(app.to_build(), false, &cancel_task).unwrap(); - - let _ = match docr.push(&build_result.build.image, false) { - Ok(_) => assert!(true), - Err(_) => assert!(false), - }; - - // TODO clean local docker cache - - let start_pull_time = SystemTime::now(); - let _ = match docr.pull(&build_result.build.image).unwrap() { - PullResult::Some(_) => assert!(true), - PullResult::None => assert!(false), - }; - - let pull_duration = SystemTime::now().duration_since(start_pull_time).unwrap(); - - let _ = match local_docker.has_cache(&build_result.build) { - Ok(CacheResult::Hit) => assert!(true), - Ok(CacheResult::Miss(_)) => assert!(false), - Ok(CacheResult::MissWithoutParentBuild) => assert!(false), - Err(_) => assert!(false), - }; - - let start_pull_time = SystemTime::now(); - let _ = match docr.pull(&image).unwrap() { - PullResult::Some(_) => assert!(true), - PullResult::None => assert!(false), - }; - - let pull_duration_2 = SystemTime::now().duration_since(start_pull_time).unwrap(); - - if pull_duration_2.as_millis() > pull_duration.as_millis() { - assert!(false); - } - - return test_name.to_string(); - }) -} - #[cfg(feature = "test-do-self-hosted")] #[named] #[test] diff --git a/tests/digitalocean/do_kubernetes.rs b/tests/digitalocean/do_kubernetes.rs index 5a262127..b91fbcb7 100644 --- a/tests/digitalocean/do_kubernetes.rs +++ b/tests/digitalocean/do_kubernetes.rs @@ -2,9 +2,7 @@ extern crate test_utilities; use self::test_utilities::common::ClusterDomain; use self::test_utilities::digitalocean::{DO_KUBERNETES_MAJOR_VERSION, DO_KUBERNETES_MINOR_VERSION}; -use self::test_utilities::utilities::{ - context, engine_run_test, generate_cluster_id, generate_id, logger, FuncTestsSecrets, -}; +use self::test_utilities::utilities::{context, engine_run_test, generate_cluster_id, generate_id, logger}; use ::function_name::named; use qovery_engine::cloud_provider::digitalocean::application::DoRegion; use qovery_engine::cloud_provider::Kind; diff --git a/tests/digitalocean/do_utility_kubernetes_doks_test_cluster.rs b/tests/digitalocean/do_utility_kubernetes_doks_test_cluster.rs index 6f1a7dfd..c1aaf385 100644 --- a/tests/digitalocean/do_utility_kubernetes_doks_test_cluster.rs +++ b/tests/digitalocean/do_utility_kubernetes_doks_test_cluster.rs @@ -2,11 +2,9 @@ extern crate test_utilities; use self::test_utilities::utilities::{context, engine_run_test, init, logger, FuncTestsSecrets}; use ::function_name::named; -use qovery_engine::cloud_provider::digitalocean::DO; -use test_utilities::digitalocean::{do_default_engine_config, DO_KUBERNETES_VERSION, DO_TEST_REGION}; +use test_utilities::digitalocean::do_default_engine_config; use tracing::{span, Level}; -use self::test_utilities::common::Cluster; use qovery_engine::transaction::{Transaction, TransactionResult}; // Warning: This test shouldn't be ran by CI diff --git a/tests/docker/multi_stage_simple/Dockerfile.buildkit b/tests/docker/multi_stage_simple/Dockerfile.buildkit new file mode 100644 index 00000000..5219fdf1 --- /dev/null +++ b/tests/docker/multi_stage_simple/Dockerfile.buildkit @@ -0,0 +1,10 @@ +FROM golang:1.16 AS build + +# ../ is not valid if using old docker engine, only allowed with buildkit +COPY ../hello.go /go/src/project/hello.go +WORKDIR /go/src/project +RUN go build hello.go + +FROM scratch +COPY --from=build /go/src/project/hello /bin/hello +ENTRYPOINT ["/bin/hello"] diff --git a/tests/docker/multi_stage_simple/hello.go b/tests/docker/multi_stage_simple/hello.go new file mode 100644 index 00000000..a932edea --- /dev/null +++ b/tests/docker/multi_stage_simple/hello.go @@ -0,0 +1,7 @@ +package main + +import "fmt" + +func main() { + fmt.Println("hello world") +} diff --git a/tests/scaleway/scw_container_registry.rs b/tests/scaleway/scw_container_registry.rs index fce2203f..ce1f918a 100644 --- a/tests/scaleway/scw_container_registry.rs +++ b/tests/scaleway/scw_container_registry.rs @@ -1,7 +1,6 @@ extern crate test_utilities; use self::test_utilities::utilities::{context, FuncTestsSecrets}; -use qovery_engine::build_platform::Image; use qovery_engine::cloud_provider::scaleway::application::ScwZone; use qovery_engine::container_registry::scaleway_container_registry::ScalewayCR; use test_utilities::utilities::logger; @@ -49,17 +48,7 @@ fn test_get_registry_namespace() { logger(), ); - let image = Image { - application_id: "1234".to_string(), - name: registry_name.to_string(), - tag: "tag123".to_string(), - commit_id: "commit_id".to_string(), - registry_name: Some(registry_name.to_string()), - registry_secret: None, - registry_url: None, - registry_docker_json_config: None, - }; - + let image = registry_name.to_string(); container_registry .create_registry_namespace(&image) .expect("error while creating registry namespace"); @@ -108,16 +97,7 @@ fn test_create_registry_namespace() { logger(), ); - let image = Image { - application_id: "1234".to_string(), - name: registry_name.to_string(), - tag: "tag123".to_string(), - commit_id: "commit_id".to_string(), - registry_name: Some(registry_name.to_string()), - registry_secret: None, - registry_url: None, - registry_docker_json_config: None, - }; + let image = registry_name.to_string(); // execute: debug!("test_create_registry_namespace - {}", region); @@ -160,17 +140,7 @@ fn test_delete_registry_namespace() { logger(), ); - let image = Image { - application_id: "1234".to_string(), - name: registry_name.to_string(), - tag: "tag123".to_string(), - commit_id: "commit_id".to_string(), - registry_name: Some(registry_name.to_string()), - registry_secret: None, - registry_url: None, - registry_docker_json_config: None, - }; - + let image = registry_name.to_string(); container_registry .create_registry_namespace(&image) .expect("error while creating registry namespace"); @@ -207,17 +177,7 @@ fn test_get_or_create_registry_namespace() { logger(), ); - let image = Image { - application_id: "1234".to_string(), - name: registry_name.to_string(), - tag: "tag123".to_string(), - commit_id: "commit_id".to_string(), - registry_name: Some(registry_name.to_string()), - registry_secret: None, - registry_url: None, - registry_docker_json_config: None, - }; - + let image = registry_name.to_string(); container_registry .create_registry_namespace(&image) .expect("error while creating registry namespace"); diff --git a/tests/scaleway/scw_databases.rs b/tests/scaleway/scw_databases.rs index acb7c597..e2f1ef13 100644 --- a/tests/scaleway/scw_databases.rs +++ b/tests/scaleway/scw_databases.rs @@ -3,7 +3,7 @@ use tracing::{span, warn, Level}; use qovery_engine::cloud_provider::{Kind as ProviderKind, Kind}; use qovery_engine::models::{ - Action, Clone2, Context, Database, DatabaseKind, DatabaseMode, Environment, EnvironmentAction, Port, Protocol, + Action, CloneForTest, Database, DatabaseKind, DatabaseMode, EnvironmentAction, Port, Protocol, }; use qovery_engine::transaction::TransactionResult; use test_utilities::utilities::{ @@ -12,12 +12,11 @@ use test_utilities::utilities::{ }; use qovery_engine::models::DatabaseMode::{CONTAINER, MANAGED}; +use test_utilities::common::test_db; use test_utilities::common::{database_test_environment, Infrastructure}; -use test_utilities::common::{test_db, working_minimal_environment}; use test_utilities::scaleway::{ - clean_environments, scw_default_engine_config, SCW_KUBERNETES_VERSION, SCW_MANAGED_DATABASE_DISK_TYPE, - SCW_MANAGED_DATABASE_INSTANCE_TYPE, SCW_SELF_HOSTED_DATABASE_DISK_TYPE, SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, - SCW_TEST_ZONE, + clean_environments, scw_default_engine_config, SCW_MANAGED_DATABASE_DISK_TYPE, SCW_MANAGED_DATABASE_INSTANCE_TYPE, + SCW_SELF_HOSTED_DATABASE_DISK_TYPE, SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, SCW_TEST_ZONE, }; /** diff --git a/tests/scaleway/scw_environment.rs b/tests/scaleway/scw_environment.rs index 7b329967..979bf888 100644 --- a/tests/scaleway/scw_environment.rs +++ b/tests/scaleway/scw_environment.rs @@ -6,16 +6,12 @@ use self::test_utilities::utilities::{ context, engine_run_test, generate_id, get_pods, get_pvc, init, is_pod_restarted_env, logger, FuncTestsSecrets, }; use ::function_name::named; -use qovery_engine::build_platform::{BuildPlatform, CacheResult}; use qovery_engine::cloud_provider::Kind; -use qovery_engine::container_registry::{ContainerRegistry, PullResult}; -use qovery_engine::models::{Action, Clone2, EnvironmentAction, Port, Protocol, Storage, StorageType}; +use qovery_engine::models::{Action, CloneForTest, EnvironmentAction, Port, Protocol, Storage, StorageType}; use qovery_engine::transaction::TransactionResult; use std::collections::BTreeMap; -use std::time::SystemTime; use test_utilities::common::Infrastructure; -use test_utilities::scaleway::{container_registry_scw, scw_default_engine_config, SCW_KUBERNETES_VERSION}; -use test_utilities::utilities::build_platform_local_docker; +use test_utilities::scaleway::scw_default_engine_config; use tracing::{span, warn, Level}; // Note: All those tests relies on a test cluster running on Scaleway infrastructure. @@ -81,97 +77,6 @@ fn scaleway_kapsule_deploy_a_working_environment_with_no_router() { }) } -#[cfg(feature = "test-scw-self-hosted")] -#[named] -#[test] -fn test_build_cache() { - let test_name = function_name!(); - engine_run_test(|| { - init(); - let span = span!(Level::INFO, "test", name = test_name); - let _enter = span.enter(); - - let secrets = FuncTestsSecrets::new(); - let context = context( - secrets - .SCALEWAY_TEST_ORGANIZATION_ID - .as_ref() - .expect("SCALEWAY_TEST_ORGANIZATION_ID") - .as_str(), - secrets - .SCALEWAY_TEST_CLUSTER_ID - .as_ref() - .expect("SCALEWAY_TEST_CLUSTER_ID") - .as_str(), - ); - - let environment = test_utilities::common::working_minimal_environment( - &context, - secrets - .DEFAULT_TEST_DOMAIN - .expect("DEFAULT_TEST_DOMAIN is not set in secrets") - .as_str(), - ); - - let scr = container_registry_scw(&context); - let local_docker = build_platform_local_docker(&context, logger()); - let app = environment.applications.first().unwrap(); - let image = app.to_image(); - - let app_build = app.to_build(); - let _ = match local_docker.has_cache(&app_build) { - Ok(CacheResult::Hit) => assert!(false), - Ok(CacheResult::Miss(_)) => assert!(true), - Ok(CacheResult::MissWithoutParentBuild) => assert!(false), - Err(_) => assert!(false), - }; - - let _ = match scr.pull(&image).unwrap() { - PullResult::Some(_) => assert!(false), - PullResult::None => assert!(true), - }; - - let cancel_task = || false; - let build_result = local_docker.build(app.to_build(), false, &cancel_task).unwrap(); - - let _ = match scr.push(&build_result.build.image, false) { - Ok(_) => assert!(true), - Err(_) => assert!(false), - }; - - // TODO clean local docker cache - - let start_pull_time = SystemTime::now(); - let _ = match scr.pull(&build_result.build.image).unwrap() { - PullResult::Some(_) => assert!(true), - PullResult::None => assert!(false), - }; - - let pull_duration = SystemTime::now().duration_since(start_pull_time).unwrap(); - - let _ = match local_docker.has_cache(&build_result.build) { - Ok(CacheResult::Hit) => assert!(true), - Ok(CacheResult::Miss(_)) => assert!(false), - Ok(CacheResult::MissWithoutParentBuild) => assert!(false), - Err(_) => assert!(false), - }; - - let start_pull_time = SystemTime::now(); - let _ = match scr.pull(&image).unwrap() { - PullResult::Some(_) => assert!(true), - PullResult::None => assert!(false), - }; - - let pull_duration_2 = SystemTime::now().duration_since(start_pull_time).unwrap(); - - if pull_duration_2.as_millis() > pull_duration.as_millis() { - assert!(false); - } - - return test_name.to_string(); - }) -} - #[cfg(feature = "test-scw-self-hosted")] #[named] #[test] diff --git a/tests/scaleway/scw_kubernetes.rs b/tests/scaleway/scw_kubernetes.rs index 58e5c9a0..0e6075be 100644 --- a/tests/scaleway/scw_kubernetes.rs +++ b/tests/scaleway/scw_kubernetes.rs @@ -1,9 +1,7 @@ extern crate test_utilities; use self::test_utilities::scaleway::{SCW_KUBERNETES_MAJOR_VERSION, SCW_KUBERNETES_MINOR_VERSION}; -use self::test_utilities::utilities::{ - context, engine_run_test, generate_cluster_id, generate_id, logger, FuncTestsSecrets, -}; +use self::test_utilities::utilities::{context, engine_run_test, generate_cluster_id, generate_id, logger}; use ::function_name::named; use qovery_engine::cloud_provider::aws::kubernetes::VpcQoveryNetworkMode; use qovery_engine::cloud_provider::scaleway::application::ScwZone; diff --git a/tests/scaleway/scw_utility_kubernetes_kapsule_test_cluster.rs b/tests/scaleway/scw_utility_kubernetes_kapsule_test_cluster.rs index 19b8753e..70816f19 100644 --- a/tests/scaleway/scw_utility_kubernetes_kapsule_test_cluster.rs +++ b/tests/scaleway/scw_utility_kubernetes_kapsule_test_cluster.rs @@ -2,11 +2,9 @@ extern crate test_utilities; use self::test_utilities::utilities::{context, engine_run_test, init, logger, FuncTestsSecrets}; use ::function_name::named; -use test_utilities::scaleway::{scw_default_engine_config, SCW_KUBERNETES_VERSION, SCW_TEST_ZONE}; +use test_utilities::scaleway::scw_default_engine_config; use tracing::{span, Level}; -use self::test_utilities::common::Cluster; -use qovery_engine::cloud_provider::scaleway::Scaleway; use qovery_engine::transaction::{Transaction, TransactionResult}; // Warning: This test shouldn't be ran by CI