jlengrand/engine
1
0
Fork 0
mirror of https://github.com/jlengrand/engine.git synced 2026-03-10 08:11:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

build_refacto (#643)

Browse Source 
* build_refacto

* build_refacto
This commit is contained in:
Erèbe - Romain Gerard
2022-03-17 10:44:56 +01:00
committed by GitHub
parent 490751032a
commit e0a1dff4b2
41 changed files with 1113 additions and 2756 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
Cargo.lock generated
View File

@@ -2119,6 +2119,7 @@ dependencies = [
"tracing-test",
"trust-dns-resolver",
"url 2.2.2",
"urlencoding",
"uuid 0.8.2",
"walkdir",
]
@@ -3283,6 +3284,7 @@ dependencies = [
"time 0.2.27",
"tracing",
"tracing-subscriber",
"url 2.2.2",
"uuid 0.8.2",
]
@@ -3939,6 +3941,12 @@ dependencies = [
"url 1.7.2",
]
[[package]]
name = "urlencoding"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68b90931029ab9b034b300b797048cf23723400aa757e8a2bfb9d748102f9821"
[[package]]
name = "uuid"
version = "0.7.4"

1
Cargo.toml
View File

@@ -30,6 +30,7 @@ function_name = "0.2.0"
thiserror = "1.0.30"
strum = "0.23"
strum_macros = "0.23"
urlencoding = "2.1.0"
# FIXME use https://crates.io/crates/blocking instead of runtime.rs

243
src/build_platform/local_docker.rs
View File

@@ -6,10 +6,11 @@ use chrono::Duration;
use git2::{Cred, CredentialType};
use sysinfo::{Disk, DiskExt, SystemExt};
use crate::build_platform::{docker, Build, BuildPlatform, BuildResult, CacheResult, Credentials, Image, Kind};
use crate::build_platform::{docker, Build, BuildPlatform, BuildResult, Credentials, Kind};
use crate::cmd::command;
use crate::cmd::command::CommandError::Killed;
use crate::cmd::command::QoveryCommand;
use crate::cmd::docker::{ContainerImage, Docker, DockerError};
use crate::errors::{CommandError, EngineError, Tag};
use crate::events::{EngineEvent, EventDetails, EventMessage, ToTransmitter, Transmitter};
use crate::fs::workspace_directory;
@@ -32,6 +33,7 @@ const BUILDPACKS_BUILDERS: [&str; 1] = [
/// use Docker in local
pub struct LocalDocker {
context: Context,
docker: Docker,
id: String,
name: String,
listeners: Listeners,
@@ -39,31 +41,25 @@ pub struct LocalDocker {
}
impl LocalDocker {
pub fn new(context: Context, id: &str, name: &str, logger: Box<dyn Logger>) -> Self {
LocalDocker {
pub fn new(
context: Context,
id: &str,
name: &str,
logger: Box<dyn Logger>,
) -> Result<Self, Box<dyn std::error::Error>> {
let docker = Docker::new_with_options(true, context.docker_tcp_socket().clone())?;
Ok(LocalDocker {
context,
docker,
id: id.to_string(),
name: name.to_string(),
listeners: vec![],
logger,
}
}
fn image_does_exist(&self, image: &Image) -> Result<bool, EngineError> {
let mut cmd = QoveryCommand::new(
"docker",
&vec!["image", "inspect", image.name_with_tag().as_str()],
&self.get_docker_host_envs(),
);
Ok(matches!(cmd.exec(), Ok(_)))
})
}
fn get_docker_host_envs(&self) -> Vec<(&str, &str)> {
match self.context.docker_tcp_socket() {
Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())],
None => vec![],
}
vec![]
}
/// Read Dockerfile content from location path and return an array of bytes
@@ -89,34 +85,20 @@ impl LocalDocker {
dockerfile_complete_path: &str,
into_dir_docker_style: &str,
env_var_args: Vec<String>,
use_build_cache: bool,
lh: &ListenersHelper,
is_task_canceled: &dyn Fn() -> bool,
) -> Result<BuildResult, EngineError> {
let mut docker_args = if !use_build_cache {
vec!["build", "--no-cache"]
} else {
vec!["build"]
let image_to_build = ContainerImage {
registry: build.image.registry_url.clone(),
name: build.image.name(),
tags: vec![build.image.tag.clone(), "latest".to_string()],
};
let args = self.context.docker_build_options();
for v in args.iter() {
for s in v.iter() {
docker_args.push(String::as_str(s));
}
}
let name_with_tag = build.image.name_with_tag();
let name_with_latest_tag = build.image.name_with_latest_tag();
docker_args.extend(vec![
"-f",
dockerfile_complete_path,
"-t",
name_with_tag.as_str(),
"-t",
name_with_latest_tag.as_str(),
]);
let image_cache = ContainerImage {
registry: build.image.registry_url.clone(),
name: build.image.name(),
tags: vec!["latest".to_string()],
};
let dockerfile_content = self.get_dockerfile_content(dockerfile_complete_path)?;
let env_var_args = match docker::match_used_env_var_args(env_var_args, dockerfile_content) {
@@ -133,27 +115,25 @@ impl LocalDocker {
}
};
let mut docker_args = if env_var_args.is_empty() {
docker_args
} else {
let mut build_args = vec![];
// FIXME: pass a Vec<(key, value)> instead of spliting always the string
let env_vars = env_var_args
.into_iter()
.map(|val| {
let (key, value) = val.rsplit_once('=').unwrap();
(key.to_string(), value.to_string())
})
.collect::<Vec<_>>();
env_var_args.iter().for_each(|arg_value| {
build_args.push("--build-arg");
build_args.push(arg_value.as_str());
});
docker_args.extend(build_args);
docker_args
};
docker_args.push(into_dir_docker_style);
// docker build
let mut cmd = QoveryCommand::new("docker", &docker_args, &self.get_docker_host_envs());
let exit_status = cmd.exec_with_abort(
Duration::minutes(BUILD_DURATION_TIMEOUT_MIN),
let exit_status = self.docker.build(
&Path::new(dockerfile_complete_path),
&Path::new(into_dir_docker_style),
&image_to_build,
&env_vars
.iter()
.map(|(k, v)| (k.as_str(), v.as_str()))
.collect::<Vec<_>>(),
&image_cache,
true,
|line| {
self.logger.log(
LogLevel::Info,
@@ -171,25 +151,26 @@ impl LocalDocker {
},
|line| {
self.logger.log(
LogLevel::Warning,
EngineEvent::Warning(self.get_event_details(), EventMessage::new_from_safe(line.to_string())),
LogLevel::Info,
EngineEvent::Info(self.get_event_details(), EventMessage::new_from_safe(line.to_string())),
);
lh.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: build.image.application_id.clone(),
},
ProgressLevel::Warn,
ProgressLevel::Info,
Some(line),
self.context.execution_id(),
));
},
Duration::minutes(BUILD_DURATION_TIMEOUT_MIN),
is_task_canceled,
);
match exit_status {
Ok(_) => Ok(BuildResult { build }),
Err(Killed(_)) => Err(EngineError::new_task_cancellation_requested(self.get_event_details())),
Err(DockerError::Aborted(_)) => Err(EngineError::new_task_cancellation_requested(self.get_event_details())),
Err(err) => Err(EngineError::new_docker_cannot_build_container_image(
self.get_event_details(),
self.name_with_id(),
@@ -207,10 +188,8 @@ impl LocalDocker {
lh: &ListenersHelper,
is_task_canceled: &dyn Fn() -> bool,
) -> Result<BuildResult, EngineError> {
let name_with_tag = build.image.name_with_tag();
let name_with_latest_tag = build.image.name_with_latest_tag();
let args = self.context.docker_build_options();
let name_with_tag = build.image.full_image_name_with_tag();
let name_with_latest_tag = format!("{}:latest", build.image.full_image_name());
let mut exit_status: Result<(), command::CommandError> = Err(command::CommandError::ExecutionError(
Error::new(ErrorKind::InvalidData, "No builder names".to_string()),
@@ -218,20 +197,13 @@ impl LocalDocker {
for builder_name in BUILDPACKS_BUILDERS.iter() {
let mut buildpacks_args = if !use_build_cache {
vec!["build", name_with_tag.as_str(), "--clear-cache"]
vec!["build", "--publish", name_with_tag.as_str(), "--clear-cache"]
} else {
vec!["build", name_with_tag.as_str()]
vec!["build", "--publish", name_with_tag.as_str()]
};
// always add 'latest' tag
buildpacks_args.extend(vec!["-t", name_with_latest_tag.as_str()]);
for v in args.iter() {
for s in v.iter() {
buildpacks_args.push(String::as_str(s));
}
}
buildpacks_args.extend(vec!["--path", into_dir_docker_style]);
let mut buildpacks_args = if env_var_args.is_empty() {
@@ -414,69 +386,7 @@ impl BuildPlatform for LocalDocker {
Ok(())
}
fn has_cache(&self, build: &Build) -> Result<CacheResult, EngineError> {
let event_details = self.get_event_details();
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe("LocalDocker.has_cache() called".to_string()),
),
);
// Check if a local cache layers for the container image exists.
let repository_root_path = self.get_repository_build_root_path(&build)?;
let parent_build = build.to_previous_build(repository_root_path).map_err(|err| {
EngineError::new_builder_get_build_error(self.get_event_details(), build.image.commit_id.to_string(), err)
})?;
let parent_build = match parent_build {
Some(parent_build) => parent_build,
None => return Ok(CacheResult::MissWithoutParentBuild),
};
// check if local layers exist
let cmd_bin = "docker";
let image_name = parent_build.image.name.clone();
let cmd_args = vec!["images", "-q", &image_name];
let mut cmd = QoveryCommand::new(cmd_bin, &cmd_args.clone(), &[]);
let mut result = CacheResult::Miss(parent_build);
let _ = cmd.exec_with_timeout(
Duration::minutes(1), // `docker images` command can be slow with tons of images - it's probably not indexed
|_| result = CacheResult::Hit, // if a line is returned, then the image is locally present
|r_err| {
self.logger.log(
LogLevel::Error,
EngineEvent::Error(
EngineError::new_docker_cannot_list_images(
event_details.clone(),
CommandError::new_from_command_line(
"Cannot list docker images".to_string(),
cmd_bin.to_string(),
cmd_args.clone().into_iter().map(|v| v.to_string()).collect(),
vec![],
None,
Some(r_err.to_string()),
),
),
None,
),
)
},
);
Ok(result)
}
fn build(
&self,
build: Build,
force_build: bool,
is_task_canceled: &dyn Fn() -> bool,
) -> Result<BuildResult, EngineError> {
fn build(&self, build: Build, is_task_canceled: &dyn Fn() -> bool) -> Result<BuildResult, EngineError> {
let event_details = self.get_event_details();
self.logger.log(
@@ -492,22 +402,6 @@ impl BuildPlatform for LocalDocker {
}
let listeners_helper = ListenersHelper::new(&self.listeners);
if !force_build && self.image_does_exist(&build.image)? {
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(format!(
"Image `{}` found on repository, container build is not required",
build.image.name_with_tag()
)),
),
);
return Ok(BuildResult { build });
}
let repository_root_path = self.get_repository_build_root_path(&build)?;
self.logger.log(
@@ -551,6 +445,7 @@ impl BuildPlatform for LocalDocker {
if is_task_canceled() {
return Err(EngineError::new_task_cancellation_requested(event_details.clone()));
}
if let Err(clone_error) = git::clone_at_commit(
&build.git_repository.url,
&build.git_repository.commit_id,
@@ -669,7 +564,6 @@ impl BuildPlatform for LocalDocker {
dockerfile_absolute_path.as_str(),
build_context_path.as_str(),
env_var_args,
!disable_build_cache,
&listeners_helper,
is_task_canceled,
)
@@ -714,38 +608,6 @@ impl BuildPlatform for LocalDocker {
result
}
fn build_error(&self, build: Build) -> Result<BuildResult, EngineError> {
let event_details = self.get_event_details();
self.logger.log(
LogLevel::Warning,
EngineEvent::Warning(
event_details.clone(),
EventMessage::new_from_safe(format!("LocalDocker.build_error() called for {}", self.name())),
),
);
let listener_helper = ListenersHelper::new(&self.listeners);
// FIXME
let message = String::from("something goes wrong (not implemented)");
listener_helper.error(ProgressInfo::new(
ProgressScope::Application {
id: build.image.application_id,
},
ProgressLevel::Error,
Some(message.as_str()),
self.context.execution_id(),
));
let err = EngineError::new_not_implemented_error(event_details);
self.logger.log(LogLevel::Error, EngineEvent::Error(err.clone(), None));
// FIXME
Err(err)
}
fn logger(&self) -> Box<dyn Logger> {
self.logger.clone()
}
@@ -814,6 +676,7 @@ fn docker_prune_images(envs: Vec<(&str, &str)>) -> Result<(), CommandError> {
vec!["image", "prune", "-a", "-f"],
vec!["builder", "prune", "-a", "-f"],
vec!["volume", "prune", "-f"],
vec!["buildx", "prune", "-a", "-f"],
];
let mut errored_commands = vec![];

114
src/build_platform/mod.rs
View File

@@ -1,15 +1,11 @@
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use crate::errors::{CommandError, EngineError};
use crate::errors::EngineError;
use crate::events::{EnvironmentStep, EventDetails, Stage, ToTransmitter};
use crate::git;
use crate::logger::Logger;
use crate::models::{Context, Listen, QoveryIdentifier};
use crate::utilities::get_image_tag;
use git2::{Cred, CredentialType};
use std::fmt::{Display, Formatter, Result as FmtResult};
use std::path::Path;
use url::Url;
pub mod docker;
pub mod local_docker;
@@ -23,14 +19,7 @@ pub trait BuildPlatform: ToTransmitter + Listen {
format!("{} ({})", self.name(), self.id())
}
fn is_valid(&self) -> Result<(), EngineError>;
fn has_cache(&self, build: &Build) -> Result<CacheResult, EngineError>;
fn build(
&self,
build: Build,
force_build: bool,
is_task_canceled: &dyn Fn() -> bool,
) -> Result<BuildResult, EngineError>;
fn build_error(&self, build: Build) -> Result<BuildResult, EngineError>;
fn build(&self, build: Build, is_task_canceled: &dyn Fn() -> bool) -> Result<BuildResult, EngineError>;
fn logger(&self) -> Box<dyn Logger>;
fn get_event_details(&self) -> EventDetails {
let context = self.context();
@@ -52,63 +41,6 @@ pub struct Build {
pub options: BuildOptions,
}
impl Build {
pub fn to_previous_build<P>(&self, clone_repo_into_dir: P) -> Result<Option<Build>, CommandError>
where
P: AsRef<Path>,
{
let parent_commit_id = git::get_parent_commit_id(
self.git_repository.url.as_str(),
self.git_repository.commit_id.as_str(),
clone_repo_into_dir,
&|_| match &self.git_repository.credentials {
None => vec![],
Some(creds) => vec![(
CredentialType::USER_PASS_PLAINTEXT,
Cred::userpass_plaintext(creds.login.as_str(), creds.password.as_str()).unwrap(),
)],
},
)
.map_err(|err| CommandError::new(err.to_string(), Some("Cannot get parent commit ID.".to_string())))?;
let parent_commit_id = match parent_commit_id {
None => return Ok(None),
Some(parent_commit_id) => parent_commit_id,
};
let mut environment_variables_map = BTreeMap::<String, String>::new();
for env in &self.options.environment_variables {
environment_variables_map.insert(env.key.clone(), env.value.clone());
}
let mut image = self.image.clone();
image.tag = get_image_tag(
&self.git_repository.root_path,
&self.git_repository.dockerfile_path,
&environment_variables_map,
&parent_commit_id,
);
image.commit_id = parent_commit_id.clone();
Ok(Some(Build {
git_repository: GitRepository {
url: self.git_repository.url.clone(),
credentials: self.git_repository.credentials.clone(),
ssh_keys: self.git_repository.ssh_keys.clone(),
commit_id: parent_commit_id,
dockerfile_path: self.git_repository.dockerfile_path.clone(),
root_path: self.git_repository.root_path.clone(),
buildpack_language: self.git_repository.buildpack_language.clone(),
},
image,
options: BuildOptions {
environment_variables: self.options.environment_variables.clone(),
},
}))
}
}
pub struct BuildOptions {
pub environment_variables: Vec<EnvironmentVariable>,
}
@@ -149,22 +81,33 @@ pub struct Image {
pub tag: String,
pub commit_id: String,
// registry name where the image has been pushed: Optional
pub registry_name: Option<String>,
pub registry_name: String,
// registry docker json config: Optional
pub registry_docker_json_config: Option<String>,
// registry secret to pull image: Optional
pub registry_secret: Option<String>,
// complete registry URL where the image has been pushed
pub registry_url: Option<String>,
pub registry_url: Url,
}
impl Image {
pub fn name_with_tag(&self) -> String {
format!("{}:{}", self.name, self.tag)
pub fn registry_host(&self) -> &str {
self.registry_url.host_str().unwrap()
}
pub fn name_with_latest_tag(&self) -> String {
format!("{}:latest", self.name)
pub fn full_image_name_with_tag(&self) -> String {
format!(
"{}/{}:{}",
self.registry_url.host_str().unwrap_or_default(),
self.name,
self.tag
)
}
pub fn full_image_name(&self) -> String {
format!("{}/{}", self.registry_url.host_str().unwrap_or_default(), self.name,)
}
pub fn name(&self) -> String {
self.name.clone()
}
}
@@ -175,10 +118,9 @@ impl Default for Image {
name: "".to_string(),
tag: "".to_string(),
commit_id: "".to_string(),
registry_name: None,
registry_name: "".to_string(),
registry_docker_json_config: None,
registry_secret: None,
registry_url: None,
registry_url: Url::parse("https://default.com").unwrap(),
}
}
}
@@ -208,11 +150,3 @@ impl BuildResult {
pub enum Kind {
LocalDocker,
}
type ParentBuild = Build;
pub enum CacheResult {
MissWithoutParentBuild,
Miss(ParentBuild),
Hit,
}

37
src/cloud_provider/aws/application.rs
View File

@@ -15,8 +15,8 @@ use crate::cloud_provider::DeploymentTarget;
use crate::cmd::helm::Timeout;
use crate::cmd::kubectl::ScalingKind::{Deployment, Statefulset};
use crate::errors::EngineError;
use crate::events::{EngineEvent, EnvironmentStep, EventMessage, Stage, ToTransmitter, Transmitter};
use crate::logger::{LogLevel, Logger};
use crate::events::{EnvironmentStep, Stage, ToTransmitter, Transmitter};
use crate::logger::Logger;
use crate::models::{Context, Listen, Listener, Listeners, ListenersHelper, Port};
use ::function_name::named;
@@ -201,26 +201,7 @@ impl Service for Application {
let commit_id = self.image().commit_id.as_str();
context.insert("helm_app_version", &commit_id[..7]);
match &self.image().registry_url {
Some(registry_url) => context.insert("image_name_with_tag", registry_url.as_str()),
None => {
let image_name_with_tag = self.image().name_with_tag();
self.logger().log(
LogLevel::Warning,
EngineEvent::Warning(
event_details.clone(),
EventMessage::new_from_safe(format!(
"there is no registry url, use image name with tag with the default container registry: {}",
image_name_with_tag.as_str()
)),
),
);
context.insert("image_name_with_tag", image_name_with_tag.as_str());
}
}
context.insert("image_name_with_tag", &self.image.full_image_name_with_tag());
let environment_variables = self
.environment_variables
@@ -233,16 +214,8 @@ impl Service for Application {
context.insert("environment_variables", &environment_variables);
context.insert("ports", &self.ports);
match self.image.registry_name.as_ref() {
Some(registry_name) => {
context.insert("is_registry_secret", &true);
context.insert("registry_secret", registry_name);
}
None => {
context.insert("is_registry_secret", &false);
}
};
context.insert("is_registry_secret", &true);
context.insert("registry_secret", self.image.registry_host());
let cpu_limits = match validate_k8s_required_cpu_and_burstable(
&ListenersHelper::new(&self.listeners),

37
src/cloud_provider/digitalocean/application.rs
View File

@@ -15,8 +15,8 @@ use crate::cloud_provider::DeploymentTarget;
use crate::cmd::helm::Timeout;
use crate::cmd::kubectl::ScalingKind::{Deployment, Statefulset};
use crate::errors::{CommandError, EngineError};
use crate::events::{EngineEvent, EnvironmentStep, EventMessage, Stage, ToTransmitter, Transmitter};
use crate::logger::{LogLevel, Logger};
use crate::events::{EnvironmentStep, Stage, ToTransmitter, Transmitter};
use crate::logger::Logger;
use crate::models::{Context, Listen, Listener, Listeners, ListenersHelper, Port};
use ::function_name::named;
use std::fmt;
@@ -205,26 +205,7 @@ impl Service for Application {
let commit_id = self.image.commit_id.as_str();
context.insert("helm_app_version", &commit_id[..7]);
match &self.image.registry_url {
Some(registry_url) => context.insert("image_name_with_tag", registry_url.as_str()),
None => {
let image_name_with_tag = self.image.name_with_tag();
self.logger().log(
LogLevel::Warning,
EngineEvent::Warning(
event_details.clone(),
EventMessage::new_from_safe(format!(
"there is no registry url, use image name with tag with the default container registry: {}",
image_name_with_tag.as_str()
)),
),
);
context.insert("image_name_with_tag", image_name_with_tag.as_str());
}
}
context.insert("image_name_with_tag", &self.image.full_image_name_with_tag());
let cpu_limits = match validate_k8s_required_cpu_and_burstable(
&ListenersHelper::new(&self.listeners),
@@ -258,16 +239,8 @@ impl Service for Application {
context.insert("environment_variables", &environment_variables);
context.insert("ports", &self.ports);
if self.image.registry_name.is_some() {
context.insert("is_registry_secret", &true);
context.insert(
"registry_secret",
&"do-container-registry-secret-for-cluster".to_string(),
);
} else {
context.insert("is_registry_secret", &false);
};
context.insert("is_registry_secret", &true);
context.insert("registry_secret", self.image.registry_host());
let storage = self
.storage

38
src/cloud_provider/scaleway/application.rs
View File

@@ -18,8 +18,8 @@ use crate::cloud_provider::DeploymentTarget;
use crate::cmd::helm::Timeout;
use crate::cmd::kubectl::ScalingKind::{Deployment, Statefulset};
use crate::errors::{CommandError, EngineError};
use crate::events::{EngineEvent, EnvironmentStep, EventMessage, Stage, ToTransmitter, Transmitter};
use crate::logger::{LogLevel, Logger};
use crate::events::{EnvironmentStep, Stage, ToTransmitter, Transmitter};
use crate::logger::Logger;
use crate::models::{Context, Listen, Listener, Listeners, ListenersHelper, Port};
use ::function_name::named;
@@ -206,27 +206,7 @@ impl Service for Application {
let commit_id = self.image().commit_id.as_str();
context.insert("helm_app_version", &commit_id[..7]);
match &self.image().registry_url {
Some(registry_url) => context.insert(
"image_name_with_tag",
format!("{}/{}", registry_url.as_str(), self.image().name_with_tag()).as_str(),
),
None => {
let image_name_with_tag = self.image().name_with_tag();
self.logger().log(
LogLevel::Warning,
EngineEvent::Warning(
event_details.clone(),
EventMessage::new_from_safe(format!(
"there is no registry url, use image name with tag with the default container registry: {}",
image_name_with_tag.as_str()
)),
),
);
context.insert("image_name_with_tag", image_name_with_tag.as_str());
}
}
context.insert("image_name_with_tag", &self.image.full_image_name_with_tag());
let environment_variables = self
.environment_variables
@@ -239,16 +219,8 @@ impl Service for Application {
context.insert("environment_variables", &environment_variables);
context.insert("ports", &self.ports);
match self.image.registry_name.as_ref() {
Some(_) => {
context.insert("is_registry_secret", &true);
context.insert("registry_secret_name", &format!("registry-token-{}", &self.id));
}
None => {
context.insert("is_registry_secret", &false);
}
};
context.insert("is_registry_secret", &true);
context.insert("registry_secret_name", &format!("registry-token-{}", &self.id));
let cpu_limits = match validate_k8s_required_cpu_and_burstable(
&ListenersHelper::new(&self.listeners),

682
src/cmd/docker.rs Normal file
View File

@@ -0,0 +1,682 @@
use crate::cmd::command::{CommandError, QoveryCommand};
use crate::errors::EngineError;
use crate::events::EventDetails;
use chrono::Duration;
use std::path::Path;
use std::process::ExitStatus;
use url::Url;
#[derive(thiserror::Error, Debug)]
pub enum DockerError {
#[error("Docker Invalid configuration: {0}")]
InvalidConfig(String),
#[error("Docker terminated with an unknown error: {0}")]
ExecutionError(#[from] std::io::Error),
#[error("Docker terminated with a non success exit status code: {0}")]
ExitStatusError(ExitStatus),
#[error("Docker aborted due to user cancel request: {0}")]
Aborted(String),
#[error("Docker command terminated due to timeout: {0}")]
Timeout(String),
}
#[derive(Debug)]
pub struct ContainerImage {
pub registry: Url,
pub name: String,
pub tags: Vec<String>,
}
impl ContainerImage {
pub fn image_names(&self) -> Vec<String> {
let host = if let Some(port) = self.registry.port() {
format!("{}:{}", self.registry.host_str().unwrap_or_default(), port)
} else {
self.registry.host_str().unwrap_or_default().to_string()
};
self.tags
.iter()
.map(|tag| format!("{}/{}:{}", host, &self.name, tag))
.collect()
}
pub fn image_name(&self) -> String {
self.image_names().remove(0)
}
}
pub struct Docker {
use_buildkit: bool,
common_envs: Vec<(String, String)>,
}
impl Docker {
pub fn new_with_options(enable_buildkit: bool, socket_location: Option<Url>) -> Result<Self, DockerError> {
let mut docker = Docker {
use_buildkit: enable_buildkit,
common_envs: vec![(
"DOCKER_BUILDKIT".to_string(),
if enable_buildkit {
"1".to_string()
} else {
"0".to_string()
},
)],
};
// Override DOCKER_HOST if we use a TCP socket
if let Some(socket_location) = socket_location {
docker
.common_envs
.push(("DOCKER_HOST".to_string(), socket_location.to_string()))
}
// If we don't use buildkit nothing more to do
if !docker.use_buildkit {
return Ok(docker);
}
// First check that the buildx plugin is correctly installed
let args = vec!["buildx", "version"];
let buildx_cmd_exist = docker_exec(
&args,
&docker.get_all_envs(&vec![]),
Some(Duration::max_value()),
&|| false,
|_| {},
|_| {},
);
if let Err(_) = buildx_cmd_exist {
return Err(DockerError::InvalidConfig(format!(
"Docker buildx plugin for buildkit is not correctly installed"
)));
}
// In order to be able to use --cache-from --cache-to for buildkit,
// we need to create our specific builder, which is not the default one (aka: the docker one)
let args = vec![
"buildx",
"create",
"--name",
"qovery-engine",
"--driver-opt",
"network=host",
"--use",
];
let _ = docker_exec(
&args,
&docker.get_all_envs(&vec![]),
Some(Duration::max_value()),
&|| false,
|_| {},
|_| {},
);
Ok(docker)
}
pub fn new(socket_location: Option<Url>) -> Result<Self, DockerError> {
Self::new_with_options(true, socket_location)
}
fn get_all_envs<'a>(&'a self, envs: &'a [(&'a str, &'a str)]) -> Vec<(&'a str, &'a str)> {
let mut all_envs: Vec<(&str, &str)> = self.common_envs.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect();
all_envs.append(&mut envs.to_vec());
all_envs
}
pub fn login(&self, registry: &Url) -> Result<(), DockerError> {
info!("Docker login {} as user {}", registry, registry.username());
let password = urlencoding::decode(&registry.password().unwrap_or_default())
.unwrap_or_default()
.to_string();
let args = vec![
"login",
registry.host_str().unwrap_or_default(),
"-u",
registry.username(),
"-p",
&password,
];
docker_exec(
&args,
&self.get_all_envs(&vec![]),
None,
&|| false,
|line| info!("{}", line),
|line| warn!("{}", line),
)?;
Ok(())
}
pub fn does_image_exist_locally(&self, image: &ContainerImage) -> Result<bool, DockerError> {
info!("Docker check locally image exist {:?}", image);
let ret = docker_exec(
&vec!["image", "inspect", &image.image_name()],
&self.get_all_envs(&vec![]),
None,
&|| false,
|line| info!("{}", line),
|line| warn!("{}", line),
);
Ok(matches!(ret, Ok(_)))
}
// Warning: this command is slow > 10 sec
pub fn does_image_exist_remotely(&self, image: &ContainerImage) -> Result<bool, DockerError> {
info!("Docker check remotely image exist {:?}", image);
let ret = docker_exec(
&vec!["manifest", "inspect", &image.image_name()],
&self.get_all_envs(&vec![]),
None,
&|| false,
|line| info!("{}", line),
|line| warn!("{}", line),
);
match ret {
Ok(_) => Ok(true),
Err(DockerError::ExitStatusError(_)) => Ok(false),
Err(err) => Err(err),
}
}
pub fn pull<Stdout, Stderr>(
&self,
image: &ContainerImage,
stdout_output: Stdout,
stderr_output: Stderr,
timeout: Duration,
should_abort: &dyn Fn() -> bool,
) -> Result<(), DockerError>
where
Stdout: FnMut(String),
Stderr: FnMut(String),
{
info!("Docker pull {:?}, timeout: {:?}", image, timeout);
docker_exec(
&vec!["pull", &image.image_name()],
&self.get_all_envs(&vec![]),
Some(timeout),
should_abort,
stdout_output,
stderr_output,
)
}
pub fn build<Stdout, Stderr>(
&self,
dockerfile: &Path,
context: &Path,
image_to_build: &ContainerImage,
build_args: &[(&str, &str)],
cache: &ContainerImage,
push_after_build: bool,
stdout_output: Stdout,
stderr_output: Stderr,
timeout: Duration,
should_abort: &dyn Fn() -> bool,
) -> Result<(), DockerError>
where
Stdout: FnMut(String),
Stderr: FnMut(String),
{
// if there is no tags, nothing to build
if image_to_build.tags.is_empty() {
return Ok(());
}
// if it is already aborted, nothing to do
if (should_abort)() {
return Err(DockerError::Aborted("build".to_string()));
}
// Do some checks
if !dockerfile.is_file() {
return Err(DockerError::InvalidConfig(format!(
"provided dockerfile `{:?}` is not a valid file",
dockerfile
)));
}
if !context.is_dir() {
return Err(DockerError::InvalidConfig(format!(
"provided docker build context `{:?}` is not a valid directory",
context
)));
}
if self.use_buildkit {
self.build_with_buildkit(
dockerfile,
context,
image_to_build,
build_args,
cache,
push_after_build,
stdout_output,
stderr_output,
timeout,
should_abort,
)
} else {
self.build_with_docker(
dockerfile,
context,
image_to_build,
build_args,
cache,
push_after_build,
stdout_output,
stderr_output,
timeout,
should_abort,
)
}
}
fn build_with_docker<Stdout, Stderr>(
&self,
dockerfile: &Path,
context: &Path,
image_to_build: &ContainerImage,
build_args: &[(&str, &str)],
cache: &ContainerImage,
push_after_build: bool,
stdout_output: Stdout,
stderr_output: Stderr,
timeout: Duration,
should_abort: &dyn Fn() -> bool,
) -> Result<(), DockerError>
where
Stdout: FnMut(String),
Stderr: FnMut(String),
{
info!("Docker build {:?}", image_to_build.image_name());
// Best effort to pull the cache, if it does not exist that's ok too
let _ = self.pull(cache, |_| {}, |_| {}, timeout, should_abort);
let mut args_string: Vec<String> = vec![
"build".to_string(),
"--network".to_string(),
"host".to_string(),
"-f".to_string(),
dockerfile.to_str().unwrap_or_default().to_string(),
];
for image_name in image_to_build.image_names() {
args_string.push("--tag".to_string());
args_string.push(image_name)
}
for img_cache_name in cache.image_names() {
args_string.push("--tag".to_string());
args_string.push(img_cache_name)
}
for (k, v) in build_args {
args_string.push("--build-arg".to_string());
args_string.push(format!("{}={}", k, v));
}
args_string.push(context.to_str().unwrap_or_default().to_string());
let _ = docker_exec(
&args_string.iter().map(|x| x.as_str()).collect::<Vec<&str>>(),
&self.get_all_envs(&vec![]),
Some(timeout),
should_abort,
stdout_output,
stderr_output,
)?;
if push_after_build {
let _ = self.push(image_to_build, |_| {}, |_| {}, timeout, should_abort)?;
}
Ok(())
}
fn build_with_buildkit<Stdout, Stderr>(
&self,
dockerfile: &Path,
context: &Path,
image_to_build: &ContainerImage,
build_args: &[(&str, &str)],
cache: &ContainerImage,
push_after_build: bool,
stdout_output: Stdout,
stderr_output: Stderr,
timeout: Duration,
should_abort: &dyn Fn() -> bool,
) -> Result<(), DockerError>
where
Stdout: FnMut(String),
Stderr: FnMut(String),
{
info!("Docker buildkit build {:?}", image_to_build.image_name());
let mut args_string: Vec<String> = vec![
"buildx".to_string(),
"build".to_string(),
"--progress=plain".to_string(),
"--network=host".to_string(),
if push_after_build {
"--output=type=registry".to_string() // tell buildkit to push image to registry
} else {
"--output=type=docker".to_string() // tell buildkit to load the image into docker after build
},
"--cache-from".to_string(),
format!("type=registry,ref={}", cache.image_name()),
// Disabled for now, because private ECR does not support it ...
// https://github.com/aws/containers-roadmap/issues/876
// "--cache-to".to_string(),
// format!("type=registry,ref={}", cache.image_name()),
"-f".to_string(),
dockerfile.to_str().unwrap_or_default().to_string(),
];
for image_name in image_to_build.image_names() {
args_string.push("--tag".to_string());
args_string.push(image_name.to_string())
}
for (k, v) in build_args {
args_string.push("--build-arg".to_string());
args_string.push(format!("{}={}", k, v));
}
args_string.push(context.to_str().unwrap_or_default().to_string());
docker_exec(
&args_string.iter().map(|x| x.as_str()).collect::<Vec<&str>>(),
&self.get_all_envs(&vec![]),
Some(timeout),
should_abort,
stdout_output,
stderr_output,
)
}
pub fn push<Stdout, Stderr>(
&self,
image: &ContainerImage,
stdout_output: Stdout,
stderr_output: Stderr,
timeout: Duration,
should_abort: &dyn Fn() -> bool,
) -> Result<(), DockerError>
where
Stdout: FnMut(String),
Stderr: FnMut(String),
{
info!("Docker push {:?}, timeout: {:?}", image, timeout);
let image_names = image.image_names();
let mut args = vec!["push"];
args.extend(image_names.iter().map(|x| x.as_str()));
docker_exec(
&args,
&self.get_all_envs(&vec![]),
Some(timeout),
should_abort,
stdout_output,
stderr_output,
)
}
}
fn docker_exec<F, X>(
args: &[&str],
envs: &[(&str, &str)],
timeout: Option<Duration>,
should_abort: &dyn Fn() -> bool,
stdout_output: F,
stderr_output: X,
) -> Result<(), DockerError>
where
F: FnMut(String),
X: FnMut(String),
{
let timeout = timeout.unwrap_or_else(|| Duration::max_value());
let mut cmd = QoveryCommand::new("docker", args, envs);
let ret = cmd.exec_with_abort(timeout, stdout_output, stderr_output, should_abort);
match ret {
Ok(_) => Ok(()),
Err(CommandError::TimeoutError(msg)) => Err(DockerError::Timeout(msg)),
Err(CommandError::Killed(msg)) => Err(DockerError::Aborted(msg)),
Err(CommandError::ExitStatusError(err)) => Err(DockerError::ExitStatusError(err)),
Err(CommandError::ExecutionError(err)) => Err(DockerError::ExecutionError(err)),
}
}
pub fn to_engine_error(event_details: &EventDetails, error: DockerError) -> EngineError {
EngineError::new_docker_error(event_details.clone(), error)
}
// start a local registry to run this test
// docker run --rm -ti -p 5000:5000 --name registry registry:2
#[cfg(feature = "test-with-docker")]
#[cfg(test)]
mod tests {
use crate::cmd::docker::{ContainerImage, Docker, DockerError};
use chrono::Duration;
use std::path::Path;
use url::Url;
fn private_registry_url() -> Url {
Url::parse("http://localhost:5000").unwrap()
}
#[test]
fn test_pull() {
let docker = Docker::new(None).unwrap();
// Invalid image should fails
let image = ContainerImage {
registry: Url::parse("https://docker.io").unwrap(),
name: "alpine".to_string(),
tags: vec!["666".to_string()],
};
let ret = docker.pull(
&image,
|msg| println!("{}", msg),
|msg| eprintln!("{}", msg),
Duration::max_value(),
&|| false,
);
assert!(matches!(ret, Err(_)));
// Valid image should be ok
let image = ContainerImage {
registry: Url::parse("https://docker.io").unwrap(),
name: "alpine".to_string(),
tags: vec!["3.15".to_string()],
};
let ret = docker.pull(
&image,
|msg| println!("{}", msg),
|msg| eprintln!("{}", msg),
Duration::max_value(),
&|| false,
);
assert!(matches!(ret, Ok(_)));
// Should timeout
let ret = docker.pull(
&image,
|msg| println!("{}", msg),
|msg| eprintln!("{}", msg),
Duration::seconds(1),
&|| false,
);
assert!(matches!(ret, Err(DockerError::Timeout(_))));
}
#[test]
fn test_docker_build() {
// start a local registry to run this test
// docker run --rm -d -p 5000:5000 --name registry registry:2
let docker = Docker::new_with_options(false, None).unwrap();
let image_to_build = ContainerImage {
registry: private_registry_url(),
name: "erebe/alpine".to_string(),
tags: vec!["3.15".to_string()],
};
let image_cache = ContainerImage {
registry: private_registry_url(),
name: "erebe/alpine".to_string(),
tags: vec!["cache".to_string()],
};
let ret = docker.build_with_docker(
Path::new("tests/docker/multi_stage_simple/Dockerfile"),
Path::new("tests/docker/multi_stage_simple/"),
&image_to_build,
&vec![],
&image_cache,
false,
|msg| println!("{}", msg),
|msg| eprintln!("{}", msg),
Duration::max_value(),
&|| false,
);
assert!(matches!(ret, Ok(_)));
// It should fails with buildkit dockerfile
let ret = docker.build_with_docker(
Path::new("tests/docker/multi_stage_simple/Dockerfile.buildkit"),
Path::new("tests/docker/multi_stage_simple/"),
&image_to_build,
&vec![],
&image_cache,
false,
|msg| println!("{}", msg),
|msg| eprintln!("{}", msg),
Duration::max_value(),
&|| false,
);
assert!(matches!(ret, Err(_)));
}
#[test]
fn test_buildkit_build() {
// start a local registry to run this test
// docker run --rm -d -p 5000:5000 --name registry registry:2
let docker = Docker::new_with_options(true, None).unwrap();
let image_to_build = ContainerImage {
registry: private_registry_url(),
name: "erebe/alpine".to_string(),
tags: vec!["3.15".to_string()],
};
let image_cache = ContainerImage {
registry: private_registry_url(),
name: "erebe/alpine".to_string(),
tags: vec!["cache".to_string()],
};
// It should work
let ret = docker.build_with_buildkit(
Path::new("tests/docker/multi_stage_simple/Dockerfile"),
Path::new("tests/docker/multi_stage_simple/"),
&image_to_build,
&vec![],
&image_cache,
false,
|msg| println!("{}", msg),
|msg| eprintln!("{}", msg),
Duration::max_value(),
&|| false,
);
assert!(matches!(ret, Ok(_)));
let ret = docker.build_with_buildkit(
Path::new("tests/docker/multi_stage_simple/Dockerfile.buildkit"),
Path::new("tests/docker/multi_stage_simple/"),
&image_to_build,
&vec![],
&image_cache,
false,
|msg| println!("{}", msg),
|msg| eprintln!("{}", msg),
Duration::max_value(),
&|| false,
);
assert!(matches!(ret, Ok(_)));
}
#[test]
fn test_push() {
// start a local registry to run this test
// docker run --rm -d -p 5000:5000 --name registry registry:2
let docker = Docker::new_with_options(true, None).unwrap();
let image_to_build = ContainerImage {
registry: private_registry_url(),
name: "erebe/alpine".to_string(),
tags: vec!["3.15".to_string()],
};
let image_cache = ContainerImage {
registry: private_registry_url(),
name: "erebe/alpine".to_string(),
tags: vec!["cache".to_string()],
};
// It should work
let ret = docker.build_with_buildkit(
Path::new("tests/docker/multi_stage_simple/Dockerfile"),
Path::new("tests/docker/multi_stage_simple/"),
&image_to_build,
&vec![],
&image_cache,
false,
|msg| println!("{}", msg),
|msg| eprintln!("{}", msg),
Duration::max_value(),
&|| false,
);
assert!(matches!(ret, Ok(_)));
let ret = docker.does_image_exist_locally(&image_to_build);
assert!(matches!(ret, Ok(true)));
let ret = docker.does_image_exist_remotely(&image_to_build);
assert!(matches!(ret, Ok(false)));
let ret = docker.push(
&image_to_build,
|msg| println!("{}", msg),
|msg| eprintln!("{}", msg),
Duration::max_value(),
&|| false,
);
assert!(matches!(ret, Ok(_)));
let ret = docker.pull(
&image_to_build,
|msg| println!("{}", msg),
|msg| eprintln!("{}", msg),
Duration::max_value(),
&|| false,
);
assert!(matches!(ret, Ok(_)));
}
}

1
src/cmd/mod.rs
View File

@@ -1,4 +1,5 @@
pub mod command;
pub mod docker;
pub mod helm;
pub mod kubectl;
pub mod structs;

462
src/container_registry/docker.rs
View File

@@ -1,462 +0,0 @@
use crate::build_platform::Image;
use crate::cmd;
use crate::cmd::command::QoveryCommand;
use crate::container_registry::Kind;
use crate::errors::CommandError;
use crate::events::{EngineEvent, EventDetails, EventMessage};
use crate::logger::{LogLevel, Logger};
use chrono::Duration;
use retry::delay::Fibonacci;
use retry::Error::Operation;
use retry::OperationResult;
#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct DockerImageManifest {
pub schema_version: i64,
pub media_type: String,
pub config: Config,
pub layers: Vec<Layer>,
}
#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Config {
pub media_type: String,
pub size: i64,
pub digest: String,
}
#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Layer {
pub media_type: String,
pub size: i64,
pub digest: String,
}
pub fn docker_manifest_inspect(
container_registry_kind: Kind,
docker_envs: Vec<(&str, &str)>,
image_name: String,
image_tag: String,
registry_url: String,
event_details: EventDetails,
logger: &dyn Logger,
) -> Result<DockerImageManifest, CommandError> {
let image_with_tag = format!("{}:{}", image_name, image_tag);
let registry_provider = match container_registry_kind {
Kind::DockerHub => "DockerHub",
Kind::Ecr => "AWS ECR",
Kind::Docr => "DigitalOcean Registry",
Kind::ScalewayCr => "Scaleway Registry",
};
// Note: `docker manifest inspect` is still experimental for the time being:
// https://docs.docker.com/engine/reference/commandline/manifest_inspect/
let mut envs = docker_envs.clone();
envs.push(("DOCKER_CLI_EXPERIMENTAL", "enabled"));
let binary = "docker";
let image_full_url = format!("{}/{}", registry_url.as_str(), &image_with_tag);
let args = vec!["manifest", "inspect", image_full_url.as_str()];
let mut raw_output: Vec<String> = vec![];
let mut cmd = QoveryCommand::new("docker", &args, &envs);
return match cmd.exec_with_timeout(Duration::minutes(1), |line| raw_output.push(line), |_| {}) {
Ok(_) => {
let joined = raw_output.join("");
match serde_json::from_str(&joined) {
Ok(extracted_manifest) => Ok(extracted_manifest),
Err(e) => {
let error = CommandError::new(
e.to_string(),
Some(format!(
"Error while trying to deserialize manifest image manifest for image {} in {} ({}).",
image_with_tag, registry_provider, registry_url,
)),
);
logger.log(
LogLevel::Warning,
EngineEvent::Warning(event_details.clone(), EventMessage::from(error.clone())),
);
Err(error)
}
}
}
Err(e) => {
let error = CommandError::new(
format!(
"Command `{}`: {:?}",
cmd::command::command_to_string(binary, &args, &envs),
e
),
Some(format!(
"Error while trying to inspect image manifest for image {} in {} ({}).",
image_with_tag, registry_provider, registry_url,
)),
);
logger.log(
LogLevel::Warning,
EngineEvent::Warning(event_details.clone(), EventMessage::from(error.clone())),
);
Err(error)
}
};
}
pub fn docker_login(
container_registry_kind: Kind,
docker_envs: Vec<(&str, &str)>,
registry_login: String,
registry_pass: String,
registry_url: String,
event_details: EventDetails,
logger: &dyn Logger,
) -> Result<(), CommandError> {
let registry_provider = match container_registry_kind {
Kind::DockerHub => "DockerHub",
Kind::Ecr => "AWS ECR",
Kind::Docr => "DigitalOcean Registry",
Kind::ScalewayCr => "Scaleway Registry",
};
let binary = "docker";
let args = vec![
"login",
registry_url.as_str(),
"-u",
registry_login.as_str(),
"-p",
registry_pass.as_str(),
];
let mut cmd = QoveryCommand::new(binary, &args, &docker_envs);
match cmd.exec() {
Ok(_) => Ok(()),
Err(e) => {
let err = CommandError::new(
format!(
"Command `{}`: {:?}",
cmd::command::command_to_string(binary, &args, &docker_envs),
e,
),
Some(format!(
"Error while trying to login to registry {} {}.",
registry_provider, registry_url,
)),
);
logger.log(
LogLevel::Warning,
EngineEvent::Warning(event_details.clone(), EventMessage::from(err.clone())),
);
Err(err)
}
}
}
pub fn docker_tag_and_push_image(
container_registry_kind: Kind,
docker_envs: Vec<(&str, &str)>,
image: &Image,
dest: String,
dest_latest_tag: String,
event_details: EventDetails,
logger: &dyn Logger,
) -> Result<(), CommandError> {
let image_with_tag = image.name_with_tag();
let registry_provider = match container_registry_kind {
Kind::DockerHub => "DockerHub",
Kind::Ecr => "AWS ECR",
Kind::Docr => "DigitalOcean Registry",
Kind::ScalewayCr => "Scaleway Registry",
};
let binary = "docker";
let args = vec!["tag", &image_with_tag, dest.as_str()];
let mut cmd = QoveryCommand::new(binary, &args, &docker_envs);
match retry::retry(Fibonacci::from_millis(3000).take(5), || match cmd.exec() {
Ok(_) => OperationResult::Ok(()),
Err(e) => {
logger.log(
LogLevel::Warning,
EngineEvent::Warning(
event_details.clone(),
EventMessage::new(
format!("Failed to tag image `{}`, retrying...", image_with_tag),
Some(format!(
"Command `{}`: {:?}",
cmd::command::command_to_string(binary, &args, &docker_envs),
e
)),
),
),
);
OperationResult::Retry(e)
}
}) {
Err(Operation { error, .. }) => {
logger.log(
LogLevel::Warning,
EngineEvent::Warning(
event_details.clone(),
EventMessage::from(CommandError::new_from_legacy_command_error(
error,
Some(format!("Error while trying to tag docker image `{}`", image_with_tag)),
)),
),
);
}
Err(retry::Error::Internal(msg)) => {
logger.log(
LogLevel::Warning,
EngineEvent::Warning(
event_details.clone(),
EventMessage::from(CommandError::new(
msg,
Some(format!("Error while trying to tag docker image `{}`", image_with_tag)),
)),
),
);
}
Ok(_) => {}
}
let mut cmd = QoveryCommand::new("docker", &vec!["push", dest.as_str()], &docker_envs);
let _ = match retry::retry(Fibonacci::from_millis(5000).take(5), || {
match cmd.exec_with_timeout(
Duration::minutes(10),
|line| {
logger.log(
LogLevel::Info,
EngineEvent::Info(event_details.clone(), EventMessage::new(line, None)),
)
},
|line| {
logger.log(
LogLevel::Warning,
EngineEvent::Warning(event_details.clone(), EventMessage::new(line, None)),
)
},
) {
Ok(_) => OperationResult::Ok(()),
Err(e) => {
logger.log(
LogLevel::Warning,
EngineEvent::Warning(
event_details.clone(),
EventMessage::new(
format!(
"Failed to push image `{}` on `{}`, retrying ...",
image_with_tag, registry_provider
),
Some(format!("{:?}", e)),
),
),
);
OperationResult::Retry(e)
}
}
}) {
Err(Operation { error, .. }) => Err(CommandError::new_from_legacy_command_error(
error,
Some(format!("Failed to push docker image `{}`", image_with_tag)),
)),
Err(retry::Error::Internal(msg)) => Err(CommandError::new(
msg,
Some(format!("Failed to push docker image `{}`", image_with_tag)),
)),
_ => {
logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(format!(
"Image {} has successfully been pushed on `{}`",
image_with_tag, registry_provider
)),
),
);
Ok(())
}
};
let image_with_latest_tag = image.name_with_latest_tag();
let mut cmd = QoveryCommand::new(
"docker",
&vec!["tag", &image_with_latest_tag, dest_latest_tag.as_str()],
&docker_envs,
);
match retry::retry(Fibonacci::from_millis(3000).take(5), || match cmd.exec() {
Ok(_) => OperationResult::Ok(()),
Err(e) => {
logger.log(
LogLevel::Warning,
EngineEvent::Warning(
event_details.clone(),
EventMessage::new(
format!("Failed to tag image `{}`, retrying ...", image_with_latest_tag),
Some(format!("{:?}", e)),
),
),
);
OperationResult::Retry(e)
}
}) {
Err(Operation { error, .. }) => {
return Err(CommandError::new_from_legacy_command_error(
error,
Some(format!("Failed to tag docker image `{}`", image_with_tag)),
))
}
Err(retry::Error::Internal(msg)) => {
return Err(CommandError::new(
msg,
Some(format!("Failed to tag docker image `{}`", image_with_tag)),
))
}
_ => {}
}
let mut cmd = QoveryCommand::new("docker", &vec!["push", dest_latest_tag.as_str()], &docker_envs);
match retry::retry(Fibonacci::from_millis(5000).take(5), || {
match cmd.exec_with_timeout(
Duration::minutes(10),
|line| {
logger.log(
LogLevel::Info,
EngineEvent::Info(event_details.clone(), EventMessage::new(line, None)),
)
},
|line| {
logger.log(
LogLevel::Warning,
EngineEvent::Warning(event_details.clone(), EventMessage::new(line, None)),
)
},
) {
Ok(_) => OperationResult::Ok(()),
Err(e) => {
logger.log(
LogLevel::Warning,
EngineEvent::Warning(
event_details.clone(),
EventMessage::new(
format!(
"Failed to push image {} on {}, retrying...",
image_with_tag, registry_provider
),
Some(format!("{:?}", e)),
),
),
);
OperationResult::Retry(e)
}
}
}) {
Err(Operation { error, .. }) => Err(CommandError::new(error.to_string(), None)),
Err(e) => Err(CommandError::new(
format!("{:?}", e),
Some(format!(
"Unknown error while trying to push image {} to {}.",
image_with_tag, registry_provider,
)),
)),
_ => {
logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(format!("image {} has successfully been pushed", image_with_tag)),
),
);
Ok(())
}
}
}
pub fn docker_pull_image(
container_registry_kind: Kind,
docker_envs: Vec<(&str, &str)>,
dest: String,
event_details: EventDetails,
logger: &dyn Logger,
) -> Result<(), CommandError> {
let registry_provider = match container_registry_kind {
Kind::DockerHub => "DockerHub",
Kind::Ecr => "AWS ECR",
Kind::Docr => "DigitalOcean Registry",
Kind::ScalewayCr => "Scaleway Registry",
};
let mut cmd = QoveryCommand::new("docker", &vec!["pull", dest.as_str()], &docker_envs);
match retry::retry(Fibonacci::from_millis(5000).take(5), || {
match cmd.exec_with_timeout(
Duration::minutes(10),
|line| {
logger.log(
LogLevel::Info,
EngineEvent::Info(event_details.clone(), EventMessage::new(line, None)),
)
},
|line| {
logger.log(
LogLevel::Warning,
EngineEvent::Warning(event_details.clone(), EventMessage::new(line, None)),
)
},
) {
Ok(_) => OperationResult::Ok(()),
Err(e) => {
logger.log(
LogLevel::Warning,
EngineEvent::Warning(
event_details.clone(),
EventMessage::new(
format!(
"failed to pull image from {} registry {}, retrying...",
registry_provider,
dest.as_str(),
),
Some(format!("{:?}", e)),
),
),
);
OperationResult::Retry(e)
}
}
}) {
Err(Operation { error, .. }) => Err(CommandError::new(error.to_string(), None)),
Err(e) => Err(CommandError::new(
format!("{:?}", e),
Some(format!(
"Unknown error while trying to pull image {} from {} registry.",
dest.as_str(),
registry_provider,
)),
)),
_ => {
logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(format!(
"Image {} has successfully been pulled from {} registry",
dest.as_str(),
registry_provider,
)),
),
);
Ok(())
}
}
}

320
src/container_registry/docker_hub.rs
View File

@@ -1,320 +0,0 @@
extern crate reqwest;
use reqwest::StatusCode;
use std::borrow::Borrow;
use crate::build_platform::Image;
use crate::cmd::command::QoveryCommand;
use crate::container_registry::docker::{docker_pull_image, docker_tag_and_push_image};
use crate::container_registry::{ContainerRegistry, Kind, PullResult, PushResult};
use crate::errors::{CommandError, EngineError};
use crate::events::{EngineEvent, EventMessage, ToTransmitter, Transmitter};
use crate::logger::{LogLevel, Logger};
use crate::models::{
Context, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope,
};
pub struct DockerHub {
context: Context,
id: String,
name: String,
login: String,
password: String,
listeners: Listeners,
logger: Box<dyn Logger>,
}
impl DockerHub {
pub fn new(context: Context, id: &str, name: &str, login: &str, password: &str, logger: Box<dyn Logger>) -> Self {
DockerHub {
context,
id: id.to_string(),
name: name.to_string(),
login: login.to_string(),
password: password.to_string(),
listeners: vec![],
logger,
}
}
pub fn exec_docker_login(&self) -> Result<(), EngineError> {
let event_details = self.get_event_details();
let envs = match self.context.docker_tcp_socket() {
Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())],
None => vec![],
};
let mut cmd = QoveryCommand::new(
"docker",
&vec!["login", "-u", self.login.as_str(), "-p", self.password.as_str()],
&envs,
);
match cmd.exec() {
Ok(_) => Ok(()),
Err(_) => Err(EngineError::new_client_invalid_cloud_provider_credentials(
event_details,
)),
}
}
fn pull_image(&self, dest: String, image: &Image) -> Result<PullResult, EngineError> {
let event_details = self.get_event_details();
match docker_pull_image(self.kind(), vec![], dest.clone(), event_details.clone(), self.logger()) {
Ok(_) => {
let mut image = image.clone();
image.registry_url = Some(dest);
Ok(PullResult::Some(image))
}
Err(e) => Err(EngineError::new_docker_pull_image_error(
event_details,
image.name.to_string(),
dest.to_string(),
e,
)),
}
}
}
impl ToTransmitter for DockerHub {
fn to_transmitter(&self) -> Transmitter {
Transmitter::ContainerRegistry(self.id().to_string(), self.name().to_string())
}
}
impl ContainerRegistry for DockerHub {
fn context(&self) -> &Context {
&self.context
}
fn kind(&self) -> Kind {
Kind::DockerHub
}
fn id(&self) -> &str {
self.id.as_str()
}
fn name(&self) -> &str {
self.name.as_str()
}
fn is_valid(&self) -> Result<(), EngineError> {
Ok(())
}
fn on_create(&self) -> Result<(), EngineError> {
Ok(())
}
fn on_create_error(&self) -> Result<(), EngineError> {
Ok(())
}
fn on_delete(&self) -> Result<(), EngineError> {
Ok(())
}
fn on_delete_error(&self) -> Result<(), EngineError> {
Ok(())
}
fn does_image_exists(&self, image: &Image) -> bool {
let event_details = self.get_event_details();
use reqwest::blocking::Client;
let client = Client::new();
let path = format!(
"https://index.docker.io/v1/repositories/{}/{}/tags",
&self.login, image.name
);
let res = client
.get(path.as_str())
.basic_auth(&self.login, Option::from(&self.password))
.send();
// TODO (mzo) no check of existing tags as in others impl ?
match res {
Ok(out) => matches!(out.status(), StatusCode::OK),
Err(e) => {
self.logger.log(
LogLevel::Error,
EngineEvent::Error(
EngineError::new_container_registry_repository_doesnt_exist(
event_details.clone(),
image.name.to_string(),
Some(CommandError::new(
e.to_string(),
Some("Error while trying to retrieve if DockerHub repository exist.".to_string()),
)),
),
None,
),
);
false
}
}
}
fn pull(&self, image: &Image) -> Result<PullResult, EngineError> {
let event_details = self.get_event_details();
let listeners_helper = ListenersHelper::new(&self.listeners);
if !self.does_image_exists(image) {
let info_message = format!(
"image {:?} does not exist in DockerHub {} repository",
image,
self.name()
);
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(info_message.to_string()),
),
);
listeners_helper.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
return Ok(PullResult::None);
}
let info_message = format!("pull image {:?} from DockerHub {} repository", image, self.name());
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(info_message.to_string()),
),
);
listeners_helper.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
let _ = self.exec_docker_login()?;
let dest = format!("{}/{}", self.login.as_str(), image.name_with_tag().as_str());
// pull image
self.pull_image(dest, image)
}
fn push(&self, image: &Image, force_push: bool) -> Result<PushResult, EngineError> {
let event_details = self.get_event_details();
let _ = self.exec_docker_login()?;
let dest = format!("{}/{}", self.login.as_str(), image.name_with_tag().as_str());
let listeners_helper = ListenersHelper::new(&self.listeners);
if !force_push && self.does_image_exists(image) {
// check if image does exist - if yes, do not upload it again
let info_message = format!(
"image {:?} found on DockerHub {} repository, container build is not required",
image,
self.name()
);
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(info_message.to_string()),
),
);
listeners_helper.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
let mut image = image.clone();
image.registry_url = Some(dest);
return Ok(PushResult { image });
}
let info_message = format!(
"image {:?} does not exist on DockerHub {} repository, starting image upload",
image,
self.name()
);
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(info_message.to_string()),
),
);
listeners_helper.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
let dest_latest_tag = format!("{}/{}:latest", self.login.as_str(), image.name);
match docker_tag_and_push_image(
self.kind(),
vec![],
&image,
dest.clone(),
dest_latest_tag,
event_details.clone(),
self.logger(),
) {
Ok(_) => {
let mut image = image.clone();
image.registry_url = Some(dest);
Ok(PushResult { image })
}
Err(e) => Err(EngineError::new_docker_push_image_error(
event_details.clone(),
image.name.to_string(),
dest.to_string(),
e,
)),
}
}
fn push_error(&self, _image: &Image) -> Result<PushResult, EngineError> {
unimplemented!()
}
fn logger(&self) -> &dyn Logger {
self.logger.borrow()
}
}
impl Listen for DockerHub {
fn listeners(&self) -> &Listeners {
&self.listeners
}
fn add_listener(&mut self, listener: Listener) {
self.listeners.push(listener);
}
}

344
src/container_registry/docr.rs
View File

@@ -6,21 +6,17 @@ use std::borrow::Borrow;
use crate::build_platform::Image;
use crate::cmd::command::QoveryCommand;
use crate::container_registry::docker::{docker_pull_image, docker_tag_and_push_image};
use crate::container_registry::{ContainerRegistry, EngineError, Kind, PullResult, PushResult};
use crate::container_registry::{ContainerRegistry, ContainerRegistryInfo, EngineError, Kind};
use crate::errors::CommandError;
use crate::events::{EngineEvent, EventDetails, EventMessage, ToTransmitter, Transmitter};
use crate::events::{EngineEvent, EventDetails, ToTransmitter, Transmitter};
use crate::logger::{LogLevel, Logger};
use crate::models::{
Context, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope,
};
use crate::models::{Context, Listen, Listener, Listeners};
use crate::utilities;
use retry::delay::Fixed;
use retry::Error::Operation;
use retry::OperationResult;
use url::Url;
const CR_API_PATH: &str = "https://api.digitalocean.com/v2/registry";
const CR_CLUSTER_API_PATH: &str = "https://api.digitalocean.com/v2/kubernetes/registry";
const CR_REGISTRY_DOMAIN: &str = "registry.digitalocean.com";
// TODO : use --output json
// see https://www.digitalocean.com/community/tutorials/how-to-use-doctl-the-official-digitalocean-command-line-client
@@ -46,32 +42,16 @@ impl DOCR {
}
}
fn get_registry_name(&self, image: &Image) -> Result<String, EngineError> {
fn create_registry(&self, registry_name: &str) -> Result<(), EngineError> {
let event_details = self.get_event_details();
let registry_name = match image.registry_name.as_ref() {
// DOCR does not support upper cases
Some(registry_name) => registry_name.to_lowercase(),
None => get_current_registry_name(self.api_key.as_str(), event_details, self.logger())?,
};
Ok(registry_name)
}
fn create_repository(&self, image: &Image) -> Result<(), EngineError> {
let event_details = self.get_event_details();
let registry_name = match image.registry_name.as_ref() {
// DOCR does not support upper cases
Some(registry_name) => registry_name.to_lowercase(),
None => self.name.clone(),
};
// DOCR does not support upper cases
let registry_name = registry_name.to_lowercase();
let headers = utilities::get_header_with_bearer(&self.api_key);
// subscription_tier_slug: https://www.digitalocean.com/products/container-registry/
// starter and basic tiers are too limited on repository creation
let repo = DoApiCreateRepository {
name: registry_name.clone(),
name: registry_name.to_string(),
subscription_tier_slug: "professional".to_string(),
};
@@ -133,77 +113,7 @@ impl DOCR {
}
}
fn push_image(&self, registry_name: String, dest: String, image: &Image) -> Result<PushResult, EngineError> {
let event_details = self.get_event_details();
let dest_latest_tag = format!(
"registry.digitalocean.com/{}/{}:latest",
registry_name.as_str(),
image.name
);
if let Err(e) = docker_tag_and_push_image(
self.kind(),
vec![],
image,
dest.clone(),
dest_latest_tag.clone(),
event_details.clone(),
self.logger(),
) {
return Err(EngineError::new_docker_push_image_error(
event_details,
image.name.to_string(),
dest.to_string(),
e,
));
}
let mut image = image.clone();
image.registry_name = Some(registry_name.clone());
// on DOCR registry secret is the same as registry name
image.registry_secret = Some(registry_name);
image.registry_url = Some(dest);
let result = retry::retry(Fixed::from_millis(10000).take(12), || {
match self.does_image_exists(&image) {
true => OperationResult::Ok(&image),
false => {
self.logger.log(
LogLevel::Warning,
EngineEvent::Warning(
self.get_event_details(),
EventMessage::new_from_safe(
"Image is not yet available on DOCR, retrying in a few seconds...".to_string(),
),
),
);
OperationResult::Retry(())
}
}
});
let image_not_reachable = Err(EngineError::new_container_registry_image_unreachable_after_push(
event_details.clone(),
image.name.to_string(),
));
match result {
Ok(_) => Ok(PushResult { image }),
Err(Operation { .. }) => image_not_reachable,
Err(retry::Error::Internal(_)) => image_not_reachable,
}
}
pub fn get_image(&self, _image: &Image) -> Option<()> {
todo!()
}
pub fn delete_image(&self, _image: &Image) -> Result<(), EngineError> {
// TODO(benjaminch): To be implemented later on, but note it must not slow down CI workflow
Ok(())
}
pub fn delete_repository(&self) -> Result<(), EngineError> {
pub fn delete_registry(&self) -> Result<(), EngineError> {
let event_details = self.get_event_details();
let headers = utilities::get_header_with_bearer(&self.api_key);
@@ -255,27 +165,6 @@ impl DOCR {
)),
}
}
fn pull_image(&self, registry_name: String, dest: String, image: &Image) -> Result<PullResult, EngineError> {
let event_details = self.get_event_details();
match docker_pull_image(self.kind(), vec![], dest.clone(), event_details.clone(), self.logger()) {
Ok(_) => {
let mut image = image.clone();
image.registry_name = Some(registry_name.clone());
// on DOCR registry secret is the same as registry name
image.registry_secret = Some(registry_name);
image.registry_url = Some(dest);
Ok(PullResult::Some(image))
}
Err(e) => Err(EngineError::new_docker_pull_image_error(
event_details,
image.name.to_string(),
dest.to_string(),
e,
)),
}
}
}
impl ToTransmitter for DOCR {
@@ -305,38 +194,39 @@ impl ContainerRegistry for DOCR {
Ok(())
}
fn on_create(&self) -> Result<(), EngineError> {
fn login(&self) -> Result<ContainerRegistryInfo, EngineError> {
let _ = self.exec_docr_login()?;
let registry_name = self.name.clone();
Ok(ContainerRegistryInfo {
endpoint: Url::parse(&format!("https://{}", CR_REGISTRY_DOMAIN)).unwrap(),
registry_name: self.name.to_string(),
registry_docker_json_config: None,
get_image_name: Box::new(move |img_name| format!("{}/{}", registry_name, img_name)),
})
}
fn create_registry(&self) -> Result<(), EngineError> {
// Digital Ocean only allow one registry per account...
if let Err(_) = get_current_registry_name(self.api_key.as_str(), self.get_event_details(), self.logger()) {
let _ = self.create_registry(self.name())?;
}
Ok(())
}
fn on_create_error(&self) -> Result<(), EngineError> {
Ok(())
}
fn on_delete(&self) -> Result<(), EngineError> {
Ok(())
}
fn on_delete_error(&self) -> Result<(), EngineError> {
fn create_repository(&self, _repository_name: &str) -> Result<(), EngineError> {
// Nothing to do, DO only allow one registry and create repository on the flight when image are pushed
Ok(())
}
fn does_image_exists(&self, image: &Image) -> bool {
let event_details = self.get_event_details();
let registry_name = match self.get_registry_name(image) {
Ok(registry_name) => registry_name,
Err(err) => {
self.logger.log(LogLevel::Error, EngineEvent::Error(err, None));
return false;
}
};
let headers = utilities::get_header_with_bearer(self.api_key.as_str());
let url = format!(
"https://api.digitalocean.com/v2/registry/{}/repositories/{}/tags",
registry_name,
image.name.as_str()
image.registry_name,
image.name()
);
let res = reqwest::blocking::Client::new()
@@ -353,10 +243,10 @@ impl ContainerRegistry for DOCR {
EngineEvent::Error(
EngineError::new_container_registry_image_doesnt_exist(
event_details.clone(),
image.name.to_string(),
image.name().to_string(),
Some(CommandError::new_from_safe_message(format!(
"While tyring to get all tags for image: `{}`, maybe this image not exist !",
image.name.to_string()
image.name().to_string()
))),
),
None,
@@ -372,10 +262,10 @@ impl ContainerRegistry for DOCR {
EngineEvent::Error(
EngineError::new_container_registry_image_doesnt_exist(
event_details.clone(),
image.name.to_string(),
image.name().to_string(),
Some(CommandError::new_from_safe_message(format!(
"While trying to communicate with DigitalOcean API to retrieve all tags for image `{}`.",
image.name.to_string()
image.name().to_string()
))),
),
None,
@@ -405,7 +295,7 @@ impl ContainerRegistry for DOCR {
EngineEvent::Error(
EngineError::new_container_registry_image_doesnt_exist(
event_details.clone(),
image.name.to_string(),
image.name().to_string(),
Some(CommandError::new(
out.to_string(),
Some(format!(
@@ -428,10 +318,10 @@ impl ContainerRegistry for DOCR {
EngineEvent::Error(
EngineError::new_container_registry_image_doesnt_exist(
event_details.clone(),
image.name.to_string(),
image.name().to_string(),
Some(CommandError::new_from_safe_message(format!(
"While retrieving tags for image `{}` Unable to get output from DigitalOcean API.",
image.name.to_string()
image.name().to_string()
))),
),
None,
@@ -443,164 +333,6 @@ impl ContainerRegistry for DOCR {
}
}
fn pull(&self, image: &Image) -> Result<PullResult, EngineError> {
let event_details = self.get_event_details();
let listeners_helper = ListenersHelper::new(&self.listeners);
if !self.does_image_exists(image) {
let info_message = format!("image {:?} does not exist in DOCR {} repository", image, self.name());
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(info_message.to_string()),
),
);
listeners_helper.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
return Ok(PullResult::None);
}
let info_message = format!("pull image {:?} from DOCR {} repository", image, self.name());
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(info_message.to_string()),
),
);
listeners_helper.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
let _ = self.exec_docr_login()?;
let registry_name = self.get_registry_name(image)?;
let dest = format!(
"registry.digitalocean.com/{}/{}",
registry_name.as_str(),
image.name_with_tag()
);
// pull image
self.pull_image(registry_name, dest, image)
}
// https://www.digitalocean.com/docs/images/container-registry/how-to/use-registry-docker-kubernetes/
fn push(&self, image: &Image, force_push: bool) -> Result<PushResult, EngineError> {
let event_details = self.get_event_details();
let registry_name = self.get_registry_name(image)?;
match self.create_repository(image) {
Ok(_) => self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(format!("DOCR {} has been created", registry_name.as_str())),
),
),
Err(e) => self.logger.log(
LogLevel::Error,
EngineEvent::Error(
e.clone(),
Some(EventMessage::new_from_safe(format!(
"DOCR {} already exists",
registry_name.as_str()
))),
),
),
};
let _ = self.exec_docr_login()?;
let dest = format!(
"registry.digitalocean.com/{}/{}",
registry_name.as_str(),
image.name_with_tag()
);
let listeners_helper = ListenersHelper::new(&self.listeners);
if !force_push && self.does_image_exists(image) {
// check if image does exist - if yes, do not upload it again
let info_message = format!(
"image {:?} found on DOCR {} repository, container build is not required",
image,
registry_name.as_str()
);
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(info_message.to_string()),
),
);
listeners_helper.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
let mut image = image.clone();
image.registry_name = Some(registry_name.clone());
// on DOCR registry secret is the same as registry name
image.registry_secret = Some(registry_name);
image.registry_url = Some(dest);
return Ok(PushResult { image });
}
let info_message = format!(
"image {:?} does not exist on DOCR {} repository, starting image upload",
image, registry_name
);
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(info_message.to_string()),
),
);
listeners_helper.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
self.push_image(registry_name, dest, image)
}
fn push_error(&self, image: &Image) -> Result<PushResult, EngineError> {
Ok(PushResult { image: image.clone() })
}
fn logger(&self) -> &dyn Logger {
self.logger.borrow()
}

326
src/container_registry/ecr.rs
View File

@@ -10,20 +10,18 @@ use rusoto_ecr::{
use rusoto_sts::{GetCallerIdentityRequest, Sts, StsClient};
use crate::build_platform::Image;
use crate::cmd::command::QoveryCommand;
use crate::container_registry::docker::{docker_pull_image, docker_tag_and_push_image};
use crate::container_registry::{ContainerRegistry, Kind, PullResult, PushResult};
use crate::cmd::docker::{to_engine_error, Docker};
use crate::container_registry::{ContainerRegistry, ContainerRegistryInfo, Kind};
use crate::errors::{CommandError, EngineError};
use crate::events::{EngineEvent, EventMessage, ToTransmitter, Transmitter};
use crate::logger::{LogLevel, Logger};
use crate::models::{
Context, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope,
};
use crate::models::{Context, Listen, Listener, Listeners};
use crate::runtime::block_on;
use retry::delay::Fixed;
use retry::Error::Operation;
use retry::OperationResult;
use serde_json::json;
use url::Url;
pub struct ECR {
context: Context,
@@ -75,9 +73,9 @@ impl ECR {
EcrClient::new_with_client(self.client(), self.region.clone())
}
fn get_repository(&self, image: &Image) -> Option<Repository> {
fn get_repository(&self, repository_name: &str) -> Option<Repository> {
let mut drr = DescribeRepositoriesRequest::default();
drr.repository_names = Some(vec![image.name.to_string()]);
drr.repository_names = Some(vec![repository_name.to_string()]);
let r = block_on(self.ecr_client().describe_repositories(drr));
@@ -93,7 +91,7 @@ impl ECR {
fn get_image(&self, image: &Image) -> Option<ImageDetail> {
let mut dir = DescribeImagesRequest::default();
dir.repository_name = image.name.to_string();
dir.repository_name = image.name().to_string();
let mut image_identifier = ImageIdentifier::default();
image_identifier.image_tag = Some(image.tag.to_string());
@@ -111,71 +109,8 @@ impl ECR {
}
}
fn docker_envs(&self) -> Vec<(&str, &str)> {
match self.context.docker_tcp_socket() {
Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())],
None => vec![],
}
}
fn push_image(&self, dest: String, dest_latest_tag: String, image: &Image) -> Result<PushResult, EngineError> {
// READ https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html
// docker tag e9ae3c220b23 aws_account_id.dkr.ecr.region.amazonaws.com/my-web-app
fn create_repository(&self, repository_name: &str) -> Result<Repository, EngineError> {
let event_details = self.get_event_details();
match docker_tag_and_push_image(
self.kind(),
self.docker_envs(),
&image,
dest.clone(),
dest_latest_tag,
event_details.clone(),
self.logger(),
) {
Ok(_) => {
let mut image = image.clone();
image.registry_url = Some(dest);
Ok(PushResult { image })
}
Err(e) => Err(EngineError::new_docker_push_image_error(
event_details,
image.name.to_string(),
dest.to_string(),
e,
)),
}
}
fn pull_image(&self, dest: String, image: &Image) -> Result<PullResult, EngineError> {
// READ https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-pull-ecr-image.html
// docker pull aws_account_id.dkr.ecr.us-west-2.amazonaws.com/amazonlinux:latest
let event_details = self.get_event_details();
match docker_pull_image(
self.kind(),
self.docker_envs(),
dest.clone(),
event_details.clone(),
self.logger(),
) {
Ok(_) => {
let mut image = image.clone();
image.registry_url = Some(dest);
Ok(PullResult::Some(image))
}
Err(e) => Err(EngineError::new_docker_pull_image_error(
event_details,
image.name.to_string(),
dest.to_string(),
e,
)),
}
}
fn create_repository(&self, image: &Image) -> Result<Repository, EngineError> {
let event_details = self.get_event_details();
let repository_name = image.name.as_str();
self.logger().log(
LogLevel::Info,
EngineEvent::Info(
@@ -314,7 +249,7 @@ impl ECR {
});
let plp = PutLifecyclePolicyRequest {
repository_name: image.name.clone(),
repository_name: repository_name.to_string(),
lifecycle_policy_text: lifecycle_policy_text.to_string(),
..Default::default()
};
@@ -327,27 +262,27 @@ impl ECR {
CommandError::new_from_safe_message(err.to_string()),
),
),
_ => Ok(self.get_repository(image).expect("cannot get repository")),
_ => Ok(self.get_repository(repository_name).expect("cannot get repository")),
}
}
fn get_or_create_repository(&self, image: &Image) -> Result<Repository, EngineError> {
fn get_or_create_repository(&self, repository_name: &str) -> Result<Repository, EngineError> {
let event_details = self.get_event_details();
// check if the repository already exists
let repository = self.get_repository(image);
let repository = self.get_repository(repository_name);
if repository.is_some() {
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(format!("ECR repository {} already exists", image.name.as_str())),
EventMessage::new_from_safe(format!("ECR repository {} already exists", repository_name)),
),
);
return Ok(repository.unwrap());
}
self.create_repository(image)
self.create_repository(repository_name)
}
fn get_credentials(&self) -> Result<ECRCredentials, EngineError> {
@@ -391,32 +326,6 @@ impl ECR {
Ok(ECRCredentials::new(access_token, password, endpoint_url))
}
fn exec_docker_login(&self) -> Result<(), EngineError> {
let event_details = self.get_event_details();
let credentials = self.get_credentials()?;
let mut cmd = QoveryCommand::new(
"docker",
&vec![
"login",
"-u",
credentials.access_token.as_str(),
"-p",
credentials.password.as_str(),
credentials.endpoint_url.as_str(),
],
&self.docker_envs(),
);
if let Err(_) = cmd.exec() {
return Err(EngineError::new_client_invalid_cloud_provider_credentials(
event_details.clone(),
));
};
Ok(())
}
}
impl ToTransmitter for ECR {
@@ -454,196 +363,41 @@ impl ContainerRegistry for ECR {
}
}
fn on_create(&self) -> Result<(), EngineError> {
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
self.get_event_details(),
EventMessage::new_from_safe("ECR.on_create() called".to_string()),
),
);
fn login(&self) -> Result<ContainerRegistryInfo, EngineError> {
let event_details = self.get_event_details();
let credentials = self.get_credentials()?;
let docker = Docker::new(self.context.docker_tcp_socket().clone())
.map_err(|err| to_engine_error(&event_details, err))?;
let mut registry_url = Url::parse(credentials.endpoint_url.as_str()).unwrap();
let _ = registry_url.set_username(&credentials.access_token);
let _ = registry_url.set_password(Some(&credentials.password));
let _ = docker
.login(&registry_url)
.map_err(|err| to_engine_error(&event_details, err))?;
Ok(ContainerRegistryInfo {
endpoint: registry_url,
registry_name: self.name.to_string(),
registry_docker_json_config: None,
get_image_name: Box::new(|img_name| img_name.to_string()),
})
}
fn create_registry(&self) -> Result<(), EngineError> {
// Nothing to do, ECR require to create only repository
Ok(())
}
fn on_create_error(&self) -> Result<(), EngineError> {
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
self.get_event_details(),
EventMessage::new_from_safe("ECR.on_create_error() called".to_string()),
),
);
unimplemented!()
}
fn on_delete(&self) -> Result<(), EngineError> {
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
self.get_event_details(),
EventMessage::new_from_safe("ECR.on_delete() called".to_string()),
),
);
unimplemented!()
}
fn on_delete_error(&self) -> Result<(), EngineError> {
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
self.get_event_details(),
EventMessage::new_from_safe("ECR.on_delete_error() called".to_string()),
),
);
unimplemented!()
fn create_repository(&self, name: &str) -> Result<(), EngineError> {
let _ = self.get_or_create_repository(name)?;
Ok(())
}
fn does_image_exists(&self, image: &Image) -> bool {
self.get_image(image).is_some()
}
fn pull(&self, image: &Image) -> Result<PullResult, EngineError> {
let event_details = self.get_event_details();
let listeners_helper = ListenersHelper::new(&self.listeners);
if !self.does_image_exists(image) {
let info_message = format!(
"image `{}` does not exist in ECR {} repository",
image.name_with_tag(),
self.name()
);
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(info_message.to_string()),
),
);
listeners_helper.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
return Ok(PullResult::None);
}
let info_message = format!(
"pull image `{:?}` from ECR {} repository",
image.name_with_tag(),
self.name()
);
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(info_message.to_string()),
),
);
listeners_helper.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
let _ = self.exec_docker_login()?;
let repository = self.get_or_create_repository(image)?;
let dest = format!("{}:{}", repository.repository_uri.unwrap(), image.tag.as_str());
// pull image
self.pull_image(dest, image)
}
fn push(&self, image: &Image, force_push: bool) -> Result<PushResult, EngineError> {
let _ = self.exec_docker_login()?;
let repository = if force_push {
self.create_repository(image)
} else {
self.get_or_create_repository(image)
}?;
let repository_uri = repository.repository_uri.expect("Error getting repository URI");
let dest = format!("{}:{}", repository_uri, image.tag.as_str());
let listeners_helper = ListenersHelper::new(&self.listeners);
if !force_push && self.does_image_exists(image) {
// check if image does exist - if yes, do not upload it again
let info_message = format!(
"image {} found on ECR {} repository, container build is not required",
image.name_with_tag(),
self.name()
);
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
self.get_event_details(),
EventMessage::new_from_safe(info_message.to_string()),
),
);
listeners_helper.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
let mut image = image.clone();
image.registry_url = Some(dest);
return Ok(PushResult { image });
}
let info_message = format!(
"image `{}` does not exist on ECR {} repository, starting image upload",
image.name_with_tag(),
self.name()
);
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
self.get_event_details(),
EventMessage::new_from_safe(info_message.to_string()),
),
);
listeners_helper.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
let dest_latest_tag = format!("{}:latest", repository_uri);
self.push_image(dest, dest_latest_tag, image)
}
fn push_error(&self, image: &Image) -> Result<PushResult, EngineError> {
// TODO change this
Ok(PushResult { image: image.clone() })
}
fn logger(&self) -> &dyn Logger {
self.logger.borrow()
}

41
src/container_registry/mod.rs
View File

@@ -1,4 +1,5 @@
use serde::{Deserialize, Serialize};
use url::Url;
use crate::build_platform::Image;
use crate::errors::EngineError;
@@ -6,8 +7,6 @@ use crate::events::{EnvironmentStep, EventDetails, Stage, ToTransmitter};
use crate::logger::Logger;
use crate::models::{Context, Listen, QoveryIdentifier};
pub mod docker;
pub mod docker_hub;
pub mod docr;
pub mod ecr;
pub mod scaleway_container_registry;
@@ -21,14 +20,26 @@ pub trait ContainerRegistry: Listen + ToTransmitter {
format!("{} ({})", self.name(), self.id())
}
fn is_valid(&self) -> Result<(), EngineError>;
fn on_create(&self) -> Result<(), EngineError>;
fn on_create_error(&self) -> Result<(), EngineError>;
fn on_delete(&self) -> Result<(), EngineError>;
fn on_delete_error(&self) -> Result<(), EngineError>;
// Login into the registry and setup everything for it
// mainly getting creds and calling docker login behind the hood
// It is poart of the ContainerRegistry only because DigitalOcean require to call doctl
// and that we can't get credentials directly
fn login(&self) -> Result<ContainerRegistryInfo, EngineError>;
// Some provider require specific action in order to allow container registry
// For now it is only digital ocean, that require 2 steps to have registries
fn create_registry(&self) -> Result<(), EngineError>;
// Call to create a specific repository in the registry
// i.e: docker.io/erebe or docker.io/qovery
// All providers requires action for that
// The convention for us is that we create one per application
fn create_repository(&self, repository_name: &str) -> Result<(), EngineError>;
// Check on the registry if a specific image already exist
fn does_image_exists(&self, image: &Image) -> bool;
fn pull(&self, image: &Image) -> Result<PullResult, EngineError>;
fn push(&self, image: &Image, force_push: bool) -> Result<PushResult, EngineError>;
fn push_error(&self, image: &Image) -> Result<PushResult, EngineError>;
fn logger(&self) -> &dyn Logger;
fn get_event_details(&self) -> EventDetails {
let context = self.context();
@@ -44,6 +55,17 @@ pub trait ContainerRegistry: Listen + ToTransmitter {
}
}
pub struct ContainerRegistryInfo {
pub endpoint: Url, // Contains username and password if necessary
pub registry_name: String,
pub registry_docker_json_config: Option<String>,
// give it the name of your image, and it returns the full name with prefix if needed
// i.e: for DigitalOcean => registry_name/image_name
// i.e: fo scaleway => image_name/image_name
// i.e: for AWS => image_name
pub get_image_name: Box<dyn Fn(&str) -> String>,
}
pub struct PushResult {
pub image: Image,
}
@@ -56,7 +78,6 @@ pub enum PullResult {
#[derive(Serialize, Deserialize, Clone, Copy, Debug)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub enum Kind {
DockerHub,
Ecr,
Docr,
ScalewayCr,

399
src/container_registry/scaleway_container_registry.rs
View File

@@ -5,21 +5,15 @@ use std::borrow::Borrow;
use self::scaleway_api_rs::models::scaleway_registry_v1_namespace::Status;
use crate::build_platform::Image;
use crate::container_registry::docker::{
docker_login, docker_manifest_inspect, docker_pull_image, docker_tag_and_push_image,
};
use crate::container_registry::{ContainerRegistry, Kind, PullResult, PushResult};
use crate::cmd::docker;
use crate::cmd::docker::Docker;
use crate::container_registry::{ContainerRegistry, ContainerRegistryInfo, Kind};
use crate::errors::{CommandError, EngineError};
use crate::events::{EngineEvent, EventMessage, ToTransmitter, Transmitter};
use crate::logger::{LogLevel, Logger};
use crate::models::{
Context, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressScope,
};
use crate::models::{Context, Listen, Listener, Listeners};
use crate::runtime::block_on;
use retry::delay::Fibonacci;
use retry::Error::Operation;
use retry::OperationResult;
use rusoto_core::param::ToParam;
use url::Url;
pub struct ScalewayCR {
context: Context,
@@ -29,6 +23,7 @@ pub struct ScalewayCR {
login: String,
secret_token: String,
zone: ScwZone,
docker: Docker,
listeners: Listeners,
logger: Box<dyn Logger>,
}
@@ -43,6 +38,8 @@ impl ScalewayCR {
zone: ScwZone,
logger: Box<dyn Logger>,
) -> ScalewayCR {
let docker = Docker::new(context.docker_tcp_socket().clone()).unwrap(); // FIXME: remove unwrap
ScalewayCR {
context,
id: id.to_string(),
@@ -51,6 +48,7 @@ impl ScalewayCR {
login: "nologin".to_string(),
secret_token: secret_token.to_string(),
zone,
docker,
listeners: Vec::new(),
logger,
}
@@ -66,16 +64,9 @@ impl ScalewayCR {
}
}
fn get_docker_envs(&self) -> Vec<(&str, &str)> {
match self.context.docker_tcp_socket() {
Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())],
None => vec![],
}
}
pub fn get_registry_namespace(
&self,
image: &Image,
namespace_name: &str,
) -> Option<scaleway_api_rs::models::ScalewayRegistryV1Namespace> {
// https://developers.scaleway.com/en/products/registry/api/#get-09e004
let scaleway_registry_namespaces = match block_on(scaleway_api_rs::apis::namespaces_api::list_namespaces(
@@ -86,7 +77,7 @@ impl ScalewayCR {
None,
None,
Some(self.default_project_id.as_str()),
image.registry_name.as_deref(),
Some(namespace_name),
)) {
Ok(res) => res.namespaces,
Err(e) => {
@@ -96,7 +87,7 @@ impl ScalewayCR {
self.get_event_details(),
EventMessage::new(
"Error while interacting with Scaleway API (list_namespaces).".to_string(),
Some(format!("error: {}, image: {}", e, &image.name)),
Some(format!("error: {}, image: {}", e, namespace_name)),
),
),
);
@@ -127,7 +118,7 @@ impl ScalewayCR {
None,
None,
None,
Some(image.name.as_str()),
Some(image.name().as_str()),
None,
Some(self.default_project_id.as_str()),
)) {
@@ -139,7 +130,7 @@ impl ScalewayCR {
self.get_event_details(),
EventMessage::new(
"Error while interacting with Scaleway API (list_namespaces).".to_string(),
Some(format!("error: {}, image: {}", e, &image.name)),
Some(format!("error: {}, image: {}", e, &image.name())),
),
),
);
@@ -168,7 +159,7 @@ impl ScalewayCR {
if image_to_delete.is_none() {
let err = EngineError::new_container_registry_image_doesnt_exist(
event_details.clone(),
image.name.to_string(),
image.name().to_string(),
None,
);
@@ -188,7 +179,7 @@ impl ScalewayCR {
Err(e) => {
let err = EngineError::new_container_registry_delete_image_error(
event_details.clone(),
image.name.to_string(),
image.name().to_string(),
Some(CommandError::new(e.to_string(), None)),
);
@@ -199,81 +190,9 @@ impl ScalewayCR {
}
}
fn push_image(&self, dest: String, dest_latest_tag: String, image: &Image) -> Result<PushResult, EngineError> {
// https://www.scaleway.com/en/docs/deploy-an-image-from-registry-to-kubernetes-kapsule/
let event_details = self.get_event_details();
if let Err(e) = docker_tag_and_push_image(
self.kind(),
self.get_docker_envs(),
image,
dest.to_string(),
dest_latest_tag.to_string(),
event_details.clone(),
self.logger(),
) {
return Err(EngineError::new_docker_push_image_error(
event_details,
image.name.to_string(),
dest.to_string(),
e,
));
}
let result = retry::retry(Fibonacci::from_millis(10000).take(10), || {
match self.does_image_exists(image) {
true => OperationResult::Ok(&image),
false => {
self.logger.log(
LogLevel::Warning,
EngineEvent::Warning(
self.get_event_details(),
EventMessage::new_from_safe(
"Image is not yet available on Scaleway Registry Namespace, retrying in a few seconds...".to_string(),
),
),
);
OperationResult::Retry(())
}
}
});
let image_not_reachable = Err(EngineError::new_container_registry_image_unreachable_after_push(
event_details.clone(),
image.name.to_string(),
));
match result {
Ok(_) => Ok(PushResult { image: image.clone() }),
Err(Operation { .. }) => image_not_reachable,
Err(retry::Error::Internal(_)) => image_not_reachable,
}
}
fn pull_image(&self, dest: String, image: &Image) -> Result<PullResult, EngineError> {
let event_details = self.get_event_details();
if let Err(e) = docker_pull_image(
self.kind(),
self.get_docker_envs(),
dest.to_string(),
event_details.clone(),
self.logger(),
) {
return Err(EngineError::new_docker_pull_image_error(
event_details,
image.name.to_string(),
dest.to_string(),
e,
));
}
Ok(PullResult::Some(image.clone()))
}
pub fn create_registry_namespace(
&self,
image: &Image,
namespace_name: &str,
) -> Result<scaleway_api_rs::models::ScalewayRegistryV1Namespace, EngineError> {
let event_details = self.get_event_details();
@@ -282,7 +201,7 @@ impl ScalewayCR {
&self.get_configuration(),
self.zone.region().to_string().as_str(),
scaleway_api_rs::models::inline_object_29::InlineObject29 {
name: image.name.clone(),
name: namespace_name.to_string(),
description: None,
project_id: Some(self.default_project_id.clone()),
is_public: Some(false),
@@ -293,7 +212,7 @@ impl ScalewayCR {
Err(e) => {
let error = EngineError::new_container_registry_namespace_creation_error(
event_details.clone(),
image.name.clone(),
namespace_name.to_string(),
self.name_with_id(),
CommandError::new(e.to_string(), Some("Can't create SCW repository".to_string())),
);
@@ -308,19 +227,15 @@ impl ScalewayCR {
pub fn delete_registry_namespace(
&self,
image: &Image,
namespace_name: &str,
) -> Result<scaleway_api_rs::models::ScalewayRegistryV1Namespace, EngineError> {
// https://developers.scaleway.com/en/products/registry/api/#delete-c1ac9b
let event_details = self.get_event_details();
let registry_to_delete = self.get_registry_namespace(image);
let repository_name = match image.registry_name.as_ref() {
None => "unknown",
Some(name) => name,
};
let registry_to_delete = self.get_registry_namespace(namespace_name);
if registry_to_delete.is_none() {
let error = EngineError::new_container_registry_repository_doesnt_exist(
event_details.clone(),
repository_name.to_string(),
namespace_name.to_string(),
None,
);
@@ -341,7 +256,7 @@ impl ScalewayCR {
Err(e) => {
let error = EngineError::new_container_registry_delete_repository_error(
event_details.clone(),
repository_name.to_string(),
namespace_name.to_string(),
Some(CommandError::new(e.to_string(), None)),
);
@@ -355,23 +270,25 @@ impl ScalewayCR {
pub fn get_or_create_registry_namespace(
&self,
image: &Image,
namespace_name: &str,
) -> Result<scaleway_api_rs::models::ScalewayRegistryV1Namespace, EngineError> {
info!("Get/Create repository for {}", namespace_name);
// check if the repository already exists
let event_details = self.get_event_details();
let registry_namespace = self.get_registry_namespace(&image);
let registry_namespace = self.get_registry_namespace(namespace_name);
if let Some(namespace) = registry_namespace {
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(format!("SCW repository {} already exists", image.name.as_str())),
EventMessage::new_from_safe(format!("SCW repository {} already exists", namespace_name)),
),
);
return Ok(namespace);
}
self.create_registry_namespace(image)
self.create_registry_namespace(namespace_name)
}
fn get_docker_json_config_raw(&self) -> String {
@@ -384,27 +301,6 @@ impl ScalewayCR {
.as_bytes(),
)
}
fn exec_docker_login(&self, registry_url: &String) -> Result<(), EngineError> {
let event_details = self.get_event_details();
if docker_login(
Kind::ScalewayCr,
self.get_docker_envs(),
self.login.clone(),
self.secret_token.clone(),
registry_url.clone(),
event_details.clone(),
self.logger(),
)
.is_err()
{
return Err(EngineError::new_client_invalid_cloud_provider_credentials(
event_details,
));
};
Ok(())
}
}
impl ToTransmitter for ScalewayCR {
@@ -434,218 +330,49 @@ impl ContainerRegistry for ScalewayCR {
Ok(())
}
fn on_create(&self) -> Result<(), EngineError> {
fn login(&self) -> Result<ContainerRegistryInfo, EngineError> {
let event_details = self.get_event_details();
let mut registry = Url::parse(&format!("https://rg.{}.scw.cloud", self.zone.region())).unwrap();
let _ = registry.set_username(&self.login);
let _ = registry.set_password(Some(&self.secret_token));
if self.docker.login(&registry).is_err() {
return Err(EngineError::new_client_invalid_cloud_provider_credentials(
event_details,
));
}
Ok(ContainerRegistryInfo {
endpoint: registry,
registry_name: self.name.to_string(),
registry_docker_json_config: Some(self.get_docker_json_config_raw()),
get_image_name: Box::new(move |img_name| format!("{}/{}", img_name, img_name)),
})
}
fn create_registry(&self) -> Result<(), EngineError> {
// Nothing to do, scaleway managed container registry per repository (aka `namespace` by the scw naming convention)
Ok(())
}
fn on_create_error(&self) -> Result<(), EngineError> {
Ok(())
}
fn on_delete(&self) -> Result<(), EngineError> {
Ok(())
}
fn on_delete_error(&self) -> Result<(), EngineError> {
fn create_repository(&self, name: &str) -> Result<(), EngineError> {
let _ = self.get_or_create_registry_namespace(name)?;
Ok(())
}
fn does_image_exists(&self, image: &Image) -> bool {
let event_details = self.get_event_details();
let registry_url = image
.registry_url
.as_ref()
.unwrap_or(&"undefined".to_string())
.to_param();
if let Err(_) = docker_login(
Kind::ScalewayCr,
self.get_docker_envs(),
self.login.clone(),
self.secret_token.clone(),
registry_url.clone(),
event_details.clone(),
self.logger(),
) {
let info = if let Ok(url) = self.login() {
url
} else {
return false;
}
};
docker_manifest_inspect(
Kind::ScalewayCr,
self.get_docker_envs(),
image.name.clone(),
image.tag.clone(),
registry_url,
event_details.clone(),
self.logger(),
)
.is_ok()
}
fn pull(&self, image: &Image) -> Result<PullResult, EngineError> {
let event_details = self.get_event_details();
let listeners_helper = ListenersHelper::new(&self.listeners);
let mut image = image.clone();
let registry_url: String;
match self.get_or_create_registry_namespace(&image) {
Ok(registry) => {
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(format!(
"Scaleway registry namespace for {} has been created",
image.name.as_str()
)),
),
);
image.registry_name = Some(image.name.clone()); // Note: Repository namespace should have the same name as the image name
image.registry_url = registry.endpoint.clone();
image.registry_secret = Some(self.secret_token.clone());
image.registry_docker_json_config = Some(self.get_docker_json_config_raw());
registry_url = registry.endpoint.unwrap_or_else(|| "undefined".to_string());
}
Err(e) => {
self.logger.log(LogLevel::Error, EngineEvent::Error(e.clone(), None));
return Err(e);
}
}
if !self.does_image_exists(&image) {
let info_message = format!("Image {:?} does not exist in SCR {} repository", image, self.name());
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(info_message.to_string()),
),
);
listeners_helper.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
return Ok(PullResult::None);
}
let info_message = format!("pull image {:?} from SCR {} repository", image, self.name());
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(info_message.to_string()),
),
);
listeners_helper.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
let _ = self.exec_docker_login(&registry_url)?;
let dest = format!("{}/{}", registry_url, image.name_with_tag());
// pull image
self.pull_image(dest, &image)
}
fn push(&self, image: &Image, force_push: bool) -> Result<PushResult, EngineError> {
let event_details = self.get_event_details();
let mut image = image.clone();
let registry_url: String;
let registry_name: String;
match self.get_or_create_registry_namespace(&image) {
Ok(registry) => {
image.registry_name = Some(image.name.clone()); // Note: Repository namespace should have the same name as the image name
image.registry_url = registry.endpoint.clone();
image.registry_secret = Some(self.secret_token.clone());
image.registry_docker_json_config = Some(self.get_docker_json_config_raw());
registry_url = registry.endpoint.unwrap_or_else(|| "undefined".to_string());
registry_name = registry.name.unwrap();
}
Err(e) => {
self.logger.log(LogLevel::Error, EngineEvent::Error(e.clone(), None));
return Err(e);
}
}
let _ = self.exec_docker_login(&registry_url)?;
let dest = format!("{}/{}", registry_url, image.name_with_tag());
let listeners_helper = ListenersHelper::new(&self.listeners);
if !force_push && self.does_image_exists(&image) {
// check if image does exist - if yes, do not upload it again
let info_message = format!(
"image {} found on Scaleway {} repository, container build is not required",
image, registry_name,
);
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(info_message.to_string()),
),
);
listeners_helper.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
return Ok(PushResult { image: image.clone() });
}
let info_message = format!(
"image {} does not exist on Scaleway {} repository, starting image upload",
image,
self.name()
);
self.logger.log(
LogLevel::Info,
EngineEvent::Info(
event_details.clone(),
EventMessage::new_from_safe(info_message.to_string()),
),
);
listeners_helper.deployment_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
let dest_latest_tag = format!("{}/{}:latest", registry_url, image.name);
self.push_image(dest, dest_latest_tag, &image)
}
fn push_error(&self, image: &Image) -> Result<PushResult, EngineError> {
Ok(PushResult { image: image.clone() })
let image = docker::ContainerImage {
registry: info.endpoint,
name: image.name().clone(),
tags: vec![image.tag.clone()],
};
self.docker.does_image_exist_remotely(&image).is_ok()
}
fn logger(&self) -> &dyn Logger {

2
src/errors/io.rs
View File

@@ -96,6 +96,7 @@ pub enum Tag {
BuilderBuildpackCannotBuildContainerImage,
BuilderGetBuildError,
BuilderCloningRepositoryError,
DockerError,
DockerPushImageError,
DockerPullImageError,
BuilderDockerCannotListImages,
@@ -206,6 +207,7 @@ impl From<errors::Tag> for Tag {
errors::Tag::ContainerRegistryRepositoryDoesntExist => Tag::ContainerRegistryRepositoryDoesntExist,
errors::Tag::ContainerRegistryDeleteRepositoryError => Tag::ContainerRegistryDeleteRepositoryError,
errors::Tag::BuilderDockerCannotListImages => Tag::BuilderDockerCannotListImages,
errors::Tag::DockerError => Tag::DockerError,
}
}
}

21
src/errors/mod.rs
View File

@@ -4,6 +4,7 @@ extern crate url;
use crate::cloud_provider::utilities::VersionsNumber;
use crate::cmd;
use crate::cmd::docker::DockerError;
use crate::cmd::helm::HelmError;
use crate::error::{EngineError as LegacyEngineError, EngineErrorCause, EngineErrorScope};
use crate::events::{EventDetails, GeneralStep, Stage, Transmitter};
@@ -253,6 +254,8 @@ pub enum Tag {
BuilderGetBuildError,
/// BuilderCloningRepositoryError: represents an error when builder is trying to clone a git repository.
BuilderCloningRepositoryError,
/// DockerError: represents an error when trying to use docker cli.
DockerError,
/// DockerPushImageError: represents an error when trying to push a docker image.
DockerPushImageError,
/// DockerPullImageError: represents an error when trying to pull a docker image.
@@ -2286,6 +2289,24 @@ impl EngineError {
)
}
/// Creates new error from an Docker error
///
/// Arguments:
///
/// * `event_details`: Error linked event details.
/// * `error`: Raw error message.
pub fn new_docker_error(event_details: EventDetails, error: DockerError) -> EngineError {
EngineError::new(
event_details,
Tag::DockerError,
error.to_string(),
error.to_string(),
None,
None,
None,
)
}
/// Creates new error when trying to push a Docker image.
///
/// Arguments:

2
src/git.rs
View File

@@ -313,7 +313,7 @@ mod tests {
fn test_git_submodule_with_ssh_key() {
// Unique Key only valid for the submodule and in read access only
// https://github.com/Qovery/dumb-logger/settings/keys
let ssh_key = String::from_utf8(base64::decode("LS0tLS1CRUdJTiBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0NCmIzQmxibk56YUMxclpYa3RkakVBQUFBQUJHNXZibVVBQUFBRWJtOXVaUUFBQUFBQUFBQUJBQUFCbHdBQUFBZHpjMmd0Y24NCk5oQUFBQUF3RUFBUUFBQVlFQTFGcS95ZGF6dU84T3ZRdjVUNEdxbndOMjhZV0EzaXlqanREMFdSQXhtdDZEV3lJRlVYZ1gNClZFZ1ZVYnZyYndKNGJQa0tTbkdqd1hZRUdJYkdYa0hKUTdvWTVSMnB6b1hqUkVYTzIzZEZ2aVp4bUpOcVdEVVJqSHhjc1INCndOYWxiOFVZZVBCRVI4TEQzWWpQd0lYNXdCWm5VSjZLWTJFbXhjSlBVUnV4bUlyTjI4QndiZ3FiejJPU3NJdWg4a1ZwSngNCldheitFc3JNM282NHpHMm0wa0dxMVI1VHE0enBPRWliUk1iY1ZXTldKUzRZR29JczdsRzB0ZHZndktNRnJsWktzSUw1Y2ENCkFOQzRXTlROMm1DVVFrVGpGSDVySDlDa0ZBZjZaZ0lqYklvN0s3TTc0L1B5RVhEcStyRW5vRWdzeEkzRi9NZHMydGM2RWkNClJaY2JrUmRLVnpaUzJCMXdKNDhrOGR3Sml5VytKSWY4ejEzK2FiUXVPNGR5MWRnM2gwbEZ6dm9qaVYxTjNBRXdHcmhjZEUNClo3TXNaeThKM3JvRElZSWZCczdkbmh2T1FrME1taEpKSEpMaVlEZWZCYUk4MVdGTGlqekUxejhqMG90cExlNkt0SVhQYk8NCmV5WWdod0U2aDlhSmNrOEU3WklYMjc4MGRQMW93T2g1dC9VaE0vdjFBQUFGZ082eU9GenVzamhjQUFBQUIzTnphQzF5YzINCkVBQUFHQkFOUmF2OG5XczdqdkRyMEwrVStCcXA4RGR2R0ZnTjRzbzQ3UTlGa1FNWnJlZzFzaUJWRjRGMVJJRlZHNzYyOEMNCmVHejVDa3B4bzhGMkJCaUd4bDVCeVVPNkdPVWRxYzZGNDBSRnp0dDNSYjRtY1ppVGFsZzFFWXg4WExFY0RXcFcvRkdIancNClJFZkN3OTJJejhDRitjQVdaMUNlaW1OaEpzWENUMUVic1ppS3pkdkFjRzRLbTg5amtyQ0xvZkpGYVNjVm1zL2hMS3pONk8NCnVNeHRwdEpCcXRVZVU2dU02VGhJbTBURzNGVmpWaVV1R0JxQ0xPNVJ0TFhiNEx5akJhNVdTckNDK1hHZ0RRdUZqVXpkcGcNCmxFSkU0eFIrYXgvUXBCUUgrbVlDSTJ5S095dXpPK1B6OGhGdzZ2cXhKNkJJTE1TTnhmekhiTnJYT2hJa1dYRzVFWFNsYzINClV0Z2RjQ2VQSlBIY0NZc2x2aVNIL005ZC9tbTBManVIY3RYWU40ZEpSYzc2STRsZFRkd0JNQnE0WEhSR2V6TEdjdkNkNjYNCkF5R0NId2JPM1o0YnprSk5ESm9TU1J5UzRtQTNud1dpUE5WaFM0bzh4TmMvSTlLTGFTM3VpclNGejJ6bnNtSUljQk9vZlcNCmlYSlBCTzJTRjl1L05IVDlhTURvZWJmMUlUUDc5UUFBQUFNQkFBRUFBQUdCQUxhR1pqRkwvV0NwQWtjV0lxM25LMHZRZzQNCjBuamxQcGxKQXVKTWprOVc1RGNpNkQrSVJGTC9BK29TeUcxTit2Qk9uTnliMmhIZnNzd0dxQWRjTVEwcmtISFZ6WitWbk4NCmxVSGFxdW5UQkR4aitPSUhXN0lEczFqSWtEZWZnQngyTmh5eDR3anRBTHBhVW1ja1B1SkhTcURSV3JvQkc1c01Uc3RwWmwNCnNtb0diTmxFK0o1dE9lMnhqYVYzNzdRNVd4L0FIemd0T09RemZNL3lTZjMzTDhCS1Y0a3J4eXV3ZW95T1Q5OU9ia0ltaUUNCnpTMEQxVERuUStmSTNjdm1aL3lvcDZ0clA0a01wdWtWdC93ZUhFWU5nZkdPdHVHMndwU3oyRmpNcUcyT1NFd3ZpRXM3U0YNCmlwTGNWc2dpUzg3ckI5ZFBRejFYTGhhdW9MTDliY3BlOE9sZW50VkI5VHFaU1lqaTJoeUNtZG5id25CS2QyMGVaUlh0S3QNCnh3SUpDdkpESGwyWk9wTVVUcnIydFcwSkVFZU1QSDJWMCs4amg3aGxlQ0NLcDhmdE1pcGVuWTdvelR1M1JVTUdNcjB4eTINCmhUalVJNkVGU0ppVGlKVE9ibGVhcGVPMVE1czdHaU5ibmdZQXFhN3h3RmJuYllrODJ3ekxPbzdEUjYzODhJbzVQcEFRQUENCkFNQUtXbURSMWU5bXlncm8wZmtQUDQ3dGsxMnF5bWpkQzVtRU1SNm9TOTNMbGRaK1ptKzBxVlBxN1BSQ3JPZlpLcFJSQ1UNCmJOUkM0ZFJhUHk0ek85cEdqdzE3ZlhjUGxGQzRaQUN1anhnRzhvazdYNEdGVlZEQ2lySFRySFhWN0ozNUtPMnR5MloyR2UNCms2L0dhMUpCMlBLN0tJZFlnMWpjY3lUR0FsZTlmcjIyU21nZHVoUmt2WlZsVU9mMHp2ZDhERzlVcktYUURWTERHd1QrWlkNClp2ODhYdGduZzZneU1jZXhZaHZZY04yMUo4ay9wNmM1ZGVuUXNNL0QxN0Qyck9iNE1BQUFEQkFPcDBJWitTVWxXY0xzbjMNCmVwQk1pTVAwdm5LUTI4UUd4NDl1bW14VXdhMTI0djk5YzhtTXZ5TXJPYnFsODdjZjQwWTlqdUhsSGZKSzd0MXhNdE5qU3QNCkJWRlNjU2E5Sk56S0hKRTJaYlJma1d1ZXpScytGbytKcjU0YVppQjNvcjNFeUtaamNZY2RFTG5ROHNjNmJXd25Ic29WSHkNCmNpTThtcUhudHRqeXJPZFdJRi9CTURlYjF5WkliYlQ0aWN3Y1N2TEJOVE95dllwakg1RWNsTXdXcWlsQ2NxVVJyTmtZVXMNCnJWZkFabDZuUmE5N0FNNDd6THhBT0RZT1FzbjZhdk5RQUFBTUVBNTk2ejRYZkxrQ09MT3drUi85NS90WEYzS3p4MjFsdC8NCllBVExmRlBKbHdNaGRxN1d2VG9LZWxNV0QwNUxXYlZxYitNOGU3SWZSQlducEp0V1RxMVBCY3ltT2k1TkprSmZnWWhqdGgNCjlqT1k4WTVCWWlvcENRUUFtTWc3SHF3a0xUSUdUU25IdDN5ZGFTK21TaVFTQUhLb1VKbmp4cEdLQ3ZyVGk5eHdxTFpZT1YNClZvOHFCZ003M1c1TWUyQWI0YnpPaEt4Tm9iTFpqWkxqZDJoeHRyWENJaityRXVRa09NT1hGTmR6NkFDR0hwQ09KTGp4clUNCmk4TGNwd2c5NlpWZkhCQUFBQUNtVnlaV0psUUhOMGVYZz0NCi0tLS0tRU5EIE9QRU5TU0ggUFJJVkFURSBLRVktLS0tLQ==").unwrap()).unwrap();
let ssh_key = String::from_utf8(base64::decode("LS0tLS1CRUdJTiBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0KYjNCbGJuTnphQzFyWlhrdGRqRUFBQUFBQkc1dmJtVUFBQUFFYm05dVpRQUFBQUFBQUFBQkFBQUFNd0FBQUF0emMyZ3RaVwpReU5UVXhPUUFBQUNBTzZlaGNrV0JrNlcwd3lTZ0FIY0dSY3JneW1IVThqRWVKRm5yQ2k1ZjZaQUFBQUpERlV0TVZ4VkxUCkZRQUFBQXR6YzJndFpXUXlOVFV4T1FBQUFDQU82ZWhja1dCazZXMHd5U2dBSGNHUmNyZ3ltSFU4akVlSkZuckNpNWY2WkEKQUFBRUQ0aGwvTmk0aGgvK3oxUm4wdWtMcm5mQ0xrN1BUWmErbVNQYk01ZS9aS0pnN3A2RnlSWUdUcGJUREpLQUFkd1pGeQp1REtZZFR5TVI0a1dlc0tMbC9wa0FBQUFDbVZ5WldKbFFITjBlWGdCQWdNPQotLS0tLUVORCBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0K").unwrap()).unwrap();
let invalid_ssh_key = String::from_utf8(base64::decode("LS0tLS1CRUdJTiBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0KYjNCbGJuTnphQzFyWlhrdGRqRUFBQUFBQ21GbGN6STFOaTFqZEhJQUFBQUdZbU55ZVhCMEFBQUFHQUFBQUJCNzZzbWIzVgp5WFB3SE12dm8zWTB5M0FBQUFFQUFBQUFFQUFBR1hBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFCZ1FDOVZHbm13cjZCClRHdWxzODhEaXRXaE5IUUoxMjV0eGxHa2EzNDNxUVB2S3dSc2VxN05SdFAzY2IxbDRMZytzdWozZ0lQYU5yM295SlBoRDIKZmIxbzF1cUFiOStkbWhwQXc4L1lCa05NZkRrdDRTWEpGZjZ3dUZwa1p4SHF3czNZUXF6cjhicVJaaHA0bXlnc2VwNFVHOApBaGxVMG5CUXFBREFhS3dBcmpLeUdBeWwwenRDYVdObm9sOVRZSmZuNEpOQW5YUDFONmMxMUVaRm5wKzJsMTVoSVdNd2NKClpCMnFFeTFSZzFVNXpuOVNSOURIVXhvN2p0ZkkrdWJWbHdnelBQaDVjZzAydVc0K0JwcFg1UGlpZ04rQlBNajc3WEJ0VTQKZzU3MmRDZHBSRjk3NjJ5SDBsY21nSkRqVnhnOTludVVGRDlwVG9nUTRrUENrdUluNmcxS3JObFdqY1R2c1hFS2JVS0xqawpkQkR2Yk1tbzZBaHJXRFhDSjZqRUN0T2Jka29XMGVjTGU4cXB3Nmh5N1NmdWppSm9QbnVsazRWenMwR2xPa3VPU0JIUmhJClhSc25NaFNiNnh2dDl6QldJcklvZDZoWnhuQ0V2SWRESzlacVBnOXJpbXc4bG8rUkFwdm1ySnRINUhsbFJiYWh4K2RUU1cKM2hCa1BlMnNDL1UvRUFBQVdBVXBEOTFIQTAzSnQyNFFSSFVXRDAvVTJGMTBzZE5WN0w4bkhMeVNibFBnSFhMc3lpSTFxOQo0NXBOUEQyNElBakNzQ08rVHREcXc3MDhlNXliUWhXUCsybkxtdGQwclEyTXh3SnZwUjlGcEV6UDFyejRYUDVUbzZDN3N1CmZpd0JPZWd6bjhQT1hGSmRvRk9Ud3E3dWhaM201NE93NHZvZkFKSHdtYWtwTGZMd2R1TnQ3S1RNQkVpT3VlM0ZXTGtCR0wKQUE1RGtoYVlpVGgyajB2YU9jUWhxZVphVEp6V2tidUcvb29DK1cwcTVXcFNZdFlxREFhWEh0bG8rZGtOMFEzZVVhcm1FTQpGcy9tdEpha3dhOVhCMVgzMndKbUpIdmN0OG4vVzA1T0N5V0U1Y2szeitRQVB3a2pGK0hKOGlOZDluVk5zckx1T010a2VQCk1aMTZreTg5WUVSZVQ1QXRJU1lRd0JQU2tsTFZKL3VaOCszK2Vyc3JrOW1aakw3ZXpISnV4ZysxUmR1T3BPeWpXMTRoTGYKblJQTDlKOXgvZWZ2MFV0L3BpR3M5NEFRcFFVZnJFdXpjL1dmejRocUtzVUxnT0VnblZBWXpuSksyWHJGeTN4aWlKVkFVUQpZcm4xak9lU1oyTWV0cjJvd05VdVM3cEhGTHZIWURRWklURmxVaFlOYUx0ejV5WU9HTCtFbEVxQm4wT1FFenNESDhROEpFCk5jWGVxUjFRTE4rTUJaMFZqQ2Q3T0ExTGpXZVVrdjNMaFJER3lPS3RjWk5OeFl5MkgwRWlmYzIvRHpLMnlpcVRQWUdMbHYKOWhZTlZZcC8xOGxhUkFOL040MlVDMjRmS0hFZ2lYVTNnL3RCZkZmbEFBWThKSE9sQUJEdXFWYjJkWHZKdXFLeUJMUElqVQo5cVl5VXNOVXhWS2M2ZWh4VU4wcVlnTmV2Z0JmMXVSZkxCY2c3SjVJVDZQQ2dSa3lNenBRakY1RkhuM0J6SVMrb3ZFSnNaCk5LNklYbDJIY3FncExTWUFkTFZlZEZOUzlkVU01blpMdlJEMjkyc0FQWm5aaU91Z3pwSWNrMllFcXpscjc2NXlUakRJdWgKR3kvdFlBQ3FIZHV4S2pMdGc0OXpjZjdNN2xESGNuVEY1MlJsazEyR2x1emZGK1dhZDF3eUFKVnNyUmtqVFZYVHhnTEV6MQo4SzF0WUtVOWoyc3grUE1Vd0JxM3lQR2lTaEgydWp6em82SUc1cnVYSTAwZXVkT2t1NVVrSHhBVnJneUI1S0M2VFRMR1BYCnhQMFN5Zk12dXJycDdvMnhsK2dkSVc0c0dudEJ2V0RHRVFSY0RxbWdLV0tuNTNsbmg5U1Urcmh2UkdhRFJueENuYkNwUEUKTE82V0lKUXVPQm54bzhWcGU0R2JLc2NmSktKSzlZV2ZIOFEvYzBncnE0ZDh5ZmRwUG1uc3hHOEpoTFVuMEhpRFEzQytaMgpzU1RPeU85TDAySUZIdDdIUEY2OWRWR3c3M0pPU1FiL05GK2g5cGRVazBScGNRdGFaTm9TMHg2a3RCQXljK0o0VUpUYTliCkdENWRaSE1KVHBvcWFZUDV0dFlnMjlBQkpUUURMa0tnbWxWRGNtK28zRTN3cTlySWFXMlhpNDQrc3RnTVJVS1J5R041d1EKM2xTWjk1QXBpWFlpRkNONUVrWitUci96TDAraVdwUHRCRzlJZmlGbmlqVlVYUnpEWHZxeGE1QTQ1YUlNWDhad2U5ckxFdAphaVRaOUI5d2tVb0tYdXlDU3plQXhMTGU2aG8wLzBDbmhSR3NoVGg1UDd6aFA4bVExRGZMYlFCRU0zOHJMWlplMExVVVhZCkZpZkFXc3BFRDk2VjBMckhxRkd0Z0dzd1NQcWRBRzBPTDBWekRUbFRucDJVWDY0SEhjUzF2MUMyQnNxbllWbkJNL3p5aUYKQXhabDB4cGRPUVVuKzV2V2VHUXZsQkhGeU0vQmtXRVhMbjc1YVNQL3JwcnlZeGdOeWx2M2NiRWNYZXoyWXdLM2UrN1NnZAoxRzFZUVVtNStqNy90Q0x5aFluL1VjRzJhTHJNc3pRY1FoWTE4Sk9IOXF6a2FacWdYckFybnE0dWluT25sbFBKaGJ3ZTVrCmgvMmdyTlVqbEsrRHYxQ2dGZUVDcm9yRHo4L3ZxZW1QNXdVWWF5bFNWWVZ3UHM1bkxDQWUrVlNobFlIOXlNb3JwanNXc3MKYlg0UlAvVGd3TmNtRnBuZ21kTXppNmtIUXhSc2pUT3VxZ3Vsb01FUVZmQ3JkNGxBeWp3eVhRaEcrd2dWMXBuempCZlR4eQpZeFBrc1VGaTg3aEVkZ1RPZ2M5MHlNamVoVGhHOGRMWGEvd0NOU0hLZ1pBbFBZbWdLd2ZvcFlBMjQxdUlxR2J0WUtqSTFSCnVHU2JqSU80dUVYbkJ5eWVZTnA3Z29iR2NVc1BGV0doY1FPV05QZnl5K1crQ0xhKzVpYkJCZEF2NStVdlZZUHFGMHhTNy8KUm1TbW9BPT0KLS0tLS1FTkQgT1BFTlNTSCBQUklWQVRFIEtFWS0tLS0t").unwrap()).unwrap();
let clone_dir = DirectoryForTests::new_with_random_suffix("/tmp/engine_test_submodule".to_string());
let get_credentials = |user: &str| {

81
src/models.rs
View File

@@ -2,16 +2,15 @@ use std::collections::BTreeMap;
use std::fmt::{Display, Formatter};
use std::hash::Hash;
use std::net::Ipv4Addr;
use std::path::Path;
use std::str::FromStr;
use std::sync::Arc;
use chrono::{DateTime, Utc};
use git2::{Cred, CredentialType, Error};
use itertools::Itertools;
use rand::distributions::Alphanumeric;
use rand::Rng;
use serde::{Deserialize, Serialize};
use url::Url;
use crate::build_platform::{Build, BuildOptions, Credentials, GitRepository, Image, SshKey};
use crate::cloud_provider::aws::databases::mongodb::MongoDB;
@@ -22,7 +21,7 @@ use crate::cloud_provider::service::{DatabaseOptions, StatefulService, Stateless
use crate::cloud_provider::utilities::VersionsNumber;
use crate::cloud_provider::CloudProvider;
use crate::cloud_provider::Kind as CPKind;
use crate::git;
use crate::container_registry::ContainerRegistryInfo;
use crate::logger::Logger;
use crate::utilities::get_image_tag;
@@ -102,17 +101,14 @@ impl Environment {
pub fn to_qe_environment(
&self,
context: &Context,
built_applications: &Vec<Box<dyn crate::cloud_provider::service::Application>>,
cloud_provider: &dyn CloudProvider,
container_registry: &ContainerRegistryInfo,
logger: Box<dyn Logger>,
) -> crate::cloud_provider::environment::Environment {
let applications = self
.applications
.iter()
.map(|x| match built_applications.iter().find(|y| x.id.as_str() == y.id()) {
Some(app) => x.to_stateless_service(context, app.image().clone(), cloud_provider, logger.clone()),
_ => x.to_stateless_service(context, x.to_image(), cloud_provider, logger.clone()),
})
.map(|x| x.to_stateless_service(context, x.to_image(container_registry), cloud_provider, logger.clone()))
.filter(|x| x.is_some())
.map(|x| x.unwrap())
.collect::<Vec<_>>();
@@ -365,52 +361,24 @@ impl Application {
}
}
pub fn to_image(&self) -> Image {
self.to_image_with_commit(&self.commit_id)
}
pub fn to_image_from_parent_commit<P>(&self, clone_repo_into_dir: P) -> Result<Option<Image>, Error>
where
P: AsRef<Path>,
{
let parent_commit_id = git::get_parent_commit_id(
self.git_url.as_str(),
self.commit_id.as_str(),
clone_repo_into_dir,
&|_| match &self.git_credentials {
None => vec![],
Some(creds) => vec![(
CredentialType::USER_PASS_PLAINTEXT,
Cred::userpass_plaintext(creds.login.as_str(), creds.access_token.as_str()).unwrap(),
)],
},
)?;
Ok(match parent_commit_id {
Some(id) => Some(self.to_image_with_commit(&id)),
None => None,
})
}
pub fn to_image_with_commit(&self, commit_id: &String) -> Image {
pub fn to_image(&self, cr_info: &ContainerRegistryInfo) -> Image {
Image {
application_id: self.id.clone(),
name: self.name.clone(),
name: (cr_info.get_image_name)(&self.name),
tag: get_image_tag(
&self.root_path,
&self.dockerfile_path,
&self.environment_vars,
commit_id,
&self.commit_id,
),
commit_id: self.commit_id.clone(),
registry_name: None,
registry_secret: None,
registry_url: None,
registry_docker_json_config: None,
registry_name: cr_info.registry_name.clone(),
registry_url: cr_info.endpoint.clone(),
registry_docker_json_config: cr_info.registry_docker_json_config.clone(),
}
}
pub fn to_build(&self) -> Build {
pub fn to_build(&self, registry_url: &ContainerRegistryInfo) -> Build {
// Retrieve ssh keys from env variables
const ENV_GIT_PREFIX: &str = "GIT_SSH_KEY";
let env_ssh_keys: Vec<(String, String)> = self
@@ -471,7 +439,7 @@ impl Application {
root_path: self.root_path.clone(),
buildpack_language: self.buildpack_language.clone(),
},
image: self.to_image(),
image: self.to_image(registry_url),
options: BuildOptions {
environment_variables: self
.environment_vars
@@ -1159,7 +1127,7 @@ pub struct Context {
workspace_root_dir: String,
lib_root_dir: String,
test_cluster: bool,
docker_host: Option<String>,
docker_host: Option<Url>,
features: Vec<Features>,
metadata: Option<Metadata>,
}
@@ -1172,13 +1140,13 @@ pub enum Features {
// trait used to reimplement clone without same fields
// this trait is used for Context struct
pub trait Clone2 {
pub trait CloneForTest {
fn clone_not_same_execution_id(&self) -> Self;
}
// for test we need to clone context but to change the directory workspace used
// to to this we just have to suffix the execution id in tests
impl Clone2 for Context {
impl CloneForTest for Context {
fn clone_not_same_execution_id(&self) -> Context {
let mut new = self.clone();
let suffix = rand::thread_rng()
@@ -1199,7 +1167,7 @@ impl Context {
workspace_root_dir: String,
lib_root_dir: String,
test_cluster: bool,
docker_host: Option<String>,
docker_host: Option<Url>,
features: Vec<Features>,
metadata: Option<Metadata>,
) -> Self {
@@ -1236,8 +1204,8 @@ impl Context {
self.lib_root_dir.as_str()
}
pub fn docker_tcp_socket(&self) -> Option<&String> {
self.docker_host.as_ref()
pub fn docker_tcp_socket(&self) -> &Option<Url> {
&self.docker_host
}
pub fn metadata(&self) -> Option<&Metadata> {
@@ -1276,16 +1244,6 @@ impl Context {
}
}
pub fn docker_build_options(&self) -> Option<Vec<String>> {
match &self.metadata {
Some(meta) => meta
.docker_build_options
.clone()
.map(|b| b.split(' ').map(|x| x.to_string()).collect()),
_ => None,
}
}
// Qovery features
pub fn is_feature_enabled(&self, name: &Features) -> bool {
for feature in &self.features {
@@ -1303,7 +1261,6 @@ impl Context {
pub struct Metadata {
pub dry_run_deploy: Option<bool>,
pub resource_expiration_in_seconds: Option<u32>,
pub docker_build_options: Option<String>,
pub forced_upgrade: Option<bool>,
pub disable_pleco: Option<bool>,
}
@@ -1312,14 +1269,12 @@ impl Metadata {
pub fn new(
dry_run_deploy: Option<bool>,
resource_expiration_in_seconds: Option<u32>,
docker_build_options: Option<String>,
forced_upgrade: Option<bool>,
disable_pleco: Option<bool>,
) -> Self {
Metadata {
dry_run_deploy,
resource_expiration_in_seconds,
docker_build_options,
forced_upgrade,
disable_pleco,
}

240
src/transaction.rs
View File

@@ -1,10 +1,7 @@
use std::collections::HashMap;
use std::thread;
use crate::build_platform::BuildResult;
use crate::cloud_provider::kubernetes::Kubernetes;
use crate::cloud_provider::service::{Application, Service};
use crate::container_registry::PushResult;
use crate::cloud_provider::service::Service;
use crate::engine::EngineConfig;
use crate::errors::{EngineError, Tag};
use crate::events::{EngineEvent, EventMessage};
@@ -102,129 +99,48 @@ impl<'a> Transaction<'a> {
Ok(())
}
fn load_build_app_cache(&self, app: &crate::models::Application) -> Result<(), EngineError> {
let container_registry = self.engine.container_registry();
let mut image = app.to_image();
image.tag = String::from("latest");
// pull image from container registry
// FIXME: if one day we use something else than LocalDocker to build image
// FIXME: we'll need to send the PullResult to the Build implementation
let _ = match container_registry.pull(&image) {
Ok(pull_result) => pull_result,
Err(err) => {
self.logger.log(
LogLevel::Error,
EngineEvent::Error(
err.clone(),
Some(EventMessage::new_from_safe(
"Something goes wrong while pulling image from container registry".to_string(),
)),
),
);
return Err(err);
}
};
Ok(())
}
fn build_applications(
fn build_and_push_applications(
&self,
environment: &Environment,
option: &DeploymentOption,
) -> Result<Vec<Box<dyn Application>>, EngineError> {
) -> Result<(), EngineError> {
// do the same for applications
let apps_to_build = environment
.applications
.iter()
// build only applications that are set with Action: Create
.filter(|app| app.action == Action::Create);
let application_and_result_tuples = apps_to_build
.map(|app| {
let image = app.to_image();
let build_result = if option.force_build || !self.engine.container_registry().does_image_exists(&image)
{
// If an error occurred we can skip it. It's not critical.
let _ = self.load_build_app_cache(app);
// only if the build is forced OR if the image does not exist in the registry
self.engine
.build_platform()
.build(app.to_build(), option.force_build, &self.is_transaction_aborted)
} else {
// use the cache
Ok(BuildResult::new(app.to_build()))
};
(app, build_result)
})
.filter(|app| app.action == Action::Create)
.collect::<Vec<_>>();
let mut applications: Vec<Box<dyn Application>> = Vec::with_capacity(application_and_result_tuples.len());
for (application, result) in application_and_result_tuples {
// catch build error, can't do it in Fn
let build_result = match result {
Err(err) => {
error!("build error for application {}: {:?}", application.id.as_str(), err);
return Err(err);
}
Ok(build_result) => build_result,
};
if let Some(app) = application.to_application(
self.engine.context(),
&build_result.build.image,
self.engine.cloud_provider(),
self.logger.clone(),
) {
applications.push(app)
}
// If nothing to build, do nothing
if apps_to_build.is_empty() {
return Ok(());
}
Ok(applications)
}
// Do setup of registry and be sure we are login to the registry
let cr_registry = self.engine.container_registry();
let _ = cr_registry.create_registry()?;
let registry = self.engine.container_registry().login()?;
fn push_applications(
&self,
applications: Vec<Box<dyn Application>>,
option: &DeploymentOption,
) -> Result<Vec<(Box<dyn Application>, PushResult)>, EngineError> {
let application_and_push_results: Vec<_> = applications
.into_iter()
.map(|mut app| {
match self.engine.container_registry().push(app.image(), option.force_push) {
Ok(push_result) => {
// I am not a big fan of doing that but it's the most effective way
app.set_image(push_result.image.clone());
Ok((app, push_result))
}
Err(err) => Err(err),
}
})
.collect();
for app in apps_to_build.into_iter() {
let app_build = app.to_build(&registry);
let mut results: Vec<(Box<dyn Application>, PushResult)> = vec![];
for result in application_and_push_results.into_iter() {
match result {
Ok(tuple) => results.push(tuple),
Err(err) => {
self.logger.log(
LogLevel::Error,
EngineEvent::Error(
err.clone(),
Some(EventMessage::new_from_safe("Error pushing docker image".to_string())),
),
);
return Err(err);
}
// If image already exist in the registry, skip the build
if !option.force_build && cr_registry.does_image_exists(&app_build.image) {
continue;
}
// Be sure that our repository exist before trying to pull/push images from it
let _ = self.engine.container_registry().create_repository(&app.name)?;
// Ok now everything is setup, we can try to build the app
let _ = self
.engine
.build_platform()
.build(app_build, &self.is_transaction_aborted)?;
}
Ok(results)
Ok(())
}
pub fn rollback(&self) -> Result<(), RollbackError> {
@@ -269,63 +185,11 @@ impl<'a> Transaction<'a> {
/// This function is a wrapper to correctly revert all changes of an attempted deployment AND
/// if a failover environment is provided, then rollback.
fn rollback_environment(&self, environment_action: &EnvironmentAction) -> Result<(), RollbackError> {
let qe_environment = |environment: &Environment| {
let mut _applications = Vec::with_capacity(environment.applications.len());
for application in environment.applications.iter() {
let build = application.to_build();
if let Some(x) = application.to_application(
self.engine.context(),
&build.image,
self.engine.cloud_provider(),
self.logger.clone(),
) {
_applications.push(x)
}
}
let qe_environment = environment.to_qe_environment(
self.engine.context(),
&_applications,
self.engine.cloud_provider(),
self.logger.clone(),
);
qe_environment
};
match environment_action {
EnvironmentAction::Environment(te) => {
// revert changes but there is no failover environment
let target_qe_environment = qe_environment(te);
let action = match te.action {
Action::Create => self
.engine
.kubernetes()
.deploy_environment_error(&target_qe_environment),
Action::Pause => self.engine.kubernetes().pause_environment_error(&target_qe_environment),
Action::Delete => self
.engine
.kubernetes()
.delete_environment_error(&target_qe_environment),
Action::Nothing => Ok(()),
};
let _ = match action {
Ok(_) => {}
Err(err) => return Err(RollbackError::CommitError(err)),
};
Err(RollbackError::NoFailoverEnvironment)
}
}
fn rollback_environment(&self, _environment_action: &EnvironmentAction) -> Result<(), RollbackError> {
Ok(())
}
pub fn commit(mut self) -> TransactionResult {
let mut applications_by_environment: HashMap<&Environment, Vec<Box<dyn Application>>> = HashMap::new();
for step in self.steps.clone().into_iter() {
// execution loop
self.executed_steps.push(step.clone());
@@ -372,7 +236,7 @@ impl<'a> Transaction<'a> {
EnvironmentAction::Environment(te) => te,
};
let applications_builds = match self.build_applications(target_environment, &option) {
match self.build_and_push_applications(target_environment, &option) {
Ok(apps) => apps,
Err(engine_err) => {
self.logger.log(
@@ -392,30 +256,6 @@ impl<'a> Transaction<'a> {
};
}
};
if (self.is_transaction_aborted)() {
return TransactionResult::Canceled;
}
let applications = match self.push_applications(applications_builds, &option) {
Ok(results) => {
let applications = results.into_iter().map(|(app, _)| app).collect::<Vec<_>>();
applications
}
Err(engine_err) => {
warn!("ROLLBACK STARTED! an error occurred {:?}", engine_err);
return match self.rollback() {
Ok(_) => TransactionResult::Rollback(engine_err),
Err(err) => {
error!("ROLLBACK FAILED! fatal error: {:?}", err);
TransactionResult::UnrecoverableError(engine_err, err)
}
};
}
};
applications_by_environment.insert(target_environment, applications);
}
Step::DeployEnvironment(environment_action) => {
if (self.is_transaction_aborted)() {
@@ -423,7 +263,7 @@ impl<'a> Transaction<'a> {
}
// deploy complete environment
match self.commit_environment(environment_action, &applications_by_environment, |qe_env| {
match self.commit_environment(environment_action, |qe_env| {
self.engine.kubernetes().deploy_environment(qe_env)
}) {
TransactionResult::Ok => {}
@@ -439,7 +279,7 @@ impl<'a> Transaction<'a> {
}
// pause complete environment
match self.commit_environment(environment_action, &applications_by_environment, |qe_env| {
match self.commit_environment(environment_action, |qe_env| {
self.engine.kubernetes().pause_environment(qe_env)
}) {
TransactionResult::Ok => {}
@@ -455,7 +295,7 @@ impl<'a> Transaction<'a> {
}
// delete complete environment
match self.commit_environment(environment_action, &applications_by_environment, |qe_env| {
match self.commit_environment(environment_action, |qe_env| {
self.engine.kubernetes().delete_environment(qe_env)
}) {
TransactionResult::Ok => {}
@@ -534,12 +374,7 @@ impl<'a> Transaction<'a> {
}
}
fn commit_environment<F>(
&self,
environment_action: &EnvironmentAction,
applications_by_environment: &HashMap<&Environment, Vec<Box<dyn Application>>>,
action_fn: F,
) -> TransactionResult
fn commit_environment<F>(&self, environment_action: &EnvironmentAction, action_fn: F) -> TransactionResult
where
F: Fn(&crate::cloud_provider::environment::Environment) -> Result<(), EngineError>,
{
@@ -547,16 +382,11 @@ impl<'a> Transaction<'a> {
EnvironmentAction::Environment(te) => te,
};
let empty_vec = Vec::with_capacity(0);
let built_applications = match applications_by_environment.get(target_environment) {
Some(applications) => applications,
None => &empty_vec,
};
let registry_info = self.engine.container_registry().login().unwrap();
let qe_environment = target_environment.to_qe_environment(
self.engine.context(),
built_applications,
self.engine.cloud_provider(),
&registry_info,
self.logger.clone(),
);

8
test_utilities/Cargo.lock generated
View File

@@ -2147,6 +2147,7 @@ dependencies = [
"tracing-subscriber",
"trust-dns-resolver",
"url 2.2.2",
"urlencoding",
"uuid 0.8.2",
"walkdir",
]
@@ -3321,6 +3322,7 @@ dependencies = [
"time 0.2.24",
"tracing",
"tracing-subscriber",
"url 2.2.2",
"uuid 0.8.2",
]
@@ -3957,6 +3959,12 @@ dependencies = [
"url 1.7.2",
]
[[package]]
name = "urlencoding"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68b90931029ab9b034b300b797048cf23723400aa757e8a2bfb9d748102f9821"
[[package]]
name = "uuid"
version = "0.7.4"

1
test_utilities/Cargo.toml
View File

@@ -28,6 +28,7 @@ hashicorp_vault = "2.0.1"
maplit = "1.0.2"
uuid = { version = "0.8", features = ["v4"] }
const_format = "0.2.22"
url = "2.2.2"
# Digital Ocean Deps
digitalocean = "0.1.1"

13
test_utilities/src/aws.rs
View File

@@ -9,7 +9,6 @@ use qovery_engine::cloud_provider::models::NodeGroups;
use qovery_engine::cloud_provider::qovery::EngineLocation::ClientSide;
use qovery_engine::cloud_provider::Kind::Aws;
use qovery_engine::cloud_provider::{CloudProvider, TerraformStateCredentials};
use qovery_engine::container_registry::docker_hub::DockerHub;
use qovery_engine::container_registry::ecr::ECR;
use qovery_engine::dns_provider::DnsProvider;
use qovery_engine::engine::EngineConfig;
@@ -54,17 +53,6 @@ pub fn container_registry_ecr(context: &Context) -> ECR {
)
}
pub fn container_registry_docker_hub(context: &Context) -> DockerHub {
DockerHub::new(
context.clone(),
"my-docker-hub-id-123",
"my-default-docker-hub",
"qoveryrd",
"3b9481fe-74e7-4d7b-bc08-e147c9fd4f24",
logger(),
)
}
pub fn aws_default_engine_config(context: &Context, logger: Box<dyn Logger>) -> EngineConfig {
AWS::docker_cr_engine(
&context,
@@ -75,7 +63,6 @@ pub fn aws_default_engine_config(context: &Context, logger: Box<dyn Logger>) ->
None,
)
}
impl Cluster<AWS, Options> for AWS {
fn docker_cr_engine(
context: &Context,

2
test_utilities/src/common.rs
View File

@@ -6,7 +6,7 @@ use chrono::Utc;
use qovery_engine::cloud_provider::utilities::sanitize_name;
use qovery_engine::dns_provider::DnsProvider;
use qovery_engine::models::{
Action, Application, Clone2, Context, Database, DatabaseKind, DatabaseMode, Environment, EnvironmentAction,
Action, Application, CloneForTest, Context, Database, DatabaseKind, DatabaseMode, Environment, EnvironmentAction,
GitCredentials, Port, Protocol, Route, Router, Storage, StorageType,
};

24
test_utilities/src/digitalocean.rs
View File

@@ -1,5 +1,4 @@
use const_format::formatcp;
use qovery_engine::build_platform::Image;
use qovery_engine::cloud_provider::aws::kubernetes::VpcQoveryNetworkMode;
use qovery_engine::cloud_provider::digitalocean::kubernetes::DoksOptions;
use qovery_engine::cloud_provider::digitalocean::network::vpc::VpcInitKind;
@@ -37,7 +36,7 @@ pub fn container_registry_digital_ocean(context: &Context) -> DOCR {
DOCR::new(
context.clone(),
DOCR_ID,
"default-docr-registry-qovery-do-test",
DOCR_ID,
secrets.DIGITAL_OCEAN_TOKEN.unwrap().as_str(),
logger(),
)
@@ -163,11 +162,11 @@ impl Cluster<DO, DoksOptions> for DO {
pub fn clean_environments(
context: &Context,
environments: Vec<Environment>,
_environments: Vec<Environment>,
secrets: FuncTestsSecrets,
_region: DoRegion,
) -> Result<(), EngineError> {
let do_cr = DOCR::new(
let _do_cr = DOCR::new(
context.clone(),
"test",
"test",
@@ -178,14 +177,23 @@ pub fn clean_environments(
logger(),
);
// FIXME: re-enable it, or let pleco do its job ?
/*
// delete images created in registry
let registry_url = do_cr.login()?;
for env in environments.iter() {
for image in env.applications.iter().map(|a| a.to_image()).collect::<Vec<Image>>() {
if let Err(e) = do_cr.delete_image(&image) {
return Err(e);
}
for image in env
.applications
.iter()
.map(|a| a.to_image(&registry_url))
.collect::<Vec<Image>>()
{
//if let Err(e) = do_cr.delete_registry(&image.name) {
// return Err(e);
//}
}
}
*/
Ok(())
}

9
test_utilities/src/scaleway.rs
View File

@@ -18,6 +18,7 @@ use qovery_engine::cloud_provider::aws::kubernetes::VpcQoveryNetworkMode;
use qovery_engine::cloud_provider::models::NodeGroups;
use qovery_engine::cloud_provider::qovery::EngineLocation;
use qovery_engine::cloud_provider::Kind::Scw;
use qovery_engine::container_registry::ContainerRegistry;
use qovery_engine::dns_provider::DnsProvider;
use qovery_engine::errors::EngineError;
use qovery_engine::logger::Logger;
@@ -239,8 +240,14 @@ pub fn clean_environments(
);
// delete images created in registry
let registry_url = container_registry_client.login()?;
for env in environments.iter() {
for image in env.applications.iter().map(|a| a.to_image()).collect::<Vec<Image>>() {
for image in env
.applications
.iter()
.map(|a| a.to_image(&registry_url))
.collect::<Vec<Image>>()
{
if let Err(e) = container_registry_client.delete_image(&image) {
return Err(e);
}

3
test_utilities/src/utilities.rs
View File

@@ -80,7 +80,6 @@ pub fn context(organization_id: &str, cluster_id: &str) -> Context {
None => Some(7200),
}
},
docker_build_options: Some("--network host".to_string()),
forced_upgrade: Option::from({
match env::var_os("forced_upgrade") {
Some(_) => true,
@@ -363,7 +362,7 @@ impl FuncTestsSecrets {
}
pub fn build_platform_local_docker(context: &Context, logger: Box<dyn Logger>) -> LocalDocker {
LocalDocker::new(context.clone(), "oxqlm3r99vwcmvuj", "qovery-local-docker", logger)
LocalDocker::new(context.clone(), "oxqlm3r99vwcmvuj", "qovery-local-docker", logger).unwrap()
}
pub fn init() -> Instant {

7
tests/aws/aws_databases.rs
View File

@@ -3,19 +3,18 @@ extern crate test_utilities;
use ::function_name::named;
use qovery_engine::cloud_provider::Kind;
use qovery_engine::models::{
Action, Clone2, Context, Database, DatabaseKind, DatabaseMode, Environment, EnvironmentAction, Port, Protocol,
Action, CloneForTest, Database, DatabaseKind, DatabaseMode, EnvironmentAction, Port, Protocol,
};
use test_utilities::aws::{aws_default_engine_config, AWS_KUBERNETES_VERSION, AWS_TEST_REGION};
use test_utilities::aws::aws_default_engine_config;
use tracing::{span, Level};
use self::test_utilities::aws::{AWS_DATABASE_DISK_TYPE, AWS_DATABASE_INSTANCE_TYPE};
use self::test_utilities::utilities::{
context, engine_run_test, generate_id, get_pods, get_svc_name, init, is_pod_restarted_env, logger, FuncTestsSecrets,
};
use qovery_engine::cloud_provider::aws::AWS;
use qovery_engine::models::DatabaseMode::{CONTAINER, MANAGED};
use qovery_engine::transaction::TransactionResult;
use test_utilities::common::{test_db, Cluster, ClusterDomain, Infrastructure};
use test_utilities::common::{test_db, Infrastructure};
/**
**

101
tests/aws/aws_environment.rs
View File

@@ -5,16 +5,13 @@ use self::test_utilities::utilities::{
engine_run_test, generate_id, get_pods, get_pvc, is_pod_restarted_env, logger, FuncTestsSecrets,
};
use ::function_name::named;
use qovery_engine::build_platform::{BuildPlatform, CacheResult};
use qovery_engine::cloud_provider::Kind;
use qovery_engine::cmd::kubectl::kubernetes_get_all_pdbs;
use qovery_engine::container_registry::{ContainerRegistry, PullResult};
use qovery_engine::models::{Action, Clone2, EnvironmentAction, Port, Protocol, Storage, StorageType};
use qovery_engine::models::{Action, CloneForTest, EnvironmentAction, Port, Protocol, Storage, StorageType};
use qovery_engine::transaction::TransactionResult;
use std::collections::BTreeMap;
use std::time::SystemTime;
use test_utilities::aws::{aws_default_engine_config, container_registry_ecr, AWS_KUBERNETES_VERSION, AWS_TEST_REGION};
use test_utilities::utilities::{build_platform_local_docker, context, init, kubernetes_config_path};
use test_utilities::aws::aws_default_engine_config;
use test_utilities::utilities::{context, init, kubernetes_config_path};
use tracing::{span, Level};
// TODO:
@@ -75,98 +72,6 @@ fn deploy_a_working_environment_with_no_router_on_aws_eks() {
})
}
#[cfg(feature = "test-aws-self-hosted")]
#[named]
#[test]
fn test_build_cache() {
let test_name = function_name!();
engine_run_test(|| {
init();
let span = span!(Level::INFO, "test", name = test_name);
let _enter = span.enter();
let secrets = FuncTestsSecrets::new();
let context = context(
secrets
.AWS_TEST_ORGANIZATION_ID
.as_ref()
.expect("AWS_TEST_ORGANIZATION_ID is not set")
.as_str(),
secrets
.AWS_TEST_CLUSTER_ID
.as_ref()
.expect("AWS_TEST_CLUSTER_ID is not set")
.as_str(),
);
let engine_config = aws_default_engine_config(&context, logger());
let environment = test_utilities::common::working_minimal_environment(
&context,
secrets
.DEFAULT_TEST_DOMAIN
.expect("DEFAULT_TEST_DOMAIN is not set in secrets")
.as_str(),
);
let ecr = container_registry_ecr(&context);
let local_docker = build_platform_local_docker(&context, logger());
let app = environment.applications.first().unwrap();
let image = app.to_image();
let app_build = app.to_build();
let _ = match local_docker.has_cache(&app_build) {
Ok(CacheResult::Hit) => assert!(false),
Ok(CacheResult::Miss(_)) => assert!(true),
Ok(CacheResult::MissWithoutParentBuild) => assert!(false),
Err(_) => assert!(false),
};
let _ = match ecr.pull(&image).unwrap() {
PullResult::Some(_) => assert!(false),
PullResult::None => assert!(true),
};
let cancel_task = || false;
let build_result = local_docker.build(app.to_build(), false, &cancel_task).unwrap();
let _ = match ecr.push(&build_result.build.image, false) {
Ok(_) => assert!(true),
Err(_) => assert!(false),
};
// TODO clean local docker cache
let start_pull_time = SystemTime::now();
let _ = match ecr.pull(&build_result.build.image).unwrap() {
PullResult::Some(_) => assert!(true),
PullResult::None => assert!(false),
};
let pull_duration = SystemTime::now().duration_since(start_pull_time).unwrap();
let _ = match local_docker.has_cache(&build_result.build) {
Ok(CacheResult::Hit) => assert!(true),
Ok(CacheResult::Miss(_)) => assert!(false),
Ok(CacheResult::MissWithoutParentBuild) => assert!(false),
Err(_) => assert!(false),
};
let start_pull_time = SystemTime::now();
let _ = match ecr.pull(&image).unwrap() {
PullResult::Some(_) => assert!(true),
PullResult::None => assert!(false),
};
let pull_duration_2 = SystemTime::now().duration_since(start_pull_time).unwrap();
if pull_duration_2.as_millis() > pull_duration.as_millis() {
assert!(false);
}
return test_name.to_string();
})
}
#[cfg(feature = "test-aws-self-hosted")]
#[named]
#[test]

3
tests/aws/aws_kubernetes.rs
View File

@@ -7,9 +7,8 @@ use self::test_utilities::utilities::{
use ::function_name::named;
use qovery_engine::cloud_provider::aws::kubernetes::VpcQoveryNetworkMode;
use qovery_engine::cloud_provider::aws::kubernetes::VpcQoveryNetworkMode::{WithNatGateways, WithoutNatGateways};
use qovery_engine::cloud_provider::aws::regions::{AwsRegion, AwsZones};
use qovery_engine::cloud_provider::aws::regions::AwsRegion;
use qovery_engine::cloud_provider::Kind;
use std::borrow::Borrow;
use std::str::FromStr;
use test_utilities::common::{cluster_test, ClusterDomain, ClusterTestType};

11
tests/digitalocean/do_databases.rs
View File

@@ -3,7 +3,7 @@ use tracing::{span, warn, Level};
use qovery_engine::cloud_provider::{Kind as ProviderKind, Kind};
use qovery_engine::models::{
Action, Clone2, Context, Database, DatabaseKind, DatabaseMode, Environment, EnvironmentAction, Port, Protocol,
Action, CloneForTest, Database, DatabaseKind, DatabaseMode, EnvironmentAction, Port, Protocol,
};
use qovery_engine::transaction::TransactionResult;
use test_utilities::utilities::{
@@ -11,11 +11,10 @@ use test_utilities::utilities::{
};
use qovery_engine::models::DatabaseMode::{CONTAINER, MANAGED};
use test_utilities::common::{database_test_environment, test_db, working_minimal_environment, Infrastructure};
use test_utilities::common::{database_test_environment, test_db, Infrastructure};
use test_utilities::digitalocean::{
clean_environments, do_default_engine_config, DO_KUBERNETES_VERSION, DO_MANAGED_DATABASE_DISK_TYPE,
DO_MANAGED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE,
DO_TEST_REGION,
clean_environments, do_default_engine_config, DO_MANAGED_DATABASE_DISK_TYPE, DO_MANAGED_DATABASE_INSTANCE_TYPE,
DO_SELF_HOSTED_DATABASE_DISK_TYPE, DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, DO_TEST_REGION,
};
/**
@@ -437,7 +436,6 @@ fn private_postgresql_v10_deploy_a_working_dev_environment() {
#[ignore]
#[named]
#[test]
#[ignore]
fn public_postgresql_v10_deploy_a_working_dev_environment() {
test_postgresql_configuration("10", function_name!(), CONTAINER, true);
}
@@ -454,7 +452,6 @@ fn private_postgresql_v11_deploy_a_working_dev_environment() {
#[ignore]
#[named]
#[test]
#[ignore]
fn public_postgresql_v11_deploy_a_working_dev_environment() {
test_postgresql_configuration("11", function_name!(), CONTAINER, true);
}

99
tests/digitalocean/do_environment.rs
View File

@@ -6,16 +6,13 @@ use self::test_utilities::utilities::{
engine_run_test, generate_id, get_pods, get_pvc, init, is_pod_restarted_env, logger, FuncTestsSecrets,
};
use ::function_name::named;
use qovery_engine::build_platform::{BuildPlatform, CacheResult};
use qovery_engine::cloud_provider::Kind;
use qovery_engine::container_registry::{ContainerRegistry, PullResult};
use qovery_engine::models::{Action, Clone2, EnvironmentAction, Port, Protocol, Storage, StorageType};
use qovery_engine::models::{Action, CloneForTest, EnvironmentAction, Port, Protocol, Storage, StorageType};
use qovery_engine::transaction::TransactionResult;
use std::collections::BTreeMap;
use std::time::SystemTime;
use test_utilities::common::Infrastructure;
use test_utilities::digitalocean::{container_registry_digital_ocean, do_default_engine_config, DO_KUBERNETES_VERSION};
use test_utilities::utilities::{build_platform_local_docker, context};
use test_utilities::digitalocean::do_default_engine_config;
use test_utilities::utilities::context;
use tracing::{span, warn, Level};
// Note: All those tests relies on a test cluster running on DigitalOcean infrastructure.
@@ -78,96 +75,6 @@ fn digitalocean_doks_deploy_a_working_environment_with_no_router() {
})
}
#[cfg(feature = "test-do-self-hosted")]
#[named]
#[test]
fn test_build_cache() {
let test_name = function_name!();
engine_run_test(|| {
init();
let span = span!(Level::INFO, "test", name = test_name);
let _enter = span.enter();
let secrets = FuncTestsSecrets::new();
let context = context(
secrets
.DIGITAL_OCEAN_TEST_ORGANIZATION_ID
.as_ref()
.expect("DIGITAL_OCEAN_TEST_ORGANIZATION_ID is not set"),
secrets
.DIGITAL_OCEAN_TEST_CLUSTER_ID
.as_ref()
.expect("DIGITAL_OCEAN_TEST_CLUSTER_ID is not set"),
);
let engine_config = do_default_engine_config(&context, logger());
let environment = test_utilities::common::working_minimal_environment(
&context,
secrets
.DEFAULT_TEST_DOMAIN
.expect("DEFAULT_TEST_DOMAIN is not set in secrets")
.as_str(),
);
let docr = container_registry_digital_ocean(&context);
let local_docker = build_platform_local_docker(&context, logger());
let app = environment.applications.first().unwrap();
let image = app.to_image();
let app_build = app.to_build();
let _ = match local_docker.has_cache(&app_build) {
Ok(CacheResult::Hit) => assert!(false),
Ok(CacheResult::Miss(_)) => assert!(true),
Ok(CacheResult::MissWithoutParentBuild) => assert!(false),
Err(_) => assert!(false),
};
let _ = match docr.pull(&image).unwrap() {
PullResult::Some(_) => assert!(false),
PullResult::None => assert!(true),
};
let cancel_task = || false;
let build_result = local_docker.build(app.to_build(), false, &cancel_task).unwrap();
let _ = match docr.push(&build_result.build.image, false) {
Ok(_) => assert!(true),
Err(_) => assert!(false),
};
// TODO clean local docker cache
let start_pull_time = SystemTime::now();
let _ = match docr.pull(&build_result.build.image).unwrap() {
PullResult::Some(_) => assert!(true),
PullResult::None => assert!(false),
};
let pull_duration = SystemTime::now().duration_since(start_pull_time).unwrap();
let _ = match local_docker.has_cache(&build_result.build) {
Ok(CacheResult::Hit) => assert!(true),
Ok(CacheResult::Miss(_)) => assert!(false),
Ok(CacheResult::MissWithoutParentBuild) => assert!(false),
Err(_) => assert!(false),
};
let start_pull_time = SystemTime::now();
let _ = match docr.pull(&image).unwrap() {
PullResult::Some(_) => assert!(true),
PullResult::None => assert!(false),
};
let pull_duration_2 = SystemTime::now().duration_since(start_pull_time).unwrap();
if pull_duration_2.as_millis() > pull_duration.as_millis() {
assert!(false);
}
return test_name.to_string();
})
}
#[cfg(feature = "test-do-self-hosted")]
#[named]
#[test]

4
tests/digitalocean/do_kubernetes.rs
View File

@@ -2,9 +2,7 @@ extern crate test_utilities;
use self::test_utilities::common::ClusterDomain;
use self::test_utilities::digitalocean::{DO_KUBERNETES_MAJOR_VERSION, DO_KUBERNETES_MINOR_VERSION};
use self::test_utilities::utilities::{
context, engine_run_test, generate_cluster_id, generate_id, logger, FuncTestsSecrets,
};
use self::test_utilities::utilities::{context, engine_run_test, generate_cluster_id, generate_id, logger};
use ::function_name::named;
use qovery_engine::cloud_provider::digitalocean::application::DoRegion;
use qovery_engine::cloud_provider::Kind;

4
tests/digitalocean/do_utility_kubernetes_doks_test_cluster.rs
View File

@@ -2,11 +2,9 @@ extern crate test_utilities;
use self::test_utilities::utilities::{context, engine_run_test, init, logger, FuncTestsSecrets};
use ::function_name::named;
use qovery_engine::cloud_provider::digitalocean::DO;
use test_utilities::digitalocean::{do_default_engine_config, DO_KUBERNETES_VERSION, DO_TEST_REGION};
use test_utilities::digitalocean::do_default_engine_config;
use tracing::{span, Level};
use self::test_utilities::common::Cluster;
use qovery_engine::transaction::{Transaction, TransactionResult};
// Warning: This test shouldn't be ran by CI

10
tests/docker/multi_stage_simple/Dockerfile.buildkit Normal file
View File

@@ -0,0 +1,10 @@
FROM golang:1.16 AS build
# ../ is not valid if using old docker engine, only allowed with buildkit
COPY ../hello.go /go/src/project/hello.go
WORKDIR /go/src/project
RUN go build hello.go
FROM scratch
COPY --from=build /go/src/project/hello /bin/hello
ENTRYPOINT ["/bin/hello"]

7
tests/docker/multi_stage_simple/hello.go Normal file
View File

@@ -0,0 +1,7 @@
package main
import "fmt"
func main() {
fmt.Println("hello world")
}

48
tests/scaleway/scw_container_registry.rs
View File

@@ -1,7 +1,6 @@
extern crate test_utilities;
use self::test_utilities::utilities::{context, FuncTestsSecrets};
use qovery_engine::build_platform::Image;
use qovery_engine::cloud_provider::scaleway::application::ScwZone;
use qovery_engine::container_registry::scaleway_container_registry::ScalewayCR;
use test_utilities::utilities::logger;
@@ -49,17 +48,7 @@ fn test_get_registry_namespace() {
logger(),
);
let image = Image {
application_id: "1234".to_string(),
name: registry_name.to_string(),
tag: "tag123".to_string(),
commit_id: "commit_id".to_string(),
registry_name: Some(registry_name.to_string()),
registry_secret: None,
registry_url: None,
registry_docker_json_config: None,
};
let image = registry_name.to_string();
container_registry
.create_registry_namespace(&image)
.expect("error while creating registry namespace");
@@ -108,16 +97,7 @@ fn test_create_registry_namespace() {
logger(),
);
let image = Image {
application_id: "1234".to_string(),
name: registry_name.to_string(),
tag: "tag123".to_string(),
commit_id: "commit_id".to_string(),
registry_name: Some(registry_name.to_string()),
registry_secret: None,
registry_url: None,
registry_docker_json_config: None,
};
let image = registry_name.to_string();
// execute:
debug!("test_create_registry_namespace - {}", region);
@@ -160,17 +140,7 @@ fn test_delete_registry_namespace() {
logger(),
);
let image = Image {
application_id: "1234".to_string(),
name: registry_name.to_string(),
tag: "tag123".to_string(),
commit_id: "commit_id".to_string(),
registry_name: Some(registry_name.to_string()),
registry_secret: None,
registry_url: None,
registry_docker_json_config: None,
};
let image = registry_name.to_string();
container_registry
.create_registry_namespace(&image)
.expect("error while creating registry namespace");
@@ -207,17 +177,7 @@ fn test_get_or_create_registry_namespace() {
logger(),
);
let image = Image {
application_id: "1234".to_string(),
name: registry_name.to_string(),
tag: "tag123".to_string(),
commit_id: "commit_id".to_string(),
registry_name: Some(registry_name.to_string()),
registry_secret: None,
registry_url: None,
registry_docker_json_config: None,
};
let image = registry_name.to_string();
container_registry
.create_registry_namespace(&image)
.expect("error while creating registry namespace");

9
tests/scaleway/scw_databases.rs
View File

@@ -3,7 +3,7 @@ use tracing::{span, warn, Level};
use qovery_engine::cloud_provider::{Kind as ProviderKind, Kind};
use qovery_engine::models::{
Action, Clone2, Context, Database, DatabaseKind, DatabaseMode, Environment, EnvironmentAction, Port, Protocol,
Action, CloneForTest, Database, DatabaseKind, DatabaseMode, EnvironmentAction, Port, Protocol,
};
use qovery_engine::transaction::TransactionResult;
use test_utilities::utilities::{
@@ -12,12 +12,11 @@ use test_utilities::utilities::{
};
use qovery_engine::models::DatabaseMode::{CONTAINER, MANAGED};
use test_utilities::common::test_db;
use test_utilities::common::{database_test_environment, Infrastructure};
use test_utilities::common::{test_db, working_minimal_environment};
use test_utilities::scaleway::{
clean_environments, scw_default_engine_config, SCW_KUBERNETES_VERSION, SCW_MANAGED_DATABASE_DISK_TYPE,
SCW_MANAGED_DATABASE_INSTANCE_TYPE, SCW_SELF_HOSTED_DATABASE_DISK_TYPE, SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE,
SCW_TEST_ZONE,
clean_environments, scw_default_engine_config, SCW_MANAGED_DATABASE_DISK_TYPE, SCW_MANAGED_DATABASE_INSTANCE_TYPE,
SCW_SELF_HOSTED_DATABASE_DISK_TYPE, SCW_SELF_HOSTED_DATABASE_INSTANCE_TYPE, SCW_TEST_ZONE,
};
/**

99
tests/scaleway/scw_environment.rs
View File

@@ -6,16 +6,12 @@ use self::test_utilities::utilities::{
context, engine_run_test, generate_id, get_pods, get_pvc, init, is_pod_restarted_env, logger, FuncTestsSecrets,
};
use ::function_name::named;
use qovery_engine::build_platform::{BuildPlatform, CacheResult};
use qovery_engine::cloud_provider::Kind;
use qovery_engine::container_registry::{ContainerRegistry, PullResult};
use qovery_engine::models::{Action, Clone2, EnvironmentAction, Port, Protocol, Storage, StorageType};
use qovery_engine::models::{Action, CloneForTest, EnvironmentAction, Port, Protocol, Storage, StorageType};
use qovery_engine::transaction::TransactionResult;
use std::collections::BTreeMap;
use std::time::SystemTime;
use test_utilities::common::Infrastructure;
use test_utilities::scaleway::{container_registry_scw, scw_default_engine_config, SCW_KUBERNETES_VERSION};
use test_utilities::utilities::build_platform_local_docker;
use test_utilities::scaleway::scw_default_engine_config;
use tracing::{span, warn, Level};
// Note: All those tests relies on a test cluster running on Scaleway infrastructure.
@@ -81,97 +77,6 @@ fn scaleway_kapsule_deploy_a_working_environment_with_no_router() {
})
}
#[cfg(feature = "test-scw-self-hosted")]
#[named]
#[test]
fn test_build_cache() {
let test_name = function_name!();
engine_run_test(|| {
init();
let span = span!(Level::INFO, "test", name = test_name);
let _enter = span.enter();
let secrets = FuncTestsSecrets::new();
let context = context(
secrets
.SCALEWAY_TEST_ORGANIZATION_ID
.as_ref()
.expect("SCALEWAY_TEST_ORGANIZATION_ID")
.as_str(),
secrets
.SCALEWAY_TEST_CLUSTER_ID
.as_ref()
.expect("SCALEWAY_TEST_CLUSTER_ID")
.as_str(),
);
let environment = test_utilities::common::working_minimal_environment(
&context,
secrets
.DEFAULT_TEST_DOMAIN
.expect("DEFAULT_TEST_DOMAIN is not set in secrets")
.as_str(),
);
let scr = container_registry_scw(&context);
let local_docker = build_platform_local_docker(&context, logger());
let app = environment.applications.first().unwrap();
let image = app.to_image();
let app_build = app.to_build();
let _ = match local_docker.has_cache(&app_build) {
Ok(CacheResult::Hit) => assert!(false),
Ok(CacheResult::Miss(_)) => assert!(true),
Ok(CacheResult::MissWithoutParentBuild) => assert!(false),
Err(_) => assert!(false),
};
let _ = match scr.pull(&image).unwrap() {
PullResult::Some(_) => assert!(false),
PullResult::None => assert!(true),
};
let cancel_task = || false;
let build_result = local_docker.build(app.to_build(), false, &cancel_task).unwrap();
let _ = match scr.push(&build_result.build.image, false) {
Ok(_) => assert!(true),
Err(_) => assert!(false),
};
// TODO clean local docker cache
let start_pull_time = SystemTime::now();
let _ = match scr.pull(&build_result.build.image).unwrap() {
PullResult::Some(_) => assert!(true),
PullResult::None => assert!(false),
};
let pull_duration = SystemTime::now().duration_since(start_pull_time).unwrap();
let _ = match local_docker.has_cache(&build_result.build) {
Ok(CacheResult::Hit) => assert!(true),
Ok(CacheResult::Miss(_)) => assert!(false),
Ok(CacheResult::MissWithoutParentBuild) => assert!(false),
Err(_) => assert!(false),
};
let start_pull_time = SystemTime::now();
let _ = match scr.pull(&image).unwrap() {
PullResult::Some(_) => assert!(true),
PullResult::None => assert!(false),
};
let pull_duration_2 = SystemTime::now().duration_since(start_pull_time).unwrap();
if pull_duration_2.as_millis() > pull_duration.as_millis() {
assert!(false);
}
return test_name.to_string();
})
}
#[cfg(feature = "test-scw-self-hosted")]
#[named]
#[test]

4
tests/scaleway/scw_kubernetes.rs
View File

@@ -1,9 +1,7 @@
extern crate test_utilities;
use self::test_utilities::scaleway::{SCW_KUBERNETES_MAJOR_VERSION, SCW_KUBERNETES_MINOR_VERSION};
use self::test_utilities::utilities::{
context, engine_run_test, generate_cluster_id, generate_id, logger, FuncTestsSecrets,
};
use self::test_utilities::utilities::{context, engine_run_test, generate_cluster_id, generate_id, logger};
use ::function_name::named;
use qovery_engine::cloud_provider::aws::kubernetes::VpcQoveryNetworkMode;
use qovery_engine::cloud_provider::scaleway::application::ScwZone;

4
tests/scaleway/scw_utility_kubernetes_kapsule_test_cluster.rs
View File

@@ -2,11 +2,9 @@ extern crate test_utilities;
use self::test_utilities::utilities::{context, engine_run_test, init, logger, FuncTestsSecrets};
use ::function_name::named;
use test_utilities::scaleway::{scw_default_engine_config, SCW_KUBERNETES_VERSION, SCW_TEST_ZONE};
use test_utilities::scaleway::scw_default_engine_config;
use tracing::{span, Level};
use self::test_utilities::common::Cluster;
use qovery_engine::cloud_provider::scaleway::Scaleway;
use qovery_engine::transaction::{Transaction, TransactionResult};
// Warning: This test shouldn't be ran by CI