first commit for Qovery engine src/*

This commit is contained in:
Romaric Philogene
2020-10-25 14:30:38 +01:00
parent 64cd43befc
commit b32182e307
70 changed files with 16601 additions and 0 deletions

11
.gitignore vendored Normal file
View File

@@ -0,0 +1,11 @@
/target
qovery-engine/Cargo.lock
docker/ci/bin_versions
docker/ci/load.sh
*.iml
.idea
.qovery-workspace
app/.qovery-workspace
.terraform/
NOTES.txt
docker/engine/providers/

3243
Cargo.lock generated Normal file

File diff suppressed because it is too large Load Diff

53
Cargo.toml Normal file
View File

@@ -0,0 +1,53 @@
[package]
name = "qovery-engine"
version = "0.1.0"
authors = ["Romaric Philogene <romaric@qovery.com>"]
edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
chrono = "0.4.11"
cmd_lib = "0.7.8"
git2 = "0.13.8"
walkdir = "2.3.1"
itertools = "0.9.0"
base64 = "0.12.3"
dirs = "3.0.1"
rust-crypto = "0.2.36"
retry = "1.0.0"
dns-lookup = "1.0.3"
rand = "0.7.3"
# FIXME use https://crates.io/crates/blocking instead of runtime.rs
# tar gz
flate2 = "1.0.16" # tar gz
tar = "0.4.29"
# logger
env_logger = "0.7.1"
log = "0.4.8"
# Docker deps
# shiplift = "0.6.0"
# Jinja2
tera = "1.3.1"
serde = "1.0.114"
serde_json = "1.0.57"
serde_derive = "1.0"
# AWS deps
tokio = "0.2.22"
rusoto_core = "0.45.0"
rusoto_sts = "0.45.0"
rusoto_credential = "0.45.0"
rusoto_ecr = "0.45.0"
rusoto_eks = "0.45.0"
rusoto_s3 = "0.45.0"
rusoto_dynamodb = "0.45.0"
# Digital Ocean Deps
digitalocean = "0.1.1"
[dev-dependencies]
test-utilities = { path = "test_utilities" }

View File

@@ -0,0 +1,4 @@
#[derive(Debug)]
pub enum BuildPlatformError {
Unexpected(String),
}

View File

@@ -0,0 +1,254 @@
use std::path::Path;
use std::rc::Rc;
use crate::build_platform::error::BuildPlatformError;
use crate::build_platform::{Build, BuildError, BuildPlatform, BuildResult, Image, Kind};
use crate::fs::workspace_directory;
use crate::git::checkout_submodules;
use crate::models::{
Context, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressListener,
ProgressScope,
};
use crate::transaction::CommitError::BuildImage;
use crate::{cmd, git};
/// use Docker in local
pub struct LocalDocker {
context: Context,
id: String,
name: String,
listeners: Listeners,
}
impl LocalDocker {
pub fn new(context: Context, id: &str, name: &str) -> Self {
LocalDocker {
context,
id: id.to_string(),
name: name.to_string(),
listeners: vec![],
}
}
fn image_does_exist(&self, image: &Image) -> Result<bool, BuildError> {
let envs = match self.context.docker_tcp_socket() {
Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())],
None => vec![],
};
Ok(
match crate::cmd::utilities::exec_with_envs(
"docker",
vec!["image", "inspect", image.name_with_tag().as_str()],
envs,
) {
Ok(_) => true,
_ => false,
},
)
}
}
impl BuildPlatform for LocalDocker {
fn context(&self) -> &Context {
&self.context
}
fn kind(&self) -> Kind {
Kind::LocalDocker
}
fn id(&self) -> &str {
self.id.as_str()
}
fn name(&self) -> &str {
self.name.as_str()
}
fn is_valid(&self) -> Result<(), BuildPlatformError> {
if !crate::cmd::utilities::does_binary_exist("docker") {
return Err(BuildPlatformError::Unexpected(
"docker binary not found".to_string(),
));
}
Ok(())
}
fn add_listener(&mut self, listener: Rc<Box<dyn ProgressListener>>) {
self.listeners.push(listener);
}
fn build(&self, build: Build, force_build: bool) -> Result<BuildResult, BuildError> {
info!("LocalDocker.build() called for {}", self.name());
let listeners_helper = ListenersHelper::new(&self.listeners);
if !force_build && self.image_does_exist(&build.image)? {
info!(
"image {:?} does already exist - no need to build it",
build.image
);
return Ok(BuildResult { build });
}
// git clone
let into_dir = workspace_directory(
self.context.workspace_root_dir(),
self.context.execution_id(),
format!("build/{}", build.image.name.as_str()),
);
info!("cloning repository: {}", build.git_repository.url);
let git_clone = git::clone(
build.git_repository.url.as_str(),
&into_dir,
&build.git_repository.credentials,
);
match git_clone {
Ok(_) => {}
Err(err) => {
error! {"Error while trying to clone repository {}", build.git_repository.url}
return Err(BuildError::Git(err));
}
}
// git checkout to given commit
let repo = &git_clone.unwrap();
let commit_id = &build.git_repository.commit_id;
match git::checkout(&repo, &commit_id, build.git_repository.url.as_str()) {
Ok(_) => {}
Err(err) => return Err(BuildError::Git(err)),
}
// git checkout submodules
let _ = checkout_submodules(&repo);
// TODO what if we can't checkout submodules? Today we ignore it
let into_dir_docker_style = format!("{}/.", into_dir.as_str());
let dockerfile_relative_path = match build.git_repository.dockerfile_path.trim() {
"" | "." | "/" | "/." | "./" | "Dockerfile" => "Dockerfile",
dockerfile_root_path => dockerfile_root_path,
};
let dockerfile_complete_path =
format!("{}/{}", into_dir.as_str(), dockerfile_relative_path);
match Path::new(dockerfile_complete_path.as_str()).exists() {
false => {
error!(
"Unable to find Dockerfile path {}",
dockerfile_complete_path.as_str()
);
return Err(BuildError::Error);
}
_ => {}
}
let env_var_args = &build
.options
.environment_variables
.iter()
.map(|ev| format!("'{}={}'", ev.key, ev.value))
.collect::<Vec<_>>();
let name_with_tag = build.image.name_with_tag();
let mut docker_args = vec![
"build",
"-f",
dockerfile_complete_path.as_str(),
"-t",
name_with_tag.as_str(),
];
let mut docker_args = if env_var_args.is_empty() {
docker_args
} else {
let mut build_args = vec![];
env_var_args.iter().for_each(|x| {
build_args.push("--build-arg");
build_args.push(x.as_str());
});
docker_args.extend(build_args);
docker_args
};
docker_args.push(into_dir_docker_style.as_str());
let envs = match self.context.docker_tcp_socket() {
Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())],
None => vec![],
};
// docker build
let exit_status = cmd::utilities::exec_with_envs_and_output(
"docker",
docker_args,
envs,
|line| {
let line_string = line.unwrap();
info!("{}", line_string.as_str());
listeners_helper.start_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: build.image.application_id.clone(),
},
ProgressLevel::Info,
Some(line_string.as_str()),
self.context.execution_id(),
));
},
|line| {
let line_string = line.unwrap();
error!("{}", line_string.as_str());
listeners_helper.error(ProgressInfo::new(
ProgressScope::Application {
id: build.image.application_id.clone(),
},
ProgressLevel::Error,
Some(line_string.as_str()),
self.context.execution_id(),
));
},
);
match exit_status {
Ok(_) => {}
Err(_) => return Err(BuildError::Error),
}
listeners_helper.start_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: build.image.application_id.clone(),
},
ProgressLevel::Info,
Some("build is done ✔"),
self.context.execution_id(),
));
Ok(BuildResult { build })
}
fn build_error(&self, build: Build) -> Result<BuildResult, BuildError> {
warn!("LocalDocker.build_error() called for {}", self.name());
let listener_helper = ListenersHelper::new(&self.listeners);
listener_helper.error(ProgressInfo::new(
ProgressScope::Application {
id: build.image.application_id,
},
ProgressLevel::Error,
Some("something goes wrong (not implemented)"),
self.context.execution_id(),
));
// FIXME
Err(BuildError::Error)
}
}

75
src/build_platform/mod.rs Normal file
View File

@@ -0,0 +1,75 @@
use std::rc::Rc;
use git2::Error;
use serde::{Deserialize, Serialize};
use crate::build_platform::error::BuildPlatformError;
use crate::git::Credentials;
use crate::models::{Context, ProgressListener};
pub mod error;
pub mod local_docker;
pub trait BuildPlatform {
fn context(&self) -> &Context;
fn kind(&self) -> Kind;
fn id(&self) -> &str;
fn name(&self) -> &str;
fn is_valid(&self) -> Result<(), BuildPlatformError>;
fn add_listener(&mut self, listener: Rc<Box<dyn ProgressListener>>);
fn build(&self, build: Build, force_build: bool) -> Result<BuildResult, BuildError>;
fn build_error(&self, build: Build) -> Result<BuildResult, BuildError>;
}
pub struct Build {
pub git_repository: GitRepository,
pub image: Image,
pub options: BuildOptions,
}
pub struct BuildOptions {
pub environment_variables: Vec<EnvironmentVariable>,
}
pub struct EnvironmentVariable {
pub key: String,
pub value: String,
}
pub struct GitRepository {
pub url: String,
pub credentials: Option<Credentials>,
pub commit_id: String,
pub dockerfile_path: String,
}
#[derive(Clone, Eq, PartialEq, Hash, Debug)]
pub struct Image {
pub application_id: String,
pub name: String,
pub tag: String,
pub commit_id: String,
pub registry_url: Option<String>,
}
impl Image {
pub fn name_with_tag(&self) -> String {
format!("{}:{}", self.name, self.tag)
}
}
pub struct BuildResult {
pub build: Build,
}
#[derive(Debug)]
pub enum BuildError {
Git(Error),
Error,
}
#[derive(Serialize, Deserialize, Clone)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub enum Kind {
LocalDocker,
}

View File

@@ -0,0 +1,419 @@
use serde::{Deserialize, Serialize};
use tera::Context as TeraContext;
use crate::build_platform::Image;
use crate::cloud_provider::aws::{common, AWS};
use crate::cloud_provider::environment::Environment;
use crate::cloud_provider::kubernetes::Kubernetes;
use crate::cloud_provider::service::{
Action, Application as CApplication, Create, Delete, Pause, Service, ServiceError, ServiceType,
StatelessService,
};
use crate::cloud_provider::DeploymentTarget;
use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY};
use crate::models::Context;
#[derive(Clone, Eq, PartialEq, Hash)]
pub struct Application {
context: Context,
id: String,
action: Action,
name: String,
private_port: Option<u16>,
total_cpus: String,
cpu_burst: String,
total_ram_in_mib: u32,
total_instances: u16,
image: Image,
storage: Vec<Storage>,
environment_variables: Vec<EnvironmentVariable>,
}
impl Application {
pub fn new(
context: Context,
id: &str,
action: Action,
name: &str,
private_port: Option<u16>,
total_cpus: String,
cpu_burst: String,
total_ram_in_mib: u32,
total_instances: u16,
image: Image,
storage: Vec<Storage>,
environment_variables: Vec<EnvironmentVariable>,
) -> Self {
Application {
context,
id: id.to_string(),
action,
name: name.to_string(),
private_port,
total_cpus,
cpu_burst,
total_ram_in_mib,
total_instances,
image,
storage,
environment_variables,
}
}
fn helm_release_name(&self) -> String {
crate::string::cut(format!("application-{}-{}", self.name(), self.id()), 50)
}
fn workspace_directory(&self) -> String {
crate::fs::workspace_directory(
self.context.workspace_root_dir(),
self.context.execution_id(),
format!("applications/{}", self.name()),
)
}
fn context(&self, kubernetes: &dyn Kubernetes, environment: &Environment) -> TeraContext {
let mut context = self.default_tera_context(kubernetes, environment);
let commit_id = self.image().commit_id.as_str();
context.insert("helm_app_version", &commit_id[..7]);
match &self.image().registry_url {
Some(registry_url) => context.insert("image_name_with_tag", registry_url.as_str()),
None => {
let image_name_with_tag = self.image().name_with_tag();
warn!("there is no registry url, use image name with tag with the default container registry: {}", image_name_with_tag.as_str());
context.insert("image_name_with_tag", image_name_with_tag.as_str());
}
}
let environment_variables = self
.environment_variables
.iter()
.map(|ev| EnvironmentVariableDataTemplate {
key: ev.key.clone(),
value: ev.value.clone(),
})
.collect::<Vec<_>>();
context.insert("environment_variables", &environment_variables);
let storage = self
.storage
.iter()
.map(|s| StorageDataTemplate {
id: s.id.clone(),
name: s.name.clone(),
storage_type: match s.storage_type {
StorageType::SC1 => "sc1",
StorageType::ST1 => "st1",
StorageType::GP2 => "gp2",
StorageType::IO1 => "io1",
}
.to_string(),
size_in_gib: s.size_in_gib,
mount_point: s.mount_point.clone(),
snapshot_retention_in_days: s.snapshot_retention_in_days,
})
.collect::<Vec<_>>();
let is_storage = storage.len() > 0;
context.insert("storage", &storage);
context.insert("is_storage", &is_storage);
context.insert("clone", &false);
context
}
fn delete(&self, target: &DeploymentTarget, is_error: bool) -> Result<(), ServiceError> {
let (kubernetes, environment) = match target {
DeploymentTarget::ManagedServices(k, env) => (*k, *env),
DeploymentTarget::SelfHosted(k, env) => (*k, *env),
};
let workspace_dir = self.workspace_directory();
let helm_release_name = self.helm_release_name();
let selector = format!("app={}", self.name());
if is_error {
let _ = common::get_stateless_resource_information(
kubernetes,
environment,
workspace_dir.as_str(),
selector.as_str(),
)?;
}
// clean the resource
let _ = common::do_stateless_service_cleanup(
kubernetes,
environment,
workspace_dir.as_str(),
helm_release_name.as_str(),
)?;
Ok(())
}
}
impl crate::cloud_provider::service::Application for Application {
fn image(&self) -> &Image {
&self.image
}
fn set_image(&mut self, image: Image) {
self.image = image;
}
}
impl StatelessService for Application {}
impl Service for Application {
fn context(&self) -> &Context {
&self.context
}
fn service_type(&self) -> ServiceType {
ServiceType::Application
}
fn id(&self) -> &str {
self.id.as_str()
}
fn name(&self) -> &str {
self.name.as_str()
}
fn version(&self) -> &str {
self.image.commit_id.as_str()
}
fn action(&self) -> &Action {
&self.action
}
fn private_port(&self) -> Option<u16> {
self.private_port
}
fn total_cpus(&self) -> String {
self.total_cpus.to_string()
}
fn total_ram_in_mib(&self) -> u32 {
self.total_ram_in_mib
}
fn total_instances(&self) -> u16 {
self.total_instances
}
}
impl Create for Application {
fn on_create(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
info!("AWS.application.on_create() called for {}", self.name());
let (kubernetes, environment) = match target {
DeploymentTarget::ManagedServices(k, env) => (*k, *env),
DeploymentTarget::SelfHosted(k, env) => (*k, *env),
};
let aws = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.unwrap();
let context = self.context(kubernetes, environment);
let workspace_dir = self.workspace_directory();
let from_dir = format!("{}/aws/charts/q-application", self.context.lib_root_dir());
let _ = crate::template::generate_and_copy_all_files_into_dir(
from_dir.as_str(),
workspace_dir.as_str(),
&context,
)?;
// render
// TODO check the rendered files?
let helm_release_name = self.helm_release_name();
let aws_credentials_envs = vec![
(AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()),
(AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()),
];
let kubernetes_config_file_path = common::kubernetes_config_path(
workspace_dir.as_str(),
environment.organization_id.as_str(),
kubernetes.id(),
aws.access_key_id.as_str(),
aws.secret_access_key.as_str(),
kubernetes.region(),
)?;
// do exec helm upgrade and return the last deployment status
let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history(
kubernetes_config_file_path.as_str(),
environment.namespace(),
helm_release_name.as_str(),
workspace_dir.as_str(),
aws_credentials_envs.clone(),
)?;
// check deployment status
if helm_history_row.is_none() || !helm_history_row.unwrap().is_successfully_deployed() {
// TODO get pod output by using kubectl and return it into the OnCreateFailed
return Err(ServiceError::OnCreateFailed);
}
// check app status
let selector = format!("app={}", self.name());
match crate::cmd::kubectl::kubectl_exec_is_pod_ready_with_retry(
kubernetes_config_file_path.as_str(),
environment.namespace(),
selector.as_str(),
aws_credentials_envs,
) {
Ok(Some(true)) => {}
_ => return Err(ServiceError::OnCreateFailed),
}
Ok(())
}
fn on_create_check(&self) -> Result<(), ServiceError> {
Ok(())
}
fn on_create_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!(
"AWS.application.on_create_error() called for {}",
self.name()
);
let (kubernetes, environment) = match target {
DeploymentTarget::ManagedServices(k, env) => (*k, *env),
DeploymentTarget::SelfHosted(k, env) => (*k, *env),
};
let workspace_dir = self.workspace_directory();
let aws = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.unwrap();
let aws_credentials_envs = vec![
(AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()),
(AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()),
];
let kubernetes_config_file_path = common::kubernetes_config_path(
workspace_dir.as_str(),
environment.organization_id.as_str(),
kubernetes.id(),
aws.access_key_id.as_str(),
aws.secret_access_key.as_str(),
kubernetes.region(),
)?;
let helm_release_name = self.helm_release_name();
let history_rows = crate::cmd::helm::helm_exec_history(
kubernetes_config_file_path.as_str(),
environment.namespace(),
helm_release_name.as_str(),
aws_credentials_envs.clone(),
)?;
if history_rows.len() == 1 {
crate::cmd::helm::helm_exec_uninstall(
kubernetes_config_file_path.as_str(),
environment.namespace(),
helm_release_name.as_str(),
aws_credentials_envs.clone(),
)?;
}
Ok(())
}
}
impl Pause for Application {
fn on_pause(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
info!("AWS.application.on_pause() called for {}", self.name());
self.delete(target, false)
}
fn on_pause_check(&self) -> Result<(), ServiceError> {
Ok(())
}
fn on_pause_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!(
"AWS.application.on_pause_error() called for {}",
self.name()
);
self.delete(target, true)
}
}
impl Delete for Application {
fn on_delete(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
info!("AWS.application.on_delete() called for {}", self.name());
self.delete(target, false)
}
fn on_delete_check(&self) -> Result<(), ServiceError> {
Ok(())
}
fn on_delete_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!(
"AWS.application.on_delete_error() called for {}",
self.name()
);
self.delete(target, true)
}
}
#[derive(Clone, Eq, PartialEq, Hash)]
pub struct EnvironmentVariable {
pub key: String,
pub value: String,
}
#[derive(Serialize, Deserialize)]
struct EnvironmentVariableDataTemplate {
pub key: String,
pub value: String,
}
#[derive(Clone, Eq, PartialEq, Hash)]
pub struct Storage {
pub id: String,
pub name: String,
pub storage_type: StorageType,
pub size_in_gib: u16,
pub mount_point: String,
pub snapshot_retention_in_days: u16,
}
#[derive(Clone, Eq, PartialEq, Hash)]
pub enum StorageType {
SC1,
ST1,
GP2,
IO1,
}
#[derive(Serialize, Deserialize)]
struct StorageDataTemplate {
pub id: String,
pub name: String,
pub storage_type: String,
pub size_in_gib: u16,
pub mount_point: String,
pub snapshot_retention_in_days: u16,
}

View File

@@ -0,0 +1,156 @@
use std::io::Error;
use std::str::FromStr;
use rusoto_core::Region;
use crate::cloud_provider::aws::AWS;
use crate::cloud_provider::environment::Environment;
use crate::cloud_provider::kubernetes::Kubernetes;
use crate::cloud_provider::service::ServiceError;
use crate::cmd::utilities::CmdError;
use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY};
pub fn kubernetes_config_path(
workspace_directory: &str,
organization_id: &str,
kubernetes_cluster_id: &str,
access_key_id: &str,
secret_access_key: &str,
region: &str,
) -> Result<String, Error> {
let kubernetes_config_bucket_name = format!("qovery-kubeconfigs-{}", kubernetes_cluster_id);
let kubernetes_config_object_key = format!("{}.yaml", kubernetes_cluster_id);
let kubernetes_config_file_path = format!(
"{}/kubernetes_config_{}",
workspace_directory, kubernetes_cluster_id
);
let _region = Region::from_str(region).unwrap();
let _ = crate::s3::get_kubernetes_config_file(
access_key_id,
secret_access_key,
&_region,
kubernetes_config_bucket_name.as_str(),
kubernetes_config_object_key.as_str(),
kubernetes_config_file_path.as_str(),
)?;
Ok(kubernetes_config_file_path)
}
pub type Logs = String;
pub type Describe = String;
/// show different output (kubectl describe, log..) for debug purpose
pub fn get_stateless_resource_information(
kubernetes: &dyn Kubernetes,
environment: &Environment,
workspace_dir: &str,
selector: &str,
) -> Result<(Describe, Logs), CmdError> {
let aws = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.unwrap();
let kubernetes_config_file_path = kubernetes_config_path(
workspace_dir,
environment.organization_id.as_str(),
kubernetes.id(),
aws.access_key_id.as_str(),
aws.secret_access_key.as_str(),
kubernetes.region(),
)?;
let aws_credentials_envs = vec![
(AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()),
(AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()),
];
// exec describe pod...
let describe = match crate::cmd::kubectl::kubectl_exec_describe_pod(
kubernetes_config_file_path.as_str(),
environment.namespace(),
selector,
aws_credentials_envs.clone(),
) {
Ok(output) => {
info!("{}", output);
output
}
Err(err) => {
error!("{:?}", err);
return Err(err);
}
};
// exec logs...
let logs = match crate::cmd::kubectl::kubectl_exec_logs(
kubernetes_config_file_path.as_str(),
environment.namespace(),
selector,
aws_credentials_envs.clone(),
) {
Ok(output) => {
info!("{}", output);
output
}
Err(err) => {
error!("{:?}", err);
return Err(err);
}
};
Ok((describe, logs))
}
pub fn do_stateless_service_cleanup(
kubernetes: &dyn Kubernetes,
environment: &Environment,
workspace_dir: &str,
helm_release_name: &str,
) -> Result<(), ServiceError> {
let aws = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.unwrap();
let kubernetes_config_file_path = kubernetes_config_path(
workspace_dir,
environment.organization_id.as_str(),
kubernetes.id(),
aws.access_key_id.as_str(),
aws.secret_access_key.as_str(),
kubernetes.region(),
)?;
let aws_credentials_envs = vec![
(AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()),
(AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()),
];
let history_rows = crate::cmd::helm::helm_exec_history(
kubernetes_config_file_path.as_str(),
environment.namespace(),
helm_release_name,
aws_credentials_envs.clone(),
)?;
// if there is no valid history - then delete the helm chart
let first_valid_history_row = history_rows.iter().find(|x| x.is_successfully_deployed());
if first_valid_history_row.is_some() {
crate::cmd::helm::helm_exec_uninstall(
kubernetes_config_file_path.as_str(),
environment.namespace(),
helm_release_name,
aws_credentials_envs,
)?;
}
Ok(())
}

View File

@@ -0,0 +1,8 @@
pub use mongodb::MongoDB;
pub use mysql::MySQL;
pub use postgresql::PostgreSQL;
mod mongodb;
mod mysql;
mod postgresql;
mod utilities;

View File

@@ -0,0 +1,478 @@
use tera::Context as TeraContext;
use crate::cloud_provider::aws::databases::utilities;
use crate::cloud_provider::aws::{common, AWS};
use crate::cloud_provider::environment::Environment;
use crate::cloud_provider::kubernetes::Kubernetes;
use crate::cloud_provider::service::{
Action, Backup, Create, Database, DatabaseOptions, DatabaseType, Delete, Downgrade, Pause,
Service, ServiceError, ServiceType, StatefulService, Upgrade,
};
use crate::cloud_provider::DeploymentTarget;
use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY};
use crate::models::Context;
pub struct MongoDB {
context: Context,
id: String,
action: Action,
name: String,
version: String,
fqdn: String,
fqdn_id: String,
total_cpus: String,
total_ram_in_mib: u32,
database_instance_type: String,
options: DatabaseOptions,
}
impl MongoDB {
pub fn new(
context: Context,
id: &str,
action: Action,
name: &str,
version: &str,
fqdn: &str,
fqdn_id: &str,
total_cpus: String,
total_ram_in_mib: u32,
database_instance_type: &str,
options: DatabaseOptions,
) -> Self {
MongoDB {
context,
action,
id: id.to_string(),
name: name.to_string(),
version: version.to_string(),
fqdn: fqdn.to_string(),
fqdn_id: fqdn_id.to_string(),
total_cpus,
total_ram_in_mib,
database_instance_type: database_instance_type.to_string(),
options,
}
}
fn helm_release_name(&self) -> String {
crate::string::cut(format!("mongodb-{}", self.id()), 50)
}
fn helm_release_external_dns(&self) -> String {
format!("{}-dns", self.helm_release_name())
}
fn workspace_directory(&self) -> String {
crate::fs::workspace_directory(
self.context.workspace_root_dir(),
self.context.execution_id(),
format!("databases/{}", self.name()),
)
}
fn tera_context(&self, kubernetes: &dyn Kubernetes, environment: &Environment) -> TeraContext {
let mut context = self.default_tera_context(kubernetes, environment);
// FIXME: is there an other way than downcast a pointer?
let cp = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.expect("Could not downcast kubernetes.cloud_provider() to AWS");
// we need the kubernetes config file to store tfstates file in kube secrets
let kubernetes_config_file_path = utilities::get_kubernetes_config_path(
self.workspace_directory().as_str(),
kubernetes,
environment,
);
match kubernetes_config_file_path {
Ok(kube_config) => {
context.insert("kubeconfig_path", &kube_config.as_str());
let aws = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.unwrap();
utilities::create_namespace(&environment.namespace(), kube_config.as_str(), aws);
}
Err(e) => error!("Failed to generate the kubernetes config file path: {}", e),
}
context.insert("namespace", environment.namespace());
context.insert("aws_access_key", &cp.access_key_id);
context.insert("aws_secret_key", &cp.secret_access_key);
context.insert("eks_cluster_id", kubernetes.id());
context.insert("eks_cluster_name", kubernetes.name());
context.insert("fqdn_id", self.fqdn_id.as_str());
context.insert("fqdn", self.fqdn.as_str());
context.insert("database_login", self.options.login.as_str());
context.insert("database_password", self.options.password.as_str());
context.insert("database_port", &self.private_port());
context.insert("database_disk_size_in_gib", &self.options.disk_size_in_gib);
context.insert("database_instance_type", &self.database_instance_type);
context.insert("database_disk_type", &self.options.database_disk_type);
context.insert("database_ram_size_in_mib", &self.total_ram_in_mib);
context.insert("database_total_cpus", &self.total_cpus);
context.insert("database_fqdn", &self.options.host.as_str());
context.insert("database_id", &self.id());
context
}
fn delete(&self, target: &DeploymentTarget, is_error: bool) -> Result<(), ServiceError> {
let workspace_dir = self.workspace_directory();
match target {
DeploymentTarget::ManagedServices(kubernetes, environment) => {
if is_error {
// do not delete if it is an error
return Ok(());
}
let context = self.tera_context(*kubernetes, *environment);
crate::template::generate_and_copy_all_files_into_dir(
format!("{}/aws/services/common", self.context.lib_root_dir()).as_str(),
&workspace_dir,
&context,
)?;
crate::template::generate_and_copy_all_files_into_dir(
format!("{}/aws/services/mongodb", self.context.lib_root_dir()).as_str(),
workspace_dir.as_str(),
&context,
)?;
crate::template::generate_and_copy_all_files_into_dir(
format!(
"{}/aws/charts/external-name-svc",
self.context.lib_root_dir()
)
.as_str(),
format!("{}/{}", workspace_dir, "external-name-svc").as_str(),
&context,
)?;
crate::template::generate_and_copy_all_files_into_dir(
format!(
"{}/aws/charts/external-name-svc",
self.context.lib_root_dir()
)
.as_str(),
workspace_dir.as_str(),
&context,
)?;
match crate::cmd::terraform::terraform_exec_with_init_validate_destroy(
workspace_dir.as_str(),
) {
Ok(o) => {
info!("Deleting secrets containing tfstates");
utilities::delete_terraform_tfstate_secret(
*kubernetes,
environment,
self.workspace_directory().as_str(),
);
}
//TODO: find a way to raise the error
Err(e) => error!("Error while destroying infrastructure {}", e),
}
}
DeploymentTarget::SelfHosted(kubernetes, environment) => {
let helm_release_name = self.helm_release_name();
let selector = format!("app={}", self.name());
if is_error {
let _ = common::get_stateless_resource_information(
*kubernetes,
*environment,
workspace_dir.as_str(),
selector.as_str(),
)?;
}
// clean the resource
let _ = common::do_stateless_service_cleanup(
*kubernetes,
*environment,
workspace_dir.as_str(),
helm_release_name.as_str(),
)?;
}
}
Ok(())
}
}
impl StatefulService for MongoDB {}
impl Service for MongoDB {
fn context(&self) -> &Context {
&self.context
}
fn service_type(&self) -> ServiceType {
ServiceType::Database(DatabaseType::MongoDB(&self.options))
}
fn id(&self) -> &str {
self.id.as_str()
}
fn name(&self) -> &str {
self.name.as_str()
}
fn version(&self) -> &str {
self.version.as_str()
}
fn action(&self) -> &Action {
&self.action
}
fn private_port(&self) -> Option<u16> {
Some(self.options.port)
}
fn total_cpus(&self) -> String {
self.total_cpus.to_string()
}
fn total_ram_in_mib(&self) -> u32 {
self.total_ram_in_mib
}
fn total_instances(&self) -> u16 {
1
}
}
impl Database for MongoDB {}
impl Create for MongoDB {
fn on_create(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
info!("AWS.MongoDB.on_create() called for {}", self.name());
let workspace_dir = self.workspace_directory();
match target {
DeploymentTarget::ManagedServices(kubernetes, environment) => {
// use terraform
info!("deploy mongodb on AWS DocumentDB for {}", self.name());
let context = self.tera_context(*kubernetes, *environment);
let workspace_dir = self.workspace_directory();
crate::template::generate_and_copy_all_files_into_dir(
format!("{}/aws/services/common", self.context.lib_root_dir()).as_str(),
&workspace_dir,
&context,
)?;
crate::template::generate_and_copy_all_files_into_dir(
format!("{}/aws/services/mongodb", self.context.lib_root_dir()).as_str(),
workspace_dir.as_str(),
&context,
)?;
crate::template::generate_and_copy_all_files_into_dir(
format!(
"{}/aws/charts/external-name-svc",
self.context.lib_root_dir()
)
.as_str(),
format!("{}/{}", workspace_dir, "external-name-svc").as_str(),
&context,
)?;
// deploy database + external DNS
crate::cmd::terraform::terraform_exec_with_init_validate_plan_apply(
workspace_dir.as_str(),
false,
)?;
}
DeploymentTarget::SelfHosted(kubernetes, environment) => {
// use helm
info!("deploy MongoDB on Kubernetes for {}", self.name());
let context = self.tera_context(*kubernetes, *environment);
let aws = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.unwrap();
let kubernetes_config_file_path = common::kubernetes_config_path(
workspace_dir.as_str(),
environment.organization_id.as_str(),
kubernetes.id(),
aws.access_key_id.as_str(),
aws.secret_access_key.as_str(),
kubernetes.region(),
)?;
let from_dir = format!("{}/common/services/mongodb", self.context.lib_root_dir());
let _ = crate::template::generate_and_copy_all_files_into_dir(
from_dir.as_str(),
workspace_dir.as_str(),
&context,
)?;
// render templates
let helm_release_name = self.helm_release_name();
let aws_credentials_envs = vec![
(AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()),
(AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()),
];
// do exec helm upgrade and return the last deployment status
let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history(
kubernetes_config_file_path.as_str(),
environment.namespace(),
helm_release_name.as_str(),
workspace_dir.as_str(),
aws_credentials_envs.clone(),
)?;
// check deployment status
if helm_history_row.is_none()
|| !helm_history_row.unwrap().is_successfully_deployed()
{
return Err(ServiceError::OnCreateFailed);
}
// check app status
let selector = format!("app={}", self.name());
match crate::cmd::kubectl::kubectl_exec_is_pod_ready_with_retry(
kubernetes_config_file_path.as_str(),
environment.namespace(),
selector.as_str(),
aws_credentials_envs,
) {
Ok(Some(true)) => {}
_ => return Err(ServiceError::OnCreateFailed),
}
}
}
Ok(())
}
fn on_create_check(&self) -> Result<(), ServiceError> {
Ok(())
}
fn on_create_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!("AWS.MongoDB.on_create_error() called for {}", self.name());
self.delete(target, true)
}
}
impl Pause for MongoDB {
fn on_pause(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
info!("AWS.MongoDB.on_pause() called for {}", self.name());
// TODO how to pause production? - the goal is to reduce cost, but it is possible to pause a production env?
// TODO how to pause development? - the goal is also to reduce cost, we can set the number of instances to 0, which will avoid to delete data :)
Ok(())
}
fn on_pause_check(&self) -> Result<(), ServiceError> {
Ok(())
}
fn on_pause_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!("AWS.MongoDB.on_pause_error() called for {}", self.name());
// TODO what to do if there is a pause error?
Ok(())
}
}
impl Delete for MongoDB {
fn on_delete(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
info!("AWS.MongoDB.on_delete() called for {}", self.name());
self.delete(target, false)
}
fn on_delete_check(&self) -> Result<(), ServiceError> {
Ok(())
}
fn on_delete_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!("AWS.MongoDB.on_create_error() called for {}", self.name());
self.delete(target, true)
}
}
impl crate::cloud_provider::service::Clone for MongoDB {
fn on_clone(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_clone_check(&self) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_clone_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
}
impl Upgrade for MongoDB {
fn on_upgrade(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_upgrade_check(&self) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_upgrade_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
}
impl Downgrade for MongoDB {
fn on_downgrade(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_downgrade_check(&self) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_downgrade_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
}
impl Backup for MongoDB {
fn on_backup(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_backup_check(&self) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_backup_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_restore(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_restore_check(&self) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_restore_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
}

View File

@@ -0,0 +1,472 @@
use tera::Context as TeraContext;
use crate::cloud_provider::aws::databases::utilities;
use crate::cloud_provider::aws::kubernetes::EKS;
use crate::cloud_provider::aws::{common, AWS};
use crate::cloud_provider::environment::Environment;
use crate::cloud_provider::kubernetes::Kubernetes;
use crate::cloud_provider::service::{
Action, Backup, Create, DatabaseOptions, DatabaseType, Delete, Downgrade, Pause, Service,
ServiceError, ServiceType, StatefulService, Upgrade,
};
use crate::cloud_provider::DeploymentTarget;
use crate::cmd::kubectl::{
kubectl_exec_create_namespace, kubectl_exec_delete_namespace, kubectl_exec_delete_secret,
};
use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY};
use crate::models::Context;
pub struct MySQL {
context: Context,
id: String,
action: Action,
name: String,
version: String,
fqdn: String,
fqdn_id: String,
total_cpus: String,
total_ram_in_mib: u32,
database_instance_type: String,
options: DatabaseOptions,
}
impl MySQL {
pub fn new(
context: Context,
id: &str,
action: Action,
name: &str,
version: &str,
fqdn: &str,
fqdn_id: &str,
total_cpus: String,
total_ram_in_mib: u32,
database_instance_type: &str,
options: DatabaseOptions,
) -> Self {
Self {
context,
action,
id: id.to_string(),
name: name.to_string(),
version: version.to_string(),
fqdn: fqdn.to_string(),
fqdn_id: fqdn_id.to_string(),
total_cpus,
total_ram_in_mib,
database_instance_type: database_instance_type.to_string(),
options,
}
}
fn helm_release_name(&self) -> String {
crate::string::cut(format!("mysql-{}", self.id()), 50)
}
fn workspace_directory(&self) -> String {
crate::fs::workspace_directory(
self.context.workspace_root_dir(),
self.context.execution_id(),
format!("databases/{}", self.name()),
)
}
fn tera_context(&self, kubernetes: &dyn Kubernetes, environment: &Environment) -> TeraContext {
let mut context = self.default_tera_context(kubernetes, environment);
// FIXME: is there an other way than downcast a pointer?
let cp = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.expect("Could not downcast kubernetes.cloud_provider() to AWS");
// we need the kubernetes config file to store tfstates file in kube secrets
let kubernetes_config_file_path = utilities::get_kubernetes_config_path(
self.workspace_directory().as_str(),
kubernetes,
environment,
);
match kubernetes_config_file_path {
Ok(kube_config) => {
context.insert("kubeconfig_path", &kube_config.as_str());
let aws = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.unwrap();
utilities::create_namespace(&environment.namespace(), kube_config.as_str(), aws);
}
Err(e) => error!("Failed to generate the kubernetes config file path: {}", e),
}
context.insert("namespace", environment.namespace());
context.insert("aws_access_key", &cp.access_key_id);
context.insert("aws_secret_key", &cp.secret_access_key);
context.insert("eks_cluster_id", kubernetes.id());
context.insert("eks_cluster_name", kubernetes.name());
context.insert("fqdn_id", self.fqdn_id.as_str());
context.insert("fqdn", self.fqdn.as_str());
context.insert("database_login", self.options.login.as_str());
context.insert("database_password", self.options.password.as_str());
context.insert("database_port", &self.private_port());
context.insert("database_disk_size_in_gib", &self.options.disk_size_in_gib);
context.insert("database_instance_type", &self.database_instance_type);
context.insert("database_disk_type", &self.options.database_disk_type);
context.insert("database_ram_size_in_mib", &self.total_ram_in_mib);
context.insert("database_total_cpus", &self.total_cpus);
context.insert("database_fqdn", &self.options.host.as_str());
context.insert("database_id", &self.id());
context
}
fn delete(&self, target: &DeploymentTarget, is_error: bool) -> Result<(), ServiceError> {
let workspace_dir = self.workspace_directory();
match target {
DeploymentTarget::ManagedServices(kubernetes, environment) => {
if is_error {
// do not delete if it is an error
return Ok(());
}
let context = self.tera_context(*kubernetes, *environment);
crate::template::generate_and_copy_all_files_into_dir(
format!("{}/aws/services/common", self.context.lib_root_dir()).as_str(),
&workspace_dir,
&context,
)?;
crate::template::generate_and_copy_all_files_into_dir(
format!("{}/aws/services/mysql", self.context.lib_root_dir()).as_str(),
workspace_dir.as_str(),
&context,
)?;
crate::template::generate_and_copy_all_files_into_dir(
format!(
"{}/aws/charts/external-name-svc",
self.context.lib_root_dir()
)
.as_str(),
format!("{}/{}", workspace_dir, "external-name-svc").as_str(),
&context,
)?;
crate::template::generate_and_copy_all_files_into_dir(
format!(
"{}/aws/charts/external-name-svc",
self.context.lib_root_dir()
)
.as_str(),
workspace_dir.as_str(),
&context,
)?;
match crate::cmd::terraform::terraform_exec_with_init_validate_destroy(
workspace_dir.as_str(),
) {
Ok(o) => {
info!("Deleting secrets containing tfstates");
utilities::delete_terraform_tfstate_secret(
*kubernetes,
environment,
self.workspace_directory().as_str(),
);
}
//TODO: find a way to raise the error
Err(e) => error!("Error while destroying infrastructure {}", e),
}
}
DeploymentTarget::SelfHosted(kubernetes, environment) => {
let helm_release_name = self.helm_release_name();
let selector = format!("app={}", self.name());
if is_error {
let _ = common::get_stateless_resource_information(
*kubernetes,
*environment,
workspace_dir.as_str(),
selector.as_str(),
)?;
}
// clean the resource
let _ = common::do_stateless_service_cleanup(
*kubernetes,
*environment,
workspace_dir.as_str(),
helm_release_name.as_str(),
)?;
}
}
Ok(())
}
}
impl StatefulService for MySQL {}
impl Service for MySQL {
fn context(&self) -> &Context {
&self.context
}
fn service_type(&self) -> ServiceType {
ServiceType::Database(DatabaseType::MySQL(&self.options))
}
fn id(&self) -> &str {
self.id.as_str()
}
fn name(&self) -> &str {
self.name.as_str()
}
fn version(&self) -> &str {
self.version.as_str()
}
fn action(&self) -> &Action {
&self.action
}
fn private_port(&self) -> Option<u16> {
Some(self.options.port)
}
fn total_cpus(&self) -> String {
self.total_cpus.to_string()
}
fn total_ram_in_mib(&self) -> u32 {
self.total_ram_in_mib
}
fn total_instances(&self) -> u16 {
1
}
}
impl Create for MySQL {
fn on_create(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
match target {
DeploymentTarget::ManagedServices(kubernetes, environment) => {
// use terraform
info!("deploy MySQL on AWS RDS for {}", self.name());
let context = self.tera_context(*kubernetes, *environment);
let workspace_dir = self.workspace_directory();
crate::template::generate_and_copy_all_files_into_dir(
format!("{}/aws/services/common", self.context.lib_root_dir()).as_str(),
&workspace_dir,
&context,
)?;
crate::template::generate_and_copy_all_files_into_dir(
format!("{}/aws/services/mysql", self.context.lib_root_dir()).as_str(),
workspace_dir.as_str(),
&context,
)?;
crate::template::generate_and_copy_all_files_into_dir(
format!(
"{}/aws/charts/external-name-svc",
self.context.lib_root_dir()
)
.as_str(),
format!("{}/{}", workspace_dir, "external-name-svc").as_str(),
&context,
)?;
crate::cmd::terraform::terraform_exec_with_init_validate_plan_apply(
workspace_dir.as_str(),
false,
)?;
}
DeploymentTarget::SelfHosted(kubernetes, environment) => {
// use helm
info!("deploy MySQL on Kubernetes for {}", self.name());
let context = self.tera_context(*kubernetes, *environment);
let workspace_dir = self.workspace_directory();
let aws = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.unwrap();
let kubernetes_config_file_path = common::kubernetes_config_path(
workspace_dir.as_str(),
environment.organization_id.as_str(),
kubernetes.id(),
aws.access_key_id.as_str(),
aws.secret_access_key.as_str(),
kubernetes.region(),
)?;
let from_dir = format!("{}/common/services/mysql", self.context.lib_root_dir());
let _ = crate::template::generate_and_copy_all_files_into_dir(
from_dir.as_str(),
workspace_dir.as_str(),
&context,
)?;
// render templates
let helm_release_name = self.helm_release_name();
let aws_credentials_envs = vec![
(AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()),
(AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()),
];
// do exec helm upgrade and return the last deployment status
let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history(
kubernetes_config_file_path.as_str(),
environment.namespace(),
helm_release_name.as_str(),
workspace_dir.as_str(),
aws_credentials_envs.clone(),
)?;
// check deployment status
if helm_history_row.is_none()
|| !helm_history_row.unwrap().is_successfully_deployed()
{
return Err(ServiceError::OnCreateFailed);
}
// check app status
let selector = format!("app={}", self.name());
match crate::cmd::kubectl::kubectl_exec_is_pod_ready_with_retry(
kubernetes_config_file_path.as_str(),
environment.namespace(),
selector.as_str(),
aws_credentials_envs,
) {
Ok(Some(true)) => {}
_ => return Err(ServiceError::OnCreateFailed),
}
}
}
Ok(())
}
fn on_create_check(&self) -> Result<(), ServiceError> {
//FIXME : perform an actual check
Ok(())
}
fn on_create_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!("AWS.MySQL.on_create_error() called for {}", self.name());
self.delete(target, true)
}
}
impl Pause for MySQL {
fn on_pause(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
info!("AWS.MySQL.on_pause() called for {}", self.name());
// TODO how to pause production? - the goal is to reduce cost, but it is possible to pause a production env?
// TODO how to pause development? - the goal is also to reduce cost, we can set the number of instances to 0, which will avoid to delete data :)
Ok(())
}
fn on_pause_check(&self) -> Result<(), ServiceError> {
Ok(())
}
fn on_pause_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!("AWS.MySQL.on_pause_error() called for {}", self.name());
// TODO what to do if there is a pause error?
Ok(())
}
}
impl Delete for MySQL {
fn on_delete(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
info!("AWS.MySQL.on_delete() called for {}", self.name());
self.delete(target, false)
}
fn on_delete_check(&self) -> Result<(), ServiceError> {
Ok(())
}
fn on_delete_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!("AWS.MySQL.on_create_error() called for {}", self.name());
self.delete(target, true)
}
}
impl crate::cloud_provider::service::Clone for MySQL {
fn on_clone(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_clone_check(&self) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_clone_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
}
impl Upgrade for MySQL {
fn on_upgrade(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_upgrade_check(&self) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_upgrade_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
}
impl Downgrade for MySQL {
fn on_downgrade(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_downgrade_check(&self) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_downgrade_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
}
impl Backup for MySQL {
fn on_backup(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_backup_check(&self) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_backup_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_restore(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_restore_check(&self) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_restore_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
}

View File

@@ -0,0 +1,482 @@
use tera::Context as TeraContext;
use crate::cloud_provider::aws::databases::utilities;
use crate::cloud_provider::aws::{common, AWS};
use crate::cloud_provider::environment::Environment;
use crate::cloud_provider::kubernetes::Kubernetes;
use crate::cloud_provider::service::{
Action, Backup, Create, Database, DatabaseOptions, DatabaseType, Delete, Downgrade, Pause,
Service, ServiceError, ServiceType, StatefulService, Upgrade,
};
use crate::cloud_provider::DeploymentTarget;
use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY};
use crate::models::Context;
pub struct PostgreSQL {
context: Context,
id: String,
action: Action,
name: String,
version: String,
fqdn: String,
fqdn_id: String,
total_cpus: String,
total_ram_in_mib: u32,
database_instance_type: String,
options: DatabaseOptions,
}
impl PostgreSQL {
pub fn new(
context: Context,
id: &str,
action: Action,
name: &str,
version: &str,
fqdn: &str,
fqdn_id: &str,
total_cpus: String,
total_ram_in_mib: u32,
database_instance_type: &str,
options: DatabaseOptions,
) -> Self {
PostgreSQL {
context,
action,
id: id.to_string(),
name: name.to_string(),
version: version.to_string(),
fqdn: fqdn.to_string(),
fqdn_id: fqdn_id.to_string(),
total_cpus,
total_ram_in_mib,
database_instance_type: database_instance_type.to_string(),
options,
}
}
fn helm_release_name(&self) -> String {
crate::string::cut(format!("postgresql-{}", self.id()), 50)
}
fn workspace_directory(&self) -> String {
crate::fs::workspace_directory(
self.context.workspace_root_dir(),
self.context.execution_id(),
format!("databases/{}", self.name()),
)
}
fn tera_context(&self, kubernetes: &dyn Kubernetes, environment: &Environment) -> TeraContext {
let mut context = self.default_tera_context(kubernetes, environment);
// FIXME: is there an other way than downcast a pointer?
let cp = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.expect("Could not downcast kubernetes.cloud_provider() to AWS");
// we need the kubernetes config file to store tfstates file in kube secrets
let kubernetes_config_file_path = utilities::get_kubernetes_config_path(
self.workspace_directory().as_str(),
kubernetes,
environment,
);
match kubernetes_config_file_path {
Ok(kube_config) => {
context.insert("kubeconfig_path", &kube_config.as_str());
let aws = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.unwrap();
utilities::create_namespace(&environment.namespace(), kube_config.as_str(), aws);
}
Err(e) => error!("Failed to generate the kubernetes config file path: {}", e),
}
context.insert("namespace", environment.namespace());
context.insert("aws_access_key", &cp.access_key_id);
context.insert("aws_secret_key", &cp.secret_access_key);
context.insert("eks_cluster_id", kubernetes.id());
context.insert("eks_cluster_name", kubernetes.name());
context.insert("fqdn_id", self.fqdn_id.as_str());
context.insert("fqdn", self.fqdn.as_str());
context.insert("database_login", self.options.login.as_str());
context.insert("database_password", self.options.password.as_str());
context.insert("database_port", &self.private_port());
context.insert("database_disk_size_in_gib", &self.options.disk_size_in_gib);
context.insert("database_instance_type", &self.database_instance_type);
context.insert("database_disk_type", &self.options.database_disk_type);
context.insert("database_ram_size_in_mib", &self.total_ram_in_mib);
context.insert("database_total_cpus", &self.total_cpus);
context.insert("database_fqdn", &self.options.host.as_str());
context.insert("database_id", &self.id());
context
}
fn delete(&self, target: &DeploymentTarget, is_error: bool) -> Result<(), ServiceError> {
let workspace_dir = self.workspace_directory();
match target {
DeploymentTarget::ManagedServices(kubernetes, environment) => {
if is_error {
// do not delete if it is an error
return Ok(());
}
let context = self.tera_context(*kubernetes, *environment);
crate::template::generate_and_copy_all_files_into_dir(
format!("{}/aws/services/common", self.context.lib_root_dir()).as_str(),
&workspace_dir,
&context,
)?;
crate::template::generate_and_copy_all_files_into_dir(
format!("{}/aws/services/postgresql", self.context.lib_root_dir()).as_str(),
workspace_dir.as_str(),
&context,
)?;
crate::template::generate_and_copy_all_files_into_dir(
format!(
"{}/aws/charts/external-name-svc",
self.context.lib_root_dir()
)
.as_str(),
format!("{}/{}", workspace_dir, "external-name-svc").as_str(),
&context,
)?;
crate::template::generate_and_copy_all_files_into_dir(
format!(
"{}/aws/charts/external-name-svc",
self.context.lib_root_dir()
)
.as_str(),
workspace_dir.as_str(),
&context,
)?;
match crate::cmd::terraform::terraform_exec_with_init_validate_destroy(
workspace_dir.as_str(),
) {
Ok(o) => {
info!("Deleting secrets containing tfstates");
utilities::delete_terraform_tfstate_secret(
*kubernetes,
environment,
self.workspace_directory().as_str(),
);
}
//TODO: find a way to raise the error
Err(e) => error!("Error while destroying infrastructure {}", e),
}
}
DeploymentTarget::SelfHosted(kubernetes, environment) => {
let helm_release_name = self.helm_release_name();
let selector = format!("app={}", self.name());
if is_error {
let _ = common::get_stateless_resource_information(
*kubernetes,
*environment,
workspace_dir.as_str(),
selector.as_str(),
)?;
}
// clean the resource
let _ = common::do_stateless_service_cleanup(
*kubernetes,
*environment,
workspace_dir.as_str(),
helm_release_name.as_str(),
)?;
}
}
Ok(())
}
}
impl StatefulService for PostgreSQL {}
impl Service for PostgreSQL {
fn context(&self) -> &Context {
&self.context
}
fn service_type(&self) -> ServiceType {
ServiceType::Database(DatabaseType::PostgreSQL(&self.options))
}
fn id(&self) -> &str {
self.id.as_str()
}
fn name(&self) -> &str {
self.name.as_str()
}
fn version(&self) -> &str {
self.version.as_str()
}
fn action(&self) -> &Action {
&self.action
}
fn private_port(&self) -> Option<u16> {
Some(self.options.port)
}
fn total_cpus(&self) -> String {
self.total_cpus.to_string()
}
fn total_ram_in_mib(&self) -> u32 {
self.total_ram_in_mib
}
fn total_instances(&self) -> u16 {
1
}
}
impl Database for PostgreSQL {}
impl Create for PostgreSQL {
fn on_create(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
info!("AWS.PostgreSQL.on_create() called for {}", self.name());
let workspace_dir = self.workspace_directory();
match target {
DeploymentTarget::ManagedServices(kubernetes, environment) => {
// use terraform
info!("deploy postgresql on AWS RDS for {}", self.name());
let context = self.tera_context(*kubernetes, *environment);
let workspace_dir = self.workspace_directory();
crate::template::generate_and_copy_all_files_into_dir(
format!("{}/aws/services/common", self.context.lib_root_dir()).as_str(),
&workspace_dir,
&context,
)?;
crate::template::generate_and_copy_all_files_into_dir(
format!("{}/aws/services/postgresql", self.context.lib_root_dir()).as_str(),
workspace_dir.as_str(),
&context,
)?;
crate::template::generate_and_copy_all_files_into_dir(
format!(
"{}/aws/charts/external-name-svc",
self.context.lib_root_dir()
)
.as_str(),
format!("{}/{}", workspace_dir, "external-name-svc").as_str(),
&context,
)?;
crate::cmd::terraform::terraform_exec_with_init_validate_plan_apply(
workspace_dir.as_str(),
false,
)?;
}
DeploymentTarget::SelfHosted(kubernetes, environment) => {
// use helm
info!("deploy PostgreSQL on Kubernetes for {}", self.name());
let context = self.tera_context(*kubernetes, *environment);
let aws = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.unwrap();
let kubernetes_config_file_path = common::kubernetes_config_path(
workspace_dir.as_str(),
environment.organization_id.as_str(),
kubernetes.id(),
aws.access_key_id.as_str(),
aws.secret_access_key.as_str(),
kubernetes.region(),
)?;
let from_dir =
format!("{}/common/services/postgresql", self.context.lib_root_dir());
let _ = crate::template::generate_and_copy_all_files_into_dir(
from_dir.as_str(),
workspace_dir.as_str(),
&context,
)?;
// render templates
let helm_release_name = self.helm_release_name();
let aws_credentials_envs = vec![
(AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()),
(AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()),
];
// do exec helm upgrade and return the last deployment status
let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history(
kubernetes_config_file_path.as_str(),
environment.namespace(),
helm_release_name.as_str(),
workspace_dir.as_str(),
aws_credentials_envs.clone(),
)?;
// check deployment status
if helm_history_row.is_none()
|| !helm_history_row.unwrap().is_successfully_deployed()
{
return Err(ServiceError::OnCreateFailed);
}
// check app status
let selector = format!("app={}", self.name());
match crate::cmd::kubectl::kubectl_exec_is_pod_ready_with_retry(
kubernetes_config_file_path.as_str(),
environment.namespace(),
selector.as_str(),
aws_credentials_envs,
) {
Ok(Some(true)) => {}
_ => return Err(ServiceError::OnCreateFailed),
}
}
}
Ok(())
}
fn on_create_check(&self) -> Result<(), ServiceError> {
Ok(())
}
fn on_create_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!(
"AWS.PostgreSQL.on_create_error() called for {}",
self.name()
);
self.delete(target, true)
}
}
impl Pause for PostgreSQL {
fn on_pause(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
info!("AWS.PostgreSQL.on_pause() called for {}", self.name());
// TODO how to pause production? - the goal is to reduce cost, but it is possible to pause a production env?
// TODO how to pause development? - the goal is also to reduce cost, we can set the number of instances to 0, which will avoid to delete data :)
Ok(())
}
fn on_pause_check(&self) -> Result<(), ServiceError> {
Ok(())
}
fn on_pause_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!("AWS.PostgreSQL.on_pause_error() called for {}", self.name());
// TODO what to do if there is a pause error?
Ok(())
}
}
impl Delete for PostgreSQL {
fn on_delete(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
info!("AWS.PostgreSQL.on_delete() called for {}", self.name());
self.delete(target, false)
}
fn on_delete_check(&self) -> Result<(), ServiceError> {
Ok(())
}
fn on_delete_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!(
"AWS.PostgreSQL.on_create_error() called for {}",
self.name()
);
self.delete(target, true)
}
}
impl crate::cloud_provider::service::Clone for PostgreSQL {
fn on_clone(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_clone_check(&self) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_clone_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
}
impl Upgrade for PostgreSQL {
fn on_upgrade(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_upgrade_check(&self) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_upgrade_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
}
impl Downgrade for PostgreSQL {
fn on_downgrade(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_downgrade_check(&self) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_downgrade_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
}
impl Backup for PostgreSQL {
fn on_backup(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_backup_check(&self) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_backup_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_restore(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_restore_check(&self) -> Result<(), ServiceError> {
unimplemented!()
}
fn on_restore_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> {
unimplemented!()
}
}

View File

@@ -0,0 +1,74 @@
use tokio::io::Error;
use crate::cloud_provider::aws::{common, AWS};
use crate::cloud_provider::environment::Environment;
use crate::cloud_provider::kubernetes::Kubernetes;
use crate::cmd::kubectl::{kubectl_exec_create_namespace, kubectl_exec_delete_secret};
use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY};
// generate the kubernetes config path
pub fn get_kubernetes_config_path(
workspace: &str,
kubernetes: &dyn Kubernetes,
environment: &Environment,
) -> Result<String, Error> {
let aws = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.unwrap();
common::kubernetes_config_path(
workspace,
environment.organization_id.as_str(),
kubernetes.id(),
aws.access_key_id.as_str(),
aws.secret_access_key.as_str(),
kubernetes.region(),
)
}
pub fn create_namespace(namespace: &str, kube_config: &str, aws: &AWS) {
let aws_credentials_envs = vec![
(AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()),
(AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()),
];
kubectl_exec_create_namespace(kube_config, namespace, aws_credentials_envs);
}
pub fn delete_terraform_tfstate_secret(
kubernetes: &dyn Kubernetes,
environment: &Environment,
workspace_dir: &str,
) -> Result<(), Error> {
let aws = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.unwrap();
let aws_credentials_envs = vec![
(AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()),
(AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()),
];
let kubernetes_config_file_path = common::kubernetes_config_path(
workspace_dir,
environment.organization_id.as_str(),
kubernetes.id(),
aws.access_key_id.as_str(),
aws.secret_access_key.as_str(),
kubernetes.region(),
);
match kubernetes_config_file_path {
Ok(kube_config) => {
//create the namespace to insert the tfstate in secrets
kubectl_exec_delete_secret(kube_config, "tfstate-default-state", aws_credentials_envs);
Ok(())
}
Err(e) => {
error!("Failed to generate the kubernetes config file path: {}", e);
Err(e)
}
}
}

View File

@@ -0,0 +1,364 @@
use serde::{Deserialize, Serialize};
use tera::Context as TeraContext;
use crate::build_platform::Image;
use crate::cloud_provider::aws::{common, AWS};
use crate::cloud_provider::environment::Environment;
use crate::cloud_provider::kubernetes::Kubernetes;
use crate::cloud_provider::service::{
Action, Application, Create, Delete, Pause, Service, ServiceError, ServiceType,
StatelessService,
};
use crate::cloud_provider::DeploymentTarget;
use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY};
use crate::models::Context;
#[derive(Clone, Eq, PartialEq, Hash)]
pub struct ExternalService {
context: Context,
id: String,
action: Action,
name: String,
total_cpus: String,
total_ram_in_mib: u32,
image: Image,
environment_variables: Vec<EnvironmentVariable>,
}
impl ExternalService {
pub fn new(
context: Context,
id: &str,
action: Action,
name: &str,
total_cpus: String,
total_ram_in_mib: u32,
image: Image,
environment_variables: Vec<EnvironmentVariable>,
) -> Self {
ExternalService {
context,
id: id.to_string(),
action,
name: name.to_string(),
total_cpus,
total_ram_in_mib,
image,
environment_variables,
}
}
fn helm_release_name(&self) -> String {
crate::string::cut(
format!("external-service-{}-{}", self.name(), self.id()),
50,
)
}
fn workspace_directory(&self) -> String {
crate::fs::workspace_directory(
self.context.workspace_root_dir(),
self.context.execution_id(),
format!("external-services/{}", self.name()),
)
}
fn context(&self, kubernetes: &dyn Kubernetes, environment: &Environment) -> TeraContext {
let mut context = self.default_tera_context(kubernetes, environment);
let commit_id = self.image().commit_id.as_str();
context.insert("helm_app_version", &commit_id[..7]);
match &self.image().registry_url {
Some(registry_url) => context.insert("image_name_with_tag", registry_url.as_str()),
None => {
let image_name_with_tag = self.image().name_with_tag();
warn!("there is no registry url, use image name with tag with the default container registry: {}", image_name_with_tag.as_str());
context.insert("image_name_with_tag", image_name_with_tag.as_str());
}
}
let environment_variables = self
.environment_variables
.iter()
.map(|ev| EnvironmentVariableDataTemplate {
key: ev.key.clone(),
value: ev.value.clone(),
})
.collect::<Vec<_>>();
context.insert("environment_variables", &environment_variables);
context
}
fn delete(&self, target: &DeploymentTarget, is_error: bool) -> Result<(), ServiceError> {
let (kubernetes, environment) = match target {
DeploymentTarget::ManagedServices(k, env) => (*k, *env),
DeploymentTarget::SelfHosted(k, env) => (*k, *env),
};
let workspace_dir = self.workspace_directory();
let helm_release_name = self.helm_release_name();
let selector = format!("app={}", self.name());
if is_error {
let _ = common::get_stateless_resource_information(
kubernetes,
environment,
workspace_dir.as_str(),
selector.as_str(),
)?;
}
// clean the resource
let _ = common::do_stateless_service_cleanup(
kubernetes,
environment,
workspace_dir.as_str(),
helm_release_name.as_str(),
)?;
Ok(())
}
}
impl crate::cloud_provider::service::ExternalService for ExternalService {}
impl crate::cloud_provider::service::Application for ExternalService {
fn image(&self) -> &Image {
&self.image
}
fn set_image(&mut self, image: Image) {
self.image = image;
}
}
impl StatelessService for ExternalService {}
impl Service for ExternalService {
fn context(&self) -> &Context {
&self.context
}
fn service_type(&self) -> ServiceType {
ServiceType::ExternalService
}
fn id(&self) -> &str {
self.id.as_str()
}
fn name(&self) -> &str {
self.name.as_str()
}
fn version(&self) -> &str {
self.image.commit_id.as_str()
}
fn action(&self) -> &Action {
&self.action
}
fn private_port(&self) -> Option<u16> {
None
}
fn total_cpus(&self) -> String {
self.total_cpus.to_string()
}
fn total_ram_in_mib(&self) -> u32 {
self.total_ram_in_mib
}
fn total_instances(&self) -> u16 {
1
}
}
impl Create for ExternalService {
fn on_create(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
info!(
"AWS.external_service.on_create() called for {}",
self.name()
);
let (kubernetes, environment) = match target {
DeploymentTarget::ManagedServices(k, env) => (*k, *env),
DeploymentTarget::SelfHosted(k, env) => (*k, *env),
};
let aws = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.unwrap();
let context = self.context(kubernetes, environment);
let workspace_dir = self.workspace_directory();
let from_dir = format!("{}/common/services/q-job", self.context.lib_root_dir());
let _ = crate::template::generate_and_copy_all_files_into_dir(
from_dir.as_str(),
workspace_dir.as_str(),
&context,
)?;
// render
// TODO check the rendered files?
let helm_release_name = self.helm_release_name();
let aws_credentials_envs = vec![
(AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()),
(AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()),
];
let kubernetes_config_file_path = common::kubernetes_config_path(
workspace_dir.as_str(),
environment.organization_id.as_str(),
kubernetes.id(),
aws.access_key_id.as_str(),
aws.secret_access_key.as_str(),
kubernetes.region(),
)?;
// do exec helm upgrade and return the last deployment status
let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history(
kubernetes_config_file_path.as_str(),
environment.namespace(),
helm_release_name.as_str(),
workspace_dir.as_str(),
aws_credentials_envs.clone(),
)?;
// check deployment status
if helm_history_row.is_none() || !helm_history_row.unwrap().is_successfully_deployed() {
// TODO get pod output by using kubectl and return it into the OnCreateFailed
return Err(ServiceError::OnCreateFailed);
}
// check job status
match crate::cmd::kubectl::kubectl_exec_is_job_ready_with_retry(
kubernetes_config_file_path.as_str(),
environment.namespace(),
self.name.as_str(),
aws_credentials_envs,
) {
Ok(Some(true)) => {}
_ => return Err(ServiceError::OnCreateFailed),
}
Ok(())
}
fn on_create_check(&self) -> Result<(), ServiceError> {
Ok(())
}
fn on_create_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!(
"AWS.external_service.on_create_error() called for {}",
self.name()
);
let (kubernetes, environment) = match target {
DeploymentTarget::ManagedServices(k, env) => (*k, *env),
DeploymentTarget::SelfHosted(k, env) => (*k, *env),
};
let workspace_dir = self.workspace_directory();
let aws = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.unwrap();
let aws_credentials_envs = vec![
(AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()),
(AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()),
];
let kubernetes_config_file_path = common::kubernetes_config_path(
workspace_dir.as_str(),
environment.organization_id.as_str(),
kubernetes.id(),
aws.access_key_id.as_str(),
aws.secret_access_key.as_str(),
kubernetes.region(),
)?;
let helm_release_name = self.helm_release_name();
let history_rows = crate::cmd::helm::helm_exec_history(
kubernetes_config_file_path.as_str(),
environment.namespace(),
helm_release_name.as_str(),
aws_credentials_envs.clone(),
)?;
if history_rows.len() == 1 {
crate::cmd::helm::helm_exec_uninstall(
kubernetes_config_file_path.as_str(),
environment.namespace(),
helm_release_name.as_str(),
aws_credentials_envs.clone(),
)?;
}
Ok(())
}
}
impl Pause for ExternalService {
fn on_pause(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
info!("AWS.external_service.on_pause() called for {}", self.name());
self.delete(target, false)
}
fn on_pause_check(&self) -> Result<(), ServiceError> {
Ok(())
}
fn on_pause_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!(
"AWS.external_service.on_pause_error() called for {}",
self.name()
);
self.delete(target, true)
}
}
impl Delete for ExternalService {
fn on_delete(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
info!(
"AWS.external_service.on_delete() called for {}",
self.name()
);
self.delete(target, false)
}
fn on_delete_check(&self) -> Result<(), ServiceError> {
Ok(())
}
fn on_delete_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!(
"AWS.external_service.on_delete_error() called for {}",
self.name()
);
self.delete(target, true)
}
}
#[derive(Clone, Eq, PartialEq, Hash)]
pub struct EnvironmentVariable {
pub key: String,
pub value: String,
}
#[derive(Serialize, Deserialize)]
struct EnvironmentVariableDataTemplate {
pub key: String,
pub value: String,
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,97 @@
use std::any::Any;
use crate::cloud_provider::kubernetes::KubernetesNode;
pub struct Node {
total_cpu: u8,
total_memory_in_gib: u16,
instance_types_table: [(u8, u16, &'static str); 6],
}
impl Node {
/// Number of CPUs and total memory wanted - the right AWS EC2 instance type is found algorithmically
/// Eg. total_cpu = 1 and total_memory_in_gib = 2 means `t2.small` instance type
/// BUT total_cpu = 1 and total_memory_in_gib = 3 does not have an existing instance - so we will pick the upper closest,
/// which is `t2.medium` with 2 cpu and 4 GiB
/// ```
/// use qovery_engine::cloud_provider::aws::kubernetes::node::Node;
/// use qovery_engine::cloud_provider::kubernetes::KubernetesNode;
///
/// let node = Node::new(2, 4);
/// assert_eq!(node.instance_type(), "t2.medium")
/// ```
pub fn new(total_cpu: u8, total_memory_in_gib: u16) -> Self {
let instance_types_table = [
(1, 1, "t2.micro"),
(1, 2, "t2.small"),
(2, 4, "t2.medium"),
(2, 8, "t2.large"),
(4, 16, "t2.xlarge"),
(8, 32, "t2.2xlarge"),
// TODO add other instance types
];
Node {
total_cpu,
total_memory_in_gib,
instance_types_table,
}
}
}
impl KubernetesNode for Node {
fn total_cpu(&self) -> u8 {
self.total_cpu
}
fn total_memory_in_gib(&self) -> u16 {
self.total_memory_in_gib
}
fn instance_type(&self) -> &str {
if self.total_cpu() == 0 || self.total_memory_in_gib() == 0 {
let (_, _, instance_type) = self.instance_types_table.first().unwrap();
return instance_type;
}
for (_cpu, mem, instance_type) in self.instance_types_table.iter() {
if self.total_memory_in_gib() <= *mem {
return instance_type;
}
}
let (_, _, instance_type) = self.instance_types_table.last().unwrap();
return instance_type;
}
fn as_any(&self) -> &dyn Any {
self
}
}
#[cfg(test)]
mod tests {
use crate::cloud_provider::aws::kubernetes::node::Node;
use crate::cloud_provider::kubernetes::KubernetesNode;
#[test]
fn test_instance_types() {
assert_eq!(Node::new(0, 0).instance_type(), "t2.micro");
assert_eq!(Node::new(1, 0).instance_type(), "t2.micro");
assert_eq!(Node::new(0, 1).instance_type(), "t2.micro");
assert_eq!(Node::new(1, 1).instance_type(), "t2.micro");
assert_eq!(Node::new(1, 2).instance_type(), "t2.small");
assert_eq!(Node::new(2, 4).instance_type(), "t2.medium");
assert_eq!(Node::new(2, 5).instance_type(), "t2.large");
assert_eq!(Node::new(1, 6).instance_type(), "t2.large");
assert_eq!(Node::new(1, 7).instance_type(), "t2.large");
assert_eq!(Node::new(2, 8).instance_type(), "t2.large");
assert_eq!(Node::new(3, 8).instance_type(), "t2.large");
assert_eq!(Node::new(3, 10).instance_type(), "t2.xlarge");
assert_eq!(Node::new(3, 12).instance_type(), "t2.xlarge");
assert_eq!(Node::new(4, 16).instance_type(), "t2.xlarge");
assert_eq!(Node::new(4, 17).instance_type(), "t2.2xlarge");
assert_eq!(Node::new(8, 32).instance_type(), "t2.2xlarge");
assert_eq!(Node::new(16, 64).instance_type(), "t2.2xlarge");
}
}

View File

@@ -0,0 +1,109 @@
use std::any::Any;
use std::rc::Rc;
use rusoto_core::{Client, HttpClient, Region};
use rusoto_credential::StaticProvider;
use rusoto_sts::{GetCallerIdentityRequest, Sts, StsClient};
use crate::cloud_provider::{CloudProvider, CloudProviderError, Kind, TerraformStateCredentials};
use crate::models::{Context, Listener, Listeners, ProgressListener};
use crate::runtime::async_run;
mod common;
pub mod application;
pub mod databases;
pub mod external_service;
pub mod kubernetes;
pub mod router;
pub struct AWS {
context: Context,
id: String,
organization_id: String,
name: String,
pub access_key_id: String,
pub secret_access_key: String,
terraform_state_credentials: TerraformStateCredentials,
listeners: Listeners,
}
impl AWS {
pub fn new(
context: Context,
id: &str,
organization_id: &str,
name: &str,
access_key_id: &str,
secret_access_key: &str,
terraform_state_credentials: TerraformStateCredentials,
) -> Self {
AWS {
context,
id: id.to_string(),
organization_id: organization_id.to_string(),
name: name.to_string(),
access_key_id: access_key_id.to_string(),
secret_access_key: secret_access_key.to_string(),
terraform_state_credentials,
listeners: vec![],
}
}
pub fn credentials(&self) -> StaticProvider {
StaticProvider::new(
self.access_key_id.to_string(),
self.secret_access_key.to_string(),
None,
None,
)
}
pub fn client(&self) -> Client {
Client::new_with(self.credentials(), HttpClient::new().unwrap())
}
}
impl CloudProvider for AWS {
fn context(&self) -> &Context {
&self.context
}
fn kind(&self) -> Kind {
Kind::AWS
}
fn id(&self) -> &str {
self.id.as_str()
}
fn organization_id(&self) -> &str {
self.organization_id.as_str()
}
fn name(&self) -> &str {
self.name.as_str()
}
fn is_valid(&self) -> Result<(), CloudProviderError> {
let client = StsClient::new_with_client(self.client(), Region::default());
let s = async_run(client.get_caller_identity(GetCallerIdentityRequest::default()));
match s {
Ok(_x) => Ok(()),
Err(err) => Err(CloudProviderError::from(err)),
}
}
fn add_listener(&mut self, listener: Listener) {
self.listeners.push(listener);
}
fn terraform_state_credentials(&self) -> &TerraformStateCredentials {
&self.terraform_state_credentials
}
fn as_any(&self) -> &dyn Any {
self
}
}

View File

@@ -0,0 +1,500 @@
use dns_lookup::lookup_host;
use retry::delay::Fibonacci;
use retry::OperationResult;
use serde::{Deserialize, Serialize};
use tera::Context as TeraContext;
use crate::cloud_provider::aws::{common, AWS};
use crate::cloud_provider::environment::Environment;
use crate::cloud_provider::kubernetes::Kubernetes;
use crate::cloud_provider::service::{
Action, Create, Delete, Pause, Router as RRouter, Service, ServiceError, ServiceType,
StatelessService,
};
use crate::cloud_provider::DeploymentTarget;
use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY};
use crate::models::{Context, Metadata};
pub struct Router {
context: Context,
id: String,
name: String,
default_domain: String,
custom_domains: Vec<CustomDomain>,
routes: Vec<Route>,
}
impl Router {
pub fn new(
context: Context,
id: &str,
name: &str,
default_domain: &str,
custom_domains: Vec<CustomDomain>,
routes: Vec<Route>,
) -> Self {
Router {
context,
id: id.to_string(),
name: name.to_string(),
default_domain: default_domain.to_string(),
custom_domains,
routes,
}
}
fn helm_release_name(&self) -> String {
crate::string::cut(format!("router-{}", self.id()), 50)
}
fn aws_credentials_envs<'x>(&self, aws: &'x AWS) -> [(&'x str, &'x str); 2] {
[
(AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()),
(AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()),
]
}
fn workspace_directory(&self) -> String {
crate::fs::workspace_directory(
self.context.workspace_root_dir(),
self.context.execution_id(),
format!("routers/{}", self.name()),
)
}
fn tera_context(&self, kubernetes: &dyn Kubernetes, environment: &Environment) -> TeraContext {
let mut context = self.default_tera_context(kubernetes, environment);
let applications = environment
.stateless_services
.iter()
.filter(|x| x.service_type() == ServiceType::Application)
.collect::<Vec<_>>();
let custom_domain_data_templates = self
.custom_domains
.iter()
.map(|cd| {
let domain_hash = crate::crypto::to_sha1_truncate_16(cd.domain.as_str());
//context.insert("target_hostname", cd.domain.clone());
CustomDomainDataTemplate {
domain: cd.domain.clone(),
domain_hash,
target_domain: cd.target_domain.clone(),
}
})
.collect::<Vec<_>>();
let route_data_templates = self
.routes
.iter()
.map(|r| {
match applications
.iter()
.find(|app| app.name() == r.application_name.as_str())
{
Some(application) => match application.private_port() {
Some(private_port) => Some(RouteDataTemplate {
path: r.path.clone(),
application_name: application.name().to_string(),
application_port: private_port,
}),
_ => None,
},
_ => None,
}
})
.filter(|x| x.is_some())
.map(|x| x.unwrap())
.collect::<Vec<_>>();
let workspace_dir = self.workspace_directory();
let aws = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.unwrap();
let kubernetes_config_file_path = common::kubernetes_config_path(
workspace_dir.as_str(),
environment.organization_id.as_str(),
kubernetes.id(),
aws.access_key_id.as_str(),
aws.secret_access_key.as_str(),
kubernetes.region(),
);
match kubernetes_config_file_path {
Ok(kubernetes_config_file_path_string) => {
// Default domain
let external_ingress_hostname_default =
crate::cmd::kubectl::kubectl_exec_get_external_ingress_hostname(
kubernetes_config_file_path_string.as_str(),
"nginx-ingress",
"app=nginx-ingress,component=controller",
self.aws_credentials_envs(aws).to_vec(),
);
match external_ingress_hostname_default {
Ok(external_ingress_hostname_default) => {
match external_ingress_hostname_default {
Some(hostname) => context
.insert("external_ingress_hostname_default", hostname.as_str()),
None => {
warn!("unable to get external_ingress_hostname_default - what's wrong? This must never happened");
}
}
}
_ => {
// FIXME really?
warn!("can't fetch kubernetes config file - what's wrong? This must never happened");
}
}
// Check if there is a custom domain first
if !self.custom_domains.is_empty() {
let external_ingress_hostname_custom =
crate::cmd::kubectl::kubectl_exec_get_external_ingress_hostname(
kubernetes_config_file_path_string.as_str(),
environment.namespace(),
"app=nginx-ingress,component=controller",
self.aws_credentials_envs(aws).to_vec(),
);
match external_ingress_hostname_custom {
Ok(external_ingress_hostname_custom) => {
match external_ingress_hostname_custom {
Some(hostname) => {
context.insert(
"external_ingress_hostname_custom",
hostname.as_str(),
);
}
None => {
warn!("unable to get external_ingress_hostname_custom - what's wrong? This must never happened");
}
}
}
_ => {
// FIXME really?
warn!("can't fetch kubernetes config file - what's wrong? This must never happened");
}
}
context.insert("app_id", kubernetes.id());
}
}
Err(_) => error!(
"can't fetch kubernetes config file - what's wrong? This must never happened"
), // FIXME should I return an Err?
}
let router_default_domain_hash =
crate::crypto::to_sha1_truncate_16(self.default_domain.as_str());
context.insert("router_default_domain", self.default_domain.as_str());
context.insert(
"router_default_domain_hash",
router_default_domain_hash.as_str(),
);
context.insert("custom_domains", &custom_domain_data_templates);
context.insert("routes", &route_data_templates);
context.insert("spec_acme_email", "tls@qovery.com"); // TODO CHANGE ME
context.insert(
"metadata_annotations_cert_manager_cluster_issuer",
"letsencrypt-qovery",
);
let lets_encrypt_url = match self.context.metadata() {
Some(meta) => match meta.test {
Some(true) => "https://acme-staging-v02.api.letsencrypt.org/directory",
_ => "https://acme-v02.api.letsencrypt.org/directory",
},
_ => "https://acme-v02.api.letsencrypt.org/directory",
};
context.insert("spec_acme_server", lets_encrypt_url);
context
}
fn delete(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
let (kubernetes, environment) = match target {
DeploymentTarget::ManagedServices(k, env) => (*k, *env),
DeploymentTarget::SelfHosted(k, env) => (*k, *env),
};
let workspace_dir = self.workspace_directory();
let helm_release_name = self.helm_release_name();
let _ = common::do_stateless_service_cleanup(
kubernetes,
environment,
workspace_dir.as_str(),
helm_release_name.as_str(),
)?;
Ok(())
}
}
impl Service for Router {
fn context(&self) -> &Context {
&self.context
}
fn service_type(&self) -> ServiceType {
ServiceType::Router
}
fn id(&self) -> &str {
self.id.as_str()
}
fn name(&self) -> &str {
self.name.as_str()
}
fn version(&self) -> &str {
"1.0"
}
fn action(&self) -> &Action {
&Action::Create
}
fn private_port(&self) -> Option<u16> {
None
}
fn total_cpus(&self) -> String {
"1".to_string()
}
fn total_ram_in_mib(&self) -> u32 {
1
}
fn total_instances(&self) -> u16 {
1
}
}
impl crate::cloud_provider::service::Router for Router {
fn check_domains(&self) -> Result<(), ServiceError> {
let check_result = retry::retry(Fibonacci::from_millis(3000).take(10), || {
// TODO send information back to the core
info!("check custom domain {}", self.default_domain.as_str());
match lookup_host(self.default_domain.as_str()) {
Ok(_) => OperationResult::Ok(()),
Err(err) => {
debug!("{:?}", err);
OperationResult::Retry(())
}
}
});
// TODO - check custom domains? if yes, why wasting time waiting for user setting up the custom domain?
match check_result {
Ok(_) => {}
Err(_) => return Err(ServiceError::CheckFailed),
}
Ok(())
}
}
impl StatelessService for Router {}
impl Create for Router {
fn on_create(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
info!("AWS.router.on_create() called for {}", self.name());
let (kubernetes, environment) = match target {
DeploymentTarget::ManagedServices(k, env) => (*k, *env),
DeploymentTarget::SelfHosted(k, env) => (*k, *env),
};
let aws = kubernetes
.cloud_provider()
.as_any()
.downcast_ref::<AWS>()
.unwrap();
let workspace_dir = self.workspace_directory();
let helm_release_name = self.helm_release_name();
let kubernetes_config_file_path = common::kubernetes_config_path(
workspace_dir.as_str(),
environment.organization_id.as_str(),
kubernetes.id(),
aws.access_key_id.as_str(),
aws.secret_access_key.as_str(),
kubernetes.region(),
)?;
// respect order - getting the context here and not before is mandatory
// the nginx-ingress must be available to get the external dns target if necessary
let mut context = self.tera_context(kubernetes, environment);
if !self.custom_domains.is_empty() {
// custom domains? create an NGINX ingress
info!("setup NGINX ingress for custom domains");
let into_dir = crate::fs::workspace_directory(
self.context.workspace_root_dir(),
self.context.execution_id(),
"routers/nginx-ingress",
);
let from_dir = format!("{}/common/chart_values", self.context.lib_root_dir());
let _ = crate::template::generate_and_copy_all_files_into_dir(
from_dir.as_str(),
into_dir.as_str(),
&context,
)?;
let _ = crate::template::copy_non_template_files(
format!(
"{}/common/charts/nginx-ingress",
self.context().lib_root_dir()
),
into_dir.as_str(),
)?;
// do exec helm upgrade and return the last deployment status
let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history_with_override(
kubernetes_config_file_path.as_str(),
environment.namespace(),
format!("custom-{}", helm_release_name).as_str(),
into_dir.as_str(),
format!("{}/nginx-ingress.yaml", into_dir.as_str()).as_str(),
self.aws_credentials_envs(aws).to_vec(),
)?;
// check deployment status
if helm_history_row.is_none() || !helm_history_row.unwrap().is_successfully_deployed() {
return Err(ServiceError::OnCreateFailed);
}
// waiting for the nlb, it should be deploy to get fqdn
let external_ingress_hostname_custom_result =
retry::retry(Fibonacci::from_millis(3000).take(10), || {
let external_ingress_hostname_custom =
crate::cmd::kubectl::kubectl_exec_get_external_ingress_hostname(
kubernetes_config_file_path.as_str(),
environment.namespace(),
format!(
"app=nginx-ingress,component=controller,release=custom-{}",
helm_release_name
)
.as_str(),
self.aws_credentials_envs(aws).to_vec(),
);
match external_ingress_hostname_custom {
Ok(external_ingress_hostname_custom) => {
OperationResult::Ok(external_ingress_hostname_custom)
}
Err(err) => {
error!(
"Waiting NLB endpoint to be available to be able to configure TLS"
);
OperationResult::Retry(err)
}
}
});
match external_ingress_hostname_custom_result {
Ok(elb) => {
//put it in the context
context.insert("nlb_ingress_hostname", &elb);
}
Err(e) => error!("Error getting the NLB endpoint to be able to configure TLS"),
}
}
let from_dir = format!("{}/aws/charts/q-ingress-tls", self.context.lib_root_dir());
let _ = crate::template::generate_and_copy_all_files_into_dir(
from_dir.as_str(),
workspace_dir.as_str(),
&context,
)?;
// do exec helm upgrade and return the last deployment status
let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history(
kubernetes_config_file_path.as_str(),
environment.namespace(),
helm_release_name.as_str(),
workspace_dir.as_str(),
self.aws_credentials_envs(aws).to_vec(),
)?;
if helm_history_row.is_none() || !helm_history_row.unwrap().is_successfully_deployed() {
return Err(ServiceError::OnCreateFailed);
}
Ok(())
}
fn on_create_check(&self) -> Result<(), ServiceError> {
// Todo: manage it properly to avoid timeouts
//self.check_domains()
Ok(())
}
fn on_create_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!("AWS.router.on_create_error() called for {}", self.name());
self.delete(target)
}
}
impl Pause for Router {
fn on_pause(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
info!("AWS.router.on_pause() called for {}", self.name());
self.delete(target)
}
fn on_pause_check(&self) -> Result<(), ServiceError> {
warn!("AWS.router.on_pause_error() called for {}", self.name());
// TODO check resource has been cleaned?
Ok(())
}
fn on_pause_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
self.delete(target)
}
}
impl Delete for Router {
fn on_delete(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
info!("AWS.router.on_delete() called for {}", self.name());
self.delete(target)
}
fn on_delete_check(&self) -> Result<(), ServiceError> {
Ok(())
}
fn on_delete_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> {
warn!("AWS.router.on_delete_error() called for {}", self.name());
self.delete(target)
}
}
pub struct CustomDomain {
pub domain: String,
pub target_domain: String,
}
#[derive(Serialize, Deserialize)]
struct CustomDomainDataTemplate {
pub domain: String,
pub domain_hash: String,
pub target_domain: String,
}
pub struct Route {
pub path: String,
pub application_name: String,
}
#[derive(Serialize, Deserialize)]
struct RouteDataTemplate {
pub path: String,
pub application_name: String,
pub application_port: u16,
}

View File

@@ -0,0 +1,67 @@
extern crate digitalocean;
use std::any::Any;
use std::rc::Rc;
use digitalocean::DigitalOcean;
use crate::cloud_provider::{CloudProvider, CloudProviderError, Kind, TerraformStateCredentials};
use crate::models::{Context, Listener, ProgressListener};
pub struct DO {
context: Context,
id: String,
pub token: String,
}
impl DO {
pub fn new(context: Context, id: &str, token: &str) -> Self {
DO {
context,
id: id.to_string(),
token: token.to_string(),
}
}
pub fn client(&self) -> DigitalOcean {
DigitalOcean::new(self.token.as_str()).unwrap()
}
}
impl CloudProvider for DO {
fn context(&self) -> &Context {
&self.context
}
fn kind(&self) -> Kind {
Kind::DO
}
fn id(&self) -> &str {
self.id.as_str()
}
fn organization_id(&self) -> &str {
unimplemented!()
}
fn name(&self) -> &str {
unimplemented!()
}
fn is_valid(&self) -> Result<(), CloudProviderError> {
unimplemented!()
}
fn add_listener(&mut self, _listener: Listener) {
unimplemented!()
}
fn terraform_state_credentials(&self) -> &TerraformStateCredentials {
unimplemented!()
}
fn as_any(&self) -> &dyn Any {
unimplemented!()
}
}

View File

@@ -0,0 +1,100 @@
use crate::cloud_provider::service::{ServiceError, StatefulService, StatelessService};
use crate::unit_conversion::cpu_string_to_float;
pub struct Environment {
namespace: String,
pub kind: Kind,
pub id: String,
pub project_id: String,
pub owner_id: String,
pub organization_id: String,
pub stateless_services: Vec<Box<dyn StatelessService>>,
pub stateful_services: Vec<Box<dyn StatefulService>>,
}
impl Environment {
pub fn new(
kind: Kind,
id: &str,
project_id: &str,
owner_id: &str,
organization_id: &str,
stateless_services: Vec<Box<dyn StatelessService>>,
stateful_services: Vec<Box<dyn StatefulService>>,
) -> Self {
Environment {
namespace: format!("{}-{}", project_id, id),
kind,
id: id.to_string(),
project_id: project_id.to_string(),
owner_id: owner_id.to_string(),
organization_id: organization_id.to_string(),
stateless_services,
stateful_services,
}
}
pub fn namespace(&self) -> &str {
self.namespace.as_str()
}
pub fn is_valid(&self) -> Result<(), ServiceError> {
for service in self.stateful_services.iter() {
match service.is_valid() {
Err(err) => return Err(err),
_ => {}
}
}
for service in self.stateless_services.iter() {
match service.is_valid() {
Err(err) => return Err(err),
_ => {}
}
}
Ok(())
}
/// compute the required resources for this environment from
/// applications, external services, routers, and databases
/// Note: Even if external services don't run on the targeted Kubernetes cluster, it requires CPU and memory resources to run the container(s)
pub fn required_resources(&self) -> EnvironmentResources {
let mut total_cpu_for_stateless_services: f32 = 0.0;
let mut total_ram_in_mib_for_stateless_services: u32 = 0;
for service in &self.stateless_services {
total_cpu_for_stateless_services += cpu_string_to_float(&service.total_cpus());
total_ram_in_mib_for_stateless_services += &service.total_ram_in_mib();
}
let mut total_cpu_for_stateful_services: f32 = 0.0;
let mut total_ram_in_mib_for_stateful_services: u32 = 0;
match self.kind {
Kind::Development => {
// development means databases are running on Kubernetes
for service in &self.stateful_services {
total_cpu_for_stateful_services += cpu_string_to_float(&service.total_cpus());
total_ram_in_mib_for_stateful_services += &service.total_ram_in_mib();
}
}
Kind::Production => {} // production means databases are running on managed services - so it consumes 0 cpu
};
EnvironmentResources {
cpu: total_cpu_for_stateless_services + total_cpu_for_stateful_services,
ram_in_mib: total_ram_in_mib_for_stateless_services
+ total_ram_in_mib_for_stateless_services,
}
}
}
pub enum Kind {
Production,
Development,
}
pub struct EnvironmentResources {
pub cpu: f32,
pub ram_in_mib: u32,
}

View File

@@ -0,0 +1,61 @@
use std::any::Any;
use std::rc::Rc;
use crate::cloud_provider::{CloudProvider, CloudProviderError, Kind, TerraformStateCredentials};
use crate::models::{Context, Listener, ProgressListener};
pub struct GCP {
context: Context,
id: String,
name: String,
p12_file_content: String,
}
impl GCP {
pub fn new(context: Context, id: &str, name: &str, p12_file_content: &str) -> Self {
GCP {
context,
id: id.to_string(),
name: name.to_string(),
p12_file_content: p12_file_content.to_string(),
}
}
}
impl<'x> CloudProvider for GCP {
fn context(&self) -> &Context {
&self.context
}
fn kind(&self) -> Kind {
Kind::GCP
}
fn id(&self) -> &str {
self.id.as_str()
}
fn organization_id(&self) -> &str {
unimplemented!()
}
fn name(&self) -> &str {
self.name.as_str()
}
fn is_valid(&self) -> Result<(), CloudProviderError> {
Ok(())
}
fn add_listener(&mut self, _listener: Listener) {
// TODO
}
fn terraform_state_credentials(&self) -> &TerraformStateCredentials {
unimplemented!()
}
fn as_any(&self) -> &dyn Any {
self
}
}

View File

@@ -0,0 +1,176 @@
use std::any::Any;
use std::process::ExitStatus;
use std::rc::Rc;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::cloud_provider::environment::Environment;
use crate::cloud_provider::service::ServiceError;
use crate::cloud_provider::CloudProvider;
use crate::cmd::utilities::CmdError;
use crate::dns_provider::DnsProvider;
use crate::models::{Context, Listener, Listeners, ProgressListener};
pub trait Kubernetes {
fn context(&self) -> &Context;
fn kind(&self) -> Kind;
fn id(&self) -> &str;
fn name(&self) -> &str;
fn version(&self) -> &str;
fn region(&self) -> &str;
fn cloud_provider(&self) -> &dyn CloudProvider;
fn dns_provider(&self) -> &dyn DnsProvider;
fn is_valid(&self) -> Result<(), KubernetesError>;
fn add_listener(&mut self, listener: Listener);
fn listeners(&self) -> &Listeners;
fn resources(&self, environment: &Environment) -> Result<Resources, KubernetesError>;
fn on_create(&self) -> Result<(), KubernetesError>;
fn on_create_error(&self) -> Result<(), KubernetesError>;
fn on_upgrade(&self) -> Result<(), KubernetesError>;
fn on_upgrade_error(&self) -> Result<(), KubernetesError>;
fn on_downgrade(&self) -> Result<(), KubernetesError>;
fn on_downgrade_error(&self) -> Result<(), KubernetesError>;
fn on_delete(&self) -> Result<(), KubernetesError>;
fn on_delete_error(&self) -> Result<(), KubernetesError>;
fn deploy_environment(&self, environment: &Environment) -> Result<(), KubernetesError>;
fn deploy_environment_error(&self, environment: &Environment) -> Result<(), KubernetesError>;
fn pause_environment(&self, environment: &Environment) -> Result<(), KubernetesError>;
fn pause_environment_error(&self, environment: &Environment) -> Result<(), KubernetesError>;
fn delete_environment(&self, environment: &Environment) -> Result<(), KubernetesError>;
fn delete_environment_error(&self, environment: &Environment) -> Result<(), KubernetesError>;
}
pub trait KubernetesNode {
fn total_cpu(&self) -> u8;
fn total_memory_in_gib(&self) -> u16;
fn instance_type(&self) -> &str;
fn as_any(&self) -> &dyn Any;
}
#[derive(Serialize, Deserialize, Clone)]
pub enum Kind {
EKS,
}
#[derive(Debug)]
pub struct Resources {
pub free_cpu: f32,
pub max_cpu: f32,
pub free_ram_in_mib: u32,
pub max_ram_in_mib: u32,
pub free_pods: u16,
pub max_pods: u16,
pub running_nodes: u16,
}
#[derive(Debug)]
pub enum KubernetesError {
Cmd(CmdError),
Io(std::io::Error),
Create(ExitStatus),
Deploy(ServiceError),
Pause(ServiceError),
Delete(ServiceError),
Error,
}
impl From<std::io::Error> for KubernetesError {
fn from(error: std::io::Error) -> Self {
KubernetesError::Io(error)
}
}
impl From<CmdError> for KubernetesError {
fn from(error: CmdError) -> Self {
KubernetesError::Cmd(error)
}
}
impl From<KubernetesError> for Option<ServiceError> {
fn from(item: KubernetesError) -> Self {
return match item {
KubernetesError::Deploy(e) | KubernetesError::Pause(e) | KubernetesError::Delete(e) => {
Option::from(e)
}
_ => None,
};
}
}
/// check that there is enough CPU and RAM, and pods resources
/// before starting to deploy stateful and stateless services
pub fn check_kubernetes_has_enough_resources_to_deploy_environment(
kubernetes: &dyn Kubernetes,
environment: &Environment,
) -> Result<(), KubernetesError> {
let resources = kubernetes.resources(environment)?;
let required_resources = environment.required_resources();
if required_resources.cpu > resources.free_cpu
&& required_resources.ram_in_mib > resources.free_ram_in_mib
{
// not enough cpu and ram to deploy environment
return Err(KubernetesError::Deploy(ServiceError::NotEnoughResources(
format!(
"There is not enough CPU and RAM resources on the Kubernetes '{}' cluster. \
{} CPU and {}mib RAM requested. \
{} CPU and {}mib RAM available. \
Consider to add one more node or upgrade your nodes configuration.",
kubernetes.name(),
required_resources.cpu,
required_resources.ram_in_mib,
resources.free_cpu,
resources.free_ram_in_mib,
),
)));
} else if required_resources.cpu > resources.free_cpu {
// not enough cpu to deploy environment
return Err(KubernetesError::Deploy(ServiceError::NotEnoughResources(
format!(
"There is not enough free CPU on the Kubernetes '{}' cluster. \
{} CPU requested. {} CPU available. \
Consider to add one more node or upgrade your nodes configuration.",
kubernetes.name(),
required_resources.cpu,
resources.free_cpu,
),
)));
} else if required_resources.ram_in_mib > resources.free_ram_in_mib {
// not enough ram to deploy environment
return Err(KubernetesError::Deploy(ServiceError::NotEnoughResources(
format!(
"There is not enough free RAM on the Kubernetes cluster '{}'. \
{}mib RAM requested. \
{}mib RAM available. \
Consider to add one more node or upgrade your nodes configuration.",
kubernetes.name(),
required_resources.ram_in_mib,
resources.free_ram_in_mib,
),
)));
}
let mut required_pods = environment.stateless_services.len() as u16;
match environment.kind {
crate::cloud_provider::environment::Kind::Production => {}
crate::cloud_provider::environment::Kind::Development => {
required_pods += environment.stateful_services.len() as u16;
}
}
if required_pods > resources.free_pods {
// not enough free pods on the cluster
return Err(KubernetesError::Deploy(ServiceError::NotEnoughResources(
format!(
"There is not enough free Pods ({} required) on the Kubernetes cluster '{}'. \
Consider to add one more node or upgrade your nodes configuration.",
required_pods,
kubernetes.name(),
),
)));
}
Ok(())
}

86
src/cloud_provider/mod.rs Normal file
View File

@@ -0,0 +1,86 @@
use std::any::Any;
use std::rc::Rc;
use rusoto_core::RusotoError;
use serde::{Deserialize, Serialize};
use crate::cloud_provider::environment::Environment;
use crate::cloud_provider::kubernetes::Kubernetes;
use crate::models::{Context, Listener, ProgressListener};
pub mod aws;
pub mod digitalocean;
pub mod environment;
pub mod gcp;
pub mod kubernetes;
pub mod service;
pub trait CloudProvider {
fn context(&self) -> &Context;
fn kind(&self) -> Kind;
fn id(&self) -> &str;
fn organization_id(&self) -> &str;
fn name(&self) -> &str;
fn is_valid(&self) -> Result<(), CloudProviderError>;
fn add_listener(&mut self, listener: Listener);
fn terraform_state_credentials(&self) -> &TerraformStateCredentials;
fn as_any(&self) -> &dyn Any;
}
#[derive(Debug)]
pub enum CloudProviderError {
Credentials,
Error(Box<dyn std::error::Error>),
Unknown,
}
impl From<Box<dyn std::error::Error>> for CloudProviderError {
fn from(error: Box<dyn std::error::Error>) -> Self {
CloudProviderError::Error(error)
}
}
impl<E> From<RusotoError<E>> for CloudProviderError {
fn from(error: RusotoError<E>) -> Self {
match error {
RusotoError::Credentials(_) => CloudProviderError::Credentials,
RusotoError::Service(_) => CloudProviderError::Unknown,
RusotoError::HttpDispatch(_) => CloudProviderError::Unknown,
RusotoError::Validation(_) => CloudProviderError::Unknown,
RusotoError::ParseError(_) => CloudProviderError::Unknown,
RusotoError::Unknown(e) => {
if e.status == 403 {
CloudProviderError::Credentials
} else {
CloudProviderError::Unknown
}
}
RusotoError::Blocking => CloudProviderError::Unknown,
}
}
}
#[derive(Debug)]
pub enum DeployError {
Error,
}
#[derive(Serialize, Deserialize, Clone)]
pub enum Kind {
AWS,
GCP,
DO,
}
pub struct TerraformStateCredentials {
pub access_key_id: String,
pub secret_access_key: String,
pub region: String,
}
pub enum DeploymentTarget<'a> {
// ManagedService = Managed by the Cloud Provider (eg. RDS, DynamoDB...)
ManagedServices(&'a dyn Kubernetes, &'a Environment),
// SelfHosted = Kubernetes or anything else that implies management on our side
SelfHosted(&'a dyn Kubernetes, &'a Environment),
}

View File

@@ -0,0 +1,255 @@
use std::io::Error;
use std::net::TcpStream;
use std::process::id;
use tera::Context as TeraContext;
use crate::build_platform::Image;
use crate::cloud_provider::environment::Environment;
use crate::cloud_provider::kubernetes::Kubernetes;
use crate::cloud_provider::DeploymentTarget;
use crate::cmd::utilities::CmdError;
use crate::models::{Context, ProgressScope};
use crate::transaction::CommitError;
pub trait Service {
fn context(&self) -> &Context;
fn service_type(&self) -> ServiceType;
fn id(&self) -> &str;
fn name(&self) -> &str;
fn version(&self) -> &str;
fn action(&self) -> &Action;
fn private_port(&self) -> Option<u16>;
fn total_cpus(&self) -> String;
fn total_ram_in_mib(&self) -> u32;
fn total_instances(&self) -> u16;
fn is_listening(&self, ip: &str) -> bool {
let private_port = match self.private_port() {
Some(private_port) => private_port,
_ => return false,
};
match TcpStream::connect(format!("{}:{}", ip, private_port)) {
Ok(_) => true,
Err(_) => false,
}
}
fn is_valid(&self) -> Result<(), ServiceError> {
let binaries = ["kubectl", "helm", "terraform", "aws-iam-authenticator"];
for binary in binaries.iter() {
if !crate::cmd::utilities::does_binary_exist(binary) {
let err = format!("{} binary not found", binary);
return Err(ServiceError::Unexpected(err));
}
}
// TODO check lib directories available
Ok(())
}
fn default_tera_context(
&self,
kubernetes: &dyn Kubernetes,
environment: &Environment,
) -> TeraContext {
let mut context = TeraContext::new();
context.insert("id", self.id());
context.insert("owner_id", environment.owner_id.as_str());
context.insert("project_id", environment.project_id.as_str());
context.insert("organization_id", environment.organization_id.as_str());
context.insert("environment_id", environment.id.as_str());
context.insert("region", kubernetes.region());
context.insert("name", self.name());
context.insert("namespace", environment.namespace());
context.insert("cluster_name", kubernetes.name());
context.insert("total_cpus", &self.total_cpus());
context.insert("total_ram_in_mib", &self.total_ram_in_mib());
context.insert("total_instances", &self.total_instances());
context.insert("is_private_port", &self.private_port().is_some());
if self.private_port().is_some() {
context.insert("private_port", &self.private_port().unwrap());
}
context.insert("version", self.version());
context
}
fn progress_scope(&self) -> ProgressScope {
let id = self.id().to_string();
match self.service_type() {
ServiceType::Application => ProgressScope::Application { id },
ServiceType::ExternalService => ProgressScope::ExternalService { id },
ServiceType::Database(_) => ProgressScope::Database { id },
ServiceType::Router => ProgressScope::Router { id },
}
}
}
pub trait StatelessService: Service + Create + Pause + Delete {
fn exec_action(&self, deployment_target: &DeploymentTarget) -> Result<(), ServiceError> {
match self.action() {
crate::cloud_provider::service::Action::Create => self.on_create(deployment_target),
crate::cloud_provider::service::Action::Delete => self.on_delete(deployment_target),
crate::cloud_provider::service::Action::Pause => self.on_pause(deployment_target),
crate::cloud_provider::service::Action::Nothing => Ok(()),
}
}
}
pub trait StatefulService:
Service + Create + Pause + Delete + Backup + Clone + Upgrade + Downgrade
{
fn exec_action(&self, deployment_target: &DeploymentTarget) -> Result<(), ServiceError> {
match self.action() {
crate::cloud_provider::service::Action::Create => self.on_create(deployment_target),
crate::cloud_provider::service::Action::Delete => self.on_delete(deployment_target),
crate::cloud_provider::service::Action::Pause => self.on_pause(deployment_target),
crate::cloud_provider::service::Action::Nothing => Ok(()),
}
}
}
pub trait Application: StatelessService {
fn image(&self) -> &Image;
fn set_image(&mut self, image: Image);
}
pub trait ExternalService: Application {}
pub trait Router: StatelessService {
fn check_domains(&self) -> Result<(), ServiceError>;
}
pub trait Database: StatefulService {}
pub trait Create {
fn on_create(&self, target: &DeploymentTarget) -> Result<(), ServiceError>;
fn on_create_check(&self) -> Result<(), ServiceError>;
fn on_create_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError>;
}
pub trait Pause {
fn on_pause(&self, target: &DeploymentTarget) -> Result<(), ServiceError>;
fn on_pause_check(&self) -> Result<(), ServiceError>;
fn on_pause_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError>;
}
pub trait Delete {
fn on_delete(&self, target: &DeploymentTarget) -> Result<(), ServiceError>;
fn on_delete_check(&self) -> Result<(), ServiceError>;
fn on_delete_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError>;
}
pub trait Backup {
fn on_backup(&self, target: &DeploymentTarget) -> Result<(), ServiceError>;
fn on_backup_check(&self) -> Result<(), ServiceError>;
fn on_backup_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError>;
fn on_restore(&self, target: &DeploymentTarget) -> Result<(), ServiceError>;
fn on_restore_check(&self) -> Result<(), ServiceError>;
fn on_restore_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError>;
}
pub trait Clone {
fn on_clone(&self, target: &DeploymentTarget) -> Result<(), ServiceError>;
fn on_clone_check(&self) -> Result<(), ServiceError>;
fn on_clone_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError>;
}
pub trait Upgrade {
fn on_upgrade(&self, target: &DeploymentTarget) -> Result<(), ServiceError>;
fn on_upgrade_check(&self) -> Result<(), ServiceError>;
fn on_upgrade_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError>;
}
pub trait Downgrade {
fn on_downgrade(&self, target: &DeploymentTarget) -> Result<(), ServiceError>;
fn on_downgrade_check(&self) -> Result<(), ServiceError>;
fn on_downgrade_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError>;
}
#[derive(Clone, Eq, PartialEq, Hash)]
pub enum Action {
Create,
Pause,
Delete,
Nothing,
}
#[derive(Eq, PartialEq)]
pub struct DatabaseOptions {
pub login: String,
pub password: String,
pub host: String,
pub port: u16,
pub disk_size_in_gib: u32,
pub database_disk_type: String,
}
#[derive(Eq, PartialEq)]
pub enum DatabaseType<'a> {
PostgreSQL(&'a DatabaseOptions),
MongoDB(&'a DatabaseOptions),
MySQL(&'a DatabaseOptions),
}
#[derive(Eq, PartialEq)]
pub enum ServiceType<'a> {
Application,
ExternalService,
Database(DatabaseType<'a>),
Router,
}
impl<'a> ServiceType<'a> {
pub fn name(&self) -> &str {
match self {
ServiceType::Application => "Application",
ServiceType::ExternalService => "ExternalService",
ServiceType::Database(_) => "Database",
ServiceType::Router => "Router",
}
}
}
#[derive(Debug)]
pub enum ServiceError {
OnCreateFailed,
CheckFailed,
Cmd(CmdError),
Io(Error),
NotEnoughResources(String),
Unexpected(String),
}
impl From<std::io::Error> for ServiceError {
fn from(err: Error) -> Self {
ServiceError::Io(err)
}
}
impl From<CmdError> for ServiceError {
fn from(err: CmdError) -> Self {
ServiceError::Cmd(err)
}
}
impl From<CommitError> for Option<ServiceError> {
fn from(err: CommitError) -> Self {
return match err {
CommitError::DeleteEnvironment(e)
| CommitError::PauseEnvironment(e)
| CommitError::DeployEnvironment(e)
| CommitError::DeleteKubernetes(e)
| CommitError::CreateKubernetes(e) => Option::from(e),
CommitError::NotValidService(e) => Option::Some(e),
_ => None,
};
}
}

387
src/cmd/helm.rs Normal file
View File

@@ -0,0 +1,387 @@
use std::ffi::OsStr;
use std::fmt::{Display, Formatter};
use std::io::Error;
use std::io::{BufRead, BufReader};
use std::path::Path;
use std::process::{Child, Command, ExitStatus, Stdio};
use dirs::home_dir;
use retry::delay::Fibonacci;
use retry::OperationResult;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::cmd::structs::{Helm, HelmHistoryRow};
use crate::cmd::utilities::{exec_with_envs_and_output, CmdError};
use crate::constants::{KUBECONFIG, TF_PLUGIN_CACHE_DIR};
pub fn helm_exec_with_upgrade_history<P>(
kubernetes_config: P,
namespace: &str,
release_name: &str,
chart_root_dir: P,
envs: Vec<(&str, &str)>,
) -> Result<Option<HelmHistoryRow>, CmdError>
where
P: AsRef<Path>,
{
// do exec helm upgrade
info!(
"exec helm upgrade for namespace {} and chart {}",
namespace,
chart_root_dir.as_ref().to_str().unwrap()
);
let _ = helm_exec_upgrade(
kubernetes_config.as_ref(),
namespace,
release_name,
chart_root_dir.as_ref(),
envs.clone(),
)?;
// list helm history
info!(
"exec helm history for namespace {} and chart {}",
namespace,
chart_root_dir.as_ref().to_str().unwrap()
);
let helm_history_rows =
helm_exec_history(kubernetes_config.as_ref(), namespace, release_name, envs)?;
// take the last deployment from helm history - or return none if there is no history
Ok(match helm_history_rows.first() {
Some(helm_history_row) => Some(helm_history_row.clone()),
None => None,
})
}
pub fn helm_exec_upgrade<P>(
kubernetes_config: P,
namespace: &str,
release_name: &str,
chart_root_dir: P,
envs: Vec<(&str, &str)>,
) -> Result<(), CmdError>
where
P: AsRef<Path>,
{
helm_exec_with_output(
vec![
"upgrade",
"--kubeconfig",
kubernetes_config.as_ref().to_str().unwrap(),
"--create-namespace",
"--install",
"--history-max",
"50",
"--wait",
"--namespace",
namespace,
release_name,
chart_root_dir.as_ref().to_str().unwrap(),
],
envs,
|out| match out {
Ok(line) => info!("{}", line.as_str()),
Err(err) => error!("{}", err),
},
|out| match out {
Ok(line) => error!("{}", line.as_str()),
Err(err) => error!("{}", err),
},
)
}
pub fn helm_exec_uninstall<P>(
kubernetes_config: P,
namespace: &str,
release_name: &str,
envs: Vec<(&str, &str)>,
) -> Result<(), CmdError>
where
P: AsRef<Path>,
{
helm_exec_with_output(
vec![
"uninstall",
"--kubeconfig",
kubernetes_config.as_ref().to_str().unwrap(),
"--namespace",
namespace,
release_name,
],
envs,
|out| match out {
Ok(line) => info!("{}", line.as_str()),
Err(err) => error!("{}", err),
},
|out| match out {
Ok(line) => error!("{}", line.as_str()),
Err(err) => error!("{}", err),
},
)
}
pub fn helm_exec_history<P>(
kubernetes_config: P,
namespace: &str,
release_name: &str,
envs: Vec<(&str, &str)>,
) -> Result<Vec<HelmHistoryRow>, CmdError>
where
P: AsRef<Path>,
{
let mut output_string = String::new();
match helm_exec_with_output(
// WARN: do not add argument --debug, otherwise JSON decoding will not work
vec![
"history",
"--kubeconfig",
kubernetes_config.as_ref().to_str().unwrap(),
"--namespace",
namespace,
"-o",
"json",
release_name,
],
envs,
|out| match out {
Ok(line) => output_string = line,
Err(err) => error!("{:?}", err),
},
|out| match out {
Ok(line) => error!("{}", line),
Err(err) => error!("{:?}", err),
},
) {
Ok(_) => info!("Helm history success for release name: {}", release_name),
Err(_) => info!("Helm history found for release name: {}", release_name),
};
// TODO better check, release not found
let mut results = match serde_json::from_str::<Vec<HelmHistoryRow>>(output_string.as_str()) {
Ok(x) => x,
Err(_) => vec![],
};
// unsort results by revision number
let _ = results.sort_by_key(|x| x.revision);
// there is no performance penalty to do it in 2 operations instead of one, but who really cares anyway
let _ = results.reverse();
Ok(results)
}
pub fn helm_uninstall_list<P>(
kubernetes_config: P,
helmlist: Vec<String>,
envs: Vec<(&str, &str)>,
) -> Result<String, CmdError>
where
P: AsRef<Path>,
{
let mut output_vec: Vec<String> = Vec::new();
let helmlist_string = helmlist.join(" ");
match helm_exec_with_output(
vec![
"uninstall",
helmlist_string.as_str(),
"--kubeconfig",
kubernetes_config.as_ref().to_str().unwrap(),
],
envs,
|out| match out {
Ok(line) => output_vec.push(line),
Err(err) => error!("{:?}", err),
},
|out| match out {
Ok(line) => error!("{}", line),
Err(err) => error!("{:?}", err),
},
) {
Ok(_) => info!("Helm uninstall fail with : {}", helmlist_string.clone()),
Err(_) => info!(
"Helm history found for release name: {}",
helmlist_string.clone()
),
};
Ok(output_vec.join(""))
}
pub fn helm_exec_upgrade_with_override_file<P>(
kubernetes_config: P,
namespace: &str,
release_name: &str,
chart_root_dir: P,
override_file: &str,
envs: Vec<(&str, &str)>,
) -> Result<(), CmdError>
where
P: AsRef<Path>,
{
helm_exec_with_output(
vec![
"upgrade",
"--kubeconfig",
kubernetes_config.as_ref().to_str().unwrap(),
"--create-namespace",
"--install",
"--history-max",
"50",
"--wait",
"--namespace",
namespace,
release_name,
chart_root_dir.as_ref().to_str().unwrap(),
"-f",
override_file,
],
envs,
|out| match out {
Ok(line) => info!("{}", line.as_str()),
Err(err) => error!("{}", err),
},
|out| match out {
// don't crash errors if releases are not found
Ok(line) if line.contains("Error: release: not found") => info!("{}", line.as_str()),
Ok(line) => error!("{}", line.as_str()),
Err(err) => error!("{}", err),
},
)
}
pub fn helm_exec_with_upgrade_history_with_override<P>(
kubernetes_config: P,
namespace: &str,
release_name: &str,
chart_root_dir: P,
override_file: &str,
envs: Vec<(&str, &str)>,
) -> Result<Option<HelmHistoryRow>, CmdError>
where
P: AsRef<Path>,
{
// do exec helm upgrade
info!(
"exec helm upgrade for namespace {} and chart {}",
namespace,
chart_root_dir.as_ref().to_str().unwrap()
);
let _ = helm_exec_upgrade_with_override_file(
kubernetes_config.as_ref(),
namespace,
release_name,
chart_root_dir.as_ref(),
override_file,
envs.clone(),
)?;
// list helm history
info!(
"exec helm history for namespace {} and chart {}",
namespace,
chart_root_dir.as_ref().to_str().unwrap()
);
let helm_history_rows =
helm_exec_history(kubernetes_config.as_ref(), namespace, release_name, envs)?;
// take the last deployment from helm history - or return none if there is no history
Ok(match helm_history_rows.first() {
Some(helm_history_row) => Some(helm_history_row.clone()),
None => None,
})
}
pub fn helm_list<P>(kubernetes_config: P, envs: Vec<(&str, &str)>) -> Result<Vec<String>, CmdError>
where
P: AsRef<Path>,
{
let mut output_vec: Vec<String> = Vec::new();
helm_exec_with_output(
vec![
"list",
"-A",
"--kubeconfig",
kubernetes_config.as_ref().to_str().unwrap(),
"-o",
"json",
],
envs,
|out| match out {
Ok(line) => output_vec.push(line),
Err(err) => error!("{}", err),
},
|out| match out {
Ok(line) => error!("{}", line.as_str()),
Err(err) => error!("{}", err),
},
);
let output_string: String = output_vec.join("");
let values = serde_json::from_str::<Vec<Helm>>(output_string.as_str());
let mut helms_name: Vec<String> = Vec::new();
match values {
Ok(all_helms) => {
for helm in all_helms {
helms_name.push(helm.name)
}
}
Err(e) => {
error!("Error while deserializing all helms names {}", e);
return Err(CmdError::Io(Error::new(std::io::ErrorKind::InvalidData, e)));
}
}
Ok(helms_name)
}
pub fn helm_exec(args: Vec<&str>, envs: Vec<(&str, &str)>) -> Result<(), CmdError> {
helm_exec_with_output(
args,
envs,
|line| {
info!("{}", line.unwrap());
},
|line| {
error!("{}", line.unwrap());
},
)
}
pub fn helm_exec_with_output<F, X>(
args: Vec<&str>,
envs: Vec<(&str, &str)>,
stdout_output: F,
stderr_output: X,
) -> Result<(), CmdError>
where
F: FnMut(Result<String, Error>),
X: FnMut(Result<String, Error>),
{
match exec_with_envs_and_output("helm", args, envs, stdout_output, stderr_output) {
Err(err) => return Err(err),
_ => {}
};
Ok(())
}
pub fn kubectl_exec_with_output<F, X>(
args: Vec<&str>,
envs: Vec<(&str, &str)>,
stdout_output: F,
stderr_output: X,
) -> Result<(), CmdError>
where
F: FnMut(Result<String, Error>),
X: FnMut(Result<String, Error>),
{
match exec_with_envs_and_output("kubectl", args, envs, stdout_output, stderr_output) {
Err(err) => return Err(err),
_ => {}
};
Ok(())
}

630
src/cmd/kubectl.rs Normal file
View File

@@ -0,0 +1,630 @@
use std::ffi::OsStr;
use std::fmt::{Display, Formatter};
use std::io::Error;
use std::io::{BufRead, BufReader};
use std::path::Path;
use std::process::{Child, Command, ExitStatus, Stdio};
use dirs::home_dir;
use retry::delay::Fibonacci;
use retry::OperationResult;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::cmd::structs::{
Item, KubernetesJob, KubernetesList, KubernetesNode, KubernetesPod, KubernetesPodStatusPhase,
KubernetesService,
};
use crate::cmd::utilities::{exec_with_envs_and_output, CmdError};
use crate::constants::{KUBECONFIG, TF_PLUGIN_CACHE_DIR};
pub fn kubectl_exec_with_output<F, X>(
args: Vec<&str>,
envs: Vec<(&str, &str)>,
stdout_output: F,
stderr_output: X,
) -> Result<(), CmdError>
where
F: FnMut(Result<String, Error>),
X: FnMut(Result<String, Error>),
{
match exec_with_envs_and_output("kubectl", args, envs, stdout_output, stderr_output) {
Err(err) => return Err(err),
_ => {}
};
Ok(())
}
pub fn kubectl_exec_get_external_ingress_hostname<P>(
kubernetes_config: P,
namespace: &str,
selector: &str,
envs: Vec<(&str, &str)>,
) -> Result<Option<String>, CmdError>
where
P: AsRef<Path>,
{
let mut _envs = Vec::with_capacity(envs.len() + 1);
_envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap()));
_envs.extend(envs);
let mut output_vec: Vec<String> = Vec::with_capacity(20);
let _ = kubectl_exec_with_output(
vec![
"get", "svc", "-o", "json", "-n", namespace, "-l", // selector
selector,
],
_envs,
|out| match out {
Ok(line) => output_vec.push(line),
Err(err) => error!("{:?}", err),
},
|out| match out {
Ok(line) => error!("{}", line),
Err(err) => error!("{:?}", err),
},
)?;
let output_string: String = output_vec.join("");
let result =
match serde_json::from_str::<KubernetesList<KubernetesService>>(output_string.as_str()) {
Ok(x) => x,
Err(err) => {
error!("{:?}", err);
error!("{}", output_string.as_str());
return Err(CmdError::Io(Error::new(
std::io::ErrorKind::InvalidData,
output_string,
)));
}
};
if result.items.is_empty()
|| result
.items
.first()
.unwrap()
.status
.load_balancer
.ingress
.is_empty()
{
return Ok(None);
}
// FIXME unsafe unwrap here?
Ok(Some(
result
.items
.first()
.unwrap()
.status
.load_balancer
.ingress
.first()
.unwrap()
.hostname
.clone(),
))
}
pub fn kubectl_exec_is_pod_ready_with_retry<P>(
kubernetes_config: P,
namespace: &str,
selector: &str,
envs: Vec<(&str, &str)>,
) -> Result<Option<bool>, CmdError>
where
P: AsRef<Path>,
{
// TODO check this
let result = retry::retry(Fibonacci::from_millis(3000).take(10), || {
let r = crate::cmd::kubectl::kubectl_exec_is_pod_ready(
kubernetes_config.as_ref(),
namespace,
selector,
envs.clone(),
);
match r {
Ok(is_ready) => match is_ready {
Some(true) => OperationResult::Ok(true),
_ => {
let t = format!("pod with selector: {} is not ready yet", selector);
info!("{}", t.as_str());
OperationResult::Retry(t)
}
},
Err(err) => OperationResult::Err(format!("command error: {:?}", err)),
}
});
match result {
Err(err) => match err {
retry::Error::Operation {
error: _,
total_delay: _,
tries: _,
} => Ok(Some(false)),
retry::Error::Internal(err) => Err(CmdError::Unexpected(err)),
},
Ok(_) => Ok(Some(true)),
}
}
pub fn kubectl_exec_is_pod_ready<P>(
kubernetes_config: P,
namespace: &str,
selector: &str,
envs: Vec<(&str, &str)>,
) -> Result<Option<bool>, CmdError>
where
P: AsRef<Path>,
{
let mut _envs = Vec::with_capacity(envs.len() + 1);
_envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap()));
_envs.extend(envs);
let mut output_vec: Vec<String> = Vec::with_capacity(20);
let _ = kubectl_exec_with_output(
vec!["get", "pod", "-o", "json", "-n", namespace, "-l", selector],
_envs,
|out| match out {
Ok(line) => output_vec.push(line),
Err(err) => error!("{:?}", err),
},
|out| match out {
Ok(line) => error!("{}", line),
Err(err) => error!("{:?}", err),
},
)?;
let output_string: String = output_vec.join("");
let result = match serde_json::from_str::<KubernetesList<KubernetesPod>>(output_string.as_str())
{
Ok(x) => x,
Err(err) => {
error!("{:?}", err);
error!("{}", output_string.as_str());
return Err(CmdError::Io(Error::new(
std::io::ErrorKind::InvalidData,
output_string,
)));
}
};
if result.items.is_empty()
|| result
.items
.first()
.unwrap()
.status
.container_statuses
.is_empty()
{
return Ok(None);
}
let first_item = result.items.first().unwrap();
let is_ready = match first_item.status.phase {
KubernetesPodStatusPhase::Running => true,
_ => false,
};
Ok(Some(is_ready))
}
pub fn kubectl_exec_is_job_ready_with_retry<P>(
kubernetes_config: P,
namespace: &str,
job_name: &str,
envs: Vec<(&str, &str)>,
) -> Result<Option<bool>, CmdError>
where
P: AsRef<Path>,
{
// TODO check this
let result = retry::retry(Fibonacci::from_millis(3000).take(10), || {
let r = crate::cmd::kubectl::kubectl_exec_is_job_ready(
kubernetes_config.as_ref(),
namespace,
job_name,
envs.clone(),
);
match r {
Ok(is_ready) => match is_ready {
Some(true) => OperationResult::Ok(true),
_ => {
let t = format!("job {} is not ready yet", job_name);
info!("{}", t.as_str());
OperationResult::Retry(t)
}
},
Err(err) => OperationResult::Err(format!("command error: {:?}", err)),
}
});
match result {
Err(err) => match err {
retry::Error::Operation {
error: _,
total_delay: _,
tries: _,
} => Ok(Some(false)),
retry::Error::Internal(err) => Err(CmdError::Unexpected(err)),
},
Ok(_) => Ok(Some(true)),
}
}
pub fn kubectl_exec_is_job_ready<P>(
kubernetes_config: P,
namespace: &str,
job_name: &str,
envs: Vec<(&str, &str)>,
) -> Result<Option<bool>, CmdError>
where
P: AsRef<Path>,
{
let mut _envs = Vec::with_capacity(envs.len() + 1);
_envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap()));
_envs.extend(envs);
let mut output_vec: Vec<String> = Vec::with_capacity(20);
let _ = kubectl_exec_with_output(
vec!["get", "job", "-o", "json", "-n", namespace, job_name],
_envs,
|out| match out {
Ok(line) => output_vec.push(line),
Err(err) => error!("{:?}", err),
},
|out| match out {
Ok(line) => error!("{}", line),
Err(err) => error!("{:?}", err),
},
)?;
let output_string: String = output_vec.join("");
let result = match serde_json::from_str::<KubernetesJob>(output_string.as_str()) {
Ok(x) => x,
Err(err) => {
error!("{:?}", err);
error!("{}", output_string.as_str());
return Err(CmdError::Io(Error::new(
std::io::ErrorKind::InvalidData,
output_string,
)));
}
};
if result.status.succeeded > 0 {
return Ok(Some(true));
}
Ok(Some(false))
}
pub fn kubectl_exec_create_namespace<P>(
kubernetes_config: P,
namespace: &str,
envs: Vec<(&str, &str)>,
) -> Result<(), CmdError>
where
P: AsRef<Path>,
{
let mut _envs = Vec::with_capacity(envs.len() + 1);
_envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap()));
_envs.extend(envs);
let _ = kubectl_exec_with_output(
vec!["create", "namespace", namespace],
_envs,
|out| match out {
Ok(line) => info!("{}", line),
Err(err) => error!("{:?}", err),
},
|out| match out {
Ok(line) => error!("{}", line),
Err(err) => error!("{:?}", err),
},
)?;
Ok(())
}
// used for testing the does_contain_terraform_tfstate
pub fn create_sample_secret_terraform_in_namespace<P>(
kubernetes_config: P,
namespace_to_override: &str,
envs: &Vec<(&str, &str)>,
) -> Result<String, CmdError>
where
P: AsRef<Path>,
{
let mut _envs = Vec::with_capacity(envs.len() + 1);
let mut output_vec: Vec<String> = Vec::new();
_envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap()));
_envs.extend(envs);
let _ = kubectl_exec_with_output(
vec![
"create",
"secret",
"tfstate-default-state",
"--from-literal=blablablabla",
"--namespace",
namespace_to_override,
],
_envs,
|out| match out {
Ok(_line) => output_vec.push(_line),
Err(err) => error!("{:?}", err),
},
|out| match out {
Ok(_line) => {}
Err(err) => error!("{:?}", err),
},
);
Ok(output_vec.join(""))
}
pub fn does_contain_terraform_tfstate<P>(
kubernetes_config: P,
namespace: &str,
envs: &Vec<(&str, &str)>,
) -> Result<bool, CmdError>
where
P: AsRef<Path>,
{
let mut _envs = Vec::with_capacity(envs.len() + 1);
_envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap()));
_envs.extend(envs);
let mut exist = true;
let _ = kubectl_exec_with_output(
vec![
"describe",
"secrets/tfstate-default-state",
"--namespace",
namespace,
],
_envs,
|out| match out {
Ok(_line) => exist = true,
Err(err) => error!("{:?}", err),
},
|out| match out {
Ok(_line) => {}
Err(err) => error!("{:?}", err),
},
)?;
Ok(exist)
}
pub fn kubectl_exec_get_all_namespaces<P>(
kubernetes_config: P,
envs: Vec<(&str, &str)>,
) -> Result<Vec<String>, Error>
where
P: AsRef<Path>,
{
let mut _envs = Vec::with_capacity(envs.len() + 1);
_envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap()));
_envs.extend(envs);
let mut output_vec: Vec<String> = Vec::new();
let _ = kubectl_exec_with_output(
vec!["get", "namespaces", "-o", "json"],
_envs,
|out| match out {
Ok(line) => output_vec.push(line),
Err(err) => error!("{:?}", err),
},
|out| match out {
Ok(line) => error!("{}", line),
Err(err) => error!("{:?}", err),
},
)?;
let mut output_string: String = output_vec.join("");
let mut to_return: Vec<String> = Vec::new();
let result = serde_json::from_str::<KubernetesList<Item>>(output_string.as_str());
match result {
Ok(out) => {
for item in out.items {
to_return.push(item.metadata.name);
}
}
Err(e) => {
error!("While deserializing Kubernetes namespaces names {}", e);
return Err(Error::from(CmdError::Io(Error::new(
std::io::ErrorKind::InvalidData,
output_string,
))));
}
};
Ok(to_return)
}
pub fn kubectl_exec_delete_namespace<P>(
kubernetes_config: P,
namespace: &str,
envs: Vec<(&str, &str)>,
) -> Result<(), CmdError>
where
P: AsRef<Path>,
{
match does_contain_terraform_tfstate(&kubernetes_config, &namespace, &envs) {
Ok(exist) => match exist {
true => {
return Err(CmdError::Io(Error::new(
std::io::ErrorKind::Other,
"Namespace contains terraform tfstates in secret, can't delete it !",
)));
}
false => info!(
"Namespace {} doesn't contain any tfstates, able to delete it",
namespace
),
},
Err(e) => warn!(
"Unable to execute describe on secrets: {}. it may not exist anymore?",
e
),
};
let mut _envs = Vec::with_capacity(envs.len() + 1);
_envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap()));
_envs.extend(envs);
let _ = kubectl_exec_with_output(
vec!["delete", "namespace", namespace],
_envs,
|out| match out {
Ok(line) => info!("{}", line),
Err(err) => error!("{:?}", err),
},
|out| match out {
Ok(line) => error!("{}", line),
Err(err) => error!("{:?}", err),
},
)?;
Ok(())
}
pub fn kubectl_exec_delete_secret<P>(
kubernetes_config: P,
secret: &str,
envs: Vec<(&str, &str)>,
) -> Result<(), CmdError>
where
P: AsRef<Path>,
{
let mut _envs = Vec::with_capacity(envs.len() + 1);
_envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap()));
_envs.extend(envs);
let _ = kubectl_exec_with_output(
vec!["delete", "secret", secret],
_envs,
|out| match out {
Ok(line) => info!("{}", line),
Err(err) => error!("{:?}", err),
},
|out| match out {
Ok(line) => error!("{}", line),
Err(err) => error!("{:?}", err),
},
)?;
Ok(())
}
pub fn kubectl_exec_logs<P>(
kubernetes_config: P,
namespace: &str,
selector: &str,
envs: Vec<(&str, &str)>,
) -> Result<String, CmdError>
where
P: AsRef<Path>,
{
let mut _envs = Vec::with_capacity(envs.len() + 1);
_envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap()));
_envs.extend(envs);
let mut output_vec: Vec<String> = Vec::with_capacity(50);
let _ = kubectl_exec_with_output(
vec!["logs", "--tail", "1000", "-n", namespace, "-l", selector],
_envs,
|out| match out {
Ok(line) => output_vec.push(line),
Err(err) => error!("{:?}", err),
},
|out| match out {
Ok(line) => error!("{}", line),
Err(err) => error!("{:?}", err),
},
)?;
Ok(output_vec.join("\n"))
}
pub fn kubectl_exec_describe_pod<P>(
kubernetes_config: P,
namespace: &str,
selector: &str,
envs: Vec<(&str, &str)>,
) -> Result<String, CmdError>
where
P: AsRef<Path>,
{
let mut _envs = Vec::with_capacity(envs.len() + 1);
_envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap()));
_envs.extend(envs);
let mut output_vec: Vec<String> = Vec::with_capacity(50);
let _ = kubectl_exec_with_output(
vec!["describe", "pod", "-n", namespace, "-l", selector],
_envs,
|out| match out {
Ok(line) => output_vec.push(line),
Err(err) => error!("{:?}", err),
},
|out| match out {
Ok(line) => error!("{}", line),
Err(err) => error!("{:?}", err),
},
)?;
Ok(output_vec.join("\n"))
}
pub fn kubectl_exec_get_node<P>(
kubernetes_config: P,
envs: Vec<(&str, &str)>,
) -> Result<KubernetesList<KubernetesNode>, CmdError>
where
P: AsRef<Path>,
{
let mut _envs = Vec::with_capacity(envs.len() + 1);
_envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap()));
_envs.extend(envs);
let mut output_vec: Vec<String> = Vec::with_capacity(50);
let _ = kubectl_exec_with_output(
vec!["get", "node", "-o", "json"],
_envs,
|out| match out {
Ok(line) => output_vec.push(line),
Err(err) => error!("{:?}", err),
},
|out| match out {
Ok(line) => error!("{}", line),
Err(err) => error!("{:?}", err),
},
)?;
let output_string: String = output_vec.join("");
let result =
match serde_json::from_str::<KubernetesList<KubernetesNode>>(output_string.as_str()) {
Ok(x) => x,
Err(err) => {
error!("{:?}", err);
error!("{}", output_string.as_str());
return Err(CmdError::Io(Error::new(
std::io::ErrorKind::InvalidData,
output_string,
)));
}
};
Ok(result)
}

5
src/cmd/mod.rs Normal file
View File

@@ -0,0 +1,5 @@
pub mod helm;
pub mod kubectl;
pub mod structs;
pub mod terraform;
pub mod utilities;

167
src/cmd/structs.rs Normal file
View File

@@ -0,0 +1,167 @@
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "camelCase")]
pub struct KubernetesList<T> {
pub items: Vec<T>,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "camelCase")]
pub struct KubernetesService {
pub status: KubernetesServiceStatus,
}
#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Labels {
pub name: String,
}
#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Spec {
pub finalizers: Vec<String>,
}
#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Status {
pub phase: String,
}
#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Metadata2 {
pub resource_version: String,
pub self_link: String,
}
#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Item {
pub api_version: String,
pub kind: String,
pub metadata: Metadata,
pub spec: Spec,
pub status: Status,
}
#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Metadata {
pub creation_timestamp: String,
pub labels: Option<Labels>,
pub name: String,
pub resource_version: String,
pub self_link: String,
pub uid: String,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "camelCase")]
pub struct KubernetesServiceStatus {
pub load_balancer: KubernetesServiceStatusLoadBalancer,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "camelCase")]
pub struct KubernetesServiceStatusLoadBalancer {
pub ingress: Vec<KubernetesServiceStatusLoadBalancerIngress>,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "camelCase")]
pub struct KubernetesServiceStatusLoadBalancerIngress {
pub hostname: String,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "camelCase")]
pub struct KubernetesPod {
pub status: KubernetesPodStatus,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "camelCase")]
pub struct KubernetesPodStatus {
pub container_statuses: Vec<KubernetesPodContainerStatus>,
// read the doc: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/
// phase can be Pending, Running, Succeeded, Failed, Unknown
pub phase: KubernetesPodStatusPhase,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
pub enum KubernetesPodStatusPhase {
Pending,
Running,
Succeeded,
Failed,
Unknown,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "camelCase")]
pub struct KubernetesPodContainerStatus {
pub ready: bool,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "camelCase")]
pub struct KubernetesJob {
pub status: KubernetesJobStatus,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "camelCase")]
pub struct KubernetesJobStatus {
pub succeeded: u32,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "camelCase")]
pub struct KubernetesNode {
pub status: KubernetesNodeStatus,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "camelCase")]
pub struct KubernetesNodeStatus {
pub allocatable: KubernetesNodeStatusResources,
pub capacity: KubernetesNodeStatusResources,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "camelCase")]
pub struct KubernetesNodeStatusResources {
pub cpu: String,
pub memory: String,
pub pods: String,
}
#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Helm {
pub name: String,
pub namespace: String,
pub revision: String,
pub updated: String,
pub status: String,
pub chart: String,
#[serde(rename = "app_version")]
pub app_version: String,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
pub struct HelmHistoryRow {
pub revision: u16,
pub status: String,
pub chart: String,
pub app_version: String,
}
impl HelmHistoryRow {
pub fn is_successfully_deployed(&self) -> bool {
self.status == "deployed"
}
}

101
src/cmd/terraform.rs Normal file
View File

@@ -0,0 +1,101 @@
use std::ffi::OsStr;
use std::fmt::{Display, Formatter};
use std::io::Error;
use std::io::{BufRead, BufReader};
use std::path::Path;
use std::process::{Child, Command, ExitStatus, Stdio};
use dirs::home_dir;
use retry::delay::Fibonacci;
use retry::OperationResult;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::cmd::utilities::{exec_with_envs_and_output, CmdError};
use crate::constants::{KUBECONFIG, TF_PLUGIN_CACHE_DIR};
fn terraform_exec_with_init_validate(
root_dir: &str,
first_time_init_terraform: bool,
) -> Result<(), CmdError> {
// terraform init
let init_args = if first_time_init_terraform {
vec!["init"]
} else {
vec!["init"]
};
//TODO print
terraform_exec(root_dir, init_args)?;
// terraform validate config
terraform_exec(root_dir, vec!["validate"])?;
Ok(())
}
fn terraform_exec_with_init_validate_plan(
root_dir: &str,
first_time_init_terraform: bool,
) -> Result<(), CmdError> {
// terraform init
let init_args = if first_time_init_terraform {
vec!["init"]
} else {
vec!["init"]
};
//TODO print
terraform_exec(root_dir, init_args)?;
// terraform validate config
terraform_exec(root_dir, vec!["validate"])?;
// terraform plan
terraform_exec(root_dir, vec!["plan", "-out", "tf_plan"])?;
Ok(())
}
pub fn terraform_exec_with_init_validate_plan_apply(
root_dir: &str,
first_time_init_terraform: bool,
) -> Result<(), CmdError> {
// terraform init and plan
terraform_exec_with_init_validate_plan(root_dir, first_time_init_terraform);
// terraform apply
terraform_exec(root_dir, vec!["apply", "-auto-approve", "tf_plan"])?;
Ok(())
}
pub fn terraform_exec_with_init_validate_destroy(root_dir: &str) -> Result<(), CmdError> {
// terraform init and plan
terraform_exec_with_init_validate(root_dir, false);
// terraform destroy
terraform_exec(root_dir, vec!["destroy", "-auto-approve"])
}
pub fn terraform_exec(root_dir: &str, args: Vec<&str>) -> Result<(), CmdError> {
let home_dir = home_dir().expect("Could not find $HOME");
let tf_plugin_cache_dir = format!("{}/.terraform.d/plugin-cache", home_dir.to_str().unwrap());
match exec_with_envs_and_output(
format!("{} terraform", root_dir).as_str(),
args,
vec![(TF_PLUGIN_CACHE_DIR, tf_plugin_cache_dir.as_str())],
|line: Result<String, std::io::Error>| {
info!("{}", line.unwrap());
},
|line: Result<String, std::io::Error>| {
error!("{}", line.unwrap());
},
) {
Err(err) => return Err(err),
_ => {}
};
Ok(())
}

272
src/cmd/utilities.rs Normal file
View File

@@ -0,0 +1,272 @@
use std::ffi::OsStr;
use std::fmt::{Display, Formatter};
use std::io::Error;
use std::io::{BufRead, BufReader};
use std::path::Path;
use std::process::{Child, Command, ExitStatus, Stdio};
use dirs::home_dir;
use retry::delay::Fibonacci;
use retry::OperationResult;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::constants::{KUBECONFIG, TF_PLUGIN_CACHE_DIR};
fn command<P>(binary: P, args: Vec<&str>, envs: Option<Vec<(&str, &str)>>) -> Command
where
P: AsRef<Path>,
{
let s_binary = binary
.as_ref()
.to_str()
.unwrap()
.split_whitespace()
.map(|x| x.to_string())
.collect::<Vec<_>>();
let (current_dir, _binary) = if s_binary.len() == 1 {
(None, s_binary.first().unwrap().clone())
} else {
(
Some(s_binary.first().unwrap().clone()),
s_binary.get(1).unwrap().clone(),
)
};
let mut cmd = Command::new(&_binary);
cmd.args(&args)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
if current_dir.is_some() {
cmd.current_dir(current_dir.unwrap());
}
if envs.is_some() {
envs.unwrap().into_iter().for_each(|(k, v)| {
cmd.env(k, v);
});
}
cmd
}
pub fn exec<P>(binary: P, args: Vec<&str>) -> Result<(), CmdError>
where
P: AsRef<Path>,
{
let command_string = command_to_string(binary.as_ref(), &args);
info!("command: {}", command_string.as_str());
let exit_status = match command(binary, args, None).spawn().unwrap().wait() {
Ok(x) => x,
Err(err) => return Err(CmdError::Io(err)),
};
if exit_status.success() {
return Ok(());
}
Err(CmdError::Exec(exit_status))
}
pub fn exec_with_envs<P>(
binary: P,
args: Vec<&str>,
envs: Vec<(&str, &str)>,
) -> Result<(), CmdError>
where
P: AsRef<Path>,
{
let command_string = command_with_envs_to_string(binary.as_ref(), &args, &envs);
info!("command: {}", command_string.as_str());
let exit_status = match command(binary, args, Some(envs)).spawn().unwrap().wait() {
Ok(x) => x,
Err(err) => return Err(CmdError::Io(err)),
};
if exit_status.success() {
return Ok(());
}
Err(CmdError::Exec(exit_status))
}
fn _with_output<F, X>(mut child: Child, mut stdout_output: F, mut stderr_output: X) -> Child
where
F: FnMut(Result<String, Error>),
X: FnMut(Result<String, Error>),
{
let stdout_reader = BufReader::new(child.stdout.as_mut().unwrap());
for line in stdout_reader.lines() {
stdout_output(line);
}
let stderr_reader = BufReader::new(child.stderr.as_mut().unwrap());
for line in stderr_reader.lines() {
stderr_output(line);
}
child
}
pub fn exec_with_output<P, F, X>(
binary: P,
args: Vec<&str>,
stdout_output: F,
stderr_output: X,
) -> Result<(), CmdError>
where
P: AsRef<Path>,
F: FnMut(Result<String, Error>),
X: FnMut(Result<String, Error>),
{
let command_string = command_to_string(binary.as_ref(), &args);
info!("command: {}", command_string.as_str());
let mut child = _with_output(
command(binary, args, None).spawn().unwrap(),
stdout_output,
stderr_output,
);
let exit_status = match child.wait() {
Ok(x) => x,
Err(err) => return Err(CmdError::Io(err)),
};
if exit_status.success() {
return Ok(());
}
Err(CmdError::Exec(exit_status))
}
pub fn exec_with_envs_and_output<P, F, X>(
binary: P,
args: Vec<&str>,
envs: Vec<(&str, &str)>,
stdout_output: F,
stderr_output: X,
) -> Result<(), CmdError>
where
P: AsRef<Path>,
F: FnMut(Result<String, Error>),
X: FnMut(Result<String, Error>),
{
let command_string = command_with_envs_to_string(binary.as_ref(), &args, &envs);
info!("command: {}", command_string.as_str());
let mut child = _with_output(
command(binary, args, Some(envs)).spawn().unwrap(),
stdout_output,
stderr_output,
);
let exit_status = match child.wait() {
Ok(x) => x,
Err(err) => return Err(CmdError::Io(err)),
};
if exit_status.success() {
return Ok(());
}
Err(CmdError::Exec(exit_status))
}
// return the output of "binary_name" --version
pub fn run_version_command_for(binary_name: &str) -> String {
let mut output_from_cmd = String::new();
exec_with_output(
binary_name,
vec!["--version"],
|r_out| match r_out {
Ok(s) => output_from_cmd.push_str(&s.to_owned()),
Err(e) => error!("Error while getting stdout from {} {}", binary_name, e),
},
|r_err| match r_err {
Ok(s) => error!("Error executing {}", binary_name),
Err(e) => error!("Error while getting stderr from {} {}", binary_name, e),
},
);
output_from_cmd
}
pub fn does_binary_exist<S>(binary: S) -> bool
where
S: AsRef<OsStr>,
{
match Command::new(binary)
.stdout(Stdio::null())
.stdin(Stdio::null())
.stderr(Stdio::null())
.spawn()
{
Ok(_) => true,
_ => false,
}
}
pub fn command_to_string<P>(binary: P, args: &Vec<&str>) -> String
where
P: AsRef<Path>,
{
format!("{} {}", binary.as_ref().to_str().unwrap(), args.join(" "))
}
pub fn command_with_envs_to_string<P>(
binary: P,
args: &Vec<&str>,
envs: &Vec<(&str, &str)>,
) -> String
where
P: AsRef<Path>,
{
let _envs = envs
.iter()
.map(|(k, v)| format!("{}={}", k, v))
.collect::<Vec<_>>();
format!(
"{} {} {}",
_envs.join(" "),
binary.as_ref().to_str().unwrap(),
args.join(" ")
)
}
#[derive(Debug)]
pub enum CmdError {
Exec(ExitStatus),
Io(Error),
Unexpected(String),
}
impl Display for CmdError {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let s = match self {
CmdError::Exec(status) => format!("CmdError: Exec({})", status),
CmdError::Io(io) => format!("CmdError: IO: {}", io),
CmdError::Unexpected(s) => format!("CmdError: Unexpected: {}", s),
};
write!(f, "{}", s)
}
}
impl std::error::Error for CmdError {}
impl From<std::io::Error> for CmdError {
fn from(err: Error) -> Self {
CmdError::Io(err)
}
}
impl From<CmdError> for std::io::Error {
fn from(e: CmdError) -> Self {
std::io::Error::new(std::io::ErrorKind::Other, e)
}
}

4
src/constants.rs Normal file
View File

@@ -0,0 +1,4 @@
pub const TF_PLUGIN_CACHE_DIR: &str = "TF_PLUGIN_CACHE_DIR";
pub const AWS_ACCESS_KEY_ID: &str = "AWS_ACCESS_KEY_ID";
pub const AWS_SECRET_ACCESS_KEY: &str = "AWS_SECRET_ACCESS_KEY";
pub const KUBECONFIG: &str = "KUBECONFIG";

View File

@@ -0,0 +1,210 @@
use std::rc::Rc;
use crate::build_platform::Image;
use crate::cmd;
use crate::cmd::utilities::CmdError;
use crate::container_registry::{
ContainerRegistry, ContainerRegistryError, Kind, PushError, PushResult,
};
use crate::models::{Context, Listener, Listeners, ProgressListener};
pub struct DockerHub {
context: Context,
id: String,
name: String,
login: String,
password: String,
listeners: Listeners,
}
impl DockerHub {
pub fn new(context: Context, id: &str, name: &str, login: &str, password: &str) -> Self {
DockerHub {
context,
id: id.to_string(),
name: name.to_string(),
login: login.to_string(),
password: password.to_string(),
listeners: vec![],
}
}
}
impl ContainerRegistry for DockerHub {
fn context(&self) -> &Context {
&self.context
}
fn kind(&self) -> Kind {
Kind::DockerHub
}
fn id(&self) -> &str {
self.id.as_str()
}
fn name(&self) -> &str {
self.name.as_str()
}
fn is_valid(&self) -> Result<(), ContainerRegistryError> {
// check the version of docker and print it as info
let mut output_from_cmd = String::new();
cmd::utilities::exec_with_output(
"docker",
vec!["--version"],
|r_out| match r_out {
Ok(s) => output_from_cmd.push_str(&s.to_owned()),
Err(e) => error!("Error while getting sdtout from docker {}", e),
},
|r_err| match r_err {
Ok(s) => error!("Error executing docker command {}", s),
Err(e) => error!("Error while getting stderr from docker {}", e),
},
);
info!("Using Docker: {}", output_from_cmd);
Ok(())
}
fn add_listener(&mut self, listener: Listener) {
self.listeners.push(listener);
}
fn on_create(&self) -> Result<(), ContainerRegistryError> {
Ok(())
}
fn on_create_error(&self) -> Result<(), ContainerRegistryError> {
Ok(())
}
fn on_delete(&self) -> Result<(), ContainerRegistryError> {
Ok(())
}
fn on_delete_error(&self) -> Result<(), ContainerRegistryError> {
Ok(())
}
fn does_image_exists(&self, image: &Image) -> bool {
let envs = match self.context.docker_tcp_socket() {
Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())],
None => vec![],
};
// login into docker hub
match cmd::utilities::exec_with_envs(
"docker",
vec![
"login",
"-u",
self.login.as_str(),
"-p",
self.password.as_str(),
],
envs.clone(),
) {
Err(err) => match err {
CmdError::Exec(exit_status) => {
error!("Cannot login into dockerhub");
return false;
}
CmdError::Io(err) => {
error!("IO error on dockerhub login: {}", err);
return false;
}
CmdError::Unexpected(err) => {
error!("Unexpected error on dockerhub login: {}", err);
return false;
}
},
_ => {}
};
// check if image and tag exist
// note: to retrieve if specific tags exist you can specify the tag at the end of the cUrl path
let curl_path = format!(
"https://index.docker.io/v1/repositories/{}/tags/",
image.name
);
let mut exist_stdoud: bool = false;
let mut exist_stderr: bool = true;
cmd::utilities::exec_with_envs_and_output(
"curl",
vec!["--silent", "-f", "-lSL", &curl_path],
envs.clone(),
|r_out| match r_out {
Ok(s) => exist_stdoud = true,
Err(e) => error!("Error while getting stdout from curl {}", e),
},
|r_err| match r_err {
Ok(s) => exist_stderr = true,
Err(e) => error!("Error while getting stderr from curl {}", e),
},
);
exist_stdoud
}
fn push(&self, image: &Image, force_push: bool) -> Result<PushResult, PushError> {
let envs = match self.context.docker_tcp_socket() {
Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())],
None => vec![],
};
match cmd::utilities::exec_with_envs(
"docker",
vec![
"login",
"-u",
self.login.as_str(),
"-p",
self.password.as_str(),
],
envs.clone(),
) {
Err(err) => match err {
CmdError::Exec(exit_status) => return Err(PushError::CredentialsError),
CmdError::Io(err) => return Err(PushError::IoError(err)),
CmdError::Unexpected(err) => return Err(PushError::Unknown(err)),
},
_ => {}
};
let dest = format!("{}/{}", self.login.as_str(), image.name_with_tag().as_str());
match cmd::utilities::exec_with_envs(
"docker",
vec![
"tag",
dest.as_str(),
format!("{}/{}", self.login.as_str(), dest.as_str()).as_str(),
],
envs.clone(),
) {
Err(err) => match err {
CmdError::Exec(exit_status) => return Err(PushError::ImageTagFailed),
CmdError::Io(err) => return Err(PushError::IoError(err)),
CmdError::Unexpected(err) => return Err(PushError::Unknown(err)),
},
_ => {}
};
match cmd::utilities::exec_with_envs("docker", vec!["push", dest.as_str()], envs) {
Err(err) => match err {
CmdError::Exec(exit_status) => return Err(PushError::ImagePushFailed),
CmdError::Io(err) => return Err(PushError::IoError(err)),
CmdError::Unexpected(err) => return Err(PushError::Unknown(err)),
},
_ => {}
};
let mut image = image.clone();
image.registry_url = Some(dest);
Ok(PushResult { image })
}
fn push_error(&self, _image: &Image) -> Result<PushResult, PushError> {
unimplemented!()
}
}

View File

@@ -0,0 +1,188 @@
extern crate digitalocean;
use std::rc::Rc;
use digitalocean::DigitalOcean;
use crate::build_platform::Image;
use crate::cmd;
use crate::cmd::utilities::CmdError;
use crate::container_registry::{
ContainerRegistry, ContainerRegistryError, Kind, PushError, PushResult,
};
use crate::models::{Context, Listener, ProgressListener};
// TODO : use --output json
// see https://www.digitalocean.com/community/tutorials/how-to-use-doctl-the-official-digitalocean-command-line-client
pub struct DOCR {
pub context: Context,
pub registry_name: String,
pub api_key: String,
}
impl DOCR {
pub fn new(context: Context, registry_name: &str, api_key: &str) -> Self {
DOCR {
context,
registry_name: registry_name.to_string(),
api_key: api_key.to_string(),
}
}
pub fn client(&self) -> DigitalOcean {
DigitalOcean::new(self.api_key.as_str()).unwrap()
}
pub fn create_repository(&self, _image: &Image) -> Result<(), ContainerRegistryError> {
match cmd::utilities::exec(
"doctl",
vec![
"registry",
"create",
self.registry_name.as_str(),
"-t",
self.api_key.as_str(),
],
) {
Err(err) => match err {
CmdError::Exec(_exit_status) => return Err(ContainerRegistryError::Unknown),
CmdError::Io(err) => return Err(ContainerRegistryError::Unknown),
CmdError::Unexpected(err) => return Err(ContainerRegistryError::Unknown),
},
_ => {}
};
Ok(())
}
pub fn push_image(&self, dest: String, image: &Image) -> Result<PushResult, PushError> {
match cmd::utilities::exec(
"docker",
vec!["tag", image.name_with_tag().as_str(), dest.as_str()],
) {
Err(err) => match err {
CmdError::Exec(_exit_status) => return Err(PushError::ImageTagFailed),
CmdError::Io(err) => return Err(PushError::IoError(err)),
CmdError::Unexpected(err) => return Err(PushError::Unknown(err)),
},
_ => {}
};
match cmd::utilities::exec("docker", vec!["push", dest.as_str()]) {
Err(err) => match err {
CmdError::Exec(_exit_status) => return Err(PushError::ImagePushFailed),
CmdError::Io(err) => return Err(PushError::IoError(err)),
CmdError::Unexpected(err) => return Err(PushError::Unknown(err)),
},
_ => {}
};
let mut image = image.clone();
image.registry_url = Some(dest);
Ok(PushResult { image })
}
fn get_or_create_repository(&self, _image: &Image) -> Result<(), ContainerRegistryError> {
// TODO check if repository really exist
self.create_repository(&_image)
}
fn delete_repository(&self, _image: &Image) -> Result<(), ContainerRegistryError> {
match cmd::utilities::exec(
"doctl",
vec![
"registry",
"delete",
self.registry_name.as_str(),
"-f",
"-t",
self.api_key.as_str(),
],
) {
Err(err) => match err {
CmdError::Exec(exit_status) => return Err(ContainerRegistryError::Unknown),
CmdError::Io(err) => return Err(ContainerRegistryError::Unknown),
CmdError::Unexpected(err) => return Err(ContainerRegistryError::Unknown),
},
_ => {}
};
Ok(())
}
}
impl ContainerRegistry for DOCR {
fn context(&self) -> &Context {
&self.context
}
fn kind(&self) -> Kind {
Kind::DOCR
}
fn id(&self) -> &str {
unimplemented!()
}
fn name(&self) -> &str {
unimplemented!()
}
fn is_valid(&self) -> Result<(), ContainerRegistryError> {
unimplemented!()
}
fn add_listener(&mut self, _listener: Listener) {
unimplemented!()
}
fn on_create(&self) -> Result<(), ContainerRegistryError> {
unimplemented!()
}
fn on_create_error(&self) -> Result<(), ContainerRegistryError> {
unimplemented!()
}
fn on_delete(&self) -> Result<(), ContainerRegistryError> {
unimplemented!()
}
fn on_delete_error(&self) -> Result<(), ContainerRegistryError> {
unimplemented!()
}
fn does_image_exists(&self, _image: &Image) -> bool {
unimplemented!()
}
// https://www.digitalocean.com/docs/images/container-registry/how-to/use-registry-docker-kubernetes/
fn push(&self, image: &Image, _force_push: bool) -> Result<PushResult, PushError> {
let image = image.clone();
//TODO instead use get_or_create_repository
self.create_repository(&image);
match cmd::utilities::exec(
"doctl",
vec![
"registry",
"login",
self.registry_name.as_str(),
"-t",
self.api_key.as_str(),
],
) {
Err(err) => match err {
CmdError::Exec(_exit_status) => return Err(PushError::CredentialsError),
CmdError::Io(err) => return Err(PushError::IoError(err)),
CmdError::Unexpected(err) => return Err(PushError::Unknown(err)),
},
_ => {}
};
//TODO check force or not
let dest = format!("{}:{}", self.registry_name.as_str(), image.tag.as_str());
self.push_image(dest, &image)
}
fn push_error(&self, _image: &Image) -> Result<PushResult, PushError> {
unimplemented!()
}
}

View File

@@ -0,0 +1,382 @@
use std::rc::Rc;
use std::str::FromStr;
use rusoto_core::{Client, HttpClient, Region, RusotoError};
use rusoto_credential::StaticProvider;
use rusoto_ecr::{
CreateRepositoryError, CreateRepositoryRequest, DescribeImagesRequest,
DescribeRepositoriesRequest, Ecr, EcrClient, GetAuthorizationTokenRequest, ImageDetail,
ImageIdentifier, PutLifecyclePolicyRequest, Repository,
};
use rusoto_sts::{GetCallerIdentityRequest, Sts, StsClient};
use crate::build_platform::Image;
use crate::cmd;
use crate::cmd::utilities::CmdError;
use crate::container_registry::{
ContainerRegistry, ContainerRegistryError, Kind, PushError, PushResult,
};
use crate::models::{
Context, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressListener,
ProgressScope,
};
use crate::runtime::async_run;
pub struct ECR {
context: Context,
id: String,
name: String,
access_key_id: String,
secret_access_key: String,
region: Region,
listeners: Listeners,
}
impl ECR {
pub fn new(
context: Context,
id: &str,
name: &str,
access_key_id: &str,
secret_access_key: &str,
region: &str,
) -> Self {
ECR {
context,
id: id.to_string(),
name: name.to_string(),
access_key_id: access_key_id.to_string(),
secret_access_key: secret_access_key.to_string(),
region: Region::from_str(region).unwrap(),
listeners: vec![],
}
}
pub fn credentials(&self) -> StaticProvider {
StaticProvider::new(
self.access_key_id.to_string(),
self.secret_access_key.to_string(),
None,
None,
)
}
pub fn client(&self) -> Client {
Client::new_with(self.credentials(), HttpClient::new().unwrap())
}
pub fn ecr_client(&self) -> EcrClient {
EcrClient::new_with_client(self.client(), self.region.clone())
}
fn get_repository(&self, image: &Image) -> Option<Repository> {
let mut drr = DescribeRepositoriesRequest::default();
drr.repository_names = Some(vec![image.name.to_string()]);
let r = async_run(self.ecr_client().describe_repositories(drr));
match r {
Err(_) => None,
Ok(res) => match res.repositories {
// assume there is only one repository returned - why? Because we set only one repository_names above
Some(repositories) => repositories.into_iter().next(),
_ => None,
},
}
}
fn get_image(&self, image: &Image) -> Option<ImageDetail> {
let mut dir = DescribeImagesRequest::default();
dir.repository_name = image.name.to_string();
let mut image_identifier = ImageIdentifier::default();
image_identifier.image_tag = Some(image.tag.to_string());
dir.image_ids = Some(vec![image_identifier]);
let r = async_run(self.ecr_client().describe_images(dir));
match r {
Err(_) => None,
Ok(res) => match res.image_details {
// assume there is only one repository returned - why? Because we set only one repository_names above
Some(image_details) => image_details.into_iter().next(),
_ => None,
},
}
}
fn docker_envs(&self) -> Vec<(&str, &str)> {
match self.context.docker_tcp_socket() {
Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())],
None => vec![],
}
}
fn push_image(&self, dest: String, image: &Image) -> Result<PushResult, PushError> {
// READ https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html
// docker tag e9ae3c220b23 aws_account_id.dkr.ecr.region.amazonaws.com/my-web-app
match cmd::utilities::exec_with_envs(
"docker",
vec!["tag", image.name_with_tag().as_str(), dest.as_str()],
self.docker_envs(),
) {
Err(err) => match err {
CmdError::Exec(_exit_status) => return Err(PushError::ImageTagFailed),
CmdError::Io(err) => return Err(PushError::IoError(err)),
CmdError::Unexpected(err) => return Err(PushError::Unknown(err)),
},
_ => {}
};
// docker push aws_account_id.dkr.ecr.region.amazonaws.com/my-web-app
match cmd::utilities::exec_with_envs(
"docker",
vec!["push", dest.as_str()],
self.docker_envs(),
) {
Err(err) => match err {
CmdError::Exec(_exit_status) => return Err(PushError::ImagePushFailed),
CmdError::Io(err) => return Err(PushError::IoError(err)),
CmdError::Unexpected(err) => return Err(PushError::Unknown(err)),
},
_ => {}
};
let mut image = image.clone();
image.registry_url = Some(dest);
Ok(PushResult { image })
}
fn create_repository(&self, image: &Image) -> Result<Repository, ContainerRegistryError> {
info!("ECR create repository {}", image.name.as_str());
let mut crr = CreateRepositoryRequest::default();
crr.repository_name = image.name.clone();
let r = async_run(self.ecr_client().create_repository(crr));
match r {
Err(err) => match err {
RusotoError::Service(ref err) => info!("{:?}", err),
_ => return Err(ContainerRegistryError::from(err)),
},
_ => {}
}
let mut plp = PutLifecyclePolicyRequest::default();
plp.repository_name = image.name.clone();
let ecr_policy = r#"
{
"rules": [
{
"action": {
"type": "expire"
},
"selection": {
"countType": "sinceImagePushed",
"countUnit": "days",
"countNumber": 1,
"tagStatus": "any"
},
"description": "Remove unit test images",
"rulePriority": 1
}
]
}
"#;
plp.lifecycle_policy_text = ecr_policy.to_string();
let r = async_run(self.ecr_client().put_lifecycle_policy(plp));
match r {
Err(err) => Err(ContainerRegistryError::from(err)),
_ => Ok(self.get_repository(&image).unwrap()),
}
}
fn get_or_create_repository(
&self,
image: &Image,
) -> Result<Repository, ContainerRegistryError> {
// check if the repository already exists
let repository = self.get_repository(&image);
if repository.is_some() {
info!("ECR repository {} already exists", image.name.as_str());
return Ok(repository.unwrap());
}
self.create_repository(&image)
}
}
impl ContainerRegistry for ECR {
fn context(&self) -> &Context {
&self.context
}
fn kind(&self) -> Kind {
Kind::ECR
}
fn id(&self) -> &str {
self.id.as_str()
}
fn name(&self) -> &str {
self.name.as_str()
}
fn is_valid(&self) -> Result<(), ContainerRegistryError> {
let client = StsClient::new_with_client(self.client(), Region::default());
let s = async_run(client.get_caller_identity(GetCallerIdentityRequest::default()));
match s {
Ok(_x) => Ok(()),
Err(err) => Err(ContainerRegistryError::from(err)),
}
}
fn add_listener(&mut self, listener: Listener) {
self.listeners.push(listener);
}
fn on_create(&self) -> Result<(), ContainerRegistryError> {
info!("ECR.on_create() called");
Ok(())
}
fn on_create_error(&self) -> Result<(), ContainerRegistryError> {
unimplemented!()
}
fn on_delete(&self) -> Result<(), ContainerRegistryError> {
unimplemented!()
}
fn on_delete_error(&self) -> Result<(), ContainerRegistryError> {
unimplemented!()
}
fn does_image_exists(&self, image: &Image) -> bool {
self.get_repository(&image).is_some()
}
fn push(&self, image: &Image, force_push: bool) -> Result<PushResult, PushError> {
let r = async_run(
self.ecr_client()
.get_authorization_token(GetAuthorizationTokenRequest::default()),
);
let (access_token, password, endpoint_url) = match r {
Ok(t) => match t.authorization_data {
Some(authorization_data) => {
let ad = authorization_data.first().unwrap();
let b64_token = ad.authorization_token.as_ref().unwrap();
let decoded_token = base64::decode(b64_token).unwrap();
let token = std::str::from_utf8(decoded_token.as_slice()).unwrap();
let s_token: Vec<&str> = token.split(":").collect::<Vec<_>>();
(
s_token.first().unwrap().to_string(),
s_token.get(1).unwrap().to_string(),
ad.clone().proxy_endpoint.unwrap(),
)
}
None => return Err(PushError::RepositoryInitFailure),
},
_ => return Err(PushError::RepositoryInitFailure),
};
let repository = match if force_push {
self.create_repository(&image)
} else {
self.get_or_create_repository(&image)
} {
Ok(r) => r,
_ => return Err(PushError::RepositoryInitFailure),
};
match cmd::utilities::exec_with_envs(
"docker",
vec![
"login",
"-u",
access_token.as_str(),
"-p",
password.as_str(),
endpoint_url.as_str(),
],
self.docker_envs(),
) {
Err(err) => match err {
CmdError::Exec(_exit_status) => return Err(PushError::CredentialsError),
CmdError::Io(err) => return Err(PushError::IoError(err)),
CmdError::Unexpected(err) => return Err(PushError::Unknown(err)),
},
_ => {}
};
let dest = format!(
"{}:{}",
repository.repository_uri.unwrap(),
image.tag.as_str()
);
let listeners_helper = ListenersHelper::new(&self.listeners);
if !force_push && self.get_image(image).is_some() {
// check if image does exist - if yes, do not upload it again
let info_message = format!(
"image {:?} does already exist into ECR {} repository - no need to upload it",
image,
self.name()
);
info!("{}", info_message.as_str());
listeners_helper.start_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
let mut image = image.clone();
image.registry_url = Some(dest);
return Ok(PushResult { image });
}
let info_message = format!(
"image {:?} does not exist into ECR {} repository - let's upload it",
image,
self.name()
);
info!("{}", info_message.as_str());
listeners_helper.start_in_progress(ProgressInfo::new(
ProgressScope::Application {
id: image.application_id.clone(),
},
ProgressLevel::Info,
Some(info_message),
self.context.execution_id(),
));
self.push_image(dest, image)
}
fn push_error(&self, image: &Image) -> Result<PushResult, PushError> {
// TODO change this
Ok(PushResult {
image: image.clone(),
})
}
}

View File

@@ -0,0 +1,76 @@
use std::error::Error;
use std::rc::Rc;
use rusoto_core::RusotoError;
use serde::{Deserialize, Serialize};
use crate::build_platform::Image;
use crate::models::{Context, Listener, ProgressListener};
pub mod docker_hub;
pub mod docr;
pub mod ecr;
pub trait ContainerRegistry {
fn context(&self) -> &Context;
fn kind(&self) -> Kind;
fn id(&self) -> &str;
fn name(&self) -> &str;
fn is_valid(&self) -> Result<(), ContainerRegistryError>;
fn add_listener(&mut self, listener: Listener);
fn on_create(&self) -> Result<(), ContainerRegistryError>;
fn on_create_error(&self) -> Result<(), ContainerRegistryError>;
fn on_delete(&self) -> Result<(), ContainerRegistryError>;
fn on_delete_error(&self) -> Result<(), ContainerRegistryError>;
fn does_image_exists(&self, image: &Image) -> bool;
fn push(&self, image: &Image, force_push: bool) -> Result<PushResult, PushError>;
fn push_error(&self, image: &Image) -> Result<PushResult, PushError>;
}
pub struct PushResult {
pub image: Image,
}
#[derive(Debug)]
pub enum PushError {
RepositoryInitFailure,
CredentialsError,
IoError(std::io::Error),
ImageTagFailed,
ImagePushFailed,
ImageAlreadyExists,
Unknown(String),
}
#[derive(Serialize, Deserialize, Clone)]
pub enum Kind {
DockerHub,
ECR,
DOCR,
}
#[derive(Debug, Eq, PartialEq)]
pub enum ContainerRegistryError {
Credentials,
Unknown,
}
impl<E> From<RusotoError<E>> for ContainerRegistryError {
fn from(error: RusotoError<E>) -> Self {
match error {
RusotoError::Credentials(_) => ContainerRegistryError::Credentials,
RusotoError::Service(_) => ContainerRegistryError::Unknown,
RusotoError::HttpDispatch(_) => ContainerRegistryError::Unknown,
RusotoError::Validation(_) => ContainerRegistryError::Unknown,
RusotoError::ParseError(_) => ContainerRegistryError::Unknown,
RusotoError::Unknown(e) => {
if e.status == 403 {
ContainerRegistryError::Credentials
} else {
ContainerRegistryError::Unknown
}
}
RusotoError::Blocking => ContainerRegistryError::Unknown,
}
}
}

14
src/crypto.rs Normal file
View File

@@ -0,0 +1,14 @@
use crypto::digest::Digest;
use crypto::sha1::Sha1;
pub fn to_sha1(input: &str) -> String {
let mut hasher = Sha1::new();
hasher.input_str(input);
hasher.result_str()
}
pub fn to_sha1_truncate_16(input: &str) -> String {
let mut hash_str = to_sha1(input);
hash_str.truncate(16);
hash_str
}

41
src/deletion_utilities.rs Normal file
View File

@@ -0,0 +1,41 @@
use crate::cmd::kubectl::{kubectl_exec_delete_namespace, kubectl_exec_get_all_namespaces};
use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY};
// this fn should implements the algorythm describe here: https://qovery.atlassian.net/secure/RapidBoard.jspa?rapidView=10&modal=detail&selectedIssue=DEV-283
pub fn get_firsts_namespaces_to_delete(namespaces: Vec<&str>) -> Vec<&str> {
// from all namesapce remove managed and never delete namespaces
let minus_managed = minus_namespaces(namespaces, get_qovery_managed_namespaces());
let minus_qovery_managed_and_never_delete =
minus_namespaces(minus_managed, get_never_delete_namespaces());
minus_qovery_managed_and_never_delete
}
fn minus_namespaces<'a>(all: Vec<&'a str>, to_remove_namespaces: Vec<&str>) -> Vec<&'a str> {
let reduced = all
.into_iter()
.filter(|item| !to_remove_namespaces.contains(item))
.collect();
return reduced;
}
// TODO: use label instead
// TODO: create enum: deletion_rule [system, qovery,..]
pub fn get_qovery_managed_namespaces() -> Vec<&'static str> {
let mut qovery_managed_namespaces = Vec::with_capacity(5);
qovery_managed_namespaces.push("logging");
qovery_managed_namespaces.push("nginx");
qovery_managed_namespaces.push("qovery");
qovery_managed_namespaces.push("cert-manager");
qovery_managed_namespaces.push("prometheus");
return qovery_managed_namespaces;
}
// TODO: use label instead
fn get_never_delete_namespaces() -> Vec<&'static str> {
let mut kubernetes_never_delete_namespaces = Vec::with_capacity(4);
kubernetes_never_delete_namespaces.push("default");
kubernetes_never_delete_namespaces.push("kube-node-lease");
kubernetes_never_delete_namespaces.push("kube-public");
kubernetes_never_delete_namespaces.push("kube-system");
return kubernetes_never_delete_namespaces;
}

View File

@@ -0,0 +1,75 @@
use std::net::Ipv4Addr;
use crate::dns_provider::{DnsProvider, DnsProviderError, Kind};
use crate::models::Context;
pub struct Cloudflare {
context: Context,
id: String,
name: String,
domain: String,
cloudflare_api_token: String,
cloudflare_email: String,
}
impl Cloudflare {
pub fn new(
context: Context,
id: String,
name: String,
domain: String,
cloudflare_api_token: String,
cloudflare_email: String,
) -> Self {
Cloudflare {
context,
id,
name,
domain,
cloudflare_api_token,
cloudflare_email,
}
}
}
impl DnsProvider for Cloudflare {
fn context(&self) -> &Context {
&self.context
}
fn kind(&self) -> Kind {
Kind::CLOUDFLARE
}
fn id(&self) -> &str {
&self.id
}
fn name(&self) -> &str {
&self.name
}
fn account(&self) -> &str {
&self.cloudflare_email
}
fn token(&self) -> &str {
&self.cloudflare_api_token
}
fn domain(&self) -> &str {
self.domain.as_str()
}
fn resolvers(&self) -> Vec<Ipv4Addr> {
vec![Ipv4Addr::new(1, 1, 1, 1), Ipv4Addr::new(1, 0, 0, 1)]
}
fn is_valid(&self) -> Result<(), DnsProviderError> {
if self.cloudflare_api_token.is_empty() || self.cloudflare_email.is_empty() {
Err(DnsProviderError::Credentials)
} else {
Ok(())
}
}
}

30
src/dns_provider/mod.rs Normal file
View File

@@ -0,0 +1,30 @@
use std::net::Ipv4Addr;
use serde::{Deserialize, Serialize};
use crate::models::Context;
pub mod cloudflare;
#[derive(Serialize, Deserialize, Clone)]
pub enum Kind {
CLOUDFLARE,
}
pub trait DnsProvider {
fn context(&self) -> &Context;
fn kind(&self) -> Kind;
fn id(&self) -> &str;
fn name(&self) -> &str;
fn account(&self) -> &str;
fn token(&self) -> &str;
fn domain(&self) -> &str;
fn resolvers(&self) -> Vec<Ipv4Addr>;
fn is_valid(&self) -> Result<(), DnsProviderError>;
}
#[derive(Debug, Eq, PartialEq)]
pub enum DnsProviderError {
Credentials,
Unknown,
}

67
src/dynamo_db.rs Normal file
View File

@@ -0,0 +1,67 @@
use std::io::{Error, ErrorKind};
use rusoto_core::{Client, HttpClient, Region, RusotoError};
use rusoto_credential::StaticProvider;
use rusoto_dynamodb::{
AttributeDefinition, CreateTableError, CreateTableInput, DynamoDb, DynamoDbClient,
KeySchemaElement,
};
use crate::runtime::async_run;
pub fn create_terraform_table(
access_key_id: &str,
secret_access_key: &str,
region: &Region,
table_name: &str,
) -> Result<(), Error> {
let access_key_id = access_key_id.to_string();
let secret_access_key = secret_access_key.to_string();
let table_name = table_name.to_string();
let credentials = StaticProvider::new(access_key_id, secret_access_key, None, None);
let client = Client::new_with(credentials, HttpClient::new().unwrap());
let ddb_client = DynamoDbClient::new_with_client(client, region.clone());
let mut cti = CreateTableInput::default();
cti.table_name = table_name;
cti.billing_mode = Some("PAY_PER_REQUEST".to_string());
cti.key_schema = vec![KeySchemaElement {
attribute_name: "LockID".to_string(),
key_type: "HASH".to_string(),
}];
cti.attribute_definitions = vec![AttributeDefinition {
attribute_name: "LockID".to_string(),
attribute_type: "S".to_string(),
}];
let r = async_run(ddb_client.create_table(cti));
// FIXME: return a custom DynamoDBError?
match r {
Err(err) => match err {
RusotoError::Unknown(r) => {
error!("{}", r.body_as_str());
Err(Error::new(ErrorKind::Other, r.body_as_str()))
}
RusotoError::Service(r) => match r {
CreateTableError::ResourceInUse(_) => Ok(()), // table already exists
_ => {
return Err(Error::new(
ErrorKind::Other,
"something goes wrong while creating terraform DynamoDB table",
));
}
},
_ => {
return Err(Error::new(
ErrorKind::Other,
"something goes wrong while creating terraform DynamoDB table",
));
}
},
Ok(_x) => Ok(()),
}
}

89
src/engine.rs Normal file
View File

@@ -0,0 +1,89 @@
use std::borrow::Borrow;
use crate::build_platform::BuildPlatform;
use crate::cloud_provider::CloudProvider;
use crate::container_registry::ContainerRegistry;
use crate::dns_provider::DnsProvider;
use crate::error::ConfigurationError;
use crate::models::Context;
use crate::session::Session;
pub struct Engine {
context: Context,
build_platform: Box<dyn BuildPlatform>,
container_registry: Box<dyn ContainerRegistry>,
cloud_provider: Box<dyn CloudProvider>,
dns_provider: Box<dyn DnsProvider>,
}
impl Engine {
pub fn new(
context: Context,
build_platform: Box<dyn BuildPlatform>,
container_registry: Box<dyn ContainerRegistry>,
cloud_provider: Box<dyn CloudProvider>,
dns_provider: Box<dyn DnsProvider>,
) -> Engine {
Engine {
context,
build_platform,
container_registry,
cloud_provider,
dns_provider,
}
}
}
impl<'a> Engine {
pub fn context(&self) -> &Context {
&self.context
}
pub fn build_platform(&self) -> &dyn BuildPlatform {
self.build_platform.borrow()
}
pub fn container_registry(&self) -> &dyn ContainerRegistry {
self.container_registry.borrow()
}
pub fn cloud_provider(&self) -> &dyn CloudProvider {
self.cloud_provider.borrow()
}
pub fn dns_provider(&self) -> &dyn DnsProvider {
self.dns_provider.borrow()
}
pub fn is_valid(&self) -> Result<(), ConfigurationError> {
match self.build_platform.is_valid() {
Ok(_) => {}
Err(err) => {
return Err(ConfigurationError::BuildPlatform(err));
}
}
match self.container_registry.is_valid() {
Ok(_) => {}
Err(err) => {
return Err(ConfigurationError::ContainerRegistry(err));
}
}
match self.cloud_provider.is_valid() {
Ok(_) => {}
Err(err) => {
return Err(ConfigurationError::CloudProvider(err));
}
}
Ok(())
}
/// check and init the connection to all the services
pub fn session(&'a self) -> Result<Session<'a>, ConfigurationError> {
match self.is_valid() {
Ok(_) => Ok(Session::<'a> { engine: self }),
Err(err) => Err(err),
}
}
}

10
src/error.rs Normal file
View File

@@ -0,0 +1,10 @@
use crate::build_platform::error::BuildPlatformError;
use crate::cloud_provider::CloudProviderError;
use crate::container_registry::ContainerRegistryError;
#[derive(Debug)]
pub enum ConfigurationError {
BuildPlatform(BuildPlatformError),
ContainerRegistry(ContainerRegistryError),
CloudProvider(CloudProviderError),
}

118
src/fs.rs Normal file
View File

@@ -0,0 +1,118 @@
use std::fs;
use std::fs::{create_dir_all, File};
use std::io::Error;
use std::path::Path;
use flate2::write::GzEncoder;
use flate2::Compression;
use walkdir::WalkDir;
pub fn copy_files(from: &Path, to: &Path, exclude_j2_files: bool) -> Result<(), Error> {
let files = WalkDir::new(from)
.follow_links(true)
.into_iter()
.filter_map(|e| e.ok());
let files = match exclude_j2_files {
true => files
.filter(|e| {
// return only non *.j2.* files
e.file_name()
.to_str()
.map(|s| !s.contains(".j2."))
.unwrap_or(false)
})
.collect::<Vec<_>>(),
false => files.collect::<Vec<_>>(),
};
let _ = fs::create_dir_all(to)?;
let from_str = from.to_str().unwrap();
for file in files {
let path_str = file.path().to_str().unwrap();
let dest = format!(
"{}{}",
to.to_str().unwrap(),
path_str.replace(from_str, "").as_str()
);
if file.metadata().unwrap().is_dir() {
let _ = fs::create_dir_all(&dest)?;
}
let _ = fs::copy(file.path(), dest);
}
Ok(())
}
pub fn root_workspace_directory<X, S>(working_root_dir: X, execution_id: S) -> String
where
X: AsRef<Path>,
S: AsRef<Path>,
{
workspace_directory(working_root_dir, execution_id, ".")
}
pub fn workspace_directory<X, S, P>(working_root_dir: X, execution_id: S, dir_name: P) -> String
where
X: AsRef<Path>,
S: AsRef<Path>,
P: AsRef<Path>,
{
let dir = format!(
"{}/.qovery-workspace/{}/{}",
working_root_dir.as_ref().to_str().unwrap(),
execution_id.as_ref().to_str().unwrap(),
dir_name.as_ref().to_str().unwrap(),
);
let _ = create_dir_all(&dir);
dir
}
fn archive_workspace_directory(
working_root_dir: &str,
execution_id: &str,
) -> Result<File, std::io::Error> {
let workspace_dir = crate::fs::root_workspace_directory(working_root_dir, execution_id);
let tar_gz_file_path = format!(
"{}/.qovery-workspace/{}.tar.gz",
working_root_dir, execution_id
);
let tar_gz_file = File::create(tar_gz_file_path.as_str())?;
let enc = GzEncoder::new(tar_gz_file, Compression::fast());
let mut tar = tar::Builder::new(enc);
tar.append_dir_all(execution_id, workspace_dir)?;
Ok(File::open(tar_gz_file_path).unwrap())
}
pub fn cleanup_workspace_directory(working_root_dir: &str, execution_id: &str) {
let workspace_dir = crate::fs::root_workspace_directory(working_root_dir, execution_id);
std::fs::remove_dir_all(workspace_dir);
}
pub fn create_workspace_archive(
working_root_dir: &str,
execution_id: &str,
) -> Result<File, std::io::Error> {
info!("archive workspace directory in progress");
match archive_workspace_directory(working_root_dir, execution_id) {
Err(err) => {
error!("archive workspace directory error: {:?}", err);
Err(err)
}
Ok(file) => {
info!("workspace directory is archived");
cleanup_workspace_directory(working_root_dir, execution_id);
Ok(file)
}
}
}

104
src/git.rs Normal file
View File

@@ -0,0 +1,104 @@
use std::path::Path;
use git2::build::RepoBuilder;
use git2::{Error, Oid, Repository, Submodule};
/// TODO support SSH repository_url - we assume that the repository URL starts with HTTPS
/// TODO support git submodules
pub fn clone<P>(
repository_url: &str,
into_dir: P,
credentials: &Option<Credentials>,
) -> Result<Repository, Error>
where
P: AsRef<Path>,
{
let final_repository_url = match credentials {
Some(c) => format!(
"https://{}:{}@{}",
c.login,
c.password,
repository_url.replace("https://", "")
),
None => repository_url.to_string(),
};
RepoBuilder::new().clone(final_repository_url.as_str(), into_dir.as_ref())
}
pub fn checkout(repo: &Repository, commit_id: &str, repo_url: &str) -> Result<(), Error> {
let oid = match Oid::from_str(&commit_id) {
Err(e) => {
let mut x = git2::Error::from_str(
format!(
"Error while trying to validate commit ID {} on repository {}: {}",
&commit_id, &repo_url, &e
)
.as_ref(),
);
return Err(x);
}
Ok(o) => o,
};
let _ = match repo.find_commit(oid) {
Err(e) => {
let mut x = git2::Error::from_str(
format!(
"Commit ID {} on repository {} was not found",
&commit_id, &repo_url
)
.as_ref(),
);
x.set_code(e.code());
x.set_class(e.class());
return Err(x);
}
Ok(c) => c,
};
let obj = match repo.revparse_single(&commit_id) {
Err(e) => {
let mut x = git2::Error::from_str(
format!(
"Wasn't able to use git object commit ID {} on repository {}: {}",
&commit_id, &repo_url, &e
)
.as_ref(),
);
return Err(x);
}
Ok(o) => o,
};
repo.checkout_tree(&obj, None);
repo.set_head(&("refs/heads/".to_owned() + &commit_id))
}
pub fn checkout_submodules(repo: &Repository) -> Result<(), Error> {
match repo.submodules() {
Ok(submodules) => {
for mut submodule in submodules {
info!(
"getting submodule {:?} from {:?}",
submodule.name(),
submodule.url()
);
match submodule.update(true, None) {
Err(e) => return Err(e),
_ => (),
}
}
}
Err(err) => return Err(err),
}
Ok(())
}
pub struct Credentials {
pub login: String,
pub password: String,
}

25
src/lib.rs Normal file
View File

@@ -0,0 +1,25 @@
#[macro_use]
extern crate log;
extern crate tera;
pub mod build_platform;
pub mod cloud_provider;
pub mod cmd;
mod constants;
pub mod container_registry;
mod crypto;
mod deletion_utilities;
pub mod dns_provider;
mod dynamo_db;
pub mod engine;
pub mod error;
pub mod fs;
mod git;
pub mod models;
mod runtime;
pub mod s3;
pub mod session;
mod string;
mod template;
pub mod transaction;
mod unit_conversion;

833
src/models.rs Normal file
View File

@@ -0,0 +1,833 @@
use std::hash::Hash;
use std::rc::Rc;
use chrono::{DateTime, Utc};
use rand::distributions::Alphanumeric;
use rand::{thread_rng, Rng};
use serde::{Deserialize, Serialize};
use crate::build_platform::{Build, BuildOptions, GitRepository, Image};
use crate::cloud_provider::aws::databases::{MongoDB, MySQL, PostgreSQL};
use crate::cloud_provider::service::{DatabaseOptions, StatefulService, StatelessService};
use crate::cloud_provider::CloudProvider;
use crate::cloud_provider::Kind as CPKind;
use crate::git::Credentials;
use crate::models::DatabaseKind::Mongodb;
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
pub enum EnvironmentAction {
Environment(TargetEnvironment),
EnvironmentWithFailover(TargetEnvironment, FailoverEnvironment),
}
pub type TargetEnvironment = Environment;
pub type FailoverEnvironment = Environment;
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
pub struct Environment {
pub execution_id: String,
pub id: String,
pub kind: Kind,
pub owner_id: String,
pub project_id: String,
pub organization_id: String,
pub action: Action,
pub applications: Vec<Application>,
pub routers: Vec<Router>,
pub databases: Vec<Database>,
pub external_services: Vec<ExternalService>,
pub clone_from_environment_id: Option<String>,
}
impl Environment {
pub fn is_valid(&self) -> Result<(), EnvironmentError> {
Ok(())
}
pub fn to_qe_environment(
&self,
context: &Context,
built_applications: &Vec<Box<dyn crate::cloud_provider::service::Application>>,
cloud_provider: &dyn CloudProvider,
) -> crate::cloud_provider::environment::Environment {
let external_services = self
.external_services
.iter()
.map(
|x| match built_applications.iter().find(|y| x.id.as_str() == y.id()) {
Some(app) => {
x.to_stateless_service(context, app.image().clone(), cloud_provider)
}
_ => x.to_stateless_service(context, x.to_image(), cloud_provider),
},
)
.filter(|x| x.is_some())
.map(|x| x.unwrap())
.collect::<Vec<_>>();
let applications = self
.applications
.iter()
.map(
|x| match built_applications.iter().find(|y| x.id.as_str() == y.id()) {
Some(app) => {
x.to_stateless_service(context, app.image().clone(), cloud_provider)
}
_ => x.to_stateless_service(context, x.to_image(), cloud_provider),
},
)
.filter(|x| x.is_some())
.map(|x| x.unwrap())
.collect::<Vec<_>>();
let routers = self
.routers
.iter()
.map(|x| x.to_stateless_service(context, cloud_provider))
.filter(|x| x.is_some())
.map(|x| x.unwrap())
.collect::<Vec<_>>();
let mut stateless_services = external_services;
stateless_services.extend(routers);
stateless_services.extend(applications);
let databases = self
.databases
.iter()
.map(|x| x.to_stateful_service(context, cloud_provider))
.filter(|x| x.is_some())
.map(|x| x.unwrap())
.collect::<Vec<_>>();
let stateful_services = databases;
crate::cloud_provider::environment::Environment::new(
match self.kind {
Kind::Production => crate::cloud_provider::environment::Kind::Production,
Kind::Development => crate::cloud_provider::environment::Kind::Development,
},
self.id.as_str(),
self.project_id.as_str(),
self.owner_id.as_str(),
self.organization_id.as_str(),
stateless_services,
stateful_services,
)
}
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub enum Kind {
Production,
Development,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub enum Action {
Create,
Pause,
Delete,
Nothing,
}
impl Action {
pub fn to_service_action(&self) -> crate::cloud_provider::service::Action {
match self {
Action::Create => crate::cloud_provider::service::Action::Create,
Action::Pause => crate::cloud_provider::service::Action::Pause,
Action::Delete => crate::cloud_provider::service::Action::Delete,
Action::Nothing => crate::cloud_provider::service::Action::Nothing,
}
}
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
pub struct Application {
pub id: String,
pub name: String,
pub action: Action,
pub git_url: String,
pub git_credentials: GitCredentials,
pub branch: String,
pub commit_id: String,
pub dockerfile_path: String,
pub private_port: Option<u16>,
pub total_cpus: String,
pub cpu_burst: String,
pub total_ram_in_mib: u32,
pub total_instances: u16,
pub storage: Vec<Storage>,
pub environment_variables: Vec<EnvironmentVariable>,
}
impl Application {
pub fn to_application<'a>(
&self,
context: &Context,
image: &Image,
cloud_provider: &dyn CloudProvider,
) -> Option<Box<(dyn crate::cloud_provider::service::Application)>> {
match cloud_provider.kind() {
CPKind::AWS => Some(Box::new(
crate::cloud_provider::aws::application::Application::new(
context.clone(),
self.id.as_str(),
self.action.to_service_action(),
self.name.as_str(),
self.private_port,
self.total_cpus.clone(),
self.cpu_burst.clone(),
self.total_ram_in_mib,
self.total_instances,
image.clone(),
self.storage
.iter()
.map(|s| s.to_aws_storage())
.collect::<Vec<_>>(),
self.environment_variables
.iter()
.map(|ev| ev.to_aws_application_environment_variable())
.collect::<Vec<_>>(),
),
)),
CPKind::GCP => None,
_ => None,
//TODO to implement
}
}
pub fn to_stateless_service(
&self,
context: &Context,
image: Image,
cloud_provider: &dyn CloudProvider,
) -> Option<Box<dyn StatelessService>> {
match cloud_provider.kind() {
CPKind::AWS => Some(Box::new(
crate::cloud_provider::aws::application::Application::new(
context.clone(),
self.id.as_str(),
self.action.to_service_action(),
self.name.as_str(),
self.private_port,
self.total_cpus.clone(),
self.cpu_burst.clone(),
self.total_ram_in_mib,
self.total_instances,
image,
self.storage
.iter()
.map(|s| s.to_aws_storage())
.collect::<Vec<_>>(),
self.environment_variables
.iter()
.map(|ev| ev.to_aws_application_environment_variable())
.collect::<Vec<_>>(),
),
)),
CPKind::GCP => None,
_ => None,
//TODO to implement
}
}
pub fn to_image(&self) -> Image {
Image {
application_id: self.id.clone(),
name: self.name.clone(),
tag: self.commit_id.clone(),
commit_id: self.commit_id.clone(),
registry_url: None,
}
}
pub fn to_build(&self) -> Build {
Build {
git_repository: GitRepository {
url: self.git_url.clone(),
credentials: Some(Credentials {
login: self.git_credentials.login.clone(),
password: self.git_credentials.access_token.clone(),
}),
commit_id: self.commit_id.clone(),
dockerfile_path: self.dockerfile_path.clone(),
},
image: self.to_image(),
options: BuildOptions {
environment_variables: self
.environment_variables
.iter()
.map(|ev| crate::build_platform::EnvironmentVariable {
key: ev.key.clone(),
value: ev.value.clone(),
})
.collect::<Vec<_>>(),
},
}
}
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
pub struct EnvironmentVariable {
pub key: String,
pub value: String,
}
impl EnvironmentVariable {
pub fn to_aws_application_environment_variable(
&self,
) -> crate::cloud_provider::aws::application::EnvironmentVariable {
crate::cloud_provider::aws::application::EnvironmentVariable {
key: self.key.clone(),
value: self.value.clone(),
}
}
pub fn to_aws_external_service_environment_variable(
&self,
) -> crate::cloud_provider::aws::external_service::EnvironmentVariable {
crate::cloud_provider::aws::external_service::EnvironmentVariable {
key: self.key.clone(),
value: self.value.clone(),
}
}
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
pub struct GitCredentials {
pub login: String,
pub access_token: String,
pub expired_at: DateTime<Utc>,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
pub struct Storage {
pub id: String,
pub name: String,
pub storage_type: StorageType,
pub size_in_gib: u16,
pub mount_point: String,
pub snapshot_retention_in_days: u16,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub enum StorageType {
SlowHdd,
Hdd,
Ssd,
FastSsd,
}
impl Storage {
pub fn to_aws_storage(&self) -> crate::cloud_provider::aws::application::Storage {
crate::cloud_provider::aws::application::Storage {
id: self.id.clone(),
name: self.name.clone(),
storage_type: match self.storage_type {
StorageType::SlowHdd => crate::cloud_provider::aws::application::StorageType::SC1,
StorageType::Hdd => crate::cloud_provider::aws::application::StorageType::ST1,
StorageType::Ssd => crate::cloud_provider::aws::application::StorageType::GP2,
StorageType::FastSsd => crate::cloud_provider::aws::application::StorageType::IO1,
},
size_in_gib: self.size_in_gib,
mount_point: self.mount_point.clone(),
snapshot_retention_in_days: self.snapshot_retention_in_days,
}
}
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
pub struct Router {
pub id: String,
pub name: String,
pub action: Action,
pub default_domain: String,
pub public_port: u16,
pub custom_domains: Vec<CustomDomain>,
pub routes: Vec<Route>,
}
impl Router {
pub fn to_stateless_service(
&self,
context: &Context,
cloud_provider: &dyn CloudProvider,
) -> Option<Box<dyn StatelessService>> {
match cloud_provider.kind() {
CPKind::AWS => {
let router: Box<dyn StatelessService> =
Box::new(crate::cloud_provider::aws::router::Router::new(
context.clone(),
self.id.as_str(),
self.name.as_str(),
self.default_domain.as_str(),
self.custom_domains
.iter()
.map(|x| crate::cloud_provider::aws::router::CustomDomain {
domain: x.domain.clone(),
target_domain: x.target_domain.clone(),
})
.collect::<Vec<_>>(),
self.routes
.iter()
.map(|x| crate::cloud_provider::aws::router::Route {
path: x.path.clone(),
application_name: x.application_name.clone(),
})
.collect::<Vec<_>>(),
));
Some(router)
}
CPKind::GCP => None,
_ => None,
//TODO to implement
}
}
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
pub struct CustomDomain {
pub domain: String,
pub target_domain: String,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
pub struct Route {
pub path: String,
pub application_name: String,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
pub struct Database {
pub kind: DatabaseKind,
pub action: Action,
pub id: String,
pub name: String,
pub version: String,
pub fqdn_id: String,
pub fqdn: String,
pub port: u16,
pub username: String,
pub password: String,
pub total_cpus: String,
pub total_ram_in_mib: u32,
pub disk_size_in_gib: u32,
pub database_instance_type: String,
pub database_disk_type: String,
}
impl Database {
pub fn to_stateful_service(
&self,
context: &Context,
cloud_provider: &dyn CloudProvider,
) -> Option<Box<dyn StatefulService>> {
let database_options = DatabaseOptions {
login: self.username.clone(),
password: self.password.clone(),
host: self.fqdn.clone(),
port: self.port,
disk_size_in_gib: self.disk_size_in_gib,
database_disk_type: self.database_disk_type.clone(),
};
match cloud_provider.kind() {
CPKind::AWS => match self.kind {
DatabaseKind::Postgresql => {
let db: Box<dyn StatefulService> = Box::new(PostgreSQL::new(
context.clone(),
self.id.as_str(),
self.action.to_service_action(),
self.name.as_str(),
self.version.as_str(),
self.fqdn.as_str(),
self.fqdn_id.as_str(),
self.total_cpus.clone(),
self.total_ram_in_mib,
self.database_instance_type.as_str(),
database_options,
));
Some(db)
}
DatabaseKind::Mysql => {
let db: Box<dyn StatefulService> = Box::new(MySQL::new(
context.clone(),
self.id.as_str(),
self.action.to_service_action(),
self.name.as_str(),
self.version.as_str(),
self.fqdn.as_str(),
self.fqdn_id.as_str(),
self.total_cpus.clone(),
self.total_ram_in_mib,
self.database_instance_type.as_str(),
database_options,
));
Some(db)
}
DatabaseKind::Mongodb => {
let db: Box<dyn StatefulService> = Box::new(MongoDB::new(
context.clone(),
self.id.as_str(),
self.action.to_service_action(),
self.name.as_str(),
self.version.as_str(),
self.fqdn.as_str(),
self.fqdn_id.as_str(),
self.total_cpus.clone(),
self.total_ram_in_mib,
self.database_instance_type.as_str(),
database_options,
));
Some(db)
}
_ => None,
},
CPKind::GCP => None,
_ => None,
//TODO to implement
}
}
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub enum DatabaseKind {
Postgresql,
Mysql,
Mongodb,
}
#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
pub struct ExternalService {
pub action: Action,
pub id: String,
pub name: String,
pub total_cpus: String,
pub total_ram_in_mib: u32,
pub git_url: String,
pub git_credentials: GitCredentials,
pub branch: String,
pub commit_id: String,
pub on_create_dockerfile_path: String,
pub on_pause_dockerfile_path: String,
pub on_delete_dockerfile_path: String,
pub environment_variables: Vec<EnvironmentVariable>,
}
impl ExternalService {
pub fn to_application<'a>(
&self,
context: &Context,
image: &Image,
cloud_provider: &dyn CloudProvider,
) -> Option<Box<(dyn crate::cloud_provider::service::Application)>> {
match cloud_provider.kind() {
CPKind::AWS => Some(Box::new(
crate::cloud_provider::aws::external_service::ExternalService::new(
context.clone(),
self.id.as_str(),
self.action.to_service_action(),
self.name.as_str(),
self.total_cpus.clone(),
self.total_ram_in_mib,
image.clone(),
self.environment_variables
.iter()
.map(|ev| ev.to_aws_external_service_environment_variable())
.collect::<Vec<_>>(),
),
)),
CPKind::GCP => None,
_ => None,
//TODO to implement
}
}
pub fn to_stateless_service<'a>(
&self,
context: &Context,
image: Image,
cloud_provider: &dyn CloudProvider,
) -> Option<Box<(dyn crate::cloud_provider::service::StatelessService)>> {
match cloud_provider.kind() {
CPKind::AWS => Some(Box::new(
crate::cloud_provider::aws::external_service::ExternalService::new(
context.clone(),
self.id.as_str(),
self.action.to_service_action(),
self.name.as_str(),
self.total_cpus.clone(),
self.total_ram_in_mib,
image,
self.environment_variables
.iter()
.map(|ev| ev.to_aws_external_service_environment_variable())
.collect::<Vec<_>>(),
),
)),
CPKind::GCP => None,
_ => None,
//TODO to implement
}
}
pub fn to_image(&self) -> Image {
Image {
application_id: self.id.clone(),
name: self.name.clone(),
tag: self.commit_id.clone(),
commit_id: self.commit_id.clone(),
registry_url: None,
}
}
pub fn to_build(&self) -> Build {
Build {
git_repository: GitRepository {
url: self.git_url.clone(),
credentials: Some(Credentials {
login: self.git_credentials.login.clone(),
password: self.git_credentials.access_token.clone(),
}),
commit_id: self.commit_id.clone(),
dockerfile_path: match self.action {
Action::Create => self.on_create_dockerfile_path.clone(),
Action::Pause => self.on_pause_dockerfile_path.clone(),
Action::Delete => self.on_delete_dockerfile_path.clone(),
Action::Nothing => self.on_create_dockerfile_path.clone(),
},
},
image: self.to_image(),
options: BuildOptions {
environment_variables: self
.environment_variables
.iter()
.map(|ev| crate::build_platform::EnvironmentVariable {
key: ev.key.clone(),
value: ev.value.clone(),
})
.collect::<Vec<_>>(),
},
}
}
}
#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub enum EnvironmentError {}
#[derive(Clone)]
pub struct ProgressInfo {
pub created_at: DateTime<Utc>,
pub scope: ProgressScope,
pub level: ProgressLevel,
pub message: Option<String>,
pub execution_id: String,
}
impl ProgressInfo {
pub fn new<T: Into<String>, X: Into<String>>(
scope: ProgressScope,
level: ProgressLevel,
message: Option<T>,
execution_id: X,
) -> Self {
ProgressInfo {
created_at: Utc::now(),
scope,
level,
message: match message {
Some(msg) => Some(msg.into()),
_ => None,
},
execution_id: execution_id.into(),
}
}
}
#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "SCREAMING_SNAKE_CASE")]
pub enum ProgressScope {
Queued,
Infrastructure { execution_id: String },
Database { id: String },
Application { id: String },
ExternalService { id: String },
Router { id: String },
Environment { id: String },
}
#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub enum ProgressLevel {
Debug,
Info,
Warn,
Error,
}
pub trait ProgressListener {
fn start_in_progress(&self, info: ProgressInfo);
fn pause_in_progress(&self, info: ProgressInfo);
fn delete_in_progress(&self, info: ProgressInfo);
fn error(&self, info: ProgressInfo);
fn started(&self, info: ProgressInfo);
fn paused(&self, info: ProgressInfo);
fn deleted(&self, info: ProgressInfo);
fn start_error(&self, info: ProgressInfo);
fn pause_error(&self, info: ProgressInfo);
fn delete_error(&self, info: ProgressInfo);
}
pub type Listener = Rc<Box<dyn ProgressListener>>;
pub type Listeners = Vec<Listener>;
pub struct ListenersHelper<'a> {
listeners: &'a Listeners,
}
impl<'a> ListenersHelper<'a> {
pub fn new(listeners: &'a Listeners) -> Self {
ListenersHelper { listeners }
}
pub fn start_in_progress(&self, info: ProgressInfo) {
self.listeners
.iter()
.for_each(|l| l.start_in_progress(info.clone()));
}
pub fn pause_in_progress(&self, info: ProgressInfo) {
self.listeners
.iter()
.for_each(|l| l.pause_in_progress(info.clone()));
}
pub fn delete_in_progress(&self, info: ProgressInfo) {
self.listeners
.iter()
.for_each(|l| l.delete_in_progress(info.clone()));
}
pub fn error(&self, info: ProgressInfo) {
self.listeners.iter().for_each(|l| l.error(info.clone()));
}
pub fn started(&self, info: ProgressInfo) {
self.listeners.iter().for_each(|l| l.started(info.clone()));
}
pub fn paused(&self, info: ProgressInfo) {
self.listeners.iter().for_each(|l| l.paused(info.clone()));
}
pub fn deleted(&self, info: ProgressInfo) {
self.listeners.iter().for_each(|l| l.deleted(info.clone()));
}
pub fn start_error(&self, info: ProgressInfo) {
self.listeners
.iter()
.for_each(|l| l.start_error(info.clone()));
}
pub fn pause_error(&self, info: ProgressInfo) {
self.listeners
.iter()
.for_each(|l| l.pause_error(info.clone()));
}
pub fn delete_error(&self, info: ProgressInfo) {
self.listeners
.iter()
.for_each(|l| l.delete_error(info.clone()));
}
}
#[derive(PartialEq, Eq, Hash, Clone)]
pub struct Context {
execution_id: String,
workspace_root_dir: String,
lib_root_dir: String,
docker_host: Option<String>,
metadata: Option<Metadata>,
}
// trait used to reimplement clone without same fields
// this trait is used for Context struct
pub trait Clone2 {
fn clone_not_same_execution_id(&self) -> Self;
}
// for test we need to clone context but to change the directory workspace used
// to to this we just have to suffix the execution id in tests
impl Clone2 for Context {
fn clone_not_same_execution_id(&self) -> Context {
let mut new = self.clone();
let suffix = rand::thread_rng()
.sample_iter(&Alphanumeric)
.take(10)
.collect::<String>();
new.execution_id = format!("{}-{}", self.execution_id, suffix);
new
}
}
impl Context {
pub fn new(
execution_id: &str,
workspace_root_dir: &str,
lib_root_dir: &str,
docker_host: Option<String>,
metadata: Option<Metadata>,
) -> Self {
Context {
execution_id: execution_id.to_string(),
workspace_root_dir: workspace_root_dir.to_string(),
lib_root_dir: lib_root_dir.to_string(),
docker_host,
metadata,
}
}
pub fn execution_id(&self) -> &str {
self.execution_id.as_str()
}
pub fn workspace_root_dir(&self) -> &str {
self.workspace_root_dir.as_str()
}
pub fn lib_root_dir(&self) -> &str {
self.lib_root_dir.as_str()
}
pub fn docker_tcp_socket(&self) -> Option<&String> {
self.docker_host.as_ref()
}
pub fn metadata(&self) -> Option<&Metadata> {
self.metadata.as_ref()
}
}
/// put everything you want here that is required to change the behaviour of the request.
/// E.g you can indicate that this request is a test, then you can adapt the behaviour as you want.
#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
pub struct Metadata {
pub test: Option<bool>,
}
impl Metadata {
pub fn new(test: Option<bool>) -> Self {
Metadata { test }
}
}

9
src/runtime.rs Normal file
View File

@@ -0,0 +1,9 @@
use std::future::Future;
use tokio::runtime::Runtime;
pub fn async_run<F: Future>(future: F) -> F::Output {
// TODO improve - is it efficient to create a Runtime at each exec?
let mut runtime = Runtime::new().expect("unable to create a tokio runtime");
runtime.block_on(future)
}

257
src/s3.rs Normal file
View File

@@ -0,0 +1,257 @@
use std::fmt::Display;
use std::fs::{read_to_string, File};
use std::io;
use std::io::{Error, ErrorKind, Read, Write};
use std::path::Path;
use std::str::FromStr;
use retry::delay::Fibonacci;
use retry::OperationResult;
use rusoto_core::{Client, HttpClient, Region, RusotoError};
use rusoto_credential::StaticProvider;
use rusoto_s3::{
CreateBucketConfiguration, CreateBucketError, CreateBucketRequest, GetObjectError,
GetObjectRequest, ListObjectsV2Output, ListObjectsV2Request, PutBucketVersioningRequest,
S3Client, VersioningConfiguration, S3,
};
use crate::cmd::utilities::{exec_with_envs, CmdError};
use crate::runtime::async_run;
pub const AWS_REGION_FOR_S3_US: &str = "ap-south-1";
pub fn create_bucket(
access_key_id: &str,
secret_access_key: &str,
bucket_name: &str,
) -> Result<(), CmdError> {
exec_with_envs(
"aws",
vec!["s3api", "create-bucket", "--bucket", &bucket_name],
vec![
("AWS_ACCESS_KEY_ID", &access_key_id),
("AWS_SECRET_ACCESS_KEY", &secret_access_key),
],
)
}
pub type FileContent = String;
pub fn get_object(
access_key_id: &str,
secret_access_key: &str,
region: &Region,
bucket_name: &str,
object_key: &str,
) -> Result<FileContent, Error> {
let credentials = StaticProvider::new(
access_key_id.to_string(),
secret_access_key.to_string(),
None,
None,
);
let client = Client::new_with(credentials, HttpClient::new().unwrap());
let s3_client = S3Client::new_with_client(client, region.clone());
let mut or = GetObjectRequest::default();
or.bucket = bucket_name.to_string();
or.key = object_key.to_string();
let get_object_output = s3_client.get_object(or);
let r = async_run(get_object_output);
let _err = Error::new(
ErrorKind::Other,
format!(
"something goes wrong while getting object {} in the S3 bucket {}",
object_key, bucket_name
),
);
match r {
Ok(x) => {
let mut s = String::new();
x.body.unwrap().into_blocking_read().read_to_string(&mut s);
if s.is_empty() {
// this handle a case where the request succeeds but contains an empty body.
// https://github.com/rusoto/rusoto/issues/1822
let r_from_aws_cli = get_object_via_aws_cli(
access_key_id,
secret_access_key,
bucket_name,
object_key,
)?;
return Ok(r_from_aws_cli);
}
Ok(s)
}
Err(err) => {
return match err {
RusotoError::Service(s) => match s {
GetObjectError::NoSuchKey(x) => {
info!("no such key '{}': {}", object_key, x.as_str());
Err(Error::new(
ErrorKind::NotFound,
format!("no such key '{}': {}", object_key, x.as_str()),
))
}
},
RusotoError::Unknown(r) => {
let r_from_aws_cli = get_object_via_aws_cli(
access_key_id,
secret_access_key,
bucket_name,
object_key,
);
match r_from_aws_cli {
Ok(..) => Ok(r_from_aws_cli.unwrap()),
Err(err) => {
error!("{}", err);
Err(_err)
}
}
}
_ => Err(_err),
};
}
}
}
/// gets an aws s3 object using aws-cli
/// used as a failover when rusoto_s3 acts up
fn get_object_via_aws_cli(
access_key_id: &str,
secret_access_key: &str,
bucket_name: &str,
object_key: &str,
) -> Result<FileContent, Error> {
let s3_url = format!("s3://{}/{}", bucket_name, object_key);
let local_path = format!("/tmp/{}", object_key);
let r = exec_with_envs(
"aws",
vec!["s3", "cp", &s3_url, &local_path],
vec![
("AWS_ACCESS_KEY_ID", &access_key_id),
("AWS_SECRET_ACCESS_KEY", &secret_access_key),
],
);
match r {
Err(e) => return Err(Error::new(ErrorKind::Other, e)),
_ => {}
};
let s = read_to_string(&local_path)?;
Ok(s)
}
pub fn get_kubernetes_config_file<P>(
access_key_id: &str,
secret_access_key: &str,
region: &Region,
kubernetes_config_bucket_name: &str,
kubernetes_config_object_key: &str,
file_path: P,
) -> Result<File, Error>
where
P: AsRef<Path>,
{
// return the file if it already exists
let _ = match File::open(file_path.as_ref()) {
Ok(f) => return Ok(f),
Err(_) => {}
};
let file_content_result = retry::retry(Fibonacci::from_millis(3000).take(5), || {
let file_content = crate::s3::get_object_via_aws_cli(
access_key_id,
secret_access_key,
kubernetes_config_bucket_name,
kubernetes_config_object_key,
);
match file_content {
Ok(file_content) => OperationResult::Ok(file_content),
Err(err) => {
warn!(
"Can't download the kubernetes config file {} stored on {}, please check access key and secrets",
kubernetes_config_object_key, kubernetes_config_bucket_name
);
OperationResult::Retry(err)
}
}
});
let file_content = match file_content_result {
Ok(file_content) => file_content,
Err(_) => {
return Err(Error::new(
ErrorKind::InvalidData,
"file content is empty (retry failed multiple times) - which is not the expected content - what's wrong?",
));
}
};
let mut kubernetes_config_file = File::create(file_path.as_ref())?;
let _ = kubernetes_config_file.write(file_content.as_bytes())?;
Ok(kubernetes_config_file)
}
pub fn list_objects_in(
access_key_id: &str,
secret_access_key: &str,
bucket_name: &str,
) -> Result<ListObjectsV2Output, Error> {
let credentials = StaticProvider::new(
access_key_id.to_string(),
secret_access_key.to_string(),
None,
None,
);
let client = Client::new_with(credentials, HttpClient::new().unwrap());
let s3_client = S3Client::new_with_client(client, get_default_region_for_us());
let mut list_request = ListObjectsV2Request::default();
list_request.bucket = bucket_name.to_string();
let lis_object = s3_client.list_objects_v2(list_request);
let objects_in = async_run(lis_object);
match objects_in {
Ok(objects) => Ok(objects),
Err(err) => Err(Error::new(ErrorKind::Other, err)),
}
}
// delete bucket implement by default objects deletion
pub fn delete_bucket(
access_key_id: &str,
secret_access_key: &str,
bucket_name: &str,
) -> Result<(), CmdError> {
info!("Deleting S3 Bucket {}", bucket_name.clone());
match exec_with_envs(
"aws",
vec![
"s3",
"rb",
"--force",
"--bucket",
format!("s3://{}", bucket_name).as_str(),
],
vec![
("AWS_ACCESS_KEY_ID", &access_key_id),
("AWS_SECRET_ACCESS_KEY", &secret_access_key),
],
) {
Ok(o) => {
info!("Successfuly delete bucket");
return Ok(o);
}
Err(e) => {
error!("while deleting bucket {}", e);
return Err(e);
}
}
}
pub fn get_default_region_for_us() -> Region {
Region::from_str(AWS_REGION_FOR_S3_US).unwrap()
}

12
src/session.rs Normal file
View File

@@ -0,0 +1,12 @@
use crate::engine::Engine;
use crate::transaction::Transaction;
pub struct Session<'a> {
pub engine: &'a Engine,
}
impl<'a> Session<'a> {
pub fn transaction(self) -> Transaction<'a> {
Transaction::new(self.engine)
}
}

11
src/string.rs Normal file
View File

@@ -0,0 +1,11 @@
pub fn cut(str: String, max_length: usize) -> String {
if str.len() <= max_length {
str
} else {
str.as_str()[..max_length - 1].to_string()
}
}
pub fn terraform_list_format(tf_vec: Vec<String>) -> String {
format!("{{{}}}", tf_vec.join(","))
}

180
src/template.rs Normal file
View File

@@ -0,0 +1,180 @@
use std::ffi::OsStr;
use std::fs;
use std::fs::File;
use std::io::{Error, ErrorKind, Write};
use std::os::unix::fs::PermissionsExt;
use std::path::Path;
use tera::Error as TeraError;
use tera::{Context, Tera};
use walkdir::WalkDir;
pub fn generate_and_copy_all_files_into_dir<S, P>(
from_dir: S,
to_dir: P,
context: &Context,
) -> Result<(), Error>
where
S: AsRef<Path> + Copy,
P: AsRef<Path> + Copy,
{
// generate j2 templates
let rendered_templates = match generate_j2_template_files(from_dir, context) {
Ok(rt) => rt,
Err(e) => {
let error_msg = match e.kind {
tera::ErrorKind::TemplateNotFound(x) => format!("template not found: {}", x),
tera::ErrorKind::Msg(x) => format!("tera error: {}", x),
tera::ErrorKind::CircularExtend {
tpl,
inheritance_chain,
} => format!(
"circular extend - template: {}, inheritance chain: {:?}",
tpl, inheritance_chain
),
tera::ErrorKind::MissingParent { current, parent } => {
format!("missing parent - current: {}, parent: {}", current, parent)
}
tera::ErrorKind::FilterNotFound(x) => format!("filter not found: {}", x),
tera::ErrorKind::TestNotFound(x) => format!("test not found: {}", x),
tera::ErrorKind::InvalidMacroDefinition(x) => {
format!("invalid macro definition: {}", x)
}
tera::ErrorKind::FunctionNotFound(x) => format!("function not found: {}", x),
tera::ErrorKind::Json(x) => format!("json error: {:?}", x),
tera::ErrorKind::CallFunction(x) => format!("call function: {}", x),
tera::ErrorKind::CallFilter(x) => format!("call filter: {}", x),
tera::ErrorKind::CallTest(x) => format!("call test: {}", x),
tera::ErrorKind::__Nonexhaustive => format!("non exhaustive error"),
};
error!("{}", error_msg.as_str());
return Err(Error::new(ErrorKind::InvalidData, error_msg));
}
};
// copy all .tf and .yaml files into our dest directory
copy_non_template_files(from_dir.as_ref(), to_dir.as_ref())?;
write_rendered_templates(&rendered_templates, to_dir.as_ref())?;
Ok(())
}
pub fn copy_non_template_files<S, P>(from: S, to: P) -> Result<(), Error>
where
S: AsRef<Path>,
P: AsRef<Path>,
{
crate::fs::copy_files(from.as_ref(), to.as_ref(), true)
}
pub fn generate_j2_template_files<P>(
root_dir: P,
context: &Context,
) -> Result<Vec<RenderedTemplate>, TeraError>
where
P: AsRef<Path>,
{
//TODO: sort on fly context should be implemented to optimize reading
debug!("context: {:#?}", context);
let root_dir_str = root_dir.as_ref().to_str().unwrap();
let tera_template_string = format!("{}/**/*.j2.*", root_dir_str);
let tera = Tera::new(tera_template_string.as_str())?;
let files = WalkDir::new(root_dir_str)
.follow_links(true)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| {
e.file_name()
.to_str()
.map(|s| s.contains(".j2."))
.unwrap_or(false)
})
.collect::<Vec<_>>();
let mut results: Vec<RenderedTemplate> = vec![];
for file in files.into_iter() {
let path_str = file.path().to_str().unwrap();
let j2_path = path_str.replace(root_dir_str, "");
let j2_file_name = file.file_name().to_str().unwrap();
let j2_path_split = j2_path.split("/").collect::<Vec<_>>();
let j2_root_path: String = j2_path_split.as_slice()[..j2_path_split.len() - 1].join("/");
let file_name = j2_file_name.replace(".j2", "");
let content = tera.render(&j2_path[1..], &context)?;
results.push(RenderedTemplate::new(j2_root_path, file_name, content));
}
Ok(results)
}
pub fn write_rendered_templates(
rendered_templates: &[RenderedTemplate],
into: &Path,
) -> Result<(), Error> {
for rt in rendered_templates {
let dest = format!("{}/{}", into.to_str().unwrap(), rt.path_and_file_name());
if dest.contains("/") {
// create the parent directories
let s_dest = dest.split("/").collect::<Vec<_>>();
let dir: String = s_dest.as_slice()[..s_dest.len() - 1].join("/");
let _ = fs::create_dir_all(dir);
}
// remove file if it already exists
let _ = fs::remove_file(dest.as_str());
// create an empty file
let mut f = fs::File::create(&dest)?;
// write rendered template into the new file
f.write_all(rt.content.as_bytes())?;
// perform spcific action based on the extension
let extension = Path::new(&dest).extension().and_then(OsStr::to_str);
match extension {
Some("sh") => set_file_permission(&f, 0o755),
_ => {}
}
}
Ok(())
}
pub fn set_file_permission(f: &File, mode: u32) {
let metadata = f.metadata().unwrap();
let mut permissions = metadata.permissions();
permissions.set_mode(mode);
f.set_permissions(permissions).unwrap();
}
pub struct RenderedTemplate {
pub path: String,
pub file_name: String,
pub content: String,
}
impl RenderedTemplate {
pub fn new(path: String, file_name: String, content: String) -> Self {
RenderedTemplate {
path,
file_name,
content,
}
}
pub fn path_and_file_name(&self) -> String {
if self.path.trim().is_empty() || self.path.as_str() == "." {
self.file_name.clone()
} else {
format!("{}/{}", self.path.as_str(), self.file_name.as_str())
}
}
}

791
src/transaction.rs Normal file
View File

@@ -0,0 +1,791 @@
use std::collections::HashMap;
use std::thread;
use serde::{Deserialize, Serialize};
use crate::build_platform::BuildError;
use crate::cloud_provider::kubernetes::{Kubernetes, KubernetesError};
use crate::cloud_provider::service::ServiceError;
use crate::cloud_provider::service::{Application, Service};
use crate::cloud_provider::DeployError;
use crate::container_registry::{PushError, PushResult};
use crate::engine::Engine;
use crate::models::{
Action, Environment, EnvironmentAction, EnvironmentError, ListenersHelper, ProgressInfo,
ProgressLevel, ProgressScope,
};
pub struct Transaction<'a> {
engine: &'a Engine,
steps: Vec<Step<'a>>,
executed_steps: Vec<Step<'a>>,
}
impl<'a> Transaction<'a> {
pub fn new(engine: &'a Engine) -> Self {
Transaction::<'a> {
engine,
steps: vec![],
executed_steps: vec![],
}
}
pub fn create_kubernetes(
&mut self,
kubernetes: &'a dyn Kubernetes,
) -> Result<(), KubernetesError> {
match kubernetes.is_valid() {
Ok(_) => {
self.steps.push(Step::CreateKubernetes(kubernetes));
Ok(())
}
Err(err) => Err(err),
}
}
pub fn delete_kubernetes(
&mut self,
kubernetes: &'a dyn Kubernetes,
) -> Result<(), KubernetesError> {
match kubernetes.is_valid() {
Ok(_) => {
self.steps.push(Step::DeleteKubernetes(kubernetes));
Ok(())
}
Err(err) => Err(err),
}
}
pub fn deploy_environment(
&mut self,
kubernetes: &'a dyn Kubernetes,
environment_action: &'a EnvironmentAction,
) -> Result<(), EnvironmentError> {
self.deploy_environment_with_options(
kubernetes,
environment_action,
DeploymentOption {
force_build: false,
force_push: false,
},
)
}
pub fn deploy_environment_with_options(
&mut self,
kubernetes: &'a dyn Kubernetes,
environment_action: &'a EnvironmentAction,
option: DeploymentOption,
) -> Result<(), EnvironmentError> {
let _ = self.check_environment_action(environment_action)?;
// add build step
self.steps
.push(Step::BuildEnvironment(environment_action, option));
// add deployment step
self.steps
.push(Step::DeployEnvironment(kubernetes, environment_action));
Ok(())
}
pub fn pause_environment(
&mut self,
kubernetes: &'a dyn Kubernetes,
environment_action: &'a EnvironmentAction,
) -> Result<(), EnvironmentError> {
let _ = self.check_environment_action(environment_action)?;
self.steps
.push(Step::PauseEnvironment(kubernetes, environment_action));
Ok(())
}
pub fn delete_environment(
&mut self,
kubernetes: &'a dyn Kubernetes,
environment_action: &'a EnvironmentAction,
) -> Result<(), EnvironmentError> {
let _ = self.check_environment_action(environment_action)?;
self.steps
.push(Step::DeleteEnvironment(kubernetes, environment_action));
Ok(())
}
fn check_environment_action(
&self,
environment_action: &EnvironmentAction,
) -> Result<(), EnvironmentError> {
match environment_action {
EnvironmentAction::Environment(te) => match te.is_valid() {
Ok(_) => {}
Err(err) => return Err(err),
},
EnvironmentAction::EnvironmentWithFailover(te, fe) => {
match te.is_valid() {
Ok(_) => {}
Err(err) => return Err(err),
};
match fe.is_valid() {
Ok(_) => {}
Err(err) => return Err(err),
};
}
};
Ok(())
}
fn _build_applications(
&self,
environment: &Environment,
option: &DeploymentOption,
) -> Result<Vec<Box<dyn Application>>, BuildError> {
let external_services_to_build = environment
.external_services
.iter()
// build only applications that are set with Action: Create
.filter(|es| es.action == Action::Create);
let external_service_and_result_tuples = external_services_to_build
.map(|es| {
(
es,
self.engine
.build_platform()
.build(es.to_build(), option.force_build),
)
})
.collect::<Vec<_>>();
// do the same for applications
let apps_to_build = environment
.applications
.iter()
// build only applications that are set with Action: Create
.filter(|app| app.action == Action::Create);
let application_and_result_tuples = apps_to_build
.map(|app| {
(
app,
self.engine
.build_platform()
.build(app.to_build(), option.force_build),
)
})
.collect::<Vec<_>>();
let mut applications: Vec<Box<dyn Application>> =
Vec::with_capacity(application_and_result_tuples.len());
for (external_service, result) in external_service_and_result_tuples {
// catch build error, can't do it in Fn
let build_result = match result {
Err(err) => {
error!(
"build error for external_service {}: {:?}",
external_service.id.as_str(),
err
);
return Err(err);
}
Ok(build_result) => build_result,
};
match external_service.to_application(
self.engine.context(),
&build_result.build.image,
self.engine.cloud_provider(),
) {
Some(x) => applications.push(x),
None => {}
}
}
for (application, result) in application_and_result_tuples {
// catch build error, can't do it in Fn
let build_result = match result {
Err(err) => {
error!(
"build error for application {}: {:?}",
application.id.as_str(),
err
);
return Err(err);
}
Ok(build_result) => build_result,
};
match application.to_application(
self.engine.context(),
&build_result.build.image,
self.engine.cloud_provider(),
) {
Some(x) => applications.push(x),
None => {}
}
}
Ok(applications)
}
fn _push_applications(
&self,
applications: Vec<Box<dyn Application>>,
option: &DeploymentOption,
) -> Result<Vec<(Box<dyn Application>, PushResult)>, PushError> {
let application_and_push_results: Vec<_> = applications
.into_iter()
.map(|mut app| {
match self
.engine
.container_registry()
.push(app.image(), option.force_push)
{
Ok(push_result) => {
// I am not a big fan of doing that but it's the most effective way
app.set_image(push_result.image.clone());
Ok((app, push_result))
}
Err(err) => Err(err),
}
})
.collect();
let mut results: Vec<(Box<dyn Application>, PushResult)> = vec![];
for result in application_and_push_results.into_iter() {
match result {
Ok(tuple) => results.push(tuple),
Err(err) => {
error!("error pushing docker image {:?}", err);
return Err(err);
}
}
}
Ok(results)
}
fn check_environment(
&self,
environment: &crate::cloud_provider::environment::Environment,
) -> TransactionResult {
match environment.is_valid() {
Err(service_error) => {
warn!("ROLLBACK STARTED! an error occurred {:?}", service_error);
return match self.rollback() {
Ok(_) => {
TransactionResult::Rollback(CommitError::NotValidService(service_error))
}
Err(err) => {
error!("ROLLBACK FAILED! fatal error: {:?}", err);
TransactionResult::UnrecoverableError(
CommitError::NotValidService(service_error),
err,
)
}
};
}
_ => {}
};
TransactionResult::Ok
}
pub fn rollback(&self) -> Result<(), RollbackError> {
for step in self.executed_steps.iter() {
match step {
Step::CreateKubernetes(kubernetes) => {
// revert kubernetes creation
match kubernetes.on_create_error() {
Err(err) => return Err(RollbackError::CreateKubernetes(err)),
_ => {}
};
}
Step::DeleteKubernetes(kubernetes) => {
// revert kubernetes deletion
match kubernetes.on_delete_error() {
Err(err) => return Err(RollbackError::DeleteKubernetes(err)),
_ => {}
};
}
Step::BuildEnvironment(_environment_action, _option) => {
// revert build applications
}
Step::DeployEnvironment(kubernetes, environment_action) => {
// revert environment deployment
self.rollback_environment(*kubernetes, *environment_action)?;
}
Step::PauseEnvironment(kubernetes, environment_action) => {
self.rollback_environment(*kubernetes, *environment_action)?;
}
Step::DeleteEnvironment(kubernetes, environment_action) => {
self.rollback_environment(*kubernetes, *environment_action)?;
}
}
}
Ok(())
}
/// This function is a wrapper to correctly revert all changes of an attempted deployment AND
/// if a failover environment is provided, then rollback.
fn rollback_environment(
&self,
kubernetes: &dyn Kubernetes,
environment_action: &EnvironmentAction,
) -> Result<(), RollbackError> {
let qe_environment = |environment: &Environment| {
let mut _applications = Vec::with_capacity(
// ExternalService impl Application (which is a StatelessService)
environment.applications.len() + environment.external_services.len(),
);
for application in environment.applications.iter() {
let build = application.to_build();
match application.to_application(
self.engine.context(),
&build.image,
self.engine.cloud_provider(),
) {
Some(x) => _applications.push(x),
None => {}
}
}
for external_service in environment.external_services.iter() {
let build = external_service.to_build();
match external_service.to_application(
self.engine.context(),
&build.image,
self.engine.cloud_provider(),
) {
Some(x) => _applications.push(x),
None => {}
}
}
let qe_environment = environment.to_qe_environment(
self.engine.context(),
&_applications,
self.engine.cloud_provider(),
);
qe_environment
};
match environment_action {
EnvironmentAction::EnvironmentWithFailover(
target_environment,
failover_environment,
) => {
// let's reverse changes and rollback on the provided failover version
let target_qe_environment = qe_environment(&target_environment);
let failover_qe_environment = qe_environment(&failover_environment);
let action = match failover_environment.action {
Action::Create => {
kubernetes.deploy_environment_error(&target_qe_environment);
kubernetes.deploy_environment(&failover_qe_environment)
}
Action::Pause => {
kubernetes.pause_environment_error(&target_qe_environment);
kubernetes.pause_environment(&failover_qe_environment)
}
Action::Delete => {
kubernetes.delete_environment_error(&target_qe_environment);
kubernetes.delete_environment(&failover_qe_environment)
}
Action::Nothing => Ok(()),
};
let _ = match action {
Ok(_) => {}
Err(err) => {
return Err(match failover_environment.action {
Action::Create => RollbackError::DeployEnvironment(err),
Action::Pause => RollbackError::PauseEnvironment(err),
Action::Delete => RollbackError::DeleteEnvironment(err),
Action::Nothing => RollbackError::Error, // it can't happens
});
}
};
Ok(())
}
EnvironmentAction::Environment(te) => {
// revert changes but there is no failover environment
let target_qe_environment = qe_environment(&te);
let action = match te.action {
Action::Create => kubernetes.deploy_environment_error(&target_qe_environment),
Action::Pause => kubernetes.pause_environment_error(&target_qe_environment),
Action::Delete => kubernetes.delete_environment_error(&target_qe_environment),
Action::Nothing => Ok(()),
};
let _ = match action {
Ok(_) => {}
Err(err) => {
return Err(match te.action {
Action::Create => RollbackError::DeployEnvironment(err),
Action::Pause => RollbackError::PauseEnvironment(err),
Action::Delete => RollbackError::DeleteEnvironment(err),
Action::Nothing => RollbackError::Error, // it can't happens
});
}
};
Err(RollbackError::NoFailoverEnvironment)
}
}
}
pub fn commit(&mut self) -> TransactionResult {
let mut applications_by_environment: HashMap<&Environment, Vec<Box<dyn Application>>> =
HashMap::new();
for step in self.steps.iter() {
// execution loop
self.executed_steps.push(step.clone());
match step {
Step::CreateKubernetes(kubernetes) => {
// create kubernetes
match kubernetes.on_create() {
Err(err) => {
warn!("ROLLBACK STARTED! an error occurred {:?}", err);
match self.rollback() {
Ok(_) => {
TransactionResult::Rollback(CommitError::CreateKubernetes(err))
}
Err(e) => {
error!("ROLLBACK FAILED! fatal error: {:?}", e);
TransactionResult::UnrecoverableError(
CommitError::CreateKubernetes(err),
e,
)
}
}
}
_ => TransactionResult::Ok,
};
}
Step::DeleteKubernetes(kubernetes) => {
// delete kubernetes
match kubernetes.on_delete() {
Err(err) => {
warn!("ROLLBACK STARTED! an error occurred {:?}", err);
match self.rollback() {
Ok(_) => {
TransactionResult::Rollback(CommitError::DeleteKubernetes(err))
}
Err(e) => {
error!("ROLLBACK FAILED! fatal error: {:?}", e);
TransactionResult::UnrecoverableError(
CommitError::DeleteKubernetes(err),
e,
)
}
}
}
_ => TransactionResult::Ok,
};
}
Step::BuildEnvironment(environment_action, option) => {
// build applications
let target_environment = match environment_action {
EnvironmentAction::Environment(te) => te,
EnvironmentAction::EnvironmentWithFailover(te, _) => te,
};
let apps_result = match self._build_applications(target_environment, option) {
Ok(applications) => match self._push_applications(applications, option) {
Ok(results) => {
let applications =
results.into_iter().map(|(app, _)| app).collect::<Vec<_>>();
Ok(applications)
}
Err(err) => Err(CommitError::PushImage(err)),
},
Err(err) => Err(CommitError::BuildImage(err)),
};
if apps_result.is_err() {
let commit_error = apps_result.err().unwrap();
warn!("ROLLBACK STARTED! an error occurred {:?}", commit_error);
return match self.rollback() {
Ok(_) => TransactionResult::Rollback(commit_error),
Err(err) => {
error!("ROLLBACK FAILED! fatal error: {:?}", err);
TransactionResult::UnrecoverableError(commit_error, err)
}
};
}
let applications = apps_result.ok().unwrap();
applications_by_environment.insert(target_environment, applications);
}
Step::DeployEnvironment(kubernetes, environment_action) => {
// deploy complete environment
match self.commit_environment(
*kubernetes,
*environment_action,
&applications_by_environment,
|qe_env| kubernetes.deploy_environment(qe_env),
|err| CommitError::DeployEnvironment(err),
) {
TransactionResult::Ok => {}
err => return err,
};
}
Step::PauseEnvironment(kubernetes, environment_action) => {
// pause complete environment
match self.commit_environment(
*kubernetes,
*environment_action,
&applications_by_environment,
|qe_env| kubernetes.pause_environment(qe_env),
|err| CommitError::PauseEnvironment(err),
) {
TransactionResult::Ok => {}
err => return err,
};
}
Step::DeleteEnvironment(kubernetes, environment_action) => {
// delete complete environment
match self.commit_environment(
*kubernetes,
*environment_action,
&applications_by_environment,
|qe_env| kubernetes.delete_environment(qe_env),
|err| CommitError::DeleteEnvironment(err),
) {
TransactionResult::Ok => {}
err => return err,
};
}
};
}
TransactionResult::Ok
}
fn commit_environment<F, E>(
&self,
kubernetes: &dyn Kubernetes,
environment_action: &EnvironmentAction,
applications_by_environment: &HashMap<&Environment, Vec<Box<dyn Application>>>,
action_fn: F,
commit_error: E,
) -> TransactionResult
where
F: Fn(&crate::cloud_provider::environment::Environment) -> Result<(), KubernetesError>,
E: Fn(KubernetesError) -> CommitError,
{
let target_environment = match environment_action {
EnvironmentAction::Environment(te) => te,
EnvironmentAction::EnvironmentWithFailover(te, _) => te,
};
let empty_vec = Vec::with_capacity(0);
let built_applications = match applications_by_environment.get(target_environment) {
Some(applications) => applications,
None => &empty_vec,
};
let qe_environment = target_environment.to_qe_environment(
self.engine.context(),
built_applications,
kubernetes.cloud_provider(),
);
let _ = match self.check_environment(&qe_environment) {
TransactionResult::Ok => {}
err => return err, // which it means that an error occurred
};
let execution_id = self.engine.context().execution_id();
// inner function - I use it instead of closure because of ?Sized
fn get_final_progress_info<T>(service: &Box<T>, execution_id: &str) -> ProgressInfo
where
T: Service + ?Sized,
{
ProgressInfo::new(
service.progress_scope(),
ProgressLevel::Info,
None::<&str>,
execution_id,
)
};
// send the back the right progress status
fn send_progress<T>(
kubernetes: &dyn Kubernetes,
action: &Action,
service: &Box<T>,
execution_id: &str,
is_error: bool,
) where
T: Service + ?Sized,
{
let lh = ListenersHelper::new(kubernetes.listeners());
let progress_info = get_final_progress_info(service, execution_id);
if !is_error {
match action {
Action::Create => lh.started(progress_info),
Action::Pause => lh.paused(progress_info),
Action::Delete => lh.deleted(progress_info),
Action::Nothing => {} // nothing to do here?
};
return;
}
match action {
Action::Create => lh.start_error(progress_info),
Action::Pause => lh.pause_error(progress_info),
Action::Delete => lh.delete_error(progress_info),
Action::Nothing => {} // nothing to do here?
};
}
// 100 ms sleep to avoid race condition on last service status update
// Otherwise, the last status sent to the CORE is (sometimes) not the right one.
// Even by storing data at the micro seconds precision
thread::sleep(std::time::Duration::from_millis(100));
let _ = match action_fn(&qe_environment) {
Err(err) => {
let rollback_result = match self.rollback() {
Ok(_) => TransactionResult::Rollback(commit_error(err)),
Err(rollback_err) => {
error!("ROLLBACK FAILED! fatal error: {:?}", rollback_err);
TransactionResult::UnrecoverableError(commit_error(err), rollback_err)
}
};
// !!! don't change the order
// terminal update
for service in &qe_environment.stateful_services {
send_progress(
kubernetes,
&target_environment.action,
service,
execution_id,
true,
);
}
for service in &qe_environment.stateless_services {
send_progress(
kubernetes,
&target_environment.action,
service,
execution_id,
true,
);
}
return rollback_result;
}
_ => {
// terminal update
for service in &qe_environment.stateful_services {
send_progress(
kubernetes,
&target_environment.action,
service,
execution_id,
false,
);
}
for service in &qe_environment.stateless_services {
send_progress(
kubernetes,
&target_environment.action,
service,
execution_id,
false,
);
}
}
};
TransactionResult::Ok
}
}
#[derive(Clone)]
pub struct DeploymentOption {
pub force_build: bool,
pub force_push: bool,
}
enum Step<'a> {
// init and create all the necessary resources (Network, Kubernetes)
CreateKubernetes(&'a dyn Kubernetes),
DeleteKubernetes(&'a dyn Kubernetes),
BuildEnvironment(&'a EnvironmentAction, DeploymentOption),
DeployEnvironment(&'a dyn Kubernetes, &'a EnvironmentAction),
PauseEnvironment(&'a dyn Kubernetes, &'a EnvironmentAction),
DeleteEnvironment(&'a dyn Kubernetes, &'a EnvironmentAction),
}
impl<'a> Clone for Step<'a> {
fn clone(&self) -> Self {
match self {
Step::CreateKubernetes(k) => Step::CreateKubernetes(*k),
Step::DeleteKubernetes(k) => Step::DeleteKubernetes(*k),
Step::BuildEnvironment(e, option) => Step::BuildEnvironment(*e, option.clone()),
Step::DeployEnvironment(k, e) => Step::DeployEnvironment(*k, *e),
Step::PauseEnvironment(k, e) => Step::PauseEnvironment(*k, *e),
Step::DeleteEnvironment(k, e) => Step::DeleteEnvironment(*k, *e),
}
}
}
#[derive(Debug)]
pub enum CommitError {
CreateKubernetes(KubernetesError),
DeleteKubernetes(KubernetesError),
DeployEnvironment(KubernetesError),
PauseEnvironment(KubernetesError),
DeleteEnvironment(KubernetesError),
NotValidService(ServiceError),
BuildImage(BuildError),
PushImage(PushError),
DeployImage(DeployError),
}
#[derive(Debug)]
pub enum RollbackError {
CreateKubernetes(KubernetesError),
DeleteKubernetes(KubernetesError),
DeployEnvironment(KubernetesError),
PauseEnvironment(KubernetesError),
DeleteEnvironment(KubernetesError),
NotValidService(ServiceError),
BuildImage(BuildError),
PushImage(PushError),
DeployImage(DeployError),
NoFailoverEnvironment,
Error,
}
pub enum TransactionResult {
Ok,
Rollback(CommitError),
UnrecoverableError(CommitError, RollbackError),
}

62
src/unit_conversion.rs Normal file
View File

@@ -0,0 +1,62 @@
use std::num::{ParseFloatError, ParseIntError};
/// convert a cpu string (kubernetes like) into a float. It supports millis cpu
/// examples:
/// 250m = 0.25 cpu
/// 500m = 0.50 cpu
/// 1000m = 1 cpu
/// 1.25 = 1.25
pub fn cpu_string_to_float<T: Into<String>>(cpu: T) -> f32 {
let cpu = cpu.into();
if cpu.is_empty() {
return 0.0;
}
if !cpu.ends_with("m") {
// the value is not in millis
return match cpu.parse::<f32>() {
Ok(v) if v >= 0.0 => v,
_ => 0.0,
};
}
// the result is in millis, so convert it to float
let cpu = cpu.replace("m", "");
match cpu.parse::<f32>() {
Ok(v) if v >= 0.0 => v / 1000.0,
_ => 0.0,
}
}
/// convert ki to mi
pub fn ki_to_mi<T: Into<String>>(ram: T) -> u32 {
let ram = ram.into().to_lowercase().replace("ki", "");
match ram.parse::<u32>() {
Ok(v) => v / 1000,
_ => 0,
}
}
#[cfg(test)]
mod tests {
use crate::unit_conversion::cpu_string_to_float;
use crate::unit_conversion::ki_to_mi;
#[test]
fn test_cpu_conversions() {
assert_eq!(cpu_string_to_float("250m"), 0.25);
assert_eq!(cpu_string_to_float("500m"), 0.5);
assert_eq!(cpu_string_to_float("1500m"), 1.5);
assert_eq!(cpu_string_to_float("1.5"), 1.5);
assert_eq!(cpu_string_to_float("0"), 0.0);
assert_eq!(cpu_string_to_float("0m"), 0.0);
assert_eq!(cpu_string_to_float("-250m"), 0.0);
assert_eq!(cpu_string_to_float("-10"), 0.0);
assert_eq!(cpu_string_to_float("1000"), 1000.0);
}
#[test]
fn test_kib_to_mib_conversions() {
assert_eq!(ki_to_mi("15564756Ki"), 15_564);
}
}

22
test_utilities/Cargo.toml Normal file
View File

@@ -0,0 +1,22 @@
[package]
name = "test-utilities"
version = "0.1.0"
authors = ["Romaric Philogene <evoxmusic@gmail.com>"]
edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
qovery-engine = { path = "../" }
chrono = "0.4.11"
dirs = "3.0.1"
env_logger = "0.7.1"
rand = "0.7.3"
serde = "1.0"
serde_json = "1.0.57"
serde_derive = "1.0"
curl = "0.4.34"
# Digital Ocean Deps
digitalocean = "0.1.1"

688
test_utilities/src/aws.rs Normal file
View File

@@ -0,0 +1,688 @@
extern crate serde;
extern crate serde_derive;
use std::borrow::Borrow;
use std::fs::File;
use std::io::Read;
use std::str::FromStr;
use chrono::Utc;
use dirs::home_dir;
use serde_json::map::Values;
use serde_json::value::Value;
use qovery_engine::build_platform::local_docker::LocalDocker;
use qovery_engine::build_platform::BuildPlatform;
use qovery_engine::cloud_provider::aws::kubernetes::node::Node;
use qovery_engine::cloud_provider::aws::kubernetes::EKS;
use qovery_engine::cloud_provider::aws::AWS;
use qovery_engine::cloud_provider::{CloudProvider, TerraformStateCredentials};
use qovery_engine::container_registry::docker_hub::DockerHub;
use qovery_engine::container_registry::ecr::ECR;
use qovery_engine::container_registry::ContainerRegistry;
use qovery_engine::dns_provider::cloudflare::Cloudflare;
use qovery_engine::dns_provider::DnsProvider;
use qovery_engine::engine::Engine;
use qovery_engine::models::{
Action, Application, Context, CustomDomain, Database, DatabaseKind, Environment,
EnvironmentVariable, GitCredentials, Kind, Metadata, Route, Router, Storage, StorageType,
};
use qovery_engine::session::Session;
use crate::cloudflare::dns_provider_cloudflare;
use crate::utilities::init;
use crate::utilities::{build_platform_local_docker, generate_id};
pub const AWS_ACCESS_KEY_ID: &str = "CHANGE ME";
pub const AWS_SECRET_ACCESS_KEY: &str = "CHANGE ME";
pub const AWS_DEFAULT_REGION: &str = "us-east-2";
pub const TERRAFORM_AWS_ACCESS_KEY_ID: &str = "CHANGE ME";
pub const TERRAFORM_AWS_SECRET_ACCESS_KEY: &str = "CHANGE ME";
pub const ORGANIZATION_ID: &str = "u8nb94c7fwxzr2jt";
pub const AWS_REGION_FOR_S3: &str = "us-east-1";
pub const AWS_KUBERNETES_VERSION: &str = "1.16";
pub fn execution_id() -> String {
Utc::now()
.to_rfc3339()
.replace(":", "-")
.replace(".", "-")
.replace("+", "-")
}
pub fn context() -> Context {
let execution_id = execution_id();
let home_dir = std::env::var("WORKSPACE_ROOT_DIR")
.unwrap_or(home_dir().unwrap().to_str().unwrap().to_string());
let lib_root_dir = std::env::var("LIB_ROOT_DIR").expect("LIB_ROOT_DIR is mandatory");
let metadata = Metadata {
test: Option::from(true),
};
Context::new(
execution_id.as_str(),
home_dir.as_str(),
lib_root_dir.as_str(),
None,
Option::from(metadata),
)
}
pub fn container_registry_ecr(context: &Context) -> ECR {
ECR::new(
context.clone(),
"default-ecr-registry-Qovery Test",
"ea59qe62xaw3wjai",
AWS_ACCESS_KEY_ID,
AWS_SECRET_ACCESS_KEY,
AWS_DEFAULT_REGION,
)
}
pub fn container_registry_docker_hub(context: &Context) -> DockerHub {
DockerHub::new(
context.clone(),
"my-docker-hub-id-123",
"my-default-docker-hub",
"qoveryrd",
"3b9481fe-74e7-4d7b-bc08-e147c9fd4f24",
)
}
pub fn aws_kubernetes_nodes() -> Vec<Node> {
vec![
Node::new(2, 16),
Node::new(2, 16),
Node::new(2, 16),
Node::new(2, 16),
]
}
pub fn cloud_provider_aws(context: &Context) -> AWS {
AWS::new(
context.clone(),
"u8nb94c7fwxzr2jt",
ORGANIZATION_ID,
"QoveryTest",
AWS_ACCESS_KEY_ID,
AWS_SECRET_ACCESS_KEY,
TerraformStateCredentials {
access_key_id: TERRAFORM_AWS_ACCESS_KEY_ID.to_string(),
secret_access_key: TERRAFORM_AWS_SECRET_ACCESS_KEY.to_string(),
region: "eu-west-3".to_string(),
},
)
}
pub fn aws_kubernetes_eks<'a>(
context: &Context,
cloud_provider: &'a AWS,
dns_provider: &'a dyn DnsProvider,
nodes: Vec<Node>,
) -> EKS<'a> {
let mut file = File::open("tests/assets/eks-options.json").expect("file not found");
let options_values = serde_json::from_reader(file).expect("JSON was not well-formatted");
EKS::<'a>::new(
context.clone(),
"dmubm9agk7sr8a8r",
"dmubm9agk7sr8a8r",
AWS_KUBERNETES_VERSION,
AWS_DEFAULT_REGION,
cloud_provider,
dns_provider,
options_values,
nodes,
)
}
pub fn docker_ecr_aws_engine(context: &Context) -> Engine {
// use ECR
let container_registry = Box::new(container_registry_ecr(context));
// use LocalDocker
let build_platform = Box::new(build_platform_local_docker(context));
// use AWS
let cloud_provider = Box::new(cloud_provider_aws(context));
let dns_provider = Box::new(dns_provider_cloudflare(context));
Engine::new(
context.clone(),
build_platform,
container_registry,
cloud_provider,
dns_provider,
)
}
pub fn environment_3_apps_3_routers_3_databases(context: &Context) -> Environment {
let app_name_1 = format!("{}-{}", "simple-app-1".to_string(), generate_id());
let app_name_2 = format!("{}-{}", "simple-app-2".to_string(), generate_id());
let app_name_3 = format!("{}-{}", "simple-app-3".to_string(), generate_id());
// mongoDB management part
let database_host_mongo = "mongodb-".to_string() + generate_id().as_str() + ".oom.sh"; // External access check
let database_port_mongo = 27017;
let database_db_name_mongo = "my-mongodb".to_string();
let database_username_mongo = "superuser".to_string();
let database_password_mongo = generate_id();
let database_uri_mongo = format!(
"mongodb://{}:{}@{}:{}/{}",
database_username_mongo,
database_password_mongo,
database_host_mongo,
database_port_mongo,
database_db_name_mongo
);
let version_mongo = "4.4";
// pSQL 1 management part
let fqdn_id = "my-postgresql-".to_string() + generate_id().as_str();
let fqdn = fqdn_id.clone() + ".oom.sh";
let database_port = 5432;
let database_username = "superuser".to_string();
let database_password = generate_id();
let database_name = "my-psql".to_string();
// pSQL 2 management part
let fqdn_id_2 = "my-postgresql-2".to_string() + generate_id().as_str();
let fqdn_2 = fqdn_id_2.clone() + ".oom.sh";
let database_username_2 = "superuser2".to_string();
let database_name_2 = "my-psql-2".to_string();
Environment {
execution_id: context.execution_id().to_string(),
id: generate_id(),
kind: Kind::Development,
owner_id: generate_id(),
project_id: generate_id(),
organization_id: ORGANIZATION_ID.to_string(),
action: Action::Create,
applications: vec![
Application {
id: generate_id(),
name: app_name_1.clone(),
git_url: "https://github.com/Qovery/engine-testing.git".to_string(),
commit_id: "5990752647af11ef21c3d46a51abbde3da1ab351".to_string(),
dockerfile_path: "Dockerfile".to_string(),
action: Action::Create,
git_credentials: GitCredentials {
login: "x-access-token".to_string(),
access_token: "CHANGE ME".to_string(),
expired_at: Utc::now(),
},
storage: vec![Storage {
id: generate_id(),
name: "photos".to_string(),
storage_type: StorageType::Ssd,
size_in_gib: 10,
mount_point: "/mnt/photos".to_string(),
snapshot_retention_in_days: 0,
}],
environment_variables: vec![
EnvironmentVariable {
key: "PG_DBNAME".to_string(),
value: database_name.clone(),
},
EnvironmentVariable {
key: "PG_HOST".to_string(),
value: fqdn.clone(),
},
EnvironmentVariable {
key: "PG_PORT".to_string(),
value: database_port.clone().to_string(),
},
EnvironmentVariable {
key: "PG_USERNAME".to_string(),
value: database_username.clone(),
},
EnvironmentVariable {
key: "PG_PASSWORD".to_string(),
value: database_password.clone(),
},
],
branch: "master".to_string(),
private_port: Some(1234),
total_cpus: "100m".to_string(),
total_ram_in_mib: 256,
total_instances: 2,
cpu_burst: "100m".to_string(),
},
Application {
id: generate_id(),
name: app_name_2.clone(),
git_url: "https://github.com/Qovery/engine-testing.git".to_string(),
commit_id: "5990752647af11ef21c3d46a51abbde3da1ab351".to_string(),
dockerfile_path: "Dockerfile".to_string(),
action: Action::Create,
git_credentials: GitCredentials {
login: "x-access-token".to_string(),
access_token: "CHANGE ME".to_string(),
expired_at: Utc::now(),
},
storage: vec![Storage {
id: generate_id(),
name: "photos".to_string(),
storage_type: StorageType::Ssd,
size_in_gib: 10,
mount_point: "/mnt/photos".to_string(),
snapshot_retention_in_days: 0,
}],
environment_variables: vec![
EnvironmentVariable {
key: "PG_DBNAME".to_string(),
value: database_name_2.clone(),
},
EnvironmentVariable {
key: "PG_HOST".to_string(),
value: fqdn_2.clone(),
},
EnvironmentVariable {
key: "PG_PORT".to_string(),
value: database_port.clone().to_string(),
},
EnvironmentVariable {
key: "PG_USERNAME".to_string(),
value: database_username_2.clone(),
},
EnvironmentVariable {
key: "PG_PASSWORD".to_string(),
value: database_password.clone(),
},
],
branch: "master".to_string(),
private_port: Some(1234),
total_cpus: "100m".to_string(),
total_ram_in_mib: 256,
total_instances: 2,
cpu_burst: "100m".to_string(),
},
Application {
id: generate_id(),
name: app_name_3.clone(),
git_url: "https://github.com/Qovery/engine-testing.git".to_string(),
commit_id: "158ea8ebc9897c50a7c56b910db33ce837ac1e61".to_string(),
dockerfile_path: format!("Dockerfile-{}", version_mongo),
action: Action::Create,
git_credentials: GitCredentials {
login: "x-access-token".to_string(),
access_token: "CHANGE ME".to_string(),
expired_at: Utc::now(),
},
storage: vec![Storage {
id: generate_id(),
name: "photos".to_string(),
storage_type: StorageType::Ssd,
size_in_gib: 10,
mount_point: "/mnt/photos".to_string(),
snapshot_retention_in_days: 0,
}],
environment_variables: vec![
EnvironmentVariable {
key: "IS_DOCUMENTDB".to_string(),
value: "false".to_string(),
},
EnvironmentVariable {
key: "QOVERY_DATABASE_TESTING_DATABASE_FQDN".to_string(),
value: database_host_mongo.clone(),
},
EnvironmentVariable {
key: "QOVERY_DATABASE_MY_DDB_CONNECTION_URI".to_string(),
value: database_uri_mongo.clone(),
},
EnvironmentVariable {
key: "QOVERY_DATABASE_TESTING_DATABASE_PORT".to_string(),
value: database_port_mongo.clone().to_string(),
},
EnvironmentVariable {
key: "MONGODB_DBNAME".to_string(),
value: database_db_name_mongo.clone(),
},
EnvironmentVariable {
key: "QOVERY_DATABASE_TESTING_DATABASE_USERNAME".to_string(),
value: database_username_mongo.clone(),
},
EnvironmentVariable {
key: "QOVERY_DATABASE_TESTING_DATABASE_PASSWORD".to_string(),
value: database_password_mongo.clone(),
},
],
branch: "master".to_string(),
private_port: Some(1234),
total_cpus: "100m".to_string(),
total_ram_in_mib: 256,
total_instances: 2,
cpu_burst: "100m".to_string(),
},
],
routers: vec![
Router {
id: generate_id(),
name: "main".to_string(),
action: Action::Create,
default_domain: generate_id() + ".oom.sh",
public_port: 443,
custom_domains: vec![],
routes: vec![Route {
path: "/app1".to_string(),
application_name: app_name_1.clone(),
}],
},
Router {
id: generate_id(),
name: "second-router".to_string(),
action: Action::Create,
default_domain: generate_id() + ".oom.sh",
public_port: 443,
custom_domains: vec![],
routes: vec![Route {
path: "/app2".to_string(),
application_name: app_name_2.clone(),
}],
},
Router {
id: generate_id(),
name: "third-router".to_string(),
action: Action::Create,
default_domain: generate_id() + ".oom.sh",
public_port: 443,
custom_domains: vec![],
routes: vec![Route {
path: "/app3".to_string(),
application_name: app_name_3.clone(),
}],
},
],
databases: vec![
Database {
kind: DatabaseKind::Postgresql,
action: Action::Create,
id: generate_id(),
name: database_name.clone(),
version: "11.8.0".to_string(),
fqdn_id: fqdn_id.clone(),
fqdn: fqdn.clone(),
port: database_port.clone(),
username: database_username.clone(),
password: database_password.clone(),
total_cpus: "100m".to_string(),
total_ram_in_mib: 512,
disk_size_in_gib: 10,
database_instance_type: "db.t2.micro".to_string(),
database_disk_type: "gp2".to_string(),
},
Database {
kind: DatabaseKind::Postgresql,
action: Action::Create,
id: generate_id(),
name: database_name_2.clone(),
version: "11.8.0".to_string(),
fqdn_id: fqdn_id_2.clone(),
fqdn: fqdn_2.clone(),
port: database_port.clone(),
username: database_username_2.clone(),
password: database_password.clone(),
total_cpus: "100m".to_string(),
total_ram_in_mib: 512,
disk_size_in_gib: 10,
database_instance_type: "db.t2.micro".to_string(),
database_disk_type: "gp2".to_string(),
},
Database {
kind: DatabaseKind::Mongodb,
action: Action::Create,
id: generate_id(),
name: database_db_name_mongo.clone(),
version: version_mongo.to_string(),
fqdn_id: "mongodb-".to_string() + generate_id().as_str(),
fqdn: database_host_mongo.clone(),
port: database_port_mongo.clone(),
username: database_username_mongo.clone(),
password: database_password_mongo.clone(),
total_cpus: "100m".to_string(),
total_ram_in_mib: 512,
disk_size_in_gib: 10,
database_instance_type: "db.t3.medium".to_string(),
database_disk_type: "gp2".to_string(),
},
],
external_services: vec![],
clone_from_environment_id: None,
}
}
pub fn working_minimal_environment(context: &Context) -> Environment {
let suffix = generate_id();
Environment {
execution_id: context.execution_id().to_string(),
id: generate_id(),
kind: Kind::Development,
owner_id: generate_id(),
project_id: generate_id(),
organization_id: ORGANIZATION_ID.to_string(),
action: Action::Create,
applications: vec![Application {
id: generate_id(),
name: format!("{}-{}", "simple-app".to_string(), &suffix),
git_url: "https://github.com/Qovery/engine-testing.git".to_string(),
commit_id: "fc575a2f3be0b9100492c8a463bf18134a8698a5".to_string(),
dockerfile_path: "Dockerfile".to_string(),
action: Action::Create,
git_credentials: GitCredentials {
login: "x-access-token".to_string(),
access_token: "CHANGE ME".to_string(),
expired_at: Utc::now(),
},
storage: vec![],
environment_variables: vec![],
branch: "basic-app-deploy".to_string(),
private_port: Some(80),
total_cpus: "100m".to_string(),
total_ram_in_mib: 256,
total_instances: 2,
cpu_burst: "100m".to_string(),
}],
routers: vec![Router {
id: generate_id(),
name: "main".to_string(),
action: Action::Create,
default_domain: generate_id() + ".oom.sh",
public_port: 443,
custom_domains: vec![],
routes: vec![Route {
path: "/".to_string(),
application_name: format!("{}-{}", "simple-app".to_string(), &suffix),
}],
}],
databases: vec![],
external_services: vec![],
clone_from_environment_id: None,
}
}
pub fn environnement_2_app_2_routers_1_psql(context: &Context) -> Environment {
let fqdn_id = "my-postgresql-".to_string() + generate_id().as_str();
let fqdn = fqdn_id.clone() + ".oom.sh";
let database_port = 5432;
let database_username = "superuser".to_string();
let database_password = generate_id();
let database_name = "my-psql".to_string();
let suffix = generate_id();
Environment {
execution_id: context.execution_id().to_string(),
id: generate_id(),
kind: Kind::Development,
owner_id: generate_id(),
project_id: generate_id(),
organization_id: ORGANIZATION_ID.to_string(),
action: Action::Create,
databases: vec![Database {
kind: DatabaseKind::Postgresql,
action: Action::Create,
id: generate_id(),
name: database_name.clone(),
version: "11.8.0".to_string(),
fqdn_id: fqdn_id.clone(),
fqdn: fqdn.clone(),
port: database_port.clone(),
username: database_username.clone(),
password: database_password.clone(),
total_cpus: "100m".to_string(),
total_ram_in_mib: 512,
disk_size_in_gib: 10,
database_instance_type: "db.t2.micro".to_string(),
database_disk_type: "gp2".to_string(),
}],
applications: vec![
Application {
id: generate_id(),
name: format!("{}-{}", "simple-app".to_string(), &suffix),
git_url: "https://github.com/Qovery/engine-testing.git".to_string(),
commit_id: "680550d1937b3f90551849c0da8f77c39916913b".to_string(),
dockerfile_path: "Dockerfile".to_string(),
action: Action::Create,
git_credentials: GitCredentials {
login: "x-access-token".to_string(),
access_token: "CHANGE ME".to_string(),
expired_at: Utc::now(),
},
storage: vec![Storage {
id: generate_id(),
name: "photos".to_string(),
storage_type: StorageType::Ssd,
size_in_gib: 10,
mount_point: "/mnt/photos".to_string(),
snapshot_retention_in_days: 0,
}],
environment_variables: vec![
EnvironmentVariable {
key: "PG_DBNAME".to_string(),
value: database_name.clone(),
},
EnvironmentVariable {
key: "PG_HOST".to_string(),
value: fqdn.clone(),
},
EnvironmentVariable {
key: "PG_PORT".to_string(),
value: database_port.clone().to_string(),
},
EnvironmentVariable {
key: "PG_USERNAME".to_string(),
value: database_username.clone(),
},
EnvironmentVariable {
key: "PG_PASSWORD".to_string(),
value: database_password.clone(),
},
],
branch: "master".to_string(),
private_port: Some(1234),
total_cpus: "100m".to_string(),
total_ram_in_mib: 256,
total_instances: 2,
cpu_burst: "100m".to_string(),
},
Application {
id: generate_id(),
name: format!("{}-{}", "simple-app-2".to_string(), &suffix),
git_url: "https://github.com/Qovery/engine-testing.git".to_string(),
commit_id: "680550d1937b3f90551849c0da8f77c39916913b".to_string(),
dockerfile_path: "Dockerfile".to_string(),
action: Action::Create,
git_credentials: GitCredentials {
login: "x-access-token".to_string(),
access_token: "CHANGE ME".to_string(),
expired_at: Utc::now(),
},
storage: vec![Storage {
id: generate_id(),
name: "photos".to_string(),
storage_type: StorageType::Ssd,
size_in_gib: 10,
mount_point: "/mnt/photos".to_string(),
snapshot_retention_in_days: 0,
}],
environment_variables: vec![
EnvironmentVariable {
key: "PG_DBNAME".to_string(),
value: database_name.clone(),
},
EnvironmentVariable {
key: "PG_HOST".to_string(),
value: fqdn.clone(),
},
EnvironmentVariable {
key: "PG_PORT".to_string(),
value: database_port.clone().to_string(),
},
EnvironmentVariable {
key: "PG_USERNAME".to_string(),
value: database_username.clone(),
},
EnvironmentVariable {
key: "PG_PASSWORD".to_string(),
value: database_password.clone(),
},
],
branch: "master".to_string(),
private_port: Some(1234),
total_cpus: "100m".to_string(),
total_ram_in_mib: 256,
total_instances: 2,
cpu_burst: "100m".to_string(),
},
],
routers: vec![
Router {
id: generate_id(),
name: "main".to_string(),
action: Action::Create,
default_domain: generate_id() + ".oom.sh",
public_port: 443,
custom_domains: vec![],
routes: vec![Route {
path: "/".to_string(),
application_name: format!("{}-{}", "simple-app".to_string(), &suffix),
}],
},
Router {
id: generate_id(),
name: "second-router".to_string(),
action: Action::Create,
default_domain: generate_id() + ".oom.sh",
public_port: 443,
custom_domains: vec![],
routes: vec![Route {
path: "/coco".to_string(),
application_name: format!("{}-{}", "simple-app-2".to_string(), &suffix),
}],
},
],
external_services: vec![],
clone_from_environment_id: None,
}
}
pub fn non_working_environment(context: &Context) -> Environment {
let mut environment = working_minimal_environment(context);
environment.applications = environment
.applications
.into_iter()
.map(|mut app| {
app.git_url = "https://github.com/Qovery/engine-testing.git".to_string();
app.branch = "bugged-image".to_string();
app.commit_id = "c2b2d7b5d96832732df25fe992721f53842b5eac".to_string();
app
})
.collect::<Vec<_>>();
environment
}

View File

@@ -0,0 +1,16 @@
use qovery_engine::dns_provider::cloudflare::Cloudflare;
use qovery_engine::models::Context;
pub const CLOUDFLARE_ID: &str = "CHANGE ME";
pub const CLOUDFLARE_TOKEN: &str = "CHANGE ME";
pub fn dns_provider_cloudflare(context: &Context) -> Cloudflare {
Cloudflare::new(
context.clone(),
"qoverytestdnsclo".to_string(),
"Qovery Test Cloudflare".to_string(),
"oom.sh".to_string(),
CLOUDFLARE_TOKEN.to_string(), // Cloudflare name: Qovery test
CLOUDFLARE_ID.to_string(),
)
}

View File

@@ -0,0 +1,42 @@
use digitalocean::DigitalOcean;
use qovery_engine::cloud_provider::digitalocean::DO;
use qovery_engine::container_registry::docr;
use qovery_engine::container_registry::docr::DOCR;
use qovery_engine::dns_provider::cloudflare::Cloudflare;
use qovery_engine::engine::Engine;
use qovery_engine::models::Context;
use crate::cloudflare::dns_provider_cloudflare;
use crate::utilities::build_platform_local_docker;
//TODO: should be environment var
pub const DIGITAL_OCEAN_TOKEN: &str = "CHANGE ME";
pub const DIGITAL_OCEAN_URL: &str = "https://api.digitalocean.com/v2/";
pub fn container_registry_digital_ocean(context: &Context) -> DOCR {
DOCR::new(context.clone(), "qovery-registry", DIGITAL_OCEAN_TOKEN)
}
pub fn docker_cr_do_engine(context: &Context) -> Engine {
// use DigitalOcean Container Registry
let container_registry = Box::new(container_registry_digital_ocean(context));
// use LocalDocker
let build_platform = Box::new(build_platform_local_docker(context));
// use Digital Ocean
let cloud_provider = Box::new(cloud_provider_digitalocean(context));
let dns_provider = Box::new(dns_provider_cloudflare(context));
Engine::new(
context.clone(),
build_platform,
container_registry,
cloud_provider,
dns_provider,
)
}
pub fn cloud_provider_digitalocean(context: &Context) -> DO {
DO::new(context.clone(), "test", DIGITAL_OCEAN_TOKEN)
}

View File

@@ -0,0 +1,4 @@
pub mod aws;
pub mod cloudflare;
pub mod digitalocean;
pub mod utilities;

View File

@@ -0,0 +1,61 @@
use curl::easy::Easy;
use curl::Error;
use rand::distributions::Alphanumeric;
use rand::{thread_rng, Rng};
use qovery_engine::build_platform::local_docker::LocalDocker;
use qovery_engine::models::{Context, Environment};
pub fn build_platform_local_docker(context: &Context) -> LocalDocker {
LocalDocker::new(context.clone(), "oxqlm3r99vwcmvuj", "qovery-local-docker")
}
pub fn init() {
env_logger::try_init();
println!(
"running from current directory: {}",
std::env::current_dir().unwrap().to_str().unwrap()
);
}
pub fn generate_id() -> String {
// Should follow DNS naming convention https://tools.ietf.org/html/rfc1035
let uuid;
loop {
let rand_string: String = thread_rng().sample_iter(Alphanumeric).take(15).collect();
if rand_string.chars().next().unwrap().is_alphabetic() {
uuid = rand_string.to_lowercase();
break;
}
}
uuid
}
pub fn check_all_connections(env: &Environment) -> Vec<bool> {
let mut checking: Vec<bool> = Vec::with_capacity(env.routers.len());
for router_to_test in &env.routers {
let path_to_test = format!(
"https://{}{}",
&router_to_test.default_domain, &router_to_test.routes[0].path
);
checking.push(curl_path(path_to_test.as_str()));
}
return checking;
}
fn curl_path(path: &str) -> bool {
let mut easy = Easy::new();
easy.url(path).unwrap();
let res = easy.perform();
match res {
Ok(out) => return true,
Err(e) => {
println!("TEST Error : while trying to call {}", e);
return false;
}
}
}

View File

@@ -0,0 +1,128 @@
{
"eks_zone_a_subnet_blocks": [
"10.0.0.0/23",
"10.0.2.0/23",
"10.0.4.0/23",
"10.0.6.0/23",
"10.0.8.0/23",
"10.0.10.0/23",
"10.0.12.0/23",
"10.0.14.0/23",
"10.0.16.0/23",
"10.0.18.0/23",
"10.0.20.0/23",
"10.0.22.0/23",
"10.0.24.0/23",
"10.0.26.0/23",
"10.0.28.0/23",
"10.0.30.0/23",
"10.0.32.0/23",
"10.0.34.0/23",
"10.0.36.0/23",
"10.0.38.0/23",
"10.0.40.0/23"
],
"eks_zone_b_subnet_blocks": [
"10.0.42.0/23",
"10.0.44.0/23",
"10.0.46.0/23",
"10.0.48.0/23",
"10.0.50.0/23",
"10.0.52.0/23",
"10.0.54.0/23",
"10.0.56.0/23",
"10.0.58.0/23",
"10.0.60.0/23",
"10.0.62.0/23",
"10.0.64.0/23",
"10.0.66.0/23",
"10.0.68.0/23",
"10.0.70.0/23",
"10.0.72.0/23",
"10.0.74.0/23",
"10.0.78.0/23",
"10.0.80.0/23",
"10.0.82.0/23",
"10.0.84.0/23"
],
"eks_zone_c_subnet_blocks": [
"10.0.86.0/23",
"10.0.88.0/23",
"10.0.90.0/23",
"10.0.92.0/23",
"10.0.94.0/23",
"10.0.96.0/23",
"10.0.98.0/23",
"10.0.100.0/23",
"10.0.102.0/23",
"10.0.104.0/23",
"10.0.106.0/23",
"10.0.108.0/23",
"10.0.110.0/23",
"10.0.112.0/23",
"10.0.114.0/23",
"10.0.116.0/23",
"10.0.118.0/23",
"10.0.120.0/23",
"10.0.122.0/23",
"10.0.124.0/23",
"10.0.126.0/23"
],
"rds_zone_a_subnet_blocks": [
"10.0.214.0/23",
"10.0.216.0/23",
"10.0.218.0/23",
"10.0.220.0/23",
"10.0.222.0/23",
"10.0.224.0/23"
],
"rds_zone_b_subnet_blocks": [
"10.0.226.0/23",
"10.0.228.0/23",
"10.0.230.0/23",
"10.0.232.0/23",
"10.0.234.0/23",
"10.0.236.0/23"
],
"rds_zone_c_subnet_blocks": [
"10.0.238.0/23",
"10.0.240.0/23",
"10.0.242.0/23",
"10.0.244.0/23",
"10.0.246.0/23",
"10.0.248.0/23"
],
"documentdb_zone_a_subnet_blocks": [
"10.0.196.0/23",
"10.0.198.0/23",
"10.0.200.0/23"
],
"documentdb_zone_b_subnet_blocks": [
"10.0.202.0/23",
"10.0.204.0/23",
"10.0.206.0/23"
],
"documentdb_zone_c_subnet_blocks": [
"10.0.208.0/23",
"10.0.210.0/23",
"10.0.212.0/23"
],
"elasticsearch_zone_a_subnet_blocks": [
"10.0.184.0/23",
"10.0.186.0/23"
],
"elasticsearch_zone_b_subnet_blocks": [
"10.0.188.0/23",
"10.0.190.0/23"
],
"elasticsearch_zone_c_subnet_blocks": [
"10.0.192.0/23",
"10.0.194.0/23"
],
"vpc_cidr_block": "10.0.0.0/16",
"eks_cidr_subnet": "23",
"qovery_api_url": "api.qovery.com",
"rds_cidr_subnet": "23",
"documentdb_cidr_subnet": "23",
"elasticsearch_cidr_subnet": "23"
}

1330
tests/aws/aws_environment.rs Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1 @@

223
tests/aws/aws_kubernetes.rs Normal file
View File

@@ -0,0 +1,223 @@
/*extern crate test_utilities;
use serde_json::value::Value;
use self::test_utilities::cloudflare::dns_provider_cloudflare;
use self::test_utilities::utilities::init;
use qovery_engine::cloud_provider::aws::kubernetes::node::Node;
use qovery_engine::cloud_provider::aws::kubernetes::EKS;
use qovery_engine::cloud_provider::aws::AWS;
use qovery_engine::cloud_provider::kubernetes::{Kubernetes, KubernetesError};
use qovery_engine::cloud_provider::CloudProvider;
use qovery_engine::dns_provider::cloudflare::Cloudflare;
use qovery_engine::models::Clone2;
use qovery_engine::transaction::TransactionResult;
use std::borrow::Borrow;
use std::fs::File;
use std::io::{BufReader, Read};
use test_utilities::aws::AWS_KUBERNETES_VERSION;
#[test]
#[ignore]
fn create_eks_cluster_in_us_east_2() {
init();
let context = test_utilities::aws::context();
let engine = test_utilities::aws::docker_ecr_aws_engine(&context);
let session = engine.session().unwrap();
let mut tx = session.transaction();
let aws = test_utilities::aws::cloud_provider_aws(&context);
let nodes = test_utilities::aws::aws_kubernetes_nodes();
let cloudflare = dns_provider_cloudflare(&context);
let mut file = File::open("tests/assets/eks-options.json").unwrap();
let mut read_buf = String::new();
file.read_to_string(&mut read_buf).unwrap();
let options_result = serde_json::from_str::<
qovery_engine::cloud_provider::aws::kubernetes::Options,
>(read_buf.as_str());
let kubernetes = EKS::new(
context,
"eks-on-us-east-2",
"eks-us-east-2",
AWS_KUBERNETES_VERSION,
"us-east-2",
&aws,
&cloudflare,
options_result.expect("Oh my god an error in test... Options options options"),
nodes,
);
match tx.create_kubernetes(&kubernetes) {
Err(err) => panic!("{:?}", err),
_ => {}
}
let _ = match tx.commit() {
TransactionResult::Ok => assert!(true),
TransactionResult::Rollback(_) => assert!(false),
TransactionResult::UnrecoverableError(_, _) => assert!(false),
};
}
pub fn read_file(filepath: &str) -> String {
let file = File::open(filepath).expect("could not open file");
let mut buffered_reader = BufReader::new(file);
let mut contents = String::new();
let _number_of_bytes: usize = match buffered_reader.read_to_string(&mut contents) {
Ok(number_of_bytes) => number_of_bytes,
Err(_err) => 0,
};
contents
}
#[test]
#[ignore]
fn create_eks_cluster_in_eu_west_3() {
init();
let context = test_utilities::aws::context();
let engine = test_utilities::aws::docker_ecr_aws_engine(&context);
let session = engine.session().unwrap();
let mut tx = session.transaction();
let aws = test_utilities::aws::cloud_provider_aws(&context);
let nodes = test_utilities::aws::aws_kubernetes_nodes();
let cloudflare = dns_provider_cloudflare(&context);
let mut file = File::open("tests/assets/eks-options.json").unwrap();
let mut read_buf = String::new();
file.read_to_string(&mut read_buf).unwrap();
let options_result = serde_json::from_str::<
qovery_engine::cloud_provider::aws::kubernetes::Options,
>(read_buf.as_str());
let kubernetes = EKS::new(
context.clone(),
"eks-on-eu-west-3",
"eks-eu-west-3",
AWS_KUBERNETES_VERSION,
"eu-west-3",
&aws,
&cloudflare,
options_result.expect("Oh my god an error in test... Options options options"),
nodes,
);
match tx.create_kubernetes(&kubernetes) {
Err(err) => panic!("{:?}", err),
_ => {}
}
let _ = match tx.commit() {
TransactionResult::Ok => assert!(true),
TransactionResult::Rollback(_) => assert!(false),
TransactionResult::UnrecoverableError(_, _) => assert!(false),
};
}
#[test]
#[ignore]
fn delete_eks_cluster_in_us_east_2() {
init();
let context = test_utilities::aws::context();
let engine = test_utilities::aws::docker_ecr_aws_engine(&context);
let session = engine.session().unwrap();
let mut tx = session.transaction();
let aws = test_utilities::aws::cloud_provider_aws(&context);
let nodes = test_utilities::aws::aws_kubernetes_nodes();
let cloudflare = dns_provider_cloudflare(&context);
let mut file = File::open("tests/assets/eks-options.json").unwrap();
let mut read_buf = String::new();
file.read_to_string(&mut read_buf).unwrap();
let options_result = serde_json::from_str::<
qovery_engine::cloud_provider::aws::kubernetes::Options,
>(read_buf.as_str());
let kubernetes = EKS::new(
context,
"eks-on-us-east-2",
"eks-us-east-2",
AWS_KUBERNETES_VERSION,
"us-east-2",
&aws,
&cloudflare,
options_result.expect("Oh my god an error in test... Options options options"),
nodes,
);
match tx.delete_kubernetes(&kubernetes) {
Err(err) => panic!("{:?}", err),
_ => {}
}
let _ = match tx.commit() {
TransactionResult::Ok => assert!(true),
TransactionResult::Rollback(_) => assert!(false),
TransactionResult::UnrecoverableError(_, _) => assert!(false),
};
}
#[test]
#[ignore]
fn delete_eks_cluster_in_eu_west_3() {
init();
// put some environments here, simulated or not
let context = test_utilities::aws::context();
let engine = test_utilities::aws::docker_ecr_aws_engine(&context);
let session = engine.session().unwrap();
let mut tx = session.transaction();
let aws = test_utilities::aws::cloud_provider_aws(&context);
let nodes = test_utilities::aws::aws_kubernetes_nodes();
let cloudflare = dns_provider_cloudflare(&context);
let mut file = File::open("tests/assets/eks-options.json").unwrap();
let mut read_buf = String::new();
file.read_to_string(&mut read_buf).unwrap();
let options_result = serde_json::from_str::<
qovery_engine::cloud_provider::aws::kubernetes::Options,
>(read_buf.as_str());
let kubernetes = EKS::new(
context,
"eks-on-eu-west-3",
"eks-eu-west-3",
AWS_KUBERNETES_VERSION,
"eu-west-3",
&aws,
&cloudflare,
options_result.expect("Oh my god an error in test... Options options options"),
nodes,
);
match tx.delete_kubernetes(&kubernetes) {
Err(err) => panic!("{:?}", err),
_ => {}
}
let _ = match tx.commit() {
TransactionResult::Ok => assert!(true),
TransactionResult::Rollback(_) => assert!(false),
TransactionResult::UnrecoverableError(_, _) => assert!(false),
};
}
*/

59
tests/aws/deletion.rs Normal file
View File

@@ -0,0 +1,59 @@
use std::fs::File;
use std::io::Read;
use qovery_engine::cloud_provider::aws::kubernetes::EKS;
use qovery_engine::cmd::kubectl::create_sample_secret_terraform_in_namespace;
use qovery_engine::transaction::TransactionResult;
use test_utilities::aws::AWS_KUBERNETES_VERSION;
use test_utilities::cloudflare::dns_provider_cloudflare;
use test_utilities::utilities::init;
pub fn do_not_delete_cluster_containing_tfstate() {
init();
// put some environments here, simulated or not
let context = test_utilities::aws::context();
let engine = test_utilities::aws::docker_ecr_aws_engine(&context);
let session = engine.session().unwrap();
let mut tx = session.transaction();
let aws = test_utilities::aws::cloud_provider_aws(&context);
let nodes = test_utilities::aws::aws_kubernetes_nodes();
let cloudflare = dns_provider_cloudflare(&context);
let mut file = File::open("qovery-engine/tests/assets/eks-options.json").unwrap();
let mut read_buf = String::new();
file.read_to_string(&mut read_buf).unwrap();
let options_result = serde_json::from_str::<
qovery_engine::cloud_provider::aws::kubernetes::Options,
>(read_buf.as_str());
let kubernetes = EKS::new(
context,
"eks-on-eu-west-3",
"eks-eu-west-3",
AWS_KUBERNETES_VERSION,
"eu-west-3",
&aws,
&cloudflare,
options_result.expect("Oh my god an error in test... Options options options"),
nodes,
);
/*
create_sample_secret_terraform_in_namespace();
*/
match tx.delete_kubernetes(&kubernetes) {
Err(err) => panic!("{:?}", err),
_ => {}
}
let _ = match tx.commit() {
TransactionResult::Ok => assert!(false),
TransactionResult::Rollback(_) => assert!(false),
TransactionResult::UnrecoverableError(_, _) => assert!(true),
};
}

3
tests/aws/mod.rs Normal file
View File

@@ -0,0 +1,3 @@
mod aws_environment;
mod aws_kubernetes;
mod deletion;

View File

@@ -0,0 +1,42 @@
extern crate test_utilities;
use qovery_engine::build_platform::Image;
use qovery_engine::container_registry::docr::DOCR;
use test_utilities::digitalocean::DIGITAL_OCEAN_TOKEN;
use self::test_utilities::aws::context;
use self::test_utilities::digitalocean::docker_cr_do_engine;
use self::test_utilities::utilities::init;
/*#[test]
#[ignore]
fn create_do_container_registry() {
init();
let context = context();
docker_cr_do_engine(&context);
let docr = DOCR {
context,
registry_name: "qoverytest".to_string(),
api_key: DIGITAL_OCEAN_TOKEN.to_string(),
};
let image_test = Image {
application_id: "to change".to_string(),
name: "imageName".to_string(),
tag: "v666".to_string(),
commit_id: "sha256".to_string(),
registry_url: None,
};
let repository = DOCR::create_repository(&docr, &image_test);
}
*/
/*#[test]
fn create_do_repository_on_container_registry() {}
#[test]
fn delete_do_repository_on_container_registry() {}
#[test]
fn push_sample_image_on_container_registry() {}*/
//
// test --package qovery-engine --test container_registry create_do_container_registry -- --exact

View File

@@ -0,0 +1 @@
mod container_registry;

3
tests/lib.rs Normal file
View File

@@ -0,0 +1,3 @@
mod aws;
mod digital_ocean;
mod unit;

1
tests/unit/mod.rs Normal file
View File

@@ -0,0 +1 @@
mod s3;

38
tests/unit/s3.rs Normal file
View File

@@ -0,0 +1,38 @@
use std::str::FromStr;
use rusoto_core::credential::StaticProvider;
use rusoto_core::{Client, Region};
use rusoto_s3::{
CreateBucketConfiguration, CreateBucketError, CreateBucketRequest, GetObjectError,
GetObjectRequest, ListObjectsV2Output, ListObjectsV2Request, PutBucketVersioningRequest,
PutObjectRequest, S3Client, VersioningConfiguration, S3,
};
use qovery_engine::s3;
use qovery_engine::s3::{delete_bucket, get_default_region_for_us};
use test_utilities::aws::{AWS_SECRET_ACCESS_KEY, AWS_DEFAULT_REGION, AWS_ACCESS_KEY_ID, AWS_REGION_FOR_S3};
use test_utilities::utilities::init;
#[test]
fn delete_s3_bucket() {
init();
let bucket_name = "my-test-bucket";
let credentials = StaticProvider::new(
AWS_ACCESS_KEY_ID.to_string(),
AWS_SECRET_ACCESS_KEY.to_string(),
None,
None,
);
let creation = s3::create_bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, bucket_name);
match creation {
Ok(out) => println!("Yippee Ki Yay"),
Err(e) => println!("While creating the bucket {}", e),
}
let delete = delete_bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, bucket_name);
match delete {
Ok(out) => println!("Yippee Ki Yay"),
Err(e) => println!("While deleting the bucket {}", e),
}
}