diff --git a/Cargo.lock b/Cargo.lock index aefb2a9f..e5e49ce1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3282,6 +3282,7 @@ dependencies = [ "serde_derive", "serde_json", "time 0.2.27", + "tokio 1.10.0", "tracing", "tracing-subscriber", "url 2.2.2", diff --git a/src/cloud_provider/digitalocean/kubernetes/doks_api.rs b/src/cloud_provider/digitalocean/kubernetes/doks_api.rs index e07688f6..28319e1d 100644 --- a/src/cloud_provider/digitalocean/kubernetes/doks_api.rs +++ b/src/cloud_provider/digitalocean/kubernetes/doks_api.rs @@ -80,6 +80,36 @@ fn get_do_kubernetes_latest_slug_version( ))) } +pub fn get_do_kubeconfig_by_cluster_name(token: &str, cluster_name: &str) -> Result, CommandError> { + let clusters_url = format!("{}/clusters", DoApiType::Doks.api_url()); + let clusters_response = do_get_from_api(token, DoApiType::Doks, clusters_url); + let clusters: Result = match clusters_response { + Ok(clusters_response) => match serde_json::from_str(clusters_response.as_str()) { + Ok(clusters) => Ok(clusters), + Err(e) => Err(CommandError::new_from_safe_message(e.to_string())), + }, + Err(e) => Err(CommandError::new_from_safe_message(e.message())), + }; + + let clusters_copy = clusters.expect("Unable to list clusters").kubernetes_clusters.clone(); + match clusters_copy + .into_iter() + .filter(|cluster| cluster.name == cluster_name.to_string()) + .collect::>() + .first() + .clone() + { + Some(cluster) => { + let kubeconfig_url = format!("{}/clusters/{}/kubeconfig", DoApiType::Doks.api_url(), cluster.id); + match do_get_from_api(token, DoApiType::Doks, kubeconfig_url) { + Ok(kubeconfig) => Ok(Some(kubeconfig)), + Err(e) => Err(CommandError::new_from_safe_message(e.message())), + } + } + None => Ok(None), + } +} + #[cfg(test)] mod tests_doks { use crate::cloud_provider::digitalocean::kubernetes::doks_api::{ diff --git a/src/cloud_provider/digitalocean/kubernetes/mod.rs b/src/cloud_provider/digitalocean/kubernetes/mod.rs index a807cdb0..fb12669e 100644 --- a/src/cloud_provider/digitalocean/kubernetes/mod.rs +++ b/src/cloud_provider/digitalocean/kubernetes/mod.rs @@ -1,5 +1,6 @@ use std::borrow::Borrow; use std::env; +use std::fs::File; use serde::{Deserialize, Serialize}; use tera::Context as TeraContext; @@ -8,7 +9,7 @@ use crate::cloud_provider::aws::regions::AwsZones; use crate::cloud_provider::digitalocean::application::DoRegion; use crate::cloud_provider::digitalocean::do_api_common::{do_get_from_api, DoApiType}; use crate::cloud_provider::digitalocean::kubernetes::doks_api::{ - get_do_latest_doks_slug_from_api, get_doks_info_from_name, + get_do_kubeconfig_by_cluster_name, get_do_latest_doks_slug_from_api, get_doks_info_from_name, }; use crate::cloud_provider::digitalocean::kubernetes::helm_charts::{do_helm_charts, ChartsConfigPrerequisites}; use crate::cloud_provider::digitalocean::kubernetes::node::DoInstancesType; @@ -36,14 +37,17 @@ use crate::deletion_utilities::{get_firsts_namespaces_to_delete, get_qovery_mana use crate::dns_provider::DnsProvider; use crate::errors::{CommandError, EngineError}; use crate::events::Stage::Infrastructure; -use crate::events::{EngineEvent, EnvironmentStep, EventDetails, EventMessage, InfrastructureStep, Stage, Transmitter}; +use crate::events::{ + EngineEvent, EnvironmentStep, EventDetails, EventMessage, GeneralStep, InfrastructureStep, Stage, Transmitter, +}; use crate::logger::{LogLevel, Logger}; use crate::models::{ Action, Context, Features, Listen, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, - ProgressScope, QoveryIdentifier, ToHelmString, + ProgressScope, QoveryIdentifier, StringPath, ToHelmString, }; use crate::object_storage::spaces::{BucketDeleteStrategy, Spaces}; use crate::object_storage::ObjectStorage; +use crate::runtime::block_on; use crate::string::terraform_list_format; use crate::{cmd, dns_provider}; use ::function_name::named; @@ -53,6 +57,7 @@ use retry::OperationResult; use std::path::Path; use std::str::FromStr; use std::sync::Arc; +use tokio::io::AsyncWriteExt; pub mod cidr; pub mod doks_api; @@ -613,25 +618,6 @@ impl DOKS { ), }; - // Kubeconfig bucket - self.logger().log( - LogLevel::Info, - EngineEvent::Deploying( - event_details.clone(), - EventMessage::new_from_safe("Create Qovery managed object storage buckets".to_string()), - ), - ); - if let Err(e) = self.spaces.create_bucket(self.kubeconfig_bucket_name().as_str()) { - let error = EngineError::new_object_storage_cannot_create_bucket_error( - event_details.clone(), - self.kubeconfig_bucket_name(), - CommandError::new(e.message.unwrap_or("No error message".to_string()), None), - ); - self.logger() - .log(LogLevel::Error, EngineEvent::Error(error.clone(), None)); - return Err(error); - } - // Logs bucket if let Err(e) = self.spaces.create_bucket(self.logs_bucket_name().as_str()) { let error = EngineError::new_object_storage_cannot_create_bucket_error( @@ -652,25 +638,8 @@ impl DOKS { )); } - // push config file to object storage let kubeconfig_path = &self.get_kubeconfig_file_path()?; let kubeconfig_path = Path::new(kubeconfig_path); - let kubeconfig_name = format!("{}.yaml", self.id()); - if let Err(e) = self.spaces.put( - self.kubeconfig_bucket_name().as_str(), - kubeconfig_name.as_str(), - kubeconfig_path.to_str().expect("No path for Kubeconfig"), - ) { - let error = EngineError::new_object_storage_cannot_put_file_into_bucket_error( - event_details.clone(), - self.logs_bucket_name(), - kubeconfig_name.to_string(), - CommandError::new(e.message.unwrap_or("No error message".to_string()), None), - ); - self.logger() - .log(LogLevel::Error, EngineEvent::Error(error.clone(), None)); - return Err(error); - } match self.check_workers_on_create() { Ok(_) => { @@ -1715,6 +1684,134 @@ impl Kubernetes for DOKS { ); Ok(()) } + + fn get_kubeconfig_file_path(&self) -> Result { + let (path, _) = self.get_kubeconfig_file()?; + Ok(path) + } + + fn get_kubeconfig_file(&self) -> Result<(String, File), EngineError> { + let event_details = self.get_event_details(Infrastructure(InfrastructureStep::LoadConfiguration)); + let bucket_name = format!("qovery-kubeconfigs-{}", self.id()); + let object_key = self.get_kubeconfig_filename(); + let stage = Stage::General(GeneralStep::RetrieveClusterConfig); + + // check if kubeconfig locally exists + let local_kubeconfig = match self.get_temp_dir(event_details.clone()) { + Ok(x) => { + let local_kubeconfig_folder_path = format!("{}/{}", &x, &bucket_name); + let local_kubeconfig_generated = format!("{}/{}", &local_kubeconfig_folder_path, &object_key); + if Path::new(&local_kubeconfig_generated).exists() { + match File::open(&local_kubeconfig_generated) { + Ok(_) => Some(local_kubeconfig_generated), + Err(err) => { + self.logger().log( + LogLevel::Debug, + EngineEvent::Debug( + self.get_event_details(stage.clone()), + EventMessage::new( + err.to_string(), + Some( + format!("Error, couldn't open {} file", &local_kubeconfig_generated,) + .to_string(), + ), + ), + ), + ); + None + } + } + } else { + None + } + } + Err(_) => None, + }; + + // otherwise, try to get it from digital ocean api + let result = match local_kubeconfig { + Some(local_kubeconfig_generated) => match File::open(&local_kubeconfig_generated) { + Ok(file) => Ok((StringPath::from(&local_kubeconfig_generated), file)), + Err(e) => Err(EngineError::new_cannot_retrieve_cluster_config_file( + event_details.clone(), + CommandError::new(e.to_string(), Some(e.to_string())), + )), + }, + None => { + let kubeconfig = match get_do_kubeconfig_by_cluster_name(self.cloud_provider.token(), self.name()) { + Ok(kubeconfig) => Ok(kubeconfig), + Err(e) => Err(EngineError::new_cannot_retrieve_cluster_config_file( + event_details.clone(), + CommandError::new(e.message(), Some(e.message())), + )), + } + .expect("Unable to get kubeconfig"); + + let workspace_directory = crate::fs::workspace_directory( + self.context().workspace_root_dir(), + self.context().execution_id(), + format!("object-storage/scaleway_os/{}", self.name()), + ) + .map_err(|err| { + EngineError::new_cannot_retrieve_cluster_config_file( + event_details.clone(), + CommandError::new(err.to_string(), Some(err.to_string())), + ) + }) + .expect("Unable to create directory"); + + let file_path = format!( + "{}/{}/{}", + workspace_directory, + format!("qovery-kubeconfigs-{}", self.id()), + format!("{}.yaml", self.id()) + ); + let path = Path::new(file_path.as_str()); + let parent_dir = path.parent().unwrap(); + let _ = block_on(tokio::fs::create_dir_all(parent_dir)); + + match block_on( + tokio::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path), + ) { + Ok(mut created_file) => match kubeconfig.is_some() { + false => Err(EngineError::new_cannot_create_file( + event_details.clone(), + CommandError::new( + "No kubeconfig found".to_string(), + Some("No kubeconfig found".to_string()), + ), + )), + true => match block_on(created_file.write_all(kubeconfig.unwrap().as_bytes())) { + Ok(_) => { + let file = File::open(path).unwrap(); + Ok((file_path, file)) + } + Err(e) => Err(EngineError::new_cannot_retrieve_cluster_config_file( + event_details.clone(), + CommandError::new(e.to_string(), Some(e.to_string())), + )), + }, + }, + Err(e) => Err(EngineError::new_cannot_create_file( + event_details.clone(), + CommandError::new(e.to_string(), Some(e.to_string())), + )), + } + } + }; + + match result { + Err(e) => Err(EngineError::new_cannot_retrieve_cluster_config_file( + event_details.clone(), + CommandError::new(e.message(), Some(e.message())), + )), + Ok((file_path, file)) => Ok((file_path, file)), + } + } } impl Listen for DOKS { diff --git a/src/cloud_provider/digitalocean/models/do_api.rs b/src/cloud_provider/digitalocean/models/do_api.rs new file mode 100644 index 00000000..e69de29b diff --git a/src/cloud_provider/digitalocean/models/doks.rs b/src/cloud_provider/digitalocean/models/doks.rs index 0fa04c8e..e7a9394d 100644 --- a/src/cloud_provider/digitalocean/models/doks.rs +++ b/src/cloud_provider/digitalocean/models/doks.rs @@ -1,11 +1,11 @@ use serde::{Deserialize, Serialize}; -#[derive(Default, Serialize, Deserialize, PartialEq, Debug)] +#[derive(Default, Serialize, Deserialize, PartialEq, Debug, Clone)] pub struct DoksList { pub kubernetes_clusters: Vec, } -#[derive(Default, Serialize, Deserialize, PartialEq, Debug)] +#[derive(Default, Serialize, Deserialize, PartialEq, Debug, Clone)] pub struct KubernetesCluster { pub id: String, pub name: String, diff --git a/src/container_registry/docr.rs b/src/container_registry/docr.rs index cd686f30..3f904fe1 100644 --- a/src/container_registry/docr.rs +++ b/src/container_registry/docr.rs @@ -26,7 +26,7 @@ pub struct DOCR { pub name: String, pub api_key: String, pub id: String, - pub registry_info: ContainerRegistryInfo, + pub registry_info: Option, pub listeners: Listeners, pub logger: Box, } @@ -55,12 +55,13 @@ impl DOCR { name: name.to_string(), api_key: api_key.into(), id: id.into(), - registry_info, listeners: vec![], logger, + registry_info: Some(registry_info), }; let event_details = cr.get_event_details(); + if cr.context.docker.login(&cr.registry_info.endpoint).is_err() { return Err(EngineError::new_client_invalid_cloud_provider_credentials( event_details, @@ -222,7 +223,8 @@ impl ContainerRegistry for DOCR { } fn registry_info(&self) -> &ContainerRegistryInfo { - &self.registry_info + // At this point the registry info should be initialize, so unwrap is safe + self.registry_info.as_ref().unwrap() } fn create_registry(&self) -> Result<(), EngineError> { diff --git a/src/errors/io.rs b/src/errors/io.rs index 17c63858..de78d756 100644 --- a/src/errors/io.rs +++ b/src/errors/io.rs @@ -28,6 +28,7 @@ pub enum Tag { CannotGetWorkspaceDirectory, UnsupportedInstanceType, CannotRetrieveClusterConfigFile, + CannotCreateFile, CannotGetClusterNodes, NotEnoughResourcesToDeployEnvironment, CannotUninstallHelmChart, @@ -116,6 +117,7 @@ impl From for Tag { errors::Tag::Unknown => Tag::Unknown, errors::Tag::UnsupportedInstanceType => Tag::UnsupportedInstanceType, errors::Tag::CannotRetrieveClusterConfigFile => Tag::CannotRetrieveClusterConfigFile, + errors::Tag::CannotCreateFile => Tag::CannotCreateFile, errors::Tag::CannotGetClusterNodes => Tag::CannotGetClusterNodes, errors::Tag::NotEnoughResourcesToDeployEnvironment => Tag::NotEnoughResourcesToDeployEnvironment, errors::Tag::MissingRequiredEnvVariable => Tag::MissingRequiredEnvVariable, diff --git a/src/errors/mod.rs b/src/errors/mod.rs index b6531c75..91c0fa2a 100644 --- a/src/errors/mod.rs +++ b/src/errors/mod.rs @@ -120,6 +120,8 @@ pub enum Tag { UnsupportedZone, /// CannotRetrieveKubernetesConfigFile: represents an error while trying to retrieve Kubernetes config file. CannotRetrieveClusterConfigFile, + /// CannotCreateFile: represents an error while trying to create a file. + CannotCreateFile, /// CannotGetClusterNodes: represents an error while trying to get cluster's nodes. CannotGetClusterNodes, /// NotEnoughResourcesToDeployEnvironment: represents an error when trying to deploy an environment but there are not enough resources available on the cluster. @@ -626,7 +628,7 @@ impl EngineError { event_details: EventDetails, error_message: CommandError, ) -> EngineError { - let message = "Cannot retrieve Kubernetes instance type is not supported"; + let message = "Cannot retrieve Kubernetes kubeconfig"; EngineError::new( event_details, Tag::CannotRetrieveClusterConfigFile, @@ -638,6 +640,25 @@ impl EngineError { ) } + /// Creates new error for file we can't create. + /// + /// Arguments: + /// + /// * `event_details`: Error linked event details. + /// * `error_message`: Raw error message. + pub fn new_cannot_create_file(event_details: EventDetails, error_message: CommandError) -> EngineError { + let message = "Cannot create file"; + EngineError::new( + event_details, + Tag::CannotCreateFile, + message.to_string(), + message.to_string(), + Some(error_message), + None, + None, + ) + } + /// Creates new error for Kubernetes cannot get nodes. /// /// Arguments: diff --git a/test_utilities/Cargo.lock b/test_utilities/Cargo.lock index 46e83ffb..e737331b 100644 --- a/test_utilities/Cargo.lock +++ b/test_utilities/Cargo.lock @@ -3320,6 +3320,7 @@ dependencies = [ "serde_derive", "serde_json", "time 0.2.24", + "tokio 1.10.0", "tracing", "tracing-subscriber", "url 2.2.2", diff --git a/test_utilities/Cargo.toml b/test_utilities/Cargo.toml index 01e81fbd..d74d5875 100644 --- a/test_utilities/Cargo.toml +++ b/test_utilities/Cargo.toml @@ -29,6 +29,7 @@ maplit = "1.0.2" uuid = { version = "0.8", features = ["v4"] } const_format = "0.2.22" url = "2.2.2" +tokio = { version = "1.10.0", features = ["full"] } # Digital Ocean Deps digitalocean = "0.1.1" diff --git a/test_utilities/src/utilities.rs b/test_utilities/src/utilities.rs index c004bc8e..df9a94e5 100644 --- a/test_utilities/src/utilities.rs +++ b/test_utilities/src/utilities.rs @@ -9,17 +9,19 @@ use curl::easy::Easy; use dirs::home_dir; use gethostname; use std::collections::BTreeMap; -use std::io::{Error, ErrorKind, Read, Write}; +use std::io::{Error, ErrorKind, Write}; use std::path::Path; use std::str::FromStr; use passwords::PasswordGenerator; +use qovery_engine::cloud_provider::digitalocean::kubernetes::doks_api::get_do_kubeconfig_by_cluster_name; use rand::distributions::Alphanumeric; use rand::{thread_rng, Rng}; use retry::delay::Fibonacci; use retry::OperationResult; use std::env; use std::fs; +use tokio::io::AsyncWriteExt; use tracing::{info, warn}; use crate::scaleway::{ @@ -44,7 +46,6 @@ use crate::digitalocean::{ DO_MANAGED_DATABASE_DISK_TYPE, DO_MANAGED_DATABASE_INSTANCE_TYPE, DO_SELF_HOSTED_DATABASE_DISK_TYPE, DO_SELF_HOSTED_DATABASE_INSTANCE_TYPE, }; -use qovery_engine::cloud_provider::digitalocean::application::DoRegion; use qovery_engine::cmd::command::QoveryCommand; use qovery_engine::cmd::docker::Docker; use qovery_engine::cmd::kubectl::{kubectl_get_pvc, kubectl_get_svc}; @@ -52,8 +53,6 @@ use qovery_engine::cmd::structs::{KubernetesList, KubernetesPod, PVC, SVC}; use qovery_engine::errors::CommandError; use qovery_engine::logger::{Logger, StdIoLogger}; use qovery_engine::models::DatabaseMode::MANAGED; -use qovery_engine::object_storage::spaces::{BucketDeleteStrategy, Spaces}; -use qovery_engine::object_storage::ObjectStorage; use qovery_engine::runtime::block_on; use time::Instant; @@ -536,64 +535,52 @@ where ) } Kind::Do => { - let region_raw = secrets - .DIGITAL_OCEAN_DEFAULT_REGION - .as_ref() - .expect(&"DIGITAL_OCEAN_DEFAULT_REGION should be set".to_string()) - .to_string(); + let cluster_name = format!("qovery-{}", context.cluster_id()); + let kubeconfig = match get_do_kubeconfig_by_cluster_name( + secrets.clone().DIGITAL_OCEAN_TOKEN.unwrap().as_str(), + cluster_name.clone().as_str(), + ) { + Ok(kubeconfig) => Ok(kubeconfig), + Err(e) => Err(CommandError::new(e.message(), Some(e.message()))), + } + .expect("Unable to get kubeconfig"); - match DoRegion::from_str(region_raw.as_str()) { - Ok(region) => { - let spaces = Spaces::new( - context.clone(), - "fake".to_string(), - "fake".to_string(), - secrets - .DIGITAL_OCEAN_SPACES_ACCESS_ID - .as_ref() - .expect(&"DIGITAL_OCEAN_SPACES_ACCESS_ID should be set".to_string()) - .to_string(), - secrets - .DIGITAL_OCEAN_SPACES_SECRET_ID - .as_ref() - .expect(&"DIGITAL_OCEAN_SPACES_SECRET_ID should be set".to_string()) - .to_string(), - region, - BucketDeleteStrategy::HardDelete, - ); + let workspace_directory = qovery_engine::fs::workspace_directory( + context.workspace_root_dir(), + context.execution_id(), + format!("object-storage/scaleway_os/{}", cluster_name.clone()), + ) + .map_err(|err| CommandError::new(err.to_string(), Some(err.to_string()))) + .expect("Unable to create directory"); - match spaces.get( - kubernetes_config_bucket_name.as_str(), - kubernetes_config_object_key.as_str(), - false, - ) { - Ok((_, mut file)) => { - let mut content = String::new(); - match file.read_to_string(&mut content) { - Ok(_) => Ok(content), - Err(e) => { - let message_safe = "Error while trying to read file"; - Err(CommandError::new( - format!("{}, error: {}", message_safe.to_string(), e), - Some(message_safe.to_string()), - )) - } - } - } - Err(e) => { - let message_safe = "Error while trying to get kubeconfig from spaces"; - Err(CommandError::new( - format!( - "{}, error: {}", - message_safe.to_string(), - e.message.unwrap_or("no error message".to_string()) - ), - Some(message_safe.to_string()), - )) - } - } - } - Err(e) => Err(e), + let file_path = format!( + "{}/{}/{}", + workspace_directory, + format!("qovery-kubeconfigs-{}", context.cluster_id()), + format!("{}.yaml", context.cluster_id()) + ); + let path = Path::new(file_path.as_str()); + let parent_dir = path.parent().unwrap(); + let _ = block_on(tokio::fs::create_dir_all(parent_dir)); + + match block_on( + tokio::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path), + ) { + Ok(mut created_file) => match kubeconfig.is_some() { + false => Err(CommandError::new( + "No kubeconfig found".to_string(), + Some("No kubeconfig found".to_string()), + )), + true => match block_on(created_file.write_all(kubeconfig.unwrap().as_bytes())) { + Ok(_) => Ok(file_path), + Err(e) => Err(CommandError::new(e.to_string(), Some(e.to_string()))), + }, + }, + Err(e) => Err(CommandError::new(e.to_string(), Some(e.to_string()))), } } Kind::Scw => {