From f5c8011da1b8090fd7e67cccbdf0b7dc9c77272a Mon Sep 17 00:00:00 2001 From: Pierre Mavro Date: Fri, 27 Aug 2021 18:09:50 +0200 Subject: [PATCH] feat: auto-repair terraform provider cache issue --- Cargo.lock | 24 +++++ Cargo.toml | 1 + lib/aws/bootstrap/tf-providers-aws.j2.tf | 1 - src/cloud_provider/aws/kubernetes/mod.rs | 2 +- .../digitalocean/kubernetes/mod.rs | 2 +- src/cmd/terraform.rs | 87 +++++++++++++++++-- test_utilities/Cargo.lock | 24 +++++ 7 files changed, 132 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d18432c..d9efa7e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2077,6 +2077,7 @@ dependencies = [ "tokio 1.10.0", "tracing", "tracing-subscriber", + "tracing-test", "trust-dns-resolver", "uuid 0.8.2", "walkdir", @@ -3614,6 +3615,29 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "tracing-test" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3b48778c2d401c6a7fcf38a0e3c55dc8e8e753cbd381044a8cdb6fd69a29f53" +dependencies = [ + "lazy_static", + "tracing-core", + "tracing-subscriber", + "tracing-test-macro", +] + +[[package]] +name = "tracing-test-macro" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c49adbab879d2e0dd7f75edace5f0ac2156939ecb7e6a1e8fa14e53728328c48" +dependencies = [ + "lazy_static", + "quote 1.0.9", + "syn 1.0.74", +] + [[package]] name = "trust-dns-proto" version = "0.20.3" diff --git a/Cargo.toml b/Cargo.toml index 0b03a57e..95a3a11b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ tar = ">=0.4.36" # logger tracing = "0.1.26" tracing-subscriber = "0.2.18" +tracing-test = "0.1.0" # Docker deps # shiplift = "0.6.0" diff --git a/lib/aws/bootstrap/tf-providers-aws.j2.tf b/lib/aws/bootstrap/tf-providers-aws.j2.tf index 0126ee5c..f3a4e85e 100644 --- a/lib/aws/bootstrap/tf-providers-aws.j2.tf +++ b/lib/aws/bootstrap/tf-providers-aws.j2.tf @@ -24,7 +24,6 @@ terraform { source = "hashicorp/random" version = "~> 2.3" } - time = { source = "hashicorp/time" version = "~> 0.3" diff --git a/src/cloud_provider/aws/kubernetes/mod.rs b/src/cloud_provider/aws/kubernetes/mod.rs index 50de6869..4d3161bd 100644 --- a/src/cloud_provider/aws/kubernetes/mod.rs +++ b/src/cloud_provider/aws/kubernetes/mod.rs @@ -786,7 +786,7 @@ impl<'a> Kubernetes for EKS<'a> { scope: EngineErrorScope::Engine, execution_id: self.context.execution_id().to_string(), message: Some(format!( - "error while trying to remove {} out of terraform state file. {:?}", + "error while trying to remove {} out of terraform state file.\n {:?}", entry, e.message )), }) diff --git a/src/cloud_provider/digitalocean/kubernetes/mod.rs b/src/cloud_provider/digitalocean/kubernetes/mod.rs index a73e91d8..c2748e7b 100644 --- a/src/cloud_provider/digitalocean/kubernetes/mod.rs +++ b/src/cloud_provider/digitalocean/kubernetes/mod.rs @@ -511,7 +511,7 @@ impl<'a> Kubernetes for DOKS<'a> { scope: EngineErrorScope::Engine, execution_id: self.context.execution_id().to_string(), message: Some(format!( - "error while trying to remove {} out of terraform state file. {:?}", + "error while trying to remove {} out of terraform state file.\n {:?}", entry, e.message )), }) diff --git a/src/cmd/terraform.rs b/src/cmd/terraform.rs index a494581b..3abd5616 100644 --- a/src/cmd/terraform.rs +++ b/src/cmd/terraform.rs @@ -8,9 +8,11 @@ use crate::error::{SimpleError, SimpleErrorKind}; use chrono::Duration; use rand::Rng; use retry::Error::Operation; -use std::{thread, time}; +use std::{fs, thread, time}; fn terraform_init_validate(root_dir: &str) -> Result<(), SimpleError> { + let terraform_provider_lock = format!("{}/.terraform.lock.hcl", &root_dir); + // terraform init let result = retry::retry(Fixed::from_millis(3000).take(5), || { match terraform_exec(root_dir, vec!["init"]) { @@ -18,19 +20,29 @@ fn terraform_init_validate(root_dir: &str) -> Result<(), SimpleError> { Err(err) => { // Error: Failed to install provider from shared cache // in order to avoid lock errors on parallel run, let's sleep a bit + // https://github.com/hashicorp/terraform/issues/28041 + debug!("{:?}", err); if err.message.is_some() { let message = err.message.clone(); if message .unwrap() .contains("Failed to install provider from shared cache") { - let sleep_time_int = rand::thread_rng().gen_range(30..75); - let sleep_time = time::Duration::from_millis(sleep_time_int); - info!( - "another terraform command is trying to use shared provider cache which is forbidden, sleeping {} before retrying...", + let sleep_time_int = rand::thread_rng().gen_range(20..45); + let sleep_time = time::Duration::from_secs(sleep_time_int); + warn!( + "failed to install provider from shared cache, cleaning and sleeping {} before retrying...", sleep_time_int ); thread::sleep(sleep_time); + + match fs::remove_file(&terraform_provider_lock) { + Ok(_) => info!("terraform lock file {} has been removed", &terraform_provider_lock), + Err(e) => error!( + "wasn't able to delete terraform lock file {}: {}", + &terraform_provider_lock, e + ), + } } }; error!("error while trying to run terraform init, retrying..."); @@ -175,23 +187,86 @@ pub fn terraform_exec(root_dir: &str, args: Vec<&str>) -> Result, Si let home_dir = home_dir().expect("Could not find $HOME"); let tf_plugin_cache_dir = format!("{}/.terraform.d/plugin-cache", home_dir.to_str().unwrap()); + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); let result = exec_with_envs_and_output( format!("{} terraform", root_dir).as_str(), args, vec![(TF_PLUGIN_CACHE_DIR, tf_plugin_cache_dir.as_str())], |line: Result| { let output = line.unwrap(); + stdout.push(output.clone()); info!("{}", &output) }, |line: Result| { let output = line.unwrap(); + stderr.push(output.clone()); error!("{}", &output); }, Duration::max_value(), ); + stdout.extend(stderr); + match result { Ok(_) => Ok(result.unwrap()), - Err(e) => Err(e), + Err(mut e) => { + e.message = Some(stdout.join("\n")); + Err(e) + } + } +} + +#[cfg(test)] +mod tests { + use crate::cmd::terraform::terraform_init_validate; + use std::fs; + //use tracing::{span, Level}; + //use tracing_test::traced_test; + + #[test] + #[ignore] + //#[traced_test] + // https://github.com/hashicorp/terraform/issues/28041 + fn test_terraform_init_lock_issue() { + //let span = span!(Level::TRACE, "terraform_test"); + //let _enter = span.enter(); + + // those 2 files are a voluntary broken config, it should detect it and auto repair + let terraform_lock_file = r#" +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/local" { + version = "1.4.0" + constraints = "~> 1.4" + hashes = [ + "h1:bZN53L85E49Pc5o3HUUCUqP5rZBziMF2KfKOaFsqN7w=", + "zh:1b265fcfdce8cc3ccb51969c6d7a61531bf8a6e1218d95c1a74c40f25595c74b", + ] +} + "#; + + let provider_file = r#" +terraform { + required_providers { + local = { + source = "hashicorp/local" + version = "~> 1.4" + } + } + required_version = ">= 0.14" +} + "#; + + let dest_dir = "/tmp/test"; + let _ = fs::create_dir_all(&dest_dir).unwrap(); + + let _ = fs::write(format!("{}/.terraform.lock.hcl", &dest_dir), terraform_lock_file); + let _ = fs::write(format!("{}/providers.tf", &dest_dir), provider_file); + + let res = terraform_init_validate(dest_dir); + + assert!(res.is_ok()); } } diff --git a/test_utilities/Cargo.lock b/test_utilities/Cargo.lock index a703c22d..4e8efa4d 100644 --- a/test_utilities/Cargo.lock +++ b/test_utilities/Cargo.lock @@ -2077,6 +2077,7 @@ dependencies = [ "tokio 1.10.0", "tracing", "tracing-subscriber", + "tracing-test", "trust-dns-resolver", "walkdir", ] @@ -3603,6 +3604,29 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "tracing-test" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3b48778c2d401c6a7fcf38a0e3c55dc8e8e753cbd381044a8cdb6fd69a29f53" +dependencies = [ + "lazy_static", + "tracing-core", + "tracing-subscriber", + "tracing-test-macro", +] + +[[package]] +name = "tracing-test-macro" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c49adbab879d2e0dd7f75edace5f0ac2156939ecb7e6a1e8fa14e53728328c48" +dependencies = [ + "lazy_static", + "quote 1.0.8", + "syn 1.0.73", +] + [[package]] name = "trust-dns-proto" version = "0.20.3"