feat: auto-repair terraform provider cache issue

Cargo.lock

@@ -2077,6 +2077,7 @@ dependencies = [
  "tokio 1.10.0",
  "tracing",
  "tracing-subscriber",
+ "tracing-test",
  "trust-dns-resolver",
  "uuid 0.8.2",
  "walkdir",
@@ -3614,6 +3615,29 @@ dependencies = [
  "tracing-serde",
 ]
 
+[[package]]
+name = "tracing-test"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3b48778c2d401c6a7fcf38a0e3c55dc8e8e753cbd381044a8cdb6fd69a29f53"
+dependencies = [
+ "lazy_static",
+ "tracing-core",
+ "tracing-subscriber",
+ "tracing-test-macro",
+]
+
+[[package]]
+name = "tracing-test-macro"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c49adbab879d2e0dd7f75edace5f0ac2156939ecb7e6a1e8fa14e53728328c48"
+dependencies = [
+ "lazy_static",
+ "quote 1.0.9",
+ "syn 1.0.74",
+]
+
 [[package]]
 name = "trust-dns-proto"
 version = "0.20.3"

Cargo.toml

@@ -33,6 +33,7 @@ tar = ">=0.4.36"
 # logger
 tracing = "0.1.26"
 tracing-subscriber = "0.2.18"
+tracing-test = "0.1.0"
 
 # Docker deps
 # shiplift = "0.6.0"

lib/aws/bootstrap/tf-providers-aws.j2.tf

@@ -24,7 +24,6 @@ terraform {
       source = "hashicorp/random"
       version = "~> 2.3"
     }
-
     time = {
       source  = "hashicorp/time"
       version = "~> 0.3"

src/cloud_provider/aws/kubernetes/mod.rs

@@ -786,7 +786,7 @@ impl<'a> Kubernetes for EKS<'a> {
                                         scope: EngineErrorScope::Engine,
                                         execution_id: self.context.execution_id().to_string(),
                                         message: Some(format!(
-                                            "error while trying to remove {} out of terraform state file. {:?}",
+                                            "error while trying to remove {} out of terraform state file.\n {:?}",
                                             entry, e.message
                                         )),
                                     })

src/cloud_provider/digitalocean/kubernetes/mod.rs

@@ -511,7 +511,7 @@ impl<'a> Kubernetes for DOKS<'a> {
                                         scope: EngineErrorScope::Engine,
                                         execution_id: self.context.execution_id().to_string(),
                                         message: Some(format!(
-                                            "error while trying to remove {} out of terraform state file. {:?}",
+                                            "error while trying to remove {} out of terraform state file.\n {:?}",
                                             entry, e.message
                                         )),
                                     })

src/cmd/terraform.rs

@@ -8,9 +8,11 @@ use crate::error::{SimpleError, SimpleErrorKind};
 use chrono::Duration;
 use rand::Rng;
 use retry::Error::Operation;
-use std::{thread, time};
+use std::{fs, thread, time};
 
 fn terraform_init_validate(root_dir: &str) -> Result<(), SimpleError> {
+    let terraform_provider_lock = format!("{}/.terraform.lock.hcl", &root_dir);
+
     // terraform init
     let result = retry::retry(Fixed::from_millis(3000).take(5), || {
         match terraform_exec(root_dir, vec!["init"]) {
@@ -18,19 +20,29 @@ fn terraform_init_validate(root_dir: &str) -> Result<(), SimpleError> {
             Err(err) => {
                 // Error: Failed to install provider from shared cache
                 // in order to avoid lock errors on parallel run, let's sleep a bit
+                // https://github.com/hashicorp/terraform/issues/28041
+                debug!("{:?}", err);
                 if err.message.is_some() {
                     let message = err.message.clone();
                     if message
                         .unwrap()
                         .contains("Failed to install provider from shared cache")
                     {
-                        let sleep_time_int = rand::thread_rng().gen_range(30..75);
+                        let sleep_time_int = rand::thread_rng().gen_range(20..45);
-                        let sleep_time = time::Duration::from_millis(sleep_time_int);
+                        let sleep_time = time::Duration::from_secs(sleep_time_int);
-                        info!(
+                        warn!(
-                            "another terraform command is trying to use shared provider cache which is forbidden, sleeping {} before retrying...",
+                            "failed to install provider from shared cache, cleaning and sleeping {} before retrying...",
                             sleep_time_int
                         );
                         thread::sleep(sleep_time);
+
+                        match fs::remove_file(&terraform_provider_lock) {
+                            Ok(_) => info!("terraform lock file {} has been removed", &terraform_provider_lock),
+                            Err(e) => error!(
+                                "wasn't able to delete terraform lock file {}: {}",
+                                &terraform_provider_lock, e
+                            ),
+                        }
                     }
                 };
                 error!("error while trying to run terraform init, retrying...");
@@ -175,23 +187,86 @@ pub fn terraform_exec(root_dir: &str, args: Vec<&str>) -> Result<Vec<String>, Si
     let home_dir = home_dir().expect("Could not find $HOME");
     let tf_plugin_cache_dir = format!("{}/.terraform.d/plugin-cache", home_dir.to_str().unwrap());
 
+    let mut stdout = Vec::new();
+    let mut stderr = Vec::new();
     let result = exec_with_envs_and_output(
         format!("{} terraform", root_dir).as_str(),
         args,
         vec![(TF_PLUGIN_CACHE_DIR, tf_plugin_cache_dir.as_str())],
         |line: Result<String, std::io::Error>| {
             let output = line.unwrap();
+            stdout.push(output.clone());
             info!("{}", &output)
         },
         |line: Result<String, std::io::Error>| {
             let output = line.unwrap();
+            stderr.push(output.clone());
             error!("{}", &output);
         },
         Duration::max_value(),
     );
 
+    stdout.extend(stderr);
+
     match result {
         Ok(_) => Ok(result.unwrap()),
-        Err(e) => Err(e),
+        Err(mut e) => {
+            e.message = Some(stdout.join("\n"));
+            Err(e)
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::cmd::terraform::terraform_init_validate;
+    use std::fs;
+    //use tracing::{span, Level};
+    //use tracing_test::traced_test;
+
+    #[test]
+    #[ignore]
+    //#[traced_test]
+    // https://github.com/hashicorp/terraform/issues/28041
+    fn test_terraform_init_lock_issue() {
+        //let span = span!(Level::TRACE, "terraform_test");
+        //let _enter = span.enter();
+
+        // those 2 files are a voluntary broken config, it should detect it and auto repair
+        let terraform_lock_file = r#"
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/local" {
+  version     = "1.4.0"
+  constraints = "~> 1.4"
+  hashes = [
+    "h1:bZN53L85E49Pc5o3HUUCUqP5rZBziMF2KfKOaFsqN7w=",
+    "zh:1b265fcfdce8cc3ccb51969c6d7a61531bf8a6e1218d95c1a74c40f25595c74b",
+  ]
+}
+        "#;
+
+        let provider_file = r#"
+terraform {
+  required_providers {
+    local = {
+      source = "hashicorp/local"
+      version = "~> 1.4"
+    }
+  }
+  required_version = ">= 0.14"
+}
+        "#;
+
+        let dest_dir = "/tmp/test";
+        let _ = fs::create_dir_all(&dest_dir).unwrap();
+
+        let _ = fs::write(format!("{}/.terraform.lock.hcl", &dest_dir), terraform_lock_file);
+        let _ = fs::write(format!("{}/providers.tf", &dest_dir), provider_file);
+
+        let res = terraform_init_validate(dest_dir);
+
+        assert!(res.is_ok());
     }
 }

test_utilities/Cargo.lock

@@ -2077,6 +2077,7 @@ dependencies = [
  "tokio 1.10.0",
  "tracing",
  "tracing-subscriber",
+ "tracing-test",
  "trust-dns-resolver",
  "walkdir",
 ]
@@ -3603,6 +3604,29 @@ dependencies = [
  "tracing-serde",
 ]
 
+[[package]]
+name = "tracing-test"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3b48778c2d401c6a7fcf38a0e3c55dc8e8e753cbd381044a8cdb6fd69a29f53"
+dependencies = [
+ "lazy_static",
+ "tracing-core",
+ "tracing-subscriber",
+ "tracing-test-macro",
+]
+
+[[package]]
+name = "tracing-test-macro"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c49adbab879d2e0dd7f75edace5f0ac2156939ecb7e6a1e8fa14e53728328c48"
+dependencies = [
+ "lazy_static",
+ "quote 1.0.8",
+ "syn 1.0.73",
+]
+
 [[package]]
 name = "trust-dns-proto"
 version = "0.20.3"