jlengrand/engine
1
0
Fork 0
mirror of https://github.com/jlengrand/engine.git synced 2026-03-10 08:11:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: auto-repair terraform provider cache issue

Browse Source
This commit is contained in:
Pierre Mavro
2021-08-27 18:09:50 +02:00
parent fd78c902d3
commit f5c8011da1
7 changed files with 132 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
Cargo.lock generated
View File

@@ -2077,6 +2077,7 @@ dependencies = [
"tokio 1.10.0",
"tracing",
"tracing-subscriber",
"tracing-test",
"trust-dns-resolver",
"uuid 0.8.2",
"walkdir",
@@ -3614,6 +3615,29 @@ dependencies = [
"tracing-serde",
]
[[package]]
name = "tracing-test"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3b48778c2d401c6a7fcf38a0e3c55dc8e8e753cbd381044a8cdb6fd69a29f53"
dependencies = [
"lazy_static",
"tracing-core",
"tracing-subscriber",
"tracing-test-macro",
]
[[package]]
name = "tracing-test-macro"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c49adbab879d2e0dd7f75edace5f0ac2156939ecb7e6a1e8fa14e53728328c48"
dependencies = [
"lazy_static",
"quote 1.0.9",
"syn 1.0.74",
]
[[package]]
name = "trust-dns-proto"
version = "0.20.3"

1
Cargo.toml
View File

@@ -33,6 +33,7 @@ tar = ">=0.4.36"
# logger
tracing = "0.1.26"
tracing-subscriber = "0.2.18"
tracing-test = "0.1.0"
# Docker deps
# shiplift = "0.6.0"

1
lib/aws/bootstrap/tf-providers-aws.j2.tf
View File

@@ -24,7 +24,6 @@ terraform {
source = "hashicorp/random"
version = "~> 2.3"
}
time = {
source = "hashicorp/time"
version = "~> 0.3"

2
src/cloud_provider/aws/kubernetes/mod.rs
View File

@@ -786,7 +786,7 @@ impl<'a> Kubernetes for EKS<'a> {
scope: EngineErrorScope::Engine,
execution_id: self.context.execution_id().to_string(),
message: Some(format!(
"error while trying to remove {} out of terraform state file. {:?}",
"error while trying to remove {} out of terraform state file.\n {:?}",
entry, e.message
)),
})

2
src/cloud_provider/digitalocean/kubernetes/mod.rs
View File

@@ -511,7 +511,7 @@ impl<'a> Kubernetes for DOKS<'a> {
scope: EngineErrorScope::Engine,
execution_id: self.context.execution_id().to_string(),
message: Some(format!(
"error while trying to remove {} out of terraform state file. {:?}",
"error while trying to remove {} out of terraform state file.\n {:?}",
entry, e.message
)),
})

87
src/cmd/terraform.rs
View File

@@ -8,9 +8,11 @@ use crate::error::{SimpleError, SimpleErrorKind};
use chrono::Duration;
use rand::Rng;
use retry::Error::Operation;
use std::{thread, time};
use std::{fs, thread, time};
fn terraform_init_validate(root_dir: &str) -> Result<(), SimpleError> {
let terraform_provider_lock = format!("{}/.terraform.lock.hcl", &root_dir);
// terraform init
let result = retry::retry(Fixed::from_millis(3000).take(5), || {
match terraform_exec(root_dir, vec!["init"]) {
@@ -18,19 +20,29 @@ fn terraform_init_validate(root_dir: &str) -> Result<(), SimpleError> {
Err(err) => {
// Error: Failed to install provider from shared cache
// in order to avoid lock errors on parallel run, let's sleep a bit
// https://github.com/hashicorp/terraform/issues/28041
debug!("{:?}", err);
if err.message.is_some() {
let message = err.message.clone();
if message
.unwrap()
.contains("Failed to install provider from shared cache")
{
let sleep_time_int = rand::thread_rng().gen_range(30..75);
let sleep_time = time::Duration::from_millis(sleep_time_int);
info!(
"another terraform command is trying to use shared provider cache which is forbidden, sleeping {} before retrying...",
let sleep_time_int = rand::thread_rng().gen_range(20..45);
let sleep_time = time::Duration::from_secs(sleep_time_int);
warn!(
"failed to install provider from shared cache, cleaning and sleeping {} before retrying...",
sleep_time_int
);
thread::sleep(sleep_time);
match fs::remove_file(&terraform_provider_lock) {
Ok(_) => info!("terraform lock file {} has been removed", &terraform_provider_lock),
Err(e) => error!(
"wasn't able to delete terraform lock file {}: {}",
&terraform_provider_lock, e
),
}
}
};
error!("error while trying to run terraform init, retrying...");
@@ -175,23 +187,86 @@ pub fn terraform_exec(root_dir: &str, args: Vec<&str>) -> Result<Vec<String>, Si
let home_dir = home_dir().expect("Could not find $HOME");
let tf_plugin_cache_dir = format!("{}/.terraform.d/plugin-cache", home_dir.to_str().unwrap());
let mut stdout = Vec::new();
let mut stderr = Vec::new();
let result = exec_with_envs_and_output(
format!("{} terraform", root_dir).as_str(),
args,
vec![(TF_PLUGIN_CACHE_DIR, tf_plugin_cache_dir.as_str())],
|line: Result<String, std::io::Error>| {
let output = line.unwrap();
stdout.push(output.clone());
info!("{}", &output)
},
|line: Result<String, std::io::Error>| {
let output = line.unwrap();
stderr.push(output.clone());
error!("{}", &output);
},
Duration::max_value(),
);
stdout.extend(stderr);
match result {
Ok(_) => Ok(result.unwrap()),
Err(e) => Err(e),
Err(mut e) => {
e.message = Some(stdout.join("\n"));
Err(e)
}
}
}
#[cfg(test)]
mod tests {
use crate::cmd::terraform::terraform_init_validate;
use std::fs;
//use tracing::{span, Level};
//use tracing_test::traced_test;
#[test]
#[ignore]
//#[traced_test]
// https://github.com/hashicorp/terraform/issues/28041
fn test_terraform_init_lock_issue() {
//let span = span!(Level::TRACE, "terraform_test");
//let _enter = span.enter();
// those 2 files are a voluntary broken config, it should detect it and auto repair
let terraform_lock_file = r#"
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/local" {
version = "1.4.0"
constraints = "~> 1.4"
hashes = [
"h1:bZN53L85E49Pc5o3HUUCUqP5rZBziMF2KfKOaFsqN7w=",
"zh:1b265fcfdce8cc3ccb51969c6d7a61531bf8a6e1218d95c1a74c40f25595c74b",
]
}
"#;
let provider_file = r#"
terraform {
required_providers {
local = {
source = "hashicorp/local"
version = "~> 1.4"
}
}
required_version = ">= 0.14"
}
"#;
let dest_dir = "/tmp/test";
let _ = fs::create_dir_all(&dest_dir).unwrap();
let _ = fs::write(format!("{}/.terraform.lock.hcl", &dest_dir), terraform_lock_file);
let _ = fs::write(format!("{}/providers.tf", &dest_dir), provider_file);
let res = terraform_init_validate(dest_dir);
assert!(res.is_ok());
}
}

24
test_utilities/Cargo.lock generated
View File

@@ -2077,6 +2077,7 @@ dependencies = [
"tokio 1.10.0",
"tracing",
"tracing-subscriber",
"tracing-test",
"trust-dns-resolver",
"walkdir",
]
@@ -3603,6 +3604,29 @@ dependencies = [
"tracing-serde",
]
[[package]]
name = "tracing-test"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3b48778c2d401c6a7fcf38a0e3c55dc8e8e753cbd381044a8cdb6fd69a29f53"
dependencies = [
"lazy_static",
"tracing-core",
"tracing-subscriber",
"tracing-test-macro",
]
[[package]]
name = "tracing-test-macro"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c49adbab879d2e0dd7f75edace5f0ac2156939ecb7e6a1e8fa14e53728328c48"
dependencies = [
"lazy_static",
"quote 1.0.8",
"syn 1.0.73",
]
[[package]]
name = "trust-dns-proto"
version = "0.20.3"