diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..34d2e845 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +/target +qovery-engine/Cargo.lock +docker/ci/bin_versions +docker/ci/load.sh +*.iml +.idea +.qovery-workspace +app/.qovery-workspace +.terraform/ +NOTES.txt +docker/engine/providers/ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 00000000..b5bb3618 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,3243 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "addr2line" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + +[[package]] +name = "aho-corasick" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b476ce7103678b0c6d3d395dbbae31d48ff910bd28be979ba5d48c6351131d0d" +dependencies = [ + "memchr", +] + +[[package]] +name = "arc-swap" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + +[[package]] +name = "async-trait" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b246867b8b3b6ae56035f1eb1ed557c1d8eae97f0d53696138a50fa0e3a3b8c0" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "backtrace" +version = "0.3.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707b586e0e2f247cbde68cdd2c3ce69ea7b7be43e1c5b426e37c9319c4b9838e" +dependencies = [ + "addr2line", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base-x" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b20b618342cf9891c292c4f5ac2cde7287cc5c87e87e9c769d617793607dec1" + +[[package]] +name = "base64" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" +dependencies = [ + "byteorder", +] + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "blake2b_simd" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array 0.12.3", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "bstr" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473fc6b38233f9af7baa94fb5852dca389e3d95b8e21c8e3719301462c5d9faf" +dependencies = [ + "memchr", +] + +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "either", + "iovec", +] + +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + +[[package]] +name = "cc" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d" +dependencies = [ + "jobserver", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "serde", + "time 0.1.44", + "winapi 0.3.9", +] + +[[package]] +name = "chrono-tz" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2554a3155fec064362507487171dcc4edc3df60cb10f3a1fb10ed8094822b120" +dependencies = [ + "chrono", + "parse-zoneinfo", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "cmd_lib" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "475bd7aa7680b4ed8f6bb59745e882bcbaeb39326532bb79ffb1716480d9a274" + +[[package]] +name = "const_fn" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce90df4c658c62f12d78f7508cf92f9173e5184a539c10bfe54a3107b3ffd0f2" + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "cookie" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5" +dependencies = [ + "time 0.1.44", + "url 1.7.2", +] + +[[package]] +name = "cookie_store" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46750b3f362965f197996c4448e4a0935e791bf7d6631bfce9ee0af3d24c919c" +dependencies = [ + "cookie", + "failure", + "idna 0.1.5", + "log", + "publicsuffix", + "serde", + "serde_json", + "time 0.1.44", + "try_from", + "url 1.7.2", +] + +[[package]] +name = "core-foundation" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" + +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg 1.0.1", + "cfg-if 0.1.10", + "crossbeam-utils", + "lazy_static", + "maybe-uninit", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-utils", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg 1.0.1", + "cfg-if 0.1.10", + "lazy_static", +] + +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array 0.14.4", + "subtle", +] + +[[package]] +name = "curl" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e268162af1a5fe89917ae25ba3b0a77c8da752bdc58e7dbb4f15b91fbd33756e" +dependencies = [ + "curl-sys", + "libc", + "openssl-probe", + "openssl-sys", + "schannel", + "socket2", + "winapi 0.3.9", +] + +[[package]] +name = "curl-sys" +version = "0.4.38+curl-7.73.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498ecfb4f59997fd40023d62a9f1e506e768b2baeb59a1d311eb9751cdcd7e3f" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", + "winapi 0.3.9", +] + +[[package]] +name = "deunicode" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "850878694b7933ca4c9569d30a34b55031b9b139ee1fc7b94a527c4ef960d690" + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array 0.12.3", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "digitalocean" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce46f423ac4d033c3f0bd032677f2a6362232435de75deacad7c1490f9eb5be" +dependencies = [ + "chrono", + "getset", + "lazy_static", + "log", + "reqwest", + "serde", + "serde_derive", + "serde_json", + "thiserror", + "url 1.7.2", + "url_serde", +] + +[[package]] +name = "dirs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" +dependencies = [ + "cfg-if 0.1.10", + "dirs-sys", +] + +[[package]] +name = "dirs" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142995ed02755914747cc6ca76fc7e4583cd18578746716d0508ea6ed558b9ff" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a" +dependencies = [ + "libc", + "redox_users", + "winapi 0.3.9", +] + +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + +[[package]] +name = "dns-lookup" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "093d88961fd18c4ecacb8c80cd0b356463ba941ba11e0e01f9cf5271380b79dc" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "socket2", + "winapi 0.3.9", +] + +[[package]] +name = "dtoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "encoding_rs" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a51b8cf747471cb9499b6d59e59b0444f4c90eba8968c4e44874e92b5b64ace2" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "error-chain" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" +dependencies = [ + "version_check", +] + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", + "synstructure", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "filetime" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed85775dcc68644b5c950ac06a2b23768d3bc9390464151aaf27136998dcf9e" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "redox_syscall", + "winapi 0.3.9", +] + +[[package]] +name = "flate2" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da80be589a72651dcda34d8b35bcdc9b7254ad06325611074d9cc0fbb19f60ee" +dependencies = [ + "cfg-if 0.1.10", + "crc32fast", + "libc", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7e4c2612746b0df8fed4ce0c69156021b704c9aefa360311c04e6e9e002eed" + +[[package]] +name = "futures" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95314d38584ffbfda215621d723e0a3906f032e03ae5551e650058dac83d4797" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0448174b01148032eed37ac4aed28963aaaa8cfa93569a08e5b479bbc6c2c151" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18eaa56102984bed2c88ea39026cff3ce3b4c7f508ca970cedf2450ea10d4e46" + +[[package]] +name = "futures-cpupool" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" +dependencies = [ + "futures 0.1.30", + "num_cpus", +] + +[[package]] +name = "futures-executor" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5f8e0c9258abaea85e78ebdda17ef9666d390e987f006be6080dfe354b708cb" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1798854a4727ff944a7b12aa999f58ce7aa81db80d2dfaaf2ba06f065ddd2b" + +[[package]] +name = "futures-macro" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36fccf3fc58563b4a14d265027c627c3b665d7fed489427e88e7cc929559efe" +dependencies = [ + "proc-macro-hack", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", +] + +[[package]] +name = "futures-sink" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e3ca3f17d6e8804ae5d3df7a7d35b2b3a6fe89dac84b31872720fc3060a0b11" + +[[package]] +name = "futures-task" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d502af37186c4fef99453df03e374683f8a1eec9dcc1e66b3b82dc8278ce3c" +dependencies = [ + "once_cell", +] + +[[package]] +name = "futures-util" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abcb44342f62e6f3e8ac427b8aa815f724fd705dfad060b18ac7866c15bb8e34" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project 1.0.1", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "gcc" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getset" +version = "0.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19fbde0fad0c1c1f9474694b1f5c9ba22b09f2f74f74e6d2bd19c43f6656e2cb" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + +[[package]] +name = "gimli" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" + +[[package]] +name = "git2" +version = "0.13.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6f1a0238d7f8f8fd5ee642f4ebac4dbc03e03d1f78fbe7a3ede35dcf7e2224" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url 2.1.1", +] + +[[package]] +name = "globset" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c152169ef1e421390738366d2f796655fec62621dabbd0fd476f905934061e4a" +dependencies = [ + "aho-corasick", + "bstr", + "fnv", + "log", + "regex", +] + +[[package]] +name = "globwalk" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "178270263374052c40502e9f607134947de75302c1348d1a0e31db67c1691446" +dependencies = [ + "bitflags", + "ignore", + "walkdir", +] + +[[package]] +name = "h2" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462" +dependencies = [ + "byteorder", + "bytes 0.4.12", + "fnv", + "futures 0.1.30", + "http 0.1.21", + "indexmap", + "log", + "slab", + "string", + "tokio-io", +] + +[[package]] +name = "h2" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.1", + "indexmap", + "slab", + "tokio 0.2.22", + "tokio-util", + "tracing", + "tracing-futures", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "hermit-abi" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +dependencies = [ + "libc", +] + +[[package]] +name = "hex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" + +[[package]] +name = "hmac" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" +dependencies = [ + "crypto-mac", + "digest 0.9.0", +] + +[[package]] +name = "http" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ccf5ede3a895d8856620237b2f02972c1bbc78d2965ad7fe8838d4a0ed41f0" +dependencies = [ + "bytes 0.4.12", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" +dependencies = [ + "bytes 0.5.6", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.30", + "http 0.1.21", + "tokio-buf", +] + +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes 0.5.6", + "http 0.2.1", +] + +[[package]] +name = "httparse" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" + +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "humansize" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6cab2627acfc432780848602f3f558f7e9dd427352224b0d9324025796d2a5e" + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "hyper" +version = "0.12.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dbe6ed1438e1f8ad955a4701e9a944938e9519f6888d12d8558b645e247d5f6" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.30", + "futures-cpupool", + "h2 0.1.26", + "http 0.1.21", + "http-body 0.1.0", + "httparse", + "iovec", + "itoa", + "log", + "net2", + "rustc_version", + "time 0.1.44", + "tokio 0.1.22", + "tokio-buf", + "tokio-executor", + "tokio-io", + "tokio-reactor", + "tokio-tcp", + "tokio-threadpool", + "tokio-timer", + "want 0.2.0", +] + +[[package]] +name = "hyper" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3afcfae8af5ad0576a31e768415edb627824129e8e5a29b8bfccb2f234e835" +dependencies = [ + "bytes 0.5.6", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.2.7", + "http 0.2.1", + "http-body 0.3.1", + "httparse", + "httpdate", + "itoa", + "pin-project 0.4.27", + "socket2", + "tokio 0.2.22", + "tower-service", + "tracing", + "want 0.3.0", +] + +[[package]] +name = "hyper-tls" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.30", + "hyper 0.12.35", + "native-tls", + "tokio-io", +] + +[[package]] +name = "hyper-tls" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" +dependencies = [ + "bytes 0.5.6", + "hyper 0.13.8", + "native-tls", + "tokio 0.2.22", + "tokio-tls", +] + +[[package]] +name = "idna" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "idna" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "ignore" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22dcbf2a4a289528dbef21686354904e1c694ac642610a9bff9e7df730d9ec72" +dependencies = [ + "crossbeam-utils", + "globset", + "lazy_static", + "log", + "memchr", + "regex", + "same-file", + "thread_local", + "walkdir", + "winapi-util", +] + +[[package]] +name = "indexmap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" +dependencies = [ + "autocfg 1.0.1", + "hashbrown", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" + +[[package]] +name = "jobserver" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2" +dependencies = [ + "libc", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743" + +[[package]] +name = "libgit2-sys" +version = "0.12.14+1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f25af58e6495f7caf2919d08f212de550cfa3ed2f5e744988938ea292b9f549" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + +[[package]] +name = "libssh2-sys" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca46220853ba1c512fc82826d0834d87b06bcd3c2a42241b7de72f3d2fe17056" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + +[[package]] +name = "memchr" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" + +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg 1.0.1", +] + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +dependencies = [ + "adler", + "autocfg 1.0.1", +] + +[[package]] +name = "mio" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow 0.2.1", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "mio-named-pipes" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" +dependencies = [ + "log", + "mio", + "miow 0.3.5", + "winapi 0.3.9", +] + +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio", +] + +[[package]] +name = "miow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "miow" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07b88fb9795d4d36d62a012dfbf49a8f5cf12751f36d31a9dbe66d528e58979e" +dependencies = [ + "socket2", + "winapi 0.3.9", +] + +[[package]] +name = "native-tls" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b0d88c06fe90d5ee94048ba40409ef1d9315d86f6f38c2efdaad4fb50c58b2d" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "net2" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "num-integer" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" +dependencies = [ + "autocfg 1.0.1", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" +dependencies = [ + "autocfg 1.0.1", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37fd5004feb2ce328a52b0b3d01dbf4ffff72583493900ed15f22d4111c51693" + +[[package]] +name = "once_cell" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openssl" +version = "0.10.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4" +dependencies = [ + "bitflags", + "cfg-if 0.1.10", + "foreign-types", + "lazy_static", + "libc", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "openssl-sys" +version = "0.9.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" +dependencies = [ + "autocfg 1.0.1", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api", + "parking_lot_core", + "rustc_version", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall", + "rustc_version", + "smallvec", + "winapi 0.3.9", +] + +[[package]] +name = "parse-zoneinfo" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" +dependencies = [ + "regex", +] + +[[package]] +name = "percent-encoding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", +] + +[[package]] +name = "pest_meta" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" +dependencies = [ + "maplit", + "pest", + "sha-1", +] + +[[package]] +name = "pin-project" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" +dependencies = [ + "pin-project-internal 0.4.27", +] + +[[package]] +name = "pin-project" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee41d838744f60d959d7074e3afb6b35c7456d0f61cad38a24e35e6553f73841" +dependencies = [ + "pin-project-internal 1.0.1", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81a4ffa594b66bff340084d4081df649a7dc049ac8d7fc458d8e628bfbbb2f86" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "ppv-lite86" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" + +[[package]] +name = "proc-macro-hack" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598" + +[[package]] +name = "proc-macro-nested" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" + +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid 0.1.0", +] + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid 0.2.1", +] + +[[package]] +name = "publicsuffix" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bbaa49075179162b49acac1c6aa45fb4dafb5f13cf6794276d77bc7fd95757b" +dependencies = [ + "error-chain", + "idna 0.2.0", + "lazy_static", + "regex", + "url 2.1.1", +] + +[[package]] +name = "qovery-engine" +version = "0.1.0" +dependencies = [ + "base64 0.12.3", + "chrono", + "cmd_lib", + "digitalocean", + "dirs 3.0.1", + "dns-lookup", + "env_logger", + "flate2", + "git2", + "itertools", + "log", + "rand 0.7.3", + "retry", + "rusoto_core", + "rusoto_credential", + "rusoto_dynamodb", + "rusoto_ecr", + "rusoto_eks", + "rusoto_s3", + "rusoto_sts", + "rust-crypto", + "serde", + "serde_derive", + "serde_json", + "tar", + "tera", + "test-utilities", + "tokio 0.2.22", + "walkdir", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2 1.0.24", +] + +[[package]] +name = "rand" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c" +dependencies = [ + "libc", + "rand 0.4.6", +] + +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi 0.3.9", +] + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi 0.3.9", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi 0.3.9", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi 0.3.9", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_users" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" +dependencies = [ + "getrandom", + "redox_syscall", + "rust-argon2", +] + +[[package]] +name = "regex" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "reqwest" +version = "0.9.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f88643aea3c1343c804950d7bf983bd2067f5ab59db6d613a08e05572f2714ab" +dependencies = [ + "base64 0.10.1", + "bytes 0.4.12", + "cookie", + "cookie_store", + "encoding_rs", + "flate2", + "futures 0.1.30", + "http 0.1.21", + "hyper 0.12.35", + "hyper-tls 0.3.2", + "log", + "mime", + "mime_guess", + "native-tls", + "serde", + "serde_json", + "serde_urlencoded 0.5.5", + "time 0.1.44", + "tokio 0.1.22", + "tokio-executor", + "tokio-io", + "tokio-threadpool", + "tokio-timer", + "url 1.7.2", + "uuid", + "winreg", +] + +[[package]] +name = "retry" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddadf3a46af916aa0fe6b8283e676b6bb5cbdf835d986d98a49d7345072341e5" +dependencies = [ + "rand 0.7.3", +] + +[[package]] +name = "rusoto_core" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e977941ee0658df96fca7291ecc6fc9a754600b21ad84b959eb1dbbc9d5abcc7" +dependencies = [ + "async-trait", + "base64 0.12.3", + "bytes 0.5.6", + "crc32fast", + "futures 0.3.7", + "http 0.2.1", + "hyper 0.13.8", + "hyper-tls 0.4.3", + "lazy_static", + "log", + "md5", + "percent-encoding 2.1.0", + "pin-project 0.4.27", + "rusoto_credential", + "rusoto_signature", + "rustc_version", + "serde", + "serde_json", + "tokio 0.2.22", + "xml-rs", +] + +[[package]] +name = "rusoto_credential" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac05563f83489b19b4d413607a30821ab08bbd9007d14fa05618da3ef09d8b" +dependencies = [ + "async-trait", + "chrono", + "dirs 2.0.2", + "futures 0.3.7", + "hyper 0.13.8", + "pin-project 0.4.27", + "regex", + "serde", + "serde_json", + "shlex", + "tokio 0.2.22", + "zeroize", +] + +[[package]] +name = "rusoto_dynamodb" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a1473bb1c1dd54f61c5e150aec47bcbf4a992963dcc3c60e12be5af3245cefc" +dependencies = [ + "async-trait", + "bytes 0.5.6", + "futures 0.3.7", + "rusoto_core", + "serde", + "serde_json", +] + +[[package]] +name = "rusoto_ecr" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76c8b6ce0d39d5dc93582b5c318698f89787e6966c7a6dbdbe733d38d30f792d" +dependencies = [ + "async-trait", + "bytes 0.5.6", + "futures 0.3.7", + "rusoto_core", + "serde", + "serde_json", +] + +[[package]] +name = "rusoto_eks" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a410b2b76a15c226a81b819d5189047be60d1db39cd46d7801942c5f0d8fef5" +dependencies = [ + "async-trait", + "bytes 0.5.6", + "futures 0.3.7", + "rusoto_core", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "rusoto_s3" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1146e37a7c1df56471ea67825fe09bbbd37984b5f6e201d8b2e0be4ee15643d8" +dependencies = [ + "async-trait", + "bytes 0.5.6", + "futures 0.3.7", + "rusoto_core", + "xml-rs", +] + +[[package]] +name = "rusoto_signature" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a740a88dde8ded81b6f2cff9cd5e054a5a2e38a38397260f7acdd2c85d17dd" +dependencies = [ + "base64 0.12.3", + "bytes 0.5.6", + "futures 0.3.7", + "hex", + "hmac", + "http 0.2.1", + "hyper 0.13.8", + "log", + "md5", + "percent-encoding 2.1.0", + "pin-project 0.4.27", + "rusoto_credential", + "rustc_version", + "serde", + "sha2", + "time 0.2.22", + "tokio 0.2.22", +] + +[[package]] +name = "rusoto_sts" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3815b8c0fc1c50caf9e87603f23daadfedb18d854de287b361c69f68dc9d49e0" +dependencies = [ + "async-trait", + "bytes 0.5.6", + "chrono", + "futures 0.3.7", + "rusoto_core", + "serde_urlencoded 0.6.1", + "tempfile", + "xml-rs", +] + +[[package]] +name = "rust-argon2" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dab61250775933275e84053ac235621dfb739556d5c54a2f2e9313b7cf43a19" +dependencies = [ + "base64 0.12.3", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils", +] + +[[package]] +name = "rust-crypto" +version = "0.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a" +dependencies = [ + "gcc", + "libc", + "rand 0.3.23", + "rustc-serialize", + "time 0.1.44", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" + +[[package]] +name = "rustc-serialize" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi 0.3.9", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "security-framework" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64808902d7d99f78eaddd2b4e2509713babc3dc3c85ad6f4c447680f3c01e535" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17bf11d99252f512695eb468de5516e5cf75455521e69dfe343f3b74e4748405" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", +] + +[[package]] +name = "serde_json" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "642dd69105886af2efd227f75a520ec9b44a820d65bc133a9131f7d229fd165a" +dependencies = [ + "dtoa", + "itoa", + "serde", + "url 1.7.2", +] + +[[package]] +name = "serde_urlencoded" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" +dependencies = [ + "dtoa", + "itoa", + "serde", + "url 2.1.1", +] + +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer 0.7.3", + "digest 0.8.1", + "fake-simd", + "opaque-debug 0.2.3", +] + +[[package]] +name = "sha1" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" + +[[package]] +name = "sha2" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 0.1.10", + "cpuid-bool", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + +[[package]] +name = "shlex" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" + +[[package]] +name = "signal-hook-registry" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e12110bc539e657a646068aaf5eb5b63af9d0c1f7b29c97113fad80e15f035" +dependencies = [ + "arc-swap", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "slug" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3bc762e6a4b6c6fcaade73e77f9ebc6991b676f88bb2358bddb56560f073373" +dependencies = [ + "deunicode", +] + +[[package]] +name = "smallvec" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" +dependencies = [ + "maybe-uninit", +] + +[[package]] +name = "socket2" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "redox_syscall", + "winapi 0.3.9", +] + +[[package]] +name = "standback" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4e0831040d2cf2bdfd51b844be71885783d489898a192f254ae25d57cce725c" +dependencies = [ + "version_check", +] + +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.7", + "serde", + "serde_derive", + "syn 1.0.48", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2 1.0.24", + "quote 1.0.7", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn 1.0.48", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + +[[package]] +name = "string" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d" +dependencies = [ + "bytes 0.4.12", +] + +[[package]] +name = "subtle" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343f3f510c2915908f155e94f17220b19ccfacf2a64a2a5d8004f2c3e311e7fd" + +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", +] + +[[package]] +name = "syn" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.7", + "unicode-xid 0.2.1", +] + +[[package]] +name = "synstructure" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", + "unicode-xid 0.2.1", +] + +[[package]] +name = "tar" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489997b7557e9a43e192c527face4feacc78bfbe6eed67fd55c4c9e381cba290" +dependencies = [ + "filetime", + "libc", + "redox_syscall", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "rand 0.7.3", + "redox_syscall", + "remove_dir_all", + "winapi 0.3.9", +] + +[[package]] +name = "tera" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1381c83828bedd5ce4e59473110afa5381ffe523406d9ade4b77c9f7be70ff9a" +dependencies = [ + "chrono", + "chrono-tz", + "globwalk", + "humansize", + "lazy_static", + "percent-encoding 2.1.0", + "pest", + "pest_derive", + "rand 0.7.3", + "regex", + "serde", + "serde_json", + "slug", + "unic-segment", +] + +[[package]] +name = "termcolor" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "test-utilities" +version = "0.1.0" +dependencies = [ + "chrono", + "curl", + "digitalocean", + "dirs 3.0.1", + "env_logger", + "qovery-engine", + "rand 0.7.3", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "thiserror" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "318234ffa22e0920fe9a40d7b8369b5f649d490980cf7aadcf1eb91594869b42" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae2447b6282786c3493999f40a9be2a6ad20cb8bd268b0a0dbf5a065535c0ab" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", +] + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi 0.3.9", +] + +[[package]] +name = "time" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55b7151c9065e80917fbf285d9a5d1432f60db41d170ccafc749a136b41a93af" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros", + "version_check", + "winapi 0.3.9", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" +dependencies = [ + "proc-macro-hack", + "proc-macro2 1.0.24", + "quote 1.0.7", + "standback", + "syn 1.0.48", +] + +[[package]] +name = "tinyvec" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117" + +[[package]] +name = "tokio" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.30", + "mio", + "num_cpus", + "tokio-current-thread", + "tokio-executor", + "tokio-io", + "tokio-reactor", + "tokio-tcp", + "tokio-threadpool", + "tokio-timer", +] + +[[package]] +name = "tokio" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d34ca54d84bf2b5b4d7d31e901a8464f7b60ac145a284fba25ceb801f2ddccd" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "iovec", + "lazy_static", + "libc", + "memchr", + "mio", + "mio-named-pipes", + "mio-uds", + "pin-project-lite", + "signal-hook-registry", + "slab", + "tokio-macros", + "winapi 0.3.9", +] + +[[package]] +name = "tokio-buf" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46" +dependencies = [ + "bytes 0.4.12", + "either", + "futures 0.1.30", +] + +[[package]] +name = "tokio-current-thread" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" +dependencies = [ + "futures 0.1.30", + "tokio-executor", +] + +[[package]] +name = "tokio-executor" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" +dependencies = [ + "crossbeam-utils", + "futures 0.1.30", +] + +[[package]] +name = "tokio-io" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.30", + "log", +] + +[[package]] +name = "tokio-macros" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" +dependencies = [ + "crossbeam-utils", + "futures 0.1.30", + "lazy_static", + "log", + "mio", + "num_cpus", + "parking_lot", + "slab", + "tokio-executor", + "tokio-io", + "tokio-sync", +] + +[[package]] +name = "tokio-sync" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" +dependencies = [ + "fnv", + "futures 0.1.30", +] + +[[package]] +name = "tokio-tcp" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.30", + "iovec", + "mio", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-threadpool" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" +dependencies = [ + "crossbeam-deque", + "crossbeam-queue", + "crossbeam-utils", + "futures 0.1.30", + "lazy_static", + "log", + "num_cpus", + "slab", + "tokio-executor", +] + +[[package]] +name = "tokio-timer" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" +dependencies = [ + "crossbeam-utils", + "futures 0.1.30", + "slab", + "tokio-executor", +] + +[[package]] +name = "tokio-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" +dependencies = [ + "native-tls", + "tokio 0.2.22", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio 0.2.22", +] + +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" + +[[package]] +name = "tracing" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27" +dependencies = [ + "cfg-if 0.1.10", + "log", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +dependencies = [ + "pin-project 0.4.27", + "tracing", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "try_from" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "283d3b89e1368717881a9d51dad843cc435380d8109c9e47d38780a324698d8b" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + +[[package]] +name = "unic-char-property" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" +dependencies = [ + "unic-char-range", +] + +[[package]] +name = "unic-char-range" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" + +[[package]] +name = "unic-common" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" + +[[package]] +name = "unic-segment" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ed5d26be57f84f176157270c112ef57b86debac9cd21daaabbe56db0f88f23" +dependencies = [ + "unic-ucd-segment", +] + +[[package]] +name = "unic-ucd-segment" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-version" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" +dependencies = [ + "unic-common", +] + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "url" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" +dependencies = [ + "idna 0.1.5", + "matches", + "percent-encoding 1.0.1", +] + +[[package]] +name = "url" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" +dependencies = [ + "idna 0.2.0", + "matches", + "percent-encoding 2.1.0", +] + +[[package]] +name = "url_serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74e7d099f1ee52f823d4bdd60c93c3602043c728f5db3b97bdb548467f7bddea" +dependencies = [ + "serde", + "url 1.7.2", +] + +[[package]] +name = "uuid" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90dbc611eb48397705a6b0f6e917da23ae517e4d127123d2cf7674206627d32a" +dependencies = [ + "rand 0.6.5", +] + +[[package]] +name = "vcpkg" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c" + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" + +[[package]] +name = "walkdir" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" +dependencies = [ + "same-file", + "winapi 0.3.9", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230" +dependencies = [ + "futures 0.1.30", + "log", + "try-lock", +] + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" +dependencies = [ + "cfg-if 0.1.10", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038" +dependencies = [ + "quote 1.0.7", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307" + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "xattr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" +dependencies = [ + "libc", +] + +[[package]] +name = "xml-rs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a" + +[[package]] +name = "zeroize" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f33972566adbd2d3588b0491eb94b98b43695c4ef897903470ede4f3f5a28a" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..3fd747ba --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,53 @@ +[package] +name = "qovery-engine" +version = "0.1.0" +authors = ["Romaric Philogene "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +chrono = "0.4.11" +cmd_lib = "0.7.8" +git2 = "0.13.8" +walkdir = "2.3.1" +itertools = "0.9.0" +base64 = "0.12.3" +dirs = "3.0.1" +rust-crypto = "0.2.36" +retry = "1.0.0" +dns-lookup = "1.0.3" +rand = "0.7.3" +# FIXME use https://crates.io/crates/blocking instead of runtime.rs + +# tar gz +flate2 = "1.0.16" # tar gz +tar = "0.4.29" + +# logger +env_logger = "0.7.1" +log = "0.4.8" + +# Docker deps +# shiplift = "0.6.0" + +# Jinja2 +tera = "1.3.1" +serde = "1.0.114" +serde_json = "1.0.57" +serde_derive = "1.0" +# AWS deps +tokio = "0.2.22" +rusoto_core = "0.45.0" +rusoto_sts = "0.45.0" +rusoto_credential = "0.45.0" +rusoto_ecr = "0.45.0" +rusoto_eks = "0.45.0" +rusoto_s3 = "0.45.0" +rusoto_dynamodb = "0.45.0" + +# Digital Ocean Deps +digitalocean = "0.1.1" + +[dev-dependencies] +test-utilities = { path = "test_utilities" } diff --git a/src/build_platform/error.rs b/src/build_platform/error.rs new file mode 100644 index 00000000..61f366e8 --- /dev/null +++ b/src/build_platform/error.rs @@ -0,0 +1,4 @@ +#[derive(Debug)] +pub enum BuildPlatformError { + Unexpected(String), +} diff --git a/src/build_platform/local_docker.rs b/src/build_platform/local_docker.rs new file mode 100644 index 00000000..e253b73b --- /dev/null +++ b/src/build_platform/local_docker.rs @@ -0,0 +1,254 @@ +use std::path::Path; +use std::rc::Rc; + +use crate::build_platform::error::BuildPlatformError; +use crate::build_platform::{Build, BuildError, BuildPlatform, BuildResult, Image, Kind}; +use crate::fs::workspace_directory; +use crate::git::checkout_submodules; +use crate::models::{ + Context, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressListener, + ProgressScope, +}; +use crate::transaction::CommitError::BuildImage; +use crate::{cmd, git}; + +/// use Docker in local +pub struct LocalDocker { + context: Context, + id: String, + name: String, + listeners: Listeners, +} + +impl LocalDocker { + pub fn new(context: Context, id: &str, name: &str) -> Self { + LocalDocker { + context, + id: id.to_string(), + name: name.to_string(), + listeners: vec![], + } + } + + fn image_does_exist(&self, image: &Image) -> Result { + let envs = match self.context.docker_tcp_socket() { + Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())], + None => vec![], + }; + + Ok( + match crate::cmd::utilities::exec_with_envs( + "docker", + vec!["image", "inspect", image.name_with_tag().as_str()], + envs, + ) { + Ok(_) => true, + _ => false, + }, + ) + } +} + +impl BuildPlatform for LocalDocker { + fn context(&self) -> &Context { + &self.context + } + + fn kind(&self) -> Kind { + Kind::LocalDocker + } + + fn id(&self) -> &str { + self.id.as_str() + } + + fn name(&self) -> &str { + self.name.as_str() + } + + fn is_valid(&self) -> Result<(), BuildPlatformError> { + if !crate::cmd::utilities::does_binary_exist("docker") { + return Err(BuildPlatformError::Unexpected( + "docker binary not found".to_string(), + )); + } + + Ok(()) + } + + fn add_listener(&mut self, listener: Rc>) { + self.listeners.push(listener); + } + + fn build(&self, build: Build, force_build: bool) -> Result { + info!("LocalDocker.build() called for {}", self.name()); + + let listeners_helper = ListenersHelper::new(&self.listeners); + + if !force_build && self.image_does_exist(&build.image)? { + info!( + "image {:?} does already exist - no need to build it", + build.image + ); + + return Ok(BuildResult { build }); + } + + // git clone + let into_dir = workspace_directory( + self.context.workspace_root_dir(), + self.context.execution_id(), + format!("build/{}", build.image.name.as_str()), + ); + + info!("cloning repository: {}", build.git_repository.url); + let git_clone = git::clone( + build.git_repository.url.as_str(), + &into_dir, + &build.git_repository.credentials, + ); + + match git_clone { + Ok(_) => {} + Err(err) => { + error! {"Error while trying to clone repository {}", build.git_repository.url} + return Err(BuildError::Git(err)); + } + } + + // git checkout to given commit + let repo = &git_clone.unwrap(); + let commit_id = &build.git_repository.commit_id; + match git::checkout(&repo, &commit_id, build.git_repository.url.as_str()) { + Ok(_) => {} + Err(err) => return Err(BuildError::Git(err)), + } + + // git checkout submodules + let _ = checkout_submodules(&repo); + // TODO what if we can't checkout submodules? Today we ignore it + + let into_dir_docker_style = format!("{}/.", into_dir.as_str()); + + let dockerfile_relative_path = match build.git_repository.dockerfile_path.trim() { + "" | "." | "/" | "/." | "./" | "Dockerfile" => "Dockerfile", + dockerfile_root_path => dockerfile_root_path, + }; + + let dockerfile_complete_path = + format!("{}/{}", into_dir.as_str(), dockerfile_relative_path); + + match Path::new(dockerfile_complete_path.as_str()).exists() { + false => { + error!( + "Unable to find Dockerfile path {}", + dockerfile_complete_path.as_str() + ); + return Err(BuildError::Error); + } + _ => {} + } + + let env_var_args = &build + .options + .environment_variables + .iter() + .map(|ev| format!("'{}={}'", ev.key, ev.value)) + .collect::>(); + + let name_with_tag = build.image.name_with_tag(); + let mut docker_args = vec![ + "build", + "-f", + dockerfile_complete_path.as_str(), + "-t", + name_with_tag.as_str(), + ]; + + let mut docker_args = if env_var_args.is_empty() { + docker_args + } else { + let mut build_args = vec![]; + env_var_args.iter().for_each(|x| { + build_args.push("--build-arg"); + build_args.push(x.as_str()); + }); + + docker_args.extend(build_args); + docker_args + }; + + docker_args.push(into_dir_docker_style.as_str()); + + let envs = match self.context.docker_tcp_socket() { + Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())], + None => vec![], + }; + + // docker build + let exit_status = cmd::utilities::exec_with_envs_and_output( + "docker", + docker_args, + envs, + |line| { + let line_string = line.unwrap(); + info!("{}", line_string.as_str()); + + listeners_helper.start_in_progress(ProgressInfo::new( + ProgressScope::Application { + id: build.image.application_id.clone(), + }, + ProgressLevel::Info, + Some(line_string.as_str()), + self.context.execution_id(), + )); + }, + |line| { + let line_string = line.unwrap(); + error!("{}", line_string.as_str()); + + listeners_helper.error(ProgressInfo::new( + ProgressScope::Application { + id: build.image.application_id.clone(), + }, + ProgressLevel::Error, + Some(line_string.as_str()), + self.context.execution_id(), + )); + }, + ); + + match exit_status { + Ok(_) => {} + Err(_) => return Err(BuildError::Error), + } + + listeners_helper.start_in_progress(ProgressInfo::new( + ProgressScope::Application { + id: build.image.application_id.clone(), + }, + ProgressLevel::Info, + Some("build is done ✔"), + self.context.execution_id(), + )); + + Ok(BuildResult { build }) + } + + fn build_error(&self, build: Build) -> Result { + warn!("LocalDocker.build_error() called for {}", self.name()); + + let listener_helper = ListenersHelper::new(&self.listeners); + listener_helper.error(ProgressInfo::new( + ProgressScope::Application { + id: build.image.application_id, + }, + ProgressLevel::Error, + Some("something goes wrong (not implemented)"), + self.context.execution_id(), + )); + + // FIXME + Err(BuildError::Error) + } +} diff --git a/src/build_platform/mod.rs b/src/build_platform/mod.rs new file mode 100644 index 00000000..8686fb99 --- /dev/null +++ b/src/build_platform/mod.rs @@ -0,0 +1,75 @@ +use std::rc::Rc; + +use git2::Error; +use serde::{Deserialize, Serialize}; + +use crate::build_platform::error::BuildPlatformError; +use crate::git::Credentials; +use crate::models::{Context, ProgressListener}; + +pub mod error; +pub mod local_docker; + +pub trait BuildPlatform { + fn context(&self) -> &Context; + fn kind(&self) -> Kind; + fn id(&self) -> &str; + fn name(&self) -> &str; + fn is_valid(&self) -> Result<(), BuildPlatformError>; + fn add_listener(&mut self, listener: Rc>); + fn build(&self, build: Build, force_build: bool) -> Result; + fn build_error(&self, build: Build) -> Result; +} + +pub struct Build { + pub git_repository: GitRepository, + pub image: Image, + pub options: BuildOptions, +} + +pub struct BuildOptions { + pub environment_variables: Vec, +} + +pub struct EnvironmentVariable { + pub key: String, + pub value: String, +} + +pub struct GitRepository { + pub url: String, + pub credentials: Option, + pub commit_id: String, + pub dockerfile_path: String, +} + +#[derive(Clone, Eq, PartialEq, Hash, Debug)] +pub struct Image { + pub application_id: String, + pub name: String, + pub tag: String, + pub commit_id: String, + pub registry_url: Option, +} + +impl Image { + pub fn name_with_tag(&self) -> String { + format!("{}:{}", self.name, self.tag) + } +} + +pub struct BuildResult { + pub build: Build, +} + +#[derive(Debug)] +pub enum BuildError { + Git(Error), + Error, +} + +#[derive(Serialize, Deserialize, Clone)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum Kind { + LocalDocker, +} diff --git a/src/cloud_provider/aws/application.rs b/src/cloud_provider/aws/application.rs new file mode 100644 index 00000000..71c99cef --- /dev/null +++ b/src/cloud_provider/aws/application.rs @@ -0,0 +1,419 @@ +use serde::{Deserialize, Serialize}; +use tera::Context as TeraContext; + +use crate::build_platform::Image; +use crate::cloud_provider::aws::{common, AWS}; +use crate::cloud_provider::environment::Environment; +use crate::cloud_provider::kubernetes::Kubernetes; +use crate::cloud_provider::service::{ + Action, Application as CApplication, Create, Delete, Pause, Service, ServiceError, ServiceType, + StatelessService, +}; +use crate::cloud_provider::DeploymentTarget; +use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY}; +use crate::models::Context; + +#[derive(Clone, Eq, PartialEq, Hash)] +pub struct Application { + context: Context, + id: String, + action: Action, + name: String, + private_port: Option, + total_cpus: String, + cpu_burst: String, + total_ram_in_mib: u32, + total_instances: u16, + image: Image, + storage: Vec, + environment_variables: Vec, +} + +impl Application { + pub fn new( + context: Context, + id: &str, + action: Action, + name: &str, + private_port: Option, + total_cpus: String, + cpu_burst: String, + total_ram_in_mib: u32, + total_instances: u16, + image: Image, + storage: Vec, + environment_variables: Vec, + ) -> Self { + Application { + context, + id: id.to_string(), + action, + name: name.to_string(), + private_port, + total_cpus, + cpu_burst, + total_ram_in_mib, + total_instances, + image, + storage, + environment_variables, + } + } + + fn helm_release_name(&self) -> String { + crate::string::cut(format!("application-{}-{}", self.name(), self.id()), 50) + } + + fn workspace_directory(&self) -> String { + crate::fs::workspace_directory( + self.context.workspace_root_dir(), + self.context.execution_id(), + format!("applications/{}", self.name()), + ) + } + + fn context(&self, kubernetes: &dyn Kubernetes, environment: &Environment) -> TeraContext { + let mut context = self.default_tera_context(kubernetes, environment); + let commit_id = self.image().commit_id.as_str(); + + context.insert("helm_app_version", &commit_id[..7]); + + match &self.image().registry_url { + Some(registry_url) => context.insert("image_name_with_tag", registry_url.as_str()), + None => { + let image_name_with_tag = self.image().name_with_tag(); + warn!("there is no registry url, use image name with tag with the default container registry: {}", image_name_with_tag.as_str()); + context.insert("image_name_with_tag", image_name_with_tag.as_str()); + } + } + + let environment_variables = self + .environment_variables + .iter() + .map(|ev| EnvironmentVariableDataTemplate { + key: ev.key.clone(), + value: ev.value.clone(), + }) + .collect::>(); + + context.insert("environment_variables", &environment_variables); + + let storage = self + .storage + .iter() + .map(|s| StorageDataTemplate { + id: s.id.clone(), + name: s.name.clone(), + storage_type: match s.storage_type { + StorageType::SC1 => "sc1", + StorageType::ST1 => "st1", + StorageType::GP2 => "gp2", + StorageType::IO1 => "io1", + } + .to_string(), + size_in_gib: s.size_in_gib, + mount_point: s.mount_point.clone(), + snapshot_retention_in_days: s.snapshot_retention_in_days, + }) + .collect::>(); + + let is_storage = storage.len() > 0; + + context.insert("storage", &storage); + context.insert("is_storage", &is_storage); + context.insert("clone", &false); + + context + } + + fn delete(&self, target: &DeploymentTarget, is_error: bool) -> Result<(), ServiceError> { + let (kubernetes, environment) = match target { + DeploymentTarget::ManagedServices(k, env) => (*k, *env), + DeploymentTarget::SelfHosted(k, env) => (*k, *env), + }; + + let workspace_dir = self.workspace_directory(); + let helm_release_name = self.helm_release_name(); + let selector = format!("app={}", self.name()); + + if is_error { + let _ = common::get_stateless_resource_information( + kubernetes, + environment, + workspace_dir.as_str(), + selector.as_str(), + )?; + } + + // clean the resource + let _ = common::do_stateless_service_cleanup( + kubernetes, + environment, + workspace_dir.as_str(), + helm_release_name.as_str(), + )?; + + Ok(()) + } +} + +impl crate::cloud_provider::service::Application for Application { + fn image(&self) -> &Image { + &self.image + } + + fn set_image(&mut self, image: Image) { + self.image = image; + } +} + +impl StatelessService for Application {} + +impl Service for Application { + fn context(&self) -> &Context { + &self.context + } + + fn service_type(&self) -> ServiceType { + ServiceType::Application + } + + fn id(&self) -> &str { + self.id.as_str() + } + + fn name(&self) -> &str { + self.name.as_str() + } + + fn version(&self) -> &str { + self.image.commit_id.as_str() + } + + fn action(&self) -> &Action { + &self.action + } + + fn private_port(&self) -> Option { + self.private_port + } + + fn total_cpus(&self) -> String { + self.total_cpus.to_string() + } + + fn total_ram_in_mib(&self) -> u32 { + self.total_ram_in_mib + } + + fn total_instances(&self) -> u16 { + self.total_instances + } +} + +impl Create for Application { + fn on_create(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + info!("AWS.application.on_create() called for {}", self.name()); + let (kubernetes, environment) = match target { + DeploymentTarget::ManagedServices(k, env) => (*k, *env), + DeploymentTarget::SelfHosted(k, env) => (*k, *env), + }; + + let aws = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + + let context = self.context(kubernetes, environment); + let workspace_dir = self.workspace_directory(); + + let from_dir = format!("{}/aws/charts/q-application", self.context.lib_root_dir()); + let _ = crate::template::generate_and_copy_all_files_into_dir( + from_dir.as_str(), + workspace_dir.as_str(), + &context, + )?; + + // render + // TODO check the rendered files? + let helm_release_name = self.helm_release_name(); + let aws_credentials_envs = vec![ + (AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()), + (AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()), + ]; + + let kubernetes_config_file_path = common::kubernetes_config_path( + workspace_dir.as_str(), + environment.organization_id.as_str(), + kubernetes.id(), + aws.access_key_id.as_str(), + aws.secret_access_key.as_str(), + kubernetes.region(), + )?; + + // do exec helm upgrade and return the last deployment status + let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history( + kubernetes_config_file_path.as_str(), + environment.namespace(), + helm_release_name.as_str(), + workspace_dir.as_str(), + aws_credentials_envs.clone(), + )?; + + // check deployment status + if helm_history_row.is_none() || !helm_history_row.unwrap().is_successfully_deployed() { + // TODO get pod output by using kubectl and return it into the OnCreateFailed + return Err(ServiceError::OnCreateFailed); + } + + // check app status + let selector = format!("app={}", self.name()); + + match crate::cmd::kubectl::kubectl_exec_is_pod_ready_with_retry( + kubernetes_config_file_path.as_str(), + environment.namespace(), + selector.as_str(), + aws_credentials_envs, + ) { + Ok(Some(true)) => {} + _ => return Err(ServiceError::OnCreateFailed), + } + + Ok(()) + } + + fn on_create_check(&self) -> Result<(), ServiceError> { + Ok(()) + } + + fn on_create_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!( + "AWS.application.on_create_error() called for {}", + self.name() + ); + + let (kubernetes, environment) = match target { + DeploymentTarget::ManagedServices(k, env) => (*k, *env), + DeploymentTarget::SelfHosted(k, env) => (*k, *env), + }; + + let workspace_dir = self.workspace_directory(); + + let aws = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + + let aws_credentials_envs = vec![ + (AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()), + (AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()), + ]; + + let kubernetes_config_file_path = common::kubernetes_config_path( + workspace_dir.as_str(), + environment.organization_id.as_str(), + kubernetes.id(), + aws.access_key_id.as_str(), + aws.secret_access_key.as_str(), + kubernetes.region(), + )?; + + let helm_release_name = self.helm_release_name(); + + let history_rows = crate::cmd::helm::helm_exec_history( + kubernetes_config_file_path.as_str(), + environment.namespace(), + helm_release_name.as_str(), + aws_credentials_envs.clone(), + )?; + if history_rows.len() == 1 { + crate::cmd::helm::helm_exec_uninstall( + kubernetes_config_file_path.as_str(), + environment.namespace(), + helm_release_name.as_str(), + aws_credentials_envs.clone(), + )?; + } + Ok(()) + } +} + +impl Pause for Application { + fn on_pause(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + info!("AWS.application.on_pause() called for {}", self.name()); + self.delete(target, false) + } + + fn on_pause_check(&self) -> Result<(), ServiceError> { + Ok(()) + } + + fn on_pause_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!( + "AWS.application.on_pause_error() called for {}", + self.name() + ); + self.delete(target, true) + } +} + +impl Delete for Application { + fn on_delete(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + info!("AWS.application.on_delete() called for {}", self.name()); + self.delete(target, false) + } + + fn on_delete_check(&self) -> Result<(), ServiceError> { + Ok(()) + } + + fn on_delete_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!( + "AWS.application.on_delete_error() called for {}", + self.name() + ); + self.delete(target, true) + } +} + +#[derive(Clone, Eq, PartialEq, Hash)] +pub struct EnvironmentVariable { + pub key: String, + pub value: String, +} + +#[derive(Serialize, Deserialize)] +struct EnvironmentVariableDataTemplate { + pub key: String, + pub value: String, +} + +#[derive(Clone, Eq, PartialEq, Hash)] +pub struct Storage { + pub id: String, + pub name: String, + pub storage_type: StorageType, + pub size_in_gib: u16, + pub mount_point: String, + pub snapshot_retention_in_days: u16, +} + +#[derive(Clone, Eq, PartialEq, Hash)] +pub enum StorageType { + SC1, + ST1, + GP2, + IO1, +} + +#[derive(Serialize, Deserialize)] +struct StorageDataTemplate { + pub id: String, + pub name: String, + pub storage_type: String, + pub size_in_gib: u16, + pub mount_point: String, + pub snapshot_retention_in_days: u16, +} diff --git a/src/cloud_provider/aws/common.rs b/src/cloud_provider/aws/common.rs new file mode 100644 index 00000000..59fc6a93 --- /dev/null +++ b/src/cloud_provider/aws/common.rs @@ -0,0 +1,156 @@ +use std::io::Error; +use std::str::FromStr; + +use rusoto_core::Region; + +use crate::cloud_provider::aws::AWS; +use crate::cloud_provider::environment::Environment; +use crate::cloud_provider::kubernetes::Kubernetes; +use crate::cloud_provider::service::ServiceError; +use crate::cmd::utilities::CmdError; +use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY}; + +pub fn kubernetes_config_path( + workspace_directory: &str, + organization_id: &str, + kubernetes_cluster_id: &str, + access_key_id: &str, + secret_access_key: &str, + region: &str, +) -> Result { + let kubernetes_config_bucket_name = format!("qovery-kubeconfigs-{}", kubernetes_cluster_id); + let kubernetes_config_object_key = format!("{}.yaml", kubernetes_cluster_id); + + let kubernetes_config_file_path = format!( + "{}/kubernetes_config_{}", + workspace_directory, kubernetes_cluster_id + ); + + let _region = Region::from_str(region).unwrap(); + + let _ = crate::s3::get_kubernetes_config_file( + access_key_id, + secret_access_key, + &_region, + kubernetes_config_bucket_name.as_str(), + kubernetes_config_object_key.as_str(), + kubernetes_config_file_path.as_str(), + )?; + + Ok(kubernetes_config_file_path) +} + +pub type Logs = String; +pub type Describe = String; + +/// show different output (kubectl describe, log..) for debug purpose +pub fn get_stateless_resource_information( + kubernetes: &dyn Kubernetes, + environment: &Environment, + workspace_dir: &str, + selector: &str, +) -> Result<(Describe, Logs), CmdError> { + let aws = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + + let kubernetes_config_file_path = kubernetes_config_path( + workspace_dir, + environment.organization_id.as_str(), + kubernetes.id(), + aws.access_key_id.as_str(), + aws.secret_access_key.as_str(), + kubernetes.region(), + )?; + + let aws_credentials_envs = vec![ + (AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()), + (AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()), + ]; + + // exec describe pod... + let describe = match crate::cmd::kubectl::kubectl_exec_describe_pod( + kubernetes_config_file_path.as_str(), + environment.namespace(), + selector, + aws_credentials_envs.clone(), + ) { + Ok(output) => { + info!("{}", output); + output + } + Err(err) => { + error!("{:?}", err); + return Err(err); + } + }; + + // exec logs... + let logs = match crate::cmd::kubectl::kubectl_exec_logs( + kubernetes_config_file_path.as_str(), + environment.namespace(), + selector, + aws_credentials_envs.clone(), + ) { + Ok(output) => { + info!("{}", output); + output + } + Err(err) => { + error!("{:?}", err); + return Err(err); + } + }; + + Ok((describe, logs)) +} + +pub fn do_stateless_service_cleanup( + kubernetes: &dyn Kubernetes, + environment: &Environment, + workspace_dir: &str, + helm_release_name: &str, +) -> Result<(), ServiceError> { + let aws = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + + let kubernetes_config_file_path = kubernetes_config_path( + workspace_dir, + environment.organization_id.as_str(), + kubernetes.id(), + aws.access_key_id.as_str(), + aws.secret_access_key.as_str(), + kubernetes.region(), + )?; + + let aws_credentials_envs = vec![ + (AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()), + (AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()), + ]; + + let history_rows = crate::cmd::helm::helm_exec_history( + kubernetes_config_file_path.as_str(), + environment.namespace(), + helm_release_name, + aws_credentials_envs.clone(), + )?; + + // if there is no valid history - then delete the helm chart + let first_valid_history_row = history_rows.iter().find(|x| x.is_successfully_deployed()); + + if first_valid_history_row.is_some() { + crate::cmd::helm::helm_exec_uninstall( + kubernetes_config_file_path.as_str(), + environment.namespace(), + helm_release_name, + aws_credentials_envs, + )?; + } + + Ok(()) +} diff --git a/src/cloud_provider/aws/databases/mod.rs b/src/cloud_provider/aws/databases/mod.rs new file mode 100644 index 00000000..647dc15a --- /dev/null +++ b/src/cloud_provider/aws/databases/mod.rs @@ -0,0 +1,8 @@ +pub use mongodb::MongoDB; +pub use mysql::MySQL; +pub use postgresql::PostgreSQL; + +mod mongodb; +mod mysql; +mod postgresql; +mod utilities; diff --git a/src/cloud_provider/aws/databases/mongodb.rs b/src/cloud_provider/aws/databases/mongodb.rs new file mode 100644 index 00000000..5512e71e --- /dev/null +++ b/src/cloud_provider/aws/databases/mongodb.rs @@ -0,0 +1,478 @@ +use tera::Context as TeraContext; + +use crate::cloud_provider::aws::databases::utilities; +use crate::cloud_provider::aws::{common, AWS}; +use crate::cloud_provider::environment::Environment; +use crate::cloud_provider::kubernetes::Kubernetes; +use crate::cloud_provider::service::{ + Action, Backup, Create, Database, DatabaseOptions, DatabaseType, Delete, Downgrade, Pause, + Service, ServiceError, ServiceType, StatefulService, Upgrade, +}; +use crate::cloud_provider::DeploymentTarget; +use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY}; +use crate::models::Context; + +pub struct MongoDB { + context: Context, + id: String, + action: Action, + name: String, + version: String, + fqdn: String, + fqdn_id: String, + total_cpus: String, + total_ram_in_mib: u32, + database_instance_type: String, + options: DatabaseOptions, +} + +impl MongoDB { + pub fn new( + context: Context, + id: &str, + action: Action, + name: &str, + version: &str, + fqdn: &str, + fqdn_id: &str, + total_cpus: String, + total_ram_in_mib: u32, + database_instance_type: &str, + options: DatabaseOptions, + ) -> Self { + MongoDB { + context, + action, + id: id.to_string(), + name: name.to_string(), + version: version.to_string(), + fqdn: fqdn.to_string(), + fqdn_id: fqdn_id.to_string(), + total_cpus, + total_ram_in_mib, + database_instance_type: database_instance_type.to_string(), + options, + } + } + + fn helm_release_name(&self) -> String { + crate::string::cut(format!("mongodb-{}", self.id()), 50) + } + + fn helm_release_external_dns(&self) -> String { + format!("{}-dns", self.helm_release_name()) + } + + fn workspace_directory(&self) -> String { + crate::fs::workspace_directory( + self.context.workspace_root_dir(), + self.context.execution_id(), + format!("databases/{}", self.name()), + ) + } + + fn tera_context(&self, kubernetes: &dyn Kubernetes, environment: &Environment) -> TeraContext { + let mut context = self.default_tera_context(kubernetes, environment); + // FIXME: is there an other way than downcast a pointer? + let cp = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .expect("Could not downcast kubernetes.cloud_provider() to AWS"); + // we need the kubernetes config file to store tfstates file in kube secrets + let kubernetes_config_file_path = utilities::get_kubernetes_config_path( + self.workspace_directory().as_str(), + kubernetes, + environment, + ); + match kubernetes_config_file_path { + Ok(kube_config) => { + context.insert("kubeconfig_path", &kube_config.as_str()); + let aws = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + + utilities::create_namespace(&environment.namespace(), kube_config.as_str(), aws); + } + Err(e) => error!("Failed to generate the kubernetes config file path: {}", e), + } + context.insert("namespace", environment.namespace()); + + context.insert("aws_access_key", &cp.access_key_id); + context.insert("aws_secret_key", &cp.secret_access_key); + context.insert("eks_cluster_id", kubernetes.id()); + context.insert("eks_cluster_name", kubernetes.name()); + + context.insert("fqdn_id", self.fqdn_id.as_str()); + context.insert("fqdn", self.fqdn.as_str()); + + context.insert("database_login", self.options.login.as_str()); + context.insert("database_password", self.options.password.as_str()); + context.insert("database_port", &self.private_port()); + context.insert("database_disk_size_in_gib", &self.options.disk_size_in_gib); + context.insert("database_instance_type", &self.database_instance_type); + context.insert("database_disk_type", &self.options.database_disk_type); + context.insert("database_ram_size_in_mib", &self.total_ram_in_mib); + context.insert("database_total_cpus", &self.total_cpus); + context.insert("database_fqdn", &self.options.host.as_str()); + context.insert("database_id", &self.id()); + + context + } + + fn delete(&self, target: &DeploymentTarget, is_error: bool) -> Result<(), ServiceError> { + let workspace_dir = self.workspace_directory(); + + match target { + DeploymentTarget::ManagedServices(kubernetes, environment) => { + if is_error { + // do not delete if it is an error + return Ok(()); + } + + let context = self.tera_context(*kubernetes, *environment); + + crate::template::generate_and_copy_all_files_into_dir( + format!("{}/aws/services/common", self.context.lib_root_dir()).as_str(), + &workspace_dir, + &context, + )?; + crate::template::generate_and_copy_all_files_into_dir( + format!("{}/aws/services/mongodb", self.context.lib_root_dir()).as_str(), + workspace_dir.as_str(), + &context, + )?; + crate::template::generate_and_copy_all_files_into_dir( + format!( + "{}/aws/charts/external-name-svc", + self.context.lib_root_dir() + ) + .as_str(), + format!("{}/{}", workspace_dir, "external-name-svc").as_str(), + &context, + )?; + crate::template::generate_and_copy_all_files_into_dir( + format!( + "{}/aws/charts/external-name-svc", + self.context.lib_root_dir() + ) + .as_str(), + workspace_dir.as_str(), + &context, + )?; + + match crate::cmd::terraform::terraform_exec_with_init_validate_destroy( + workspace_dir.as_str(), + ) { + Ok(o) => { + info!("Deleting secrets containing tfstates"); + utilities::delete_terraform_tfstate_secret( + *kubernetes, + environment, + self.workspace_directory().as_str(), + ); + } + //TODO: find a way to raise the error + Err(e) => error!("Error while destroying infrastructure {}", e), + } + } + DeploymentTarget::SelfHosted(kubernetes, environment) => { + let helm_release_name = self.helm_release_name(); + let selector = format!("app={}", self.name()); + + if is_error { + let _ = common::get_stateless_resource_information( + *kubernetes, + *environment, + workspace_dir.as_str(), + selector.as_str(), + )?; + } + + // clean the resource + let _ = common::do_stateless_service_cleanup( + *kubernetes, + *environment, + workspace_dir.as_str(), + helm_release_name.as_str(), + )?; + } + } + + Ok(()) + } +} + +impl StatefulService for MongoDB {} + +impl Service for MongoDB { + fn context(&self) -> &Context { + &self.context + } + + fn service_type(&self) -> ServiceType { + ServiceType::Database(DatabaseType::MongoDB(&self.options)) + } + + fn id(&self) -> &str { + self.id.as_str() + } + + fn name(&self) -> &str { + self.name.as_str() + } + + fn version(&self) -> &str { + self.version.as_str() + } + + fn action(&self) -> &Action { + &self.action + } + + fn private_port(&self) -> Option { + Some(self.options.port) + } + + fn total_cpus(&self) -> String { + self.total_cpus.to_string() + } + + fn total_ram_in_mib(&self) -> u32 { + self.total_ram_in_mib + } + + fn total_instances(&self) -> u16 { + 1 + } +} + +impl Database for MongoDB {} + +impl Create for MongoDB { + fn on_create(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + info!("AWS.MongoDB.on_create() called for {}", self.name()); + + let workspace_dir = self.workspace_directory(); + + match target { + DeploymentTarget::ManagedServices(kubernetes, environment) => { + // use terraform + info!("deploy mongodb on AWS DocumentDB for {}", self.name()); + let context = self.tera_context(*kubernetes, *environment); + let workspace_dir = self.workspace_directory(); + + crate::template::generate_and_copy_all_files_into_dir( + format!("{}/aws/services/common", self.context.lib_root_dir()).as_str(), + &workspace_dir, + &context, + )?; + crate::template::generate_and_copy_all_files_into_dir( + format!("{}/aws/services/mongodb", self.context.lib_root_dir()).as_str(), + workspace_dir.as_str(), + &context, + )?; + crate::template::generate_and_copy_all_files_into_dir( + format!( + "{}/aws/charts/external-name-svc", + self.context.lib_root_dir() + ) + .as_str(), + format!("{}/{}", workspace_dir, "external-name-svc").as_str(), + &context, + )?; + + // deploy database + external DNS + crate::cmd::terraform::terraform_exec_with_init_validate_plan_apply( + workspace_dir.as_str(), + false, + )?; + } + DeploymentTarget::SelfHosted(kubernetes, environment) => { + // use helm + info!("deploy MongoDB on Kubernetes for {}", self.name()); + let context = self.tera_context(*kubernetes, *environment); + + let aws = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + + let kubernetes_config_file_path = common::kubernetes_config_path( + workspace_dir.as_str(), + environment.organization_id.as_str(), + kubernetes.id(), + aws.access_key_id.as_str(), + aws.secret_access_key.as_str(), + kubernetes.region(), + )?; + + let from_dir = format!("{}/common/services/mongodb", self.context.lib_root_dir()); + + let _ = crate::template::generate_and_copy_all_files_into_dir( + from_dir.as_str(), + workspace_dir.as_str(), + &context, + )?; + + // render templates + let helm_release_name = self.helm_release_name(); + let aws_credentials_envs = vec![ + (AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()), + (AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()), + ]; + + // do exec helm upgrade and return the last deployment status + let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history( + kubernetes_config_file_path.as_str(), + environment.namespace(), + helm_release_name.as_str(), + workspace_dir.as_str(), + aws_credentials_envs.clone(), + )?; + + // check deployment status + if helm_history_row.is_none() + || !helm_history_row.unwrap().is_successfully_deployed() + { + return Err(ServiceError::OnCreateFailed); + } + + // check app status + let selector = format!("app={}", self.name()); + + match crate::cmd::kubectl::kubectl_exec_is_pod_ready_with_retry( + kubernetes_config_file_path.as_str(), + environment.namespace(), + selector.as_str(), + aws_credentials_envs, + ) { + Ok(Some(true)) => {} + _ => return Err(ServiceError::OnCreateFailed), + } + } + } + + Ok(()) + } + + fn on_create_check(&self) -> Result<(), ServiceError> { + Ok(()) + } + + fn on_create_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!("AWS.MongoDB.on_create_error() called for {}", self.name()); + + self.delete(target, true) + } +} + +impl Pause for MongoDB { + fn on_pause(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + info!("AWS.MongoDB.on_pause() called for {}", self.name()); + + // TODO how to pause production? - the goal is to reduce cost, but it is possible to pause a production env? + // TODO how to pause development? - the goal is also to reduce cost, we can set the number of instances to 0, which will avoid to delete data :) + + Ok(()) + } + + fn on_pause_check(&self) -> Result<(), ServiceError> { + Ok(()) + } + + fn on_pause_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!("AWS.MongoDB.on_pause_error() called for {}", self.name()); + + // TODO what to do if there is a pause error? + + Ok(()) + } +} + +impl Delete for MongoDB { + fn on_delete(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + info!("AWS.MongoDB.on_delete() called for {}", self.name()); + self.delete(target, false) + } + + fn on_delete_check(&self) -> Result<(), ServiceError> { + Ok(()) + } + + fn on_delete_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!("AWS.MongoDB.on_create_error() called for {}", self.name()); + self.delete(target, true) + } +} + +impl crate::cloud_provider::service::Clone for MongoDB { + fn on_clone(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_clone_check(&self) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_clone_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } +} + +impl Upgrade for MongoDB { + fn on_upgrade(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_upgrade_check(&self) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_upgrade_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } +} + +impl Downgrade for MongoDB { + fn on_downgrade(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_downgrade_check(&self) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_downgrade_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } +} + +impl Backup for MongoDB { + fn on_backup(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_backup_check(&self) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_backup_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_restore(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_restore_check(&self) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_restore_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } +} diff --git a/src/cloud_provider/aws/databases/mysql.rs b/src/cloud_provider/aws/databases/mysql.rs new file mode 100644 index 00000000..53674604 --- /dev/null +++ b/src/cloud_provider/aws/databases/mysql.rs @@ -0,0 +1,472 @@ +use tera::Context as TeraContext; + +use crate::cloud_provider::aws::databases::utilities; +use crate::cloud_provider::aws::kubernetes::EKS; +use crate::cloud_provider::aws::{common, AWS}; +use crate::cloud_provider::environment::Environment; +use crate::cloud_provider::kubernetes::Kubernetes; +use crate::cloud_provider::service::{ + Action, Backup, Create, DatabaseOptions, DatabaseType, Delete, Downgrade, Pause, Service, + ServiceError, ServiceType, StatefulService, Upgrade, +}; +use crate::cloud_provider::DeploymentTarget; +use crate::cmd::kubectl::{ + kubectl_exec_create_namespace, kubectl_exec_delete_namespace, kubectl_exec_delete_secret, +}; +use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY}; +use crate::models::Context; + +pub struct MySQL { + context: Context, + id: String, + action: Action, + name: String, + version: String, + fqdn: String, + fqdn_id: String, + total_cpus: String, + total_ram_in_mib: u32, + database_instance_type: String, + options: DatabaseOptions, +} + +impl MySQL { + pub fn new( + context: Context, + id: &str, + action: Action, + name: &str, + version: &str, + fqdn: &str, + fqdn_id: &str, + total_cpus: String, + total_ram_in_mib: u32, + database_instance_type: &str, + options: DatabaseOptions, + ) -> Self { + Self { + context, + action, + id: id.to_string(), + name: name.to_string(), + version: version.to_string(), + fqdn: fqdn.to_string(), + fqdn_id: fqdn_id.to_string(), + total_cpus, + total_ram_in_mib, + database_instance_type: database_instance_type.to_string(), + options, + } + } + fn helm_release_name(&self) -> String { + crate::string::cut(format!("mysql-{}", self.id()), 50) + } + fn workspace_directory(&self) -> String { + crate::fs::workspace_directory( + self.context.workspace_root_dir(), + self.context.execution_id(), + format!("databases/{}", self.name()), + ) + } + fn tera_context(&self, kubernetes: &dyn Kubernetes, environment: &Environment) -> TeraContext { + let mut context = self.default_tera_context(kubernetes, environment); + // FIXME: is there an other way than downcast a pointer? + let cp = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .expect("Could not downcast kubernetes.cloud_provider() to AWS"); + // we need the kubernetes config file to store tfstates file in kube secrets + let kubernetes_config_file_path = utilities::get_kubernetes_config_path( + self.workspace_directory().as_str(), + kubernetes, + environment, + ); + match kubernetes_config_file_path { + Ok(kube_config) => { + context.insert("kubeconfig_path", &kube_config.as_str()); + let aws = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + + utilities::create_namespace(&environment.namespace(), kube_config.as_str(), aws); + } + Err(e) => error!("Failed to generate the kubernetes config file path: {}", e), + } + context.insert("namespace", environment.namespace()); + + context.insert("aws_access_key", &cp.access_key_id); + context.insert("aws_secret_key", &cp.secret_access_key); + context.insert("eks_cluster_id", kubernetes.id()); + context.insert("eks_cluster_name", kubernetes.name()); + + context.insert("fqdn_id", self.fqdn_id.as_str()); + context.insert("fqdn", self.fqdn.as_str()); + + context.insert("database_login", self.options.login.as_str()); + context.insert("database_password", self.options.password.as_str()); + context.insert("database_port", &self.private_port()); + context.insert("database_disk_size_in_gib", &self.options.disk_size_in_gib); + context.insert("database_instance_type", &self.database_instance_type); + context.insert("database_disk_type", &self.options.database_disk_type); + context.insert("database_ram_size_in_mib", &self.total_ram_in_mib); + context.insert("database_total_cpus", &self.total_cpus); + context.insert("database_fqdn", &self.options.host.as_str()); + context.insert("database_id", &self.id()); + + context + } + + fn delete(&self, target: &DeploymentTarget, is_error: bool) -> Result<(), ServiceError> { + let workspace_dir = self.workspace_directory(); + + match target { + DeploymentTarget::ManagedServices(kubernetes, environment) => { + if is_error { + // do not delete if it is an error + return Ok(()); + } + + let context = self.tera_context(*kubernetes, *environment); + + crate::template::generate_and_copy_all_files_into_dir( + format!("{}/aws/services/common", self.context.lib_root_dir()).as_str(), + &workspace_dir, + &context, + )?; + crate::template::generate_and_copy_all_files_into_dir( + format!("{}/aws/services/mysql", self.context.lib_root_dir()).as_str(), + workspace_dir.as_str(), + &context, + )?; + crate::template::generate_and_copy_all_files_into_dir( + format!( + "{}/aws/charts/external-name-svc", + self.context.lib_root_dir() + ) + .as_str(), + format!("{}/{}", workspace_dir, "external-name-svc").as_str(), + &context, + )?; + crate::template::generate_and_copy_all_files_into_dir( + format!( + "{}/aws/charts/external-name-svc", + self.context.lib_root_dir() + ) + .as_str(), + workspace_dir.as_str(), + &context, + )?; + + match crate::cmd::terraform::terraform_exec_with_init_validate_destroy( + workspace_dir.as_str(), + ) { + Ok(o) => { + info!("Deleting secrets containing tfstates"); + utilities::delete_terraform_tfstate_secret( + *kubernetes, + environment, + self.workspace_directory().as_str(), + ); + } + //TODO: find a way to raise the error + Err(e) => error!("Error while destroying infrastructure {}", e), + } + } + DeploymentTarget::SelfHosted(kubernetes, environment) => { + let helm_release_name = self.helm_release_name(); + let selector = format!("app={}", self.name()); + + if is_error { + let _ = common::get_stateless_resource_information( + *kubernetes, + *environment, + workspace_dir.as_str(), + selector.as_str(), + )?; + } + + // clean the resource + let _ = common::do_stateless_service_cleanup( + *kubernetes, + *environment, + workspace_dir.as_str(), + helm_release_name.as_str(), + )?; + } + } + + Ok(()) + } +} + +impl StatefulService for MySQL {} + +impl Service for MySQL { + fn context(&self) -> &Context { + &self.context + } + + fn service_type(&self) -> ServiceType { + ServiceType::Database(DatabaseType::MySQL(&self.options)) + } + + fn id(&self) -> &str { + self.id.as_str() + } + + fn name(&self) -> &str { + self.name.as_str() + } + + fn version(&self) -> &str { + self.version.as_str() + } + + fn action(&self) -> &Action { + &self.action + } + + fn private_port(&self) -> Option { + Some(self.options.port) + } + + fn total_cpus(&self) -> String { + self.total_cpus.to_string() + } + + fn total_ram_in_mib(&self) -> u32 { + self.total_ram_in_mib + } + + fn total_instances(&self) -> u16 { + 1 + } +} + +impl Create for MySQL { + fn on_create(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + match target { + DeploymentTarget::ManagedServices(kubernetes, environment) => { + // use terraform + info!("deploy MySQL on AWS RDS for {}", self.name()); + let context = self.tera_context(*kubernetes, *environment); + + let workspace_dir = self.workspace_directory(); + + crate::template::generate_and_copy_all_files_into_dir( + format!("{}/aws/services/common", self.context.lib_root_dir()).as_str(), + &workspace_dir, + &context, + )?; + crate::template::generate_and_copy_all_files_into_dir( + format!("{}/aws/services/mysql", self.context.lib_root_dir()).as_str(), + workspace_dir.as_str(), + &context, + )?; + crate::template::generate_and_copy_all_files_into_dir( + format!( + "{}/aws/charts/external-name-svc", + self.context.lib_root_dir() + ) + .as_str(), + format!("{}/{}", workspace_dir, "external-name-svc").as_str(), + &context, + )?; + + crate::cmd::terraform::terraform_exec_with_init_validate_plan_apply( + workspace_dir.as_str(), + false, + )?; + } + DeploymentTarget::SelfHosted(kubernetes, environment) => { + // use helm + info!("deploy MySQL on Kubernetes for {}", self.name()); + + let context = self.tera_context(*kubernetes, *environment); + let workspace_dir = self.workspace_directory(); + + let aws = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + + let kubernetes_config_file_path = common::kubernetes_config_path( + workspace_dir.as_str(), + environment.organization_id.as_str(), + kubernetes.id(), + aws.access_key_id.as_str(), + aws.secret_access_key.as_str(), + kubernetes.region(), + )?; + + let from_dir = format!("{}/common/services/mysql", self.context.lib_root_dir()); + + let _ = crate::template::generate_and_copy_all_files_into_dir( + from_dir.as_str(), + workspace_dir.as_str(), + &context, + )?; + + // render templates + let helm_release_name = self.helm_release_name(); + let aws_credentials_envs = vec![ + (AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()), + (AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()), + ]; + + // do exec helm upgrade and return the last deployment status + let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history( + kubernetes_config_file_path.as_str(), + environment.namespace(), + helm_release_name.as_str(), + workspace_dir.as_str(), + aws_credentials_envs.clone(), + )?; + + // check deployment status + if helm_history_row.is_none() + || !helm_history_row.unwrap().is_successfully_deployed() + { + return Err(ServiceError::OnCreateFailed); + } + + // check app status + let selector = format!("app={}", self.name()); + + match crate::cmd::kubectl::kubectl_exec_is_pod_ready_with_retry( + kubernetes_config_file_path.as_str(), + environment.namespace(), + selector.as_str(), + aws_credentials_envs, + ) { + Ok(Some(true)) => {} + _ => return Err(ServiceError::OnCreateFailed), + } + } + } + + Ok(()) + } + + fn on_create_check(&self) -> Result<(), ServiceError> { + //FIXME : perform an actual check + Ok(()) + } + + fn on_create_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!("AWS.MySQL.on_create_error() called for {}", self.name()); + + self.delete(target, true) + } +} + +impl Pause for MySQL { + fn on_pause(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + info!("AWS.MySQL.on_pause() called for {}", self.name()); + + // TODO how to pause production? - the goal is to reduce cost, but it is possible to pause a production env? + // TODO how to pause development? - the goal is also to reduce cost, we can set the number of instances to 0, which will avoid to delete data :) + + Ok(()) + } + + fn on_pause_check(&self) -> Result<(), ServiceError> { + Ok(()) + } + + fn on_pause_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!("AWS.MySQL.on_pause_error() called for {}", self.name()); + + // TODO what to do if there is a pause error? + + Ok(()) + } +} + +impl Delete for MySQL { + fn on_delete(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + info!("AWS.MySQL.on_delete() called for {}", self.name()); + self.delete(target, false) + } + + fn on_delete_check(&self) -> Result<(), ServiceError> { + Ok(()) + } + + fn on_delete_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!("AWS.MySQL.on_create_error() called for {}", self.name()); + self.delete(target, true) + } +} + +impl crate::cloud_provider::service::Clone for MySQL { + fn on_clone(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_clone_check(&self) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_clone_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } +} + +impl Upgrade for MySQL { + fn on_upgrade(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_upgrade_check(&self) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_upgrade_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } +} + +impl Downgrade for MySQL { + fn on_downgrade(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_downgrade_check(&self) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_downgrade_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } +} + +impl Backup for MySQL { + fn on_backup(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_backup_check(&self) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_backup_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_restore(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_restore_check(&self) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_restore_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } +} diff --git a/src/cloud_provider/aws/databases/postgresql.rs b/src/cloud_provider/aws/databases/postgresql.rs new file mode 100644 index 00000000..ef1a04d5 --- /dev/null +++ b/src/cloud_provider/aws/databases/postgresql.rs @@ -0,0 +1,482 @@ +use tera::Context as TeraContext; + +use crate::cloud_provider::aws::databases::utilities; +use crate::cloud_provider::aws::{common, AWS}; +use crate::cloud_provider::environment::Environment; +use crate::cloud_provider::kubernetes::Kubernetes; +use crate::cloud_provider::service::{ + Action, Backup, Create, Database, DatabaseOptions, DatabaseType, Delete, Downgrade, Pause, + Service, ServiceError, ServiceType, StatefulService, Upgrade, +}; +use crate::cloud_provider::DeploymentTarget; +use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY}; +use crate::models::Context; + +pub struct PostgreSQL { + context: Context, + id: String, + action: Action, + name: String, + version: String, + fqdn: String, + fqdn_id: String, + total_cpus: String, + total_ram_in_mib: u32, + database_instance_type: String, + options: DatabaseOptions, +} + +impl PostgreSQL { + pub fn new( + context: Context, + id: &str, + action: Action, + name: &str, + version: &str, + fqdn: &str, + fqdn_id: &str, + total_cpus: String, + total_ram_in_mib: u32, + database_instance_type: &str, + options: DatabaseOptions, + ) -> Self { + PostgreSQL { + context, + action, + id: id.to_string(), + name: name.to_string(), + version: version.to_string(), + fqdn: fqdn.to_string(), + fqdn_id: fqdn_id.to_string(), + total_cpus, + total_ram_in_mib, + database_instance_type: database_instance_type.to_string(), + options, + } + } + + fn helm_release_name(&self) -> String { + crate::string::cut(format!("postgresql-{}", self.id()), 50) + } + + fn workspace_directory(&self) -> String { + crate::fs::workspace_directory( + self.context.workspace_root_dir(), + self.context.execution_id(), + format!("databases/{}", self.name()), + ) + } + + fn tera_context(&self, kubernetes: &dyn Kubernetes, environment: &Environment) -> TeraContext { + let mut context = self.default_tera_context(kubernetes, environment); + // FIXME: is there an other way than downcast a pointer? + let cp = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .expect("Could not downcast kubernetes.cloud_provider() to AWS"); + // we need the kubernetes config file to store tfstates file in kube secrets + let kubernetes_config_file_path = utilities::get_kubernetes_config_path( + self.workspace_directory().as_str(), + kubernetes, + environment, + ); + match kubernetes_config_file_path { + Ok(kube_config) => { + context.insert("kubeconfig_path", &kube_config.as_str()); + let aws = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + + utilities::create_namespace(&environment.namespace(), kube_config.as_str(), aws); + } + Err(e) => error!("Failed to generate the kubernetes config file path: {}", e), + } + context.insert("namespace", environment.namespace()); + + context.insert("aws_access_key", &cp.access_key_id); + context.insert("aws_secret_key", &cp.secret_access_key); + context.insert("eks_cluster_id", kubernetes.id()); + context.insert("eks_cluster_name", kubernetes.name()); + + context.insert("fqdn_id", self.fqdn_id.as_str()); + context.insert("fqdn", self.fqdn.as_str()); + + context.insert("database_login", self.options.login.as_str()); + context.insert("database_password", self.options.password.as_str()); + context.insert("database_port", &self.private_port()); + context.insert("database_disk_size_in_gib", &self.options.disk_size_in_gib); + context.insert("database_instance_type", &self.database_instance_type); + context.insert("database_disk_type", &self.options.database_disk_type); + context.insert("database_ram_size_in_mib", &self.total_ram_in_mib); + context.insert("database_total_cpus", &self.total_cpus); + context.insert("database_fqdn", &self.options.host.as_str()); + context.insert("database_id", &self.id()); + + context + } + + fn delete(&self, target: &DeploymentTarget, is_error: bool) -> Result<(), ServiceError> { + let workspace_dir = self.workspace_directory(); + + match target { + DeploymentTarget::ManagedServices(kubernetes, environment) => { + if is_error { + // do not delete if it is an error + return Ok(()); + } + + let context = self.tera_context(*kubernetes, *environment); + + crate::template::generate_and_copy_all_files_into_dir( + format!("{}/aws/services/common", self.context.lib_root_dir()).as_str(), + &workspace_dir, + &context, + )?; + crate::template::generate_and_copy_all_files_into_dir( + format!("{}/aws/services/postgresql", self.context.lib_root_dir()).as_str(), + workspace_dir.as_str(), + &context, + )?; + crate::template::generate_and_copy_all_files_into_dir( + format!( + "{}/aws/charts/external-name-svc", + self.context.lib_root_dir() + ) + .as_str(), + format!("{}/{}", workspace_dir, "external-name-svc").as_str(), + &context, + )?; + crate::template::generate_and_copy_all_files_into_dir( + format!( + "{}/aws/charts/external-name-svc", + self.context.lib_root_dir() + ) + .as_str(), + workspace_dir.as_str(), + &context, + )?; + + match crate::cmd::terraform::terraform_exec_with_init_validate_destroy( + workspace_dir.as_str(), + ) { + Ok(o) => { + info!("Deleting secrets containing tfstates"); + utilities::delete_terraform_tfstate_secret( + *kubernetes, + environment, + self.workspace_directory().as_str(), + ); + } + //TODO: find a way to raise the error + Err(e) => error!("Error while destroying infrastructure {}", e), + } + } + DeploymentTarget::SelfHosted(kubernetes, environment) => { + let helm_release_name = self.helm_release_name(); + let selector = format!("app={}", self.name()); + + if is_error { + let _ = common::get_stateless_resource_information( + *kubernetes, + *environment, + workspace_dir.as_str(), + selector.as_str(), + )?; + } + + // clean the resource + let _ = common::do_stateless_service_cleanup( + *kubernetes, + *environment, + workspace_dir.as_str(), + helm_release_name.as_str(), + )?; + } + } + + Ok(()) + } +} + +impl StatefulService for PostgreSQL {} + +impl Service for PostgreSQL { + fn context(&self) -> &Context { + &self.context + } + + fn service_type(&self) -> ServiceType { + ServiceType::Database(DatabaseType::PostgreSQL(&self.options)) + } + + fn id(&self) -> &str { + self.id.as_str() + } + + fn name(&self) -> &str { + self.name.as_str() + } + + fn version(&self) -> &str { + self.version.as_str() + } + + fn action(&self) -> &Action { + &self.action + } + + fn private_port(&self) -> Option { + Some(self.options.port) + } + + fn total_cpus(&self) -> String { + self.total_cpus.to_string() + } + + fn total_ram_in_mib(&self) -> u32 { + self.total_ram_in_mib + } + + fn total_instances(&self) -> u16 { + 1 + } +} + +impl Database for PostgreSQL {} + +impl Create for PostgreSQL { + fn on_create(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + info!("AWS.PostgreSQL.on_create() called for {}", self.name()); + + let workspace_dir = self.workspace_directory(); + + match target { + DeploymentTarget::ManagedServices(kubernetes, environment) => { + // use terraform + info!("deploy postgresql on AWS RDS for {}", self.name()); + let context = self.tera_context(*kubernetes, *environment); + + let workspace_dir = self.workspace_directory(); + + crate::template::generate_and_copy_all_files_into_dir( + format!("{}/aws/services/common", self.context.lib_root_dir()).as_str(), + &workspace_dir, + &context, + )?; + crate::template::generate_and_copy_all_files_into_dir( + format!("{}/aws/services/postgresql", self.context.lib_root_dir()).as_str(), + workspace_dir.as_str(), + &context, + )?; + crate::template::generate_and_copy_all_files_into_dir( + format!( + "{}/aws/charts/external-name-svc", + self.context.lib_root_dir() + ) + .as_str(), + format!("{}/{}", workspace_dir, "external-name-svc").as_str(), + &context, + )?; + + crate::cmd::terraform::terraform_exec_with_init_validate_plan_apply( + workspace_dir.as_str(), + false, + )?; + } + DeploymentTarget::SelfHosted(kubernetes, environment) => { + // use helm + info!("deploy PostgreSQL on Kubernetes for {}", self.name()); + + let context = self.tera_context(*kubernetes, *environment); + + let aws = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + + let kubernetes_config_file_path = common::kubernetes_config_path( + workspace_dir.as_str(), + environment.organization_id.as_str(), + kubernetes.id(), + aws.access_key_id.as_str(), + aws.secret_access_key.as_str(), + kubernetes.region(), + )?; + + let from_dir = + format!("{}/common/services/postgresql", self.context.lib_root_dir()); + + let _ = crate::template::generate_and_copy_all_files_into_dir( + from_dir.as_str(), + workspace_dir.as_str(), + &context, + )?; + + // render templates + let helm_release_name = self.helm_release_name(); + let aws_credentials_envs = vec![ + (AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()), + (AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()), + ]; + + // do exec helm upgrade and return the last deployment status + let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history( + kubernetes_config_file_path.as_str(), + environment.namespace(), + helm_release_name.as_str(), + workspace_dir.as_str(), + aws_credentials_envs.clone(), + )?; + + // check deployment status + if helm_history_row.is_none() + || !helm_history_row.unwrap().is_successfully_deployed() + { + return Err(ServiceError::OnCreateFailed); + } + + // check app status + let selector = format!("app={}", self.name()); + + match crate::cmd::kubectl::kubectl_exec_is_pod_ready_with_retry( + kubernetes_config_file_path.as_str(), + environment.namespace(), + selector.as_str(), + aws_credentials_envs, + ) { + Ok(Some(true)) => {} + _ => return Err(ServiceError::OnCreateFailed), + } + } + } + + Ok(()) + } + + fn on_create_check(&self) -> Result<(), ServiceError> { + Ok(()) + } + + fn on_create_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!( + "AWS.PostgreSQL.on_create_error() called for {}", + self.name() + ); + + self.delete(target, true) + } +} + +impl Pause for PostgreSQL { + fn on_pause(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + info!("AWS.PostgreSQL.on_pause() called for {}", self.name()); + + // TODO how to pause production? - the goal is to reduce cost, but it is possible to pause a production env? + // TODO how to pause development? - the goal is also to reduce cost, we can set the number of instances to 0, which will avoid to delete data :) + + Ok(()) + } + + fn on_pause_check(&self) -> Result<(), ServiceError> { + Ok(()) + } + + fn on_pause_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!("AWS.PostgreSQL.on_pause_error() called for {}", self.name()); + + // TODO what to do if there is a pause error? + + Ok(()) + } +} + +impl Delete for PostgreSQL { + fn on_delete(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + info!("AWS.PostgreSQL.on_delete() called for {}", self.name()); + self.delete(target, false) + } + + fn on_delete_check(&self) -> Result<(), ServiceError> { + Ok(()) + } + + fn on_delete_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!( + "AWS.PostgreSQL.on_create_error() called for {}", + self.name() + ); + self.delete(target, true) + } +} + +impl crate::cloud_provider::service::Clone for PostgreSQL { + fn on_clone(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_clone_check(&self) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_clone_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } +} + +impl Upgrade for PostgreSQL { + fn on_upgrade(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_upgrade_check(&self) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_upgrade_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } +} + +impl Downgrade for PostgreSQL { + fn on_downgrade(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_downgrade_check(&self) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_downgrade_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } +} + +impl Backup for PostgreSQL { + fn on_backup(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_backup_check(&self) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_backup_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_restore(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_restore_check(&self) -> Result<(), ServiceError> { + unimplemented!() + } + + fn on_restore_error(&self, _target: &DeploymentTarget) -> Result<(), ServiceError> { + unimplemented!() + } +} diff --git a/src/cloud_provider/aws/databases/utilities.rs b/src/cloud_provider/aws/databases/utilities.rs new file mode 100644 index 00000000..a000355d --- /dev/null +++ b/src/cloud_provider/aws/databases/utilities.rs @@ -0,0 +1,74 @@ +use tokio::io::Error; + +use crate::cloud_provider::aws::{common, AWS}; +use crate::cloud_provider::environment::Environment; +use crate::cloud_provider::kubernetes::Kubernetes; +use crate::cmd::kubectl::{kubectl_exec_create_namespace, kubectl_exec_delete_secret}; +use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY}; + +// generate the kubernetes config path +pub fn get_kubernetes_config_path( + workspace: &str, + kubernetes: &dyn Kubernetes, + environment: &Environment, +) -> Result { + let aws = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + + common::kubernetes_config_path( + workspace, + environment.organization_id.as_str(), + kubernetes.id(), + aws.access_key_id.as_str(), + aws.secret_access_key.as_str(), + kubernetes.region(), + ) +} + +pub fn create_namespace(namespace: &str, kube_config: &str, aws: &AWS) { + let aws_credentials_envs = vec![ + (AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()), + (AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()), + ]; + kubectl_exec_create_namespace(kube_config, namespace, aws_credentials_envs); +} + +pub fn delete_terraform_tfstate_secret( + kubernetes: &dyn Kubernetes, + environment: &Environment, + workspace_dir: &str, +) -> Result<(), Error> { + let aws = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + let aws_credentials_envs = vec![ + (AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()), + (AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()), + ]; + + let kubernetes_config_file_path = common::kubernetes_config_path( + workspace_dir, + environment.organization_id.as_str(), + kubernetes.id(), + aws.access_key_id.as_str(), + aws.secret_access_key.as_str(), + kubernetes.region(), + ); + + match kubernetes_config_file_path { + Ok(kube_config) => { + //create the namespace to insert the tfstate in secrets + kubectl_exec_delete_secret(kube_config, "tfstate-default-state", aws_credentials_envs); + Ok(()) + } + Err(e) => { + error!("Failed to generate the kubernetes config file path: {}", e); + Err(e) + } + } +} diff --git a/src/cloud_provider/aws/external_service.rs b/src/cloud_provider/aws/external_service.rs new file mode 100644 index 00000000..a90bdd30 --- /dev/null +++ b/src/cloud_provider/aws/external_service.rs @@ -0,0 +1,364 @@ +use serde::{Deserialize, Serialize}; +use tera::Context as TeraContext; + +use crate::build_platform::Image; +use crate::cloud_provider::aws::{common, AWS}; +use crate::cloud_provider::environment::Environment; +use crate::cloud_provider::kubernetes::Kubernetes; +use crate::cloud_provider::service::{ + Action, Application, Create, Delete, Pause, Service, ServiceError, ServiceType, + StatelessService, +}; +use crate::cloud_provider::DeploymentTarget; +use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY}; +use crate::models::Context; + +#[derive(Clone, Eq, PartialEq, Hash)] +pub struct ExternalService { + context: Context, + id: String, + action: Action, + name: String, + total_cpus: String, + total_ram_in_mib: u32, + image: Image, + environment_variables: Vec, +} + +impl ExternalService { + pub fn new( + context: Context, + id: &str, + action: Action, + name: &str, + total_cpus: String, + total_ram_in_mib: u32, + image: Image, + environment_variables: Vec, + ) -> Self { + ExternalService { + context, + id: id.to_string(), + action, + name: name.to_string(), + total_cpus, + total_ram_in_mib, + image, + environment_variables, + } + } + + fn helm_release_name(&self) -> String { + crate::string::cut( + format!("external-service-{}-{}", self.name(), self.id()), + 50, + ) + } + + fn workspace_directory(&self) -> String { + crate::fs::workspace_directory( + self.context.workspace_root_dir(), + self.context.execution_id(), + format!("external-services/{}", self.name()), + ) + } + + fn context(&self, kubernetes: &dyn Kubernetes, environment: &Environment) -> TeraContext { + let mut context = self.default_tera_context(kubernetes, environment); + let commit_id = self.image().commit_id.as_str(); + + context.insert("helm_app_version", &commit_id[..7]); + + match &self.image().registry_url { + Some(registry_url) => context.insert("image_name_with_tag", registry_url.as_str()), + None => { + let image_name_with_tag = self.image().name_with_tag(); + warn!("there is no registry url, use image name with tag with the default container registry: {}", image_name_with_tag.as_str()); + context.insert("image_name_with_tag", image_name_with_tag.as_str()); + } + } + + let environment_variables = self + .environment_variables + .iter() + .map(|ev| EnvironmentVariableDataTemplate { + key: ev.key.clone(), + value: ev.value.clone(), + }) + .collect::>(); + + context.insert("environment_variables", &environment_variables); + + context + } + + fn delete(&self, target: &DeploymentTarget, is_error: bool) -> Result<(), ServiceError> { + let (kubernetes, environment) = match target { + DeploymentTarget::ManagedServices(k, env) => (*k, *env), + DeploymentTarget::SelfHosted(k, env) => (*k, *env), + }; + + let workspace_dir = self.workspace_directory(); + let helm_release_name = self.helm_release_name(); + let selector = format!("app={}", self.name()); + + if is_error { + let _ = common::get_stateless_resource_information( + kubernetes, + environment, + workspace_dir.as_str(), + selector.as_str(), + )?; + } + + // clean the resource + let _ = common::do_stateless_service_cleanup( + kubernetes, + environment, + workspace_dir.as_str(), + helm_release_name.as_str(), + )?; + + Ok(()) + } +} + +impl crate::cloud_provider::service::ExternalService for ExternalService {} + +impl crate::cloud_provider::service::Application for ExternalService { + fn image(&self) -> &Image { + &self.image + } + + fn set_image(&mut self, image: Image) { + self.image = image; + } +} + +impl StatelessService for ExternalService {} + +impl Service for ExternalService { + fn context(&self) -> &Context { + &self.context + } + + fn service_type(&self) -> ServiceType { + ServiceType::ExternalService + } + + fn id(&self) -> &str { + self.id.as_str() + } + + fn name(&self) -> &str { + self.name.as_str() + } + + fn version(&self) -> &str { + self.image.commit_id.as_str() + } + + fn action(&self) -> &Action { + &self.action + } + + fn private_port(&self) -> Option { + None + } + + fn total_cpus(&self) -> String { + self.total_cpus.to_string() + } + + fn total_ram_in_mib(&self) -> u32 { + self.total_ram_in_mib + } + + fn total_instances(&self) -> u16 { + 1 + } +} + +impl Create for ExternalService { + fn on_create(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + info!( + "AWS.external_service.on_create() called for {}", + self.name() + ); + let (kubernetes, environment) = match target { + DeploymentTarget::ManagedServices(k, env) => (*k, *env), + DeploymentTarget::SelfHosted(k, env) => (*k, *env), + }; + + let aws = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + + let context = self.context(kubernetes, environment); + let workspace_dir = self.workspace_directory(); + + let from_dir = format!("{}/common/services/q-job", self.context.lib_root_dir()); + let _ = crate::template::generate_and_copy_all_files_into_dir( + from_dir.as_str(), + workspace_dir.as_str(), + &context, + )?; + + // render + // TODO check the rendered files? + let helm_release_name = self.helm_release_name(); + let aws_credentials_envs = vec![ + (AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()), + (AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()), + ]; + + let kubernetes_config_file_path = common::kubernetes_config_path( + workspace_dir.as_str(), + environment.organization_id.as_str(), + kubernetes.id(), + aws.access_key_id.as_str(), + aws.secret_access_key.as_str(), + kubernetes.region(), + )?; + + // do exec helm upgrade and return the last deployment status + let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history( + kubernetes_config_file_path.as_str(), + environment.namespace(), + helm_release_name.as_str(), + workspace_dir.as_str(), + aws_credentials_envs.clone(), + )?; + + // check deployment status + if helm_history_row.is_none() || !helm_history_row.unwrap().is_successfully_deployed() { + // TODO get pod output by using kubectl and return it into the OnCreateFailed + return Err(ServiceError::OnCreateFailed); + } + + // check job status + match crate::cmd::kubectl::kubectl_exec_is_job_ready_with_retry( + kubernetes_config_file_path.as_str(), + environment.namespace(), + self.name.as_str(), + aws_credentials_envs, + ) { + Ok(Some(true)) => {} + _ => return Err(ServiceError::OnCreateFailed), + } + + Ok(()) + } + + fn on_create_check(&self) -> Result<(), ServiceError> { + Ok(()) + } + + fn on_create_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!( + "AWS.external_service.on_create_error() called for {}", + self.name() + ); + let (kubernetes, environment) = match target { + DeploymentTarget::ManagedServices(k, env) => (*k, *env), + DeploymentTarget::SelfHosted(k, env) => (*k, *env), + }; + + let workspace_dir = self.workspace_directory(); + + let aws = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + + let aws_credentials_envs = vec![ + (AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()), + (AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()), + ]; + + let kubernetes_config_file_path = common::kubernetes_config_path( + workspace_dir.as_str(), + environment.organization_id.as_str(), + kubernetes.id(), + aws.access_key_id.as_str(), + aws.secret_access_key.as_str(), + kubernetes.region(), + )?; + + let helm_release_name = self.helm_release_name(); + + let history_rows = crate::cmd::helm::helm_exec_history( + kubernetes_config_file_path.as_str(), + environment.namespace(), + helm_release_name.as_str(), + aws_credentials_envs.clone(), + )?; + + if history_rows.len() == 1 { + crate::cmd::helm::helm_exec_uninstall( + kubernetes_config_file_path.as_str(), + environment.namespace(), + helm_release_name.as_str(), + aws_credentials_envs.clone(), + )?; + } + + Ok(()) + } +} + +impl Pause for ExternalService { + fn on_pause(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + info!("AWS.external_service.on_pause() called for {}", self.name()); + self.delete(target, false) + } + + fn on_pause_check(&self) -> Result<(), ServiceError> { + Ok(()) + } + + fn on_pause_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!( + "AWS.external_service.on_pause_error() called for {}", + self.name() + ); + self.delete(target, true) + } +} + +impl Delete for ExternalService { + fn on_delete(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + info!( + "AWS.external_service.on_delete() called for {}", + self.name() + ); + self.delete(target, false) + } + + fn on_delete_check(&self) -> Result<(), ServiceError> { + Ok(()) + } + + fn on_delete_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!( + "AWS.external_service.on_delete_error() called for {}", + self.name() + ); + self.delete(target, true) + } +} + +#[derive(Clone, Eq, PartialEq, Hash)] +pub struct EnvironmentVariable { + pub key: String, + pub value: String, +} + +#[derive(Serialize, Deserialize)] +struct EnvironmentVariableDataTemplate { + pub key: String, + pub value: String, +} diff --git a/src/cloud_provider/aws/kubernetes/mod.rs b/src/cloud_provider/aws/kubernetes/mod.rs new file mode 100644 index 00000000..f3d03d91 --- /dev/null +++ b/src/cloud_provider/aws/kubernetes/mod.rs @@ -0,0 +1,1245 @@ +use std::borrow::Borrow; +use std::env; +use std::iter::FromIterator; +use std::ops::Deref; +use std::rc::Rc; +use std::str::FromStr; + +use itertools::Itertools; +use rusoto_core::Region; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tera::Context as TeraContext; + +use crate::cloud_provider::aws::common::{do_stateless_service_cleanup, kubernetes_config_path}; +use crate::cloud_provider::aws::kubernetes::node::Node; +use crate::cloud_provider::aws::{common, AWS}; +use crate::cloud_provider::environment::Environment; +use crate::cloud_provider::kubernetes::{ + check_kubernetes_has_enough_resources_to_deploy_environment, Kind, Kubernetes, KubernetesError, + KubernetesNode, Resources, +}; +use crate::cloud_provider::service::{Service, ServiceType}; +use crate::cloud_provider::{CloudProvider, DeploymentTarget}; +use crate::cmd; +use crate::cmd::helm::helm_uninstall_list; +use crate::cmd::kubectl::{kubectl_exec_delete_namespace, kubectl_exec_get_all_namespaces}; +use crate::cmd::utilities::CmdError; +use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY}; +use crate::deletion_utilities::{get_firsts_namespaces_to_delete, get_qovery_managed_namespaces}; +use crate::dns_provider::cloudflare::Cloudflare; +use crate::dns_provider::DnsProvider; +use crate::dns_provider::Kind::CLOUDFLARE; +use crate::fs::workspace_directory; +use crate::models::{ + Context, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressListener, + ProgressScope, +}; +use crate::string::terraform_list_format; +use crate::unit_conversion::{cpu_string_to_float, ki_to_mi}; +use crate::{dns_provider, dynamo_db, s3}; + +pub mod node; + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Options { + pub eks_zone_a_subnet_blocks: Vec, + pub eks_zone_b_subnet_blocks: Vec, + pub eks_zone_c_subnet_blocks: Vec, + pub rds_zone_a_subnet_blocks: Vec, + pub rds_zone_b_subnet_blocks: Vec, + pub rds_zone_c_subnet_blocks: Vec, + pub documentdb_zone_a_subnet_blocks: Vec, + pub documentdb_zone_b_subnet_blocks: Vec, + pub documentdb_zone_c_subnet_blocks: Vec, + pub elasticsearch_zone_a_subnet_blocks: Vec, + pub elasticsearch_zone_b_subnet_blocks: Vec, + pub elasticsearch_zone_c_subnet_blocks: Vec, + pub vpc_cidr_block: String, + pub eks_cidr_subnet: String, + pub qovery_api_url: String, + pub tls_email_report: String, + pub rds_cidr_subnet: String, + pub documentdb_cidr_subnet: String, + pub elasticsearch_cidr_subnet: String, + pub engine_version_controller_token: String, +} + +pub struct EKS<'a> { + context: Context, + id: String, + name: String, + version: String, + region: Region, + cloud_provider: &'a AWS, + dns_provider: &'a DnsProvider, + nodes: Vec, + template_directory: String, + options: Options, + listeners: Listeners, +} + +impl<'a> EKS<'a> { + pub fn new( + context: Context, + id: &str, + name: &str, + version: &str, + region: &str, + cloud_provider: &'a AWS, + dns_provider: &'a DnsProvider, + options: Options, + nodes: Vec, + ) -> Self { + let template_directory = format!("{}/aws/bootstrap", context.lib_root_dir()); + + EKS { + context, + id: id.to_string(), + name: name.to_string(), + version: version.to_string(), + region: Region::from_str(region).unwrap(), + cloud_provider, + dns_provider, + options, + nodes, + template_directory, + listeners: cloud_provider.listeners.clone(), // copy listeners from CloudProvider + } + } + + fn tera_context(&self) -> TeraContext { + let format_ips = |ips: &Vec| -> Vec { + ips.iter() + .map(|ip| format!("\"{}\"", ip)) + .collect::>() + }; + + let eks_zone_a_subnet_blocks = format_ips(&self.options.eks_zone_a_subnet_blocks); + let eks_zone_b_subnet_blocks = format_ips(&self.options.eks_zone_b_subnet_blocks); + let eks_zone_c_subnet_blocks = format_ips(&self.options.eks_zone_c_subnet_blocks); + let rds_zone_a_subnet_blocks = format_ips(&self.options.rds_zone_a_subnet_blocks); + let rds_zone_b_subnet_blocks = format_ips(&self.options.rds_zone_b_subnet_blocks); + let rds_zone_c_subnet_blocks = format_ips(&self.options.rds_zone_c_subnet_blocks); + + let documentdb_zone_a_subnet_blocks = + format_ips(&self.options.documentdb_zone_a_subnet_blocks); + + let documentdb_zone_b_subnet_blocks = + format_ips(&self.options.documentdb_zone_b_subnet_blocks); + + let documentdb_zone_c_subnet_blocks = + format_ips(&self.options.documentdb_zone_c_subnet_blocks); + + let elasticsearch_zone_a_subnet_blocks = + format_ips(&self.options.elasticsearch_zone_a_subnet_blocks); + + let elasticsearch_zone_b_subnet_blocks = + format_ips(&self.options.elasticsearch_zone_b_subnet_blocks); + + let elasticsearch_zone_c_subnet_blocks = + format_ips(&self.options.elasticsearch_zone_c_subnet_blocks); + + let region_cluster_id = format!("{}-{}", self.region(), self.id()); + let vpc_cidr_block = self.options.vpc_cidr_block.clone(); + let eks_cloudwatch_log_group = format!("/aws/eks/{}/cluster", self.id()); + let eks_cidr_subnet = self.options.eks_cidr_subnet.clone(); + let worker_nodes = self + .nodes + .iter() + .group_by(|e| e.instance_type()) + .into_iter() + .map(|(instance_type, group)| (instance_type, group.collect::>())) + .map(|(instance_type, nodes)| WorkerNodeDataTemplate { + instance_type: instance_type.to_string(), + desired_size: nodes.len().to_string(), + max_size: nodes.len().to_string(), + min_size: nodes.len().to_string(), + }) + .collect::>(); + + let s3_kubeconfig_bucket = get_s3_kubeconfig_bucket_name(self.id.clone()); + + let qovery_api_url = self.options.qovery_api_url.clone(); + let rds_cidr_subnet = self.options.rds_cidr_subnet.clone(); + let documentdb_cidr_subnet = self.options.documentdb_cidr_subnet.clone(); + let elasticsearch_cidr_subnet = self.options.elasticsearch_cidr_subnet.clone(); + + let managed_dns_list = vec![self.dns_provider.name()]; + let managed_dns_domains_helm_format = vec![format!("\"{}\"", self.dns_provider.domain())]; + let managed_dns_domains_terraform_format = + terraform_list_format(vec![self.dns_provider.domain().to_string()]); + let managed_dns_resolvers: Vec = self + .dns_provider + .resolvers() + .iter() + .map(|x| format!("{}", x.clone().to_string())) + .collect(); + let managed_dns_resolvers_terraform_format = terraform_list_format(managed_dns_resolvers); + + let mut context = TeraContext::new(); + // Qovery + context.insert("organization_id", self.cloud_provider.organization_id()); + context.insert("qovery_api_url", &qovery_api_url); + context.insert( + "engine_version_controller_token", + &self.options.engine_version_controller_token, + ); + + // DNS configuration + context.insert("managed_dns", &managed_dns_list); + context.insert( + "managed_dns_domains_helm_format", + &managed_dns_domains_helm_format, + ); + context.insert( + "managed_dns_domains_terraform_format", + &managed_dns_domains_terraform_format, + ); + context.insert( + "managed_dns_resolvers_terraform_format", + &managed_dns_resolvers_terraform_format, + ); + + match self.dns_provider.kind() { + dns_provider::Kind::CLOUDFLARE => { + context.insert("external_dns_provider", "cloudflare"); + context.insert("cloudflare_api_token", self.dns_provider.token()); + context.insert("cloudflare_email", self.dns_provider.account()); + } + }; + + context.insert("dns_email_report", &self.options.tls_email_report); // Pierre suggested renaming to tls_email_report + + // TLS + let lets_encrypt_url = match self.context.metadata() { + Some(meta) => match meta.test { + Some(true) => "https://acme-staging-v02.api.letsencrypt.org/directory", + _ => "https://acme-v02.api.letsencrypt.org/directory", + }, + _ => "https://acme-v02.api.letsencrypt.org/directory", + }; + context.insert("acme_server_url", lets_encrypt_url); + + // AWS + context.insert("aws_access_key", &self.cloud_provider.access_key_id); + context.insert("aws_secret_key", &self.cloud_provider.secret_access_key); + + // AWS S3 tfstate storage + context.insert( + "aws_access_key_tfstates_account", + self.cloud_provider() + .terraform_state_credentials() + .access_key_id + .as_str(), + ); + + context.insert( + "aws_secret_key_tfstates_account", + self.cloud_provider() + .terraform_state_credentials() + .secret_access_key + .as_str(), + ); + context.insert( + "aws_region_tfstates_account", + self.cloud_provider() + .terraform_state_credentials() + .region + .as_str(), + ); + + // TODO URGENT change the behavior of self.bucket_name() + context.insert("aws_region", &self.region.name()); + context.insert("aws_terraform_backend_bucket", "qovery-terrafom-tfstates"); + context.insert( + "aws_terraform_backend_dynamodb_table", + "qovery-terrafom-tfstates", + ); + context.insert("vpc_cidr_block", &vpc_cidr_block.clone()); + context.insert("s3_kubeconfig_bucket", &s3_kubeconfig_bucket); + + // AWS - EKS + context.insert("eks_cidr_subnet", &eks_cidr_subnet.clone()); + context.insert("eks_cluster_name", &self.name()); + context.insert("eks_cluster_id", self.id()); + context.insert("eks_region_cluster_id", region_cluster_id.as_str()); + context.insert("eks_worker_nodes", &worker_nodes); + context.insert("eks_zone_a_subnet_blocks", &eks_zone_a_subnet_blocks); + context.insert("eks_zone_b_subnet_blocks", &eks_zone_b_subnet_blocks); + context.insert("eks_zone_c_subnet_blocks", &eks_zone_c_subnet_blocks); + context.insert("eks_masters_version", &self.version()); + context.insert("eks_workers_version", &self.version()); + context.insert("eks_cloudwatch_log_group", &eks_cloudwatch_log_group); + + // AWS - RDS + context.insert("rds_cidr_subnet", &rds_cidr_subnet); + context.insert("rds_zone_a_subnet_blocks", &rds_zone_a_subnet_blocks); + context.insert("rds_zone_b_subnet_blocks", &rds_zone_b_subnet_blocks); + context.insert("rds_zone_c_subnet_blocks", &rds_zone_c_subnet_blocks); + + // AWS - DocumentDB + context.insert("documentdb_cidr_subnet", &documentdb_cidr_subnet); + context.insert( + "documentdb_zone_a_subnet_blocks", + &documentdb_zone_a_subnet_blocks, + ); + context.insert( + "documentdb_zone_b_subnet_blocks", + &documentdb_zone_b_subnet_blocks, + ); + context.insert( + "documentdb_zone_c_subnet_blocks", + &documentdb_zone_c_subnet_blocks, + ); + + // AWS - Elasticsearch + context.insert( + "elasticsearch_cidr_subnet", + &elasticsearch_cidr_subnet.clone(), + ); + context.insert( + "elasticsearch_zone_a_subnet_blocks", + &elasticsearch_zone_a_subnet_blocks, + ); + context.insert( + "elasticsearch_zone_b_subnet_blocks", + &elasticsearch_zone_b_subnet_blocks, + ); + context.insert( + "elasticsearch_zone_c_subnet_blocks", + &elasticsearch_zone_c_subnet_blocks, + ); + + context + } +} + +impl<'a> Kubernetes for EKS<'a> { + fn context(&self) -> &Context { + &self.context + } + + fn kind(&self) -> Kind { + Kind::EKS + } + + fn id(&self) -> &str { + self.id.as_str() + } + + fn name(&self) -> &str { + self.name.as_str() + } + + fn version(&self) -> &str { + self.version.as_str() + } + + fn region(&self) -> &str { + self.region.name() + } + + fn cloud_provider(&self) -> &dyn CloudProvider { + self.cloud_provider + } + + fn dns_provider(&self) -> &dyn DnsProvider { + self.dns_provider + } + + fn is_valid(&self) -> Result<(), KubernetesError> { + Ok(()) + } + + fn add_listener(&mut self, listener: Listener) { + self.listeners.push(listener); + } + + fn listeners(&self) -> &Listeners { + &self.listeners + } + + fn resources(&self, environment: &Environment) -> Result { + let aws = self + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + + let kubernetes_config_file_path = common::kubernetes_config_path( + self.context.workspace_root_dir(), + environment.organization_id.as_str(), + &self.id, + aws.access_key_id.as_str(), + aws.secret_access_key.as_str(), + self.region(), + )?; + + let aws_credentials_envs = vec![ + (AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()), + (AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()), + ]; + + let nodes = + cmd::kubectl::kubectl_exec_get_node(kubernetes_config_file_path, aws_credentials_envs)?; + + let mut resources = Resources { + free_cpu: 0.0, + max_cpu: 0.0, + free_ram_in_mib: 0, + max_ram_in_mib: 0, + free_pods: 0, + max_pods: 0, + running_nodes: 0, + }; + + for node in nodes.items { + resources.free_cpu += cpu_string_to_float(node.status.allocatable.cpu); + resources.max_cpu += cpu_string_to_float(node.status.capacity.cpu); + resources.free_ram_in_mib += ki_to_mi(node.status.allocatable.memory); + resources.max_ram_in_mib += ki_to_mi(node.status.capacity.memory); + resources.free_pods = match node.status.allocatable.pods.parse::() { + Ok(v) => v, + _ => 0, + }; + resources.max_pods = match node.status.capacity.pods.parse::() { + Ok(v) => v, + _ => 0, + }; + resources.running_nodes += 1; + } + + Ok(resources) + } + + fn on_create(&self) -> Result<(), KubernetesError> { + info!("EKS.on_create() called for {}", self.name()); + + let listeners_helper = ListenersHelper::new(&self.listeners); + + listeners_helper.start_in_progress(ProgressInfo::new( + ProgressScope::Infrastructure { + execution_id: self.context.execution_id().to_string(), + }, + ProgressLevel::Info, + Some(format!( + "start to delete EKS cluster {} with id {}", + self.name(), + self.id() + )), + self.context.execution_id(), + )); + + let temp_dir = workspace_directory( + self.context.workspace_root_dir(), + self.context.execution_id(), + format!("bootstrap/{}", self.name()), + ); + + // generate terraform files and copy them into temp dir + let context = self.tera_context(); + let _ = crate::template::generate_and_copy_all_files_into_dir( + self.template_directory.as_str(), + temp_dir.as_str(), + &context, + )?; + + // copy lib/common/bootstrap/charts directory (and sub directory) into the lib/aws/bootstrap/common/charts directory. + // this is due to the required dependencies of lib/aws/bootstrap/*.tf files + let common_charts_temp_dir = format!("{}/common/charts", temp_dir.as_str()); + let _ = crate::template::copy_non_template_files( + format!("{}/common/bootstrap/charts", self.context.lib_root_dir()), + common_charts_temp_dir.as_str(), + )?; + + let _ = crate::cmd::terraform::terraform_exec_with_init_validate_plan_apply( + temp_dir.as_str(), + true, + )?; + + Ok(()) + } + + fn on_create_error(&self) -> Result<(), KubernetesError> { + warn!("EKS.on_create_error() called for {}", self.name()); + // FIXME + Err(KubernetesError::Error) + } + + fn on_upgrade(&self) -> Result<(), KubernetesError> { + info!("EKS.on_upgrade() called for {}", self.name()); + unimplemented!() + } + + fn on_upgrade_error(&self) -> Result<(), KubernetesError> { + warn!("EKS.on_upgrade_error() called for {}", self.name()); + unimplemented!() + } + + fn on_downgrade(&self) -> Result<(), KubernetesError> { + info!("EKS.on_downgrade() called for {}", self.name()); + unimplemented!() + } + + fn on_downgrade_error(&self) -> Result<(), KubernetesError> { + warn!("EKS.on_downgrade_error() called for {}", self.name()); + unimplemented!() + } + + fn on_delete(&self) -> Result<(), KubernetesError> { + info!("EKS.on_delete() called for {}", self.name()); + let listeners_helper = ListenersHelper::new(&self.listeners); + listeners_helper.delete_in_progress(ProgressInfo::new( + ProgressScope::Infrastructure { + execution_id: self.context.execution_id().to_string(), + }, + ProgressLevel::Warn, + Some(format!( + "start to delete EKS cluster {} with id {}", + self.name(), + self.id() + )), + self.context.execution_id(), + )); + + let temp_dir = workspace_directory( + self.context.workspace_root_dir(), + self.context.execution_id(), + format!("bootstrap/{}", self.name()), + ); + + // generate terraform files and copy them into temp dir + let context = self.tera_context(); + let _ = crate::template::generate_and_copy_all_files_into_dir( + self.template_directory.as_str(), + temp_dir.as_str(), + &context, + )?; + + // copy lib/common/bootstrap/charts directory (and sub directory) into the lib/aws/bootstrap/common/charts directory. + // this is due to the required dependencies of lib/aws/bootstrap/*.tf files + let common_charts_temp_dir = format!("{}/common/charts", temp_dir.as_str()); + let _ = crate::template::copy_non_template_files( + format!("{}/common/bootstrap/charts", self.context.lib_root_dir()), + common_charts_temp_dir.as_str(), + )?; + let aws_credentials_envs = vec![ + ( + AWS_ACCESS_KEY_ID, + self.cloud_provider.access_key_id.as_str(), + ), + ( + AWS_SECRET_ACCESS_KEY, + &self.cloud_provider.secret_access_key.as_str(), + ), + ]; + + let kubernetes_config_file_path = kubernetes_config_path( + self.context.workspace_root_dir(), + self.cloud_provider.organization_id.as_str(), + self.id(), + self.cloud_provider.access_key_id.as_str(), + self.cloud_provider.secret_access_key.as_str(), + self.region(), + )?; + let all_namespaces = kubectl_exec_get_all_namespaces( + kubernetes_config_file_path, + aws_credentials_envs.clone(), + ); + + // should make the diff between all namespaces and qovery managed namespaces + match all_namespaces { + Ok(namespace_vec) => { + let namespaces_as_str = namespace_vec.iter().map(std::ops::Deref::deref).collect(); + let namespaces_to_delete = get_firsts_namespaces_to_delete(namespaces_as_str); + info!("Deleting non Qovery Namespaces"); + for namespace_to_delete in namespaces_to_delete.iter() { + let kubernetes_config_file_path0 = kubernetes_config_path( + self.context.workspace_root_dir(), + self.cloud_provider.organization_id.as_str(), + self.id(), + self.cloud_provider.access_key_id.as_str(), + self.cloud_provider.secret_access_key.as_str(), + self.region(), + )?; + + let deletion = cmd::kubectl::kubectl_exec_delete_namespace( + &kubernetes_config_file_path0, + namespace_to_delete, + aws_credentials_envs.clone(), + ); + match deletion { + Ok(out) => info!("Namespace {} is deleted", namespace_to_delete), + Err(e) => { + error!( + "Can't delete the namespace {}, quiting now", + namespace_to_delete + ); + } + } + } + } + Err(e) => error!("Error while getting all namespaces {}", e), + } + info!("Deleting Qovery managed Namespaces"); + let kubernetes_config_file_path2 = kubernetes_config_path( + self.context.workspace_root_dir(), + self.cloud_provider.organization_id.as_str(), + self.id(), + self.cloud_provider.access_key_id.as_str(), + self.cloud_provider.secret_access_key.as_str(), + self.region(), + )?; + // TODO use label instead fixed names + let mut qovery_namespaces = get_qovery_managed_namespaces(); + for qovery_namespace in qovery_namespaces.iter() { + let deletion = cmd::kubectl::kubectl_exec_delete_namespace( + &kubernetes_config_file_path2, + qovery_namespace, + aws_credentials_envs.clone(), + ); + match deletion { + Ok(out) => info!("Namespace {} is fully deleted", qovery_namespace), + Err(e) => { + error!( + "Can't delete the namespace {}, quiting now", + qovery_namespace + ); + } + } + } + info!("Delete all remaining deployed helm applications"); + + match cmd::helm::helm_list(&kubernetes_config_file_path2, aws_credentials_envs.clone()) { + Ok(helm_list) => { + cmd::helm::helm_uninstall_list( + &kubernetes_config_file_path2, + helm_list, + aws_credentials_envs.clone(), + ); + } + Err(e) => error!("Unable to get helm list"), + } + info!("Running Terraform destroy"); + let terraform_result = + cmd::terraform::terraform_exec_with_init_validate_destroy(temp_dir.as_str())?; + // we should delete the bucket containing the kubeconfig after + // to prevent to loose connection from terraform to kube cluster + match terraform_result { + () => { + info!("Deleting S3 Bucket containing Kubeconfig"); + let s3_kubeconfig_bucket = get_s3_kubeconfig_bucket_name(self.id.clone()); + let _region = Region::from_str(self.region()).unwrap(); + s3::delete_bucket( + self.cloud_provider.access_key_id.as_str(), + self.cloud_provider.secret_access_key.as_str(), + s3_kubeconfig_bucket.clone().as_str(), + ); + } + _ => { + error!("Something is wrong with terraform destroy, Kubeconfig S3 location will not be deleting preventing to loose kube accessibility"); + } + } + + Ok(()) + } + + fn on_delete_error(&self) -> Result<(), KubernetesError> { + warn!("EKS.on_delete_error() called for {}", self.name()); + + // FIXME What should we do if something goes wrong while deleting the cluster? + + Ok(()) + } + + fn deploy_environment(&self, environment: &Environment) -> Result<(), KubernetesError> { + info!("EKS.deploy_environment() called for {}", self.name()); + + let listeners_helper = ListenersHelper::new(&self.listeners); + + let stateful_deployment_target = match environment.kind { + crate::cloud_provider::environment::Kind::Production => { + DeploymentTarget::ManagedServices(self, environment) + } + crate::cloud_provider::environment::Kind::Development => { + DeploymentTarget::SelfHosted(self, environment) + } + }; + + // do not deploy if there is not enough resources + check_kubernetes_has_enough_resources_to_deploy_environment(self, environment)?; + + // create all stateful services (database) + for service in &environment.stateful_services { + let progress_scope = service.progress_scope(); + + listeners_helper.start_in_progress(ProgressInfo::new( + progress_scope.clone(), + ProgressLevel::Info, + Some(format!( + "let's deploy {} {}", + service.service_type().name().to_lowercase(), + service.name() + )), + self.context.execution_id(), + )); + + match service.exec_action(&stateful_deployment_target) { + Err(err) => { + error!( + "error with stateful service {} , id: {} => {:?}", + service.name(), + service.id(), + err + ); + + listeners_helper.error(ProgressInfo::new( + progress_scope, + ProgressLevel::Error, + Some(format!( + "error while deploying {} {} : error => {:?}", + service.service_type().name().to_lowercase(), + service.name(), + err + )), + self.context.execution_id(), + )); + + return Err(KubernetesError::Deploy(err)); + } + _ => { + listeners_helper.start_in_progress(ProgressInfo::new( + progress_scope, + ProgressLevel::Info, + Some(format!( + "deployment succeeded for {} {}", + service.service_type().name().to_lowercase(), + service.name() + )), + self.context.execution_id(), + )); + } + } + } + + // stateless services are deployed on kubernetes, that's why we choose the deployment target SelfHosted. + let stateless_deployment_target = DeploymentTarget::SelfHosted(self, environment); + // create all stateless services (router, application...) + for service in &environment.stateless_services { + let progress_scope = service.progress_scope(); + + listeners_helper.start_in_progress(ProgressInfo::new( + progress_scope.clone(), + ProgressLevel::Info, + Some(format!( + "let's deploy {} {}", + service.service_type().name().to_lowercase(), + service.name() + )), + self.context.execution_id(), + )); + + match service.exec_action(&stateless_deployment_target) { + Err(err) => { + error!( + "error with stateless service {} , id: {} => {:?}", + service.name(), + service.id(), + err + ); + + listeners_helper.error(ProgressInfo::new( + progress_scope, + ProgressLevel::Error, + Some(format!( + "error while deploying {} {} : error => {:?}", + service.service_type().name().to_lowercase(), + service.name(), + err + )), + self.context.execution_id(), + )); + + return Err(KubernetesError::Deploy(err)); + } + _ => { + listeners_helper.start_in_progress(ProgressInfo::new( + progress_scope, + ProgressLevel::Info, + Some(format!( + "deployment succeeded for {} {}", + service.service_type().name().to_lowercase(), + service.name() + )), + self.context.execution_id(), + )); + } + } + } + + // check all deployed services + for service in &environment.stateful_services { + let progress_scope = service.progress_scope(); + + listeners_helper.start_in_progress(ProgressInfo::new( + progress_scope.clone(), + ProgressLevel::Info, + Some(format!( + "check {} {}", + service.service_type().name().to_lowercase(), + service.name() + )), + self.context.execution_id(), + )); + + match service.on_create_check() { + Err(err) => { + error!( + "error with stateful service while checking it {} , id: {} => {:?}", + service.name(), + service.id(), + err + ); + + listeners_helper.error(ProgressInfo::new( + progress_scope, + ProgressLevel::Error, + Some(format!( + "error while checking {} {} : error => {:?}", + service.service_type().name().to_lowercase(), + service.name(), + err + )), + self.context.execution_id(), + )); + + return Err(KubernetesError::Deploy(err)); + } + _ => { + listeners_helper.started(ProgressInfo::new( + progress_scope, + ProgressLevel::Info, + Some(format!( + "{} {} is up and running", + service.service_type().name().to_lowercase(), + service.name() + )), + self.context.execution_id(), + )); + } + } + } + + for service in &environment.stateless_services { + let progress_scope = service.progress_scope(); + + listeners_helper.start_in_progress(ProgressInfo::new( + progress_scope.clone(), + ProgressLevel::Info, + Some(format!( + "check {} {}", + service.service_type().name().to_lowercase(), + service.name() + )), + self.context.execution_id(), + )); + + match service.on_create_check() { + Err(err) => { + error!( + "error with stateless service while checking it {} , id: {} => {:?}", + service.name(), + service.id(), + err + ); + + listeners_helper.error(ProgressInfo::new( + progress_scope, + ProgressLevel::Error, + Some(format!( + "error while checking {} {} : error => {:?}", + service.service_type().name().to_lowercase(), + service.name(), + err + )), + self.context.execution_id(), + )); + + return Err(KubernetesError::Deploy(err)); + } + _ => { + listeners_helper.start_in_progress(ProgressInfo::new( + progress_scope, + ProgressLevel::Info, + Some(format!( + "{} {} is up and running", + service.service_type().name().to_lowercase(), + service.name() + )), + self.context.execution_id(), + )); + } + } + } + + Ok(()) + } + + fn deploy_environment_error(&self, environment: &Environment) -> Result<(), KubernetesError> { + warn!("EKS.deploy_environment_error() called for {}", self.name()); + + let listeners_helper = ListenersHelper::new(&self.listeners); + + listeners_helper.start_in_progress(ProgressInfo::new( + ProgressScope::Environment { + id: self.context.execution_id().to_string(), + }, + ProgressLevel::Warn, + Some( + "An error occurred while trying to deploy the environment, so let's revert changes", + ), + self.context.execution_id(), + )); + + let stateful_deployment_target = match environment.kind { + crate::cloud_provider::environment::Kind::Production => { + DeploymentTarget::ManagedServices(self, environment) + } + crate::cloud_provider::environment::Kind::Development => { + DeploymentTarget::SelfHosted(self, environment) + } + }; + + // clean up all stateful services (database) + for service in &environment.stateful_services { + let progress_scope = service.progress_scope(); + + listeners_helper.start_in_progress(ProgressInfo::new( + progress_scope.clone(), + ProgressLevel::Info, + Some(format!( + "reverting changes for {} {}", + service.service_type().name().to_lowercase(), + service.name() + )), + self.context.execution_id(), + )); + + match service.on_create_error(&stateful_deployment_target) { + Err(err) => { + error!( + "error with stateful service {} , id: {} => {:?}", + service.name(), + service.id(), + err + ); + + listeners_helper.error(ProgressInfo::new( + progress_scope, + ProgressLevel::Error, + Some(format!( + "error while reverting changes for {} {} : error => {:?}", + service.service_type().name().to_lowercase(), + service.name(), + err + )), + self.context.execution_id(), + )); + + return Err(KubernetesError::Deploy(err)); + } + _ => { + listeners_helper.start_in_progress(ProgressInfo::new( + progress_scope, + ProgressLevel::Info, + Some(format!( + "reverting changes succeeded for {} {}", + service.service_type().name().to_lowercase(), + service.name() + )), + self.context.execution_id(), + )); + } + } + } + + // stateless services are deployed on kubernetes, that's why we choose the deployment target SelfHosted. + let stateless_deployment_target = DeploymentTarget::SelfHosted(self, environment); + // clean up all stateless services (router, application...) + for service in &environment.stateless_services { + let progress_scope = service.progress_scope(); + + listeners_helper.start_in_progress(ProgressInfo::new( + progress_scope.clone(), + ProgressLevel::Info, + Some(format!( + "reverting changes for {} {}", + service.service_type().name().to_lowercase(), + service.name() + )), + self.context.execution_id(), + )); + + match service.on_create_error(&stateless_deployment_target) { + Err(err) => { + error!( + "error with stateless service {} , id: {} => {:?}", + service.name(), + service.id(), + err + ); + + listeners_helper.error(ProgressInfo::new( + progress_scope, + ProgressLevel::Error, + Some(format!( + "error while reverting changes for {} {} : error => {:?}", + service.service_type().name().to_lowercase(), + service.name(), + err + )), + self.context.execution_id(), + )); + + return Err(KubernetesError::Deploy(err)); + } + _ => { + listeners_helper.start_in_progress(ProgressInfo::new( + progress_scope, + ProgressLevel::Info, + Some(format!( + "reverting changes succeeded for {} {}", + service.service_type().name().to_lowercase(), + service.name() + )), + self.context.execution_id(), + )); + } + } + } + + Ok(()) + } + + fn pause_environment(&self, environment: &Environment) -> Result<(), KubernetesError> { + info!("EKS.pause_environment() called for {}", self.name()); + + let listeners_helper = ListenersHelper::new(&self.listeners); + + let stateful_deployment_target = match environment.kind { + crate::cloud_provider::environment::Kind::Production => { + DeploymentTarget::ManagedServices(self, environment) + } + crate::cloud_provider::environment::Kind::Development => { + DeploymentTarget::SelfHosted(self, environment) + } + }; + + // create all stateful services (database) + for stateful_service in &environment.stateful_services { + match stateful_service.on_pause(&stateful_deployment_target) { + Err(err) => { + error!( + "error with stateful service {} , id: {} => {:?}", + stateful_service.name(), + stateful_service.id(), + err + ); + + return Err(KubernetesError::Pause(err)); + } + _ => {} + } + } + + // stateless services are deployed on kubernetes, that's why we choose the deployment target SelfHosted. + let stateless_deployment_target = DeploymentTarget::SelfHosted(self, environment); + // create all stateless services (router, application...) + for stateless_service in &environment.stateless_services { + match stateless_service.on_pause(&stateless_deployment_target) { + Err(err) => { + error!( + "error with stateless service {} , id: {} => {:?}", + stateless_service.name(), + stateless_service.id(), + err + ); + + return Err(KubernetesError::Pause(err)); + } + _ => {} + } + } + + // check all deployed services + for stateful_service in &environment.stateful_services { + match stateful_service.on_pause_check() { + Err(err) => { + error!( + "error with stateful service while checking it {} , id: {} => {:?}", + stateful_service.name(), + stateful_service.id(), + err + ); + + return Err(KubernetesError::Pause(err)); + } + _ => {} + } + } + + for stateless_service in &environment.stateless_services { + match stateless_service.on_pause_check() { + Err(err) => { + error!( + "error with stateless service while checking it {} , id: {} => {:?}", + stateless_service.name(), + stateless_service.id(), + err + ); + + return Err(KubernetesError::Pause(err)); + } + _ => {} + } + } + + Ok(()) + } + + fn pause_environment_error(&self, _environment: &Environment) -> Result<(), KubernetesError> { + warn!("EKS.pause_environment_error() called for {}", self.name()); + Ok(()) + } + + fn delete_environment(&self, environment: &Environment) -> Result<(), KubernetesError> { + info!("EKS.delete_environment() called for {}", self.name()); + + let listeners_helper = ListenersHelper::new(&self.listeners); + // TODO use listeners_helper !!!! Don't be so shy Marc + Pierre + + let stateful_deployment_target = match environment.kind { + crate::cloud_provider::environment::Kind::Production => { + DeploymentTarget::ManagedServices(self, environment) + } + crate::cloud_provider::environment::Kind::Development => { + DeploymentTarget::SelfHosted(self, environment) + } + }; + + // delete all stateful services (database) + for stateful_service in &environment.stateful_services { + match stateful_service.on_delete(&stateful_deployment_target) { + Err(err) => { + error!( + "error with stateful service {} , id: {} => {:?}", + stateful_service.name(), + stateful_service.id(), + err + ); + + return Err(KubernetesError::Delete(err)); + } + _ => {} + } + } + + // stateless services are deployed on kubernetes, that's why we choose the deployment target SelfHosted. + let stateless_deployment_target = DeploymentTarget::SelfHosted(self, environment); + // delete all stateless services (router, application...) + for stateless_service in &environment.stateless_services { + match stateless_service.on_delete(&stateless_deployment_target) { + Err(err) => { + error!( + "error with stateless service {} , id: {} => {:?}", + stateless_service.name(), + stateless_service.id(), + err + ); + + return Err(KubernetesError::Delete(err)); + } + _ => {} + } + } + + // check all deployed services + for stateful_service in &environment.stateful_services { + match stateful_service.on_delete_check() { + Err(err) => { + error!( + "error with stateful service while checking it {} , id: {} => {:?}", + stateful_service.name(), + stateful_service.id(), + err + ); + + return Err(KubernetesError::Delete(err)); + } + _ => {} + } + } + + for stateless_service in &environment.stateless_services { + match stateless_service.on_delete_check() { + Err(err) => { + error!( + "error with stateless service while checking it {} , id: {} => {:?}", + stateless_service.name(), + stateless_service.id(), + err + ); + + return Err(KubernetesError::Delete(err)); + } + _ => {} + } + } + + let aws_credentials_envs = vec![ + ( + AWS_ACCESS_KEY_ID, + self.cloud_provider.access_key_id.as_str(), + ), + ( + AWS_SECRET_ACCESS_KEY, + &self.cloud_provider.secret_access_key.as_str(), + ), + ]; + + let kubernetes_config_file_path = common::kubernetes_config_path( + &self.context.workspace_root_dir(), + &environment.organization_id.as_str(), + &self.id, + &self.cloud_provider.access_key_id.as_str(), + &self.cloud_provider.secret_access_key.as_str(), + &self.region.name(), + )?; + + kubectl_exec_delete_namespace( + kubernetes_config_file_path, + &environment.namespace(), + aws_credentials_envs, + ); + + Ok(()) + } + + fn delete_environment_error(&self, _environment: &Environment) -> Result<(), KubernetesError> { + warn!("EKS.delete_environment_error() called for {}", self.name()); + Ok(()) + } +} + +fn get_s3_kubeconfig_bucket_name(id: String) -> String { + format!("qovery-kubeconfigs-{}", id) +} + +#[derive(Serialize, Deserialize)] +struct WorkerNodeDataTemplate { + instance_type: String, + desired_size: String, + max_size: String, + min_size: String, +} diff --git a/src/cloud_provider/aws/kubernetes/node.rs b/src/cloud_provider/aws/kubernetes/node.rs new file mode 100644 index 00000000..754baa66 --- /dev/null +++ b/src/cloud_provider/aws/kubernetes/node.rs @@ -0,0 +1,97 @@ +use std::any::Any; + +use crate::cloud_provider::kubernetes::KubernetesNode; + +pub struct Node { + total_cpu: u8, + total_memory_in_gib: u16, + instance_types_table: [(u8, u16, &'static str); 6], +} + +impl Node { + /// Number of CPUs and total memory wanted - the right AWS EC2 instance type is found algorithmically + /// Eg. total_cpu = 1 and total_memory_in_gib = 2 means `t2.small` instance type + /// BUT total_cpu = 1 and total_memory_in_gib = 3 does not have an existing instance - so we will pick the upper closest, + /// which is `t2.medium` with 2 cpu and 4 GiB + /// ``` + /// use qovery_engine::cloud_provider::aws::kubernetes::node::Node; + /// use qovery_engine::cloud_provider::kubernetes::KubernetesNode; + /// + /// let node = Node::new(2, 4); + /// assert_eq!(node.instance_type(), "t2.medium") + /// ``` + pub fn new(total_cpu: u8, total_memory_in_gib: u16) -> Self { + let instance_types_table = [ + (1, 1, "t2.micro"), + (1, 2, "t2.small"), + (2, 4, "t2.medium"), + (2, 8, "t2.large"), + (4, 16, "t2.xlarge"), + (8, 32, "t2.2xlarge"), + // TODO add other instance types + ]; + + Node { + total_cpu, + total_memory_in_gib, + instance_types_table, + } + } +} + +impl KubernetesNode for Node { + fn total_cpu(&self) -> u8 { + self.total_cpu + } + + fn total_memory_in_gib(&self) -> u16 { + self.total_memory_in_gib + } + + fn instance_type(&self) -> &str { + if self.total_cpu() == 0 || self.total_memory_in_gib() == 0 { + let (_, _, instance_type) = self.instance_types_table.first().unwrap(); + return instance_type; + } + + for (_cpu, mem, instance_type) in self.instance_types_table.iter() { + if self.total_memory_in_gib() <= *mem { + return instance_type; + } + } + + let (_, _, instance_type) = self.instance_types_table.last().unwrap(); + return instance_type; + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +#[cfg(test)] +mod tests { + use crate::cloud_provider::aws::kubernetes::node::Node; + use crate::cloud_provider::kubernetes::KubernetesNode; + + #[test] + fn test_instance_types() { + assert_eq!(Node::new(0, 0).instance_type(), "t2.micro"); + assert_eq!(Node::new(1, 0).instance_type(), "t2.micro"); + assert_eq!(Node::new(0, 1).instance_type(), "t2.micro"); + assert_eq!(Node::new(1, 1).instance_type(), "t2.micro"); + assert_eq!(Node::new(1, 2).instance_type(), "t2.small"); + assert_eq!(Node::new(2, 4).instance_type(), "t2.medium"); + assert_eq!(Node::new(2, 5).instance_type(), "t2.large"); + assert_eq!(Node::new(1, 6).instance_type(), "t2.large"); + assert_eq!(Node::new(1, 7).instance_type(), "t2.large"); + assert_eq!(Node::new(2, 8).instance_type(), "t2.large"); + assert_eq!(Node::new(3, 8).instance_type(), "t2.large"); + assert_eq!(Node::new(3, 10).instance_type(), "t2.xlarge"); + assert_eq!(Node::new(3, 12).instance_type(), "t2.xlarge"); + assert_eq!(Node::new(4, 16).instance_type(), "t2.xlarge"); + assert_eq!(Node::new(4, 17).instance_type(), "t2.2xlarge"); + assert_eq!(Node::new(8, 32).instance_type(), "t2.2xlarge"); + assert_eq!(Node::new(16, 64).instance_type(), "t2.2xlarge"); + } +} diff --git a/src/cloud_provider/aws/mod.rs b/src/cloud_provider/aws/mod.rs new file mode 100644 index 00000000..3e56f6fb --- /dev/null +++ b/src/cloud_provider/aws/mod.rs @@ -0,0 +1,109 @@ +use std::any::Any; +use std::rc::Rc; + +use rusoto_core::{Client, HttpClient, Region}; +use rusoto_credential::StaticProvider; +use rusoto_sts::{GetCallerIdentityRequest, Sts, StsClient}; + +use crate::cloud_provider::{CloudProvider, CloudProviderError, Kind, TerraformStateCredentials}; +use crate::models::{Context, Listener, Listeners, ProgressListener}; +use crate::runtime::async_run; + +mod common; + +pub mod application; +pub mod databases; +pub mod external_service; +pub mod kubernetes; +pub mod router; + +pub struct AWS { + context: Context, + id: String, + organization_id: String, + name: String, + pub access_key_id: String, + pub secret_access_key: String, + terraform_state_credentials: TerraformStateCredentials, + listeners: Listeners, +} + +impl AWS { + pub fn new( + context: Context, + id: &str, + organization_id: &str, + name: &str, + access_key_id: &str, + secret_access_key: &str, + terraform_state_credentials: TerraformStateCredentials, + ) -> Self { + AWS { + context, + id: id.to_string(), + organization_id: organization_id.to_string(), + name: name.to_string(), + access_key_id: access_key_id.to_string(), + secret_access_key: secret_access_key.to_string(), + terraform_state_credentials, + listeners: vec![], + } + } + + pub fn credentials(&self) -> StaticProvider { + StaticProvider::new( + self.access_key_id.to_string(), + self.secret_access_key.to_string(), + None, + None, + ) + } + + pub fn client(&self) -> Client { + Client::new_with(self.credentials(), HttpClient::new().unwrap()) + } +} + +impl CloudProvider for AWS { + fn context(&self) -> &Context { + &self.context + } + + fn kind(&self) -> Kind { + Kind::AWS + } + + fn id(&self) -> &str { + self.id.as_str() + } + + fn organization_id(&self) -> &str { + self.organization_id.as_str() + } + + fn name(&self) -> &str { + self.name.as_str() + } + + fn is_valid(&self) -> Result<(), CloudProviderError> { + let client = StsClient::new_with_client(self.client(), Region::default()); + let s = async_run(client.get_caller_identity(GetCallerIdentityRequest::default())); + + match s { + Ok(_x) => Ok(()), + Err(err) => Err(CloudProviderError::from(err)), + } + } + + fn add_listener(&mut self, listener: Listener) { + self.listeners.push(listener); + } + + fn terraform_state_credentials(&self) -> &TerraformStateCredentials { + &self.terraform_state_credentials + } + + fn as_any(&self) -> &dyn Any { + self + } +} diff --git a/src/cloud_provider/aws/router.rs b/src/cloud_provider/aws/router.rs new file mode 100644 index 00000000..676924ab --- /dev/null +++ b/src/cloud_provider/aws/router.rs @@ -0,0 +1,500 @@ +use dns_lookup::lookup_host; +use retry::delay::Fibonacci; +use retry::OperationResult; +use serde::{Deserialize, Serialize}; +use tera::Context as TeraContext; + +use crate::cloud_provider::aws::{common, AWS}; +use crate::cloud_provider::environment::Environment; +use crate::cloud_provider::kubernetes::Kubernetes; +use crate::cloud_provider::service::{ + Action, Create, Delete, Pause, Router as RRouter, Service, ServiceError, ServiceType, + StatelessService, +}; +use crate::cloud_provider::DeploymentTarget; +use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY}; +use crate::models::{Context, Metadata}; + +pub struct Router { + context: Context, + id: String, + name: String, + default_domain: String, + custom_domains: Vec, + routes: Vec, +} + +impl Router { + pub fn new( + context: Context, + id: &str, + name: &str, + default_domain: &str, + custom_domains: Vec, + routes: Vec, + ) -> Self { + Router { + context, + id: id.to_string(), + name: name.to_string(), + default_domain: default_domain.to_string(), + custom_domains, + routes, + } + } + + fn helm_release_name(&self) -> String { + crate::string::cut(format!("router-{}", self.id()), 50) + } + + fn aws_credentials_envs<'x>(&self, aws: &'x AWS) -> [(&'x str, &'x str); 2] { + [ + (AWS_ACCESS_KEY_ID, aws.access_key_id.as_str()), + (AWS_SECRET_ACCESS_KEY, aws.secret_access_key.as_str()), + ] + } + + fn workspace_directory(&self) -> String { + crate::fs::workspace_directory( + self.context.workspace_root_dir(), + self.context.execution_id(), + format!("routers/{}", self.name()), + ) + } + + fn tera_context(&self, kubernetes: &dyn Kubernetes, environment: &Environment) -> TeraContext { + let mut context = self.default_tera_context(kubernetes, environment); + + let applications = environment + .stateless_services + .iter() + .filter(|x| x.service_type() == ServiceType::Application) + .collect::>(); + + let custom_domain_data_templates = self + .custom_domains + .iter() + .map(|cd| { + let domain_hash = crate::crypto::to_sha1_truncate_16(cd.domain.as_str()); + //context.insert("target_hostname", cd.domain.clone()); + CustomDomainDataTemplate { + domain: cd.domain.clone(), + domain_hash, + target_domain: cd.target_domain.clone(), + } + }) + .collect::>(); + + let route_data_templates = self + .routes + .iter() + .map(|r| { + match applications + .iter() + .find(|app| app.name() == r.application_name.as_str()) + { + Some(application) => match application.private_port() { + Some(private_port) => Some(RouteDataTemplate { + path: r.path.clone(), + application_name: application.name().to_string(), + application_port: private_port, + }), + _ => None, + }, + _ => None, + } + }) + .filter(|x| x.is_some()) + .map(|x| x.unwrap()) + .collect::>(); + + let workspace_dir = self.workspace_directory(); + let aws = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + + let kubernetes_config_file_path = common::kubernetes_config_path( + workspace_dir.as_str(), + environment.organization_id.as_str(), + kubernetes.id(), + aws.access_key_id.as_str(), + aws.secret_access_key.as_str(), + kubernetes.region(), + ); + + match kubernetes_config_file_path { + Ok(kubernetes_config_file_path_string) => { + // Default domain + let external_ingress_hostname_default = + crate::cmd::kubectl::kubectl_exec_get_external_ingress_hostname( + kubernetes_config_file_path_string.as_str(), + "nginx-ingress", + "app=nginx-ingress,component=controller", + self.aws_credentials_envs(aws).to_vec(), + ); + + match external_ingress_hostname_default { + Ok(external_ingress_hostname_default) => { + match external_ingress_hostname_default { + Some(hostname) => context + .insert("external_ingress_hostname_default", hostname.as_str()), + None => { + warn!("unable to get external_ingress_hostname_default - what's wrong? This must never happened"); + } + } + } + _ => { + // FIXME really? + warn!("can't fetch kubernetes config file - what's wrong? This must never happened"); + } + } + + // Check if there is a custom domain first + if !self.custom_domains.is_empty() { + let external_ingress_hostname_custom = + crate::cmd::kubectl::kubectl_exec_get_external_ingress_hostname( + kubernetes_config_file_path_string.as_str(), + environment.namespace(), + "app=nginx-ingress,component=controller", + self.aws_credentials_envs(aws).to_vec(), + ); + + match external_ingress_hostname_custom { + Ok(external_ingress_hostname_custom) => { + match external_ingress_hostname_custom { + Some(hostname) => { + context.insert( + "external_ingress_hostname_custom", + hostname.as_str(), + ); + } + None => { + warn!("unable to get external_ingress_hostname_custom - what's wrong? This must never happened"); + } + } + } + _ => { + // FIXME really? + warn!("can't fetch kubernetes config file - what's wrong? This must never happened"); + } + } + context.insert("app_id", kubernetes.id()); + } + } + Err(_) => error!( + "can't fetch kubernetes config file - what's wrong? This must never happened" + ), // FIXME should I return an Err? + } + + let router_default_domain_hash = + crate::crypto::to_sha1_truncate_16(self.default_domain.as_str()); + + context.insert("router_default_domain", self.default_domain.as_str()); + context.insert( + "router_default_domain_hash", + router_default_domain_hash.as_str(), + ); + context.insert("custom_domains", &custom_domain_data_templates); + context.insert("routes", &route_data_templates); + context.insert("spec_acme_email", "tls@qovery.com"); // TODO CHANGE ME + context.insert( + "metadata_annotations_cert_manager_cluster_issuer", + "letsencrypt-qovery", + ); + + let lets_encrypt_url = match self.context.metadata() { + Some(meta) => match meta.test { + Some(true) => "https://acme-staging-v02.api.letsencrypt.org/directory", + _ => "https://acme-v02.api.letsencrypt.org/directory", + }, + _ => "https://acme-v02.api.letsencrypt.org/directory", + }; + context.insert("spec_acme_server", lets_encrypt_url); + + context + } + + fn delete(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + let (kubernetes, environment) = match target { + DeploymentTarget::ManagedServices(k, env) => (*k, *env), + DeploymentTarget::SelfHosted(k, env) => (*k, *env), + }; + + let workspace_dir = self.workspace_directory(); + let helm_release_name = self.helm_release_name(); + + let _ = common::do_stateless_service_cleanup( + kubernetes, + environment, + workspace_dir.as_str(), + helm_release_name.as_str(), + )?; + + Ok(()) + } +} + +impl Service for Router { + fn context(&self) -> &Context { + &self.context + } + + fn service_type(&self) -> ServiceType { + ServiceType::Router + } + + fn id(&self) -> &str { + self.id.as_str() + } + + fn name(&self) -> &str { + self.name.as_str() + } + + fn version(&self) -> &str { + "1.0" + } + + fn action(&self) -> &Action { + &Action::Create + } + + fn private_port(&self) -> Option { + None + } + + fn total_cpus(&self) -> String { + "1".to_string() + } + + fn total_ram_in_mib(&self) -> u32 { + 1 + } + + fn total_instances(&self) -> u16 { + 1 + } +} + +impl crate::cloud_provider::service::Router for Router { + fn check_domains(&self) -> Result<(), ServiceError> { + let check_result = retry::retry(Fibonacci::from_millis(3000).take(10), || { + // TODO send information back to the core + info!("check custom domain {}", self.default_domain.as_str()); + match lookup_host(self.default_domain.as_str()) { + Ok(_) => OperationResult::Ok(()), + Err(err) => { + debug!("{:?}", err); + OperationResult::Retry(()) + } + } + }); + + // TODO - check custom domains? if yes, why wasting time waiting for user setting up the custom domain? + + match check_result { + Ok(_) => {} + Err(_) => return Err(ServiceError::CheckFailed), + } + + Ok(()) + } +} + +impl StatelessService for Router {} + +impl Create for Router { + fn on_create(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + info!("AWS.router.on_create() called for {}", self.name()); + let (kubernetes, environment) = match target { + DeploymentTarget::ManagedServices(k, env) => (*k, *env), + DeploymentTarget::SelfHosted(k, env) => (*k, *env), + }; + + let aws = kubernetes + .cloud_provider() + .as_any() + .downcast_ref::() + .unwrap(); + + let workspace_dir = self.workspace_directory(); + let helm_release_name = self.helm_release_name(); + + let kubernetes_config_file_path = common::kubernetes_config_path( + workspace_dir.as_str(), + environment.organization_id.as_str(), + kubernetes.id(), + aws.access_key_id.as_str(), + aws.secret_access_key.as_str(), + kubernetes.region(), + )?; + + // respect order - getting the context here and not before is mandatory + // the nginx-ingress must be available to get the external dns target if necessary + let mut context = self.tera_context(kubernetes, environment); + + if !self.custom_domains.is_empty() { + // custom domains? create an NGINX ingress + info!("setup NGINX ingress for custom domains"); + + let into_dir = crate::fs::workspace_directory( + self.context.workspace_root_dir(), + self.context.execution_id(), + "routers/nginx-ingress", + ); + + let from_dir = format!("{}/common/chart_values", self.context.lib_root_dir()); + let _ = crate::template::generate_and_copy_all_files_into_dir( + from_dir.as_str(), + into_dir.as_str(), + &context, + )?; + + let _ = crate::template::copy_non_template_files( + format!( + "{}/common/charts/nginx-ingress", + self.context().lib_root_dir() + ), + into_dir.as_str(), + )?; + // do exec helm upgrade and return the last deployment status + let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history_with_override( + kubernetes_config_file_path.as_str(), + environment.namespace(), + format!("custom-{}", helm_release_name).as_str(), + into_dir.as_str(), + format!("{}/nginx-ingress.yaml", into_dir.as_str()).as_str(), + self.aws_credentials_envs(aws).to_vec(), + )?; + + // check deployment status + if helm_history_row.is_none() || !helm_history_row.unwrap().is_successfully_deployed() { + return Err(ServiceError::OnCreateFailed); + } + // waiting for the nlb, it should be deploy to get fqdn + let external_ingress_hostname_custom_result = + retry::retry(Fibonacci::from_millis(3000).take(10), || { + let external_ingress_hostname_custom = + crate::cmd::kubectl::kubectl_exec_get_external_ingress_hostname( + kubernetes_config_file_path.as_str(), + environment.namespace(), + format!( + "app=nginx-ingress,component=controller,release=custom-{}", + helm_release_name + ) + .as_str(), + self.aws_credentials_envs(aws).to_vec(), + ); + match external_ingress_hostname_custom { + Ok(external_ingress_hostname_custom) => { + OperationResult::Ok(external_ingress_hostname_custom) + } + Err(err) => { + error!( + "Waiting NLB endpoint to be available to be able to configure TLS" + ); + OperationResult::Retry(err) + } + } + }); + match external_ingress_hostname_custom_result { + Ok(elb) => { + //put it in the context + context.insert("nlb_ingress_hostname", &elb); + } + Err(e) => error!("Error getting the NLB endpoint to be able to configure TLS"), + } + } + let from_dir = format!("{}/aws/charts/q-ingress-tls", self.context.lib_root_dir()); + let _ = crate::template::generate_and_copy_all_files_into_dir( + from_dir.as_str(), + workspace_dir.as_str(), + &context, + )?; + + // do exec helm upgrade and return the last deployment status + let helm_history_row = crate::cmd::helm::helm_exec_with_upgrade_history( + kubernetes_config_file_path.as_str(), + environment.namespace(), + helm_release_name.as_str(), + workspace_dir.as_str(), + self.aws_credentials_envs(aws).to_vec(), + )?; + + if helm_history_row.is_none() || !helm_history_row.unwrap().is_successfully_deployed() { + return Err(ServiceError::OnCreateFailed); + } + + Ok(()) + } + + fn on_create_check(&self) -> Result<(), ServiceError> { + // Todo: manage it properly to avoid timeouts + //self.check_domains() + Ok(()) + } + + fn on_create_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!("AWS.router.on_create_error() called for {}", self.name()); + self.delete(target) + } +} + +impl Pause for Router { + fn on_pause(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + info!("AWS.router.on_pause() called for {}", self.name()); + self.delete(target) + } + + fn on_pause_check(&self) -> Result<(), ServiceError> { + warn!("AWS.router.on_pause_error() called for {}", self.name()); + // TODO check resource has been cleaned? + Ok(()) + } + + fn on_pause_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + self.delete(target) + } +} + +impl Delete for Router { + fn on_delete(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + info!("AWS.router.on_delete() called for {}", self.name()); + self.delete(target) + } + + fn on_delete_check(&self) -> Result<(), ServiceError> { + Ok(()) + } + + fn on_delete_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError> { + warn!("AWS.router.on_delete_error() called for {}", self.name()); + self.delete(target) + } +} + +pub struct CustomDomain { + pub domain: String, + pub target_domain: String, +} + +#[derive(Serialize, Deserialize)] +struct CustomDomainDataTemplate { + pub domain: String, + pub domain_hash: String, + pub target_domain: String, +} + +pub struct Route { + pub path: String, + pub application_name: String, +} + +#[derive(Serialize, Deserialize)] +struct RouteDataTemplate { + pub path: String, + pub application_name: String, + pub application_port: u16, +} diff --git a/src/cloud_provider/digitalocean/mod.rs b/src/cloud_provider/digitalocean/mod.rs new file mode 100644 index 00000000..f2cb5fe8 --- /dev/null +++ b/src/cloud_provider/digitalocean/mod.rs @@ -0,0 +1,67 @@ +extern crate digitalocean; + +use std::any::Any; +use std::rc::Rc; + +use digitalocean::DigitalOcean; + +use crate::cloud_provider::{CloudProvider, CloudProviderError, Kind, TerraformStateCredentials}; +use crate::models::{Context, Listener, ProgressListener}; + +pub struct DO { + context: Context, + id: String, + pub token: String, +} + +impl DO { + pub fn new(context: Context, id: &str, token: &str) -> Self { + DO { + context, + id: id.to_string(), + token: token.to_string(), + } + } + + pub fn client(&self) -> DigitalOcean { + DigitalOcean::new(self.token.as_str()).unwrap() + } +} + +impl CloudProvider for DO { + fn context(&self) -> &Context { + &self.context + } + + fn kind(&self) -> Kind { + Kind::DO + } + + fn id(&self) -> &str { + self.id.as_str() + } + + fn organization_id(&self) -> &str { + unimplemented!() + } + + fn name(&self) -> &str { + unimplemented!() + } + + fn is_valid(&self) -> Result<(), CloudProviderError> { + unimplemented!() + } + + fn add_listener(&mut self, _listener: Listener) { + unimplemented!() + } + + fn terraform_state_credentials(&self) -> &TerraformStateCredentials { + unimplemented!() + } + + fn as_any(&self) -> &dyn Any { + unimplemented!() + } +} diff --git a/src/cloud_provider/environment.rs b/src/cloud_provider/environment.rs new file mode 100644 index 00000000..e2c33bd9 --- /dev/null +++ b/src/cloud_provider/environment.rs @@ -0,0 +1,100 @@ +use crate::cloud_provider::service::{ServiceError, StatefulService, StatelessService}; +use crate::unit_conversion::cpu_string_to_float; + +pub struct Environment { + namespace: String, + pub kind: Kind, + pub id: String, + pub project_id: String, + pub owner_id: String, + pub organization_id: String, + pub stateless_services: Vec>, + pub stateful_services: Vec>, +} + +impl Environment { + pub fn new( + kind: Kind, + id: &str, + project_id: &str, + owner_id: &str, + organization_id: &str, + stateless_services: Vec>, + stateful_services: Vec>, + ) -> Self { + Environment { + namespace: format!("{}-{}", project_id, id), + kind, + id: id.to_string(), + project_id: project_id.to_string(), + owner_id: owner_id.to_string(), + organization_id: organization_id.to_string(), + stateless_services, + stateful_services, + } + } + + pub fn namespace(&self) -> &str { + self.namespace.as_str() + } + + pub fn is_valid(&self) -> Result<(), ServiceError> { + for service in self.stateful_services.iter() { + match service.is_valid() { + Err(err) => return Err(err), + _ => {} + } + } + + for service in self.stateless_services.iter() { + match service.is_valid() { + Err(err) => return Err(err), + _ => {} + } + } + + Ok(()) + } + + /// compute the required resources for this environment from + /// applications, external services, routers, and databases + /// Note: Even if external services don't run on the targeted Kubernetes cluster, it requires CPU and memory resources to run the container(s) + pub fn required_resources(&self) -> EnvironmentResources { + let mut total_cpu_for_stateless_services: f32 = 0.0; + let mut total_ram_in_mib_for_stateless_services: u32 = 0; + + for service in &self.stateless_services { + total_cpu_for_stateless_services += cpu_string_to_float(&service.total_cpus()); + total_ram_in_mib_for_stateless_services += &service.total_ram_in_mib(); + } + + let mut total_cpu_for_stateful_services: f32 = 0.0; + let mut total_ram_in_mib_for_stateful_services: u32 = 0; + match self.kind { + Kind::Development => { + // development means databases are running on Kubernetes + for service in &self.stateful_services { + total_cpu_for_stateful_services += cpu_string_to_float(&service.total_cpus()); + total_ram_in_mib_for_stateful_services += &service.total_ram_in_mib(); + } + } + Kind::Production => {} // production means databases are running on managed services - so it consumes 0 cpu + }; + + EnvironmentResources { + cpu: total_cpu_for_stateless_services + total_cpu_for_stateful_services, + ram_in_mib: total_ram_in_mib_for_stateless_services + + total_ram_in_mib_for_stateless_services, + } + } +} + +pub enum Kind { + Production, + Development, +} + +pub struct EnvironmentResources { + pub cpu: f32, + pub ram_in_mib: u32, +} diff --git a/src/cloud_provider/gcp/mod.rs b/src/cloud_provider/gcp/mod.rs new file mode 100644 index 00000000..a752a74a --- /dev/null +++ b/src/cloud_provider/gcp/mod.rs @@ -0,0 +1,61 @@ +use std::any::Any; +use std::rc::Rc; + +use crate::cloud_provider::{CloudProvider, CloudProviderError, Kind, TerraformStateCredentials}; +use crate::models::{Context, Listener, ProgressListener}; + +pub struct GCP { + context: Context, + id: String, + name: String, + p12_file_content: String, +} + +impl GCP { + pub fn new(context: Context, id: &str, name: &str, p12_file_content: &str) -> Self { + GCP { + context, + id: id.to_string(), + name: name.to_string(), + p12_file_content: p12_file_content.to_string(), + } + } +} + +impl<'x> CloudProvider for GCP { + fn context(&self) -> &Context { + &self.context + } + + fn kind(&self) -> Kind { + Kind::GCP + } + + fn id(&self) -> &str { + self.id.as_str() + } + + fn organization_id(&self) -> &str { + unimplemented!() + } + + fn name(&self) -> &str { + self.name.as_str() + } + + fn is_valid(&self) -> Result<(), CloudProviderError> { + Ok(()) + } + + fn add_listener(&mut self, _listener: Listener) { + // TODO + } + + fn terraform_state_credentials(&self) -> &TerraformStateCredentials { + unimplemented!() + } + + fn as_any(&self) -> &dyn Any { + self + } +} diff --git a/src/cloud_provider/kubernetes.rs b/src/cloud_provider/kubernetes.rs new file mode 100644 index 00000000..4402ad3e --- /dev/null +++ b/src/cloud_provider/kubernetes.rs @@ -0,0 +1,176 @@ +use std::any::Any; +use std::process::ExitStatus; +use std::rc::Rc; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::cloud_provider::environment::Environment; +use crate::cloud_provider::service::ServiceError; +use crate::cloud_provider::CloudProvider; +use crate::cmd::utilities::CmdError; +use crate::dns_provider::DnsProvider; +use crate::models::{Context, Listener, Listeners, ProgressListener}; + +pub trait Kubernetes { + fn context(&self) -> &Context; + fn kind(&self) -> Kind; + fn id(&self) -> &str; + fn name(&self) -> &str; + fn version(&self) -> &str; + fn region(&self) -> &str; + fn cloud_provider(&self) -> &dyn CloudProvider; + fn dns_provider(&self) -> &dyn DnsProvider; + fn is_valid(&self) -> Result<(), KubernetesError>; + fn add_listener(&mut self, listener: Listener); + fn listeners(&self) -> &Listeners; + fn resources(&self, environment: &Environment) -> Result; + fn on_create(&self) -> Result<(), KubernetesError>; + fn on_create_error(&self) -> Result<(), KubernetesError>; + fn on_upgrade(&self) -> Result<(), KubernetesError>; + fn on_upgrade_error(&self) -> Result<(), KubernetesError>; + fn on_downgrade(&self) -> Result<(), KubernetesError>; + fn on_downgrade_error(&self) -> Result<(), KubernetesError>; + fn on_delete(&self) -> Result<(), KubernetesError>; + fn on_delete_error(&self) -> Result<(), KubernetesError>; + fn deploy_environment(&self, environment: &Environment) -> Result<(), KubernetesError>; + fn deploy_environment_error(&self, environment: &Environment) -> Result<(), KubernetesError>; + fn pause_environment(&self, environment: &Environment) -> Result<(), KubernetesError>; + fn pause_environment_error(&self, environment: &Environment) -> Result<(), KubernetesError>; + fn delete_environment(&self, environment: &Environment) -> Result<(), KubernetesError>; + fn delete_environment_error(&self, environment: &Environment) -> Result<(), KubernetesError>; +} + +pub trait KubernetesNode { + fn total_cpu(&self) -> u8; + fn total_memory_in_gib(&self) -> u16; + fn instance_type(&self) -> &str; + fn as_any(&self) -> &dyn Any; +} + +#[derive(Serialize, Deserialize, Clone)] +pub enum Kind { + EKS, +} + +#[derive(Debug)] +pub struct Resources { + pub free_cpu: f32, + pub max_cpu: f32, + pub free_ram_in_mib: u32, + pub max_ram_in_mib: u32, + pub free_pods: u16, + pub max_pods: u16, + pub running_nodes: u16, +} + +#[derive(Debug)] +pub enum KubernetesError { + Cmd(CmdError), + Io(std::io::Error), + Create(ExitStatus), + Deploy(ServiceError), + Pause(ServiceError), + Delete(ServiceError), + Error, +} + +impl From for KubernetesError { + fn from(error: std::io::Error) -> Self { + KubernetesError::Io(error) + } +} + +impl From for KubernetesError { + fn from(error: CmdError) -> Self { + KubernetesError::Cmd(error) + } +} + +impl From for Option { + fn from(item: KubernetesError) -> Self { + return match item { + KubernetesError::Deploy(e) | KubernetesError::Pause(e) | KubernetesError::Delete(e) => { + Option::from(e) + } + _ => None, + }; + } +} + +/// check that there is enough CPU and RAM, and pods resources +/// before starting to deploy stateful and stateless services +pub fn check_kubernetes_has_enough_resources_to_deploy_environment( + kubernetes: &dyn Kubernetes, + environment: &Environment, +) -> Result<(), KubernetesError> { + let resources = kubernetes.resources(environment)?; + let required_resources = environment.required_resources(); + + if required_resources.cpu > resources.free_cpu + && required_resources.ram_in_mib > resources.free_ram_in_mib + { + // not enough cpu and ram to deploy environment + return Err(KubernetesError::Deploy(ServiceError::NotEnoughResources( + format!( + "There is not enough CPU and RAM resources on the Kubernetes '{}' cluster. \ + {} CPU and {}mib RAM requested. \ + {} CPU and {}mib RAM available. \ + Consider to add one more node or upgrade your nodes configuration.", + kubernetes.name(), + required_resources.cpu, + required_resources.ram_in_mib, + resources.free_cpu, + resources.free_ram_in_mib, + ), + ))); + } else if required_resources.cpu > resources.free_cpu { + // not enough cpu to deploy environment + return Err(KubernetesError::Deploy(ServiceError::NotEnoughResources( + format!( + "There is not enough free CPU on the Kubernetes '{}' cluster. \ + {} CPU requested. {} CPU available. \ + Consider to add one more node or upgrade your nodes configuration.", + kubernetes.name(), + required_resources.cpu, + resources.free_cpu, + ), + ))); + } else if required_resources.ram_in_mib > resources.free_ram_in_mib { + // not enough ram to deploy environment + return Err(KubernetesError::Deploy(ServiceError::NotEnoughResources( + format!( + "There is not enough free RAM on the Kubernetes cluster '{}'. \ + {}mib RAM requested. \ + {}mib RAM available. \ + Consider to add one more node or upgrade your nodes configuration.", + kubernetes.name(), + required_resources.ram_in_mib, + resources.free_ram_in_mib, + ), + ))); + } + + let mut required_pods = environment.stateless_services.len() as u16; + + match environment.kind { + crate::cloud_provider::environment::Kind::Production => {} + crate::cloud_provider::environment::Kind::Development => { + required_pods += environment.stateful_services.len() as u16; + } + } + + if required_pods > resources.free_pods { + // not enough free pods on the cluster + return Err(KubernetesError::Deploy(ServiceError::NotEnoughResources( + format!( + "There is not enough free Pods ({} required) on the Kubernetes cluster '{}'. \ + Consider to add one more node or upgrade your nodes configuration.", + required_pods, + kubernetes.name(), + ), + ))); + } + + Ok(()) +} diff --git a/src/cloud_provider/mod.rs b/src/cloud_provider/mod.rs new file mode 100644 index 00000000..9dfd64c8 --- /dev/null +++ b/src/cloud_provider/mod.rs @@ -0,0 +1,86 @@ +use std::any::Any; +use std::rc::Rc; + +use rusoto_core::RusotoError; +use serde::{Deserialize, Serialize}; + +use crate::cloud_provider::environment::Environment; +use crate::cloud_provider::kubernetes::Kubernetes; +use crate::models::{Context, Listener, ProgressListener}; + +pub mod aws; +pub mod digitalocean; +pub mod environment; +pub mod gcp; +pub mod kubernetes; +pub mod service; + +pub trait CloudProvider { + fn context(&self) -> &Context; + fn kind(&self) -> Kind; + fn id(&self) -> &str; + fn organization_id(&self) -> &str; + fn name(&self) -> &str; + fn is_valid(&self) -> Result<(), CloudProviderError>; + fn add_listener(&mut self, listener: Listener); + fn terraform_state_credentials(&self) -> &TerraformStateCredentials; + fn as_any(&self) -> &dyn Any; +} + +#[derive(Debug)] +pub enum CloudProviderError { + Credentials, + Error(Box), + Unknown, +} + +impl From> for CloudProviderError { + fn from(error: Box) -> Self { + CloudProviderError::Error(error) + } +} + +impl From> for CloudProviderError { + fn from(error: RusotoError) -> Self { + match error { + RusotoError::Credentials(_) => CloudProviderError::Credentials, + RusotoError::Service(_) => CloudProviderError::Unknown, + RusotoError::HttpDispatch(_) => CloudProviderError::Unknown, + RusotoError::Validation(_) => CloudProviderError::Unknown, + RusotoError::ParseError(_) => CloudProviderError::Unknown, + RusotoError::Unknown(e) => { + if e.status == 403 { + CloudProviderError::Credentials + } else { + CloudProviderError::Unknown + } + } + RusotoError::Blocking => CloudProviderError::Unknown, + } + } +} + +#[derive(Debug)] +pub enum DeployError { + Error, +} + +#[derive(Serialize, Deserialize, Clone)] +pub enum Kind { + AWS, + GCP, + DO, +} + +pub struct TerraformStateCredentials { + pub access_key_id: String, + pub secret_access_key: String, + pub region: String, +} + +pub enum DeploymentTarget<'a> { + // ManagedService = Managed by the Cloud Provider (eg. RDS, DynamoDB...) + ManagedServices(&'a dyn Kubernetes, &'a Environment), + // SelfHosted = Kubernetes or anything else that implies management on our side + SelfHosted(&'a dyn Kubernetes, &'a Environment), +} diff --git a/src/cloud_provider/service.rs b/src/cloud_provider/service.rs new file mode 100644 index 00000000..c27c5337 --- /dev/null +++ b/src/cloud_provider/service.rs @@ -0,0 +1,255 @@ +use std::io::Error; +use std::net::TcpStream; +use std::process::id; + +use tera::Context as TeraContext; + +use crate::build_platform::Image; +use crate::cloud_provider::environment::Environment; +use crate::cloud_provider::kubernetes::Kubernetes; +use crate::cloud_provider::DeploymentTarget; +use crate::cmd::utilities::CmdError; +use crate::models::{Context, ProgressScope}; +use crate::transaction::CommitError; + +pub trait Service { + fn context(&self) -> &Context; + fn service_type(&self) -> ServiceType; + fn id(&self) -> &str; + fn name(&self) -> &str; + fn version(&self) -> &str; + fn action(&self) -> &Action; + fn private_port(&self) -> Option; + fn total_cpus(&self) -> String; + fn total_ram_in_mib(&self) -> u32; + fn total_instances(&self) -> u16; + fn is_listening(&self, ip: &str) -> bool { + let private_port = match self.private_port() { + Some(private_port) => private_port, + _ => return false, + }; + + match TcpStream::connect(format!("{}:{}", ip, private_port)) { + Ok(_) => true, + Err(_) => false, + } + } + + fn is_valid(&self) -> Result<(), ServiceError> { + let binaries = ["kubectl", "helm", "terraform", "aws-iam-authenticator"]; + + for binary in binaries.iter() { + if !crate::cmd::utilities::does_binary_exist(binary) { + let err = format!("{} binary not found", binary); + return Err(ServiceError::Unexpected(err)); + } + } + + // TODO check lib directories available + + Ok(()) + } + + fn default_tera_context( + &self, + kubernetes: &dyn Kubernetes, + environment: &Environment, + ) -> TeraContext { + let mut context = TeraContext::new(); + + context.insert("id", self.id()); + context.insert("owner_id", environment.owner_id.as_str()); + context.insert("project_id", environment.project_id.as_str()); + context.insert("organization_id", environment.organization_id.as_str()); + context.insert("environment_id", environment.id.as_str()); + context.insert("region", kubernetes.region()); + context.insert("name", self.name()); + context.insert("namespace", environment.namespace()); + context.insert("cluster_name", kubernetes.name()); + context.insert("total_cpus", &self.total_cpus()); + context.insert("total_ram_in_mib", &self.total_ram_in_mib()); + context.insert("total_instances", &self.total_instances()); + + context.insert("is_private_port", &self.private_port().is_some()); + if self.private_port().is_some() { + context.insert("private_port", &self.private_port().unwrap()); + } + + context.insert("version", self.version()); + + context + } + + fn progress_scope(&self) -> ProgressScope { + let id = self.id().to_string(); + + match self.service_type() { + ServiceType::Application => ProgressScope::Application { id }, + ServiceType::ExternalService => ProgressScope::ExternalService { id }, + ServiceType::Database(_) => ProgressScope::Database { id }, + ServiceType::Router => ProgressScope::Router { id }, + } + } +} + +pub trait StatelessService: Service + Create + Pause + Delete { + fn exec_action(&self, deployment_target: &DeploymentTarget) -> Result<(), ServiceError> { + match self.action() { + crate::cloud_provider::service::Action::Create => self.on_create(deployment_target), + crate::cloud_provider::service::Action::Delete => self.on_delete(deployment_target), + crate::cloud_provider::service::Action::Pause => self.on_pause(deployment_target), + crate::cloud_provider::service::Action::Nothing => Ok(()), + } + } +} + +pub trait StatefulService: + Service + Create + Pause + Delete + Backup + Clone + Upgrade + Downgrade +{ + fn exec_action(&self, deployment_target: &DeploymentTarget) -> Result<(), ServiceError> { + match self.action() { + crate::cloud_provider::service::Action::Create => self.on_create(deployment_target), + crate::cloud_provider::service::Action::Delete => self.on_delete(deployment_target), + crate::cloud_provider::service::Action::Pause => self.on_pause(deployment_target), + crate::cloud_provider::service::Action::Nothing => Ok(()), + } + } +} + +pub trait Application: StatelessService { + fn image(&self) -> &Image; + fn set_image(&mut self, image: Image); +} + +pub trait ExternalService: Application {} + +pub trait Router: StatelessService { + fn check_domains(&self) -> Result<(), ServiceError>; +} + +pub trait Database: StatefulService {} + +pub trait Create { + fn on_create(&self, target: &DeploymentTarget) -> Result<(), ServiceError>; + fn on_create_check(&self) -> Result<(), ServiceError>; + fn on_create_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError>; +} + +pub trait Pause { + fn on_pause(&self, target: &DeploymentTarget) -> Result<(), ServiceError>; + fn on_pause_check(&self) -> Result<(), ServiceError>; + fn on_pause_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError>; +} + +pub trait Delete { + fn on_delete(&self, target: &DeploymentTarget) -> Result<(), ServiceError>; + fn on_delete_check(&self) -> Result<(), ServiceError>; + fn on_delete_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError>; +} + +pub trait Backup { + fn on_backup(&self, target: &DeploymentTarget) -> Result<(), ServiceError>; + fn on_backup_check(&self) -> Result<(), ServiceError>; + fn on_backup_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError>; + fn on_restore(&self, target: &DeploymentTarget) -> Result<(), ServiceError>; + fn on_restore_check(&self) -> Result<(), ServiceError>; + fn on_restore_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError>; +} + +pub trait Clone { + fn on_clone(&self, target: &DeploymentTarget) -> Result<(), ServiceError>; + fn on_clone_check(&self) -> Result<(), ServiceError>; + fn on_clone_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError>; +} + +pub trait Upgrade { + fn on_upgrade(&self, target: &DeploymentTarget) -> Result<(), ServiceError>; + fn on_upgrade_check(&self) -> Result<(), ServiceError>; + fn on_upgrade_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError>; +} + +pub trait Downgrade { + fn on_downgrade(&self, target: &DeploymentTarget) -> Result<(), ServiceError>; + fn on_downgrade_check(&self) -> Result<(), ServiceError>; + fn on_downgrade_error(&self, target: &DeploymentTarget) -> Result<(), ServiceError>; +} + +#[derive(Clone, Eq, PartialEq, Hash)] +pub enum Action { + Create, + Pause, + Delete, + Nothing, +} + +#[derive(Eq, PartialEq)] +pub struct DatabaseOptions { + pub login: String, + pub password: String, + pub host: String, + pub port: u16, + pub disk_size_in_gib: u32, + pub database_disk_type: String, +} + +#[derive(Eq, PartialEq)] +pub enum DatabaseType<'a> { + PostgreSQL(&'a DatabaseOptions), + MongoDB(&'a DatabaseOptions), + MySQL(&'a DatabaseOptions), +} + +#[derive(Eq, PartialEq)] +pub enum ServiceType<'a> { + Application, + ExternalService, + Database(DatabaseType<'a>), + Router, +} + +impl<'a> ServiceType<'a> { + pub fn name(&self) -> &str { + match self { + ServiceType::Application => "Application", + ServiceType::ExternalService => "ExternalService", + ServiceType::Database(_) => "Database", + ServiceType::Router => "Router", + } + } +} + +#[derive(Debug)] +pub enum ServiceError { + OnCreateFailed, + CheckFailed, + Cmd(CmdError), + Io(Error), + NotEnoughResources(String), + Unexpected(String), +} + +impl From for ServiceError { + fn from(err: Error) -> Self { + ServiceError::Io(err) + } +} + +impl From for ServiceError { + fn from(err: CmdError) -> Self { + ServiceError::Cmd(err) + } +} + +impl From for Option { + fn from(err: CommitError) -> Self { + return match err { + CommitError::DeleteEnvironment(e) + | CommitError::PauseEnvironment(e) + | CommitError::DeployEnvironment(e) + | CommitError::DeleteKubernetes(e) + | CommitError::CreateKubernetes(e) => Option::from(e), + CommitError::NotValidService(e) => Option::Some(e), + _ => None, + }; + } +} diff --git a/src/cmd/helm.rs b/src/cmd/helm.rs new file mode 100644 index 00000000..72bd4739 --- /dev/null +++ b/src/cmd/helm.rs @@ -0,0 +1,387 @@ +use std::ffi::OsStr; +use std::fmt::{Display, Formatter}; +use std::io::Error; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::process::{Child, Command, ExitStatus, Stdio}; + +use dirs::home_dir; +use retry::delay::Fibonacci; +use retry::OperationResult; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::cmd::structs::{Helm, HelmHistoryRow}; +use crate::cmd::utilities::{exec_with_envs_and_output, CmdError}; +use crate::constants::{KUBECONFIG, TF_PLUGIN_CACHE_DIR}; + +pub fn helm_exec_with_upgrade_history

( + kubernetes_config: P, + namespace: &str, + release_name: &str, + chart_root_dir: P, + envs: Vec<(&str, &str)>, +) -> Result, CmdError> +where + P: AsRef, +{ + // do exec helm upgrade + info!( + "exec helm upgrade for namespace {} and chart {}", + namespace, + chart_root_dir.as_ref().to_str().unwrap() + ); + + let _ = helm_exec_upgrade( + kubernetes_config.as_ref(), + namespace, + release_name, + chart_root_dir.as_ref(), + envs.clone(), + )?; + + // list helm history + info!( + "exec helm history for namespace {} and chart {}", + namespace, + chart_root_dir.as_ref().to_str().unwrap() + ); + + let helm_history_rows = + helm_exec_history(kubernetes_config.as_ref(), namespace, release_name, envs)?; + + // take the last deployment from helm history - or return none if there is no history + Ok(match helm_history_rows.first() { + Some(helm_history_row) => Some(helm_history_row.clone()), + None => None, + }) +} + +pub fn helm_exec_upgrade

( + kubernetes_config: P, + namespace: &str, + release_name: &str, + chart_root_dir: P, + envs: Vec<(&str, &str)>, +) -> Result<(), CmdError> +where + P: AsRef, +{ + helm_exec_with_output( + vec![ + "upgrade", + "--kubeconfig", + kubernetes_config.as_ref().to_str().unwrap(), + "--create-namespace", + "--install", + "--history-max", + "50", + "--wait", + "--namespace", + namespace, + release_name, + chart_root_dir.as_ref().to_str().unwrap(), + ], + envs, + |out| match out { + Ok(line) => info!("{}", line.as_str()), + Err(err) => error!("{}", err), + }, + |out| match out { + Ok(line) => error!("{}", line.as_str()), + Err(err) => error!("{}", err), + }, + ) +} + +pub fn helm_exec_uninstall

( + kubernetes_config: P, + namespace: &str, + release_name: &str, + envs: Vec<(&str, &str)>, +) -> Result<(), CmdError> +where + P: AsRef, +{ + helm_exec_with_output( + vec![ + "uninstall", + "--kubeconfig", + kubernetes_config.as_ref().to_str().unwrap(), + "--namespace", + namespace, + release_name, + ], + envs, + |out| match out { + Ok(line) => info!("{}", line.as_str()), + Err(err) => error!("{}", err), + }, + |out| match out { + Ok(line) => error!("{}", line.as_str()), + Err(err) => error!("{}", err), + }, + ) +} + +pub fn helm_exec_history

( + kubernetes_config: P, + namespace: &str, + release_name: &str, + envs: Vec<(&str, &str)>, +) -> Result, CmdError> +where + P: AsRef, +{ + let mut output_string = String::new(); + match helm_exec_with_output( + // WARN: do not add argument --debug, otherwise JSON decoding will not work + vec![ + "history", + "--kubeconfig", + kubernetes_config.as_ref().to_str().unwrap(), + "--namespace", + namespace, + "-o", + "json", + release_name, + ], + envs, + |out| match out { + Ok(line) => output_string = line, + Err(err) => error!("{:?}", err), + }, + |out| match out { + Ok(line) => error!("{}", line), + Err(err) => error!("{:?}", err), + }, + ) { + Ok(_) => info!("Helm history success for release name: {}", release_name), + Err(_) => info!("Helm history found for release name: {}", release_name), + }; + // TODO better check, release not found + + let mut results = match serde_json::from_str::>(output_string.as_str()) { + Ok(x) => x, + Err(_) => vec![], + }; + + // unsort results by revision number + let _ = results.sort_by_key(|x| x.revision); + // there is no performance penalty to do it in 2 operations instead of one, but who really cares anyway + let _ = results.reverse(); + + Ok(results) +} + +pub fn helm_uninstall_list

( + kubernetes_config: P, + helmlist: Vec, + envs: Vec<(&str, &str)>, +) -> Result +where + P: AsRef, +{ + let mut output_vec: Vec = Vec::new(); + let helmlist_string = helmlist.join(" "); + match helm_exec_with_output( + vec![ + "uninstall", + helmlist_string.as_str(), + "--kubeconfig", + kubernetes_config.as_ref().to_str().unwrap(), + ], + envs, + |out| match out { + Ok(line) => output_vec.push(line), + Err(err) => error!("{:?}", err), + }, + |out| match out { + Ok(line) => error!("{}", line), + Err(err) => error!("{:?}", err), + }, + ) { + Ok(_) => info!("Helm uninstall fail with : {}", helmlist_string.clone()), + Err(_) => info!( + "Helm history found for release name: {}", + helmlist_string.clone() + ), + }; + Ok(output_vec.join("")) +} + +pub fn helm_exec_upgrade_with_override_file

( + kubernetes_config: P, + namespace: &str, + release_name: &str, + chart_root_dir: P, + override_file: &str, + envs: Vec<(&str, &str)>, +) -> Result<(), CmdError> +where + P: AsRef, +{ + helm_exec_with_output( + vec![ + "upgrade", + "--kubeconfig", + kubernetes_config.as_ref().to_str().unwrap(), + "--create-namespace", + "--install", + "--history-max", + "50", + "--wait", + "--namespace", + namespace, + release_name, + chart_root_dir.as_ref().to_str().unwrap(), + "-f", + override_file, + ], + envs, + |out| match out { + Ok(line) => info!("{}", line.as_str()), + Err(err) => error!("{}", err), + }, + |out| match out { + // don't crash errors if releases are not found + Ok(line) if line.contains("Error: release: not found") => info!("{}", line.as_str()), + Ok(line) => error!("{}", line.as_str()), + Err(err) => error!("{}", err), + }, + ) +} + +pub fn helm_exec_with_upgrade_history_with_override

( + kubernetes_config: P, + namespace: &str, + release_name: &str, + chart_root_dir: P, + override_file: &str, + envs: Vec<(&str, &str)>, +) -> Result, CmdError> +where + P: AsRef, +{ + // do exec helm upgrade + info!( + "exec helm upgrade for namespace {} and chart {}", + namespace, + chart_root_dir.as_ref().to_str().unwrap() + ); + + let _ = helm_exec_upgrade_with_override_file( + kubernetes_config.as_ref(), + namespace, + release_name, + chart_root_dir.as_ref(), + override_file, + envs.clone(), + )?; + + // list helm history + info!( + "exec helm history for namespace {} and chart {}", + namespace, + chart_root_dir.as_ref().to_str().unwrap() + ); + + let helm_history_rows = + helm_exec_history(kubernetes_config.as_ref(), namespace, release_name, envs)?; + + // take the last deployment from helm history - or return none if there is no history + Ok(match helm_history_rows.first() { + Some(helm_history_row) => Some(helm_history_row.clone()), + None => None, + }) +} + +pub fn helm_list

(kubernetes_config: P, envs: Vec<(&str, &str)>) -> Result, CmdError> +where + P: AsRef, +{ + let mut output_vec: Vec = Vec::new(); + helm_exec_with_output( + vec![ + "list", + "-A", + "--kubeconfig", + kubernetes_config.as_ref().to_str().unwrap(), + "-o", + "json", + ], + envs, + |out| match out { + Ok(line) => output_vec.push(line), + Err(err) => error!("{}", err), + }, + |out| match out { + Ok(line) => error!("{}", line.as_str()), + Err(err) => error!("{}", err), + }, + ); + let output_string: String = output_vec.join(""); + let values = serde_json::from_str::>(output_string.as_str()); + let mut helms_name: Vec = Vec::new(); + match values { + Ok(all_helms) => { + for helm in all_helms { + helms_name.push(helm.name) + } + } + Err(e) => { + error!("Error while deserializing all helms names {}", e); + return Err(CmdError::Io(Error::new(std::io::ErrorKind::InvalidData, e))); + } + } + Ok(helms_name) +} + +pub fn helm_exec(args: Vec<&str>, envs: Vec<(&str, &str)>) -> Result<(), CmdError> { + helm_exec_with_output( + args, + envs, + |line| { + info!("{}", line.unwrap()); + }, + |line| { + error!("{}", line.unwrap()); + }, + ) +} + +pub fn helm_exec_with_output( + args: Vec<&str>, + envs: Vec<(&str, &str)>, + stdout_output: F, + stderr_output: X, +) -> Result<(), CmdError> +where + F: FnMut(Result), + X: FnMut(Result), +{ + match exec_with_envs_and_output("helm", args, envs, stdout_output, stderr_output) { + Err(err) => return Err(err), + _ => {} + }; + + Ok(()) +} + +pub fn kubectl_exec_with_output( + args: Vec<&str>, + envs: Vec<(&str, &str)>, + stdout_output: F, + stderr_output: X, +) -> Result<(), CmdError> +where + F: FnMut(Result), + X: FnMut(Result), +{ + match exec_with_envs_and_output("kubectl", args, envs, stdout_output, stderr_output) { + Err(err) => return Err(err), + _ => {} + }; + + Ok(()) +} diff --git a/src/cmd/kubectl.rs b/src/cmd/kubectl.rs new file mode 100644 index 00000000..91483e9c --- /dev/null +++ b/src/cmd/kubectl.rs @@ -0,0 +1,630 @@ +use std::ffi::OsStr; +use std::fmt::{Display, Formatter}; +use std::io::Error; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::process::{Child, Command, ExitStatus, Stdio}; + +use dirs::home_dir; +use retry::delay::Fibonacci; +use retry::OperationResult; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::cmd::structs::{ + Item, KubernetesJob, KubernetesList, KubernetesNode, KubernetesPod, KubernetesPodStatusPhase, + KubernetesService, +}; +use crate::cmd::utilities::{exec_with_envs_and_output, CmdError}; +use crate::constants::{KUBECONFIG, TF_PLUGIN_CACHE_DIR}; + +pub fn kubectl_exec_with_output( + args: Vec<&str>, + envs: Vec<(&str, &str)>, + stdout_output: F, + stderr_output: X, +) -> Result<(), CmdError> +where + F: FnMut(Result), + X: FnMut(Result), +{ + match exec_with_envs_and_output("kubectl", args, envs, stdout_output, stderr_output) { + Err(err) => return Err(err), + _ => {} + }; + + Ok(()) +} + +pub fn kubectl_exec_get_external_ingress_hostname

( + kubernetes_config: P, + namespace: &str, + selector: &str, + envs: Vec<(&str, &str)>, +) -> Result, CmdError> +where + P: AsRef, +{ + let mut _envs = Vec::with_capacity(envs.len() + 1); + _envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap())); + _envs.extend(envs); + + let mut output_vec: Vec = Vec::with_capacity(20); + let _ = kubectl_exec_with_output( + vec![ + "get", "svc", "-o", "json", "-n", namespace, "-l", // selector + selector, + ], + _envs, + |out| match out { + Ok(line) => output_vec.push(line), + Err(err) => error!("{:?}", err), + }, + |out| match out { + Ok(line) => error!("{}", line), + Err(err) => error!("{:?}", err), + }, + )?; + + let output_string: String = output_vec.join(""); + + let result = + match serde_json::from_str::>(output_string.as_str()) { + Ok(x) => x, + Err(err) => { + error!("{:?}", err); + error!("{}", output_string.as_str()); + return Err(CmdError::Io(Error::new( + std::io::ErrorKind::InvalidData, + output_string, + ))); + } + }; + + if result.items.is_empty() + || result + .items + .first() + .unwrap() + .status + .load_balancer + .ingress + .is_empty() + { + return Ok(None); + } + + // FIXME unsafe unwrap here? + Ok(Some( + result + .items + .first() + .unwrap() + .status + .load_balancer + .ingress + .first() + .unwrap() + .hostname + .clone(), + )) +} + +pub fn kubectl_exec_is_pod_ready_with_retry

( + kubernetes_config: P, + namespace: &str, + selector: &str, + envs: Vec<(&str, &str)>, +) -> Result, CmdError> +where + P: AsRef, +{ + // TODO check this + let result = retry::retry(Fibonacci::from_millis(3000).take(10), || { + let r = crate::cmd::kubectl::kubectl_exec_is_pod_ready( + kubernetes_config.as_ref(), + namespace, + selector, + envs.clone(), + ); + + match r { + Ok(is_ready) => match is_ready { + Some(true) => OperationResult::Ok(true), + _ => { + let t = format!("pod with selector: {} is not ready yet", selector); + info!("{}", t.as_str()); + OperationResult::Retry(t) + } + }, + Err(err) => OperationResult::Err(format!("command error: {:?}", err)), + } + }); + + match result { + Err(err) => match err { + retry::Error::Operation { + error: _, + total_delay: _, + tries: _, + } => Ok(Some(false)), + retry::Error::Internal(err) => Err(CmdError::Unexpected(err)), + }, + Ok(_) => Ok(Some(true)), + } +} + +pub fn kubectl_exec_is_pod_ready

( + kubernetes_config: P, + namespace: &str, + selector: &str, + envs: Vec<(&str, &str)>, +) -> Result, CmdError> +where + P: AsRef, +{ + let mut _envs = Vec::with_capacity(envs.len() + 1); + _envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap())); + _envs.extend(envs); + + let mut output_vec: Vec = Vec::with_capacity(20); + let _ = kubectl_exec_with_output( + vec!["get", "pod", "-o", "json", "-n", namespace, "-l", selector], + _envs, + |out| match out { + Ok(line) => output_vec.push(line), + Err(err) => error!("{:?}", err), + }, + |out| match out { + Ok(line) => error!("{}", line), + Err(err) => error!("{:?}", err), + }, + )?; + + let output_string: String = output_vec.join(""); + + let result = match serde_json::from_str::>(output_string.as_str()) + { + Ok(x) => x, + Err(err) => { + error!("{:?}", err); + error!("{}", output_string.as_str()); + return Err(CmdError::Io(Error::new( + std::io::ErrorKind::InvalidData, + output_string, + ))); + } + }; + + if result.items.is_empty() + || result + .items + .first() + .unwrap() + .status + .container_statuses + .is_empty() + { + return Ok(None); + } + + let first_item = result.items.first().unwrap(); + + let is_ready = match first_item.status.phase { + KubernetesPodStatusPhase::Running => true, + _ => false, + }; + + Ok(Some(is_ready)) +} + +pub fn kubectl_exec_is_job_ready_with_retry

( + kubernetes_config: P, + namespace: &str, + job_name: &str, + envs: Vec<(&str, &str)>, +) -> Result, CmdError> +where + P: AsRef, +{ + // TODO check this + let result = retry::retry(Fibonacci::from_millis(3000).take(10), || { + let r = crate::cmd::kubectl::kubectl_exec_is_job_ready( + kubernetes_config.as_ref(), + namespace, + job_name, + envs.clone(), + ); + + match r { + Ok(is_ready) => match is_ready { + Some(true) => OperationResult::Ok(true), + _ => { + let t = format!("job {} is not ready yet", job_name); + info!("{}", t.as_str()); + OperationResult::Retry(t) + } + }, + Err(err) => OperationResult::Err(format!("command error: {:?}", err)), + } + }); + + match result { + Err(err) => match err { + retry::Error::Operation { + error: _, + total_delay: _, + tries: _, + } => Ok(Some(false)), + retry::Error::Internal(err) => Err(CmdError::Unexpected(err)), + }, + Ok(_) => Ok(Some(true)), + } +} + +pub fn kubectl_exec_is_job_ready

( + kubernetes_config: P, + namespace: &str, + job_name: &str, + envs: Vec<(&str, &str)>, +) -> Result, CmdError> +where + P: AsRef, +{ + let mut _envs = Vec::with_capacity(envs.len() + 1); + _envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap())); + _envs.extend(envs); + + let mut output_vec: Vec = Vec::with_capacity(20); + let _ = kubectl_exec_with_output( + vec!["get", "job", "-o", "json", "-n", namespace, job_name], + _envs, + |out| match out { + Ok(line) => output_vec.push(line), + Err(err) => error!("{:?}", err), + }, + |out| match out { + Ok(line) => error!("{}", line), + Err(err) => error!("{:?}", err), + }, + )?; + + let output_string: String = output_vec.join(""); + + let result = match serde_json::from_str::(output_string.as_str()) { + Ok(x) => x, + Err(err) => { + error!("{:?}", err); + error!("{}", output_string.as_str()); + return Err(CmdError::Io(Error::new( + std::io::ErrorKind::InvalidData, + output_string, + ))); + } + }; + + if result.status.succeeded > 0 { + return Ok(Some(true)); + } + + Ok(Some(false)) +} + +pub fn kubectl_exec_create_namespace

( + kubernetes_config: P, + namespace: &str, + envs: Vec<(&str, &str)>, +) -> Result<(), CmdError> +where + P: AsRef, +{ + let mut _envs = Vec::with_capacity(envs.len() + 1); + _envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap())); + _envs.extend(envs); + + let _ = kubectl_exec_with_output( + vec!["create", "namespace", namespace], + _envs, + |out| match out { + Ok(line) => info!("{}", line), + Err(err) => error!("{:?}", err), + }, + |out| match out { + Ok(line) => error!("{}", line), + Err(err) => error!("{:?}", err), + }, + )?; + + Ok(()) +} + +// used for testing the does_contain_terraform_tfstate + +pub fn create_sample_secret_terraform_in_namespace

( + kubernetes_config: P, + namespace_to_override: &str, + envs: &Vec<(&str, &str)>, +) -> Result +where + P: AsRef, +{ + let mut _envs = Vec::with_capacity(envs.len() + 1); + let mut output_vec: Vec = Vec::new(); + _envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap())); + _envs.extend(envs); + let _ = kubectl_exec_with_output( + vec![ + "create", + "secret", + "tfstate-default-state", + "--from-literal=blablablabla", + "--namespace", + namespace_to_override, + ], + _envs, + |out| match out { + Ok(_line) => output_vec.push(_line), + Err(err) => error!("{:?}", err), + }, + |out| match out { + Ok(_line) => {} + Err(err) => error!("{:?}", err), + }, + ); + Ok(output_vec.join("")) +} + +pub fn does_contain_terraform_tfstate

( + kubernetes_config: P, + namespace: &str, + envs: &Vec<(&str, &str)>, +) -> Result +where + P: AsRef, +{ + let mut _envs = Vec::with_capacity(envs.len() + 1); + _envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap())); + _envs.extend(envs); + let mut exist = true; + let _ = kubectl_exec_with_output( + vec![ + "describe", + "secrets/tfstate-default-state", + "--namespace", + namespace, + ], + _envs, + |out| match out { + Ok(_line) => exist = true, + Err(err) => error!("{:?}", err), + }, + |out| match out { + Ok(_line) => {} + Err(err) => error!("{:?}", err), + }, + )?; + Ok(exist) +} + +pub fn kubectl_exec_get_all_namespaces

( + kubernetes_config: P, + envs: Vec<(&str, &str)>, +) -> Result, Error> +where + P: AsRef, +{ + let mut _envs = Vec::with_capacity(envs.len() + 1); + _envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap())); + _envs.extend(envs); + + let mut output_vec: Vec = Vec::new(); + let _ = kubectl_exec_with_output( + vec!["get", "namespaces", "-o", "json"], + _envs, + |out| match out { + Ok(line) => output_vec.push(line), + Err(err) => error!("{:?}", err), + }, + |out| match out { + Ok(line) => error!("{}", line), + Err(err) => error!("{:?}", err), + }, + )?; + + let mut output_string: String = output_vec.join(""); + let mut to_return: Vec = Vec::new(); + let result = serde_json::from_str::>(output_string.as_str()); + match result { + Ok(out) => { + for item in out.items { + to_return.push(item.metadata.name); + } + } + Err(e) => { + error!("While deserializing Kubernetes namespaces names {}", e); + return Err(Error::from(CmdError::Io(Error::new( + std::io::ErrorKind::InvalidData, + output_string, + )))); + } + }; + Ok(to_return) +} + +pub fn kubectl_exec_delete_namespace

( + kubernetes_config: P, + namespace: &str, + envs: Vec<(&str, &str)>, +) -> Result<(), CmdError> +where + P: AsRef, +{ + match does_contain_terraform_tfstate(&kubernetes_config, &namespace, &envs) { + Ok(exist) => match exist { + true => { + return Err(CmdError::Io(Error::new( + std::io::ErrorKind::Other, + "Namespace contains terraform tfstates in secret, can't delete it !", + ))); + } + false => info!( + "Namespace {} doesn't contain any tfstates, able to delete it", + namespace + ), + }, + Err(e) => warn!( + "Unable to execute describe on secrets: {}. it may not exist anymore?", + e + ), + }; + + let mut _envs = Vec::with_capacity(envs.len() + 1); + _envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap())); + _envs.extend(envs); + + let _ = kubectl_exec_with_output( + vec!["delete", "namespace", namespace], + _envs, + |out| match out { + Ok(line) => info!("{}", line), + Err(err) => error!("{:?}", err), + }, + |out| match out { + Ok(line) => error!("{}", line), + Err(err) => error!("{:?}", err), + }, + )?; + + Ok(()) +} + +pub fn kubectl_exec_delete_secret

( + kubernetes_config: P, + secret: &str, + envs: Vec<(&str, &str)>, +) -> Result<(), CmdError> +where + P: AsRef, +{ + let mut _envs = Vec::with_capacity(envs.len() + 1); + _envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap())); + _envs.extend(envs); + + let _ = kubectl_exec_with_output( + vec!["delete", "secret", secret], + _envs, + |out| match out { + Ok(line) => info!("{}", line), + Err(err) => error!("{:?}", err), + }, + |out| match out { + Ok(line) => error!("{}", line), + Err(err) => error!("{:?}", err), + }, + )?; + + Ok(()) +} + +pub fn kubectl_exec_logs

( + kubernetes_config: P, + namespace: &str, + selector: &str, + envs: Vec<(&str, &str)>, +) -> Result +where + P: AsRef, +{ + let mut _envs = Vec::with_capacity(envs.len() + 1); + _envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap())); + _envs.extend(envs); + + let mut output_vec: Vec = Vec::with_capacity(50); + let _ = kubectl_exec_with_output( + vec!["logs", "--tail", "1000", "-n", namespace, "-l", selector], + _envs, + |out| match out { + Ok(line) => output_vec.push(line), + Err(err) => error!("{:?}", err), + }, + |out| match out { + Ok(line) => error!("{}", line), + Err(err) => error!("{:?}", err), + }, + )?; + + Ok(output_vec.join("\n")) +} + +pub fn kubectl_exec_describe_pod

( + kubernetes_config: P, + namespace: &str, + selector: &str, + envs: Vec<(&str, &str)>, +) -> Result +where + P: AsRef, +{ + let mut _envs = Vec::with_capacity(envs.len() + 1); + _envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap())); + _envs.extend(envs); + + let mut output_vec: Vec = Vec::with_capacity(50); + let _ = kubectl_exec_with_output( + vec!["describe", "pod", "-n", namespace, "-l", selector], + _envs, + |out| match out { + Ok(line) => output_vec.push(line), + Err(err) => error!("{:?}", err), + }, + |out| match out { + Ok(line) => error!("{}", line), + Err(err) => error!("{:?}", err), + }, + )?; + + Ok(output_vec.join("\n")) +} + +pub fn kubectl_exec_get_node

( + kubernetes_config: P, + envs: Vec<(&str, &str)>, +) -> Result, CmdError> +where + P: AsRef, +{ + let mut _envs = Vec::with_capacity(envs.len() + 1); + _envs.push((KUBECONFIG, kubernetes_config.as_ref().to_str().unwrap())); + _envs.extend(envs); + + let mut output_vec: Vec = Vec::with_capacity(50); + let _ = kubectl_exec_with_output( + vec!["get", "node", "-o", "json"], + _envs, + |out| match out { + Ok(line) => output_vec.push(line), + Err(err) => error!("{:?}", err), + }, + |out| match out { + Ok(line) => error!("{}", line), + Err(err) => error!("{:?}", err), + }, + )?; + + let output_string: String = output_vec.join(""); + + let result = + match serde_json::from_str::>(output_string.as_str()) { + Ok(x) => x, + Err(err) => { + error!("{:?}", err); + error!("{}", output_string.as_str()); + return Err(CmdError::Io(Error::new( + std::io::ErrorKind::InvalidData, + output_string, + ))); + } + }; + + Ok(result) +} diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs new file mode 100644 index 00000000..eabeb043 --- /dev/null +++ b/src/cmd/mod.rs @@ -0,0 +1,5 @@ +pub mod helm; +pub mod kubectl; +pub mod structs; +pub mod terraform; +pub mod utilities; diff --git a/src/cmd/structs.rs b/src/cmd/structs.rs new file mode 100644 index 00000000..ed51bf1b --- /dev/null +++ b/src/cmd/structs.rs @@ -0,0 +1,167 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "camelCase")] +pub struct KubernetesList { + pub items: Vec, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "camelCase")] +pub struct KubernetesService { + pub status: KubernetesServiceStatus, +} + +#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Labels { + pub name: String, +} + +#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Spec { + pub finalizers: Vec, +} + +#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Status { + pub phase: String, +} + +#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Metadata2 { + pub resource_version: String, + pub self_link: String, +} + +#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Item { + pub api_version: String, + pub kind: String, + pub metadata: Metadata, + pub spec: Spec, + pub status: Status, +} + +#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Metadata { + pub creation_timestamp: String, + pub labels: Option, + pub name: String, + pub resource_version: String, + pub self_link: String, + pub uid: String, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "camelCase")] +pub struct KubernetesServiceStatus { + pub load_balancer: KubernetesServiceStatusLoadBalancer, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "camelCase")] +pub struct KubernetesServiceStatusLoadBalancer { + pub ingress: Vec, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "camelCase")] +pub struct KubernetesServiceStatusLoadBalancerIngress { + pub hostname: String, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "camelCase")] +pub struct KubernetesPod { + pub status: KubernetesPodStatus, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "camelCase")] +pub struct KubernetesPodStatus { + pub container_statuses: Vec, + // read the doc: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + // phase can be Pending, Running, Succeeded, Failed, Unknown + pub phase: KubernetesPodStatusPhase, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +pub enum KubernetesPodStatusPhase { + Pending, + Running, + Succeeded, + Failed, + Unknown, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "camelCase")] +pub struct KubernetesPodContainerStatus { + pub ready: bool, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "camelCase")] +pub struct KubernetesJob { + pub status: KubernetesJobStatus, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "camelCase")] +pub struct KubernetesJobStatus { + pub succeeded: u32, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "camelCase")] +pub struct KubernetesNode { + pub status: KubernetesNodeStatus, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "camelCase")] +pub struct KubernetesNodeStatus { + pub allocatable: KubernetesNodeStatusResources, + pub capacity: KubernetesNodeStatusResources, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "camelCase")] +pub struct KubernetesNodeStatusResources { + pub cpu: String, + pub memory: String, + pub pods: String, +} + +#[derive(Default, Debug, Clone, PartialEq, serde_derive::Serialize, serde_derive::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Helm { + pub name: String, + pub namespace: String, + pub revision: String, + pub updated: String, + pub status: String, + pub chart: String, + #[serde(rename = "app_version")] + pub app_version: String, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +pub struct HelmHistoryRow { + pub revision: u16, + pub status: String, + pub chart: String, + pub app_version: String, +} + +impl HelmHistoryRow { + pub fn is_successfully_deployed(&self) -> bool { + self.status == "deployed" + } +} diff --git a/src/cmd/terraform.rs b/src/cmd/terraform.rs new file mode 100644 index 00000000..f8039301 --- /dev/null +++ b/src/cmd/terraform.rs @@ -0,0 +1,101 @@ +use std::ffi::OsStr; +use std::fmt::{Display, Formatter}; +use std::io::Error; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::process::{Child, Command, ExitStatus, Stdio}; + +use dirs::home_dir; +use retry::delay::Fibonacci; +use retry::OperationResult; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::cmd::utilities::{exec_with_envs_and_output, CmdError}; +use crate::constants::{KUBECONFIG, TF_PLUGIN_CACHE_DIR}; + +fn terraform_exec_with_init_validate( + root_dir: &str, + first_time_init_terraform: bool, +) -> Result<(), CmdError> { + // terraform init + let init_args = if first_time_init_terraform { + vec!["init"] + } else { + vec!["init"] + }; + + //TODO print + terraform_exec(root_dir, init_args)?; + + // terraform validate config + terraform_exec(root_dir, vec!["validate"])?; + + Ok(()) +} + +fn terraform_exec_with_init_validate_plan( + root_dir: &str, + first_time_init_terraform: bool, +) -> Result<(), CmdError> { + // terraform init + let init_args = if first_time_init_terraform { + vec!["init"] + } else { + vec!["init"] + }; + + //TODO print + terraform_exec(root_dir, init_args)?; + + // terraform validate config + terraform_exec(root_dir, vec!["validate"])?; + + // terraform plan + terraform_exec(root_dir, vec!["plan", "-out", "tf_plan"])?; + + Ok(()) +} + +pub fn terraform_exec_with_init_validate_plan_apply( + root_dir: &str, + first_time_init_terraform: bool, +) -> Result<(), CmdError> { + // terraform init and plan + terraform_exec_with_init_validate_plan(root_dir, first_time_init_terraform); + + // terraform apply + terraform_exec(root_dir, vec!["apply", "-auto-approve", "tf_plan"])?; + + Ok(()) +} + +pub fn terraform_exec_with_init_validate_destroy(root_dir: &str) -> Result<(), CmdError> { + // terraform init and plan + terraform_exec_with_init_validate(root_dir, false); + + // terraform destroy + terraform_exec(root_dir, vec!["destroy", "-auto-approve"]) +} + +pub fn terraform_exec(root_dir: &str, args: Vec<&str>) -> Result<(), CmdError> { + let home_dir = home_dir().expect("Could not find $HOME"); + let tf_plugin_cache_dir = format!("{}/.terraform.d/plugin-cache", home_dir.to_str().unwrap()); + + match exec_with_envs_and_output( + format!("{} terraform", root_dir).as_str(), + args, + vec![(TF_PLUGIN_CACHE_DIR, tf_plugin_cache_dir.as_str())], + |line: Result| { + info!("{}", line.unwrap()); + }, + |line: Result| { + error!("{}", line.unwrap()); + }, + ) { + Err(err) => return Err(err), + _ => {} + }; + + Ok(()) +} diff --git a/src/cmd/utilities.rs b/src/cmd/utilities.rs new file mode 100644 index 00000000..8195d463 --- /dev/null +++ b/src/cmd/utilities.rs @@ -0,0 +1,272 @@ +use std::ffi::OsStr; +use std::fmt::{Display, Formatter}; +use std::io::Error; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::process::{Child, Command, ExitStatus, Stdio}; + +use dirs::home_dir; +use retry::delay::Fibonacci; +use retry::OperationResult; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::constants::{KUBECONFIG, TF_PLUGIN_CACHE_DIR}; + +fn command

(binary: P, args: Vec<&str>, envs: Option>) -> Command +where + P: AsRef, +{ + let s_binary = binary + .as_ref() + .to_str() + .unwrap() + .split_whitespace() + .map(|x| x.to_string()) + .collect::>(); + + let (current_dir, _binary) = if s_binary.len() == 1 { + (None, s_binary.first().unwrap().clone()) + } else { + ( + Some(s_binary.first().unwrap().clone()), + s_binary.get(1).unwrap().clone(), + ) + }; + + let mut cmd = Command::new(&_binary); + + cmd.args(&args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + if current_dir.is_some() { + cmd.current_dir(current_dir.unwrap()); + } + + if envs.is_some() { + envs.unwrap().into_iter().for_each(|(k, v)| { + cmd.env(k, v); + }); + } + + cmd +} + +pub fn exec

(binary: P, args: Vec<&str>) -> Result<(), CmdError> +where + P: AsRef, +{ + let command_string = command_to_string(binary.as_ref(), &args); + info!("command: {}", command_string.as_str()); + + let exit_status = match command(binary, args, None).spawn().unwrap().wait() { + Ok(x) => x, + Err(err) => return Err(CmdError::Io(err)), + }; + + if exit_status.success() { + return Ok(()); + } + + Err(CmdError::Exec(exit_status)) +} + +pub fn exec_with_envs

( + binary: P, + args: Vec<&str>, + envs: Vec<(&str, &str)>, +) -> Result<(), CmdError> +where + P: AsRef, +{ + let command_string = command_with_envs_to_string(binary.as_ref(), &args, &envs); + info!("command: {}", command_string.as_str()); + + let exit_status = match command(binary, args, Some(envs)).spawn().unwrap().wait() { + Ok(x) => x, + Err(err) => return Err(CmdError::Io(err)), + }; + + if exit_status.success() { + return Ok(()); + } + + Err(CmdError::Exec(exit_status)) +} + +fn _with_output(mut child: Child, mut stdout_output: F, mut stderr_output: X) -> Child +where + F: FnMut(Result), + X: FnMut(Result), +{ + let stdout_reader = BufReader::new(child.stdout.as_mut().unwrap()); + for line in stdout_reader.lines() { + stdout_output(line); + } + + let stderr_reader = BufReader::new(child.stderr.as_mut().unwrap()); + for line in stderr_reader.lines() { + stderr_output(line); + } + + child +} + +pub fn exec_with_output( + binary: P, + args: Vec<&str>, + stdout_output: F, + stderr_output: X, +) -> Result<(), CmdError> +where + P: AsRef, + F: FnMut(Result), + X: FnMut(Result), +{ + let command_string = command_to_string(binary.as_ref(), &args); + info!("command: {}", command_string.as_str()); + + let mut child = _with_output( + command(binary, args, None).spawn().unwrap(), + stdout_output, + stderr_output, + ); + + let exit_status = match child.wait() { + Ok(x) => x, + Err(err) => return Err(CmdError::Io(err)), + }; + + if exit_status.success() { + return Ok(()); + } + + Err(CmdError::Exec(exit_status)) +} + +pub fn exec_with_envs_and_output( + binary: P, + args: Vec<&str>, + envs: Vec<(&str, &str)>, + stdout_output: F, + stderr_output: X, +) -> Result<(), CmdError> +where + P: AsRef, + F: FnMut(Result), + X: FnMut(Result), +{ + let command_string = command_with_envs_to_string(binary.as_ref(), &args, &envs); + info!("command: {}", command_string.as_str()); + + let mut child = _with_output( + command(binary, args, Some(envs)).spawn().unwrap(), + stdout_output, + stderr_output, + ); + + let exit_status = match child.wait() { + Ok(x) => x, + Err(err) => return Err(CmdError::Io(err)), + }; + + if exit_status.success() { + return Ok(()); + } + + Err(CmdError::Exec(exit_status)) +} + +// return the output of "binary_name" --version +pub fn run_version_command_for(binary_name: &str) -> String { + let mut output_from_cmd = String::new(); + exec_with_output( + binary_name, + vec!["--version"], + |r_out| match r_out { + Ok(s) => output_from_cmd.push_str(&s.to_owned()), + Err(e) => error!("Error while getting stdout from {} {}", binary_name, e), + }, + |r_err| match r_err { + Ok(s) => error!("Error executing {}", binary_name), + Err(e) => error!("Error while getting stderr from {} {}", binary_name, e), + }, + ); + output_from_cmd +} + +pub fn does_binary_exist(binary: S) -> bool +where + S: AsRef, +{ + match Command::new(binary) + .stdout(Stdio::null()) + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + { + Ok(_) => true, + _ => false, + } +} + +pub fn command_to_string

(binary: P, args: &Vec<&str>) -> String +where + P: AsRef, +{ + format!("{} {}", binary.as_ref().to_str().unwrap(), args.join(" ")) +} + +pub fn command_with_envs_to_string

( + binary: P, + args: &Vec<&str>, + envs: &Vec<(&str, &str)>, +) -> String +where + P: AsRef, +{ + let _envs = envs + .iter() + .map(|(k, v)| format!("{}={}", k, v)) + .collect::>(); + + format!( + "{} {} {}", + _envs.join(" "), + binary.as_ref().to_str().unwrap(), + args.join(" ") + ) +} + +#[derive(Debug)] +pub enum CmdError { + Exec(ExitStatus), + Io(Error), + Unexpected(String), +} + +impl Display for CmdError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let s = match self { + CmdError::Exec(status) => format!("CmdError: Exec({})", status), + CmdError::Io(io) => format!("CmdError: IO: {}", io), + CmdError::Unexpected(s) => format!("CmdError: Unexpected: {}", s), + }; + write!(f, "{}", s) + } +} + +impl std::error::Error for CmdError {} + +impl From for CmdError { + fn from(err: Error) -> Self { + CmdError::Io(err) + } +} + +impl From for std::io::Error { + fn from(e: CmdError) -> Self { + std::io::Error::new(std::io::ErrorKind::Other, e) + } +} diff --git a/src/constants.rs b/src/constants.rs new file mode 100644 index 00000000..34ee639a --- /dev/null +++ b/src/constants.rs @@ -0,0 +1,4 @@ +pub const TF_PLUGIN_CACHE_DIR: &str = "TF_PLUGIN_CACHE_DIR"; +pub const AWS_ACCESS_KEY_ID: &str = "AWS_ACCESS_KEY_ID"; +pub const AWS_SECRET_ACCESS_KEY: &str = "AWS_SECRET_ACCESS_KEY"; +pub const KUBECONFIG: &str = "KUBECONFIG"; diff --git a/src/container_registry/docker_hub.rs b/src/container_registry/docker_hub.rs new file mode 100644 index 00000000..409f2dbf --- /dev/null +++ b/src/container_registry/docker_hub.rs @@ -0,0 +1,210 @@ +use std::rc::Rc; + +use crate::build_platform::Image; +use crate::cmd; +use crate::cmd::utilities::CmdError; +use crate::container_registry::{ + ContainerRegistry, ContainerRegistryError, Kind, PushError, PushResult, +}; +use crate::models::{Context, Listener, Listeners, ProgressListener}; + +pub struct DockerHub { + context: Context, + id: String, + name: String, + login: String, + password: String, + listeners: Listeners, +} + +impl DockerHub { + pub fn new(context: Context, id: &str, name: &str, login: &str, password: &str) -> Self { + DockerHub { + context, + id: id.to_string(), + name: name.to_string(), + login: login.to_string(), + password: password.to_string(), + listeners: vec![], + } + } +} + +impl ContainerRegistry for DockerHub { + fn context(&self) -> &Context { + &self.context + } + + fn kind(&self) -> Kind { + Kind::DockerHub + } + + fn id(&self) -> &str { + self.id.as_str() + } + + fn name(&self) -> &str { + self.name.as_str() + } + + fn is_valid(&self) -> Result<(), ContainerRegistryError> { + // check the version of docker and print it as info + let mut output_from_cmd = String::new(); + cmd::utilities::exec_with_output( + "docker", + vec!["--version"], + |r_out| match r_out { + Ok(s) => output_from_cmd.push_str(&s.to_owned()), + Err(e) => error!("Error while getting sdtout from docker {}", e), + }, + |r_err| match r_err { + Ok(s) => error!("Error executing docker command {}", s), + Err(e) => error!("Error while getting stderr from docker {}", e), + }, + ); + info!("Using Docker: {}", output_from_cmd); + Ok(()) + } + + fn add_listener(&mut self, listener: Listener) { + self.listeners.push(listener); + } + + fn on_create(&self) -> Result<(), ContainerRegistryError> { + Ok(()) + } + + fn on_create_error(&self) -> Result<(), ContainerRegistryError> { + Ok(()) + } + + fn on_delete(&self) -> Result<(), ContainerRegistryError> { + Ok(()) + } + + fn on_delete_error(&self) -> Result<(), ContainerRegistryError> { + Ok(()) + } + + fn does_image_exists(&self, image: &Image) -> bool { + let envs = match self.context.docker_tcp_socket() { + Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())], + None => vec![], + }; + + // login into docker hub + match cmd::utilities::exec_with_envs( + "docker", + vec![ + "login", + "-u", + self.login.as_str(), + "-p", + self.password.as_str(), + ], + envs.clone(), + ) { + Err(err) => match err { + CmdError::Exec(exit_status) => { + error!("Cannot login into dockerhub"); + return false; + } + CmdError::Io(err) => { + error!("IO error on dockerhub login: {}", err); + return false; + } + CmdError::Unexpected(err) => { + error!("Unexpected error on dockerhub login: {}", err); + return false; + } + }, + _ => {} + }; + + // check if image and tag exist + // note: to retrieve if specific tags exist you can specify the tag at the end of the cUrl path + let curl_path = format!( + "https://index.docker.io/v1/repositories/{}/tags/", + image.name + ); + let mut exist_stdoud: bool = false; + let mut exist_stderr: bool = true; + + cmd::utilities::exec_with_envs_and_output( + "curl", + vec!["--silent", "-f", "-lSL", &curl_path], + envs.clone(), + |r_out| match r_out { + Ok(s) => exist_stdoud = true, + Err(e) => error!("Error while getting stdout from curl {}", e), + }, + |r_err| match r_err { + Ok(s) => exist_stderr = true, + Err(e) => error!("Error while getting stderr from curl {}", e), + }, + ); + exist_stdoud + } + + fn push(&self, image: &Image, force_push: bool) -> Result { + let envs = match self.context.docker_tcp_socket() { + Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())], + None => vec![], + }; + + match cmd::utilities::exec_with_envs( + "docker", + vec![ + "login", + "-u", + self.login.as_str(), + "-p", + self.password.as_str(), + ], + envs.clone(), + ) { + Err(err) => match err { + CmdError::Exec(exit_status) => return Err(PushError::CredentialsError), + CmdError::Io(err) => return Err(PushError::IoError(err)), + CmdError::Unexpected(err) => return Err(PushError::Unknown(err)), + }, + _ => {} + }; + + let dest = format!("{}/{}", self.login.as_str(), image.name_with_tag().as_str()); + match cmd::utilities::exec_with_envs( + "docker", + vec![ + "tag", + dest.as_str(), + format!("{}/{}", self.login.as_str(), dest.as_str()).as_str(), + ], + envs.clone(), + ) { + Err(err) => match err { + CmdError::Exec(exit_status) => return Err(PushError::ImageTagFailed), + CmdError::Io(err) => return Err(PushError::IoError(err)), + CmdError::Unexpected(err) => return Err(PushError::Unknown(err)), + }, + _ => {} + }; + + match cmd::utilities::exec_with_envs("docker", vec!["push", dest.as_str()], envs) { + Err(err) => match err { + CmdError::Exec(exit_status) => return Err(PushError::ImagePushFailed), + CmdError::Io(err) => return Err(PushError::IoError(err)), + CmdError::Unexpected(err) => return Err(PushError::Unknown(err)), + }, + _ => {} + }; + + let mut image = image.clone(); + image.registry_url = Some(dest); + + Ok(PushResult { image }) + } + + fn push_error(&self, _image: &Image) -> Result { + unimplemented!() + } +} diff --git a/src/container_registry/docr.rs b/src/container_registry/docr.rs new file mode 100644 index 00000000..b93a4225 --- /dev/null +++ b/src/container_registry/docr.rs @@ -0,0 +1,188 @@ +extern crate digitalocean; + +use std::rc::Rc; + +use digitalocean::DigitalOcean; + +use crate::build_platform::Image; +use crate::cmd; +use crate::cmd::utilities::CmdError; +use crate::container_registry::{ + ContainerRegistry, ContainerRegistryError, Kind, PushError, PushResult, +}; +use crate::models::{Context, Listener, ProgressListener}; + +// TODO : use --output json +// see https://www.digitalocean.com/community/tutorials/how-to-use-doctl-the-official-digitalocean-command-line-client + +pub struct DOCR { + pub context: Context, + pub registry_name: String, + pub api_key: String, +} + +impl DOCR { + pub fn new(context: Context, registry_name: &str, api_key: &str) -> Self { + DOCR { + context, + registry_name: registry_name.to_string(), + api_key: api_key.to_string(), + } + } + pub fn client(&self) -> DigitalOcean { + DigitalOcean::new(self.api_key.as_str()).unwrap() + } + + pub fn create_repository(&self, _image: &Image) -> Result<(), ContainerRegistryError> { + match cmd::utilities::exec( + "doctl", + vec![ + "registry", + "create", + self.registry_name.as_str(), + "-t", + self.api_key.as_str(), + ], + ) { + Err(err) => match err { + CmdError::Exec(_exit_status) => return Err(ContainerRegistryError::Unknown), + CmdError::Io(err) => return Err(ContainerRegistryError::Unknown), + CmdError::Unexpected(err) => return Err(ContainerRegistryError::Unknown), + }, + _ => {} + }; + Ok(()) + } + + pub fn push_image(&self, dest: String, image: &Image) -> Result { + match cmd::utilities::exec( + "docker", + vec!["tag", image.name_with_tag().as_str(), dest.as_str()], + ) { + Err(err) => match err { + CmdError::Exec(_exit_status) => return Err(PushError::ImageTagFailed), + CmdError::Io(err) => return Err(PushError::IoError(err)), + CmdError::Unexpected(err) => return Err(PushError::Unknown(err)), + }, + _ => {} + }; + + match cmd::utilities::exec("docker", vec!["push", dest.as_str()]) { + Err(err) => match err { + CmdError::Exec(_exit_status) => return Err(PushError::ImagePushFailed), + CmdError::Io(err) => return Err(PushError::IoError(err)), + CmdError::Unexpected(err) => return Err(PushError::Unknown(err)), + }, + _ => {} + }; + + let mut image = image.clone(); + image.registry_url = Some(dest); + + Ok(PushResult { image }) + } + + fn get_or_create_repository(&self, _image: &Image) -> Result<(), ContainerRegistryError> { + // TODO check if repository really exist + self.create_repository(&_image) + } + + fn delete_repository(&self, _image: &Image) -> Result<(), ContainerRegistryError> { + match cmd::utilities::exec( + "doctl", + vec![ + "registry", + "delete", + self.registry_name.as_str(), + "-f", + "-t", + self.api_key.as_str(), + ], + ) { + Err(err) => match err { + CmdError::Exec(exit_status) => return Err(ContainerRegistryError::Unknown), + CmdError::Io(err) => return Err(ContainerRegistryError::Unknown), + CmdError::Unexpected(err) => return Err(ContainerRegistryError::Unknown), + }, + _ => {} + }; + Ok(()) + } +} + +impl ContainerRegistry for DOCR { + fn context(&self) -> &Context { + &self.context + } + + fn kind(&self) -> Kind { + Kind::DOCR + } + + fn id(&self) -> &str { + unimplemented!() + } + + fn name(&self) -> &str { + unimplemented!() + } + + fn is_valid(&self) -> Result<(), ContainerRegistryError> { + unimplemented!() + } + + fn add_listener(&mut self, _listener: Listener) { + unimplemented!() + } + + fn on_create(&self) -> Result<(), ContainerRegistryError> { + unimplemented!() + } + + fn on_create_error(&self) -> Result<(), ContainerRegistryError> { + unimplemented!() + } + + fn on_delete(&self) -> Result<(), ContainerRegistryError> { + unimplemented!() + } + + fn on_delete_error(&self) -> Result<(), ContainerRegistryError> { + unimplemented!() + } + + fn does_image_exists(&self, _image: &Image) -> bool { + unimplemented!() + } + + // https://www.digitalocean.com/docs/images/container-registry/how-to/use-registry-docker-kubernetes/ + fn push(&self, image: &Image, _force_push: bool) -> Result { + let image = image.clone(); + //TODO instead use get_or_create_repository + self.create_repository(&image); + match cmd::utilities::exec( + "doctl", + vec![ + "registry", + "login", + self.registry_name.as_str(), + "-t", + self.api_key.as_str(), + ], + ) { + Err(err) => match err { + CmdError::Exec(_exit_status) => return Err(PushError::CredentialsError), + CmdError::Io(err) => return Err(PushError::IoError(err)), + CmdError::Unexpected(err) => return Err(PushError::Unknown(err)), + }, + _ => {} + }; + //TODO check force or not + let dest = format!("{}:{}", self.registry_name.as_str(), image.tag.as_str()); + self.push_image(dest, &image) + } + + fn push_error(&self, _image: &Image) -> Result { + unimplemented!() + } +} diff --git a/src/container_registry/ecr.rs b/src/container_registry/ecr.rs new file mode 100644 index 00000000..1d780197 --- /dev/null +++ b/src/container_registry/ecr.rs @@ -0,0 +1,382 @@ +use std::rc::Rc; +use std::str::FromStr; + +use rusoto_core::{Client, HttpClient, Region, RusotoError}; +use rusoto_credential::StaticProvider; +use rusoto_ecr::{ + CreateRepositoryError, CreateRepositoryRequest, DescribeImagesRequest, + DescribeRepositoriesRequest, Ecr, EcrClient, GetAuthorizationTokenRequest, ImageDetail, + ImageIdentifier, PutLifecyclePolicyRequest, Repository, +}; +use rusoto_sts::{GetCallerIdentityRequest, Sts, StsClient}; + +use crate::build_platform::Image; +use crate::cmd; +use crate::cmd::utilities::CmdError; +use crate::container_registry::{ + ContainerRegistry, ContainerRegistryError, Kind, PushError, PushResult, +}; +use crate::models::{ + Context, Listener, Listeners, ListenersHelper, ProgressInfo, ProgressLevel, ProgressListener, + ProgressScope, +}; +use crate::runtime::async_run; + +pub struct ECR { + context: Context, + id: String, + name: String, + access_key_id: String, + secret_access_key: String, + region: Region, + listeners: Listeners, +} + +impl ECR { + pub fn new( + context: Context, + id: &str, + name: &str, + access_key_id: &str, + secret_access_key: &str, + region: &str, + ) -> Self { + ECR { + context, + id: id.to_string(), + name: name.to_string(), + access_key_id: access_key_id.to_string(), + secret_access_key: secret_access_key.to_string(), + region: Region::from_str(region).unwrap(), + listeners: vec![], + } + } + + pub fn credentials(&self) -> StaticProvider { + StaticProvider::new( + self.access_key_id.to_string(), + self.secret_access_key.to_string(), + None, + None, + ) + } + + pub fn client(&self) -> Client { + Client::new_with(self.credentials(), HttpClient::new().unwrap()) + } + + pub fn ecr_client(&self) -> EcrClient { + EcrClient::new_with_client(self.client(), self.region.clone()) + } + + fn get_repository(&self, image: &Image) -> Option { + let mut drr = DescribeRepositoriesRequest::default(); + drr.repository_names = Some(vec![image.name.to_string()]); + + let r = async_run(self.ecr_client().describe_repositories(drr)); + + match r { + Err(_) => None, + Ok(res) => match res.repositories { + // assume there is only one repository returned - why? Because we set only one repository_names above + Some(repositories) => repositories.into_iter().next(), + _ => None, + }, + } + } + + fn get_image(&self, image: &Image) -> Option { + let mut dir = DescribeImagesRequest::default(); + dir.repository_name = image.name.to_string(); + + let mut image_identifier = ImageIdentifier::default(); + image_identifier.image_tag = Some(image.tag.to_string()); + dir.image_ids = Some(vec![image_identifier]); + + let r = async_run(self.ecr_client().describe_images(dir)); + + match r { + Err(_) => None, + Ok(res) => match res.image_details { + // assume there is only one repository returned - why? Because we set only one repository_names above + Some(image_details) => image_details.into_iter().next(), + _ => None, + }, + } + } + + fn docker_envs(&self) -> Vec<(&str, &str)> { + match self.context.docker_tcp_socket() { + Some(tcp_socket) => vec![("DOCKER_HOST", tcp_socket.as_str())], + None => vec![], + } + } + + fn push_image(&self, dest: String, image: &Image) -> Result { + // READ https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html + // docker tag e9ae3c220b23 aws_account_id.dkr.ecr.region.amazonaws.com/my-web-app + + match cmd::utilities::exec_with_envs( + "docker", + vec!["tag", image.name_with_tag().as_str(), dest.as_str()], + self.docker_envs(), + ) { + Err(err) => match err { + CmdError::Exec(_exit_status) => return Err(PushError::ImageTagFailed), + CmdError::Io(err) => return Err(PushError::IoError(err)), + CmdError::Unexpected(err) => return Err(PushError::Unknown(err)), + }, + _ => {} + }; + + // docker push aws_account_id.dkr.ecr.region.amazonaws.com/my-web-app + match cmd::utilities::exec_with_envs( + "docker", + vec!["push", dest.as_str()], + self.docker_envs(), + ) { + Err(err) => match err { + CmdError::Exec(_exit_status) => return Err(PushError::ImagePushFailed), + CmdError::Io(err) => return Err(PushError::IoError(err)), + CmdError::Unexpected(err) => return Err(PushError::Unknown(err)), + }, + _ => {} + }; + + let mut image = image.clone(); + image.registry_url = Some(dest); + + Ok(PushResult { image }) + } + + fn create_repository(&self, image: &Image) -> Result { + info!("ECR create repository {}", image.name.as_str()); + let mut crr = CreateRepositoryRequest::default(); + crr.repository_name = image.name.clone(); + + let r = async_run(self.ecr_client().create_repository(crr)); + match r { + Err(err) => match err { + RusotoError::Service(ref err) => info!("{:?}", err), + _ => return Err(ContainerRegistryError::from(err)), + }, + _ => {} + } + + let mut plp = PutLifecyclePolicyRequest::default(); + plp.repository_name = image.name.clone(); + + let ecr_policy = r#" + { + "rules": [ + { + "action": { + "type": "expire" + }, + "selection": { + "countType": "sinceImagePushed", + "countUnit": "days", + "countNumber": 1, + "tagStatus": "any" + }, + "description": "Remove unit test images", + "rulePriority": 1 + } + ] + } + "#; + + plp.lifecycle_policy_text = ecr_policy.to_string(); + + let r = async_run(self.ecr_client().put_lifecycle_policy(plp)); + + match r { + Err(err) => Err(ContainerRegistryError::from(err)), + _ => Ok(self.get_repository(&image).unwrap()), + } + } + + fn get_or_create_repository( + &self, + image: &Image, + ) -> Result { + // check if the repository already exists + let repository = self.get_repository(&image); + if repository.is_some() { + info!("ECR repository {} already exists", image.name.as_str()); + return Ok(repository.unwrap()); + } + + self.create_repository(&image) + } +} + +impl ContainerRegistry for ECR { + fn context(&self) -> &Context { + &self.context + } + + fn kind(&self) -> Kind { + Kind::ECR + } + + fn id(&self) -> &str { + self.id.as_str() + } + + fn name(&self) -> &str { + self.name.as_str() + } + + fn is_valid(&self) -> Result<(), ContainerRegistryError> { + let client = StsClient::new_with_client(self.client(), Region::default()); + let s = async_run(client.get_caller_identity(GetCallerIdentityRequest::default())); + + match s { + Ok(_x) => Ok(()), + Err(err) => Err(ContainerRegistryError::from(err)), + } + } + + fn add_listener(&mut self, listener: Listener) { + self.listeners.push(listener); + } + + fn on_create(&self) -> Result<(), ContainerRegistryError> { + info!("ECR.on_create() called"); + Ok(()) + } + + fn on_create_error(&self) -> Result<(), ContainerRegistryError> { + unimplemented!() + } + + fn on_delete(&self) -> Result<(), ContainerRegistryError> { + unimplemented!() + } + + fn on_delete_error(&self) -> Result<(), ContainerRegistryError> { + unimplemented!() + } + + fn does_image_exists(&self, image: &Image) -> bool { + self.get_repository(&image).is_some() + } + + fn push(&self, image: &Image, force_push: bool) -> Result { + let r = async_run( + self.ecr_client() + .get_authorization_token(GetAuthorizationTokenRequest::default()), + ); + + let (access_token, password, endpoint_url) = match r { + Ok(t) => match t.authorization_data { + Some(authorization_data) => { + let ad = authorization_data.first().unwrap(); + let b64_token = ad.authorization_token.as_ref().unwrap(); + + let decoded_token = base64::decode(b64_token).unwrap(); + let token = std::str::from_utf8(decoded_token.as_slice()).unwrap(); + + let s_token: Vec<&str> = token.split(":").collect::>(); + + ( + s_token.first().unwrap().to_string(), + s_token.get(1).unwrap().to_string(), + ad.clone().proxy_endpoint.unwrap(), + ) + } + None => return Err(PushError::RepositoryInitFailure), + }, + _ => return Err(PushError::RepositoryInitFailure), + }; + + let repository = match if force_push { + self.create_repository(&image) + } else { + self.get_or_create_repository(&image) + } { + Ok(r) => r, + _ => return Err(PushError::RepositoryInitFailure), + }; + + match cmd::utilities::exec_with_envs( + "docker", + vec![ + "login", + "-u", + access_token.as_str(), + "-p", + password.as_str(), + endpoint_url.as_str(), + ], + self.docker_envs(), + ) { + Err(err) => match err { + CmdError::Exec(_exit_status) => return Err(PushError::CredentialsError), + CmdError::Io(err) => return Err(PushError::IoError(err)), + CmdError::Unexpected(err) => return Err(PushError::Unknown(err)), + }, + _ => {} + }; + + let dest = format!( + "{}:{}", + repository.repository_uri.unwrap(), + image.tag.as_str() + ); + + let listeners_helper = ListenersHelper::new(&self.listeners); + + if !force_push && self.get_image(image).is_some() { + // check if image does exist - if yes, do not upload it again + let info_message = format!( + "image {:?} does already exist into ECR {} repository - no need to upload it", + image, + self.name() + ); + + info!("{}", info_message.as_str()); + + listeners_helper.start_in_progress(ProgressInfo::new( + ProgressScope::Application { + id: image.application_id.clone(), + }, + ProgressLevel::Info, + Some(info_message), + self.context.execution_id(), + )); + + let mut image = image.clone(); + image.registry_url = Some(dest); + + return Ok(PushResult { image }); + } + + let info_message = format!( + "image {:?} does not exist into ECR {} repository - let's upload it", + image, + self.name() + ); + + info!("{}", info_message.as_str()); + + listeners_helper.start_in_progress(ProgressInfo::new( + ProgressScope::Application { + id: image.application_id.clone(), + }, + ProgressLevel::Info, + Some(info_message), + self.context.execution_id(), + )); + + self.push_image(dest, image) + } + + fn push_error(&self, image: &Image) -> Result { + // TODO change this + Ok(PushResult { + image: image.clone(), + }) + } +} diff --git a/src/container_registry/mod.rs b/src/container_registry/mod.rs new file mode 100644 index 00000000..531cb77b --- /dev/null +++ b/src/container_registry/mod.rs @@ -0,0 +1,76 @@ +use std::error::Error; +use std::rc::Rc; + +use rusoto_core::RusotoError; +use serde::{Deserialize, Serialize}; + +use crate::build_platform::Image; +use crate::models::{Context, Listener, ProgressListener}; + +pub mod docker_hub; +pub mod docr; +pub mod ecr; + +pub trait ContainerRegistry { + fn context(&self) -> &Context; + fn kind(&self) -> Kind; + fn id(&self) -> &str; + fn name(&self) -> &str; + fn is_valid(&self) -> Result<(), ContainerRegistryError>; + fn add_listener(&mut self, listener: Listener); + fn on_create(&self) -> Result<(), ContainerRegistryError>; + fn on_create_error(&self) -> Result<(), ContainerRegistryError>; + fn on_delete(&self) -> Result<(), ContainerRegistryError>; + fn on_delete_error(&self) -> Result<(), ContainerRegistryError>; + fn does_image_exists(&self, image: &Image) -> bool; + fn push(&self, image: &Image, force_push: bool) -> Result; + fn push_error(&self, image: &Image) -> Result; +} + +pub struct PushResult { + pub image: Image, +} + +#[derive(Debug)] +pub enum PushError { + RepositoryInitFailure, + CredentialsError, + IoError(std::io::Error), + ImageTagFailed, + ImagePushFailed, + ImageAlreadyExists, + Unknown(String), +} + +#[derive(Serialize, Deserialize, Clone)] +pub enum Kind { + DockerHub, + ECR, + DOCR, +} + +#[derive(Debug, Eq, PartialEq)] +pub enum ContainerRegistryError { + Credentials, + Unknown, +} + +impl From> for ContainerRegistryError { + fn from(error: RusotoError) -> Self { + match error { + RusotoError::Credentials(_) => ContainerRegistryError::Credentials, + RusotoError::Service(_) => ContainerRegistryError::Unknown, + RusotoError::HttpDispatch(_) => ContainerRegistryError::Unknown, + RusotoError::Validation(_) => ContainerRegistryError::Unknown, + RusotoError::ParseError(_) => ContainerRegistryError::Unknown, + RusotoError::Unknown(e) => { + if e.status == 403 { + ContainerRegistryError::Credentials + } else { + ContainerRegistryError::Unknown + } + } + RusotoError::Blocking => ContainerRegistryError::Unknown, + } + } +} diff --git a/src/crypto.rs b/src/crypto.rs new file mode 100644 index 00000000..a2b8afad --- /dev/null +++ b/src/crypto.rs @@ -0,0 +1,14 @@ +use crypto::digest::Digest; +use crypto::sha1::Sha1; + +pub fn to_sha1(input: &str) -> String { + let mut hasher = Sha1::new(); + hasher.input_str(input); + hasher.result_str() +} + +pub fn to_sha1_truncate_16(input: &str) -> String { + let mut hash_str = to_sha1(input); + hash_str.truncate(16); + hash_str +} diff --git a/src/deletion_utilities.rs b/src/deletion_utilities.rs new file mode 100644 index 00000000..ac96f210 --- /dev/null +++ b/src/deletion_utilities.rs @@ -0,0 +1,41 @@ +use crate::cmd::kubectl::{kubectl_exec_delete_namespace, kubectl_exec_get_all_namespaces}; +use crate::constants::{AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY}; + +// this fn should implements the algorythm describe here: https://qovery.atlassian.net/secure/RapidBoard.jspa?rapidView=10&modal=detail&selectedIssue=DEV-283 +pub fn get_firsts_namespaces_to_delete(namespaces: Vec<&str>) -> Vec<&str> { + // from all namesapce remove managed and never delete namespaces + let minus_managed = minus_namespaces(namespaces, get_qovery_managed_namespaces()); + let minus_qovery_managed_and_never_delete = + minus_namespaces(minus_managed, get_never_delete_namespaces()); + minus_qovery_managed_and_never_delete +} + +fn minus_namespaces<'a>(all: Vec<&'a str>, to_remove_namespaces: Vec<&str>) -> Vec<&'a str> { + let reduced = all + .into_iter() + .filter(|item| !to_remove_namespaces.contains(item)) + .collect(); + return reduced; +} + +// TODO: use label instead +// TODO: create enum: deletion_rule [system, qovery,..] +pub fn get_qovery_managed_namespaces() -> Vec<&'static str> { + let mut qovery_managed_namespaces = Vec::with_capacity(5); + qovery_managed_namespaces.push("logging"); + qovery_managed_namespaces.push("nginx"); + qovery_managed_namespaces.push("qovery"); + qovery_managed_namespaces.push("cert-manager"); + qovery_managed_namespaces.push("prometheus"); + return qovery_managed_namespaces; +} + +// TODO: use label instead +fn get_never_delete_namespaces() -> Vec<&'static str> { + let mut kubernetes_never_delete_namespaces = Vec::with_capacity(4); + kubernetes_never_delete_namespaces.push("default"); + kubernetes_never_delete_namespaces.push("kube-node-lease"); + kubernetes_never_delete_namespaces.push("kube-public"); + kubernetes_never_delete_namespaces.push("kube-system"); + return kubernetes_never_delete_namespaces; +} diff --git a/src/dns_provider/cloudflare.rs b/src/dns_provider/cloudflare.rs new file mode 100644 index 00000000..42d56e99 --- /dev/null +++ b/src/dns_provider/cloudflare.rs @@ -0,0 +1,75 @@ +use std::net::Ipv4Addr; + +use crate::dns_provider::{DnsProvider, DnsProviderError, Kind}; +use crate::models::Context; + +pub struct Cloudflare { + context: Context, + id: String, + name: String, + domain: String, + cloudflare_api_token: String, + cloudflare_email: String, +} + +impl Cloudflare { + pub fn new( + context: Context, + id: String, + name: String, + domain: String, + cloudflare_api_token: String, + cloudflare_email: String, + ) -> Self { + Cloudflare { + context, + id, + name, + domain, + cloudflare_api_token, + cloudflare_email, + } + } +} + +impl DnsProvider for Cloudflare { + fn context(&self) -> &Context { + &self.context + } + + fn kind(&self) -> Kind { + Kind::CLOUDFLARE + } + + fn id(&self) -> &str { + &self.id + } + + fn name(&self) -> &str { + &self.name + } + + fn account(&self) -> &str { + &self.cloudflare_email + } + + fn token(&self) -> &str { + &self.cloudflare_api_token + } + + fn domain(&self) -> &str { + self.domain.as_str() + } + + fn resolvers(&self) -> Vec { + vec![Ipv4Addr::new(1, 1, 1, 1), Ipv4Addr::new(1, 0, 0, 1)] + } + + fn is_valid(&self) -> Result<(), DnsProviderError> { + if self.cloudflare_api_token.is_empty() || self.cloudflare_email.is_empty() { + Err(DnsProviderError::Credentials) + } else { + Ok(()) + } + } +} diff --git a/src/dns_provider/mod.rs b/src/dns_provider/mod.rs new file mode 100644 index 00000000..faa653d3 --- /dev/null +++ b/src/dns_provider/mod.rs @@ -0,0 +1,30 @@ +use std::net::Ipv4Addr; + +use serde::{Deserialize, Serialize}; + +use crate::models::Context; + +pub mod cloudflare; + +#[derive(Serialize, Deserialize, Clone)] +pub enum Kind { + CLOUDFLARE, +} + +pub trait DnsProvider { + fn context(&self) -> &Context; + fn kind(&self) -> Kind; + fn id(&self) -> &str; + fn name(&self) -> &str; + fn account(&self) -> &str; + fn token(&self) -> &str; + fn domain(&self) -> &str; + fn resolvers(&self) -> Vec; + fn is_valid(&self) -> Result<(), DnsProviderError>; +} + +#[derive(Debug, Eq, PartialEq)] +pub enum DnsProviderError { + Credentials, + Unknown, +} diff --git a/src/dynamo_db.rs b/src/dynamo_db.rs new file mode 100644 index 00000000..fcf72e6d --- /dev/null +++ b/src/dynamo_db.rs @@ -0,0 +1,67 @@ +use std::io::{Error, ErrorKind}; + +use rusoto_core::{Client, HttpClient, Region, RusotoError}; +use rusoto_credential::StaticProvider; +use rusoto_dynamodb::{ + AttributeDefinition, CreateTableError, CreateTableInput, DynamoDb, DynamoDbClient, + KeySchemaElement, +}; + +use crate::runtime::async_run; + +pub fn create_terraform_table( + access_key_id: &str, + secret_access_key: &str, + region: &Region, + table_name: &str, +) -> Result<(), Error> { + let access_key_id = access_key_id.to_string(); + let secret_access_key = secret_access_key.to_string(); + let table_name = table_name.to_string(); + + let credentials = StaticProvider::new(access_key_id, secret_access_key, None, None); + let client = Client::new_with(credentials, HttpClient::new().unwrap()); + let ddb_client = DynamoDbClient::new_with_client(client, region.clone()); + + let mut cti = CreateTableInput::default(); + cti.table_name = table_name; + cti.billing_mode = Some("PAY_PER_REQUEST".to_string()); + + cti.key_schema = vec![KeySchemaElement { + attribute_name: "LockID".to_string(), + key_type: "HASH".to_string(), + }]; + + cti.attribute_definitions = vec![AttributeDefinition { + attribute_name: "LockID".to_string(), + attribute_type: "S".to_string(), + }]; + + let r = async_run(ddb_client.create_table(cti)); + + // FIXME: return a custom DynamoDBError? + match r { + Err(err) => match err { + RusotoError::Unknown(r) => { + error!("{}", r.body_as_str()); + Err(Error::new(ErrorKind::Other, r.body_as_str())) + } + RusotoError::Service(r) => match r { + CreateTableError::ResourceInUse(_) => Ok(()), // table already exists + _ => { + return Err(Error::new( + ErrorKind::Other, + "something goes wrong while creating terraform DynamoDB table", + )); + } + }, + _ => { + return Err(Error::new( + ErrorKind::Other, + "something goes wrong while creating terraform DynamoDB table", + )); + } + }, + Ok(_x) => Ok(()), + } +} diff --git a/src/engine.rs b/src/engine.rs new file mode 100644 index 00000000..a2f7d76e --- /dev/null +++ b/src/engine.rs @@ -0,0 +1,89 @@ +use std::borrow::Borrow; + +use crate::build_platform::BuildPlatform; +use crate::cloud_provider::CloudProvider; +use crate::container_registry::ContainerRegistry; +use crate::dns_provider::DnsProvider; +use crate::error::ConfigurationError; +use crate::models::Context; +use crate::session::Session; + +pub struct Engine { + context: Context, + build_platform: Box, + container_registry: Box, + cloud_provider: Box, + dns_provider: Box, +} + +impl Engine { + pub fn new( + context: Context, + build_platform: Box, + container_registry: Box, + cloud_provider: Box, + dns_provider: Box, + ) -> Engine { + Engine { + context, + build_platform, + container_registry, + cloud_provider, + dns_provider, + } + } +} + +impl<'a> Engine { + pub fn context(&self) -> &Context { + &self.context + } + + pub fn build_platform(&self) -> &dyn BuildPlatform { + self.build_platform.borrow() + } + + pub fn container_registry(&self) -> &dyn ContainerRegistry { + self.container_registry.borrow() + } + + pub fn cloud_provider(&self) -> &dyn CloudProvider { + self.cloud_provider.borrow() + } + pub fn dns_provider(&self) -> &dyn DnsProvider { + self.dns_provider.borrow() + } + + pub fn is_valid(&self) -> Result<(), ConfigurationError> { + match self.build_platform.is_valid() { + Ok(_) => {} + Err(err) => { + return Err(ConfigurationError::BuildPlatform(err)); + } + } + + match self.container_registry.is_valid() { + Ok(_) => {} + Err(err) => { + return Err(ConfigurationError::ContainerRegistry(err)); + } + } + + match self.cloud_provider.is_valid() { + Ok(_) => {} + Err(err) => { + return Err(ConfigurationError::CloudProvider(err)); + } + } + + Ok(()) + } + + /// check and init the connection to all the services + pub fn session(&'a self) -> Result, ConfigurationError> { + match self.is_valid() { + Ok(_) => Ok(Session::<'a> { engine: self }), + Err(err) => Err(err), + } + } +} diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 00000000..51780c59 --- /dev/null +++ b/src/error.rs @@ -0,0 +1,10 @@ +use crate::build_platform::error::BuildPlatformError; +use crate::cloud_provider::CloudProviderError; +use crate::container_registry::ContainerRegistryError; + +#[derive(Debug)] +pub enum ConfigurationError { + BuildPlatform(BuildPlatformError), + ContainerRegistry(ContainerRegistryError), + CloudProvider(CloudProviderError), +} diff --git a/src/fs.rs b/src/fs.rs new file mode 100644 index 00000000..17eb75fc --- /dev/null +++ b/src/fs.rs @@ -0,0 +1,118 @@ +use std::fs; +use std::fs::{create_dir_all, File}; +use std::io::Error; +use std::path::Path; + +use flate2::write::GzEncoder; +use flate2::Compression; +use walkdir::WalkDir; + +pub fn copy_files(from: &Path, to: &Path, exclude_j2_files: bool) -> Result<(), Error> { + let files = WalkDir::new(from) + .follow_links(true) + .into_iter() + .filter_map(|e| e.ok()); + + let files = match exclude_j2_files { + true => files + .filter(|e| { + // return only non *.j2.* files + e.file_name() + .to_str() + .map(|s| !s.contains(".j2.")) + .unwrap_or(false) + }) + .collect::>(), + false => files.collect::>(), + }; + + let _ = fs::create_dir_all(to)?; + let from_str = from.to_str().unwrap(); + + for file in files { + let path_str = file.path().to_str().unwrap(); + let dest = format!( + "{}{}", + to.to_str().unwrap(), + path_str.replace(from_str, "").as_str() + ); + + if file.metadata().unwrap().is_dir() { + let _ = fs::create_dir_all(&dest)?; + } + + let _ = fs::copy(file.path(), dest); + } + + Ok(()) +} + +pub fn root_workspace_directory(working_root_dir: X, execution_id: S) -> String +where + X: AsRef, + S: AsRef, +{ + workspace_directory(working_root_dir, execution_id, ".") +} + +pub fn workspace_directory(working_root_dir: X, execution_id: S, dir_name: P) -> String +where + X: AsRef, + S: AsRef, + P: AsRef, +{ + let dir = format!( + "{}/.qovery-workspace/{}/{}", + working_root_dir.as_ref().to_str().unwrap(), + execution_id.as_ref().to_str().unwrap(), + dir_name.as_ref().to_str().unwrap(), + ); + + let _ = create_dir_all(&dir); + + dir +} + +fn archive_workspace_directory( + working_root_dir: &str, + execution_id: &str, +) -> Result { + let workspace_dir = crate::fs::root_workspace_directory(working_root_dir, execution_id); + + let tar_gz_file_path = format!( + "{}/.qovery-workspace/{}.tar.gz", + working_root_dir, execution_id + ); + + let tar_gz_file = File::create(tar_gz_file_path.as_str())?; + + let enc = GzEncoder::new(tar_gz_file, Compression::fast()); + let mut tar = tar::Builder::new(enc); + tar.append_dir_all(execution_id, workspace_dir)?; + + Ok(File::open(tar_gz_file_path).unwrap()) +} + +pub fn cleanup_workspace_directory(working_root_dir: &str, execution_id: &str) { + let workspace_dir = crate::fs::root_workspace_directory(working_root_dir, execution_id); + std::fs::remove_dir_all(workspace_dir); +} + +pub fn create_workspace_archive( + working_root_dir: &str, + execution_id: &str, +) -> Result { + info!("archive workspace directory in progress"); + + match archive_workspace_directory(working_root_dir, execution_id) { + Err(err) => { + error!("archive workspace directory error: {:?}", err); + Err(err) + } + Ok(file) => { + info!("workspace directory is archived"); + cleanup_workspace_directory(working_root_dir, execution_id); + Ok(file) + } + } +} diff --git a/src/git.rs b/src/git.rs new file mode 100644 index 00000000..dd9df640 --- /dev/null +++ b/src/git.rs @@ -0,0 +1,104 @@ +use std::path::Path; + +use git2::build::RepoBuilder; +use git2::{Error, Oid, Repository, Submodule}; + +/// TODO support SSH repository_url - we assume that the repository URL starts with HTTPS +/// TODO support git submodules +pub fn clone

( + repository_url: &str, + into_dir: P, + credentials: &Option, +) -> Result +where + P: AsRef, +{ + let final_repository_url = match credentials { + Some(c) => format!( + "https://{}:{}@{}", + c.login, + c.password, + repository_url.replace("https://", "") + ), + None => repository_url.to_string(), + }; + + RepoBuilder::new().clone(final_repository_url.as_str(), into_dir.as_ref()) +} + +pub fn checkout(repo: &Repository, commit_id: &str, repo_url: &str) -> Result<(), Error> { + let oid = match Oid::from_str(&commit_id) { + Err(e) => { + let mut x = git2::Error::from_str( + format!( + "Error while trying to validate commit ID {} on repository {}: {}", + &commit_id, &repo_url, &e + ) + .as_ref(), + ); + return Err(x); + } + Ok(o) => o, + }; + + let _ = match repo.find_commit(oid) { + Err(e) => { + let mut x = git2::Error::from_str( + format!( + "Commit ID {} on repository {} was not found", + &commit_id, &repo_url + ) + .as_ref(), + ); + x.set_code(e.code()); + x.set_class(e.class()); + return Err(x); + } + Ok(c) => c, + }; + + let obj = match repo.revparse_single(&commit_id) { + Err(e) => { + let mut x = git2::Error::from_str( + format!( + "Wasn't able to use git object commit ID {} on repository {}: {}", + &commit_id, &repo_url, &e + ) + .as_ref(), + ); + return Err(x); + } + Ok(o) => o, + }; + + repo.checkout_tree(&obj, None); + + repo.set_head(&("refs/heads/".to_owned() + &commit_id)) +} + +pub fn checkout_submodules(repo: &Repository) -> Result<(), Error> { + match repo.submodules() { + Ok(submodules) => { + for mut submodule in submodules { + info!( + "getting submodule {:?} from {:?}", + submodule.name(), + submodule.url() + ); + + match submodule.update(true, None) { + Err(e) => return Err(e), + _ => (), + } + } + } + Err(err) => return Err(err), + } + + Ok(()) +} + +pub struct Credentials { + pub login: String, + pub password: String, +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 00000000..b9bb7f4a --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,25 @@ +#[macro_use] +extern crate log; +extern crate tera; + +pub mod build_platform; +pub mod cloud_provider; +pub mod cmd; +mod constants; +pub mod container_registry; +mod crypto; +mod deletion_utilities; +pub mod dns_provider; +mod dynamo_db; +pub mod engine; +pub mod error; +pub mod fs; +mod git; +pub mod models; +mod runtime; +pub mod s3; +pub mod session; +mod string; +mod template; +pub mod transaction; +mod unit_conversion; diff --git a/src/models.rs b/src/models.rs new file mode 100644 index 00000000..1763a2a4 --- /dev/null +++ b/src/models.rs @@ -0,0 +1,833 @@ +use std::hash::Hash; +use std::rc::Rc; + +use chrono::{DateTime, Utc}; +use rand::distributions::Alphanumeric; +use rand::{thread_rng, Rng}; +use serde::{Deserialize, Serialize}; + +use crate::build_platform::{Build, BuildOptions, GitRepository, Image}; +use crate::cloud_provider::aws::databases::{MongoDB, MySQL, PostgreSQL}; +use crate::cloud_provider::service::{DatabaseOptions, StatefulService, StatelessService}; +use crate::cloud_provider::CloudProvider; +use crate::cloud_provider::Kind as CPKind; +use crate::git::Credentials; +use crate::models::DatabaseKind::Mongodb; + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +pub enum EnvironmentAction { + Environment(TargetEnvironment), + EnvironmentWithFailover(TargetEnvironment, FailoverEnvironment), +} + +pub type TargetEnvironment = Environment; +pub type FailoverEnvironment = Environment; + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +pub struct Environment { + pub execution_id: String, + pub id: String, + pub kind: Kind, + pub owner_id: String, + pub project_id: String, + pub organization_id: String, + pub action: Action, + pub applications: Vec, + pub routers: Vec, + pub databases: Vec, + pub external_services: Vec, + pub clone_from_environment_id: Option, +} + +impl Environment { + pub fn is_valid(&self) -> Result<(), EnvironmentError> { + Ok(()) + } + + pub fn to_qe_environment( + &self, + context: &Context, + built_applications: &Vec>, + cloud_provider: &dyn CloudProvider, + ) -> crate::cloud_provider::environment::Environment { + let external_services = self + .external_services + .iter() + .map( + |x| match built_applications.iter().find(|y| x.id.as_str() == y.id()) { + Some(app) => { + x.to_stateless_service(context, app.image().clone(), cloud_provider) + } + _ => x.to_stateless_service(context, x.to_image(), cloud_provider), + }, + ) + .filter(|x| x.is_some()) + .map(|x| x.unwrap()) + .collect::>(); + + let applications = self + .applications + .iter() + .map( + |x| match built_applications.iter().find(|y| x.id.as_str() == y.id()) { + Some(app) => { + x.to_stateless_service(context, app.image().clone(), cloud_provider) + } + _ => x.to_stateless_service(context, x.to_image(), cloud_provider), + }, + ) + .filter(|x| x.is_some()) + .map(|x| x.unwrap()) + .collect::>(); + + let routers = self + .routers + .iter() + .map(|x| x.to_stateless_service(context, cloud_provider)) + .filter(|x| x.is_some()) + .map(|x| x.unwrap()) + .collect::>(); + + let mut stateless_services = external_services; + stateless_services.extend(routers); + stateless_services.extend(applications); + + let databases = self + .databases + .iter() + .map(|x| x.to_stateful_service(context, cloud_provider)) + .filter(|x| x.is_some()) + .map(|x| x.unwrap()) + .collect::>(); + + let stateful_services = databases; + + crate::cloud_provider::environment::Environment::new( + match self.kind { + Kind::Production => crate::cloud_provider::environment::Kind::Production, + Kind::Development => crate::cloud_provider::environment::Kind::Development, + }, + self.id.as_str(), + self.project_id.as_str(), + self.owner_id.as_str(), + self.organization_id.as_str(), + stateless_services, + stateful_services, + ) + } +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum Kind { + Production, + Development, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum Action { + Create, + Pause, + Delete, + Nothing, +} + +impl Action { + pub fn to_service_action(&self) -> crate::cloud_provider::service::Action { + match self { + Action::Create => crate::cloud_provider::service::Action::Create, + Action::Pause => crate::cloud_provider::service::Action::Pause, + Action::Delete => crate::cloud_provider::service::Action::Delete, + Action::Nothing => crate::cloud_provider::service::Action::Nothing, + } + } +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +pub struct Application { + pub id: String, + pub name: String, + pub action: Action, + pub git_url: String, + pub git_credentials: GitCredentials, + pub branch: String, + pub commit_id: String, + pub dockerfile_path: String, + pub private_port: Option, + pub total_cpus: String, + pub cpu_burst: String, + pub total_ram_in_mib: u32, + pub total_instances: u16, + pub storage: Vec, + pub environment_variables: Vec, +} + +impl Application { + pub fn to_application<'a>( + &self, + context: &Context, + image: &Image, + cloud_provider: &dyn CloudProvider, + ) -> Option> { + match cloud_provider.kind() { + CPKind::AWS => Some(Box::new( + crate::cloud_provider::aws::application::Application::new( + context.clone(), + self.id.as_str(), + self.action.to_service_action(), + self.name.as_str(), + self.private_port, + self.total_cpus.clone(), + self.cpu_burst.clone(), + self.total_ram_in_mib, + self.total_instances, + image.clone(), + self.storage + .iter() + .map(|s| s.to_aws_storage()) + .collect::>(), + self.environment_variables + .iter() + .map(|ev| ev.to_aws_application_environment_variable()) + .collect::>(), + ), + )), + CPKind::GCP => None, + _ => None, + //TODO to implement + } + } + + pub fn to_stateless_service( + &self, + context: &Context, + image: Image, + cloud_provider: &dyn CloudProvider, + ) -> Option> { + match cloud_provider.kind() { + CPKind::AWS => Some(Box::new( + crate::cloud_provider::aws::application::Application::new( + context.clone(), + self.id.as_str(), + self.action.to_service_action(), + self.name.as_str(), + self.private_port, + self.total_cpus.clone(), + self.cpu_burst.clone(), + self.total_ram_in_mib, + self.total_instances, + image, + self.storage + .iter() + .map(|s| s.to_aws_storage()) + .collect::>(), + self.environment_variables + .iter() + .map(|ev| ev.to_aws_application_environment_variable()) + .collect::>(), + ), + )), + CPKind::GCP => None, + _ => None, + //TODO to implement + } + } + + pub fn to_image(&self) -> Image { + Image { + application_id: self.id.clone(), + name: self.name.clone(), + tag: self.commit_id.clone(), + commit_id: self.commit_id.clone(), + registry_url: None, + } + } + + pub fn to_build(&self) -> Build { + Build { + git_repository: GitRepository { + url: self.git_url.clone(), + credentials: Some(Credentials { + login: self.git_credentials.login.clone(), + password: self.git_credentials.access_token.clone(), + }), + commit_id: self.commit_id.clone(), + dockerfile_path: self.dockerfile_path.clone(), + }, + image: self.to_image(), + options: BuildOptions { + environment_variables: self + .environment_variables + .iter() + .map(|ev| crate::build_platform::EnvironmentVariable { + key: ev.key.clone(), + value: ev.value.clone(), + }) + .collect::>(), + }, + } + } +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +pub struct EnvironmentVariable { + pub key: String, + pub value: String, +} + +impl EnvironmentVariable { + pub fn to_aws_application_environment_variable( + &self, + ) -> crate::cloud_provider::aws::application::EnvironmentVariable { + crate::cloud_provider::aws::application::EnvironmentVariable { + key: self.key.clone(), + value: self.value.clone(), + } + } + + pub fn to_aws_external_service_environment_variable( + &self, + ) -> crate::cloud_provider::aws::external_service::EnvironmentVariable { + crate::cloud_provider::aws::external_service::EnvironmentVariable { + key: self.key.clone(), + value: self.value.clone(), + } + } +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +pub struct GitCredentials { + pub login: String, + pub access_token: String, + pub expired_at: DateTime, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +pub struct Storage { + pub id: String, + pub name: String, + pub storage_type: StorageType, + pub size_in_gib: u16, + pub mount_point: String, + pub snapshot_retention_in_days: u16, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum StorageType { + SlowHdd, + Hdd, + Ssd, + FastSsd, +} + +impl Storage { + pub fn to_aws_storage(&self) -> crate::cloud_provider::aws::application::Storage { + crate::cloud_provider::aws::application::Storage { + id: self.id.clone(), + name: self.name.clone(), + storage_type: match self.storage_type { + StorageType::SlowHdd => crate::cloud_provider::aws::application::StorageType::SC1, + StorageType::Hdd => crate::cloud_provider::aws::application::StorageType::ST1, + StorageType::Ssd => crate::cloud_provider::aws::application::StorageType::GP2, + StorageType::FastSsd => crate::cloud_provider::aws::application::StorageType::IO1, + }, + size_in_gib: self.size_in_gib, + mount_point: self.mount_point.clone(), + snapshot_retention_in_days: self.snapshot_retention_in_days, + } + } +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +pub struct Router { + pub id: String, + pub name: String, + pub action: Action, + pub default_domain: String, + pub public_port: u16, + pub custom_domains: Vec, + pub routes: Vec, +} + +impl Router { + pub fn to_stateless_service( + &self, + context: &Context, + cloud_provider: &dyn CloudProvider, + ) -> Option> { + match cloud_provider.kind() { + CPKind::AWS => { + let router: Box = + Box::new(crate::cloud_provider::aws::router::Router::new( + context.clone(), + self.id.as_str(), + self.name.as_str(), + self.default_domain.as_str(), + self.custom_domains + .iter() + .map(|x| crate::cloud_provider::aws::router::CustomDomain { + domain: x.domain.clone(), + target_domain: x.target_domain.clone(), + }) + .collect::>(), + self.routes + .iter() + .map(|x| crate::cloud_provider::aws::router::Route { + path: x.path.clone(), + application_name: x.application_name.clone(), + }) + .collect::>(), + )); + Some(router) + } + CPKind::GCP => None, + _ => None, + //TODO to implement + } + } +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +pub struct CustomDomain { + pub domain: String, + pub target_domain: String, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +pub struct Route { + pub path: String, + pub application_name: String, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +pub struct Database { + pub kind: DatabaseKind, + pub action: Action, + pub id: String, + pub name: String, + pub version: String, + pub fqdn_id: String, + pub fqdn: String, + pub port: u16, + pub username: String, + pub password: String, + pub total_cpus: String, + pub total_ram_in_mib: u32, + pub disk_size_in_gib: u32, + pub database_instance_type: String, + pub database_disk_type: String, +} + +impl Database { + pub fn to_stateful_service( + &self, + context: &Context, + cloud_provider: &dyn CloudProvider, + ) -> Option> { + let database_options = DatabaseOptions { + login: self.username.clone(), + password: self.password.clone(), + host: self.fqdn.clone(), + port: self.port, + disk_size_in_gib: self.disk_size_in_gib, + database_disk_type: self.database_disk_type.clone(), + }; + + match cloud_provider.kind() { + CPKind::AWS => match self.kind { + DatabaseKind::Postgresql => { + let db: Box = Box::new(PostgreSQL::new( + context.clone(), + self.id.as_str(), + self.action.to_service_action(), + self.name.as_str(), + self.version.as_str(), + self.fqdn.as_str(), + self.fqdn_id.as_str(), + self.total_cpus.clone(), + self.total_ram_in_mib, + self.database_instance_type.as_str(), + database_options, + )); + + Some(db) + } + DatabaseKind::Mysql => { + let db: Box = Box::new(MySQL::new( + context.clone(), + self.id.as_str(), + self.action.to_service_action(), + self.name.as_str(), + self.version.as_str(), + self.fqdn.as_str(), + self.fqdn_id.as_str(), + self.total_cpus.clone(), + self.total_ram_in_mib, + self.database_instance_type.as_str(), + database_options, + )); + + Some(db) + } + DatabaseKind::Mongodb => { + let db: Box = Box::new(MongoDB::new( + context.clone(), + self.id.as_str(), + self.action.to_service_action(), + self.name.as_str(), + self.version.as_str(), + self.fqdn.as_str(), + self.fqdn_id.as_str(), + self.total_cpus.clone(), + self.total_ram_in_mib, + self.database_instance_type.as_str(), + database_options, + )); + + Some(db) + } + _ => None, + }, + CPKind::GCP => None, + _ => None, + //TODO to implement + } + } +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum DatabaseKind { + Postgresql, + Mysql, + Mongodb, +} + +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +pub struct ExternalService { + pub action: Action, + pub id: String, + pub name: String, + pub total_cpus: String, + pub total_ram_in_mib: u32, + pub git_url: String, + pub git_credentials: GitCredentials, + pub branch: String, + pub commit_id: String, + pub on_create_dockerfile_path: String, + pub on_pause_dockerfile_path: String, + pub on_delete_dockerfile_path: String, + pub environment_variables: Vec, +} + +impl ExternalService { + pub fn to_application<'a>( + &self, + context: &Context, + image: &Image, + cloud_provider: &dyn CloudProvider, + ) -> Option> { + match cloud_provider.kind() { + CPKind::AWS => Some(Box::new( + crate::cloud_provider::aws::external_service::ExternalService::new( + context.clone(), + self.id.as_str(), + self.action.to_service_action(), + self.name.as_str(), + self.total_cpus.clone(), + self.total_ram_in_mib, + image.clone(), + self.environment_variables + .iter() + .map(|ev| ev.to_aws_external_service_environment_variable()) + .collect::>(), + ), + )), + CPKind::GCP => None, + _ => None, + //TODO to implement + } + } + + pub fn to_stateless_service<'a>( + &self, + context: &Context, + image: Image, + cloud_provider: &dyn CloudProvider, + ) -> Option> { + match cloud_provider.kind() { + CPKind::AWS => Some(Box::new( + crate::cloud_provider::aws::external_service::ExternalService::new( + context.clone(), + self.id.as_str(), + self.action.to_service_action(), + self.name.as_str(), + self.total_cpus.clone(), + self.total_ram_in_mib, + image, + self.environment_variables + .iter() + .map(|ev| ev.to_aws_external_service_environment_variable()) + .collect::>(), + ), + )), + CPKind::GCP => None, + _ => None, + //TODO to implement + } + } + + pub fn to_image(&self) -> Image { + Image { + application_id: self.id.clone(), + name: self.name.clone(), + tag: self.commit_id.clone(), + commit_id: self.commit_id.clone(), + registry_url: None, + } + } + + pub fn to_build(&self) -> Build { + Build { + git_repository: GitRepository { + url: self.git_url.clone(), + credentials: Some(Credentials { + login: self.git_credentials.login.clone(), + password: self.git_credentials.access_token.clone(), + }), + commit_id: self.commit_id.clone(), + dockerfile_path: match self.action { + Action::Create => self.on_create_dockerfile_path.clone(), + Action::Pause => self.on_pause_dockerfile_path.clone(), + Action::Delete => self.on_delete_dockerfile_path.clone(), + Action::Nothing => self.on_create_dockerfile_path.clone(), + }, + }, + image: self.to_image(), + options: BuildOptions { + environment_variables: self + .environment_variables + .iter() + .map(|ev| crate::build_platform::EnvironmentVariable { + key: ev.key.clone(), + value: ev.value.clone(), + }) + .collect::>(), + }, + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum EnvironmentError {} + +#[derive(Clone)] +pub struct ProgressInfo { + pub created_at: DateTime, + pub scope: ProgressScope, + pub level: ProgressLevel, + pub message: Option, + pub execution_id: String, +} + +impl ProgressInfo { + pub fn new, X: Into>( + scope: ProgressScope, + level: ProgressLevel, + message: Option, + execution_id: X, + ) -> Self { + ProgressInfo { + created_at: Utc::now(), + scope, + level, + message: match message { + Some(msg) => Some(msg.into()), + _ => None, + }, + execution_id: execution_id.into(), + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "SCREAMING_SNAKE_CASE")] +pub enum ProgressScope { + Queued, + Infrastructure { execution_id: String }, + Database { id: String }, + Application { id: String }, + ExternalService { id: String }, + Router { id: String }, + Environment { id: String }, +} + +#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum ProgressLevel { + Debug, + Info, + Warn, + Error, +} + +pub trait ProgressListener { + fn start_in_progress(&self, info: ProgressInfo); + fn pause_in_progress(&self, info: ProgressInfo); + fn delete_in_progress(&self, info: ProgressInfo); + fn error(&self, info: ProgressInfo); + fn started(&self, info: ProgressInfo); + fn paused(&self, info: ProgressInfo); + fn deleted(&self, info: ProgressInfo); + fn start_error(&self, info: ProgressInfo); + fn pause_error(&self, info: ProgressInfo); + fn delete_error(&self, info: ProgressInfo); +} + +pub type Listener = Rc>; +pub type Listeners = Vec; + +pub struct ListenersHelper<'a> { + listeners: &'a Listeners, +} + +impl<'a> ListenersHelper<'a> { + pub fn new(listeners: &'a Listeners) -> Self { + ListenersHelper { listeners } + } + + pub fn start_in_progress(&self, info: ProgressInfo) { + self.listeners + .iter() + .for_each(|l| l.start_in_progress(info.clone())); + } + + pub fn pause_in_progress(&self, info: ProgressInfo) { + self.listeners + .iter() + .for_each(|l| l.pause_in_progress(info.clone())); + } + + pub fn delete_in_progress(&self, info: ProgressInfo) { + self.listeners + .iter() + .for_each(|l| l.delete_in_progress(info.clone())); + } + + pub fn error(&self, info: ProgressInfo) { + self.listeners.iter().for_each(|l| l.error(info.clone())); + } + + pub fn started(&self, info: ProgressInfo) { + self.listeners.iter().for_each(|l| l.started(info.clone())); + } + + pub fn paused(&self, info: ProgressInfo) { + self.listeners.iter().for_each(|l| l.paused(info.clone())); + } + + pub fn deleted(&self, info: ProgressInfo) { + self.listeners.iter().for_each(|l| l.deleted(info.clone())); + } + + pub fn start_error(&self, info: ProgressInfo) { + self.listeners + .iter() + .for_each(|l| l.start_error(info.clone())); + } + + pub fn pause_error(&self, info: ProgressInfo) { + self.listeners + .iter() + .for_each(|l| l.pause_error(info.clone())); + } + + pub fn delete_error(&self, info: ProgressInfo) { + self.listeners + .iter() + .for_each(|l| l.delete_error(info.clone())); + } +} + +#[derive(PartialEq, Eq, Hash, Clone)] +pub struct Context { + execution_id: String, + workspace_root_dir: String, + lib_root_dir: String, + docker_host: Option, + metadata: Option, +} + +// trait used to reimplement clone without same fields +// this trait is used for Context struct +pub trait Clone2 { + fn clone_not_same_execution_id(&self) -> Self; +} + +// for test we need to clone context but to change the directory workspace used +// to to this we just have to suffix the execution id in tests +impl Clone2 for Context { + fn clone_not_same_execution_id(&self) -> Context { + let mut new = self.clone(); + let suffix = rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(10) + .collect::(); + new.execution_id = format!("{}-{}", self.execution_id, suffix); + new + } +} + +impl Context { + pub fn new( + execution_id: &str, + workspace_root_dir: &str, + lib_root_dir: &str, + docker_host: Option, + metadata: Option, + ) -> Self { + Context { + execution_id: execution_id.to_string(), + workspace_root_dir: workspace_root_dir.to_string(), + lib_root_dir: lib_root_dir.to_string(), + docker_host, + metadata, + } + } + + pub fn execution_id(&self) -> &str { + self.execution_id.as_str() + } + + pub fn workspace_root_dir(&self) -> &str { + self.workspace_root_dir.as_str() + } + + pub fn lib_root_dir(&self) -> &str { + self.lib_root_dir.as_str() + } + + pub fn docker_tcp_socket(&self) -> Option<&String> { + self.docker_host.as_ref() + } + + pub fn metadata(&self) -> Option<&Metadata> { + self.metadata.as_ref() + } +} + +/// put everything you want here that is required to change the behaviour of the request. +/// E.g you can indicate that this request is a test, then you can adapt the behaviour as you want. +#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +pub struct Metadata { + pub test: Option, +} + +impl Metadata { + pub fn new(test: Option) -> Self { + Metadata { test } + } +} diff --git a/src/runtime.rs b/src/runtime.rs new file mode 100644 index 00000000..e0ef66c4 --- /dev/null +++ b/src/runtime.rs @@ -0,0 +1,9 @@ +use std::future::Future; + +use tokio::runtime::Runtime; + +pub fn async_run(future: F) -> F::Output { + // TODO improve - is it efficient to create a Runtime at each exec? + let mut runtime = Runtime::new().expect("unable to create a tokio runtime"); + runtime.block_on(future) +} diff --git a/src/s3.rs b/src/s3.rs new file mode 100644 index 00000000..5eb430ff --- /dev/null +++ b/src/s3.rs @@ -0,0 +1,257 @@ +use std::fmt::Display; +use std::fs::{read_to_string, File}; +use std::io; +use std::io::{Error, ErrorKind, Read, Write}; +use std::path::Path; +use std::str::FromStr; + +use retry::delay::Fibonacci; +use retry::OperationResult; +use rusoto_core::{Client, HttpClient, Region, RusotoError}; +use rusoto_credential::StaticProvider; +use rusoto_s3::{ + CreateBucketConfiguration, CreateBucketError, CreateBucketRequest, GetObjectError, + GetObjectRequest, ListObjectsV2Output, ListObjectsV2Request, PutBucketVersioningRequest, + S3Client, VersioningConfiguration, S3, +}; + +use crate::cmd::utilities::{exec_with_envs, CmdError}; +use crate::runtime::async_run; + +pub const AWS_REGION_FOR_S3_US: &str = "ap-south-1"; + +pub fn create_bucket( + access_key_id: &str, + secret_access_key: &str, + bucket_name: &str, +) -> Result<(), CmdError> { + exec_with_envs( + "aws", + vec!["s3api", "create-bucket", "--bucket", &bucket_name], + vec![ + ("AWS_ACCESS_KEY_ID", &access_key_id), + ("AWS_SECRET_ACCESS_KEY", &secret_access_key), + ], + ) +} + +pub type FileContent = String; + +pub fn get_object( + access_key_id: &str, + secret_access_key: &str, + region: &Region, + bucket_name: &str, + object_key: &str, +) -> Result { + let credentials = StaticProvider::new( + access_key_id.to_string(), + secret_access_key.to_string(), + None, + None, + ); + let client = Client::new_with(credentials, HttpClient::new().unwrap()); + let s3_client = S3Client::new_with_client(client, region.clone()); + + let mut or = GetObjectRequest::default(); + or.bucket = bucket_name.to_string(); + or.key = object_key.to_string(); + + let get_object_output = s3_client.get_object(or); + let r = async_run(get_object_output); + + let _err = Error::new( + ErrorKind::Other, + format!( + "something goes wrong while getting object {} in the S3 bucket {}", + object_key, bucket_name + ), + ); + + match r { + Ok(x) => { + let mut s = String::new(); + x.body.unwrap().into_blocking_read().read_to_string(&mut s); + + if s.is_empty() { + // this handle a case where the request succeeds but contains an empty body. + // https://github.com/rusoto/rusoto/issues/1822 + let r_from_aws_cli = get_object_via_aws_cli( + access_key_id, + secret_access_key, + bucket_name, + object_key, + )?; + return Ok(r_from_aws_cli); + } + Ok(s) + } + Err(err) => { + return match err { + RusotoError::Service(s) => match s { + GetObjectError::NoSuchKey(x) => { + info!("no such key '{}': {}", object_key, x.as_str()); + Err(Error::new( + ErrorKind::NotFound, + format!("no such key '{}': {}", object_key, x.as_str()), + )) + } + }, + RusotoError::Unknown(r) => { + let r_from_aws_cli = get_object_via_aws_cli( + access_key_id, + secret_access_key, + bucket_name, + object_key, + ); + + match r_from_aws_cli { + Ok(..) => Ok(r_from_aws_cli.unwrap()), + Err(err) => { + error!("{}", err); + Err(_err) + } + } + } + _ => Err(_err), + }; + } + } +} + +/// gets an aws s3 object using aws-cli +/// used as a failover when rusoto_s3 acts up +fn get_object_via_aws_cli( + access_key_id: &str, + secret_access_key: &str, + bucket_name: &str, + object_key: &str, +) -> Result { + let s3_url = format!("s3://{}/{}", bucket_name, object_key); + let local_path = format!("/tmp/{}", object_key); + let r = exec_with_envs( + "aws", + vec!["s3", "cp", &s3_url, &local_path], + vec![ + ("AWS_ACCESS_KEY_ID", &access_key_id), + ("AWS_SECRET_ACCESS_KEY", &secret_access_key), + ], + ); + match r { + Err(e) => return Err(Error::new(ErrorKind::Other, e)), + _ => {} + }; + let s = read_to_string(&local_path)?; + Ok(s) +} + +pub fn get_kubernetes_config_file

( + access_key_id: &str, + secret_access_key: &str, + region: &Region, + kubernetes_config_bucket_name: &str, + kubernetes_config_object_key: &str, + file_path: P, +) -> Result +where + P: AsRef, +{ + // return the file if it already exists + let _ = match File::open(file_path.as_ref()) { + Ok(f) => return Ok(f), + Err(_) => {} + }; + + let file_content_result = retry::retry(Fibonacci::from_millis(3000).take(5), || { + let file_content = crate::s3::get_object_via_aws_cli( + access_key_id, + secret_access_key, + kubernetes_config_bucket_name, + kubernetes_config_object_key, + ); + match file_content { + Ok(file_content) => OperationResult::Ok(file_content), + Err(err) => { + warn!( + "Can't download the kubernetes config file {} stored on {}, please check access key and secrets", + kubernetes_config_object_key, kubernetes_config_bucket_name + ); + OperationResult::Retry(err) + } + } + }); + + let file_content = match file_content_result { + Ok(file_content) => file_content, + Err(_) => { + return Err(Error::new( + ErrorKind::InvalidData, + "file content is empty (retry failed multiple times) - which is not the expected content - what's wrong?", + )); + } + }; + + let mut kubernetes_config_file = File::create(file_path.as_ref())?; + let _ = kubernetes_config_file.write(file_content.as_bytes())?; + + Ok(kubernetes_config_file) +} + +pub fn list_objects_in( + access_key_id: &str, + secret_access_key: &str, + bucket_name: &str, +) -> Result { + let credentials = StaticProvider::new( + access_key_id.to_string(), + secret_access_key.to_string(), + None, + None, + ); + let client = Client::new_with(credentials, HttpClient::new().unwrap()); + let s3_client = S3Client::new_with_client(client, get_default_region_for_us()); + let mut list_request = ListObjectsV2Request::default(); + list_request.bucket = bucket_name.to_string(); + let lis_object = s3_client.list_objects_v2(list_request); + let objects_in = async_run(lis_object); + match objects_in { + Ok(objects) => Ok(objects), + Err(err) => Err(Error::new(ErrorKind::Other, err)), + } +} + +// delete bucket implement by default objects deletion +pub fn delete_bucket( + access_key_id: &str, + secret_access_key: &str, + bucket_name: &str, +) -> Result<(), CmdError> { + info!("Deleting S3 Bucket {}", bucket_name.clone()); + match exec_with_envs( + "aws", + vec![ + "s3", + "rb", + "--force", + "--bucket", + format!("s3://{}", bucket_name).as_str(), + ], + vec![ + ("AWS_ACCESS_KEY_ID", &access_key_id), + ("AWS_SECRET_ACCESS_KEY", &secret_access_key), + ], + ) { + Ok(o) => { + info!("Successfuly delete bucket"); + return Ok(o); + } + Err(e) => { + error!("while deleting bucket {}", e); + return Err(e); + } + } +} + +pub fn get_default_region_for_us() -> Region { + Region::from_str(AWS_REGION_FOR_S3_US).unwrap() +} diff --git a/src/session.rs b/src/session.rs new file mode 100644 index 00000000..d5265107 --- /dev/null +++ b/src/session.rs @@ -0,0 +1,12 @@ +use crate::engine::Engine; +use crate::transaction::Transaction; + +pub struct Session<'a> { + pub engine: &'a Engine, +} + +impl<'a> Session<'a> { + pub fn transaction(self) -> Transaction<'a> { + Transaction::new(self.engine) + } +} diff --git a/src/string.rs b/src/string.rs new file mode 100644 index 00000000..69dd4611 --- /dev/null +++ b/src/string.rs @@ -0,0 +1,11 @@ +pub fn cut(str: String, max_length: usize) -> String { + if str.len() <= max_length { + str + } else { + str.as_str()[..max_length - 1].to_string() + } +} + +pub fn terraform_list_format(tf_vec: Vec) -> String { + format!("{{{}}}", tf_vec.join(",")) +} diff --git a/src/template.rs b/src/template.rs new file mode 100644 index 00000000..a9094212 --- /dev/null +++ b/src/template.rs @@ -0,0 +1,180 @@ +use std::ffi::OsStr; +use std::fs; +use std::fs::File; +use std::io::{Error, ErrorKind, Write}; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; + +use tera::Error as TeraError; +use tera::{Context, Tera}; +use walkdir::WalkDir; + +pub fn generate_and_copy_all_files_into_dir( + from_dir: S, + to_dir: P, + context: &Context, +) -> Result<(), Error> +where + S: AsRef + Copy, + P: AsRef + Copy, +{ + // generate j2 templates + let rendered_templates = match generate_j2_template_files(from_dir, context) { + Ok(rt) => rt, + Err(e) => { + let error_msg = match e.kind { + tera::ErrorKind::TemplateNotFound(x) => format!("template not found: {}", x), + tera::ErrorKind::Msg(x) => format!("tera error: {}", x), + tera::ErrorKind::CircularExtend { + tpl, + inheritance_chain, + } => format!( + "circular extend - template: {}, inheritance chain: {:?}", + tpl, inheritance_chain + ), + tera::ErrorKind::MissingParent { current, parent } => { + format!("missing parent - current: {}, parent: {}", current, parent) + } + tera::ErrorKind::FilterNotFound(x) => format!("filter not found: {}", x), + tera::ErrorKind::TestNotFound(x) => format!("test not found: {}", x), + tera::ErrorKind::InvalidMacroDefinition(x) => { + format!("invalid macro definition: {}", x) + } + tera::ErrorKind::FunctionNotFound(x) => format!("function not found: {}", x), + tera::ErrorKind::Json(x) => format!("json error: {:?}", x), + tera::ErrorKind::CallFunction(x) => format!("call function: {}", x), + tera::ErrorKind::CallFilter(x) => format!("call filter: {}", x), + tera::ErrorKind::CallTest(x) => format!("call test: {}", x), + tera::ErrorKind::__Nonexhaustive => format!("non exhaustive error"), + }; + + error!("{}", error_msg.as_str()); + return Err(Error::new(ErrorKind::InvalidData, error_msg)); + } + }; + + // copy all .tf and .yaml files into our dest directory + copy_non_template_files(from_dir.as_ref(), to_dir.as_ref())?; + + write_rendered_templates(&rendered_templates, to_dir.as_ref())?; + + Ok(()) +} + +pub fn copy_non_template_files(from: S, to: P) -> Result<(), Error> +where + S: AsRef, + P: AsRef, +{ + crate::fs::copy_files(from.as_ref(), to.as_ref(), true) +} + +pub fn generate_j2_template_files

( + root_dir: P, + context: &Context, +) -> Result, TeraError> +where + P: AsRef, +{ + //TODO: sort on fly context should be implemented to optimize reading + debug!("context: {:#?}", context); + let root_dir_str = root_dir.as_ref().to_str().unwrap(); + let tera_template_string = format!("{}/**/*.j2.*", root_dir_str); + + let tera = Tera::new(tera_template_string.as_str())?; + + let files = WalkDir::new(root_dir_str) + .follow_links(true) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| { + e.file_name() + .to_str() + .map(|s| s.contains(".j2.")) + .unwrap_or(false) + }) + .collect::>(); + + let mut results: Vec = vec![]; + + for file in files.into_iter() { + let path_str = file.path().to_str().unwrap(); + let j2_path = path_str.replace(root_dir_str, ""); + + let j2_file_name = file.file_name().to_str().unwrap(); + let j2_path_split = j2_path.split("/").collect::>(); + let j2_root_path: String = j2_path_split.as_slice()[..j2_path_split.len() - 1].join("/"); + let file_name = j2_file_name.replace(".j2", ""); + + let content = tera.render(&j2_path[1..], &context)?; + + results.push(RenderedTemplate::new(j2_root_path, file_name, content)); + } + + Ok(results) +} + +pub fn write_rendered_templates( + rendered_templates: &[RenderedTemplate], + into: &Path, +) -> Result<(), Error> { + for rt in rendered_templates { + let dest = format!("{}/{}", into.to_str().unwrap(), rt.path_and_file_name()); + + if dest.contains("/") { + // create the parent directories + let s_dest = dest.split("/").collect::>(); + let dir: String = s_dest.as_slice()[..s_dest.len() - 1].join("/"); + let _ = fs::create_dir_all(dir); + } + + // remove file if it already exists + let _ = fs::remove_file(dest.as_str()); + + // create an empty file + let mut f = fs::File::create(&dest)?; + + // write rendered template into the new file + f.write_all(rt.content.as_bytes())?; + + // perform spcific action based on the extension + let extension = Path::new(&dest).extension().and_then(OsStr::to_str); + match extension { + Some("sh") => set_file_permission(&f, 0o755), + _ => {} + } + } + + Ok(()) +} + +pub fn set_file_permission(f: &File, mode: u32) { + let metadata = f.metadata().unwrap(); + let mut permissions = metadata.permissions(); + permissions.set_mode(mode); + f.set_permissions(permissions).unwrap(); +} + +pub struct RenderedTemplate { + pub path: String, + pub file_name: String, + pub content: String, +} + +impl RenderedTemplate { + pub fn new(path: String, file_name: String, content: String) -> Self { + RenderedTemplate { + path, + file_name, + content, + } + } + + pub fn path_and_file_name(&self) -> String { + if self.path.trim().is_empty() || self.path.as_str() == "." { + self.file_name.clone() + } else { + format!("{}/{}", self.path.as_str(), self.file_name.as_str()) + } + } +} diff --git a/src/transaction.rs b/src/transaction.rs new file mode 100644 index 00000000..74c5bff3 --- /dev/null +++ b/src/transaction.rs @@ -0,0 +1,791 @@ +use std::collections::HashMap; +use std::thread; + +use serde::{Deserialize, Serialize}; + +use crate::build_platform::BuildError; +use crate::cloud_provider::kubernetes::{Kubernetes, KubernetesError}; +use crate::cloud_provider::service::ServiceError; +use crate::cloud_provider::service::{Application, Service}; +use crate::cloud_provider::DeployError; +use crate::container_registry::{PushError, PushResult}; +use crate::engine::Engine; +use crate::models::{ + Action, Environment, EnvironmentAction, EnvironmentError, ListenersHelper, ProgressInfo, + ProgressLevel, ProgressScope, +}; + +pub struct Transaction<'a> { + engine: &'a Engine, + steps: Vec>, + executed_steps: Vec>, +} + +impl<'a> Transaction<'a> { + pub fn new(engine: &'a Engine) -> Self { + Transaction::<'a> { + engine, + steps: vec![], + executed_steps: vec![], + } + } + + pub fn create_kubernetes( + &mut self, + kubernetes: &'a dyn Kubernetes, + ) -> Result<(), KubernetesError> { + match kubernetes.is_valid() { + Ok(_) => { + self.steps.push(Step::CreateKubernetes(kubernetes)); + Ok(()) + } + Err(err) => Err(err), + } + } + + pub fn delete_kubernetes( + &mut self, + kubernetes: &'a dyn Kubernetes, + ) -> Result<(), KubernetesError> { + match kubernetes.is_valid() { + Ok(_) => { + self.steps.push(Step::DeleteKubernetes(kubernetes)); + Ok(()) + } + Err(err) => Err(err), + } + } + + pub fn deploy_environment( + &mut self, + kubernetes: &'a dyn Kubernetes, + environment_action: &'a EnvironmentAction, + ) -> Result<(), EnvironmentError> { + self.deploy_environment_with_options( + kubernetes, + environment_action, + DeploymentOption { + force_build: false, + force_push: false, + }, + ) + } + + pub fn deploy_environment_with_options( + &mut self, + kubernetes: &'a dyn Kubernetes, + environment_action: &'a EnvironmentAction, + option: DeploymentOption, + ) -> Result<(), EnvironmentError> { + let _ = self.check_environment_action(environment_action)?; + + // add build step + self.steps + .push(Step::BuildEnvironment(environment_action, option)); + + // add deployment step + self.steps + .push(Step::DeployEnvironment(kubernetes, environment_action)); + + Ok(()) + } + + pub fn pause_environment( + &mut self, + kubernetes: &'a dyn Kubernetes, + environment_action: &'a EnvironmentAction, + ) -> Result<(), EnvironmentError> { + let _ = self.check_environment_action(environment_action)?; + + self.steps + .push(Step::PauseEnvironment(kubernetes, environment_action)); + Ok(()) + } + + pub fn delete_environment( + &mut self, + kubernetes: &'a dyn Kubernetes, + environment_action: &'a EnvironmentAction, + ) -> Result<(), EnvironmentError> { + let _ = self.check_environment_action(environment_action)?; + + self.steps + .push(Step::DeleteEnvironment(kubernetes, environment_action)); + Ok(()) + } + + fn check_environment_action( + &self, + environment_action: &EnvironmentAction, + ) -> Result<(), EnvironmentError> { + match environment_action { + EnvironmentAction::Environment(te) => match te.is_valid() { + Ok(_) => {} + Err(err) => return Err(err), + }, + EnvironmentAction::EnvironmentWithFailover(te, fe) => { + match te.is_valid() { + Ok(_) => {} + Err(err) => return Err(err), + }; + + match fe.is_valid() { + Ok(_) => {} + Err(err) => return Err(err), + }; + } + }; + + Ok(()) + } + + fn _build_applications( + &self, + environment: &Environment, + option: &DeploymentOption, + ) -> Result>, BuildError> { + let external_services_to_build = environment + .external_services + .iter() + // build only applications that are set with Action: Create + .filter(|es| es.action == Action::Create); + + let external_service_and_result_tuples = external_services_to_build + .map(|es| { + ( + es, + self.engine + .build_platform() + .build(es.to_build(), option.force_build), + ) + }) + .collect::>(); + + // do the same for applications + + let apps_to_build = environment + .applications + .iter() + // build only applications that are set with Action: Create + .filter(|app| app.action == Action::Create); + + let application_and_result_tuples = apps_to_build + .map(|app| { + ( + app, + self.engine + .build_platform() + .build(app.to_build(), option.force_build), + ) + }) + .collect::>(); + + let mut applications: Vec> = + Vec::with_capacity(application_and_result_tuples.len()); + + for (external_service, result) in external_service_and_result_tuples { + // catch build error, can't do it in Fn + let build_result = match result { + Err(err) => { + error!( + "build error for external_service {}: {:?}", + external_service.id.as_str(), + err + ); + return Err(err); + } + Ok(build_result) => build_result, + }; + + match external_service.to_application( + self.engine.context(), + &build_result.build.image, + self.engine.cloud_provider(), + ) { + Some(x) => applications.push(x), + None => {} + } + } + + for (application, result) in application_and_result_tuples { + // catch build error, can't do it in Fn + let build_result = match result { + Err(err) => { + error!( + "build error for application {}: {:?}", + application.id.as_str(), + err + ); + return Err(err); + } + Ok(build_result) => build_result, + }; + + match application.to_application( + self.engine.context(), + &build_result.build.image, + self.engine.cloud_provider(), + ) { + Some(x) => applications.push(x), + None => {} + } + } + + Ok(applications) + } + + fn _push_applications( + &self, + applications: Vec>, + option: &DeploymentOption, + ) -> Result, PushResult)>, PushError> { + let application_and_push_results: Vec<_> = applications + .into_iter() + .map(|mut app| { + match self + .engine + .container_registry() + .push(app.image(), option.force_push) + { + Ok(push_result) => { + // I am not a big fan of doing that but it's the most effective way + app.set_image(push_result.image.clone()); + Ok((app, push_result)) + } + Err(err) => Err(err), + } + }) + .collect(); + + let mut results: Vec<(Box, PushResult)> = vec![]; + for result in application_and_push_results.into_iter() { + match result { + Ok(tuple) => results.push(tuple), + Err(err) => { + error!("error pushing docker image {:?}", err); + return Err(err); + } + } + } + + Ok(results) + } + + fn check_environment( + &self, + environment: &crate::cloud_provider::environment::Environment, + ) -> TransactionResult { + match environment.is_valid() { + Err(service_error) => { + warn!("ROLLBACK STARTED! an error occurred {:?}", service_error); + return match self.rollback() { + Ok(_) => { + TransactionResult::Rollback(CommitError::NotValidService(service_error)) + } + Err(err) => { + error!("ROLLBACK FAILED! fatal error: {:?}", err); + TransactionResult::UnrecoverableError( + CommitError::NotValidService(service_error), + err, + ) + } + }; + } + _ => {} + }; + + TransactionResult::Ok + } + + pub fn rollback(&self) -> Result<(), RollbackError> { + for step in self.executed_steps.iter() { + match step { + Step::CreateKubernetes(kubernetes) => { + // revert kubernetes creation + match kubernetes.on_create_error() { + Err(err) => return Err(RollbackError::CreateKubernetes(err)), + _ => {} + }; + } + Step::DeleteKubernetes(kubernetes) => { + // revert kubernetes deletion + match kubernetes.on_delete_error() { + Err(err) => return Err(RollbackError::DeleteKubernetes(err)), + _ => {} + }; + } + Step::BuildEnvironment(_environment_action, _option) => { + // revert build applications + } + Step::DeployEnvironment(kubernetes, environment_action) => { + // revert environment deployment + self.rollback_environment(*kubernetes, *environment_action)?; + } + Step::PauseEnvironment(kubernetes, environment_action) => { + self.rollback_environment(*kubernetes, *environment_action)?; + } + Step::DeleteEnvironment(kubernetes, environment_action) => { + self.rollback_environment(*kubernetes, *environment_action)?; + } + } + } + + Ok(()) + } + + /// This function is a wrapper to correctly revert all changes of an attempted deployment AND + /// if a failover environment is provided, then rollback. + fn rollback_environment( + &self, + kubernetes: &dyn Kubernetes, + environment_action: &EnvironmentAction, + ) -> Result<(), RollbackError> { + let qe_environment = |environment: &Environment| { + let mut _applications = Vec::with_capacity( + // ExternalService impl Application (which is a StatelessService) + environment.applications.len() + environment.external_services.len(), + ); + + for application in environment.applications.iter() { + let build = application.to_build(); + + match application.to_application( + self.engine.context(), + &build.image, + self.engine.cloud_provider(), + ) { + Some(x) => _applications.push(x), + None => {} + } + } + + for external_service in environment.external_services.iter() { + let build = external_service.to_build(); + + match external_service.to_application( + self.engine.context(), + &build.image, + self.engine.cloud_provider(), + ) { + Some(x) => _applications.push(x), + None => {} + } + } + + let qe_environment = environment.to_qe_environment( + self.engine.context(), + &_applications, + self.engine.cloud_provider(), + ); + + qe_environment + }; + + match environment_action { + EnvironmentAction::EnvironmentWithFailover( + target_environment, + failover_environment, + ) => { + // let's reverse changes and rollback on the provided failover version + let target_qe_environment = qe_environment(&target_environment); + let failover_qe_environment = qe_environment(&failover_environment); + + let action = match failover_environment.action { + Action::Create => { + kubernetes.deploy_environment_error(&target_qe_environment); + kubernetes.deploy_environment(&failover_qe_environment) + } + Action::Pause => { + kubernetes.pause_environment_error(&target_qe_environment); + kubernetes.pause_environment(&failover_qe_environment) + } + Action::Delete => { + kubernetes.delete_environment_error(&target_qe_environment); + kubernetes.delete_environment(&failover_qe_environment) + } + Action::Nothing => Ok(()), + }; + + let _ = match action { + Ok(_) => {} + Err(err) => { + return Err(match failover_environment.action { + Action::Create => RollbackError::DeployEnvironment(err), + Action::Pause => RollbackError::PauseEnvironment(err), + Action::Delete => RollbackError::DeleteEnvironment(err), + Action::Nothing => RollbackError::Error, // it can't happens + }); + } + }; + + Ok(()) + } + EnvironmentAction::Environment(te) => { + // revert changes but there is no failover environment + let target_qe_environment = qe_environment(&te); + + let action = match te.action { + Action::Create => kubernetes.deploy_environment_error(&target_qe_environment), + Action::Pause => kubernetes.pause_environment_error(&target_qe_environment), + Action::Delete => kubernetes.delete_environment_error(&target_qe_environment), + Action::Nothing => Ok(()), + }; + + let _ = match action { + Ok(_) => {} + Err(err) => { + return Err(match te.action { + Action::Create => RollbackError::DeployEnvironment(err), + Action::Pause => RollbackError::PauseEnvironment(err), + Action::Delete => RollbackError::DeleteEnvironment(err), + Action::Nothing => RollbackError::Error, // it can't happens + }); + } + }; + + Err(RollbackError::NoFailoverEnvironment) + } + } + } + + pub fn commit(&mut self) -> TransactionResult { + let mut applications_by_environment: HashMap<&Environment, Vec>> = + HashMap::new(); + + for step in self.steps.iter() { + // execution loop + self.executed_steps.push(step.clone()); + + match step { + Step::CreateKubernetes(kubernetes) => { + // create kubernetes + match kubernetes.on_create() { + Err(err) => { + warn!("ROLLBACK STARTED! an error occurred {:?}", err); + match self.rollback() { + Ok(_) => { + TransactionResult::Rollback(CommitError::CreateKubernetes(err)) + } + Err(e) => { + error!("ROLLBACK FAILED! fatal error: {:?}", e); + TransactionResult::UnrecoverableError( + CommitError::CreateKubernetes(err), + e, + ) + } + } + } + _ => TransactionResult::Ok, + }; + } + Step::DeleteKubernetes(kubernetes) => { + // delete kubernetes + match kubernetes.on_delete() { + Err(err) => { + warn!("ROLLBACK STARTED! an error occurred {:?}", err); + match self.rollback() { + Ok(_) => { + TransactionResult::Rollback(CommitError::DeleteKubernetes(err)) + } + Err(e) => { + error!("ROLLBACK FAILED! fatal error: {:?}", e); + TransactionResult::UnrecoverableError( + CommitError::DeleteKubernetes(err), + e, + ) + } + } + } + _ => TransactionResult::Ok, + }; + } + Step::BuildEnvironment(environment_action, option) => { + // build applications + let target_environment = match environment_action { + EnvironmentAction::Environment(te) => te, + EnvironmentAction::EnvironmentWithFailover(te, _) => te, + }; + + let apps_result = match self._build_applications(target_environment, option) { + Ok(applications) => match self._push_applications(applications, option) { + Ok(results) => { + let applications = + results.into_iter().map(|(app, _)| app).collect::>(); + + Ok(applications) + } + Err(err) => Err(CommitError::PushImage(err)), + }, + Err(err) => Err(CommitError::BuildImage(err)), + }; + + if apps_result.is_err() { + let commit_error = apps_result.err().unwrap(); + warn!("ROLLBACK STARTED! an error occurred {:?}", commit_error); + + return match self.rollback() { + Ok(_) => TransactionResult::Rollback(commit_error), + Err(err) => { + error!("ROLLBACK FAILED! fatal error: {:?}", err); + TransactionResult::UnrecoverableError(commit_error, err) + } + }; + } + + let applications = apps_result.ok().unwrap(); + applications_by_environment.insert(target_environment, applications); + } + Step::DeployEnvironment(kubernetes, environment_action) => { + // deploy complete environment + match self.commit_environment( + *kubernetes, + *environment_action, + &applications_by_environment, + |qe_env| kubernetes.deploy_environment(qe_env), + |err| CommitError::DeployEnvironment(err), + ) { + TransactionResult::Ok => {} + err => return err, + }; + } + Step::PauseEnvironment(kubernetes, environment_action) => { + // pause complete environment + match self.commit_environment( + *kubernetes, + *environment_action, + &applications_by_environment, + |qe_env| kubernetes.pause_environment(qe_env), + |err| CommitError::PauseEnvironment(err), + ) { + TransactionResult::Ok => {} + err => return err, + }; + } + Step::DeleteEnvironment(kubernetes, environment_action) => { + // delete complete environment + match self.commit_environment( + *kubernetes, + *environment_action, + &applications_by_environment, + |qe_env| kubernetes.delete_environment(qe_env), + |err| CommitError::DeleteEnvironment(err), + ) { + TransactionResult::Ok => {} + err => return err, + }; + } + }; + } + + TransactionResult::Ok + } + + fn commit_environment( + &self, + kubernetes: &dyn Kubernetes, + environment_action: &EnvironmentAction, + applications_by_environment: &HashMap<&Environment, Vec>>, + action_fn: F, + commit_error: E, + ) -> TransactionResult + where + F: Fn(&crate::cloud_provider::environment::Environment) -> Result<(), KubernetesError>, + E: Fn(KubernetesError) -> CommitError, + { + let target_environment = match environment_action { + EnvironmentAction::Environment(te) => te, + EnvironmentAction::EnvironmentWithFailover(te, _) => te, + }; + + let empty_vec = Vec::with_capacity(0); + let built_applications = match applications_by_environment.get(target_environment) { + Some(applications) => applications, + None => &empty_vec, + }; + + let qe_environment = target_environment.to_qe_environment( + self.engine.context(), + built_applications, + kubernetes.cloud_provider(), + ); + + let _ = match self.check_environment(&qe_environment) { + TransactionResult::Ok => {} + err => return err, // which it means that an error occurred + }; + + let execution_id = self.engine.context().execution_id(); + + // inner function - I use it instead of closure because of ?Sized + fn get_final_progress_info(service: &Box, execution_id: &str) -> ProgressInfo + where + T: Service + ?Sized, + { + ProgressInfo::new( + service.progress_scope(), + ProgressLevel::Info, + None::<&str>, + execution_id, + ) + }; + + // send the back the right progress status + fn send_progress( + kubernetes: &dyn Kubernetes, + action: &Action, + service: &Box, + execution_id: &str, + is_error: bool, + ) where + T: Service + ?Sized, + { + let lh = ListenersHelper::new(kubernetes.listeners()); + let progress_info = get_final_progress_info(service, execution_id); + + if !is_error { + match action { + Action::Create => lh.started(progress_info), + Action::Pause => lh.paused(progress_info), + Action::Delete => lh.deleted(progress_info), + Action::Nothing => {} // nothing to do here? + }; + return; + } + + match action { + Action::Create => lh.start_error(progress_info), + Action::Pause => lh.pause_error(progress_info), + Action::Delete => lh.delete_error(progress_info), + Action::Nothing => {} // nothing to do here? + }; + } + + // 100 ms sleep to avoid race condition on last service status update + // Otherwise, the last status sent to the CORE is (sometimes) not the right one. + // Even by storing data at the micro seconds precision + thread::sleep(std::time::Duration::from_millis(100)); + + let _ = match action_fn(&qe_environment) { + Err(err) => { + let rollback_result = match self.rollback() { + Ok(_) => TransactionResult::Rollback(commit_error(err)), + Err(rollback_err) => { + error!("ROLLBACK FAILED! fatal error: {:?}", rollback_err); + TransactionResult::UnrecoverableError(commit_error(err), rollback_err) + } + }; + + // !!! don't change the order + // terminal update + for service in &qe_environment.stateful_services { + send_progress( + kubernetes, + &target_environment.action, + service, + execution_id, + true, + ); + } + + for service in &qe_environment.stateless_services { + send_progress( + kubernetes, + &target_environment.action, + service, + execution_id, + true, + ); + } + + return rollback_result; + } + _ => { + // terminal update + for service in &qe_environment.stateful_services { + send_progress( + kubernetes, + &target_environment.action, + service, + execution_id, + false, + ); + } + + for service in &qe_environment.stateless_services { + send_progress( + kubernetes, + &target_environment.action, + service, + execution_id, + false, + ); + } + } + }; + + TransactionResult::Ok + } +} + +#[derive(Clone)] +pub struct DeploymentOption { + pub force_build: bool, + pub force_push: bool, +} + +enum Step<'a> { + // init and create all the necessary resources (Network, Kubernetes) + CreateKubernetes(&'a dyn Kubernetes), + DeleteKubernetes(&'a dyn Kubernetes), + BuildEnvironment(&'a EnvironmentAction, DeploymentOption), + DeployEnvironment(&'a dyn Kubernetes, &'a EnvironmentAction), + PauseEnvironment(&'a dyn Kubernetes, &'a EnvironmentAction), + DeleteEnvironment(&'a dyn Kubernetes, &'a EnvironmentAction), +} + +impl<'a> Clone for Step<'a> { + fn clone(&self) -> Self { + match self { + Step::CreateKubernetes(k) => Step::CreateKubernetes(*k), + Step::DeleteKubernetes(k) => Step::DeleteKubernetes(*k), + Step::BuildEnvironment(e, option) => Step::BuildEnvironment(*e, option.clone()), + Step::DeployEnvironment(k, e) => Step::DeployEnvironment(*k, *e), + Step::PauseEnvironment(k, e) => Step::PauseEnvironment(*k, *e), + Step::DeleteEnvironment(k, e) => Step::DeleteEnvironment(*k, *e), + } + } +} + +#[derive(Debug)] +pub enum CommitError { + CreateKubernetes(KubernetesError), + DeleteKubernetes(KubernetesError), + DeployEnvironment(KubernetesError), + PauseEnvironment(KubernetesError), + DeleteEnvironment(KubernetesError), + NotValidService(ServiceError), + BuildImage(BuildError), + PushImage(PushError), + DeployImage(DeployError), +} + +#[derive(Debug)] +pub enum RollbackError { + CreateKubernetes(KubernetesError), + DeleteKubernetes(KubernetesError), + DeployEnvironment(KubernetesError), + PauseEnvironment(KubernetesError), + DeleteEnvironment(KubernetesError), + NotValidService(ServiceError), + BuildImage(BuildError), + PushImage(PushError), + DeployImage(DeployError), + NoFailoverEnvironment, + Error, +} + +pub enum TransactionResult { + Ok, + Rollback(CommitError), + UnrecoverableError(CommitError, RollbackError), +} diff --git a/src/unit_conversion.rs b/src/unit_conversion.rs new file mode 100644 index 00000000..4c0ccc8e --- /dev/null +++ b/src/unit_conversion.rs @@ -0,0 +1,62 @@ +use std::num::{ParseFloatError, ParseIntError}; + +/// convert a cpu string (kubernetes like) into a float. It supports millis cpu +/// examples: +/// 250m = 0.25 cpu +/// 500m = 0.50 cpu +/// 1000m = 1 cpu +/// 1.25 = 1.25 +pub fn cpu_string_to_float>(cpu: T) -> f32 { + let cpu = cpu.into(); + if cpu.is_empty() { + return 0.0; + } + + if !cpu.ends_with("m") { + // the value is not in millis + return match cpu.parse::() { + Ok(v) if v >= 0.0 => v, + _ => 0.0, + }; + } + + // the result is in millis, so convert it to float + let cpu = cpu.replace("m", ""); + match cpu.parse::() { + Ok(v) if v >= 0.0 => v / 1000.0, + _ => 0.0, + } +} + +/// convert ki to mi +pub fn ki_to_mi>(ram: T) -> u32 { + let ram = ram.into().to_lowercase().replace("ki", ""); + match ram.parse::() { + Ok(v) => v / 1000, + _ => 0, + } +} + +#[cfg(test)] +mod tests { + use crate::unit_conversion::cpu_string_to_float; + use crate::unit_conversion::ki_to_mi; + + #[test] + fn test_cpu_conversions() { + assert_eq!(cpu_string_to_float("250m"), 0.25); + assert_eq!(cpu_string_to_float("500m"), 0.5); + assert_eq!(cpu_string_to_float("1500m"), 1.5); + assert_eq!(cpu_string_to_float("1.5"), 1.5); + assert_eq!(cpu_string_to_float("0"), 0.0); + assert_eq!(cpu_string_to_float("0m"), 0.0); + assert_eq!(cpu_string_to_float("-250m"), 0.0); + assert_eq!(cpu_string_to_float("-10"), 0.0); + assert_eq!(cpu_string_to_float("1000"), 1000.0); + } + + #[test] + fn test_kib_to_mib_conversions() { + assert_eq!(ki_to_mi("15564756Ki"), 15_564); + } +} diff --git a/test_utilities/Cargo.toml b/test_utilities/Cargo.toml new file mode 100644 index 00000000..c534ce93 --- /dev/null +++ b/test_utilities/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "test-utilities" +version = "0.1.0" +authors = ["Romaric Philogene "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +qovery-engine = { path = "../" } +chrono = "0.4.11" +dirs = "3.0.1" +env_logger = "0.7.1" +rand = "0.7.3" +serde = "1.0" +serde_json = "1.0.57" +serde_derive = "1.0" +curl = "0.4.34" + + +# Digital Ocean Deps +digitalocean = "0.1.1" \ No newline at end of file diff --git a/test_utilities/src/aws.rs b/test_utilities/src/aws.rs new file mode 100644 index 00000000..faace59d --- /dev/null +++ b/test_utilities/src/aws.rs @@ -0,0 +1,688 @@ +extern crate serde; +extern crate serde_derive; + +use std::borrow::Borrow; +use std::fs::File; +use std::io::Read; +use std::str::FromStr; + +use chrono::Utc; +use dirs::home_dir; +use serde_json::map::Values; +use serde_json::value::Value; + +use qovery_engine::build_platform::local_docker::LocalDocker; +use qovery_engine::build_platform::BuildPlatform; +use qovery_engine::cloud_provider::aws::kubernetes::node::Node; +use qovery_engine::cloud_provider::aws::kubernetes::EKS; +use qovery_engine::cloud_provider::aws::AWS; +use qovery_engine::cloud_provider::{CloudProvider, TerraformStateCredentials}; +use qovery_engine::container_registry::docker_hub::DockerHub; +use qovery_engine::container_registry::ecr::ECR; +use qovery_engine::container_registry::ContainerRegistry; +use qovery_engine::dns_provider::cloudflare::Cloudflare; +use qovery_engine::dns_provider::DnsProvider; +use qovery_engine::engine::Engine; +use qovery_engine::models::{ + Action, Application, Context, CustomDomain, Database, DatabaseKind, Environment, + EnvironmentVariable, GitCredentials, Kind, Metadata, Route, Router, Storage, StorageType, +}; +use qovery_engine::session::Session; + +use crate::cloudflare::dns_provider_cloudflare; +use crate::utilities::init; +use crate::utilities::{build_platform_local_docker, generate_id}; + +pub const AWS_ACCESS_KEY_ID: &str = "CHANGE ME"; +pub const AWS_SECRET_ACCESS_KEY: &str = "CHANGE ME"; +pub const AWS_DEFAULT_REGION: &str = "us-east-2"; +pub const TERRAFORM_AWS_ACCESS_KEY_ID: &str = "CHANGE ME"; +pub const TERRAFORM_AWS_SECRET_ACCESS_KEY: &str = "CHANGE ME"; +pub const ORGANIZATION_ID: &str = "u8nb94c7fwxzr2jt"; +pub const AWS_REGION_FOR_S3: &str = "us-east-1"; +pub const AWS_KUBERNETES_VERSION: &str = "1.16"; + +pub fn execution_id() -> String { + Utc::now() + .to_rfc3339() + .replace(":", "-") + .replace(".", "-") + .replace("+", "-") +} + +pub fn context() -> Context { + let execution_id = execution_id(); + let home_dir = std::env::var("WORKSPACE_ROOT_DIR") + .unwrap_or(home_dir().unwrap().to_str().unwrap().to_string()); + let lib_root_dir = std::env::var("LIB_ROOT_DIR").expect("LIB_ROOT_DIR is mandatory"); + let metadata = Metadata { + test: Option::from(true), + }; + + Context::new( + execution_id.as_str(), + home_dir.as_str(), + lib_root_dir.as_str(), + None, + Option::from(metadata), + ) +} + +pub fn container_registry_ecr(context: &Context) -> ECR { + ECR::new( + context.clone(), + "default-ecr-registry-Qovery Test", + "ea59qe62xaw3wjai", + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + AWS_DEFAULT_REGION, + ) +} + +pub fn container_registry_docker_hub(context: &Context) -> DockerHub { + DockerHub::new( + context.clone(), + "my-docker-hub-id-123", + "my-default-docker-hub", + "qoveryrd", + "3b9481fe-74e7-4d7b-bc08-e147c9fd4f24", + ) +} + +pub fn aws_kubernetes_nodes() -> Vec { + vec![ + Node::new(2, 16), + Node::new(2, 16), + Node::new(2, 16), + Node::new(2, 16), + ] +} + +pub fn cloud_provider_aws(context: &Context) -> AWS { + AWS::new( + context.clone(), + "u8nb94c7fwxzr2jt", + ORGANIZATION_ID, + "QoveryTest", + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + TerraformStateCredentials { + access_key_id: TERRAFORM_AWS_ACCESS_KEY_ID.to_string(), + secret_access_key: TERRAFORM_AWS_SECRET_ACCESS_KEY.to_string(), + region: "eu-west-3".to_string(), + }, + ) +} + +pub fn aws_kubernetes_eks<'a>( + context: &Context, + cloud_provider: &'a AWS, + dns_provider: &'a dyn DnsProvider, + nodes: Vec, +) -> EKS<'a> { + let mut file = File::open("tests/assets/eks-options.json").expect("file not found"); + let options_values = serde_json::from_reader(file).expect("JSON was not well-formatted"); + EKS::<'a>::new( + context.clone(), + "dmubm9agk7sr8a8r", + "dmubm9agk7sr8a8r", + AWS_KUBERNETES_VERSION, + AWS_DEFAULT_REGION, + cloud_provider, + dns_provider, + options_values, + nodes, + ) +} + +pub fn docker_ecr_aws_engine(context: &Context) -> Engine { + // use ECR + let container_registry = Box::new(container_registry_ecr(context)); + + // use LocalDocker + let build_platform = Box::new(build_platform_local_docker(context)); + + // use AWS + let cloud_provider = Box::new(cloud_provider_aws(context)); + + let dns_provider = Box::new(dns_provider_cloudflare(context)); + + Engine::new( + context.clone(), + build_platform, + container_registry, + cloud_provider, + dns_provider, + ) +} + +pub fn environment_3_apps_3_routers_3_databases(context: &Context) -> Environment { + let app_name_1 = format!("{}-{}", "simple-app-1".to_string(), generate_id()); + let app_name_2 = format!("{}-{}", "simple-app-2".to_string(), generate_id()); + let app_name_3 = format!("{}-{}", "simple-app-3".to_string(), generate_id()); + + // mongoDB management part + let database_host_mongo = "mongodb-".to_string() + generate_id().as_str() + ".oom.sh"; // External access check + let database_port_mongo = 27017; + let database_db_name_mongo = "my-mongodb".to_string(); + let database_username_mongo = "superuser".to_string(); + let database_password_mongo = generate_id(); + let database_uri_mongo = format!( + "mongodb://{}:{}@{}:{}/{}", + database_username_mongo, + database_password_mongo, + database_host_mongo, + database_port_mongo, + database_db_name_mongo + ); + let version_mongo = "4.4"; + + // pSQL 1 management part + let fqdn_id = "my-postgresql-".to_string() + generate_id().as_str(); + let fqdn = fqdn_id.clone() + ".oom.sh"; + let database_port = 5432; + let database_username = "superuser".to_string(); + let database_password = generate_id(); + let database_name = "my-psql".to_string(); + + // pSQL 2 management part + let fqdn_id_2 = "my-postgresql-2".to_string() + generate_id().as_str(); + let fqdn_2 = fqdn_id_2.clone() + ".oom.sh"; + let database_username_2 = "superuser2".to_string(); + let database_name_2 = "my-psql-2".to_string(); + + Environment { + execution_id: context.execution_id().to_string(), + id: generate_id(), + kind: Kind::Development, + owner_id: generate_id(), + project_id: generate_id(), + organization_id: ORGANIZATION_ID.to_string(), + action: Action::Create, + applications: vec![ + Application { + id: generate_id(), + name: app_name_1.clone(), + git_url: "https://github.com/Qovery/engine-testing.git".to_string(), + commit_id: "5990752647af11ef21c3d46a51abbde3da1ab351".to_string(), + dockerfile_path: "Dockerfile".to_string(), + action: Action::Create, + git_credentials: GitCredentials { + login: "x-access-token".to_string(), + access_token: "CHANGE ME".to_string(), + expired_at: Utc::now(), + }, + storage: vec![Storage { + id: generate_id(), + name: "photos".to_string(), + storage_type: StorageType::Ssd, + size_in_gib: 10, + mount_point: "/mnt/photos".to_string(), + snapshot_retention_in_days: 0, + }], + environment_variables: vec![ + EnvironmentVariable { + key: "PG_DBNAME".to_string(), + value: database_name.clone(), + }, + EnvironmentVariable { + key: "PG_HOST".to_string(), + value: fqdn.clone(), + }, + EnvironmentVariable { + key: "PG_PORT".to_string(), + value: database_port.clone().to_string(), + }, + EnvironmentVariable { + key: "PG_USERNAME".to_string(), + value: database_username.clone(), + }, + EnvironmentVariable { + key: "PG_PASSWORD".to_string(), + value: database_password.clone(), + }, + ], + branch: "master".to_string(), + private_port: Some(1234), + total_cpus: "100m".to_string(), + total_ram_in_mib: 256, + total_instances: 2, + cpu_burst: "100m".to_string(), + }, + Application { + id: generate_id(), + name: app_name_2.clone(), + git_url: "https://github.com/Qovery/engine-testing.git".to_string(), + commit_id: "5990752647af11ef21c3d46a51abbde3da1ab351".to_string(), + dockerfile_path: "Dockerfile".to_string(), + action: Action::Create, + git_credentials: GitCredentials { + login: "x-access-token".to_string(), + access_token: "CHANGE ME".to_string(), + expired_at: Utc::now(), + }, + storage: vec![Storage { + id: generate_id(), + name: "photos".to_string(), + storage_type: StorageType::Ssd, + size_in_gib: 10, + mount_point: "/mnt/photos".to_string(), + snapshot_retention_in_days: 0, + }], + environment_variables: vec![ + EnvironmentVariable { + key: "PG_DBNAME".to_string(), + value: database_name_2.clone(), + }, + EnvironmentVariable { + key: "PG_HOST".to_string(), + value: fqdn_2.clone(), + }, + EnvironmentVariable { + key: "PG_PORT".to_string(), + value: database_port.clone().to_string(), + }, + EnvironmentVariable { + key: "PG_USERNAME".to_string(), + value: database_username_2.clone(), + }, + EnvironmentVariable { + key: "PG_PASSWORD".to_string(), + value: database_password.clone(), + }, + ], + branch: "master".to_string(), + private_port: Some(1234), + total_cpus: "100m".to_string(), + total_ram_in_mib: 256, + total_instances: 2, + cpu_burst: "100m".to_string(), + }, + Application { + id: generate_id(), + name: app_name_3.clone(), + git_url: "https://github.com/Qovery/engine-testing.git".to_string(), + commit_id: "158ea8ebc9897c50a7c56b910db33ce837ac1e61".to_string(), + dockerfile_path: format!("Dockerfile-{}", version_mongo), + action: Action::Create, + git_credentials: GitCredentials { + login: "x-access-token".to_string(), + access_token: "CHANGE ME".to_string(), + expired_at: Utc::now(), + }, + storage: vec![Storage { + id: generate_id(), + name: "photos".to_string(), + storage_type: StorageType::Ssd, + size_in_gib: 10, + mount_point: "/mnt/photos".to_string(), + snapshot_retention_in_days: 0, + }], + environment_variables: vec![ + EnvironmentVariable { + key: "IS_DOCUMENTDB".to_string(), + value: "false".to_string(), + }, + EnvironmentVariable { + key: "QOVERY_DATABASE_TESTING_DATABASE_FQDN".to_string(), + value: database_host_mongo.clone(), + }, + EnvironmentVariable { + key: "QOVERY_DATABASE_MY_DDB_CONNECTION_URI".to_string(), + value: database_uri_mongo.clone(), + }, + EnvironmentVariable { + key: "QOVERY_DATABASE_TESTING_DATABASE_PORT".to_string(), + value: database_port_mongo.clone().to_string(), + }, + EnvironmentVariable { + key: "MONGODB_DBNAME".to_string(), + value: database_db_name_mongo.clone(), + }, + EnvironmentVariable { + key: "QOVERY_DATABASE_TESTING_DATABASE_USERNAME".to_string(), + value: database_username_mongo.clone(), + }, + EnvironmentVariable { + key: "QOVERY_DATABASE_TESTING_DATABASE_PASSWORD".to_string(), + value: database_password_mongo.clone(), + }, + ], + branch: "master".to_string(), + private_port: Some(1234), + total_cpus: "100m".to_string(), + total_ram_in_mib: 256, + total_instances: 2, + cpu_burst: "100m".to_string(), + }, + ], + routers: vec![ + Router { + id: generate_id(), + name: "main".to_string(), + action: Action::Create, + default_domain: generate_id() + ".oom.sh", + public_port: 443, + custom_domains: vec![], + routes: vec![Route { + path: "/app1".to_string(), + application_name: app_name_1.clone(), + }], + }, + Router { + id: generate_id(), + name: "second-router".to_string(), + action: Action::Create, + default_domain: generate_id() + ".oom.sh", + public_port: 443, + custom_domains: vec![], + routes: vec![Route { + path: "/app2".to_string(), + application_name: app_name_2.clone(), + }], + }, + Router { + id: generate_id(), + name: "third-router".to_string(), + action: Action::Create, + default_domain: generate_id() + ".oom.sh", + public_port: 443, + custom_domains: vec![], + routes: vec![Route { + path: "/app3".to_string(), + application_name: app_name_3.clone(), + }], + }, + ], + databases: vec![ + Database { + kind: DatabaseKind::Postgresql, + action: Action::Create, + id: generate_id(), + name: database_name.clone(), + version: "11.8.0".to_string(), + fqdn_id: fqdn_id.clone(), + fqdn: fqdn.clone(), + port: database_port.clone(), + username: database_username.clone(), + password: database_password.clone(), + total_cpus: "100m".to_string(), + total_ram_in_mib: 512, + disk_size_in_gib: 10, + database_instance_type: "db.t2.micro".to_string(), + database_disk_type: "gp2".to_string(), + }, + Database { + kind: DatabaseKind::Postgresql, + action: Action::Create, + id: generate_id(), + name: database_name_2.clone(), + version: "11.8.0".to_string(), + fqdn_id: fqdn_id_2.clone(), + fqdn: fqdn_2.clone(), + port: database_port.clone(), + username: database_username_2.clone(), + password: database_password.clone(), + total_cpus: "100m".to_string(), + total_ram_in_mib: 512, + disk_size_in_gib: 10, + database_instance_type: "db.t2.micro".to_string(), + database_disk_type: "gp2".to_string(), + }, + Database { + kind: DatabaseKind::Mongodb, + action: Action::Create, + id: generate_id(), + name: database_db_name_mongo.clone(), + version: version_mongo.to_string(), + fqdn_id: "mongodb-".to_string() + generate_id().as_str(), + fqdn: database_host_mongo.clone(), + port: database_port_mongo.clone(), + username: database_username_mongo.clone(), + password: database_password_mongo.clone(), + total_cpus: "100m".to_string(), + total_ram_in_mib: 512, + disk_size_in_gib: 10, + database_instance_type: "db.t3.medium".to_string(), + database_disk_type: "gp2".to_string(), + }, + ], + external_services: vec![], + clone_from_environment_id: None, + } +} + +pub fn working_minimal_environment(context: &Context) -> Environment { + let suffix = generate_id(); + Environment { + execution_id: context.execution_id().to_string(), + id: generate_id(), + kind: Kind::Development, + owner_id: generate_id(), + project_id: generate_id(), + organization_id: ORGANIZATION_ID.to_string(), + action: Action::Create, + applications: vec![Application { + id: generate_id(), + name: format!("{}-{}", "simple-app".to_string(), &suffix), + git_url: "https://github.com/Qovery/engine-testing.git".to_string(), + commit_id: "fc575a2f3be0b9100492c8a463bf18134a8698a5".to_string(), + dockerfile_path: "Dockerfile".to_string(), + action: Action::Create, + git_credentials: GitCredentials { + login: "x-access-token".to_string(), + access_token: "CHANGE ME".to_string(), + expired_at: Utc::now(), + }, + storage: vec![], + environment_variables: vec![], + branch: "basic-app-deploy".to_string(), + private_port: Some(80), + total_cpus: "100m".to_string(), + total_ram_in_mib: 256, + total_instances: 2, + cpu_burst: "100m".to_string(), + }], + routers: vec![Router { + id: generate_id(), + name: "main".to_string(), + action: Action::Create, + default_domain: generate_id() + ".oom.sh", + public_port: 443, + custom_domains: vec![], + routes: vec![Route { + path: "/".to_string(), + application_name: format!("{}-{}", "simple-app".to_string(), &suffix), + }], + }], + databases: vec![], + external_services: vec![], + clone_from_environment_id: None, + } +} + +pub fn environnement_2_app_2_routers_1_psql(context: &Context) -> Environment { + let fqdn_id = "my-postgresql-".to_string() + generate_id().as_str(); + let fqdn = fqdn_id.clone() + ".oom.sh"; + + let database_port = 5432; + let database_username = "superuser".to_string(); + let database_password = generate_id(); + let database_name = "my-psql".to_string(); + + let suffix = generate_id(); + + Environment { + execution_id: context.execution_id().to_string(), + id: generate_id(), + kind: Kind::Development, + owner_id: generate_id(), + project_id: generate_id(), + organization_id: ORGANIZATION_ID.to_string(), + action: Action::Create, + databases: vec![Database { + kind: DatabaseKind::Postgresql, + + action: Action::Create, + id: generate_id(), + name: database_name.clone(), + version: "11.8.0".to_string(), + fqdn_id: fqdn_id.clone(), + fqdn: fqdn.clone(), + port: database_port.clone(), + username: database_username.clone(), + password: database_password.clone(), + total_cpus: "100m".to_string(), + total_ram_in_mib: 512, + disk_size_in_gib: 10, + database_instance_type: "db.t2.micro".to_string(), + database_disk_type: "gp2".to_string(), + }], + applications: vec![ + Application { + id: generate_id(), + name: format!("{}-{}", "simple-app".to_string(), &suffix), + git_url: "https://github.com/Qovery/engine-testing.git".to_string(), + commit_id: "680550d1937b3f90551849c0da8f77c39916913b".to_string(), + dockerfile_path: "Dockerfile".to_string(), + action: Action::Create, + git_credentials: GitCredentials { + login: "x-access-token".to_string(), + access_token: "CHANGE ME".to_string(), + expired_at: Utc::now(), + }, + storage: vec![Storage { + id: generate_id(), + name: "photos".to_string(), + storage_type: StorageType::Ssd, + size_in_gib: 10, + mount_point: "/mnt/photos".to_string(), + snapshot_retention_in_days: 0, + }], + environment_variables: vec![ + EnvironmentVariable { + key: "PG_DBNAME".to_string(), + value: database_name.clone(), + }, + EnvironmentVariable { + key: "PG_HOST".to_string(), + value: fqdn.clone(), + }, + EnvironmentVariable { + key: "PG_PORT".to_string(), + value: database_port.clone().to_string(), + }, + EnvironmentVariable { + key: "PG_USERNAME".to_string(), + value: database_username.clone(), + }, + EnvironmentVariable { + key: "PG_PASSWORD".to_string(), + value: database_password.clone(), + }, + ], + branch: "master".to_string(), + private_port: Some(1234), + total_cpus: "100m".to_string(), + total_ram_in_mib: 256, + total_instances: 2, + cpu_burst: "100m".to_string(), + }, + Application { + id: generate_id(), + name: format!("{}-{}", "simple-app-2".to_string(), &suffix), + git_url: "https://github.com/Qovery/engine-testing.git".to_string(), + commit_id: "680550d1937b3f90551849c0da8f77c39916913b".to_string(), + dockerfile_path: "Dockerfile".to_string(), + action: Action::Create, + git_credentials: GitCredentials { + login: "x-access-token".to_string(), + access_token: "CHANGE ME".to_string(), + expired_at: Utc::now(), + }, + storage: vec![Storage { + id: generate_id(), + name: "photos".to_string(), + storage_type: StorageType::Ssd, + size_in_gib: 10, + mount_point: "/mnt/photos".to_string(), + snapshot_retention_in_days: 0, + }], + environment_variables: vec![ + EnvironmentVariable { + key: "PG_DBNAME".to_string(), + value: database_name.clone(), + }, + EnvironmentVariable { + key: "PG_HOST".to_string(), + value: fqdn.clone(), + }, + EnvironmentVariable { + key: "PG_PORT".to_string(), + value: database_port.clone().to_string(), + }, + EnvironmentVariable { + key: "PG_USERNAME".to_string(), + value: database_username.clone(), + }, + EnvironmentVariable { + key: "PG_PASSWORD".to_string(), + value: database_password.clone(), + }, + ], + branch: "master".to_string(), + private_port: Some(1234), + total_cpus: "100m".to_string(), + total_ram_in_mib: 256, + total_instances: 2, + cpu_burst: "100m".to_string(), + }, + ], + routers: vec![ + Router { + id: generate_id(), + name: "main".to_string(), + action: Action::Create, + default_domain: generate_id() + ".oom.sh", + public_port: 443, + custom_domains: vec![], + routes: vec![Route { + path: "/".to_string(), + application_name: format!("{}-{}", "simple-app".to_string(), &suffix), + }], + }, + Router { + id: generate_id(), + name: "second-router".to_string(), + action: Action::Create, + default_domain: generate_id() + ".oom.sh", + public_port: 443, + custom_domains: vec![], + routes: vec![Route { + path: "/coco".to_string(), + application_name: format!("{}-{}", "simple-app-2".to_string(), &suffix), + }], + }, + ], + + external_services: vec![], + clone_from_environment_id: None, + } +} + +pub fn non_working_environment(context: &Context) -> Environment { + let mut environment = working_minimal_environment(context); + + environment.applications = environment + .applications + .into_iter() + .map(|mut app| { + app.git_url = "https://github.com/Qovery/engine-testing.git".to_string(); + app.branch = "bugged-image".to_string(); + app.commit_id = "c2b2d7b5d96832732df25fe992721f53842b5eac".to_string(); + app + }) + .collect::>(); + + environment +} diff --git a/test_utilities/src/cloudflare.rs b/test_utilities/src/cloudflare.rs new file mode 100644 index 00000000..7837456e --- /dev/null +++ b/test_utilities/src/cloudflare.rs @@ -0,0 +1,16 @@ +use qovery_engine::dns_provider::cloudflare::Cloudflare; +use qovery_engine::models::Context; + +pub const CLOUDFLARE_ID: &str = "CHANGE ME"; +pub const CLOUDFLARE_TOKEN: &str = "CHANGE ME"; + +pub fn dns_provider_cloudflare(context: &Context) -> Cloudflare { + Cloudflare::new( + context.clone(), + "qoverytestdnsclo".to_string(), + "Qovery Test Cloudflare".to_string(), + "oom.sh".to_string(), + CLOUDFLARE_TOKEN.to_string(), // Cloudflare name: Qovery test + CLOUDFLARE_ID.to_string(), + ) +} diff --git a/test_utilities/src/digitalocean.rs b/test_utilities/src/digitalocean.rs new file mode 100644 index 00000000..719cc584 --- /dev/null +++ b/test_utilities/src/digitalocean.rs @@ -0,0 +1,42 @@ +use digitalocean::DigitalOcean; + +use qovery_engine::cloud_provider::digitalocean::DO; +use qovery_engine::container_registry::docr; +use qovery_engine::container_registry::docr::DOCR; +use qovery_engine::dns_provider::cloudflare::Cloudflare; +use qovery_engine::engine::Engine; +use qovery_engine::models::Context; + +use crate::cloudflare::dns_provider_cloudflare; +use crate::utilities::build_platform_local_docker; + +//TODO: should be environment var +pub const DIGITAL_OCEAN_TOKEN: &str = "CHANGE ME"; +pub const DIGITAL_OCEAN_URL: &str = "https://api.digitalocean.com/v2/"; + +pub fn container_registry_digital_ocean(context: &Context) -> DOCR { + DOCR::new(context.clone(), "qovery-registry", DIGITAL_OCEAN_TOKEN) +} + +pub fn docker_cr_do_engine(context: &Context) -> Engine { + // use DigitalOcean Container Registry + let container_registry = Box::new(container_registry_digital_ocean(context)); + // use LocalDocker + let build_platform = Box::new(build_platform_local_docker(context)); + // use Digital Ocean + let cloud_provider = Box::new(cloud_provider_digitalocean(context)); + + let dns_provider = Box::new(dns_provider_cloudflare(context)); + + Engine::new( + context.clone(), + build_platform, + container_registry, + cloud_provider, + dns_provider, + ) +} + +pub fn cloud_provider_digitalocean(context: &Context) -> DO { + DO::new(context.clone(), "test", DIGITAL_OCEAN_TOKEN) +} diff --git a/test_utilities/src/lib.rs b/test_utilities/src/lib.rs new file mode 100644 index 00000000..b5b2ee6d --- /dev/null +++ b/test_utilities/src/lib.rs @@ -0,0 +1,4 @@ +pub mod aws; +pub mod cloudflare; +pub mod digitalocean; +pub mod utilities; diff --git a/test_utilities/src/utilities.rs b/test_utilities/src/utilities.rs new file mode 100644 index 00000000..49f50263 --- /dev/null +++ b/test_utilities/src/utilities.rs @@ -0,0 +1,61 @@ +use curl::easy::Easy; +use curl::Error; +use rand::distributions::Alphanumeric; +use rand::{thread_rng, Rng}; + +use qovery_engine::build_platform::local_docker::LocalDocker; +use qovery_engine::models::{Context, Environment}; + +pub fn build_platform_local_docker(context: &Context) -> LocalDocker { + LocalDocker::new(context.clone(), "oxqlm3r99vwcmvuj", "qovery-local-docker") +} + +pub fn init() { + env_logger::try_init(); + println!( + "running from current directory: {}", + std::env::current_dir().unwrap().to_str().unwrap() + ); +} + +pub fn generate_id() -> String { + // Should follow DNS naming convention https://tools.ietf.org/html/rfc1035 + let uuid; + + loop { + let rand_string: String = thread_rng().sample_iter(Alphanumeric).take(15).collect(); + if rand_string.chars().next().unwrap().is_alphabetic() { + uuid = rand_string.to_lowercase(); + break; + } + } + uuid +} + +pub fn check_all_connections(env: &Environment) -> Vec { + let mut checking: Vec = Vec::with_capacity(env.routers.len()); + + for router_to_test in &env.routers { + let path_to_test = format!( + "https://{}{}", + &router_to_test.default_domain, &router_to_test.routes[0].path + ); + + checking.push(curl_path(path_to_test.as_str())); + } + return checking; +} + +fn curl_path(path: &str) -> bool { + let mut easy = Easy::new(); + easy.url(path).unwrap(); + let res = easy.perform(); + match res { + Ok(out) => return true, + + Err(e) => { + println!("TEST Error : while trying to call {}", e); + return false; + } + } +} diff --git a/tests/assets/eks-options.json b/tests/assets/eks-options.json new file mode 100644 index 00000000..9bb8c949 --- /dev/null +++ b/tests/assets/eks-options.json @@ -0,0 +1,128 @@ +{ + "eks_zone_a_subnet_blocks": [ + "10.0.0.0/23", + "10.0.2.0/23", + "10.0.4.0/23", + "10.0.6.0/23", + "10.0.8.0/23", + "10.0.10.0/23", + "10.0.12.0/23", + "10.0.14.0/23", + "10.0.16.0/23", + "10.0.18.0/23", + "10.0.20.0/23", + "10.0.22.0/23", + "10.0.24.0/23", + "10.0.26.0/23", + "10.0.28.0/23", + "10.0.30.0/23", + "10.0.32.0/23", + "10.0.34.0/23", + "10.0.36.0/23", + "10.0.38.0/23", + "10.0.40.0/23" + ], + "eks_zone_b_subnet_blocks": [ + "10.0.42.0/23", + "10.0.44.0/23", + "10.0.46.0/23", + "10.0.48.0/23", + "10.0.50.0/23", + "10.0.52.0/23", + "10.0.54.0/23", + "10.0.56.0/23", + "10.0.58.0/23", + "10.0.60.0/23", + "10.0.62.0/23", + "10.0.64.0/23", + "10.0.66.0/23", + "10.0.68.0/23", + "10.0.70.0/23", + "10.0.72.0/23", + "10.0.74.0/23", + "10.0.78.0/23", + "10.0.80.0/23", + "10.0.82.0/23", + "10.0.84.0/23" + ], + "eks_zone_c_subnet_blocks": [ + "10.0.86.0/23", + "10.0.88.0/23", + "10.0.90.0/23", + "10.0.92.0/23", + "10.0.94.0/23", + "10.0.96.0/23", + "10.0.98.0/23", + "10.0.100.0/23", + "10.0.102.0/23", + "10.0.104.0/23", + "10.0.106.0/23", + "10.0.108.0/23", + "10.0.110.0/23", + "10.0.112.0/23", + "10.0.114.0/23", + "10.0.116.0/23", + "10.0.118.0/23", + "10.0.120.0/23", + "10.0.122.0/23", + "10.0.124.0/23", + "10.0.126.0/23" + ], + "rds_zone_a_subnet_blocks": [ + "10.0.214.0/23", + "10.0.216.0/23", + "10.0.218.0/23", + "10.0.220.0/23", + "10.0.222.0/23", + "10.0.224.0/23" + ], + "rds_zone_b_subnet_blocks": [ + "10.0.226.0/23", + "10.0.228.0/23", + "10.0.230.0/23", + "10.0.232.0/23", + "10.0.234.0/23", + "10.0.236.0/23" + ], + "rds_zone_c_subnet_blocks": [ + "10.0.238.0/23", + "10.0.240.0/23", + "10.0.242.0/23", + "10.0.244.0/23", + "10.0.246.0/23", + "10.0.248.0/23" + ], + "documentdb_zone_a_subnet_blocks": [ + "10.0.196.0/23", + "10.0.198.0/23", + "10.0.200.0/23" + ], + "documentdb_zone_b_subnet_blocks": [ + "10.0.202.0/23", + "10.0.204.0/23", + "10.0.206.0/23" + ], + "documentdb_zone_c_subnet_blocks": [ + "10.0.208.0/23", + "10.0.210.0/23", + "10.0.212.0/23" + ], + "elasticsearch_zone_a_subnet_blocks": [ + "10.0.184.0/23", + "10.0.186.0/23" + ], + "elasticsearch_zone_b_subnet_blocks": [ + "10.0.188.0/23", + "10.0.190.0/23" + ], + "elasticsearch_zone_c_subnet_blocks": [ + "10.0.192.0/23", + "10.0.194.0/23" + ], + "vpc_cidr_block": "10.0.0.0/16", + "eks_cidr_subnet": "23", + "qovery_api_url": "api.qovery.com", + "rds_cidr_subnet": "23", + "documentdb_cidr_subnet": "23", + "elasticsearch_cidr_subnet": "23" +} diff --git a/tests/aws/aws_environment.rs b/tests/aws/aws_environment.rs new file mode 100644 index 00000000..cedbcdd4 --- /dev/null +++ b/tests/aws/aws_environment.rs @@ -0,0 +1,1330 @@ +extern crate test_utilities; + +use chrono::Utc; +use rusoto_core::region::Region::Custom; + +use qovery_engine::cloud_provider::service::Router; +use qovery_engine::cmd; +use qovery_engine::models::Kind::Production; +use qovery_engine::models::{ + Action, Clone2, Context, CustomDomain, Database, DatabaseKind, Environment, EnvironmentAction, + EnvironmentVariable, ExternalService, GitCredentials, Kind, Storage, StorageType, +}; +use qovery_engine::transaction::{DeploymentOption, TransactionResult}; +use test_utilities::aws::context; +use test_utilities::utilities::init; + +use self::test_utilities::cloudflare::dns_provider_cloudflare; +use self::test_utilities::utilities::generate_id; + +// insert how many actions you will use in tests +// args are function you want to use and how many context you want to have +// it permit you to create several different workspaces for each steps +// TODO implement it well +fn generate_contexts_and_environments( + number: u8, + func: fn(&Context) -> Environment, +) -> (Vec, Vec) { + let mut context_vec: Vec = Vec::new(); + let mut env_vec: Vec = Vec::new(); + let context = context(); + for i in std::iter::repeat(number) { + context_vec.push(context.clone_not_same_execution_id()); + let mut environment = func(&context); + env_vec.push(environment); + } + (context_vec, env_vec) +} + +fn deploy_environment( + context: &Context, + environment_action: &EnvironmentAction, +) -> TransactionResult { + let engine = test_utilities::aws::docker_ecr_aws_engine(&context); + let session = engine.session().unwrap(); + let mut tx = session.transaction(); + + let cp = test_utilities::aws::cloud_provider_aws(&context); + let nodes = test_utilities::aws::aws_kubernetes_nodes(); + let dns_provider = dns_provider_cloudflare(context); + let k = test_utilities::aws::aws_kubernetes_eks(&context, &cp, &dns_provider, nodes); + + tx.deploy_environment_with_options( + &k, + &environment_action, + DeploymentOption { + force_build: true, + force_push: true, + }, + ); + + tx.commit() +} + +fn pause_environment( + context: &Context, + environment_action: &EnvironmentAction, +) -> TransactionResult { + let engine = test_utilities::aws::docker_ecr_aws_engine(&context); + let session = engine.session().unwrap(); + let mut tx = session.transaction(); + + let cp = test_utilities::aws::cloud_provider_aws(&context); + let nodes = test_utilities::aws::aws_kubernetes_nodes(); + let dns_provider = dns_provider_cloudflare(context); + let k = test_utilities::aws::aws_kubernetes_eks(&context, &cp, &dns_provider, nodes); + + tx.pause_environment(&k, &environment_action); + + tx.commit() +} + +fn delete_environment( + context: &Context, + environment_action: &EnvironmentAction, +) -> TransactionResult { + let engine = test_utilities::aws::docker_ecr_aws_engine(&context); + let session = engine.session().unwrap(); + let mut tx = session.transaction(); + + let cp = test_utilities::aws::cloud_provider_aws(&context); + let nodes = test_utilities::aws::aws_kubernetes_nodes(); + let dns_provider = dns_provider_cloudflare(context); + let k = test_utilities::aws::aws_kubernetes_eks(&context, &cp, &dns_provider, nodes); + + tx.delete_environment(&k, &environment_action); + + tx.commit() +} + +#[test] +fn deploy_a_working_environment_with_no_router_on_aws_eks() { + init(); + + let context = context(); + let context_for_delete = context.clone_not_same_execution_id(); + let mut environment = test_utilities::aws::working_minimal_environment(&context); + let mut environment_for_delete = test_utilities::aws::working_minimal_environment(&context); + environment.routers = vec![]; + environment_for_delete.routers = vec![]; + environment_for_delete.action = Action::Delete; + let ea = EnvironmentAction::Environment(environment); + let ea_delete = EnvironmentAction::Environment(environment_for_delete); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + + match delete_environment(&context_for_delete, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} + +#[test] +fn deploy_dockerfile_not_exist() { + init(); + let context = context(); + let context2 = context.clone_not_same_execution_id(); + + let mut environment = test_utilities::aws::working_minimal_environment(&context); + // working env + let mut not_working_env = test_utilities::aws::working_minimal_environment(&context2); + + not_working_env.applications = environment + .applications + .into_iter() + .map(|mut app| { + app.git_url = "https://github.com/Qovery/engine-testing.git".to_string(); + app.branch = "dockerfile-not-exist".to_string(); + app.commit_id = "5cd900a07a17c7aa3c14cb5cb82c62e19219d57c".to_string(); + app.environment_variables = vec![]; + app.dockerfile_path = "".to_string(); + app + }) + .collect::>(); + + let ea = EnvironmentAction::Environment(not_working_env); + + match deploy_environment(&context2, &ea) { + TransactionResult::Ok => assert!(false), + TransactionResult::Rollback(_) => assert!(true), + TransactionResult::UnrecoverableError(_, _) => assert!(true), + }; +} + +#[test] +fn deploy_a_not_working_environment_with_no_router_on_aws_eks() { + init(); + + let context = context(); + let context_for_deletion = context.clone_not_same_execution_id(); + let mut environment = test_utilities::aws::non_working_environment(&context); + + environment.routers = vec![]; + + let mut environment_delete = + test_utilities::aws::non_working_environment(&context_for_deletion); + environment_delete.routers = vec![]; + environment_delete.action = Action::Delete; + let ea = EnvironmentAction::Environment(environment); + let ea_delete = EnvironmentAction::Environment(environment_delete); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(false), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(true), + }; + + match delete_environment(&context_for_deletion, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(true), + }; + + //Todo: remove the namespace (or project) +} + +// to check overload between several databases and apps +#[test] +#[ignore] +fn deploy_an_environment_with_3_databases_and_3_apps() { + init(); + let context = context(); + let context_for_deletion = context.clone_not_same_execution_id(); + let mut environment = test_utilities::aws::environment_3_apps_3_routers_3_databases(&context); + + let mut environment_delete = environment.clone(); + environment_delete.action = Action::Delete; + let ea = EnvironmentAction::Environment(environment); + let ea_delete = EnvironmentAction::Environment(environment_delete); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + + // TODO: should be uncommented as soon as cert-manager is fixed + // for the moment this assert report a SSL issue on the second router, so it's works well + /* let connections = test_utilities::utilities::check_all_connections(&env_to_check); + for con in connections { + assert_eq!(con, true); + }*/ + + match delete_environment(&context_for_deletion, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} + +#[test] +#[ignore] +fn deploy_a_working_environment_with_domain() { + init(); + + let context = context(); + let context_for_deletion = context.clone_not_same_execution_id(); + + let mut environment = test_utilities::aws::working_minimal_environment(&context); + + let mut environment_delete = environment.clone(); + environment_delete.action = Action::Delete; + let ea = EnvironmentAction::Environment(environment); + let ea_delete = EnvironmentAction::Environment(environment_delete); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + + match delete_environment(&context_for_deletion, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} + +#[test] +#[ignore] +fn deploy_a_working_environment_with_custom_domain() { + init(); + + let context = context(); + let context_for_delete = context.clone_not_same_execution_id(); + + let mut environment = test_utilities::aws::working_minimal_environment(&context); + // Todo: fix domains + environment.routers = environment + .routers + .into_iter() + .map(|mut router| { + router.custom_domains = vec![CustomDomain { + // should be the client domain + domain: "test-domain.qvy.io".to_string(), + // should be our domain + target_domain: "target-domain.oom.sh".to_string(), + }]; + router + }) + .collect::>(); + + let mut environment_delete = + test_utilities::aws::working_minimal_environment(&context_for_delete); + environment_delete.routers = environment_delete + .routers + .into_iter() + .map(|mut router| { + router.custom_domains = vec![CustomDomain { + // should be the client domain + domain: "test-domain.qvy.io".to_string(), + // should be our domain + target_domain: "target-domain.oom.sh".to_string(), + }]; + router + }) + .collect::>(); + environment_delete.action = Action::Delete; + let ea = EnvironmentAction::Environment(environment); + let ea_delete = EnvironmentAction::Environment(environment_delete); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + match delete_environment(&context_for_delete, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} + +#[test] +#[ignore] +fn deploy_a_working_environment_with_storage_on_aws_eks() { + init(); + + let context = context(); + let context_for_deletion = context.clone_not_same_execution_id(); + + let mut environment = test_utilities::aws::working_minimal_environment(&context); + + // Todo: make an image that check there is a mounted disk + environment.applications = environment + .applications + .into_iter() + .map(|mut app| { + app.storage = vec![Storage { + id: generate_id(), + name: "photos".to_string(), + storage_type: StorageType::Ssd, + size_in_gib: 10, + mount_point: "/mnt/photos".to_string(), + snapshot_retention_in_days: 0, + }]; + app + }) + .collect::>(); + + let mut environment_delete = + test_utilities::aws::working_minimal_environment(&context_for_deletion); + environment_delete.action = Action::Delete; + environment_delete.applications = environment_delete + .applications + .into_iter() + .map(|mut app| { + app.storage = vec![Storage { + id: generate_id(), + name: "photos".to_string(), + storage_type: StorageType::Ssd, + size_in_gib: 10, + mount_point: "/mnt/photos".to_string(), + snapshot_retention_in_days: 0, + }]; + app + }) + .collect::>(); + let ea = EnvironmentAction::Environment(environment); + let ea_delete = EnvironmentAction::Environment(environment_delete); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + + // todo: check the disk is here and with correct size + + match delete_environment(&context_for_deletion, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + + //Todo: remove the namespace (or project) +} + +#[test] +fn deploy_a_working_environment_with_postgresql() { + init(); + + let context = context(); + let context_for_delete = context.clone_not_same_execution_id(); + + let mut environment = test_utilities::aws::working_minimal_environment(&context); + + let database_host = "postgresql-".to_string() + generate_id().as_str() + ".oom.sh"; // External access check + let database_port = 5432; + let database_db_name = "my-postgres".to_string(); + let database_username = "superuser".to_string(); + let database_password = generate_id(); + environment.databases = vec![Database { + kind: DatabaseKind::Postgresql, + action: Action::Create, + id: generate_id(), + name: database_db_name.clone(), + version: "11.8.0".to_string(), + fqdn_id: "postgresql-".to_string() + generate_id().as_str(), + fqdn: database_host.clone(), + port: database_port.clone(), + username: database_username.clone(), + password: database_password.clone(), + total_cpus: "500m".to_string(), + total_ram_in_mib: 512, + disk_size_in_gib: 10, + database_instance_type: "db.t2.micro".to_string(), + database_disk_type: "gp2".to_string(), + }]; + environment.applications = environment + .applications + .into_iter() + .map(|mut app| { + app.branch = "postgres-app".to_string(); + app.commit_id = "5990752647af11ef21c3d46a51abbde3da1ab351".to_string(); + app.private_port = Some(1234); + app.environment_variables = vec![ + EnvironmentVariable { + key: "PG_HOST".to_string(), + value: database_host.clone(), + }, + EnvironmentVariable { + key: "PG_PORT".to_string(), + value: database_port.clone().to_string(), + }, + EnvironmentVariable { + key: "PG_DBNAME".to_string(), + value: database_db_name.clone(), + }, + EnvironmentVariable { + key: "PG_USERNAME".to_string(), + value: database_username.clone(), + }, + EnvironmentVariable { + key: "PG_PASSWORD".to_string(), + value: database_password.clone(), + }, + ]; + app + }) + .collect::>(); + environment.routers[0].routes[0].application_name = "postgres-app".to_string(); + + let mut environment_delete = environment.clone(); + environment_delete.action = Action::Delete; + let ea = EnvironmentAction::Environment(environment); + let ea_delete = EnvironmentAction::Environment(environment_delete); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + + // todo: check the database disk is here and with correct size + + match delete_environment(&context_for_delete, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(true), + }; +} + +#[test] +#[ignore] +fn deploy_a_working_production_environment_with_postgresql() { + init(); + + let context = context(); + let context_for_delete = context.clone_not_same_execution_id(); + + let mut environment = test_utilities::aws::working_minimal_environment(&context); + environment.kind = Kind::Production; + + let database_host = "postgresql-".to_string() + generate_id().as_str() + ".oom.sh"; // External access check + let database_port = 5432; + let database_db_name = "postgres".to_string(); + let database_username = "superuser".to_string(); + let database_password = generate_id(); + environment.databases = vec![Database { + kind: DatabaseKind::Postgresql, + action: Action::Create, + id: generate_id(), + name: database_db_name.clone(), + version: "12.4".to_string(), + fqdn_id: "postgresql-".to_string() + generate_id().as_str(), + fqdn: database_host.clone(), + port: database_port.clone(), + username: database_username.clone(), + password: database_password.clone(), + total_cpus: "100m".to_string(), + total_ram_in_mib: 512, + disk_size_in_gib: 10, + database_instance_type: "db.t2.micro".to_string(), + database_disk_type: "gp2".to_string(), + }]; + environment.applications = environment + .applications + .into_iter() + .map(|mut app| { + app.branch = "postgres-app".to_string(); + app.commit_id = "5990752647af11ef21c3d46a51abbde3da1ab351".to_string(); + app.private_port = Some(1234); + app.environment_variables = vec![ + EnvironmentVariable { + key: "PG_HOST".to_string(), + value: database_host.clone(), + }, + EnvironmentVariable { + key: "PG_PORT".to_string(), + value: database_port.clone().to_string(), + }, + EnvironmentVariable { + key: "PG_DBNAME".to_string(), + value: database_db_name.clone(), + }, + EnvironmentVariable { + key: "PG_USERNAME".to_string(), + value: database_username.clone(), + }, + EnvironmentVariable { + key: "PG_PASSWORD".to_string(), + value: database_password.clone(), + }, + ]; + app + }) + .collect::>(); + environment.routers[0].routes[0].application_name = "postgres-app".to_string(); + + let mut environment_delete = environment.clone(); + environment_delete.action = Action::Delete; + let ea = EnvironmentAction::Environment(environment); + let ea_delete = EnvironmentAction::Environment(environment_delete); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + + // todo: check the database disk is here and with correct size + + match delete_environment(&context_for_delete, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(true), + }; +} + +fn test_mongodb_configuration(context: Context, mut environment: Environment, version: &str) { + init(); + + let context_for_delete = context.clone_not_same_execution_id(); + + let database_host = "mongodb-".to_string() + generate_id().as_str() + ".oom.sh"; // External access check + let database_port = 27017; + let database_db_name = "my-mongodb".to_string(); + let database_username = "superuser".to_string(); + let database_password = generate_id(); + let database_uri = format!( + "mongodb://{}:{}@{}:{}/{}", + database_username, database_password, database_host, database_port, database_db_name + ); + // while waiting the info to be given directly in the database info, we're using this + let is_documentdb = match environment.kind { + Kind::Production => true, + Kind::Development => false, + }; + + environment.databases = vec![Database { + kind: DatabaseKind::Mongodb, + action: Action::Create, + id: generate_id(), + name: database_db_name.clone(), + version: version.to_string(), + fqdn_id: "mongodb-".to_string() + generate_id().as_str(), + fqdn: database_host.clone(), + port: database_port.clone(), + username: database_username.clone(), + password: database_password.clone(), + total_cpus: "500m".to_string(), + total_ram_in_mib: 512, + disk_size_in_gib: 10, + database_instance_type: "db.t3.medium".to_string(), + database_disk_type: "gp2".to_string(), + }]; + environment.applications = environment + .applications + .into_iter() + .map(|mut app| { + app.branch = "mongodb-app".to_string(); + app.commit_id = "158ea8ebc9897c50a7c56b910db33ce837ac1e61".to_string(); + app.private_port = Some(1234); + app.dockerfile_path = format!("Dockerfile-{}", version); + app.environment_variables = vec![ + // EnvironmentVariable { + // key: "ENABLE_DEBUG".to_string(), + // value: "true".to_string(), + // }, + // EnvironmentVariable { + // key: "DEBUG_PAUSE".to_string(), + // value: "true".to_string(), + // }, + EnvironmentVariable { + key: "IS_DOCUMENTDB".to_string(), + value: is_documentdb.to_string(), + }, + EnvironmentVariable { + key: "QOVERY_DATABASE_TESTING_DATABASE_FQDN".to_string(), + value: database_host.clone(), + }, + EnvironmentVariable { + key: "QOVERY_DATABASE_MY_DDB_CONNECTION_URI".to_string(), + value: database_uri.clone(), + }, + EnvironmentVariable { + key: "QOVERY_DATABASE_TESTING_DATABASE_PORT".to_string(), + value: database_port.clone().to_string(), + }, + EnvironmentVariable { + key: "MONGODB_DBNAME".to_string(), + value: database_db_name.clone(), + }, + EnvironmentVariable { + key: "QOVERY_DATABASE_TESTING_DATABASE_USERNAME".to_string(), + value: database_username.clone(), + }, + EnvironmentVariable { + key: "QOVERY_DATABASE_TESTING_DATABASE_PASSWORD".to_string(), + value: database_password.clone(), + }, + ]; + app + }) + .collect::>(); + environment.routers[0].routes[0].application_name = "mongodb-app".to_string(); + + let mut environment_delete = environment.clone(); + environment_delete.action = Action::Delete; + let ea = EnvironmentAction::Environment(environment); + let ea_delete = EnvironmentAction::Environment(environment_delete); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + + // todo: check the database disk is here and with correct size + + match delete_environment(&context_for_delete, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(true), + }; +} + +/// test mongodb v3.6 with development environment +#[test] +fn deploy_a_working_environment_with_mongodb_v3_6() { + let context = context(); + let mut environment = test_utilities::aws::working_minimal_environment(&context); + test_mongodb_configuration(context, environment, "3.6"); +} + +#[test] +#[ignore] +fn deploy_a_working_environment_with_mongodb_v4_0() { + let context = context(); + let mut environment = test_utilities::aws::working_minimal_environment(&context); + test_mongodb_configuration(context, environment, "4.0"); +} + +/// test mongodb v4.2 with development environment +#[test] +#[ignore] +fn deploy_a_working_environment_with_mongodb_v4_2() { + let context = context(); + let mut environment = test_utilities::aws::working_minimal_environment(&context); + test_mongodb_configuration(context, environment, "4.2"); +} + +/// test mongodb v4.4 with development environment +#[test] +fn deploy_a_working_environment_with_mongodb_v4_4() { + let context = context(); + let mut environment = test_utilities::aws::working_minimal_environment(&context); + test_mongodb_configuration(context, environment, "4.4"); +} + +/// test mongodb v3.6 with production environment (DocumentDB) +#[test] +#[ignore] +fn deploy_a_working_environment_with_production_mongodb_v3_6() { + let context = context(); + + let mut environment = test_utilities::aws::working_minimal_environment(&context); + environment.kind = Kind::Production; + + test_mongodb_configuration(context, environment, "3.6"); +} + +// #[test] +// fn deploy_a_working_environment_with_external_service() { +// init(); +// +// let context = context(); +// let deletion_context = context.clone_not_same_execution_id(); +// +// let mut environment = test_utilities::aws::working_minimal_environment(&context); +// +// // no apps +// environment.applications = vec![]; +// +// environment.external_services = vec![ExternalService { +// id: generate_id(), +// action: Action::Create, +// name: "my-external-service".to_string(), +// total_cpus: "500m".to_string(), +// total_ram_in_mib: 512, +// git_url: "https://github.com/evoxmusic/qovery-external-service-example.git".to_string(), +// git_credentials: GitCredentials { +// login: "x-access-token".to_string(), +// access_token: "CHANGE ME".to_string(), // fake one +// expired_at: Utc::now(), +// }, +// branch: "master".to_string(), +// commit_id: "db322f2f4ac70933f16e8a422ea9f72e1e14df22".to_string(), +// on_create_dockerfile_path: "extsvc/Dockerfile.on-create".to_string(), +// on_pause_dockerfile_path: "extsvc/Dockerfile.on-pause".to_string(), +// on_delete_dockerfile_path: "extsvc/Dockerfile.on-delete".to_string(), +// environment_variables: vec![], +// }]; +// +// let mut environment_delete = environment.clone(); +// environment_delete.action = Action::Delete; +// +// let ea = EnvironmentAction::Environment(environment); +// let ea_delete = EnvironmentAction::Environment(environment_delete); +// +// match deploy_environment(&context, &ea) { +// TransactionResult::Ok => assert!(true), +// TransactionResult::Rollback(_) => assert!(false), +// TransactionResult::UnrecoverableError(_, _) => assert!(false), +// }; +// +// match delete_environment(&deletion_context, &ea_delete) { +// TransactionResult::Ok => assert!(true), +// TransactionResult::Rollback(_) => assert!(false), +// TransactionResult::UnrecoverableError(_, _) => assert!(false), +// }; +// +// // TODO: remove the namespace (or project) +// } + +#[test] +#[ignore] +fn deploy_a_working_environment_with_mysql() { + init(); + + let context = context(); + let deletion_context = context.clone_not_same_execution_id(); + + let mut environment = test_utilities::aws::working_minimal_environment(&context); + + let database_host = "mysql-".to_string() + generate_id().as_str() + ".oom.sh"; // External access check + let database_port = 3306; + let database_db_name = "mydb".to_string(); + let database_username = "superuser".to_string(); + let database_password = generate_id(); + environment.databases = vec![Database { + kind: DatabaseKind::Mysql, + action: Action::Create, + id: generate_id(), + name: database_db_name.clone(), + version: "5.7.30".to_string(), + fqdn_id: "mysql-".to_string() + generate_id().as_str(), + fqdn: database_host.clone(), + port: database_port.clone(), + username: database_username.clone(), + password: database_password.clone(), + total_cpus: "500m".to_string(), + total_ram_in_mib: 512, + disk_size_in_gib: 10, + database_instance_type: "db.t2.micro".to_string(), + database_disk_type: "gp2".to_string(), + }]; + environment.applications = environment + .applications + .into_iter() + .map(|mut app| { + app.branch = "mysql-app".to_string(); + app.commit_id = "222295112d58d78227c21060d3a707687302e86f".to_string(); + app.private_port = Some(1234); + app.environment_variables = vec![ + EnvironmentVariable { + key: "MYSQL_HOST".to_string(), + value: database_host.clone(), + }, + EnvironmentVariable { + key: "MYSQL_PORT".to_string(), + value: database_port.clone().to_string(), + }, + EnvironmentVariable { + key: "MYSQL_DBNAME".to_string(), + value: database_db_name.clone(), + }, + EnvironmentVariable { + key: "MYSQL_USERNAME".to_string(), + value: database_username.clone(), + }, + EnvironmentVariable { + key: "MYSQL_PASSWORD".to_string(), + value: database_password.clone(), + }, + ]; + app + }) + .collect::>(); + environment.routers[0].routes[0].application_name = "mysql-app".to_string(); + + let mut environment_delete = environment.clone(); + environment_delete.action = Action::Delete; + let ea = EnvironmentAction::Environment(environment); + let ea_delete = EnvironmentAction::Environment(environment_delete); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + + // todo: check the database disk is here and with correct size + + match delete_environment(&deletion_context, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + + //Todo: remove the namespace (or project) +} + +#[test] +#[ignore] +/// Tests the creation of a simple environment on AWS, with the DB provisioned on RDS. +fn deploy_a_working_production_environment_with_mysql() { + init(); + + let context = context(); + let deletion_context = context.clone_not_same_execution_id(); + + let mut environment = test_utilities::aws::working_minimal_environment(&context); + environment.kind = Production; + + let database_host = "mysql-app-".to_string() + generate_id().as_str() + "-svc.oom.sh"; // External access check + let database_port = 3306; + let database_db_name = "mysql".to_string(); + let database_username = "superuser".to_string(); + let database_password = generate_id(); + environment.databases = vec![Database { + kind: DatabaseKind::Mysql, + action: Action::Create, + id: generate_id(), + name: database_db_name.clone(), + version: "5.7.30".to_string(), + fqdn_id: "mysql-".to_string() + generate_id().as_str(), + fqdn: database_host.clone(), + port: database_port.clone(), + username: database_username.clone(), + password: database_password.clone(), + total_cpus: "500m".to_string(), + total_ram_in_mib: 512, + disk_size_in_gib: 10, + database_instance_type: "db.t2.micro".to_string(), + database_disk_type: "gp2".to_string(), + }]; + environment.applications = environment + .applications + .into_iter() + .map(|mut app| { + app.branch = "mysql-app".to_string(); + app.commit_id = "222295112d58d78227c21060d3a707687302e86f".to_string(); + app.private_port = Some(1234); + app.environment_variables = vec![ + EnvironmentVariable { + key: "MYSQL_HOST".to_string(), + value: database_host.clone(), + }, + EnvironmentVariable { + key: "MYSQL_PORT".to_string(), + value: database_port.clone().to_string(), + }, + EnvironmentVariable { + key: "MYSQL_DBNAME".to_string(), + value: database_db_name.clone(), + }, + EnvironmentVariable { + key: "MYSQL_USERNAME".to_string(), + value: database_username.clone(), + }, + EnvironmentVariable { + key: "MYSQL_PASSWORD".to_string(), + value: database_password.clone(), + }, + ]; + app + }) + .collect::>(); + environment.routers[0].routes[0].application_name = "mysql-app".to_string(); + + let mut environment_delete = environment.clone(); + environment_delete.action = Action::Delete; + let ea = EnvironmentAction::Environment(environment); + let ea_delete = EnvironmentAction::Environment(environment_delete); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + + // todo: check the database disk is here and with correct size + + match delete_environment(&deletion_context, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} + +#[test] +#[ignore] +fn deploy_a_working_development_environment_with_all_options_and_psql() { + init(); + + let context = context(); + let context_for_deletion = context.clone_not_same_execution_id(); + + let mut environment = test_utilities::aws::environnement_2_app_2_routers_1_psql(&context); + let mut env_to_check = environment.clone(); + let mut environment_delete = + test_utilities::aws::environnement_2_app_2_routers_1_psql(&context_for_deletion); + + environment.kind = Kind::Development; + environment_delete.kind = Kind::Development; + environment_delete.action = Action::Delete; + + let ea = EnvironmentAction::Environment(environment); + let ea_for_deletion = EnvironmentAction::Environment(environment_delete); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + // TODO: should be uncommented as soon as cert-manager is fixed + // for the moment this assert report a SSL issue on the second router, so it's works well + /* let connections = test_utilities::utilities::check_all_connections(&env_to_check); + for con in connections { + assert_eq!(con, true); + }*/ + + match delete_environment(&context_for_deletion, &ea_for_deletion) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} + +/*#[test] +#[ignore] +fn deploy_a_working_production_environment_with_all_options_on_aws_eks() { + init(); + + let context = context(); + + let mut environment = test_utilities::aws::working_environment(&context); + environment.kind = Kind::Production; + let environment_delete = environment.clone_not_same_execution_id(); + let ea = EnvironmentAction::Environment(environment); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + let ea_delete = EnvironmentAction::Environment(environment_delete); + match delete_environment(&context, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +}*/ + +#[test] +fn deploy_a_not_working_environment_and_after_working_environment() { + init(); + + // let mut contex_envs = generate_contexts_and_environments(3, test_utilities::aws::working_minimal_environment); + let context = context(); + let context_for_not_working = context.clone_not_same_execution_id(); + let context_for_delete = context.clone_not_same_execution_id(); + // env part generation + let mut environment = test_utilities::aws::working_minimal_environment(&context); + let mut environment_for_not_working = + test_utilities::aws::working_minimal_environment(&context_for_not_working); + // this environment is broken by container exit + environment_for_not_working.applications = environment_for_not_working + .applications + .into_iter() + .map(|mut app| { + app.git_url = "https://gitlab.com/maathor/my-exit-container".to_string(); + app.branch = "master".to_string(); + app.commit_id = "55bc95a23fbf91a7699c28c5f61722d4f48201c9".to_string(); + app.environment_variables = vec![]; + app + }) + .collect::>(); + + let mut environment_for_delete = + test_utilities::aws::working_minimal_environment(&context_for_delete); + environment_for_delete.action = Action::Delete; + // environment actions + let ea = EnvironmentAction::Environment(environment); + let ea_not_working = EnvironmentAction::Environment(environment_for_not_working); + let ea_delete = EnvironmentAction::Environment(environment_for_delete); + + match deploy_environment(&context_for_not_working, &ea_not_working) { + TransactionResult::Ok => assert!(false), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(true), + }; + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + match delete_environment(&context_for_delete, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} + +#[test] +fn deploy_ok_fail_fail_ok_environment() { + init(); + // working env + let context = context(); + let mut environment = test_utilities::aws::working_minimal_environment(&context); + let ea = EnvironmentAction::Environment(environment); + // not working 1 + let context_for_not_working = context.clone_not_same_execution_id(); + let mut not_working_env = + test_utilities::aws::working_minimal_environment(&context_for_not_working); + // not working 2 + let context_for_not_working2 = context.clone_not_same_execution_id(); + let mut not_working_env2 = + test_utilities::aws::working_minimal_environment(&context_for_not_working2); + // final env is working + let context_for_working2 = context.clone_not_same_execution_id(); + let mut working_env_2 = test_utilities::aws::working_minimal_environment(&context_for_working2); + let ea2 = EnvironmentAction::Environment(working_env_2); + // work for delete + let context_for_delete = context.clone_not_same_execution_id(); + let mut delete_env = test_utilities::aws::working_minimal_environment(&context_for_delete); + delete_env.action = Action::Delete; + let ea_delete = EnvironmentAction::Environment(delete_env); + // override application to make envs to be not working + not_working_env.applications = not_working_env + .applications + .into_iter() + .map(|mut app| { + app.git_url = "https://gitlab.com/maathor/my-exit-container".to_string(); + app.branch = "master".to_string(); + app.commit_id = "55bc95a23fbf91a7699c28c5f61722d4f48201c9".to_string(); + app.environment_variables = vec![]; + app + }) + .collect::>(); + not_working_env2.applications = not_working_env2 + .applications + .into_iter() + .map(|mut app| { + app.git_url = "https://gitlab.com/maathor/my-exit-container".to_string(); + app.branch = "master".to_string(); + app.commit_id = "55bc95a23fbf91a7699c28c5f61722d4f48201c9".to_string(); + app.environment_variables = vec![]; + app + }) + .collect::>(); + + let ea_not_working = EnvironmentAction::Environment(not_working_env); + let ea_not_working2 = EnvironmentAction::Environment(not_working_env2); + + // OK + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + // FAIL and rollback + match deploy_environment(&context_for_not_working, &ea_not_working) { + TransactionResult::Ok => assert!(false), + TransactionResult::Rollback(_) => assert!(true), + TransactionResult::UnrecoverableError(_, _) => assert!(true), + }; + // FAIL and Rollback again + match deploy_environment(&context_for_not_working2, &ea_not_working2) { + TransactionResult::Ok => assert!(false), + TransactionResult::Rollback(_) => assert!(true), + TransactionResult::UnrecoverableError(_, _) => assert!(true), + }; + // Should be working + match deploy_environment(&context_for_working2, &ea2) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + match delete_environment(&context_for_delete, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} + +#[test] +fn deploy_a_non_working_environment_with_no_failover_on_aws_eks() { + init(); + + let context = context(); + + let mut environment = test_utilities::aws::non_working_environment(&context); + + let ea = EnvironmentAction::Environment(environment); + + let context_for_delete = context.clone_not_same_execution_id(); + let mut delete_env = test_utilities::aws::non_working_environment(&context_for_delete); + delete_env.action = Action::Delete; + let ea_delete = EnvironmentAction::Environment(delete_env); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(false), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(true), + }; + match delete_environment(&context_for_delete, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} + +#[test] +#[ignore] +fn deploy_a_non_working_environment_with_a_working_failover_on_aws_eks() { + init(); + // context for non working environment + let context = context(); + + let mut environment = test_utilities::aws::non_working_environment(&context); + let mut failover_environment = test_utilities::aws::working_minimal_environment(&context); + // context for deletion + let context_deletion = context.clone_not_same_execution_id(); + let mut delete_env = test_utilities::aws::working_minimal_environment(&context_deletion); + delete_env.action = Action::Delete; + let ea_delete = EnvironmentAction::Environment(delete_env); + let ea = EnvironmentAction::EnvironmentWithFailover(environment, failover_environment); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(false), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(true), + }; + match delete_environment(&context_deletion, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} + +#[test] +#[ignore] +fn deploy_a_non_working_environment_with_a_non_working_failover_on_aws_eks() { + init(); + + let context = context(); + + let mut environment = test_utilities::aws::non_working_environment(&context); + let mut failover_environment = test_utilities::aws::non_working_environment(&context); + + let context_for_deletion = context.clone_not_same_execution_id(); + let mut delete_env = test_utilities::aws::non_working_environment(&context_for_deletion); + delete_env.action = Action::Delete; + // environment action initialize + let ea_delete = EnvironmentAction::Environment(delete_env); + let ea = EnvironmentAction::EnvironmentWithFailover(environment, failover_environment); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(false), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(true), + }; + match delete_environment(&context_for_deletion, &ea_delete) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(true), + }; +} + +/*#[test] +#[ignore] +fn deploy_a_working_environment_with_a_failing_default_domain_on_aws_eks() { + init(); + + // TODO +} + +#[test] +#[ignore] +fn deploy_but_fail_to_push_image_on_container_registry() { + init(); + + // TODO +}*/ +/* +fn pause_a_working_development_environment_on_aws_eks() { + init(); + + let context = test_utilities::aws::context(); + + let mut environment = test_utilities::aws::working_environment(&context); + environment.kind = Kind::Development; + + let ea = EnvironmentAction::Environment(environment); + + match pause_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} + +#[test] +#[ignore] +fn pause_a_working_production_environment_on_aws_eks() { + init(); + + let context = test_utilities::aws::context(); + + let mut environment = test_utilities::aws::working_environment(&context); + environment.kind = Kind::Production; + + let ea = EnvironmentAction::Environment(environment); + + match pause_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} + +#[test] +#[ignore] +fn pause_a_non_working_environment_on_aws_eks() { + init(); + + let context = test_utilities::aws::context(); + + let mut environment = test_utilities::aws::non_working_environment(&context); + + let ea = EnvironmentAction::Environment(environment); + + match pause_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} + +#[test] +#[ignore] +fn start_and_pause_and_start_and_delete_a_working_environment_on_aws_eks() { + init(); + + // START + let context = test_utilities::aws::context(); + + let mut environment = test_utilities::aws::working_environment(&context); + let ea = EnvironmentAction::Environment(environment); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + + // PAUSE + let context = test_utilities::aws::context(); + + let mut environment = test_utilities::aws::working_environment(&context); + let ea = EnvironmentAction::Environment(environment); + + match pause_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + + // START + let context = test_utilities::aws::context(); + + let mut environment = test_utilities::aws::working_environment(&context); + let ea = EnvironmentAction::Environment(environment); + + match deploy_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; + + // DELETE + let context = test_utilities::aws::context(); + + let mut environment = test_utilities::aws::working_environment(&context); + let ea = EnvironmentAction::Environment(environment); + + match delete_environment(&context, &ea) { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} +*/ diff --git a/tests/aws/aws_kube_secret_tfstate.rs b/tests/aws/aws_kube_secret_tfstate.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tests/aws/aws_kube_secret_tfstate.rs @@ -0,0 +1 @@ + diff --git a/tests/aws/aws_kubernetes.rs b/tests/aws/aws_kubernetes.rs new file mode 100644 index 00000000..a34c6ddb --- /dev/null +++ b/tests/aws/aws_kubernetes.rs @@ -0,0 +1,223 @@ +/*extern crate test_utilities; +use serde_json::value::Value; + +use self::test_utilities::cloudflare::dns_provider_cloudflare; +use self::test_utilities::utilities::init; +use qovery_engine::cloud_provider::aws::kubernetes::node::Node; +use qovery_engine::cloud_provider::aws::kubernetes::EKS; +use qovery_engine::cloud_provider::aws::AWS; +use qovery_engine::cloud_provider::kubernetes::{Kubernetes, KubernetesError}; +use qovery_engine::cloud_provider::CloudProvider; +use qovery_engine::dns_provider::cloudflare::Cloudflare; +use qovery_engine::models::Clone2; +use qovery_engine::transaction::TransactionResult; +use std::borrow::Borrow; +use std::fs::File; +use std::io::{BufReader, Read}; +use test_utilities::aws::AWS_KUBERNETES_VERSION; + +#[test] +#[ignore] +fn create_eks_cluster_in_us_east_2() { + init(); + + let context = test_utilities::aws::context(); + + let engine = test_utilities::aws::docker_ecr_aws_engine(&context); + let session = engine.session().unwrap(); + let mut tx = session.transaction(); + + let aws = test_utilities::aws::cloud_provider_aws(&context); + let nodes = test_utilities::aws::aws_kubernetes_nodes(); + + let cloudflare = dns_provider_cloudflare(&context); + + let mut file = File::open("tests/assets/eks-options.json").unwrap(); + let mut read_buf = String::new(); + file.read_to_string(&mut read_buf).unwrap(); + + let options_result = serde_json::from_str::< + qovery_engine::cloud_provider::aws::kubernetes::Options, + >(read_buf.as_str()); + + let kubernetes = EKS::new( + context, + "eks-on-us-east-2", + "eks-us-east-2", + AWS_KUBERNETES_VERSION, + "us-east-2", + &aws, + &cloudflare, + options_result.expect("Oh my god an error in test... Options options options"), + nodes, + ); + + match tx.create_kubernetes(&kubernetes) { + Err(err) => panic!("{:?}", err), + _ => {} + } + + let _ = match tx.commit() { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} + +pub fn read_file(filepath: &str) -> String { + let file = File::open(filepath).expect("could not open file"); + let mut buffered_reader = BufReader::new(file); + let mut contents = String::new(); + let _number_of_bytes: usize = match buffered_reader.read_to_string(&mut contents) { + Ok(number_of_bytes) => number_of_bytes, + Err(_err) => 0, + }; + + contents +} + +#[test] +#[ignore] +fn create_eks_cluster_in_eu_west_3() { + init(); + + let context = test_utilities::aws::context(); + + let engine = test_utilities::aws::docker_ecr_aws_engine(&context); + let session = engine.session().unwrap(); + let mut tx = session.transaction(); + + let aws = test_utilities::aws::cloud_provider_aws(&context); + let nodes = test_utilities::aws::aws_kubernetes_nodes(); + + let cloudflare = dns_provider_cloudflare(&context); + + let mut file = File::open("tests/assets/eks-options.json").unwrap(); + let mut read_buf = String::new(); + file.read_to_string(&mut read_buf).unwrap(); + + let options_result = serde_json::from_str::< + qovery_engine::cloud_provider::aws::kubernetes::Options, + >(read_buf.as_str()); + + let kubernetes = EKS::new( + context.clone(), + "eks-on-eu-west-3", + "eks-eu-west-3", + AWS_KUBERNETES_VERSION, + "eu-west-3", + &aws, + &cloudflare, + options_result.expect("Oh my god an error in test... Options options options"), + nodes, + ); + + match tx.create_kubernetes(&kubernetes) { + Err(err) => panic!("{:?}", err), + _ => {} + } + + let _ = match tx.commit() { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} + +#[test] +#[ignore] +fn delete_eks_cluster_in_us_east_2() { + init(); + + let context = test_utilities::aws::context(); + + let engine = test_utilities::aws::docker_ecr_aws_engine(&context); + let session = engine.session().unwrap(); + let mut tx = session.transaction(); + + let aws = test_utilities::aws::cloud_provider_aws(&context); + let nodes = test_utilities::aws::aws_kubernetes_nodes(); + + let cloudflare = dns_provider_cloudflare(&context); + + let mut file = File::open("tests/assets/eks-options.json").unwrap(); + let mut read_buf = String::new(); + file.read_to_string(&mut read_buf).unwrap(); + + let options_result = serde_json::from_str::< + qovery_engine::cloud_provider::aws::kubernetes::Options, + >(read_buf.as_str()); + + let kubernetes = EKS::new( + context, + "eks-on-us-east-2", + "eks-us-east-2", + AWS_KUBERNETES_VERSION, + "us-east-2", + &aws, + &cloudflare, + options_result.expect("Oh my god an error in test... Options options options"), + nodes, + ); + + match tx.delete_kubernetes(&kubernetes) { + Err(err) => panic!("{:?}", err), + _ => {} + } + + let _ = match tx.commit() { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} + +#[test] +#[ignore] +fn delete_eks_cluster_in_eu_west_3() { + init(); + // put some environments here, simulated or not + + let context = test_utilities::aws::context(); + + let engine = test_utilities::aws::docker_ecr_aws_engine(&context); + let session = engine.session().unwrap(); + let mut tx = session.transaction(); + + let aws = test_utilities::aws::cloud_provider_aws(&context); + let nodes = test_utilities::aws::aws_kubernetes_nodes(); + + let cloudflare = dns_provider_cloudflare(&context); + + let mut file = File::open("tests/assets/eks-options.json").unwrap(); + let mut read_buf = String::new(); + file.read_to_string(&mut read_buf).unwrap(); + + let options_result = serde_json::from_str::< + qovery_engine::cloud_provider::aws::kubernetes::Options, + >(read_buf.as_str()); + + let kubernetes = EKS::new( + context, + "eks-on-eu-west-3", + "eks-eu-west-3", + AWS_KUBERNETES_VERSION, + "eu-west-3", + &aws, + &cloudflare, + options_result.expect("Oh my god an error in test... Options options options"), + nodes, + ); + + match tx.delete_kubernetes(&kubernetes) { + Err(err) => panic!("{:?}", err), + _ => {} + } + + let _ = match tx.commit() { + TransactionResult::Ok => assert!(true), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(false), + }; +} +*/ diff --git a/tests/aws/deletion.rs b/tests/aws/deletion.rs new file mode 100644 index 00000000..beef0b5c --- /dev/null +++ b/tests/aws/deletion.rs @@ -0,0 +1,59 @@ +use std::fs::File; +use std::io::Read; + +use qovery_engine::cloud_provider::aws::kubernetes::EKS; +use qovery_engine::cmd::kubectl::create_sample_secret_terraform_in_namespace; +use qovery_engine::transaction::TransactionResult; +use test_utilities::aws::AWS_KUBERNETES_VERSION; +use test_utilities::cloudflare::dns_provider_cloudflare; +use test_utilities::utilities::init; + +pub fn do_not_delete_cluster_containing_tfstate() { + init(); + // put some environments here, simulated or not + + let context = test_utilities::aws::context(); + + let engine = test_utilities::aws::docker_ecr_aws_engine(&context); + let session = engine.session().unwrap(); + let mut tx = session.transaction(); + + let aws = test_utilities::aws::cloud_provider_aws(&context); + let nodes = test_utilities::aws::aws_kubernetes_nodes(); + + let cloudflare = dns_provider_cloudflare(&context); + + let mut file = File::open("qovery-engine/tests/assets/eks-options.json").unwrap(); + let mut read_buf = String::new(); + file.read_to_string(&mut read_buf).unwrap(); + + let options_result = serde_json::from_str::< + qovery_engine::cloud_provider::aws::kubernetes::Options, + >(read_buf.as_str()); + + let kubernetes = EKS::new( + context, + "eks-on-eu-west-3", + "eks-eu-west-3", + AWS_KUBERNETES_VERSION, + "eu-west-3", + &aws, + &cloudflare, + options_result.expect("Oh my god an error in test... Options options options"), + nodes, + ); + + /* + create_sample_secret_terraform_in_namespace(); + */ + match tx.delete_kubernetes(&kubernetes) { + Err(err) => panic!("{:?}", err), + _ => {} + } + + let _ = match tx.commit() { + TransactionResult::Ok => assert!(false), + TransactionResult::Rollback(_) => assert!(false), + TransactionResult::UnrecoverableError(_, _) => assert!(true), + }; +} diff --git a/tests/aws/mod.rs b/tests/aws/mod.rs new file mode 100644 index 00000000..ca587568 --- /dev/null +++ b/tests/aws/mod.rs @@ -0,0 +1,3 @@ +mod aws_environment; +mod aws_kubernetes; +mod deletion; diff --git a/tests/digital_ocean/container_registry.rs b/tests/digital_ocean/container_registry.rs new file mode 100644 index 00000000..4664bcb3 --- /dev/null +++ b/tests/digital_ocean/container_registry.rs @@ -0,0 +1,42 @@ +extern crate test_utilities; + +use qovery_engine::build_platform::Image; +use qovery_engine::container_registry::docr::DOCR; +use test_utilities::digitalocean::DIGITAL_OCEAN_TOKEN; + +use self::test_utilities::aws::context; +use self::test_utilities::digitalocean::docker_cr_do_engine; +use self::test_utilities::utilities::init; + +/*#[test] +#[ignore] +fn create_do_container_registry() { + init(); + let context = context(); + docker_cr_do_engine(&context); + let docr = DOCR { + context, + registry_name: "qoverytest".to_string(), + api_key: DIGITAL_OCEAN_TOKEN.to_string(), + }; + let image_test = Image { + application_id: "to change".to_string(), + name: "imageName".to_string(), + tag: "v666".to_string(), + commit_id: "sha256".to_string(), + registry_url: None, + }; + let repository = DOCR::create_repository(&docr, &image_test); +} +*/ +/*#[test] +fn create_do_repository_on_container_registry() {} + +#[test] +fn delete_do_repository_on_container_registry() {} + +#[test] +fn push_sample_image_on_container_registry() {}*/ + +// +// test --package qovery-engine --test container_registry create_do_container_registry -- --exact diff --git a/tests/digital_ocean/mod.rs b/tests/digital_ocean/mod.rs new file mode 100644 index 00000000..228aba23 --- /dev/null +++ b/tests/digital_ocean/mod.rs @@ -0,0 +1 @@ +mod container_registry; diff --git a/tests/lib.rs b/tests/lib.rs new file mode 100644 index 00000000..3d96e5da --- /dev/null +++ b/tests/lib.rs @@ -0,0 +1,3 @@ +mod aws; +mod digital_ocean; +mod unit; diff --git a/tests/unit/mod.rs b/tests/unit/mod.rs new file mode 100644 index 00000000..038fa38b --- /dev/null +++ b/tests/unit/mod.rs @@ -0,0 +1 @@ +mod s3; \ No newline at end of file diff --git a/tests/unit/s3.rs b/tests/unit/s3.rs new file mode 100644 index 00000000..60380c59 --- /dev/null +++ b/tests/unit/s3.rs @@ -0,0 +1,38 @@ +use std::str::FromStr; + +use rusoto_core::credential::StaticProvider; +use rusoto_core::{Client, Region}; +use rusoto_s3::{ + CreateBucketConfiguration, CreateBucketError, CreateBucketRequest, GetObjectError, + GetObjectRequest, ListObjectsV2Output, ListObjectsV2Request, PutBucketVersioningRequest, + PutObjectRequest, S3Client, VersioningConfiguration, S3, +}; + +use qovery_engine::s3; +use qovery_engine::s3::{delete_bucket, get_default_region_for_us}; +use test_utilities::aws::{AWS_SECRET_ACCESS_KEY, AWS_DEFAULT_REGION, AWS_ACCESS_KEY_ID, AWS_REGION_FOR_S3}; +use test_utilities::utilities::init; + +#[test] +fn delete_s3_bucket() { + init(); + let bucket_name = "my-test-bucket"; + let credentials = StaticProvider::new( + AWS_ACCESS_KEY_ID.to_string(), + AWS_SECRET_ACCESS_KEY.to_string(), + None, + None, + ); + + let creation = s3::create_bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, bucket_name); + match creation { + Ok(out) => println!("Yippee Ki Yay"), + Err(e) => println!("While creating the bucket {}", e), + } + + let delete = delete_bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, bucket_name); + match delete { + Ok(out) => println!("Yippee Ki Yay"), + Err(e) => println!("While deleting the bucket {}", e), + } +}