From e16c15f7ec4f86a6c422c472a9c92e3bea705bfd Mon Sep 17 00:00:00 2001 From: Romain GERARD Date: Mon, 15 Mar 2021 17:21:58 +0100 Subject: [PATCH] Fix get pod events --- src/cloud_provider/service.rs | 67 ++++---- src/cmd/kubectl.rs | 2 +- src/cmd/structs.rs | 303 +++++++++++++++++++++++++++++++++- tests/assets/eks-options.json | 1 - 4 files changed, 338 insertions(+), 35 deletions(-) diff --git a/src/cloud_provider/service.rs b/src/cloud_provider/service.rs index 274cedaa..793e3946 100644 --- a/src/cloud_provider/service.rs +++ b/src/cloud_provider/service.rs @@ -53,18 +53,34 @@ pub trait Service { // used to retrieve logs by using Kubernetes labels (selector) fn selector(&self) -> String; fn debug_logs(&self, deployment_target: &DeploymentTarget) -> Vec { - debug_logs(self, deployment_target) + match deployment_target { + // TODO retrieve logs from managed service? + DeploymentTarget::ManagedServices(_, _) => vec![], + + DeploymentTarget::SelfHosted(kubernetes, environment) => { + match get_stateless_resource_information_for_user(*kubernetes, *environment, self) { + Ok(lines) => lines, + Err(err) => { + error!( + "error while retrieving debug logs from {} {}; error: {:?}", + self.service_type().name(), + self.name_with_id(), + err + ); + vec![] + } + } + } + } } + fn is_listening(&self, ip: &str) -> bool { let private_port = match self.private_port() { Some(private_port) => private_port, _ => return false, }; - match TcpStream::connect(format!("{}:{}", ip, private_port)) { - Ok(_) => true, - Err(_) => false, - } + TcpStream::connect(format!("{}:{}", ip, private_port)).is_ok() } fn engine_error_scope(&self) -> EngineErrorScope; fn engine_error(&self, cause: EngineErrorCause, message: String) -> EngineError { @@ -143,7 +159,7 @@ pub trait Router: StatelessService + Listen { check_domain_for( ListenersHelper::new(self.listeners()), self.domains(), - self.id().into(), + self.id(), self.context().execution_id(), )?; Ok(()) @@ -155,7 +171,7 @@ pub trait Database: StatefulService { check_domain_for( ListenersHelper::new(&listeners), domains, - self.id().into(), + self.id(), self.context().execution_id(), )?; Ok(()) @@ -269,29 +285,6 @@ impl<'a> ServiceType<'a> { } } -pub fn debug_logs(service: &T, deployment_target: &DeploymentTarget) -> Vec -where - T: Service + ?Sized, -{ - match deployment_target { - DeploymentTarget::ManagedServices(_, _) => Vec::new(), // TODO retrieve logs from managed service? - DeploymentTarget::SelfHosted(kubernetes, environment) => { - match get_stateless_resource_information_for_user(*kubernetes, *environment, service) { - Ok(lines) => lines, - Err(err) => { - error!( - "error while retrieving debug logs from {} {}; error: {:?}", - service.service_type().name(), - service.name_with_id(), - err - ); - Vec::new() - } - } - } - } -} - pub fn default_tera_context( service: &dyn Service, kubernetes: &dyn Kubernetes, @@ -874,7 +867,7 @@ where } let debug_logs = service.debug_logs(deployment_target); - let debug_logs_string = if debug_logs.len() > 0 { + let debug_logs_string = if !debug_logs.is_empty() { debug_logs.join("\n") } else { String::from("") @@ -965,7 +958,17 @@ where )?; for pod in pods.items { - for container_status in pod.status.container_statuses { + for container_condition in pod.status.conditions { + if container_condition.status.to_ascii_lowercase() == "false" { + result.push(format!( + "Condition not met to start the container: {} -> {}: {}", + container_condition.typee, + container_condition.reason.unwrap_or_default(), + container_condition.message.unwrap_or_default() + )) + } + } + for container_status in pod.status.container_statuses.unwrap_or_default() { if let Some(last_state) = container_status.last_state { if let Some(terminated) = last_state.terminated { if let Some(message) = terminated.message { diff --git a/src/cmd/kubectl.rs b/src/cmd/kubectl.rs index 471adce9..67468c14 100644 --- a/src/cmd/kubectl.rs +++ b/src/cmd/kubectl.rs @@ -277,7 +277,7 @@ where { let result = kubectl_exec_get_pod(kubernetes_config, namespace, selector, envs)?; - if result.items.is_empty() || result.items.first().unwrap().status.container_statuses.is_empty() { + if result.items.is_empty() || result.items.first().unwrap().status.container_statuses.is_none() { return Ok(None); } diff --git a/src/cmd/structs.rs b/src/cmd/structs.rs index 168ccd80..fe15af74 100644 --- a/src/cmd/structs.rs +++ b/src/cmd/structs.rs @@ -90,12 +90,23 @@ pub struct KubernetesPod { #[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] #[serde(rename_all = "camelCase")] pub struct KubernetesPodStatus { - pub container_statuses: Vec, + pub container_statuses: Option>, + pub conditions: Vec, // read the doc: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ // phase can be Pending, Running, Succeeded, Failed, Unknown pub phase: KubernetesPodStatusPhase, } +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(rename_all = "camelCase")] +pub struct KubernetesPodCondition { + pub status: String, + #[serde(rename = "type")] + pub typee: String, + pub message: Option, + pub reason: Option, +} + #[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] pub enum KubernetesPodStatusPhase { Pending, @@ -230,3 +241,293 @@ impl HelmHistoryRow { self.status == "deployed" } } + +#[cfg(test)] +mod tests { + use crate::cmd::structs::{KubernetesList, KubernetesPod}; + + #[test] + fn test_pod_status_deserialize() { + let payload = r#" +{ "apiVersion": "v1", "items": [ { "apiVersion": "v1", "kind": "Pod", "metadata": { "annotations": { "kubernetes.io/psp": "eks.privileged" }, "creationTimestamp": "2021-03-15T15:41:56Z", "generateName": "postgresqlpostgres-", "labels": { "app": "postgresqlpostgres", "chart": "postgresql-8.9.8", "controller-revision-hash": "postgresqlpostgres-8db988cfd", "heritage": "Helm", "release": "postgresql-atx9frcbbrlphzu", "role": "master", "statefulset.kubernetes.io/pod-name": "postgresqlpostgres-0" }, "name": "postgresqlpostgres-0", "namespace": "lbxmwiibzi9lbla-ah5bbhekjarxta5", "ownerReferences": [ { "apiVersion": "apps/v1", "blockOwnerDeletion": true, "controller": true, "kind": "StatefulSet", "name": "postgresqlpostgres", "uid": "507ca7da-7d2c-4fdd-90f8-890c8a0d9491" } ], "resourceVersion": "53444298", "selfLink": "/api/v1/namespaces/lbxmwiibzi9lbla-ah5bbhekjarxta5/pods/postgresqlpostgres-0", "uid": "baf9e257-f517-49f5-b530-392a690f5231" }, "spec": { "containers": [ { "env": [ { "name": "BITNAMI_DEBUG", "value": "false" }, { "name": "POSTGRESQL_PORT_NUMBER", "value": "5432" }, { "name": "POSTGRESQL_VOLUME_DIR", "value": "/bitnami/postgresql" }, { "name": "POSTGRESQL_INITSCRIPTS_USERNAME", "value": "postgres" }, { "name": "POSTGRESQL_INITSCRIPTS_PASSWORD", "value": "cvbwtt8tzt6jtli" }, { "name": "PGDATA", "value": "/bitnami/postgresql/data" }, { "name": "POSTGRES_POSTGRES_PASSWORD", "valueFrom": { "secretKeyRef": { "key": "postgresql-postgres-password", "name": "postgresqlpostgres" } } }, { "name": "POSTGRES_USER", "value": "superuser" }, { "name": "POSTGRES_PASSWORD", "valueFrom": { "secretKeyRef": { "key": "postgresql-password", "name": "postgresqlpostgres" } } }, { "name": "POSTGRES_DB", "value": "postgres" }, { "name": "POSTGRESQL_ENABLE_LDAP", "value": "no" } ], "image": "quay.io/bitnami/postgresql:10.16.0", "imagePullPolicy": "IfNotPresent", "livenessProbe": { "exec": { "command": [ "/bin/sh", "-c", "exec pg_isready -U \"superuser\" -d \"postgres\" -h 127.0.0.1 -p 5432" ] }, "failureThreshold": 6, "initialDelaySeconds": 30, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 5 }, "name": "postgresqlpostgres", "ports": [ { "containerPort": 5432, "name": "tcp-postgresql", "protocol": "TCP" } ], "readinessProbe": { "exec": { "command": [ "/bin/sh", "-c", "-e", "exec pg_isready -U \"superuser\" -d \"postgres\" -h 127.0.0.1 -p 5432\n[ -f /opt/bitnami/postgresql/tmp/.initialized ] || [ -f /bitnami/postgresql/.initialized ]\n" ] }, "failureThreshold": 6, "initialDelaySeconds": 5, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 5 }, "resources": { "requests": { "cpu": "100m", "memory": "50Gi" } }, "securityContext": { "runAsUser": 1001 }, "terminationMessagePath": "/dev/termination-log", "terminationMessagePolicy": "File", "volumeMounts": [ { "mountPath": "/dev/shm", "name": "dshm" }, { "mountPath": "/bitnami/postgresql", "name": "data" }, { "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", "name": "default-token-n6bkr", "readOnly": true } ] } ], "dnsPolicy": "ClusterFirst", "enableServiceLinks": true, "hostname": "postgresqlpostgres-0", "initContainers": [ { "command": [ "/bin/sh", "-cx", "mkdir -p /bitnami/postgresql/data\nchmod 700 /bitnami/postgresql/data\nfind /bitnami/postgresql -mindepth 1 -maxdepth 1 -not -name \"conf\" -not -name \".snapshot\" -not -name \"lost+found\" | \\\n xargs chown -R 1001:1001\nchmod -R 777 /dev/shm\n" ], "image": "docker.io/bitnami/minideb:buster", "imagePullPolicy": "IfNotPresent", "name": "init-chmod-data", "resources": { "requests": { "cpu": "100m", "memory": "50Gi" } }, "securityContext": { "runAsUser": 0 }, "terminationMessagePath": "/dev/termination-log", "terminationMessagePolicy": "File", "volumeMounts": [ { "mountPath": "/bitnami/postgresql", "name": "data" }, { "mountPath": "/dev/shm", "name": "dshm" }, { "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", "name": "default-token-n6bkr", "readOnly": true } ] } ], "priority": 0, "restartPolicy": "Always", "schedulerName": "default-scheduler", "securityContext": { "fsGroup": 1001 }, "serviceAccount": "default", "serviceAccountName": "default", "subdomain": "postgresqlpostgres-headless", "terminationGracePeriodSeconds": 30, "tolerations": [ { "effect": "NoExecute", "key": "node.kubernetes.io/not-ready", "operator": "Exists", "tolerationSeconds": 300 }, { "effect": "NoExecute", "key": "node.kubernetes.io/unreachable", "operator": "Exists", "tolerationSeconds": 300 } ], "volumes": [ { "name": "data", "persistentVolumeClaim": { "claimName": "data-postgresqlpostgres-0" } }, { "emptyDir": { "medium": "Memory", "sizeLimit": "1Gi" }, "name": "dshm" }, { "name": "default-token-n6bkr", "secret": { "defaultMode": 420, "secretName": "default-token-n6bkr" } } ] }, "status": { "conditions": [ { "lastProbeTime": null, "lastTransitionTime": "2021-03-15T15:41:56Z", "message": "0/5 nodes are available: 5 Insufficient memory.", "reason": "Unschedulable", "status": "False", "type": "PodScheduled" } ], "phase": "Pending", "qosClass": "Burstable" } } ], "kind": "List", "metadata": { "resourceVersion": "", "selfLink": "" }} + "#; + + let pod_status = serde_json::from_str::>(payload); + assert_eq!(pod_status.is_ok(), true); + assert_eq!(pod_status.unwrap().items[0].status.conditions[0].status, "False"); + + let payload = r#" + { + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2021-02-26T10:11:37Z", + "generateName": "gradle-deployment-5654f49c5f-", + "labels": { + "app": "gradle", + "pod-template-hash": "5654f49c5f" + }, + "managedFields": [ + { + "apiVersion": "v1", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:generateName": {}, + "f:labels": { + ".": {}, + "f:app": {}, + "f:pod-template-hash": {} + }, + "f:ownerReferences": { + ".": {}, + "k:{\"uid\":\"e6c07d77-5b1c-497a-bafa-e24e945dccda\"}": { + ".": {}, + "f:apiVersion": {}, + "f:blockOwnerDeletion": {}, + "f:controller": {}, + "f:kind": {}, + "f:name": {}, + "f:uid": {} + } + } + }, + "f:spec": { + "f:containers": { + "k:{\"name\":\"gradle\"}": { + ".": {}, + "f:args": {}, + "f:command": {}, + "f:image": {}, + "f:imagePullPolicy": {}, + "f:name": {}, + "f:ports": { + ".": {}, + "k:{\"containerPort\":80,\"protocol\":\"TCP\"}": { + ".": {}, + "f:containerPort": {}, + "f:protocol": {} + } + }, + "f:resources": {}, + "f:terminationMessagePath": {}, + "f:terminationMessagePolicy": {} + } + }, + "f:dnsPolicy": {}, + "f:enableServiceLinks": {}, + "f:restartPolicy": {}, + "f:schedulerName": {}, + "f:securityContext": {}, + "f:terminationGracePeriodSeconds": {} + } + }, + "manager": "kube-controller-manager", + "operation": "Update", + "time": "2021-02-26T10:11:37Z" + }, + { + "apiVersion": "v1", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:status": { + "f:conditions": { + "k:{\"type\":\"ContainersReady\"}": { + ".": {}, + "f:lastProbeTime": {}, + "f:lastTransitionTime": {}, + "f:status": {}, + "f:type": {} + }, + "k:{\"type\":\"Initialized\"}": { + ".": {}, + "f:lastProbeTime": {}, + "f:lastTransitionTime": {}, + "f:status": {}, + "f:type": {} + }, + "k:{\"type\":\"Ready\"}": { + ".": {}, + "f:lastProbeTime": {}, + "f:lastTransitionTime": {}, + "f:status": {}, + "f:type": {} + } + }, + "f:containerStatuses": {}, + "f:hostIP": {}, + "f:phase": {}, + "f:podIP": {}, + "f:podIPs": { + ".": {}, + "k:{\"ip\":\"10.244.0.68\"}": { + ".": {}, + "f:ip": {} + } + }, + "f:startTime": {} + } + }, + "manager": "kubelet", + "operation": "Update", + "time": "2021-02-26T10:11:43Z" + } + ], + "name": "gradle-deployment-5654f49c5f-dw8zl", + "namespace": "default", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "gradle-deployment-5654f49c5f", + "uid": "e6c07d77-5b1c-497a-bafa-e24e945dccda" + } + ], + "resourceVersion": "9095811", + "selfLink": "/api/v1/namespaces/default/pods/gradle-deployment-5654f49c5f-dw8zl", + "uid": "c10f29f2-35d6-42dc-b9e8-71c99d7123e2" + }, + "spec": { + "containers": [ + { + "args": [ + "-c", + "sleep 6000000" + ], + "command": [ + "/bin/sh" + ], + "image": "ubuntu:latest", + "imagePullPolicy": "IfNotPresent", + "name": "gradle", + "ports": [ + { + "containerPort": 80, + "protocol": "TCP" + } + ], + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "default-token-p85k2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "imagePullSecrets": [ + { + "name": "default-docr-registry-qovery-do-test" + } + ], + "nodeName": "qovery-gqgyb7zy4ykwumak-3zl08", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "default-token-p85k2", + "secret": { + "defaultMode": 420, + "secretName": "default-token-p85k2" + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2021-02-26T10:11:37Z", + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2021-02-26T10:11:43Z", + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2021-02-26T10:11:43Z", + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2021-02-26T10:11:37Z", + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "docker://3afa93048e28f823becac70f17546a6bd7d83a8c50c25e22b8c0a1ca6b91aa21", + "image": "ubuntu:latest", + "imageID": "docker-pullable://ubuntu@sha256:703218c0465075f4425e58fac086e09e1de5c340b12976ab9eb8ad26615c3715", + "lastState": {}, + "name": "gradle", + "ready": true, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2021-02-26T10:11:42Z" + } + } + } + ], + "hostIP": "10.20.0.3", + "phase": "Running", + "podIP": "10.244.0.68", + "podIPs": [ + { + "ip": "10.244.0.68" + } + ], + "qosClass": "BestEffort", + "startTime": "2021-02-26T10:11:37Z" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "", + "selfLink": "" + } +} + "#; + + let pod_status = serde_json::from_str::>(payload); + assert_eq!(pod_status.is_ok(), true); + } +} diff --git a/tests/assets/eks-options.json b/tests/assets/eks-options.json index 191a5e0d..ab92b841 100644 --- a/tests/assets/eks-options.json +++ b/tests/assets/eks-options.json @@ -146,7 +146,6 @@ "vault_address": "CHANGE-ME/VAULT_ADDRESS", "vault_token": "CHANGE-ME/VAULT_TOKEN", "discord_api_key": "CHANGE-ME/DISCORD_API_URL", - "discord_api_key": "CHANGE-ME/DISCORD_API_URL", "qovery_nats_url": "CHANGE-ME/QOVERY_NATS_URL", "qovery_nats_user": "CHANGE-ME/QOVERY_NATS_USERNNAME", "qovery_nats_password": "CHANGE-ME/QOVERY_NATS_PASSWORD",