From be2175a95fb722fcc6f320a9499ce0bba72c0c71 Mon Sep 17 00:00:00 2001 From: Pierre Mavro Date: Mon, 7 Dec 2020 17:07:32 +0100 Subject: [PATCH] feat: adding autoscaler for testing purpose --- Cargo.lock | 7 +- lib/aws/bootstrap/eks-workers-nodes.j2.tf | 5 + .../bootstrap/helm-alertmanager-discord.tf | 1 + lib/aws/bootstrap/helm-aws-limits-exporter.tf | 2 +- .../bootstrap/helm-aws-node-term-handler.tf | 1 + lib/aws/bootstrap/helm-cert-manager.j2.tf | 2 +- lib/aws/bootstrap/helm-cluster-autoscaler.tf | 122 ++++++ lib/aws/bootstrap/helm-external-dns.tf | 1 + lib/aws/bootstrap/helm-grafana.tf | 2 +- lib/aws/bootstrap/helm-loki.tf | 1 + lib/aws/bootstrap/helm-metrics-server.tf | 1 + lib/aws/bootstrap/helm-nginx-ingress.tf | 2 +- lib/aws/bootstrap/helm-pleco.j2.tf | 1 + .../bootstrap/helm-prometheus-operator.j2.tf | 12 +- lib/aws/bootstrap/helm-qovery-agent.tf | 1 + lib/aws/bootstrap/helm-qovery-engine.tf | 1 + lib/aws/bootstrap/tf-providers-aws.j2.tf | 19 + .../charts/cluster-autoscaler/.helmignore | 23 + .../charts/cluster-autoscaler/Chart.yaml | 19 + .../charts/cluster-autoscaler/README.md | 395 ++++++++++++++++++ .../cluster-autoscaler/README.md.gotmpl | 324 ++++++++++++++ .../cluster-autoscaler/templates/_helpers.tpl | 87 ++++ .../templates/clusterrole.yaml | 147 +++++++ .../templates/clusterrolebinding.yaml | 16 + .../templates/deployment.yaml | 246 +++++++++++ .../cluster-autoscaler/templates/pdb.yaml | 15 + .../templates/podsecuritypolicy.yaml | 46 ++ .../priority-expander-configmap.yaml | 13 + .../cluster-autoscaler/templates/role.yaml | 25 ++ .../templates/rolebinding.yaml | 16 + .../cluster-autoscaler/templates/secret.yaml | 20 + .../cluster-autoscaler/templates/service.yaml | 36 ++ .../templates/serviceaccount.yaml | 11 + .../templates/servicemonitor.yaml | 24 ++ .../charts/cluster-autoscaler/values.yaml | 295 +++++++++++++ lib/helm-freeze.yaml | 5 + src/cloud_provider/aws/databases/redis.rs | 2 +- src/cloud_provider/aws/kubernetes/mod.rs | 4 +- .../digitalocean/kubernetes/mod.rs | 4 +- test_utilities/src/aws.rs | 14 +- test_utilities/src/digitalocean.rs | 5 + 41 files changed, 1956 insertions(+), 17 deletions(-) create mode 100644 lib/aws/bootstrap/helm-cluster-autoscaler.tf create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/.helmignore create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/Chart.yaml create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/README.md create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/README.md.gotmpl create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/templates/_helpers.tpl create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/templates/clusterrole.yaml create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/templates/clusterrolebinding.yaml create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/templates/deployment.yaml create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/templates/pdb.yaml create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/templates/podsecuritypolicy.yaml create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/templates/priority-expander-configmap.yaml create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/templates/role.yaml create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/templates/rolebinding.yaml create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/templates/secret.yaml create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/templates/service.yaml create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/templates/serviceaccount.yaml create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/templates/servicemonitor.yaml create mode 100644 lib/common/bootstrap/charts/cluster-autoscaler/values.yaml diff --git a/Cargo.lock b/Cargo.lock index 78155ad9..276b4e03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2285,7 +2285,7 @@ dependencies = [ "rustc_version", "serde", "sha2", - "time 0.2.22", + "time 0.2.23", "tokio 0.2.22", ] @@ -2750,6 +2750,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "time 0.2.23", "tracing", "tracing-subscriber", ] @@ -2796,9 +2797,9 @@ dependencies = [ [[package]] name = "time" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55b7151c9065e80917fbf285d9a5d1432f60db41d170ccafc749a136b41a93af" +checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b" dependencies = [ "const_fn", "libc", diff --git a/lib/aws/bootstrap/eks-workers-nodes.j2.tf b/lib/aws/bootstrap/eks-workers-nodes.j2.tf index 879942e8..47709e04 100644 --- a/lib/aws/bootstrap/eks-workers-nodes.j2.tf +++ b/lib/aws/bootstrap/eks-workers-nodes.j2.tf @@ -28,6 +28,11 @@ resource "aws_eks_node_group" "eks_cluster_workers_{{ loop.index }}" { update = "60m" } + lifecycle { + // don't update the desired size and let the cluster-autoscaler do the job + ignore_changes = [scaling_config[0].desired_size] + } + # Ensure that IAM Role permissions are created before and deleted after EKS Node Group handling. # Otherwise, EKS will not be able to properly delete EC2 Instances and Elastic Network Interfaces. depends_on = [ diff --git a/lib/aws/bootstrap/helm-alertmanager-discord.tf b/lib/aws/bootstrap/helm-alertmanager-discord.tf index 9aba1832..e1895366 100644 --- a/lib/aws/bootstrap/helm-alertmanager-discord.tf +++ b/lib/aws/bootstrap/helm-alertmanager-discord.tf @@ -46,5 +46,6 @@ resource "helm_release" "alertmanager_discord" { depends_on = [ aws_eks_cluster.eks_cluster, helm_release.aws_vpc_cni, + helm_release.cluster_autoscaler, ] } \ No newline at end of file diff --git a/lib/aws/bootstrap/helm-aws-limits-exporter.tf b/lib/aws/bootstrap/helm-aws-limits-exporter.tf index 78f2422b..c22be216 100644 --- a/lib/aws/bootstrap/helm-aws-limits-exporter.tf +++ b/lib/aws/bootstrap/helm-aws-limits-exporter.tf @@ -57,7 +57,7 @@ resource "helm_release" "iam_aws_limits_exporter" { depends_on = [ aws_eks_cluster.eks_cluster, - helm_release.prometheus_operator, + helm_release.cluster_autoscaler, helm_release.aws_vpc_cni, ] } diff --git a/lib/aws/bootstrap/helm-aws-node-term-handler.tf b/lib/aws/bootstrap/helm-aws-node-term-handler.tf index 569ed7e2..b6f75d2b 100644 --- a/lib/aws/bootstrap/helm-aws-node-term-handler.tf +++ b/lib/aws/bootstrap/helm-aws-node-term-handler.tf @@ -59,5 +59,6 @@ resource "helm_release" "aws_node_term_handler" { depends_on = [ aws_eks_cluster.eks_cluster, helm_release.aws_vpc_cni, + helm_release.cluster_autoscaler, ] } \ No newline at end of file diff --git a/lib/aws/bootstrap/helm-cert-manager.j2.tf b/lib/aws/bootstrap/helm-cert-manager.j2.tf index 99c0c6ce..d2aaac09 100644 --- a/lib/aws/bootstrap/helm-cert-manager.j2.tf +++ b/lib/aws/bootstrap/helm-cert-manager.j2.tf @@ -104,7 +104,7 @@ resource "helm_release" "cert_manager" { depends_on = [ aws_eks_cluster.eks_cluster, - helm_release.prometheus_operator, + helm_release.cluster_autoscaler, helm_release.aws_vpc_cni, ] } diff --git a/lib/aws/bootstrap/helm-cluster-autoscaler.tf b/lib/aws/bootstrap/helm-cluster-autoscaler.tf new file mode 100644 index 00000000..97e8e337 --- /dev/null +++ b/lib/aws/bootstrap/helm-cluster-autoscaler.tf @@ -0,0 +1,122 @@ +resource "aws_iam_user" "iam_eks_cluster_autoscaler" { + name = "qovery-clustauto-${var.kubernetes_cluster_id}" +} + +resource "aws_iam_access_key" "iam_eks_cluster_autoscaler" { + user = aws_iam_user.iam_eks_cluster_autoscaler.name +} + +resource "aws_iam_policy" "cluster_autoscaler_policy" { + name = aws_iam_user.iam_eks_cluster_autoscaler.name + description = "Policy for cluster autoscaler" + + policy = < + +# Method 2 - Specifying groups manually +$ helm install my-release autoscaler/cluster-autoscaler \ +--set "autoscalingGroups[0].name=your-asg-name" \ +--set "autoscalingGroups[0].maxSize=10" \ +--set "autoscalingGroups[0].minSize=1" +``` + +## Introduction + +This chart bootstraps a cluster-autoscaler deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Helm 3+ +- Kubernetes 1.8+ + - [Older versions](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler#releases) may work by overriding the `image`. Cluster autoscaler internally simulates the scheduler and bugs between mismatched versions may be subtle. +- Azure AKS specific Prerequisites: + - Kubernetes 1.10+ with RBAC-enabled. + +## Previous Helm Chart + +The previous `cluster-autoscaler` Helm chart hosted at [helm/charts](https://github.com/helm/charts) has been moved to this repository in accordance with the [Deprecation timeline](https://github.com/helm/charts#deprecation-timeline). Note that a few things have changed between this version and the old version: + +- This repository **only** supports Helm chart installations using Helm 3+ since the `apiVersion` on the charts has been marked as `v2`. +- Previous versions of the Helm chart have not been migrated, and the version was reset to `1.0.0` initially. If you are looking for old versions of the chart, it's best to run `helm pull stable/cluster-autoscaler --version ` until you are ready to move to this repository's version. +- The previous versioning scheme has been returned to as of version `9.0.0` for ease of migration from the previous chart location. + +## Migration from 1.X to 9.X+ versions of this Chart + +On initial adoption of this chart this chart was renamed from `cluster-autoscaler` to `cluster-autoscaler-chart` due to technical limitations. This affects all `1.X` releases of the chart. + +Releases of the chart from `9.0.0` onwards return the naming of the chart to `cluster-autoscaler` and return to following the versioning established by the chart's previous location. + +To migrate from a 1.X release of the chart to a `9.0.0` or later release, you should first uninstall your `1.X` install of the `cluster-autoscaler-chart` chart, before performing the installation of the new `cluster-autoscaler` chart. + +## Migration from 9.0 to 9.1 + +Starting from `9.1.0` the `envFromConfigMap` value is expected to contain the name of a ConfigMap that is used as ref for `envFrom`, similar to `envFromSecret`. If you want to keep the previous behaviour of `envFromConfigMap` you must rename it to `extraEnvConfigMaps`. + +## Installing the Chart + +**By default, no deployment is created and nothing will autoscale**. + +You must provide some minimal configuration, either to specify instance groups or enable auto-discovery. It is not recommended to do both. + +Either: + +- Set `autoDiscovery.clusterName` and provide additional autodiscovery options if necessary **or** +- Set static node group configurations for one or more node groups (using `autoscalingGroups` or `autoscalingGroupsnamePrefix`). + +To create a valid configuration, follow instructions for your cloud provider: + +* [AWS](#aws---using-auto-discovery-of-tagged-instance-groups) +* [GCE](#gce) +* [Azure AKS](#azure-aks) +* [OpenStack Magnum](#openstack-magnum) + +### AWS - Using auto-discovery of tagged instance groups + +Auto-discovery finds ASGs tags as below and automatically manages them based on the min and max size specified in the ASG. `cloudProvider=aws` only. + +- Tag the ASGs with keys to match `.Values.autoDiscovery.tags`, by default: `k8s.io/cluster-autoscaler/enabled` and `k8s.io/cluster-autoscaler/` +- Verify the [IAM Permissions](#aws---iam) +- Set `autoDiscovery.clusterName=` +- Set `awsRegion=` +- Set `awsAccessKeyID=` and `awsSecretAccessKey=` if you want to [use AWS credentials directly instead of an instance role](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials) + +```console +$ helm install my-release autoscaler/cluster-autoscaler --set autoDiscovery.clusterName= +``` + +#### Specifying groups manually + +Without autodiscovery, specify an array of elements each containing ASG name, min size, max size. The sizes specified here will be applied to the ASG, assuming IAM permissions are correctly configured. + +- Verify the [IAM Permissions](#aws---iam) +- Either provide a yaml file setting `autoscalingGroups` (see values.yaml) or use `--set` e.g.: + +```console +$ helm install my-release autoscaler/cluster-autoscaler \ +--set "autoscalingGroups[0].name=your-asg-name" \ +--set "autoscalingGroups[0].maxSize=10" \ +--set "autoscalingGroups[0].minSize=1" +``` + +#### Auto-discovery + +For auto-discovery of instances to work, they must be tagged with the keys in `.Values.autoDiscovery.tags`, which by default are +`k8s.io/cluster-autoscaler/enabled` and `k8s.io/cluster-autoscaler/` + +The value of the tag does not matter, only the key. + +An example kops spec excerpt: + +```yaml +apiVersion: kops/v1alpha2 +kind: Cluster +metadata: + name: my.cluster.internal +spec: + additionalPolicies: + node: | + [ + {"Effect":"Allow","Action":["autoscaling:DescribeAutoScalingGroups","autoscaling:DescribeAutoScalingInstances","autoscaling:DescribeLaunchConfigurations","autoscaling:DescribeTags","autoscaling:SetDesiredCapacity","autoscaling:TerminateInstanceInAutoScalingGroup"],"Resource":"*"} + ] + ... +--- +apiVersion: kops/v1alpha2 +kind: InstanceGroup +metadata: + labels: + kops.k8s.io/cluster: my.cluster.internal + name: my-instances +spec: + cloudLabels: + k8s.io/cluster-autoscaler/enabled: "" + k8s.io/cluster-autoscaler/my.cluster.internal: "" + image: kops.io/k8s-1.8-debian-jessie-amd64-hvm-ebs-2018-01-14 + machineType: r4.large + maxSize: 4 + minSize: 0 +``` + +In this example you would need to `--set autoDiscovery.clusterName=my.cluster.internal` when installing. + +It is not recommended to try to mix this with setting `autoscalingGroups` + +See [autoscaler AWS documentation](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup) for a more discussion of the setup. + +### GCE + +The following parameters are required: + +- `autoDiscovery.clusterName=any-name` +- `cloud-provider=gce` +- `autoscalingGroupsnamePrefix[0].name=your-ig-prefix,autoscalingGroupsnamePrefix[0].maxSize=10,autoscalingGroupsnamePrefix[0].minSize=1` + +To use Managed Instance Group (MIG) auto-discovery, provide a YAML file setting `autoscalingGroupsnamePrefix` (see values.yaml) or use `--set` when installing the Chart - e.g. + +```console +$ helm install my-release autoscaler/cluster-autoscaler \ +--set "autoscalingGroupsnamePrefix[0].name=your-ig-prefix,autoscalingGroupsnamePrefix[0].maxSize=10,autoscalingGroupsnamePrefi[0].minSize=1" \ +--set autoDiscovery.clusterName= \ +--set cloudProvider=gce +``` + +Note that `your-ig-prefix` should be a _prefix_ matching one or more MIGs, and _not_ the full name of the MIG. For example, to match multiple instance groups - `k8s-node-group-a-standard`, `k8s-node-group-b-gpu`, you would use a prefix of `k8s-node-group-`. + +In the event you want to explicitly specify MIGs instead of using auto-discovery, set members of the `autoscalingGroups` array directly - e.g. + +``` +# where 'n' is the index, starting at 0 +-- set autoscalingGroups[n].name=https://content.googleapis.com/compute/v1/projects/$PROJECTID/zones/$ZONENAME/instanceGroupManagers/$FULL-MIG-NAME,autoscalingGroups[n].maxSize=$MAXSIZE,autoscalingGroups[n].minSize=$MINSIZE +``` + +### Azure AKS + +The following parameters are required: + +- `cloudProvider=azure` +- `autoscalingGroups[0].name=your-agent-pool,autoscalingGroups[0].maxSize=10,autoscalingGroups[0].minSize=1` +- `azureClientID: "your-service-principal-app-id"` +- `azureClientSecret: "your-service-principal-client-secret"` +- `azureSubscriptionID: "your-azure-subscription-id"` +- `azureTenantID: "your-azure-tenant-id"` +- `azureClusterName: "your-aks-cluster-name"` +- `azureResourceGroup: "your-aks-cluster-resource-group-name"` +- `azureVMType: "AKS"` +- `azureNodeResourceGroup: "your-aks-cluster-node-resource-group"` + +### OpenStack Magnum + +`cloudProvider: magnum` must be set, and then one of + +- `magnumClusterName=` and `autoscalingGroups` with the names of node groups and min/max node counts +- or `autoDiscovery.clusterName=` with one or more `autoDiscovery.roles`. + +Additionally, `cloudConfigPath: "/etc/kubernetes/cloud-config"` must be set as this should be the location +of the cloud-config file on the host. + +Example values files can be found [here](../../cluster-autoscaler/cloudprovider/magnum/examples). + +Install the chart with + +``` +$ helm install my-release autoscaler/cluster-autoscaler -f myvalues.yaml +``` + +## Uninstalling the Chart + +To uninstall `my-release`: + +```console +$ helm uninstall my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +> **Tip**: List all releases using `helm list` or start clean with `helm uninstall my-release` + +## Additional Configuration + +### AWS - IAM + +The worker running the cluster autoscaler will need access to certain resources and actions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup" + ], + "Resource": "*" + } + ] +} +``` + +- `DescribeTags` is required for autodiscovery. +- `DescribeLaunchConfigurations` is required to scale up an ASG from 0. + +If you would like to limit the scope of the Cluster Autoscaler to ***only*** modify ASGs for a particular cluster, use the following policy instead: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeLaunchTemplateVersions" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:UpdateAutoScalingGroup" + ], + "Resource": [ + "arn:aws:autoscaling:::autoScalingGroup::autoScalingGroupName/node-group-1", + "arn:aws:autoscaling:::autoScalingGroup::autoScalingGroupName/node-group-2", + "arn:aws:autoscaling:::autoScalingGroup::autoScalingGroupName/node-group-3" + ], + "Condition": { + "StringEquals": { + "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled": "true", + "autoscaling:ResourceTag/kubernetes.io/cluster/": "owned" + } + } + } + ] +} +``` + +Make sure to replace the variables ``, ``, ``, and the ARNs of the ASGs where applicable. + +### AWS - IAM Roles for Service Accounts (IRSA) + +For Kubernetes clusters that use Amazon EKS, the service account can be configured with an IAM role using [IAM Roles for Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) to avoid needing to grant access to the worker nodes for AWS resources. + +In order to accomplish this, you will first need to create a new IAM role with the above mentions policies. Take care in [configuring the trust relationship](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html#iam-role-configuration) to restrict access just to the service account used by cluster autoscaler. + +Once you have the IAM role configured, you would then need to `--set rbac.serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=arn:aws:iam::123456789012:role/MyRoleName` when installing. + +## Troubleshooting + +The chart will succeed even if the container arguments are incorrect. A few minutes after starting +`kubectl logs -l "app=aws-cluster-autoscaler" --tail=50` should loop through something like + +``` +polling_autoscaler.go:111] Poll finished +static_autoscaler.go:97] Starting main loop +utils.go:435] No pod using affinity / antiaffinity found in cluster, disabling affinity predicate for this loop +static_autoscaler.go:230] Filtering out schedulables +``` + +If not, find a pod that the deployment created and `describe` it, paying close attention to the arguments under `Command`. e.g.: + +``` +Containers: + cluster-autoscaler: + Command: + ./cluster-autoscaler + --cloud-provider=aws +# if specifying ASGs manually + --nodes=1:10:your-scaling-group-name +# if using autodiscovery + --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/ + --v=4 +``` + +### PodSecurityPolicy + +Though enough for the majority of installations, the default PodSecurityPolicy _could_ be too restrictive depending on the specifics of your release. Please make sure to check that the template fits with any customizations made or disable it by setting `rbac.pspEnabled` to `false`. + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| additionalLabels | object | `{}` | Labels to add to each object of the chart. | +| affinity | object | `{}` | Affinity for pod assignment | +| autoDiscovery.clusterName | string | `nil` | Enable autodiscovery for `cloudProvider=aws`, for groups matching `autoDiscovery.tags`. Enable autodiscovery for `cloudProvider=gce`, but no MIG tagging required. Enable autodiscovery for `cloudProvider=magnum`, for groups matching `autoDiscovery.roles`. | +| autoDiscovery.roles | list | `["worker"]` | Magnum node group roles to match. | +| autoDiscovery.tags | list | `["k8s.io/cluster-autoscaler/enabled","k8s.io/cluster-autoscaler/{{ .Values.autoDiscovery.clusterName }}"]` | ASG tags to match, run through `tpl`. | +| autoscalingGroups | list | `[]` | For AWS, Azure AKS or Magnum. At least one element is required if not using `autoDiscovery`. For example: 
 - name: asg1
   maxSize: 2
   minSize: 1
| +| autoscalingGroupsnamePrefix | list | `[]` | For GCE. At least one element is required if not using `autoDiscovery`. For example: 
 - name: ig01
   maxSize: 10
   minSize: 0
| +| awsAccessKeyID | string | `""` | AWS access key ID ([if AWS user keys used](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials)) | +| awsRegion | string | `"us-east-1"` | AWS region (required if `cloudProvider=aws`) | +| awsSecretAccessKey | string | `""` | AWS access secret key ([if AWS user keys used](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials)) | +| azureClientID | string | `""` | Service Principal ClientID with contributor permission to Cluster and Node ResourceGroup. Required if `cloudProvider=azure` | +| azureClientSecret | string | `""` | Service Principal ClientSecret with contributor permission to Cluster and Node ResourceGroup. Required if `cloudProvider=azure` | +| azureClusterName | string | `""` | Azure AKS cluster name. Required if `cloudProvider=azure` | +| azureNodeResourceGroup | string | `""` | Azure resource group where the cluster's nodes are located, typically set as `MC___`. Required if `cloudProvider=azure` | +| azureResourceGroup | string | `""` | Azure resource group that the cluster is located. Required if `cloudProvider=azure` | +| azureSubscriptionID | string | `""` | Azure subscription where the resources are located. Required if `cloudProvider=azure` | +| azureTenantID | string | `""` | Azure tenant where the resources are located. Required if `cloudProvider=azure` | +| azureUseManagedIdentityExtension | bool | `false` | Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID and resource group are set. | +| azureVMType | string | `"AKS"` | Azure VM type. | +| cloudConfigPath | string | `"/etc/gce.conf"` | Configuration file for cloud provider. | +| cloudProvider | string | `"aws"` | The cloud provider where the autoscaler runs. Currently only `gce`, `aws`, `azure` and `magnum` are supported. `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS. `magnum` for OpenStack Magnum. | +| containerSecurityContext | object | `{}` | [Security context for container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) | +| dnsPolicy | string | `"ClusterFirst"` | Defaults to `ClusterFirst`. Valid values are: `ClusterFirstWithHostNet`, `ClusterFirst`, `Default` or `None`. If autoscaler does not depend on cluster DNS, recommended to set this to `Default`. | +| envFromConfigMap | string | `""` | ConfigMap name to use as envFrom. | +| envFromSecret | string | `""` | Secret name to use as envFrom. | +| expanderPriorities | object | `{}` | The expanderPriorities is used if `extraArgs.expander` is set to `priority` and expanderPriorities is also set with the priorities. If `extraArgs.expander` is set to `priority`, then expanderPriorities is used to define cluster-autoscaler-priority-expander priorities. See: https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/expander/priority/readme.md | +| extraArgs | object | `{"logtostderr":true,"stderrthreshold":"info","v":4}` | Additional container arguments. | +| extraEnv | object | `{}` | Additional container environment variables. | +| extraEnvConfigMaps | object | `{}` | Additional container environment variables from ConfigMaps. | +| extraEnvSecrets | object | `{}` | Additional container environment variables from Secrets. | +| extraVolumeSecrets | object | `{}` | Additional volumes to mount from Secrets. | +| fullnameOverride | string | `""` | String to fully override `cluster-autoscaler.fullname` template. | +| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | +| image.pullSecrets | list | `[]` | Image pull secrets | +| image.repository | string | `"us.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler"` | Image repository | +| image.tag | string | `"v1.18.1"` | Image tag | +| kubeTargetVersionOverride | string | `""` | Allow overriding the `.Capabilities.KubeVersion.GitVersion` check. Useful for `helm template` commands. | +| magnumCABundlePath | string | `"/etc/kubernetes/ca-bundle.crt"` | Path to the host's CA bundle, from `ca-file` in the cloud-config file. | +| magnumClusterName | string | `""` | Cluster name or ID in Magnum. Required if `cloudProvider=magnum` and not setting `autoDiscovery.clusterName`. | +| nameOverride | string | `""` | String to partially override `cluster-autoscaler.fullname` template (will maintain the release name) | +| nodeSelector | object | `{}` | Node labels for pod assignment. Ref: https://kubernetes.io/docs/user-guide/node-selection/. | +| podAnnotations | object | `{}` | Annotations to add to each pod. | +| podDisruptionBudget | object | `{"maxUnavailable":1}` | Pod disruption budget. | +| podLabels | object | `{}` | Labels to add to each pod. | +| priorityClassName | string | `""` | priorityClassName | +| rbac.create | bool | `true` | If `true`, create and use RBAC resources. | +| rbac.pspEnabled | bool | `false` | If `true`, creates and uses RBAC resources required in the cluster with [Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) enabled. Must be used with `rbac.create` set to `true`. | +| rbac.serviceAccount.annotations | object | `{}` | Additional Service Account annotations. | +| rbac.serviceAccount.create | bool | `true` | If `true` and `rbac.create` is also true, a Service Account will be created. | +| rbac.serviceAccount.name | string | `""` | The name of the ServiceAccount to use. If not set and create is `true`, a name is generated using the fullname template. | +| replicaCount | int | `1` | Desired number of pods | +| resources | object | `{}` | Pod resource requests and limits. | +| securityContext | object | `{}` | [Security context for pod](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) | +| service.annotations | object | `{}` | Annotations to add to service | +| service.externalIPs | list | `[]` | List of IP addresses at which the service is available. Ref: https://kubernetes.io/docs/user-guide/services/#external-ips. | +| service.labels | object | `{}` | Labels to add to service | +| service.loadBalancerIP | string | `""` | IP address to assign to load balancer (if supported). | +| service.loadBalancerSourceRanges | list | `[]` | List of IP CIDRs allowed access to load balancer (if supported). | +| service.portName | string | `"http"` | Name for service port. | +| service.servicePort | int | `8085` | Service port to expose. | +| service.type | string | `"ClusterIP"` | Type of service to create. | +| serviceMonitor.enabled | bool | `false` | If true, creates a Prometheus Operator ServiceMonitor. | +| serviceMonitor.interval | string | `"10s"` | Interval that Prometheus scrapes Cluster Autoscaler metrics. | +| serviceMonitor.namespace | string | `"monitoring"` | Namespace which Prometheus is running in. | +| serviceMonitor.path | string | `"/metrics"` | The path to scrape for metrics; autoscaler exposes `/metrics` (this is standard) | +| serviceMonitor.selector | object | `{"release":"prometheus-operator"}` | Default to kube-prometheus install (CoreOS recommended), but should be set according to Prometheus install. | +| tolerations | list | `[]` | List of node taints to tolerate (requires Kubernetes >= 1.6). | +| updateStrategy | object | `{}` | [Deployment update strategy](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy) | diff --git a/lib/common/bootstrap/charts/cluster-autoscaler/README.md.gotmpl b/lib/common/bootstrap/charts/cluster-autoscaler/README.md.gotmpl new file mode 100644 index 00000000..3969d454 --- /dev/null +++ b/lib/common/bootstrap/charts/cluster-autoscaler/README.md.gotmpl @@ -0,0 +1,324 @@ +{{ template "chart.header" . }} + +{{ template "chart.description" . }} + +## TL;DR: + +```console +$ helm repo add autoscaler https://kubernetes.github.io/autoscaler + +# Method 1 - Using Autodiscovery +$ helm install my-release autoscaler/cluster-autoscaler \ +--set 'autoDiscovery.clusterName'= + +# Method 2 - Specifying groups manually +$ helm install my-release autoscaler/cluster-autoscaler \ +--set "autoscalingGroups[0].name=your-asg-name" \ +--set "autoscalingGroups[0].maxSize=10" \ +--set "autoscalingGroups[0].minSize=1" +``` + +## Introduction + +This chart bootstraps a cluster-autoscaler deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Helm 3+ +- Kubernetes 1.8+ + - [Older versions](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler#releases) may work by overriding the `image`. Cluster autoscaler internally simulates the scheduler and bugs between mismatched versions may be subtle. +- Azure AKS specific Prerequisites: + - Kubernetes 1.10+ with RBAC-enabled. + +## Previous Helm Chart + +The previous `cluster-autoscaler` Helm chart hosted at [helm/charts](https://github.com/helm/charts) has been moved to this repository in accordance with the [Deprecation timeline](https://github.com/helm/charts#deprecation-timeline). Note that a few things have changed between this version and the old version: + +- This repository **only** supports Helm chart installations using Helm 3+ since the `apiVersion` on the charts has been marked as `v2`. +- Previous versions of the Helm chart have not been migrated, and the version was reset to `1.0.0` initially. If you are looking for old versions of the chart, it's best to run `helm pull stable/cluster-autoscaler --version ` until you are ready to move to this repository's version. +- The previous versioning scheme has been returned to as of version `9.0.0` for ease of migration from the previous chart location. + +## Migration from 1.X to 9.X+ versions of this Chart + +On initial adoption of this chart this chart was renamed from `cluster-autoscaler` to `cluster-autoscaler-chart` due to technical limitations. This affects all `1.X` releases of the chart. + +Releases of the chart from `9.0.0` onwards return the naming of the chart to `cluster-autoscaler` and return to following the versioning established by the chart's previous location. + +To migrate from a 1.X release of the chart to a `9.0.0` or later release, you should first uninstall your `1.X` install of the `cluster-autoscaler-chart` chart, before performing the installation of the new `cluster-autoscaler` chart. + +## Migration from 9.0 to 9.1 + +Starting from `9.1.0` the `envFromConfigMap` value is expected to contain the name of a ConfigMap that is used as ref for `envFrom`, similar to `envFromSecret`. If you want to keep the previous behaviour of `envFromConfigMap` you must rename it to `extraEnvConfigMaps`. + +## Installing the Chart + +**By default, no deployment is created and nothing will autoscale**. + +You must provide some minimal configuration, either to specify instance groups or enable auto-discovery. It is not recommended to do both. + +Either: + +- Set `autoDiscovery.clusterName` and provide additional autodiscovery options if necessary **or** +- Set static node group configurations for one or more node groups (using `autoscalingGroups` or `autoscalingGroupsnamePrefix`). + +To create a valid configuration, follow instructions for your cloud provider: + +* [AWS](#aws---using-auto-discovery-of-tagged-instance-groups) +* [GCE](#gce) +* [Azure AKS](#azure-aks) +* [OpenStack Magnum](#openstack-magnum) + +### AWS - Using auto-discovery of tagged instance groups + +Auto-discovery finds ASGs tags as below and automatically manages them based on the min and max size specified in the ASG. `cloudProvider=aws` only. + +- Tag the ASGs with keys to match `.Values.autoDiscovery.tags`, by default: `k8s.io/cluster-autoscaler/enabled` and `k8s.io/cluster-autoscaler/` +- Verify the [IAM Permissions](#aws---iam) +- Set `autoDiscovery.clusterName=` +- Set `awsRegion=` +- Set `awsAccessKeyID=` and `awsSecretAccessKey=` if you want to [use AWS credentials directly instead of an instance role](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials) + +```console +$ helm install my-release autoscaler/cluster-autoscaler --set autoDiscovery.clusterName= +``` + +#### Specifying groups manually + +Without autodiscovery, specify an array of elements each containing ASG name, min size, max size. The sizes specified here will be applied to the ASG, assuming IAM permissions are correctly configured. + +- Verify the [IAM Permissions](#aws---iam) +- Either provide a yaml file setting `autoscalingGroups` (see values.yaml) or use `--set` e.g.: + +```console +$ helm install my-release autoscaler/cluster-autoscaler \ +--set "autoscalingGroups[0].name=your-asg-name" \ +--set "autoscalingGroups[0].maxSize=10" \ +--set "autoscalingGroups[0].minSize=1" +``` + +#### Auto-discovery + +For auto-discovery of instances to work, they must be tagged with the keys in `.Values.autoDiscovery.tags`, which by default are +`k8s.io/cluster-autoscaler/enabled` and `k8s.io/cluster-autoscaler/` + +The value of the tag does not matter, only the key. + +An example kops spec excerpt: + +```yaml +apiVersion: kops/v1alpha2 +kind: Cluster +metadata: + name: my.cluster.internal +spec: + additionalPolicies: + node: | + [ + {"Effect":"Allow","Action":["autoscaling:DescribeAutoScalingGroups","autoscaling:DescribeAutoScalingInstances","autoscaling:DescribeLaunchConfigurations","autoscaling:DescribeTags","autoscaling:SetDesiredCapacity","autoscaling:TerminateInstanceInAutoScalingGroup"],"Resource":"*"} + ] + ... +--- +apiVersion: kops/v1alpha2 +kind: InstanceGroup +metadata: + labels: + kops.k8s.io/cluster: my.cluster.internal + name: my-instances +spec: + cloudLabels: + k8s.io/cluster-autoscaler/enabled: "" + k8s.io/cluster-autoscaler/my.cluster.internal: "" + image: kops.io/k8s-1.8-debian-jessie-amd64-hvm-ebs-2018-01-14 + machineType: r4.large + maxSize: 4 + minSize: 0 +``` + +In this example you would need to `--set autoDiscovery.clusterName=my.cluster.internal` when installing. + +It is not recommended to try to mix this with setting `autoscalingGroups` + +See [autoscaler AWS documentation](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup) for a more discussion of the setup. + +### GCE + +The following parameters are required: + +- `autoDiscovery.clusterName=any-name` +- `cloud-provider=gce` +- `autoscalingGroupsnamePrefix[0].name=your-ig-prefix,autoscalingGroupsnamePrefix[0].maxSize=10,autoscalingGroupsnamePrefix[0].minSize=1` + +To use Managed Instance Group (MIG) auto-discovery, provide a YAML file setting `autoscalingGroupsnamePrefix` (see values.yaml) or use `--set` when installing the Chart - e.g. + +```console +$ helm install my-release autoscaler/cluster-autoscaler \ +--set "autoscalingGroupsnamePrefix[0].name=your-ig-prefix,autoscalingGroupsnamePrefix[0].maxSize=10,autoscalingGroupsnamePrefi[0].minSize=1" \ +--set autoDiscovery.clusterName= \ +--set cloudProvider=gce +``` + +Note that `your-ig-prefix` should be a _prefix_ matching one or more MIGs, and _not_ the full name of the MIG. For example, to match multiple instance groups - `k8s-node-group-a-standard`, `k8s-node-group-b-gpu`, you would use a prefix of `k8s-node-group-`. + +In the event you want to explicitly specify MIGs instead of using auto-discovery, set members of the `autoscalingGroups` array directly - e.g. + +``` +# where 'n' is the index, starting at 0 +-- set autoscalingGroups[n].name=https://content.googleapis.com/compute/v1/projects/$PROJECTID/zones/$ZONENAME/instanceGroupManagers/$FULL-MIG-NAME,autoscalingGroups[n].maxSize=$MAXSIZE,autoscalingGroups[n].minSize=$MINSIZE +``` + +### Azure AKS + +The following parameters are required: + +- `cloudProvider=azure` +- `autoscalingGroups[0].name=your-agent-pool,autoscalingGroups[0].maxSize=10,autoscalingGroups[0].minSize=1` +- `azureClientID: "your-service-principal-app-id"` +- `azureClientSecret: "your-service-principal-client-secret"` +- `azureSubscriptionID: "your-azure-subscription-id"` +- `azureTenantID: "your-azure-tenant-id"` +- `azureClusterName: "your-aks-cluster-name"` +- `azureResourceGroup: "your-aks-cluster-resource-group-name"` +- `azureVMType: "AKS"` +- `azureNodeResourceGroup: "your-aks-cluster-node-resource-group"` + +### OpenStack Magnum + +`cloudProvider: magnum` must be set, and then one of + +- `magnumClusterName=` and `autoscalingGroups` with the names of node groups and min/max node counts +- or `autoDiscovery.clusterName=` with one or more `autoDiscovery.roles`. + +Additionally, `cloudConfigPath: "/etc/kubernetes/cloud-config"` must be set as this should be the location +of the cloud-config file on the host. + +Example values files can be found [here](../../cluster-autoscaler/cloudprovider/magnum/examples). + +Install the chart with + +``` +$ helm install my-release autoscaler/cluster-autoscaler -f myvalues.yaml +``` + +## Uninstalling the Chart + +To uninstall `my-release`: + +```console +$ helm uninstall my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +> **Tip**: List all releases using `helm list` or start clean with `helm uninstall my-release` + +## Additional Configuration + +### AWS - IAM + +The worker running the cluster autoscaler will need access to certain resources and actions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup" + ], + "Resource": "*" + } + ] +} +``` + +- `DescribeTags` is required for autodiscovery. +- `DescribeLaunchConfigurations` is required to scale up an ASG from 0. + +If you would like to limit the scope of the Cluster Autoscaler to ***only*** modify ASGs for a particular cluster, use the following policy instead: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeLaunchTemplateVersions" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:UpdateAutoScalingGroup" + ], + "Resource": [ + "arn:aws:autoscaling:::autoScalingGroup::autoScalingGroupName/node-group-1", + "arn:aws:autoscaling:::autoScalingGroup::autoScalingGroupName/node-group-2", + "arn:aws:autoscaling:::autoScalingGroup::autoScalingGroupName/node-group-3" + ], + "Condition": { + "StringEquals": { + "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled": "true", + "autoscaling:ResourceTag/kubernetes.io/cluster/": "owned" + } + } + } + ] +} +``` + +Make sure to replace the variables ``, ``, ``, and the ARNs of the ASGs where applicable. + +### AWS - IAM Roles for Service Accounts (IRSA) + +For Kubernetes clusters that use Amazon EKS, the service account can be configured with an IAM role using [IAM Roles for Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) to avoid needing to grant access to the worker nodes for AWS resources. + +In order to accomplish this, you will first need to create a new IAM role with the above mentions policies. Take care in [configuring the trust relationship](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html#iam-role-configuration) to restrict access just to the service account used by cluster autoscaler. + +Once you have the IAM role configured, you would then need to `--set rbac.serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=arn:aws:iam::123456789012:role/MyRoleName` when installing. + +## Troubleshooting + +The chart will succeed even if the container arguments are incorrect. A few minutes after starting +`kubectl logs -l "app=aws-cluster-autoscaler" --tail=50` should loop through something like + +``` +polling_autoscaler.go:111] Poll finished +static_autoscaler.go:97] Starting main loop +utils.go:435] No pod using affinity / antiaffinity found in cluster, disabling affinity predicate for this loop +static_autoscaler.go:230] Filtering out schedulables +``` + +If not, find a pod that the deployment created and `describe` it, paying close attention to the arguments under `Command`. e.g.: + +``` +Containers: + cluster-autoscaler: + Command: + ./cluster-autoscaler + --cloud-provider=aws +# if specifying ASGs manually + --nodes=1:10:your-scaling-group-name +# if using autodiscovery + --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/ + --v=4 +``` + +### PodSecurityPolicy + +Though enough for the majority of installations, the default PodSecurityPolicy _could_ be too restrictive depending on the specifics of your release. Please make sure to check that the template fits with any customizations made or disable it by setting `rbac.pspEnabled` to `false`. + +{{ template "chart.valuesSection" . }} diff --git a/lib/common/bootstrap/charts/cluster-autoscaler/templates/_helpers.tpl b/lib/common/bootstrap/charts/cluster-autoscaler/templates/_helpers.tpl new file mode 100644 index 00000000..07230593 --- /dev/null +++ b/lib/common/bootstrap/charts/cluster-autoscaler/templates/_helpers.tpl @@ -0,0 +1,87 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "cluster-autoscaler.name" -}} +{{- default (printf "%s-%s" .Values.cloudProvider .Chart.Name) .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "cluster-autoscaler.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default (printf "%s-%s" .Values.cloudProvider .Chart.Name) .Values.nameOverride -}} +{{- if ne $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cluster-autoscaler.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Return instance and name labels. +*/}} +{{- define "cluster-autoscaler.instance-name" -}} +app.kubernetes.io/instance: {{ .Release.Name | quote }} +app.kubernetes.io/name: {{ include "cluster-autoscaler.name" . | quote }} +{{- end -}} + + +{{/* +Return labels, including instance and name. +*/}} +{{- define "cluster-autoscaler.labels" -}} +{{ include "cluster-autoscaler.instance-name" . }} +app.kubernetes.io/managed-by: {{ .Release.Service | quote }} +helm.sh/chart: {{ include "cluster-autoscaler.chart" . | quote }} +{{- if .Values.additionalLabels }} +{{ toYaml .Values.additionalLabels }} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "deployment.apiVersion" -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "<1.9-0" $kubeTargetVersion -}} +{{- print "apps/v1beta2" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for podsecuritypolicy. +*/}} +{{- define "podsecuritypolicy.apiVersion" -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "<1.10-0" $kubeTargetVersion -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "policy/v1beta1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the service account name used by the pod. +*/}} +{{- define "cluster-autoscaler.serviceAccountName" -}} +{{- if .Values.rbac.serviceAccount.create -}} + {{ default (include "cluster-autoscaler.fullname" .) .Values.rbac.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.rbac.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/lib/common/bootstrap/charts/cluster-autoscaler/templates/clusterrole.yaml b/lib/common/bootstrap/charts/cluster-autoscaler/templates/clusterrole.yaml new file mode 100644 index 00000000..2ebadc3b --- /dev/null +++ b/lib/common/bootstrap/charts/cluster-autoscaler/templates/clusterrole.yaml @@ -0,0 +1,147 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + name: {{ template "cluster-autoscaler.fullname" . }} +rules: + - apiGroups: + - "" + resources: + - events + - endpoints + verbs: + - create + - patch + - apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create + - apiGroups: + - "" + resources: + - pods/status + verbs: + - update + - apiGroups: + - "" + resources: + - endpoints + resourceNames: + - cluster-autoscaler + verbs: + - get + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - watch + - list + - get + - update + - apiGroups: + - "" + resources: + - pods + - services + - replicationcontrollers + - persistentvolumeclaims + - persistentvolumes + verbs: + - watch + - list + - get + - apiGroups: + - batch + resources: + - jobs + - cronjobs + verbs: + - watch + - list + - get + - apiGroups: + - batch + - extensions + resources: + - jobs + verbs: + - get + - list + - patch + - watch + - apiGroups: + - extensions + resources: + - replicasets + - daemonsets + verbs: + - watch + - list + - get + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - watch + - list + - apiGroups: + - apps + resources: + - daemonsets + - replicasets + - statefulsets + verbs: + - watch + - list + - get + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - watch + - list + - get + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - apiGroups: + - coordination.k8s.io + resourceNames: + - cluster-autoscaler + resources: + - leases + verbs: + - get + - update +{{- if .Values.rbac.pspEnabled }} + - apiGroups: + - extensions + - policy + resources: + - podsecuritypolicies + resourceNames: + - {{ template "cluster-autoscaler.fullname" . }} + verbs: + - use +{{- end -}} + +{{- end -}} diff --git a/lib/common/bootstrap/charts/cluster-autoscaler/templates/clusterrolebinding.yaml b/lib/common/bootstrap/charts/cluster-autoscaler/templates/clusterrolebinding.yaml new file mode 100644 index 00000000..d1e8308a --- /dev/null +++ b/lib/common/bootstrap/charts/cluster-autoscaler/templates/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + name: {{ template "cluster-autoscaler.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "cluster-autoscaler.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "cluster-autoscaler.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/lib/common/bootstrap/charts/cluster-autoscaler/templates/deployment.yaml b/lib/common/bootstrap/charts/cluster-autoscaler/templates/deployment.yaml new file mode 100644 index 00000000..9d976be2 --- /dev/null +++ b/lib/common/bootstrap/charts/cluster-autoscaler/templates/deployment.yaml @@ -0,0 +1,246 @@ +{{- if or .Values.autoDiscovery.clusterName .Values.autoscalingGroups }} +{{/* one of the above is required */}} +apiVersion: {{ template "deployment.apiVersion" . }} +kind: Deployment +metadata: + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + name: {{ template "cluster-autoscaler.fullname" . }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: +{{ include "cluster-autoscaler.instance-name" . | indent 6 }} + {{- if .Values.podLabels }} +{{ toYaml .Values.podLabels | indent 6 }} + {{- end }} +{{- if .Values.updateStrategy }} + strategy: + {{ toYaml .Values.updateStrategy | nindent 4 | trim }} +{{- end }} + template: + metadata: + {{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} + {{- end }} + labels: +{{ include "cluster-autoscaler.instance-name" . | indent 8 }} + {{- if .Values.additionalLabels }} +{{ toYaml .values.additionalLabels | indent 8 }} + {{- end }} + {{- if .Values.podLabels }} +{{ toYaml .Values.podLabels | indent 8 }} + {{- end }} + spec: + {{- if .Values.priorityClassName }} + priorityClassName: "{{ .Values.priorityClassName }}" + {{- end }} + {{- if .Values.dnsPolicy }} + dnsPolicy: "{{ .Values.dnsPolicy }}" + {{- end }} + containers: + - name: {{ template "cluster-autoscaler.name" . }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + command: + - ./cluster-autoscaler + - --cloud-provider={{ .Values.cloudProvider }} + - --namespace={{ .Release.Namespace }} + {{- if .Values.autoscalingGroups }} + {{- range .Values.autoscalingGroups }} + - --nodes={{ .minSize }}:{{ .maxSize }}:{{ .name }} + {{- end }} + {{- end }} + {{- if eq .Values.cloudProvider "aws" }} + {{- if .Values.autoDiscovery.clusterName }} + - --node-group-auto-discovery=asg:tag={{ tpl (join "," .Values.autoDiscovery.tags) . }} + {{- end }} + {{- else if eq .Values.cloudProvider "gce" }} + {{- if .Values.autoscalingGroupsnamePrefix }} + {{- range .Values.autoscalingGroupsnamePrefix }} + - --node-group-auto-discovery=mig:namePrefix={{ .name }},min={{ .minSize }},max={{ .maxSize }} + {{- end }} + {{- end }} + {{- else if eq .Values.cloudProvider "magnum" }} + {{- if .Values.autoDiscovery.clusterName }} + - --cluster-name={{ .Values.autoDiscovery.clusterName }} + - --node-group-auto-discovery=magnum:role={{ tpl (join "," .Values.autoDiscovery.roles) . }} + {{- else }} + - --cluster-name={{ .Values.magnumClusterName }} + {{- end }} + {{- end }} + {{- if or (eq .Values.cloudProvider "gce") (eq .Values.cloudProvider "magnum") }} + - --cloud-config={{ .Values.cloudConfigPath }} + {{- end }} + {{- range $key, $value := .Values.extraArgs }} + - --{{ $key }}={{ $value }} + {{- end }} + env: + {{- if and (eq .Values.cloudProvider "aws") (ne .Values.awsRegion "") }} + - name: AWS_REGION + value: "{{ .Values.awsRegion }}" + {{- if .Values.awsAccessKeyID }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: AwsAccessKeyId + name: {{ template "cluster-autoscaler.fullname" . }} + {{- end }} + {{- if .Values.awsSecretAccessKey }} + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: AwsSecretAccessKey + name: {{ template "cluster-autoscaler.fullname" . }} + {{- end }} + {{- else if eq .Values.cloudProvider "azure" }} + - name: ARM_SUBSCRIPTION_ID + valueFrom: + secretKeyRef: + key: SubscriptionID + name: {{ template "cluster-autoscaler.fullname" . }} + - name: ARM_RESOURCE_GROUP + valueFrom: + secretKeyRef: + key: ResourceGroup + name: {{ template "cluster-autoscaler.fullname" . }} + - name: ARM_VM_TYPE + valueFrom: + secretKeyRef: + key: VMType + name: {{ template "cluster-autoscaler.fullname" . }} + {{- if .Values.azureUseManagedIdentityExtension }} + - name: ARM_USE_MANAGED_IDENTITY_EXTENSION + value: "true" + {{- else }} + - name: ARM_TENANT_ID + valueFrom: + secretKeyRef: + key: TenantID + name: {{ template "cluster-autoscaler.fullname" . }} + - name: ARM_CLIENT_ID + valueFrom: + secretKeyRef: + key: ClientID + name: {{ template "cluster-autoscaler.fullname" . }} + - name: ARM_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: ClientSecret + name: {{ template "cluster-autoscaler.fullname" . }} + - name: AZURE_CLUSTER_NAME + valueFrom: + secretKeyRef: + key: ClusterName + name: {{ template "cluster-autoscaler.fullname" . }} + - name: AZURE_NODE_RESOURCE_GROUP + valueFrom: + secretKeyRef: + key: NodeResourceGroup + name: {{ template "cluster-autoscaler.fullname" . }} + {{- end }} + {{- end }} + {{- range $key, $value := .Values.extraEnv }} + - name: {{ $key }} + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.extraEnvConfigMaps }} + - name: {{ $key }} + valueFrom: + configMapKeyRef: + name: {{ default (include "cluster-autoscaler.fullname" $) $value.name }} + key: {{ required "Must specify key!" $value.key }} + {{- end }} + {{- range $key, $value := .Values.extraEnvSecrets }} + - name: {{ $key }} + valueFrom: + secretKeyRef: + name: {{ default (include "cluster-autoscaler.fullname" $) $value.name }} + key: {{ required "Must specify key!" $value.key }} + {{- end }} + {{- if or .Values.envFromSecret .Values.envFromConfigMap }} + envFrom: + {{- if .Values.envFromSecret }} + - secretRef: + name: {{ .Values.envFromSecret }} + {{- end }} + {{- if .Values.envFromConfigMap }} + - configMapRef: + name: {{ .Values.envFromConfigMap }} + {{- end }} + {{- end }} + livenessProbe: + httpGet: + path: /health-check + port: 8085 + ports: + - containerPort: 8085 + resources: +{{ toYaml .Values.resources | indent 12 }} + {{- if .Values.containerSecurityContext }} + securityContext: + {{ toYaml .Values.containerSecurityContext | nindent 12 | trim }} + {{- end }} + {{- if or (eq .Values.cloudProvider "gce") (eq .Values.cloudProvider "magnum") .Values.extraVolumeSecrets }} + volumeMounts: + {{- if or (eq .Values.cloudProvider "gce") (eq .Values.cloudProvider "magnum") }} + - name: cloudconfig + mountPath: {{ .Values.cloudConfigPath }} + readOnly: true + {{- end }} + {{- if and (eq .Values.cloudProvider "magnum") (.Values.magnumCABundlePath) }} + - name: ca-bundle + mountPath: {{ .Values.magnumCABundlePath }} + readOnly: true + {{- end }} + {{- range $key, $value := .Values.extraVolumeSecrets }} + - name: {{ $key }} + mountPath: {{ required "Must specify mountPath!" $value.mountPath }} + readOnly: true + {{- end }} + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + serviceAccountName: {{ template "cluster-autoscaler.serviceAccountName" . }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + {{- if .Values.securityContext }} + securityContext: + {{ toYaml .Values.securityContext | nindent 8 | trim }} + {{- end }} + {{- if or (eq .Values.cloudProvider "gce") (eq .Values.cloudProvider "magnum") .Values.extraVolumeSecrets }} + volumes: + {{- if or (eq .Values.cloudProvider "gce") (eq .Values.cloudProvider "magnum") }} + - name: cloudconfig + hostPath: + path: {{ .Values.cloudConfigPath }} + {{- end }} + {{- if and (eq .Values.cloudProvider "magnum") (.Values.magnumCABundlePath) }} + - name: ca-bundle + hostPath: + path: {{ .Values.magnumCABundlePath }} + {{- end }} + {{- range $key, $value := .Values.extraVolumeSecrets }} + - name: {{ $key }} + secret: + secretName: {{ default (include "cluster-autoscaler.fullname" $) $value.name }} + {{- if $value.items }} + items: + {{- toYaml $value.items | nindent 14 }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end }} diff --git a/lib/common/bootstrap/charts/cluster-autoscaler/templates/pdb.yaml b/lib/common/bootstrap/charts/cluster-autoscaler/templates/pdb.yaml new file mode 100644 index 00000000..da5bd564 --- /dev/null +++ b/lib/common/bootstrap/charts/cluster-autoscaler/templates/pdb.yaml @@ -0,0 +1,15 @@ +{{- if .Values.podDisruptionBudget -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + name: {{ template "cluster-autoscaler.fullname" . }} +spec: + selector: + matchLabels: +{{ include "cluster-autoscaler.instance-name" . | indent 6 }} +{{- if .Values.podDisruptionBudget }} + {{ toYaml .Values.podDisruptionBudget | nindent 2 }} +{{- end }} +{{- end -}} diff --git a/lib/common/bootstrap/charts/cluster-autoscaler/templates/podsecuritypolicy.yaml b/lib/common/bootstrap/charts/cluster-autoscaler/templates/podsecuritypolicy.yaml new file mode 100644 index 00000000..28369bf9 --- /dev/null +++ b/lib/common/bootstrap/charts/cluster-autoscaler/templates/podsecuritypolicy.yaml @@ -0,0 +1,46 @@ +{{- if .Values.rbac.pspEnabled }} +apiVersion: {{ template "podsecuritypolicy.apiVersion" . }} +kind: PodSecurityPolicy +metadata: + name: {{ template "cluster-autoscaler.fullname" . }} + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} +spec: + # Prevents running in privileged mode + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + volumes: + - 'configMap' + - 'secret' + - 'hostPath' + - 'emptyDir' + - 'projected' + - 'downwardAPI' +{{- if eq .Values.cloudProvider "gce" }} + allowedHostPaths: + - pathPrefix: {{ .Values.cloudConfigPath }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/lib/common/bootstrap/charts/cluster-autoscaler/templates/priority-expander-configmap.yaml b/lib/common/bootstrap/charts/cluster-autoscaler/templates/priority-expander-configmap.yaml new file mode 100644 index 00000000..08655d0b --- /dev/null +++ b/lib/common/bootstrap/charts/cluster-autoscaler/templates/priority-expander-configmap.yaml @@ -0,0 +1,13 @@ +{{- if hasKey .Values.extraArgs "expander" }} +{{- if and (.Values.expanderPriorities) (eq .Values.extraArgs.expander "priority") -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-autoscaler-priority-expander + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} +data: + priorities: |- +{{ .Values.expanderPriorities | indent 4 }} +{{- end -}} +{{- end -}} diff --git a/lib/common/bootstrap/charts/cluster-autoscaler/templates/role.yaml b/lib/common/bootstrap/charts/cluster-autoscaler/templates/role.yaml new file mode 100644 index 00000000..e6af27d5 --- /dev/null +++ b/lib/common/bootstrap/charts/cluster-autoscaler/templates/role.yaml @@ -0,0 +1,25 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + name: {{ template "cluster-autoscaler.fullname" . }} +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - cluster-autoscaler-status + verbs: + - delete + - get + - update +{{- end -}} diff --git a/lib/common/bootstrap/charts/cluster-autoscaler/templates/rolebinding.yaml b/lib/common/bootstrap/charts/cluster-autoscaler/templates/rolebinding.yaml new file mode 100644 index 00000000..938bc03a --- /dev/null +++ b/lib/common/bootstrap/charts/cluster-autoscaler/templates/rolebinding.yaml @@ -0,0 +1,16 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + name: {{ template "cluster-autoscaler.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "cluster-autoscaler.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "cluster-autoscaler.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/lib/common/bootstrap/charts/cluster-autoscaler/templates/secret.yaml b/lib/common/bootstrap/charts/cluster-autoscaler/templates/secret.yaml new file mode 100644 index 00000000..3f0ef09e --- /dev/null +++ b/lib/common/bootstrap/charts/cluster-autoscaler/templates/secret.yaml @@ -0,0 +1,20 @@ +{{- if or (eq .Values.cloudProvider "azure") (and (eq .Values.cloudProvider "aws") (not (has "" (list .Values.awsAccessKeyID .Values.awsSecretAccessKey)))) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "cluster-autoscaler.fullname" . }} +data: +{{- if eq .Values.cloudProvider "azure" }} + ClientID: "{{ .Values.azureClientID | b64enc }}" + ClientSecret: "{{ .Values.azureClientSecret | b64enc }}" + ResourceGroup: "{{ .Values.azureResourceGroup | b64enc }}" + SubscriptionID: "{{ .Values.azureSubscriptionID | b64enc }}" + TenantID: "{{ .Values.azureTenantID | b64enc }}" + VMType: "{{ .Values.azureVMType | b64enc }}" + ClusterName: "{{ .Values.azureClusterName | b64enc }}" + NodeResourceGroup: "{{ .Values.azureNodeResourceGroup | b64enc }}" +{{- else if eq .Values.cloudProvider "aws" }} + AwsAccessKeyId: "{{ .Values.awsAccessKeyID | b64enc }}" + AwsSecretAccessKey: "{{ .Values.awsSecretAccessKey | b64enc }}" +{{- end }} +{{- end }} diff --git a/lib/common/bootstrap/charts/cluster-autoscaler/templates/service.yaml b/lib/common/bootstrap/charts/cluster-autoscaler/templates/service.yaml new file mode 100644 index 00000000..dd8903d9 --- /dev/null +++ b/lib/common/bootstrap/charts/cluster-autoscaler/templates/service.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.service.annotations }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +{{- end }} + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} +{{- if .Values.service.labels }} +{{ toYaml .Values.service.labels | indent 4 }} +{{- end }} + name: {{ template "cluster-autoscaler.fullname" . }} +spec: +{{- if .Values.service.clusterIP }} + clusterIP: "{{ .Values.service.clusterIP }}" +{{- end }} +{{- if .Values.service.externalIPs }} + externalIPs: +{{ toYaml .Values.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" +{{- end }} +{{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml .Values.service.loadBalancerSourceRanges | indent 4 }} +{{- end }} + ports: + - port: {{ .Values.service.servicePort }} + protocol: TCP + targetPort: 8085 + name: {{ .Values.service.portName }} + selector: +{{ include "cluster-autoscaler.instance-name" . | indent 4 }} + type: "{{ .Values.service.type }}" diff --git a/lib/common/bootstrap/charts/cluster-autoscaler/templates/serviceaccount.yaml b/lib/common/bootstrap/charts/cluster-autoscaler/templates/serviceaccount.yaml new file mode 100644 index 00000000..088b858b --- /dev/null +++ b/lib/common/bootstrap/charts/cluster-autoscaler/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.rbac.create .Values.rbac.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + name: {{ template "cluster-autoscaler.serviceAccountName" . }} +{{- if .Values.rbac.serviceAccount.annotations }} + annotations: {{ toYaml .Values.rbac.serviceAccount.annotations | nindent 4 }} +{{- end }} +{{- end }} diff --git a/lib/common/bootstrap/charts/cluster-autoscaler/templates/servicemonitor.yaml b/lib/common/bootstrap/charts/cluster-autoscaler/templates/servicemonitor.yaml new file mode 100644 index 00000000..a0b9d25d --- /dev/null +++ b/lib/common/bootstrap/charts/cluster-autoscaler/templates/servicemonitor.yaml @@ -0,0 +1,24 @@ +{{ if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "cluster-autoscaler.fullname" . }} + {{- if .Values.serviceMonitor.namespace }} + namespace: {{ .Values.serviceMonitor.namespace }} + {{- end }} + labels: + {{- range $key, $value := .Values.serviceMonitor.selector }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + selector: + matchLabels: +{{ include "cluster-autoscaler.instance-name" . | indent 6 }} + endpoints: + - port: {{ .Values.service.portName }} + interval: {{ .Values.serviceMonitor.interval }} + path: {{ .Values.serviceMonitor.path }} + namespaceSelector: + matchNames: + - {{.Release.Namespace}} +{{ end }} diff --git a/lib/common/bootstrap/charts/cluster-autoscaler/values.yaml b/lib/common/bootstrap/charts/cluster-autoscaler/values.yaml new file mode 100644 index 00000000..92fee784 --- /dev/null +++ b/lib/common/bootstrap/charts/cluster-autoscaler/values.yaml @@ -0,0 +1,295 @@ +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +# affinity -- Affinity for pod assignment +affinity: {} + +autoDiscovery: + # cloudProviders `aws`, `gce` and `magnum` are supported by auto-discovery at this time + # AWS: Set tags as described in https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup + + # autoDiscovery.clusterName -- Enable autodiscovery for `cloudProvider=aws`, for groups matching `autoDiscovery.tags`. + # Enable autodiscovery for `cloudProvider=gce`, but no MIG tagging required. + # Enable autodiscovery for `cloudProvider=magnum`, for groups matching `autoDiscovery.roles`. + clusterName: # cluster.local + + # autoDiscovery.tags -- ASG tags to match, run through `tpl`. + tags: + - k8s.io/cluster-autoscaler/enabled + - k8s.io/cluster-autoscaler/{{ .Values.autoDiscovery.clusterName }} + # - kubernetes.io/cluster/{{ .Values.autoDiscovery.clusterName }} + + # autoDiscovery.roles -- Magnum node group roles to match. + roles: + - worker + +# autoscalingGroups -- For AWS, Azure AKS or Magnum. At least one element is required if not using `autoDiscovery`. For example: +# 
+# - name: asg1

+#   maxSize: 2

+#   minSize: 1
+#
+autoscalingGroups: [] +# - name: asg1 +# maxSize: 2 +# minSize: 1 +# - name: asg2 +# maxSize: 2 +# minSize: 1 + +# autoscalingGroupsnamePrefix -- For GCE. At least one element is required if not using `autoDiscovery`. For example: +# 
+# - name: ig01

+#   maxSize: 10

+#   minSize: 0
+#
+autoscalingGroupsnamePrefix: [] +# - name: ig01 +# maxSize: 10 +# minSize: 0 +# - name: ig02 +# maxSize: 10 +# minSize: 0 + +# awsAccessKeyID -- AWS access key ID ([if AWS user keys used](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials)) +awsAccessKeyID: "" + +# awsRegion -- AWS region (required if `cloudProvider=aws`) +awsRegion: us-east-1 + +# awsSecretAccessKey -- AWS access secret key ([if AWS user keys used](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials)) +awsSecretAccessKey: "" + +# azureClientID -- Service Principal ClientID with contributor permission to Cluster and Node ResourceGroup. +# Required if `cloudProvider=azure` +azureClientID: "" + +# azureClientSecret -- Service Principal ClientSecret with contributor permission to Cluster and Node ResourceGroup. +# Required if `cloudProvider=azure` +azureClientSecret: "" + +# azureResourceGroup -- Azure resource group that the cluster is located. +# Required if `cloudProvider=azure` +azureResourceGroup: "" + +# azureSubscriptionID -- Azure subscription where the resources are located. +# Required if `cloudProvider=azure` +azureSubscriptionID: "" + +# azureTenantID -- Azure tenant where the resources are located. +# Required if `cloudProvider=azure` +azureTenantID: "" + +# azureVMType -- Azure VM type. +azureVMType: "AKS" + +# azureClusterName -- Azure AKS cluster name. +# Required if `cloudProvider=azure` +azureClusterName: "" + +# azureNodeResourceGroup -- Azure resource group where the cluster's nodes are located, typically set as `MC___`. +# Required if `cloudProvider=azure` +azureNodeResourceGroup: "" + +# azureUseManagedIdentityExtension -- Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID and resource group are set. +azureUseManagedIdentityExtension: false + +# magnumClusterName -- Cluster name or ID in Magnum. +# Required if `cloudProvider=magnum` and not setting `autoDiscovery.clusterName`. +magnumClusterName: "" + +# magnumCABundlePath -- Path to the host's CA bundle, from `ca-file` in the cloud-config file. +magnumCABundlePath: "/etc/kubernetes/ca-bundle.crt" + +# cloudConfigPath -- Configuration file for cloud provider. +cloudConfigPath: /etc/gce.conf + +# cloudProvider -- The cloud provider where the autoscaler runs. +# Currently only `gce`, `aws`, `azure` and `magnum` are supported. +# `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS. +# `magnum` for OpenStack Magnum. +cloudProvider: aws + +# containerSecurityContext -- [Security context for container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) +containerSecurityContext: {} + # capabilities: + # drop: + # - ALL + +# dnsPolicy -- Defaults to `ClusterFirst`. Valid values are: +# `ClusterFirstWithHostNet`, `ClusterFirst`, `Default` or `None`. +# If autoscaler does not depend on cluster DNS, recommended to set this to `Default`. +dnsPolicy: ClusterFirst + +## Priorities Expander +# expanderPriorities -- The expanderPriorities is used if `extraArgs.expander` is set to `priority` and expanderPriorities is also set with the priorities. +# If `extraArgs.expander` is set to `priority`, then expanderPriorities is used to define cluster-autoscaler-priority-expander priorities. +# See: https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/expander/priority/readme.md +expanderPriorities: {} + +# extraArgs -- Additional container arguments. +extraArgs: + logtostderr: true + stderrthreshold: info + v: 4 + # write-status-configmap: true + # leader-elect: true + # skip-nodes-with-local-storage: false + # expander: least-waste + # scale-down-enabled: true + # balance-similar-node-groups: true + # min-replica-count: 2 + # scale-down-utilization-threshold: 0.5 + # scale-down-non-empty-candidates-count: 5 + # max-node-provision-time: 15m0s + # scan-interval: 10s + # scale-down-delay-after-add: 10m + # scale-down-delay-after-delete: 0s + # scale-down-delay-after-failure: 3m + # scale-down-unneeded-time: 10m + # skip-nodes-with-system-pods: true + +# extraEnv -- Additional container environment variables. +extraEnv: {} + +# extraEnvConfigMaps -- Additional container environment variables from ConfigMaps. +extraEnvConfigMaps: {} + +# extraEnvSecrets -- Additional container environment variables from Secrets. +extraEnvSecrets: {} + +# envFromConfigMap -- ConfigMap name to use as envFrom. +envFromConfigMap: "" + +# envFromSecret -- Secret name to use as envFrom. +envFromSecret: "" + +# extraVolumeSecrets -- Additional volumes to mount from Secrets. +extraVolumeSecrets: {} + # autoscaler-vol: + # mountPath: /data/autoscaler/ + # custom-vol: + # name: custom-secret + # mountPath: /data/custom/ + # items: + # - key: subkey + # path: mypath + +# fullnameOverride -- String to fully override `cluster-autoscaler.fullname` template. +fullnameOverride: "" + +image: + # image.repository -- Image repository + repository: us.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler + # image.tag -- Image tag + tag: v1.18.1 + # image.pullPolicy -- Image pull policy + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # image.pullSecrets -- Image pull secrets + pullSecrets: [] + # - myRegistrKeySecretName + +# kubeTargetVersionOverride -- Allow overriding the `.Capabilities.KubeVersion.GitVersion` check. Useful for `helm template` commands. +kubeTargetVersionOverride: "" + +# nameOverride -- String to partially override `cluster-autoscaler.fullname` template (will maintain the release name) +nameOverride: "" + +# nodeSelector -- Node labels for pod assignment. Ref: https://kubernetes.io/docs/user-guide/node-selection/. +nodeSelector: {} + +# podAnnotations -- Annotations to add to each pod. +podAnnotations: {} + +# podDisruptionBudget -- Pod disruption budget. +podDisruptionBudget: + maxUnavailable: 1 + # minAvailable: 2 + +# podLabels -- Labels to add to each pod. +podLabels: {} + +# additionalLabels -- Labels to add to each object of the chart. +additionalLabels: {} + +# priorityClassName -- priorityClassName +priorityClassName: "" + +rbac: + # rbac.create -- If `true`, create and use RBAC resources. + create: true + # rbac.pspEnabled -- If `true`, creates and uses RBAC resources required in the cluster with [Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) enabled. + # Must be used with `rbac.create` set to `true`. + pspEnabled: false + serviceAccount: + # rbac.serviceAccount.annotations -- Additional Service Account annotations. + annotations: {} + # rbac.serviceAccount.create -- If `true` and `rbac.create` is also true, a Service Account will be created. + create: true + # rbac.serviceAccount.name -- The name of the ServiceAccount to use. If not set and create is `true`, a name is generated using the fullname template. + name: "" + +# replicaCount -- Desired number of pods +replicaCount: 1 + +# resources -- Pod resource requests and limits. +resources: {} + # limits: + # cpu: 100m + # memory: 300Mi + # requests: + # cpu: 100m + # memory: 300Mi + +# securityContext -- [Security context for pod](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) +securityContext: {} + # runAsNonRoot: true + # runAsUser: 1001 + # runAsGroup: 1001 + +service: + # service.annotations -- Annotations to add to service + annotations: {} + # service.labels -- Labels to add to service + labels: {} + # service.externalIPs -- List of IP addresses at which the service is available. Ref: https://kubernetes.io/docs/user-guide/services/#external-ips. + externalIPs: [] + + # service.loadBalancerIP -- IP address to assign to load balancer (if supported). + loadBalancerIP: "" + # service.loadBalancerSourceRanges -- List of IP CIDRs allowed access to load balancer (if supported). + loadBalancerSourceRanges: [] + # service.servicePort -- Service port to expose. + servicePort: 8085 + # service.portName -- Name for service port. + portName: http + # service.type -- Type of service to create. + type: ClusterIP + +## Are you using Prometheus Operator? +serviceMonitor: + # serviceMonitor.enabled -- If true, creates a Prometheus Operator ServiceMonitor. + enabled: false + # serviceMonitor.interval -- Interval that Prometheus scrapes Cluster Autoscaler metrics. + interval: 10s + # serviceMonitor.namespace -- Namespace which Prometheus is running in. + namespace: monitoring + ## [Prometheus Selector Label](https://github.com/helm/charts/tree/master/stable/prometheus-operator#prometheus-operator-1) + ## [Kube Prometheus Selector Label](https://github.com/helm/charts/tree/master/stable/prometheus-operator#exporters) + # serviceMonitor.selector -- Default to kube-prometheus install (CoreOS recommended), but should be set according to Prometheus install. + selector: + release: prometheus-operator + # serviceMonitor.path -- The path to scrape for metrics; autoscaler exposes `/metrics` (this is standard) + path: /metrics + +# tolerations -- List of node taints to tolerate (requires Kubernetes >= 1.6). +tolerations: [] + +# updateStrategy -- [Deployment update strategy](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy) +updateStrategy: {} + # rollingUpdate: + # maxSurge: 1 + # maxUnavailable: 0 + # type: RollingUpdate diff --git a/lib/helm-freeze.yaml b/lib/helm-freeze.yaml index c66a2a59..039065b4 100644 --- a/lib/helm-freeze.yaml +++ b/lib/helm-freeze.yaml @@ -27,6 +27,9 @@ charts: repo_name: aws dest: aws-bootstrap version: 0.3.1 + - name: cluster-autoscaler + repo_name: cluster-autoscaler + version: 9.3.0 - name: metrics-server repo_name: bitnami version: 4.3.1 @@ -77,6 +80,8 @@ repos: url: https://grafana.github.io/loki/charts - name: pleco url: https://qovery.github.io/pleco/ + - name: cluster-autoscaler + url: https://kubernetes.github.io/autoscaler - name: do-k8s-token-rotate url: https://qovery.github.io/do-k8s-token-rotate/ diff --git a/src/cloud_provider/aws/databases/redis.rs b/src/cloud_provider/aws/databases/redis.rs index b476ef68..1e8ef5b1 100644 --- a/src/cloud_provider/aws/databases/redis.rs +++ b/src/cloud_provider/aws/databases/redis.rs @@ -421,4 +421,4 @@ mod tests { "Redis 1.0 version is not supported" ); } -} +} \ No newline at end of file diff --git a/src/cloud_provider/aws/kubernetes/mod.rs b/src/cloud_provider/aws/kubernetes/mod.rs index 22c149af..656d1441 100644 --- a/src/cloud_provider/aws/kubernetes/mod.rs +++ b/src/cloud_provider/aws/kubernetes/mod.rs @@ -180,9 +180,9 @@ impl<'a> EKS<'a> { .map(|(instance_type, group)| (instance_type, group.collect::>())) .map(|(instance_type, nodes)| WorkerNodeDataTemplate { instance_type: instance_type.to_string(), - desired_size: nodes.len().to_string(), + desired_size: "1".to_string(), max_size: nodes.len().to_string(), - min_size: nodes.len().to_string(), + min_size: "1".to_string(), }) .collect::>(); diff --git a/src/cloud_provider/digitalocean/kubernetes/mod.rs b/src/cloud_provider/digitalocean/kubernetes/mod.rs index 9e2403c3..f0738f5f 100644 --- a/src/cloud_provider/digitalocean/kubernetes/mod.rs +++ b/src/cloud_provider/digitalocean/kubernetes/mod.rs @@ -246,9 +246,9 @@ impl<'a> DOKS<'a> { .map(|(instance_type, group)| (instance_type, group.collect::>())) .map(|(instance_type, nodes)| WorkerNodeDataTemplate { instance_type: instance_type.to_string(), - desired_size: nodes.len().to_string(), + desired_size: "1".to_string(), max_size: nodes.len().to_string(), - min_size: nodes.len().to_string(), + min_size: "1".to_string(), }) .collect::>(); diff --git a/test_utilities/src/aws.rs b/test_utilities/src/aws.rs index 82eda0dd..1361e7c3 100644 --- a/test_utilities/src/aws.rs +++ b/test_utilities/src/aws.rs @@ -79,10 +79,16 @@ pub fn container_registry_docker_hub(context: &Context) -> DockerHub { pub fn aws_kubernetes_nodes() -> Vec { vec![ - Node::new_with_cpu_and_mem(2, 16), - Node::new_with_cpu_and_mem(2, 16), - Node::new_with_cpu_and_mem(2, 16), - Node::new_with_cpu_and_mem(2, 16), + Node::new_with_cpu_and_mem(2, 8), + Node::new_with_cpu_and_mem(2, 8), + Node::new_with_cpu_and_mem(2, 8), + Node::new_with_cpu_and_mem(2, 8), + Node::new_with_cpu_and_mem(2, 8), + Node::new_with_cpu_and_mem(2, 8), + Node::new_with_cpu_and_mem(2, 8), + Node::new_with_cpu_and_mem(2, 8), + Node::new_with_cpu_and_mem(2, 8), + Node::new_with_cpu_and_mem(2, 8), ] } diff --git a/test_utilities/src/digitalocean.rs b/test_utilities/src/digitalocean.rs index 0e2d18d4..6fe9565a 100644 --- a/test_utilities/src/digitalocean.rs +++ b/test_utilities/src/digitalocean.rs @@ -90,6 +90,11 @@ pub fn do_kubernetes_nodes() -> Vec { Node::new_with_cpu_and_mem(4, 8), Node::new_with_cpu_and_mem(4, 8), Node::new_with_cpu_and_mem(4, 8), + Node::new_with_cpu_and_mem(4, 8), + Node::new_with_cpu_and_mem(4, 8), + Node::new_with_cpu_and_mem(4, 8), + Node::new_with_cpu_and_mem(4, 8), + Node::new_with_cpu_and_mem(4, 8), ] }