From 8cad7552a0aacd0b2304ed3c4a71c1a159834530 Mon Sep 17 00:00:00 2001 From: juan131 Date: Fri, 16 Nov 2018 13:59:22 +0100 Subject: [PATCH] Add Examples to run Cluster AutoScaler on AWS --- bitnami/cluster-autoscaler/README.md | 24 ++-- .../cluster-autoscaler/aws-examples/README.md | 30 +++++ .../cluster-autoscaler-autodiscover.yaml | 45 ++++++++ .../cluster-autoscaler-multi-asg.yaml | 46 ++++++++ .../cluster-autoscaler-one-asg.yaml | 44 ++++++++ .../cluster-autoscaler-run-on-master.yaml | 51 +++++++++ .../aws-examples/rbac-requirements.yaml | 103 ++++++++++++++++++ 7 files changed, 336 insertions(+), 7 deletions(-) create mode 100644 bitnami/cluster-autoscaler/aws-examples/README.md create mode 100644 bitnami/cluster-autoscaler/aws-examples/cluster-autoscaler-autodiscover.yaml create mode 100644 bitnami/cluster-autoscaler/aws-examples/cluster-autoscaler-multi-asg.yaml create mode 100644 bitnami/cluster-autoscaler/aws-examples/cluster-autoscaler-one-asg.yaml create mode 100644 bitnami/cluster-autoscaler/aws-examples/cluster-autoscaler-run-on-master.yaml create mode 100644 bitnami/cluster-autoscaler/aws-examples/rbac-requirements.yaml diff --git a/bitnami/cluster-autoscaler/README.md b/bitnami/cluster-autoscaler/README.md index a61e0b4c6c1d..36b01e21cfe5 100644 --- a/bitnami/cluster-autoscaler/README.md +++ b/bitnami/cluster-autoscaler/README.md @@ -9,16 +9,26 @@ Cluster Autoscaler is a tool that automatically adjusts the size of the Kubernet [https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) -# Deployment +# How to deploy Cluster Autoscaler in Kubernetes? -Cluster Autoscaler runs on the Kubernetes master node (at least in the default setup on GCE and GKE). It is possible to run customized Cluster Autoscaler inside of the cluster but then extra care needs to be taken to ensure that Cluster Autoscaler is up and running. User can put it into kube-system namespace (Cluster Autoscaler doesn't scale down node with non-manifest based kube-system pods running on them) and mark with scheduler.alpha.kubernetes.io/critical-pod annotation (so that the rescheduler, if enabled, will kill other pods to make space for it to run). +Cluster Autoscaler runs on the Kubernetes master node on most K8s cloud offerings. -Right now it is possible to run Cluster Autoscaler on: +> NOTE: It is possible to run customized Cluster Autoscaler inside of the cluster but then extra care needs to be taken to ensure that Cluster Autoscaler is up and running. User can put it into kube-system namespace (Cluster Autoscaler doesn't scale down node with non-manifest based kube-system pods running on them) and mark with scheduler.alpha.kubernetes.io/critical-pod annotation (so that the rescheduler, if enabled, will kill other pods to make space for it to run). -* GCE: https://kubernetes.io/docs/concepts/cluster-administration/cluster-management/ -* GKE: https://cloud.google.com/container-engine/docs/cluster-autoscaler -* AWS: https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md -* Azure: https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/azure/README.md +Currently, it is possible to run Cluster Autoscaler on: + +* **AliCloud**: Consult [Cluster Autoscaler on AliCloud docs](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/alicloud/README.md). +* **AWS**: Consult [Cluster Autoscaler on AWS docs](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md). +* **Azure**: Consult [Cluster Autoscaler on Azure docs](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/azure/README.md). +* **GCE**: Consult [Cluster Autoscaler on GCE docs](https://kubernetes.io/docs/concepts/cluster-administration/cluster-management#upgrading-google-compute-engine-clusters). +* **GKE**: Consult [Cluster Autoscaler on GKE docs](https://cloud.google.com/container-engine/docs/cluster-autoscaler). + +Please note that Cluster Autoscaler a series of permissions/privileges to adjusts the size of the K8s cluster. For instance, to run it on AWS, you need to: + +* Provide the K8s worker node which runs the cluster autoscaler with a minimum IAM policy (check [permissions docs](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/aws#permissions) for more information). +* Create a service account for Cluster Autoscaler's deployment and bind to it some roles and cluster roles that provide the corresponding RBAC privileges. + +> NOTE: Find resources to deploy Cluster Autoscaler on AWS in the [aws-examples](./aws-examples) directory. # Why use Bitnami Images? diff --git a/bitnami/cluster-autoscaler/aws-examples/README.md b/bitnami/cluster-autoscaler/aws-examples/README.md new file mode 100644 index 000000000000..012391e855d3 --- /dev/null +++ b/bitnami/cluster-autoscaler/aws-examples/README.md @@ -0,0 +1,30 @@ +# Deploy Cluster Autoscaler on AWS + +Run the command below to create the RBAC requirements to deploy Cluster Autoscaler on your cluster: + +```bash +kubectl apply -f rbac-requirements.yaml +``` + +The following K8s resources will be created: + +- A **serviceAccount** with name cluster-autoscaler in the `kube-system` namespace. +- A **role** in the `kube-system` namespace. +- A **roleBinding** which binds the serviceAccount created with the corresponding role. +- A **clusterRole**. +- A **clusterRoleBinding** which binds the serviceAccount created with the corresponding clusterRole. + +Once you accomplish RBAC requirements, deploy Cluster Autoscaler on the cluster with one of the specifications below: + +- 1 ASG Setup (use cluster-autoscaler-one-asg.yaml) +- Multiple ASG Setup (use cluster-autoscaler-multi-asg.yaml +- Master Node Setup (use cluster-autoscaler-run-on-master.yaml) +- Auto-Discovery Setup (use cluster-autoscaler-autodiscover.yaml) + +You just need to run the command below: + +```bash +kubectl apply -f DEPLOYMENT-SPECIFICATIONS.yaml +``` + +Find more information about deployments specifications in the [official docs](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/aws#deployment-specification). diff --git a/bitnami/cluster-autoscaler/aws-examples/cluster-autoscaler-autodiscover.yaml b/bitnami/cluster-autoscaler/aws-examples/cluster-autoscaler-autodiscover.yaml new file mode 100644 index 000000000000..65239dad771d --- /dev/null +++ b/bitnami/cluster-autoscaler/aws-examples/cluster-autoscaler-autodiscover.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + spec: + serviceAccountName: cluster-autoscaler + containers: + - image: bitnami/cluster-autoscaler:latest + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./cluster-autoscaler + - --v=4 + - --stderrthreshold=info + - --cloud-provider=aws + - --skip-nodes-with-local-storage=false + - --expander=least-waste + - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/ + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs/ca-certificates.crt + readOnly: true + imagePullPolicy: "Always" + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs/ca-certificates.crt" diff --git a/bitnami/cluster-autoscaler/aws-examples/cluster-autoscaler-multi-asg.yaml b/bitnami/cluster-autoscaler/aws-examples/cluster-autoscaler-multi-asg.yaml new file mode 100644 index 000000000000..b5add22219b9 --- /dev/null +++ b/bitnami/cluster-autoscaler/aws-examples/cluster-autoscaler-multi-asg.yaml @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + spec: + serviceAccountName: cluster-autoscaler + containers: + - image: bitnami/cluster-autoscaler:latest + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./cluster-autoscaler + - --v=4 + - --stderrthreshold=info + - --cloud-provider=aws + - --skip-nodes-with-local-storage=false + - --expander=least-waste + - --nodes=1:10:k8s-worker-asg-1 + - --nodes=1:3:k8s-worker-asg-2 + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs/ca-certificates.crt + readOnly: true + imagePullPolicy: "Always" + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs/ca-certificates.crt" diff --git a/bitnami/cluster-autoscaler/aws-examples/cluster-autoscaler-one-asg.yaml b/bitnami/cluster-autoscaler/aws-examples/cluster-autoscaler-one-asg.yaml new file mode 100644 index 000000000000..e2accb71576f --- /dev/null +++ b/bitnami/cluster-autoscaler/aws-examples/cluster-autoscaler-one-asg.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + spec: + serviceAccountName: cluster-autoscaler + containers: + - image: bitnami/cluster-autoscaler:latest + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./cluster-autoscaler + - --v=4 + - --stderrthreshold=info + - --cloud-provider=aws + - --skip-nodes-with-local-storage=false + - --nodes=1:10:k8s-worker-asg-1 + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs/ca-certificates.crt + readOnly: true + imagePullPolicy: "Always" + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs/ca-certificates.crt" diff --git a/bitnami/cluster-autoscaler/aws-examples/cluster-autoscaler-run-on-master.yaml b/bitnami/cluster-autoscaler/aws-examples/cluster-autoscaler-run-on-master.yaml new file mode 100644 index 000000000000..b5bf670bab63 --- /dev/null +++ b/bitnami/cluster-autoscaler/aws-examples/cluster-autoscaler-run-on-master.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + spec: + serviceAccountName: cluster-autoscaler + tolerations: + - effect: NoSchedule + operator: "Equal" + value: "true" + key: node-role.kubernetes.io/master + nodeSelector: + kubernetes.io/role: master + containers: + - image: bitnami/cluster-autoscaler:latest + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./cluster-autoscaler + - --v=4 + - --stderrthreshold=info + - --cloud-provider=aws + - --skip-nodes-with-local-storage=false + - --nodes={{ node_asg_min }}:{{ node_asg_max }}:{{ name }} + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs/ca-certificates.crt + readOnly: true + imagePullPolicy: "Always" + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs/ca-certificates.crt" diff --git a/bitnami/cluster-autoscaler/aws-examples/rbac-requirements.yaml b/bitnami/cluster-autoscaler/aws-examples/rbac-requirements.yaml new file mode 100644 index 000000000000..9c36f9f213cc --- /dev/null +++ b/bitnami/cluster-autoscaler/aws-examples/rbac-requirements.yaml @@ -0,0 +1,103 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: +- apiGroups: [""] + resources: ["events","endpoints"] + verbs: ["create", "patch"] +- apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] +- apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] +- apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get","update"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["watch","list","get","update"] +- apiGroups: [""] + resources: ["pods","services","replicationcontrollers","persistentvolumeclaims","persistentvolumes"] + verbs: ["watch","list","get"] +- apiGroups: ["extensions"] + resources: ["replicasets","daemonsets"] + verbs: ["watch","list","get"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch","list"] +- apiGroups: ["apps"] + resources: ["statefulsets", "replicasets"] + verbs: ["watch","list","get"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["watch","list","get"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create"] +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-autoscaler-status"] + verbs: ["delete","get","update"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system +