Kubernetes Cloud Autoscalar in Terraform - Part 1
Here's an example of a Kubernetes Cluster that creates the Cloud Autoscaler deployment using Terraform dynamic Modules instead of using a YAML file.
By
·
·
Tutorial
Likes
(3)
Likes
There are no likes...yet! 👀
Be the first to like this post!
It looks like you're not logged in.
Sign in to see who liked this post!
Comment
Save
3.7K Views
Join the DZone community and get the full member experience.
Join For FreeCluster Autoscaler - It is a component that automatically adjusts the size of a Kubernetes Cluster so that all pods have a place to run and there are no unneeded nodes. It can be created by using the YAML file, Helm Chart, or Terraform. The following example creates the Cloud Autoscalar deployment using Terraform dynamic Modules.
Corresponding to https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yamlResource kubernetes_deployment:-
Plain Text
xxxxxxxxxx
1
75
1
cluster-autoscaler.tf:-
2
resource "kubernetes_deployment" "cluster-autoscaler" {
3
metadata {
4
name = var.cluster-autoscaler-data["cluster-autoscaler-name"]
5
namespace = var.cluster-autoscaler-data["namespace"]
6
labels = {
7
"app" = var.cluster-autoscaler-data["cluster-autoscaler-label"]
8
}
9
}
10
11
spec {
12
replicas = 1
13
selector {
14
match_labels = {
15
"app" = var.cluster-autoscaler-data["cluster-autoscaler-label"]
16
}
17
}
18
19
template {
20
metadata {
21
labels = {
22
"app" = var.cluster-autoscaler-data["cluster-autoscaler-label"]
23
}
24
annotations = {
25
"prometheus.io/port" = "8085"
26
"prometheus.io/scrape" = "true"
27
}
28
}
29
30
spec {
31
automount_service_account_token = true
32
termination_grace_period_seconds = 300
33
service_account_name = var.service-account-name
34
35
container {
36
image = var.cluster-autoscaler-data["image_name"]
37
name = "cluster-autoscaler"
38
command = ["./cluster-autoscaler",
39
"--v=4",
40
" --stderrthreshold=info",
41
"--cloud-provider=aws",
42
"--skip-nodes-with-local-storage=false",
43
"--expander=least-waste",
44
"--node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/${var.eks_cluster_name}"
45
]
46
resources {
47
limits {
48
cpu = "100m"
49
memory = "300Mi"
50
}
51
requests {
52
cpu = "100m"
53
memory = "300Mi"
54
}
55
}
56
volume_mount {
57
name = "ssl-certs"
58
mount_path = "/etc/ssl/certs/ca-certificates.crt"
59
read_only = "true"
60
}
61
}
62
volume {
63
name = "ssl-certs"
64
host_path {
65
path = "/etc/ssl/certs/ca-bundle.crt"
66
}
67
}
68
}
69
}
70
}
71
}
72
variable.tf :-
73
variable "cluster-autoscaler-data" {}
74
variable "service-account-name" {}
75
variable "eks_cluster_name" {}
Service Account Module:-
Plain Text
xxxxxxxxxx
1
24
1
service-account.tf:-
2
resource "kubernetes_service_account" "service-account" {
3
metadata {
4
name = lower(var.service_account_name)
5
namespace = var.namespace
6
7
labels = {
8
"${var.label1}" = var.label-value1
9
}
10
}
11
}
12
data "kubernetes_secret" "service-account-secret" {
13
metadata {
14
name = kubernetes_service_account.service-account.default_secret_name
15
namespace = var.namespace
16
}
17
}
18
variable.tf:-
19
variable "service_account_name" {}
20
variable "namespace" {}
21
variable "label-value1" {}
22
variable "label1" {
23
default = "name"
24
}
Cluster Role Binding Module:-
Plain Text
xxxxxxxxxx
1
30
1
clusterrolebinding-resource.tf:-
2
resource "kubernetes_cluster_role_binding" "clusterrolebinding-res" {
3
metadata {
4
name = lower(var.cluster-role-binding-name)
5
labels = {
6
"${var.label1}" = var.label-value1
7
}
8
}
9
role_ref {
10
api_group = "rbac.authorization.k8s.io"
11
kind = "ClusterRole"
12
name = var.cluster-role
13
}
14
subject {
15
kind = "ServiceAccount"
16
name = var.service-account-name
17
namespace = var.namespace
18
}
19
20
}
21
variable.tf:-
22
variable "cluster-role-binding-name" {}
23
variable "cluster-role" {}
24
variable "service-account-name" {}
25
variable "namespace" {}
26
27
variable "label-value1" {}
28
variable "label1" {
29
default = "name"
30
}
Role Binding Module in Terraform:-
Plain Text
xxxxxxxxxx
1
30
1
rolebinding-res.tf File:-
2
resource "kubernetes_role_binding" "rolebinding-res" {
3
metadata {
4
name = lower(var.role-binding-name)
5
namespace = var.namespace
6
labels = {
7
"${var.label1}" = var.label-value1
8
}
9
}
10
role_ref {
11
api_group = "rbac.authorization.k8s.io"
12
kind = "Role"
13
name = var.role-name
14
}
15
subject {
16
kind = "ServiceAccount"
17
name = var.service-account-name
18
namespace = var.namespace
19
}
20
}
21
variable.tf :-
22
variable "role-binding-name" {}
23
variable "role-name" {}
24
variable "service-account-name" {}
25
variable "namespace" {}
26
27
variable "label-value1" {}
28
variable "label1" {
29
default = "name"
30
}
Role Module:-
Plain Text
xxxxxxxxxx
1
29
1
k8role.tf:-
2
resource "kubernetes_role" "k8role" {
3
metadata {
4
name = lower(var.role-name)
5
namespace = var.namespace
6
labels = {
7
"${var.label1}" = var.label-value1
8
}
9
}
10
11
dynamic "rule" {
12
for_each = var.rules
13
14
content {
15
api_groups = rule.value.api_groups
16
resources = rule.value.resources
17
verbs = rule.value.verbs
18
}
19
}
20
}
21
22
variable.tf File:-
23
variable "role-name" {}
24
variable "namespace" {}
25
variable "rules" {}
26
variable "label-value1" {}
27
variable "label1" {
28
default = "name"
29
}
Cluster Role Module:-
Plain Text
xxxxxxxxxx
1
26
1
clusterrole.tf:-
2
resource "kubernetes_cluster_role" "clusterrole" {
3
metadata {
4
name = lower(var.cluster-role-name)
5
labels = {
6
"${var.label1}" = var.label-value1
7
}
8
}
9
10
dynamic "rule" {
11
for_each = var.rules
12
13
content {
14
api_groups = rule.value.api_groups
15
resources = rule.value.resources
16
verbs = rule.value.verbs
17
}
18
}
19
}
20
variable.tf :-
21
variable "cluster-role-name" {}
22
variable "rules" {}
23
variable "label-value1" {}
24
variable "label1" {
25
default = "name"
26
}
Calling above-defined Modules:-
Plain Text
xxxxxxxxxx
1
128
1
module "cluster-autoscaler-sa" {
2
source = "./../service-account"
3
service_account_name = "${var.cluster-autoscaler-data["cluster-autoscaler-name"]}-sa"
4
label1 = var.cluster-autoscaler-data["label1"]
5
label-value1 = var.cluster-autoscaler-data["label-value1"]
6
namespace = var.cluster-autoscaler-data["namespace"]
7
}
8
module "autoscaler-deployment" {
9
source = "./../cluster-autoscaler-deployment"
10
cluster-autoscaler-data = var.cluster-autoscaler-data
11
service-account-name = module.cluster-autoscaler-sa.service-account-name
12
eks_cluster_name = var.eks_cluster_name
13
}
14
15
module "cluster-autoscaler-cluster-role" {
16
source = "./../clusterrole"
17
cluster-role-name = "${var.cluster-autoscaler-data["cluster-autoscaler-name"]}-clusterrole"
18
label1 = var.cluster-autoscaler-data["label1"]
19
label-value1 = var.cluster-autoscaler-data["label-value1"]
20
rules = [
21
{
22
api_groups = [""]
23
resources = ["events", "endpoints"]
24
verbs = ["create", "patch"]
25
},
26
{
27
api_groups = [""]
28
resources = ["pods/eviction"]
29
verbs = ["create"]
30
},
31
{
32
api_groups = [""]
33
resources = ["pods/status"]
34
verbs = ["update"]
35
},
36
{
37
api_groups = [""]
38
resources = ["endpoints"]
39
resource_names = ["cluster-autoscaler"]
40
verbs = ["get", "update"]
41
},
42
{
43
api_groups = [""]
44
resources = ["nodes"]
45
verbs = ["watch", "list", "get", "update"]
46
},
47
{
48
api_groups = [""]
49
resources = ["pods", "services", "replicationcontrollers", "persistentvolumeclaims", "persistentvolumes"]
50
verbs = ["watch", "list", "get"]
51
}, {
52
api_groups = ["extensions"]
53
resources = ["replicasets", "daemonsets"]
54
verbs = ["watch", "list", "get"]
55
}
56
, {
57
api_groups = ["policy"]
58
resources = ["poddisruptionbudgets"]
59
verbs = ["watch", "list"]
60
}
61
, {
62
api_groups = ["apps"]
63
resources = ["statefulsets", "replicasets", "daemonsets"]
64
verbs = ["watch", "list", "get"]
65
}
66
, {
67
api_groups = ["storage.k8s.io"]
68
resources = ["storageclasses", "csinodes"]
69
verbs = ["watch", "list", "get"]
70
}
71
, {
72
api_groups = ["batch", "extensions"]
73
resources = ["jobs"]
74
verbs = ["get", "list", "watch", "patch"]
75
}
76
, {
77
api_groups = ["coordination.k8s.io"]
78
resources = ["leases"]
79
verbs = ["create"]
80
},
81
{
82
api_groups = ["coordination.k8s.io"]
83
resource_names = ["cluster-autoscaler"]
84
resources = ["leases"]
85
verbs = ["get", "update"]
86
}
87
]
88
}
89
module "cluster-autoscaler-clusterrolebinding" {
90
source = "./../clusterrolebinding"
91
cluster-role-binding-name = "${var.cluster-autoscaler-data["cluster-autoscaler-name"]}-clusterrolebinding"
92
cluster-role = module.cluster-autoscaler-cluster-role.cluster-role-name
93
service-account-name = module.cluster-autoscaler-sa.service-account-name
94
label1 = var.cluster-autoscaler-data["label1"]
95
label-value1 = var.cluster-autoscaler-data["label-value1"]
96
namespace = var.cluster-autoscaler-data["namespace"]
97
}
98
99
module "autoscaler-role" {
100
source = "./../role"
101
role-name = "${var.cluster-autoscaler-data["cluster-autoscaler-name"]}-role"
102
label1 = var.cluster-autoscaler-data["label1"]
103
label-value1 = var.cluster-autoscaler-data["label-value1"]
104
namespace = var.cluster-autoscaler-data["namespace"]
105
rules = [
106
{
107
api_groups = [""]
108
resources = ["configmaps"]
109
verbs = ["create", "list", "watch"]
110
},
111
{
112
api_groups = [""]
113
resources = ["configmaps"]
114
resource_names = ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
115
verbs = ["delete", "get", "update", "watch"]
116
}
117
]
118
}
119
120
module "autoscaler-rolebinding" {
121
source = "./../rolebinding"
122
role-binding-name = "${var.cluster-autoscaler-data["cluster-autoscaler-name"]}-rolebinding"
123
role-name = module.autoscaler-role.role-name
124
service-account-name = module.cluster-autoscaler-sa.service-account-name
125
label1 = var.cluster-autoscaler-data["label1"]
126
label-value1 = var.cluster-autoscaler-data["label-value1"]
127
namespace = var.cluster-autoscaler-data["namespace"]
128
}
Data passed in parameters:-
Plain Text
xxxxxxxxxx
1
10
1
cluster-autoscaler-data = {
2
cluster-autoscaler-name = "k8demo-cluster-autoscaler"
3
cluster-autoscaler-label = "k8demo-cluster-autoscaler"
4
label1 = "k8s-addon"
5
label-value1 = "cluster-autoscaler.addons.k8s.io"
6
namespace = "kube-system"
7
image_name = "k8s.gcr.io/cluster-autoscaler:v1.14.7"
8
}
9
service-account-name = "k8demo-service-account"
10
eks_cluster_name = "k8demo-Cluster"
Stay tuned for the second article.
Reference Article:- https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md
Kubernetes
Plain text
Terraform (software)
Cloud
Opinions expressed by DZone contributors are their own.
Comments