-
Notifications
You must be signed in to change notification settings - Fork 13
/
main.tf
345 lines (293 loc) · 11.8 KB
/
main.tf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
# Azure Infrastructure Resources
resource "tls_private_key" "global_key" {
algorithm = "RSA"
rsa_bits = 2048
}
resource "local_file" "ssh_private_key_pem" {
filename = "${path.module}/id_rsa"
sensitive_content = tls_private_key.global_key.private_key_pem
file_permission = "0600"
}
resource "local_file" "ssh_public_key_openssh" {
filename = "${path.module}/id_rsa.pub"
content = tls_private_key.global_key.public_key_openssh
}
# Resource group containing all resources
resource "azurerm_resource_group" "rke2-cluster" {
name = "${var.prefix}-${var.resource_group_name}"
location = var.azure_location
tags = {
Resource_owner = var.tag_resource_owner,
Group = var.tag_group,
Department = var.tag_department,
Stakeholder = var.tag_stakeholder,
Environment = var.tag_environment,
Project = var.tag_project
}
}
# Azure virtual network space for the resource group
resource "azurerm_virtual_network" "rke2-cluster" {
name = "${var.prefix}-rke2-cluster-vnet"
address_space = ["10.0.0.0/16"]
location = azurerm_resource_group.rke2-cluster.location
resource_group_name = azurerm_resource_group.rke2-cluster.name
}
# ----------------------------------------------------------------
# Azure Load Balancer
# ----------------------------------------------------------------
# Azure Load Balancer - Public IP
resource "azurerm_public_ip" "rke2-lb-pip" {
name = "rke2-lb-pip"
location = azurerm_resource_group.rke2-cluster.location
resource_group_name = azurerm_resource_group.rke2-cluster.name
allocation_method = "Static"
}
# Define Azure Load Balancer with frontend IP configuration
resource "azurerm_lb" "rke2-lb" {
name = "rke2-lb"
location = azurerm_resource_group.rke2-cluster.location
resource_group_name = azurerm_resource_group.rke2-cluster.name
frontend_ip_configuration {
name = "rke2-lb-frontend_ip_configuration"
public_ip_address_id = azurerm_public_ip.rke2-lb-pip.id
}
}
# Define backend pool for LB
resource "azurerm_lb_backend_address_pool" "rke2-lb" {
loadbalancer_id = azurerm_lb.rke2-lb.id
name = "rke2-lb-backend_address_pool"
}
# Create an LB probe on port 80
resource "azurerm_lb_probe" "rke2-lb" {
resource_group_name = azurerm_resource_group.rke2-cluster.name
loadbalancer_id = azurerm_lb.rke2-lb.id
name = "rke2-lb-probe-tcp80"
port = 80
}
# Create an LB rule for port 6443 (KubeAPI)
resource "azurerm_lb_rule" "rke2-lb-rule-6443" {
resource_group_name = azurerm_resource_group.rke2-cluster.name
loadbalancer_id = azurerm_lb.rke2-lb.id
name = "rke2-lb-rule-6443"
protocol = "Tcp"
frontend_port = 6443
backend_port = 6443
frontend_ip_configuration_name = "rke2-lb-frontend_ip_configuration"
backend_address_pool_ids = [azurerm_lb_backend_address_pool.rke2-lb.id]
}
# Create an LB rule for port 443 (https)
resource "azurerm_lb_rule" "rke2-lb-rule-443" {
resource_group_name = azurerm_resource_group.rke2-cluster.name
loadbalancer_id = azurerm_lb.rke2-lb.id
name = "rke2-lb-rule-443"
protocol = "Tcp"
frontend_port = 443
backend_port = 443
frontend_ip_configuration_name = "rke2-lb-frontend_ip_configuration"
backend_address_pool_ids = [azurerm_lb_backend_address_pool.rke2-lb.id]
}
# --------------------------------
# RKE2 cluster on Azure
# --------------------------------
# Azure subnet for rke2-cluster
resource "azurerm_subnet" "rke2-cluster-subnet" {
name = "rke2-cluster-subnet"
resource_group_name = azurerm_resource_group.rke2-cluster.name
virtual_network_name = azurerm_virtual_network.rke2-cluster.name
address_prefixes = ["10.0.0.0/24"]
}
# Public IP of RKE2 nodes
resource "azurerm_public_ip" "rke2-nodes-pip" {
count = var.rke2-cluster-size
name = format("rke2-node%d-pip", count.index + 1)
location = azurerm_resource_group.rke2-cluster.location
resource_group_name = azurerm_resource_group.rke2-cluster.name
allocation_method = "Dynamic"
}
# RKE2 node network security group
resource "azurerm_network_security_group" "rke2-node-nsg" {
name = "rke2-node-nsg"
location = azurerm_resource_group.rke2-cluster.location
resource_group_name = azurerm_resource_group.rke2-cluster.name
}
# RKE2 node network security group rules
resource "azurerm_network_security_rule" "rke2-node-nsg-rule-allowall" {
name = "rke2-node-nsg-rule-allowall"
priority = 100
direction = "Inbound"
access = "Allow"
protocol = "Tcp"
source_port_range = "*"
destination_port_range = "*"
source_address_prefix = "*"
destination_address_prefix = "*"
resource_group_name = azurerm_resource_group.rke2-cluster.name
network_security_group_name = azurerm_network_security_group.rke2-node-nsg.name
}
# Azure network interface for RKE2 Node
resource "azurerm_network_interface" "rke2-nodes-nic" {
count = var.rke2-cluster-size
name = format("rke2-node%d-nic", count.index + 1)
location = azurerm_resource_group.rke2-cluster.location
resource_group_name = azurerm_resource_group.rke2-cluster.name
ip_configuration {
name = format("rke2-node%d_ip_config", count.index + 1)
subnet_id = azurerm_subnet.rke2-cluster-subnet.id
private_ip_address_allocation = "Dynamic"
public_ip_address_id = azurerm_public_ip.rke2-nodes-pip[count.index].id
}
}
# Associate network interface with RKE2 Network Security Group
resource "azurerm_network_interface_security_group_association" "rke2-nodes-nic-nsg" {
count = var.rke2-cluster-size
network_interface_id = azurerm_network_interface.rke2-nodes-nic[count.index].id
network_security_group_id = azurerm_network_security_group.rke2-node-nsg.id
}
# Associate network interface with LB backend pool
resource "azurerm_network_interface_backend_address_pool_association" "rke2-nodes-nic-lb-backend-pool" {
count = var.rke2-cluster-size
network_interface_id = azurerm_network_interface.rke2-nodes-nic[count.index].id
ip_configuration_name = format("rke2-node%d_ip_config", count.index + 1)
backend_address_pool_id = azurerm_lb_backend_address_pool.rke2-lb.id
}
# Availability Set for RKE2 VMs
resource "azurerm_availability_set" "rke2-nodes-as" {
name = "rke2-nodes-as"
location = azurerm_resource_group.rke2-cluster.location
resource_group_name = azurerm_resource_group.rke2-cluster.name
platform_update_domain_count = 3
platform_fault_domain_count = 3
}
# Azure linux virtual machine for RKE2 nodes
resource "azurerm_linux_virtual_machine" "rke2_node" {
count = var.rke2-cluster-size
name = format("${var.prefix}-rke2-node%d", count.index + 1)
computer_name = format("rke2-node%d", count.index + 1) // ensure computer_name meets 15 character limit
location = azurerm_resource_group.rke2-cluster.location
resource_group_name = azurerm_resource_group.rke2-cluster.name
network_interface_ids = [azurerm_network_interface.rke2-nodes-nic[count.index].id]
size = var.rke2_node_vm_size
admin_username = local.node_username
availability_set_id = azurerm_availability_set.rke2-nodes-as.id
license_type = "SLES_BYOS"
source_image_reference {
publisher = "SUSE"
offer = "sles-15-sp3"
sku = "gen1"
version = "latest"
}
admin_ssh_key {
username = local.node_username
public_key = tls_private_key.global_key.public_key_openssh
}
os_disk {
caching = "ReadWrite"
storage_account_type = "Premium_LRS"
}
provisioner "remote-exec" {
inline = [
"echo 'Waiting for cloud-init to complete...'",
"cloud-init status --wait > /dev/null",
"echo 'Completed cloud-init!'",
format("sleep %d", count.index * 60), # wait 60s between nodes to avoid join node simulatenously which is error prone.
templatefile(join("/", [path.module, "files/install_rke2.sh"]), {
first_instance = (count.index == 0) ? "yes" : "no",
rke2_first_node_public_ip = azurerm_linux_virtual_machine.rke2_node[0].public_ip_address
rke2_loadbalancer_public_ip = azurerm_public_ip.rke2-lb-pip.ip_address
})
]
connection {
type = "ssh"
host = self.public_ip_address
user = local.node_username
private_key = tls_private_key.global_key.private_key_pem
}
}
}
# ssh command file for RKE2 nodes VM
resource "local_file" "ssh_to_rke2_nodes" {
count = var.rke2-cluster-size
filename = format("${path.module}/ssh_rke2-node%d.sh", count.index + 1)
content = join(" ", ["ssh", "-i id_rsa", "-o StrictHostKeyChecking=no", "${local.node_username}@${azurerm_linux_virtual_machine.rke2_node[count.index].public_ip_address}"])
file_permission = "0755"
}
resource "ssh_resource" "retrieve_config" {
depends_on = [azurerm_linux_virtual_machine.rke2_node]
host = azurerm_linux_virtual_machine.rke2_node[0].public_ip_address
commands = [
"sudo sed \"s/127.0.0.1/${azurerm_public_ip.rke2-lb-pip.ip_address}/g\" /etc/rancher/rke2/rke2.yaml"
]
user = local.node_username
private_key = tls_private_key.global_key.private_key_pem
}
# Save kubeconfig file for interacting with the RKE cluster on your local machine
resource "local_file" "kube_config_server_yaml" {
depends_on = [azurerm_linux_virtual_machine.rke2_node]
filename = format("%s/%s", path.root, "kube_config_server.yaml")
content = ssh_resource.retrieve_config.result
}
# Rancher bootstrap password
resource "random_password" "rancher_server_password" {
length = 16
special = true
override_special = "_%@"
}
locals {
rancher_server_dns = join(".", ["rancher", azurerm_public_ip.rke2-lb-pip.ip_address, "sslip.io"])
rancher_password = random_password.rancher_server_password.result
}
# Install cert-manager helm chart
resource "helm_release" "cert_manager" {
repository = "https://charts.jetstack.io"
name = "cert-manager"
chart = "cert-manager"
version = "v${var.cert_manager_version}"
namespace = "cert-manager"
create_namespace = true
wait = true
wait_for_jobs = true
verify = false
set {
name = "installCRDs"
value = "true"
}
}
# Wait for 30 seconds till cert-manager is fully initialized
resource "time_sleep" "wait_30_seconds" {
depends_on = [helm_release.cert_manager]
create_duration = "30s"
}
# Install Rancher helm chart
resource "helm_release" "rancher_server" {
depends_on = [time_sleep.wait_30_seconds]
repository = "https://releases.rancher.com/server-charts/latest"
name = "rancher"
chart = "rancher"
version = var.rancher_version
namespace = "cattle-system"
create_namespace = true
wait = true
wait_for_jobs = true
set {
name = "hostname"
value = local.rancher_server_dns
}
set {
name = "replicas"
value = "1"
}
set {
name = "bootstrapPassword"
value = local.rancher_password
}
}
# Initialize Rancher server
resource "rancher2_bootstrap" "admin" {
depends_on = [
helm_release.rancher_server
]
provider = rancher2.bootstrap
initial_password = local.rancher_password
password = local.rancher_password
telemetry = true
}