-
Notifications
You must be signed in to change notification settings - Fork 1
/
data-transfer-ch-mch.tf
160 lines (145 loc) · 5.3 KB
/
data-transfer-ch-mch.tf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
# Infrastructure for the Yandex Cloud Managed Service for ClickHouse cluster and Data Transfer
#
# RU: https://yandex.cloud/ru/docs/managed-clickhouse/tutorials/data-migration
# EN: https://yandex.cloud/en/docs/managed-clickhouse/tutorials/data-migration
#
# Specify the following settings:
locals {
# Source ClickHouse server settings:
source_user = "" # ClickHouse server username
source_db_name = "" # ClickHouse server database name
source_pwd = "" # ClickHouse server password
source_host = "" # ClickHouse server IP address or FQDN
source_shard = "" # ClickHouse server shard name
source_http_port = "8123" # TCP port number for the HTTP interface of the ClickHouse server
source_native_port = "9000" # TCP port number for the native interface of the ClickHouse server
# Target cluster settings:
target_clickhouse_version = "" # Desired version of ClickHouse. For available versions, see the documentation main page: https://yandex.cloud/en/docs/managed-clickhouse/.
target_user = "" # Username of the ClickHouse cluster
target_password = "" # ClickHouse user's password
# Setting for the YC CLI that allows running CLI command to activate the transfer
profile_name = "" # Name of the YC CLI profile
}
resource "yandex_vpc_network" "network" {
description = "Network for the Managed Service for ClickHouse cluster"
name = "network"
}
resource "yandex_vpc_subnet" "subnet-a" {
description = "Subnet in the ru-central1-a availability zone"
name = "subnet-a"
zone = "ru-central1-a"
network_id = yandex_vpc_network.network.id
v4_cidr_blocks = ["10.1.0.0/16"]
}
resource "yandex_vpc_security_group" "security-group" {
description = "Security group for the Managed Service for ClickHouse cluster"
name = "ch-mch-sg"
network_id = yandex_vpc_network.network.id
ingress {
description = "Allow connections to the cluster from the Internet"
protocol = "TCP"
port = local.source_http_port
v4_cidr_blocks = ["0.0.0.0/0"]
}
ingress {
description = "Allow connections to the cluster from the Internet"
protocol = "TCP"
port = local.source_native_port
v4_cidr_blocks = ["0.0.0.0/0"]
}
ingress {
description = "Allow incoming traffic on port 8443 from any IP address"
protocol = "TCP"
port = "8443"
v4_cidr_blocks = ["0.0.0.0/0"]
}
ingress {
description = "Allow incoming traffic on port 9440 from any IP address"
protocol = "TCP"
port = "9440"
v4_cidr_blocks = ["0.0.0.0/0"]
}
}
resource "yandex_mdb_clickhouse_cluster" "clickhouse-cluster" {
name = "clickhouse-cluster"
description = "Managed Service for ClickHouse cluster"
version = local.target_clickhouse_version
environment = "PRODUCTION"
network_id = yandex_vpc_network.network.id
security_group_ids = [yandex_vpc_security_group.security-group.id]
clickhouse {
resources {
resource_preset_id = "s2.micro" # 2 vCPU, 8 GB RAM
disk_type_id = "network-hdd"
disk_size = 10 # GB
}
}
host {
type = "CLICKHOUSE"
zone = "ru-central1-a"
subnet_id = yandex_vpc_subnet.subnet-a.id
}
database {
name = local.source_db_name
}
user {
name = local.target_user
password = local.target_password
permission {
database_name = local.source_db_name
}
}
}
resource "yandex_datatransfer_endpoint" "clickhouse-source" {
description = "Source endpoint for ClickHouse server"
name = "clickhouse-source"
settings {
clickhouse_source {
connection {
connection_options {
on_premise {
shards {
name = local.source_shard
hosts = [local.source_host]
}
http_port = local.source_http_port
native_port = local.source_native_port
}
database = local.source_db_name
user = local.source_user
password {
raw = local.source_pwd
}
}
}
}
}
}
resource "yandex_datatransfer_endpoint" "managed-clickhouse-target" {
description = "Target endpoint for the Managed Service for ClickHouse cluster"
name = "managed-clickhouse-target"
settings {
clickhouse_target {
connection {
connection_options {
mdb_cluster_id = yandex_mdb_clickhouse_cluster.clickhouse-cluster.id
database = local.source_db_name
user = local.target_user
password {
raw = local.target_password
}
}
}
}
}
}
resource "yandex_datatransfer_transfer" "clickhouse-transfer" {
description = "Transfer from ClickHouse server to the Managed Service for ClickHouse cluster"
name = "transfer-from-onpremise-clickhouse-to-managed-clickhouse"
source_id = yandex_datatransfer_endpoint.clickhouse-source.id
target_id = yandex_datatransfer_endpoint.managed-clickhouse-target.id
type = "SNAPSHOT_ONLY" # Copy all data from the source server
provisioner "local-exec" {
command = "yc --profile ${local.profile_name} datatransfer transfer activate ${yandex_datatransfer_transfer.clickhouse-transfer.id}"
}
}