From 99fed1a64df0d9c44bfaa54280a455a5a64c4800 Mon Sep 17 00:00:00 2001 From: ankush-sqops Date: Fri, 22 Nov 2024 19:00:53 +0530 Subject: [PATCH 1/6] Release 2.2.1 (#17) * added custom password functionality and print database name * added custom password functionality and print database name * fixed security group inbound & outbound rule * Update main.tf * changed the version * fixed security group * version updated * version updated --------- Co-authored-by: amanravi-squareops Co-authored-by: Aman <146931382+amanravi-squareops@users.noreply.github.com> --- examples/aurora-global/main.tf | 2 +- examples/aurora/main.tf | 3 +-- main.tf | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/aurora-global/main.tf b/examples/aurora-global/main.tf index 5c11532..346c3f9 100644 --- a/examples/aurora-global/main.tf +++ b/examples/aurora-global/main.tf @@ -103,7 +103,7 @@ module "secondary_vpc" { module "aurora" { source = "squareops/rds-aurora/aws" - version = "2.1.1" + version = "2.2.1" environment = local.environment global_cluster_enable = true port = local.port diff --git a/examples/aurora/main.tf b/examples/aurora/main.tf index 84766fd..0e08b98 100644 --- a/examples/aurora/main.tf +++ b/examples/aurora/main.tf @@ -86,7 +86,7 @@ module "vpc" { module "aurora" { source = "squareops/rds-aurora/aws" - version = "2.1.1" + version = "2.2.1" role_arn = local.role_arn external_id = local.external_id environment = local.environment @@ -123,5 +123,4 @@ module "aurora" { autoscaling_scale_in_cooldown = 60 autoscaling_scale_out_cooldown = 30 allowed_cidr_blocks = local.allowed_cidr_blocks - allowed_security_groups = local.allowed_security_groups } diff --git a/main.tf b/main.tf index ca0ba27..6f6f3ee 100644 --- a/main.tf +++ b/main.tf @@ -65,8 +65,8 @@ module "aurora" { ingress_postgresql = { description = "Allow inbound PostgreSQL traffic from trusted CIDR blocks" type = "ingress" - from_port = 5432 - to_port = 5432 + from_port = var.port + to_port = var.port protocol = "tcp" cidr_blocks = var.allowed_cidr_blocks } From 3eb1f1ff7f31a463a431eb9da5f4cc8c27bab4a5 Mon Sep 17 00:00:00 2001 From: amanravi-squareops Date: Mon, 20 Jan 2025 21:08:35 +0530 Subject: [PATCH 2/6] added backup & restore functionality --- examples/aurora/helm/values.yaml | 48 +++++++ examples/aurora/main.tf | 33 ++++- examples/aurora/provider.tf | 22 ++++ helm/values/backup/values.yaml | 37 ++++++ helm/values/restore/values.yaml | 35 ++++++ main.tf | 42 ++++++- modules/db-backup-restore/backup/.helmignore | 21 ++++ modules/db-backup-restore/backup/Chart.yaml | 4 + .../backup/templates/cronjob.yaml | 36 ++++++ .../backup/templates/service_account.yaml | 7 ++ modules/db-backup-restore/main.tf | 64 ++++++++++ modules/db-backup-restore/restore/.helmignore | 21 ++++ modules/db-backup-restore/restore/Chart.yaml | 4 + .../restore/templates/job.yaml | 49 ++++++++ .../restore/templates/restore-secret.yaml | 8 ++ .../restore/templates/service_account.yaml | 6 + modules/db-backup-restore/roles.tf | 100 +++++++++++++++ modules/db-backup-restore/variables.tf | 119 ++++++++++++++++++ variables.tf | 64 ++++++++++ 19 files changed, 708 insertions(+), 12 deletions(-) create mode 100644 examples/aurora/helm/values.yaml create mode 100644 helm/values/backup/values.yaml create mode 100644 helm/values/restore/values.yaml create mode 100644 modules/db-backup-restore/backup/.helmignore create mode 100644 modules/db-backup-restore/backup/Chart.yaml create mode 100644 modules/db-backup-restore/backup/templates/cronjob.yaml create mode 100644 modules/db-backup-restore/backup/templates/service_account.yaml create mode 100644 modules/db-backup-restore/main.tf create mode 100644 modules/db-backup-restore/restore/.helmignore create mode 100644 modules/db-backup-restore/restore/Chart.yaml create mode 100644 modules/db-backup-restore/restore/templates/job.yaml create mode 100644 modules/db-backup-restore/restore/templates/restore-secret.yaml create mode 100644 modules/db-backup-restore/restore/templates/service_account.yaml create mode 100644 modules/db-backup-restore/roles.tf create mode 100644 modules/db-backup-restore/variables.tf diff --git a/examples/aurora/helm/values.yaml b/examples/aurora/helm/values.yaml new file mode 100644 index 0000000..7371739 --- /dev/null +++ b/examples/aurora/helm/values.yaml @@ -0,0 +1,48 @@ +primary: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Addons-Services" + operator: In + values: + - "true" + +secondary: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Addons-Services" + operator: In + values: + - "true" + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Addons-Services" + operator: In + values: + - "true" +backupjob: + resources: + requests: + memory: 100Mi + cpu: 50m + limits: + memory: 200Mi + cpu: 100m + +restorejob: + resources: + requests: + memory: 100Mi + cpu: 50m + limits: + memory: 200Mi + cpu: 100m \ No newline at end of file diff --git a/examples/aurora/main.tf b/examples/aurora/main.tf index 0e08b98..f8f215c 100644 --- a/examples/aurora/main.tf +++ b/examples/aurora/main.tf @@ -5,13 +5,16 @@ locals { assume_role_config = length(local.role_arn) > 0 ? { role_arn = local.role_arn } : null name = "skaf" region = "us-east-2" - port = 5432 #/3306 - family = "aurora-postgresql15" #/aurora-mysql5.7" - engine = "aurora-postgresql" #/aurora-mysql" + port = 5432 # 3306 + family = "aurora-postgresql15" # aurora-mysql5.7" + engine = "aurora-postgresql" # aurora-mysql" vpc_cidr = "10.0.0.0/16" - environment = "production" - db_engine_version = "15.2" #/5.7" + environment = "prod" + db_engine_version = "15.2" # 5.7" db_instance_class = "db.r5.large" + cluster_name = "" + create_namespace = false + namespace = "mydb" master_password = "" # Leave this field empty to have a password automatically generated. additional_aws_tags = { Owner = "Organization_Name" @@ -19,7 +22,7 @@ locals { Department = "Engineering" } current_identity = data.aws_caller_identity.current.arn - allowed_cidr_blocks = ["10.10.0.0/16"] + allowed_cidr_blocks = ["10.0.0.0/16"] } data "aws_caller_identity" "current" {} @@ -87,6 +90,8 @@ module "vpc" { module "aurora" { source = "squareops/rds-aurora/aws" version = "2.2.1" + name = local.name + region = local.region role_arn = local.role_arn external_id = local.external_id environment = local.environment @@ -123,4 +128,20 @@ module "aurora" { autoscaling_scale_in_cooldown = 60 autoscaling_scale_out_cooldown = 30 allowed_cidr_blocks = local.allowed_cidr_blocks + ######### + cluster_name = local.cluster_name # cluster name + namespace = local.namespace + create_namespace = local.create_namespace + bucket_provider_type = "s3" + db_backup_enabled = false + db_backup_config = { + mysql_database_name = "" # Specify the database name or Leave empty if you wish to backup all databases + cron_for_full_backup = "*/2 * * * *" # set cronjob for backup + bucket_uri = "s3://aurora-backup-atmosly" # S3 bucket URI (without a trailing slash /) + } + db_restore_enabled = false + db_restore_config = { + bucket_uri = "s3://aurora-backup-atmosly" # S3 bucket URI (without a trailing slash /) containing the backup dump file. + file_name = "mysqldump_20250120_090803.zip" # Give .sql or .zip file for restore + } } diff --git a/examples/aurora/provider.tf b/examples/aurora/provider.tf index 67b50fa..e0c9697 100644 --- a/examples/aurora/provider.tf +++ b/examples/aurora/provider.tf @@ -9,3 +9,25 @@ provider "aws" { } } } + +data "aws_eks_cluster" "cluster" { + name = "" + +} +data "aws_eks_cluster_auth" "cluster" { + name = "" +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token +} + +provider "helm" { + kubernetes { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token + } +} diff --git a/helm/values/backup/values.yaml b/helm/values/backup/values.yaml new file mode 100644 index 0000000..4276d8f --- /dev/null +++ b/helm/values/backup/values.yaml @@ -0,0 +1,37 @@ +## Enable Full backup +backup: + bucket_uri: ${bucket_uri} + cron_for_full_backup: "${cron_for_full_backup}" + database_name: "${mysql_database_name}" + database_endpoint: "${db_endpoint}" + database_password: "${db_password}" + database_user: "${db_username}" + engine: "${engine}" + + +annotations: + ${annotations} + +auth: + username: "${custom_user_username}" + +bucket_provider_type: ${bucket_provider_type} + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Addons-Services" + operator: In + values: + - "true" + +backupjob: + resources: + requests: + memory: 100Mi + cpu: 50m + limits: + memory: 200Mi + cpu: 100m diff --git a/helm/values/restore/values.yaml b/helm/values/restore/values.yaml new file mode 100644 index 0000000..b98c684 --- /dev/null +++ b/helm/values/restore/values.yaml @@ -0,0 +1,35 @@ +restore: + file_name: ${file_name} + bucket_uri: ${bucket_uri} + database_endpoint: "${db_endpoint}" + database_password: "${db_password}" + database_user: "${db_username}" + engine: "${engine}" + # port: 5432 + +auth: + username: "${custom_user_username}" + +annotations: + ${annotations} + +bucket_provider_type: ${bucket_provider_type} + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Addons-Services" + operator: In + values: + - "true" + +restorejob: + resources: + requests: + memory: 100Mi + cpu: 50m + limits: + memory: 200Mi + cpu: 100m diff --git a/main.tf b/main.tf index 6f6f3ee..1b230a2 100644 --- a/main.tf +++ b/main.tf @@ -1,10 +1,10 @@ locals { - tags = { + tags = { Automation = "true" Environment = var.environment } region = var.region - secondary_region = var.secondary_region + secondary_region = var.secondary_region != "null" ? var.secondary_region : null # Check if secondary_region is null role_arn = var.role_arn external_id = var.external_id assume_role_config = length(var.role_arn) > 0 ? { role_arn = var.role_arn } : null @@ -23,7 +23,7 @@ provider "aws" { provider "aws" { alias = "secondary" - region = local.secondary_region + region = local.secondary_region != null ? local.secondary_region : var.region # Fallback to primary region if secondary is null } data "aws_caller_identity" "current" {} @@ -36,7 +36,7 @@ module "aurora" { source = "terraform-aws-modules/rds-aurora/aws" version = "8.3.0" name = format("%s-%s", var.environment, var.rds_instance_name) - + # region = var.region engine = var.engine engine_mode = var.engine_mode engine_version = var.engine_mode == "serverless" ? null : var.engine_version @@ -161,7 +161,7 @@ resource "aws_rds_cluster_parameter_group" "rds_cluster_parameter_group" { } resource "aws_secretsmanager_secret" "secret_master_db" { - name = format("%s/%s/%s-%s", var.environment, var.rds_instance_name, var.engine, "aurora-pass") + name = format("%s/%s/%s-%s", var.environment, var.rds_instance_name, var.engine, "aurora-password") tags = merge( { "Name" = format("%s/%s/%s-%s", var.environment, var.rds_instance_name, var.engine, "aurora-pass") }, local.tags, @@ -199,7 +199,7 @@ module "aurora_secondary" { source = "terraform-aws-modules/rds-aurora/aws" count = var.global_cluster_enable ? 1 : 0 version = "8.3.0" - providers = { aws = aws.secondary } + # providers = { aws = aws.secondary } is_primary_cluster = false @@ -239,3 +239,33 @@ module "aurora_secondary" { local.tags, ) } + + +module "backup_restore" { + depends_on = [module.aurora] + source = "./modules/db-backup-restore" + name = var.name + cluster_name = var.cluster_name + namespace = var.namespace + create_namespace = var.create_namespace + bucket_provider_type = var.bucket_provider_type + engine = var.engine + db_backup_enabled = var.db_backup_enabled + db_backup_config = { + db_username = module.aurora.cluster_master_username + db_password = var.master_password != "" ? var.master_password : nonsensitive(random_password.master[0].result) + mysql_database_name = var.db_backup_config.mysql_database_name + cron_for_full_backup = var.db_backup_config.cron_for_full_backup + bucket_uri = var.db_backup_config.bucket_uri + db_endpoint = module.aurora.cluster_endpoint + } + + db_restore_enabled = var.db_restore_enabled + db_restore_config = { + db_endpoint = module.aurora.cluster_endpoint + db_username = module.aurora.cluster_master_username + db_password = var.master_password != "" ? var.master_password : nonsensitive(random_password.master[0].result) + bucket_uri = var.db_restore_config.bucket_uri + file_name = var.db_restore_config.file_name + } +} diff --git a/modules/db-backup-restore/backup/.helmignore b/modules/db-backup-restore/backup/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/modules/db-backup-restore/backup/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/modules/db-backup-restore/backup/Chart.yaml b/modules/db-backup-restore/backup/Chart.yaml new file mode 100644 index 0000000..3394465 --- /dev/null +++ b/modules/db-backup-restore/backup/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A helm chart for Backup of mysql and stored in S3 +name: mysql-backup +version: 1.0.0 diff --git a/modules/db-backup-restore/backup/templates/cronjob.yaml b/modules/db-backup-restore/backup/templates/cronjob.yaml new file mode 100644 index 0000000..b4832bd --- /dev/null +++ b/modules/db-backup-restore/backup/templates/cronjob.yaml @@ -0,0 +1,36 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: backup-job +spec: + schedule: {{ .Values.backup.cron_for_full_backup | quote }} + concurrencyPolicy: Forbid + suspend: false + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 1 + + jobTemplate: + spec: + template: + spec: + affinity: {{ .Values.affinity | toYaml | nindent 10 }} + restartPolicy: OnFailure + imagePullSecrets: + - name: regcred + serviceAccountName: sa-db-backup + containers: + - name: backup-database + image: {{ if eq .Values.backup.engine "aurora-mysql" }}amanravi12/rds-mysql-backup:v1{{ else }}squareops01/rds-postgresql-backup:v1{{ end }} + imagePullPolicy: Always + env: + - name: {{if eq .Values.backup.engine "aurora-mysql"}}MYSQL_HOST{{ else }}DB_HOST{{ end }} + value: {{ .Values.backup.database_endpoint }} + - name: {{if eq .Values.backup.engine "aurora-mysql"}}DATABASE{{ else }}DB_NAME{{ end }} + value: {{ .Values.backup.database_name }} + - name: {{if eq .Values.backup.engine "aurora-mysql"}}MYSQL_USER{{ else }}DB_USER{{ end }} + value: {{ .Values.backup.database_user }} + - name: {{if eq .Values.backup.engine "aurora-mysql"}}MYSQL_PASSWORD{{ else }}DB_PASSWORD{{ end }} + value: {{ .Values.backup.database_password }} + - name: {{if eq .Values.backup.engine "aurora-mysql"}}MYSQL_BUCKET_URI{{ else }}S3_BUCKET{{ end }} + value: {{ .Values.backup.bucket_uri }} + resources: {{ .Values.backupjob.resources | toYaml | nindent 12 }} \ No newline at end of file diff --git a/modules/db-backup-restore/backup/templates/service_account.yaml b/modules/db-backup-restore/backup/templates/service_account.yaml new file mode 100644 index 0000000..d43c007 --- /dev/null +++ b/modules/db-backup-restore/backup/templates/service_account.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sa-db-backup + namespace: {{ .Release.Namespace }} + annotations: + {{ toYaml .Values.annotations | indent 4 }} diff --git a/modules/db-backup-restore/main.tf b/modules/db-backup-restore/main.tf new file mode 100644 index 0000000..3b54d1f --- /dev/null +++ b/modules/db-backup-restore/main.tf @@ -0,0 +1,64 @@ +resource "kubernetes_namespace" "db" { + count = var.create_namespace ? 1 : 0 + metadata { + annotations = {} + name = var.namespace + } +} + +resource "helm_release" "db_backup" { + count = var.db_backup_enabled ? 1 : 0 + depends_on = [kubernetes_namespace.db] + name = "db-backup" + chart = "${path.module}/../../modules/db-backup-restore/backup" + timeout = 600 + namespace = var.namespace + values = [ + templatefile("${path.module}/../../helm/values/backup/values.yaml", { + bucket_uri = var.db_backup_config.bucket_uri, + engine = var.engine, + mysql_database_name = var.bucket_provider_type == "s3" ? var.db_backup_config.mysql_database_name : "", + db_endpoint = var.bucket_provider_type == "s3" ? var.db_backup_config.db_endpoint : "", + db_password = var.bucket_provider_type == "s3" ? var.db_backup_config.db_password : "", + db_username = var.bucket_provider_type == "s3" ? var.db_backup_config.db_username : "", + cron_for_full_backup = var.db_backup_config.cron_for_full_backup, + custom_user_username = "admin", + bucket_provider_type = var.bucket_provider_type, + azure_storage_account_name = var.bucket_provider_type == "azure" ? var.azure_storage_account_name : "" + azure_storage_account_key = var.bucket_provider_type == "azure" ? var.azure_storage_account_key : "" + azure_container_name = var.bucket_provider_type == "azure" ? var.azure_container_name : "" + annotations = var.bucket_provider_type == "s3" ? "eks.amazonaws.com/role-arn: ${aws_iam_role.mysql_backup_role[count.index].arn}" : "iam.gke.io/gcp-service-account: ${var.service_account_backup}" + }) + ] +} + + +## DB dump restore +resource "helm_release" "db_restore" { + count = var.db_restore_enabled ? 1 : 0 + depends_on = [kubernetes_namespace.db] + name = "db-restore" + chart = "${path.module}/../../modules/db-backup-restore/restore" + timeout = 600 + namespace = var.namespace + values = [ + templatefile("${path.module}/../../helm/values/restore/values.yaml", { + bucket_uri = var.db_restore_config.bucket_uri, + file_name = var.db_restore_config.file_name, + engine = var.engine, + db_endpoint = var.bucket_provider_type == "s3" ? var.db_restore_config.db_endpoint : "", + db_password = var.bucket_provider_type == "s3" ? var.db_restore_config.db_password : "", + db_username = var.bucket_provider_type == "s3" ? var.db_restore_config.db_username : "", + custom_user_username = "admin", + bucket_provider_type = var.bucket_provider_type, + azure_storage_account_name = var.bucket_provider_type == "azure" ? var.azure_storage_account_name : "" + azure_storage_account_key = var.bucket_provider_type == "azure" ? var.azure_storage_account_key : "" + azure_container_name = var.bucket_provider_type == "azure" ? var.azure_container_name : "" + annotations = var.bucket_provider_type == "s3" ? "eks.amazonaws.com/role-arn: ${aws_iam_role.mysql_restore_role[count.index].arn}" : "iam.gke.io/gcp-service-account: ${var.service_account_restore}" + }) + ] +} + + + + diff --git a/modules/db-backup-restore/restore/.helmignore b/modules/db-backup-restore/restore/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/modules/db-backup-restore/restore/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/modules/db-backup-restore/restore/Chart.yaml b/modules/db-backup-restore/restore/Chart.yaml new file mode 100644 index 0000000..661e06b --- /dev/null +++ b/modules/db-backup-restore/restore/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A helm chart for restore of mysql and stored in S3 +name: mysql-restore +version: 1.0.0 diff --git a/modules/db-backup-restore/restore/templates/job.yaml b/modules/db-backup-restore/restore/templates/job.yaml new file mode 100644 index 0000000..8ff3274 --- /dev/null +++ b/modules/db-backup-restore/restore/templates/job.yaml @@ -0,0 +1,49 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: restore +spec: + template: + spec: + affinity: {{ .Values.affinity | toYaml | nindent 6 }} + serviceAccountName: sa-db-restore + containers: + - name: restore-job + image: {{ if eq .Values.restore.engine "aurora-mysql" }}amanravi12/rds-aurora-mysql-restore:v1{{ else }}squareops01/rds-postgresql-restore:v1{{ end }} + imagePullPolicy: Always + env: + - name: {{if eq .Values.restore.engine "aurora-mysql"}}MYSQL_HOST{{ else }}DB_HOST{{ end }} + value: {{ .Values.restore.database_endpoint }} + - name: {{if eq .Values.restore.engine "aurora-mysql"}}DATABASE{{ else }}DB_NAME{{ end }} + value: {{ .Values.restore.database_name }} + - name: {{if eq .Values.restore.engine "aurora-mysql"}}MYSQL_USER{{ else }}DB_USER{{ end }} + value: {{ .Values.restore.database_user }} + - name: {{if eq .Values.restore.engine "aurora-mysql"}}MYSQL_PASSWORD{{ else }}DB_PASSWORD{{ end }} + value: {{ .Values.restore.database_password }} + - name: {{if eq .Values.restore.engine "aurora-mysql"}}MYSQL_BUCKET_RESTORE_URI{{ else }}POSTGRESQL_BUCKET_RESTORE_URI{{ end }} + value: {{ .Values.restore.bucket_uri }} + - name: {{if eq .Values.restore.engine "aurora-mysql"}}RESTORE_FILE_NAME{{ else }}RESTORE_FILE_NAME{{ end }} + value: {{ .Values.restore.file_name}} + resources: {{ .Values.restorejob.resources | toYaml | nindent 12 }} + {{- if eq .Values.restore.engine "aurora-mysql" }} + initContainers: + - name: grant-system-variable-admin + image: mysql:8.0 + command: + - /bin/bash + - -c + - | + # Connect to the MySQL server using the MySQL client + mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" -e "GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO '$MYSQL_USER'@'%'; FLUSH PRIVILEGES;" + # Exit with status code 0 to indicate success + exit 0 + env: + - name: MYSQL_HOST + value: {{ .Values.restore.database_endpoint }} + - name: MYSQL_USER + value: {{ .Values.restore.database_user }} + - name: MYSQL_PASSWORD + value: {{ .Values.restore.database_password }} + {{- end }} + restartPolicy: Never + backoffLimit: 4 diff --git a/modules/db-backup-restore/restore/templates/restore-secret.yaml b/modules/db-backup-restore/restore/templates/restore-secret.yaml new file mode 100644 index 0000000..8ead489 --- /dev/null +++ b/modules/db-backup-restore/restore/templates/restore-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: mysql-bucket-uri-restore + namespace: {{ .Release.Namespace }} + labels: +data: + MYSQL_BUCKET_URI: {{ .Values.restore.bucket_uri | b64enc | quote }} diff --git a/modules/db-backup-restore/restore/templates/service_account.yaml b/modules/db-backup-restore/restore/templates/service_account.yaml new file mode 100644 index 0000000..8b4e659 --- /dev/null +++ b/modules/db-backup-restore/restore/templates/service_account.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sa-db-restore + annotations: + {{ toYaml .Values.annotations | indent 4 }} diff --git a/modules/db-backup-restore/roles.tf b/modules/db-backup-restore/roles.tf new file mode 100644 index 0000000..e08ac2c --- /dev/null +++ b/modules/db-backup-restore/roles.tf @@ -0,0 +1,100 @@ +locals { + oidc_provider = replace( + data.aws_eks_cluster.kubernetes_cluster.identity[0].oidc[0].issuer, + "/^https:///", + "" + ) +} + +data "aws_caller_identity" "current" {} + +data "aws_eks_cluster" "kubernetes_cluster" { + name = var.cluster_name +} + +resource "aws_iam_role" "mysql_backup_role" { + count = var.db_backup_enabled ? 1 : 0 + name = format("%s-%s-%s", var.cluster_name, var.name, "db-backup") + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Principal = { + Federated = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.oidc_provider}" + }, + Action = "sts:AssumeRoleWithWebIdentity", + Condition = { + StringEquals = { + "${local.oidc_provider}:aud" = "sts.amazonaws.com", + "${local.oidc_provider}:sub" = "system:serviceaccount:${var.namespace}:sa-db-backup" + } + } + } + ] + }) + inline_policy { + name = "AllowS3PutObject" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ] + Effect = "Allow" + Resource = "*" + } + ] + }) + } +} + + +resource "aws_iam_role" "mysql_restore_role" { + count = var.db_restore_enabled ? 1 : 0 + name = format("%s-%s-%s", var.cluster_name, var.name, "db-restore") + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Principal = { + Federated = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.oidc_provider}" + }, + Action = "sts:AssumeRoleWithWebIdentity", + Condition = { + StringEquals = { + "${local.oidc_provider}:aud" = "sts.amazonaws.com", + "${local.oidc_provider}:sub" = "system:serviceaccount:${var.namespace}:sa-db-restore" + } + } + } + ] + }) + inline_policy { + name = "AllowS3PutObject" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ] + Effect = "Allow" + Resource = "*" + } + ] + }) + } +} \ No newline at end of file diff --git a/modules/db-backup-restore/variables.tf b/modules/db-backup-restore/variables.tf new file mode 100644 index 0000000..3fbc0bb --- /dev/null +++ b/modules/db-backup-restore/variables.tf @@ -0,0 +1,119 @@ +variable "iam_role_arn_backup" { + description = "IAM role ARN for backup (AWS)" + type = string + default = "" +} + +variable "service_account_backup" { + description = "Service account for backup (GCP)" + type = string + default = "" +} + +variable "azure_storage_account_name" { + description = "Azure storage account name" + type = string + default = "" +} + +variable "azure_storage_account_key" { + description = "Azure storage account key" + type = string + default = "" +} + +variable "azure_container_name" { + description = "Azure container name" + type = string + default = "" +} + +variable "namespace" { + type = string + default = "db" + description = "Name of the Kubernetes namespace where the MYSQL deployment will be deployed." +} + +variable "create_namespace" { + type = string + description = "Specify whether or not to create the namespace if it does not already exist. Set it to true to create the namespace." + default = false +} + +variable "iam_role_arn_restore" { + description = "IAM role ARN for restore (AWS)" + type = string + default = "" +} + +variable "service_account_restore" { + description = "Service account for restore (GCP)" + type = string + default = "" +} + +# two variable of clustername and name +variable "name" { + description = "Name identifier for module to be added as suffix to resources" + type = string + default = "test" +} + +variable "cluster_name" { + type = string + default = "" + description = "Specifies the name of the EKS cluster to deploy the MySQL application on." +} + +variable "db_permission" { + default = false + description = "access" + type = bool +} + +variable "bucket_provider_type" { + type = string + default = "s3" + description = "Choose what type of provider you want (s3, gcs)" +} + + +variable "db_backup_enabled" { + type = bool + default = false + description = "Specifies whether to enable backups for MySQL database." +} + +variable "db_restore_enabled" { + type = bool + default = false + description = "Specifies whether to enable restoring dump to the MySQL database." +} + +variable "db_backup_config" { + type = map(string) + default = { + bucket_uri = "" + # s3_bucket_region = "" + cron_for_full_backup = "" + mysql_database_name = "" + # port = "" + } + description = "configuration options for MySQL database backups. It includes properties such as the S3 bucket URI, the S3 bucket region, and the cron expression for full backups." +} + +variable "db_restore_config" { + type = any + default = { + bucket_uri = "" + file_name = "" + # s3_bucket_region = "" + } + description = "Configuration options for restoring dump to the MySQL database." +} + +variable "engine" { + description = "The name of the database engine to be used for this DB cluster" + type = string + default = "aurora" +} \ No newline at end of file diff --git a/variables.tf b/variables.tf index 647b0b0..18ac65b 100644 --- a/variables.tf +++ b/variables.tf @@ -1,3 +1,8 @@ +variable "name" { + description = "The name of the RDS instance" + default = "" + type = string +} variable "allowed_cidr_blocks" { description = "A list of CIDR blocks which are allowed to access the database" type = any @@ -389,3 +394,62 @@ variable "external_id" { type = string default = "" # Default to empty string if not provided } + +### backup & restore + +variable "cluster_name" { + type = string + default = "" + description = "Specifies the name of the EKS cluster to deploy the MySQL application on." +} + +variable "create_namespace" { + type = string + description = "Specify whether or not to create the namespace if it does not already exist. Set it to true to create the namespace." + default = false +} + +variable "namespace" { + type = string + default = "db" + description = "Name of the Kubernetes namespace where the MYSQL deployment will be deployed." +} + +variable "db_backup_enabled" { + type = bool + default = false + description = "Specifies whether to enable backups for MySQL database." +} + +variable "db_restore_enabled" { + type = bool + default = false + description = "Specifies whether to enable restoring dump to the MySQL database." +} +variable "bucket_provider_type" { + type = string + default = "s3" + description = "Choose what type of provider you want (s3, gcs)" +} + +variable "db_backup_config" { + type = map(string) + default = { + bucket_uri = "" + # s3_bucket_region = "" + cron_for_full_backup = "" + mysql_database_name = "" + # port ="" + } + description = "configuration options for MySQL database backups. It includes properties such as the S3 bucket URI, the S3 bucket region, and the cron expression for full backups." +} + +variable "db_restore_config" { + type = any + default = { + bucket_uri = "" + file_name = "" + # s3_bucket_region = "" + } + description = "Configuration options for restoring dump to the MySQL database." +} \ No newline at end of file From 7bf6d2899c82af6ce3085bf1cdc075d7b1a39fc4 Mon Sep 17 00:00:00 2001 From: amanravi-squareops Date: Fri, 7 Feb 2025 15:21:53 +0530 Subject: [PATCH 3/6] Fix formatting issues --- README.md | 12 ++++ examples/aurora-global/README.md | 2 +- examples/aurora/README.md | 5 +- examples/aurora/helm/values.yaml | 2 +- examples/aurora/main.tf | 36 +++++----- examples/aurora/provider.tf | 4 +- main.tf | 72 +++++++++---------- modules/db-backup-restore/README.md | 58 +++++++++++++++ .../backup/templates/cronjob.yaml | 4 +- modules/db-backup-restore/main.tf | 4 -- .../restore/templates/job.yaml | 2 +- modules/db-backup-restore/roles.tf | 10 +-- modules/db-backup-restore/variables.tf | 14 ++-- outputs.tf | 2 +- variables.tf | 8 +-- 15 files changed, 152 insertions(+), 83 deletions(-) create mode 100644 modules/db-backup-restore/README.md diff --git a/README.md b/README.md index 855991b..26833a4 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,7 @@ Security scanning is graciously provided by Prowler. Proowler is the leading ful |------|--------|---------| | [aurora](#module\_aurora) | terraform-aws-modules/rds-aurora/aws | 8.3.0 | | [aurora\_secondary](#module\_aurora\_secondary) | terraform-aws-modules/rds-aurora/aws | 8.3.0 | +| [backup\_restore](#module\_backup\_restore) | ./modules/db-backup-restore | n/a | ## Resources @@ -125,10 +126,17 @@ Security scanning is graciously provided by Prowler. Proowler is the leading ful | [autoscaling\_scale\_out\_cooldown](#input\_autoscaling\_scale\_out\_cooldown) | Cooldown in seconds before allowing further scaling operations after a scale out | `number` | `300` | no | | [autoscaling\_target\_connections](#input\_autoscaling\_target\_connections) | No of connections on which aurora has to scale if predefined\_metric\_type is RDSReaderAverageDatabaseConnections | `number` | `50` | no | | [backup\_retention\_period](#input\_backup\_retention\_period) | The number of days to retain backups for | `number` | `null` | no | +| [bucket\_provider\_type](#input\_bucket\_provider\_type) | Choose what type of provider you want (s3, gcs) | `string` | `"s3"` | no | +| [cluster\_name](#input\_cluster\_name) | Specifies the name of the EKS cluster to deploy the MySQL application on. | `string` | `""` | no | | [create\_monitoring\_role](#input\_create\_monitoring\_role) | Set it to true to create IAM role for Enhanced monitoring. | `bool` | `false` | no | +| [create\_namespace](#input\_create\_namespace) | Specify whether or not to create the namespace if it does not already exist. Set it to true to create the namespace. | `string` | `false` | no | | [create\_random\_password](#input\_create\_random\_password) | Whether to create a random password for the primary database cluster | `bool` | `true` | no | | [create\_security\_group](#input\_create\_security\_group) | Whether to create a security group or not | `bool` | `true` | no | | [database\_name](#input\_database\_name) | The name for an automatically created database on cluster creation | `string` | `""` | no | +| [db\_backup\_config](#input\_db\_backup\_config) | configuration options for MySQL database backups. It includes properties such as the S3 bucket URI, the S3 bucket region, and the cron expression for full backups. | `map(string)` | 
{
  "bucket_uri": "",
  "cron_for_full_backup": "",
  "mysql_database_name": ""
}
| no | +| [db\_backup\_enabled](#input\_db\_backup\_enabled) | Specifies whether to enable backups for MySQL database. | `bool` | `false` | no | +| [db\_restore\_config](#input\_db\_restore\_config) | Configuration options for restoring dump to the MySQL database. | `any` | 
{
  "bucket_uri": "",
  "file_name": ""
}
| no | +| [db\_restore\_enabled](#input\_db\_restore\_enabled) | Specifies whether to enable restoring dump to the MySQL database. | `bool` | `false` | no | | [deletion\_protection](#input\_deletion\_protection) | Whether accidental deletion protection is enabled | `bool` | `true` | no | | [enable\_egress](#input\_enable\_egress) | Set it true if allow outbound traffic in rds security group | `bool` | `true` | no | | [enable\_http\_endpoint](#input\_enable\_http\_endpoint) | Whether or not to enable the Data API for a serverless Aurora database engine | `bool` | `false` | no | @@ -148,8 +156,11 @@ Security scanning is graciously provided by Prowler. Proowler is the leading ful | [kms\_key\_arn](#input\_kms\_key\_arn) | The ARN for the KMS encryption key. If creating an encrypted replica, set this to the destination KMS ARN. If storage\_encrypted is set to true and kms\_key\_id is not specified the default KMS key created in your account will be used | `string` | `null` | no | | [long\_query\_time](#input\_long\_query\_time) | To prevent fast-running queries from being logged in the slow query log, specify a value for the shortest query runtime to be logged, in seconds | `number` | `10` | no | | [manage\_master\_user\_password](#input\_manage\_master\_user\_password) | Set to true to allow RDS to manage the master user password in Secrets Manager. Cannot be set if `master_password` is provided | `bool` | `false` | no | +| [master\_password](#input\_master\_password) | The password for the primary cluster | `string` | `null` | no | | [master\_username](#input\_master\_username) | The username for the primary cluster | `string` | `"root"` | no | | [monitoring\_interval](#input\_monitoring\_interval) | The interval, in seconds, between points when Enhanced Monitoring metrics are collected for instances. Set to 0 to disble. Default is 0 | `number` | `0` | no | +| [name](#input\_name) | The name of the RDS instance | `string` | `""` | no | +| [namespace](#input\_namespace) | Name of the Kubernetes namespace where the MYSQL deployment will be deployed. | `string` | `"db"` | no | | [performance\_insights\_enabled](#input\_performance\_insights\_enabled) | Specifies whether Performance Insights is enabled or not | `bool` | `null` | no | | [performance\_insights\_kms\_key\_id](#input\_performance\_insights\_kms\_key\_id) | ARN of KMS key to encrypt performance insights data. | `string` | `null` | no | | [performance\_insights\_retention\_period](#input\_performance\_insights\_retention\_period) | Retention period for performance insights data, Either 7 (7 days) or 731 (2 years). | `number` | `null` | no | @@ -182,6 +193,7 @@ Security scanning is graciously provided by Prowler. Proowler is the leading ful | Name | Description | |------|-------------| +| [rds\_cluster\_database\_name](#output\_rds\_cluster\_database\_name) | Name for an automatically created database on cluster creation | | [rds\_cluster\_endpoint](#output\_rds\_cluster\_endpoint) | The endpoint URL of the Aurora cluster | | [rds\_cluster\_master\_password](#output\_rds\_cluster\_master\_password) | The master password for the Aurora cluster | | [rds\_cluster\_master\_username](#output\_rds\_cluster\_master\_username) | The master username for the Aurora cluster | diff --git a/examples/aurora-global/README.md b/examples/aurora-global/README.md index d767c4d..111ef74 100644 --- a/examples/aurora-global/README.md +++ b/examples/aurora-global/README.md @@ -17,7 +17,7 @@ | Name | Source | Version | |------|--------|---------| -| [aurora](#module\_aurora) | squareops/rds-aurora/aws | n/a | +| [aurora](#module\_aurora) | squareops/rds-aurora/aws | 2.2.1 | | [kms](#module\_kms) | terraform-aws-modules/kms/aws | n/a | | [secondary\_vpc](#module\_secondary\_vpc) | squareops/vpc/aws | n/a | | [vpc](#module\_vpc) | squareops/vpc/aws | n/a | diff --git a/examples/aurora/README.md b/examples/aurora/README.md index 3772d22..8c379c0 100644 --- a/examples/aurora/README.md +++ b/examples/aurora/README.md @@ -16,7 +16,7 @@ | Name | Source | Version | |------|--------|---------| -| [aurora](#module\_aurora) | squareops/rds-aurora/aws | n/a | +| [aurora](#module\_aurora) | squareops/rds-aurora/aws | 2.2.1 | | [kms](#module\_kms) | terraform-aws-modules/kms/aws | n/a | | [vpc](#module\_vpc) | squareops/vpc/aws | n/a | @@ -25,6 +25,8 @@ | Name | Type | |------|------| | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | +| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | ## Inputs @@ -35,6 +37,7 @@ No inputs. | Name | Description | |------|-------------| +| [aurora\_cluster\_database\_name](#output\_aurora\_cluster\_database\_name) | The reader endpoint URL of the Aurora cluster | | [aurora\_cluster\_endpoint](#output\_aurora\_cluster\_endpoint) | The endpoint URL of the Aurora cluster | | [aurora\_cluster\_master\_password](#output\_aurora\_cluster\_master\_password) | The master password for the Aurora cluster | | [aurora\_cluster\_master\_username](#output\_aurora\_cluster\_master\_username) | The master username for the Aurora cluster | diff --git a/examples/aurora/helm/values.yaml b/examples/aurora/helm/values.yaml index 7371739..bfd3de9 100644 --- a/examples/aurora/helm/values.yaml +++ b/examples/aurora/helm/values.yaml @@ -45,4 +45,4 @@ restorejob: cpu: 50m limits: memory: 200Mi - cpu: 100m \ No newline at end of file + cpu: 100m diff --git a/examples/aurora/main.tf b/examples/aurora/main.tf index f8f215c..552098f 100644 --- a/examples/aurora/main.tf +++ b/examples/aurora/main.tf @@ -4,25 +4,25 @@ locals { external_id = "" # Define your external ID here assume_role_config = length(local.role_arn) > 0 ? { role_arn = local.role_arn } : null name = "skaf" - region = "us-east-2" - port = 5432 # 3306 + region = "us-east-1" + port = 5432 # 3306 for MySQL family = "aurora-postgresql15" # aurora-mysql5.7" engine = "aurora-postgresql" # aurora-mysql" vpc_cidr = "10.0.0.0/16" environment = "prod" - db_engine_version = "15.2" # 5.7" + db_engine_version = "15.7" # 5.7" db_instance_class = "db.r5.large" - cluster_name = "" + cluster_name = "non-prod-07-feb" create_namespace = false namespace = "mydb" - master_password = "" # Leave this field empty to have a password automatically generated. + master_password = "Amanrajj12" # Leave this field empty to have a password automatically generated. additional_aws_tags = { Owner = "Organization_Name" Expires = "Never" Department = "Engineering" } - current_identity = data.aws_caller_identity.current.arn - allowed_cidr_blocks = ["10.0.0.0/16"] + current_identity = data.aws_caller_identity.current.arn + allowed_cidr_blocks = ["10.0.0.0/16"] } data "aws_caller_identity" "current" {} @@ -90,7 +90,7 @@ module "vpc" { module "aurora" { source = "squareops/rds-aurora/aws" version = "2.2.1" - name = local.name + name = local.name region = local.region role_arn = local.role_arn external_id = local.external_id @@ -129,19 +129,19 @@ module "aurora" { autoscaling_scale_out_cooldown = 30 allowed_cidr_blocks = local.allowed_cidr_blocks ######### - cluster_name = local.cluster_name # cluster name - namespace = local.namespace - create_namespace = local.create_namespace - bucket_provider_type = "s3" - db_backup_enabled = false + cluster_name = local.cluster_name # cluster name where your backup or restore job will run. + namespace = local.namespace + create_namespace = local.create_namespace + bucket_provider_type = "s3" + db_backup_enabled = false db_backup_config = { - mysql_database_name = "" # Specify the database name or Leave empty if you wish to backup all databases - cron_for_full_backup = "*/2 * * * *" # set cronjob for backup - bucket_uri = "s3://aurora-backup-atmosly" # S3 bucket URI (without a trailing slash /) + mysql_database_name = "atmosly_db4" # Specify the database name or Leave empty if you wish to backup all databases + cron_for_full_backup = "*/2 * * * *" # set cronjob for backup + bucket_uri = "s3://my-backup-dumps-databases" # S3 bucket URI (without a trailing slash /) } db_restore_enabled = false db_restore_config = { - bucket_uri = "s3://aurora-backup-atmosly" # S3 bucket URI (without a trailing slash /) containing the backup dump file. - file_name = "mysqldump_20250120_090803.zip" # Give .sql or .zip file for restore + bucket_uri = "s3://my-backup-dumps-databases" # S3 bucket URI (without a trailing slash /) containing the backup dump file. + file_name = "atmosly_db1.sql" # Give .sql or .zip file for restore } } diff --git a/examples/aurora/provider.tf b/examples/aurora/provider.tf index e0c9697..50e7931 100644 --- a/examples/aurora/provider.tf +++ b/examples/aurora/provider.tf @@ -11,11 +11,11 @@ provider "aws" { } data "aws_eks_cluster" "cluster" { - name = "" + name = local.cluster_name } data "aws_eks_cluster_auth" "cluster" { - name = "" + name = local.cluster_name } provider "kubernetes" { diff --git a/main.tf b/main.tf index 1b230a2..ac58f0f 100644 --- a/main.tf +++ b/main.tf @@ -1,10 +1,10 @@ locals { - tags = { + tags = { Automation = "true" Environment = var.environment } region = var.region - secondary_region = var.secondary_region != "null" ? var.secondary_region : null # Check if secondary_region is null + secondary_region = var.secondary_region != "null" ? var.secondary_region : null # Check if secondary_region is null role_arn = var.role_arn external_id = var.external_id assume_role_config = length(var.role_arn) > 0 ? { role_arn = var.role_arn } : null @@ -23,7 +23,7 @@ provider "aws" { provider "aws" { alias = "secondary" - region = local.secondary_region != null ? local.secondary_region : var.region # Fallback to primary region if secondary is null + region = local.secondary_region != null ? local.secondary_region : var.region # Fallback to primary region if secondary is null } data "aws_caller_identity" "current" {} @@ -62,23 +62,23 @@ module "aurora" { # cidr_blocks = var.allowed_cidr_blocks # security_groups = var.allowed_security_groups security_group_rules = { - ingress_postgresql = { - description = "Allow inbound PostgreSQL traffic from trusted CIDR blocks" - type = "ingress" - from_port = var.port - to_port = var.port - protocol = "tcp" - cidr_blocks = var.allowed_cidr_blocks - } - egress_allow_all = { - description = "Allow all outbound traffic" - type = "egress" - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] + ingress_postgresql = { + description = "Allow inbound PostgreSQL traffic from trusted CIDR blocks" + type = "ingress" + from_port = var.port + to_port = var.port + protocol = "tcp" + cidr_blocks = var.allowed_cidr_blocks + } + egress_allow_all = { + description = "Allow all outbound traffic" + type = "egress" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } } -} subnets = var.subnets master_password = var.master_password != "" ? var.master_password : (length(random_password.master) > 0 ? random_password.master[0].result : null) @@ -196,9 +196,9 @@ resource "aws_rds_global_cluster" "this" { } module "aurora_secondary" { - source = "terraform-aws-modules/rds-aurora/aws" - count = var.global_cluster_enable ? 1 : 0 - version = "8.3.0" + source = "terraform-aws-modules/rds-aurora/aws" + count = var.global_cluster_enable ? 1 : 0 + version = "8.3.0" # providers = { aws = aws.secondary } is_primary_cluster = false @@ -242,15 +242,15 @@ module "aurora_secondary" { module "backup_restore" { - depends_on = [module.aurora] - source = "./modules/db-backup-restore" - name = var.name - cluster_name = var.cluster_name - namespace = var.namespace - create_namespace = var.create_namespace - bucket_provider_type = var.bucket_provider_type - engine = var.engine - db_backup_enabled = var.db_backup_enabled + depends_on = [module.aurora] + source = "./modules/db-backup-restore" + name = var.name + cluster_name = var.cluster_name + namespace = var.namespace + create_namespace = var.create_namespace + bucket_provider_type = var.bucket_provider_type + engine = var.engine + db_backup_enabled = var.db_backup_enabled db_backup_config = { db_username = module.aurora.cluster_master_username db_password = var.master_password != "" ? var.master_password : nonsensitive(random_password.master[0].result) @@ -262,10 +262,10 @@ module "backup_restore" { db_restore_enabled = var.db_restore_enabled db_restore_config = { - db_endpoint = module.aurora.cluster_endpoint - db_username = module.aurora.cluster_master_username - db_password = var.master_password != "" ? var.master_password : nonsensitive(random_password.master[0].result) - bucket_uri = var.db_restore_config.bucket_uri - file_name = var.db_restore_config.file_name + db_endpoint = module.aurora.cluster_endpoint + db_username = module.aurora.cluster_master_username + db_password = var.master_password != "" ? var.master_password : nonsensitive(random_password.master[0].result) + bucket_uri = var.db_restore_config.bucket_uri + file_name = var.db_restore_config.file_name } } diff --git a/modules/db-backup-restore/README.md b/modules/db-backup-restore/README.md new file mode 100644 index 0000000..01d3e3f --- /dev/null +++ b/modules/db-backup-restore/README.md @@ -0,0 +1,58 @@ +# db-backup-restore + + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | n/a | +| [helm](#provider\_helm) | n/a | +| [kubernetes](#provider\_kubernetes) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_role.mysql_backup_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.mysql_restore_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [helm_release.db_backup](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.db_restore](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubernetes_namespace.db](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_eks_cluster.kubernetes_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [azure\_container\_name](#input\_azure\_container\_name) | Azure container name | `string` | `""` | no | +| [azure\_storage\_account\_key](#input\_azure\_storage\_account\_key) | Azure storage account key | `string` | `""` | no | +| [azure\_storage\_account\_name](#input\_azure\_storage\_account\_name) | Azure storage account name | `string` | `""` | no | +| [bucket\_provider\_type](#input\_bucket\_provider\_type) | Choose what type of provider you want (s3, gcs) | `string` | `"s3"` | no | +| [cluster\_name](#input\_cluster\_name) | Specifies the name of the EKS cluster to deploy the MySQL application on. | `string` | `""` | no | +| [create\_namespace](#input\_create\_namespace) | Specify whether or not to create the namespace if it does not already exist. Set it to true to create the namespace. | `string` | `false` | no | +| [db\_backup\_config](#input\_db\_backup\_config) | configuration options for MySQL database backups. It includes properties such as the S3 bucket URI, the S3 bucket region, and the cron expression for full backups. | `map(string)` | 
{
  "bucket_uri": "",
  "cron_for_full_backup": "",
  "mysql_database_name": ""
}
| no | +| [db\_backup\_enabled](#input\_db\_backup\_enabled) | Specifies whether to enable backups for MySQL database. | `bool` | `false` | no | +| [db\_permission](#input\_db\_permission) | access | `bool` | `false` | no | +| [db\_restore\_config](#input\_db\_restore\_config) | Configuration options for restoring dump to the MySQL database. | `any` | 
{
  "bucket_uri": "",
  "file_name": ""
}
| no | +| [db\_restore\_enabled](#input\_db\_restore\_enabled) | Specifies whether to enable restoring dump to the MySQL database. | `bool` | `false` | no | +| [engine](#input\_engine) | The name of the database engine to be used for this DB cluster | `string` | `"aurora"` | no | +| [iam\_role\_arn\_backup](#input\_iam\_role\_arn\_backup) | IAM role ARN for backup (AWS) | `string` | `""` | no | +| [iam\_role\_arn\_restore](#input\_iam\_role\_arn\_restore) | IAM role ARN for restore (AWS) | `string` | `""` | no | +| [name](#input\_name) | Name identifier for module to be added as suffix to resources | `string` | `"test"` | no | +| [namespace](#input\_namespace) | Name of the Kubernetes namespace where the MYSQL deployment will be deployed. | `string` | `"db"` | no | +| [service\_account\_backup](#input\_service\_account\_backup) | Service account for backup (GCP) | `string` | `""` | no | +| [service\_account\_restore](#input\_service\_account\_restore) | Service account for restore (GCP) | `string` | `""` | no | + +## Outputs + +No outputs. + diff --git a/modules/db-backup-restore/backup/templates/cronjob.yaml b/modules/db-backup-restore/backup/templates/cronjob.yaml index b4832bd..3445e8d 100644 --- a/modules/db-backup-restore/backup/templates/cronjob.yaml +++ b/modules/db-backup-restore/backup/templates/cronjob.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: sa-db-backup containers: - name: backup-database - image: {{ if eq .Values.backup.engine "aurora-mysql" }}amanravi12/rds-mysql-backup:v1{{ else }}squareops01/rds-postgresql-backup:v1{{ end }} + image: {{ if eq .Values.backup.engine "aurora-mysql" }}squareops01/rds-mysql-backup:v1{{ else }}squareops01/rds-postgresql-backup:v2{{ end }} imagePullPolicy: Always env: - name: {{if eq .Values.backup.engine "aurora-mysql"}}MYSQL_HOST{{ else }}DB_HOST{{ end }} @@ -33,4 +33,4 @@ spec: value: {{ .Values.backup.database_password }} - name: {{if eq .Values.backup.engine "aurora-mysql"}}MYSQL_BUCKET_URI{{ else }}S3_BUCKET{{ end }} value: {{ .Values.backup.bucket_uri }} - resources: {{ .Values.backupjob.resources | toYaml | nindent 12 }} \ No newline at end of file + resources: {{ .Values.backupjob.resources | toYaml | nindent 12 }} diff --git a/modules/db-backup-restore/main.tf b/modules/db-backup-restore/main.tf index 3b54d1f..32f801e 100644 --- a/modules/db-backup-restore/main.tf +++ b/modules/db-backup-restore/main.tf @@ -58,7 +58,3 @@ resource "helm_release" "db_restore" { }) ] } - - - - diff --git a/modules/db-backup-restore/restore/templates/job.yaml b/modules/db-backup-restore/restore/templates/job.yaml index 8ff3274..423844a 100644 --- a/modules/db-backup-restore/restore/templates/job.yaml +++ b/modules/db-backup-restore/restore/templates/job.yaml @@ -9,7 +9,7 @@ spec: serviceAccountName: sa-db-restore containers: - name: restore-job - image: {{ if eq .Values.restore.engine "aurora-mysql" }}amanravi12/rds-aurora-mysql-restore:v1{{ else }}squareops01/rds-postgresql-restore:v1{{ end }} + image: {{ if eq .Values.restore.engine "aurora-mysql" }}squareops01/rds-mysql-restore:v1{{ else }}squareops01/rds-postgresql-restore:v2{{ end }} imagePullPolicy: Always env: - name: {{if eq .Values.restore.engine "aurora-mysql"}}MYSQL_HOST{{ else }}DB_HOST{{ end }} diff --git a/modules/db-backup-restore/roles.tf b/modules/db-backup-restore/roles.tf index e08ac2c..617c0cc 100644 --- a/modules/db-backup-restore/roles.tf +++ b/modules/db-backup-restore/roles.tf @@ -13,8 +13,8 @@ data "aws_eks_cluster" "kubernetes_cluster" { } resource "aws_iam_role" "mysql_backup_role" { - count = var.db_backup_enabled ? 1 : 0 - name = format("%s-%s-%s", var.cluster_name, var.name, "db-backup") + count = var.db_backup_enabled ? 1 : 0 + name = format("%s-%s-%s", var.cluster_name, var.name, "db-backup") assume_role_policy = jsonencode({ Version = "2012-10-17", Statement = [ @@ -57,8 +57,8 @@ resource "aws_iam_role" "mysql_backup_role" { resource "aws_iam_role" "mysql_restore_role" { - count = var.db_restore_enabled ? 1 : 0 - name = format("%s-%s-%s", var.cluster_name, var.name, "db-restore") + count = var.db_restore_enabled ? 1 : 0 + name = format("%s-%s-%s", var.cluster_name, var.name, "db-restore") assume_role_policy = jsonencode({ Version = "2012-10-17", Statement = [ @@ -97,4 +97,4 @@ resource "aws_iam_role" "mysql_restore_role" { ] }) } -} \ No newline at end of file +} diff --git a/modules/db-backup-restore/variables.tf b/modules/db-backup-restore/variables.tf index 3fbc0bb..b8042ba 100644 --- a/modules/db-backup-restore/variables.tf +++ b/modules/db-backup-restore/variables.tf @@ -66,9 +66,9 @@ variable "cluster_name" { } variable "db_permission" { - default = false + default = false description = "access" - type = bool + type = bool } variable "bucket_provider_type" { @@ -93,11 +93,11 @@ variable "db_restore_enabled" { variable "db_backup_config" { type = map(string) default = { - bucket_uri = "" + bucket_uri = "" # s3_bucket_region = "" cron_for_full_backup = "" mysql_database_name = "" - # port = "" + # port = "" } description = "configuration options for MySQL database backups. It includes properties such as the S3 bucket URI, the S3 bucket region, and the cron expression for full backups." } @@ -105,8 +105,8 @@ variable "db_backup_config" { variable "db_restore_config" { type = any default = { - bucket_uri = "" - file_name = "" + bucket_uri = "" + file_name = "" # s3_bucket_region = "" } description = "Configuration options for restoring dump to the MySQL database." @@ -116,4 +116,4 @@ variable "engine" { description = "The name of the database engine to be used for this DB cluster" type = string default = "aurora" -} \ No newline at end of file +} diff --git a/outputs.tf b/outputs.tf index ad98641..d348ceb 100644 --- a/outputs.tf +++ b/outputs.tf @@ -18,7 +18,7 @@ output "secondary_rds_cluster_reader_endpoint" { value = var.global_cluster_enable ? module.aurora_secondary[0].cluster_reader_endpoint : null } -output "rds_cluster_database_name"{ +output "rds_cluster_database_name" { description = "Name for an automatically created database on cluster creation" value = module.aurora.cluster_database_name } diff --git a/variables.tf b/variables.tf index 18ac65b..bc59db6 100644 --- a/variables.tf +++ b/variables.tf @@ -435,7 +435,7 @@ variable "bucket_provider_type" { variable "db_backup_config" { type = map(string) default = { - bucket_uri = "" + bucket_uri = "" # s3_bucket_region = "" cron_for_full_backup = "" mysql_database_name = "" @@ -447,9 +447,9 @@ variable "db_backup_config" { variable "db_restore_config" { type = any default = { - bucket_uri = "" - file_name = "" + bucket_uri = "" + file_name = "" # s3_bucket_region = "" } description = "Configuration options for restoring dump to the MySQL database." -} \ No newline at end of file +} From b23716eb8a9afbd021eb50fcb75352e62289718d Mon Sep 17 00:00:00 2001 From: Aman <146931382+amanravi-squareops@users.noreply.github.com> Date: Fri, 7 Feb 2025 15:48:24 +0530 Subject: [PATCH 4/6] Update main.tf --- examples/aurora/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/aurora/main.tf b/examples/aurora/main.tf index 552098f..aa00f5d 100644 --- a/examples/aurora/main.tf +++ b/examples/aurora/main.tf @@ -12,7 +12,7 @@ locals { environment = "prod" db_engine_version = "15.7" # 5.7" db_instance_class = "db.r5.large" - cluster_name = "non-prod-07-feb" + cluster_name = "" create_namespace = false namespace = "mydb" master_password = "Amanrajj12" # Leave this field empty to have a password automatically generated. From 044762ef31460647eed20076175629597f982630 Mon Sep 17 00:00:00 2001 From: Aman <146931382+amanravi-squareops@users.noreply.github.com> Date: Wed, 19 Feb 2025 14:55:57 +0530 Subject: [PATCH 5/6] Update main.tf --- examples/aurora/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/aurora/main.tf b/examples/aurora/main.tf index aa00f5d..4885bd3 100644 --- a/examples/aurora/main.tf +++ b/examples/aurora/main.tf @@ -15,7 +15,7 @@ locals { cluster_name = "" create_namespace = false namespace = "mydb" - master_password = "Amanrajj12" # Leave this field empty to have a password automatically generated. + master_password = "" # Leave this field empty to have a password automatically generated. additional_aws_tags = { Owner = "Organization_Name" Expires = "Never" From 5b852c3c34607b3f101fab81c083bacd0b81e6d8 Mon Sep 17 00:00:00 2001 From: Aman <146931382+amanravi-squareops@users.noreply.github.com> Date: Wed, 19 Feb 2025 14:58:15 +0530 Subject: [PATCH 6/6] Update main.tf version --- examples/aurora/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/aurora/main.tf b/examples/aurora/main.tf index 4885bd3..5c59f3f 100644 --- a/examples/aurora/main.tf +++ b/examples/aurora/main.tf @@ -89,7 +89,7 @@ module "vpc" { module "aurora" { source = "squareops/rds-aurora/aws" - version = "2.2.1" + version = "3.0.0" name = local.name region = local.region role_arn = local.role_arn