Add deployer type node to the codebase

Author: andysingleton

ansible/roles/dataiku-dss/files/dataiku-api-manager/dataiku_api_manager/main.py

@@ -25,7 +25,7 @@ def main():
     system_config.load_system_config()
     dss_config = Config(
         aws_region=system_config.aws_settings["aws_region"],
-        sub_categories=['api', 'design', 'automation'],
+        sub_categories=['api', 'design', 'automation', 'deployer'],
         my_category=system_config.node_type
     )
     dss_config.load_data(system_config.get_config_data())
@@ -77,6 +77,8 @@ def main():
     if system_config.node_type == 'api':
         configurator.action_store_admin_api_token()
 
+    if system_config.node_type == 'deployer':
+        configurator.action_store_admin_api_token()
 
 if __name__ == "__main__":
     # execute only if run as a script

terraform/environments/codebuild/dss/deployer/data.tf

@@ -0,0 +1,9 @@
+# Uncomment to retrieve PAT from AWS Parameter Store
+#data "aws_ssm_parameter" "github_pat" {
+#  name = "/dss/github_personal_access_token"
+#}
+
+# Uncomment to retrieve PAT from Hasihcorp Vault
+#data "vault_generic_secret" "github_pat" {
+#  path = var.vault_path_github_api_key
+#}

terraform/environments/codebuild/dss/deployer/main.tf

@@ -0,0 +1,20 @@
+module "codebuild_automation" {
+  source                     = "../../../../modules/codebuild-packer"
+  # source                     = "osodevops/dataiku-dss/codebuild-packer#123456"
+  additional_build_variables = { "NODE_TYPE": "deployer" }
+  encrypt_ami                = var.encrypt_ami
+  kms_key_arn                = var.kms_key_arn
+  instance_type              = var.packer_instance_type
+  packer_file_location       = var.packer_file_location
+  project_name               = var.project_name
+  region                     = var.aws_region
+  root_volume_size           = var.root_volume_size
+  shared_ami_users           = var.shared_ami_users
+  source_image_account_no    = var.source_image_account_no
+  source_image_name          = var.source_image_name
+  source_repository_url      = var.source_repository_url
+  subnet_name_filter         = var.build_subnet_name_filter
+  vpc_name                   = var.vpc_name
+  # github_api_token           = data.aws_ssm_parameter.github_pat.value  # Uncomment to provide PAT from Parameter Store
+  # github_api_token           = data.vault_generic_secret.github_pat  # Uncomment to provide PAT from Vault
+}

terraform/environments/codebuild/dss/deployer/provider.tf

@@ -0,0 +1,18 @@
+provider "aws" {
+  region = var.aws_region
+  default_tags {
+    tags = var.common_tags
+  }
+}
+
+#provider "vault" {
+#  address = var.vault_address
+#  auth_login {
+#    path   = "auth/${var.vault_env}/login"
+#    method = "aws"
+#    parameters = {
+#      role = var.vault_role
+#    }
+#  }
+#  skip_tls_verify = true
+#}

terraform/environments/codebuild/dss/deployer/terraform.tf

@@ -0,0 +1,18 @@
+terraform {
+  required_version = "~> 1.7.0"
+
+  backend "s3" {
+    bucket         = "MY-BUCKET-NAME"
+    dynamodb_table = "MY-DYNAMODB-LOCKING-TABLE-tf-state-lock"
+    region         = "MY-AWS-REGION"
+    key            = "dss/codebuild/deployer/terraform.tfstate"
+    encrypt        = true
+  }
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+  }
+}

terraform/environments/codebuild/dss/deployer/terraform.tfvars

@@ -0,0 +1,47 @@
+# AWS region for resources
+aws_region = "YOUR-AWS-REGION"
+
+# Name of the VPC subnets to use in building
+build_subnet_name_filter = "*Private*"
+
+# Tags to add to all Terraform resources that support them
+common_tags = {
+  Application = "CodeBuild"
+  Environment = "tooling"
+  Function    = "Dataiku AMI building"
+  Tooling     = "Terraform"
+}
+
+# Size and type of the instance used to build the image.
+packer_instance_type = "t3.xlarge"
+
+# Name of the project as it will appear in CodeBuild
+project_name = "build-dss-ami-deployer"
+
+# Volume size for the root partition during the build process
+root_volume_size = "40"
+
+# Additional AWS account numbers to share the final AMI to
+shared_ami_users = ""
+
+# Owning account of the source image
+source_image_account_no = "amazon"
+
+# Name of the source image to base the DSS build on
+source_image_name = "amzn2-ami-hvm-*.*.*.*-*-gp2"
+
+# Repository used to get the ansible and packer build code
+source_repository_url = "https://github.com/osodevops/terraform-aws-dataiku-platform"
+
+# Name of the VPC to use for AMI builds
+vpc_name = "MY-VPC"
+
+# Vault configuration
+# To enable vault as a store for the github token, uncomment vault blocks in these files:
+# - provider.tf
+# - data.tf
+# - main.tf
+vault_env      = "MY-VAULT-ENVIRONMENT"
+vault_role     = "MY-VAULT-ROLE"
+vault_address  = "https://MY-VAULT-URL"
+vault_skip_tls = true

terraform/environments/codebuild/dss/deployer/variables.tf

@@ -0,0 +1,105 @@
+variable "aws_region" {
+  description = "The AWS region in which all resources will be created"
+  type        = string
+}
+
+variable "build_subnet_name_filter" {
+  description = "Used to filter by 'Name' tag for the subnets we will be building in"
+  type        = string
+  default     = "Private*"
+}
+
+variable "common_tags" {
+  description = "A collection of common tags to be added to resources."
+  type        = map(string)
+}
+
+variable "encrypt_ami" {
+  description = "Encrypt AMI after successful build."
+  type        = string
+  default     = false
+}
+
+variable "kms_key_arn" {
+  description = "If Encrypt_ami set to true then you must pass in the arn of the key you wish to encrypt disk with."
+  type        = string
+  default     = ""
+}
+
+variable "packer_file_location" {
+  description = "The file path of the .json packer to build."
+  type        = string
+  default     = "packer/dataiku-dss.json"
+}
+
+variable "packer_instance_type" {
+  description = "Instance type used by packer to build"
+  type        = string
+  default     = "m5a.2xlarge"
+}
+
+variable "project_name" {
+  description = "Name of the CodeBuild Project"
+  type        = string
+}
+
+variable "root_volume_size" {
+  description = "Specify the root volume size for the built image"
+  type        = string
+  default     = "150"
+}
+
+variable "shared_ami_users" {
+  description = "List of user accounts to share the built AMI with"
+  type        = string
+  default     = ""
+}
+
+variable "source_image_account_no" {
+  description = "Account number owning the source image for the packer build"
+  type        = string
+  default     = "amazon"
+}
+
+variable "source_image_name" {
+  description = "Name of an AMI to base the build on"
+  type        = string
+  default     = "amzn2-ami-hvm-*.*.*.*-*-gp2"
+}
+
+variable "source_repository_url" {
+  description = "The source repository for the packer and ansible build code"
+  type        = string
+}
+
+variable "vpc_name" {
+  description = "Name of the VPC we will be building in"
+  type        = string
+}
+
+variable "vault_env" {
+  description = "the vault environment variable."
+  type        = string
+}
+
+variable "vault_role" {
+  description = "the vault role to enable permissions to access the vault installation."
+  type        = string
+}
+
+variable "vault_address" {
+  description = "the vault host address pointing to the vault installation."
+  type        = string
+}
+
+variable "vault_path_github_api_key" {
+  description = "Vault path of the github api key secret"
+  type        = string
+  default     = ""
+}
+
+variable "vault_skip_tls" {
+  description = "Whether to skip TLS verification for simple Vault installations"
+  type        = bool
+  default     = true
+}

terraform/environments/common/dss/iam/terraform.tfvars

@@ -6,7 +6,7 @@ common_tags = {
 }
 
 # Creates security groups named "<item>-access" for use in cross-node access
-security_group_target_names = ["automation", "api", "design"]
+security_group_target_names = ["automation", "api", "design", "deployer"]
 
 # The name (not ID) of the VPC we are deploying to
 vpc_name = "MY-VPC-NAME"

terraform/environments/pre-prod/dss/deployer/main.tf

@@ -0,0 +1,51 @@
+module "dss_design" {
+  source                        = "../../../../modules/dss-node"
+  ami_name_filter               = var.ami_name_filter
+  ami_owner_account             = var.ami_owner_account
+  asg_desired_capacity          = var.asg_desired_capacity
+  aws_region                    = var.aws_region
+  cloudwatch_alarm_topic_name   = var.cloudwatch_alarm_sns_topic_name
+  data_volume_device_name       = var.data_volume_device_name
+  data_volume_encrypt           = var.data_volume_encrypt
+  data_volume_iops              = var.data_volume_iops
+  data_volume_kms_key           = var.data_volume_kms_key
+  data_volume_mount_point       = var.data_volume_mount_point
+  data_volume_size              = var.data_volume_size
+  data_volume_type              = var.data_volume_type
+  dlm_target_instance_tag       = var.dlm_target_instance_tag
+  dr_target_instance_tag        = var.dr_target_instance_tag
+  dss_node_type                 = var.dss_node_type
+  dss_s3_config_bucket          = var.dss_s3_config_bucket
+  dss_s3_config_key             = var.dss_s3_config_key
+  dss_service_port              = var.dss_service_port
+  dss_service_protocol          = var.dss_service_protocol
+  environment                   = var.environment
+  instance_allowed_ips          = var.instance_allowed_ips
+  instance_type                 = var.instance_type
+  lb_allowed_ips                = var.lb_allowed_ips
+  lb_allow_security_groups      = var.lb_allow_security_groups
+  lb_certificate_arn            = var.lb_certificate_arn
+  lb_enable_load_balancer       = var.lb_enable_load_balancer
+  lb_enable_deletion_protection = var.lb_enable_deletion_protection
+  lb_health_check_path          = var.lb_health_check_path
+  lb_health_check_port          = var.lb_health_check_port
+  lb_health_check_protocol      = var.lb_health_check_protocol
+  lb_health_check_response_code = var.lb_health_check_response_code
+  lb_https_port                 = var.lb_https_port
+  lb_internal                   = var.lb_internal
+  lb_logs_s3_enabled            = var.lb_logs_s3_enabled
+  lb_log_s3_bucket_name         = var.lb_log_s3_bucket_name
+  private_subnet_name_filter    = var.private_subnet_name_filter
+  public_subnet_name_filter     = var.public_subnet_name_filter
+  r53_enable_private_zone       = var.r53_enable_private_zone
+  r53_enable_public_zone        = var.r53_enable_public_zone
+  r53_zone_name                 = var.r53_zone_name
+  root_volume_size              = var.root_volume_size
+  s3_allow_instance_bucket      = var.s3_allow_instance_bucket
+  s3_create_instance_bucket     = var.s3_create_instance_bucket
+  s3_instance_bucket_name       = var.s3_instance_bucket_name
+  s3_session_logging_bucket_arn = var.s3_session_logging_bucket_arn
+  ssh_key_name                  = var.ssh_key_name
+  vpc_name                      = var.vpc_name
+}
+

terraform/environments/pre-prod/dss/deployer/providers.tf

@@ -0,0 +1,6 @@
+provider "aws" {
+  region = var.aws_region
+  default_tags {
+    tags = var.common_tags
+  }
+}

terraform/environments/pre-prod/dss/deployer/terraform.tf

@@ -0,0 +1,18 @@
+terraform {
+  required_version = "~> 1.7.0"
+
+  backend "s3" {
+    bucket         = "MY-BUCKET-NAME"
+    dynamodb_table = "MY-DYNAMODB-LOCKING-TABLE-tf-state-lock"
+    region         = "MY-AWS-REGION"
+    key            = "dss/preprod/deployer/terraform.tfstate"
+    encrypt        = true
+  }
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+  }
+}