init

.github/workflows/main.yml

@@ -0,0 +1,27 @@
+name: kafka golden-image
+
+on:
+  push:
+    branches: 
+      - 'test-workflow'
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+*'
+      - '[0-9]+.[0-9]+.[0-9]+*'
+      - 'v[0-9]+.[0-9]+.[0-9]+-(CR|Beta|milestone)[0-9]*'
+      - '[0-9]+.[0-9]+.[0-9]+-(CR|Beta|milestone)[0-9]*'
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  goldenimage:
+    uses: inqwise/ansible-automation-toolkit/.github/workflows/goldenimage-workflow.yml@default
+    with:
+      S3_BASE_PATH: "s3://bootstrap-pension-stg/playbooks"
+      ROLE: "arn:aws:iam::339712742264:role/ansible-playbook-workflow-role"
+      REGION: "eu-west-1"
+      CPU_ARCH: "arm64"
+      OS: "amzn2023"
+      APP: "kafka"
+      TOOLKIT_VERSION: "default"

.gitignore

@@ -0,0 +1,7 @@
+/**/.*
+/**/.*/
+!/.gitignore
+!/.github
+manifest.json
+test_main.sh
+test_main.yml

Vagrantfile

@@ -0,0 +1,90 @@
+# spot support: 
+# vagrant plugin install vagrant-aws-mkubenka --plugin-version "0.7.2.pre.24"
+# classic:
+# vagrant plugin install vagrant-aws 
+# vagrant up --provider=aws
+# vagrant destroy -f && vagrant up --provider=aws
+
+## optional:
+# export COMMON_COLLECTION_PATH='~/git/inqwise/ansible/ansible-common-collection'
+# export STACKTREK_COLLECTION_PATH='~/git/inqwise/ansible/ansible-stack-trek'
+
+MAIN_SCRIPT_URL = "https://raw.githubusercontent.com/inqwise/ansible-automation-toolkit/default/main_amzn2023.sh"
+TOPIC_NAME = "pre_playbook_errors"
+ACCOUNT_ID = "339712742264"
+AWS_REGION = "eu-west-1"
+MAIN_SH_ARGS = <<MARKER
+-e "playbook_name=ansible-kafka discord_message_owner_name=#{Etc.getpwuid(Process.uid).name}"
+MARKER
+NODE_COUNT = 1
+CLUSTER_NAME = "#{Etc.getpwuid(Process.uid).name}-test"
+Vagrant.configure("2") do |config|
+  (1..NODE_COUNT).each do |i|
+    config.vm.define "node#{i}" do |subconfig|
+      subconfig.vm.provision "shell", inline: <<-SHELL
+      #   set -euxo pipefail
+      #   echo "start vagrant file"
+      #   source /deployment/ansibleenv/bin/activate
+      #   cd /deployment/playbook
+      #   export ANSIBLE_VERBOSITY=0
+      #   export ANSIBLE_DISPLAY_SKIPPED_HOSTS=false
+      #   export VAULT_PASSWORD=#{`op read "op://Security/ansible-vault tamal-pension-stg/password"`.strip!}
+      #   echo "$VAULT_PASSWORD" > vault_password
+      #   bash main.sh #{MAIN_SH_ARGS}
+      #   rm vault_password
+      # ---------------------
+      
+        set -euxo pipefail
+        echo "start vagrant file"
+        cd /vagrant
+        python3 -m venv /tmp/ansibleenv
+        source /tmp/ansibleenv/bin/activate
+        aws s3 cp s3://resource-pension-stg/get-pip.py - | python3
+        cd /vagrant
+        export VAULT_PASSWORD=#{`op read "op://Security/ansible-vault tamal-pension-stg/password"`.strip!}
+        echo "$VAULT_PASSWORD" > vault_password
+        export ANSIBLE_VERBOSITY=0
+        export ANSIBLE_DISPLAY_SKIPPED_HOSTS=false
+        if [ -f "main.sh" ]; then
+          echo "Local main.sh found. Run the local main.sh script..."
+          bash main.sh #{MAIN_SH_ARGS}
+        else
+          echo "Local main.sh not found. running the main.sh script from the URL..."
+          curl -s https://raw.githubusercontent.com/inqwise/ansible-automation-toolkit/default/main_amzn2023.sh | bash -s -- #{MAIN_SH_ARGS}
+        fi
+        rm vault_password
+      SHELL
+
+      subconfig.vm.provider :aws do |aws, override|
+        override.vm.box = "dummy"
+        override.ssh.username = "ec2-user"
+        override.ssh.private_key_path = "~/.ssh/id_rsa"
+        aws.access_key_id             = `op read "op://Security/aws pension-stg/Security/Access key ID"`.strip!
+        aws.secret_access_key         = `op read "op://Security/aws pension-stg/Security/Secret access key"`.strip!
+        aws.keypair_name = Etc.getpwuid(Process.uid).name
+        override.vm.allowed_synced_folder_types = [:rsync]
+        override.vm.synced_folder ".", "/vagrant", type: :rsync, rsync__exclude: ['.git/','inqwise/'], disabled: false
+        common_collection_path = ENV['COMMON_COLLECTION_PATH'] || '~/git/ansible-common-collection'
+        stacktrek_collection_path = ENV['STACKTREK_COLLECTION_PATH'] || '~/git/ansible-stack-trek'
+        override.vm.synced_folder common_collection_path, '/vagrant/collections/ansible_collections/inqwise/common', type: :rsync, rsync__exclude: '.git/', disabled: false
+        override.vm.synced_folder stacktrek_collection_path, '/vagrant/collections/ansible_collections/inqwise/stacktrek', type: :rsync, rsync__exclude: '.git/', disabled: false
+
+        aws.region = AWS_REGION
+        aws.security_groups = ["sg-077f8d7d58d420467","sg-0e5812f76f107c47a", "sg-01707e90d708616d7"]
+            # public-ssh, kafka, consul
+        aws.ami = "ami-0fa86d752d8b7d1ff"
+        aws.instance_type = "r6g.medium"
+        aws.subnet_id = "subnet-0331d92e81f166c9f"
+        aws.associate_public_ip = true
+        aws.iam_instance_profile_name = "bootstrap-role"
+        aws.tags = {
+          Name: "kafka-test#{i}-#{Etc.getpwuid(Process.uid).name}",
+          kafka_cluster: "#{CLUSTER_NAME}",
+          private_dns: "kafka-test#{i}-#{Etc.getpwuid(Process.uid).name}",
+          node_id: "#{i}",
+          quorum_voters: "1@kafka-test1-#{Etc.getpwuid(Process.uid).name}.opinion-stg.local:9093,2@kafka-test2-#{Etc.getpwuid(Process.uid).name}.opinion-stg.local:9093,3@kafka-test3-#{Etc.getpwuid(Process.uid).name}.opinion-stg.local:9093"
+        }
+      end
+    end
+  end
+end

goldenimage.pkr.hcl

@@ -0,0 +1,230 @@
+//-on-error=abort
+// packer build --only=amazon-ebs.amzn2023_arm64 -var cpu_arch=arm64 -var 'aws_profile=opinion-stg' -var 'tag=latest' -var app=consul -var 'aws_region=il-central-1' .
+packer {
+  required_plugins {
+    amazon = {
+      version = ">= 1.2.2" # preferably "~> 1.2.0" for latest patch version
+      source = "github.com/hashicorp/amazon"
+    }
+  }
+}
+
+
+
+
+######## 
+
+
+
+
+variable "cpu_arch" {
+  description = "The CPU architecture type (e.g., arm64 or x86)."
+  type        = string
+  default     = "arm64"
+}
+
+variable "instance_type" {
+  type = string
+  default = ""
+}
+
+variable "base_path" {
+  description = "The s3 base path to playbooks (e.g., s3://bootstrap-inqwise-org/playbooks)."
+  type = string
+  default = "s3://bootstrap-opinion-stg/playbooks"
+}
+
+variable "tag" {
+  description = "The version of image"
+  type    = string
+}
+
+variable "aws_region" {
+  type    = string
+}
+
+variable "aws_iam_instance_profile" {
+  type    = string
+  default = "PackerRole"
+}
+
+variable "aws_profile" {
+  type    = string
+  default = ""
+}
+
+variable "app" {
+  description = "The app name. for example 'consul'"
+  type    = string
+}
+
+variable "toolkit_version" {
+  description = "automation toolkit repository release version. for example 'v1'"
+  type    = string
+  default = "default"
+}
+
+variable "verbose" {
+  type    = bool
+  default = false
+}
+
+variable "skip_remote_requirements" {
+  type    = bool
+  default = false
+}
+
+######## 
+
+data "amazon-secretsmanager" "vault_secret" {
+  name = "vault_secret"
+  region = "${var.aws_region}"
+  profile = "${var.aws_profile}"
+
+}
+
+######## 
+
+locals {
+  instance_types = {
+    arm64 = var.instance_type != "" ? var.instance_type : "t4g.small"
+    x86   = var.instance_type != "" ? var.instance_type : "t3.small"
+  }
+
+  playbook_name = "ansible-${var.app}"
+  common_build_settings = {
+    shell_provisioners = {
+        inline = !fileexists("goldenimage-test.sh") ? [
+        "curl --connect-timeout 2.37 -m 20 -o /tmp/goldenimage.sh https://raw.githubusercontent.com/inqwise/ansible-automation-toolkit/${var.toolkit_version}/packer/goldenimage.sh",
+        "bash /tmp/goldenimage.sh",
+        ] : [],
+        scripts = fileexists("goldenimage-test.sh") ? [
+            "goldenimage-test.sh"
+        ] : []
+        environment_vars = [
+          "PLAYBOOK_NAME=${local.playbook_name}",
+          "PLAYBOOK_BASE_URL=${var.base_path}",
+          "REGION=${var.aws_region}",
+          "VAULT_PASSWORD=${data.amazon-secretsmanager.vault_secret.value}",
+          "PLAYBOOK_VERSION=${var.tag}",
+          "TOOLKIT_VERSION=${var.toolkit_version}",
+          "VERBOSE=${var.verbose}",
+          "SKIP_REMOTE_REQUIREMENTS=${var.skip_remote_requirements}"
+        ]
+
+    }
+
+    post_processors = {
+      manifest = {
+        type       = "manifest"
+        output     = "manifest.json"
+        strip_path = true
+        custom_data = {
+          app       = var.app
+          version   = var.tag
+          profile   = var.aws_profile
+          region    = var.aws_region
+        }
+      }
+    }
+  }
+
+  timestamp = formatdate("YYYYMMDDhhmm", timestamp())
+}
+
+
+
+######## 
+
+
+
+source "amazon-ebs" "common" {
+  force_deregister      = true
+  force_delete_snapshot = true
+  ami_name              = "${var.app}-${var.tag}"
+  ami_description       = "Image of ${var.app} version ${var.tag}"
+  spot_instance_types   = ["${local.instance_types[var.cpu_arch]}"]
+  region                = "${var.aws_region}"
+  #ami_regions           = ["us-west-2"]
+  #ami_users             = ["123456789012", "987654321098"]  # List of AWS Account IDs granted launch permissions for the created AMI
+  encrypt_boot          = false
+  profile               = "${var.aws_profile}"
+  iam_instance_profile  = "${var.aws_iam_instance_profile}"
+  ssh_username          = "ec2-user"
+  spot_price            = "auto"
+  skip_create_ami       = false # for debug
+
+  metadata_options {
+    instance_metadata_tags = "enabled"
+    http_endpoint               = "enabled"
+    http_put_response_hop_limit = "1"
+    http_tokens                 = "required"
+  }
+
+  run_tags = {
+    Name      = "${var.app}-${var.tag}-packer"
+    app       = "${var.app}"
+    version   = "${var.tag}"
+    timestamp = "${local.timestamp}"
+    playbook_name = "${local.playbook_name}"
+  }
+
+  tags = {
+    Name      = "${var.app}-${var.tag}"
+    app       = "${var.app}"
+    version   = "${var.tag}"
+    timestamp = "${local.timestamp}"
+  }
+}
+
+build {
+  source "source.amazon-ebs.common" {
+    name = "amzn2023_arm64"
+    source_ami_filter {
+      filters={
+        name                = "al2023-ami-2023.*-kernel-6.1-arm64"
+        root-device-type    = "ebs"
+        virtualization-type = "hvm"
+      }
+      most_recent = true 
+      owners      = ["amazon"]
+    }
+  }
+
+  source "source.amazon-ebs.common" {
+    name = "amzn2_x86"
+    source_ami_filter {
+      filters={
+        name                = "amzn2-ami-kernel-5.*-x86_64-gp2"
+        root-device-type    = "ebs"
+        virtualization-type = "hvm"
+      }
+      most_recent = true 
+      owners      = ["amazon"]
+    }
+  }
+
+  provisioner "shell" {
+    scripts = local.common_build_settings.shell_provisioners.scripts
+    inline = local.common_build_settings.shell_provisioners.inline
+    environment_vars = local.common_build_settings.shell_provisioners.environment_vars
+  }
+
+  post-processor "manifest" {
+    output     = local.common_build_settings.post_processors.manifest.output
+    strip_path = local.common_build_settings.post_processors.manifest.strip_path
+    custom_data = local.common_build_settings.post_processors.manifest.custom_data
+  }
+
+  post-processor "shell-local" {
+    inline = [
+      "if [ -f ./goldenimage-postprocess-test.sh ]; then",
+      "    echo 'Executing local script: goldenimage-postprocess-test.sh';",
+      "    bash ./goldenimage-postprocess-test.sh;",
+      "else",
+      "    echo 'Local script not found. Executing remote script: https://raw.githubusercontent.com/inqwise/ansible-automation-toolkit/default/packer/goldenimage-postprocess.sh';",
+      "    curl -s https://raw.githubusercontent.com/inqwise/ansible-automation-toolkit/default/packer/goldenimage-postprocess.sh | bash;",
+      "fi"
+    ]
+  }
+}

goldenimage.txt

@@ -0,0 +1 @@
+curl -O https://raw.githubusercontent.com/inqwise/ansible-automation-toolkit/default/packer/goldenimage.pkr.hcl && packer build --only=amazon-ebs.amzn2023_arm64 -var cpu_arch=arm64 -var 'aws_profile=pension-stg' -var 'tag=latest' -var app=kafka -var 'aws_region=eu-west-1' -var 'base_path=s3://bootstrap-pension-stg/playbooks' goldenimage.pkr.hcl

requirements.yml

@@ -0,0 +1,8 @@
+---
+roles:
+  - name: users
+    src: robertdebock.users
+
+collections:
+  - name: https://github.com/inqwise/ansible-common-collection/releases/download/v1/inqwise-common-1.0.0.tar.gz
+  - name: https://github.com/inqwise/ansible-stack-trek/releases/download/v1/inqwise-stacktrek-1.0.0.tar.gz