chore(jenkinsfile) adds garbage collector as distinct job for azure v…

…ms (#1526) * feat(jenkinsfile):Added garbage collector and respective aws cleanup scripts Signed-off-by: jayfranco999 <[email protected]> * chore(jenkinsfile) adds garbage collector for azure vms Signed-off-by: jayfranco999 <[email protected]> * Apply suggestions from code review Co-authored-by: Damien Duportal <[email protected]> * fixup Signed-off-by: jayfranco999 <[email protected]> * fixup Signed-off-by: jayfranco999 <[email protected]> --------- Signed-off-by: jayfranco999 <[email protected]> Co-authored-by: Damien Duportal <[email protected]>
jenkins-infra · Nov 15, 2024 · df1c46d · df1c46d
1 parent 385caea
commit df1c46d
Show file tree

Hide file tree

Showing 4 changed files with 230 additions and 0 deletions.
diff --git a/Jenkinsfile_gc b/Jenkinsfile_gc
@@ -0,0 +1,25 @@
+pipeline {
+    agent {
+        label "linux-arm64-docker"
+    }
+    stages {
+        stage('Garbage Collection') {
+            parallel {
+                stage('GC on Azure') {
+                    environment {
+                        PACKER_AZURE = credentials('packer-azure-serviceprincipal-sponsorship')
+                    }
+                    steps {
+                            sh 'az login --service-principal -u "$PACKER_AZURE_CLIENT_ID" -p "$PACKER_AZURE_CLIENT_SECRET" -t "$PACKER_AZURE_TENANT_ID"'
+                            sh 'az account set -s "$PACKER_AZURE_SUBSCRIPTION_ID"'
+                            sh './cleanup/azure_gallery_images.sh 1 dev'
+                            sh './cleanup/azure_gallery_images.sh 7 staging'
+                            sh './cleanup/azure.sh 1 dev'
+                            sh './cleanup/azure.sh 1 staging'
+                            sh './cleanup/azure.sh 1 prod'
+                        }
+                }
+            }
+        }
+    }
+}
diff --git a/cleanup/aws.sh b/cleanup/aws.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+run_aws_ec2_deletion_command() {
+  # Check the DRYRUN environment variable
+  if [ "${DRYRUN:-true}" = "false" ] || [ "${DRYRUN:-true}" = "no" ]
+  then
+    # Execute command "as it"
+    aws ec2 "$@"
+  else
+    # Execute command with the "--dry-run"
+    ec2_subcommand="$1"
+    shift
+    echo "== DRY RUN:"
+    echo "aws ec2 ${ec2_subcommand} --dry-run" "$@"
+    aws ec2 "${ec2_subcommand}" --dry-run "$@" 2>&1 | grep -v 'Request would have succeeded' || true
+  fi
+}
+
+## Check for presence of required CLIs
+for cli in aws jq date xargs
+do
+  command -v "${cli}" >/dev/null || { echo "[ERROR] no '${cli}' command found."; exit 1; }
+done
+
+## When is yesterday exactly?
+yesterday=""
+timeshift_days="1"
+if date -v-${timeshift_days}m > /dev/null 2>&1; then
+    # BSD systems (Mac OS X)
+    yesterday="$(date -v-${timeshift_days}d +%Y-%m-%d)"
+else
+    # GNU systems (Linux)
+    yesterday="$(date --date="-${timeshift_days} days" +%Y-%m-%d)"
+fi
+
+## Check for aws API reachibility (is it configured?)
+aws sts get-caller-identity >/dev/null || \
+  { echo "[ERROR] Unable to request the AWS API: the command 'sts get-caller-identity' failed. Please check your AWS credentials"; exit 1; }
+
+## Remove running instances older than 24 hours
+
+INSTANCE_IDS="$(aws ec2 describe-instances --filters 'Name=tag:Name,Values=*Packer*' \
+  --query 'Reservations[].Instances[?LaunchTime<=`'"${yesterday}"'`][].InstanceId' | jq -r '.[]' | xargs)"
+
+if [ -n "${INSTANCE_IDS}" ]
+then
+  #shellcheck disable=SC2086
+  run_aws_ec2_deletion_command terminate-instances --instance-ids ${INSTANCE_IDS}
+else
+  echo "== No dangling instance found to terminate."
+fi
+
+## Remove security groups older than 24 hours
+for secgroup_id in $(aws ec2 describe-security-groups --filters 'Name=group-name,Values=*packer*' \
+  | jq -r '.SecurityGroups[].GroupId')
+do
+  # Each security group which name matches the pattern '*packer*' is deleted if it is orphaned (not use by any network interface)
+  if [ "0" = "$(aws ec2 describe-network-interfaces --filters "Name=group-id,Values=${secgroup_id}" | jq -r '.NetworkInterfaces | length')" ]
+  then
+    #shellcheck disable=SC2086
+    run_aws_ec2_deletion_command delete-security-group --group-id ${secgroup_id}
+  fi
+done
+
+echo "== AWS Packer Cleanup finished."
diff --git a/cleanup/aws_images.sh b/cleanup/aws_images.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Check input parameteres
+if ! [[ "${1:-1}" =~ ^[0-9]+$ ]]; then
+  echo "ERROR: argument $1 is not a number" >&2
+  exit 1
+fi
+timeshift_day="${1}"
+build_type="${2:-dev}"
+
+set -eux -o pipefail
+
+run_aws_ec2_command() {
+  # Check the DRYRUN environment variable
+  if [ "${DRYRUN:-true}" = "false" ] || [ "${DRYRUN:-true}" = "no" ]
+  then
+    echo "== INFO: Running command AMI aws ec2 ${*}"
+    aws ec2 "$@"
+  else
+    # Execute command with the "--dry-run"
+    ec2_subcommand="$1"
+    shift
+    echo "== DRY RUN:"
+    echo "aws ec2 ${ec2_subcommand} --dry-run" "$@"
+    aws ec2 "${ec2_subcommand}" --dry-run "$@" 2>&1 | grep -v 'Request would have succeeded' || true
+  fi
+}
+
+## Check for presence of required CLIs
+for cli in aws jq date xargs curl cut grep
+do
+  command -v "${cli}" >/dev/null || { echo "[ERROR] no '${cli}' command found."; exit 1; }
+done
+
+creation_date_threshold=""
+if date -v-"${timeshift_day}d" > /dev/null 2>&1; then
+    # BSD systems (Mac OS X)
+    creation_date_threshold="$(date -v-"${timeshift_day}"d +%Y-%m-%d)"
+else
+    # GNU systems (Linux)
+    creation_date_threshold="$(date --date="-${timeshift_day} days" +%Y-%m-%d)"
+fi
+
+## Check for aws API reachibility (is it configured?)
+aws sts get-caller-identity >/dev/null || \
+  { echo "[ERROR] Unable to request the AWS API: the command 'sts get-caller-identity' failed. Please check your AWS credentials"; exit 1; }
+
+## STEP 1
+## Remove images older than $1 day(s) of build_type $2
+found_ami_ids="$(aws ec2 describe-images --owners self --filters "Name=tag:build_type,Values=${build_type}" \
+  --query 'Images[?CreationDate<=`'"${creation_date_threshold}"'`][].ImageId' | jq -r '.[]' | sort -u)"
+
+echo "== Checking the following AMIs (before creation_date_threshold=${creation_date_threshold}, with tag build_type=${build_type}) for deletion:"
+echo "${found_ami_ids}"
+echo "======"
+
+# Exclude AMI defined in production for Puppet VMs (non puppet managed controllers should have the same, if not then it's sad)
+production_amis="$(curl --silent --show-error --location https://raw.githubusercontent.com/jenkins-infra/jenkins-infra/production/hieradata/common.yaml | grep ami- | cut -d '"' -f2)" || true # force error_code to true to handle no usage of aws ami images in production
+ami_ids=()
+for ami_id in ${found_ami_ids}
+do
+  # Add this AMI to the list if not present in the "production_amis"
+  if echo "${production_amis}" | grep -v "${ami_id}"
+  then
+    ami_ids+=("${ami_id}")
+  fi
+done
+
+if [ -n "${!ami_ids[*]}" ] # ! to avoid error with empty ami list
+then
+  echo "== Deleting the following AMIs:"
+  echo "${ami_ids[*]}"
+  echo "======"
+  export -f run_aws_ec2_command
+  parallel --halt-on-error never --no-run-if-empty run_aws_ec2_command deregister-image --image-id {} ::: "${ami_ids[@]}"
+else
+  echo "== No dangling images found to delete."
+fi
+
+echo "== AWS Packer Cleanup IMAGES finished."
diff --git a/cleanup/aws_snapshots.sh b/cleanup/aws_snapshots.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+run_aws_ec2_command() {
+  # Check the DRYRUN environment variable
+  if [ "${DRYRUN:-true}" = "false" ] || [ "${DRYRUN:-true}" = "no" ]
+  then
+    # Execute command "as it"
+    aws ec2 "$@"
+  else
+    # Execute command with the "--dry-run"
+    ec2_subcommand="$1"
+    shift
+    echo "== DRY RUN:"
+    echo "aws ec2 ${ec2_subcommand} --dry-run" "$@"
+    aws ec2 "${ec2_subcommand}" --dry-run "$@" 2>&1 | grep -v 'Request would have succeeded' || true
+  fi
+}
+
+## Check for presence of required CLIs
+for cli in aws jq date xargs
+do
+  command -v "${cli}" >/dev/null || { echo "[ERROR] no '${cli}' command found."; exit 1; }
+done
+
+start_time_threshold=""
+timeshift_days="1"
+if date -v-${timeshift_days}d > /dev/null 2>&1; then
+    # BSD systems (Mac OS X)
+    start_time_threshold="$(date -v-${timeshift_days}d +%Y-%m-%d)"
+else
+    # GNU systems (Linux)
+    start_time_threshold="$(date --date="-${timeshift_days} days" +%Y-%m-%d)"
+fi
+
+## Check for aws API reachibility (is it configured?)
+aws sts get-caller-identity >/dev/null || \
+  { echo "[ERROR] Unable to request the AWS API: the command 'sts get-caller-identity' failed. Please check your AWS credentials"; exit 1; }
+
+## STEP 1
+## Remove snapshots older than <timeshift_days> day(s)
+snapshot_ids="$(aws ec2 describe-snapshots \
+  --owner-ids self \
+  --query "Snapshots[?StartTime<='${start_time_threshold}'].[SnapshotId]" \
+  --no-paginate \
+  --region=us-east-2 \
+  | jq -r '.[][]')"
+
+if [ -n "${snapshot_ids}" ]
+then
+  export -f run_aws_ec2_command
+  echo "${snapshot_ids}" | parallel --halt-on-error never --no-run-if-empty run_aws_ec2_command delete-snapshot --snapshot-id {} :::
+else
+  echo "== No dangling snapshots found to delete."
+fi
+
+echo "== AWS Packer Cleanup SNAPSHOTS finished."