From d64df0335367e999d5304d945898049528b6030e Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Tue, 17 Dec 2024 08:59:45 +0530
Subject: [PATCH 01/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 .github/workflows/awscleanup.yaml |  69 +++++++++++++++
 .github/workflows/codeql.yml      |  10 ++-
 .github/workflows/e2e.yml         |  13 +--
 .github/workflows/go.yml          |  10 ++-
 .github/workflows/image.yml       |  10 ++-
 scripts/awscleanup.sh             | 142 ++++++++++++++++++++++++++++++
 tests/aws_test.go                 |   3 +-
 tests/common/common.go            |  21 ++++-
 8 files changed, 257 insertions(+), 21 deletions(-)
 create mode 100644 .github/workflows/awscleanup.yaml
 create mode 100755 scripts/awscleanup.sh

diff --git a/.github/workflows/awscleanup.yaml b/.github/workflows/awscleanup.yaml
new file mode 100644
index 00000000..78dda0d4
--- /dev/null
+++ b/.github/workflows/awscleanup.yaml
@@ -0,0 +1,69 @@
+name: Daily AWS Cleanup Bot
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+    branches:
+      - test-awsresourcecleanup
+  push:
+    branches:
+      - test-awsresourcecleanup
+
+jobs:
+  cleanup:
+    runs-on: linux-amd64-cpu4
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up AWS CLI
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-1
+
+      - name: Identify resources for deletion
+        id: identify-resources
+        run: |
+          # Find vpcs with names ci*
+          vpcs=$(aws ec2 describe-vpcs \
+            --filters "Name=tag:Name,Values=ci*" \
+            --query "Vpcs[].VpcId" \
+            --output text | tr -d '\r' | tr '\n' ' ')
+          echo "Found VPCs: $vpcs"
+          echo "vpcs=$vpcs" >> $GITHUB_ENV
+
+      - name: Clean up VPCs
+        if: env.vpcs != ''
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          get_tag_value(){
+            local vpc_id=$1
+            local key=$2
+            aws ec2 describe-tags --filters "Name=resource-id,Values=$vpc_id" "Name=key,Values=$key" \
+              --query "Tags[0].Value" --output text
+          }
+          for vpc in $vpcs; do
+            github_repository=$(get_tag_value $vpc "GitHubRepository")
+            run_id=$(get_tag_value $vpc "GitHubRunId")
+            job_name=$(get_tag_value $vpc "GitHubJob")
+            response=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" \
+              "https://api.github.com/repos/${github_repository}/actions/runs/${run_id}/jobs")
+            if [[ -z "$response" || "$response" == "null" ]]; then
+              continue
+            fi
+            status=$(echo "$response" | jq -r ".jobs? // [] | map(select(.name | test(\"^$job_name\"))) | .[0]?.status // \"null\"" 2>/dev/null || echo "null")
+            if [[ "$status" != "null" && ! -z "$status" && $(echo "$status" | grep -qvE '^(queued|in_progress)$'; echo $?) -eq 0 ]]; then
+              echo "Holodeck e2e Job status is not in running stage , Delete the dependent resources"
+              scripts/awscleanup.sh $vpc
+            fi
+          done
+
+      - name: Post cleanup
+        run: |
+          echo "Cleanup completed."
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
index 674cb838..21c41265 100644
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@@ -20,12 +20,14 @@ on:
       - opened
       - synchronize
     branches:
-      - main
-      - release-*
+      - main-no
+      - release-no
+      - test-awsresourcecleanup
   push:
     branches:
-      - main
-      - release-*
+      - main-no
+      - release-no
+      - test-awsresourcecleanup
   schedule:
     - cron: '31 11 * * 4'
 
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index a186b981..fa3b322e 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -15,14 +15,15 @@
 name: End-to-end Tests
 
 on:
-  workflow_run:
-    workflows: [Go]
+  pull_request:
     types:
-      - completed
+      - opened
+      - synchronize
     branches:
-      - "pull-request/[0-9]+"
-      - main
-      - release-*
+      - test-awsresourcecleanup
+  push:
+    branches:
+      - test-awsresourcecleanup
 
 jobs:
   e2e-test:
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index 9e215402..b18e5bc2 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -17,12 +17,14 @@ name: Go
 on:
   push:
     branches: 
-      - main
-      - release-*
+      - main-no
+      - release-no
+      - test-awsresourcecleanup
   pull_request:
     branches:
-      - main
-      - release-*
+      - main-no
+      - release-no
+      - test-awsresourcecleanup
 
 jobs:
   build:
diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml
index f57058f8..ed35c894 100644
--- a/.github/workflows/image.yml
+++ b/.github/workflows/image.yml
@@ -17,14 +17,16 @@ name: Image
 on:
   pull_request:
     branches:
-      - 'main'
-      - 'release-*'
+      - 'main-no'
+      - 'release-no'
+      - test-awsresourcecleanup
   push:
     tags:
       - 'v*.*.*'  
     branches:
-      - 'main'
-      - 'release-*'
+      - 'main-no'
+      - 'release-no'
+      - test-awsresourcecleanup
 
 jobs:
   docker:
diff --git a/scripts/awscleanup.sh b/scripts/awscleanup.sh
new file mode 100755
index 00000000..27e809c8
--- /dev/null
+++ b/scripts/awscleanup.sh
@@ -0,0 +1,142 @@
+#!/bin/bash
+
+if [[ $# -ne 1 ]]; then
+	echo " vpcid required for deletion"
+	exit 1
+fi
+export vpc=$1
+
+echo "Start Deleting  VPC: $vpc resource"
+
+# Delete Instance
+instances=$(aws ec2 describe-instances \
+    --filters "Name=vpc-id,Values=$vpc" \
+    --query "Reservations[].Instances[].InstanceId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for instance in $instances; do
+    echo "Terminating instance: $instance"
+    aws ec2 terminate-instances --instance-ids "$instance"
+done
+
+# Delete Internet Gateway
+internet_gateways=$(aws ec2 describe-internet-gateways \
+    --filters Name=attachment.vpc-id,Values=$vpc \
+    --query "InternetGateways[].InternetGatewayId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for igw in $internet_gateways; do
+    aws ec2 detach-internet-gateway --internet-gateway-id "$igw" --vpc-id "$vpc"
+    aws ec2 delete-internet-gateway --internet-gateway-id "$igw"
+done
+
+# Delete NAT Gateways
+nat_gateways=$(aws ec2 describe-nat-gateways \
+    --filter Name=vpc-id,Values=$vpc \
+    --query "NatGateways[].NatGatewayId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for ngw in $nat_gateways; do
+    aws ec2 delete-nat-gateway --nat-gateway-id "$ngw"
+done
+
+# Delete Elastic IPs
+eips=$(aws ec2 describe-addresses \
+  --filters Name=domain,Values=vpc \
+  --query "Addresses[].[AllocationId,Association.VpcId]" \
+  --output text | grep "$vpc" | awk '{print $1}' | tr -d '\r' | tr '\n' ' ')
+for eip in $eips; do
+    aws ec2 release-address --allocation-id "$eip"
+done
+
+# Detach and Delete Security Groups
+security_groups=$(aws ec2 describe-security-groups \
+    --filters Name=vpc-id,Values=$vpc \
+    --query "SecurityGroups[?GroupName!='default'].GroupId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for sg in $security_groups; do
+    enis=$(aws ec2 describe-network-interfaces \
+        --filters Name=group-id,Values=$sg \
+        --query "NetworkInterfaces[].NetworkInterfaceId" \
+        --output text | tr -d '\r' | tr '\n' ' ')
+    for eni in $enis; do
+        aws ec2 modify-network-interface-attribute \
+            --network-interface-id "$eni" \
+            --groups "$(aws ec2 describe-security-groups \
+                --query 'SecurityGroups[?GroupName==`default`].GroupId' \
+                --output text)"
+    done
+    aws ec2 delete-security-group --group-id "$sg"
+done
+
+# Delete Route Tables
+# 1. Make first rt as Main , as we cannot delete vpcs attached with main
+# 2. replace all rt with first rt
+# 3. delete rt
+# 4. Main table(first_rt) will be deleted once vpc deleted
+first_rt=""
+route_tables=$(aws ec2 describe-route-tables \
+    --filters Name=vpc-id,Values=$vpc \
+    --query "RouteTables[].RouteTableId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for rt in $route_tables; do
+    associations=$(aws ec2 describe-route-tables \
+            --route-table-ids "$rt" \
+            --query "RouteTables[].Associations[].RouteTableAssociationId" \
+            --output text | tr -d '\r' | tr '\n' ' ')
+    for assoc_id in $associations; do
+        if [ -z "$first_rt" ]; then
+            aws ec2 replace-route-table-association --association-id $assoc_id --route-table-id $rt
+            first_rt=$rt
+        else
+            aws ec2 replace-route-table-association --association-id $assoc_id --route-table-id $first_rt
+            aws ec2 delete-route-table --route-table-id "$rt"
+        fi
+    done
+done
+
+# Delete Subnets
+subnets=$(aws ec2 describe-subnets \
+    --filters Name=vpc-id,Values=$vpc \
+    --query "Subnets[].SubnetId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for subnet in $subnets; do
+    aws ec2 delete-subnet --subnet-id "$subnet"
+done
+
+# Delete Network Interfaces
+eni_ids=$(aws ec2 describe-network-interfaces \
+    --filters Name=vpc-id,Values=$vpc \
+    --query "NetworkInterfaces[].NetworkInterfaceId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for eni in $eni_ids; do
+    aws ec2 delete-network-interface --network-interface-id "$eni"
+done
+
+# Delete Network ACLs
+nw_acls=$(aws ec2 describe-network-acls \
+    --filters "Name=vpc-id,Values=$vpc" \
+    --query "NetworkAcls[?IsDefault==false].NetworkAclId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for acl in $nw_acls; do
+    echo "Deleting Network ACL: $acl"
+    aws ec2 delete-network-acl --network-acl-id $acl
+done
+
+# Delete vpc
+# try 3 times with 30 seconds interval
+attempts=0
+echo "All resource Deleted for VPC: $vpc , now delete vpc"
+while [ $attempts -lt 3 ]; do
+    echo "Attempting to delete VPC: $vpc (Attempt $((attempts+1)))"
+    if aws ec2 delete-vpc --vpc-id $vpc; then
+        echo "Successfully deleted VPC: $vpc"
+        break
+    else
+        attempts=$((attempts + 1))
+        if [ $attempts -lt 3 ]; then
+            echo "Failed to delete VPC: $vpc. Retrying in 30 seconds..."
+            sleep 30
+        fi
+    fi
+done
+if [ $attempts -eq 3 ]; then
+    echo "Failed to delete VPC: $vpc after 3 attempts. Continue the loop to delete other vpc"
+fi
diff --git a/tests/aws_test.go b/tests/aws_test.go
index 541c5d7d..d168b59e 100644
--- a/tests/aws_test.go
+++ b/tests/aws_test.go
@@ -29,7 +29,6 @@ import (
 	"github.com/NVIDIA/holodeck/pkg/jyaml"
 	"github.com/NVIDIA/holodeck/pkg/provider"
 	"github.com/NVIDIA/holodeck/pkg/provisioner"
-	"github.com/NVIDIA/holodeck/tests/common"
 )
 
 // Actual test suite
@@ -56,7 +55,7 @@ var _ = Describe("AWS", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			// Set unique name for the environment
-			opts.cfg.Name = opts.cfg.Name + "-" + common.GenerateUID()
+			common.SetCfgName(opts.cfg)
 			// set cache path
 			opts.cachePath = LogArtifactDir
 			// set cache file
diff --git a/tests/common/common.go b/tests/common/common.go
index 254c0dd1..973476c3 100644
--- a/tests/common/common.go
+++ b/tests/common/common.go
@@ -16,7 +16,13 @@
 
 package common
 
-import "math/rand"
+import (
+	"fmt"
+	"math/rand"
+	"os"
+
+	"github.com/NVIDIA/holodeck/api/holodeck/v1alpha1"
+)
 
 func GenerateUID() string {
 	const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
@@ -28,3 +34,16 @@ func GenerateUID() string {
 
 	return string(b)
 }
+
+func SetCfgName(cfg *v1alpha1.Environment) {
+	sha := os.Getenv("GITHUB_SHA")
+	attempt := os.Getenv("GITHUB_RUN_ATTEMPT")
+	// short sha
+	if len(sha) > 8 {
+		sha = sha[:8]
+	}
+	// uid is unique for each run
+	uid := GenerateUID()
+
+	cfg.Name = fmt.Sprintf("ci%s-%s-%s", attempt, sha, uid)
+}

From 54aafd9bc3066ad5fdae65afe31e649c9e639d5a Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 13:29:59 +0530
Subject: [PATCH 02/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 tests/aws_test.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/aws_test.go b/tests/aws_test.go
index d168b59e..e2090826 100644
--- a/tests/aws_test.go
+++ b/tests/aws_test.go
@@ -29,6 +29,7 @@ import (
 	"github.com/NVIDIA/holodeck/pkg/jyaml"
 	"github.com/NVIDIA/holodeck/pkg/provider"
 	"github.com/NVIDIA/holodeck/pkg/provisioner"
+  "github.com/NVIDIA/holodeck/tests/common"
 )
 
 // Actual test suite

From 7dc26d88daf2c3b5ad46a6e8be587e8627a512a1 Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 13:33:52 +0530
Subject: [PATCH 03/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 tests/aws_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/aws_test.go b/tests/aws_test.go
index e2090826..81cb6df7 100644
--- a/tests/aws_test.go
+++ b/tests/aws_test.go
@@ -29,7 +29,7 @@ import (
 	"github.com/NVIDIA/holodeck/pkg/jyaml"
 	"github.com/NVIDIA/holodeck/pkg/provider"
 	"github.com/NVIDIA/holodeck/pkg/provisioner"
-  "github.com/NVIDIA/holodeck/tests/common"
+	"github.com/NVIDIA/holodeck/tests/common"
 )
 
 // Actual test suite
@@ -56,7 +56,7 @@ var _ = Describe("AWS", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			// Set unique name for the environment
-			common.SetCfgName(opts.cfg)
+			common.SetCfgName(&opts.cfg)
 			// set cache path
 			opts.cachePath = LogArtifactDir
 			// set cache file

From 1410d048f9b6e7f35df9c3c10873ba4db9b58a0e Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 13:39:03 +0530
Subject: [PATCH 04/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 scripts/awscleanup.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/awscleanup.sh b/scripts/awscleanup.sh
index 27e809c8..f0dfc584 100755
--- a/scripts/awscleanup.sh
+++ b/scripts/awscleanup.sh
@@ -6,6 +6,7 @@ if [[ $# -ne 1 ]]; then
 fi
 export vpc=$1
 
+exit 0
 echo "Start Deleting  VPC: $vpc resource"
 
 # Delete Instance

From 82291ce6bbb5015e3eb7a987b578a4f6eba4b026 Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 15:35:37 +0530
Subject: [PATCH 05/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 scripts/awscleanup.sh | 72 +++++++++++++++++++++++--------------------
 1 file changed, 38 insertions(+), 34 deletions(-)

diff --git a/scripts/awscleanup.sh b/scripts/awscleanup.sh
index f0dfc584..0b61119e 100755
--- a/scripts/awscleanup.sh
+++ b/scripts/awscleanup.sh
@@ -19,34 +19,6 @@ for instance in $instances; do
     aws ec2 terminate-instances --instance-ids "$instance"
 done
 
-# Delete Internet Gateway
-internet_gateways=$(aws ec2 describe-internet-gateways \
-    --filters Name=attachment.vpc-id,Values=$vpc \
-    --query "InternetGateways[].InternetGatewayId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for igw in $internet_gateways; do
-    aws ec2 detach-internet-gateway --internet-gateway-id "$igw" --vpc-id "$vpc"
-    aws ec2 delete-internet-gateway --internet-gateway-id "$igw"
-done
-
-# Delete NAT Gateways
-nat_gateways=$(aws ec2 describe-nat-gateways \
-    --filter Name=vpc-id,Values=$vpc \
-    --query "NatGateways[].NatGatewayId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for ngw in $nat_gateways; do
-    aws ec2 delete-nat-gateway --nat-gateway-id "$ngw"
-done
-
-# Delete Elastic IPs
-eips=$(aws ec2 describe-addresses \
-  --filters Name=domain,Values=vpc \
-  --query "Addresses[].[AllocationId,Association.VpcId]" \
-  --output text | grep "$vpc" | awk '{print $1}' | tr -d '\r' | tr '\n' ' ')
-for eip in $eips; do
-    aws ec2 release-address --allocation-id "$eip"
-done
-
 # Detach and Delete Security Groups
 security_groups=$(aws ec2 describe-security-groups \
     --filters Name=vpc-id,Values=$vpc \
@@ -67,6 +39,15 @@ for sg in $security_groups; do
     aws ec2 delete-security-group --group-id "$sg"
 done
 
+# Delete Subnets
+subnets=$(aws ec2 describe-subnets \
+    --filters Name=vpc-id,Values=$vpc \
+    --query "Subnets[].SubnetId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for subnet in $subnets; do
+    aws ec2 delete-subnet --subnet-id "$subnet"
+done
+
 # Delete Route Tables
 # 1. Make first rt as Main , as we cannot delete vpcs attached with main
 # 2. replace all rt with first rt
@@ -93,15 +74,37 @@ for rt in $route_tables; do
     done
 done
 
-# Delete Subnets
-subnets=$(aws ec2 describe-subnets \
-    --filters Name=vpc-id,Values=$vpc \
-    --query "Subnets[].SubnetId" \
+# Detach and Delete Internet Gateway
+internet_gateways=$(aws ec2 describe-internet-gateways \
+    --filters Name=attachment.vpc-id,Values=$vpc \
+    --query "InternetGateways[].InternetGatewayId" \
     --output text | tr -d '\r' | tr '\n' ' ')
-for subnet in $subnets; do
-    aws ec2 delete-subnet --subnet-id "$subnet"
+for igw in $internet_gateways; do
+    aws ec2 detach-internet-gateway --internet-gateway-id "$igw" --vpc-id "$vpc"
+    aws ec2 delete-internet-gateway --internet-gateway-id "$igw"
+done
+
+# SHIVA
+# Delete NAT Gateways
+nat_gateways=$(aws ec2 describe-nat-gateways \
+    --filter Name=vpc-id,Values=$vpc \
+    --query "NatGateways[].NatGatewayId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for ngw in $nat_gateways; do
+    aws ec2 delete-nat-gateway --nat-gateway-id "$ngw"
+done
+
+# SHIVA
+# Delete Elastic IPs
+eips=$(aws ec2 describe-addresses \
+  --filters Name=domain,Values=vpc \
+  --query "Addresses[].[AllocationId,Association.VpcId]" \
+  --output text | grep "$vpc" | awk '{print $1}' | tr -d '\r' | tr '\n' ' ')
+for eip in $eips; do
+    aws ec2 release-address --allocation-id "$eip"
 done
 
+# SHIVA
 # Delete Network Interfaces
 eni_ids=$(aws ec2 describe-network-interfaces \
     --filters Name=vpc-id,Values=$vpc \
@@ -111,6 +114,7 @@ for eni in $eni_ids; do
     aws ec2 delete-network-interface --network-interface-id "$eni"
 done
 
+# SHIVA
 # Delete Network ACLs
 nw_acls=$(aws ec2 describe-network-acls \
     --filters "Name=vpc-id,Values=$vpc" \

From 0d6f55c4c5dc3bc25783396ffc524f2201545f2c Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 16:10:54 +0530
Subject: [PATCH 06/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 scripts/awscleanup.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/awscleanup.sh b/scripts/awscleanup.sh
index 0b61119e..7962a125 100755
--- a/scripts/awscleanup.sh
+++ b/scripts/awscleanup.sh
@@ -6,7 +6,6 @@ if [[ $# -ne 1 ]]; then
 fi
 export vpc=$1
 
-exit 0
 echo "Start Deleting  VPC: $vpc resource"
 
 # Delete Instance

From ce7a9d0a8bcc59c66eb9c9205890e3a369a38825 Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 16:22:20 +0530
Subject: [PATCH 07/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 scripts/awscleanup.sh | 94 +++++++++++++++++++++----------------------
 1 file changed, 47 insertions(+), 47 deletions(-)

diff --git a/scripts/awscleanup.sh b/scripts/awscleanup.sh
index 7962a125..9c411ca4 100755
--- a/scripts/awscleanup.sh
+++ b/scripts/awscleanup.sh
@@ -18,6 +18,36 @@ for instance in $instances; do
     aws ec2 terminate-instances --instance-ids "$instance"
 done
 
+# SHIVA
+# Delete Elastic IPs
+eips=$(aws ec2 describe-addresses \
+  --filters Name=domain,Values=vpc \
+  --query "Addresses[].[AllocationId,Association.VpcId]" \
+  --output text | grep "$vpc" | awk '{print $1}' | tr -d '\r' | tr '\n' ' ')
+for eip in $eips; do
+    aws ec2 release-address --allocation-id "$eip"
+done
+
+# SHIVA
+# Delete NAT Gateways
+nat_gateways=$(aws ec2 describe-nat-gateways \
+    --filter Name=vpc-id,Values=$vpc \
+    --query "NatGateways[].NatGatewayId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for ngw in $nat_gateways; do
+    aws ec2 delete-nat-gateway --nat-gateway-id "$ngw"
+done
+
+# SHIVA
+# Delete Network Interfaces
+eni_ids=$(aws ec2 describe-network-interfaces \
+    --filters Name=vpc-id,Values=$vpc \
+    --query "NetworkInterfaces[].NetworkInterfaceId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for eni in $eni_ids; do
+    aws ec2 delete-network-interface --network-interface-id "$eni"
+done
+
 # Detach and Delete Security Groups
 security_groups=$(aws ec2 describe-security-groups \
     --filters Name=vpc-id,Values=$vpc \
@@ -38,13 +68,15 @@ for sg in $security_groups; do
     aws ec2 delete-security-group --group-id "$sg"
 done
 
-# Delete Subnets
-subnets=$(aws ec2 describe-subnets \
-    --filters Name=vpc-id,Values=$vpc \
-    --query "Subnets[].SubnetId" \
+# SHIVA
+# Delete Network ACLs
+nw_acls=$(aws ec2 describe-network-acls \
+    --filters "Name=vpc-id,Values=$vpc" \
+    --query "NetworkAcls[?IsDefault==false].NetworkAclId" \
     --output text | tr -d '\r' | tr '\n' ' ')
-for subnet in $subnets; do
-    aws ec2 delete-subnet --subnet-id "$subnet"
+for acl in $nw_acls; do
+    echo "Deleting Network ACL: $acl"
+    aws ec2 delete-network-acl --network-acl-id $acl
 done
 
 # Delete Route Tables
@@ -73,6 +105,15 @@ for rt in $route_tables; do
     done
 done
 
+# Delete Subnets
+subnets=$(aws ec2 describe-subnets \
+    --filters Name=vpc-id,Values=$vpc \
+    --query "Subnets[].SubnetId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for subnet in $subnets; do
+    aws ec2 delete-subnet --subnet-id "$subnet"
+done
+
 # Detach and Delete Internet Gateway
 internet_gateways=$(aws ec2 describe-internet-gateways \
     --filters Name=attachment.vpc-id,Values=$vpc \
@@ -83,47 +124,6 @@ for igw in $internet_gateways; do
     aws ec2 delete-internet-gateway --internet-gateway-id "$igw"
 done
 
-# SHIVA
-# Delete NAT Gateways
-nat_gateways=$(aws ec2 describe-nat-gateways \
-    --filter Name=vpc-id,Values=$vpc \
-    --query "NatGateways[].NatGatewayId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for ngw in $nat_gateways; do
-    aws ec2 delete-nat-gateway --nat-gateway-id "$ngw"
-done
-
-# SHIVA
-# Delete Elastic IPs
-eips=$(aws ec2 describe-addresses \
-  --filters Name=domain,Values=vpc \
-  --query "Addresses[].[AllocationId,Association.VpcId]" \
-  --output text | grep "$vpc" | awk '{print $1}' | tr -d '\r' | tr '\n' ' ')
-for eip in $eips; do
-    aws ec2 release-address --allocation-id "$eip"
-done
-
-# SHIVA
-# Delete Network Interfaces
-eni_ids=$(aws ec2 describe-network-interfaces \
-    --filters Name=vpc-id,Values=$vpc \
-    --query "NetworkInterfaces[].NetworkInterfaceId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for eni in $eni_ids; do
-    aws ec2 delete-network-interface --network-interface-id "$eni"
-done
-
-# SHIVA
-# Delete Network ACLs
-nw_acls=$(aws ec2 describe-network-acls \
-    --filters "Name=vpc-id,Values=$vpc" \
-    --query "NetworkAcls[?IsDefault==false].NetworkAclId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for acl in $nw_acls; do
-    echo "Deleting Network ACL: $acl"
-    aws ec2 delete-network-acl --network-acl-id $acl
-done
-
 # Delete vpc
 # try 3 times with 30 seconds interval
 attempts=0

From f32956ddf125fccf65fde5a5c8f720b46fd50954 Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 16:29:31 +0530
Subject: [PATCH 08/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 scripts/awscleanup.sh | 55 ++++++-------------------------------------
 1 file changed, 7 insertions(+), 48 deletions(-)

diff --git a/scripts/awscleanup.sh b/scripts/awscleanup.sh
index 9c411ca4..b2dbf018 100755
--- a/scripts/awscleanup.sh
+++ b/scripts/awscleanup.sh
@@ -18,34 +18,14 @@ for instance in $instances; do
     aws ec2 terminate-instances --instance-ids "$instance"
 done
 
-# SHIVA
-# Delete Elastic IPs
-eips=$(aws ec2 describe-addresses \
-  --filters Name=domain,Values=vpc \
-  --query "Addresses[].[AllocationId,Association.VpcId]" \
-  --output text | grep "$vpc" | awk '{print $1}' | tr -d '\r' | tr '\n' ' ')
-for eip in $eips; do
-    aws ec2 release-address --allocation-id "$eip"
-done
-
-# SHIVA
-# Delete NAT Gateways
-nat_gateways=$(aws ec2 describe-nat-gateways \
-    --filter Name=vpc-id,Values=$vpc \
-    --query "NatGateways[].NatGatewayId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for ngw in $nat_gateways; do
-    aws ec2 delete-nat-gateway --nat-gateway-id "$ngw"
-done
-
-# SHIVA
-# Delete Network Interfaces
-eni_ids=$(aws ec2 describe-network-interfaces \
-    --filters Name=vpc-id,Values=$vpc \
-    --query "NetworkInterfaces[].NetworkInterfaceId" \
+# Delete Internet Gateway
+internet_gateways=$(aws ec2 describe-internet-gateways \
+    --filters Name=attachment.vpc-id,Values=$vpc \
+    --query "InternetGateways[].InternetGatewayId" \
     --output text | tr -d '\r' | tr '\n' ' ')
-for eni in $eni_ids; do
-    aws ec2 delete-network-interface --network-interface-id "$eni"
+for igw in $internet_gateways; do
+    aws ec2 detach-internet-gateway --internet-gateway-id "$igw" --vpc-id "$vpc"
+    aws ec2 delete-internet-gateway --internet-gateway-id "$igw"
 done
 
 # Detach and Delete Security Groups
@@ -68,17 +48,6 @@ for sg in $security_groups; do
     aws ec2 delete-security-group --group-id "$sg"
 done
 
-# SHIVA
-# Delete Network ACLs
-nw_acls=$(aws ec2 describe-network-acls \
-    --filters "Name=vpc-id,Values=$vpc" \
-    --query "NetworkAcls[?IsDefault==false].NetworkAclId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for acl in $nw_acls; do
-    echo "Deleting Network ACL: $acl"
-    aws ec2 delete-network-acl --network-acl-id $acl
-done
-
 # Delete Route Tables
 # 1. Make first rt as Main , as we cannot delete vpcs attached with main
 # 2. replace all rt with first rt
@@ -114,16 +83,6 @@ for subnet in $subnets; do
     aws ec2 delete-subnet --subnet-id "$subnet"
 done
 
-# Detach and Delete Internet Gateway
-internet_gateways=$(aws ec2 describe-internet-gateways \
-    --filters Name=attachment.vpc-id,Values=$vpc \
-    --query "InternetGateways[].InternetGatewayId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for igw in $internet_gateways; do
-    aws ec2 detach-internet-gateway --internet-gateway-id "$igw" --vpc-id "$vpc"
-    aws ec2 delete-internet-gateway --internet-gateway-id "$igw"
-done
-
 # Delete vpc
 # try 3 times with 30 seconds interval
 attempts=0

From 1704b447ca6e640d960e295c39a6632f1fad2dc8 Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 16:40:18 +0530
Subject: [PATCH 09/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 scripts/awscleanup.sh | 44 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/scripts/awscleanup.sh b/scripts/awscleanup.sh
index b2dbf018..dd1596b9 100755
--- a/scripts/awscleanup.sh
+++ b/scripts/awscleanup.sh
@@ -24,10 +24,31 @@ internet_gateways=$(aws ec2 describe-internet-gateways \
     --query "InternetGateways[].InternetGatewayId" \
     --output text | tr -d '\r' | tr '\n' ' ')
 for igw in $internet_gateways; do
+    echo "Terminating delete-internet-gateway: $igw"
     aws ec2 detach-internet-gateway --internet-gateway-id "$igw" --vpc-id "$vpc"
     aws ec2 delete-internet-gateway --internet-gateway-id "$igw"
 done
 
+# Delete NAT Gateways
+nat_gateways=$(aws ec2 describe-nat-gateways \
+    --filter Name=vpc-id,Values=$vpc \
+    --query "NatGateways[].NatGatewayId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for ngw in $nat_gateways; do
+    echo "Terminating delete-nat-gateway: $ngw"
+    aws ec2 delete-nat-gateway --nat-gateway-id "$ngw"
+done
+
+# Delete Elastic IPs
+eips=$(aws ec2 describe-addresses \
+  --filters Name=domain,Values=vpc \
+  --query "Addresses[].[AllocationId,Association.VpcId]" \
+  --output text | grep "$vpc" | awk '{print $1}' | tr -d '\r' | tr '\n' ' ')
+for eip in $eips; do
+    echo "Terminating eip: $eip"
+    aws ec2 release-address --allocation-id "$eip"
+done
+
 # Detach and Delete Security Groups
 security_groups=$(aws ec2 describe-security-groups \
     --filters Name=vpc-id,Values=$vpc \
@@ -39,12 +60,14 @@ for sg in $security_groups; do
         --query "NetworkInterfaces[].NetworkInterfaceId" \
         --output text | tr -d '\r' | tr '\n' ' ')
     for eni in $enis; do
+        echo "Terminating delete-security-group: $eni"
         aws ec2 modify-network-interface-attribute \
             --network-interface-id "$eni" \
             --groups "$(aws ec2 describe-security-groups \
                 --query 'SecurityGroups[?GroupName==`default`].GroupId' \
                 --output text)"
     done
+    echo "Terminating delete-security-group: $sg"
     aws ec2 delete-security-group --group-id "$sg"
 done
 
@@ -70,6 +93,7 @@ for rt in $route_tables; do
         else
             aws ec2 replace-route-table-association --association-id $assoc_id --route-table-id $first_rt
             aws ec2 delete-route-table --route-table-id "$rt"
+            echo "Terminating delete-route-table: $rt"
         fi
     done
 done
@@ -80,9 +104,29 @@ subnets=$(aws ec2 describe-subnets \
     --query "Subnets[].SubnetId" \
     --output text | tr -d '\r' | tr '\n' ' ')
 for subnet in $subnets; do
+    echo "Terminating delete-subnet: $subnet"
     aws ec2 delete-subnet --subnet-id "$subnet"
 done
 
+# Delete Network Interfaces
+eni_ids=$(aws ec2 describe-network-interfaces \
+    --filters Name=vpc-id,Values=$vpc \
+    --query "NetworkInterfaces[].NetworkInterfaceId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for eni in $eni_ids; do
+    aws ec2 delete-network-interface --network-interface-id "$eni"
+done
+
+# Delete Network ACLs
+nw_acls=$(aws ec2 describe-network-acls \
+    --filters "Name=vpc-id,Values=$vpc" \
+    --query "NetworkAcls[?IsDefault==false].NetworkAclId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for acl in $nw_acls; do
+    echo "Deleting Network ACL: $acl"
+    aws ec2 delete-network-acl --network-acl-id $acl
+done
+
 # Delete vpc
 # try 3 times with 30 seconds interval
 attempts=0

From 31e23a565b8a19fdf0b493c6b12772af17ea6477 Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 17:09:41 +0530
Subject: [PATCH 10/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 scripts/awscleanup.sh | 75 +++++++++++--------------------------------
 1 file changed, 18 insertions(+), 57 deletions(-)

diff --git a/scripts/awscleanup.sh b/scripts/awscleanup.sh
index dd1596b9..36893939 100755
--- a/scripts/awscleanup.sh
+++ b/scripts/awscleanup.sh
@@ -18,37 +18,6 @@ for instance in $instances; do
     aws ec2 terminate-instances --instance-ids "$instance"
 done
 
-# Delete Internet Gateway
-internet_gateways=$(aws ec2 describe-internet-gateways \
-    --filters Name=attachment.vpc-id,Values=$vpc \
-    --query "InternetGateways[].InternetGatewayId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for igw in $internet_gateways; do
-    echo "Terminating delete-internet-gateway: $igw"
-    aws ec2 detach-internet-gateway --internet-gateway-id "$igw" --vpc-id "$vpc"
-    aws ec2 delete-internet-gateway --internet-gateway-id "$igw"
-done
-
-# Delete NAT Gateways
-nat_gateways=$(aws ec2 describe-nat-gateways \
-    --filter Name=vpc-id,Values=$vpc \
-    --query "NatGateways[].NatGatewayId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for ngw in $nat_gateways; do
-    echo "Terminating delete-nat-gateway: $ngw"
-    aws ec2 delete-nat-gateway --nat-gateway-id "$ngw"
-done
-
-# Delete Elastic IPs
-eips=$(aws ec2 describe-addresses \
-  --filters Name=domain,Values=vpc \
-  --query "Addresses[].[AllocationId,Association.VpcId]" \
-  --output text | grep "$vpc" | awk '{print $1}' | tr -d '\r' | tr '\n' ' ')
-for eip in $eips; do
-    echo "Terminating eip: $eip"
-    aws ec2 release-address --allocation-id "$eip"
-done
-
 # Detach and Delete Security Groups
 security_groups=$(aws ec2 describe-security-groups \
     --filters Name=vpc-id,Values=$vpc \
@@ -71,6 +40,16 @@ for sg in $security_groups; do
     aws ec2 delete-security-group --group-id "$sg"
 done
 
+# Delete Subnets
+subnets=$(aws ec2 describe-subnets \
+    --filters Name=vpc-id,Values=$vpc \
+    --query "Subnets[].SubnetId" \
+    --output text | tr -d '\r' | tr '\n' ' ')
+for subnet in $subnets; do
+    echo "Terminating delete-subnet: $subnet"
+    aws ec2 delete-subnet --subnet-id "$subnet"
+done
+
 # Delete Route Tables
 # 1. Make first rt as Main , as we cannot delete vpcs attached with main
 # 2. replace all rt with first rt
@@ -98,33 +77,15 @@ for rt in $route_tables; do
     done
 done
 
-# Delete Subnets
-subnets=$(aws ec2 describe-subnets \
-    --filters Name=vpc-id,Values=$vpc \
-    --query "Subnets[].SubnetId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for subnet in $subnets; do
-    echo "Terminating delete-subnet: $subnet"
-    aws ec2 delete-subnet --subnet-id "$subnet"
-done
-
-# Delete Network Interfaces
-eni_ids=$(aws ec2 describe-network-interfaces \
-    --filters Name=vpc-id,Values=$vpc \
-    --query "NetworkInterfaces[].NetworkInterfaceId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for eni in $eni_ids; do
-    aws ec2 delete-network-interface --network-interface-id "$eni"
-done
-
-# Delete Network ACLs
-nw_acls=$(aws ec2 describe-network-acls \
-    --filters "Name=vpc-id,Values=$vpc" \
-    --query "NetworkAcls[?IsDefault==false].NetworkAclId" \
+# Delete Internet Gateway
+internet_gateways=$(aws ec2 describe-internet-gateways \
+    --filters Name=attachment.vpc-id,Values=$vpc \
+    --query "InternetGateways[].InternetGatewayId" \
     --output text | tr -d '\r' | tr '\n' ' ')
-for acl in $nw_acls; do
-    echo "Deleting Network ACL: $acl"
-    aws ec2 delete-network-acl --network-acl-id $acl
+for igw in $internet_gateways; do
+    echo "Terminating delete-internet-gateway: $igw"
+    aws ec2 detach-internet-gateway --internet-gateway-id "$igw" --vpc-id "$vpc"
+    aws ec2 delete-internet-gateway --internet-gateway-id "$igw"
 done
 
 # Delete vpc

From 4d05d00381e9d3d365e976f8853bdccf6620d809 Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 17:15:22 +0530
Subject: [PATCH 11/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 scripts/awscleanup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/awscleanup.sh b/scripts/awscleanup.sh
index 36893939..b4a386f1 100755
--- a/scripts/awscleanup.sh
+++ b/scripts/awscleanup.sh
@@ -71,10 +71,10 @@ for rt in $route_tables; do
             first_rt=$rt
         else
             aws ec2 replace-route-table-association --association-id $assoc_id --route-table-id $first_rt
-            aws ec2 delete-route-table --route-table-id "$rt"
             echo "Terminating delete-route-table: $rt"
         fi
     done
+    aws ec2 delete-route-table --route-table-id "$rt"
 done
 
 # Delete Internet Gateway

From b6dabe2ee2e7da3915a6aa2b89c99aac6e83916c Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 17:37:42 +0530
Subject: [PATCH 12/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 scripts/awscleanup.sh | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/scripts/awscleanup.sh b/scripts/awscleanup.sh
index b4a386f1..113524f5 100755
--- a/scripts/awscleanup.sh
+++ b/scripts/awscleanup.sh
@@ -14,7 +14,6 @@ instances=$(aws ec2 describe-instances \
     --query "Reservations[].Instances[].InstanceId" \
     --output text | tr -d '\r' | tr '\n' ' ')
 for instance in $instances; do
-    echo "Terminating instance: $instance"
     aws ec2 terminate-instances --instance-ids "$instance"
 done
 
@@ -29,14 +28,12 @@ for sg in $security_groups; do
         --query "NetworkInterfaces[].NetworkInterfaceId" \
         --output text | tr -d '\r' | tr '\n' ' ')
     for eni in $enis; do
-        echo "Terminating delete-security-group: $eni"
         aws ec2 modify-network-interface-attribute \
             --network-interface-id "$eni" \
             --groups "$(aws ec2 describe-security-groups \
                 --query 'SecurityGroups[?GroupName==`default`].GroupId' \
                 --output text)"
     done
-    echo "Terminating delete-security-group: $sg"
     aws ec2 delete-security-group --group-id "$sg"
 done
 
@@ -46,7 +43,6 @@ subnets=$(aws ec2 describe-subnets \
     --query "Subnets[].SubnetId" \
     --output text | tr -d '\r' | tr '\n' ' ')
 for subnet in $subnets; do
-    echo "Terminating delete-subnet: $subnet"
     aws ec2 delete-subnet --subnet-id "$subnet"
 done
 
@@ -71,10 +67,9 @@ for rt in $route_tables; do
             first_rt=$rt
         else
             aws ec2 replace-route-table-association --association-id $assoc_id --route-table-id $first_rt
-            echo "Terminating delete-route-table: $rt"
         fi
     done
-    aws ec2 delete-route-table --route-table-id "$rt"
+    aws ec2 delete-route-table --route-table-id "$rt" 2>>/dev/null
 done
 
 # Delete Internet Gateway
@@ -83,7 +78,6 @@ internet_gateways=$(aws ec2 describe-internet-gateways \
     --query "InternetGateways[].InternetGatewayId" \
     --output text | tr -d '\r' | tr '\n' ' ')
 for igw in $internet_gateways; do
-    echo "Terminating delete-internet-gateway: $igw"
     aws ec2 detach-internet-gateway --internet-gateway-id "$igw" --vpc-id "$vpc"
     aws ec2 delete-internet-gateway --internet-gateway-id "$igw"
 done
@@ -93,7 +87,6 @@ done
 attempts=0
 echo "All resource Deleted for VPC: $vpc , now delete vpc"
 while [ $attempts -lt 3 ]; do
-    echo "Attempting to delete VPC: $vpc (Attempt $((attempts+1)))"
     if aws ec2 delete-vpc --vpc-id $vpc; then
         echo "Successfully deleted VPC: $vpc"
         break

From bc1c896f431f285e7e7cd18a20862c71a1dcaa45 Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 18:14:06 +0530
Subject: [PATCH 13/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 .github/workflows/awscleanup.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/awscleanup.yaml b/.github/workflows/awscleanup.yaml
index 78dda0d4..a4e8024f 100644
--- a/.github/workflows/awscleanup.yaml
+++ b/.github/workflows/awscleanup.yaml
@@ -57,6 +57,8 @@ jobs:
             if [[ -z "$response" || "$response" == "null" ]]; then
               continue
             fi
+            echo $response
+            exit 0
             status=$(echo "$response" | jq -r ".jobs? // [] | map(select(.name | test(\"^$job_name\"))) | .[0]?.status // \"null\"" 2>/dev/null || echo "null")
             if [[ "$status" != "null" && ! -z "$status" && $(echo "$status" | grep -qvE '^(queued|in_progress)$'; echo $?) -eq 0 ]]; then
               echo "Holodeck e2e Job status is not in running stage , Delete the dependent resources"

From 385fcb78b19faecca803fdab52dd6f25f4ec4911 Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 18:50:52 +0530
Subject: [PATCH 14/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 .github/workflows/awscleanup.yaml | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/awscleanup.yaml b/.github/workflows/awscleanup.yaml
index a4e8024f..2b0c5dc6 100644
--- a/.github/workflows/awscleanup.yaml
+++ b/.github/workflows/awscleanup.yaml
@@ -57,10 +57,13 @@ jobs:
             if [[ -z "$response" || "$response" == "null" ]]; then
               continue
             fi
-            echo $response
-            exit 0
-            status=$(echo "$response" | jq -r ".jobs? // [] | map(select(.name | test(\"^$job_name\"))) | .[0]?.status // \"null\"" 2>/dev/null || echo "null")
-            if [[ "$status" != "null" && ! -z "$status" && $(echo "$status" | grep -qvE '^(queued|in_progress)$'; echo $?) -eq 0 ]]; then
+            echo "SHIVA $response"
+            is_job_running=$(echo "$response" | jq -r "
+              .jobs? // [] |
+              map(select(.status == \"in_progress\" or .status == \"queued\")) |
+              length
+            ")
+            if [[ "$is_job_running" -eq 0 ]]; then
               echo "Holodeck e2e Job status is not in running stage , Delete the dependent resources"
               scripts/awscleanup.sh $vpc
             fi

From 34988d4fde66f294051120efa95af06eb650089a Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 19:05:21 +0530
Subject: [PATCH 15/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 .github/workflows/awscleanup.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/awscleanup.yaml b/.github/workflows/awscleanup.yaml
index 2b0c5dc6..e0797fb2 100644
--- a/.github/workflows/awscleanup.yaml
+++ b/.github/workflows/awscleanup.yaml
@@ -57,13 +57,13 @@ jobs:
             if [[ -z "$response" || "$response" == "null" ]]; then
               continue
             fi
-            echo "SHIVA $response"
-            is_job_running=$(echo "$response" | jq -r "
+            
+            is_jobs_not_completed=$(echo "$response" | jq -r "
               .jobs? // [] |
-              map(select(.status == \"in_progress\" or .status == \"queued\")) |
-              length
-            ")
-            if [[ "$is_job_running" -eq 0 ]]; then
+              map(select(.status != \"completed\")) |
+              length")
+
+            if [[ "$is_jobs_not_completed" -eq 0 ]]; then
               echo "Holodeck e2e Job status is not in running stage , Delete the dependent resources"
               scripts/awscleanup.sh $vpc
             fi

From 9264ab26ec8e3e66a25eab2a85972b869892399d Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 19:12:45 +0530
Subject: [PATCH 16/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 .github/workflows/awscleanup.yaml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/awscleanup.yaml b/.github/workflows/awscleanup.yaml
index e0797fb2..55f91831 100644
--- a/.github/workflows/awscleanup.yaml
+++ b/.github/workflows/awscleanup.yaml
@@ -58,13 +58,20 @@ jobs:
               continue
             fi
             
+            echo "SHIVA $response"
+            echo "$response" | jq -r "
+              .jobs? // [] |
+              map(select(.status != \"completed\")) |
+              length")
+            echo "SHIVA1"
             is_jobs_not_completed=$(echo "$response" | jq -r "
               .jobs? // [] |
               map(select(.status != \"completed\")) |
               length")
 
+            echo "SHIVA2 $is_jobs_not_completed"
             if [[ "$is_jobs_not_completed" -eq 0 ]]; then
-              echo "Holodeck e2e Job status is not in running stage , Delete the dependent resources"
+              echo "Holodeck e2e Job status is not in running stage , Delete the vpc $vpc and dependent resources"
               scripts/awscleanup.sh $vpc
             fi
           done

From af705c723a3f520748c2c93c76bec9bac049e9ba Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 19:15:58 +0530
Subject: [PATCH 17/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 .github/workflows/awscleanup.yaml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.github/workflows/awscleanup.yaml b/.github/workflows/awscleanup.yaml
index 55f91831..2af2e579 100644
--- a/.github/workflows/awscleanup.yaml
+++ b/.github/workflows/awscleanup.yaml
@@ -59,11 +59,7 @@ jobs:
             fi
             
             echo "SHIVA $response"
-            echo "$response" | jq -r "
-              .jobs? // [] |
-              map(select(.status != \"completed\")) |
-              length")
-            echo "SHIVA1"
+      
             is_jobs_not_completed=$(echo "$response" | jq -r "
               .jobs? // [] |
               map(select(.status != \"completed\")) |

From edd8729ddee75bbea1c56da248da3744d8c2c6a7 Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 19:29:13 +0530
Subject: [PATCH 18/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 .github/workflows/awscleanup.yaml | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/awscleanup.yaml b/.github/workflows/awscleanup.yaml
index 2af2e579..d654b0b3 100644
--- a/.github/workflows/awscleanup.yaml
+++ b/.github/workflows/awscleanup.yaml
@@ -58,14 +58,16 @@ jobs:
               continue
             fi
             
-            echo "SHIVA $response"
-      
-            is_jobs_not_completed=$(echo "$response" | jq -r "
-              .jobs? // [] |
+            # 1. make sure .jobs exist in response
+            # e.g. { "message": "Not Found", "documentation_url": "https://docs.github.com/rest", "status": "404" }
+            # 2. check if all jobs completed
+            is_jobs_not_completed=1
+            jobs_exist=$(echo "$response" | jq -e '.jobs? // null' >/dev/null 2>&1; echo $?)
+            if [[ "$jobs_exist" -ne 0 ]]; then
+              is_jobs_not_completed=$(echo "$response" | jq -r ".jobs? // [] |
               map(select(.status != \"completed\")) |
               length")
-
-            echo "SHIVA2 $is_jobs_not_completed"
+            fi
             if [[ "$is_jobs_not_completed" -eq 0 ]]; then
               echo "Holodeck e2e Job status is not in running stage , Delete the vpc $vpc and dependent resources"
               scripts/awscleanup.sh $vpc

From bac82fc040b47b108de8b6ad7a2256bbd826aab2 Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 19:39:49 +0530
Subject: [PATCH 19/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 .github/workflows/awscleanup.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/awscleanup.yaml b/.github/workflows/awscleanup.yaml
index d654b0b3..7459d250 100644
--- a/.github/workflows/awscleanup.yaml
+++ b/.github/workflows/awscleanup.yaml
@@ -68,6 +68,7 @@ jobs:
               map(select(.status != \"completed\")) |
               length")
             fi
+            echo "SHIVA $vpc $jobs_exist $is_jobs_not_completed"
             if [[ "$is_jobs_not_completed" -eq 0 ]]; then
               echo "Holodeck e2e Job status is not in running stage , Delete the vpc $vpc and dependent resources"
               scripts/awscleanup.sh $vpc

From 69264e6e83e45ac1f85cacee38a0463f6f71d133 Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 19:43:00 +0530
Subject: [PATCH 20/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 .github/workflows/awscleanup.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/awscleanup.yaml b/.github/workflows/awscleanup.yaml
index 7459d250..a66544d1 100644
--- a/.github/workflows/awscleanup.yaml
+++ b/.github/workflows/awscleanup.yaml
@@ -62,6 +62,9 @@ jobs:
             # e.g. { "message": "Not Found", "documentation_url": "https://docs.github.com/rest", "status": "404" }
             # 2. check if all jobs completed
             is_jobs_not_completed=1
+            echo "SHIVA $response"
+            echo "$response" | jq -e '.jobs? // null'
+            echo "SHIVA==="
             jobs_exist=$(echo "$response" | jq -e '.jobs? // null' >/dev/null 2>&1; echo $?)
             if [[ "$jobs_exist" -ne 0 ]]; then
               is_jobs_not_completed=$(echo "$response" | jq -r ".jobs? // [] |

From dd0d16c2de9cfa45211fb3cef938b2d668b27649 Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 20:08:27 +0530
Subject: [PATCH 21/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 .github/workflows/awscleanup.yaml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/awscleanup.yaml b/.github/workflows/awscleanup.yaml
index a66544d1..b6b06f51 100644
--- a/.github/workflows/awscleanup.yaml
+++ b/.github/workflows/awscleanup.yaml
@@ -63,15 +63,12 @@ jobs:
             # 2. check if all jobs completed
             is_jobs_not_completed=1
             echo "SHIVA $response"
-            echo "$response" | jq -e '.jobs? // null'
-            echo "SHIVA==="
             jobs_exist=$(echo "$response" | jq -e '.jobs? // null' >/dev/null 2>&1; echo $?)
             if [[ "$jobs_exist" -ne 0 ]]; then
               is_jobs_not_completed=$(echo "$response" | jq -r ".jobs? // [] |
               map(select(.status != \"completed\")) |
               length")
             fi
-            echo "SHIVA $vpc $jobs_exist $is_jobs_not_completed"
             if [[ "$is_jobs_not_completed" -eq 0 ]]; then
               echo "Holodeck e2e Job status is not in running stage , Delete the vpc $vpc and dependent resources"
               scripts/awscleanup.sh $vpc

From c07f700104aa4032f7f662bd3a948009f69aa57a Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Thu, 19 Dec 2024 21:13:48 +0530
Subject: [PATCH 22/25] unused aws instace and vpcs cleanup

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 .github/workflows/awscleanup.yaml | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/awscleanup.yaml b/.github/workflows/awscleanup.yaml
index b6b06f51..172902d6 100644
--- a/.github/workflows/awscleanup.yaml
+++ b/.github/workflows/awscleanup.yaml
@@ -61,14 +61,15 @@ jobs:
             # 1. make sure .jobs exist in response
             # e.g. { "message": "Not Found", "documentation_url": "https://docs.github.com/rest", "status": "404" }
             # 2. check if all jobs completed
-            is_jobs_not_completed=1
-            echo "SHIVA $response"
-            jobs_exist=$(echo "$response" | jq -e '.jobs? // null' >/dev/null 2>&1; echo $?)
-            if [[ "$jobs_exist" -ne 0 ]]; then
-              is_jobs_not_completed=$(echo "$response" | jq -r ".jobs? // [] |
+            
+            if ! echo "$response" | jq -e '.jobs != null' >/dev/null 2>&1; then
+              continue
+            fi
+            
+            is_jobs_not_completed=$(echo "$response" | jq -r ".jobs? // [] |
               map(select(.status != \"completed\")) |
               length")
-            fi
+           
             if [[ "$is_jobs_not_completed" -eq 0 ]]; then
               echo "Holodeck e2e Job status is not in running stage , Delete the vpc $vpc and dependent resources"
               scripts/awscleanup.sh $vpc

From 014a6e07e043ed586f8b01b79625094d07f4c38a Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Fri, 20 Dec 2024 12:54:58 +0530
Subject: [PATCH 23/25] Add script to clean-up unused AWS EC2 Instances and
 VPCs

---
 .github/workflows/awscleanup.yaml | 2 +-
 scripts/awscleanup.sh             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/awscleanup.yaml b/.github/workflows/awscleanup.yaml
index 172902d6..0cb55c21 100644
--- a/.github/workflows/awscleanup.yaml
+++ b/.github/workflows/awscleanup.yaml
@@ -35,7 +35,7 @@ jobs:
             --query "Vpcs[].VpcId" \
             --output text | tr -d '\r' | tr '\n' ' ')
           echo "Found VPCs: $vpcs"
-          echo "vpcs=$vpcs" >> $GITHUB_ENV
+          echo "AWS_VPC_IDS=$vpcs" >> $GITHUB_ENV
 
       - name: Clean up VPCs
         if: env.vpcs != ''
diff --git a/scripts/awscleanup.sh b/scripts/awscleanup.sh
index 113524f5..1e137ade 100755
--- a/scripts/awscleanup.sh
+++ b/scripts/awscleanup.sh
@@ -6,7 +6,7 @@ if [[ $# -ne 1 ]]; then
 fi
 export vpc=$1
 
-echo "Start Deleting  VPC: $vpc resource"
+echo "Start cleanup of resources in  VPC: $vpc"
 
 # Delete Instance
 instances=$(aws ec2 describe-instances \

From aa1d57e698ee7dac9fcb050ee1b4fd997c13eea5 Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Fri, 20 Dec 2024 13:10:19 +0530
Subject: [PATCH 24/25] Add script to clean-up unused AWS EC2 Instances and
 VPCs

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 .github/workflows/awscleanup.yaml |  36 +----
 scripts/awscleanup.sh             | 224 ++++++++++++++++++------------
 2 files changed, 136 insertions(+), 124 deletions(-)

diff --git a/.github/workflows/awscleanup.yaml b/.github/workflows/awscleanup.yaml
index 0cb55c21..841ba2d7 100644
--- a/.github/workflows/awscleanup.yaml
+++ b/.github/workflows/awscleanup.yaml
@@ -38,42 +38,12 @@ jobs:
           echo "AWS_VPC_IDS=$vpcs" >> $GITHUB_ENV
 
       - name: Clean up VPCs
-        if: env.vpcs != ''
+        if: env.AWS_VPC_IDS != ''
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          get_tag_value(){
-            local vpc_id=$1
-            local key=$2
-            aws ec2 describe-tags --filters "Name=resource-id,Values=$vpc_id" "Name=key,Values=$key" \
-              --query "Tags[0].Value" --output text
-          }
-          for vpc in $vpcs; do
-            github_repository=$(get_tag_value $vpc "GitHubRepository")
-            run_id=$(get_tag_value $vpc "GitHubRunId")
-            job_name=$(get_tag_value $vpc "GitHubJob")
-            response=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" \
-              "https://api.github.com/repos/${github_repository}/actions/runs/${run_id}/jobs")
-            if [[ -z "$response" || "$response" == "null" ]]; then
-              continue
-            fi
-            
-            # 1. make sure .jobs exist in response
-            # e.g. { "message": "Not Found", "documentation_url": "https://docs.github.com/rest", "status": "404" }
-            # 2. check if all jobs completed
-            
-            if ! echo "$response" | jq -e '.jobs != null' >/dev/null 2>&1; then
-              continue
-            fi
-            
-            is_jobs_not_completed=$(echo "$response" | jq -r ".jobs? // [] |
-              map(select(.status != \"completed\")) |
-              length")
-           
-            if [[ "$is_jobs_not_completed" -eq 0 ]]; then
-              echo "Holodeck e2e Job status is not in running stage , Delete the vpc $vpc and dependent resources"
-              scripts/awscleanup.sh $vpc
-            fi
+          for vpcid in $AWS_VPC_IDS; do
+            scripts/awscleanup.sh $vpcid
           done
 
       - name: Post cleanup
diff --git a/scripts/awscleanup.sh b/scripts/awscleanup.sh
index 1e137ade..3e0b3bcb 100755
--- a/scripts/awscleanup.sh
+++ b/scripts/awscleanup.sh
@@ -1,103 +1,145 @@
 #!/bin/bash
 
 if [[ $# -ne 1 ]]; then
-	echo " vpcid required for deletion"
-	exit 1
+    echo " vpcid required for deletion"
+    exit 1
 fi
-export vpc=$1
-
-echo "Start cleanup of resources in  VPC: $vpc"
-
-# Delete Instance
-instances=$(aws ec2 describe-instances \
-    --filters "Name=vpc-id,Values=$vpc" \
-    --query "Reservations[].Instances[].InstanceId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for instance in $instances; do
-    aws ec2 terminate-instances --instance-ids "$instance"
-done
-
-# Detach and Delete Security Groups
-security_groups=$(aws ec2 describe-security-groups \
-    --filters Name=vpc-id,Values=$vpc \
-    --query "SecurityGroups[?GroupName!='default'].GroupId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for sg in $security_groups; do
-    enis=$(aws ec2 describe-network-interfaces \
-        --filters Name=group-id,Values=$sg \
-        --query "NetworkInterfaces[].NetworkInterfaceId" \
+
+export vpcid=$1
+
+get_tag_value(){
+    local vpc=$1
+    local key=$2
+    aws ec2 describe-tags --filters "Name=resource-id,Values=$vpcid" "Name=key,Values=$key" \
+        --query "Tags[0].Value" --output text
+}
+
+delete_vpc_resources() {
+    if [[ $# -ne 1 ]]; then
+        echo " vpcid required for deletion"
+        exit 1
+    fi
+    export vpcid=$1
+
+    echo "Start cleanup of resources in  VPC: $vpcid"
+
+    # Delete Instance
+    instances=$(aws ec2 describe-instances \
+        --filters "Name=vpc-id,Values=$vpcid" \
+        --query "Reservations[].Instances[].InstanceId" \
         --output text | tr -d '\r' | tr '\n' ' ')
-    for eni in $enis; do
-        aws ec2 modify-network-interface-attribute \
-            --network-interface-id "$eni" \
-            --groups "$(aws ec2 describe-security-groups \
-                --query 'SecurityGroups[?GroupName==`default`].GroupId' \
-                --output text)"
+    for instance in $instances; do
+        aws ec2 terminate-instances --instance-ids "$instance"
     done
-    aws ec2 delete-security-group --group-id "$sg"
-done
-
-# Delete Subnets
-subnets=$(aws ec2 describe-subnets \
-    --filters Name=vpc-id,Values=$vpc \
-    --query "Subnets[].SubnetId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for subnet in $subnets; do
-    aws ec2 delete-subnet --subnet-id "$subnet"
-done
-
-# Delete Route Tables
-# 1. Make first rt as Main , as we cannot delete vpcs attached with main
-# 2. replace all rt with first rt
-# 3. delete rt
-# 4. Main table(first_rt) will be deleted once vpc deleted
-first_rt=""
-route_tables=$(aws ec2 describe-route-tables \
-    --filters Name=vpc-id,Values=$vpc \
-    --query "RouteTables[].RouteTableId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for rt in $route_tables; do
-    associations=$(aws ec2 describe-route-tables \
-            --route-table-ids "$rt" \
-            --query "RouteTables[].Associations[].RouteTableAssociationId" \
+
+    # Detach and Delete Security Groups
+    security_groups=$(aws ec2 describe-security-groups \
+        --filters Name=vpc-id,Values=$vpcid \
+        --query "SecurityGroups[?GroupName!='default'].GroupId" \
+        --output text | tr -d '\r' | tr '\n' ' ')
+    for sg in $security_groups; do
+        enis=$(aws ec2 describe-network-interfaces \
+            --filters Name=group-id,Values=$sg \
+            --query "NetworkInterfaces[].NetworkInterfaceId" \
             --output text | tr -d '\r' | tr '\n' ' ')
-    for assoc_id in $associations; do
-        if [ -z "$first_rt" ]; then
-            aws ec2 replace-route-table-association --association-id $assoc_id --route-table-id $rt
-            first_rt=$rt
+        for eni in $enis; do
+            aws ec2 modify-network-interface-attribute \
+                --network-interface-id "$eni" \
+                --groups "$(aws ec2 describe-security-groups \
+                    --query 'SecurityGroups[?GroupName==`default`].GroupId' \
+                    --output text)"
+        done
+        aws ec2 delete-security-group --group-id "$sg"
+    done
+
+    # Delete Subnets
+    subnets=$(aws ec2 describe-subnets \
+        --filters Name=vpc-id,Values=$vpcid \
+        --query "Subnets[].SubnetId" \
+        --output text | tr -d '\r' | tr '\n' ' ')
+    for subnet in $subnets; do
+        aws ec2 delete-subnet --subnet-id "$subnet"
+    done
+
+    # Delete Route Tables
+    # 1. Make first rt as Main , as we cannot delete vpcs attached with main
+    # 2. replace all rt with first rt
+    # 3. delete rt
+    # 4. Main table(first_rt) will be deleted once vpc deleted
+    first_rt=""
+    route_tables=$(aws ec2 describe-route-tables \
+        --filters Name=vpc-id,Values=$vpcid \
+        --query "RouteTables[].RouteTableId" \
+        --output text | tr -d '\r' | tr '\n' ' ')
+    for rt in $route_tables; do
+        associations=$(aws ec2 describe-route-tables \
+                --route-table-ids "$rt" \
+                --query "RouteTables[].Associations[].RouteTableAssociationId" \
+                --output text | tr -d '\r' | tr '\n' ' ')
+        for assoc_id in $associations; do
+            if [ -z "$first_rt" ]; then
+                aws ec2 replace-route-table-association --association-id $assoc_id --route-table-id $rt
+                first_rt=$rt
+            else
+                aws ec2 replace-route-table-association --association-id $assoc_id --route-table-id $first_rt
+            fi
+        done
+        aws ec2 delete-route-table --route-table-id "$rt" 2>>/dev/null
+    done
+
+    # Delete Internet Gateway
+    internet_gateways=$(aws ec2 describe-internet-gateways \
+        --filters Name=attachment.vpc-id,Values=$vpcid \
+        --query "InternetGateways[].InternetGatewayId" \
+        --output text | tr -d '\r' | tr '\n' ' ')
+    for igw in $internet_gateways; do
+        aws ec2 detach-internet-gateway --internet-gateway-id "$igw" --vpc-id "$vpcid"
+        aws ec2 delete-internet-gateway --internet-gateway-id "$igw"
+    done
+
+    # Delete vpc
+    # try 3 times with 30 seconds interval
+    attempts=0
+    echo "All resource Deleted for VPC: $vpcid , now delete vpc"
+    while [ $attempts -lt 3 ]; do
+        if aws ec2 delete-vpc --vpc-id $vpcid; then
+            echo "Successfully deleted VPC: $vpcid"
+            break
         else
-            aws ec2 replace-route-table-association --association-id $assoc_id --route-table-id $first_rt
+            attempts=$((attempts + 1))
+            if [ $attempts -lt 3 ]; then
+                echo "Failed to delete VPC: $vpcid. Retrying in 30 seconds..."
+                sleep 30
+            fi
         fi
     done
-    aws ec2 delete-route-table --route-table-id "$rt" 2>>/dev/null
-done
-
-# Delete Internet Gateway
-internet_gateways=$(aws ec2 describe-internet-gateways \
-    --filters Name=attachment.vpc-id,Values=$vpc \
-    --query "InternetGateways[].InternetGatewayId" \
-    --output text | tr -d '\r' | tr '\n' ' ')
-for igw in $internet_gateways; do
-    aws ec2 detach-internet-gateway --internet-gateway-id "$igw" --vpc-id "$vpc"
-    aws ec2 delete-internet-gateway --internet-gateway-id "$igw"
-done
-
-# Delete vpc
-# try 3 times with 30 seconds interval
-attempts=0
-echo "All resource Deleted for VPC: $vpc , now delete vpc"
-while [ $attempts -lt 3 ]; do
-    if aws ec2 delete-vpc --vpc-id $vpc; then
-        echo "Successfully deleted VPC: $vpc"
-        break
-    else
-        attempts=$((attempts + 1))
-        if [ $attempts -lt 3 ]; then
-            echo "Failed to delete VPC: $vpc. Retrying in 30 seconds..."
-            sleep 30
-        fi
+    if [ $attempts -eq 3 ]; then
+        echo "Failed to delete VPC: $vpcid after 3 attempts. Continue the loop to delete other vpc"
     fi
-done
-if [ $attempts -eq 3 ]; then
-    echo "Failed to delete VPC: $vpc after 3 attempts. Continue the loop to delete other vpc"
+}
+
+github_repository=$(get_tag_value $vpcid "GitHubRepository")
+run_id=$(get_tag_value $vpcid "GitHubRunId")
+job_name=$(get_tag_value $vpcid "GitHubJob")
+response=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" \
+    "https://api.github.com/repos/${github_repository}/actions/runs/${run_id}/jobs")
+if [[ -z "$response" || "$response" == "null" ]]; then
+    exit 0
+fi
+
+# 1. make sure .jobs exist in response
+# e.g. { "message": "Not Found", "documentation_url": "https://docs.github.com/rest", "status": "404" }
+# 2. check if all jobs completed
+
+if ! echo "$response" | jq -e '.jobs != null' >/dev/null 2>&1; then
+    exit 0
+fi
+
+is_jobs_not_completed=$(echo "$response" | jq -r ".jobs? // [] |
+    map(select(.status != \"completed\")) |
+    length")
+
+if [[ "$is_jobs_not_completed" -eq 0 ]]; then
+    echo "Holodeck e2e Job status is not in running stage , Delete the vpc $vpcid and dependent resources"
+    delete_vpc_resources $vpcid
 fi

From d306460d188ddf096ce049422a363eeea7cb91f7 Mon Sep 17 00:00:00 2001
From: shiva kumar <shivaku@nvidia.com>
Date: Fri, 20 Dec 2024 13:29:56 +0530
Subject: [PATCH 25/25] Add script to clean-up unused AWS EC2 Instances and
 VPCs

Signed-off-by: shiva kumar <shivaku@nvidia.com>
---
 scripts/awscleanup.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scripts/awscleanup.sh b/scripts/awscleanup.sh
index 3e0b3bcb..a7a66bb8 100755
--- a/scripts/awscleanup.sh
+++ b/scripts/awscleanup.sh
@@ -8,6 +8,10 @@ fi
 export vpcid=$1
 
 get_tag_value(){
+    if [[ $# -ne 2 ]]; then
+        echo " vpcid and key required to get tag value"
+        exit 1
+    fi
     local vpc=$1
     local key=$2
     aws ec2 describe-tags --filters "Name=resource-id,Values=$vpcid" "Name=key,Values=$key" \
@@ -19,7 +23,7 @@ delete_vpc_resources() {
         echo " vpcid required for deletion"
         exit 1
     fi
-    export vpcid=$1
+    local vpcid=$1
 
     echo "Start cleanup of resources in  VPC: $vpcid"