Optionally delay destroy in github actions to allow investigation (#407)

* Optionally delay destroy in github actions to allow investigation * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time * Optionally delay destroy in github actions to allow investigation time
imperva · May 1, 2024 · 68da10a · 68da10a
1 parent 3c7fc91
commit 68da10a
Show file tree

Hide file tree

Showing 7 changed files with 211 additions and 69 deletions.
diff --git a/.github/workflows/dsf_poc_cli.yml b/.github/workflows/dsf_poc_cli.yml
@@ -9,6 +9,11 @@ on:
       explicit_ref:
         required: true
         type: string
+      delay_destroy:
+        description: 'Delay the destroy step and subsequent steps to allow investigation'
+        type: boolean
+        default: false
+        required: true
     secrets:
       AWS_ACCESS_KEY_ID:
         required: true
@@ -55,6 +60,7 @@ env:
   TF_INPUT: 0
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  DESTROY_DELAY_SECONDS: 1800
 
 permissions:
   contents: read
@@ -211,13 +217,37 @@ jobs:
       run: terraform -chdir=$EXAMPLE_DIR output -json
 
     - name: Collect Artifacts
+      id: collect-artifacts
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
-        name: collected-keys
+        name: collected-keys-${{ env.TF_WORKSPACE }}
         path: |
           ${{ env.EXAMPLE_DIR }}/ssh_keys
 
+    - name: Check how was the workflow run
+      id: check-trigger
+      if: ${{ failure() }}
+      run: |
+        if [ "${{ github.event_name }}" == "schedule" ]; then
+          echo "run-by=Automation" >> $GITHUB_OUTPUT
+        else
+          echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
+        fi
+
+    # This step allows time for investigation of the failed resources before destroying them
+    - name: Conditional Delay
+      if: ${{ failure() }}
+      run: |
+        echo "delay_destroy: ${{ inputs.delay_destroy }}"
+        if [ "${{ inputs.delay_destroy }}" == "true" ]; then
+          echo "Terraform workspace: $TF_WORKSPACE"
+          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
+          echo ""
+          echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"
+          sleep $DESTROY_DELAY_SECONDS
+        fi
+
     - name: Terraform Destroy
       id: destroy
       if: always()
@@ -235,16 +265,6 @@ jobs:
       env:
         TF_WORKSPACE: default
 
-    - name: Check how was the workflow run
-      id: check-trigger
-      if: ${{ failure() }}
-      run: |
-        if [ "${{ github.event_name }}" == "schedule" ]; then
-          echo "run-by=Automation" >> $GITHUB_OUTPUT
-        else
-          echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
-        fi
-
       # Send job failure to Slack
     - name: Send Slack When Failure
       run: |

diff --git a/.github/workflows/dsf_poc_cli_azure.yml b/.github/workflows/dsf_poc_cli_azure.yml
@@ -9,6 +9,12 @@ on:
       explicit_ref:
         required: true
         type: string
+      delay_destroy:
+        description: 'Delay the destroy step and subsequent steps to allow investigation'
+        type: boolean
+        default: false
+        required: true
+
     secrets:
       AWS_ACCESS_KEY_ID:
         required: true
@@ -55,6 +61,7 @@ env:
   ARM_CLIENT_ID: ${{ vars.ARM_CLIENT_ID }}
   ARM_TENANT_ID: ${{ vars.ARM_TENANT_ID }}
   ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
+  DESTROY_DELAY_SECONDS: 1800
 permissions:
   contents: read
 
@@ -237,13 +244,37 @@ jobs:
       run: terraform -chdir=$EXAMPLE_DIR output -json
 
     - name: Collect Artifacts
+      id: collect-artifacts
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
-        name: collected-keys
+        name: collected-keys-${{ env.TF_WORKSPACE }}
         path: |
           ${{ env.EXAMPLE_DIR }}/ssh_keys
 
+    - name: Check how was the workflow run
+      id: check-trigger
+      if: ${{ failure() }}
+      run: |
+        if [ "${{ github.event_name }}" == "schedule" ]; then
+          echo "run-by=Automation" >> $GITHUB_OUTPUT
+        else
+          echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
+        fi
+
+    # This step allows time for investigation of the failed resources before destroying them
+    - name: Conditional Delay
+      if: ${{ failure() }}
+      run: |
+        echo "delay_destroy: ${{ inputs.delay_destroy }}"
+        if [ "${{ inputs.delay_destroy }}" == "true" ]; then
+          echo "Terraform workspace: $TF_WORKSPACE"
+          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
+          echo ""
+          echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"
+          sleep $DESTROY_DELAY_SECONDS
+        fi
+
     - name: Terraform Destroy
       id: destroy
       if: always()
@@ -261,16 +292,6 @@ jobs:
       env:
         TF_WORKSPACE: default
 
-    - name: Check how was the workflow run
-      id: check-trigger
-      if: ${{ failure() }}
-      run: |
-        if [ "${{ github.event_name }}" == "schedule" ]; then
-          echo "run-by=Automation" >> $GITHUB_OUTPUT
-        else
-          echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
-        fi
-
       # Send job failure to Slack
     - name: Send Slack When Failure
       run: |

diff --git a/.github/workflows/dsf_single_account_cli.yml b/.github/workflows/dsf_single_account_cli.yml
@@ -9,6 +9,11 @@ on:
       branch:
         required: true
         type: string
+      delay_destroy:
+        description: 'Delay the destroy step and subsequent steps to allow investigation'
+        type: boolean
+        default: false
+        required: true
 
   workflow_call:
     inputs:
@@ -73,7 +78,8 @@ env:
   AWS_ACCESS_KEY_ID_STAGE: ${{ secrets.AWS_ACCESS_KEY_ID_STAGE }}
   AWS_SECRET_ACCESS_KEY_STAGE: ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGE }}
   JUMP_SERVER_KEY: ${{ secrets.JUMP_SERVER_KEY }}
-  AWS_KEY_PATH: "aws_key.cer"
+  AWS_KEY_PATH: "jump_server_key.cer"
+  DESTROY_DELAY_SECONDS: 1800
 
 permissions:
   contents: read
@@ -213,13 +219,37 @@ jobs:
       run: terraform -chdir=$EXAMPLE_DIR output -json
 
     - name: Collect Artifacts
+      id: collect-artifacts
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
-        name: collected-keys
+        name: collected-keys-${{ env.TF_WORKSPACE }}
         path: |
           ${{ env.EXAMPLE_DIR }}/ssh_keys
 
+    - name: Check how was the workflow run
+      id: check-trigger
+      if: ${{ failure() }}
+      run: |
+        if [ "${{ github.event_name }}" == "schedule" ]; then
+          echo "run-by=Automation" >> $GITHUB_OUTPUT
+        else
+          echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
+        fi
+
+    # This step allows time for investigation of the failed resources before destroying them
+    - name: Conditional Delay
+      if: ${{ failure() }}
+      run: |
+        echo "delay_destroy: ${{ inputs.delay_destroy }}"
+        if [ "${{ inputs.delay_destroy }}" == "true" ]; then
+          echo "Terraform workspace: $TF_WORKSPACE"
+          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
+          echo ""
+          echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"
+          sleep $DESTROY_DELAY_SECONDS
+        fi
+
     - name: Terraform Destroy
       if: always()
       run: terraform -chdir=$EXAMPLE_DIR destroy -auto-approve
@@ -243,16 +273,6 @@ jobs:
       run: |
         aws ec2 revoke-security-group-ingress --group-id ${{ vars.SYDNEY_SG_ID }} --security-group-rule-ids ${{ env.sg_id_for_sydney }}
 
-    - name: Check how was the workflow run
-      id: check-trigger
-      if: ${{ failure() }}
-      run: |
-        if [ "${{ github.event_name }}" == "schedule" ]; then
-          echo "run-by=Automation" >> $GITHUB_OUTPUT
-        else
-          echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
-        fi
-
     # Send job failure to Slack
     - name: Send Slack When Failure
       run: |

diff --git a/.github/workflows/sonar_multi_account_cli.yml b/.github/workflows/sonar_multi_account_cli.yml
@@ -9,6 +9,11 @@ on:
       branch:
         required: true
         type: string
+      delay_destroy:
+        description: 'Delay the destroy step and subsequent steps to allow investigation'
+        type: boolean
+        default: false
+        required: true
 
   workflow_call:
     inputs:
@@ -39,7 +44,8 @@ env:
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   TF_WORKSPACE: "multi_account"
   JUMP_SERVER_KEY: ${{ secrets.JUMP_SERVER_KEY }}
-  AWS_KEY_PATH: "aws_key.cer"
+  AWS_KEY_PATH: "jump_server_key.cer"
+  DESTROY_DELAY_SECONDS: 1800
 
 permissions:
   contents: read
@@ -154,6 +160,29 @@ jobs:
         path: |
           ${{ env.EXAMPLE_DIR }}/ssh_keys
 
+    - name: Check how was the workflow run
+      id: check-trigger
+      if: ${{ failure() }}
+      run: |
+        if [ "${{ github.event_name }}" == "schedule" ]; then
+          echo "run-by=Automation" >> $GITHUB_OUTPUT
+        else
+          echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
+        fi
+
+    # This step allows time for investigation of the failed resources before destroying them
+    - name: Conditional Delay
+      if: ${{ failure() }}
+      run: |
+        echo "delay_destroy: ${{ inputs.delay_destroy }}"
+        if [ "${{ inputs.delay_destroy }}" == "true" ]; then
+          echo "Terraform workspace: $TF_WORKSPACE"
+          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
+          echo ""
+          echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"
+          sleep $DESTROY_DELAY_SECONDS
+        fi
+
     - name: Terraform Destroy
       if: always()
       run: terraform -chdir=$EXAMPLE_DIR destroy -auto-approve
@@ -166,22 +195,12 @@ jobs:
       if: always()
       run: aws ec2 revoke-security-group-ingress --group-id ${{ vars.JUMP_SERVER_SG_ID }} --security-group-rule-ids ${{ env.sg_id }}
 
-    - name: Check how was the workflow run
-      id: check-trigger
-      if: ${{ failure() }}
-      run: |
-        if [ "${{ github.event_name }}" == "schedule" ]; then
-          echo "run-by=Automation" >> $GITHUB_OUTPUT
-        else
-          echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
-        fi
-
     # Send job failure to Slack
     - name: Send Slack When Failure
       run: |
         if [ ${{ inputs.branch }} == 'master' ]; then
-          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*Multi Account Prod ${{ inputs.workspace }} automation Failed*\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#dsfkit-prod"}' ${{ secrets.SLACK_WEBHOOK_URL }}
+          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*Sonar Multi Account Prod ${{ inputs.workspace }} automation Failed*\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#dsfkit-prod"}' ${{ secrets.SLACK_WEBHOOK_URL }}
         else
-          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*Multi Account Dev nightly ${{ inputs.workspace }} automation Failed*\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
+          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*Sonar Multi Account Dev nightly ${{ inputs.workspace }} automation Failed*\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
         fi
       if: ${{ failure() }}
diff --git a/.github/workflows/sonar_poc_cli.yml b/.github/workflows/sonar_poc_cli.yml
@@ -9,6 +9,12 @@ on:
       explicit_ref:
         required: true
         type: string
+      delay_destroy:
+        description: 'Delay the destroy step and subsequent steps to allow investigation'
+        type: boolean
+        default: false
+        required: true
+
     secrets:
       AWS_ACCESS_KEY_ID:
         required: true
@@ -49,6 +55,7 @@ env:
   TF_VAR_allowed_ssh_cidrs: "[\"82.0.0.0/8\", \"62.0.0.0/8\", \"94.0.0.0/8\"]"
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  DESTROY_DELAY_SECONDS: 1800
 
 permissions:
   contents: read
@@ -59,12 +66,14 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - workspace: basic_cli-${{ github.run_number }}-${{ github.run_attempt }}
+          - name: Sonar Basic CLI
+            workspace: sonar_basic_cli-${{ github.run_number }}-${{ github.run_attempt }}
             example_dir: ./examples/aws/poc/sonar_basic_deployment
             aws_region: eu-west-2
             hub_output: '.dsf_hub.value.public_dns'
             gw_output: '.dsf_agentless_gw.value."agentless-gw-0".private_ip'
-          - workspace: hadr_cli-${{ github.run_number }}-${{ github.run_attempt }}
+          - name: Sonar HADR CLI
+            workspace: sonar_hadr_cli-${{ github.run_number }}-${{ github.run_attempt }}
             example_dir: ./examples/aws/poc/sonar_hadr_deployment
             aws_region: eu-west-1
             hub_output: '.dsf_hubs.value.main.public_dns'
@@ -156,10 +165,11 @@ jobs:
       run: terraform -chdir=$EXAMPLE_DIR output -json
 
     - name: Collect Artifacts
+      id: collect-artifacts
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
-        name: collected-keys
+        name: collected-keys-${{ env.TF_WORKSPACE }}
         path: |
           ${{ env.EXAMPLE_DIR }}/ssh_keys
 
@@ -183,6 +193,29 @@ jobs:
         port: 22
         script: ip addr | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1'
 
+    - name: Check how was the workflow run
+      id: check-trigger
+      if: ${{ failure() }}
+      run: |
+        if [ "${{ github.event_name }}" == "schedule" ]; then
+          echo "run-by=Automation" >> $GITHUB_OUTPUT
+        else
+          echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
+        fi
+
+    # This step allows time for investigation of the failed resources before destroying them
+    - name: Conditional Delay
+      if: ${{ failure() }}
+      run: |
+        echo "delay_destroy: ${{ inputs.delay_destroy }}"
+        if [ "${{ inputs.delay_destroy }}" == "true" ]; then
+          echo "Terraform workspace: $TF_WORKSPACE"
+          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
+          echo ""
+          echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"
+          sleep $DESTROY_DELAY_SECONDS
+        fi
+
     - name: Terraform Destroy
       id: destroy
       if: always()
@@ -206,4 +239,4 @@ jobs:
         fields: author,action,eventName,ref,workflow,pullRequest
       env:
         SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-      if: ${{ failure() }} 
+      if: ${{ failure() }}