Optionally delay destroy in github actions to allow investigation time

imperva · Apr 25, 2024 · c89d6d4 · c89d6d4
1 parent 01eaa72
commit c89d6d4
Show file tree

Hide file tree

Showing 6 changed files with 99 additions and 65 deletions.
diff --git a/.github/workflows/dsf_poc_cli.yml b/.github/workflows/dsf_poc_cli.yml
@@ -224,13 +224,28 @@ jobs:
         path: |
           ${{ env.EXAMPLE_DIR }}/ssh_keys
 
+    - name: Check how was the workflow run
+      id: check-trigger
+      if: ${{ failure() }}
+      run: |
+        if [ "${{ github.event_name }}" == "schedule" ]; then
+          echo "run-by=Automation" >> $GITHUB_OUTPUT
+        else
+          echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
+        fi
+
     # This step allows time for investigation of the failed resources before destroying them
     - name: Conditional Delay
-      if: ${{ failure() && github.event.inputs.delay_destory == true }}
+      if: ${{ failure() }}
       run: |
-        destroy_delay_minutes = $DESTROY_DELAY_SECONDS / 60
-        curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*DSF Single Account ${{ inputs.workspace }} automation Failed*\nYou have $destroy_delay_minutes minutes to investigate the environment before it is destroyed.\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
-        sleep $DESTROY_DELAY_SECONDS
+        echo "delay_destroy: ${{ inputs.delay_destroy }}"
+        if [ "${{ inputs.delay_destroy }}" == "true" ]; then
+          echo "Terraform workspace: $TF_WORKSPACE"
+          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
+          echo ""
+          echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"
+          sleep $DESTROY_DELAY_SECONDS
+        fi
 
     - name: Terraform Destroy
       id: destroy
@@ -249,16 +264,6 @@ jobs:
       env:
         TF_WORKSPACE: default
 
-    - name: Check how was the workflow run
-      id: check-trigger
-      if: ${{ failure() }}
-      run: |
-        if [ "${{ github.event_name }}" == "schedule" ]; then
-          echo "run-by=Automation" >> $GITHUB_OUTPUT
-        else
-          echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
-        fi
-
       # Send job failure to Slack
     - name: Send Slack When Failure
       run: |

diff --git a/.github/workflows/dsf_poc_cli_azure.yml b/.github/workflows/dsf_poc_cli_azure.yml
@@ -251,13 +251,28 @@ jobs:
         path: |
           ${{ env.EXAMPLE_DIR }}/ssh_keys
 
+    - name: Check how was the workflow run
+      id: check-trigger
+      if: ${{ failure() }}
+      run: |
+        if [ "${{ github.event_name }}" == "schedule" ]; then
+          echo "run-by=Automation" >> $GITHUB_OUTPUT
+        else
+          echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
+        fi
+
     # This step allows time for investigation of the failed resources before destroying them
     - name: Conditional Delay
-      if: ${{ failure() && github.event.inputs.delay_destory == true }}
+      if: ${{ failure() }}
       run: |
-        destroy_delay_minutes = $DESTROY_DELAY_SECONDS / 60
-        curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*DSF Single Account ${{ inputs.workspace }} automation Failed*\nYou have $destroy_delay_minutes minutes to investigate the environment before it is destroyed.\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
-        sleep $DESTROY_DELAY_SECONDS
+        echo "delay_destroy: ${{ inputs.delay_destroy }}"
+        if [ "${{ inputs.delay_destroy }}" == "true" ]; then
+          echo "Terraform workspace: $TF_WORKSPACE"
+          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
+          echo ""
+          echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"
+          sleep $DESTROY_DELAY_SECONDS
+        fi
 
     - name: Terraform Destroy
       id: destroy
@@ -276,16 +291,6 @@ jobs:
       env:
         TF_WORKSPACE: default
 
-    - name: Check how was the workflow run
-      id: check-trigger
-      if: ${{ failure() }}
-      run: |
-        if [ "${{ github.event_name }}" == "schedule" ]; then
-          echo "run-by=Automation" >> $GITHUB_OUTPUT
-        else
-          echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
-        fi
-
       # Send job failure to Slack
     - name: Send Slack When Failure
       run: |

diff --git a/.github/workflows/dsf_single_account_cli.yml b/.github/workflows/dsf_single_account_cli.yml
@@ -105,7 +105,6 @@ jobs:
         find ./examples/ -type f -exec sed -i -f sed.expr {} \;
 
     - name: Set Workspace
-      if: always()
       run: |
         if [ ${{ inputs.branch }} == 'master' ] ||  [ ${{ inputs.branch }} == 'dev' ]; then
           echo "TF_WORKSPACE=dsf_single_account_${{ inputs.branch }}" >> $GITHUB_ENV
@@ -243,7 +242,7 @@ jobs:
       run: |
         echo "delay_destroy: ${{ inputs.delay_destroy }}"
         if [ "${{ inputs.delay_destroy }}" == "true" ]; then
-          echo "Workspace is: $TF_WORKSPACE"
+          echo "Terraform workspace: $TF_WORKSPACE"
           curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
           echo ""
           echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"

diff --git a/.github/workflows/sonar_multi_account_cli.yml b/.github/workflows/sonar_multi_account_cli.yml
@@ -160,13 +160,28 @@ jobs:
         path: |
           ${{ env.EXAMPLE_DIR }}/ssh_keys
 
+    - name: Check how was the workflow run
+      id: check-trigger
+      if: ${{ failure() }}
+      run: |
+        if [ "${{ github.event_name }}" == "schedule" ]; then
+          echo "run-by=Automation" >> $GITHUB_OUTPUT
+        else
+          echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
+        fi
+
     # This step allows time for investigation of the failed resources before destroying them
     - name: Conditional Delay
-      if: ${{ failure() && github.event.inputs.delay_destory == true }}
+      if: ${{ failure() }}
       run: |
-        destroy_delay_minutes = $DESTROY_DELAY_SECONDS / 60
-        curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*DSF Single Account ${{ inputs.workspace }} automation Failed*\nYou have $destroy_delay_minutes minutes to investigate the environment before it is destroyed.\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
-        sleep $DESTROY_DELAY_SECONDS
+        echo "delay_destroy: ${{ inputs.delay_destroy }}"
+        if [ "${{ inputs.delay_destroy }}" == "true" ]; then
+          echo "Terraform workspace: $TF_WORKSPACE"
+          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
+          echo ""
+          echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"
+          sleep $DESTROY_DELAY_SECONDS
+        fi
 
     - name: Terraform Destroy
       if: always()
@@ -180,22 +195,12 @@ jobs:
       if: always()
       run: aws ec2 revoke-security-group-ingress --group-id ${{ vars.JUMP_SERVER_SG_ID }} --security-group-rule-ids ${{ env.sg_id }}
 
-    - name: Check how was the workflow run
-      id: check-trigger
-      if: ${{ failure() }}
-      run: |
-        if [ "${{ github.event_name }}" == "schedule" ]; then
-          echo "run-by=Automation" >> $GITHUB_OUTPUT
-        else
-          echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
-        fi
-
     # Send job failure to Slack
     - name: Send Slack When Failure
       run: |
         if [ ${{ inputs.branch }} == 'master' ]; then
-          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*Multi Account Prod ${{ inputs.workspace }} automation Failed*\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#dsfkit-prod"}' ${{ secrets.SLACK_WEBHOOK_URL }}
+          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*Sonar Multi Account Prod ${{ inputs.workspace }} automation Failed*\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#dsfkit-prod"}' ${{ secrets.SLACK_WEBHOOK_URL }}
         else
-          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*Multi Account Dev nightly ${{ inputs.workspace }} automation Failed*\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
+          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*Sonar Multi Account Dev nightly ${{ inputs.workspace }} automation Failed*\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
         fi
       if: ${{ failure() }}
diff --git a/.github/workflows/sonar_poc_cli.yml b/.github/workflows/sonar_poc_cli.yml
@@ -190,13 +190,28 @@ jobs:
         port: 22
         script: ip addr | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1'
 
+    - name: Check how was the workflow run
+      id: check-trigger
+      if: ${{ failure() }}
+      run: |
+        if [ "${{ github.event_name }}" == "schedule" ]; then
+          echo "run-by=Automation" >> $GITHUB_OUTPUT
+        else
+          echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
+        fi
+
     # This step allows time for investigation of the failed resources before destroying them
     - name: Conditional Delay
-      if: ${{ failure() && github.event.inputs.delay_destory == true }}
+      if: ${{ failure() }}
       run: |
-        destroy_delay_minutes = $DESTROY_DELAY_SECONDS / 60
-        curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*DSF Single Account ${{ inputs.workspace }} automation Failed*\nYou have $destroy_delay_minutes minutes to investigate the environment before it is destroyed.\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
-        sleep $DESTROY_DELAY_SECONDS
+        echo "delay_destroy: ${{ inputs.delay_destroy }}"
+        if [ "${{ inputs.delay_destroy }}" == "true" ]; then
+          echo "Terraform workspace: $TF_WORKSPACE"
+          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
+          echo ""
+          echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"
+          sleep $DESTROY_DELAY_SECONDS
+        fi
 
     - name: Terraform Destroy
       id: destroy

diff --git a/.github/workflows/sonar_single_account_cli.yml b/.github/workflows/sonar_single_account_cli.yml
@@ -233,13 +233,28 @@ jobs:
           envs: REMOTE_EXAMPLE_DIR,TF_WORKSPACE
           script: terraform -chdir=$REMOTE_EXAMPLE_DIR output -json
 
+      - name: Check how was the workflow run
+        if: ${{ failure() }}
+        id: check-trigger
+        run: |
+          if [ "${{ github.event_name }}" == "schedule" ]; then
+            echo "run-by=Automation" >> $GITHUB_OUTPUT
+          else
+            echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
+          fi
+
       # This step allows time for investigation of the failed resources before destroying them
       - name: Conditional Delay
-        if: ${{ failure() && github.event.inputs.delay_destory == true }}
-        run: |
-          destroy_delay_minutes = $DESTROY_DELAY_SECONDS / 60
-          curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*DSF Single Account ${{ inputs.workspace }} automation Failed*\nYou have $destroy_delay_minutes minutes to investigate the environment before it is destroyed.\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
-          sleep $DESTROY_DELAY_SECONDS
+      if: ${{ failure() }}
+      run: |
+        echo "delay_destroy: ${{ inputs.delay_destroy }}"
+        if [ "${{ inputs.delay_destroy }}" == "true" ]; then
+        echo "Terraform workspace: $TF_WORKSPACE"
+        curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
+        echo ""
+        echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"
+        sleep $DESTROY_DELAY_SECONDS
+        fi
 
       - name: Terraform Destroy
         #   if: always()
@@ -258,16 +273,6 @@ jobs:
         if: always()
         run: aws ec2 revoke-security-group-ingress --group-id ${{ vars.JUMP_SERVER_SG_ID }} --security-group-rule-ids ${{ env.sg_id }}
 
-      - name: Check how was the workflow run
-        if: ${{ failure() }}
-        id: check-trigger
-        run: |
-          if [ "${{ github.event_name }}" == "schedule" ]; then
-            echo "run-by=Automation" >> $GITHUB_OUTPUT
-          else
-            echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
-          fi
-
         # Send job failure to Slack
       - name: Send Slack When Failure
         run: |