Skip to content

Commit

Permalink
Optionally delay destroy in github actions to allow investigation time
Browse files Browse the repository at this point in the history
  • Loading branch information
linda.nasredin committed Apr 25, 2024
1 parent 01eaa72 commit c89d6d4
Show file tree
Hide file tree
Showing 6 changed files with 99 additions and 65 deletions.
33 changes: 19 additions & 14 deletions .github/workflows/dsf_poc_cli.yml
Original file line number Diff line number Diff line change
Expand Up @@ -224,13 +224,28 @@ jobs:
path: |
${{ env.EXAMPLE_DIR }}/ssh_keys
- name: Check how was the workflow run
id: check-trigger
if: ${{ failure() }}
run: |
if [ "${{ github.event_name }}" == "schedule" ]; then
echo "run-by=Automation" >> $GITHUB_OUTPUT
else
echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
fi
# This step allows time for investigation of the failed resources before destroying them
- name: Conditional Delay
if: ${{ failure() && github.event.inputs.delay_destory == true }}
if: ${{ failure() }}
run: |
destroy_delay_minutes = $DESTROY_DELAY_SECONDS / 60
curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*DSF Single Account ${{ inputs.workspace }} automation Failed*\nYou have $destroy_delay_minutes minutes to investigate the environment before it is destroyed.\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
sleep $DESTROY_DELAY_SECONDS
echo "delay_destroy: ${{ inputs.delay_destroy }}"
if [ "${{ inputs.delay_destroy }}" == "true" ]; then
echo "Terraform workspace: $TF_WORKSPACE"
curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
echo ""
echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"
sleep $DESTROY_DELAY_SECONDS
fi
- name: Terraform Destroy
id: destroy
Expand All @@ -249,16 +264,6 @@ jobs:
env:
TF_WORKSPACE: default

- name: Check how was the workflow run
id: check-trigger
if: ${{ failure() }}
run: |
if [ "${{ github.event_name }}" == "schedule" ]; then
echo "run-by=Automation" >> $GITHUB_OUTPUT
else
echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
fi
# Send job failure to Slack
- name: Send Slack When Failure
run: |
Expand Down
33 changes: 19 additions & 14 deletions .github/workflows/dsf_poc_cli_azure.yml
Original file line number Diff line number Diff line change
Expand Up @@ -251,13 +251,28 @@ jobs:
path: |
${{ env.EXAMPLE_DIR }}/ssh_keys
- name: Check how was the workflow run
id: check-trigger
if: ${{ failure() }}
run: |
if [ "${{ github.event_name }}" == "schedule" ]; then
echo "run-by=Automation" >> $GITHUB_OUTPUT
else
echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
fi
# This step allows time for investigation of the failed resources before destroying them
- name: Conditional Delay
if: ${{ failure() && github.event.inputs.delay_destory == true }}
if: ${{ failure() }}
run: |
destroy_delay_minutes = $DESTROY_DELAY_SECONDS / 60
curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*DSF Single Account ${{ inputs.workspace }} automation Failed*\nYou have $destroy_delay_minutes minutes to investigate the environment before it is destroyed.\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
sleep $DESTROY_DELAY_SECONDS
echo "delay_destroy: ${{ inputs.delay_destroy }}"
if [ "${{ inputs.delay_destroy }}" == "true" ]; then
echo "Terraform workspace: $TF_WORKSPACE"
curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
echo ""
echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"
sleep $DESTROY_DELAY_SECONDS
fi
- name: Terraform Destroy
id: destroy
Expand All @@ -276,16 +291,6 @@ jobs:
env:
TF_WORKSPACE: default

- name: Check how was the workflow run
id: check-trigger
if: ${{ failure() }}
run: |
if [ "${{ github.event_name }}" == "schedule" ]; then
echo "run-by=Automation" >> $GITHUB_OUTPUT
else
echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
fi
# Send job failure to Slack
- name: Send Slack When Failure
run: |
Expand Down
3 changes: 1 addition & 2 deletions .github/workflows/dsf_single_account_cli.yml
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,6 @@ jobs:
find ./examples/ -type f -exec sed -i -f sed.expr {} \;
- name: Set Workspace
if: always()
run: |
if [ ${{ inputs.branch }} == 'master' ] || [ ${{ inputs.branch }} == 'dev' ]; then
echo "TF_WORKSPACE=dsf_single_account_${{ inputs.branch }}" >> $GITHUB_ENV
Expand Down Expand Up @@ -243,7 +242,7 @@ jobs:
run: |
echo "delay_destroy: ${{ inputs.delay_destroy }}"
if [ "${{ inputs.delay_destroy }}" == "true" ]; then
echo "Workspace is: $TF_WORKSPACE"
echo "Terraform workspace: $TF_WORKSPACE"
curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
echo ""
echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"
Expand Down
37 changes: 21 additions & 16 deletions .github/workflows/sonar_multi_account_cli.yml
Original file line number Diff line number Diff line change
Expand Up @@ -160,13 +160,28 @@ jobs:
path: |
${{ env.EXAMPLE_DIR }}/ssh_keys
- name: Check how was the workflow run
id: check-trigger
if: ${{ failure() }}
run: |
if [ "${{ github.event_name }}" == "schedule" ]; then
echo "run-by=Automation" >> $GITHUB_OUTPUT
else
echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
fi
# This step allows time for investigation of the failed resources before destroying them
- name: Conditional Delay
if: ${{ failure() && github.event.inputs.delay_destory == true }}
if: ${{ failure() }}
run: |
destroy_delay_minutes = $DESTROY_DELAY_SECONDS / 60
curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*DSF Single Account ${{ inputs.workspace }} automation Failed*\nYou have $destroy_delay_minutes minutes to investigate the environment before it is destroyed.\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
sleep $DESTROY_DELAY_SECONDS
echo "delay_destroy: ${{ inputs.delay_destroy }}"
if [ "${{ inputs.delay_destroy }}" == "true" ]; then
echo "Terraform workspace: $TF_WORKSPACE"
curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
echo ""
echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"
sleep $DESTROY_DELAY_SECONDS
fi
- name: Terraform Destroy
if: always()
Expand All @@ -180,22 +195,12 @@ jobs:
if: always()
run: aws ec2 revoke-security-group-ingress --group-id ${{ vars.JUMP_SERVER_SG_ID }} --security-group-rule-ids ${{ env.sg_id }}

- name: Check how was the workflow run
id: check-trigger
if: ${{ failure() }}
run: |
if [ "${{ github.event_name }}" == "schedule" ]; then
echo "run-by=Automation" >> $GITHUB_OUTPUT
else
echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
fi
# Send job failure to Slack
- name: Send Slack When Failure
run: |
if [ ${{ inputs.branch }} == 'master' ]; then
curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*Multi Account Prod ${{ inputs.workspace }} automation Failed*\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#dsfkit-prod"}' ${{ secrets.SLACK_WEBHOOK_URL }}
curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*Sonar Multi Account Prod ${{ inputs.workspace }} automation Failed*\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#dsfkit-prod"}' ${{ secrets.SLACK_WEBHOOK_URL }}
else
curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*Multi Account Dev nightly ${{ inputs.workspace }} automation Failed*\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*Sonar Multi Account Dev nightly ${{ inputs.workspace }} automation Failed*\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
fi
if: ${{ failure() }}
23 changes: 19 additions & 4 deletions .github/workflows/sonar_poc_cli.yml
Original file line number Diff line number Diff line change
Expand Up @@ -190,13 +190,28 @@ jobs:
port: 22
script: ip addr | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1'

- name: Check how was the workflow run
id: check-trigger
if: ${{ failure() }}
run: |
if [ "${{ github.event_name }}" == "schedule" ]; then
echo "run-by=Automation" >> $GITHUB_OUTPUT
else
echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
fi
# This step allows time for investigation of the failed resources before destroying them
- name: Conditional Delay
if: ${{ failure() && github.event.inputs.delay_destory == true }}
if: ${{ failure() }}
run: |
destroy_delay_minutes = $DESTROY_DELAY_SECONDS / 60
curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*DSF Single Account ${{ inputs.workspace }} automation Failed*\nYou have $destroy_delay_minutes minutes to investigate the environment before it is destroyed.\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
sleep $DESTROY_DELAY_SECONDS
echo "delay_destroy: ${{ inputs.delay_destroy }}"
if [ "${{ inputs.delay_destroy }}" == "true" ]; then
echo "Terraform workspace: $TF_WORKSPACE"
curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
echo ""
echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"
sleep $DESTROY_DELAY_SECONDS
fi
- name: Terraform Destroy
id: destroy
Expand Down
35 changes: 20 additions & 15 deletions .github/workflows/sonar_single_account_cli.yml
Original file line number Diff line number Diff line change
Expand Up @@ -233,13 +233,28 @@ jobs:
envs: REMOTE_EXAMPLE_DIR,TF_WORKSPACE
script: terraform -chdir=$REMOTE_EXAMPLE_DIR output -json

- name: Check how was the workflow run
if: ${{ failure() }}
id: check-trigger
run: |
if [ "${{ github.event_name }}" == "schedule" ]; then
echo "run-by=Automation" >> $GITHUB_OUTPUT
else
echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
fi
# This step allows time for investigation of the failed resources before destroying them
- name: Conditional Delay
if: ${{ failure() && github.event.inputs.delay_destory == true }}
run: |
destroy_delay_minutes = $DESTROY_DELAY_SECONDS / 60
curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*DSF Single Account ${{ inputs.workspace }} automation Failed*\nYou have $destroy_delay_minutes minutes to investigate the environment before it is destroyed.\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
sleep $DESTROY_DELAY_SECONDS
if: ${{ failure() }}
run: |
echo "delay_destroy: ${{ inputs.delay_destroy }}"
if [ "${{ inputs.delay_destroy }}" == "true" ]; then
echo "Terraform workspace: $TF_WORKSPACE"
curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Please check the job!>\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }}
echo ""
echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment"
sleep $DESTROY_DELAY_SECONDS
fi
- name: Terraform Destroy
# if: always()
Expand All @@ -258,16 +273,6 @@ jobs:
if: always()
run: aws ec2 revoke-security-group-ingress --group-id ${{ vars.JUMP_SERVER_SG_ID }} --security-group-rule-ids ${{ env.sg_id }}

- name: Check how was the workflow run
if: ${{ failure() }}
id: check-trigger
run: |
if [ "${{ github.event_name }}" == "schedule" ]; then
echo "run-by=Automation" >> $GITHUB_OUTPUT
else
echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT
fi
# Send job failure to Slack
- name: Send Slack When Failure
run: |
Expand Down

0 comments on commit c89d6d4

Please sign in to comment.