diff --git a/.github/workflows/dsf_poc_cli.yml b/.github/workflows/dsf_poc_cli.yml index eea682dd1..0f4f1bc95 100644 --- a/.github/workflows/dsf_poc_cli.yml +++ b/.github/workflows/dsf_poc_cli.yml @@ -9,6 +9,11 @@ on: explicit_ref: required: true type: string + delay_destroy: + description: 'Delay the destroy step and subsequent steps to allow investigation' + type: boolean + default: false + required: true secrets: AWS_ACCESS_KEY_ID: required: true @@ -55,6 +60,7 @@ env: TF_INPUT: 0 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DESTROY_DELAY_SECONDS: 1800 permissions: contents: read @@ -211,13 +217,37 @@ jobs: run: terraform -chdir=$EXAMPLE_DIR output -json - name: Collect Artifacts + id: collect-artifacts if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: - name: collected-keys + name: collected-keys-${{ env.TF_WORKSPACE }} path: | ${{ env.EXAMPLE_DIR }}/ssh_keys + - name: Check how was the workflow run + id: check-trigger + if: ${{ failure() }} + run: | + if [ "${{ github.event_name }}" == "schedule" ]; then + echo "run-by=Automation" >> $GITHUB_OUTPUT + else + echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT + fi + + # This step allows time for investigation of the failed resources before destroying them + - name: Conditional Delay + if: ${{ failure() }} + run: | + echo "delay_destroy: ${{ inputs.delay_destroy }}" + if [ "${{ inputs.delay_destroy }}" == "true" ]; then + echo "Terraform workspace: $TF_WORKSPACE" + curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }} + echo "" + echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment" + sleep $DESTROY_DELAY_SECONDS + fi + - name: Terraform Destroy id: destroy if: always() @@ -235,16 +265,6 @@ jobs: env: TF_WORKSPACE: default - - name: Check how was the workflow run - id: check-trigger - if: ${{ failure() }} - run: | - if [ "${{ github.event_name }}" == "schedule" ]; then - echo "run-by=Automation" >> $GITHUB_OUTPUT - else - echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT - fi - # Send job failure to Slack - name: Send Slack When Failure run: | diff --git a/.github/workflows/dsf_poc_cli_azure.yml b/.github/workflows/dsf_poc_cli_azure.yml index 466c9a890..9e9c4846f 100644 --- a/.github/workflows/dsf_poc_cli_azure.yml +++ b/.github/workflows/dsf_poc_cli_azure.yml @@ -9,6 +9,12 @@ on: explicit_ref: required: true type: string + delay_destroy: + description: 'Delay the destroy step and subsequent steps to allow investigation' + type: boolean + default: false + required: true + secrets: AWS_ACCESS_KEY_ID: required: true @@ -55,6 +61,7 @@ env: ARM_CLIENT_ID: ${{ vars.ARM_CLIENT_ID }} ARM_TENANT_ID: ${{ vars.ARM_TENANT_ID }} ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }} + DESTROY_DELAY_SECONDS: 1800 permissions: contents: read @@ -237,13 +244,37 @@ jobs: run: terraform -chdir=$EXAMPLE_DIR output -json - name: Collect Artifacts + id: collect-artifacts if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: - name: collected-keys + name: collected-keys-${{ env.TF_WORKSPACE }} path: | ${{ env.EXAMPLE_DIR }}/ssh_keys + - name: Check how was the workflow run + id: check-trigger + if: ${{ failure() }} + run: | + if [ "${{ github.event_name }}" == "schedule" ]; then + echo "run-by=Automation" >> $GITHUB_OUTPUT + else + echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT + fi + + # This step allows time for investigation of the failed resources before destroying them + - name: Conditional Delay + if: ${{ failure() }} + run: | + echo "delay_destroy: ${{ inputs.delay_destroy }}" + if [ "${{ inputs.delay_destroy }}" == "true" ]; then + echo "Terraform workspace: $TF_WORKSPACE" + curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }} + echo "" + echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment" + sleep $DESTROY_DELAY_SECONDS + fi + - name: Terraform Destroy id: destroy if: always() @@ -261,16 +292,6 @@ jobs: env: TF_WORKSPACE: default - - name: Check how was the workflow run - id: check-trigger - if: ${{ failure() }} - run: | - if [ "${{ github.event_name }}" == "schedule" ]; then - echo "run-by=Automation" >> $GITHUB_OUTPUT - else - echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT - fi - # Send job failure to Slack - name: Send Slack When Failure run: | diff --git a/.github/workflows/dsf_single_account_cli.yml b/.github/workflows/dsf_single_account_cli.yml index 56caece23..546a21ae0 100644 --- a/.github/workflows/dsf_single_account_cli.yml +++ b/.github/workflows/dsf_single_account_cli.yml @@ -9,6 +9,11 @@ on: branch: required: true type: string + delay_destroy: + description: 'Delay the destroy step and subsequent steps to allow investigation' + type: boolean + default: false + required: true workflow_call: inputs: @@ -73,7 +78,8 @@ env: AWS_ACCESS_KEY_ID_STAGE: ${{ secrets.AWS_ACCESS_KEY_ID_STAGE }} AWS_SECRET_ACCESS_KEY_STAGE: ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGE }} JUMP_SERVER_KEY: ${{ secrets.JUMP_SERVER_KEY }} - AWS_KEY_PATH: "aws_key.cer" + AWS_KEY_PATH: "jump_server_key.cer" + DESTROY_DELAY_SECONDS: 1800 permissions: contents: read @@ -213,13 +219,37 @@ jobs: run: terraform -chdir=$EXAMPLE_DIR output -json - name: Collect Artifacts + id: collect-artifacts if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: - name: collected-keys + name: collected-keys-${{ env.TF_WORKSPACE }} path: | ${{ env.EXAMPLE_DIR }}/ssh_keys + - name: Check how was the workflow run + id: check-trigger + if: ${{ failure() }} + run: | + if [ "${{ github.event_name }}" == "schedule" ]; then + echo "run-by=Automation" >> $GITHUB_OUTPUT + else + echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT + fi + + # This step allows time for investigation of the failed resources before destroying them + - name: Conditional Delay + if: ${{ failure() }} + run: | + echo "delay_destroy: ${{ inputs.delay_destroy }}" + if [ "${{ inputs.delay_destroy }}" == "true" ]; then + echo "Terraform workspace: $TF_WORKSPACE" + curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }} + echo "" + echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment" + sleep $DESTROY_DELAY_SECONDS + fi + - name: Terraform Destroy if: always() run: terraform -chdir=$EXAMPLE_DIR destroy -auto-approve @@ -243,16 +273,6 @@ jobs: run: | aws ec2 revoke-security-group-ingress --group-id ${{ vars.SYDNEY_SG_ID }} --security-group-rule-ids ${{ env.sg_id_for_sydney }} - - name: Check how was the workflow run - id: check-trigger - if: ${{ failure() }} - run: | - if [ "${{ github.event_name }}" == "schedule" ]; then - echo "run-by=Automation" >> $GITHUB_OUTPUT - else - echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT - fi - # Send job failure to Slack - name: Send Slack When Failure run: | diff --git a/.github/workflows/sonar_multi_account_cli.yml b/.github/workflows/sonar_multi_account_cli.yml index 40d229753..30f490432 100644 --- a/.github/workflows/sonar_multi_account_cli.yml +++ b/.github/workflows/sonar_multi_account_cli.yml @@ -9,6 +9,11 @@ on: branch: required: true type: string + delay_destroy: + description: 'Delay the destroy step and subsequent steps to allow investigation' + type: boolean + default: false + required: true workflow_call: inputs: @@ -39,7 +44,8 @@ env: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} TF_WORKSPACE: "multi_account" JUMP_SERVER_KEY: ${{ secrets.JUMP_SERVER_KEY }} - AWS_KEY_PATH: "aws_key.cer" + AWS_KEY_PATH: "jump_server_key.cer" + DESTROY_DELAY_SECONDS: 1800 permissions: contents: read @@ -154,6 +160,29 @@ jobs: path: | ${{ env.EXAMPLE_DIR }}/ssh_keys + - name: Check how was the workflow run + id: check-trigger + if: ${{ failure() }} + run: | + if [ "${{ github.event_name }}" == "schedule" ]; then + echo "run-by=Automation" >> $GITHUB_OUTPUT + else + echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT + fi + + # This step allows time for investigation of the failed resources before destroying them + - name: Conditional Delay + if: ${{ failure() }} + run: | + echo "delay_destroy: ${{ inputs.delay_destroy }}" + if [ "${{ inputs.delay_destroy }}" == "true" ]; then + echo "Terraform workspace: $TF_WORKSPACE" + curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }} + echo "" + echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment" + sleep $DESTROY_DELAY_SECONDS + fi + - name: Terraform Destroy if: always() run: terraform -chdir=$EXAMPLE_DIR destroy -auto-approve @@ -166,22 +195,12 @@ jobs: if: always() run: aws ec2 revoke-security-group-ingress --group-id ${{ vars.JUMP_SERVER_SG_ID }} --security-group-rule-ids ${{ env.sg_id }} - - name: Check how was the workflow run - id: check-trigger - if: ${{ failure() }} - run: | - if [ "${{ github.event_name }}" == "schedule" ]; then - echo "run-by=Automation" >> $GITHUB_OUTPUT - else - echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT - fi - # Send job failure to Slack - name: Send Slack When Failure run: | if [ ${{ inputs.branch }} == 'master' ]; then - curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*Multi Account Prod ${{ inputs.workspace }} automation Failed*\n\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#dsfkit-prod"}' ${{ secrets.SLACK_WEBHOOK_URL }} + curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*Sonar Multi Account Prod ${{ inputs.workspace }} automation Failed*\n\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#dsfkit-prod"}' ${{ secrets.SLACK_WEBHOOK_URL }} else - curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*Multi Account Dev nightly ${{ inputs.workspace }} automation Failed*\n\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }} + curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*Sonar Multi Account Dev nightly ${{ inputs.workspace }} automation Failed*\n\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }} fi if: ${{ failure() }} diff --git a/.github/workflows/sonar_poc_cli.yml b/.github/workflows/sonar_poc_cli.yml index b88c8a729..b1c4d3768 100644 --- a/.github/workflows/sonar_poc_cli.yml +++ b/.github/workflows/sonar_poc_cli.yml @@ -9,6 +9,12 @@ on: explicit_ref: required: true type: string + delay_destroy: + description: 'Delay the destroy step and subsequent steps to allow investigation' + type: boolean + default: false + required: true + secrets: AWS_ACCESS_KEY_ID: required: true @@ -49,6 +55,7 @@ env: TF_VAR_allowed_ssh_cidrs: "[\"82.0.0.0/8\", \"62.0.0.0/8\", \"94.0.0.0/8\"]" AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DESTROY_DELAY_SECONDS: 1800 permissions: contents: read @@ -59,12 +66,14 @@ jobs: fail-fast: false matrix: include: - - workspace: basic_cli-${{ github.run_number }}-${{ github.run_attempt }} + - name: Sonar Basic CLI + workspace: sonar_basic_cli-${{ github.run_number }}-${{ github.run_attempt }} example_dir: ./examples/aws/poc/sonar_basic_deployment aws_region: eu-west-2 hub_output: '.dsf_hub.value.public_dns' gw_output: '.dsf_agentless_gw.value."agentless-gw-0".private_ip' - - workspace: hadr_cli-${{ github.run_number }}-${{ github.run_attempt }} + - name: Sonar HADR CLI + workspace: sonar_hadr_cli-${{ github.run_number }}-${{ github.run_attempt }} example_dir: ./examples/aws/poc/sonar_hadr_deployment aws_region: eu-west-1 hub_output: '.dsf_hubs.value.main.public_dns' @@ -156,10 +165,11 @@ jobs: run: terraform -chdir=$EXAMPLE_DIR output -json - name: Collect Artifacts + id: collect-artifacts if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: - name: collected-keys + name: collected-keys-${{ env.TF_WORKSPACE }} path: | ${{ env.EXAMPLE_DIR }}/ssh_keys @@ -183,6 +193,29 @@ jobs: port: 22 script: ip addr | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1' + - name: Check how was the workflow run + id: check-trigger + if: ${{ failure() }} + run: | + if [ "${{ github.event_name }}" == "schedule" ]; then + echo "run-by=Automation" >> $GITHUB_OUTPUT + else + echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT + fi + + # This step allows time for investigation of the failed resources before destroying them + - name: Conditional Delay + if: ${{ failure() }} + run: | + echo "delay_destroy: ${{ inputs.delay_destroy }}" + if [ "${{ inputs.delay_destroy }}" == "true" ]; then + echo "Terraform workspace: $TF_WORKSPACE" + curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }} + echo "" + echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment" + sleep $DESTROY_DELAY_SECONDS + fi + - name: Terraform Destroy id: destroy if: always() @@ -206,4 +239,4 @@ jobs: fields: author,action,eventName,ref,workflow,pullRequest env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - if: ${{ failure() }} + if: ${{ failure() }} diff --git a/.github/workflows/sonar_single_account_cli.yml b/.github/workflows/sonar_single_account_cli.yml index 5a2527514..5f40b90ff 100644 --- a/.github/workflows/sonar_single_account_cli.yml +++ b/.github/workflows/sonar_single_account_cli.yml @@ -9,6 +9,11 @@ on: branch: required: true type: string + delay_destroy: + description: 'Delay the destroy step and subsequent steps to allow investigation' + type: boolean + default: false + required: true workflow_call: inputs: @@ -33,7 +38,8 @@ env: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGE }} JUMP_SERVER_KEY: ${{ secrets.JUMP_SERVER_KEY }} AWS_REGION: ap-southeast-1 - TF_WORKSPACE: single_account + TF_WORKSPACE: sonar_single_account_cli + DESTROY_DELAY_SECONDS: 1800 permissions: contents: read @@ -227,6 +233,38 @@ jobs: envs: REMOTE_EXAMPLE_DIR,TF_WORKSPACE script: terraform -chdir=$REMOTE_EXAMPLE_DIR output -json + - name: Collect Artifacts + id: collect-artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: collected-keys-${{ env.TF_WORKSPACE }} + path: | + ${{ env.EXAMPLE_DIR }}/ssh_keys + + - name: Check how was the workflow run + if: ${{ failure() }} + id: check-trigger + run: | + if [ "${{ github.event_name }}" == "schedule" ]; then + echo "run-by=Automation" >> $GITHUB_OUTPUT + else + echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT + fi + + # This step allows time for investigation of the failed resources before destroying them + - name: Conditional Delay + if: ${{ failure() }} + run: | + echo "delay_destroy: ${{ inputs.delay_destroy }}" + if [ "${{ inputs.delay_destroy }}" == "true" ]; then + echo "Terraform workspace: $TF_WORKSPACE" + curl -X POST -H 'Content-type: application/json' --data '{"text":":exclamation: :exclamation: :exclamation:\n*${{ github.workflow }} ${{ env.TF_WORKSPACE }} automation Failed*\n You have ${{ env.DESTROY_DELAY_SECONDS }} seconds to investigate the environment before it is destroyed :alarm_clock:\n\nRun by: ${{ steps.check-trigger.outputs.run-by }}", "channel": "#edsf_automation"}' ${{ secrets.SLACK_WEBHOOK_URL }} + echo "" + echo "Sleeping for $((DESTROY_DELAY_SECONDS / 60)) minutes before destroying the environment" + sleep $DESTROY_DELAY_SECONDS + fi + - name: Terraform Destroy # if: always() uses: appleboy/ssh-action@v1.0.0 @@ -244,16 +282,6 @@ jobs: if: always() run: aws ec2 revoke-security-group-ingress --group-id ${{ vars.JUMP_SERVER_SG_ID }} --security-group-rule-ids ${{ env.sg_id }} - - name: Check how was the workflow run - if: ${{ failure() }} - id: check-trigger - run: | - if [ "${{ github.event_name }}" == "schedule" ]; then - echo "run-by=Automation" >> $GITHUB_OUTPUT - else - echo "run-by=${{ github.actor }}" >> $GITHUB_OUTPUT - fi - # Send job failure to Slack - name: Send Slack When Failure run: | diff --git a/.github/workflows/sonar_upgrade.yml b/.github/workflows/sonar_upgrade.yml index 01c360c46..8e43ad6ee 100644 --- a/.github/workflows/sonar_upgrade.yml +++ b/.github/workflows/sonar_upgrade.yml @@ -200,11 +200,12 @@ jobs: echo agentless_gw_0_ip=$(terraform -chdir=$DEPLOYMENT_EXAMPLE_DIR output -json | jq -r '.sonar.value.agentless_gw_main[0].private_ip') >> $GITHUB_ENV echo dsf_ssh_key_file_name=../../../$DEPLOYMENT_EXAMPLE_DIR/$(terraform -chdir=$DEPLOYMENT_EXAMPLE_DIR output -json | jq -r '.dsf_private_ssh_key_file_path.value') >> $GITHUB_ENV - - name: Collect Artifacts for deployment + - name: Collect Artifacts of deployment + id: collect-artifacts if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: - name: collected-keys + name: collected-keys-${{ env.TF_WORKSPACE }} path: | ${{ env.DEPLOYMENT_EXAMPLE_DIR }}/ssh_keys