fix: add retry step to terragrunt apply

Signed-off-by: samuelarogbonlo <[email protected]>

.github/workflows/deploy-daily-snapshot.yml

@@ -44,6 +44,8 @@ jobs:
           r2_access_key: ${{ secrets.R2_ACCESS_KEY }}
           r2_secret_key: ${{ secrets.R2_SECRET_KEY }}
           slack_token: ${{ secrets.SLACK_TOKEN }}
+          max_retries: '5'
+          retry_delay_seconds: '60s'
           working_directory: tf-managed/live/environments/prod/applications/${{ matrix.replica }}
           service_name: ${{ matrix.replica }}
           new_relic_account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}

.github/workflows/deploy-forest-butterflynet.yml

@@ -33,6 +33,8 @@ jobs:
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           working_directory: tf-managed/live/environments/prod/applications/forest-butterflynet
           service_name: forest-butterflynet
+          max_retries: '5'
+          retry_delay_seconds: '60s'
           new_relic_account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
           new_relic_api_key: ${{ secrets.NEW_RELIC_API_KEY }}
           ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}

.github/workflows/deploy-forest-calibnet.yml

@@ -33,6 +33,8 @@ jobs:
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           working_directory: tf-managed/live/environments/prod/applications/forest-calibnet
           service_name: forest-calibnet
+          max_retries: '5'
+          retry_delay_seconds: '60s'
           new_relic_account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
           new_relic_api_key: ${{ secrets.NEW_RELIC_API_KEY }}
           ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}

.github/workflows/deploy-forest-mainnet.yml

@@ -33,6 +33,8 @@ jobs:
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           working_directory: tf-managed/live/environments/prod/applications/forest-mainnet
           service_name: forest-mainnet
+          max_retries: '5'
+          retry_delay_seconds: '60s'
           new_relic_account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
           new_relic_api_key: ${{ secrets.NEW_RELIC_API_KEY }}
           ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}

.github/workflows/deploy-snapshot-monitoring.yml

@@ -45,6 +45,8 @@ jobs:
           slack_token: ${{ secrets.SLACK_TOKEN }}
           working_directory: tf-managed/live/environments/prod/applications/snapshot-monitoring
           service_name: Snapshot Service
+          max_retries: '5'
+          retry_delay_seconds: '60s'
           new_relic_account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
           new_relic_api_key: ${{ secrets.NEW_RELIC_API_KEY }}
           ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}

.github/workflows/deploy-sync-check.yml

@@ -41,6 +41,8 @@ jobs:
           slack_token: ${{ secrets.SLACK_TOKEN }}
           working_directory: tf-managed/live/environments/prod/applications/sync-check
           service_name: Sync Check Service
+          max_retries: '5'
+          retry_delay_seconds: '60s'
           new_relic_account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
           new_relic_api_key: ${{ secrets.NEW_RELIC_API_KEY }}
           ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}

composite-action/terragrunt/action.yml

@@ -33,6 +33,12 @@ inputs:
     description: 'CloudFlare R2 access key id'
   r2_secret_key:
     description: 'CloudFlare R2 private access key'
+  max_retries:
+    description: 'The maximum number of retry attempts for the Terragrunt apply command if it fails initially.'
+    required: true
+  retry_delay_seconds:
+    description: 'The number of seconds to wait between each retry attempt.'
+    required: true
 
 runs:
   using: "composite"
@@ -155,7 +161,22 @@ runs:
         else
           echo "Changes detected. Redeploying everything..."
           terragrunt destroy -auto-approve --terragrunt-non-interactive
-          terragrunt apply -auto-approve --terragrunt-non-interactive
+          max_retries=${{ inputs.max_retries }}
+          delay_seconds=${{ inputs.retry_delay_seconds }}
+          n=0
+          while [ $n -lt $max_retries ]; do
+            echo "Attempt $(($n + 1)) of $max_retries"
+            terragrunt apply -auto-approve --terragrunt-non-interactive && break
+            n=$((n+1))
+            if [ $n -lt $max_retries ]; then
+              echo "Retry $n/$max_retries of terragrunt apply failed. Retrying in $delay_seconds seconds..."
+              sleep $delay_seconds
+            fi
+          done
+          if [ $n -eq $max_retries ]; then
+            echo "Failed to apply changes after $max_retries attempts."
+            exit 1
+          fi
         fi
       working-directory: ${{ inputs.working_directory }}
       env:
@@ -170,6 +191,7 @@ runs:
         TF_VAR_new_relic_api_key: ${{ inputs.new_relic_api_key }}
         TF_VAR_new_relic_account_id: ${{ inputs.new_relic_account_id }}
 
+
     - name: Terragrunt Force Apply
       if: github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch'
       working-directory: ${{ inputs.working_directory }}

tf-managed/modules/daily-snapshot/main.tf

@@ -78,7 +78,6 @@ resource "digitalocean_droplet" "forest" {
     host    = self.ipv4_address
     user    = "root"
     type    = "ssh"
-    timeout = "30m"
   }
 
   # Push the sources.tar file to the newly booted droplet

tf-managed/modules/forest-droplet/main.tf

@@ -41,7 +41,6 @@ resource "digitalocean_droplet" "forest" {
     host    = self.ipv4_address
     user    = "root"
     type    = "ssh"
-    timeout = "30m"
   }
 
   provisioner "file" {

tf-managed/modules/sync-check/main.tf

@@ -77,7 +77,6 @@ resource "digitalocean_droplet" "forest" {
     host    = self.ipv4_address
     user    = "root"
     type    = "ssh"
-    timeout = "30m"
   }
 
   # Push the sources.tar file to the newly booted droplet