From c4f6fab2ed5e4917dd76f5b2ad3014f8e3176191 Mon Sep 17 00:00:00 2001
From: Noach Magedman <noach@seekingalpha.com>
Date: Mon, 8 Apr 2024 11:52:11 +0300
Subject: [PATCH 01/13] Escape *_ENVSUBST variable expansions

The `envsubst` template expander uses placeholder variables
that *look* like Bash variables, e.g. `$FOO_ENVSUBST`.

However, `envsubst` is not Bash, and it is not Bash-aware.
It does not do any intelligent escaping/quoting of the values.
It is on us to ensure that the results of the template expansion
are valid Bash code.  If the value contains spaces, dollar signs,
or quotes, those will be inserted as-is into the script.

Here, we single-quote the template variable so that the expanded value
appears within single-quotes.  This will protect us from spaces breaking
our script.  It will also protect against dollar signs and double-quotes.
Single-quotes will still break things, but will at least be obvious
about it (and they are unlikely).
---
 pre_install.sh.tpl    | 10 +++++-----
 rotate_version.sh.tpl |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/pre_install.sh.tpl b/pre_install.sh.tpl
index 0cd95c86149b..99e5b80c4246 100644
--- a/pre_install.sh.tpl
+++ b/pre_install.sh.tpl
@@ -1,9 +1,9 @@
 #!/bin/bash
-AWS="/usr/local/bin/aws"
-AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION_ENVSUBST
-RC_DIR=$RC_DIR_ENVSUBST
-RC_FILE=$RC_TARBALL_ENVSUBST
-S3_BUCKET=$S3_BUCKET_ENVSUBST
+AWS=/usr/local/bin/aws
+AWS_DEFAULT_REGION='$AWS_DEFAULT_REGION_ENVSUBST'
+RC_DIR='$RC_DIR_ENVSUBST'
+RC_FILE='$RC_TARBALL_ENVSUBST'
+S3_BUCKET='$S3_BUCKET_ENVSUBST'
 
 set -e
 
diff --git a/rotate_version.sh.tpl b/rotate_version.sh.tpl
index 92dd66aec3c2..7c30894e2921 100755
--- a/rotate_version.sh.tpl
+++ b/rotate_version.sh.tpl
@@ -1,5 +1,5 @@
 #!/bin/bash
-RC_DIR=$RC_DIR_ENVSUBST
+RC_DIR='$RC_DIR_ENVSUBST'
 
 set -e
 

From bae64b1ac1a3cf71ee5f92350e1690e9362d1ffa Mon Sep 17 00:00:00 2001
From: Noach Magedman <noach@seekingalpha.com>
Date: Mon, 8 Apr 2024 12:54:37 +0300
Subject: [PATCH 02/13] Minor cleanup of github.sh

- The GitHub workflow passes us the $environment directly.
  No need to parse it out.
  cf. Jenkins which defined $JOB_BASE_NAME which we had to parse.
- Document our environment-variable inputs
- Hoist function definitions to top of script
- whitespace
- ALAETM (Alphabetized Lists Are Easier To Maintain!)
---
 .github/workflows/deploy.yml |  2 +-
 github.sh                    | 34 ++++++++++++++--------------------
 2 files changed, 15 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 06394d44a909..7748a67bb7e0 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -69,5 +69,5 @@ jobs:
         run: "$GITHUB_WORKSPACE/github.sh"
         shell: bash
         env:
-          ENVIRONMENT_NAME: ${{ inputs.environment }}
+          environment: ${{ inputs.environment }}
           version: ${{ inputs.version }}
diff --git a/github.sh b/github.sh
index fc462eed9775..f77d536e6d57 100755
--- a/github.sh
+++ b/github.sh
@@ -1,32 +1,26 @@
 #!/bin/bash
+## This script is invoked by the GitHub workflow ".github/workflows/deploy.yml".
+## It expects the following environment variables to be defined:
+## - $environment : Either `staging` or `production`
+## - $version     : Either the full tarball filename or just its {rc_version}.{commit_hash} substring
 
 set -o errexit
 
-case $ENVIRONMENT_NAME in
-  *staging*)    environment=staging ;;
-  *production*) environment=production ;;
-  *)            echo "ERROR: Can’t infer environment from job name!"; exit 99 ;;
-esac
-
-echo $environment
+function hr() {
+  echo "==========================================================================="
+}
 
-rc_dir="/opt/rocket-chat"
-s3_bucket="seekingalpha-rocketchat-builds"
+rc_dir=/opt/rocket-chat
+s3_bucket=seekingalpha-rocketchat-builds
 
-## Note: $version is a Jenkins job parameter.
-## We accept either the full tarball filename or just its version substring.
-if [[ "$version" == rocket.chat-*.tgz ]]
-then
+# The RC installation tarball is named "rocket.chat-{rc_version}.{commit_hash}.tgz".
+# The $version parameter may be either the full tarball filename or just its {rc_version}.{commit_hash} substring.
+if [[ "$version" == rocket.chat-*.tgz ]] ; then
   rc_tarball="$version"
 else
   rc_tarball="rocket.chat-$version.tgz"
 fi
 
-function hr() {
-  echo "==========================================================================="
-}
-
-
 ## Strip off the trailing letter from the region: Use us-west-2, not us-west-2a
 export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | awk '{print substr($0,1,length($0)-1)}')
 
@@ -34,8 +28,8 @@ export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | awk '{print substr
 export AWS_DEFAULT_REGION_ENVSUBST=$AWS_DEFAULT_REGION
 export ENV_ENVSUBST=$environment
 export RC_DIR_ENVSUBST=$rc_dir
-export S3_BUCKET_ENVSUBST=$s3_bucket
 export RC_TARBALL_ENVSUBST=$rc_tarball
+export S3_BUCKET_ENVSUBST=$s3_bucket
 
 #append all "_ENVSUBST" env vars keys to a online commas separated.
 b=""; for i in $(printenv | grep "_ENVSUBST" | sed 's;=.*;;'); do echo "$i"; b="$b\$$i,"; done; b=${b::-1};
@@ -83,4 +77,4 @@ hr
 echo "Flushing $environment CDN"
 FASTLY_SERVICE=$(aws ssm get-parameter --name /rocketchat/fastly_service_id --with-decryption --query Parameter.Value --output text)
 FASTLY_TOKEN=$(aws ssm get-parameter --name /rocketchat/fastly_api_key --with-decryption --query Parameter.Value --output text)
-curl -X POST -H "Fastly-Key: $FASTLY_TOKEN" "https://api.fastly.com/service/$FASTLY_SERVICE/purge/$environment"
\ No newline at end of file
+curl -X POST -H "Fastly-Key: $FASTLY_TOKEN" "https://api.fastly.com/service/$FASTLY_SERVICE/purge/$environment"

From eff659718958d7b9548c0230df2c967c9a41a617 Mon Sep 17 00:00:00 2001
From: Noach Magedman <noach@seekingalpha.com>
Date: Mon, 8 Apr 2024 15:17:06 +0300
Subject: [PATCH 03/13] Re-organize github.sh

- Gather all needed data before beginning the deployment.
  e.g. Get our Fastly credentials from SSM before deploying
  the new build to our RC EC2 instances, lest we find belatedly
  that we lack permissions and are left in a broken state.
- Flush the Fastly CDN **ASAP** after deploying the new build,
  to minimize any downtime.  We can record the S3 marker file
  afterward.
---
 github.sh | 68 +++++++++++++++++++++++++++++++++----------------------
 1 file changed, 41 insertions(+), 27 deletions(-)

diff --git a/github.sh b/github.sh
index f77d536e6d57..da198c81d5b2 100755
--- a/github.sh
+++ b/github.sh
@@ -10,6 +10,11 @@ function hr() {
   echo "==========================================================================="
 }
 
+hr
+
+
+## Gather all needed data
+
 rc_dir=/opt/rocket-chat
 s3_bucket=seekingalpha-rocketchat-builds
 
@@ -21,10 +26,30 @@ else
   rc_tarball="rocket.chat-$version.tgz"
 fi
 
-## Strip off the trailing letter from the region: Use us-west-2, not us-west-2a
+# Strip off the trailing letter from the region: Use us-west-2, not us-west-2a
 export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | awk '{print substr($0,1,length($0)-1)}')
 
-## EXPORTED variables ending in _ENVSUBST are for expansion in the .tpl template files.
+# We will need to flush the Fastly cache post-deployment because it caches 404s returned by an _old_ server
+# which was asked for a versionized file referenced by a _new_ server.
+# (Now that we are HUPing servers in parallel rather than serially, the window for this race condition is much smaller,
+# however it might not be completely eliminated.)
+fastly_service=$(aws ssm get-parameter --name /rocketchat/fastly_service_id --with-decryption --query Parameter.Value --output text)
+fastly_token=$(aws ssm get-parameter --name /rocketchat/fastly_api_key --with-decryption --query Parameter.Value --output text)
+
+# Get RC EC2 instance IPs (one per line, as a multiline string, since `parallel-ssh --hosts` expects them in that format)
+rc_instance_ips=$(
+  aws ec2 describe-instances \
+      --filters Name=instance-state-name,Values=running \
+                Name=tag:aws:autoscaling:groupName,Values=rocketchat \
+      --query "Reservations[*].Instances[*].NetworkInterfaces[0].PrivateIpAddress" \
+      --output text
+)
+
+
+## Generate scripts to run on the RC EC2 instances, using the `envsubst` template renderer,
+## part of the GNU gettext package.
+
+# *Exported* variables ending in _ENVSUBST will be expanded in the *.tpl template files.
 export AWS_DEFAULT_REGION_ENVSUBST=$AWS_DEFAULT_REGION
 export ENV_ENVSUBST=$environment
 export RC_DIR_ENVSUBST=$rc_dir
@@ -34,47 +59,36 @@ export S3_BUCKET_ENVSUBST=$s3_bucket
 #append all "_ENVSUBST" env vars keys to a online commas separated.
 b=""; for i in $(printenv | grep "_ENVSUBST" | sed 's;=.*;;'); do echo "$i"; b="$b\$$i,"; done; b=${b::-1};
 envsubst_varlist="$b"
+envsubst "$envsubst_varlist" < pre_install.sh.tpl    > pre_install.sh
+envsubst "$envsubst_varlist" < rotate_version.sh.tpl > rotate_version.sh
 
-## Render Script Templates
-envsubst "$envsubst_varlist" < ./pre_install.sh.tpl  > ./pre_install.sh
-envsubst "$envsubst_varlist" < ./rotate_version.sh.tpl > ./rotate_version.sh
-
-## Get instance IPs one per line (multiline string)
-rc_instance_ips=$(
-  aws ec2 describe-instances \
-      --filters Name=instance-state-name,Values=running \
-                Name=tag:aws:autoscaling:groupName,Values=rocketchat \
-      --query "Reservations[*].Instances[*].NetworkInterfaces[0].PrivateIpAddress" \
-      --output text
-)
 
 ## Install RC tarball (and its dependencies) onto all RC nodes
-hr
-echo "Installing new build onto all RC nodes:"
+echo "Installing new build onto all RC nodes..."
 parallel-ssh \
   -x  "-o StrictHostKeyChecking=no" \
   --inline --timeout 600 --user deploy \
   --hosts <(echo "$rc_instance_ips") \
-  --send-input < ./pre_install.sh
+  --send-input < pre_install.sh            # TODO: Rename to install_tarball.sh
 hr
 
 ## Activate new version
-echo "Activating new build on all RC nodes:"
+echo "Activating new build on all RC nodes..."
 parallel-ssh \
   -x  "-o StrictHostKeyChecking=no" \
   --inline --timeout 600 --user deploy \
   --hosts <(echo "$rc_instance_ips") \
-  --send-input < ./rotate_version.sh
+  --send-input < rotate_version.sh         # TODO: Rename to activate_new_build.sh
+hr
+
+## Flush CDN
+echo "Flushing $environment CDN..."
+curl -X POST -H "Fastly-Key: $fastly_token" "https://api.fastly.com/service/$fastly_service/purge/$environment"
 hr
 
 ## Update the version marker file
-echo "Mark which RC build is now active..."
-current_marker_file="rocket.chat-$environment.tgz"
-aws s3 cp "s3://$s3_bucket/$rc_tarball" "s3://$s3_bucket/$current_marker_file" --acl public-read
+echo "Mark (in S3) which RC build is now active on $environment..."
+aws s3 cp "s3://$s3_bucket/$rc_tarball" "s3://$s3_bucket/rocket.chat-$environment.tgz" --acl public-read
 hr
 
-## Flush CDN
-echo "Flushing $environment CDN"
-FASTLY_SERVICE=$(aws ssm get-parameter --name /rocketchat/fastly_service_id --with-decryption --query Parameter.Value --output text)
-FASTLY_TOKEN=$(aws ssm get-parameter --name /rocketchat/fastly_api_key --with-decryption --query Parameter.Value --output text)
-curl -X POST -H "Fastly-Key: $FASTLY_TOKEN" "https://api.fastly.com/service/$FASTLY_SERVICE/purge/$environment"
+echo Done!

From 4fc20de4b715ee07c49731f5db724bd90944336c Mon Sep 17 00:00:00 2001
From: Noach Magedman <noach@seekingalpha.com>
Date: Mon, 8 Apr 2024 16:00:30 +0300
Subject: [PATCH 04/13] Rewrite the github.sh `envsubst_varlist` generation
 code

---
 github.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/github.sh b/github.sh
index da198c81d5b2..34d3bde537a3 100755
--- a/github.sh
+++ b/github.sh
@@ -56,9 +56,11 @@ export RC_DIR_ENVSUBST=$rc_dir
 export RC_TARBALL_ENVSUBST=$rc_tarball
 export S3_BUCKET_ENVSUBST=$s3_bucket
 
-#append all "_ENVSUBST" env vars keys to a online commas separated.
-b=""; for i in $(printenv | grep "_ENVSUBST" | sed 's;=.*;;'); do echo "$i"; b="$b\$$i,"; done; b=${b::-1};
-envsubst_varlist="$b"
+# envsubst only expands template variables it is told about via
+# an argument with format '$foo,$bar,$baz'
+for dollar__varname__non_final_comma in $(printenv | grep '^\w*_ENVSUBST=' | sed 's/=.*//; s/^/$/; $!s/$/,/') ; do
+  envsubst_varlist+=$dollar__varname__non_final_comma
+done
 envsubst "$envsubst_varlist" < pre_install.sh.tpl    > pre_install.sh
 envsubst "$envsubst_varlist" < rotate_version.sh.tpl > rotate_version.sh
 

From f4a32a3a95b739224bf7ee22840ecb81279ad56c Mon Sep 17 00:00:00 2001
From: Noach Magedman <noach@seekingalpha.com>
Date: Mon, 8 Apr 2024 16:28:53 +0300
Subject: [PATCH 05/13] DRY!  Extract function run_script_on_ec2_instances

---
 github.sh | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/github.sh b/github.sh
index 34d3bde537a3..8a137c8757ce 100755
--- a/github.sh
+++ b/github.sh
@@ -10,7 +10,16 @@ function hr() {
   echo "==========================================================================="
 }
 
-hr
+function run_script_on_ec2_instances() {
+  local script_file=${1:?}
+  local ip_list_as_multiline_string=${2:?}
+  parallel-ssh \
+    -x "-o StrictHostKeyChecking=no" \
+    --inline --timeout 600 \
+    --user deploy \
+    --hosts <(echo "$ip_list_as_multiline_string") \
+    --send-input < "$script_file"
+}
 
 
 ## Gather all needed data
@@ -67,20 +76,12 @@ envsubst "$envsubst_varlist" < rotate_version.sh.tpl > rotate_version.sh
 
 ## Install RC tarball (and its dependencies) onto all RC nodes
 echo "Installing new build onto all RC nodes..."
-parallel-ssh \
-  -x  "-o StrictHostKeyChecking=no" \
-  --inline --timeout 600 --user deploy \
-  --hosts <(echo "$rc_instance_ips") \
-  --send-input < pre_install.sh            # TODO: Rename to install_tarball.sh
+run_script_on_ec2_instances pre_install.sh "$rc_instance_ips"  # TODO: Rename to install_tarball.sh
 hr
 
 ## Activate new version
 echo "Activating new build on all RC nodes..."
-parallel-ssh \
-  -x  "-o StrictHostKeyChecking=no" \
-  --inline --timeout 600 --user deploy \
-  --hosts <(echo "$rc_instance_ips") \
-  --send-input < rotate_version.sh         # TODO: Rename to activate_new_build.sh
+run_script_on_ec2_instances rotate_version.sh "$rc_instance_ips"  # TODO: Rename to activate_new_build.sh
 hr
 
 ## Flush CDN

From 1e0414de661d380baf10e26ba75dd4eacd5615b5 Mon Sep 17 00:00:00 2001
From: Noach Magedman <noach@seekingalpha.com>
Date: Mon, 8 Apr 2024 16:35:58 +0300
Subject: [PATCH 06/13] Remove Jenkinsfile

We are no longer using Jenkins to deploy RocketChat.
Use GitHub instead.
---
 Jenkinsfile    |  23 -----------
 Jenkinsfile.sh | 103 -------------------------------------------------
 2 files changed, 126 deletions(-)
 delete mode 100644 Jenkinsfile
 delete mode 100755 Jenkinsfile.sh

diff --git a/Jenkinsfile b/Jenkinsfile
deleted file mode 100644
index c5b7f243aad7..000000000000
--- a/Jenkinsfile
+++ /dev/null
@@ -1,23 +0,0 @@
-pipeline {
-    
-    agent any
-
-    options { timestamps () }
-
-    stages {
-        stage('Checkout') {
-            steps {
-                checkout scm
-            }
-        }
-        stage('Build and deploy') {
-            steps {
-                script {
-                    withEnv(["version=$version"]) {
-                        sh './Jenkinsfile.sh'
-                    }
-                }
-            }
-        }
-    }
-}
diff --git a/Jenkinsfile.sh b/Jenkinsfile.sh
deleted file mode 100755
index 6b0550f166f9..000000000000
--- a/Jenkinsfile.sh
+++ /dev/null
@@ -1,103 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-
-case $JOB_BASE_NAME in
-  *staging*)    environment=staging ;;
-  *production*) environment=production ;;
-  *)            echo "ERROR: Can’t infer environment from job name!"; exit 99 ;;
-esac
-
-rc_dir="/opt/rocket-chat"
-s3_bucket="seekingalpha-rocketchat-builds"
-
-## Note: $version is a Jenkins job parameter.
-## We accept either the full tarball filename or just its version substring.
-if [[ "$version" == rocket.chat-*.tgz ]]
-then
-  rc_tarball="$version"
-else
-  rc_tarball="rocket.chat-$version.tgz"
-fi
-
-function hr() {
-  echo "==========================================================================="
-}
-
-
-## Strip off the trailing letter from the region: Use us-west-2, not us-west-2a
-export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | awk '{print substr($0,1,length($0)-1)}')
-
-## EXPORTED variables ending in _ENVSUBST are for expansion in the .tpl template files.
-export AWS_DEFAULT_REGION_ENVSUBST=$AWS_DEFAULT_REGION
-export ENV_ENVSUBST=$environment
-export RC_DIR_ENVSUBST=$rc_dir
-export S3_BUCKET_ENVSUBST=$s3_bucket
-export RC_TARBALL_ENVSUBST=$rc_tarball
-
-## Render Script Templates
-envsubst_varlist=$( ruby -e 'puts ENV.keys.select{ |name| name.end_with?("_ENVSUBST") }.map{ |name| "$#{name}" }.join(",")' )
-envsubst "$envsubst_varlist" < ./pre_install.sh.tpl  > ./pre_install.sh
-envsubst "$envsubst_varlist" < ./rotate_version.sh.tpl > ./rotate_version.sh
-
-## When deploying to production, run using the "rocketchat-deploy" role
-if [[ $environment == production ]] ; then
-  assumed_role_json=$(
-    aws \
-      --output json \
-      sts assume-role \
-      --role-arn arn:aws:iam::618678420696:role/switch-account-deploy-rocket-chat \
-      --role-session-name rocketchat-deploy
-  )
-  assumed_role_variables=$(
-    echo "${assumed_role_json}" | jq -r \
-    '
-      "export AWS_SESSION_TOKEN=" + .Credentials.SessionToken + "\n" +
-      "export AWS_ACCESS_KEY_ID=" + .Credentials.AccessKeyId + "\n" +
-      "export AWS_SECRET_ACCESS_KEY=" + .Credentials.SecretAccessKey + "\n"
-    '
-  )
-  eval "$assumed_role_variables"
-fi
-
-## Get instance IPs one per line (multiline string)
-rc_instance_ips=$(
-  aws ec2 describe-instances \
-      --filters Name=instance-state-name,Values=running \
-                Name=tag:aws:autoscaling:groupName,Values=rocketchat \
-      --query "Reservations[*].Instances[*].NetworkInterfaces[0].PrivateIpAddress" \
-      --output text
-)
-
-
-## Install RC tarball (and its dependencies) onto all RC nodes
-hr
-echo "Installing new build onto all RC nodes:"
-parallel-ssh \
-  --inline --timeout 600 --user deploy \
-  --hosts <(echo "$rc_instance_ips") \
-  --send-input < ./pre_install.sh
-hr
-
-## Activate new version
-echo "Activating new build on all RC nodes:"
-parallel-ssh \
-  --inline --timeout 600 --user deploy \
-  --hosts <(echo "$rc_instance_ips") \
-  --send-input < ./rotate_version.sh
-hr
-
-## Update the version marker file
-echo "Mark which RC build is now active..."
-current_marker_file="rocket.chat-$environment.tgz"
-aws s3 cp "s3://$s3_bucket/$rc_tarball" "s3://$s3_bucket/$current_marker_file" --acl public-read
-hr
-
-## Flush CDN
-echo "Flushing $environment CDN"
-unset AWS_SESSION_TOKEN
-unset AWS_ACCESS_KEY_ID
-unset AWS_SECRET_ACCESS_KEY
-FASTLY_SERVICE=$(aws ssm get-parameter --name /rocketchat/fastly_service_id --with-decryption --query Parameter.Value --output text)
-FASTLY_TOKEN=$(aws ssm get-parameter --name /rocketchat/fastly_api_key --with-decryption --query Parameter.Value --output text)
-curl -X POST -H "Fastly-Key: $FASTLY_TOKEN" "https://api.fastly.com/service/$FASTLY_SERVICE/purge/$environment"

From 26ef12b56b0652136e79c23647328ae4590a3b89 Mon Sep 17 00:00:00 2001
From: Noach Magedman <noach@seekingalpha.com>
Date: Mon, 8 Apr 2024 17:22:31 +0300
Subject: [PATCH 07/13] Minor cleanup of the "deploy" GitHub Workflow

- Tighten input descriptions
- Clarify that `runs-on` is re: GitHub runner, not the EC2 role
- Tighten comments and step names
- Run github.sh from CWD

n.b. I was going to DRY the `environment` droplist by converting
it from `type: choice` to `type: environment`.  GitHub is already
configured to know our deployment environments, so no need to repeat
it here.

The problem with using `type: environment`, however, is that we don't control
the sort order, and the first option ("production") is the default.
I do not want "Deploy to Production" to be the default target!!
---
 .github/workflows/deploy.yml | 43 ++++++++++--------------------------
 1 file changed, 12 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 7748a67bb7e0..086ae1a2eb43 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -2,33 +2,16 @@ on:
   workflow_dispatch:
     inputs:
       version:
+        description: Tarball (either the full filename or just its {rc_version}.{commit_hash} substring)
         type: string
-        description: |
-          version:
-          
-            RC version + git commit number.
-
-            For a list of available versions, look in
-
-            s3://seekingalpha-rocketchat-builds/
-
-            for rocket.chat-VERSION.tgz
-
-            Special versions:
-
-            latest  = version last built
-
-            staging = version last deployed to staging
-
-            production = version last deployed to production
         required: true
-
       environment:
+        description: Deploy to
         type: choice
-        description: chose the AWS environment
         options:
           - staging
           - production
+        required: true
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.environment }}
@@ -39,35 +22,33 @@ defaults:
 
 jobs:
   deploy:
-    name: deploy
+    name: Deploy to EC2
     environment: ${{ inputs.environment }}
-    runs-on: [rocketchat]
+    runs-on: rocketchat  # https://github.com/seekingalpha/Rocket.Chat/settings/actions/runners
     steps:
-      - name: Checkout code
+      - name: Checkout deploy script branch
         uses: actions/checkout@v4
 
-      - name: install dependencies.
+      - name: Install dependencies
         run: |
           sudo apt-get update -y
           sudo apt-get install -y cloud-utils
           sudo apt-get install -y pssh
-          sudo apt-get install -y gettext
+          sudo apt-get install -y gettext      # provides envsubst(1) template renderer
 
-      - name: Configure AWS Credentials
+      - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME }}
           aws-region: ${{ vars.AWS_REGION }}
 
-      #this is the private key for ci user, used by jenkins slave, can be found in ssm parameter staging jenkins slave. that allows ssh to airflow.
-      - name: install the ci private key
+      - name: Obtain the SSH private key for logging in to RC EC2 instances
         uses: webfactory/ssh-agent@v0.9.0
         with:
           ssh-private-key: '${{ secrets.CI_SSH_PRIVATE_KEY }}'
 
-      - name: Run deployment
-        run: "$GITHUB_WORKSPACE/github.sh"
-        shell: bash
+      - name: Run deploy script
+        run: ./github.sh
         env:
           environment: ${{ inputs.environment }}
           version: ${{ inputs.version }}

From bacbe7b1eb833731e2fe8500d1788d4f986952d8 Mon Sep 17 00:00:00 2001
From: Noach Magedman <noach@seekingalpha.com>
Date: Mon, 8 Apr 2024 18:13:35 +0300
Subject: [PATCH 08/13] Copy "Rotate AWS secrets" cronjob from
 "scheduled-events" branch
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

First, the BRANCH:
  GitHub has a concept of the "default branch" which is used for several
  purposes:
    - Default target for new PRs
    - Definition of scheduled cronjob workflows
    - Adds a "Run workflow" button to the Workflow run-history page
      if a `on: workflow_dispatch:` is defined for this workflow.
      (May choose another branch to actually run workflow code,
      but the button only appears if the default branch has it)

  Until now, we had a "scheduled-events" branch as the default
  to facilitate cronjobs.  Now that we also want a "Run workflow"
  button for the "Deploy to EC2" workflow (defined here on the
  "sa_devops" branch) we need to merge these two branches.

  Unfortunately, git won't let us merge the branches:

    ```
    $ git merge scheduled-events
    fatal: refusing to merge unrelated histories
    ```

  Both sa_devops and scheduled-events have their own independent
  init commit.  Apparently you can only merge branches if they
  share a common ancestor, which these don't.

  SO, I'm copying scheduled-events’ two files over manually.
  - .github/workflows/rotate_aws_secrets.yml
  - README.md

Second, the FILE:

  For the prior history of the workflow file, you'll have to
  see the "scheduled-events" branch.

  At some point, the job started failing.  I don't know why.
  Related Jira tasks:
    - DO-5008 Rotate RC GitHub Secrets
    - DO-4591 Update all "rotate aws github keys" jobs to use upstream action
---
 .github/workflows/rotate_aws_secrets.yml | 22 +++++++++++++++
 README.md                                | 35 ++++++++++++++++++------
 2 files changed, 49 insertions(+), 8 deletions(-)
 create mode 100644 .github/workflows/rotate_aws_secrets.yml

diff --git a/.github/workflows/rotate_aws_secrets.yml b/.github/workflows/rotate_aws_secrets.yml
new file mode 100644
index 000000000000..9ce1ec588329
--- /dev/null
+++ b/.github/workflows/rotate_aws_secrets.yml
@@ -0,0 +1,22 @@
+name: Rotate AWS secrets
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 1 * *'  # monthly
+
+
+jobs:
+  rotate-aws-secrets:
+    name: Rotate AWS secrets
+    runs-on: ubuntu-latest
+    steps:
+      - name: Rotate AWS secrets
+        uses: kneemaa/github-action-rotate-aws-secrets@v1.1.0
+        env:
+          OWNER_REPOSITORY: ${{ github.repository }}
+          PERSONAL_ACCESS_TOKEN: ${{ secrets.ROCKETCHAT_GITHUB_TOKEN }}
+          AWS_ACCESS_KEY_ID:      ${{ secrets.AWS_USER_GITHUB_ROCKETCHAT_ACCESS_KEY_ID }}
+          GITHUB_ACCESS_KEY_NAME:             AWS_USER_GITHUB_ROCKETCHAT_ACCESS_KEY_ID
+          AWS_SECRET_ACCESS_KEY:  ${{ secrets.AWS_USER_GITHUB_ROCKETCHAT_SECRET_ACCESS_KEY }}
+          GITHUB_SECRET_KEY_NAME:             AWS_USER_GITHUB_ROCKETCHAT_SECRET_ACCESS_KEY
diff --git a/README.md b/README.md
index de2ffc07b0c5..4980bf12c606 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,29 @@
-# RC deploy scripts
+This repo contains Seeking Alpha's customized version
+of the Rocket.Chat server and web client.  That code
+may be found in any of the `sa_patches_1.2.3` branches,
+which are forks of the `1.2.3` release tags, with our
+custom patches re-cherry-picked onto each new release.
 
-Overview of the deploy process:
-- The Jenkins job is configured to run a Groovy script named `Jenkinsfile`,
-  passing it the job parameters (currently, just `version`)
-- The `Jenkinsfile` script invokes `Jenkinsfile.sh`, passing on all job parameters as environment variables
-- `Jenkinsfile.sh` uses the `*.sh.tpl` template files to prepare `*.sh` files which are executed on
-  the rocketchat nodes via ssh:
+This branch (`sa_devops`), however, is a mostly empty
+branch which serves two purposes:
+  - The "Deploy to EC2" workflow, which copies a tarball
+    from S3 onto the various RC EC2 instances and relaunches
+    the RocketChat servers from it.
+  - The "Rotate AWS secrets" monthly cronjob
+
+GitHub requires a branch to be marked as the "default" branch,
+which is used for three purposes:
+  - Default target for new PRs
+  - Definition of the scheduled cronjob workflows
+  - Adds a "Run workflow" button to the Workflow run-history page
+    if a `on: workflow_dispatch:` is defined for this workflow.
+    (You may choose another branch to actually *run* workflow code,
+    but the button only appears if the default branch defines it.)
+
+# The Deployment Process
+- The Deploy job (defined in .github/workflows/deploy.yml)
+  obtains credentials and runs `./github.sh`
+- `github.sh` uses the `*.sh.tpl` template files to prepare `*.sh` files
+  which are executed on the rocketchat EC2 nodes via ssh:
   - `pre_install.sh` installs the RC bundle tarball and its dependencies into a temporary folder
-  - `rotate_version.sh` swaps out the old installation folder with the new one
+  - `rotate_version.sh` swaps out the old installation folder with the new one and HUPs the daemon

From ed78ce2fcb0cb5b1898b3212b5e5e5d56e1b8e58 Mon Sep 17 00:00:00 2001
From: Noach Magedman <noach@seekingalpha.com>
Date: Mon, 8 Apr 2024 23:36:47 +0300
Subject: [PATCH 09/13] Rename pssh helper scripts

- pre_install.sh    => install_tarball.sh
- rotate_version.sh => activate_new_build.sh
---
 README.md                                          | 4 ++--
 rotate_version.sh.tpl => activate_new_build.sh.tpl | 0
 github.sh                                          | 8 ++++----
 pre_install.sh.tpl => install_tarball.sh.tpl       | 0
 4 files changed, 6 insertions(+), 6 deletions(-)
 rename rotate_version.sh.tpl => activate_new_build.sh.tpl (100%)
 mode change 100755 => 100644
 rename pre_install.sh.tpl => install_tarball.sh.tpl (100%)

diff --git a/README.md b/README.md
index 4980bf12c606..a3db1051fe17 100644
--- a/README.md
+++ b/README.md
@@ -25,5 +25,5 @@ which is used for three purposes:
   obtains credentials and runs `./github.sh`
 - `github.sh` uses the `*.sh.tpl` template files to prepare `*.sh` files
   which are executed on the rocketchat EC2 nodes via ssh:
-  - `pre_install.sh` installs the RC bundle tarball and its dependencies into a temporary folder
-  - `rotate_version.sh` swaps out the old installation folder with the new one and HUPs the daemon
+  - `install_tarball.sh` installs the RC bundle tarball and its dependencies into a temporary folder
+  - `activate_new_build.sh` swaps out the old installation folder with the new one and HUPs the daemon
diff --git a/rotate_version.sh.tpl b/activate_new_build.sh.tpl
old mode 100755
new mode 100644
similarity index 100%
rename from rotate_version.sh.tpl
rename to activate_new_build.sh.tpl
diff --git a/github.sh b/github.sh
index 8a137c8757ce..89f696b69b71 100755
--- a/github.sh
+++ b/github.sh
@@ -70,18 +70,18 @@ export S3_BUCKET_ENVSUBST=$s3_bucket
 for dollar__varname__non_final_comma in $(printenv | grep '^\w*_ENVSUBST=' | sed 's/=.*//; s/^/$/; $!s/$/,/') ; do
   envsubst_varlist+=$dollar__varname__non_final_comma
 done
-envsubst "$envsubst_varlist" < pre_install.sh.tpl    > pre_install.sh
-envsubst "$envsubst_varlist" < rotate_version.sh.tpl > rotate_version.sh
+envsubst "$envsubst_varlist" < install_tarball.sh.tpl    > install_tarball.sh
+envsubst "$envsubst_varlist" < activate_new_build.sh.tpl > activate_new_build.sh
 
 
 ## Install RC tarball (and its dependencies) onto all RC nodes
 echo "Installing new build onto all RC nodes..."
-run_script_on_ec2_instances pre_install.sh "$rc_instance_ips"  # TODO: Rename to install_tarball.sh
+run_script_on_ec2_instances install_tarball.sh "$rc_instance_ips"
 hr
 
 ## Activate new version
 echo "Activating new build on all RC nodes..."
-run_script_on_ec2_instances rotate_version.sh "$rc_instance_ips"  # TODO: Rename to activate_new_build.sh
+run_script_on_ec2_instances activate_new_build.sh "$rc_instance_ips"
 hr
 
 ## Flush CDN
diff --git a/pre_install.sh.tpl b/install_tarball.sh.tpl
similarity index 100%
rename from pre_install.sh.tpl
rename to install_tarball.sh.tpl

From 3baec16d9b059069dec6ca32d007b8e69fb9ecc1 Mon Sep 17 00:00:00 2001
From: Noach Magedman <noach@seekingalpha.com>
Date: Tue, 9 Apr 2024 11:44:42 +0300
Subject: [PATCH 10/13] Deploy: Log our config inputs

The Deploy workflow accepts configuration variables:
  - version
  - environment

I have found it hard to check which values a given run was configured
with since their display is buried in the "Run deploy script" step
in an arrow-collapsed section ("Run ./guthub.sh") containing a list
of exported ENV variables.

Instead, show them loud and clear right at the start of the workflow.
---
 .github/workflows/deploy.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 086ae1a2eb43..ac6fbc3476e2 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -26,6 +26,16 @@ jobs:
     environment: ${{ inputs.environment }}
     runs-on: rocketchat  # https://github.com/seekingalpha/Rocket.Chat/settings/actions/runners
     steps:
+      - name: CONFIG VARS  # Log our configuration inputs right here at the top, so they are easy to find
+        env:
+          environment: ${{ inputs.environment }}
+          version: ${{ inputs.version }}
+        run: |
+          echo
+          echo 'Running the "Deploy to EC2" workflow with the following config vars:'
+          echo "  - version: $version"
+          echo "  - environment: $environment"
+
       - name: Checkout deploy script branch
         uses: actions/checkout@v4
 

From a45b901731bc5c4dd20d6f9b29b83374267ba32a Mon Sep 17 00:00:00 2001
From: Noach Magedman <noach@seekingalpha.com>
Date: Tue, 9 Apr 2024 12:19:50 +0300
Subject: [PATCH 11/13] Give the "Deploy to EC2" workflow a name!

Without one, it defaults to the filename ".github/workflows/deploy.yml"
---
 .github/workflows/deploy.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index ac6fbc3476e2..03b586c0f620 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -1,3 +1,5 @@
+name: Deploy to EC2
+
 on:
   workflow_dispatch:
     inputs:

From 67af4b22e939f81596ea3c4669d787d40cc51af3 Mon Sep 17 00:00:00 2001
From: Noach Magedman <noach@seekingalpha.com>
Date: Tue, 9 Apr 2024 12:51:44 +0300
Subject: [PATCH 12/13] Give the specific workflow RUN a name

This will enable easy viewing of deployment history for each environment
---
 .github/workflows/deploy.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 03b586c0f620..0dc319afc769 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -1,4 +1,5 @@
 name: Deploy to EC2
+run-name: Deploy to ${{ inputs.environment }} (${{ inputs.version }})
 
 on:
   workflow_dispatch:

From 7df4c3d976b1d2fdc592cc044dd3fe54ed02289b Mon Sep 17 00:00:00 2001
From: Noach Magedman <noach@seekingalpha.com>
Date: Tue, 9 Apr 2024 16:48:33 +0300
Subject: [PATCH 13/13] Fix mismatched variable names RC_TARBALL vs. RC_FILE

The template variable is RC_TARBALL_ENVSUBST,
which implies that the local variable is RC_TARBALL,
but it is actually named RC_FILE.

Rename them to be consistent.  I chose to go with RC_TARBALL.
---
 install_tarball.sh.tpl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/install_tarball.sh.tpl b/install_tarball.sh.tpl
index 99e5b80c4246..094dd714622a 100644
--- a/install_tarball.sh.tpl
+++ b/install_tarball.sh.tpl
@@ -2,7 +2,7 @@
 AWS=/usr/local/bin/aws
 AWS_DEFAULT_REGION='$AWS_DEFAULT_REGION_ENVSUBST'
 RC_DIR='$RC_DIR_ENVSUBST'
-RC_FILE='$RC_TARBALL_ENVSUBST'
+RC_TARBALL='$RC_TARBALL_ENVSUBST'
 S3_BUCKET='$S3_BUCKET_ENVSUBST'
 
 set -e
@@ -17,14 +17,14 @@ prepare_directories () {
 # Download given RocketChat build
 build_rc () {
     local s3_path
-    s3_path="s3://$S3_BUCKET/$RC_FILE"
+    s3_path="s3://$S3_BUCKET/$RC_TARBALL"
     tmprcdir=$(mktemp -d)
     cd $tmprcdir
     echo "Downloading Rocket.Chat installation files.."
     $AWS s3 cp --quiet "$s3_path" .
     echo "Unpacking Rocket.Chat files..."
-    tar zxf "$RC_FILE"
-    rm -f "$RC_FILE"
+    tar zxf "$RC_TARBALL"
+    rm -f "$RC_TARBALL"
     echo "Installing modules..."
     # loglevels (high to low): error, warn, notice, http, timing, info, verbose, silly
     #   default = notice