[platform, crons-python] deploy non-GAE projects, starting with Crons

auto-deploy.exclude

@@ -3,6 +3,5 @@
 vue
 ruby
 laravel
-crons-python
 tda
 postgres

bin/validate_env.sh

@@ -33,8 +33,8 @@ for var in $(grep -v '^#' validate_env.list | xargs); do
   fi
 done 
 
-proj=$(basename $(pwd))
-. get_proj_var.sh "%s_APP_DSN" $proj
-. get_proj_var.sh "%s_SENTRY_PROJECT" $proj
-error_message=$(var_name.sh "%s_SENTRY_PROJECT and %s_APP_DSN point to different projects." $proj $proj)
+#proj=$(basename $(pwd))
+#. get_proj_var.sh "%s_APP_DSN" $proj
+#. get_proj_var.sh "%s_SENTRY_PROJECT" $proj
+#error_message=$(var_name.sh "%s_SENTRY_PROJECT and %s_APP_DSN point to different projects." $proj $proj)
 #validate_dsn.sh $sentry_project $app_dsn "$error_message"

bin/validate_project.sh

@@ -15,8 +15,8 @@ if [ "$proj" != "spring-boot" ]; then
         echo "[ERROR] project '$proj' contains legacy app.yaml file that is no longer used. Please delete this file,
         it is no longer be needed and has been replaced by app.yaml.template."
         exit 1
-    elif [ ! -f "$path/app.yaml.template" ]; then
-        echo "[ERROR] Missing $proj/app.yaml.template with '<SERVICE>' placeholder in place of actual service name."
+    elif [ ! -f "$path/app.yaml.template" -a ! -f "$path/deploy_project.sh" ]; then
+        echo "[ERROR] Project $proj must contain either app.yaml.template with '<SERVICE>' placeholder OR a deploy_project.sh script."
         exit 1
     fi
-fi
+fi

crons-python/crontab.template

@@ -0,0 +1 @@
+*/5 * * * * <CRONSPYTHON_DEPLOY_DIR>/run.sh

crons-python/deploy_project.sh

@@ -1,10 +1,15 @@
 #!/bin/bash
 
-HOST=empower-tda-and-crons.us-central1-a.sales-engineering-sf
-DIR=/home/kosty/empower-crons-python
-CRON_EXPR="*/5 * * * *"
-CRON_USER=kosty
-COMMAND="$DIR/run.sh"
+if [ ! -f .env ]; then
+  >&2 echo "[ERROR] Missing .env file. Project deploy scripts are not supposed to be run \
+directly, run the following command instead (from parent dir): ./deploy.sh --env=production crons-python"
+  exit 1
+fi
+source .env
+
+HOST="$CRONSPYTHON_DEPLOY_HOST"
+DIR="$CRONSPYTHON_DEPLOY_DIR"
+CRONTAB_USER=$CRONSPYTHON_CRONTAB_USER
 
 function cleanup {
   echo "NOTE: if ssh check hangs, re-run 'gcloud compute config-ssh; ssh $HOST exit' to fix"
@@ -32,7 +37,7 @@ if ssh $HOST '[[ -d '"$DIR/env"' ]] && [[ ! -z `ls -A '"$DIR/env"'` ]]'; then
 fi
 
 echo "Copying code to remote directory..."
-rsync -rz --exclude env * $HOST:$DIR/
+rsync -rz --exclude env * .env $HOST:$DIR/
 if [ $? != 0 ]; then
   echo "[ERROR] Failed to rsync code to remote directory."
   exit 1
@@ -53,18 +58,19 @@ fi
 
 # handle case when crontab is empty
 echo "EXISTING crontab:"
-ssh $HOST 'sudo crontab -u '$CRON_USER' -l'
+ssh $HOST 'sudo crontab -u '$CRONTAB_USER' -l'
 if [ $? != 0 ]; then
   ssh $HOST 'echo "" | crontab -'
 fi
 echo "---"
 
 # set up cron job if not set up already
 # NOTE: this will overwrite any existing cron jobs from same project directory
-ssh $HOST '(sudo crontab -u '$CRON_USER' -l | grep -v '"$DIR"'; echo "'"$CRON_EXPR $COMMAND"'") | sort - | uniq - | sudo crontab -u '$CRON_USER' -'
+ssh $HOST '(sudo crontab -u '$CRONTAB_USER' -l | grep -v '"$DIR"'; cat '$DIR'/crontab) | sort - | uniq - | sudo crontab -u '$CRONTAB_USER' -'
 
 echo "UPDATED crontab:"
-ssh $HOST 'sudo crontab -u '$CRON_USER' -l'
+ssh $HOST 'sudo crontab -u '$CRONTAB_USER' -l'
 echo "---"
 
 echo "Done. New code should be picked up when cron job runs next time."
+echo "If not working, ssh into the host ($HOST) and check the logs in $DIR/crons-python.log"

crons-python/main.py

@@ -3,17 +3,22 @@
 import random
 import urllib.request
 import time
+import dotenv
+import os
 
-# crons-python
-DSN = "https://[email protected]/4505197623705600"
+dotenv.load_dotenv()
+
+# set in ../env-config/*.env
+DSN = os.environ["CRONSPYTHON_APP_DSN"]
+MONITOR_SLUG = os.environ["CRONSPYTHON_MONITOR_SLUG"]
 FAILURE_PERCENT_CHANCE = 5 
 STUCK_PERCENT_CHANCE = 7 
 STUCK_SLEEP_MIN_MINUTES = 6
 STUCK_SLEEP_MAX_MINUTES = 20
 
 random_number = random.randint(0, 99)
 
-@monitor(monitor_slug='crons-python-monitor')
+@monitor(monitor_slug=MONITOR_SLUG)
 def job():
     if random_number < FAILURE_PERCENT_CHANCE:
         print("Attempting to call an API that is down.")
@@ -29,7 +34,7 @@ def job():
 if __name__ == "__main__":
     sentry_sdk.init(dsn=DSN)
     sentry_sdk.set_context("monitor", {
-        "slug": "cron-job-monitor-python",
+        "slug": MONITOR_SLUG,
     })
     job()
     exit(0)

crons-python/requirements.txt

@@ -1 +1,2 @@
-sentry-sdk==1.23.0
+sentry-sdk==1.23.0
+python-dotenv==1.0.0

crons-python/run.sh

@@ -4,4 +4,4 @@ set -e
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 cd $SCRIPT_DIR
 source env/bin/activate
-python3.10 main.py
+python3.10 main.py 2>&1 | tee crons-python.log

crons-python/validate_env.list

@@ -0,0 +1,5 @@
+CRONSPYTHON_APP_DSN
+CRONSPYTHON_MONITOR_SLUG
+CRONSPYTHON_DEPLOY_HOST
+CRONSPYTHON_DEPLOY_DIR
+CRONSPYTHON_CRONTAB_USER

deploy.sh

@@ -83,7 +83,7 @@ fi
 be_projects=""
 fe_projects=""
 for proj in $projects; do
-  if [[ $proj =~ ^(flask|express|ruby|spring-boot|aspnetcore|laravel|ruby-on-rails)$ ]]; then
+  if [[ $proj =~ ^(flask|express|ruby|spring-boot|aspnetcore|laravel|ruby-on-rails|crons-python)$ ]]; then
     be_projects+="$proj "
   else
     fe_projects+="$proj "
@@ -101,6 +101,7 @@ function cleanup {
   rm -f $top/*/.app.yaml
   rm -f $top/spring-boot/src/main/appengine/app.yaml
   rm -f $top/spring-boot/src/main/resources/application.properties
+  rm -f $top/crons-python/crontab
   if [ "$generated_envs" != "" ]; then
     rm -f $generated_envs # bash only (passed as separate args)
   fi
@@ -197,6 +198,13 @@ for proj in $projects; do # bash only
         exit 1
       fi
     fi
+  elif [ -f deploy_project.sh ]; then
+    if [[ "$proj" =~ ^crons- ]]; then
+      . get_proj_var.sh "%s_DEPLOY_DIR" $proj
+      escaped_deploy_dir=$(echo "$deploy_dir" | sed 's_/_\\/_g')
+      sed -e 's/<CRONSPYTHON_DEPLOY_DIR>/'$escaped_deploy_dir'/g' crontab.template > crontab 
+    fi
+    ./deploy_project.sh
   else
 
     # Replace <SERVICE> in app.yaml.template with <PROJECT>_APP_ENGINE_SERVICE

env-config/example.env

@@ -73,3 +73,9 @@ RUBYONRAILS_LOCAL_PORT=3000 # only needed in local.env
 RUBYONRAILS_RAILS_ENV=
 RUBYONRAILS_SECRET_KEY_BASE=
 SECRET_KEY_BASE=
+
+CRONSPYTHON_APP_DSN=
+CRONSPYTHON_MONITOR_SLUG=
+CRONSPYTHON_DEPLOY_HOST=
+CRONSPYTHON_DEPLOY_DIR=
+CRONSPYTHON_CRONTAB_USER= # if change this must clear crontab for previus user or will have 2 running simult.

env.sh

@@ -53,13 +53,15 @@ echo "" >> .env # in case no newline
 unset RELEASE # so we don't accidentally pick up RELEASE from another project (deploy.sh)
 # Ignore both comment lines and inline comments
 export $(grep -v '^#' .env | sed 's/ #.*//' | xargs) # just for *_RELEASE_PACKAGE_NAME and RELEASE
-if [ "$RELEASE" == "" ]; then
-    . get_proj_var.sh "%s_RELEASE_PACKAGE_NAME" $proj
-    release="${release_package_name}@"`release.sh`
-    >&2 echo $release
-    # deploy.sh script itself and non-React projects expect RELEASE
-    echo "RELEASE=$release" >> .env
-    export RELEASE="$release"
+if ! [[ $proj =~ ^(crons|tda)- ]]; then
+    if [ "$RELEASE" == "" ]; then
+        . get_proj_var.sh "%s_RELEASE_PACKAGE_NAME" $proj
+        release="${release_package_name}@"`release.sh`
+        >&2 echo $release
+        # deploy.sh script itself and non-React projects expect RELEASE
+        echo "RELEASE=$release" >> .env
+        export RELEASE="$release"
+    fi
 fi
 if [ "$proj" == "react" ]; then
     echo "REACT_APP_RELEASE=$RELEASE" >> .env