Updates for 2023 AL

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: cimg/ruby:2.6.6-node
+      - image: cimg/ruby:3.3.6-node
         environment:
           RAILS_ENV: test
           AWS_DEFAULT_REGION: us-west-2
@@ -11,12 +11,12 @@ jobs:
           # less and waiting for interactive
           # input. https://stackoverflow.com/a/53055116/14487
           PAGER: cat
-      - image: mdillon/postgis:11
+      - image: safecast/postgres:16.1
         environment:
           POSTGRES_USER: safecast
           POSTGRES_DB: safecast
-          POSTGRES_PASSWORD: ""
-      - image: docker.elastic.co/elasticsearch/elasticsearch:7.5.1
+          POSTGRES_PASSWORD: safecast
+      - image: docker.elastic.co/elasticsearch/elasticsearch:8.14.2
         environment:
           discovery.type: single-node
     steps:
@@ -25,12 +25,10 @@ jobs:
           name: Install build tools
           command: |
             sudo apt-get update
-            sudo apt-get install -y postgresql-client python3-pip
-            sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 1
-            sudo update-alternatives --set python /usr/bin/python3
-            pip3 install wheel
-            pip3 install cryptography==3.3.2 awsebcli awscli
-            gem install bundler:1.17.3
+            sudo apt-get install -y postgresql-client python3 python3-pip
+            pip install --upgrade pip
+            pip install awsebcli==3.21.0 awscli==1.35.7
+            gem install bundler:2.5.21
       - restore_cache:
           keys:
             - ingest-{{ .Environment.CACHE_VERSION }}-{{ checksum "Gemfile.lock" }}
@@ -53,8 +51,7 @@ jobs:
       - run:
           name: Load schema
           command: |
-            psql -h localhost -p 5432 -U safecast safecast -c 'DROP EXTENSION postgis CASCADE'
-            bundle exec rake db:structure:load --trace
+            bundle exec rake db:setup
       - run:
           name: Run rspec
           command: |

.ebextensions/00_base.config

@@ -7,4 +7,4 @@ commands:
     command: |
       /opt/aws/bin/cfn-get-metadata --region `{"Ref": "AWS::Region"}` --stack `{"Ref": "AWS::StackName"}` \
         --resource AWSEBBeanstalkMetadata --key AWS::ElasticBeanstalk::Ext |
-        jq .Parameters > /etc/elasticbeanstalk/parameters-cache
+        jq .Parameters > /var/elasticbeanstalk/parameters-cache

.ebextensions/50-post-deploy.config

@@ -1,37 +1,41 @@
 commands:
   create_post_dir:
     command: "mkdir -p /opt/elasticbeanstalk/hooks/appdeploy/post"
+
 files:
   "/opt/elasticbeanstalk/hooks/appdeploy/post/50_restart_workers.sh":
     mode: "000755"
     owner: root
     group: root
     content: |
       #!/usr/bin/env bash
-
-      source /opt/elasticbeanstalk/support/envvars
 
-      if grep -q ELASTIC_CLOUD_INPUT_QUEUE_URL /etc/elasticbeanstalk/parameters-cache; then
-        if status worker_elastic_cloud | grep -q running; then
+      source /opt/elasticbeanstalk/deployment/env
+
+      if grep -q ELASTIC_CLOUD_INPUT_QUEUE_URL /var/elasticbeanstalk/parameters-cache; then
+        if systemctl is-active --quiet worker_elastic_cloud; then
           logger -t worker_elastic_cloud 'Restarting worker after deploy.'
-          restart worker_elastic_cloud
+          systemctl restart worker_elastic_cloud
         else
           logger -t worker_elastic_cloud 'Starting worker after deploy.'
-          start worker_elastic_cloud
+          systemctl start worker_elastic_cloud
         fi
       else
-        logger -t worker_elastic_cloud 'This does not appear to be a worker node. Setting worker to manual after deploy.'
-        echo manual > /etc/init/worker_elastic_cloud.override
+        logger -t worker_elastic_cloud 'This does not appear to be a worker node. Disabling worker after deploy.'
+        systemctl disable worker_elastic_cloud
+        systemctl stop worker_elastic_cloud
       fi
-      if grep -q S3_RAW_INPUT_QUEUE_URL /etc/elasticbeanstalk/parameters-cache; then
-        if status worker_s3_raw | grep -q running; then
+
+      if grep -q S3_RAW_INPUT_QUEUE_URL /var/elasticbeanstalk/parameters-cache; then
+        if systemctl is-active --quiet worker_s3_raw; then
           logger -t worker_s3_raw 'Restarting worker after deploy.'
-          restart worker_s3_raw
+          systemctl restart worker_s3_raw
         else
           logger -t worker_s3_raw 'Starting worker after deploy.'
-          start worker_s3_raw
+          systemctl start worker_s3_raw
         fi
       else
-        logger -t worker_s3_raw 'This does not appear to be a worker node. Setting worker to manual after deploy.'
-        echo manual > /etc/init/worker_s3_raw.override
+        logger -t worker_s3_raw 'This does not appear to be a worker node. Disabling worker after deploy.'
+        systemctl disable worker_s3_raw
+        systemctl stop worker_s3_raw
       fi

.ebextensions/db.config

@@ -1,12 +1,7 @@
 commands:
   upgrade_postgres:
     command: |
-      /usr/bin/yum -y remove 'postgres*'
-      /usr/bin/yum -y install \
-        'https://yum.postgresql.org/11/redhat/rhel-6-x86_64/postgresql11-libs-11.5-1PGDG.rhel6.x86_64.rpm' \
-        'https://yum.postgresql.org/11/redhat/rhel-6-x86_64/postgresql11-11.5-1PGDG.rhel6.x86_64.rpm' \
-        'https://yum.postgresql.org/11/redhat/rhel-6-x86_64/postgresql11-devel-11.5-1PGDG.rhel6.x86_64.rpm'
-      /bin/ln -sf /usr/pgsql-11/bin/pg_config /usr/bin/pg_config
+      /usr/bin/yum -y install postgresql16 postgresql-libs
 
 files:
   "/etc/profile.d/z_psql.sh":

.ebextensions/workers.config

@@ -1,38 +1,41 @@
 files:
-  "/etc/init/worker_s3_raw.conf":
+  "/etc/systemd/system/worker_s3_raw.service":
     content: |
-      description "Start worker for writing raw points to S3"
-
-      start on runlevel [2345]
-      stop on runlevel [!2345]
-
-      respawn
-      respawn limit unlimited
-
-      script
-        cd /var/app/current
-        sudo -u webapp bash -lc 'rake workers:s3_raw' | logger -t worker_s3_raw
-      end script
-
-  "/etc/init/worker_elastic_cloud.conf":
+      [Unit]
+      Description=Start worker for writing raw points to S3
+      After=network.target
+
+      [Service]
+      User=webapp
+      WorkingDirectory=/var/app/current
+      ExecStart=/bin/bash -lc 'bundle exec rake workers:s3_raw'
+      Restart=always
+      StandardOutput=journal
+      StandardError=journal
+      SyslogIdentifier=worker_s3_raw
+
+      [Install]
+      WantedBy=multi-user.target
+
+  "/etc/systemd/system/worker_elastic_cloud.service":
     content: |
-      description "Start worker for writing points to elastic cloud"
-
-      start on runlevel [2345]
-      stop on runlevel [!2345]
+      [Unit]
+      Description=Start worker for writing points to elastic cloud
+      After=network.target
 
-      respawn
-      respawn limit unlimited
+      [Service]
+      User=webapp
+      WorkingDirectory=/var/app/current
+      ExecStart=/bin/bash -lc 'bundle exec rake workers:elastic_cloud'
+      Restart=always
 
-      script
-        cd /var/app/current
-        sudo -u webapp bash -lc 'rake workers:elastic_cloud' | logger -t worker_elastic_cloud
-      end script
+      [Install]
+      WantedBy=multi-user.target
 
 commands:
   01_match_nginx_timeout_to_sqs_timeout:
     command: |
-      VISIBILITY_TIMEOUT=$(jq -r .AWSEBVisibilityTimeout /etc/elasticbeanstalk/parameters-cache)
+      VISIBILITY_TIMEOUT=$(jq -r .AWSEBVisibilityTimeout /var/elasticbeanstalk/parameters-cache)
       if [[ "${VISIBILITY_TIMEOUT}" != "null" ]]; then
         echo "proxy_read_timeout ${VISIBILITY_TIMEOUT}s;" > /etc/nginx/conf.d/worker.conf
         service nginx restart

.elasticbeanstalk/package.sh

@@ -32,15 +32,4 @@ PACKAGE="${VERSION}.zip"
 
 cp config/database.yml.beanstalk config/database.yml
 
-.elasticbeanstalk/package.py "${PACKAGE}"
-
-aws s3 cp --no-progress ".elasticbeanstalk/app_versions/${PACKAGE}" "s3://${S3_BUCKET_NAME}/${EB_APP_NAME}/"
-
-aws elasticbeanstalk create-application-version \
-    --debug \
-    --region "${AWS_DEFAULT_REGION}" \
-    --application-name "${EB_APP_NAME}" \
-    --version-label "${VERSION}" \
-    --source-bundle "S3Bucket=${S3_BUCKET_NAME},S3Key=${EB_APP_NAME}/${PACKAGE}" \
-    --description "${CLEAN_DESCRIPTION}" \
-    --no-process
+eb appversion --create --label "${VERSION}" --message "${CLEAN_DESCRIPTION}"

.gitignore

@@ -7,3 +7,4 @@ test_csv_dump.last
 /config/docker/postgresql/*.conf
 /script/rackup.pid
 .elasticbeanstalk/saved_configs
+.bundle/config

.tool-versions

@@ -0,0 +1 @@
+ruby 3.3.6

Dockerfile.postgresql

@@ -1,11 +1,11 @@
-FROM postgres:9.6
+FROM postgres:16.1
 
-ENV POSTGIS_MAJOR 2.5
-ENV POSTGIS_VERSION 2.5.1+dfsg-1.pgdg90+1
+ENV POSTGIS_MAJOR=3
+ENV POSTGIS_VERSION=3.5.0+dfsg-1.pgdg120+1
 
 RUN apt-get update \
   && apt-get install -y --no-install-recommends \
-    postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION \
-    postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts=$POSTGIS_VERSION \
-    postgis=$POSTGIS_VERSION \
+  postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION \
+  postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts=$POSTGIS_VERSION \
+  postgis=$POSTGIS_VERSION \
   && rm -rf /var/lib/apt/lists/*

Gemfile

@@ -1,52 +1,52 @@
-source 'https://rubygems.org' do
-  # Puma with Ruby 2.6 running on 64bit Amazon Linux/2.11.7
-  ruby '2.6.6'
-
-  gem 'puma'
-  gem 'rake'
-
-  # api
-  gem 'grape'
-  gem 'grape_logging'
-  gem 'actionview', '>= 5.0.7.2'
-  gem 'active_model_serializers'
-
-  # database/orms
-  gem 'pg'
-  gem 'otr-activerecord'
-  gem 'activerecord-postgis-adapter', '>= 5.2.2'
-
-  # data pipeline
-  gem 'aws-sdk-sns', '~> 1'
-  gem 'aws-sdk-sqs', '~> 1'
-  gem 'aws-sdk-s3', '~> 1'
-  gem 'elasticsearch', '~> 5.0'
-
-  gem 'dotenv'
-  gem 'newrelic_rpm'
-  gem 'rison'
-  gem 'thor'
-  gem 'faraday'
-
-  # elasticbeanstalk rake task
-  gem 'aws-sdk-elasticbeanstalk'
-
-  group :development do
-    gem 'rerun'
-  end
-
-  group :test do
-    gem 'database_cleaner'
-    gem 'rack-test'
-    gem 'rspec'
-    gem 'rspec-json_matcher'
-    gem 'rspec_junit_formatter'
-    gem 'shoulda-matchers'
-  end
-
-  group :development, :test do
-    gem 'annotate'
-    gem 'factory_bot'
-    gem 'pry'
-  end
+source 'https://rubygems.org'
+
+# Ruby 3.3 running on 64bit Amazon Linux 2023/4.2.0
+ruby '3.3.6'
+
+gem 'puma'
+gem 'rake'
+
+# api
+gem 'grape'
+gem 'grape_logging'
+gem 'actionview', '>= 5.0.7.2'
+gem 'active_model_serializers'
+
+# database/orms
+gem 'pg'
+gem 'otr-activerecord'
+gem 'activerecord-postgis-adapter', '>= 5.2.2'
+
+# data pipeline
+gem 'aws-sdk-sns', '~> 1'
+gem 'aws-sdk-sqs', '~> 1'
+gem 'aws-sdk-s3', '~> 1'
+gem 'elasticsearch', '~> 5.0'
+
+gem 'dotenv'
+gem 'newrelic_rpm'
+gem 'rison'
+gem 'thor'
+gem 'faraday'
+
+# elasticbeanstalk rake task
+gem 'aws-sdk-elasticbeanstalk'
+
+group :development do
+  gem 'rerun'
+end
+
+group :test do
+  gem 'database_cleaner'
+  gem 'rack-test'
+  gem 'rspec'
+  gem 'rspec-json_matcher'
+  gem 'rspec_junit_formatter'
+  gem 'shoulda-matchers'
+end
+
+group :development, :test do
+  gem 'annotate'
+  gem 'factory_bot'
+  gem 'pry'
 end