Incorporate JSONB elements and staging deployment

deployment/.gitignore

@@ -1,2 +1 @@
-config-aws.mk
-docker-compose.deploy.yml
+config-*.mk

deployment/Makefile

@@ -1,71 +1,65 @@
-include config-aws.mk  # Variables for AWS options
+include config-deployment.mk
 
-# The osmesa container
-LOCAL_IMG := quay.io/geotrellis/osm-stat-server:latest
+# If the user is on master branch, see if we should deploy to production
+VERSION_TAG=$(shell ./scripts/get-tag.sh)
+ifeq ($(VERSION_TAG), production)
+	DATABASE=${PRODUCTION_DB}
+	ECS_CLUSTER=${CLUSTER_NAME_DEPLOYMENT}
+	TASK_SUFFIX=
+else
+	DATABASE=${STAGING_DB}
+	ECS_CLUSTER=${CLUSTER_NAME_STAGING}
+	TASK_SUFFIX=-staging
+endif
+DB_URI=${DB_BASE_URI}/${DATABASE}
 
+.EXPORT_ALL_VARIABLES:
 
-#########
-#  AWS  #
-#########
+#############################
+#  Docker image management  #
+#############################
+
+.PHONY: login-aws-registry tag-image push-image
+
+build-container:
+	./build-container.sh
 
 login-aws-registry:
 	eval `aws ecr get-login --no-include-email --region ${AWS_REGION}`
 
-tag-image:
-	docker tag ${LOCAL_IMG} ${ECR_REPO}
+tag-image: build-container
+	docker tag osm_stat_server:${VERSION_TAG} ${ECR_IMAGE}:${VERSION_TAG}
 
 push-image: login-aws-registry tag-image
-	docker push ${ECR_REPO}
-
-.PHONY: docker-compose.deploy.yml
-
-docker-compose.deploy.yml:
-	./expand.sh docker-compose.deploy.yml.tpl > docker-compose.deploy.yml
-
-configure-cluster:
-	ecs-cli configure \
-	  --cluster ${CLUSTER_NAME} \
-	  --region ${AWS_REGION} \
-	  --config-name ${CONFIG_NAME}
-
-cluster-up:
-	ecs-cli up \
-	  --keypair ${KEYPAIR} \
-	  --instance-role ${INSTANCE_ROLE} \
-	  --size 1 \
-	  --instance-type ${INSTANCE_TYPE} \
-	  --cluster-config ${CONFIG_NAME} \
-	  --subnets ${SUBNETS} \
-	  --vpc ${VPC} \
-	  --force \
-	  --verbose
-
-cluster-down:
-	ecs-cli down --cluster-config ${CONFIG_NAME}
+	docker push ${ECR_IMAGE}:${VERSION_TAG}
 
-.PHONY: create-service
+#######################
+# Streaming AWS Tasks #
+#######################
 
-create-service: docker-compose.deploy.yml configure-cluster
-	ecs-cli compose \
-	  --file $< create \
-	  --cluster ${CLUSTER_NAME}
+.PHONY: create-log-groups define-production-tasks define-staging-tasks stop-stat-server deploy-stat-server
 
-start-service: docker-compose.deploy.yml configure-cluster create-service
-	ecs-cli compose --file $< service up \
-	  --deployment-min-healthy-percent 0 \
-	  --create-log-groups \
-	  --cluster ${CLUSTER_NAME}
+create-log-groups:
+	./scripts/create-log-groups.sh
 
-stop-service:
-	ecs-cli compose down
+define-staging-tasks:
+	./scripts/define-staging-tasks.sh
 
+define-production-tasks:
+	./scripts/define-production-tasks.sh
 
-#########
-#  ALL  #
-#########
-build-image:
-	make -C .. build
+stop-stat-server:
+	./scripts/stop-stat-server.sh
 
-clean:
-	rm -f docker-compose.deploy.yml
+deploy-stat-server: stop-stat-server
+	aws ecs create-service \
+	  --cluster "${ECS_CLUSTER}" \
+	  --service-name "osmesa-stats-server" \
+	  --task-definition "osmesa-stat-server${TASK_SUFFIX}" \
+	  --desired-count 1 \
+	  --launch-type FARGATE \
+	  --scheduling-strategy REPLICA \
+	  --network-configuration ${NETWORK_CONFIGURATION}
 
+deploy-stats-refresher:
+	./scripts/deploy-stats-refresher.sh

deployment/build-container.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+if [ -z ${VERSION_TAG+x} ]; then
+    echo "No version tag has been set.  Do not run this script directly; instead, issue"
+    echo "                              make build-container"
+    echo "from the 'streaming' directory."
+    exit 1
+else
+    echo "Version tag is set to '${VERSION_TAG}'"
+fi
+
+set -xe
+SBT_DIR=$(pwd)/..
+JAR_DIR=${SBT_DIR}/target/scala-2.11/
+DOCKER_DIR=$(pwd)/docker
+
+cd ${SBT_DIR}
+./sbt clean assembly
+cp ${JAR_DIR}/osm-stat-server.jar ${DOCKER_DIR}/osm-stat-server.jar
+
+cd ${DOCKER_DIR}
+docker build -f Dockerfile --tag osm_stat_server:${VERSION_TAG} .

deployment/config-deployment.mk.template

@@ -0,0 +1,23 @@
+################################################################################
+# AWS properties
+################################################################################
+export KEYPAIR :=
+export SUBNET :=
+export AWS_REGION := us-east-1
+export IAM_ACCOUNT :=
+
+################################################################################
+#                                                 Streaming resource definitions
+################################################################################
+export STREAMING_INSTANCE_TYPE := m4.xlarge
+export ECR_IMAGE :=
+export AWS_LOG_GROUP := osmesa-stats-server
+export ECS_SUBNET := ${SUBNET}
+export ECS_SECURITY_GROUP :=
+
+export CLUSTER_NAME_DEPLOYMENT :=
+export CLUSTER_NAME_STAGING :=
+
+export DB_BASE_URI :=
+export PRODUCTION_DB :=
+export STAGING_DB :=

deployment/docker/.gitignore

@@ -0,0 +1 @@
+*.jar

docker/osm-stat-server/Dockerfile → deployment/docker/Dockerfile

@@ -9,4 +9,4 @@ RUN \
 COPY osm-stat-server.jar /opt/osm-stat-server.jar
 COPY refresh-views.sh /usr/local/bin/refresh-views.sh
 
-WORKDIR /opt
+WORKDIR /opt

docker/osm-stat-server/refresh-views.sh → deployment/docker/refresh-views.sh

@@ -1,12 +1,16 @@
 #!/usr/bin/env bash
 
+echo "$(date --iso-8601=seconds): Starting view refreshment"
+
 if [ "$(psql -Aqtc "select count(pid) from pg_stat_activity where query ilike 'refresh materialized view concurrently user_statistics%' and state='active'" $DATABASE_URL 2> /dev/null)" == "0" ]; then
   echo "$(date --iso-8601=seconds): Refreshing user statistics"
   # refresh in the background to return immediately
   psql -Aqt \
     -c "REFRESH MATERIALIZED VIEW CONCURRENTLY user_statistics" \
     -c "UPDATE refreshments SET updated_at=now() where mat_view='user_statistics'" \
     $DATABASE_URL &
+else
+  echo "$(date --iso-8601=seconds): User stats table already refreshing"
 fi
 
 if [ "$(psql -Aqtc "select count(pid) from pg_stat_activity where query ilike 'refresh materialized view concurrently hashtag_statistics%' and state='active'" $DATABASE_URL 2> /dev/null)" == "0" ]; then
@@ -16,6 +20,8 @@ if [ "$(psql -Aqtc "select count(pid) from pg_stat_activity where query ilike 'r
     -c "REFRESH MATERIALIZED VIEW CONCURRENTLY hashtag_statistics" \
     -c "UPDATE refreshments SET updated_at=now() where mat_view='hashtag_statistics'" \
     $DATABASE_URL &
+else
+  echo "$(date --iso-8601=seconds): Hashtag stats table already refreshing"
 fi
 
 if [ "$(psql -Aqtc "select count(pid) from pg_stat_activity where query ilike 'refresh materialized view concurrently country_statistics%' and state='active'" $DATABASE_URL 2> /dev/null)" == "0" ]; then
@@ -25,6 +31,17 @@ if [ "$(psql -Aqtc "select count(pid) from pg_stat_activity where query ilike 'r
     -c "REFRESH MATERIALIZED VIEW CONCURRENTLY country_statistics" \
     -c "UPDATE refreshments SET updated_at=now() where mat_view='country_statistics'" \
     $DATABASE_URL &
+else
+  echo "$(date --iso-8601=seconds): Country stats table already refreshing"
+fi
+
+if [ "$(psql -Aqtc "select count(pid) from pg_stat_activity where query ilike 'refresh materialized view concurrently hashtag_user_statistics%' and state='active'" $DATABASE_URL 2> /dev/null)" == "0" ]; then
+  # refresh in the background to return immediately
+  echo "$(date --iso-8601=seconds): Refreshing hashtag/user statistics"
+  psql -Aqt \
+    -c "REFRESH MATERIALIZED VIEW CONCURRENTLY hashtag_user_statistics" \
+    -c "UPDATE refreshments SET updated_at=now() where mat_view='hashtag_user_statistics'" \
+    $DATABASE_URL &
 fi
 
 wait

deployment/scripts/create-log-groups.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ -z ${VERSION_TAG+x} ]; then
+    echo "Do not run this script directly.  Use the Makefile in the parent directory."
+    exit 1
+fi
+
+DEFINED_GROUPS=$(aws logs describe-log-groups | jq '.logGroups[].logGroupName' | sed -e 's/"//g')
+
+if [[ $DEFINED_GROUPS != *"/ecs/${AWS_LOG_GROUP}"* ]]; then
+    aws logs create-log-group \
+        --log-group-name /ecs/${AWS_LOG_GROUP}
+fi
+
+if [[ $DEFINED_GROUPS != *"/ecs/${AWS_LOG_GROUP}-staging"* ]]; then
+    aws logs create-log-group \
+        --log-group-name /ecs/${AWS_LOG_GROUP}-staging
+fi