add Vagrant build environment for docker images and local java builds…

herd-build/herd-scripts/herd-docker-vagrant/Vagrantfile

@@ -0,0 +1,87 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure("2") do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://vagrantcloud.com/search.
+  config.vm.box = "generic/centos7"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  # config.vm.box_check_update = false
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  # NOTE: This will enable public access to the opened port
+  # config.vm.network "forwarded_port", guest: 80, host: 8080
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine and only allow access
+  # via 127.0.0.1 to disable public access
+  # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1"
+  config.vm.network "forwarded_port", guest: 8080, host: 8080, host_ip: "127.0.0.1"
+  config.vm.network "forwarded_port", guest: 5443, host: 5443, host_ip: "127.0.0.1"
+  config.vm.network "forwarded_port", guest: 5432, host: 5432, host_ip: "127.0.0.1"
+  config.vm.network "forwarded_port", guest: 18630, host: 18630, host_ip: "127.0.0.1"
+  config.vm.network "forwarded_port", guest: 18631, host: 18631, host_ip: "127.0.0.1"
+  config.vm.network "forwarded_port", guest: 18632, host: 18632, host_ip: "127.0.0.1"
+
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  # config.vm.network "private_network", ip: "192.168.33.10"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  # config.vm.synced_folder "../data", "/vagrant_data"
+  config.vm.synced_folder ".", "/home/vagrant/external"
+
+  # Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+  config.vm.provider "virtualbox" do |vb|
+  #   # Display the VirtualBox GUI when booting the machine
+  #   vb.gui = true
+  #
+    # Customize the amount of memory on the VM:
+    vb.memory = "8192"
+    vb.linked_clone = true
+    vb.cpus = 2
+  end
+  #
+  # View the documentation for the provider you are using for more
+  # information on available options.
+
+  # Enable provisioning with a shell script. Additional provisioners such as
+  # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
+  # documentation for more information about their specific syntax and use.
+  config.vm.provision "shell", inline: <<-SHELL
+    yum update -y
+    yum install -y git
+    yum install -y yum-utils device-mapper-persistent-data lvm2
+    yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+    yum install -y docker-ce docker-ce-cli containerd.io docker-compose
+    usermod -a -G docker vagrant
+    systemctl enable docker
+    systemctl start docker
+  SHELL
+end

herd-build/herd-scripts/herd-docker-vagrant/docker-compose-build.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+# Copyright 2015 herd contributors
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Builds herd 0.119 image
+
+HERD_VER="0.119.0"
+
+### CHANGE THESE ###
+echo "**** WARNING **** CHANGE VARIABLES IN THE SCRIPT FOR YOUR S3 BUCKETS AND SQS QUEUES !!!!!"
+S3BUCKET="222222:s3::::test-bucket"
+INCOMING_SQSQ="22222:sqs:::::herd-incoming"
+IDXUPTD_SQSQ="22222:sqs:::::herd-idxupdt"
+
+# here to save time
+if [ ! -e herd/herd-war-$HERD_VER.war ] ; then 
+	curl https://oss.sonatype.org/service/local/repositories/releases/content/org/finra/herd/herd-war/$HERD_VER/herd-war-$HERD_VER.war > herd/herd-war-$HERD_VER.war
+
+fi ;
+
+if [ ! -e herd-scripts-sql-$HERD_VER.jar ] ; then 
+	curl https://oss.sonatype.org/service/local/repositories/releases/content/org/finra/herd/herd-scripts-sql/$HERD_VER/herd-scripts-sql-$HERD_VER.jar > herd-scripts-sql-$HERD_VER.jar
+fi ; 
+
+if [ ! -d herd-setul-sql ] ; then
+	mkdir herd-setup-sql
+	cd herd-setup-sql
+	unzip ../herd-scripts-sql-$HERD_VER.jar
+	# fix file orders per cloudformation script
+	mv herd.postgres.0.1.0.create.sql 1herd.postgres.0.1.0.create.sql
+	mv herd.postgres.0.1.0.refdata.sql 2herd.postgres.0.1.0.refdata.sql
+	mv herd.postgres.0.1.0.cnfgn.sql 3herd.postgres.0.1.0.cnfgn.sql
+	rm herd.postgres.clean.refdata.sql
+
+	for f in *.upgrade.sql ; do 
+		mv $f 4$f ; 
+	done ;
+
+	for f in activiti*sql ; do
+		mv $f 5$f ; 
+	done ; 
+
+	for f in quartz*sql ; do
+		# moved to last, this borks in the current release and the initializer then stops running, so set variable
+		# more scripts
+
+		echo "\set ON_ERROR_STOP 0" >> 99$f
+		cat $f >> 99$f
+		rm -f $f
+	done ; 
+
+	for f in elasticsearch*sql ; do
+		mv $f 7$f ; 
+	done ;
+
+	# manual additional data
+
+	cat > 98last.sql << EOF
+
+DELETE FROM cnfgn WHERE cnfgn_key_nm = 's3.managed.bucket.name';
+INSERT INTO cnfgn VALUES ('s3.managed.bucket.name','${S3BUCKET}', NULL);
+DELETE FROM cnfgn WHERE cnfgn_key_nm = 'herd.notification.sqs.incoming.queue.name';
+INSERT INTO cnfgn VALUES ('herd.notification.sqs.incoming.queue.name','${INCOMING_SQSQ}', NULL);
+DELETE FROM cnfgn WHERE cnfgn_key_nm = 'search.index.update.sqs.queue.name';
+INSERT INTO cnfgn VALUES ('search.index.update.sqs.queue.name','${IDXUPTD_SQSQ}', NULL);
+DELETE FROM cnfgn WHERE cnfgn_key_nm = 'security.enabled.spel.expression';
+INSERT INTO cnfgn VALUES ('security.enabled.spel.expression','false', NULL);
+INSERT INTO cnfgn VALUES ('elasticsearch.best.fields.query.boost','100', NULL);
+INSERT INTO cnfgn VALUES ('elasticsearch.phrase.prefix.query.boost','1', NULL);
+INSERT INTO cnfgn VALUES ('elasticsearch.phrase.query.boost','1000', NULL);
+INSERT INTO cnfgn VALUES ('elasticsearch.rest.client.hostname','herd-elasticsearch', NULL);
+INSERT INTO cnfgn VALUES ('elasticsearch.rest.client.scheme','http', NULL);
+INSERT INTO cnfgn VALUES ('elasticsearch.rest.client.port','9200', NULL);
+INSERT INTO cnfgn VALUES ('elasticsearch.domain.rest.client.hostname','herd-elasticsearch', NULL);
+INSERT INTO cnfgn VALUES ('elasticsearch.domain.rest.client.port','9200', NULL);
+INSERT INTO cnfgn VALUES ('elasticsearch.domain.rest.client.scheme','http', NULL);
+
+EOF
+fi ;
+#if behind a proxy, remember --build-arg http_proxy --build-arg https_proxy
+
+echo "You're now ready to run 'docker-compose build' if you haven't already, or re-initialize the database."
+
+# FIXME: add uname check, this isn't needed unless it's on linux
+# check sysctl value to make sure elasticsearch is going to be happy
+echo "*********** WARNING ****************"
+echo "Make sure vm.max_map_count = 262144 or Elasticsearch will not run. Current value is "`sysctl vm.max_map_count`

herd-build/herd-scripts/herd-docker-vagrant/docker-compose.yml

@@ -0,0 +1,48 @@
+version: '3.5'
+
+volumes:
+  herd-pgsql-data:
+    driver: local
+  herd-elasticsearch-data:
+    driver: local
+
+services:
+  herd-pgsql:
+    image: postgres:10
+    restart: always
+    environment:
+      - POSTGRES_USER=finraherd
+      - POSTGRES_PASSWORD=Changeme123
+      - POSTGRES_DB=herd
+    volumes:
+      - herd-pgsql-data:/var/lib/postgresql/data
+      - ./herd-setup-sql:/docker-entrypoint-initdb.d
+
+  herd-elasticsearch:
+    image: herd-elasticsearch:latest
+    build:
+      context: elasticsearch/
+      dockerfile: Dockerfile
+    volumes:
+      - herd-elasticsearch-data:/var/lib/elasticsearch
+
+  herd:
+    image: herdsvc:latest
+    build:
+      context: herd/
+      dockerfile: Dockerfile
+      args:
+        - HERD_RELEASE=0.119.0
+    depends_on:
+      - herd-pgsql
+      - herd-elasticsearch
+    environment:
+      - AWS_ACCESS_KEY_ID
+      - AWS_SECRET_KEY
+      - HERD_UI_HOST=localhost:5443
+    links:
+      - "herd-pgsql"
+      - "herd-elasticsearch"
+    ports:
+      - "8080:8080"
+      - "5443:80"

herd-build/herd-scripts/herd-docker-vagrant/elasticsearch/Dockerfile

@@ -0,0 +1,10 @@
+# vim:set ft=dockerfile:
+#FROM debian:stretch-slim
+FROM openjdk:8-jdk
+
+ADD setup-elasticsearch.sh /setup/setup-elasticsearch.sh
+RUN set -ex ; \
+	chmod a+x /setup/setup-elasticsearch.sh ; cd /setup ; bash ./setup-elasticsearch.sh
+EXPOSE 9200 9300
+USER elasticsearch
+ENTRYPOINT ["/usr/share/elasticsearch/bin/elasticsearch"]

herd-build/herd-scripts/herd-docker-vagrant/elasticsearch/setup-elasticsearch.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Copyright 2015 herd contributors
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# setup for elasticache locally
+
+set -ex ; 
+curl https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.0.1-amd64.deb  > esearch.deb ; 
+apt-get install -y ./esearch.deb ; 
+rm -f ./esearch.deb ;
+echo "network.host : 0.0.0.0" >> /etc/elasticsearch/elasticsearch.yml
+echo "http.port : 9200"  >> /etc/elasticsearch/elasticsearch.yml
+echo "node.name: node-1" >> /etc/elasticsearch/elasticsearch.yml
+echo "cluster.initial_master_nodes: [\"node-1\"]" >> /etc/elasticsearch/elasticsearch.yml
+# the below is in the Cloudformation, but doing it here causes things to bork
+#echo "script.engine.groovy.inline.search: on" >> /etc/elasticsearch/elasticsearch.yml

herd-build/herd-scripts/herd-docker-vagrant/herd/Dockerfile

@@ -0,0 +1,26 @@
+# vim:set ft=dockerfile:
+#FROM debian:stretch-slim
+FROM tomcat:jre8
+
+
+ADD setup-os.sh /setup/setup-os.sh
+RUN set -ex ; \ 
+	chmod a+x /setup/setup-os.sh ; \
+	cd /setup/ ; \
+	bash /setup/setup-os.sh ;
+
+ADD setup-herd.sh /setup/setup-herd.sh
+ARG HERD_RELEASE
+ADD herd-war-${HERD_RELEASE}.war /herd-app.war
+RUN set -ex ; \
+	chmod a+x /setup/setup-herd.sh; \
+	cd /setup/ ; \
+	bash /setup/setup-herd.sh ;
+
+ADD run-herd.sh /usr/local/tomcat
+RUN chmod a+x /usr/local/tomcat/run-herd.sh
+
+ENTRYPOINT /usr/local/tomcat/run-herd.sh
+
+EXPOSE 80 8080
+

herd-build/herd-scripts/herd-docker-vagrant/herd/run-herd.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+# Copyright 2015 herd contributors
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# need to start apache first and then run catalina
+# the way the container did before
+
+# FIRST RUN
+
+if [ ! -e /var/www/html/configuration.json ] ; then
+
+	echo "Determining host IP for configuration ..."
+	if [ -z $HERD_UI_HOST ] ; then 
+		# check if we're in AWS, or bork
+		herd_ui_host=$(/usr/bin/curl http://169.254.169.254/latest/meta-data/public-hostname)
+		if [ -z herd_ui_host ] ; then
+			echo "ERROR: unable to determine host IP or name; please set HERD_UI_HOST env variable for container."
+			exit 2
+		fi;
+
+	else
+		herd_ui_host=$HERD_UI_HOST
+	fi;
+	echo "Setting UI host to $herd_ui_host"
+
+echo > /var/www/html/configuration.json << EOF3
+{
+  'restBaseUri': 'http://'$herd_ui_host':8080/herd-app/rest',
+  'basicAuthRestBaseUri': 'basicAuthRestBaseUri',
+  'helpUrl': 'helpUrl',
+  'supportEmail': 'orgSupportEmail',
+  'brandHeader': 'Herd-UI',
+  'brandMotto': 'Locate and understand data available in HERD',
+  'docTitlePrefix': 'Herd-UI',
+  'useBasicAuth': 'false',
+  'alertDelayInSeconds': '10',
+  'trackAnalytics': 'false',
+  'ga': {
+    'key': 'key',
+    'iv': 'iv',
+    'trackingId': 'trackingId'
+  }
+}
+
+EOF3
+
+fi ;
+
+
+service apache2 start
+catalina.sh run