cloud-config.yml

#cloud-config

# Instance

apt_sources:
- source: "ppa:openjdk-r/ppa"
- source: "deb http://packages.elasticsearch.org/elasticsearch/1.7/debian stable main"
  key: |
    -----BEGIN PGP PUBLIC KEY BLOCK-----
    Version: GnuPG v2.0.14 (GNU/Linux)

    mQENBFI3HsoBCADXDtbNJnxbPqB1vDNtCsqhe49vFYsZN9IOZsZXgp7aHjh6CJBD
    A+bGFOwyhbd7at35jQjWAw1O3cfYsKAmFy+Ar3LHCMkV3oZspJACTIgCrwnkic/9
    CUliQe324qvObU2QRtP4Fl0zWcfb/S8UYzWXWIFuJqMvE9MaRY1bwUBvzoqavLGZ
    j3SF1SPO+TB5QrHkrQHBsmX+Jda6d4Ylt8/t6CvMwgQNlrlzIO9WT+YN6zS+sqHd
    1YK/aY5qhoLNhp9G/HxhcSVCkLq8SStj1ZZ1S9juBPoXV1ZWNbxFNGwOh/NYGldD
    2kmBf3YgCqeLzHahsAEpvAm8TBa7Q9W21C8vABEBAAG0RUVsYXN0aWNzZWFyY2gg
    KEVsYXN0aWNzZWFyY2ggU2lnbmluZyBLZXkpIDxkZXZfb3BzQGVsYXN0aWNzZWFy
    Y2gub3JnPokBOAQTAQIAIgUCUjceygIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgEC
    F4AACgkQ0n1mbNiOQrRzjAgAlTUQ1mgo3nK6BGXbj4XAJvuZDG0HILiUt+pPnz75
    nsf0NWhqR4yGFlmpuctgCmTD+HzYtV9fp9qW/bwVuJCNtKXk3sdzYABY+Yl0Cez/
    7C2GuGCOlbn0luCNT9BxJnh4mC9h/cKI3y5jvZ7wavwe41teqG14V+EoFSn3NPKm
    TxcDTFrV7SmVPxCBcQze00cJhprKxkuZMPPVqpBS+JfDQtzUQD/LSFfhHj9eD+Xe
    8d7sw+XvxB2aN4gnTlRzjL1nTRp0h2/IOGkqYfIG9rWmSLNlxhB2t+c0RsjdGM4/
    eRlPWylFbVMc5pmDpItrkWSnzBfkmXL3vO2X3WvwmSFiQbkBDQRSNx7KAQgA5JUl
    zcMW5/cuyZR8alSacKqhSbvoSqqbzHKcUQZmlzNMKGTABFG1yRx9r+wa/fvqP6OT
    RzRDvVS/cycws8YX7Ddum7x8uI95b9ye1/Xy5noPEm8cD+hplnpU+PBQZJ5XJ2I+
    1l9Nixx47wPGXeClLqcdn0ayd+v+Rwf3/XUJrvccG2YZUiQ4jWZkoxsA07xx7Bj+
    Lt8/FKG7sHRFvePFU0ZS6JFx9GJqjSBbHRRkam+4emW3uWgVfZxuwcUCn1ayNgRt
    KiFv9jQrg2TIWEvzYx9tywTCxc+FFMWAlbCzi+m4WD+QUWWfDQ009U/WM0ks0Kww
    EwSk/UDuToxGnKU2dQARAQABiQEfBBgBAgAJBQJSNx7KAhsMAAoJENJ9ZmzYjkK0
    c3MIAIE9hAR20mqJWLcsxLtrRs6uNF1VrpB+4n/55QU7oxA1iVBO6IFu4qgsF12J
    TavnJ5MLaETlggXY+zDef9syTPXoQctpzcaNVDmedwo1SiL03uMoblOvWpMR/Y0j
    6rm7IgrMWUDXDPvoPGjMl2q1iTeyHkMZEyUJ8SKsaHh4jV9wp9KmC8C+9CwMukL7
    vM5w8cgvJoAwsp3Fn59AxWthN3XJYcnMfStkIuWgR7U2r+a210W6vnUxU4oN0PmM
    cursYPyeV0NX/KQeUeNMwGTFB6QHS/anRaGQewijkrYYoTNtfllxIu9XYmiBERQ/
    qPDlGRlOgVTd9xUfHFkzB52c70E=
    =92oX
    -----END PGP PUBLIC KEY BLOCK-----
bootcmd:
- cloud-init-per once ssh-users-ca echo "TrustedUserCAKeys /etc/ssh/users_ca.pub" >> /etc/ssh/sshd_config
- cloud-init-per once fallocate-swapfile fallocate -l 4G /swapfile
- cloud-init-per once chmod-swapfile chmod 600 /swapfile
- cloud-init-per once mkswap-swapfile mkswap /swapfile
- sysctl "vm.swappiness=0"
- swapon /swapfile

package_upgrade: true

packages:
- apache2-mpm-worker
- build-essential
- elasticsearch
- git
- graphviz
- libapache2-mod-wsgi-py3
- libevent-dev
- libfreetype6-dev
- libjpeg-dev
- liblcms2-dev
- libmagic-dev
- libpq-dev
- libssl-dev
- libtiff5-dev
- libwebp-dev
- libxml2-dev
- libxslt1-dev
- lzop
- ntp
- openjdk-8-jdk
- pv
- python2.7-dev
- python3.4-dev
- python-software-properties
- python-virtualenv
- ruby-dev
- unattended-upgrades
- update-notifier-common
- zlib1g-dev

power_state:
  mode: reboot

output:
  all: '| tee -a /var/log/cloud-init-output.log'

runcmd:
# Ideally this would build as a different user so clincoded only has read
# permissions
- set -ex
- echo "America/Los_Angeles" | sudo tee /etc/timezone
- sudo dpkg-reconfigure --frontend noninteractive tzdata
- sudo echo "deb https://apt-archive.postgresql.org/pub/repos/apt/ trusty-pgdg main" >> /etc/apt/sources.list.d/pgdg.list
- wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
- wget --quiet -O - https://packages.confluent.io/deb/4.1/archive.key | sudo apt-key add -
- sudo add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/4.1 stable main"
- sudo apt-get update
- sudo mv /etc/postgresql/9.4/main/recovery.conf /etc/postgresql/9.4/main/recovery-demo.conf
- sudo apt-get -y install postgresql-9.4
- curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash -
- sudo apt-get install -y nodejs
- update-rc.d elasticsearch defaults
- service elasticsearch start
- chown postgres:postgres /etc/postgresql/9.4/main/*.conf
- echo "include 'custom.conf'" >> /etc/postgresql/9.4/main/postgresql.conf
- if test "%(ROLE)s" = "demo"
- then
-   echo "standby_mode = off" >> /etc/postgresql/9.4/main/recovery-demo.conf
-   echo "include 'demo.conf'" >> /etc/postgresql/9.4/main/postgresql.conf
- fi
- sudo -u postgres createuser clincoded
- sudo -u postgres createdb --owner=clincoded clincoded
- mkdir /srv/clincoded
- chown clincoded:clincoded /srv/clincoded
- cd /srv/clincoded
- sudo -u clincoded git clone --no-checkout https://github.com/ClinGen/clincoded.git .
- sudo -u clincoded git checkout %(COMMIT)s
# Enable messaging/publication code (committed as comment in order to pass Trusty-based Travis build testing)
- sudo -u clincoded git checkout 46742b6326baa3db660d0cebd26e9eb8f4116f77 src/clincoded/__init__.py
- mkdir /opt/cloudwatchmon
- chown build:build /opt/cloudwatchmon
- sudo -u build virtualenv --python=python2.7 /opt/cloudwatchmon
- sudo -u build /opt/cloudwatchmon/bin/pip install -r cloudwatchmon-requirements.txt
- mkdir /opt/wal-e
- chown postgres:postgres /opt/wal-e
- sudo -u postgres virtualenv --python=python2.7 /opt/wal-e
- sudo -u postgres /opt/wal-e/bin/pip install -r wal-e-requirements.txt
# Restore DB
# (load data from s3)
- if test "%(BACKUPDB)s" = "s3_data"
- then
-   echo "====== restoring data from backup"
-   /etc/init.d/postgresql stop
-   sudo -u postgres /opt/wal-e/bin/wal-e --aws-instance-profile --s3-prefix="$(cat /etc/postgresql/9.4/main/wale_s3_prefix | tr -d "\n")" backup-fetch /var/lib/postgresql/9.4/main LATEST
-   sudo -u postgres ln -s /etc/postgresql/9.4/main/recovery-demo.conf /var/lib/postgresql/9.4/main/recovery.conf
-   /etc/init.d/postgresql start
# (load local test data plus all genes and diseases)
- elif test "%(BACKUPDB)s" = "test_gene_and_disease_data"
- then
-   echo "====== restoring local test data plus gene and disease data"
-   sudo -u clincoded psql clincoded < DEV_TEST_DB_DUMP_GENES_9.4
-   sudo -u clincoded psql -c 'update transactions set xid = NULL'
- else
# (load local test data [minimal])
-   echo "====== restoring local test data"
-   sudo -u clincoded psql clincoded < DEV_TEST_DB_DUMP_9.4
-   sudo -u clincoded psql -c 'update transactions set xid = NULL'
- fi
- sudo -u clincoded python3.4 bootstrap.py -v 2.9.5 --setuptools-version 15.2
- sudo -u clincoded LANG=en_US.UTF-8 bin/buildout -c %(ROLE)s.cfg
# Install Kafka client after buildout to avoid package version conflicts
- sudo apt-get -y install librdkafka-dev python-dev python3-pip
- sudo /usr/bin/pip3 install confluent-kafka==0.11.4
- until sudo -u postgres psql postgres -c ""; do sleep 10; done
- sudo -u clincoded sh -c 'cat /dev/urandom | head -c 256 | base64 > session-secret.b64'
- sudo -u clincoded bin/create-mapping production.ini --app-name app
- ln -s /srv/clincoded/etc/clincoded-apache.conf /etc/apache2/sites-available/clincoded.conf
- ln -s /srv/clincoded/etc/logging-apache.conf /etc/apache2/conf-available/logging.conf
- a2enmod headers
- a2enmod proxy_http
- a2enmod rewrite
- a2enmod ssl
- a2ensite clincoded.conf
- a2dissite 000-default
- a2enconf logging
- a2disconf charset
# - a2disconf javascript-common
- a2disconf localized-error-pages
- a2disconf other-vhosts-access-log
- a2disconf serve-cgi-bin
- if test "%(ROLE)s" = "candidate"
- then
-   sudo curl https://s3.amazonaws.com/aws-cloudwatch/downloads/latest/awslogs-agent-setup.py -O
-   sudo python ./awslogs-agent-setup.py -n --region us-west-2 -c /var/awslogs/etc/awslogsconf.conf
- fi
- if test "%(ROLE)s" = "demo"
- then
-   sudo -u clincoded bin/batchupgrade production.ini --app-name app
- fi

users:
- default
- name: build
  gecos: Build user
  inactive: true
  system: true
- name: clincoded
  gecos: ClinGen Gene Curation Database daemon user
  inactive: true
  system: true
  # Specified homedir must exist
  # https://github.com/rubygems/rubygems/issues/689
  homedir: /srv/clincoded

write_files:
- path: /etc/apt/apt.conf.d/20auto-upgrades
  content: |
    APT::Periodic::Update-Package-Lists "1";
    APT::Periodic::Unattended-Upgrade "1";

- path: /etc/apt/apt.conf.d/50unattended-upgrades
  content: |
    Unattended-Upgrade::Allowed-Origins {
        "${distro_id} ${distro_codename}-security";
    };
    Unattended-Upgrade::Automatic-Reboot "true";
    Unattended-Upgrade::Automatic-Reboot-Time "01:00";

- path: /etc/cron.d/cloudwatchmon
  content: |
    */5 * * * * nobody /opt/cloudwatchmon/bin/mon-put-instance-stats.py --mem-util --swap-util --disk-space-util --disk-path=/ --from-cron

- path: /etc/default/elasticsearch
  content: |
    # Use c4.8xlarge instance type for prod instance
    # Use c4.4xlarge instance type for test instance with prod data
    if test "%(BACKUPDB)s" = "s3_data"
    then
      ES_HEAP_SIZE=16g
    else
      ES_HEAP_SIZE=2g
    fi

- path: /var/awslogs/etc/awslogsconf.conf
  content: |
      [general]
      state_file = /var/awslogs/state/agent-state

      [/var/log/elasticsearch]
      file = /var/log/elasticsearch/elasticsearch.log
      buffer_duration = 5000
      log_stream_name = {hostname}
      initial_position = end_of_file
      log_group_name = Elasticsearch-prod

      [/var/log/apache2]
      file = /var/log/apache2/access.log
      buffer_duration = 5000
      log_stream_name = {hostname}
      initial_position = end_of_file
      log_group_name = Apache2-prod

      [/var/log/apache2]
      file = /var/log/apache2/error.log
      buffer_duration = 5000
      log_stream_name = {hostname}
      initial_position = end_of_file
      log_group_name = Apache2-error-log-prod

      [/var/log/postgresql]
      file = /var/log/postgresql/postgresql-9.4-main.log
      buffer_duration = 5000
      log_stream_name = {hostname}
      initial_position = end_of_file
      log_group_name = Postgresql-prod

- path: /etc/postgresql/9.4/main/custom.conf
  content: |
    hot_standby = on
    max_standby_archive_delay = -1
    wal_level = hot_standby
    archive_mode = on
    archive_timeout = 60
    # http://www.postgresql.org/message-id/CAOycyLTm6X3mVLz+sLCex+W==WSMgu9giteV7efPoPXYDhPtzQ@mail.gmail.com
    checkpoint_timeout = 1h

- path: /etc/postgresql/9.4/main/demo.conf
  content: |
    archive_mode = off
    hot_standby = off

- path: /etc/postgresql/9.4/main/master.conf
  content: |
    archive_command = '/opt/wal-e/bin/wal-e --aws-profile default --s3-prefix="$(cat /etc/postgresql/9.4/main/wale_s3_prefix | tr -d "\n")" wal-push "%%p"'

- path: /etc/postgresql/9.4/main/recovery.conf
  content: |
    # recovery.conf must be linked into data dir to do anything
    recovery_target_timeline = 'latest'
    restore_command = '/opt/wal-e/bin/wal-e --aws-instance-profile --s3-prefix="$(cat /etc/postgresql/9.4/main/wale_s3_prefix | tr -d "\n")" wal-fetch "%%f" "%%p"'
    standby_mode = on

- path: /etc/postgresql/9.4/main/wale_s3_prefix
  content: "%(WALE_S3_PREFIX)s"

- path: /etc/ssh/users_ca.pub
  content: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCmiM5UBd3Rant92xxhCVZFW+U+gUN3aLkICO1gzOGr/Ps173YgzgVPmdKdiI6vBzCZ8BXMG/aeiBHk2LKA+vFjh1/sFRA51nA+hnBzXuIbWYpsTHaGG3BFhnAP8tzDm/7MYRkIeXLwZRwTeFtrMd9MR/HGBVG5HmbM/jtrvTRWZVwFnXRxLQ3Rs5Y9v1QKOrZs4w5tt3iKBiBr+kJKhDHV5O8COowxjcfSqCZmfafVJQNR+8Dg6cvaizqY+ykHpgzc+a7oXJfo1LDDQELl0DGIPDIa340jMDjSSVV0o+RpjbIXTtH4m3TDpKRmZsTQrnHCMNSp5Uk7mMkhKwIwX1SP clincoded-dev@clinicalgenome.org