diff --git a/CHANGELOG.md b/CHANGELOG.md index af95680b..d90c44ef 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## Unreleased + +### Features +- Added new microservice_flask base image +- Added flask template + + ## 2.7.1 (2018-03-29) We do best effort to support docker versions 1.12.0 - 17.12.1 with this release. diff --git a/armada_command/command_create.py b/armada_command/command_create.py index 679111a5..406c6a65 100644 --- a/armada_command/command_create.py +++ b/armada_command/command_create.py @@ -12,7 +12,7 @@ def add_arguments(parser): parser.add_argument('name', help='Name of the created microservice.') parser.add_argument('-b', '--base-template', default='python3', - help='Base microservice template. Possible choices: python, python3, node') + help='Base microservice template. Possible choices: python, python3, node, flask') def _replace_in_file_content(file_path, old, new): diff --git a/docker-containers/microservice_flask/.dockerignore b/docker-containers/microservice_flask/.dockerignore new file mode 100644 index 00000000..46a5858a --- /dev/null +++ b/docker-containers/microservice_flask/.dockerignore @@ -0,0 +1,4 @@ +.git +**/*.BAK +**/*.pyc +example-rum-counter diff --git a/docker-containers/microservice_flask/.gitignore b/docker-containers/microservice_flask/.gitignore new file mode 100644 index 00000000..649ca9e9 --- /dev/null +++ b/docker-containers/microservice_flask/.gitignore @@ -0,0 +1,2 @@ +*.BAK +*.pyc diff --git a/docker-containers/microservice_flask/Dockerfile b/docker-containers/microservice_flask/Dockerfile new file mode 100644 index 00000000..8bfc2f9c --- /dev/null +++ b/docker-containers/microservice_flask/Dockerfile @@ -0,0 +1,20 @@ +FROM microservice +MAINTAINER Cerebro + +ENV MICROSERVICE_FLASK_APT_GET_UPDATE_DATE 2018-03-26 +RUN apt-get update + +RUN apt-get install -y python3 python3-dev python3-pip build-essential apache2 libapache2-mod-wsgi-py3 + +RUN pip3 install -U pip +RUN pip3 install -U requests armada Flask + +# Apache configuration. +ADD ./apache2_vhost.conf /etc/apache2/sites-available/apache2_vhost.conf +RUN ln -s /etc/apache2/sites-available/apache2_vhost.conf /etc/apache2/sites-enabled/apache2_vhost.conf +RUN rm -f /etc/apache2/sites-enabled/000-default.conf + +ADD ./supervisor/* /etc/supervisor/conf.d/ +ADD . /opt/microservice_flask + +EXPOSE 80 diff --git a/docker-containers/microservice_flask/README.md b/docker-containers/microservice_flask/README.md new file mode 100644 index 00000000..6171cd09 --- /dev/null +++ b/docker-containers/microservice_flask/README.md @@ -0,0 +1,76 @@ +# microservice-flask + +This is a base microservice image for Armada services written using [Flask](http://flask.pocoo.org/) python framework. + + +# Idea + +Main intent of this base image is to provide ability to run Flask app in concurrent way, e.g. in production environment. +This is achieved by running it under apache2 + mod_wsgi. + +Single threaded developer's environment may also be configured. + + +# Example + +We'll build example flask app [example-rum-counter](./example-rum-counter/) based on `coffee-counter` example +from Armada documentation. It has endpoint `/delay/{n}` which we can use to simulate long running request +that takes `n` seconds to complete. + + armada build microservice_flask + cd ./example-rum-counter + armada build example-rum-counter -d local + + # Run in development mode. + armada run example-rum-counter -d local -p 1129:80 --env dev + for i in `seq 7`; do curl http://localhost:1129/delay/5 2>&1 | grep rnd= & done + # ^ The results will be received one by one every 5 seconds. + + # Run in production mode (under apache2 with max. 4 concurrent requests). + armada run example-rum-counter -d local -p 2911:80 --env production + for i in `seq 11`; do curl http://localhost:2911/delay/5 2>&1 | grep rnd= & done + # ^ The results will be received in batches of 4 proving that they were run in concurrent fashion. + + +# Configuration + +`microservice_flask` reads its configuration by hermes from file `config.json`. +Following variables are supported: + +- `use_apache` (`false`/`true`, default: `false`) + + Whether to run the application under apache2+mod_wsgi or as a standalone flask application. + The latter is recommended during development. That mode has variable `FLASK_DEBUG` set to allow live reload + of the application code. You can then edit code & instantly refresh much like in PHP development model. + +- `apache_config` (dictionary) + + Dictionary of Apache variables that will be included in web server config. + Following variables are taken into account by `microservice_flask` itself: + + - `wsgi_worker_threads_count` (default: 17) + + Number of concurrent requests that can be served by the service at any one time. + + +# Flask app development + +`microservice_flask` looks for application to run in the folder `/opt/{MICROSERVICE_NAME}/src/`, +where `{MICROSERVICE_NAME}` is the value of the environment variable. +Thus, so far, the base image doesn't support container renaming, and the name of the service should match +the name of its image. +The main app file has to called `main.py`. + + +# Other + +The requests to Flask app are not timed-out by Apache, so there is a risk that all worker threads will hang +and the service will become unresponsive. Few of the possible solutions to this situation: + +* Move to Ubuntu 16.10 and install python 3.6 with newer libapache2-mod-wsgi-py3. +Then we could use `request-timeout` configuration option in `WSGIDaemonProcess`. + +* Write simple watchdog script, that will restart apache2 as soon as some health-check request takes longer +than previously set threshold. + +* Use nginx instead of apache2. diff --git a/docker-containers/microservice_flask/apache2_vhost.conf b/docker-containers/microservice_flask/apache2_vhost.conf new file mode 100644 index 00000000..564bc94c --- /dev/null +++ b/docker-containers/microservice_flask/apache2_vhost.conf @@ -0,0 +1,19 @@ +IncludeOptional /etc/apache2/defines*.conf + + + Define wsgi_worker_threads_count 17 + + + + ServerName localhost + + WSGIDaemonProcess flaskapp user=www-data group=www-data threads=${wsgi_worker_threads_count} + WSGIScriptAlias / /opt/microservice_flask/src/app.wsgi + + + WSGIProcessGroup flaskapp + WSGIApplicationGroup %{GLOBAL} + WSGIScriptReloading On + Require all granted + + diff --git a/docker-containers/microservice_flask/health-checks/http-ok b/docker-containers/microservice_flask/health-checks/http-ok new file mode 100755 index 00000000..c25e5165 --- /dev/null +++ b/docker-containers/microservice_flask/health-checks/http-ok @@ -0,0 +1,13 @@ +#!/bin/bash + +url=http://localhost +# -c /dev/null is for websites that redirect with new cookies set. +http_status_code=$(curl -sL -w "%{http_code}" -o /dev/null -c /dev/null ${url}) + +if [ ${http_status_code} -ne '200' ]; then + echo "HTTP health check failed" + exit 2 +fi + +echo "HTTP health check OK" +exit 0 diff --git a/docker-containers/microservice_flask/run_app.py b/docker-containers/microservice_flask/run_app.py new file mode 100644 index 00000000..5840d6eb --- /dev/null +++ b/docker-containers/microservice_flask/run_app.py @@ -0,0 +1,35 @@ +import os +from armada import hermes + + +def main(): + + config = hermes.get_config('config.json', {}) + + if not config.get('use_apache', False): + os.environ["FLASK_APP"] = "main.py" + os.environ["FLASK_DEBUG"] = "1" + + os.chdir("/opt/{0}/src".format(os.environ.get("MICROSERVICE_NAME", ""))) + command = "python3 -m flask run --port 80 --host 0.0.0.0" + args = command.split() + os.execvp(args[0], args) + + else: + with open("/etc/apache2/envvars", "a") as f: + for env_var in ['MICROSERVICE_NAME', 'MICROSERVICE_ENV', 'MICROSERVICE_APP_ID', 'CONFIG_PATH']: + f.write("export {env_var}=\"{env_value}\"\n".format(env_var=env_var, env_value=os.environ.get(env_var, ""))) + + apache_config = config.get('apache_config', {}) + if apache_config: + with open("/etc/apache2/defines.conf", "w") as f: + for k, v in apache_config.items(): + f.write("Define {key} {value}\n".format(key=k, value=v)) + + command = "service apache2 start" + args = command.split() + os.execvp(args[0], args) + + +if __name__ == '__main__': + main() diff --git a/docker-containers/microservice_flask/src/app.wsgi b/docker-containers/microservice_flask/src/app.wsgi new file mode 100644 index 00000000..4887ee23 --- /dev/null +++ b/docker-containers/microservice_flask/src/app.wsgi @@ -0,0 +1,7 @@ +import sys +import os + +microservice_name = os.environ.get('MICROSERVICE_NAME', 'microservice_flask') + +sys.path.insert(0, '/opt/{0}/src/'.format(microservice_name)) +from main import app as application diff --git a/docker-containers/microservice_flask/supervisor/run_app.conf b/docker-containers/microservice_flask/supervisor/run_app.conf new file mode 100644 index 00000000..3e7ab3e7 --- /dev/null +++ b/docker-containers/microservice_flask/supervisor/run_app.conf @@ -0,0 +1,2 @@ +[program:app] +command=python3 /opt/microservice_flask/run_app.py diff --git a/microservice_templates/microservice_flask_template/.dockerignore b/microservice_templates/microservice_flask_template/.dockerignore new file mode 100644 index 00000000..2d2ecd68 --- /dev/null +++ b/microservice_templates/microservice_flask_template/.dockerignore @@ -0,0 +1 @@ +.git/ diff --git a/microservice_templates/microservice_flask_template/.gitignore b/microservice_templates/microservice_flask_template/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/microservice_templates/microservice_flask_template/Dockerfile b/microservice_templates/microservice_flask_template/Dockerfile new file mode 100644 index 00000000..00e9cebb --- /dev/null +++ b/microservice_templates/microservice_flask_template/Dockerfile @@ -0,0 +1,10 @@ +FROM microservice_flask +MAINTAINER Cerebro + +ENV IMAGE_NAME=_MICROSERVICE_FLASK_TEMPLATE_ + +RUN pip3 install -U raven[flask] + +ADD . /opt/_MICROSERVICE_FLASK_TEMPLATE_ + +EXPOSE 80 diff --git a/microservice_templates/microservice_flask_template/README.md b/microservice_templates/microservice_flask_template/README.md new file mode 100644 index 00000000..e69de29b diff --git a/microservice_templates/microservice_flask_template/Vagrantfile b/microservice_templates/microservice_flask_template/Vagrantfile new file mode 100644 index 00000000..166012a9 --- /dev/null +++ b/microservice_templates/microservice_flask_template/Vagrantfile @@ -0,0 +1,8 @@ +require 'open-uri' +armada_vagrantfile_path = File.join(Dir.tmpdir, 'ArmadaVagrantfile.rb') +IO.write(armada_vagrantfile_path, open('http://vagrant.armada.sh/ArmadaVagrantfile.rb').read) +load armada_vagrantfile_path + +armada_vagrantfile( + :microservice_name => '_MICROSERVICE_FLASK_TEMPLATE_' +) diff --git a/microservice_templates/microservice_flask_template/config/dev/config.json b/microservice_templates/microservice_flask_template/config/dev/config.json new file mode 100644 index 00000000..3de262ec --- /dev/null +++ b/microservice_templates/microservice_flask_template/config/dev/config.json @@ -0,0 +1,3 @@ +{ + "sentry_url": "" +} diff --git a/microservice_templates/microservice_flask_template/config/production/config.json b/microservice_templates/microservice_flask_template/config/production/config.json new file mode 100644 index 00000000..05e87e95 --- /dev/null +++ b/microservice_templates/microservice_flask_template/config/production/config.json @@ -0,0 +1,8 @@ +{ + "sentry_url": "", + "use_apache": true, + "apache_config": + { + "wsgi_worker_threads_count": 4 + } +} diff --git a/microservice_templates/microservice_flask_template/run_app.py b/microservice_templates/microservice_flask_template/run_app.py new file mode 100644 index 00000000..5840d6eb --- /dev/null +++ b/microservice_templates/microservice_flask_template/run_app.py @@ -0,0 +1,35 @@ +import os +from armada import hermes + + +def main(): + + config = hermes.get_config('config.json', {}) + + if not config.get('use_apache', False): + os.environ["FLASK_APP"] = "main.py" + os.environ["FLASK_DEBUG"] = "1" + + os.chdir("/opt/{0}/src".format(os.environ.get("MICROSERVICE_NAME", ""))) + command = "python3 -m flask run --port 80 --host 0.0.0.0" + args = command.split() + os.execvp(args[0], args) + + else: + with open("/etc/apache2/envvars", "a") as f: + for env_var in ['MICROSERVICE_NAME', 'MICROSERVICE_ENV', 'MICROSERVICE_APP_ID', 'CONFIG_PATH']: + f.write("export {env_var}=\"{env_value}\"\n".format(env_var=env_var, env_value=os.environ.get(env_var, ""))) + + apache_config = config.get('apache_config', {}) + if apache_config: + with open("/etc/apache2/defines.conf", "w") as f: + for k, v in apache_config.items(): + f.write("Define {key} {value}\n".format(key=k, value=v)) + + command = "service apache2 start" + args = command.split() + os.execvp(args[0], args) + + +if __name__ == '__main__': + main() diff --git a/microservice_templates/microservice_flask_template/src/main.py b/microservice_templates/microservice_flask_template/src/main.py new file mode 100644 index 00000000..bbf83a9b --- /dev/null +++ b/microservice_templates/microservice_flask_template/src/main.py @@ -0,0 +1,26 @@ +from armada import hermes + +import logging +from flask import Flask + +from raven.contrib.flask import Sentry + +config = hermes.get_config('config.json', {}) + +formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(name)s: %(message)s') + +handler = logging.StreamHandler() +handler.setLevel(logging.INFO) +handler.setFormatter(formatter) + +logging.getLogger('werkzeug').handlers = [] +logging.getLogger('werkzeug').addHandler(handler) + +app = Flask(__name__) + +sentry = Sentry(app, dsn=config.get('sentry_url')) + +@app.route('/') +def status(): + return "OKej" + diff --git a/microservice_templates/microservice_flask_template/src/run-app-win32.cmd b/microservice_templates/microservice_flask_template/src/run-app-win32.cmd new file mode 100644 index 00000000..e15d0a6f --- /dev/null +++ b/microservice_templates/microservice_flask_template/src/run-app-win32.cmd @@ -0,0 +1,3 @@ +set FLASK_APP=main.py +set FLASK_DEBUG=1 +start python -m flask run