From dfe30833dee43cb73c6e155c4b61732d5284b693 Mon Sep 17 00:00:00 2001 From: ael-code Date: Fri, 26 Feb 2016 04:52:02 -0600 Subject: [PATCH 01/10] preparing json deconding for new Flask version related to #247 --- webant/api/users_api.py | 32 ++++++++++++++++++++------------ webant/api/util.py | 4 +++- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/webant/api/users_api.py b/webant/api/users_api.py index f9933f2..f6cb6fc 100644 --- a/webant/api/users_api.py +++ b/webant/api/users_api.py @@ -29,7 +29,8 @@ def delete_user(userID): @route('/users/', methods=['POST']) def add_user(): request.on_json_loading_failed = on_json_load_error - userData = request.json + userData = request.get_json() + # the next two lines should be removed when Flask version is >= 1.0 if not userData: raise ApiError("Unsupported media type", 415) name = userData.get('name', None) @@ -52,10 +53,12 @@ def add_user(): @route('/users/', methods=['PATCH']) def update_user(userID): request.on_json_loading_failed = on_json_load_error - if not request.json: + userData = request.get_json() + # the next two lines should be removed when Flask version is >= 1.0 + if not userData: raise ApiError("Unsupported media type", 415) try: - users.api.update_user(userID, request.json) + users.api.update_user(userID, userData) except users.api.NotFoundException, e: raise ApiError("Not found", 404, details=str(e)) return make_success_response("user has been successfully updated") @@ -82,7 +85,8 @@ def delete_group(groupID): @route('/groups/', methods=['POST']) def add_group(): request.on_json_loading_failed = on_json_load_error - groupData = request.json + groupData = request.get_json() + # the next two lines should be removed when Flask version is >= 1.0 if not groupData: raise ApiError("Unsupported media type", 415) name = groupData.get('name', None) @@ -102,10 +106,12 @@ def add_group(): @route('/groups/', methods=['PATCH']) def update_group(groupID): request.on_json_loading_failed = on_json_load_error - if not request.json: + groupData = request.get_json() + # the next two lines should be removed when Flask version is >= 1.0 + if not groupData: raise ApiError("Unsupported media type", 415) try: - users.api.update_group(groupID, request.json) + users.api.update_group(groupID, groupData) except users.api.NotFoundException, e: raise ApiError("Not found", 404, details=str(e)) return make_success_response("group has been successfully updated") @@ -171,7 +177,8 @@ def delete_capability(capID): @route('/capabilities/', methods=['POST']) def add_capability(): request.on_json_loading_failed = on_json_load_error - capData = request.json + capData = request.get_json() + # the next two lines should be removed when Flask version is >= 1.0 if not capData: raise ApiError("Unsupported media type", 415) domain = capData.get('domain', None) @@ -194,13 +201,14 @@ def add_capability(): @route('/capabilities/', methods=['PATCH']) def update_capability(capID): request.on_json_loading_failed = on_json_load_error - if not request.json: + capData = request.get_json() + # the next two lines should be removed when Flask version is >= 1.0 + if not capData: raise ApiError("Unsupported media type", 415) - updates = request.json - if 'actions' in updates: - updates['action'] = Action.from_list(updates.pop('actions')) + if 'actions' in capData: + capData['action'] = Action.from_list(capData.pop('actions')) try: - users.api.update_capability(capID, updates) + users.api.update_capability(capID, capData) except users.api.NotFoundException, e: raise ApiError("Not found", 404, details=str(e)) return make_success_response("capability has been successfully updated") diff --git a/webant/api/util.py b/webant/api/util.py index 91fb03e..02a4a43 100644 --- a/webant/api/util.py +++ b/webant/api/util.py @@ -1,4 +1,4 @@ -from flask import jsonify +from flask import jsonify, request class ApiError(Exception): @@ -14,6 +14,8 @@ def __str__(self): def on_json_load_error(e): + if request.mimetype != 'application/json': + raise ApiError("Wrong media type", 415, details="Expected 'application/json' encoded data" ) raise ApiError("Bad request", 400, details=str(e)) From 20890d2ee1ecc49d8a429416b075a521b424be89 Mon Sep 17 00:00:00 2001 From: ael-code Date: Fri, 11 Dec 2015 03:03:06 -0600 Subject: [PATCH 02/10] import werkzeug reloader from the original source run_with_reloader is going to be deprecated from werkzeug/serving --- webant/webserver_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webant/webserver_utils.py b/webant/webserver_utils.py index f5b5ca7..9da15aa 100644 --- a/webant/webserver_utils.py +++ b/webant/webserver_utils.py @@ -20,7 +20,7 @@ def run_server(): http_server.serve_forever() if app.config['DEBUG']: - from werkzeug.serving import run_with_reloader + from werkzeug._reloader import run_with_reloader run_with_reloader(run_server) else: run_server() From 9a74cdd1c33810db186e5503d2dafe5e14929f54 Mon Sep 17 00:00:00 2001 From: ael-code Date: Fri, 11 Dec 2015 03:06:31 -0600 Subject: [PATCH 03/10] changed default log level to INFO --- cli/agherant.py | 2 +- cli/libreant.py | 2 +- cli/libreant_db.py | 2 +- utils/loggers.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/agherant.py b/cli/agherant.py index 2e5edc6..a0f215f 100644 --- a/cli/agherant.py +++ b/cli/agherant.py @@ -28,7 +28,7 @@ def agherant(settings, debug, port, address, agherant_descriptions): if agherant_descriptions: cliConf['AGHERANT_DESCRIPTIONS'] = agherant_descriptions conf.update(cliConf) - initLoggers(logging.DEBUG if conf.get('DEBUG', False) else logging.WARNING) + initLoggers(logging.DEBUG if conf.get('DEBUG', False) else logging.INFO) try: main(conf) except Exception as e: diff --git a/cli/libreant.py b/cli/libreant.py index deb43c9..abf1d70 100644 --- a/cli/libreant.py +++ b/cli/libreant.py @@ -50,7 +50,7 @@ def libreant(settings, debug, port, address, fsdb_path, es_indexname, es_hosts, click.echo(json.dumps(conf, indent=3)) exit(0) - initLoggers(logging.DEBUG if conf.get('DEBUG', False) else logging.WARNING) + initLoggers(logging.DEBUG if conf.get('DEBUG', False) else logging.INFO) try: main(conf) except Exception as e: diff --git a/cli/libreant_db.py b/cli/libreant_db.py index 128312e..e5e961d 100644 --- a/cli/libreant_db.py +++ b/cli/libreant_db.py @@ -35,7 +35,7 @@ def libreant_db(debug, settings, fsdb_path, es_indexname, es_hosts): if es_hosts: cliConf['ES_HOSTS'] = es_hosts conf.update(cliConf) - initLoggers(logging.DEBUG if conf.get('DEBUG', False) else logging.WARNING) + initLoggers(logging.DEBUG if conf.get('DEBUG', False) else logging.INFO) try: global arc diff --git a/utils/loggers.py b/utils/loggers.py index 0b57b9b..8df4dd5 100644 --- a/utils/loggers.py +++ b/utils/loggers.py @@ -4,7 +4,7 @@ LOG_NAMES = ['webant', 'fsdb', 'presets', 'agherant', 'config_utils', 'libreantdb', 'archivant', 'users'] -def initLoggers(logLevel=logging.WARNING, logNames=LOG_NAMES): +def initLoggers(logLevel=logging.INFO, logNames=LOG_NAMES): streamHandler = logging.StreamHandler() formatter = logging.Formatter( '%(asctime)s [%(name)s] [%(levelname)s] %(message)s') From 528287066c875f5851df424d99ddaf65f8cfaa91 Mon Sep 17 00:00:00 2001 From: ael-code Date: Fri, 11 Dec 2015 02:50:11 -0600 Subject: [PATCH 04/10] added werkzeug to handled loggers --- utils/loggers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/loggers.py b/utils/loggers.py index 8df4dd5..a4f77db 100644 --- a/utils/loggers.py +++ b/utils/loggers.py @@ -1,7 +1,7 @@ import logging -LOG_NAMES = ['webant', 'fsdb', 'presets', 'agherant', 'config_utils', 'libreantdb', 'archivant', 'users'] +LOG_NAMES = ['webant', 'fsdb', 'presets', 'agherant', 'config_utils', 'libreantdb', 'archivant', 'users', 'werkzeug'] def initLoggers(logLevel=logging.INFO, logNames=LOG_NAMES): From 9f60475ec84557a1a1021d4c95ee5fcfb0c888db Mon Sep 17 00:00:00 2001 From: ael-code Date: Fri, 11 Dec 2015 03:07:34 -0600 Subject: [PATCH 05/10] use logger for printing address:port --- webant/webserver_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/webant/webserver_utils.py b/webant/webserver_utils.py index 9da15aa..5698782 100644 --- a/webant/webserver_utils.py +++ b/webant/webserver_utils.py @@ -13,9 +13,10 @@ def gevent_run(app): run_app = DebuggedApplication(app) def run_server(): + import logging port = int(app.config.get('PORT', 5000)) address = app.config.get('ADDRESS', '') - print('Listening on http://%s:%d/' % (address or '0.0.0.0', port)) + logging.getLogger('webant').info('Listening on http://{}:{}/'.format(address or '0.0.0.0', port)) http_server = WSGIServer((address, port), run_app) http_server.serve_forever() From 1ba0d9a4f445b68d19105f405be223e44eb524af Mon Sep 17 00:00:00 2001 From: ael-code Date: Sun, 28 Feb 2016 05:24:23 -0600 Subject: [PATCH 06/10] Use custom logger on gevent>=1.1 --- webant/webserver_utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/webant/webserver_utils.py b/webant/webserver_utils.py index 5698782..dc11cde 100644 --- a/webant/webserver_utils.py +++ b/webant/webserver_utils.py @@ -13,11 +13,17 @@ def gevent_run(app): run_app = DebuggedApplication(app) def run_server(): - import logging + from gevent import version_info + + logger = app._logger port = int(app.config.get('PORT', 5000)) address = app.config.get('ADDRESS', '') - logging.getLogger('webant').info('Listening on http://{}:{}/'.format(address or '0.0.0.0', port)) - http_server = WSGIServer((address, port), run_app) + logger.info('Listening on http://{}:{}/'.format(address or '0.0.0.0', port)) + server_params = dict() + #starting from gevent version 1.1b1 we can pass custom logger to gevent + if version_info[:2] >= (1,1): + server_params['log'] = logger + http_server = WSGIServer((address, port), run_app, **server_params) http_server.serve_forever() if app.config['DEBUG']: From 286a3ecbc245fe83b55039da519e105641ff0499 Mon Sep 17 00:00:00 2001 From: ael-code Date: Tue, 8 Mar 2016 12:33:52 +0100 Subject: [PATCH 07/10] fix gevent supported version to [1.0.1, 1.1.0] on setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 60fe534..8b16706 100644 --- a/setup.py +++ b/setup.py @@ -104,7 +104,7 @@ def read(fname): 'cli', 'conf'], install_requires=[ - 'gevent', + 'gevent >=1.0.1, <=1.1', # gevent version 1.0.0 do not support pyhton 2.7.8 https://github.com/gevent/gevent/issues/513 'elasticsearch >=1, <2', 'flask-bootstrap', 'Flask-Babel', From 9eb35aab25f9d4d8c10270cac85fb0bd943d57cc Mon Sep 17 00:00:00 2001 From: ael-code Date: Sun, 28 Feb 2016 04:35:52 -0600 Subject: [PATCH 08/10] CI: Test against different gevent versions we now test also the brand new gevent-1.1.0 so this commit fixes #259 --- .travis.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 27ef003..31aa69c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,14 +6,23 @@ python: services: - elasticsearch +env: + - VERSION_GEVENT="1.0.1" + - VERSION_GEVENT="1.0.2" + - VERSION_GEVENT="1.1.0" + +before_install: + - pip install --force gevent==$VERSION_GEVENT + + install: - - "pip install flake8 sphinx" - - "python setup.py install" + - pip install flake8 sphinx + - python setup.py install before_script: - sleep 10 # wait for elasticsearch script: - - 'flake8' + - flake8 - python setup.py test - - "python setup.py build_sphinx" + - python setup.py build_sphinx From 6fbb179ac4e58ebd7ed33e7a65b4f746760c2732 Mon Sep 17 00:00:00 2001 From: blallo Date: Fri, 11 Mar 2016 18:30:12 +0100 Subject: [PATCH 09/10] Minor doc change to warn about es2 issues --- doc/source/sysadmin.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/source/sysadmin.rst b/doc/source/sysadmin.rst index ceb9978..3c2085d 100644 --- a/doc/source/sysadmin.rst +++ b/doc/source/sysadmin.rst @@ -6,6 +6,9 @@ Installation System dependencies ^^^^^^^^^^^^^^^^^^^^ +.. note:: + In this moment we do *not* support elasticsearch v2. + There are plans to do it shortly! Debian wheezy / Debian jessie / Ubuntu ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -35,7 +38,15 @@ Arch Install all necessary packages:: - sudo pacman -Sy python2 python2-virtualenv elasticsearch + sudo pacman -Sy python2 python2-virtualenv + +And take care to install elasticsearch<2.x:: + + wget https://archive.archlinux.org/packages/e/elasticsearch/elasticsearch-1.7.3-1-x86_64.pkg.tar.xz + wget https://archive.archlinux.org/packages/e/elasticsearch/elasticsearch-1.7.3-1-x86_64.pkg.tar.xz.sig + sudo pacman-key --verify elasticsearch-1.7.3-1-x86_64.pkg.tar.xz elasticsearch-1.7.3-1-x86_64.pkg.tar.xz.sig + sudo pacman -U elasticsearch-1.7.3.1-x86_64.pkg.tar.xz + echo "IgnorePkg elasticsearch" | sudo tee -a /etc/pacman.conf Python dependencies ^^^^^^^^^^^^^^^^^^^^ From ae4cafdbdf79e71d784e64ffc5d45796ae2e8986 Mon Sep 17 00:00:00 2001 From: boyska Date: Tue, 16 Feb 2016 01:55:08 -0500 Subject: [PATCH 10/10] install docs updated to jessie and stretch --- doc/source/sysadmin.rst | 44 +++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/doc/source/sysadmin.rst b/doc/source/sysadmin.rst index 3c2085d..1c01342 100644 --- a/doc/source/sysadmin.rst +++ b/doc/source/sysadmin.rst @@ -1,31 +1,40 @@ Sysadmin ========= +.. highlight:: bash + Installation ------------- +We only maintain packages for debian stretch. They work fine in debian jessie, +too. Still, installing libreant is easy on most distributions. + System dependencies ^^^^^^^^^^^^^^^^^^^^ .. note:: In this moment we do *not* support elasticsearch v2. There are plans to do it shortly! -Debian wheezy / Debian jessie / Ubuntu +Debian jessie / Debian stretch ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. highlight:: bash -Download and install the Public Signing Key for elasticsearch repo:: +You need to add two custom repositories: one for elasticsearch, the other for +libreant itself:: + + wget -qO - 'http://packages.elasticsearch.org/GPG-KEY-elasticsearch' | sudo apt-key add - + wget -qO - 'http://deb.libreant.ga/repokey.gpg' | sudo apt-key add - + echo 'deb http://packages.elasticsearch.org/elasticsearch/1.7/debian stable main' | sudo tee '/etc/apt/sources.list.d/elasticsearch.list' + echo 'deb http://deb.libreant.ga/libreant-debian/ testing/' | sudo tee '/etc/apt/sources.list.d/libreant.list' - wget -qO - http://packages.elasticsearch.org/GPG-KEY-elasticsearch | sudo apt-key add - -Add elasticsearch repos in /etc/apt/sources.list.d/elasticsearch.list:: +If you are using debian jessie, you need to add stretch repositories:: - echo "deb http://packages.elasticsearch.org/elasticsearch/1.7/debian stable main" | sudo tee /etc/apt/sources.list.d/elasticsearch.list + echo "deb http://ftp.debian.org/debian stretch main" | sudo tee /etc/apt/sources.list.d/stretch.list -Install requirements:: +Install everything:: - sudo apt-get update && sudo apt-get install python2.7 gcc python2.7-dev python-virtualenv openjdk-7-jre-headless elasticsearch + sudo apt-get update && sudo apt-get install openjdk-7-jre-headless elasticsearch python-libreant .. note:: @@ -51,7 +60,7 @@ And take care to install elasticsearch<2.x:: Python dependencies ^^^^^^^^^^^^^^^^^^^^ -Create a virtual env:: +Clone libreant repository and create a virtual env:: virtualenv -p /usr/bin/python2 ve @@ -59,7 +68,7 @@ Install libreant and all python dependencies:: ./ve/bin/pip install libreant -Execution +Running ---------- Start elsticsearch @@ -78,8 +87,8 @@ Start elasticsearch service:: sudo update-rc.d elasticsearch defaults 95 10 -Arch / Debian jessie -~~~~~~~~~~~~~~~~~~~~ +Arch / Debian jessie / Debian stretch +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Start elasticsearch service:: @@ -90,11 +99,20 @@ Start elasticsearch service:: If you want to automatically start elasticsearch during bootup:: sudo systemctl enable elasticsearch + + These instructions apply both to arch and debian, but should not be needed + on debian: after the installation, elasticsearch is enabled and started + automatically. Start libreant ^^^^^^^^^^^^^^ -To execute libreant:: + +On debian stretch, the systemd unit ``libreant.service`` will take care +of starting libreant. You will find it automatically started and enabled after +the installation, so you have nothing to do. + +On debian wheezy and archlinux, this is what you need to run:: ./ve/bin/libreant