From a9c89f836c5305f62774b92748035052bd79d91b Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 22 Feb 2022 18:20:59 +0100 Subject: [PATCH 01/58] Update python version to 3.9 (cherry picked from commit 75692e94f67edbf257907204839fd70f22258ca8) --- docker/lifemonitor.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/lifemonitor.Dockerfile b/docker/lifemonitor.Dockerfile index e444d0cd4..d1a747b05 100644 --- a/docker/lifemonitor.Dockerfile +++ b/docker/lifemonitor.Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7-buster as base +FROM python:3.9-buster as base # Install base requirements RUN apt-get update -q \ From e4fda47ff1054a1f99529ef9e1d3babcaf49d5bd Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 23 Feb 2022 13:13:17 +0000 Subject: [PATCH 02/58] Update Flask to version 2.0.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 14b532e83..f02af0fa4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ Flask-APScheduler==1.12.2 Flask-SQLAlchemy==2.5.1 Flask-Migrate==3.1.0 Flask-Mail~=0.9.1 -Flask>=1.1.4,<2.0.0 +Flask~=2.0.3 gunicorn~=20.1.0 jwt==1.2.0 loginpass==0.5 From 918cabbd92a8b988c59698ea7d775cc845f54e83 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 23 Feb 2022 13:15:12 +0000 Subject: [PATCH 03/58] Move to connexion >= 2.11.* (Flask 2 compatible) --- requirements.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index f02af0fa4..f19cd9e60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,4 @@ -Authlib~=0.15.3 -apscheduler==3.8.0 -connexion[swagger-ui]==2.9.0 -dramatiq[redis,watch]==1.11.0 +connexion[swagger-ui]~=2.11.2 email-validator~=1.1.3 flask-bcrypt==0.7.1 flask-cors==3.0.10 From 1c97bf07e41a689b72ed841ee88bb58c6ab15530 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 23 Feb 2022 13:17:52 +0000 Subject: [PATCH 04/58] Bump 'flask-wtf' version to 1.0.0 --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f19cd9e60..dde560d93 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,8 +6,7 @@ flask-marshmallow~=0.14.0 flask-restful==0.3.9 flask-login~=0.5.0 flask-shell-ipython==0.4.1 -flask-wtf~=0.15.1 -Flask-APScheduler==1.12.2 +flask-wtf~=1.0.0 Flask-SQLAlchemy==2.5.1 Flask-Migrate==3.1.0 Flask-Mail~=0.9.1 From dc5f0e38dc1e6d38461b9551d2a36b1af1f0dc37 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 23 Feb 2022 13:20:15 +0000 Subject: [PATCH 05/58] Bump 'redis' version to 4.1.4 --- requirements.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index dde560d93..b310144f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,8 +26,4 @@ python-redis-lock~=3.7.0 PyGithub~=1.55 PyYAML~=5.4.1 pika~=1.2.0 -redis~=3.5.3 -requests~=2.26.0 -rocrate==0.5.2 -SQLAlchemy~=1.3.23 -wheel~=0.37.0 +redis~=4.1.4 From f4df8866d5bb588662e33571e98b593a3b3df442 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 23 Feb 2022 13:25:37 +0000 Subject: [PATCH 06/58] Add maintenance updates of libraries for async tasks --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index b310144f5..134613679 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ +apscheduler==3.8.1 connexion[swagger-ui]~=2.11.2 +dramatiq[redis,watch]==1.12.3 email-validator~=1.1.3 flask-bcrypt==0.7.1 flask-cors==3.0.10 @@ -7,6 +9,7 @@ flask-restful==0.3.9 flask-login~=0.5.0 flask-shell-ipython==0.4.1 flask-wtf~=1.0.0 +Flask-APScheduler==1.12.3 Flask-SQLAlchemy==2.5.1 Flask-Migrate==3.1.0 Flask-Mail~=0.9.1 From a97718e4b450c83f190d7b338cc4ca46fd3ab22a Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 23 Feb 2022 13:28:00 +0000 Subject: [PATCH 07/58] Add maintenance updates of auth libraries --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 134613679..473242108 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +Authlib~=0.15.4 apscheduler==3.8.1 connexion[swagger-ui]~=2.11.2 dramatiq[redis,watch]==1.12.3 @@ -15,7 +16,7 @@ Flask-Migrate==3.1.0 Flask-Mail~=0.9.1 Flask~=2.0.3 gunicorn~=20.1.0 -jwt==1.2.0 +jwt==1.3.1 loginpass==0.5 marshmallow-sqlalchemy~=0.26.1 prometheus-flask-exporter>=0.18,<0.19 From 139cea7bda6607f46069cb7fa76dcd26d6786085 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 23 Feb 2022 13:33:07 +0000 Subject: [PATCH 08/58] Bump 'pyopenssl' version to 22.0 --- requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 473242108..b2afe7458 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,9 +21,7 @@ loginpass==0.5 marshmallow-sqlalchemy~=0.26.1 prometheus-flask-exporter>=0.18,<0.19 psycopg2~=2.9.1 -pyopenssl==21.0.0 -pytest-mock~=3.6.1 -pytest~=6.2.5 +pyopenssl==22.0.0 python-dotenv~=0.19.0 python-jenkins==1.7.0 python-redis-lock~=3.7.0 From 605a52840b1f0fb69aee75608b0acb237ed6aa6b Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 23 Feb 2022 13:33:35 +0000 Subject: [PATCH 09/58] Bump pytest version to 7.0.1 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index b2afe7458..7e9e09bb7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,7 @@ marshmallow-sqlalchemy~=0.26.1 prometheus-flask-exporter>=0.18,<0.19 psycopg2~=2.9.1 pyopenssl==22.0.0 +pytest~=7.0.1 python-dotenv~=0.19.0 python-jenkins==1.7.0 python-redis-lock~=3.7.0 From c068a3a28418407729a4bffa0e0de28061222ea5 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 23 Feb 2022 13:36:32 +0000 Subject: [PATCH 10/58] Replace outdated/unmaintained 'flask-bcrypt' with Bcrypt-Flask --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7e9e09bb7..988eb5fc4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ apscheduler==3.8.1 connexion[swagger-ui]~=2.11.2 dramatiq[redis,watch]==1.12.3 email-validator~=1.1.3 -flask-bcrypt==0.7.1 +Bcrypt-Flask==1.0.2 flask-cors==3.0.10 flask-marshmallow~=0.14.0 flask-restful==0.3.9 From 0c0a46c8e5b3aab9b101f554d68389b3cb211887 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 23 Feb 2022 13:37:11 +0000 Subject: [PATCH 11/58] Bump 'ro-crate-py' version to 0.5.5 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 988eb5fc4..e8d50870d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,3 +30,4 @@ PyGithub~=1.55 PyYAML~=5.4.1 pika~=1.2.0 redis~=4.1.4 +rocrate==0.5.5 From c943a4af6ab3724853b9f90b855b81f1822589bb Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 23 Feb 2022 13:41:01 +0000 Subject: [PATCH 12/58] Add maintenance updates of basic libraries --- requirements.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e8d50870d..39dff6a69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,12 +16,14 @@ Flask-Migrate==3.1.0 Flask-Mail~=0.9.1 Flask~=2.0.3 gunicorn~=20.1.0 +itsdangerous~=2.1.0 jwt==1.3.1 loginpass==0.5 -marshmallow-sqlalchemy~=0.26.1 +marshmallow-sqlalchemy~=0.27.0 prometheus-flask-exporter>=0.18,<0.19 psycopg2~=2.9.1 pyopenssl==22.0.0 +pytest-mock~=3.7.0 pytest~=7.0.1 python-dotenv~=0.19.0 python-jenkins==1.7.0 @@ -30,4 +32,7 @@ PyGithub~=1.55 PyYAML~=5.4.1 pika~=1.2.0 redis~=4.1.4 +requests~=2.27.1 rocrate==0.5.5 +SQLAlchemy~=1.3.24 +wheel~=0.37.1 From 972296f689b8d765a985e20c41e7a148872f771c Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 12:10:00 +0100 Subject: [PATCH 13/58] Remove obsolete username param --- lifemonitor/commands/api_key.py | 50 ++++++++++++++++----------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/lifemonitor/commands/api_key.py b/lifemonitor/commands/api_key.py index d61ee7356..1f779f030 100644 --- a/lifemonitor/commands/api_key.py +++ b/lifemonitor/commands/api_key.py @@ -35,16 +35,16 @@ @blueprint.cli.command('create') -@click.argument("username") -@click.option("--scope", "scope", # type=click.Choice(ApiKey.SCOPES), +@click.option("--scope", "scope", default="read", show_default=True) @click.option("--length", "length", default=40, type=int, show_default=True) @with_appcontext -def api_key_create(username, scope="read", length=40): +def api_key_create(scope="read", length=40): """ - Create an API Key for a given user (identified by username) + Create an API Key for the 'admin' user """ - logger.debug("Finding User '%s'...", username) + username = "admin" + logger.debug("Finding user '%s'...", username) user = User.find_by_username(username) if not user: print("User not found", file=sys.stderr) @@ -52,25 +52,25 @@ def api_key_create(username, scope="read", length=40): logger.debug("User found: %r", user) api_key = generate_new_api_key(user, scope, length) print("%r" % api_key) - logger.debug("ApiKey created") + logger.debug("Api key created") @blueprint.cli.command('list') -@click.argument("username") @with_appcontext -def api_key_list(username): +def api_key_list(): """ - Create an API Key for a given user (identified by username) + Create an API Key for the 'admin' user """ - logger.debug("Finding User '%s'...", username) + username = "admin" + logger.debug("Finding user '%s'...", username) user = User.find_by_username(username) if not user: print("User not found", file=sys.stderr) sys.exit(99) logger.debug("User found: %r", user) - logger.info('-' * 82) - logger.info("User '%s' ApiKeys", user.username) - logger.info('-' * 82) + print('-' * 82) + print("Api keys of user '%s'" % user.username) + print('-' * 82) for key in user.api_keys: print(key) @@ -80,27 +80,27 @@ def api_key_list(username): @with_appcontext def api_key_delete(api_key): """ - Create an API Key for a given user (identified by username) + Create an API Key for the 'admin' user """ - logger.debug("Finding ApiKey '%s'...", api_key) + logger.debug("Finding Api key '%s'...", api_key) key = ApiKey.find(api_key) if not key: - print("ApiKey not found", file=sys.stderr) + print("Api key not found", file=sys.stderr) sys.exit(99) - logger.debug("ApiKey found: %r", key) + logger.debug("Api key found: %r", key) key.delete() - print("ApiKey '%s' deleted!" % api_key) - logger.debug("ApiKey created") + print("Api key '%s' deleted!" % api_key) + logger.debug("Api key created") @blueprint.cli.command('clean') -@click.argument("username") @with_appcontext -def api_key_clean(username): +def api_key_clean(): """ - Create an API Key for a given user (identified by username) + Create an API Key for the 'admin' user """ - logger.debug("Finding User '%s'...", username) + username = "admin" + logger.debug("Finding user '%s'...", username) user = User.find_by_username(username) if not user: print("User not found", file=sys.stderr) @@ -109,7 +109,7 @@ def api_key_clean(username): count = 0 for key in user.api_keys: key.delete() - print("ApiKey '%s' deleted!" % key.key) + print("Api key '%s' deleted!" % key.key) count += 1 print("%d ApiKeys deleted!" % count, file=sys.stderr) - logger.debug("ApiKeys of User '%s' deleted!", user.username) + logger.debug("ApiKeys of user '%s' deleted!", user.username) From a0933f57a37f7027a7b35ae7c41be6431e034ac7 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 12:14:21 +0100 Subject: [PATCH 14/58] Define help of CLI commands --- lifemonitor/commands/api_key.py | 3 +++ lifemonitor/commands/cache.py | 2 ++ lifemonitor/commands/db.py | 4 ++-- lifemonitor/commands/oauth.py | 2 ++ lifemonitor/commands/registry.py | 3 +++ lifemonitor/commands/tasks.py | 3 +++ 6 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lifemonitor/commands/api_key.py b/lifemonitor/commands/api_key.py index 1f779f030..d6c77d6fc 100644 --- a/lifemonitor/commands/api_key.py +++ b/lifemonitor/commands/api_key.py @@ -33,6 +33,9 @@ # define the blueprint for DB commands blueprint = Blueprint('api-key', __name__) +# set CLI help +blueprint.cli.help = "Manage admin API keys" + @blueprint.cli.command('create') @click.option("--scope", "scope", diff --git a/lifemonitor/commands/cache.py b/lifemonitor/commands/cache.py index 36180b668..2e6cc7ef9 100644 --- a/lifemonitor/commands/cache.py +++ b/lifemonitor/commands/cache.py @@ -29,6 +29,8 @@ # define the blueprint for DB commands blueprint = Blueprint('cache', __name__) +# set help for the CLI command +blueprint.cli.help = "Manage cache" @blueprint.cli.command('clear') @with_appcontext diff --git a/lifemonitor/commands/db.py b/lifemonitor/commands/db.py index d4114f424..5d9034d61 100644 --- a/lifemonitor/commands/db.py +++ b/lifemonitor/commands/db.py @@ -31,8 +31,8 @@ # set module level logger logger = logging.getLogger() -# define the blueprint for DB commands -blueprint = Blueprint('init', __name__) +# update help for the DB command +cli.db.help = "Manage database" # set initial revision number initial_revision = '8b2e530dc029' diff --git a/lifemonitor/commands/oauth.py b/lifemonitor/commands/oauth.py index 04692e6e7..8a479c054 100644 --- a/lifemonitor/commands/oauth.py +++ b/lifemonitor/commands/oauth.py @@ -33,6 +33,8 @@ # define the blueprint for DB commands blueprint = Blueprint('oauth', __name__) +# set CLI help +blueprint.cli.help = "Manage credentials for OAuth2 clients" def invalidate_token(token): invalid_token = token.copy() diff --git a/lifemonitor/commands/registry.py b/lifemonitor/commands/registry.py index 40ab39df8..b01a3f515 100644 --- a/lifemonitor/commands/registry.py +++ b/lifemonitor/commands/registry.py @@ -33,6 +33,9 @@ # define the blueprint for DB commands blueprint = Blueprint('registry', __name__) +# set CLI help +blueprint.cli.help = "Manage workflow registries" + # instance of LifeMonitor service lm = LifeMonitor.get_instance() diff --git a/lifemonitor/commands/tasks.py b/lifemonitor/commands/tasks.py index ef35a4a96..b2ab57823 100644 --- a/lifemonitor/commands/tasks.py +++ b/lifemonitor/commands/tasks.py @@ -29,6 +29,9 @@ # define the blueprint for DB commands blueprint = Blueprint('task-queue', __name__) +# set CLI help +blueprint.cli.help = "Manage task queue" + @blueprint.cli.command('reset') @with_appcontext From ea6a2179d31731490dcf8c013f52a8bca44db836 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 12:17:45 +0100 Subject: [PATCH 15/58] Redefine a global DB command --- lifemonitor/commands/db.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/lifemonitor/commands/db.py b/lifemonitor/commands/db.py index 5d9034d61..0312df501 100644 --- a/lifemonitor/commands/db.py +++ b/lifemonitor/commands/db.py @@ -20,17 +20,22 @@ import logging +import os +import sys +from datetime import datetime import click from flask import current_app -from flask.blueprints import Blueprint from flask.cli import with_appcontext -from flask_migrate import current, stamp, upgrade +from flask_migrate import cli, current, stamp, upgrade from lifemonitor.auth.models import User # set module level logger logger = logging.getLogger() +# export from this module +commands = [cli.db] + # update help for the DB command cli.db.help = "Manage database" @@ -38,12 +43,12 @@ initial_revision = '8b2e530dc029' -@blueprint.cli.command('db') +@cli.db.command() @click.option("-r", "--revision", default="head") @with_appcontext -def init_db(revision): +def init(revision): """ - Initialize LifeMonitor App + Initialize app database """ from lifemonitor.db import create_db, db, db_initialized, db_revision @@ -77,11 +82,11 @@ def init_db(revision): db.session.commit() -@blueprint.cli.command('wait-for-db') +@cli.db.command() @with_appcontext def wait_for_db(): """ - Wait until that DB is initialized + Wait until that DBMS service is up and running """ from lifemonitor.db import db_initialized, db_revision From 9163cbe38801fe01eed0bb989e0867d3d470cfea Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 12:19:03 +0100 Subject: [PATCH 16/58] Add commands to backup/restore DB --- lifemonitor/commands/db.py | 82 ++++++++++++++++++++++++++++++++++++++ lifemonitor/db.py | 20 ++++++++-- 2 files changed, 99 insertions(+), 3 deletions(-) diff --git a/lifemonitor/commands/db.py b/lifemonitor/commands/db.py index 0312df501..ef3dbce53 100644 --- a/lifemonitor/commands/db.py +++ b/lifemonitor/commands/db.py @@ -99,3 +99,85 @@ def wait_for_db(): while current_revision is None: current_revision = db_revision() logger.info(f"Current revision: {current_revision}") + + +@cli.db.command() +@click.option("-f", "--file", default=None, help="Filename (default hhmmss_yyyymmdd.tar") +@with_appcontext +def backup(file): + """ + Make a backup of the current app database + """ + from lifemonitor.db import db_connection_params + params = db_connection_params() + if not file: + file = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}.tar" + cmd = f"PGPASSWORD={params['password']} pg_dump -h {params['host']} -U {params['user']} -F t {params['dbname']} > {file}" + os.system(cmd) + msg = f"Created backup of database {params['dbname']} on {file}" + logger.debug(msg) + print(msg) + + +@cli.db.command() +@click.argument("file") +@click.option("-s", "--safe", default=False, is_flag=True, + help="Preserve the current database renaming it as '_yyyymmdd_hhmmss'") +@with_appcontext +def restore(file, safe=False): + """ + Restore a backup of the app database + """ + from lifemonitor.db import (create_db, db_connection_params, db_exists, + drop_db, rename_db) + params = db_connection_params() + db_copied = False + if db_exists(params['dbname']): + if safe: + answer = input(f"The database '{params['dbname']}' will be renamed. Continue? (y/n): ") + if not answer.lower() in ('y', 'yes'): + sys.exit(0) + new_db_name = f"{params['dbname']}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + rename_db(params['dbname'], new_db_name) + db_copied = True + logger.debug(f"Current database '{params['dbname']}' renamed as '{new_db_name}'") + else: + answer = input(f"The database '{params['dbname']}' will be delete. Continue? (y/n): ") + if not answer.lower() in ('y', 'yes'): + sys.exit(0) + drop_db() + logger.debug(f"Current database '{params['dbname']}' deleted") + create_db(current_app.config) + cmd = f"PGPASSWORD={params['password']} pg_restore -h {params['host']} -U {params['user']} -d {params['dbname']} -v {file}" + os.system(cmd) + if db_copied: + print(f"Existing database '{params['dbname']}' renamed as '{new_db_name}'") + msg = f"Backup {file} restored to database '{params['dbname']}'" + logger.debug(msg) + print(msg) + + +@cli.db.command() +@click.argument("snapshot", default="current") +@with_appcontext +def drop(snapshot): + """ + Drop (a snapshot of) the app database. + + A snapshot is specified by a datetime formatted as yyyymmdd_hhmmss: e.g., 20220324_100137. + + If no snaphot is provided the current app database will be removed. + """ + from lifemonitor.db import db_connection_params, drop_db + db_name = db_connection_params()['dbname'] + if snapshot and snapshot != "current": + if not snapshot.startswith(db_name): + db_name = f"{db_name}_{snapshot}" + else: + db_name = snapshot + answer = input(f"The database '{db_name}' will be removed. Are you sure? (y/n): ") + if answer.lower() in ('y', 'yes'): + drop_db(db_name=db_name) + print(f"Database '{db_name}' removed") + else: + print("Database deletion aborted") diff --git a/lifemonitor/db.py b/lifemonitor/db.py index 77411633d..1af238303 100644 --- a/lifemonitor/db.py +++ b/lifemonitor/db.py @@ -154,11 +154,25 @@ def create_db(settings=None, drop=False): logger.debug('DB %s created.', new_db_name.string) -def drop_db(settings=None): +def rename_db(old_name: str, new_name: str, settings=None): + + db.engine.dispose() + con = db_connect(settings=settings, override_db_name='postgres') + try: + con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + with con.cursor() as cur: + cur.execute(f'ALTER DATABASE {old_name} RENAME TO {new_name}') + finally: + con.close() + + logger.debug('DB %s renamed to %s.', old_name, new_name) + + +def drop_db(db_name: str=None, settings=None): """Clear existing data and create new tables.""" - actual_db_name = get_db_connection_param("POSTGRESQL_DATABASE", settings) + actual_db_name = db_name or get_db_connection_param("POSTGRESQL_DATABASE", settings) logger.debug("Actual DB name: %r", actual_db_name) - + db.engine.dispose() con = db_connect(settings=settings, override_db_name='postgres') try: con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) From 9563332002c01b71bda01046b330a014ffad882e Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 14:39:53 +0100 Subject: [PATCH 17/58] Fix missing param of 'drop_db' signature --- lifemonitor/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lifemonitor/db.py b/lifemonitor/db.py index 1af238303..94e784a97 100644 --- a/lifemonitor/db.py +++ b/lifemonitor/db.py @@ -168,7 +168,7 @@ def rename_db(old_name: str, new_name: str, settings=None): logger.debug('DB %s renamed to %s.', old_name, new_name) -def drop_db(db_name: str=None, settings=None): +def drop_db(db_name: str = None, settings=None): """Clear existing data and create new tables.""" actual_db_name = db_name or get_db_connection_param("POSTGRESQL_DATABASE", settings) logger.debug("Actual DB name: %r", actual_db_name) From 71a7686ddd7233c3606657b59015b316b3f80b13 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 14:43:03 +0100 Subject: [PATCH 18/58] Add getter for default config of current env --- lifemonitor/config.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lifemonitor/config.py b/lifemonitor/config.py index 24fca28ff..c5d008331 100644 --- a/lifemonitor/config.py +++ b/lifemonitor/config.py @@ -130,6 +130,17 @@ class TestingSupportConfig(TestingConfig): _config_by_name = {cfg.CONFIG_NAME: cfg for cfg in _EXPORT_CONFIGS} +def get_config(settings=None): + # set app env + app_env = os.environ.get("FLASK_ENV", "production") + if app_env != 'production': + # Set the DEBUG_METRICS env var to also enable the + # prometheus metrics exporter when running in development mode + os.environ['DEBUG_METRICS'] = 'true' + # load app config + return get_config_by_name(app_env, settings=settings) + + def get_config_by_name(name, settings=None): try: config = type(f"AppConfigInstance{name}".title(), (_config_by_name[name],), {}) From 97f851811a96286be96f81e55ffdbecca3740655 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 15:16:53 +0100 Subject: [PATCH 19/58] Add a configurable filter for logs --- lifemonitor/config.py | 36 +++++++++++++++++++++++++++++++++++- setup.cfg | 3 +++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/lifemonitor/config.py b/lifemonitor/config.py index c5d008331..a28c82c45 100644 --- a/lifemonitor/config.py +++ b/lifemonitor/config.py @@ -18,6 +18,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import configparser import logging import os from logging.config import dictConfig @@ -163,6 +164,32 @@ def get_config_by_name(name, settings=None): return ProductionConfig +class LogFilter(logging.Filter): + def __init__(self, param=None): + self.param = param + try: + config = configparser.ConfigParser() + config.read('setup.cfg') + self.filters = [_.strip() for _ in config.get('logging', 'filters').split(',')] + except Exception: + self.filters = [] + + def filter(self, record): + try: + filtered = False + for k in self.filters: + if k in record.name: + filtered = True + if not filtered: + if 'Requester' in record.name: + record.msg = "Request to Github API" + logger.debug("Request args: %r %r %r %r", record.args[0], record.args[1], record.args[2], record.args[3]) + record.args = None + except Exception as e: + logger.exception(e) + return not filtered + + def configure_logging(app): level_str = app.config.get('LOG_LEVEL', 'INFO') error = False @@ -177,10 +204,17 @@ def configure_logging(app): 'formatters': {'default': { 'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s', }}, + 'filters': { + 'myfilter': { + '()': LogFilter, + # 'param': '', + } + }, 'handlers': {'wsgi': { 'class': 'logging.StreamHandler', 'stream': 'ext://flask.logging.wsgi_errors_stream', - 'formatter': 'default' + 'formatter': 'default', + 'filters': ['myfilter'] }}, 'response': { 'level': logging.INFO, diff --git a/setup.cfg b/setup.cfg index 9d913ef01..78e3f2167 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,3 +6,6 @@ VCS = git style = pep440 versionfile_source = lifemonitor/_version.py tag_prefix = + +[logging] +filters = connexion,validator \ No newline at end of file From 6d97e031131a547dc6cda0890fd1d479bb856cf2 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 15:20:46 +0100 Subject: [PATCH 20/58] Add basic logging color formatter --- lifemonitor/config.py | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/lifemonitor/config.py b/lifemonitor/config.py index a28c82c45..9a6946b09 100644 --- a/lifemonitor/config.py +++ b/lifemonitor/config.py @@ -190,6 +190,44 @@ def filter(self, record): return not filtered +BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) + +# These are the sequences need to get colored ouput +RESET_SEQ = "\033[0m" +COLOR_SEQ = "\033[1;%dm" +BOLD_SEQ = "\033[1m" + + +def formatter_message(message, use_color=True): + if use_color: + message = message.replace("$RESET", RESET_SEQ).replace("$BOLD", BOLD_SEQ) + else: + message = message.replace("$RESET", "").replace("$BOLD", "") + return message + + +COLORS = { + 'WARNING': YELLOW, + 'INFO': WHITE, + 'DEBUG': BLUE, + 'CRITICAL': YELLOW, + 'ERROR': RED +} + + +class ColorFormatter(logging.Formatter): + def __init__(self, format, use_color=True): + logging.Formatter.__init__(self, format) + self.use_color = use_color + + def format(self, record): + levelname = record.levelname + if self.use_color and levelname in COLORS: + record.levelname = f"{COLOR_SEQ % (30 + COLORS[levelname])}{levelname}{RESET_SEQ}" + record.module = f"{COLOR_SEQ % (30 + COLORS[levelname])}{record.module}{RESET_SEQ}" + return logging.Formatter.format(self, record) + + def configure_logging(app): level_str = app.config.get('LOG_LEVEL', 'INFO') error = False @@ -202,7 +240,9 @@ def configure_logging(app): dictConfig({ 'version': 1, 'formatters': {'default': { - 'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s', + '()': ColorFormatter, + 'format': + f'[{COLOR_SEQ % (90)}%(asctime)s{RESET_SEQ}] %(levelname)s in %(module)s: {COLOR_SEQ % (90)}%(message)s{RESET_SEQ}', }}, 'filters': { 'myfilter': { From e54cf4719861b7d92620b00c5a1145c61028f253 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 15:25:05 +0100 Subject: [PATCH 21/58] Add manage.py script to the Docker image --- docker/lifemonitor.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/lifemonitor.Dockerfile b/docker/lifemonitor.Dockerfile index d1a747b05..202c7c9dd 100644 --- a/docker/lifemonitor.Dockerfile +++ b/docker/lifemonitor.Dockerfile @@ -60,7 +60,7 @@ RUN mkdir -p /var/data/lm \ USER lm # Copy lifemonitor app -COPY --chown=lm:lm app.py gunicorn.conf.py /lm/ +COPY --chown=lm:lm app.py manage.py gunicorn.conf.py /lm/ COPY --chown=lm:lm specs /lm/specs COPY --chown=lm:lm lifemonitor /lm/lifemonitor COPY --chown=lm:lm migrations /lm/migrations From b1caf3a641e03ca60cf0a3a965bf80a4efb08914 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 15:33:03 +0100 Subject: [PATCH 22/58] Ignore node_modules --- .gitignore | 1 + docker-compose.base.yml | 2 +- docker-compose.test.yml | 2 +- k8s/templates/backend-deployment.yaml | 2 +- k8s/templates/job-init.yaml | 2 +- k8s/templates/worker-deployment.yaml | 2 +- 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 5071ad7f7..d726e57aa 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ instance **/node_modules data lifemonitor/static/dist +lifemonitor/static/src/node_modules docker-compose.yml utils/certs/data tests/config/data/crates/*.zip diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 0965a24a8..553a6df6d 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -52,7 +52,7 @@ services: entrypoint: /bin/bash restart: "no" command: | - -c "wait-for-postgres.sh && flask init db" + -c "wait-for-postgres.sh && ./manage.py db init" depends_on: - "db" env_file: *env_file diff --git a/docker-compose.test.yml b/docker-compose.test.yml index d95648482..a04877991 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -10,7 +10,7 @@ services: restart: "no" entrypoint: /bin/bash command: | - -c "wait-for-postgres.sh && flask init db && /usr/local/bin/lm_entrypoint.sh" + -c "wait-for-postgres.sh && ./manage.py db init && /usr/local/bin/lm_entrypoint.sh" environment: - "FLASK_ENV=testingSupport" - "HOME=/lm" diff --git a/k8s/templates/backend-deployment.yaml b/k8s/templates/backend-deployment.yaml index 098bb3282..5a2117778 100644 --- a/k8s/templates/backend-deployment.yaml +++ b/k8s/templates/backend-deployment.yaml @@ -38,7 +38,7 @@ spec: image: {{ include "chart.lifemonitor.image" . }} imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} command: ["/bin/sh","-c"] - args: ["wait-for-redis.sh && wait-for-postgres.sh && flask init wait-for-db"] + args: ["wait-for-redis.sh && wait-for-postgres.sh && ./manage.py db wait-for-db"] env: {{- include "lifemonitor.common-env" . | nindent 12 }} volumeMounts: diff --git a/k8s/templates/job-init.yaml b/k8s/templates/job-init.yaml index ead9c3fc5..3be4e6941 100644 --- a/k8s/templates/job-init.yaml +++ b/k8s/templates/job-init.yaml @@ -20,7 +20,7 @@ spec: image: {{ include "chart.lifemonitor.image" . }} imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} command: ["/bin/sh","-c"] - args: ["wait-for-redis.sh && wait-for-postgres.sh && flask init db && flask task-queue reset"] + args: ["wait-for-redis.sh && wait-for-postgres.sh && ./manage.py db initt db && flask task-queue reset"] env: {{- include "lifemonitor.common-env" . | nindent 10 }} volumeMounts: diff --git a/k8s/templates/worker-deployment.yaml b/k8s/templates/worker-deployment.yaml index c0309e564..cf735fc15 100644 --- a/k8s/templates/worker-deployment.yaml +++ b/k8s/templates/worker-deployment.yaml @@ -35,7 +35,7 @@ spec: image: {{ include "chart.lifemonitor.image" . }} imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} command: ["/bin/sh","-c"] - args: ["wait-for-redis.sh && wait-for-postgres.sh && flask init wait-for-db"] + args: ["wait-for-redis.sh && wait-for-postgres.sh && ./manage.py db wait-for-db"] env: {{- include "lifemonitor.common-env" . | nindent 12 }} volumeMounts: From 626a71e7f30f3d5c9122a3934dc2d10dd48d33ec Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 15:36:10 +0100 Subject: [PATCH 23/58] Fix E302 (missing blank lines) --- lifemonitor/commands/cache.py | 1 + lifemonitor/commands/oauth.py | 1 + 2 files changed, 2 insertions(+) diff --git a/lifemonitor/commands/cache.py b/lifemonitor/commands/cache.py index 2e6cc7ef9..35d43a9af 100644 --- a/lifemonitor/commands/cache.py +++ b/lifemonitor/commands/cache.py @@ -32,6 +32,7 @@ # set help for the CLI command blueprint.cli.help = "Manage cache" + @blueprint.cli.command('clear') @with_appcontext def clear(): diff --git a/lifemonitor/commands/oauth.py b/lifemonitor/commands/oauth.py index 8a479c054..f69add16e 100644 --- a/lifemonitor/commands/oauth.py +++ b/lifemonitor/commands/oauth.py @@ -36,6 +36,7 @@ # set CLI help blueprint.cli.help = "Manage credentials for OAuth2 clients" + def invalidate_token(token): invalid_token = token.copy() invalid_token["expires_in"] = 10 From 7abc44f7abbcf205e30d92e35f49c42dd569c771 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 15:41:55 +0100 Subject: [PATCH 24/58] Add 'lftp' client to the base Docker image --- docker/lifemonitor.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/lifemonitor.Dockerfile b/docker/lifemonitor.Dockerfile index 202c7c9dd..e1e791fb9 100644 --- a/docker/lifemonitor.Dockerfile +++ b/docker/lifemonitor.Dockerfile @@ -3,7 +3,7 @@ FROM python:3.9-buster as base # Install base requirements RUN apt-get update -q \ && apt-get install -y --no-install-recommends \ - bash \ + bash lftp \ redis-tools \ postgresql-client-11 \ && apt-get clean -y && rm -rf /var/lib/apt/lists From b7886ff6e2d9ade660e082b9057e1e3508a4cae8 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 15:43:29 +0100 Subject: [PATCH 25/58] Fix missing manage.py script --- manage.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100755 manage.py diff --git a/manage.py b/manage.py new file mode 100755 index 000000000..aff50bcdc --- /dev/null +++ b/manage.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2020-2021 CRS4 +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from app import application + + +def main(): + application.cli.main() + + +if __name__ == '__main__': + main() From 07613cf0ccc51106cd84ec52437553efd3338894 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 15:59:09 +0100 Subject: [PATCH 26/58] Add --directory option to the backup command --- lifemonitor/commands/db.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lifemonitor/commands/db.py b/lifemonitor/commands/db.py index ef3dbce53..30159303b 100644 --- a/lifemonitor/commands/db.py +++ b/lifemonitor/commands/db.py @@ -102,9 +102,10 @@ def wait_for_db(): @cli.db.command() -@click.option("-f", "--file", default=None, help="Filename (default hhmmss_yyyymmdd.tar") +@click.option("-f", "--file", default=None, help="Backup filename (default 'hhmmss_yyyymmdd.tar')") +@click.option("-d", "--directory", default="./", help="Directory path for the backup file (default '.')") @with_appcontext -def backup(file): +def backup(file, directory): """ Make a backup of the current app database """ @@ -112,9 +113,10 @@ def backup(file): params = db_connection_params() if not file: file = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}.tar" - cmd = f"PGPASSWORD={params['password']} pg_dump -h {params['host']} -U {params['user']} -F t {params['dbname']} > {file}" + target_path = os.path.join(directory, file) + cmd = f"PGPASSWORD={params['password']} pg_dump -h {params['host']} -U {params['user']} -F t {params['dbname']} > {target_path}" os.system(cmd) - msg = f"Created backup of database {params['dbname']} on {file}" + msg = f"Created backup of database {params['dbname']} on {target_path}" logger.debug(msg) print(msg) From da4be6a484f26a6e3d2a93a692979cc414593594 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 16:24:07 +0100 Subject: [PATCH 27/58] Fix job init command --- k8s/templates/job-init.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/templates/job-init.yaml b/k8s/templates/job-init.yaml index 3be4e6941..857c7d94c 100644 --- a/k8s/templates/job-init.yaml +++ b/k8s/templates/job-init.yaml @@ -20,7 +20,7 @@ spec: image: {{ include "chart.lifemonitor.image" . }} imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} command: ["/bin/sh","-c"] - args: ["wait-for-redis.sh && wait-for-postgres.sh && ./manage.py db initt db && flask task-queue reset"] + args: ["wait-for-redis.sh && wait-for-postgres.sh && ./manage.py db init db && ./manage.py task-queue reset"] env: {{- include "lifemonitor.common-env" . | nindent 10 }} volumeMounts: From 347897af8feb057a66545c65093fcc592ff0eca1 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 17:23:15 +0100 Subject: [PATCH 28/58] Fix command of init job --- k8s/templates/job-init.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/templates/job-init.yaml b/k8s/templates/job-init.yaml index 857c7d94c..2429576a2 100644 --- a/k8s/templates/job-init.yaml +++ b/k8s/templates/job-init.yaml @@ -20,7 +20,7 @@ spec: image: {{ include "chart.lifemonitor.image" . }} imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} command: ["/bin/sh","-c"] - args: ["wait-for-redis.sh && wait-for-postgres.sh && ./manage.py db init db && ./manage.py task-queue reset"] + args: ["wait-for-redis.sh && wait-for-postgres.sh && ./manage.py db init && ./manage.py task-queue reset"] env: {{- include "lifemonitor.common-env" . | nindent 10 }} volumeMounts: From a0b456abd300827f4a6b8cae5b0452f0772818f8 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 17:25:14 +0100 Subject: [PATCH 29/58] Add k8s cronjob to schedule automatic backups --- k8s/pvc-backend-backup.yaml | 10 ++++++ k8s/templates/_helpers.tpl | 13 ++++++++ k8s/templates/job-backup.yaml | 58 +++++++++++++++++++++++++++++++++++ k8s/values.yaml | 16 ++++++++++ 4 files changed, 97 insertions(+) create mode 100644 k8s/pvc-backend-backup.yaml create mode 100644 k8s/templates/job-backup.yaml diff --git a/k8s/pvc-backend-backup.yaml b/k8s/pvc-backend-backup.yaml new file mode 100644 index 000000000..2dc8a2b83 --- /dev/null +++ b/k8s/pvc-backend-backup.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: data-api-backup +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi diff --git a/k8s/templates/_helpers.tpl b/k8s/templates/_helpers.tpl index 1468ad3a3..04ed70e42 100644 --- a/k8s/templates/_helpers.tpl +++ b/k8s/templates/_helpers.tpl @@ -132,3 +132,16 @@ Define mount points shared by some pods. - name: lifemonitor-data mountPath: "/var/data/lm" {{- end -}} + + +{{/* +Define command to mirror (cluster) local backup to a remote site via SFTP +*/}} +{{- define "backup.remote.command" -}} +{{- if and .Values.backup.remote .Values.backup.remote.enabled }} +{{- printf "lftp -c \"open -u %s,%s sftp://%s; mirror -e /var/data/backup %s \"" + .Values.backup.remote.user .Values.backup.remote.password + .Values.backup.remote.host .Values.backup.remote.path +}} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/k8s/templates/job-backup.yaml b/k8s/templates/job-backup.yaml new file mode 100644 index 000000000..a676e4a3b --- /dev/null +++ b/k8s/templates/job-backup.yaml @@ -0,0 +1,58 @@ +{{- if .Values.backup.enabled -}} +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: {{ include "chart.fullname" . }}-backup + labels: + app.kubernetes.io/name: {{ include "chart.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + schedule: "{{ .Values.backup.schedule }}" + successfulJobsHistoryLimit: {{ .Values.backup.successfulJobsHistoryLimit }} + failedJobsHistoryLimit: {{ .Values.backup.failedJobsHistoryLimit }} + jobTemplate: + spec: + template: + spec: + containers: + - name: lifemonitor-backup + image: {{ include "chart.lifemonitor.image" . }} + imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} + command: ["/bin/sh","-c"] + args: + - base_path="/var/data/backup" ; + db_backups_path="${base_path}/db" ; + crate_backups_path="${base_path}/crates" ; + mkdir -p ${db_backups_path} ; + mkdir -p ${crate_backups_path} ; + ./manage.py db backup -d ${db_backups_path} ; + find ${db_backups_path} -type f -mtime +60 -name '*.tar' -execdir rm -- '{}' \; ; + cp -urv /var/data/lm ${crate_backups_path} ; + {{ include "backup.remote.command" . }} + env: + {{- include "lifemonitor.common-env" . | nindent 12 }} + volumeMounts: + {{- include "lifemonitor.common-volume-mounts" . | nindent 12 }} + - name: lifemonitor-backup + mountPath: "/var/data/backup" + restartPolicy: OnFailure + volumes: + {{- include "lifemonitor.common-volume" . | nindent 10 }} + - name: lifemonitor-backup + persistentVolumeClaim: + claimName: {{ .Values.backup.existingClaim }} + {{- with .Values.lifemonitor.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.lifemonitor.affinity }} + affinity: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.lifemonitor.tolerations }} + tolerations: + {{- toYaml . | nindent 10 }} + {{- end }} + backoffLimit: 4 +{{- end }} \ No newline at end of file diff --git a/k8s/values.yaml b/k8s/values.yaml index 4a4fdd7ef..ef2fd0cf9 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -81,6 +81,22 @@ mail: ssl: true default_sender: "" +# Backup settings +backup: + enabled: false + schedule: "* 3 * * *" + successfulJobsHistoryLimit: 30 + failedJobsHistoryLimit: 30 + existingClaim: data-api-backup + # Settings to mirror the (cluster) local backup + # to a remote site via SFTP + remote: + enabled: false + user: username + password: password + host: 10.0.1.135 + path: /user/home/lm-backups + lifemonitor: replicaCount: 1 From 8d79c80466a04727c7429c005af1857311647a35 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 17:28:46 +0100 Subject: [PATCH 30/58] Bump app version number to 0.7.2 --- k8s/Chart.yaml | 2 +- lifemonitor/static/src/package.json | 2 +- specs/api.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/k8s/Chart.yaml b/k8s/Chart.yaml index 2e2097cce..4f8f3eb37 100644 --- a/k8s/Chart.yaml +++ b/k8s/Chart.yaml @@ -12,7 +12,7 @@ version: 0.7.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 0.7.1 +appVersion: 0.7.2 # Chart dependencies dependencies: diff --git a/lifemonitor/static/src/package.json b/lifemonitor/static/src/package.json index 1b2f658d9..f5d2cfb7d 100644 --- a/lifemonitor/static/src/package.json +++ b/lifemonitor/static/src/package.json @@ -1,7 +1,7 @@ { "name": "lifemonitor", "description": "Workflow Testing Service", - "version": "0.7.1", + "version": "0.7.2", "license": "MIT", "author": "CRS4", "main": "../dist/js/lifemonitor.min.js", diff --git a/specs/api.yaml b/specs/api.yaml index f340670a3..299a4d8b4 100644 --- a/specs/api.yaml +++ b/specs/api.yaml @@ -3,7 +3,7 @@ openapi: "3.0.0" info: - version: "0.7.1" + version: "0.7.2" title: "Life Monitor API" description: | *Workflow sustainability service* @@ -18,7 +18,7 @@ info: servers: - url: / description: > - Version 0.7.1 of API. + Version 0.7.2 of API. tags: - name: Registries From d6d679cb62d7e92afb91f80d66afb4b3169ba40c Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 24 Mar 2022 17:29:20 +0100 Subject: [PATCH 31/58] Bump chart version number to 0.8.0 --- k8s/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/Chart.yaml b/k8s/Chart.yaml index 4f8f3eb37..3475dc8ed 100644 --- a/k8s/Chart.yaml +++ b/k8s/Chart.yaml @@ -7,7 +7,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.7.0 +version: 0.8.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From c2b37fd14b589b5ba995a5231793be12da36c1d1 Mon Sep 17 00:00:00 2001 From: simleo Date: Fri, 25 Mar 2022 15:34:01 +0100 Subject: [PATCH 32/58] fix copyright boilerplate --- manage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manage.py b/manage.py index aff50bcdc..e1ac8cf80 100755 --- a/manage.py +++ b/manage.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright (c) 2020-2021 CRS4 +# Copyright (c) 2022 CRS4 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal From 663cd5c515c1ea398acd94401fa6a92be401022e Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 29 Mar 2022 15:33:33 +0200 Subject: [PATCH 33/58] Fix and update sync command with support for FTPs --- k8s/templates/_helpers.tpl | 11 ++++++++++- k8s/values.yaml | 4 +++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/k8s/templates/_helpers.tpl b/k8s/templates/_helpers.tpl index 04ed70e42..7187f27c3 100644 --- a/k8s/templates/_helpers.tpl +++ b/k8s/templates/_helpers.tpl @@ -139,9 +139,18 @@ Define command to mirror (cluster) local backup to a remote site via SFTP */}} {{- define "backup.remote.command" -}} {{- if and .Values.backup.remote .Values.backup.remote.enabled }} -{{- printf "lftp -c \"open -u %s,%s sftp://%s; mirror -e /var/data/backup %s \"" +{{- if eq (.Values.backup.remote.protocol | lower) "sftp" }} +{{- printf "lftp -c \"open -u %s,%s sftp://%s; mirror -e -R /var/data/backup %s \"" .Values.backup.remote.user .Values.backup.remote.password .Values.backup.remote.host .Values.backup.remote.path }} +{{- else if eq (.Values.backup.remote.protocol | lower) "ftps" }} +{{- printf "lftp -c \"%s %s open -u %s,%s ftp://%s; mirror -e -R /var/data/backup %s \"" + "set ftp:ssl-auth TLS; set ftp:ssl-force true;" + "set ftp:ssl-protect-list yes; set ftp:ssl-protect-data yes;" + .Values.backup.remote.user .Values.backup.remote.password + .Values.backup.remote.host .Values.backup.remote.path +}} +{{- end }} {{- end }} {{- end }} \ No newline at end of file diff --git a/k8s/values.yaml b/k8s/values.yaml index ef2fd0cf9..d725ac9bf 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -89,8 +89,10 @@ backup: failedJobsHistoryLimit: 30 existingClaim: data-api-backup # Settings to mirror the (cluster) local backup - # to a remote site via SFTP + # to a remote site via FTPS or SFTP remote: + # supported protocols ftps | sftp + protocol: ftps enabled: false user: username password: password From 05a955ba55e7965dd90ca2d02dd52374fa12e067 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 29 Mar 2022 15:35:39 +0200 Subject: [PATCH 34/58] Allow to configure the number of days to retain backups --- k8s/templates/job-backup.yaml | 5 +---- k8s/values.yaml | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/k8s/templates/job-backup.yaml b/k8s/templates/job-backup.yaml index a676e4a3b..45cbdfa8a 100644 --- a/k8s/templates/job-backup.yaml +++ b/k8s/templates/job-backup.yaml @@ -26,10 +26,7 @@ spec: crate_backups_path="${base_path}/crates" ; mkdir -p ${db_backups_path} ; mkdir -p ${crate_backups_path} ; - ./manage.py db backup -d ${db_backups_path} ; - find ${db_backups_path} -type f -mtime +60 -name '*.tar' -execdir rm -- '{}' \; ; - cp -urv /var/data/lm ${crate_backups_path} ; - {{ include "backup.remote.command" . }} + find ${db_backups_path} -type f -mtime +{{ .Values.backup.retain_days }} -name '*.tar' -execdir rm -- '{}' \; ; env: {{- include "lifemonitor.common-env" . | nindent 12 }} volumeMounts: diff --git a/k8s/values.yaml b/k8s/values.yaml index d725ac9bf..926ad8fe7 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -85,6 +85,7 @@ mail: backup: enabled: false schedule: "* 3 * * *" + retain_days: 30 successfulJobsHistoryLimit: 30 failedJobsHistoryLimit: 30 existingClaim: data-api-backup From 3a72fa2e633938d929053fa94f8df26a9cf9958c Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 29 Mar 2022 15:42:45 +0200 Subject: [PATCH 35/58] Catch, handle and report errors --- k8s/templates/job-backup.yaml | 12 +++++-- lifemonitor/commands/db.py | 63 ++++++++++++++++++++++++++++++----- 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/k8s/templates/job-backup.yaml b/k8s/templates/job-backup.yaml index 45cbdfa8a..4b9ce00b9 100644 --- a/k8s/templates/job-backup.yaml +++ b/k8s/templates/job-backup.yaml @@ -20,13 +20,21 @@ spec: image: {{ include "chart.lifemonitor.image" . }} imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} command: ["/bin/sh","-c"] - args: - - base_path="/var/data/backup" ; + args: + - wait-for-redis.sh && wait-for-postgres.sh ; + base_path="/var/data/backup" ; db_backups_path="${base_path}/db" ; crate_backups_path="${base_path}/crates" ; mkdir -p ${db_backups_path} ; mkdir -p ${crate_backups_path} ; + ./manage.py db backup -d ${db_backups_path} 2>&1 ; + exit_code = $?; + if [[ ${exit_code} == 0 ]]; then find ${db_backups_path} -type f -mtime +{{ .Values.backup.retain_days }} -name '*.tar' -execdir rm -- '{}' \; ; + cp -urv /var/data/lm/* ${crate_backups_path}/ ; + {{ include "backup.remote.command" . }} ; + else echo "Unable to complete the backup" ; exit ${exit_code} ; + fi ; env: {{- include "lifemonitor.common-env" . | nindent 12 }} volumeMounts: diff --git a/lifemonitor/commands/db.py b/lifemonitor/commands/db.py index 30159303b..b98fd028f 100644 --- a/lifemonitor/commands/db.py +++ b/lifemonitor/commands/db.py @@ -21,6 +21,7 @@ import logging import os +import subprocess import sys from datetime import datetime @@ -115,10 +116,18 @@ def backup(file, directory): file = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}.tar" target_path = os.path.join(directory, file) cmd = f"PGPASSWORD={params['password']} pg_dump -h {params['host']} -U {params['user']} -F t {params['dbname']} > {target_path}" - os.system(cmd) + result = subprocess.run(cmd, shell=True, capture_output=True) + logger.debug("Backup result: %r", result) + if result.returncode == 0: msg = f"Created backup of database {params['dbname']} on {target_path}" logger.debug(msg) print(msg) + else: + print("ERROR: Unable to backup the database") + if verbose and result.stderr: + print("ERROR [stderr]: %s" % result.stderr.decode()) + # report exit code to the main process + sys.exit(result.returncode) @cli.db.command() @@ -132,6 +141,13 @@ def restore(file, safe=False): """ from lifemonitor.db import (create_db, db_connection_params, db_exists, drop_db, rename_db) + + # check if DB file exists + if not os.path.isfile(file): + print("File '%s' not found!" % file) + sys.exit(128) + # check if delete or preserve the current app database (if exists) + new_db_name = None params = db_connection_params() db_copied = False if db_exists(params['dbname']): @@ -139,24 +155,53 @@ def restore(file, safe=False): answer = input(f"The database '{params['dbname']}' will be renamed. Continue? (y/n): ") if not answer.lower() in ('y', 'yes'): sys.exit(0) - new_db_name = f"{params['dbname']}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" - rename_db(params['dbname'], new_db_name) - db_copied = True - logger.debug(f"Current database '{params['dbname']}' renamed as '{new_db_name}'") else: answer = input(f"The database '{params['dbname']}' will be delete. Continue? (y/n): ") if not answer.lower() in ('y', 'yes'): sys.exit(0) - drop_db() - logger.debug(f"Current database '{params['dbname']}' deleted") + # create a snapshot of the current database + new_db_name = f"{params['dbname']}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + rename_db(params['dbname'], new_db_name) + db_copied = True + msg = f"Created a DB snapshot: data '{params['dbname']}' temporarily renamed as '{new_db_name}'" + logger.debug(msg) create_db(current_app.config) cmd = f"PGPASSWORD={params['password']} pg_restore -h {params['host']} -U {params['user']} -d {params['dbname']} -v {file}" - os.system(cmd) - if db_copied: + result = subprocess.run(cmd, shell=True) + logger.debug("Restore result: %r", result) + if result.returncode == 0: + if db_copied and safe: print(f"Existing database '{params['dbname']}' renamed as '{new_db_name}'") msg = f"Backup {file} restored to database '{params['dbname']}'" logger.debug(msg) print(msg) + # if mode is set to 'not safe' + # delete the temp snapshot of the current database + if not safe: + drop_db(db_name=new_db_name) + msg = f"Current database '{params['dbname']}' deleted" + logger.debug(msg) + if verbose: + print(msg) + else: + # if any error occurs + # restore the previous latest version of the DB + # previously saved as temp snapshot + if new_db_name: + # delete the db just created + drop_db() + # restore the old database snapshot + rename_db(new_db_name, params['dbname']) + db_copied = True + msg = f"Database restored '{params['dbname']}' renamed as '{new_db_name}'" + logger.debug(msg) + if verbose: + print(msg) + print("ERROR: Unable to restore the database backup") + if verbose and result.stderr: + print("ERROR [stderr]: %s" % result.stderr.decode()) + # report exit code to the main process + sys.exit(result.returncode) @cli.db.command() From b75930c46668833c62c382eade40d3f46e0d53db Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 29 Mar 2022 15:43:27 +0200 Subject: [PATCH 36/58] Enable verbose mode --- lifemonitor/commands/db.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/lifemonitor/commands/db.py b/lifemonitor/commands/db.py index b98fd028f..cefc26e36 100644 --- a/lifemonitor/commands/db.py +++ b/lifemonitor/commands/db.py @@ -21,6 +21,7 @@ import logging import os +import re import subprocess import sys from datetime import datetime @@ -105,8 +106,9 @@ def wait_for_db(): @cli.db.command() @click.option("-f", "--file", default=None, help="Backup filename (default 'hhmmss_yyyymmdd.tar')") @click.option("-d", "--directory", default="./", help="Directory path for the backup file (default '.')") +@click.option("-v", "--verbose", default=False, is_flag=True, help="Enable verbose mode") @with_appcontext -def backup(file, directory): +def backup(file, directory, verbose): """ Make a backup of the current app database """ @@ -116,12 +118,15 @@ def backup(file, directory): file = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}.tar" target_path = os.path.join(directory, file) cmd = f"PGPASSWORD={params['password']} pg_dump -h {params['host']} -U {params['user']} -F t {params['dbname']} > {target_path}" + if verbose: + print("Output file: %s" % target_path) + print("Backup command: %s" % re.sub("PGPASSWORD=(\S+)", "PGPASSWORD=****", cmd)) result = subprocess.run(cmd, shell=True, capture_output=True) logger.debug("Backup result: %r", result) if result.returncode == 0: - msg = f"Created backup of database {params['dbname']} on {target_path}" - logger.debug(msg) - print(msg) + msg = f"Created backup of database {params['dbname']} on {target_path}" + logger.debug(msg) + print(msg) else: print("ERROR: Unable to backup the database") if verbose and result.stderr: @@ -134,8 +139,9 @@ def backup(file, directory): @click.argument("file") @click.option("-s", "--safe", default=False, is_flag=True, help="Preserve the current database renaming it as '_yyyymmdd_hhmmss'") +@click.option("-v", "--verbose", default=False, is_flag=True, help="Enable verbose mode") @with_appcontext -def restore(file, safe=False): +def restore(file, safe, verbose): """ Restore a backup of the app database """ @@ -165,16 +171,22 @@ def restore(file, safe=False): db_copied = True msg = f"Created a DB snapshot: data '{params['dbname']}' temporarily renamed as '{new_db_name}'" logger.debug(msg) + if verbose: + print(msg) + # restore database create_db(current_app.config) cmd = f"PGPASSWORD={params['password']} pg_restore -h {params['host']} -U {params['user']} -d {params['dbname']} -v {file}" + if verbose: + print("Dabaset file: %s" % file) + print("Backup command: %s" % re.sub("PGPASSWORD=(\S+)", "PGPASSWORD=****", cmd)) result = subprocess.run(cmd, shell=True) logger.debug("Restore result: %r", result) if result.returncode == 0: if db_copied and safe: - print(f"Existing database '{params['dbname']}' renamed as '{new_db_name}'") - msg = f"Backup {file} restored to database '{params['dbname']}'" - logger.debug(msg) - print(msg) + print(f"Existing database '{params['dbname']}' renamed as '{new_db_name}'") + msg = f"Backup {file} restored to database '{params['dbname']}'" + logger.debug(msg) + print(msg) # if mode is set to 'not safe' # delete the temp snapshot of the current database if not safe: From 778dc59d1656aa87cf052bbb1661d438b0b6da4e Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 29 Mar 2022 15:52:42 +0200 Subject: [PATCH 37/58] Rename admin script to 'lm-admin.py' --- docker-compose.base.yml | 2 +- docker-compose.test.yml | 2 +- docker/lifemonitor.Dockerfile | 2 +- k8s/templates/backend-deployment.yaml | 2 +- k8s/templates/job-backup.yaml | 2 +- k8s/templates/job-init.yaml | 2 +- k8s/templates/worker-deployment.yaml | 2 +- manage.py => lm-admin.py | 0 8 files changed, 7 insertions(+), 7 deletions(-) rename manage.py => lm-admin.py (100%) diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 553a6df6d..122b75d62 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -52,7 +52,7 @@ services: entrypoint: /bin/bash restart: "no" command: | - -c "wait-for-postgres.sh && ./manage.py db init" + -c "wait-for-postgres.sh && ./lm-admin.py db init" depends_on: - "db" env_file: *env_file diff --git a/docker-compose.test.yml b/docker-compose.test.yml index a04877991..84a688c60 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -10,7 +10,7 @@ services: restart: "no" entrypoint: /bin/bash command: | - -c "wait-for-postgres.sh && ./manage.py db init && /usr/local/bin/lm_entrypoint.sh" + -c "wait-for-postgres.sh && ./lm-admin.py db init && /usr/local/bin/lm_entrypoint.sh" environment: - "FLASK_ENV=testingSupport" - "HOME=/lm" diff --git a/docker/lifemonitor.Dockerfile b/docker/lifemonitor.Dockerfile index e1e791fb9..347e9f1db 100644 --- a/docker/lifemonitor.Dockerfile +++ b/docker/lifemonitor.Dockerfile @@ -60,7 +60,7 @@ RUN mkdir -p /var/data/lm \ USER lm # Copy lifemonitor app -COPY --chown=lm:lm app.py manage.py gunicorn.conf.py /lm/ +COPY --chown=lm:lm app.py lmadmin.py gunicorn.conf.py /lm/ COPY --chown=lm:lm specs /lm/specs COPY --chown=lm:lm lifemonitor /lm/lifemonitor COPY --chown=lm:lm migrations /lm/migrations diff --git a/k8s/templates/backend-deployment.yaml b/k8s/templates/backend-deployment.yaml index 5a2117778..14e13703e 100644 --- a/k8s/templates/backend-deployment.yaml +++ b/k8s/templates/backend-deployment.yaml @@ -38,7 +38,7 @@ spec: image: {{ include "chart.lifemonitor.image" . }} imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} command: ["/bin/sh","-c"] - args: ["wait-for-redis.sh && wait-for-postgres.sh && ./manage.py db wait-for-db"] + args: ["wait-for-redis.sh && wait-for-postgres.sh && ./lm-admin.py db wait-for-db"] env: {{- include "lifemonitor.common-env" . | nindent 12 }} volumeMounts: diff --git a/k8s/templates/job-backup.yaml b/k8s/templates/job-backup.yaml index 4b9ce00b9..a80c1427a 100644 --- a/k8s/templates/job-backup.yaml +++ b/k8s/templates/job-backup.yaml @@ -27,7 +27,7 @@ spec: crate_backups_path="${base_path}/crates" ; mkdir -p ${db_backups_path} ; mkdir -p ${crate_backups_path} ; - ./manage.py db backup -d ${db_backups_path} 2>&1 ; + ./lm-admin.py db backup -d ${db_backups_path} 2>&1 ; exit_code = $?; if [[ ${exit_code} == 0 ]]; then find ${db_backups_path} -type f -mtime +{{ .Values.backup.retain_days }} -name '*.tar' -execdir rm -- '{}' \; ; diff --git a/k8s/templates/job-init.yaml b/k8s/templates/job-init.yaml index 2429576a2..a82e1b4f8 100644 --- a/k8s/templates/job-init.yaml +++ b/k8s/templates/job-init.yaml @@ -20,7 +20,7 @@ spec: image: {{ include "chart.lifemonitor.image" . }} imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} command: ["/bin/sh","-c"] - args: ["wait-for-redis.sh && wait-for-postgres.sh && ./manage.py db init && ./manage.py task-queue reset"] + args: ["wait-for-redis.sh && wait-for-postgres.sh && ./lm-admin.py db init && ./lm-admin.py task-queue reset"] env: {{- include "lifemonitor.common-env" . | nindent 10 }} volumeMounts: diff --git a/k8s/templates/worker-deployment.yaml b/k8s/templates/worker-deployment.yaml index cf735fc15..1c61bc484 100644 --- a/k8s/templates/worker-deployment.yaml +++ b/k8s/templates/worker-deployment.yaml @@ -35,7 +35,7 @@ spec: image: {{ include "chart.lifemonitor.image" . }} imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} command: ["/bin/sh","-c"] - args: ["wait-for-redis.sh && wait-for-postgres.sh && ./manage.py db wait-for-db"] + args: ["wait-for-redis.sh && wait-for-postgres.sh && ./lm-admin.py db wait-for-db"] env: {{- include "lifemonitor.common-env" . | nindent 12 }} volumeMounts: diff --git a/manage.py b/lm-admin.py similarity index 100% rename from manage.py rename to lm-admin.py From 81778566fbb0af56906e2db8bed735780622690d Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 29 Mar 2022 15:57:14 +0200 Subject: [PATCH 38/58] Fix flake8 issues --- lifemonitor/commands/db.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lifemonitor/commands/db.py b/lifemonitor/commands/db.py index cefc26e36..58c70ff20 100644 --- a/lifemonitor/commands/db.py +++ b/lifemonitor/commands/db.py @@ -120,7 +120,7 @@ def backup(file, directory, verbose): cmd = f"PGPASSWORD={params['password']} pg_dump -h {params['host']} -U {params['user']} -F t {params['dbname']} > {target_path}" if verbose: print("Output file: %s" % target_path) - print("Backup command: %s" % re.sub("PGPASSWORD=(\S+)", "PGPASSWORD=****", cmd)) + print("Backup command: %s" % re.sub(r"PGPASSWORD=(\S+)", "PGPASSWORD=****", cmd)) result = subprocess.run(cmd, shell=True, capture_output=True) logger.debug("Backup result: %r", result) if result.returncode == 0: @@ -178,7 +178,7 @@ def restore(file, safe, verbose): cmd = f"PGPASSWORD={params['password']} pg_restore -h {params['host']} -U {params['user']} -d {params['dbname']} -v {file}" if verbose: print("Dabaset file: %s" % file) - print("Backup command: %s" % re.sub("PGPASSWORD=(\S+)", "PGPASSWORD=****", cmd)) + print("Backup command: %s" % re.sub(r"PGPASSWORD=(\S+)", "PGPASSWORD=****", cmd)) result = subprocess.run(cmd, shell=True) logger.debug("Restore result: %r", result) if result.returncode == 0: @@ -189,7 +189,7 @@ def restore(file, safe, verbose): print(msg) # if mode is set to 'not safe' # delete the temp snapshot of the current database - if not safe: + if not safe: drop_db(db_name=new_db_name) msg = f"Current database '{params['dbname']}' deleted" logger.debug(msg) From a1d6a1680e26d415e0129849b7676a6d4a0ea6e1 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 29 Mar 2022 16:09:23 +0200 Subject: [PATCH 39/58] Fix Docker image: rename admin script --- docker/lifemonitor.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/lifemonitor.Dockerfile b/docker/lifemonitor.Dockerfile index 347e9f1db..2361bfc50 100644 --- a/docker/lifemonitor.Dockerfile +++ b/docker/lifemonitor.Dockerfile @@ -60,7 +60,7 @@ RUN mkdir -p /var/data/lm \ USER lm # Copy lifemonitor app -COPY --chown=lm:lm app.py lmadmin.py gunicorn.conf.py /lm/ +COPY --chown=lm:lm app.py lm-admin.py gunicorn.conf.py /lm/ COPY --chown=lm:lm specs /lm/specs COPY --chown=lm:lm lifemonitor /lm/lifemonitor COPY --chown=lm:lm migrations /lm/migrations From 4f0609403724511161d9920952d0c58cda599750 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 29 Mar 2022 14:40:26 +0000 Subject: [PATCH 40/58] Freeze Werkzeug requirement --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 39dff6a69..130b1beef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,3 +36,4 @@ requests~=2.27.1 rocrate==0.5.5 SQLAlchemy~=1.3.24 wheel~=0.37.1 +Werkzeug~=2.0.0 From 16880042a15f12550038002a8d58291e2bbdb3e9 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 29 Mar 2022 17:29:36 +0200 Subject: [PATCH 41/58] Fix script --- k8s/templates/job-backup.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/templates/job-backup.yaml b/k8s/templates/job-backup.yaml index a80c1427a..f80890e37 100644 --- a/k8s/templates/job-backup.yaml +++ b/k8s/templates/job-backup.yaml @@ -19,7 +19,7 @@ spec: - name: lifemonitor-backup image: {{ include "chart.lifemonitor.image" . }} imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} - command: ["/bin/sh","-c"] + command: ["/bin/bash","-c"] args: - wait-for-redis.sh && wait-for-postgres.sh ; base_path="/var/data/backup" ; @@ -28,7 +28,7 @@ spec: mkdir -p ${db_backups_path} ; mkdir -p ${crate_backups_path} ; ./lm-admin.py db backup -d ${db_backups_path} 2>&1 ; - exit_code = $?; + exit_code=$? ; if [[ ${exit_code} == 0 ]]; then find ${db_backups_path} -type f -mtime +{{ .Values.backup.retain_days }} -name '*.tar' -execdir rm -- '{}' \; ; cp -urv /var/data/lm/* ${crate_backups_path}/ ; From 3a8510bb5bdd38991f19d4b977f50dd29465acf1 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 29 Mar 2022 18:28:01 +0200 Subject: [PATCH 42/58] Mask passwd on log messages --- lifemonitor/commands/db.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lifemonitor/commands/db.py b/lifemonitor/commands/db.py index 58c70ff20..4ba096547 100644 --- a/lifemonitor/commands/db.py +++ b/lifemonitor/commands/db.py @@ -45,6 +45,11 @@ initial_revision = '8b2e530dc029' +def hide_pgpasswd(cmd) -> str: + assert cmd, cmd + return re.sub(r"PGPASSWORD=(\S+)", "PGPASSWORD=****", str(cmd)) + + @cli.db.command() @click.option("-r", "--revision", default="head") @with_appcontext @@ -120,9 +125,9 @@ def backup(file, directory, verbose): cmd = f"PGPASSWORD={params['password']} pg_dump -h {params['host']} -U {params['user']} -F t {params['dbname']} > {target_path}" if verbose: print("Output file: %s" % target_path) - print("Backup command: %s" % re.sub(r"PGPASSWORD=(\S+)", "PGPASSWORD=****", cmd)) + print("Backup command: %s" % hide_pgpasswd(cmd)) result = subprocess.run(cmd, shell=True, capture_output=True) - logger.debug("Backup result: %r", result) + logger.debug("Backup result: %r", hide_pgpasswd(result)) if result.returncode == 0: msg = f"Created backup of database {params['dbname']} on {target_path}" logger.debug(msg) @@ -178,9 +183,9 @@ def restore(file, safe, verbose): cmd = f"PGPASSWORD={params['password']} pg_restore -h {params['host']} -U {params['user']} -d {params['dbname']} -v {file}" if verbose: print("Dabaset file: %s" % file) - print("Backup command: %s" % re.sub(r"PGPASSWORD=(\S+)", "PGPASSWORD=****", cmd)) + print("Backup command: %s" % hide_pgpasswd(cmd)) result = subprocess.run(cmd, shell=True) - logger.debug("Restore result: %r", result) + logger.debug("Restore result: %r", hide_pgpasswd(result)) if result.returncode == 0: if db_copied and safe: print(f"Existing database '{params['dbname']}' renamed as '{new_db_name}'") From 04d4f98f3706cbbd01d8add7d3b8a6eb6c1c08be Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 29 Mar 2022 18:29:18 +0200 Subject: [PATCH 43/58] Add more log messages on backup job --- k8s/templates/job-backup.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/k8s/templates/job-backup.yaml b/k8s/templates/job-backup.yaml index f80890e37..d1a2a088b 100644 --- a/k8s/templates/job-backup.yaml +++ b/k8s/templates/job-backup.yaml @@ -32,7 +32,17 @@ spec: if [[ ${exit_code} == 0 ]]; then find ${db_backups_path} -type f -mtime +{{ .Values.backup.retain_days }} -name '*.tar' -execdir rm -- '{}' \; ; cp -urv /var/data/lm/* ${crate_backups_path}/ ; + exit_code=$? ; + if [[ ${exit_code} == 0 ]]; then + echo "RO-Crates locally synched @ ${crate_backups_path}" ; + else echo "Error when copying RO-Crates (code ${exit_code})" ; + fi ; {{ include "backup.remote.command" . }} ; + exit_code=$? ; + if [[ ${exit_code} == 0 ]]; then + echo "Backup folder synched with the remote site '{{ .Values.backup.remote.host }}' (path {{.Values.backup.remote.path}})" ; + else echo "Unable to synch backup folder with the remote site '{{ .Values.backup.remote.host }}'" ; exit ${exit_code} ; + fi ; else echo "Unable to complete the backup" ; exit ${exit_code} ; fi ; env: From d5ec8af87c8a047b014fe0a1551bd705d8102df1 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 31 Mar 2022 18:04:58 +0200 Subject: [PATCH 44/58] Refactor 'db' subcommands --- lifemonitor/commands/db.py | 55 ++++++++++++++++++++++++-------------- lifemonitor/utils.py | 5 ++++ 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/lifemonitor/commands/db.py b/lifemonitor/commands/db.py index 4ba096547..adf3d510b 100644 --- a/lifemonitor/commands/db.py +++ b/lifemonitor/commands/db.py @@ -21,7 +21,6 @@ import logging import os -import re import subprocess import sys from datetime import datetime @@ -31,6 +30,7 @@ from flask.cli import with_appcontext from flask_migrate import cli, current, stamp, upgrade from lifemonitor.auth.models import User +from lifemonitor.utils import hide_secret # set module level logger logger = logging.getLogger() @@ -45,11 +45,6 @@ initial_revision = '8b2e530dc029' -def hide_pgpasswd(cmd) -> str: - assert cmd, cmd - return re.sub(r"PGPASSWORD=(\S+)", "PGPASSWORD=****", str(cmd)) - - @cli.db.command() @click.option("-r", "--revision", default="head") @with_appcontext @@ -108,43 +103,63 @@ def wait_for_db(): logger.info(f"Current revision: {current_revision}") -@cli.db.command() -@click.option("-f", "--file", default=None, help="Backup filename (default 'hhmmss_yyyymmdd.tar')") -@click.option("-d", "--directory", default="./", help="Directory path for the backup file (default '.')") -@click.option("-v", "--verbose", default=False, is_flag=True, help="Enable verbose mode") +# define common options +verbose_option = click.option("-v", "--verbose", default=False, is_flag=True, help="Enable verbose mode") + + +def backup_options(func): + # backup command options (evaluated in reverse order!) + func = verbose_option(func) + func = click.option("-f", "--file", default=None, help="Backup filename (default 'hhmmss_yyyymmdd.tar')")(func) + func = click.option("-d", "--directory", default="./", help="Directory path for the backup file (default '.')")(func) + return func + + +@cli.db.command("backup") +@backup_options @with_appcontext -def backup(file, directory, verbose): +def backup_cmd(directory, file, verbose): + """ + Make a backup of the current app database + """ + result = backup(directory, file, verbose) + # report exit code to the main process + sys.exit(result.returncode) + + +def backup(directory, file=None, verbose=False) -> subprocess.CompletedProcess: """ Make a backup of the current app database """ + logger.debug("%r - %r - %r", file, directory, verbose) from lifemonitor.db import db_connection_params params = db_connection_params() if not file: file = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}.tar" + os.makedirs(directory, exist_ok=True) target_path = os.path.join(directory, file) cmd = f"PGPASSWORD={params['password']} pg_dump -h {params['host']} -U {params['user']} -F t {params['dbname']} > {target_path}" if verbose: print("Output file: %s" % target_path) - print("Backup command: %s" % hide_pgpasswd(cmd)) + print("Backup command: %s" % hide_secret(cmd, params['password'])) result = subprocess.run(cmd, shell=True, capture_output=True) - logger.debug("Backup result: %r", hide_pgpasswd(result)) + logger.debug("Backup result: %r", hide_secret(result, params['password'])) if result.returncode == 0: - msg = f"Created backup of database {params['dbname']} on {target_path}" + msg = f"Created backup of database {params['dbname']} @ {target_path}" logger.debug(msg) print(msg) else: - print("ERROR: Unable to backup the database") + click.echo("\nERROR Unable to backup the database: %s" % result.stderr.decode()) if verbose and result.stderr: print("ERROR [stderr]: %s" % result.stderr.decode()) - # report exit code to the main process - sys.exit(result.returncode) + return result @cli.db.command() @click.argument("file") @click.option("-s", "--safe", default=False, is_flag=True, help="Preserve the current database renaming it as '_yyyymmdd_hhmmss'") -@click.option("-v", "--verbose", default=False, is_flag=True, help="Enable verbose mode") +@verbose_option @with_appcontext def restore(file, safe, verbose): """ @@ -183,9 +198,9 @@ def restore(file, safe, verbose): cmd = f"PGPASSWORD={params['password']} pg_restore -h {params['host']} -U {params['user']} -d {params['dbname']} -v {file}" if verbose: print("Dabaset file: %s" % file) - print("Backup command: %s" % hide_pgpasswd(cmd)) + print("Backup command: %s" % hide_secret(cmd, params['password'])) result = subprocess.run(cmd, shell=True) - logger.debug("Restore result: %r", hide_pgpasswd(result)) + logger.debug("Restore result: %r", hide_secret(cmd, params['password'])) if result.returncode == 0: if db_copied and safe: print(f"Existing database '{params['dbname']}' renamed as '{new_db_name}'") diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index da0868932..8c2014677 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -131,6 +131,11 @@ def sizeof_fmt(num, suffix='B'): return "%.1f%s%s" % (num, 'Yi', suffix) +def hide_secret(text: str, secret: str, replace_with="*****") -> str: + text = str(text) if not isinstance(text, str) else text + return text if not text else text.replace(secret, replace_with) + + def decodeBase64(str, as_object=False, encoding='utf-8'): result = base64.b64decode(str) if not result: From 08ce2d89064d507d681050eb082b53959dbdcbb0 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 31 Mar 2022 18:08:23 +0200 Subject: [PATCH 45/58] Add Python native FTP(s) client --- lifemonitor/utils.py | 123 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index 8c2014677..58b98cc01 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -20,6 +20,7 @@ import base64 +import ftplib import functools import glob import json @@ -41,6 +42,7 @@ import flask import requests import yaml +from dateutil import parser from . import exceptions as lm_exceptions @@ -510,3 +512,124 @@ def encode_object(cls, obj: object) -> str: @classmethod def decode(cls, data: str) -> object: return base64.b64decode(data.encode()) + + +class FtpUtils(): + + def __init__(self, host, user, password, enable_tls) -> None: + self._ftp = None + self.host = host + self.user = user + self.passwd = password + self.tls_enabled = enable_tls + + def __del__(self): + if self._ftp: + try: + logger.warning("Closing remote connection...") + self._ftp.close() + logger.warning("Closing remote connection... DONE") + except Exception as e: + logger.debug(e) + + @property + def ftp(self) -> ftplib.FTP_TLS: + if not self._ftp: + cls = ftplib.FTP_TLS if self.tls_enabled else ftplib.FTP + self._ftp = cls(self.host) + self._ftp.login(self.user, self.passwd) + return self._ftp + + def is_dir(self, path) -> bool: + """ Check whether a remote path is a directory """ + cwd = self.ftp.pwd() + try: + self.ftp.cwd(path) + return True + except: + return False + finally: + self.ftp.cwd(cwd) + + def sync(self, source, target): + for root, dirs, files in os.walk(source, topdown=True): + for name in dirs: + local_path = os.path.join(root, name) + logger.debug("Local directory path: %s", local_path) + remote_file_path = local_path.replace(source, target) + logger.debug("Remote directory path: %s", remote_file_path) + try: + self.ftp.mkd(remote_file_path) + logger.debug("Created remote directory: %s", remote_file_path) + except Exception as e: + logger.debug("Unable to create remote directory: %s", remote_file_path) + logger.debug(str(e)) + + for name in files: + local_path = os.path.join(root, name) + remote_file_path = f"{target}/{local_path.replace(source + '/', '')}" + logger.debug("Local filepath: %s", local_path) + logger.debug("Remote filepath: %s", remote_file_path) + upload_file = True + try: + timestamp = self.ftp.voidcmd(f"MDTM {remote_file_path}")[4:].strip() + remote_time = parser.parse(timestamp).timestamp() + local_time = os.path.getmtime(local_path) + logger.debug("Checking: %r - %r", remote_time, local_time) + if local_time <= remote_time: + upload_file = False + else: + logger.warning("Not changed %s", remote_file_path) + except: + logger.debug("File %s doesn't exist @ remote path %s", name, remote_file_path) + if upload_file: + with open(local_path, 'rb') as fh: + self.ftp.storbinary('STOR %s' % remote_file_path, fh) + logger.debug("Local file '%s' uploaded on remote @ %s", local_path, remote_file_path) + # remove obsolete files on the remote target + self.remove_obsolete_remote_files(source, target) + + def remove_obsolete_remote_files(self, source, target): + """ Remove obsolete files on the remote target """ + for path in self.ftp.nlst(target): + logger.debug("Checking remote path: %r", path) + local_path = path.replace(target, source) + logger.debug("Local path corresponding to remote %s is: %s", path, local_path) + if self.is_dir(path): + logger.warning("Is dir: %s", path) + self.remove_obsolete_remote_files(local_path, path) + # remove remote folder if empty + if len(self.ftp.nlst(path)) == 0: + self.ftp.rmd(path) + logger.debug("Removed remote folder '%s'", path) + else: + if not os.path.isfile(local_path): + logger.debug("Removing remote file '%s'...", path) + try: + self.ftp.delete(path) + logger.debug("Removed remote file '%s'", path) + except Exception as e: + logger.debug(e) + else: + logger.debug("File %s exists @ %s", path, local_path) + + def rm_tree(self, path): + """Recursively delete a directory tree on a remote server.""" + try: + names = self.ftp.nlst(path) + except ftplib.all_errors as e: + logger.debug('Could not remove {0}: {1}'.format(path, e)) + return + + for name in names: + if os.path.split(name)[1] in ('.', '..'): + continue + logger.debug('Checking {0}'.format(name)) + if self.is_dir(name): + self.rm_tree(name) + else: + self.ftp.delete(name) + try: + self.ftp.rmd(path) + except ftplib.all_errors as e: + logger.debug('Could not remove {0}: {1}'.format(path, e)) From 4dc591f45bd2a78ac3ea3a00659a7760e8b47542 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 31 Mar 2022 18:14:20 +0200 Subject: [PATCH 46/58] Add CLI command to automate full backups. Allow to make (and optionally synch with a remote site via FTP(s)) backups of both RO-Crates and database --- lifemonitor/commands/backup.py | 215 +++++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 lifemonitor/commands/backup.py diff --git a/lifemonitor/commands/backup.py b/lifemonitor/commands/backup.py new file mode 100644 index 000000000..baa8e6a3f --- /dev/null +++ b/lifemonitor/commands/backup.py @@ -0,0 +1,215 @@ +# Copyright (c) 2022 CRS4 +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import logging +import os +import subprocess +import sys +import time +from pathlib import Path + +import click +from click_option_group import GroupedOption, optgroup +from flask import current_app +from flask.blueprints import Blueprint +from flask.cli import with_appcontext +from flask.config import Config +from lifemonitor.utils import FtpUtils + +from .db import backup, backup_options + +# set module level logger +logger = logging.getLogger() + +# define the blueprint for DB commands +_blueprint = Blueprint('backup', __name__) + +# set help for the CLI command +_blueprint.cli.help = "Manage backups of database and RO-Crates" + + +class RequiredIf(GroupedOption): + def __init__(self, *args, **kwargs): + self.required_if = kwargs.pop('required_if') + assert self.required_if, "'required_if' parameter required" + kwargs['help'] = (kwargs.get('help', '') + + " (NOTE: This argument is required if '%s' is True)" % + self.required_if).strip() + super(RequiredIf, self).__init__(*args, **kwargs) + + def handle_parse_result(self, ctx, opts, args): + we_are_present = self.name in opts + other_present = self.required_if in opts + + if other_present: + if not we_are_present: + raise click.UsageError( + "Illegal usage: '%s' is required when '%s' is True" % ( + self.name, self.required_if)) + else: + self.prompt = None + + return super(RequiredIf, self).handle_parse_result( + ctx, opts, args) + + +def synch_otptions(func): + func = optgroup.option('--enable-tls', default=False, is_flag=True, show_default=True, + help="Enable FTP over TLS")(func) + func = optgroup.option('-t', '--target', default="/", show_default=True, + help="Remote target path")(func) + func = optgroup.option('-p', '--password', cls=RequiredIf, required_if='synch', + help="Password fot the FTPS account")(func) + func = optgroup.option('-u', '--user', cls=RequiredIf, required_if='synch', + help="Username of the FTPS account")(func) + func = optgroup.option('-h', '--host', cls=RequiredIf, required_if='synch', + help="Hostame of the FTPS server")(func) + func = optgroup.group('\nSettings to connect with a remote site via FTPS')(func) + func = click.option('-s', '--synch', default=False, show_default=True, + is_flag=True, help="Enable sync with a remote FTPS server")(func) + return func + + +def __remote_synch__(source: str, target: str, + host: str, user: str, password: str, + enable_tls: bool): + try: + ftp_utils = FtpUtils(host, user, password, enable_tls) + ftp_utils.sync(source, target) + return 0 + except Exception as e: + logger.debug(e) + print("Unable to synch remote site. ERROR: %s" % str(e)) + return 1 + + +@_blueprint.cli.group(name="backup", invoke_without_command=True) +@with_appcontext +@click.pass_context +def bck(ctx): + if not ctx.invoked_subcommand: + auto(current_app.config) + + +@bck.command("db") +@backup_options +@synch_otptions +@with_appcontext +def db_cmd(file, directory, verbose, *args, **kwargs): + """ + Make a backup of the database + """ + result = backup_db(directory, file, verbose, *args, **kwargs) + sys.exit(result) + + +def backup_db(directory, file=None, verbose=False, *args, **kwargs): + logger.debug(sys.argv) + result = backup(directory, file, verbose) + if result.returncode == 0: + synch = kwargs.pop('synch', False) + if synch: + return __remote_synch__(source=directory, **kwargs) + return result.returncode + + +@bck.command("crates") +@click.option("-d", "--directory", default="./", show_default=True, + help="Local path to store RO-Crates") +@synch_otptions +@with_appcontext +def crates_cmd(directory, *args, **kwargs): + """ + Make a backup of the registered workflow RO-Crates + """ + result = backup_crates(current_app.config, directory, *args, **kwargs) + sys.exit(result) + + +def backup_crates(config, directory, *args, **kwargs): + assert config.get("DATA_WORKFLOWS", None), "DATA_WORKFLOWS not configured" + rocrate_source_path = config.get("DATA_WORKFLOWS").strip('/') + os.makedirs(directory, exist_ok=True) + result = subprocess.run(f'rsync -avh --delete {rocrate_source_path}/ {directory} ', shell=True, capture_output=True) + if result.returncode == 0: + print("Created backup of workflow RO-Crate @ '%s'" % rocrate_source_path) + synch = kwargs.pop('synch', False) + if synch: + logger.debug("Remaining args: %r", kwargs) + return __remote_synch__(source=directory, **kwargs) + else: + print("Unable to backup workflow RO-Crates\n%s", result.stderr.decode()) + return result.returncode + + +def auto(config: Config): + logger.debug("Current app config: %r", config) + base_path = config.get("BACKUP_LOCAL_PATH", None) + if not base_path: + click.echo("No BACKUP_LOCAL_PATH found in your settings") + sys.exit(0) + + # set paths + base_path = base_path.strip('/') # remove trailing '/' + db_backups = f"{base_path}/db" + rc_backups = f"{base_path}/crates" + logger.debug("Backup paths: %r - %r - %r", base_path, db_backups, rc_backups) + # backup database + result = backup(db_backups) + if result.returncode != 0: + sys.exit(result.returncode) + # backup crates + result = backup_crates(config, rc_backups) + if result != 0: + sys.exit(result) + # clean up old files + retain_days = int(config.get("BACKUP_RETAIN_DAYS", -1)) + logger.debug("RETAIN DAYS: %d", retain_days) + if retain_days > -1: + now = time.time() + for file in Path(db_backups).glob('*'): + if file.is_file(): + logger.debug("Check st_mtime of file %s: %r < %r", + file.absolute(), os.path.getmtime(file), now - int(retain_days) * 86400) + if os.path.getmtime(file) < now - int(retain_days) * 86400: + logger.warning("Removing %s", file.absolute()) + os.remove(file.absolute()) + # synch with a remote site + if config.get("BACKUP_REMOTE_PATH", None): + # check REMOTE_* params + required_params = ["BACKUP_REMOTE_PATH", "BACKUP_REMOTE_HOST", + "BACKUP_REMOTE_USER", "BACKUP_REMOTE_PASSWORD", + "BACKUP_REMOTE_ENABLE_TLS"] + for p in required_params: + if not config.get(p, None): + print(f"Missing '{p}' on your settings!") + print("Required params are: %s", ", ".join(required_params)) + sys.exit(128) + __remote_synch__(base_path, config.get("BACKUP_REMOTE_PATH"), + config.get("BACKUP_REMOTE_HOST"), + config.get("BACKUP_REMOTE_USER"), config.get("BACKUP_REMOTE_PASSWORD"), + config.get("BACKUP_REMOTE_ENABLE_TLS", False)) + else: + logger.warning("Remote backup not configured") + + +# export backup command +commands = [bck] From ee8401ae6fe6b5b7cef96aca49977668c57e77c3 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 31 Mar 2022 18:16:20 +0200 Subject: [PATCH 47/58] Use `lm-admin backup` on k8s backup job --- k8s/templates/job-backup.yaml | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/k8s/templates/job-backup.yaml b/k8s/templates/job-backup.yaml index d1a2a088b..109e8d2dc 100644 --- a/k8s/templates/job-backup.yaml +++ b/k8s/templates/job-backup.yaml @@ -22,29 +22,7 @@ spec: command: ["/bin/bash","-c"] args: - wait-for-redis.sh && wait-for-postgres.sh ; - base_path="/var/data/backup" ; - db_backups_path="${base_path}/db" ; - crate_backups_path="${base_path}/crates" ; - mkdir -p ${db_backups_path} ; - mkdir -p ${crate_backups_path} ; - ./lm-admin.py db backup -d ${db_backups_path} 2>&1 ; - exit_code=$? ; - if [[ ${exit_code} == 0 ]]; then - find ${db_backups_path} -type f -mtime +{{ .Values.backup.retain_days }} -name '*.tar' -execdir rm -- '{}' \; ; - cp -urv /var/data/lm/* ${crate_backups_path}/ ; - exit_code=$? ; - if [[ ${exit_code} == 0 ]]; then - echo "RO-Crates locally synched @ ${crate_backups_path}" ; - else echo "Error when copying RO-Crates (code ${exit_code})" ; - fi ; - {{ include "backup.remote.command" . }} ; - exit_code=$? ; - if [[ ${exit_code} == 0 ]]; then - echo "Backup folder synched with the remote site '{{ .Values.backup.remote.host }}' (path {{.Values.backup.remote.path}})" ; - else echo "Unable to synch backup folder with the remote site '{{ .Values.backup.remote.host }}'" ; exit ${exit_code} ; - fi ; - else echo "Unable to complete the backup" ; exit ${exit_code} ; - fi ; + ./lm-admin backup ; env: {{- include "lifemonitor.common-env" . | nindent 12 }} volumeMounts: From d89094ea42e17b3dc5c0c1031d0ccb14c115cc8c Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 31 Mar 2022 18:18:17 +0200 Subject: [PATCH 48/58] Update requirements --- docker/lifemonitor.Dockerfile | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/lifemonitor.Dockerfile b/docker/lifemonitor.Dockerfile index 2361bfc50..ffda678ec 100644 --- a/docker/lifemonitor.Dockerfile +++ b/docker/lifemonitor.Dockerfile @@ -3,7 +3,7 @@ FROM python:3.9-buster as base # Install base requirements RUN apt-get update -q \ && apt-get install -y --no-install-recommends \ - bash lftp \ + bash lftp rsync \ redis-tools \ postgresql-client-11 \ && apt-get clean -y && rm -rf /var/lib/apt/lists diff --git a/requirements.txt b/requirements.txt index 130b1beef..aac634e40 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ connexion[swagger-ui]~=2.11.2 dramatiq[redis,watch]==1.12.3 email-validator~=1.1.3 Bcrypt-Flask==1.0.2 +click-option-group~=0.5.3 flask-cors==3.0.10 flask-marshmallow~=0.14.0 flask-restful==0.3.9 From 5f2e59289bfaddd895b9b736c44ee53a342aee7a Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 31 Mar 2022 18:26:13 +0200 Subject: [PATCH 49/58] Add default backup settings --- settings.conf | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/settings.conf b/settings.conf index fdaa9c056..06e7511a6 100644 --- a/settings.conf +++ b/settings.conf @@ -72,7 +72,15 @@ CACHE_DEFAULT_TIMEOUT=300 CACHE_REQUEST_TIMEOUT=15 CACHE_SESSION_TIMEOUT=3600 CACHE_WORKFLOW_TIMEOUT=1800 -CACHE_BUILD_TIMEOUT=84600 + +# Backup settings +BACKUP_LOCAL_PATH="./backups" +BACKUP_RETAIN_DAYS=30 +# BACKUP_REMOTE_PATH="lm-backups" +# BACKUP_REMOTE_HOST="ftp-site.domain.it" +# BACKUP_REMOTE_USER="lm" +# BACKUP_REMOTE_PASSWORD="foobar" +# BACKUP_REMOTE_ENABLE_TLS=True # Github OAuth2 settings #GITHUB_CLIENT_ID="___YOUR_GITHUB_OAUTH2_CLIENT_ID___" From 33f00067548b403fd8c9bbb86b277f71dbbb61b7 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 31 Mar 2022 18:26:34 +0200 Subject: [PATCH 50/58] Update ignore files --- .dockerignore | 2 ++ .gitignore | 1 + 2 files changed, 3 insertions(+) diff --git a/.dockerignore b/.dockerignore index 8ce907d8e..5b0628378 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,7 +8,9 @@ **/__pycache__ **/.npm *.pyc +backups data docker/Dockerfile +docker*.yml package-lock.json **/node_modules diff --git a/.gitignore b/.gitignore index d726e57aa..09448f7b7 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ instance *.pyc ./certs **/node_modules +backups data lifemonitor/static/dist lifemonitor/static/src/node_modules From 7931ddb0977e5e64468aad3de547882f3ab53839 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 31 Mar 2022 18:28:32 +0200 Subject: [PATCH 51/58] Update default k8s values --- k8s/templates/secret.yaml | 16 ++++++++++++++++ k8s/values.yaml | 5 ++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/k8s/templates/secret.yaml b/k8s/templates/secret.yaml index 5bee20989..f3d736dfb 100644 --- a/k8s/templates/secret.yaml +++ b/k8s/templates/secret.yaml @@ -69,6 +69,22 @@ stringData: MAIL_USE_SSL={{- if .Values.mail.ssl -}}True{{- else -}}False{{- end }} MAIL_DEFAULT_SENDER={{ .Values.mail.default_sender }} + {{- if .Values.backup.enabled }} + # Backups + BACKUP_LOCAL_PATH="/var/data/backup" + {{- if .Values.backup.retain_days }} + BACKUP_RETAIN_DAYS={{ .Values.backup.retain_days }} + {{- end }} + {{- if .Values.backup.remote.enabled }} + BACKUP_REMOTE_PATH={{ .Values.backup.remote.path }} + BACKUP_REMOTE_HOST={{ .Values.backup.remote.host }} + BACKUP_REMOTE_USER={{ .Values.backup.remote.user }} + BACKUP_REMOTE_PASSWORD={{ .Values.backup.remote.password }} + BACKUP_REMOTE_ENABLE_TLS={{- if .Values.backup.remote.tls }}True{{-else -}}False{{- end }} + {{- end }} + {{- end }} + + # Set admin credentials LIFEMONITOR_ADMIN_PASSWORD={{ .Values.lifemonitor.administrator.password }} diff --git a/k8s/values.yaml b/k8s/values.yaml index 926ad8fe7..ab62d2a68 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -91,14 +91,13 @@ backup: existingClaim: data-api-backup # Settings to mirror the (cluster) local backup # to a remote site via FTPS or SFTP - remote: - # supported protocols ftps | sftp - protocol: ftps + remote: enabled: false user: username password: password host: 10.0.1.135 path: /user/home/lm-backups + tls: true lifemonitor: replicaCount: 1 From 31c3289024a14d677f3207c7e952372ddc8c3d5b Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 31 Mar 2022 18:34:35 +0200 Subject: [PATCH 52/58] Fix flake8 issues --- lifemonitor/commands/backup.py | 5 ++--- lifemonitor/utils.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lifemonitor/commands/backup.py b/lifemonitor/commands/backup.py index baa8e6a3f..34659b581 100644 --- a/lifemonitor/commands/backup.py +++ b/lifemonitor/commands/backup.py @@ -50,9 +50,8 @@ class RequiredIf(GroupedOption): def __init__(self, *args, **kwargs): self.required_if = kwargs.pop('required_if') assert self.required_if, "'required_if' parameter required" - kwargs['help'] = (kwargs.get('help', '') + - " (NOTE: This argument is required if '%s' is True)" % - self.required_if).strip() + kwargs['help'] = ("%s (NOTE: This argument is required if '%s' is True)" % + (kwargs.get('help', ''), self.required_if)).strip() super(RequiredIf, self).__init__(*args, **kwargs) def handle_parse_result(self, ctx, opts, args): diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index 58b98cc01..c25c39064 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -546,7 +546,7 @@ def is_dir(self, path) -> bool: try: self.ftp.cwd(path) return True - except: + except Exception: return False finally: self.ftp.cwd(cwd) @@ -580,7 +580,7 @@ def sync(self, source, target): upload_file = False else: logger.warning("Not changed %s", remote_file_path) - except: + except Exception: logger.debug("File %s doesn't exist @ remote path %s", name, remote_file_path) if upload_file: with open(local_path, 'rb') as fh: From 1d6ccd27686287bc2f33b9ae74a0271e40c1d8db Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 31 Mar 2022 18:58:06 +0200 Subject: [PATCH 53/58] Fix type of some log messages --- lifemonitor/commands/backup.py | 2 +- lifemonitor/utils.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lifemonitor/commands/backup.py b/lifemonitor/commands/backup.py index 34659b581..75d1f3127 100644 --- a/lifemonitor/commands/backup.py +++ b/lifemonitor/commands/backup.py @@ -189,7 +189,7 @@ def auto(config: Config): logger.debug("Check st_mtime of file %s: %r < %r", file.absolute(), os.path.getmtime(file), now - int(retain_days) * 86400) if os.path.getmtime(file) < now - int(retain_days) * 86400: - logger.warning("Removing %s", file.absolute()) + logger.debug("Removing %s", file.absolute()) os.remove(file.absolute()) # synch with a remote site if config.get("BACKUP_REMOTE_PATH", None): diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index c25c39064..9e26e114f 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -526,9 +526,9 @@ def __init__(self, host, user, password, enable_tls) -> None: def __del__(self): if self._ftp: try: - logger.warning("Closing remote connection...") + logger.debug("Closing remote connection...") self._ftp.close() - logger.warning("Closing remote connection... DONE") + logger.debug("Closing remote connection... DONE") except Exception as e: logger.debug(e) @@ -579,7 +579,7 @@ def sync(self, source, target): if local_time <= remote_time: upload_file = False else: - logger.warning("Not changed %s", remote_file_path) + logger.debug("Not changed %s", remote_file_path) except Exception: logger.debug("File %s doesn't exist @ remote path %s", name, remote_file_path) if upload_file: @@ -596,7 +596,7 @@ def remove_obsolete_remote_files(self, source, target): local_path = path.replace(target, source) logger.debug("Local path corresponding to remote %s is: %s", path, local_path) if self.is_dir(path): - logger.warning("Is dir: %s", path) + logger.debug("Is dir: %s", path) self.remove_obsolete_remote_files(local_path, path) # remove remote folder if empty if len(self.ftp.nlst(path)) == 0: From 0395359204d358d3cbc12fe1fbc4bb9b00723d75 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 31 Mar 2022 19:25:04 +0200 Subject: [PATCH 54/58] Fix typos --- k8s/templates/job-backup.yaml | 2 +- k8s/templates/secret.yaml | 2 +- lifemonitor/commands/backup.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/k8s/templates/job-backup.yaml b/k8s/templates/job-backup.yaml index 109e8d2dc..67de94b27 100644 --- a/k8s/templates/job-backup.yaml +++ b/k8s/templates/job-backup.yaml @@ -22,7 +22,7 @@ spec: command: ["/bin/bash","-c"] args: - wait-for-redis.sh && wait-for-postgres.sh ; - ./lm-admin backup ; + ./lm-admin.py backup ; env: {{- include "lifemonitor.common-env" . | nindent 12 }} volumeMounts: diff --git a/k8s/templates/secret.yaml b/k8s/templates/secret.yaml index f3d736dfb..b1868307a 100644 --- a/k8s/templates/secret.yaml +++ b/k8s/templates/secret.yaml @@ -80,7 +80,7 @@ stringData: BACKUP_REMOTE_HOST={{ .Values.backup.remote.host }} BACKUP_REMOTE_USER={{ .Values.backup.remote.user }} BACKUP_REMOTE_PASSWORD={{ .Values.backup.remote.password }} - BACKUP_REMOTE_ENABLE_TLS={{- if .Values.backup.remote.tls }}True{{-else -}}False{{- end }} + BACKUP_REMOTE_ENABLE_TLS={{- if .Values.backup.remote.tls }}True{{- else -}}False{{- end }} {{- end }} {{- end }} diff --git a/lifemonitor/commands/backup.py b/lifemonitor/commands/backup.py index 75d1f3127..919c05ec1 100644 --- a/lifemonitor/commands/backup.py +++ b/lifemonitor/commands/backup.py @@ -145,7 +145,7 @@ def crates_cmd(directory, *args, **kwargs): def backup_crates(config, directory, *args, **kwargs): assert config.get("DATA_WORKFLOWS", None), "DATA_WORKFLOWS not configured" - rocrate_source_path = config.get("DATA_WORKFLOWS").strip('/') + rocrate_source_path = config.get("DATA_WORKFLOWS").removesuffix('/') os.makedirs(directory, exist_ok=True) result = subprocess.run(f'rsync -avh --delete {rocrate_source_path}/ {directory} ', shell=True, capture_output=True) if result.returncode == 0: @@ -167,7 +167,7 @@ def auto(config: Config): sys.exit(0) # set paths - base_path = base_path.strip('/') # remove trailing '/' + base_path = base_path.removesuffix('/') # remove trailing '/' db_backups = f"{base_path}/db" rc_backups = f"{base_path}/crates" logger.debug("Backup paths: %r - %r - %r", base_path, db_backups, rc_backups) From dffa602993a37c34d01bdd9d5f6921093b5bf87b Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 1 Apr 2022 00:18:24 +0200 Subject: [PATCH 55/58] Update output messages --- lifemonitor/commands/backup.py | 4 +++- lifemonitor/utils.py | 7 +++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lifemonitor/commands/backup.py b/lifemonitor/commands/backup.py index 919c05ec1..a40b62432 100644 --- a/lifemonitor/commands/backup.py +++ b/lifemonitor/commands/backup.py @@ -93,6 +93,7 @@ def __remote_synch__(source: str, target: str, try: ftp_utils = FtpUtils(host, user, password, enable_tls) ftp_utils.sync(source, target) + print("Synch of local '%s' with remote '%s' completed!" % (source, target)) return 0 except Exception as e: logger.debug(e) @@ -149,7 +150,7 @@ def backup_crates(config, directory, *args, **kwargs): os.makedirs(directory, exist_ok=True) result = subprocess.run(f'rsync -avh --delete {rocrate_source_path}/ {directory} ', shell=True, capture_output=True) if result.returncode == 0: - print("Created backup of workflow RO-Crate @ '%s'" % rocrate_source_path) + print("Created backup of workflow RO-Crates @ '%s'" % directory) synch = kwargs.pop('synch', False) if synch: logger.debug("Remaining args: %r", kwargs) @@ -191,6 +192,7 @@ def auto(config: Config): if os.path.getmtime(file) < now - int(retain_days) * 86400: logger.debug("Removing %s", file.absolute()) os.remove(file.absolute()) + logger.info("File %s removed from remote site", file.absolute()) # synch with a remote site if config.get("BACKUP_REMOTE_PATH", None): # check REMOTE_* params diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index 9e26e114f..ba05b1a67 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -578,14 +578,13 @@ def sync(self, source, target): logger.debug("Checking: %r - %r", remote_time, local_time) if local_time <= remote_time: upload_file = False + logger.debug("File %s not changed... skip upload", remote_file_path) else: - logger.debug("Not changed %s", remote_file_path) - except Exception: - logger.debug("File %s doesn't exist @ remote path %s", name, remote_file_path) + logger.debug("File %s changed... it requires to be reuploaded", remote_file_path) if upload_file: with open(local_path, 'rb') as fh: self.ftp.storbinary('STOR %s' % remote_file_path, fh) - logger.debug("Local file '%s' uploaded on remote @ %s", local_path, remote_file_path) + logger.info("Local file '%s' uploaded on remote @ %s", local_path, remote_file_path) # remove obsolete files on the remote target self.remove_obsolete_remote_files(source, target) From 316b71bc2e7ed1ead70f30a6ef0733b2e69774c8 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 1 Apr 2022 11:18:38 +0200 Subject: [PATCH 56/58] Optimize and fix timestamp check --- lifemonitor/utils.py | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index ba05b1a67..4e8ab5224 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -35,6 +35,7 @@ import urllib import uuid import zipfile +from datetime import datetime from importlib import import_module from os.path import basename, dirname, isfile, join from typing import List @@ -522,6 +523,7 @@ def __init__(self, host, user, password, enable_tls) -> None: self.user = user self.passwd = password self.tls_enabled = enable_tls + self._metadata_remote_files = {} def __del__(self): if self._ftp: @@ -551,6 +553,18 @@ def is_dir(self, path) -> bool: finally: self.ftp.cwd(cwd) + def get_file_metadata(self, directory, filename, use_cache=False): + metadata = self._metadata_remote_files.get(directory, False) if use_cache else None + if not metadata: + metadata = [_ for _ in self.ftp.mlsd(directory)] + self._metadata_remote_files[directory] = metadata + for f in metadata: + if f[0] == filename: + fmeta = f[1] + logger.debug("File metadata: %r", fmeta) + return fmeta + return None + def sync(self, source, target): for root, dirs, files in os.walk(source, topdown=True): for name in dirs: @@ -572,15 +586,24 @@ def sync(self, source, target): logger.debug("Remote filepath: %s", remote_file_path) upload_file = True try: - timestamp = self.ftp.voidcmd(f"MDTM {remote_file_path}")[4:].strip() - remote_time = parser.parse(timestamp).timestamp() - local_time = os.path.getmtime(local_path) - logger.debug("Checking: %r - %r", remote_time, local_time) - if local_time <= remote_time: - upload_file = False + metadata = self.get_file_metadata( + os.path.dirname(remote_file_path), name, use_cache=True) + if metadata: + timestamp = metadata['modify'] + remote_time = parser.parse(timestamp).isoformat(' ', 'seconds') + local_time = datetime.utcfromtimestamp(os.path.getmtime(local_path)).isoformat(' ', 'seconds') + logger.debug("Checking: %r - %r", remote_time, local_time) + if local_time <= remote_time: + upload_file = False logger.debug("File %s not changed... skip upload", remote_file_path) - else: + else: + self.ftp.delete(remote_file_path) logger.debug("File %s changed... it requires to be reuploaded", remote_file_path) + else: + logger.debug("File %s doesn't exist @ remote path %s", name, remote_file_path) + except Exception as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) if upload_file: with open(local_path, 'rb') as fh: self.ftp.storbinary('STOR %s' % remote_file_path, fh) From 7cafa4ef12ed485fe1645a086229dfb40f666baa Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 1 Apr 2022 14:40:33 +0200 Subject: [PATCH 57/58] Fix help --- lifemonitor/commands/backup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lifemonitor/commands/backup.py b/lifemonitor/commands/backup.py index a40b62432..1b0b11471 100644 --- a/lifemonitor/commands/backup.py +++ b/lifemonitor/commands/backup.py @@ -76,12 +76,12 @@ def synch_otptions(func): func = optgroup.option('-t', '--target', default="/", show_default=True, help="Remote target path")(func) func = optgroup.option('-p', '--password', cls=RequiredIf, required_if='synch', - help="Password fot the FTPS account")(func) + help="Password of the FTP account")(func) func = optgroup.option('-u', '--user', cls=RequiredIf, required_if='synch', - help="Username of the FTPS account")(func) + help="Username of the FTP account")(func) func = optgroup.option('-h', '--host', cls=RequiredIf, required_if='synch', - help="Hostame of the FTPS server")(func) - func = optgroup.group('\nSettings to connect with a remote site via FTPS')(func) + help="Hostame of the FTP server")(func) + func = optgroup.group('\nSettings to connect with a remote site via FTP or FTPS')(func) func = click.option('-s', '--synch', default=False, show_default=True, is_flag=True, help="Enable sync with a remote FTPS server")(func) return func From 71beeffec75f5d07942abb6fa3f6124e374d98dd Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 1 Apr 2022 14:57:01 +0200 Subject: [PATCH 58/58] Fix missing DATA_WORKFLOWS property on default settings --- settings.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/settings.conf b/settings.conf index 06e7511a6..834da225b 100644 --- a/settings.conf +++ b/settings.conf @@ -66,6 +66,9 @@ MAIL_USE_TLS=False MAIL_USE_SSL=True MAIL_DEFAULT_SENDER='' +# Storage path of workflow RO-Crates +# DATA_WORKFLOWS = "./data" + # Cache settings CACHE_REDIS_DB=0 CACHE_DEFAULT_TIMEOUT=300