Update backend dependencies

.circleci/config.yml

@@ -27,7 +27,7 @@ jobs:
       POSTGRES_PASSWORD: postgres
       TEST: "True"
     docker:
-      - image: python:3.7.5
+      - image: python:3.10
       - image: postgres:12
         environment:
           POSTGRES_USER: postgres

garden.yml

@@ -18,8 +18,8 @@ environments:
     providers:
       - name: local-kubernetes
     variables:
-      postgres-user: admin
-      postgres-password: admin
+      postgres-user: postgres
+      postgres-password: mypw
       node-hosts: ""
       node-name: minikube
       schedulerDependencies: ['postgres']

services/postgres/garden.yaml

@@ -1,14 +1,16 @@
 kind: Module
 description: Helm chart for the postgresql database
 type: helm
+atomicInstall: false
 name: postgres
 repo: https://charts.bitnami.com/bitnami
 chart: postgresql
 releaseName: db
-version: 8.6.2
 values:
-  postgresqlPassword: ${var.postgres-password}
-  postgresqlUsername: ${var.postgres-user}
+  global:
+    postgresql:
+      auth:
+        postgresPassword: mypw
   master:
     nodeSelector:
       "kubernetes.io/hostname": ${var.node-name}

services/python-images/backend.Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7
+FROM python:3.10
 
 WORKDIR /app
 

services/python-images/requirements.txt

@@ -1,132 +1,131 @@
-absl-py==0.11.0
-aiohttp==3.6.2
+absl-py==1.3.0
+aiohttp==3.8.3
 alabaster==0.7.12
-alembic==1.0.6
-aniso8601==4.0.1
-appnope==0.1.2
-async-timeout==3.0.1
-atomicwrites==1.2.1
-attrs==18.2.0
-Babel==2.9.0
+alembic==1.9.1
+aniso8601==9.0.1
+appnope==0.1.3
+async-timeout==4.0.2
+atomicwrites==1.4.1
+attrs==22.2.0
+Babel==2.11.0
 backcall==0.2.0
-bidict==0.21.2
-cachetools==4.2.0
+bidict==0.22.1
+cachetools==5.2.1
 causaldag==0.1a162
-certifi==2018.11.29
-chardet==3.0.4
-Click==7.0
-codecov==2.0.15
+certifi==2022.12.7
+chardet==5.1.0
+Click==8.1.3
+codecov==2.1.12
 conditional-independence==0.1a5
-coverage==5.3
-cycler==0.10.0
+coverage==7.0.4
+cycler==0.11.0
 dataclasses==0.6
-decorator==4.3.2
-dnspython==2.0.0
-docutils==0.16
-eventlet==0.25.1
-factory-boy==2.11.1
-Faker==1.0.0
-flake8==3.8.4
-Flask==1.0.2
-Flask-Migrate==2.3.1
-Flask-RESTful==0.3.6
+decorator==5.1.1
+dnspython==2.2.1
+docutils
+eventlet==0.33.2
+factory-boy==3.2.1
+Faker==15.3.4
+flake8==6.0.0
+Flask==2.2.2
+Flask-Migrate==4.0.1
+Flask-RESTful==0.3.9
 flask-restful-swagger-2==0.35
-Flask-SocketIO==4.2.1
-Flask-SQLAlchemy==2.4.4
-frozendict==1.2
+Flask-SocketIO==5.3.2
+Flask-SQLAlchemy==3.0.2
+flask_marshmallow
+frozendict==2.3.4
 future==0.18.2
-google-auth==1.23.0
+google-auth==2.15.0
 graphical-model-learning==0.1a7
 graphical-models==0.1a5
-greenlet==0.4.17
-idna==2.7
-ijson==2.3
-imagesize==1.2.0
-importlib-metadata==3.1.1
-ipdb==0.13.4
-ipython==7.20.0
+greenlet==2.0.1
+idna==3.4
+ijson==3.1.4
+imagesize==1.4.1
+importlib-metadata==6.0.0
+ipdb==0.13.11
+ipython==8.8.0
 ipython-genutils==0.2.0
-itsdangerous==1.1.0
-jedi==0.18.0
-Jinja2==2.10
-joblib==1.0.0
-kiwisolver==1.3.1
-kubernetes==10.0.1
-Mako==1.0.7
-MarkupSafe==1.1.0
-marshmallow==2.16.3
-marshmallow-sqlalchemy==0.15.0
-matplotlib==3.3.4
-mccabe==0.6.1
-monotonic==1.5
-more-itertools==4.3.0
-multidict==4.7.6
-netrd==0.2.2
-networkx==2.5
-numexpr==2.7.2
-numpy==1.16.0
-numpydoc==1.1.0
-oauthlib==3.1.0
-ortools==8.1.8487
-packaging==20.9
-pandas==1.1.4
-parso==0.8.1
+itsdangerous==2.1.2
+jedi==0.18.2
+Jinja2==3.1.2
+joblib==1.2.0
+kiwisolver==1.4.4
+kubernetes==25.3.0
+Mako==1.2.4
+MarkupSafe==2.1.1
+marshmallow==3.19.0
+marshmallow-sqlalchemy==0.28.1
+matplotlib==3.6.2
+mccabe==0.7.0
+monotonic==1.6
+more-itertools==9.0.0
+multidict==6.0.4
+networkx==3.0
+numexpr==2.8.4
+numpy==1.24.1
+numpydoc==1.5.0
+oauthlib==3.2.2
+ortools==9.5.2237
+packaging==23.0
+pandas==1.5.2
+parso==0.8.3
 pexpect==4.8.0
 pickleshare==0.7.5
-Pillow==8.1.0
-pluggy==0.8.0
-progressbar2==3.53.1
-prompt-toolkit==3.0.16
-protobuf==3.14.0
-psutil==5.4.8
-psycopg2==2.8.6
-psycopg2-binary==2.8.6
+Pillow==9.4.0
+pluggy==1.0.0
+progressbar2==4.2.0
+prompt-toolkit==3.0.36
+protobuf==4.21.12
+psutil==5.9.4
+psycopg2-binary==2.9.5
 ptyprocess==0.7.0
-py==1.7.0
+py==1.11.0
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
-pycodestyle==2.6.0
-pyflakes==2.2.0
+pycodestyle==2.10.0
+pyflakes==3.0.1
 pygam==0.8.0
-Pygments==2.7.4
+Pygments==2.14.0
 pyhdb==0.3.4
-pyparsing==2.4.7
-pytest==3.10.1
-pytest-cov==2.6.1
+pyparsing==3.0.9
+pytest==7.2.0
+pytest-cov==4.0.0
 pytest-ordering==0.6
-python-dateutil==2.7.5
-python-editor==1.0.3
-python-engineio==4.0.0
-python-socketio==5.0.1
-python-utils==2.5.6
-pytz==2018.7
-PyYAML==5.3.1
-requests==2.20.1
-requests-oauthlib==1.3.0
-rsa==4.6
-scikit-learn==0.24.1
-scipy==1.5.4
-six==1.11.0
-snowballstemmer==2.1.0
-Sphinx==2.0.1
-sphinx-rtd-theme==0.5.1
-sphinxcontrib-applehelp==1.0.2
+python-dateutil==2.8.2
+python-editor==1.0.4
+python-engineio==4.3.4
+python-socketio==5.7.2
+python-utils==3.4.5
+pytz==2022.7
+PyYAML==6.0
+requests==2.28.1
+requests-oauthlib==1.3.1
+rsa==4.9
+scikit-learn==1.2.0
+scipy==1.10.0
+six==1.16.0
+snowballstemmer==2.2.0
+Sphinx
+sphinx-rtd-theme==1.1.1
+sphinxcontrib-applehelp==1.0.3
 sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-htmlhelp==2.0.0
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.4
-SQLAlchemy==1.3.20
-sqlalchemy-hana==0.3.0
-text-unidecode==1.2
-threadpoolctl==2.1.0
-tqdm==4.57.0
-traitlets==5.0.5
+sphinxcontrib-serializinghtml==1.1.5
+SQLAlchemy==1.4.46
+sqlalchemy-hana==0.5.0
+text-unidecode==1.3
+threadpoolctl==3.1.0
+tqdm==4.64.1
+traitlets==5.8.1
 typing==3.7.4.3
-typing-extensions==3.7.4.3
-urllib3==1.24.2
+typing-extensions==4.4.0
+urllib3==1.26.13
 wcwidth==0.2.5
-websocket-client==0.55.0
-Werkzeug==0.14.1
-yarl==1.6.3
-zipp==3.4.0
+websocket-client==1.4.2
+Werkzeug==2.2.2
+yarl==1.8.2
+zipp==3.11.0

services/python-images/scheduler.Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7
+FROM python:3.10
 
 WORKDIR /app
 

services/python-images/setup_algorithms.py

@@ -12,22 +12,24 @@ def set_up_algorithms(db):
         # The check here is necessary, because without running the migrations,
         # the DB might be empty. To be able to run the migrations though, it is necessary to be able
         # to initialize the app.
-        if db.engine.dialect.has_table(db.session, Algorithm.__table__.name):
+        if db.engine.dialect.has_table(db.session.connection(), Algorithm.__table__.name):
             with open('conf/algorithms.json') as f:
                 algorithms = json.load(f)
                 for algorithm in algorithms:
-                    data, errors = AlgorithmSchema().load(algorithm)
-                    if len(errors) > 0:
-                        raise InvalidInputData(payload=errors)
-                    alg_in_db = db.session.query(Algorithm).filter(Algorithm.package == data['package'])\
-                        .filter(Algorithm.function == data['function']).one_or_none()
-                    if not alg_in_db:
-                        alg = Algorithm(**data)
-                        db.session.add(alg)
-                        print(f'==> Function `{data["function"]}` from Package `{data["package"]}` added…')
-                    else:
-                        alg_in_db.update(data)
-                        print(f'==> Function `{data["function"]}` from Package `{data["package"]}` updated…')
+                    print(AlgorithmSchema().load(algorithm))
+                    try:
+                        data = AlgorithmSchema().load(algorithm)
+                        alg_in_db = db.session.query(Algorithm).filter(Algorithm.package == data['package'])\
+                            .filter(Algorithm.function == data['function']).one_or_none()
+                        if not alg_in_db:
+                            alg = Algorithm(**data)
+                            db.session.add(alg)
+                            print(f'==> Function `{data["function"]}` from Package `{data["package"]}` added…')
+                        else:
+                            alg_in_db.update(data)
+                            print(f'==> Function `{data["function"]}` from Package `{data["package"]}` updated…')
+                    except Exception as err:
+                        raise InvalidInputData(payload=err)
                 db.session.commit()
 
 

services/python-images/src/app.py

@@ -9,3 +9,4 @@
 fact = AppFactory()
 [app, socketio] = fact.up()
 db = fact.db
+ma = fact.ma

services/python-images/src/db.py

@@ -1,3 +1,5 @@
+from flask_marshmallow import Marshmallow
 from flask_sqlalchemy import SQLAlchemy
 
 db = SQLAlchemy()
+ma = Marshmallow()

services/python-images/src/master/appfactory.py

@@ -6,7 +6,7 @@
 from flask_socketio import SocketIO
 from sqlalchemy import event
 
-from src.db import db
+from src.db import db, ma
 from src.master.helpers.io import InvalidInputData
 from .routes import set_up_routes
 from src.models import Experiment, Dataset, ExperimentJob
@@ -20,6 +20,10 @@ def set_up_db(self):
 
         self.migrate = Migrate(self.app, db)
 
+    def set_up_ma(self):
+        self.ma = ma
+        self.ma.init_app(self.app)
+
     def set_up_app(self):
         self.app = Flask(__name__, static_folder=os.path.join(os.getcwd(), 'static'), static_url_path='/static')
         self.app.config.from_object('src.master.config')
@@ -111,19 +115,21 @@ def set_up_api(self):
     def set_up_error_handlers(self):
         @self.app.errorhandler(InvalidInputData)
         def handle_invalid_usage(error):
-            response = jsonify(error.to_dict())
+            response = jsonify(error)
             response.status_code = error.status_code
             return response
 
     def up(self):
         self.set_up_app()
         self.set_up_api()
         self.set_up_db()
+        self.set_up_ma()
         self.set_up_error_handlers()
         self.set_up_socketio()
         return [self.app, self.socketio]
 
     def migration_up(self):
         self.set_up_app()
         self.set_up_db()
+        self.set_up_ma()
         return self.app

services/python-images/src/master/helpers/graph.py

@@ -18,7 +18,7 @@ def creates_collider(graph, x, fixed_parents, bi_parents, combination):
                            if not graph.has_edge(par, n)}  # No bidirectional edges
         papa = crazy_shit - {x}
         if len(papa) > 0:
-            res = any([not(graph.has_edge(x, e) or graph.has_edge(e, x)) for e in papa])
+            res = any([not (graph.has_edge(x, e) or graph.has_edge(e, x)) for e in papa])
     return res