release v0.5.1

docker/lifemonitor.Dockerfile

@@ -4,6 +4,7 @@ FROM python:3.7-buster as base
 RUN apt-get update -q \
  && apt-get install -y --no-install-recommends \
         bash \
+        redis-tools \
         postgresql-client-11 \
  && apt-get clean -y && rm -rf /var/lib/apt/lists
 
@@ -31,12 +32,15 @@ WORKDIR /lm
 
 # Copy utility scripts
 COPY --chown=root:root \
-    docker/wait-for-postgres.sh docker/lm_entrypoint.sh docker/worker_entrypoint.sh \
+    docker/wait-for-postgres.sh \
+    docker/wait-for-redis.sh \
+    docker/lm_entrypoint.sh docker/worker_entrypoint.sh \
     /usr/local/bin/
 
 # Update permissions and install optional certificates
 RUN chmod 755 \
       /usr/local/bin/wait-for-postgres.sh \
+      /usr/local/bin/wait-for-redis.sh \
       /usr/local/bin/lm_entrypoint.sh \
       /usr/local/bin/worker_entrypoint.sh \
     && certs=$(ls *.crt 2> /dev/null) \

docker/wait-for-redis.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+HOST=${REDIS_HOST:-redis}
+PORT=${REDIS_PORT:-6379}
+PASSWORD=${REDIS_PASSWORD:-foobar}
+
+echo "Waiting for Redis @$HOST:$PORT ..."
+echo "Try ping Redis... "
+PONG=`redis-cli -h $HOST -p $PORT -a $PASSWORD ping | grep PONG`
+while [ -z "$PONG" ]; do
+    sleep 1
+    echo "Retry Redis ping... "
+    PONG=`redis-cli -h $HOST -p $PORT -a $PASSWORD ping | grep PONG`
+done
+echo "Redis at host '$HOST', port '$PORT' fully started."

k8s/Chart.yaml

@@ -12,7 +12,7 @@ version: 0.5.0
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 0.5.0
+appVersion: 0.5.1
 
 # Chart dependencies
 dependencies:

k8s/templates/_helpers.tpl

@@ -92,6 +92,10 @@ Define environment variables shared by some pods.
   value: "{{ .Values.postgresql.postgresqlDatabase }}"
 - name: REDIS_HOST
   value: "{{ .Release.Name }}-redis-master"
+- name: REDIS_PORT
+  value: "{{ .Values.redis.master.service.port }}"
+- name: REDIS_PASSWORD
+  value: "{{ .Values.redis.auth.password }}"
 - name: WORKER_PROCESSES
   value: "{{ .Values.worker.processes }}"
 - name: LIFEMONITOR_TLS_KEY

k8s/templates/backend-deployment.yaml

@@ -33,9 +33,16 @@ spec:
         {{- toYaml .Values.lifemonitor.podSecurityContext | nindent 8 }}
       initContainers:
         - name: init
-          image: "crs4/k8s-wait-for:latest"
-          imagePullPolicy: IfNotPresent
-          args: ["job", "{{ include "chart.fullname" . }}-init"]
+          securityContext:
+            {{- toYaml .Values.lifemonitor.securityContext | nindent 12 }}
+          image: {{ include "chart.lifemonitor.image" . }}
+          imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }}
+          command: ["/bin/sh","-c"]
+          args: ["wait-for-redis.sh && wait-for-postgres.sh && flask init wait-for-db"]
+          env:
+          {{- include "lifemonitor.common-env" . | nindent 12 }}
+          volumeMounts:
+          {{- include "lifemonitor.common-volume-mounts" . | nindent 12 }}
       containers:
         - name: app
           securityContext:

k8s/templates/job-init.yaml

@@ -6,6 +6,12 @@ metadata:
     app.kubernetes.io/name: {{ include "chart.name" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
+  annotations:
+    # This is what defines this resource as a hook. Without this line, the
+    # job is considered part of the release.
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": hook-succeeded    
 spec:
   template:
     spec:
@@ -14,7 +20,7 @@ spec:
         image: {{ include "chart.lifemonitor.image" . }}
         imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }}
         command: ["/bin/sh","-c"]
-        args: ["wait-for-postgres.sh && flask init db && flask task-queue reset"]
+        args: ["wait-for-redis.sh && wait-for-postgres.sh && flask init db && flask task-queue reset"]
         env:
         {{- include "lifemonitor.common-env" . | nindent 10 }}
         volumeMounts:

k8s/templates/worker-deployment.yaml

@@ -30,9 +30,16 @@ spec:
         {{- toYaml .Values.worker.podSecurityContext | nindent 8 }}
       initContainers:
         - name: init
-          image: "crs4/k8s-wait-for:latest"
-          imagePullPolicy: IfNotPresent
-          args: ["job", "{{ include "chart.fullname" . }}-init"]
+          securityContext:
+            {{- toYaml .Values.lifemonitor.securityContext | nindent 12 }}
+          image: {{ include "chart.lifemonitor.image" . }}
+          imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }}
+          command: ["/bin/sh","-c"]
+          args: ["wait-for-redis.sh && wait-for-postgres.sh && flask init wait-for-db"]
+          env:
+          {{- include "lifemonitor.common-env" . | nindent 12 }}
+          volumeMounts:
+          {{- include "lifemonitor.common-volume-mounts" . | nindent 12 }}
       containers:
         - name: worker
           securityContext:

lifemonitor/api/models/testsuites/testinstance.py

@@ -107,7 +107,7 @@ def last_test_build(self):
 
     @cached(timeout=Timeout.NONE, client_scope=False, transactional_update=True)
     def get_last_test_build(self):
-        builds = self.get_test_builds()
+        builds = self.get_test_builds(limit=10)
         return builds[0] if builds and len(builds) > 0 else None
 
     @cached(timeout=Timeout.NONE, client_scope=False, transactional_update=True)

lifemonitor/cache.py

@@ -226,6 +226,10 @@ class Cache(object):
     _ignore_cache_values = False
     # Reference to Redis back-end
     __cache__ = None
+    # Reference to the hash function to be used
+    _hash_function = None
+    # Reference to the current app
+    app: Flask = None
 
     @classmethod
     def init_backend(cls, config):
@@ -250,17 +254,35 @@ def get_backend(cls) -> redis.Redis:
     @classmethod
     def init_app(cls, app: Flask):
         cls.init_backend(app.config)
+        cls.app = app
+        cls._hash_function = cls.hash_function()
         if cls.__cache__ is not None:
             cls.reset_locks()
 
     def __init__(self, parent: Cache = None) -> None:
         self._local = _current_transaction
         self._parent = parent
 
-    @staticmethod
-    def _make_key(key: str, prefix: str = CACHE_PREFIX) -> str:
+    @classmethod
+    def _make_key(cls, key: str, prefix: str = CACHE_PREFIX) -> str:
+        if cls._hash_function:
+            key = cls._hash_function(key.encode()).hexdigest()
         return f"{prefix}{key}"
 
+    @classmethod
+    def hash_function(cls):
+        if not cls.app:
+            raise IllegalStateException("App not configured")
+        hash_algorithm = cls.app.config.get("CACHE_KEY_HASH_ALGORITHM", 'sha256')
+        if not hash_algorithm:
+            return None
+        try:
+            import hashlib
+            return getattr(hashlib, hash_algorithm)
+        except Exception:
+            logger.error("hashlib module doesn't support the hash algorithm '%r'", hash_algorithm)
+            return None
+
     @property
     def parent(self) -> Cache:
         return self._parent
@@ -427,15 +449,12 @@ def init_cache(app: Flask):
 
 def make_cache_key(func=None, client_scope=True, args=None, kwargs=None) -> str:
     from lifemonitor.auth import current_registry, current_user
-
-    hash_enabled = not logger.isEnabledFor(logging.DEBUG)
     fname = "" if func is None \
         else func if isinstance(func, str) \
         else f"{func.__module__}.{func.__name__}" if callable(func) else str(func)
     logger.debug("make_key func: %r", fname)
     logger.debug("make_key args: %r", args)
     logger.debug("make_key kwargs: %r", kwargs)
-    logger.debug("make_key hash enabled: %r", hash_enabled)
     result = ""
     if client_scope:
         client_id = ""
@@ -447,15 +466,15 @@ def make_cache_key(func=None, client_scope=True, args=None, kwargs=None) -> str:
             client_id += "anonymous"
         if request:
             client_id += f"@{request.remote_addr}"
-        result += f"{hash(client_id) if hash_enabled else client_id}::"
+        result += f"{client_id}::"
     if func:
         result += fname
     if args:
         args_str = "-".join([str(_) for _ in args])
-        result += f"#{hash(args_str) if hash_enabled else args_str}"
+        result += f"#{args_str}"
     if kwargs:
         kwargs_str = "-".join([f"{k}={str(v)}" for k, v in kwargs.items()])
-        result += f"#{hash(kwargs_str) if hash_enabled else kwargs_str}"
+        result += f"#{kwargs_str}"
     logger.debug("make_key calculated key: %r", result)
     return result
 
@@ -547,3 +566,11 @@ def cache(self) -> Cache:
         if self._cache is None:
             self._cache = Cache(parent=cache)
         return self._cache
+
+    def __getstate__(self):
+        data = self.__dict__.copy()
+        try:
+            del data['_cache']
+        except KeyError:
+            pass
+        return data

lifemonitor/commands/db.py

@@ -75,3 +75,22 @@ def init_db(revision):
             admin.password = current_app.config["LIFEMONITOR_ADMIN_PASSWORD"]
             db.session.add(admin)
             db.session.commit()
+
+
+@blueprint.cli.command('wait-for-db')
+@with_appcontext
+def wait_for_db():
+    """
+    Wait until that DB is initialized
+    """
+    from lifemonitor.db import db_initialized, db_revision
+
+    is_initialized = False
+    while not is_initialized:
+        is_initialized = db_initialized()
+    logger.info("DB initialized")
+
+    current_revision = None
+    while current_revision is None:
+        current_revision = db_revision()
+    logger.info(f"Current revision: {current_revision}")

lifemonitor/static/src/package.json

@@ -1,7 +1,7 @@
 {
     "name": "lifemonitor",
     "description": "Workflow Testing Service",
-    "version": "0.5.0",
+    "version": "0.5.1",
     "license": "MIT",
     "author": "CRS4",
     "main": "../dist/js/lifemonitor.min.js",

lifemonitor/tasks/tasks.py

@@ -93,11 +93,11 @@ def check_last_build():
                 logger.info("Updating workflow: %r", w)
                 for i in s.test_instances:
                     with i.cache.transaction(str(i)):
-                        builds = i.get_test_builds()
-                        logger.debug("Updating latest builds: %r", builds)
+                        builds = i.get_test_builds(limit=10)
+                        logger.info("Updating latest builds: %r", builds)
                         for b in builds:
-                            logger.debug("Updating build: %r", i.get_test_build(b.id))
-                        logger.debug("Updating latest build: %r", i.last_test_build)
+                            logger.info("Updating build: %r", i.get_test_build(b.id))
+                        logger.info("Updating latest build: %r", i.last_test_build)
         except Exception as e:
             logger.error("Error when executing task 'check_last_build': %s", str(e))
             if logger.isEnabledFor(logging.DEBUG):

specs/api.yaml

@@ -3,7 +3,7 @@
 openapi: "3.0.0"
 
 info:
-  version: "0.5.0"
+  version: "0.5.1"
   title: "Life Monitor API"
   description: |
     *Workflow sustainability service*
@@ -18,7 +18,7 @@ info:
 servers:
   - url: /
     description: >
-      Version 0.5.0 of API.
+      Version 0.5.1 of API.
 
 tags:
   - name: Registries

tests/unit/cache/test_cache.py

@@ -181,14 +181,14 @@ def cache_last_build_update(app, w, user1, check_cache_size=True, index=0,
                     assert cache.get_current_transaction() == t, "Unexpected transaction"
                     assert i.cache.get_current_transaction() == t, "Unexpected transaction"
 
-                    cache_key = make_cache_key(i.get_test_builds, client_scope=False, args=[i])
+                    cache_key = make_cache_key(i.get_test_builds, client_scope=False, args=[i], kwargs={'limit': 10})
                     logger.debug("The cache key: %r", cache_key)
 
                     #############################################################################
                     # latest builds (first call)
                     #############################################################################
                     logger.debug("\n\nGetting latest builds (first call)...")
-                    builds = i.get_test_builds()
+                    builds = i.get_test_builds(limit=10)
                     logger.debug("Getting latest builds (first call): %r\n", builds)
                     assert t.has(cache_key), "The key should be in the current transaction"
                     cache_size = cache.size()
@@ -204,7 +204,7 @@ def cache_last_build_update(app, w, user1, check_cache_size=True, index=0,
                     # latest builds (second call)
                     #############################################################################
                     logger.debug("\n\nGetting latest builds (second call)...")
-                    builds = i.get_test_builds()
+                    builds = i.get_test_builds(limit=10)
                     logger.debug("Getting latest builds (second call): %r\n", builds)
                     assert i.cache.get_current_transaction() == t, "Unexpected transaction"
                     assert t.has(cache_key), "The key should be in the current transaction"
@@ -219,7 +219,7 @@ def cache_last_build_update(app, w, user1, check_cache_size=True, index=0,
                     # latest builds (third call)
                     #############################################################################
                     logger.debug("\n\nGetting latest builds (third call)...")
-                    builds = i.get_test_builds()
+                    builds = i.get_test_builds(limit=10)
                     logger.debug("Getting latest builds (third call): %r\n", builds)
                     assert i.cache.get_current_transaction() == t, "Unexpected transaction"
                     assert t.has(cache_key), "The key should be in the current transaction"