diff --git a/.gitignore b/.gitignore index 034bc54b5..5071ad7f7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Created by .ignore support plugin (hsz.mobi) .DS_Store +.sync .idea .vscode .cache diff --git a/k8s/Chart.yaml b/k8s/Chart.yaml index c17577b74..e0715ded5 100644 --- a/k8s/Chart.yaml +++ b/k8s/Chart.yaml @@ -7,12 +7,12 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.5.0 +version: 0.6.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 0.5.1 +appVersion: 0.6.0 # Chart dependencies dependencies: diff --git a/k8s/templates/secret.yaml b/k8s/templates/secret.yaml index 73af94f14..5bee20989 100644 --- a/k8s/templates/secret.yaml +++ b/k8s/templates/secret.yaml @@ -17,6 +17,9 @@ stringData: EXTERNAL_SERVER_URL={{ .Values.externalServerName }} {{- end }} + # Base URL of the LifeMonitor web app associated with this back-end instance + WEBAPP_URL={{ .Values.webappUrl }} + # Normally, OAuthLib will raise an InsecureTransportError if you attempt to use OAuth2 over HTTP, # rather than HTTPS. Setting this environment variable will prevent this error from being raised. # This is mostly useful for local testing, or automated tests. Never set this variable in production. @@ -57,6 +60,15 @@ stringData: CACHE_WORKFLOW_TIMEOUT={{ .Values.cache.timeout.workflow }} CACHE_BUILD_TIMEOUT={{ .Values.cache.timeout.build }} + # Email sender + MAIL_SERVER={{ .Values.mail.server }} + MAIL_PORT={{ .Values.mail.port }} + MAIL_USERNAME={{ .Values.mail.username }} + MAIL_PASSWORD={{ .Values.mail.password }} + MAIL_USE_TLS={{- if .Values.mail.tls -}}True{{- else -}}False{{- end }} + MAIL_USE_SSL={{- if .Values.mail.ssl -}}True{{- else -}}False{{- end }} + MAIL_DEFAULT_SENDER={{ .Values.mail.default_sender }} + # Set admin credentials LIFEMONITOR_ADMIN_PASSWORD={{ .Values.lifemonitor.administrator.password }} diff --git a/k8s/values.yaml b/k8s/values.yaml index 151902555..4a4fdd7ef 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -10,6 +10,9 @@ fullnameOverride: "" # used as base_url on all the links returned by the API externalServerName: &hostname api.lifemonitor.eu +# Base URL of the LifeMonitor web app associated with this back-end instance +webappUrl: &webapp_url https://app.lifemonitor.eu + # global storage class storageClass: &storageClass "-" @@ -68,6 +71,16 @@ cache: workflow: 1800 build: 84600 +# Email settings +mail: + server: "" + port: 465 + username: "" + password: "" + tls: false + ssl: true + default_sender: "" + lifemonitor: replicaCount: 1 diff --git a/lifemonitor/api/controllers.py b/lifemonitor/api/controllers.py index 097e66c29..6977cb3db 100644 --- a/lifemonitor/api/controllers.py +++ b/lifemonitor/api/controllers.py @@ -18,9 +18,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import os import base64 import logging +import os import tempfile import connexion @@ -29,8 +29,10 @@ from flask import Response, request from lifemonitor.api import serializers from lifemonitor.api.services import LifeMonitor -from lifemonitor.auth import authorized, current_registry, current_user +from lifemonitor.auth import (EventType, authorized, current_registry, + current_user) from lifemonitor.auth import serializers as auth_serializers +from lifemonitor.auth.models import Subscription from lifemonitor.auth.oauth2.client.models import \ OAuthIdentityNotFoundException from lifemonitor.cache import Timeout, cached, clear_cache @@ -270,10 +272,41 @@ def user_workflow_subscribe(wf_uuid): response = _get_workflow_or_problem(wf_uuid) if isinstance(response, Response): return response + subscribed = current_user.is_subscribed_to(response.workflow) subscription = lm.subscribe_user_resource(current_user, response.workflow) logger.debug("Created new subscription: %r", subscription) clear_cache() - return auth_serializers.SubscriptionSchema(exclude=('meta', 'links')).dump(subscription), 201 + if not subscribed: + return auth_serializers.SubscriptionSchema(exclude=('meta', 'links')).dump(subscription), 201 + else: + return connexion.NoContent, 204 + + +@authorized +def user_workflow_subscribe_events(wf_uuid, body): + workflow_version = _get_workflow_or_problem(wf_uuid) + + if body is None or not isinstance(body, list): + return lm_exceptions.report_problem(400, "Bad request", + detail=messages.invalid_event_type.format(EventType.all_names())) + try: + events = EventType.from_strings(body) + if isinstance(workflow_version, Response): + return workflow_version + subscription: Subscription = current_user.get_subscription(workflow_version.workflow) + if subscription and subscription.events == events: + return connexion.NoContent, 204 + subscription = lm.subscribe_user_resource(current_user, workflow_version.workflow, events=events) + logger.debug("Updated subscription events: %r", subscription) + clear_cache() + return auth_serializers.SubscriptionSchema(exclude=('meta', 'links')).dump(subscription), 201 + except ValueError as e: + logger.debug(e) + return lm_exceptions.report_problem(400, "Bad request", + detail=messages.invalid_event_type.format(EventType.all_names())) + except Exception as e: + logger.debug(e) + return lm_exceptions.report_problem_from_exception(e) @authorized @@ -513,7 +546,7 @@ def suites_get_by_uuid(suite_uuid): def suites_get_status(suite_uuid): response = _get_suite_or_problem(suite_uuid) return response if isinstance(response, Response) \ - else serializers.SuiteStatusSchema().dump(response.status) + else serializers.SuiteStatusSchema().dump(response) @cached(timeout=Timeout.REQUEST) diff --git a/lifemonitor/api/models/rocrate.py b/lifemonitor/api/models/rocrate.py index 6f512208c..0cdc4ae9b 100644 --- a/lifemonitor/api/models/rocrate.py +++ b/lifemonitor/api/models/rocrate.py @@ -100,7 +100,8 @@ def dataset_name(self): @property def main_entity_name(self): - return self._roc_helper.mainEntity['name'] + mainEntity = self._roc_helper.mainEntity + return mainEntity.get("name", mainEntity.id) if mainEntity else None @property def _roc_helper(self): diff --git a/lifemonitor/api/models/services/jenkins.py b/lifemonitor/api/models/services/jenkins.py index 842080c77..9de557862 100644 --- a/lifemonitor/api/models/services/jenkins.py +++ b/lifemonitor/api/models/services/jenkins.py @@ -27,8 +27,8 @@ import lifemonitor.api.models as models import lifemonitor.exceptions as lm_exceptions - from lifemonitor.lang import messages +from requests.exceptions import ConnectionError import jenkins @@ -114,6 +114,8 @@ def get_project_metadata(self, test_instance: models.TestInstance, fetch_all_bui self.get_job_name(test_instance.resource), fetch_all_builds=fetch_all_builds) except jenkins.JenkinsException as e: raise lm_exceptions.TestingServiceException(f"{self}: {e}") + except ConnectionError as e: + raise lm_exceptions.TestingServiceException(f"Unable to connect to {self}", detail=str(e)) return test_instance._raw_metadata def get_test_builds(self, test_instance: models.TestInstance, limit=10) -> list: diff --git a/lifemonitor/api/models/services/service.py b/lifemonitor/api/models/services/service.py index e0634883b..5b37c61da 100644 --- a/lifemonitor/api/models/services/service.py +++ b/lifemonitor/api/models/services/service.py @@ -128,6 +128,10 @@ def base_url(self): def api_base_url(self): return self.url + @property + def type(self): + return self._type.replace('_testing_service', '').capitalize() + @property def token(self) -> TestingServiceToken: if not self._token: diff --git a/lifemonitor/api/models/status.py b/lifemonitor/api/models/status.py index a4854497a..ffacd6bd3 100644 --- a/lifemonitor/api/models/status.py +++ b/lifemonitor/api/models/status.py @@ -106,6 +106,14 @@ def check_status(suites): "issue": str(e) }) logger.exception(e) + except Exception as e: + availability_issues.append({ + "service": test_instance.testing_service.url, + "resource": test_instance.resource, + "issue": str(e) + }) + logger.exception(e) + # update the current status return status, latest_builds, availability_issues diff --git a/lifemonitor/api/models/workflows.py b/lifemonitor/api/models/workflows.py index a85c8d146..7c60650d5 100644 --- a/lifemonitor/api/models/workflows.py +++ b/lifemonitor/api/models/workflows.py @@ -49,6 +49,7 @@ class Workflow(Resource): public = db.Column(db.Boolean, nullable=True, default=False) external_ns = "external-id:" + _uuidAutoGenerated = True __mapper_args__ = { 'polymorphic_identity': 'workflow' @@ -59,6 +60,7 @@ def __init__(self, uri=None, uuid=None, identifier=None, super().__init__(uri=uri or f"{self.external_ns}", uuid=uuid, version=version, name=name) self.public = public + self._uuidAutoGenerated = uuid is None if identifier is not None: self.external_id = identifier @@ -82,12 +84,13 @@ def latest_version(self) -> WorkflowVersion: def add_version(self, version, uri, submitter: User, uuid=None, name=None, hosting_service: models.WorkflowRegistry = None): if hosting_service: - if self.external_id and hasattr(hosting_service, 'get_external_uuid'): - try: - self.uuid = hosting_service.get_external_uuid(self.external_id, version, submitter) - except RuntimeError as e: - raise lm_exceptions.NotAuthorizedException(details=str(e)) - elif not self.external_id and hasattr(hosting_service, 'get_external_id'): + if self.external_id: + if self._uuidAutoGenerated and hasattr(hosting_service, 'get_external_uuid'): + try: + self.uuid = hosting_service.get_external_uuid(self.external_id, version, submitter) + except RuntimeError as e: + raise lm_exceptions.NotAuthorizedException(details=str(e)) + elif hasattr(hosting_service, 'get_external_id'): try: self.external_id = hosting_service.get_external_id(self.uuid, version, submitter) except lm_exceptions.EntityNotFoundException: diff --git a/lifemonitor/api/serializers.py b/lifemonitor/api/serializers.py index 0162ec5da..ce6f28fe2 100644 --- a/lifemonitor/api/serializers.py +++ b/lifemonitor/api/serializers.py @@ -267,7 +267,7 @@ def get_testing_service(self, obj): logger.debug("Test current obj: %r", obj) assert obj.testing_service, "Missing testing service" return { - 'uuid': obj.testing_service.uuid, + 'uuid': str(obj.testing_service.uuid), 'url': obj.testing_service.url, 'type': obj.testing_service._type.replace('_testing_service', '') } @@ -280,6 +280,14 @@ def remove_skip_values(self, data, **kwargs): } +def format_availability_issues(status: models.WorkflowStatus): + issues = status.availability_issues + logger.info(issues) + if 'not_available' == status.aggregated_status and len(issues) > 0: + return ', '.join([f"{i['issue']}: Unable to get resource '{i['resource']}' from service '{i['service']}'" if 'service' in i and 'resource' in i else i['issue'] for i in issues]) + return None + + class BuildSummarySchema(ResourceMetadataSchema): __envelope__ = {"single": None, "many": None} __model__ = models.TestBuild @@ -287,13 +295,24 @@ class BuildSummarySchema(ResourceMetadataSchema): class Meta: model = models.TestBuild + def __init__(self, *args, self_link: bool = True, exclude_nested=True, **kwargs): + exclude = set(kwargs.pop('exclude', ())) + if exclude_nested: + exclude = exclude.union(('suite', 'workflow')) + super().__init__(*args, self_link=self_link, exclude=tuple(exclude), **kwargs) + build_id = fields.String(attribute="id") suite_uuid = fields.String(attribute="test_instance.test_suite.uuid") status = fields.String() - instance = ma.Nested(TestInstanceSchema(self_link=False, exclude=('meta',)), attribute="test_instance") + instance = ma.Nested(TestInstanceSchema(self_link=False, exclude=('meta',)), + attribute="test_instance") timestamp = fields.String() duration = fields.Integer() links = fields.Method('get_links') + suite = ma.Nested(TestInstanceSchema(self_link=False, + only=('uuid', 'name')), attribute="test_instance.test_suite") + workflow = ma.Nested(WorkflowVersionSchema(self_link=False, only=('uuid', 'name', 'version')), + attribute="test_instance.test_suite.workflow_version") def get_links(self, obj): links = { @@ -304,14 +323,6 @@ def get_links(self, obj): return links -def format_availability_issues(status: models.WorkflowStatus): - issues = status.availability_issues - logger.info(issues) - if 'not_available' == status.aggregated_status and len(issues) > 0: - return ', '.join([f"{i['issue']}: Unable to get resource '{i['resource']}' from service '{i['service']}'" if 'service' in i and 'resource' in i else i['issue'] for i in issues]) - return None - - class WorkflowStatusSchema(WorkflowVersionSchema): __envelope__ = {"single": None, "many": "items"} __model__ = models.WorkflowStatus @@ -319,13 +330,36 @@ class WorkflowStatusSchema(WorkflowVersionSchema): class Meta: model = models.WorkflowStatus - aggregate_test_status = fields.String(attribute="status.aggregated_status") - latest_builds = ma.Nested(BuildSummarySchema(exclude=('meta', 'links')), - attribute="status.latest_builds", many=True) + _errors = [] + + aggregate_test_status = fields.Method("get_aggregate_test_status") + latest_builds = fields.Method("get_latest_builds") reason = fields.Method("get_reason") + def get_aggregate_test_status(self, workflow_version): + try: + return workflow_version.status.aggregated_status + except Exception as e: + logger.debug(e) + self._errors.append(str(e)) + return "not_available" + + def get_latest_builds(self, workflow_version): + try: + return BuildSummarySchema(exclude=('meta', 'links'), many=True).dump( + workflow_version.status.latest_builds) + except Exception as e: + logger.debug(e) + self._errors.append(str(e)) + return [] + def get_reason(self, workflow_version): - return format_availability_issues(workflow_version.status) + try: + if(len(self._errors) > 0): + return ', '.join([str(i) for i in self._errors]) + return format_availability_issues(workflow_version.status) + except Exception as e: + return str(e) @post_dump def remove_skip_values(self, data, **kwargs): @@ -361,15 +395,26 @@ def get_status(self, workflow): logger.debug(e) return { "aggregate_test_status": "not_available", - "latest_build": [], + "latest_build": None, + "reason": str(e) + } + except Exception as e: + logger.debug(e) + return { + "aggregate_test_status": "not_available", + "latest_build": None, "reason": str(e) } def get_latest_build(self, workflow): - latest_builds = workflow.latest_version.status.latest_builds - if latest_builds and len(latest_builds) > 0: - return BuildSummarySchema(exclude=('meta', 'links')).dump(latest_builds[0]) - return None + try: + latest_builds = workflow.latest_version.status.latest_builds + if latest_builds and len(latest_builds) > 0: + return BuildSummarySchema(exclude=('meta', 'links')).dump(latest_builds[0]) + return None + except Exception as e: + logger.debug(e) + return None def get_subscriptions(self, w: models.Workflow): result = [] @@ -434,18 +479,40 @@ class ListOfSuites(ListOfItems): class SuiteStatusSchema(ResourceMetadataSchema): __envelope__ = {"single": None, "many": "items"} - __model__ = models.SuiteStatus + __model__ = models.TestSuite class Meta: - model = models.SuiteStatus + model = models.TestSuite - suite_uuid = fields.String(attribute="suite.uuid") - status = fields.String(attribute="aggregated_status") - latest_builds = fields.Nested(BuildSummarySchema(exclude=('meta', 'links')), many=True) + suite_uuid = fields.String(attribute="uuid") + status = fields.Method("get_aggregated_status") + latest_builds = fields.Method("get_builds") reason = fields.Method("get_reason") + _errors = [] + + def get_builds(self, suite): + try: + return BuildSummarySchema( + exclude=('meta', 'links'), many=True).dump(suite.status.latest_builds) + except Exception as e: + self._errors.append(str(e)) + logger.debug(e) + return [] - def get_reason(self, status): - return format_availability_issues(status) + def get_reason(self, suite): + if(len(self._errors) > 0): + return ", ".join(self._errors) + try: + return format_availability_issues(suite.status) + except Exception as e: + return str(e) + + def get_aggregated_status(self, suite): + try: + return suite.status.aggregated_status + except Exception as e: + self._errors.append(str(e)) + return 'not_available' @post_dump def remove_skip_values(self, data, **kwargs): diff --git a/lifemonitor/api/services.py b/lifemonitor/api/services.py index 4c37af45e..80800dcbb 100644 --- a/lifemonitor/api/services.py +++ b/lifemonitor/api/services.py @@ -21,13 +21,14 @@ from __future__ import annotations import logging +from datetime import datetime from typing import List, Optional, Union import lifemonitor.exceptions as lm_exceptions from lifemonitor.api import models -from lifemonitor.auth.models import (ExternalServiceAuthorizationHeader, - Permission, Resource, RoleType, - Subscription, User) +from lifemonitor.auth.models import (EventType, ExternalServiceAuthorizationHeader, + Notification, Permission, Resource, + RoleType, Subscription, User) from lifemonitor.auth.oauth2.client import providers from lifemonitor.auth.oauth2.client.models import OAuthIdentity from lifemonitor.auth.oauth2.server import server @@ -106,10 +107,13 @@ def register_workflow(cls, roc_link, workflow_submitter: User, workflow_version, workflow_uuid=None, workflow_identifier=None, workflow_registry: Optional[models.WorkflowRegistry] = None, authorization=None, name=None, public=False): - # find or create a user workflow + w = None if workflow_registry: - w = workflow_registry.get_workflow(workflow_uuid or workflow_identifier) + if workflow_uuid: + w = workflow_registry.get_workflow(workflow_uuid) + else: + w = workflow_registry.get_workflow(workflow_identifier) else: w = models.Workflow.get_user_workflow(workflow_submitter, workflow_uuid) if not w: @@ -133,6 +137,8 @@ def register_workflow(cls, roc_link, workflow_submitter: User, workflow_version, if workflow_submitter: wv.permissions.append(Permission(user=workflow_submitter, roles=[RoleType.owner])) + # automatically register submitter's subscription to workflow events + workflow_submitter.subscribe(w) if authorization: auth = ExternalServiceAuthorizationHeader(workflow_submitter, header=authorization) auth.resources.append(wv) @@ -140,8 +146,8 @@ def register_workflow(cls, roc_link, workflow_submitter: User, workflow_version, if name is None: if wv.workflow_name is None: raise lm_exceptions.LifeMonitorException(title="Missing attribute 'name'", - detail="Attribute 'name' is not defined and it cannot be retrieved ' \ - 'from the workflow RO-Crate (name of 'mainEntity' and '/' dataset not set)", + detail="Attribute 'name' is not defined and it cannot be retrieved \ + from the workflow RO-Crate (name of 'mainEntity' not found)", status=400) w.name = wv.workflow_name wv.name = wv.workflow_name @@ -230,10 +236,12 @@ def _init_test_suite_from_json(wv: models.WorkflowVersion, submitter: models.Use raise lm_exceptions.SpecificationNotValidException(f"Missing property: {e}") @staticmethod - def subscribe_user_resource(user: User, resource: Resource) -> Subscription: + def subscribe_user_resource(user: User, resource: Resource, events: List[EventType] = None) -> Subscription: assert user and not user.is_anonymous, "Invalid user" assert resource, "Invalid resource" subscription = user.subscribe(resource) + if events: + subscription.events = events user.save() return subscription @@ -495,3 +503,33 @@ def get_workflow_registries() -> models.WorkflowRegistry: @staticmethod def get_workflow_registry(uuid) -> models.WorkflowRegistry: return models.WorkflowRegistry.find_by_uuid(uuid) + + @staticmethod + def setUserNotificationReadingTime(user: User, notifications: List[dict]): + for n in notifications: + un = user.get_user_notification(n['uuid']) + if un is None: + return lm_exceptions.EntityNotFoundException(Notification, entity_id=n['uuid']) + un.read = datetime.utcnow() + user.save() + + @staticmethod + def deleteUserNotification(user: User, notitification_uuid: str): + if notitification_uuid is not None: + n = user.get_user_notification(notitification_uuid) + logger.debug("Search result notification %r ...", n) + if n is None: + return lm_exceptions.EntityNotFoundException(Notification, entity_id=notitification_uuid) + user.notifications.remove(n) + user.save() + + @staticmethod + def deleteUserNotifications(user: User, list_of_uuids: List[str]): + for n_uuid in list_of_uuids: + logger.debug("Searching notification %r ...", n_uuid) + n = user.get_user_notification(n_uuid) + logger.debug("Search result notification %r ...", n) + if n is None: + return lm_exceptions.EntityNotFoundException(Notification, entity_id=n_uuid) + user.notifications.remove(n) + user.save() diff --git a/lifemonitor/app.py b/lifemonitor/app.py index 0f5633167..40fb7a158 100644 --- a/lifemonitor/app.py +++ b/lifemonitor/app.py @@ -32,6 +32,7 @@ from lifemonitor.tasks.task_queue import init_task_queue from . import commands +from .mail import init_mail from .cache import init_cache from .db import db from .exceptions import handle_exception @@ -130,6 +131,8 @@ def initialize_app(app, app_context, prom_registry=None): commands.register_commands(app) # init scheduler/worker for async tasks init_task_queue(app) + # init mail system + init_mail(app) # configure prometheus exporter # must be configured after the routes are registered diff --git a/lifemonitor/auth/__init__.py b/lifemonitor/auth/__init__.py index 54d06aa3c..601d78574 100644 --- a/lifemonitor/auth/__init__.py +++ b/lifemonitor/auth/__init__.py @@ -22,6 +22,7 @@ import lifemonitor.auth.oauth2 as oauth2 +from .models import User, UserNotification, Notification, EventType from .controllers import blueprint as auth_blueprint from .services import (NotAuthorizedException, authorized, current_registry, current_user, login_manager, login_registry, login_user, @@ -40,6 +41,7 @@ def register_api(app, specs_dir): __all__ = [ + User, UserNotification, Notification, EventType, register_api, current_user, current_registry, authorized, login_user, logout_user, login_registry, logout_registry, NotAuthorizedException diff --git a/lifemonitor/auth/controllers.py b/lifemonitor/auth/controllers.py index e92e39538..ad5e7d59b 100644 --- a/lifemonitor/auth/controllers.py +++ b/lifemonitor/auth/controllers.py @@ -20,17 +20,19 @@ import logging +import connexion import flask from flask import flash, redirect, render_template, request, session, url_for from flask_login import login_required, login_user, logout_user -from lifemonitor.cache import cached, Timeout, clear_cache +from lifemonitor.cache import Timeout, cached, clear_cache from lifemonitor.utils import (NextRouteRegistry, next_route_aware, split_by_crlf) from .. import exceptions from ..utils import OpenApiSpecs from . import serializers -from .forms import LoginForm, Oauth2ClientForm, RegisterForm, SetPasswordForm +from .forms import (EmailForm, LoginForm, NotificationsForm, Oauth2ClientForm, + RegisterForm, SetPasswordForm) from .models import db from .oauth2.client.services import (get_current_user_identity, get_providers, merge_users, save_current_user_identity) @@ -49,6 +51,11 @@ login_manager.login_view = "auth.login" +def __lifemonitor_service__(): + from lifemonitor.api.services import LifeMonitor + return LifeMonitor.get_instance() + + @authorized @cached(timeout=Timeout.SESSION) def show_current_user_profile(): @@ -60,6 +67,64 @@ def show_current_user_profile(): return exceptions.report_problem_from_exception(e) +@authorized +@cached(timeout=Timeout.REQUEST) +def user_notifications_get(): + try: + if current_user and not current_user.is_anonymous: + return serializers.ListOfNotifications().dump(current_user.notifications) + raise exceptions.Forbidden(detail="Client type unknown") + except Exception as e: + return exceptions.report_problem_from_exception(e) + + +@authorized +@cached(timeout=Timeout.REQUEST) +def user_notifications_put(body): + try: + if not current_user or current_user.is_anonymous: + raise exceptions.Forbidden(detail="Client type unknown") + __lifemonitor_service__().setUserNotificationReadingTime(current_user, body.get('items', [])) + clear_cache() + return connexion.NoContent, 204 + except Exception as e: + logger.debug(e) + return exceptions.report_problem_from_exception(e) + + +@authorized +@cached(timeout=Timeout.REQUEST) +def user_notifications_patch(body): + try: + if not current_user or current_user.is_anonymous: + raise exceptions.Forbidden(detail="Client type unknown") + logger.debug("PATCH BODY: %r", body) + __lifemonitor_service__().deleteUserNotifications(current_user, body) + clear_cache() + return connexion.NoContent, 204 + except exceptions.EntityNotFoundException as e: + return exceptions.report_problem_from_exception(e) + except Exception as e: + logger.debug(e) + return exceptions.report_problem_from_exception(e) + + +@authorized +@cached(timeout=Timeout.REQUEST) +def user_notifications_delete(notification_uuid): + try: + if not current_user or current_user.is_anonymous: + raise exceptions.Forbidden(detail="Client type unknown") + __lifemonitor_service__().deleteUserNotification(current_user, notification_uuid) + clear_cache() + return connexion.NoContent, 204 + except exceptions.EntityNotFoundException as e: + return exceptions.report_problem_from_exception(e) + except Exception as e: + logger.debug(e) + return exceptions.report_problem_from_exception(e) + + @authorized def user_subscriptions_get(): return serializers.ListOfSubscriptions().dump(current_user.subscriptions) @@ -97,7 +162,8 @@ def index(): @blueprint.route("/profile", methods=("GET",)) -def profile(form=None, passwordForm=None, currentView=None): +def profile(form=None, passwordForm=None, currentView=None, + emailForm=None, notificationsForm=None): currentView = currentView or request.args.get("currentView", 'accountsTab') logger.debug(OpenApiSpecs.get_instance().authorization_code_scopes) back_param = request.args.get('back', None) @@ -111,6 +177,8 @@ def profile(form=None, passwordForm=None, currentView=None): logger.debug("detected back param: %s", back_param) return render_template("auth/profile.j2", passwordForm=passwordForm or SetPasswordForm(), + emailForm=emailForm or EmailForm(), + notificationsForm=notificationsForm or NotificationsForm(), oauth2ClientForm=form or Oauth2ClientForm(), providers=get_providers(), currentView=currentView, oauth2_generic_client_scopes=OpenApiSpecs.get_instance().authorization_code_scopes, @@ -197,6 +265,77 @@ def set_password(): return profile(passwordForm=form) +@blueprint.route("/set_email", methods=("GET", "POST")) +@login_required +def set_email(): + form = EmailForm() + if request.method == "GET": + return profile(emailForm=form, currentView='notificationsTab') + if form.validate_on_submit(): + if form.email.data == current_user.email: + flash("email address not changed") + else: + current_user.email = form.email.data + db.session.add(current_user) + db.session.commit() + from lifemonitor.mail import send_email_validation_message + send_email_validation_message(current_user) + flash("email address registered") + return redirect(url_for("auth.profile", emailForm=form, currentView='notificationsTab')) + return profile(emailForm=form, currentView='notificationsTab') + + +@blueprint.route("/send_verification_email", methods=("GET", "POST")) +@login_required +def send_verification_email(): + try: + current_user.generate_email_verification_code() + from lifemonitor.mail import send_email_validation_message + send_email_validation_message(current_user) + current_user.save() + flash("Confirmation email sent") + logger.info("Confirmation email sent %r", current_user.id) + except Exception as e: + logger.error("An error occurred when sending email verification message for user %r", + current_user.id) + logger.debug(e) + return redirect(url_for("auth.profile", currentView='notificationsTab')) + + +@blueprint.route("/validate_email", methods=("GET", "POST")) +@login_required +def validate_email(): + validated = False + try: + code = request.args.get("code", None) + current_user.verify_email(code) + current_user.save() + flash("Email address validated") + except exceptions.LifeMonitorException as e: + logger.debug(e) + logger.info("Email validated for user %r: %r", current_user.id, validated) + return redirect(url_for("auth.profile", currentView='notificationsTab')) + + +@blueprint.route("/update_notifications_switch", methods=("GET", "POST")) +@login_required +def update_notifications_switch(): + logger.debug("Updating notifications") + form = NotificationsForm() + if request.method == "GET": + return redirect(url_for('auth.profile', notificationsForm=form, currentView='notificationsTab')) + enable_notifications = form.enable_notifications.data + logger.debug("Enable notifications: %r", enable_notifications) + if enable_notifications: + current_user.enable_email_notifications() + else: + current_user.disable_email_notifications() + current_user.save() + enabled_str = "enabled" if current_user.email_notifications_enabled else "disabled" + flash(f"email notifications {enabled_str}") + return redirect(url_for("auth.profile", notificationsForm=form, currentView='notificationsTab')) + + @blueprint.route("/merge", methods=("GET", "POST")) @login_required def merge(): diff --git a/lifemonitor/auth/forms.py b/lifemonitor/auth/forms.py index 4ae88053a..8062970b5 100644 --- a/lifemonitor/auth/forms.py +++ b/lifemonitor/auth/forms.py @@ -29,7 +29,7 @@ from sqlalchemy.exc import IntegrityError from wtforms import (BooleanField, HiddenField, PasswordField, SelectField, SelectMultipleField, StringField) -from wtforms.validators import URL, DataRequired, EqualTo, Optional +from wtforms.validators import URL, DataRequired, Email, EqualTo, Optional from .models import User, db @@ -108,6 +108,27 @@ class SetPasswordForm(FlaskForm): repeat_password = PasswordField("Repeat Password") +class NotificationsForm(FlaskForm): + enable_notifications = BooleanField( + "enable_notifications", + validators=[ + DataRequired() + ], + ) + + +class EmailForm(FlaskForm): + email = StringField( + "Email", + validators=[ + DataRequired(), + Email(), + EqualTo("repeat_email", message="email addresses do not match"), + ], + ) + repeat_email = StringField("Repeat Email") + + class Oauth2ClientForm(FlaskForm): clientId = HiddenField("clientId") name = StringField("Client Name", validators=[DataRequired()]) diff --git a/lifemonitor/auth/models.py b/lifemonitor/auth/models.py index 8d28e8d80..a460c5b55 100644 --- a/lifemonitor/auth/models.py +++ b/lifemonitor/auth/models.py @@ -21,18 +21,26 @@ from __future__ import annotations import abc +import base64 import datetime +import json import logging +import random +import string import uuid as _uuid +from enum import Enum from typing import List from authlib.integrations.sqla_oauth2 import OAuth2TokenMixin from flask_bcrypt import check_password_hash, generate_password_hash from flask_login import AnonymousUserMixin, UserMixin +from lifemonitor import exceptions as lm_exceptions from lifemonitor import utils as lm_utils from lifemonitor.db import db -from lifemonitor.models import UUID, ModelMixin +from lifemonitor.models import JSON, UUID, IntegerSet, ModelMixin +from sqlalchemy import null from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.ext.mutable import MutableSet # Set the module level logger logger = logging.getLogger(__name__) @@ -55,7 +63,12 @@ class User(db.Model, UserMixin): username = db.Column(db.String(256), unique=True, nullable=False) password_hash = db.Column(db.LargeBinary, nullable=True) picture = db.Column(db.String(), nullable=True) - + _email_notifications_enabled = db.Column("email_notifications", db.Boolean, + nullable=False, default=True) + _email = db.Column("email", db.String(), nullable=True) + _email_verification_code = None + _email_verification_hash = db.Column("email_verification_hash", db.String(256), nullable=True) + _email_verified = db.Column("email_verified", db.Boolean, nullable=True, default=False) permissions = db.relationship("Permission", back_populates="user", cascade="all, delete-orphan") authorizations = db.relationship("ExternalServiceAccessAuthorization", @@ -63,6 +76,10 @@ class User(db.Model, UserMixin): subscriptions = db.relationship("Subscription", cascade="all, delete-orphan") + notifications: List[UserNotification] = db.relationship("UserNotification", + back_populates="user", + cascade="all, delete-orphan") + def __init__(self, username=None) -> None: super().__init__() self.username = username @@ -106,18 +123,121 @@ def password(self): def has_password(self): return bool(self.password_hash) + def verify_password(self, password): + return check_password_hash(self.password_hash, password) + + def _generate_random_code(self, chars=string.ascii_uppercase + string.digits): + return base64.b64encode( + json.dumps( + { + "email": self._email, + "code": ''.join(random.choice(chars) for _ in range(16)), + "expires": (datetime.datetime.now() + datetime.timedelta(hours=1)).timestamp() + } + ).encode('ascii') + ).decode() + + @staticmethod + def _decode_random_code(code): + try: + code = code.encode() if isinstance(code, str) else code + return json.loads(base64.b64decode(code.decode('ascii'))) + except Exception as e: + logger.debug(e) + return None + + @property + def email_notifications_enabled(self): + return self._email_notifications_enabled + + def disable_email_notifications(self): + self._email_notifications_enabled = False + + def enable_email_notifications(self): + self._email_notifications_enabled = True + + @property + def email(self) -> str: + return self._email + + @email.setter + def email(self, email: str): + if email and email != self._email: + self._email = email + self.generate_email_verification_code() + + @email.deleter + def email(self): + self._email = None + self._email_verified = False + + def generate_email_verification_code(self): + self._email_verified = False + code = self._generate_random_code() + self._email_verification_code = code + self._email_verification_hash = generate_password_hash(code).decode('utf-8') + return self._email_verification_code + + @property + def email_verification_code(self) -> str: + return self._email_verification_code + + @property + def email_verified(self) -> bool: + return self._email_verified + + def verify_email(self, code): + if not self._email: + raise lm_exceptions.IllegalStateException(detail="No notification email found") + # verify integrity + if not code or \ + not check_password_hash( + self._email_verification_hash, code): + raise lm_exceptions.LifeMonitorException(detail="Invalid verification code") + try: + data = self._decode_random_code(code) + except Exception as e: + logger.debug(e) + raise lm_exceptions.LifeMonitorException(detail="Invalid verification code") + if data['email'] != self._email: + raise lm_exceptions.LifeMonitorException(detail="Notification email not valid") + if data['expires'] < datetime.datetime.now().timestamp(): + raise lm_exceptions.LifeMonitorException(detail="Verification code expired") + self._email_verified = True + return True + + def get_user_notification(self, notification_uuid: str) -> UserNotification: + return next((n for n in self.notifications if str(n.notification.uuid) == notification_uuid), None) + + def get_notification(self, notification_uuid: str) -> Notification: + user_notification = self.get_user_notification(notification_uuid) + return None if not user_notification else user_notification.notification + + def remove_notification(self, n: Notification | UserNotification): + user_notification = None + try: + user_notification = self.get_user_notification(n.uuid) + if user_notification is None: + raise ValueError(f"notification {n.uuid} not associated to this user") + except Exception: + user_notification = n + if n is None: + raise ValueError("notification cannot be None") + self.notifications.remove(user_notification) + logger.debug("User notification %r removed", user_notification) + def has_permission(self, resource: Resource) -> bool: return self.get_permission(resource) is not None def get_permission(self, resource: Resource) -> Permission: return next((p for p in self.permissions if p.resource == resource), None) - def verify_password(self, password): - return check_password_hash(self.password_hash, password) - def get_subscription(self, resource: Resource) -> Subscription: return next((s for s in self.subscriptions if s.resource == resource), None) + def is_subscribed_to(self, resource: Resource) -> bool: + return self.get_subscription(resource) is not None + def subscribe(self, resource: Resource) -> Subscription: s = self.get_subscription(resource) if not s: @@ -199,6 +319,45 @@ def all(cls) -> List[ApiKey]: return cls.query.all() +class EventType(Enum): + ALL = 0 + BUILD_FAILED = 1 + BUILD_RECOVERED = 2 + + @classmethod + def all(cls): + return list(map(lambda c: c, cls)) + + @classmethod + def all_names(cls): + return list(map(lambda c: c.name, cls)) + + @classmethod + def all_values(cls): + return list(map(lambda c: c.value, cls)) + + @classmethod + def to_string(cls, event: EventType) -> str: + return event.name if event else None + + @classmethod + def to_strings(cls, event_list: List[EventType]) -> List[str]: + return [cls.to_string(_) for _ in event_list if _] if event_list else [] + + @classmethod + def from_string(cls, event_name: str) -> EventType: + try: + return cls[event_name] + except KeyError: + raise ValueError("'%s' is not a valid EventType", event_name) + + @classmethod + def from_strings(cls, event_name_list: List[str]) -> List[EventType]: + if not event_name_list: + return [] + return [cls.from_string(_) for _ in event_name_list] + + class Resource(db.Model, ModelMixin): id = db.Column('id', db.Integer, primary_key=True) @@ -248,6 +407,11 @@ def get_authorization(self, user: User): def find_by_uuid(cls, uuid): return cls.query.filter(cls.uuid == lm_utils.uuid_param(uuid)).first() + def get_subscribers(self, event: EventType = EventType.ALL) -> List[User]: + users = {s.user for s in self.subscriptions if s.has_event(event)} + users.update({s.user for s in self.subscriptions if s.has_event(event)}) + return users + resource_authorization_table = db.Table( 'resource_authorization', db.Model.metadata, @@ -271,11 +435,122 @@ class Subscription(db.Model, ModelMixin): resource: Resource = db.relationship("Resource", uselist=False, backref=db.backref("subscriptions", cascade="all, delete-orphan"), foreign_keys=[resource_id]) + _events = db.Column("events", MutableSet.as_mutable(IntegerSet()), default={0}) def __init__(self, resource: Resource, user: User) -> None: self.resource = resource self.user = user + def __get_events(self): + if self._events is None: + self._events = {0} + return self._events + + @property + def events(self) -> set: + return [EventType(e) for e in self.__get_events()] + + @events.setter + def events(self, events: List[EventType]): + self.__get_events().clear() + if events: + for e in events: + if not isinstance(e, EventType): + raise ValueError(f"Not valid event value: expected {EventType.__class__}, got {type(e)}") + self.__get_events().add(e.value) + + def has_event(self, event: EventType) -> bool: + return False if event is None else \ + EventType.ALL.value in self.__get_events() or \ + event.value in self.__get_events() + + def has_events(self, events: List[EventType]) -> bool: + if events: + for e in events: + if not self.has_event(e): + return False + return True + + +class Notification(db.Model, ModelMixin): + + id = db.Column(db.Integer, primary_key=True) + uuid = db.Column(UUID, default=_uuid.uuid4, nullable=False, index=True) + created = db.Column(db.DateTime, default=datetime.datetime.utcnow) + name = db.Column("name", db.String, nullable=True, index=True) + _event = db.Column("event", db.Integer, nullable=False) + _data = db.Column("data", JSON, nullable=True) + + users: List[UserNotification] = db.relationship("UserNotification", + back_populates="notification", cascade="all, delete-orphan") + + def __init__(self, event: EventType, name: str, data: object, users: List[User]) -> None: + self.name = name + self._event = event.value + self._data = data + for u in users: + self.add_user(u) + + @property + def event(self) -> EventType: + return EventType(self._event) + + @property + def data(self) -> object: + return self._data + + def add_user(self, user: User): + if user and user not in self.users: + UserNotification(user, self) + + def remove_user(self, user: User): + self.users.remove(user) + + @classmethod + def find_by_name(cls, name: str) -> List[Notification]: + return cls.query.filter(cls.name == name).all() + + @classmethod + def not_read(cls) -> List[Notification]: + return cls.query.join(UserNotification, UserNotification.notification_id == cls.id)\ + .filter(UserNotification.read == null()).all() + + @classmethod + def not_emailed(cls) -> List[Notification]: + return cls.query.join(UserNotification, UserNotification.notification_id == cls.id)\ + .filter(UserNotification.emailed == null()).all() + + +class UserNotification(db.Model): + + emailed = db.Column(db.DateTime, default=None, nullable=True) + read = db.Column(db.DateTime, default=None, nullable=True) + + user_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False, primary_key=True) + + notification_id = db.Column(db.Integer, db.ForeignKey("notification.id"), nullable=False, primary_key=True) + + user: User = db.relationship("User", uselist=False, + back_populates="notifications", foreign_keys=[user_id], + cascade="save-update") + + notification: Notification = db.relationship("Notification", uselist=False, + back_populates="users", + foreign_keys=[notification_id], + cascade="save-update") + + def __init__(self, user: User, notification: Notification) -> None: + self.user = user + self.notification = notification + + def save(self): + db.session.add(self) + db.session.commit() + + def delete(self): + db.session.delete(self) + db.session.commit() + class HostingService(Resource): diff --git a/lifemonitor/auth/serializers.py b/lifemonitor/auth/serializers.py index 5361947b5..7f41a1ab1 100644 --- a/lifemonitor/auth/serializers.py +++ b/lifemonitor/auth/serializers.py @@ -25,9 +25,10 @@ from lifemonitor.serializers import (BaseSchema, ListOfItems, ResourceMetadataSchema, ma) from marshmallow import fields +from marshmallow.decorators import post_dump -from . import models from ..utils import get_external_server_url +from . import models # Config a module level logger logger = logging.getLogger(__name__) @@ -118,6 +119,7 @@ class Meta: modified = fields.String(attribute='modified') resource = fields.Method("get_resource") + events = fields.Method("get_events") def get_resource(self, obj: models.Subscription): return { @@ -125,6 +127,36 @@ def get_resource(self, obj: models.Subscription): 'type': obj.resource.type } + def get_events(self, obj: models.Subscription): + return models.EventType.to_strings(obj.events) + class ListOfSubscriptions(ListOfItems): __item_scheme__ = SubscriptionSchema + + +class NotificationSchema(ResourceMetadataSchema): + __envelope__ = {"single": None, "many": "items"} + __model__ = models.UserNotification + + class Meta: + model = models.UserNotification + + uuid = fields.String(attribute='notification.uuid') + created = fields.DateTime(attribute='notification.created') + emailed = fields.DateTime(attribute='emailed') + read = fields.DateTime(attribute='read') + name = fields.String(attribute="notification.name") + event = fields.String(attribute="notification.event.name") + data = fields.Dict(attribute="notification.data") + + @post_dump + def remove_skip_values(self, data, **kwargs): + return { + key: value for key, value in data.items() + if value is not None + } + + +class ListOfNotifications(ListOfItems): + __item_scheme__ = NotificationSchema diff --git a/lifemonitor/auth/templates/auth/base.j2 b/lifemonitor/auth/templates/auth/base.j2 index 3c388e855..0f85f53cc 100644 --- a/lifemonitor/auth/templates/auth/base.j2 +++ b/lifemonitor/auth/templates/auth/base.j2 @@ -44,6 +44,7 @@ + {% endblock stylesheets %} @@ -76,6 +77,8 @@ + + {# Enable notifications #} {{ macros.messages() }} diff --git a/lifemonitor/auth/templates/auth/macros.j2 b/lifemonitor/auth/templates/auth/macros.j2 index 15eb4fbd6..7cbddace2 100644 --- a/lifemonitor/auth/templates/auth/macros.j2 +++ b/lifemonitor/auth/templates/auth/macros.j2 @@ -113,7 +113,7 @@
{% if field.name == 'username' %} - {% elif field.name == 'email' %} + {% elif field.name == 'email' or field.name == 'repeat_email' %} {% elif field.name == 'password' %} diff --git a/lifemonitor/auth/templates/auth/notifications.j2 b/lifemonitor/auth/templates/auth/notifications.j2 new file mode 100644 index 000000000..742d26c86 --- /dev/null +++ b/lifemonitor/auth/templates/auth/notifications.j2 @@ -0,0 +1,70 @@ +{% import 'auth/macros.j2' as macros %} + +
+ {{ notificationsForm.hidden_tag() }} +
+
+
+ + +
+
+
+ + +
+ +
+

Email

+
+ +
+ {% if current_user.email and not current_user.email_verified %} +
+
+
+
+ +
+
+
+ Your current email address {{ current_user.email }} is not validated! + Check your email inbox or click here + if you do not have received the verification email. +
+
+
+ {% endif %} + +
+ Use the form below to + {% if not current_user.email %}set{% endif %} + {% if current_user.email %}update{% endif %} + your email. +
+ +
+ {{ emailForm.hidden_tag() }} +
+ + {{ macros.render_custom_field(emailForm.email, value=current_user.email) }} +
+
+ + {{ macros.render_custom_field(emailForm.repeat_email) }} +
+
+ +
+
+
diff --git a/lifemonitor/auth/templates/auth/profile.j2 b/lifemonitor/auth/templates/auth/profile.j2 index f7d9278ae..58aaa0324 100644 --- a/lifemonitor/auth/templates/auth/profile.j2 +++ b/lifemonitor/auth/templates/auth/profile.j2 @@ -53,6 +53,10 @@ OAuth Apps +
@@ -71,6 +75,9 @@ {% include 'auth/oauth2_clients_tab.j2' %}
+
+ {% include 'auth/notifications.j2' %} +
diff --git a/lifemonitor/cache.py b/lifemonitor/cache.py index 68db70f63..c8adadec1 100644 --- a/lifemonitor/cache.py +++ b/lifemonitor/cache.py @@ -266,7 +266,9 @@ def __init__(self, parent: Cache = None) -> None: @classmethod def _make_key(cls, key: str, prefix: str = CACHE_PREFIX) -> str: if cls._hash_function: - key = cls._hash_function(key.encode()).hexdigest() + parts = key.split("::") + if len(parts) > 1 and parts[1] != '*': + key = f"{parts[0]}::{cls._hash_function(parts[1].encode()).hexdigest()}" return f"{prefix}{key}" @classmethod diff --git a/lifemonitor/config.py b/lifemonitor/config.py index c5318d842..24fca28ff 100644 --- a/lifemonitor/config.py +++ b/lifemonitor/config.py @@ -79,6 +79,8 @@ class BaseConfig: CACHE_DEFAULT_TIMEOUT = 60 # Workflow Data Folder DATA_WORKFLOWS = "./data" + # Base URL of the LifeMonitor web app associated with this back-end instance + WEBAPP_URL = "https://app.lifemonitor.eu" class DevelopmentConfig(BaseConfig): diff --git a/lifemonitor/exceptions.py b/lifemonitor/exceptions.py index f0cda481b..4d124317e 100644 --- a/lifemonitor/exceptions.py +++ b/lifemonitor/exceptions.py @@ -181,6 +181,13 @@ def __init__(self, detail=None, detail=detail, status=status, **kwargs) +class IllegalStateException(LifeMonitorException): + def __init__(self, detail=None, + type="about:blank", status=403, instance=None, **kwargs): + super().__init__(title="Illegal State Exception", + detail=detail, status=status, **kwargs) + + def handle_exception(e: Exception): """Return JSON instead of HTML for HTTP errors.""" # start with the correct headers and status code from the error diff --git a/lifemonitor/lang/messages.py b/lifemonitor/lang/messages.py index 36246be48..a4df4be7c 100644 --- a/lifemonitor/lang/messages.py +++ b/lifemonitor/lang/messages.py @@ -56,3 +56,5 @@ "to start the authorization flow") invalid_log_offset = "Invalid offset: it should be a positive integer" invalid_log_limit = "Invalid limit: it should be a positive integer" +invalid_event_type = "Invalid event type. Accepted values are {}" +notification_not_found = "Notification '{}' not found" diff --git a/lifemonitor/mail.py b/lifemonitor/mail.py new file mode 100644 index 000000000..8f4404cda --- /dev/null +++ b/lifemonitor/mail.py @@ -0,0 +1,115 @@ +# Copyright (c) 2020-2021 CRS4 +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import logging +from datetime import datetime +from typing import List, Optional + +from flask import Flask, render_template +from flask_mail import Mail, Message +from sqlalchemy.exc import InternalError + +from lifemonitor.api.models import TestInstance +from lifemonitor.auth.models import EventType, Notification, User +from lifemonitor.db import db +from lifemonitor.utils import Base64Encoder, get_external_server_url, boolean_value + +# set logger +logger = logging.getLogger(__name__) + +# instantiate the mail class +mail = Mail() + + +def init_mail(app: Flask): + mail_server = app.config.get('MAIL_SERVER', None) + if mail_server: + app.config['MAIL_USE_TLS'] = boolean_value(app.config.get('MAIL_USE_TLS', False)) + app.config['MAIL_USE_SSL'] = boolean_value(app.config.get('MAIL_USE_SSL', False)) + mail.init_app(app) + logger.info("Mail service bound to server '%s'", mail_server) + mail.disabled = False + mail.webapp_url = app.config.get('WEBAPP_URL') + else: + mail.disabled = True + + +def send_email_validation_message(user: User): + if mail.disabled: + logger.info("Mail notifications are disabled") + if user is None or user.is_anonymous: + logger.warning("An authenticated user is required") + with mail.connect() as conn: + confirmation_address = f"{get_external_server_url()}/validate_email?code={user.email_verification_code}" + logo = Base64Encoder.encode_file('lifemonitor/static/img/logo/lm/LifeMonitorLogo.png') + msg = Message( + 'Confirm your email address', + recipients=[user.email], + reply_to="noreply-lifemonitor@crs4.it" + ) + msg.html = render_template("mail/validate_email.j2", + confirmation_address=confirmation_address, user=user, logo=logo) + conn.send(msg) + + +def send_notification(n: Notification, recipients: List[str]) -> Optional[datetime]: + if mail.disabled: + logger.info("Mail notifications are disabled") + else: + with mail.connect() as conn: + logger.debug("Mail recipients for notification '%r': %r", n.id, recipients) + if len(recipients) > 0: + build_data = n.data['build'] + try: + i = TestInstance.find_by_uuid(build_data['instance']['uuid']) + if i is not None: + wv = i.test_suite.workflow_version + b = i.get_test_build(build_data['build_id']) + suite = i.test_suite + logo = Base64Encoder.encode_file('lifemonitor/static/img/logo/lm/LifeMonitorLogo.png') + icon_path = 'lifemonitor/static/img/icons/' \ + + ('times-circle-solid.svg' + if n.event == EventType.BUILD_FAILED else 'check-circle-solid.svg') + icon = Base64Encoder.encode_file(icon_path) + suite.url_param = Base64Encoder.encode_object({ + 'workflow': str(wv.workflow.uuid), + 'suite': str(suite.uuid) + }) + instance_status = "is failing" \ + if n.event == EventType.BUILD_FAILED else "has recovered" + msg = Message( + f'Workflow "{wv.name} ({wv.version})": test instance {i.name} {instance_status}', + bcc=recipients, + reply_to="noreply-lifemonitor@crs4.it" + ) + msg.html = render_template("mail/instance_status_notification.j2", + webapp_url=mail.webapp_url, + workflow_version=wv, build=b, + test_instance=i, + suite=suite, + json_data=build_data, + logo=logo, icon=icon) + conn.send(msg) + return datetime.utcnow() + except InternalError as e: + logger.debug(e) + db.session.rollback() + return None diff --git a/lifemonitor/models.py b/lifemonitor/models.py index b38ac3863..4bbecf841 100644 --- a/lifemonitor/models.py +++ b/lifemonitor/models.py @@ -23,9 +23,10 @@ import uuid from typing import List -from lifemonitor.db import db +from sqlalchemy import VARCHAR, types + from lifemonitor.cache import CacheMixin -from sqlalchemy import types +from lifemonitor.db import db class ModelMixin(CacheMixin): @@ -98,3 +99,39 @@ def load_dialect_impl(self, dialect): return dialect.type_descriptor(JSONB()) else: return dialect.type_descriptor(types.JSON()) + + +class CustomSet(types.TypeDecorator): + """Represents an immutable structure as a json-encoded string.""" + impl = VARCHAR + + def process_bind_param(self, value, dialect): + if value is not None: + if not isinstance(value, set): + raise ValueError("Invalid value type. Got %r", type(value)) + value = ",".join(value) + return value + + def process_result_value(self, value, dialect): + return set() if value is None or len(value) == 0 \ + else set(value.split(',')) + + +class StringSet(CustomSet): + """Represents an immutable structure as a json-encoded string.""" + pass + + +class IntegerSet(CustomSet): + """Represents an immutable structure as a json-encoded string.""" + + def process_bind_param(self, value, dialect): + if value is not None: + if not isinstance(value, set): + raise ValueError("Invalid value type. Got %r", type(value)) + value = ",".join([str(_) for _ in value]) + return value + + def process_result_value(self, value, dialect): + return set() if value is None or len(value) == 0 \ + else set({int(_) for _ in value.split(',')}) diff --git a/lifemonitor/static/img/icons/times-circle-solid.svg b/lifemonitor/static/img/icons/times-circle-solid.svg new file mode 100644 index 000000000..81c289fe1 --- /dev/null +++ b/lifemonitor/static/img/icons/times-circle-solid.svg @@ -0,0 +1,3 @@ + \ No newline at end of file diff --git a/lifemonitor/static/src/package.json b/lifemonitor/static/src/package.json index d2bbda102..6bed5ba02 100644 --- a/lifemonitor/static/src/package.json +++ b/lifemonitor/static/src/package.json @@ -1,7 +1,7 @@ { "name": "lifemonitor", "description": "Workflow Testing Service", - "version": "0.5.1", + "version": "0.6.0", "license": "MIT", "author": "CRS4", "main": "../dist/js/lifemonitor.min.js", diff --git a/lifemonitor/tasks/tasks.py b/lifemonitor/tasks/tasks.py index 047133a3b..f5e0ab4c3 100644 --- a/lifemonitor/tasks/tasks.py +++ b/lifemonitor/tasks/tasks.py @@ -1,15 +1,23 @@ +import datetime import logging import dramatiq import flask from apscheduler.triggers.cron import CronTrigger from apscheduler.triggers.interval import IntervalTrigger +from lifemonitor.api.models.testsuites.testbuild import BuildStatus +from lifemonitor.api.serializers import BuildSummarySchema +from lifemonitor.auth.models import EventType, Notification from lifemonitor.cache import Timeout +from lifemonitor.mail import send_notification # set module level logger logger = logging.getLogger(__name__) +# set expiration time (in msec) of tasks +TASK_EXPIRATION_TIME = 30000 + def schedule(trigger): """ @@ -38,13 +46,13 @@ def decorator(actor): @schedule(CronTrigger(second=0)) -@dramatiq.actor(max_retries=3) +@dramatiq.actor(max_retries=3, max_age=TASK_EXPIRATION_TIME) def heartbeat(): logger.info("Heartbeat!") @schedule(IntervalTrigger(seconds=Timeout.WORKFLOW * 3 / 4)) -@dramatiq.actor(max_retries=3) +@dramatiq.actor(max_retries=3, max_age=TASK_EXPIRATION_TIME) def check_workflows(): from flask import current_app from lifemonitor.api.controllers import workflows_rocrate_download @@ -82,14 +90,15 @@ def check_workflows(): @schedule(IntervalTrigger(seconds=Timeout.BUILD * 3 / 4)) -@dramatiq.actor(max_retries=3) +@dramatiq.actor(max_retries=3, max_age=TASK_EXPIRATION_TIME) def check_last_build(): from lifemonitor.api.models import Workflow logger.info("Starting 'check_last build' task...") for w in Workflow.all(): try: - for s in w.latest_version.test_suites: + latest_version = w.latest_version + for s in latest_version.test_suites: logger.info("Updating workflow: %r", w) for i in s.test_instances: with i.cache.transaction(str(i)): @@ -97,9 +106,65 @@ def check_last_build(): logger.info("Updating latest builds: %r", builds) for b in builds: logger.info("Updating build: %r", i.get_test_build(b.id)) - logger.info("Updating latest build: %r", i.last_test_build) + last_build = i.last_test_build + # check state transition + failed = last_build.status == BuildStatus.FAILED + if len(builds) == 1 and failed or \ + len(builds) > 1 and builds[1].status != last_build.status: + logger.info("Updating latest build: %r", last_build) + notification_name = f"{last_build} {'FAILED' if failed else 'RECOVERED'}" + if len(Notification.find_by_name(notification_name)) == 0: + users = latest_version.workflow.get_subscribers() + n = Notification(EventType.BUILD_FAILED if failed else EventType.BUILD_RECOVERED, + notification_name, + {'build': BuildSummarySchema(exclude_nested=False).dump(last_build)}, + users) + n.save() except Exception as e: logger.error("Error when executing task 'check_last_build': %s", str(e)) if logger.isEnabledFor(logging.DEBUG): logger.exception(e) logger.info("Checking last build: DONE!") + + +@schedule(IntervalTrigger(seconds=60)) +@dramatiq.actor(max_retries=0, max_age=TASK_EXPIRATION_TIME) +def send_email_notifications(): + notifications = Notification.not_emailed() + logger.info("Found %r notifications to send by email", len(notifications)) + count = 0 + for n in notifications: + logger.debug("Processing notification %r ...", n) + recipients = [ + u.user.email for u in n.users + if u.emailed is None and u.user.email_notifications_enabled and u.user.email is not None + ] + sent = send_notification(n, recipients) + logger.debug("Notification email sent: %r", sent is not None) + if sent: + logger.debug("Notification '%r' sent by email @ %r", n.id, sent) + for u in n.users: + if u.user.email in recipients: + u.emailed = sent + n.save() + count += 1 + logger.debug("Processing notification %r ... DONE", n) + logger.info("%r notifications sent by email", count) + + +@schedule(CronTrigger(minute=0, hour=1)) +@dramatiq.actor(max_retries=0, max_age=TASK_EXPIRATION_TIME) +def cleanup_notifications(): + logger.info("Starting notification cleanup") + count = 0 + current_time = datetime.datetime.utcnow() + one_week_ago = current_time - datetime.timedelta(days=0) + notifications = Notification.older_than(one_week_ago) + for n in notifications: + try: + n.delete() + count += 1 + except Exception as e: + logger.debug(e) + logger.error("Error when deleting notification %r", n) + logger.info("Notification cleanup completed: deleted %r notifications", count) diff --git a/lifemonitor/templates/mail/instance_status_notification.j2 b/lifemonitor/templates/mail/instance_status_notification.j2 new file mode 100644 index 000000000..4fc674643 --- /dev/null +++ b/lifemonitor/templates/mail/instance_status_notification.j2 @@ -0,0 +1,112 @@ + + + + + + + +
+ My Image + +

+ {{workflow_version.name}}
+ (version {{workflow_version.version}}) +

+ +

+ {% if build.status == 'failed' %} + Some builds on instance {{test_instance.name}} were not successful + {% else %} + Test instance {{test_instance.name}} has recovered + {% endif %} +

+ +
+ triangle with all three sides equal +
+ +

+ + Test Build #{{build.id}} + {{ 'failed' if build.status == 'failed' else 'passed' }} !!! +

+ +
+ test suite + + {{suite.name}} + + running on the + + {{test_instance.testing_service.type}} + service +
+ through the + + {{test_instance.name}} + + instance +
+ +
+ + View on LifeMonitor + +
+
+ + diff --git a/lifemonitor/templates/mail/validate_email.j2 b/lifemonitor/templates/mail/validate_email.j2 new file mode 100644 index 000000000..285d1fed4 --- /dev/null +++ b/lifemonitor/templates/mail/validate_email.j2 @@ -0,0 +1,86 @@ + + + + + + + +
+ My Image + +

+ email updated +

+ +

+ Hi, {{user.username}} +

+ +
+ Click the button below to confirm your email address + +
+
+ + Confirm your email + +
+ +
+ or copy and paste the following link into your browser + {{confirmation_address}} +
+
+ + diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index 78f124375..da0868932 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -85,6 +85,16 @@ def values_as_string(values, in_separator='\\s?,\\s?|\\s+', out_separator=" "): raise ValueError("Invalid format") +def boolean_value(value) -> bool: + if value is None or value == "": + return False + if isinstance(value, bool): + return value + if isinstance(value, str): + return bool_from_string(value) + raise ValueError(f"Invalid value for boolean. Got '{value}'") + + def bool_from_string(s) -> bool: if s is None or s == "": return None @@ -468,3 +478,30 @@ def get_class(self, concrete_type): def get_classes(self): return [_[0] for _ in self._concrete_types.values()] + + +class Base64Encoder(object): + + _cache = {} + + @classmethod + def encode_file(cls, file: str) -> str: + data = cls._cache.get(file, None) + if data is None: + with open(file, "rb") as f: + data = base64.b64encode(f.read()) + cls._cache[file] = data + return data.decode() + + @classmethod + def encode_object(cls, obj: object) -> str: + key = hash(frozenset(obj.items())) + data = cls._cache.get(key, None) + if data is None: + data = base64.b64encode(json.dumps(obj).encode()) + cls._cache[key] = data + return data.decode() + + @classmethod + def decode(cls, data: str) -> object: + return base64.b64decode(data.encode()) diff --git a/migrations/versions/24c34681f538_add_list_of_events_to_the_subscription_.py b/migrations/versions/24c34681f538_add_list_of_events_to_the_subscription_.py new file mode 100644 index 000000000..4d091cf15 --- /dev/null +++ b/migrations/versions/24c34681f538_add_list_of_events_to_the_subscription_.py @@ -0,0 +1,25 @@ +"""Add list of events to the subscription model + +Revision ID: 24c34681f538 +Revises: d5da43a38a6a +Create Date: 2022-01-21 15:29:19.648485 + +""" +from alembic import op +import sqlalchemy as sa +from lifemonitor.models import IntegerSet + +# revision identifiers, used by Alembic. +revision = '24c34681f538' +down_revision = 'd5da43a38a6a' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('subscription', sa.Column('events', IntegerSet(), nullable=True)) + op.execute("UPDATE subscription SET events = '0'") + + +def downgrade(): + op.drop_column('subscription', 'events') diff --git a/migrations/versions/296634f13bc4_automatically_register_submitter_.py b/migrations/versions/296634f13bc4_automatically_register_submitter_.py new file mode 100644 index 000000000..b085e5ede --- /dev/null +++ b/migrations/versions/296634f13bc4_automatically_register_submitter_.py @@ -0,0 +1,35 @@ +"""Automatically register submitter subscription to workflows + +Revision ID: 296634f13bc4 +Revises: a46c90bedbbf +Create Date: 2022-01-24 14:07:00.098626 + +""" +import logging +from datetime import datetime + +from alembic import op + +# set logger +logger = logging.getLogger('alembic.env') + + +# revision identifiers, used by Alembic. +revision = '296634f13bc4' +down_revision = 'a46c90bedbbf' +branch_labels = None +depends_on = None + + +def upgrade(): + bind = op.get_bind() + res = bind.execute('SELECT * from workflow_version WHERE workflow_id NOT IN (SELECT resource_id FROM subscription)') + logger.info(res) + for wdata in res: + logger.info("(v_id,submitter,wf_id)=(%d,%d,%d)", wdata[0], wdata[1], wdata[2]) + now = datetime.utcnow() + bind.execute(f"INSERT INTO subscription (user_id,created,modified,resource_id,events) VALUES ({wdata[1]},TIMESTAMP '{now}',TIMESTAMP '{now}',{wdata[2]},'0')") + + +def downgrade(): + pass diff --git a/migrations/versions/505e4e6976de_add_uuid_property_to_notification_model.py b/migrations/versions/505e4e6976de_add_uuid_property_to_notification_model.py new file mode 100644 index 000000000..7122b57b5 --- /dev/null +++ b/migrations/versions/505e4e6976de_add_uuid_property_to_notification_model.py @@ -0,0 +1,36 @@ +"""Add uuid property to notification model + +Revision ID: 505e4e6976de +Revises: 296634f13bc4 +Create Date: 2022-01-27 11:43:27.119562 + +""" +from alembic import op +import sqlalchemy as sa +from lifemonitor.models import UUID +import uuid as _uuid + +# revision identifiers, used by Alembic. +revision = '505e4e6976de' +down_revision = '296634f13bc4' +branch_labels = None +depends_on = None + + +def upgrade(): + bind = op.get_bind() + notifications = bind.execute("SELECT id FROM notification") + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('notification', sa.Column('uuid', UUID(), nullable=True)) + for n in notifications: + bind.execute(f"UPDATE notification SET uuid = '{_uuid.uuid4()}' WHERE id = {n[0]}") + op.alter_column('notification', 'uuid', nullable=False) + op.create_index(op.f('ix_notification_uuid'), 'notification', ['uuid'], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_notification_uuid'), table_name='notification') + op.drop_column('notification', 'uuid') + # ### end Alembic commands ### diff --git a/migrations/versions/97e77a9c44b2_add_user_notification_model.py b/migrations/versions/97e77a9c44b2_add_user_notification_model.py new file mode 100644 index 000000000..51ebf5790 --- /dev/null +++ b/migrations/versions/97e77a9c44b2_add_user_notification_model.py @@ -0,0 +1,48 @@ +"""Add user notification model + +Revision ID: 97e77a9c44b2 +Revises: f4cbfe20075f +Create Date: 2022-01-17 13:21:37.565495 + +""" +from alembic import op +import sqlalchemy as sa +from lifemonitor.models import JSON + + +# revision identifiers, used by Alembic. +revision = '97e77a9c44b2' +down_revision = 'f4cbfe20075f' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table('notification', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created', sa.DateTime(), nullable=True), + sa.Column('name', sa.String(), nullable=True), + sa.Column('type', sa.String(), nullable=False), + sa.Column('data', JSON(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('user_notification', + sa.Column('emailed', sa.DateTime(), nullable=True), + sa.Column('read', sa.DateTime(), nullable=True), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('notification_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['notification_id'], ['notification.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('user_id', 'notification_id') + ) + op.add_column('user', sa.Column('email', sa.String(), nullable=True)) + op.add_column('user', sa.Column('email_verification_hash', sa.String(length=256), nullable=True)) + op.add_column('user', sa.Column('email_verified', sa.Boolean(), nullable=True)) + + +def downgrade(): + op.drop_column('user', 'email_verified') + op.drop_column('user', 'email_verification_hash') + op.drop_column('user', 'email') + op.drop_table('user_notification') + op.drop_table('notification') diff --git a/migrations/versions/a46c90bedbbf_change_notification_type_str_to_.py b/migrations/versions/a46c90bedbbf_change_notification_type_str_to_.py new file mode 100644 index 000000000..00c4e0e97 --- /dev/null +++ b/migrations/versions/a46c90bedbbf_change_notification_type_str_to_.py @@ -0,0 +1,30 @@ +"""Change Notification.type::str to Notification.event::EventType + +Revision ID: a46c90bedbbf +Revises: 24c34681f538 +Create Date: 2022-01-24 09:27:57.815975 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'a46c90bedbbf' +down_revision = '24c34681f538' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('notification', sa.Column('event', sa.Integer(), nullable=False)) + op.drop_column('notification', 'type') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('notification', sa.Column('type', sa.VARCHAR(), autoincrement=False, nullable=False)) + op.drop_column('notification', 'event') + # ### end Alembic commands ### diff --git a/migrations/versions/d1387a6fe551_add_name_index_on_notification_model.py b/migrations/versions/d1387a6fe551_add_name_index_on_notification_model.py new file mode 100644 index 000000000..7629c681d --- /dev/null +++ b/migrations/versions/d1387a6fe551_add_name_index_on_notification_model.py @@ -0,0 +1,23 @@ +"""Add name index on notification model + +Revision ID: d1387a6fe551 +Revises: 97e77a9c44b2 +Create Date: 2022-01-18 15:41:53.769530 + +""" +from alembic import op + + +# revision identifiers, used by Alembic. +revision = 'd1387a6fe551' +down_revision = '97e77a9c44b2' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_index(op.f('ix_notification_name'), 'notification', ['name'], unique=False) + + +def downgrade(): + op.drop_index(op.f('ix_notification_name'), table_name='notification') diff --git a/migrations/versions/d5da43a38a6a_add_flag_to_enable_disable_mail_.py b/migrations/versions/d5da43a38a6a_add_flag_to_enable_disable_mail_.py new file mode 100644 index 000000000..9004634a4 --- /dev/null +++ b/migrations/versions/d5da43a38a6a_add_flag_to_enable_disable_mail_.py @@ -0,0 +1,26 @@ +"""Add flag to enable/disable mail notifications + +Revision ID: d5da43a38a6a +Revises: d1387a6fe551 +Create Date: 2022-01-20 13:27:41.016651 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'd5da43a38a6a' +down_revision = 'd1387a6fe551' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('user', sa.Column('email_notifications', sa.Boolean())) + op.execute("UPDATE \"user\" SET email_notifications = true") + op.alter_column('user', 'email_notifications', nullable=False) + + +def downgrade(): + op.drop_column('user', 'email_notifications') diff --git a/requirements.txt b/requirements.txt index 45fdeb9c9..14b532e83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ flask-wtf~=0.15.1 Flask-APScheduler==1.12.2 Flask-SQLAlchemy==2.5.1 Flask-Migrate==3.1.0 +Flask-Mail~=0.9.1 Flask>=1.1.4,<2.0.0 gunicorn~=20.1.0 jwt==1.2.0 diff --git a/settings.conf b/settings.conf index 33119552e..fdaa9c056 100644 --- a/settings.conf +++ b/settings.conf @@ -16,6 +16,9 @@ FLASK_ENV=development # used as base_url on all the links returned by the API #EXTERNAL_SERVER_URL=https://lifemonitor.eu +# Base URL of the LifeMonitor web app associated with this back-end instance +WEBAPP_URL=https://app.lifemonitor.eu + # Normally, OAuthLib will raise an InsecureTransportError if you attempt to use OAuth2 over HTTP, # rather than HTTPS. Setting this environment variable will prevent this error from being raised. # This is mostly useful for local testing, or automated tests. Never set this variable in production. @@ -54,6 +57,15 @@ REDIS_HOST=redis REDIS_PASSWORD=foobar REDIS_PORT_NUMBER=6379 +# Email settings +MAIL_SERVER='' +MAIL_PORT=465 +MAIL_USERNAME='' +MAIL_PASSWORD='' +MAIL_USE_TLS=False +MAIL_USE_SSL=True +MAIL_DEFAULT_SENDER='' + # Cache settings CACHE_REDIS_DB=0 CACHE_DEFAULT_TIMEOUT=300 diff --git a/specs/api.yaml b/specs/api.yaml index d69518bc1..f45de7ef0 100644 --- a/specs/api.yaml +++ b/specs/api.yaml @@ -3,7 +3,7 @@ openapi: "3.0.0" info: - version: "0.5.1" + version: "0.6.0" title: "Life Monitor API" description: | *Workflow sustainability service* @@ -18,7 +18,7 @@ info: servers: - url: / description: > - Version 0.5.1 of API. + Version 0.6.0 of API. tags: - name: Registries @@ -170,6 +170,122 @@ paths: "404": $ref: "#/components/responses/NotFound" + /users/current/notifications: + get: + tags: ["Users"] + x-openapi-router-controller: lifemonitor.auth.controllers + operationId: "user_notifications_get" + summary: List notifications for the current user + description: | + List all notifications for the current (authenticated) user + security: + - apiKey: ["user.profile"] + - RegistryCodeFlow: ["user.profile"] + - AuthorizationCodeFlow: ["user.profile"] + responses: + "200": + description: User profile + content: + application/json: + schema: + $ref: "#/components/schemas/ListOfNotifications" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + + put: + tags: ["Users"] + x-openapi-router-controller: lifemonitor.auth.controllers + operationId: "user_notifications_put" + summary: Mark as read notifications for the current user + description: | + Mark as read notifications for the current (authenticated) user + security: + - apiKey: ["user.profile"] + - RegistryCodeFlow: ["user.profile"] + - AuthorizationCodeFlow: ["user.profile"] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ListOfNotifications" + responses: + "204": + description: "Notifications updated" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + + patch: + tags: ["Users"] + x-openapi-router-controller: lifemonitor.auth.controllers + operationId: "user_notifications_patch" + summary: Delete notifications for the current user + description: | + Delete notifications for the current (authenticated) user + security: + - apiKey: ["user.profile"] + - RegistryCodeFlow: ["user.profile"] + - AuthorizationCodeFlow: ["user.profile"] + requestBody: + required: true + content: + application/json: + schema: + description: List of UUIDs + type: array + items: + type: string + example: "21ac72ec-b9a5-49e0-b5a6-1322b8b54552" + responses: + "204": + description: "Notifications updated" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + + /users/current/notifications/{notification_uuid}: + delete: + tags: ["Users"] + x-openapi-router-controller: lifemonitor.auth.controllers + operationId: "user_notifications_delete" + summary: Delete notification for the current user + description: | + Delete notification for the current (authenticated) user + security: + - apiKey: ["user.profile"] + - RegistryCodeFlow: ["user.profile"] + - AuthorizationCodeFlow: ["user.profile"] + parameters: + - $ref: "#/components/parameters/notification_uuid" + responses: + "204": + description: "Notifications updated" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + /registries/current: get: tags: ["Registry Client Operations"] @@ -474,7 +590,7 @@ paths: tags: ["Users"] x-openapi-router-controller: lifemonitor.auth.controllers operationId: "user_subscriptions_get" - summary: "List all subscriptions for the current user" + summary: "List subscriptions for the current user" description: | List all subscriptions for the current user security: @@ -625,6 +741,42 @@ paths: "404": $ref: "#/components/responses/NotFound" + put: + summary: Subscribe user to workflow events + description: "Subscribe the current (authenticated) user to a list of events for the specified workflow" + x-openapi-router-controller: lifemonitor.api.controllers + operationId: "user_workflow_subscribe_events" + tags: ["Users"] + security: + - apiKey: ["workflow.write"] + - AuthorizationCodeFlow: ["workflow.write"] + - RegistryCodeFlow: ["workflow.write"] + parameters: + - $ref: "#/components/parameters/wf_uuid" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ListOfEvents" + responses: + "201": + description: "Subscription to events created" + content: + application/json: + schema: + $ref: "#/components/schemas/Subscription" + "204": + description: "Subscription updated with events" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + /workflows/{wf_uuid}/unsubscribe: post: summary: Unsubscribe user from a workflow @@ -1290,6 +1442,14 @@ components: minimum: 1 default: 10 description: "Maximum number of items to retrieve" + notification_uuid: + name: "notification_uuid" + description: "Universal unique identifier of the user notification" + in: path + schema: + type: string + required: true + example: 123e4567-e89b-12d3-a456-426614174000 responses: NotFound: @@ -1462,11 +1622,29 @@ components: description: | A timestamp for the modification time of the subscription example: 1616427512.0 + events: + $ref: "#/components/schemas/ListOfEvents" required: - user - resource - created - modified + - events + + EventType: + type: string + enum: + - ALL + - BUILD_FAILED + - BUILD_RECOVERED + + ListOfEvents: + type: array + items: + $ref: "#/components/schemas/EventType" + description: | + List of events related to the specified workflow + example: ["all"] ListOfSubscriptions: type: object @@ -1478,6 +1656,49 @@ components: required: - items + Notification: + type: object + description: A notification for the current user. + properties: + uuid: + type: string + description: Universal unique identifier of notification + example: aa16c6a9-571f-4a62-976f-60ea514ad2c1 + data: + type: object + description: "Notification payload" + readOnly: true + created: + type: string + description: | + A timestamp for the creation time of the notification + example: 1616427012.0 + readOnly: true + read: + type: string + description: | + A timestamp for the read time of the notification + example: 1616427012.0 + emailed: + type: string + description: | + A timestamp for the time when the email was sent + example: 1616427012.0 + readOnly: true + required: + - uuid + - created + + ListOfNotifications: + type: object + properties: + items: + type: array + items: + $ref: "#/components/schemas/Notification" + required: + - items + IdentityProviderType: type: string enum: @@ -1638,13 +1859,13 @@ components: latest_version: type: string description: The workflow identifier on the registry - example: '1.0' + example: "1.0" versions: description: The list of workflow versions type: array items: type: string - example: ['1.0', "1.0-dev"] + example: ["1.0", "1.0-dev"] ListOfRegistryWorkflows: type: object diff --git a/tests/integration/api/controllers/test_user_subscriptions.py b/tests/integration/api/controllers/test_user_subscriptions.py index 9236fb441..275b91e12 100644 --- a/tests/integration/api/controllers/test_user_subscriptions.py +++ b/tests/integration/api/controllers/test_user_subscriptions.py @@ -23,7 +23,7 @@ import pytest from lifemonitor.api.services import LifeMonitor -from lifemonitor.auth.models import User +from lifemonitor.auth.models import EventType, User from tests import utils from tests.conftest_helpers import enable_auto_login from tests.conftest_types import ClientAuthenticationMethod @@ -62,12 +62,78 @@ def test_user_unsubscribe_not_authorized(app_client, client_auth_method, user1, ClientAuthenticationMethod.REGISTRY_CODE_FLOW ], indirect=True) @pytest.mark.parametrize("user1", [True], indirect=True) -def test_user_subscribe_workflow(app_client, client_auth_method, user1, user1_auth, valid_workflow): +def test_submitter_subscribe_workflow(app_client, client_auth_method, + user1, user1_auth, valid_workflow): workflow = utils.pick_workflow(user1, valid_workflow) logger.debug("User1 Auth Headers: %r", user1_auth) enable_auto_login(user1['user']) r = app_client.post( - utils.build_workflow_path(workflow, include_version=False, subpath='subscribe'), headers=user1_auth + utils.build_workflow_path(workflow, include_version=False, subpath='subscribe'), + headers=user1_auth + ) + assert r.status_code == 204, f"Error when subscribing to the workflow {workflow}" + + +@pytest.mark.parametrize("client_auth_method", [ + # ClientAuthenticationMethod.BASIC, + ClientAuthenticationMethod.API_KEY, + ClientAuthenticationMethod.AUTHORIZATION_CODE, + ClientAuthenticationMethod.REGISTRY_CODE_FLOW +], indirect=True) +@pytest.mark.parametrize("user1", [True], indirect=True) +def test_user_subscribe_workflow(app_client, client_auth_method, + user1, user1_auth, user2, user2_auth, + valid_workflow): + workflow = utils.pick_workflow(user1, valid_workflow) + logger.debug("User1 Auth Headers: %r", user1_auth) + enable_auto_login(user1['user']) + r = app_client.post( + utils.build_workflow_path(workflow, include_version=False, subpath='subscribe'), + headers=user2_auth + ) + assert r.status_code == 201, f"Error when subscribing to the workflow {workflow}" + data = json.loads(r.data.decode()) + logger.debug(data) + + +@pytest.mark.parametrize("client_auth_method", [ + # ClientAuthenticationMethod.BASIC, + ClientAuthenticationMethod.API_KEY, + ClientAuthenticationMethod.AUTHORIZATION_CODE, + ClientAuthenticationMethod.REGISTRY_CODE_FLOW +], indirect=True) +@pytest.mark.parametrize("user1", [True], indirect=True) +def test_user_subscribe_workflow_old_events(app_client, client_auth_method, + user1, user1_auth, valid_workflow): + workflow = utils.pick_workflow(user1, valid_workflow) + logger.debug("User1 Auth Headers: %r", user1_auth) + enable_auto_login(user1['user']) + body = [EventType.ALL.name] + r = app_client.put( + utils.build_workflow_path(workflow, include_version=False, subpath='subscribe'), + headers=user1_auth, + json=body + ) + assert r.status_code == 204, f"Error when subscribing to the workflow {workflow}" + + +@pytest.mark.parametrize("client_auth_method", [ + # ClientAuthenticationMethod.BASIC, + ClientAuthenticationMethod.API_KEY, + ClientAuthenticationMethod.AUTHORIZATION_CODE, + ClientAuthenticationMethod.REGISTRY_CODE_FLOW +], indirect=True) +@pytest.mark.parametrize("user1", [True], indirect=True) +def test_user_subscribe_workflow_new_events(app_client, client_auth_method, + user1, user1_auth, valid_workflow): + workflow = utils.pick_workflow(user1, valid_workflow) + logger.debug("User1 Auth Headers: %r", user1_auth) + enable_auto_login(user1['user']) + body = [EventType.BUILD_FAILED.name, EventType.BUILD_RECOVERED.name] + r = app_client.put( + utils.build_workflow_path(workflow, include_version=False, subpath='subscribe'), + headers=user1_auth, + json=body ) assert r.status_code == 201, f"Error when subscribing to the workflow {workflow}" data = json.loads(r.data.decode()) @@ -88,8 +154,25 @@ def test_user_unsubscribe_workflow(app_client, client_auth_method, user1, user1_ # register a subscription workflow = lm.get_workflow(wdata['uuid']) assert workflow, "Invalid workflow" - lm.subscribe_user_resource(user, workflow) - assert len(user.subscriptions) == 1, "Invalid number of subscriptions" + subscription = user.get_subscription(workflow) + assert subscription, "User should be subscribed to the workflow" + + # check number of subscriptions before unsubscribing + r = app_client.get('/users/current/subscriptions', headers=user1_auth) + assert r.status_code == 200, f"Error when getting the list of subscriptions to the workflow {workflow}" + data = r.get_json() + logger.debug("Current list of subscriptions: %r", data) + assert 'items' in data, "Unexpected response type: missing 'items' property" + number_of_subscriptions = len(data['items']) + logger.debug("Current number of subscriptions: %r", number_of_subscriptions) + + # check if there exists a subscription for the workflow + found = False + for s in data['items']: + if s['resource']['uuid'] == str(workflow.uuid): + found = True + break + assert found, "Unable to find the workflow among subscriptions" # try to delete the subscription via API r = app_client.post( @@ -97,6 +180,13 @@ def test_user_unsubscribe_workflow(app_client, client_auth_method, user1, user1_ ) assert r.status_code == 204, f"Error when unsubscribing to the workflow {workflow}" + # check number of subscriptions after unsubscribing + r = app_client.get('/users/current/subscriptions', headers=user1_auth) + assert r.status_code == 200, f"Error when getting the list of subscriptions to the workflow {workflow}" + data = r.get_json() + assert 'items' in data, "Unexpected response type: missing 'items' property" + assert len(data['items']) == number_of_subscriptions - 1, "Unexpected number of subscriptions" + @pytest.mark.parametrize("client_auth_method", [ # ClientAuthenticationMethod.BASIC, @@ -105,9 +195,13 @@ def test_user_unsubscribe_workflow(app_client, client_auth_method, user1, user1_ ClientAuthenticationMethod.REGISTRY_CODE_FLOW ], indirect=True) @pytest.mark.parametrize("user1", [True], indirect=True) -def test_user_subscriptions(app_client, client_auth_method, user1, user1_auth, valid_workflow, lm: LifeMonitor): +def test_user_subscriptions(app_client, client_auth_method, + user1, user1_auth, user2, user2_auth, + valid_workflow, lm: LifeMonitor): + # pick user1 workflow wdata = utils.pick_workflow(user1, valid_workflow) - user: User = user1['user'] + # set user2 as current user + user: User = user2['user'] enable_auto_login(user) # register a subscription workflow = lm.get_workflow(wdata['uuid']) @@ -117,7 +211,7 @@ def test_user_subscriptions(app_client, client_auth_method, user1, user1_auth, v # get subscriptions of the current user r = app_client.get( - '/users/current/subscriptions', headers=user1_auth + '/users/current/subscriptions', headers=user2_auth ) assert r.status_code == 200, "Error when trying to get user subscriptions" data = json.loads(r.data.decode()) diff --git a/tests/integration/api/services/test_workflow_subscriptions.py b/tests/integration/api/services/test_workflow_subscriptions.py index 84fbcffa2..df9f237ae 100644 --- a/tests/integration/api/services/test_workflow_subscriptions.py +++ b/tests/integration/api/services/test_workflow_subscriptions.py @@ -20,25 +20,63 @@ import logging -from lifemonitor.auth.models import User, Subscription +from lifemonitor.auth.models import EventType, User, Subscription from lifemonitor.api.services import LifeMonitor from tests import utils logger = logging.getLogger() -def test_user_workflow_subscription(app_client, lm: LifeMonitor, user1: dict, valid_workflow: str): - _, workflow = utils.pick_and_register_workflow(user1, valid_workflow) +def test_submitter_workflow_subscription(app_client, lm: LifeMonitor, user1: dict, valid_workflow: str): + _, workflow_version = utils.pick_and_register_workflow(user1, valid_workflow) + user: User = user1['user'] + # check number of subscriptions + assert len(user.subscriptions) == 1, "Unexpected number of subscriptions" + # subscribe to the workflow + s: Subscription = user.subscriptions[0] + assert s, "Subscription should not be empty" + assert s.resource.uuid == workflow_version.workflow.uuid, "Unexpected resource UUID" + assert s.resource == workflow_version.workflow, "Unexpected resource instance" + assert len(s.events) == 1, "Unexpected number of subscription events" + + +def test_submitter_workflow_unsubscription(app_client, lm: LifeMonitor, user1: dict, valid_workflow: str): + _, workflow_version = utils.pick_and_register_workflow(user1, valid_workflow) user: User = user1['user'] # check number of subscriptions + assert len(user.subscriptions) == 1, "Unexpected number of subscriptions" + # unsubscribe to workflow + s: Subscription = lm.unsubscribe_user_resource(user, workflow_version.workflow) + assert s, "Subscription should not be empty" + # check number of subscriptions + assert len(user.subscriptions) == 0, "Unexpected number of subscriptions" + + +def test_user_workflow_subscription(app_client, lm: LifeMonitor, user1: dict, user2: dict, valid_workflow: str): + _, workflow = utils.pick_and_register_workflow(user1, valid_workflow) + user: User = user2['user'] + # check number of subscriptions assert len(user.subscriptions) == 0, "Unexpected number of subscriptions" + # subscribe to the workflow - s: Subscription = lm.subscribe_user_resource(user, workflow) + events = [EventType.BUILD_FAILED, EventType.BUILD_RECOVERED] + s: Subscription = lm.subscribe_user_resource(user, workflow, events) assert s, "Subscription should not be empty" assert s.resource.uuid == workflow.uuid, "Unexpected resource UUID" assert s.resource == workflow, "Unexpected resource instance" + assert len(s.events) == len(events), "Unexpected number of subscription events" + for e in events: + assert s.has_event(e), f"Subscription should be have event {e}" # check number of subscriptions assert len(user.subscriptions) == 1, "Unexpected number of subscriptions" + + # update subscription events + events = [EventType.BUILD_FAILED] + s: Subscription = lm.subscribe_user_resource(user, workflow, events) + assert len(s.events) == len(events), "Unexpected number of subscription events" + for e in events: + assert s.has_event(e), f"Subscription should be have event {e}" + # unsubscribe to workflow s: Subscription = lm.unsubscribe_user_resource(user, workflow) assert s, "Subscription should not be empty" diff --git a/tests/settings.conf b/tests/settings.conf index 77c5aa288..b2732fdaf 100644 --- a/tests/settings.conf +++ b/tests/settings.conf @@ -8,6 +8,9 @@ # It is only used to build the links returned by the API #EXTERNAL_ACCESS_BASE_URL="https://api.lifemonitor.eu" +# Base URL of the LifeMonitor web app associated with this back-end instance +# WEBAPP_URL=https://app.lifemonitor.eu + # Normally, OAuthLib will raise an InsecureTransportError if you attempt to use OAuth2 over HTTP, # rather than HTTPS. Setting this environment variable will prevent this error from being raised. # This is mostly useful for local testing, or automated tests. Never set this variable in production. @@ -43,6 +46,15 @@ CACHE_SESSION_TIMEOUT=3600 CACHE_WORKFLOW_TIMEOUT=1800 CACHE_BUILD_TIMEOUT=84600 +# Email settings +MAIL_SERVER='' +MAIL_PORT=465 +MAIL_USERNAME='' +MAIL_PASSWORD='' +MAIL_USE_TLS=False +MAIL_USE_SSL=True +MAIL_DEFAULT_SENDER='' + # PostgreSQL DBMS settings #POSTGRESQL_HOST=0.0.0.0 #POSTGRESQL_PORT=5432 diff --git a/tests/unit/api/models/test_subscriptions.py b/tests/unit/api/models/test_subscriptions.py index c864d3bd5..b373e6d76 100644 --- a/tests/unit/api/models/test_subscriptions.py +++ b/tests/unit/api/models/test_subscriptions.py @@ -22,21 +22,48 @@ import logging -from lifemonitor.auth.models import Subscription, User +from lifemonitor.auth.models import User, EventType from tests import utils logger = logging.getLogger() def test_workflow_subscription(user1: dict, valid_workflow: str): - _, workflow = utils.pick_and_register_workflow(user1, valid_workflow) + _, workflow_version = utils.pick_and_register_workflow(user1, valid_workflow) user: User = user1['user'] - s: Subscription = user.subscribe(workflow) - logger.debug("Subscription: %r", s) - assert s, "Subscription should not be empty" - assert len(user.subscriptions) == 1, "Unexpected number of subscriptions" - - s: Subscription = user.unsubscribe(workflow) - logger.debug("Subscription: %r", s) - assert s, "Subscription should not be empty" + + # check default subscription + s = user.get_subscription(workflow_version.workflow) + assert s, "The submitter subscription should be automatically registered" + # check default events + assert len(s.events) == 1, "Invalid number of events" + assert s.has_event(EventType.ALL), f"Event '{EventType.ALL.name}' not registered on the subscription" + for event in EventType.all(): + assert s.has_event(event), f"Event '{event.name}' should be included" + + # check delete all events + s.events = None + assert len(s.events) == 0, "Invalid number of events" + s.save() + + # check event udpate + s.events = [EventType.BUILD_FAILED, EventType.BUILD_RECOVERED] + s.save() + assert len(s.events) == 2, "Invalid number of events" + assert not s.has_event(EventType.ALL), f"Event '{EventType.ALL.name}' should not be registered on the subscription" + + +def test_workflow_unsubscription(user1: dict, valid_workflow: str): + _, workflow_version = utils.pick_and_register_workflow(user1, valid_workflow) + user: User = user1['user'] + + # check default subscription + s = user.get_subscription(workflow_version.workflow) + assert s, "The submitter subscription should be automatically registered" + + # test unsubscription + user.unsubscribe(workflow_version.workflow) + user.save() assert len(user.subscriptions) == 0, "Unexpected number of subscriptions" + s = user.get_subscription(workflow_version.workflow) + assert s is None, "Subscription should be empty"