From 84b30556d4e8201c508b35a8c343313cc1d5ec10 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 2 May 2023 16:17:44 +0200 Subject: [PATCH 01/39] fix(ctrl): fix form to manage registry integration --- lifemonitor/auth/templates/auth/registry_settings.j2 | 8 +++++--- prova | 0 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 prova diff --git a/lifemonitor/auth/templates/auth/registry_settings.j2 b/lifemonitor/auth/templates/auth/registry_settings.j2 index 9590f8e05..06bd7500c 100644 --- a/lifemonitor/auth/templates/auth/registry_settings.j2 +++ b/lifemonitor/auth/templates/auth/registry_settings.j2 @@ -15,17 +15,19 @@ +
Registry Name
Registry URL
-
Registry Type
- +
Registry Type
+ + {% for registry in registrySettingsForm.available_registries %}
- {% set registry_enabled = registry.client_name in registrySettingsForm.registries.data %} + {% set registry_enabled = registry.client_name in registrySettingsForm.registries.data.split(',') %}
{{ registrySettingsForm.csrf_token() }} diff --git a/prova b/prova new file mode 100644 index 000000000..e69de29bb From 0b3aaf6e8d891c2b734af8b1d104cda1e10b7a82 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 2 May 2023 17:35:34 +0200 Subject: [PATCH 02/39] feat(ui): better feedback for users --- .../auth/templates/auth/registry_settings.j2 | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/lifemonitor/auth/templates/auth/registry_settings.j2 b/lifemonitor/auth/templates/auth/registry_settings.j2 index 06bd7500c..6153e2b3c 100644 --- a/lifemonitor/auth/templates/auth/registry_settings.j2 +++ b/lifemonitor/auth/templates/auth/registry_settings.j2 @@ -22,6 +22,26 @@
Registry Type
+ + {% for registry in registrySettingsForm.available_registries %}
@@ -34,10 +54,10 @@ - + data-toggle="toggle" {% if registry_enabled %}checked{% endif%} + data-off='' onchange="handleClick('{{ registry.client_name }}')">
From 0ab49573dad319220d5d71b30fd293eeb4896b20 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 2 May 2023 17:41:15 +0200 Subject: [PATCH 03/39] fix(ui): replace icon with text 'off' --- lifemonitor/auth/templates/auth/registry_settings.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lifemonitor/auth/templates/auth/registry_settings.j2 b/lifemonitor/auth/templates/auth/registry_settings.j2 index 6153e2b3c..77bdbba92 100644 --- a/lifemonitor/auth/templates/auth/registry_settings.j2 +++ b/lifemonitor/auth/templates/auth/registry_settings.j2 @@ -57,7 +57,7 @@ + data-off='Off' onchange="handleClick('{{ registry.client_name }}')">
From d55f60ad802cb19fbf7a906660e13bea43ddc192 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 2 May 2023 17:41:45 +0200 Subject: [PATCH 04/39] style: update margin --- lifemonitor/auth/templates/auth/registry_settings.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lifemonitor/auth/templates/auth/registry_settings.j2 b/lifemonitor/auth/templates/auth/registry_settings.j2 index 77bdbba92..c86f61725 100644 --- a/lifemonitor/auth/templates/auth/registry_settings.j2 +++ b/lifemonitor/auth/templates/auth/registry_settings.j2 @@ -63,7 +63,7 @@
-
+
{{ registry.name }}
From 14758fca6808729652111b00f689bd4b4faebc45 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 2 May 2023 18:45:50 +0200 Subject: [PATCH 05/39] fix: rename property to denote workflow name on lm config schema --- lifemonitor/schemas/lifemonitor.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lifemonitor/schemas/lifemonitor.json b/lifemonitor/schemas/lifemonitor.json index 863e211de..37c270d5e 100644 --- a/lifemonitor/schemas/lifemonitor.json +++ b/lifemonitor/schemas/lifemonitor.json @@ -4,7 +4,7 @@ "title": "LifeMonitor Configuration File", "type": "object", "properties": { - "workflow_name": { + "name": { "description": "worfklow name (override name defined on the RO-Crate metadata)", "type": "string", "minLength": 1 From 3b970a94b79d251e84895aee5408c5f186d0624e Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 2 May 2023 18:47:33 +0200 Subject: [PATCH 06/39] fix: add some missing values on auto-generated lm config file --- lifemonitor/api/models/repositories/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lifemonitor/api/models/repositories/base.py b/lifemonitor/api/models/repositories/base.py index d65e8ba97..fc3453209 100644 --- a/lifemonitor/api/models/repositories/base.py +++ b/lifemonitor/api/models/repositories/base.py @@ -301,7 +301,10 @@ def generate_config(self, ignore_existing=False) -> WorkflowFile: current_config = self.config if current_config and not ignore_existing: raise IllegalStateException("Config exists") - self._config = WorkflowRepositoryConfig.new(self.local_path, workflow_title=self.metadata.main_entity_name if self.metadata else None) + self._config = WorkflowRepositoryConfig.new(self.local_path, + workflow_title=self.metadata.main_entity_name if self.metadata else None, + main_branch=self.default_branch, + public=self.public) return self._config def write_zip(self, target_path: str): From c102b3f542f64eaa1db23f1b51b27888917ac554 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 2 May 2023 18:55:28 +0200 Subject: [PATCH 07/39] fix: report repo loading errors --- cli/client/issues.py | 3 ++- cli/client/utils.py | 21 ++++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/cli/client/issues.py b/cli/client/issues.py index 4097a51ac..71a24a141 100644 --- a/cli/client/issues.py +++ b/cli/client/issues.py @@ -147,7 +147,8 @@ def check(config, repository, output_path): console.print(f"[{message.type.value}]{message.type.name}:[/{message.type.value}] {message.text}") console.print("\n\n") except Exception as e: - logger.exception(e) + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) error_console.print(str(e)) diff --git a/cli/client/utils.py b/cli/client/utils.py index affd0152e..042e9533c 100644 --- a/cli/client/utils.py +++ b/cli/client/utils.py @@ -42,15 +42,18 @@ def is_url(value): def get_repository(repository: str, local_path: str): assert repository, repository - if is_url(repository): - remote_repo_url = repository - if remote_repo_url.endswith('.git'): - return GithubWorkflowRepository.from_url(remote_repo_url, auto_cleanup=False, local_path=local_path) - else: - local_copy_path = os.path.join(local_path, os.path.basename(repository)) - shutil.copytree(repository, local_copy_path) - return LocalWorkflowRepository(local_copy_path) - raise ValueError("Repository type not supported") + try: + if is_url(repository): + remote_repo_url = repository + if remote_repo_url.endswith('.git'): + return GithubWorkflowRepository.from_url(remote_repo_url, auto_cleanup=False, local_path=local_path) + else: + local_copy_path = os.path.join(local_path, os.path.basename(repository)) + shutil.copytree(repository, local_copy_path) + return LocalWorkflowRepository(local_copy_path) + raise ValueError("Repository type not supported") + except Exception as e: + raise ValueError("Error while loading the repository: %s" % e) def init_output_path(output_path): From 07d1e6927685ce0fd9691f8397fd498e4d5a6547 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 2 May 2023 23:08:21 +0200 Subject: [PATCH 08/39] feat(model): add minimal local git repo model --- cli/client/utils.py | 4 +++- lifemonitor/api/models/repositories/local.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/cli/client/utils.py b/cli/client/utils.py index 042e9533c..a47d8a713 100644 --- a/cli/client/utils.py +++ b/cli/client/utils.py @@ -25,7 +25,7 @@ from urllib.parse import urlparse from lifemonitor.api.models.repositories.github import GithubWorkflowRepository -from lifemonitor.api.models.repositories.local import LocalWorkflowRepository +from lifemonitor.api.models.repositories.local import LocalWorkflowRepository, LocalGitRepository from rich.prompt import Prompt # Set module logger @@ -50,6 +50,8 @@ def get_repository(repository: str, local_path: str): else: local_copy_path = os.path.join(local_path, os.path.basename(repository)) shutil.copytree(repository, local_copy_path) + if LocalGitRepository.is_git_repo(local_copy_path): + return LocalGitRepository(local_copy_path) return LocalWorkflowRepository(local_copy_path) raise ValueError("Repository type not supported") except Exception as e: diff --git a/lifemonitor/api/models/repositories/local.py b/lifemonitor/api/models/repositories/local.py index f9345cab9..890a2c93c 100644 --- a/lifemonitor/api/models/repositories/local.py +++ b/lifemonitor/api/models/repositories/local.py @@ -174,3 +174,20 @@ def __init__(self, base64_rocrate: str) -> None: except Exception as e: logger.debug(e) raise DecodeROCrateException(detail=str(e)) + + +class LocalGitRepository(LocalWorkflowRepository): + + def __init__(self, local_path: str | None = None, exclude: List[str] | None = None) -> None: + super().__init__(local_path, exclude) + assert self.is_git_repo(self.local_path), f"Local path {local_path} is not a git repository" + + @property + def main_branch(self) -> str: + from git import Repo + repo = Repo(self.local_path) + return repo.active_branch.name + + @staticmethod + def is_git_repo(local_path: str) -> bool: + return os.path.isdir(os.path.join(local_path, '.git')) From 49287a07753083c848247807ef7aa841810196fb Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 2 May 2023 23:11:21 +0200 Subject: [PATCH 09/39] fix: auto injection of default values to init config --- lifemonitor/api/models/repositories/base.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lifemonitor/api/models/repositories/base.py b/lifemonitor/api/models/repositories/base.py index fc3453209..d1e0aa3ce 100644 --- a/lifemonitor/api/models/repositories/base.py +++ b/lifemonitor/api/models/repositories/base.py @@ -297,14 +297,17 @@ def config(self) -> WorkflowRepositoryConfig: pass return self._config - def generate_config(self, ignore_existing=False) -> WorkflowFile: + def generate_config(self, ignore_existing=False, + workflow_title: Optional[str] = None, + public: bool = False, main_branch: Optional[str] = None) -> WorkflowFile: current_config = self.config if current_config and not ignore_existing: raise IllegalStateException("Config exists") self._config = WorkflowRepositoryConfig.new(self.local_path, - workflow_title=self.metadata.main_entity_name if self.metadata else None, - main_branch=self.default_branch, - public=self.public) + workflow_title=workflow_title if workflow_title is not None + else self.metadata.main_entity_name if self.metadata else None, + main_branch=main_branch if main_branch else getattr(self, "main_branch", ""), + public=public) return self._config def write_zip(self, target_path: str): From d243d76219a60ef5152ac5b6bb8bf47404483be5 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 3 May 2023 09:44:55 +0200 Subject: [PATCH 10/39] refactor: check github url --- lifemonitor/api/models/rocrate/__init__.py | 38 +++++++++++----------- lifemonitor/integrations/github/utils.py | 16 +++++++++ 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/lifemonitor/api/models/rocrate/__init__.py b/lifemonitor/api/models/rocrate/__init__.py index cb4863895..d26bcaafd 100644 --- a/lifemonitor/api/models/rocrate/__init__.py +++ b/lifemonitor/api/models/rocrate/__init__.py @@ -28,9 +28,13 @@ from pathlib import Path from typing import Dict, List, Optional, Tuple -import lifemonitor.exceptions as lm_exceptions from flask import current_app from github.GithubException import GithubException, RateLimitExceededException +from rocrate import rocrate +from sqlalchemy import inspect +from sqlalchemy.ext.hybrid import hybrid_property + +import lifemonitor.exceptions as lm_exceptions from lifemonitor.api.models import db, repositories from lifemonitor.api.models.repositories.base import ( WorkflowRepository, WorkflowRepositoryMetadata) @@ -43,10 +47,6 @@ from lifemonitor.models import JSON from lifemonitor.storage import RemoteStorage from lifemonitor.utils import download_url, get_current_ref -from sqlalchemy import inspect -from sqlalchemy.ext.hybrid import hybrid_property - -from rocrate import rocrate # set module level logger logger = logging.getLogger(__name__) @@ -111,7 +111,7 @@ def storage_path(self) -> str: @property def revision(self) -> Optional[GithubRepositoryRevision]: - if self._is_github_crate_(self.uri) and isinstance(self.repository, GithubWorkflowRepository): + if self._get_normalized_github_url_(self.uri) and isinstance(self.repository, GithubWorkflowRepository): return self.repository.get_revision(self.version) return None @@ -172,7 +172,7 @@ def repository(self) -> repositories.WorkflowRepository: logger.warning(f"Getting path {self.storage_path} from remote storage.... DONE!!!") # instantiate a local ROCrate repository - if self._is_github_crate_(self.uri): + if self._get_normalized_github_url_(self.uri): authorizations = self.authorizations + [None] token = None for authorization in authorizations: @@ -192,8 +192,13 @@ def repository(self) -> repositories.WorkflowRepository: self._repository = repositories.ZippedWorkflowRepository(self.local_path) # set metadata - self._metadata = self._repository.metadata.to_json() - self._metadata_loaded = True + try: + self._metadata = self._repository.metadata.to_json() + self._metadata_loaded = True + except Exception as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) + raise lm_exceptions.NotValidROCrateException("Unable to load ROCrate metadata") return self._repository @property @@ -215,13 +220,13 @@ def __get_attribute_from_crate_reader__(self, if logger.isEnabledFor(logging.DEBUG): logger.exception(e) if not ignore_errors: - raise e + raise except Exception as e: logger.error(str(e)) if logger.isEnabledFor(logging.DEBUG): logger.exception(e) if not ignore_errors: - raise e + raise return None def get_authors(self, suite_id: str = None) -> List[Dict] | None: @@ -284,14 +289,9 @@ def download(self, target_path: str) -> str: return (tmpdir_path / 'rocrate.zip').as_posix() @staticmethod - def _is_github_crate_(uri: str) -> str: - # FIXME: replace with a better detection mechanism - if uri.startswith('https://github.com'): - # normalize uri as clone URL - if not uri.endswith('.git'): - uri += '.git' - return uri - return None + def _get_normalized_github_url_(uri: str) -> Optional[str]: + from lifemonitor.integrations.github.utils import normalized_github_url + return normalized_github_url(uri) @staticmethod def _find_git_ref_(repo: GithubWorkflowRepository, version: str) -> str: diff --git a/lifemonitor/integrations/github/utils.py b/lifemonitor/integrations/github/utils.py index 70c835a20..f2454dba9 100644 --- a/lifemonitor/integrations/github/utils.py +++ b/lifemonitor/integrations/github/utils.py @@ -47,6 +47,22 @@ logger = logging.getLogger(__name__) +def is_github_url(url): + """ + Returns True if the given URL is a GitHub URL, False otherwise. + """ + pattern = r"^https?://(www\.)?github\.com/.*$" + return bool(re.match(pattern, url)) + + +def normalized_github_url(url): + if not is_github_url(url): + return None + if not url.endswith('.git'): + url += '.git' + return url + + def crate_branch(repo: Repository, branch_name: str, rev: str = None) -> GitRef: head = repo.get_commit(rev or repo.rev or 'HEAD') logger.debug("HEAD commit: %r", head.sha) From 20fe74cb15721e5c5934a298e8f818a7a9b57d9e Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 3 May 2023 10:16:24 +0200 Subject: [PATCH 11/39] fix: set defaults --- lifemonitor/api/models/repositories/base.py | 2 +- lifemonitor/templates/repositories/base/.lifemonitor.yaml.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lifemonitor/api/models/repositories/base.py b/lifemonitor/api/models/repositories/base.py index d1e0aa3ce..80114ba34 100644 --- a/lifemonitor/api/models/repositories/base.py +++ b/lifemonitor/api/models/repositories/base.py @@ -306,7 +306,7 @@ def generate_config(self, ignore_existing=False, self._config = WorkflowRepositoryConfig.new(self.local_path, workflow_title=workflow_title if workflow_title is not None else self.metadata.main_entity_name if self.metadata else None, - main_branch=main_branch if main_branch else getattr(self, "main_branch", ""), + main_branch=main_branch if main_branch else getattr(self, "main_branch", "main"), public=public) return self._config diff --git a/lifemonitor/templates/repositories/base/.lifemonitor.yaml.j2 b/lifemonitor/templates/repositories/base/.lifemonitor.yaml.j2 index 9d88148f6..a392db824 100644 --- a/lifemonitor/templates/repositories/base/.lifemonitor.yaml.j2 +++ b/lifemonitor/templates/repositories/base/.lifemonitor.yaml.j2 @@ -2,7 +2,7 @@ {% if workflow_name %}name: {{ workflow_name }} {% else %}# name: MyWorkflow{%endif %} # worfklow visibility -public: {{ public }} +public: {{ "true" if public else "false" }} # Issue Checker Settings issues: From 66c93ef95e73a073de5a703f41fd06aff328f91a Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 3 May 2023 11:25:28 +0200 Subject: [PATCH 12/39] fix: uncommitted line on d243d76 --- lifemonitor/api/models/rocrate/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lifemonitor/api/models/rocrate/__init__.py b/lifemonitor/api/models/rocrate/__init__.py index d26bcaafd..0b1aff921 100644 --- a/lifemonitor/api/models/rocrate/__init__.py +++ b/lifemonitor/api/models/rocrate/__init__.py @@ -327,7 +327,7 @@ def download_from_source(self, target_path: str = None, uri: str = None, version # try either with authorization header and without authorization for authorization in self._get_authorizations(extra_auth=extra_auth): try: - git_url = self._is_github_crate_(uri) + git_url = self._get_normalized_github_url_(uri) if git_url: token = None if authorization and isinstance(authorization, ExternalServiceAuthorizationHeader): From d74fb9b155abd8ed2b15bbac7d304ff2cd753d75 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 3 May 2023 12:40:54 +0200 Subject: [PATCH 13/39] fix(srv): make not blocking query to unavailable registries --- lifemonitor/api/models/registries/registry.py | 2 ++ lifemonitor/api/services.py | 9 +++++++++ lifemonitor/utils.py | 14 ++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/lifemonitor/api/models/registries/registry.py b/lifemonitor/api/models/registries/registry.py index 85381d3de..cb7c312d7 100644 --- a/lifemonitor/api/models/registries/registry.py +++ b/lifemonitor/api/models/registries/registry.py @@ -440,6 +440,8 @@ def delete_workflow(self, submitter: auth_models.User, external_id: str) -> bool @property def client(self) -> WorkflowRegistryClient: + if not is_service_alive(self.uri): + raise lm_exceptions.ServiceUnavailableException(f"Service {self.url} is not available", service=self) if self._client is None: rtype = self.__class__.__name__.replace("WorkflowRegistry", "").lower() return WorkflowRegistryClient.get_client_class(rtype)(self) diff --git a/lifemonitor/api/services.py b/lifemonitor/api/services.py index 09ba086c7..69bd446aa 100644 --- a/lifemonitor/api/services.py +++ b/lifemonitor/api/services.py @@ -58,6 +58,9 @@ def __init__(self): def _find_and_check_shared_workflow_version(user: User, uuid, version=None) -> models.WorkflowVersion: for svc in models.WorkflowRegistry.all(): try: + if not is_service_alive(svc.uri): + logger.warning(f"Service {svc.uri} is not alive") + continue if svc.get_user(user.id): for w in svc.get_user_workflows(user): if str(w.uuid) == str(uuid): @@ -559,6 +562,9 @@ def get_user_workflows(user: User, workflows = [w for w in models.Workflow.get_user_workflows(user, include_subscriptions=include_subscriptions)] for svc in models.WorkflowRegistry.all(): + if not is_service_alive(svc.uri): + logger.warning("Service %r is not alive, skipping", svc.uri) + continue if svc.get_user(user.id): try: workflows.extend([w for w in svc.get_user_workflows(user) @@ -570,6 +576,9 @@ def get_user_workflows(user: User, @staticmethod def get_user_registry_workflows(user: User, registry: models.WorkflowRegistry) -> List[models.Workflow]: workflows = [] + if not is_service_alive(registry.uri): + logger.warning("Service %r is not alive, skipping", registry.uri) + raise lm_exceptions.UnavailableServiceException(registry.uri, service=registry) if registry.get_user(user.id): try: workflows.extend([w for w in registry.get_user_workflows(user) diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index fe3d2785f..bc3e970f0 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -226,6 +226,20 @@ def validate_url(url: str) -> bool: return False +def is_service_alive(url: str, timeout: int = 5) -> bool: + try: + response = requests.get(url, timeout=timeout) + if response.status_code == 200: + return True + else: + return False + except requests.exceptions.RequestException as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) + logger.error(f'Error checking service availability: {e}') + return False + + def get_last_update(path: str): return time.ctime(max(os.stat(root).st_mtime for root, _, _ in os.walk(path))) From 2e886b15c0d1e784890e07c304bb72ce16644540 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 3 May 2023 12:50:54 +0200 Subject: [PATCH 14/39] perf: cache service availability --- lifemonitor/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index bc3e970f0..35fa4d52f 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -226,6 +226,7 @@ def validate_url(url: str) -> bool: return False +@cached(client_scope=False) def is_service_alive(url: str, timeout: int = 5) -> bool: try: response = requests.get(url, timeout=timeout) From a5fb68e85f9e9705855f63c7de204e69c1c2206c Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 3 May 2023 12:56:58 +0200 Subject: [PATCH 15/39] fix: add missing imports --- lifemonitor/api/models/registries/registry.py | 11 ++++++----- lifemonitor/api/services.py | 2 +- lifemonitor/utils.py | 2 ++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lifemonitor/api/models/registries/registry.py b/lifemonitor/api/models/registries/registry.py index cb7c312d7..021f6a87c 100644 --- a/lifemonitor/api/models/registries/registry.py +++ b/lifemonitor/api/models/registries/registry.py @@ -24,10 +24,13 @@ from abc import ABC, abstractmethod from typing import List, Tuple, Union -import lifemonitor.api.models as models -import lifemonitor.exceptions as lm_exceptions import requests from authlib.integrations.base_client import RemoteApp +from sqlalchemy.orm.collections import attribute_mapped_collection +from sqlalchemy.orm.exc import NoResultFound + +import lifemonitor.api.models as models +import lifemonitor.exceptions as lm_exceptions from lifemonitor import utils as lm_utils from lifemonitor.api.models import db from lifemonitor.api.models.repositories.base import WorkflowRepository @@ -35,9 +38,7 @@ from lifemonitor.auth.models import Resource from lifemonitor.auth.oauth2.client.models import OAuthIdentity from lifemonitor.auth.oauth2.client.services import oauth2_registry -from lifemonitor.utils import ClassManager, download_url -from sqlalchemy.orm.collections import attribute_mapped_collection -from sqlalchemy.orm.exc import NoResultFound +from lifemonitor.utils import ClassManager, download_url, is_service_alive # set module level logger logger = logging.getLogger(__name__) diff --git a/lifemonitor/api/services.py b/lifemonitor/api/services.py index 69bd446aa..2f7f71910 100644 --- a/lifemonitor/api/services.py +++ b/lifemonitor/api/services.py @@ -34,7 +34,7 @@ from lifemonitor.auth.oauth2.client.models import OAuthIdentity from lifemonitor.auth.oauth2.server import server from lifemonitor.tasks.models import Job -from lifemonitor.utils import OpenApiSpecs, ROCrateLinkContext, to_snake_case +from lifemonitor.utils import OpenApiSpecs, ROCrateLinkContext, is_service_alive, to_snake_case from lifemonitor.ws import io logger = logging.getLogger() diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index 35fa4d52f..620a8f61f 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -49,6 +49,8 @@ import yaml from dateutil import parser +from lifemonitor.cache import cached + from . import exceptions as lm_exceptions logger = logging.getLogger() From 9aae495503617d5bc082617d6e8b7b5dbdbe128a Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 3 May 2023 15:41:30 +0200 Subject: [PATCH 16/39] test: count an additional cache key --- tests/unit/cache/test_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/cache/test_cache.py b/tests/unit/cache/test_cache.py index 44d42ecb7..3fe580bf8 100644 --- a/tests/unit/cache/test_cache.py +++ b/tests/unit/cache/test_cache.py @@ -309,7 +309,7 @@ def cache_last_build_update(app, w, user1, check_cache_size=True, index=0, # check the cache after the transaction is completed if check_cache_size: cache_size = cache.size() - assert len(transaction_keys) == cache_size, "Unpexpected cache size: it should be equal to the transaction size" + assert len(transaction_keys) + 1 == cache_size, "Unpexpected cache size: it should be equal to the transaction size" sleep(2) assert cache.size() > 0, "Cache should not be empty" logger.debug(cache.keys()) From b09b96b34d17fe45a720adbbd4a9bc68bc6ce6f5 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 3 May 2023 16:26:12 +0200 Subject: [PATCH 17/39] perf: set to 2s the default timeout to check srv availability --- lifemonitor/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index 620a8f61f..1f89bdea9 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -229,7 +229,7 @@ def validate_url(url: str) -> bool: @cached(client_scope=False) -def is_service_alive(url: str, timeout: int = 5) -> bool: +def is_service_alive(url: str, timeout: int = 2) -> bool: try: response = requests.get(url, timeout=timeout) if response.status_code == 200: From 4d429af5d9f746e9ef6e01eadf6cb8de49b9af0f Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 3 May 2023 16:45:21 +0200 Subject: [PATCH 18/39] fix: missing exception --- lifemonitor/api/models/registries/registry.py | 2 +- lifemonitor/exceptions.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lifemonitor/api/models/registries/registry.py b/lifemonitor/api/models/registries/registry.py index 021f6a87c..09992edd3 100644 --- a/lifemonitor/api/models/registries/registry.py +++ b/lifemonitor/api/models/registries/registry.py @@ -442,7 +442,7 @@ def delete_workflow(self, submitter: auth_models.User, external_id: str) -> bool @property def client(self) -> WorkflowRegistryClient: if not is_service_alive(self.uri): - raise lm_exceptions.ServiceUnavailableException(f"Service {self.url} is not available", service=self) + raise lm_exceptions.UnavailableServiceException(f"Service {self.uri} is not available", service=self) if self._client is None: rtype = self.__class__.__name__.replace("WorkflowRegistry", "").lower() return WorkflowRegistryClient.get_client_class(rtype)(self) diff --git a/lifemonitor/exceptions.py b/lifemonitor/exceptions.py index f793281ef..59939e734 100644 --- a/lifemonitor/exceptions.py +++ b/lifemonitor/exceptions.py @@ -130,6 +130,14 @@ def __str__(self): return self.detail +class UnavailableServiceException(LifeMonitorException): + + def __init__(self, detail=None, + type="about:blank", status=503, service=None, **kwargs): + super().__init__(title="External service not available", + detail=detail, status=status, service=getattr(service, "uri", None) or str(service), **kwargs) + + class WorkflowVersionConflictException(LifeMonitorException): def __init__(self, workflow_uuid, workflow_version, detail=None, **kwargs) -> None: From 707faafbd929cee0b311a2dafa5c3936581aed83 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 4 May 2023 13:04:13 +0200 Subject: [PATCH 19/39] feat(ctrl): enable `next` parameter on `/logout` route --- lifemonitor/auth/controllers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lifemonitor/auth/controllers.py b/lifemonitor/auth/controllers.py index 3ac44c569..65723f41e 100644 --- a/lifemonitor/auth/controllers.py +++ b/lifemonitor/auth/controllers.py @@ -276,7 +276,9 @@ def logout(): session.pop('_flashes', None) flash("You have logged out", category="success") NextRouteRegistry.clear() - return redirect('/') + next_route = request.args.get('next', '/') + logger.debug("Next route after logout: %r", next_route) + return redirect(next_route) @blueprint.route("/delete_account", methods=("POST",)) From ec72dda88d232ce307c26027a08e22b6d0ac346a Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 4 May 2023 15:13:26 +0200 Subject: [PATCH 20/39] fix(wss/o): add custom encoder --- lifemonitor/ws/io.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lifemonitor/ws/io.py b/lifemonitor/ws/io.py index cd133fb5b..3785148ea 100644 --- a/lifemonitor/ws/io.py +++ b/lifemonitor/ws/io.py @@ -4,6 +4,7 @@ import logging import time from typing import List +from uuid import UUID from flask import Flask @@ -27,6 +28,14 @@ def __format_timestamp__(timestamp: datetime) -> str: return timestamp.strftime('%a %b %d %Y %H:%M:%S ') + 'GMT' + tz_offset[:3] + ':' + tz_offset[3:] +class _CustomEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, UUID): + return str(obj) + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) + + def publish_message(message, channel: str = __CHANNEL__, target_ids: List[str] = None, target_rooms: List[str] = None, delay: int = 0): now = datetime.datetime.now(datetime.timezone.utc) @@ -39,7 +48,7 @@ def publish_message(message, channel: str = __CHANNEL__, "target_rooms": target_rooms, # "timestamp": datetime_as_timestamp_with_msecs(now), "payload": message - })) + }, cls=_CustomEncoder)) def start_reading(app: Flask, channel: str = __CHANNEL__, max_age: int = __MAX_AGE__): From c87a3b340a4eea5f0535cc7fd214ec61966dfdd4 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 4 May 2023 15:19:55 +0200 Subject: [PATCH 21/39] fix: do not enable integrations on wss server --- ws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ws.py b/ws.py index 95e90d018..944854dc3 100644 --- a/ws.py +++ b/ws.py @@ -29,6 +29,6 @@ # create an app instance -application = create_app(init_app=True, load_jobs=False) +application = create_app(init_app=True, load_jobs=False, init_integrations=False) socketIO = initialise_ws(application) start_brodcaster(application, max_age=5) # use default ws_channel From c6bc66169adaba14320958accdb32a56eea6ec56 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 4 May 2023 15:23:05 +0200 Subject: [PATCH 22/39] fix(model): auto update modification datetime --- lifemonitor/tasks/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lifemonitor/tasks/models.py b/lifemonitor/tasks/models.py index d8eccd2ac..c8b361178 100644 --- a/lifemonitor/tasks/models.py +++ b/lifemonitor/tasks/models.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from datetime import datetime +from datetime import datetime, timezone from flask import Flask from git import List @@ -73,8 +73,8 @@ def update_status(self, status, save: bool = False): def save(self): if not self._data.get('created', None): - self._data['created'] = datetime.utcnow().timestamp() # .replace(tzinfo=timezone.utc).timestamp() - self._data['modified'] = datetime.utcnow().timestamp() + self._data['created'] = datetime.now(tz=timezone.utc).timestamp() + self._data['modified'] = datetime.now(tz=timezone.utc).timestamp() set_job_data(self._job_id, self._data) notify_update(self._job_id, target_ids=self.listening_ids, target_rooms=self.listening_rooms) From 3edbb2addbc98d7c4dfdd51e6bc9ef7bec817d67 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 4 May 2023 15:24:02 +0200 Subject: [PATCH 23/39] fix: missing wss events --- lifemonitor/ws/events.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/lifemonitor/ws/events.py b/lifemonitor/ws/events.py index 6bc26ef99..b4327b5ff 100644 --- a/lifemonitor/ws/events.py +++ b/lifemonitor/ws/events.py @@ -23,7 +23,7 @@ from typing import Dict from flask import request -from flask_socketio import disconnect, emit, join_room +from flask_socketio import disconnect, emit, join_room, leave_room from lifemonitor.cache import cache @@ -84,7 +84,27 @@ def handle_message(message): logger.debug(f"Joining SID {request.sid} to room {message['data']['user']}") join_room(str(message['data']['user'])) logger.warning(f"SID {request.sid} joined to room {message['data']['user']}") - elif message['type'] == 'SYNC': + emit('message', { + 'payload': { + 'type': 'joined', + 'data': { + 'user': str(message['data']['user']) + }, + } + }) + elif message['type'] == 'leave': + logger.debug(f"Leaving SID {request.sid} room {message['data']['user']}") + leave_room(str(message['data']['user'])) + logger.warning(f"SID {request.sid} left room {message['data']['user']}") + emit('message', { + 'payload': { + 'type': 'joined', + 'data': { + 'user': str(message['data']['user']) + }, + } + }) + elif message['type'] == 'sync': emit("message", build_sync_message()) From f4cd890c90c256b4979b2870f61ca701fff28475 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 4 May 2023 15:25:06 +0200 Subject: [PATCH 24/39] fix(util): build url from workflow or workflow version --- tests/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index 4b58a70d3..8ee66fda2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -23,6 +23,7 @@ import lifemonitor.db as lm_db from lifemonitor.api import models +from lifemonitor.api.models.workflows import WorkflowVersion from lifemonitor.api.services import LifeMonitor logger = logging.getLogger(__name__) @@ -60,7 +61,10 @@ def _get_attr(obj, name, default=None): def build_workflow_path(workflow=None, version_as_subpath=False, subpath=None, include_version=True): if workflow: - w = f"{_WORKFLOWS_ENDPOINT}/{_get_attr(workflow, 'uuid')}" + if isinstance(workflow, WorkflowVersion): + w = f"{_WORKFLOWS_ENDPOINT}/{_get_attr(workflow.workflow, 'uuid')}" + else: + w = f"{_WORKFLOWS_ENDPOINT}/{_get_attr(workflow, 'uuid')}" if include_version and version_as_subpath: w = f"{w}/versions/{_get_attr(workflow, 'version')}" if subpath: From 4db6384cb0715e87f64e2fab0eb2b7e96b6aa9ae Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 4 May 2023 15:26:40 +0200 Subject: [PATCH 25/39] fix(cfg): make optional initialisation of integrations --- lifemonitor/app.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lifemonitor/app.py b/lifemonitor/app.py index d37a60628..edb8ceb13 100644 --- a/lifemonitor/app.py +++ b/lifemonitor/app.py @@ -45,7 +45,8 @@ logger = logging.getLogger(__name__) -def create_app(env=None, settings=None, init_app=True, worker=False, load_jobs=True, **kwargs): +def create_app(env=None, settings=None, init_app=True, init_integrations=True, + worker=False, load_jobs=True, **kwargs): """ App factory method :param env: @@ -81,7 +82,7 @@ def create_app(env=None, settings=None, init_app=True, worker=False, load_jobs=T # initialize the application if init_app: with app.app_context() as ctx: - initialize_app(app, ctx, load_jobs=load_jobs) + initialize_app(app, ctx, load_jobs=load_jobs, load_integrations=init_integrations) @app.route("/") def index(): @@ -127,7 +128,7 @@ def log_response(response): return app -def initialize_app(app: Flask, app_context, prom_registry=None, load_jobs: bool = True): +def initialize_app(app: Flask, app_context, prom_registry=None, load_jobs: bool = True, load_integrations: bool = True): # init tmp folder os.makedirs(app.config.get('BASE_TEMP_FOLDER'), exist_ok=True) # enable CORS @@ -151,7 +152,8 @@ def initialize_app(app: Flask, app_context, prom_registry=None, load_jobs: bool # init mail system init_mail(app) # initialize integrations - init_integrations(app) + if load_integrations: + init_integrations(app) # initialize metrics engine init_metrics(app, prom_registry) # register commands From 05b107b79f58dbe14f605a8d8e814c5fe42dc196 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 4 May 2023 19:32:17 +0200 Subject: [PATCH 26/39] fix: enable partial user profile if some provider is unavailable --- lifemonitor/auth/oauth2/client/models.py | 8 +++++++- lifemonitor/utils.py | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/lifemonitor/auth/oauth2/client/models.py b/lifemonitor/auth/oauth2/client/models.py index 559693a75..90118bbfc 100644 --- a/lifemonitor/auth/oauth2/client/models.py +++ b/lifemonitor/auth/oauth2/client/models.py @@ -40,7 +40,7 @@ LifeMonitorException, NotAuthorizedException) from lifemonitor.models import JSON, ModelMixin -from lifemonitor.utils import to_snake_case +from lifemonitor.utils import assert_service_is_alive, to_snake_case from sqlalchemy import DateTime, inspect from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm.attributes import flag_modified @@ -206,6 +206,8 @@ def get_token(self, scope: Optional[str] = None): return OAuth2Token(token, provider=self.provider) def fetch_token(self, scope: Optional[str] = None): + # ensure that the service is alive + assert_service_is_alive(self.provider.api_base_url) # enable dynamic refresh only if the identity # has been already stored in the database if inspect(self).persistent: @@ -226,6 +228,8 @@ def refresh_token(self, token: OAuth2Token = None): logger.debug("Refresh token requested...") token = token or self.token if token and token.to_be_refreshed(): + # ensure that the service is alive + assert_service_is_alive(self.provider.api_base_url) with self.cache.lock(str(self), timeout=Timeout.NONE): # fetch current token from database self.refresh(attribute_names=['_tokens']) @@ -424,6 +428,7 @@ def token_type(self): return "Bearer" def get_user_info(self, provider_user_id, token, normalized=True): + assert_service_is_alive(self.api_base_url) access_token = token['access_token'] if isinstance(token, dict) else token response = requests.get(urljoin(self.api_base_url, self.userinfo_endpoint), headers={'Authorization': f'Bearer {access_token}'}) @@ -505,6 +510,7 @@ def find_identity_by_provider_user_id(self, provider_user_id): def refresh_token(self, token: OAuth2Token) -> OAuth2Token: logger.debug(f"Trying to refresh the token: {token}...") + assert_service_is_alive(self.api_base_url) # reference to the token associated with the identity instance oauth2session = OAuth2Session( self.client_id, self.client_secret, token=token) diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index 1f89bdea9..f4d80b308 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -229,7 +229,7 @@ def validate_url(url: str) -> bool: @cached(client_scope=False) -def is_service_alive(url: str, timeout: int = 2) -> bool: +def is_service_alive(url: str, timeout: int = 1) -> bool: try: response = requests.get(url, timeout=timeout) if response.status_code == 200: @@ -243,6 +243,11 @@ def is_service_alive(url: str, timeout: int = 2) -> bool: return False +def assert_service_is_alive(url: str, timeout: int = 1): + if not is_service_alive(url, timeout=timeout): + raise lm_exceptions.ServiceNotAvailableException(detail=f"Service not available: {url}", service=url) + + def get_last_update(path: str): return time.ctime(max(os.stat(root).st_mtime for root, _, _ in os.walk(path))) From c17d5a57e0c621251f94c68f17a9d28b4b856d25 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 4 May 2023 19:32:48 +0200 Subject: [PATCH 27/39] fix: remove redundant import --- lifemonitor/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index f4d80b308..ffcc07333 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -270,7 +270,6 @@ def match_ref(ref: str, refs: List[str]) -> Optional[Tuple[str, str]]: def notify_updates(workflows: List, type: str = 'sync', delay: int = 0): from lifemonitor.ws import io - from datetime import timezone io.publish_message({ "type": type, "data": [{ From 6f8b0d0048c38774d7aad3388e41e4f8a3d514d8 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 4 May 2023 19:47:20 +0200 Subject: [PATCH 28/39] fix: clear flash msg on login --- lifemonitor/auth/controllers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lifemonitor/auth/controllers.py b/lifemonitor/auth/controllers.py index 65723f41e..9510a7029 100644 --- a/lifemonitor/auth/controllers.py +++ b/lifemonitor/auth/controllers.py @@ -259,6 +259,7 @@ def login(): form = LoginForm() flask.session["confirm_user_details"] = True flask.session["sign_in"] = True + flask.session.pop('_flashes', None) if form.validate_on_submit(): user = form.get_user() if user: From aede06bb34efc091b56355fd5dcc680debc7fc02 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 4 May 2023 20:42:42 +0200 Subject: [PATCH 29/39] fix(util): expand set of status codes to detect service availability --- lifemonitor/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index ffcc07333..f5bac8399 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -232,7 +232,7 @@ def validate_url(url: str) -> bool: def is_service_alive(url: str, timeout: int = 1) -> bool: try: response = requests.get(url, timeout=timeout) - if response.status_code == 200: + if response.status_code < 500: return True else: return False From 4050311b39607f325111b5e055f313f395961627 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 4 May 2023 21:02:28 +0200 Subject: [PATCH 30/39] feat(auth): auto disable sign{in,up} through unavailable services --- lifemonitor/auth/controllers.py | 8 +++++--- lifemonitor/auth/oauth2/client/controllers.py | 13 ++++++++++--- lifemonitor/auth/templates/auth/login.j2 | 19 ++++++++++--------- lifemonitor/auth/templates/auth/register.j2 | 15 ++++++++++----- lifemonitor/templates/macros.j2 | 12 ++++++++---- 5 files changed, 43 insertions(+), 24 deletions(-) diff --git a/lifemonitor/auth/controllers.py b/lifemonitor/auth/controllers.py index 9510a7029..4c075a812 100644 --- a/lifemonitor/auth/controllers.py +++ b/lifemonitor/auth/controllers.py @@ -29,7 +29,7 @@ split_by_crlf) from .. import exceptions -from ..utils import OpenApiSpecs, boolean_value +from ..utils import OpenApiSpecs, boolean_value, is_service_alive from . import serializers from .forms import (EmailForm, LoginForm, NotificationsForm, Oauth2ClientForm, RegisterForm, SetPasswordForm) @@ -210,7 +210,8 @@ def register(): clear_cache() return redirect(url_for("auth.index")) return render_template("auth/register.j2", form=form, - action=url_for('auth.register'), providers=get_providers()) + action=url_for('auth.register'), + providers=get_providers(), is_service_available=is_service_alive) @blueprint.route("/identity_not_found", methods=("GET", "POST")) @@ -267,7 +268,8 @@ def login(): session.pop('_flashes', None) flash("You have logged in", category="success") return redirect(NextRouteRegistry.pop(url_for("auth.profile"))) - return render_template("auth/login.j2", form=form, providers=get_providers()) + return render_template("auth/login.j2", form=form, + providers=get_providers(), is_service_available=is_service_alive) @blueprint.route("/logout") diff --git a/lifemonitor/auth/oauth2/client/controllers.py b/lifemonitor/auth/oauth2/client/controllers.py index 1e6682808..9ba70b8fb 100644 --- a/lifemonitor/auth/oauth2/client/controllers.py +++ b/lifemonitor/auth/oauth2/client/controllers.py @@ -24,17 +24,21 @@ from authlib.integrations.base_client.errors import OAuthError from authlib.integrations.flask_client import FlaskRemoteApp -from flask import (Blueprint, abort, current_app, flash, redirect, request, session, url_for) +from flask import (Blueprint, abort, current_app, flash, redirect, request, + session, url_for) from flask_login import current_user, login_user + from lifemonitor import exceptions, utils from lifemonitor.auth.models import User from lifemonitor.auth.oauth2.client.models import ( OAuth2IdentityProvider, OAuthIdentityNotFoundException) from lifemonitor.db import db -from lifemonitor.utils import NextRouteRegistry, next_route_aware +from lifemonitor.utils import (NextRouteRegistry, is_service_alive, + next_route_aware) from .models import OAuthIdentity, OAuthUserProfile -from .services import config_oauth2_registry, oauth2_registry, save_current_user_identity +from .services import (config_oauth2_registry, oauth2_registry, + save_current_user_identity) from .utils import RequestHelper # Config a module level logger @@ -94,6 +98,9 @@ def login(name, scope: str = None): remote = oauth2_registry.create_client(name) if remote is None: abort(404) + logger.debug("config: %r", remote.OAUTH_APP_CONFIG) + if not is_service_alive(remote.OAUTH_APP_CONFIG['api_base_url']): + abort(503) action = request.args.get('action', False) if action and action == 'sign-in': session['sign_in'] = True diff --git a/lifemonitor/auth/templates/auth/login.j2 b/lifemonitor/auth/templates/auth/login.j2 index 4bf2bd459..6bbf61da4 100644 --- a/lifemonitor/auth/templates/auth/login.j2 +++ b/lifemonitor/auth/templates/auth/login.j2 @@ -40,24 +40,25 @@

diff --git a/lifemonitor/auth/templates/auth/register.j2 b/lifemonitor/auth/templates/auth/register.j2 index 2656f794d..0d495f2a2 100644 --- a/lifemonitor/auth/templates/auth/register.j2 +++ b/lifemonitor/auth/templates/auth/register.j2 @@ -66,16 +66,21 @@

- OR -

{% for p in providers %} {% if p.client_name != 'lsaai' %} - {{ macros.render_provider_signup_button(p) }} + {{ macros.render_provider_signup_button(p,not is_service_available(p.oauth_config['api_base_url'])) }} {% endif %} {% endfor %}
{% endif %} diff --git a/lifemonitor/templates/macros.j2 b/lifemonitor/templates/macros.j2 index 3a7fc3230..d5b5367e2 100644 --- a/lifemonitor/templates/macros.j2 +++ b/lifemonitor/templates/macros.j2 @@ -217,14 +217,18 @@ btn-info {%- endmacro %} -{% macro render_provider_signin_button(provider) -%} - +{% macro render_provider_signin_button(provider, disabled) -%} + {{render_provider_fa_icon(provider)}} Sign in with {{provider.name}} {%- endmacro %} -{% macro render_provider_signup_button(provider) -%} - +{% macro render_provider_signup_button(provider, disabled) -%} + {{render_provider_fa_icon(provider)}} Sign up with {{provider.name}} {%- endmacro %} \ No newline at end of file From 54f4e0cdf4d6ac5716008d0253ad59dcf3c8b7d8 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 4 May 2023 21:23:25 +0200 Subject: [PATCH 31/39] fix(ui): make links to API explorer relative to a configurable base url --- lifemonitor/auth/controllers.py | 1 + lifemonitor/auth/templates/auth/apikeys_tab.j2 | 2 +- lifemonitor/auth/templates/auth/oauth2_clients_tab.j2 | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lifemonitor/auth/controllers.py b/lifemonitor/auth/controllers.py index 4c075a812..8ffa3b853 100644 --- a/lifemonitor/auth/controllers.py +++ b/lifemonitor/auth/controllers.py @@ -190,6 +190,7 @@ def profile(form=None, passwordForm=None, currentView=None, registrySettingsForm=registrySettingsForm or RegistrySettingsForm.from_model(current_user), providers=get_providers(), currentView=currentView, oauth2_generic_client_scopes=OpenApiSpecs.get_instance().authorization_code_scopes, + api_base_url=current_app.config['EXTERNAL_SERVER_URL'], back_param=back_param) diff --git a/lifemonitor/auth/templates/auth/apikeys_tab.j2 b/lifemonitor/auth/templates/auth/apikeys_tab.j2 index e0ab14daf..6366bb225 100644 --- a/lifemonitor/auth/templates/auth/apikeys_tab.j2 +++ b/lifemonitor/auth/templates/auth/apikeys_tab.j2 @@ -7,7 +7,7 @@
- +
diff --git a/lifemonitor/auth/templates/auth/oauth2_clients_tab.j2 b/lifemonitor/auth/templates/auth/oauth2_clients_tab.j2 index c09974d0b..507964eb5 100644 --- a/lifemonitor/auth/templates/auth/oauth2_clients_tab.j2 +++ b/lifemonitor/auth/templates/auth/oauth2_clients_tab.j2 @@ -6,7 +6,7 @@
- +
From 6a7dab29f234473ba0b12776dea3d928458d9fe0 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 4 May 2023 22:12:21 +0200 Subject: [PATCH 32/39] fix(test): increase cache size --- tests/unit/cache/test_cache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/cache/test_cache.py b/tests/unit/cache/test_cache.py index 3fe580bf8..64279b147 100644 --- a/tests/unit/cache/test_cache.py +++ b/tests/unit/cache/test_cache.py @@ -309,7 +309,8 @@ def cache_last_build_update(app, w, user1, check_cache_size=True, index=0, # check the cache after the transaction is completed if check_cache_size: cache_size = cache.size() - assert len(transaction_keys) + 1 == cache_size, "Unpexpected cache size: it should be equal to the transaction size" + # +2 because of the checks for the service availability + assert len(transaction_keys) + 2 == cache_size, "Unpexpected cache size: it should be equal to the transaction size + 2" sleep(2) assert cache.size() > 0, "Cache should not be empty" logger.debug(cache.keys()) From 5d426ed68b11448adbe6e69663a0adbfbc72afd0 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 4 May 2023 22:45:04 +0200 Subject: [PATCH 33/39] build: update version number --- lifemonitor/static/src/package.json | 2 +- specs/api.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lifemonitor/static/src/package.json b/lifemonitor/static/src/package.json index 72e95c17c..0edf2ae84 100644 --- a/lifemonitor/static/src/package.json +++ b/lifemonitor/static/src/package.json @@ -1,7 +1,7 @@ { "name": "lifemonitor", "description": "Workflow Testing Service", - "version": "0.9.0", + "version": "0.10.0", "license": "MIT", "author": "CRS4", "main": "../dist/js/lifemonitor.min.js", diff --git a/specs/api.yaml b/specs/api.yaml index 2e39c9174..f70913958 100644 --- a/specs/api.yaml +++ b/specs/api.yaml @@ -3,7 +3,7 @@ openapi: "3.0.0" info: - version: "0.9.0" + version: "0.10.0" title: "Life Monitor API" description: | *Workflow sustainability service* @@ -18,7 +18,7 @@ info: servers: - url: / description: > - Version 0.9.0 of API. + Version 0.10.0 of API. tags: - name: GitHub Integration From d750d16b0ef79c785f4e70dde557d9dbd6780f8c Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 4 May 2023 23:59:06 +0200 Subject: [PATCH 34/39] feat: make srv availability timeout configurable --- lifemonitor/config.py | 2 ++ lifemonitor/utils.py | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lifemonitor/config.py b/lifemonitor/config.py index 721fd20f7..b596fc0e8 100644 --- a/lifemonitor/config.py +++ b/lifemonitor/config.py @@ -126,6 +126,8 @@ class BaseConfig: # Enable/disable integrations ENABLE_GITHUB_INTEGRATION = False ENABLE_REGISTRY_INTEGRATION = False + # Service Availability Timeout + SERVICE_AVAILABILITY_TIMEOUT = 1 class DevelopmentConfig(BaseConfig): diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index f5bac8399..3c5cfcf8f 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -229,8 +229,12 @@ def validate_url(url: str) -> bool: @cached(client_scope=False) -def is_service_alive(url: str, timeout: int = 1) -> bool: +def is_service_alive(url: str, timeout: Optional[int] = None) -> bool: try: + try: + timeout = timeout or flask.current_app.config.get("SERVICE_ALIVE_TIMEOUT", 1) + except Exception: + timeout = 1 response = requests.get(url, timeout=timeout) if response.status_code < 500: return True @@ -243,7 +247,7 @@ def is_service_alive(url: str, timeout: int = 1) -> bool: return False -def assert_service_is_alive(url: str, timeout: int = 1): +def assert_service_is_alive(url: str, timeout: Optional[int] = None): if not is_service_alive(url, timeout=timeout): raise lm_exceptions.ServiceNotAvailableException(detail=f"Service not available: {url}", service=url) From 5f3e31025c68a253622fb12d22c3752706ff29c0 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 5 May 2023 00:00:37 +0200 Subject: [PATCH 35/39] fix: increase srv availability timeout on testing env --- lifemonitor/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lifemonitor/config.py b/lifemonitor/config.py index b596fc0e8..8bde89049 100644 --- a/lifemonitor/config.py +++ b/lifemonitor/config.py @@ -158,6 +158,8 @@ class TestingConfig(BaseConfig): # CACHE_TYPE = "flask_caching.backends.nullcache.NullCache" CACHE_TYPE = "flask_caching.backends.rediscache.RedisCache" DATA_WORKFLOWS = f"{BaseConfig.BASE_TEMP_FOLDER}/lm_tests_data" + # Service Availability Timeout + SERVICE_AVAILABILITY_TIMEOUT = 120 class TestingSupportConfig(TestingConfig): @@ -166,6 +168,8 @@ class TestingSupportConfig(TestingConfig): TESTING = False LOG_LEVEL = "DEBUG" DATA_WORKFLOWS = f"{BaseConfig.BASE_TEMP_FOLDER}/lm_tests_data" + # Service Availability Timeout + SERVICE_AVAILABILITY_TIMEOUT = 120 _EXPORT_CONFIGS: List[Type[BaseConfig]] = [ From 6025f91d139b8ea7a7aae02fab70368e41140a65 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 5 May 2023 10:59:36 +0200 Subject: [PATCH 36/39] fix: use utility function to get external server URL --- lifemonitor/auth/controllers.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lifemonitor/auth/controllers.py b/lifemonitor/auth/controllers.py index 8ffa3b853..dfa49a652 100644 --- a/lifemonitor/auth/controllers.py +++ b/lifemonitor/auth/controllers.py @@ -22,14 +22,17 @@ import connexion import flask -from flask import current_app, flash, redirect, render_template, request, session, url_for +from flask import (current_app, flash, redirect, render_template, request, + session, url_for) from flask_login import login_required, login_user, logout_user + from lifemonitor.cache import Timeout, cached, clear_cache from lifemonitor.utils import (NextRouteRegistry, next_route_aware, split_by_crlf) from .. import exceptions -from ..utils import OpenApiSpecs, boolean_value, is_service_alive +from ..utils import (OpenApiSpecs, boolean_value, get_external_server_url, + is_service_alive) from . import serializers from .forms import (EmailForm, LoginForm, NotificationsForm, Oauth2ClientForm, RegisterForm, SetPasswordForm) @@ -190,7 +193,7 @@ def profile(form=None, passwordForm=None, currentView=None, registrySettingsForm=registrySettingsForm or RegistrySettingsForm.from_model(current_user), providers=get_providers(), currentView=currentView, oauth2_generic_client_scopes=OpenApiSpecs.get_instance().authorization_code_scopes, - api_base_url=current_app.config['EXTERNAL_SERVER_URL'], + api_base_url=get_external_server_url(), back_param=back_param) From 0d1a79cbe686d1a3377a96d1c96526deeb67b866 Mon Sep 17 00:00:00 2001 From: simleo Date: Fri, 5 May 2023 11:04:17 +0200 Subject: [PATCH 37/39] generate_crate: fall back to defaults --- lifemonitor/api/models/rocrate/generators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lifemonitor/api/models/rocrate/generators.py b/lifemonitor/api/models/rocrate/generators.py index f1c0cdabc..a257d8721 100644 --- a/lifemonitor/api/models/rocrate/generators.py +++ b/lifemonitor/api/models/rocrate/generators.py @@ -37,8 +37,8 @@ def generate_crate(workflow_type: str, workflow_version: str, local_repo_path: str, repo_url: Optional[str], - license: Optional[str] = "MIT", - ci_workflow: Optional[str] = "main.yml", + license: Optional[str] = None, + ci_workflow: Optional[str] = None, lang_version: Optional[str] = None, **kwargs): make_crate = get_crate_generator(workflow_type) From 9cda157afecb0339d8dc624764b04d58dff76aa7 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 5 May 2023 14:32:28 +0200 Subject: [PATCH 38/39] fix(test): use a different fake site --- tests/unit/test_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 68c20eca0..217557484 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -29,9 +29,9 @@ def test_download_url_404(): with tempfile.TemporaryDirectory() as d: - with pytest.raises(lm_exceptions.DownloadException) as exec_info: - _ = utils.download_url('http://httpbin.org/status/404', os.path.join(d, 'get_404')) - assert exec_info.value.status == 404 + with pytest.raises(lm_exceptions.DownloadException) as excinfo: + _ = utils.download_url('https://github.com/crs4/life_monitor/fake_path', os.path.join(d, 'fake_path')) + assert excinfo.value.status == 404 def test_datetime_to_isoformat(): From d58c3a20991f72f6f68435a766f4f4b4e772d015 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 5 May 2023 15:31:08 +0200 Subject: [PATCH 39/39] fix: typo --- lifemonitor/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index 3c5cfcf8f..7ec310ba7 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -249,7 +249,7 @@ def is_service_alive(url: str, timeout: Optional[int] = None) -> bool: def assert_service_is_alive(url: str, timeout: Optional[int] = None): if not is_service_alive(url, timeout=timeout): - raise lm_exceptions.ServiceNotAvailableException(detail=f"Service not available: {url}", service=url) + raise lm_exceptions.UnavailableServiceException(detail=f"Service not available: {url}", service=url) def get_last_update(path: str):