diff --git a/cli/client/issues.py b/cli/client/issues.py index 4097a51ac..71a24a141 100644 --- a/cli/client/issues.py +++ b/cli/client/issues.py @@ -147,7 +147,8 @@ def check(config, repository, output_path): console.print(f"[{message.type.value}]{message.type.name}:[/{message.type.value}] {message.text}") console.print("\n\n") except Exception as e: - logger.exception(e) + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) error_console.print(str(e)) diff --git a/cli/client/utils.py b/cli/client/utils.py index affd0152e..a47d8a713 100644 --- a/cli/client/utils.py +++ b/cli/client/utils.py @@ -25,7 +25,7 @@ from urllib.parse import urlparse from lifemonitor.api.models.repositories.github import GithubWorkflowRepository -from lifemonitor.api.models.repositories.local import LocalWorkflowRepository +from lifemonitor.api.models.repositories.local import LocalWorkflowRepository, LocalGitRepository from rich.prompt import Prompt # Set module logger @@ -42,15 +42,20 @@ def is_url(value): def get_repository(repository: str, local_path: str): assert repository, repository - if is_url(repository): - remote_repo_url = repository - if remote_repo_url.endswith('.git'): - return GithubWorkflowRepository.from_url(remote_repo_url, auto_cleanup=False, local_path=local_path) - else: - local_copy_path = os.path.join(local_path, os.path.basename(repository)) - shutil.copytree(repository, local_copy_path) - return LocalWorkflowRepository(local_copy_path) - raise ValueError("Repository type not supported") + try: + if is_url(repository): + remote_repo_url = repository + if remote_repo_url.endswith('.git'): + return GithubWorkflowRepository.from_url(remote_repo_url, auto_cleanup=False, local_path=local_path) + else: + local_copy_path = os.path.join(local_path, os.path.basename(repository)) + shutil.copytree(repository, local_copy_path) + if LocalGitRepository.is_git_repo(local_copy_path): + return LocalGitRepository(local_copy_path) + return LocalWorkflowRepository(local_copy_path) + raise ValueError("Repository type not supported") + except Exception as e: + raise ValueError("Error while loading the repository: %s" % e) def init_output_path(output_path): diff --git a/lifemonitor/api/models/registries/registry.py b/lifemonitor/api/models/registries/registry.py index 85381d3de..09992edd3 100644 --- a/lifemonitor/api/models/registries/registry.py +++ b/lifemonitor/api/models/registries/registry.py @@ -24,10 +24,13 @@ from abc import ABC, abstractmethod from typing import List, Tuple, Union -import lifemonitor.api.models as models -import lifemonitor.exceptions as lm_exceptions import requests from authlib.integrations.base_client import RemoteApp +from sqlalchemy.orm.collections import attribute_mapped_collection +from sqlalchemy.orm.exc import NoResultFound + +import lifemonitor.api.models as models +import lifemonitor.exceptions as lm_exceptions from lifemonitor import utils as lm_utils from lifemonitor.api.models import db from lifemonitor.api.models.repositories.base import WorkflowRepository @@ -35,9 +38,7 @@ from lifemonitor.auth.models import Resource from lifemonitor.auth.oauth2.client.models import OAuthIdentity from lifemonitor.auth.oauth2.client.services import oauth2_registry -from lifemonitor.utils import ClassManager, download_url -from sqlalchemy.orm.collections import attribute_mapped_collection -from sqlalchemy.orm.exc import NoResultFound +from lifemonitor.utils import ClassManager, download_url, is_service_alive # set module level logger logger = logging.getLogger(__name__) @@ -440,6 +441,8 @@ def delete_workflow(self, submitter: auth_models.User, external_id: str) -> bool @property def client(self) -> WorkflowRegistryClient: + if not is_service_alive(self.uri): + raise lm_exceptions.UnavailableServiceException(f"Service {self.uri} is not available", service=self) if self._client is None: rtype = self.__class__.__name__.replace("WorkflowRegistry", "").lower() return WorkflowRegistryClient.get_client_class(rtype)(self) diff --git a/lifemonitor/api/models/repositories/base.py b/lifemonitor/api/models/repositories/base.py index d65e8ba97..80114ba34 100644 --- a/lifemonitor/api/models/repositories/base.py +++ b/lifemonitor/api/models/repositories/base.py @@ -297,11 +297,17 @@ def config(self) -> WorkflowRepositoryConfig: pass return self._config - def generate_config(self, ignore_existing=False) -> WorkflowFile: + def generate_config(self, ignore_existing=False, + workflow_title: Optional[str] = None, + public: bool = False, main_branch: Optional[str] = None) -> WorkflowFile: current_config = self.config if current_config and not ignore_existing: raise IllegalStateException("Config exists") - self._config = WorkflowRepositoryConfig.new(self.local_path, workflow_title=self.metadata.main_entity_name if self.metadata else None) + self._config = WorkflowRepositoryConfig.new(self.local_path, + workflow_title=workflow_title if workflow_title is not None + else self.metadata.main_entity_name if self.metadata else None, + main_branch=main_branch if main_branch else getattr(self, "main_branch", "main"), + public=public) return self._config def write_zip(self, target_path: str): diff --git a/lifemonitor/api/models/repositories/local.py b/lifemonitor/api/models/repositories/local.py index f9345cab9..890a2c93c 100644 --- a/lifemonitor/api/models/repositories/local.py +++ b/lifemonitor/api/models/repositories/local.py @@ -174,3 +174,20 @@ def __init__(self, base64_rocrate: str) -> None: except Exception as e: logger.debug(e) raise DecodeROCrateException(detail=str(e)) + + +class LocalGitRepository(LocalWorkflowRepository): + + def __init__(self, local_path: str | None = None, exclude: List[str] | None = None) -> None: + super().__init__(local_path, exclude) + assert self.is_git_repo(self.local_path), f"Local path {local_path} is not a git repository" + + @property + def main_branch(self) -> str: + from git import Repo + repo = Repo(self.local_path) + return repo.active_branch.name + + @staticmethod + def is_git_repo(local_path: str) -> bool: + return os.path.isdir(os.path.join(local_path, '.git')) diff --git a/lifemonitor/api/models/rocrate/__init__.py b/lifemonitor/api/models/rocrate/__init__.py index cb4863895..0b1aff921 100644 --- a/lifemonitor/api/models/rocrate/__init__.py +++ b/lifemonitor/api/models/rocrate/__init__.py @@ -28,9 +28,13 @@ from pathlib import Path from typing import Dict, List, Optional, Tuple -import lifemonitor.exceptions as lm_exceptions from flask import current_app from github.GithubException import GithubException, RateLimitExceededException +from rocrate import rocrate +from sqlalchemy import inspect +from sqlalchemy.ext.hybrid import hybrid_property + +import lifemonitor.exceptions as lm_exceptions from lifemonitor.api.models import db, repositories from lifemonitor.api.models.repositories.base import ( WorkflowRepository, WorkflowRepositoryMetadata) @@ -43,10 +47,6 @@ from lifemonitor.models import JSON from lifemonitor.storage import RemoteStorage from lifemonitor.utils import download_url, get_current_ref -from sqlalchemy import inspect -from sqlalchemy.ext.hybrid import hybrid_property - -from rocrate import rocrate # set module level logger logger = logging.getLogger(__name__) @@ -111,7 +111,7 @@ def storage_path(self) -> str: @property def revision(self) -> Optional[GithubRepositoryRevision]: - if self._is_github_crate_(self.uri) and isinstance(self.repository, GithubWorkflowRepository): + if self._get_normalized_github_url_(self.uri) and isinstance(self.repository, GithubWorkflowRepository): return self.repository.get_revision(self.version) return None @@ -172,7 +172,7 @@ def repository(self) -> repositories.WorkflowRepository: logger.warning(f"Getting path {self.storage_path} from remote storage.... DONE!!!") # instantiate a local ROCrate repository - if self._is_github_crate_(self.uri): + if self._get_normalized_github_url_(self.uri): authorizations = self.authorizations + [None] token = None for authorization in authorizations: @@ -192,8 +192,13 @@ def repository(self) -> repositories.WorkflowRepository: self._repository = repositories.ZippedWorkflowRepository(self.local_path) # set metadata - self._metadata = self._repository.metadata.to_json() - self._metadata_loaded = True + try: + self._metadata = self._repository.metadata.to_json() + self._metadata_loaded = True + except Exception as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) + raise lm_exceptions.NotValidROCrateException("Unable to load ROCrate metadata") return self._repository @property @@ -215,13 +220,13 @@ def __get_attribute_from_crate_reader__(self, if logger.isEnabledFor(logging.DEBUG): logger.exception(e) if not ignore_errors: - raise e + raise except Exception as e: logger.error(str(e)) if logger.isEnabledFor(logging.DEBUG): logger.exception(e) if not ignore_errors: - raise e + raise return None def get_authors(self, suite_id: str = None) -> List[Dict] | None: @@ -284,14 +289,9 @@ def download(self, target_path: str) -> str: return (tmpdir_path / 'rocrate.zip').as_posix() @staticmethod - def _is_github_crate_(uri: str) -> str: - # FIXME: replace with a better detection mechanism - if uri.startswith('https://github.com'): - # normalize uri as clone URL - if not uri.endswith('.git'): - uri += '.git' - return uri - return None + def _get_normalized_github_url_(uri: str) -> Optional[str]: + from lifemonitor.integrations.github.utils import normalized_github_url + return normalized_github_url(uri) @staticmethod def _find_git_ref_(repo: GithubWorkflowRepository, version: str) -> str: @@ -327,7 +327,7 @@ def download_from_source(self, target_path: str = None, uri: str = None, version # try either with authorization header and without authorization for authorization in self._get_authorizations(extra_auth=extra_auth): try: - git_url = self._is_github_crate_(uri) + git_url = self._get_normalized_github_url_(uri) if git_url: token = None if authorization and isinstance(authorization, ExternalServiceAuthorizationHeader): diff --git a/lifemonitor/api/models/rocrate/generators.py b/lifemonitor/api/models/rocrate/generators.py index f1c0cdabc..a257d8721 100644 --- a/lifemonitor/api/models/rocrate/generators.py +++ b/lifemonitor/api/models/rocrate/generators.py @@ -37,8 +37,8 @@ def generate_crate(workflow_type: str, workflow_version: str, local_repo_path: str, repo_url: Optional[str], - license: Optional[str] = "MIT", - ci_workflow: Optional[str] = "main.yml", + license: Optional[str] = None, + ci_workflow: Optional[str] = None, lang_version: Optional[str] = None, **kwargs): make_crate = get_crate_generator(workflow_type) diff --git a/lifemonitor/api/services.py b/lifemonitor/api/services.py index 09ba086c7..2f7f71910 100644 --- a/lifemonitor/api/services.py +++ b/lifemonitor/api/services.py @@ -34,7 +34,7 @@ from lifemonitor.auth.oauth2.client.models import OAuthIdentity from lifemonitor.auth.oauth2.server import server from lifemonitor.tasks.models import Job -from lifemonitor.utils import OpenApiSpecs, ROCrateLinkContext, to_snake_case +from lifemonitor.utils import OpenApiSpecs, ROCrateLinkContext, is_service_alive, to_snake_case from lifemonitor.ws import io logger = logging.getLogger() @@ -58,6 +58,9 @@ def __init__(self): def _find_and_check_shared_workflow_version(user: User, uuid, version=None) -> models.WorkflowVersion: for svc in models.WorkflowRegistry.all(): try: + if not is_service_alive(svc.uri): + logger.warning(f"Service {svc.uri} is not alive") + continue if svc.get_user(user.id): for w in svc.get_user_workflows(user): if str(w.uuid) == str(uuid): @@ -559,6 +562,9 @@ def get_user_workflows(user: User, workflows = [w for w in models.Workflow.get_user_workflows(user, include_subscriptions=include_subscriptions)] for svc in models.WorkflowRegistry.all(): + if not is_service_alive(svc.uri): + logger.warning("Service %r is not alive, skipping", svc.uri) + continue if svc.get_user(user.id): try: workflows.extend([w for w in svc.get_user_workflows(user) @@ -570,6 +576,9 @@ def get_user_workflows(user: User, @staticmethod def get_user_registry_workflows(user: User, registry: models.WorkflowRegistry) -> List[models.Workflow]: workflows = [] + if not is_service_alive(registry.uri): + logger.warning("Service %r is not alive, skipping", registry.uri) + raise lm_exceptions.UnavailableServiceException(registry.uri, service=registry) if registry.get_user(user.id): try: workflows.extend([w for w in registry.get_user_workflows(user) diff --git a/lifemonitor/app.py b/lifemonitor/app.py index d37a60628..edb8ceb13 100644 --- a/lifemonitor/app.py +++ b/lifemonitor/app.py @@ -45,7 +45,8 @@ logger = logging.getLogger(__name__) -def create_app(env=None, settings=None, init_app=True, worker=False, load_jobs=True, **kwargs): +def create_app(env=None, settings=None, init_app=True, init_integrations=True, + worker=False, load_jobs=True, **kwargs): """ App factory method :param env: @@ -81,7 +82,7 @@ def create_app(env=None, settings=None, init_app=True, worker=False, load_jobs=T # initialize the application if init_app: with app.app_context() as ctx: - initialize_app(app, ctx, load_jobs=load_jobs) + initialize_app(app, ctx, load_jobs=load_jobs, load_integrations=init_integrations) @app.route("/") def index(): @@ -127,7 +128,7 @@ def log_response(response): return app -def initialize_app(app: Flask, app_context, prom_registry=None, load_jobs: bool = True): +def initialize_app(app: Flask, app_context, prom_registry=None, load_jobs: bool = True, load_integrations: bool = True): # init tmp folder os.makedirs(app.config.get('BASE_TEMP_FOLDER'), exist_ok=True) # enable CORS @@ -151,7 +152,8 @@ def initialize_app(app: Flask, app_context, prom_registry=None, load_jobs: bool # init mail system init_mail(app) # initialize integrations - init_integrations(app) + if load_integrations: + init_integrations(app) # initialize metrics engine init_metrics(app, prom_registry) # register commands diff --git a/lifemonitor/auth/controllers.py b/lifemonitor/auth/controllers.py index 3ac44c569..dfa49a652 100644 --- a/lifemonitor/auth/controllers.py +++ b/lifemonitor/auth/controllers.py @@ -22,14 +22,17 @@ import connexion import flask -from flask import current_app, flash, redirect, render_template, request, session, url_for +from flask import (current_app, flash, redirect, render_template, request, + session, url_for) from flask_login import login_required, login_user, logout_user + from lifemonitor.cache import Timeout, cached, clear_cache from lifemonitor.utils import (NextRouteRegistry, next_route_aware, split_by_crlf) from .. import exceptions -from ..utils import OpenApiSpecs, boolean_value +from ..utils import (OpenApiSpecs, boolean_value, get_external_server_url, + is_service_alive) from . import serializers from .forms import (EmailForm, LoginForm, NotificationsForm, Oauth2ClientForm, RegisterForm, SetPasswordForm) @@ -190,6 +193,7 @@ def profile(form=None, passwordForm=None, currentView=None, registrySettingsForm=registrySettingsForm or RegistrySettingsForm.from_model(current_user), providers=get_providers(), currentView=currentView, oauth2_generic_client_scopes=OpenApiSpecs.get_instance().authorization_code_scopes, + api_base_url=get_external_server_url(), back_param=back_param) @@ -210,7 +214,8 @@ def register(): clear_cache() return redirect(url_for("auth.index")) return render_template("auth/register.j2", form=form, - action=url_for('auth.register'), providers=get_providers()) + action=url_for('auth.register'), + providers=get_providers(), is_service_available=is_service_alive) @blueprint.route("/identity_not_found", methods=("GET", "POST")) @@ -259,6 +264,7 @@ def login(): form = LoginForm() flask.session["confirm_user_details"] = True flask.session["sign_in"] = True + flask.session.pop('_flashes', None) if form.validate_on_submit(): user = form.get_user() if user: @@ -266,7 +272,8 @@ def login(): session.pop('_flashes', None) flash("You have logged in", category="success") return redirect(NextRouteRegistry.pop(url_for("auth.profile"))) - return render_template("auth/login.j2", form=form, providers=get_providers()) + return render_template("auth/login.j2", form=form, + providers=get_providers(), is_service_available=is_service_alive) @blueprint.route("/logout") @@ -276,7 +283,9 @@ def logout(): session.pop('_flashes', None) flash("You have logged out", category="success") NextRouteRegistry.clear() - return redirect('/') + next_route = request.args.get('next', '/') + logger.debug("Next route after logout: %r", next_route) + return redirect(next_route) @blueprint.route("/delete_account", methods=("POST",)) diff --git a/lifemonitor/auth/oauth2/client/controllers.py b/lifemonitor/auth/oauth2/client/controllers.py index 1e6682808..9ba70b8fb 100644 --- a/lifemonitor/auth/oauth2/client/controllers.py +++ b/lifemonitor/auth/oauth2/client/controllers.py @@ -24,17 +24,21 @@ from authlib.integrations.base_client.errors import OAuthError from authlib.integrations.flask_client import FlaskRemoteApp -from flask import (Blueprint, abort, current_app, flash, redirect, request, session, url_for) +from flask import (Blueprint, abort, current_app, flash, redirect, request, + session, url_for) from flask_login import current_user, login_user + from lifemonitor import exceptions, utils from lifemonitor.auth.models import User from lifemonitor.auth.oauth2.client.models import ( OAuth2IdentityProvider, OAuthIdentityNotFoundException) from lifemonitor.db import db -from lifemonitor.utils import NextRouteRegistry, next_route_aware +from lifemonitor.utils import (NextRouteRegistry, is_service_alive, + next_route_aware) from .models import OAuthIdentity, OAuthUserProfile -from .services import config_oauth2_registry, oauth2_registry, save_current_user_identity +from .services import (config_oauth2_registry, oauth2_registry, + save_current_user_identity) from .utils import RequestHelper # Config a module level logger @@ -94,6 +98,9 @@ def login(name, scope: str = None): remote = oauth2_registry.create_client(name) if remote is None: abort(404) + logger.debug("config: %r", remote.OAUTH_APP_CONFIG) + if not is_service_alive(remote.OAUTH_APP_CONFIG['api_base_url']): + abort(503) action = request.args.get('action', False) if action and action == 'sign-in': session['sign_in'] = True diff --git a/lifemonitor/auth/oauth2/client/models.py b/lifemonitor/auth/oauth2/client/models.py index 559693a75..90118bbfc 100644 --- a/lifemonitor/auth/oauth2/client/models.py +++ b/lifemonitor/auth/oauth2/client/models.py @@ -40,7 +40,7 @@ LifeMonitorException, NotAuthorizedException) from lifemonitor.models import JSON, ModelMixin -from lifemonitor.utils import to_snake_case +from lifemonitor.utils import assert_service_is_alive, to_snake_case from sqlalchemy import DateTime, inspect from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm.attributes import flag_modified @@ -206,6 +206,8 @@ def get_token(self, scope: Optional[str] = None): return OAuth2Token(token, provider=self.provider) def fetch_token(self, scope: Optional[str] = None): + # ensure that the service is alive + assert_service_is_alive(self.provider.api_base_url) # enable dynamic refresh only if the identity # has been already stored in the database if inspect(self).persistent: @@ -226,6 +228,8 @@ def refresh_token(self, token: OAuth2Token = None): logger.debug("Refresh token requested...") token = token or self.token if token and token.to_be_refreshed(): + # ensure that the service is alive + assert_service_is_alive(self.provider.api_base_url) with self.cache.lock(str(self), timeout=Timeout.NONE): # fetch current token from database self.refresh(attribute_names=['_tokens']) @@ -424,6 +428,7 @@ def token_type(self): return "Bearer" def get_user_info(self, provider_user_id, token, normalized=True): + assert_service_is_alive(self.api_base_url) access_token = token['access_token'] if isinstance(token, dict) else token response = requests.get(urljoin(self.api_base_url, self.userinfo_endpoint), headers={'Authorization': f'Bearer {access_token}'}) @@ -505,6 +510,7 @@ def find_identity_by_provider_user_id(self, provider_user_id): def refresh_token(self, token: OAuth2Token) -> OAuth2Token: logger.debug(f"Trying to refresh the token: {token}...") + assert_service_is_alive(self.api_base_url) # reference to the token associated with the identity instance oauth2session = OAuth2Session( self.client_id, self.client_secret, token=token) diff --git a/lifemonitor/auth/templates/auth/apikeys_tab.j2 b/lifemonitor/auth/templates/auth/apikeys_tab.j2 index e0ab14daf..6366bb225 100644 --- a/lifemonitor/auth/templates/auth/apikeys_tab.j2 +++ b/lifemonitor/auth/templates/auth/apikeys_tab.j2 @@ -7,7 +7,7 @@
- +
diff --git a/lifemonitor/auth/templates/auth/login.j2 b/lifemonitor/auth/templates/auth/login.j2 index 4bf2bd459..6bbf61da4 100644 --- a/lifemonitor/auth/templates/auth/login.j2 +++ b/lifemonitor/auth/templates/auth/login.j2 @@ -40,24 +40,25 @@

diff --git a/lifemonitor/auth/templates/auth/oauth2_clients_tab.j2 b/lifemonitor/auth/templates/auth/oauth2_clients_tab.j2 index c09974d0b..507964eb5 100644 --- a/lifemonitor/auth/templates/auth/oauth2_clients_tab.j2 +++ b/lifemonitor/auth/templates/auth/oauth2_clients_tab.j2 @@ -6,7 +6,7 @@
- +
diff --git a/lifemonitor/auth/templates/auth/register.j2 b/lifemonitor/auth/templates/auth/register.j2 index 2656f794d..0d495f2a2 100644 --- a/lifemonitor/auth/templates/auth/register.j2 +++ b/lifemonitor/auth/templates/auth/register.j2 @@ -66,16 +66,21 @@

- OR -

{% for p in providers %} {% if p.client_name != 'lsaai' %} - {{ macros.render_provider_signup_button(p) }} + {{ macros.render_provider_signup_button(p,not is_service_available(p.oauth_config['api_base_url'])) }} {% endif %} {% endfor %}
{% endif %} diff --git a/lifemonitor/auth/templates/auth/registry_settings.j2 b/lifemonitor/auth/templates/auth/registry_settings.j2 index 9590f8e05..c86f61725 100644 --- a/lifemonitor/auth/templates/auth/registry_settings.j2 +++ b/lifemonitor/auth/templates/auth/registry_settings.j2 @@ -15,33 +15,55 @@
+
Registry Name
Registry URL
-
Registry Type
- +
Registry Type
+ + + + {% for registry in registrySettingsForm.available_registries %}
- {% set registry_enabled = registry.client_name in registrySettingsForm.registries.data %} + {% set registry_enabled = registry.client_name in registrySettingsForm.registries.data.split(',') %}
{{ registrySettingsForm.csrf_token() }} - + data-toggle="toggle" {% if registry_enabled %}checked{% endif%} + data-off='Off' onchange="handleClick('{{ registry.client_name }}')">
-
+
{{ registry.name }}
diff --git a/lifemonitor/config.py b/lifemonitor/config.py index 721fd20f7..8bde89049 100644 --- a/lifemonitor/config.py +++ b/lifemonitor/config.py @@ -126,6 +126,8 @@ class BaseConfig: # Enable/disable integrations ENABLE_GITHUB_INTEGRATION = False ENABLE_REGISTRY_INTEGRATION = False + # Service Availability Timeout + SERVICE_AVAILABILITY_TIMEOUT = 1 class DevelopmentConfig(BaseConfig): @@ -156,6 +158,8 @@ class TestingConfig(BaseConfig): # CACHE_TYPE = "flask_caching.backends.nullcache.NullCache" CACHE_TYPE = "flask_caching.backends.rediscache.RedisCache" DATA_WORKFLOWS = f"{BaseConfig.BASE_TEMP_FOLDER}/lm_tests_data" + # Service Availability Timeout + SERVICE_AVAILABILITY_TIMEOUT = 120 class TestingSupportConfig(TestingConfig): @@ -164,6 +168,8 @@ class TestingSupportConfig(TestingConfig): TESTING = False LOG_LEVEL = "DEBUG" DATA_WORKFLOWS = f"{BaseConfig.BASE_TEMP_FOLDER}/lm_tests_data" + # Service Availability Timeout + SERVICE_AVAILABILITY_TIMEOUT = 120 _EXPORT_CONFIGS: List[Type[BaseConfig]] = [ diff --git a/lifemonitor/exceptions.py b/lifemonitor/exceptions.py index f793281ef..59939e734 100644 --- a/lifemonitor/exceptions.py +++ b/lifemonitor/exceptions.py @@ -130,6 +130,14 @@ def __str__(self): return self.detail +class UnavailableServiceException(LifeMonitorException): + + def __init__(self, detail=None, + type="about:blank", status=503, service=None, **kwargs): + super().__init__(title="External service not available", + detail=detail, status=status, service=getattr(service, "uri", None) or str(service), **kwargs) + + class WorkflowVersionConflictException(LifeMonitorException): def __init__(self, workflow_uuid, workflow_version, detail=None, **kwargs) -> None: diff --git a/lifemonitor/integrations/github/utils.py b/lifemonitor/integrations/github/utils.py index 70c835a20..f2454dba9 100644 --- a/lifemonitor/integrations/github/utils.py +++ b/lifemonitor/integrations/github/utils.py @@ -47,6 +47,22 @@ logger = logging.getLogger(__name__) +def is_github_url(url): + """ + Returns True if the given URL is a GitHub URL, False otherwise. + """ + pattern = r"^https?://(www\.)?github\.com/.*$" + return bool(re.match(pattern, url)) + + +def normalized_github_url(url): + if not is_github_url(url): + return None + if not url.endswith('.git'): + url += '.git' + return url + + def crate_branch(repo: Repository, branch_name: str, rev: str = None) -> GitRef: head = repo.get_commit(rev or repo.rev or 'HEAD') logger.debug("HEAD commit: %r", head.sha) diff --git a/lifemonitor/schemas/lifemonitor.json b/lifemonitor/schemas/lifemonitor.json index 863e211de..37c270d5e 100644 --- a/lifemonitor/schemas/lifemonitor.json +++ b/lifemonitor/schemas/lifemonitor.json @@ -4,7 +4,7 @@ "title": "LifeMonitor Configuration File", "type": "object", "properties": { - "workflow_name": { + "name": { "description": "worfklow name (override name defined on the RO-Crate metadata)", "type": "string", "minLength": 1 diff --git a/lifemonitor/static/src/package.json b/lifemonitor/static/src/package.json index 72e95c17c..0edf2ae84 100644 --- a/lifemonitor/static/src/package.json +++ b/lifemonitor/static/src/package.json @@ -1,7 +1,7 @@ { "name": "lifemonitor", "description": "Workflow Testing Service", - "version": "0.9.0", + "version": "0.10.0", "license": "MIT", "author": "CRS4", "main": "../dist/js/lifemonitor.min.js", diff --git a/lifemonitor/tasks/models.py b/lifemonitor/tasks/models.py index d8eccd2ac..c8b361178 100644 --- a/lifemonitor/tasks/models.py +++ b/lifemonitor/tasks/models.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from datetime import datetime +from datetime import datetime, timezone from flask import Flask from git import List @@ -73,8 +73,8 @@ def update_status(self, status, save: bool = False): def save(self): if not self._data.get('created', None): - self._data['created'] = datetime.utcnow().timestamp() # .replace(tzinfo=timezone.utc).timestamp() - self._data['modified'] = datetime.utcnow().timestamp() + self._data['created'] = datetime.now(tz=timezone.utc).timestamp() + self._data['modified'] = datetime.now(tz=timezone.utc).timestamp() set_job_data(self._job_id, self._data) notify_update(self._job_id, target_ids=self.listening_ids, target_rooms=self.listening_rooms) diff --git a/lifemonitor/templates/macros.j2 b/lifemonitor/templates/macros.j2 index 3a7fc3230..d5b5367e2 100644 --- a/lifemonitor/templates/macros.j2 +++ b/lifemonitor/templates/macros.j2 @@ -217,14 +217,18 @@ btn-info {%- endmacro %} -{% macro render_provider_signin_button(provider) -%} - +{% macro render_provider_signin_button(provider, disabled) -%} + {{render_provider_fa_icon(provider)}} Sign in with {{provider.name}} {%- endmacro %} -{% macro render_provider_signup_button(provider) -%} - +{% macro render_provider_signup_button(provider, disabled) -%} + {{render_provider_fa_icon(provider)}} Sign up with {{provider.name}} {%- endmacro %} \ No newline at end of file diff --git a/lifemonitor/templates/repositories/base/.lifemonitor.yaml.j2 b/lifemonitor/templates/repositories/base/.lifemonitor.yaml.j2 index 9d88148f6..a392db824 100644 --- a/lifemonitor/templates/repositories/base/.lifemonitor.yaml.j2 +++ b/lifemonitor/templates/repositories/base/.lifemonitor.yaml.j2 @@ -2,7 +2,7 @@ {% if workflow_name %}name: {{ workflow_name }} {% else %}# name: MyWorkflow{%endif %} # worfklow visibility -public: {{ public }} +public: {{ "true" if public else "false" }} # Issue Checker Settings issues: diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index fe3d2785f..7ec310ba7 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -49,6 +49,8 @@ import yaml from dateutil import parser +from lifemonitor.cache import cached + from . import exceptions as lm_exceptions logger = logging.getLogger() @@ -226,6 +228,30 @@ def validate_url(url: str) -> bool: return False +@cached(client_scope=False) +def is_service_alive(url: str, timeout: Optional[int] = None) -> bool: + try: + try: + timeout = timeout or flask.current_app.config.get("SERVICE_ALIVE_TIMEOUT", 1) + except Exception: + timeout = 1 + response = requests.get(url, timeout=timeout) + if response.status_code < 500: + return True + else: + return False + except requests.exceptions.RequestException as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) + logger.error(f'Error checking service availability: {e}') + return False + + +def assert_service_is_alive(url: str, timeout: Optional[int] = None): + if not is_service_alive(url, timeout=timeout): + raise lm_exceptions.UnavailableServiceException(detail=f"Service not available: {url}", service=url) + + def get_last_update(path: str): return time.ctime(max(os.stat(root).st_mtime for root, _, _ in os.walk(path))) @@ -248,7 +274,6 @@ def match_ref(ref: str, refs: List[str]) -> Optional[Tuple[str, str]]: def notify_updates(workflows: List, type: str = 'sync', delay: int = 0): from lifemonitor.ws import io - from datetime import timezone io.publish_message({ "type": type, "data": [{ diff --git a/lifemonitor/ws/events.py b/lifemonitor/ws/events.py index 6bc26ef99..b4327b5ff 100644 --- a/lifemonitor/ws/events.py +++ b/lifemonitor/ws/events.py @@ -23,7 +23,7 @@ from typing import Dict from flask import request -from flask_socketio import disconnect, emit, join_room +from flask_socketio import disconnect, emit, join_room, leave_room from lifemonitor.cache import cache @@ -84,7 +84,27 @@ def handle_message(message): logger.debug(f"Joining SID {request.sid} to room {message['data']['user']}") join_room(str(message['data']['user'])) logger.warning(f"SID {request.sid} joined to room {message['data']['user']}") - elif message['type'] == 'SYNC': + emit('message', { + 'payload': { + 'type': 'joined', + 'data': { + 'user': str(message['data']['user']) + }, + } + }) + elif message['type'] == 'leave': + logger.debug(f"Leaving SID {request.sid} room {message['data']['user']}") + leave_room(str(message['data']['user'])) + logger.warning(f"SID {request.sid} left room {message['data']['user']}") + emit('message', { + 'payload': { + 'type': 'joined', + 'data': { + 'user': str(message['data']['user']) + }, + } + }) + elif message['type'] == 'sync': emit("message", build_sync_message()) diff --git a/lifemonitor/ws/io.py b/lifemonitor/ws/io.py index cd133fb5b..3785148ea 100644 --- a/lifemonitor/ws/io.py +++ b/lifemonitor/ws/io.py @@ -4,6 +4,7 @@ import logging import time from typing import List +from uuid import UUID from flask import Flask @@ -27,6 +28,14 @@ def __format_timestamp__(timestamp: datetime) -> str: return timestamp.strftime('%a %b %d %Y %H:%M:%S ') + 'GMT' + tz_offset[:3] + ':' + tz_offset[3:] +class _CustomEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, UUID): + return str(obj) + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) + + def publish_message(message, channel: str = __CHANNEL__, target_ids: List[str] = None, target_rooms: List[str] = None, delay: int = 0): now = datetime.datetime.now(datetime.timezone.utc) @@ -39,7 +48,7 @@ def publish_message(message, channel: str = __CHANNEL__, "target_rooms": target_rooms, # "timestamp": datetime_as_timestamp_with_msecs(now), "payload": message - })) + }, cls=_CustomEncoder)) def start_reading(app: Flask, channel: str = __CHANNEL__, max_age: int = __MAX_AGE__): diff --git a/prova b/prova new file mode 100644 index 000000000..e69de29bb diff --git a/specs/api.yaml b/specs/api.yaml index 2e39c9174..f70913958 100644 --- a/specs/api.yaml +++ b/specs/api.yaml @@ -3,7 +3,7 @@ openapi: "3.0.0" info: - version: "0.9.0" + version: "0.10.0" title: "Life Monitor API" description: | *Workflow sustainability service* @@ -18,7 +18,7 @@ info: servers: - url: / description: > - Version 0.9.0 of API. + Version 0.10.0 of API. tags: - name: GitHub Integration diff --git a/tests/unit/cache/test_cache.py b/tests/unit/cache/test_cache.py index 44d42ecb7..64279b147 100644 --- a/tests/unit/cache/test_cache.py +++ b/tests/unit/cache/test_cache.py @@ -309,7 +309,8 @@ def cache_last_build_update(app, w, user1, check_cache_size=True, index=0, # check the cache after the transaction is completed if check_cache_size: cache_size = cache.size() - assert len(transaction_keys) == cache_size, "Unpexpected cache size: it should be equal to the transaction size" + # +2 because of the checks for the service availability + assert len(transaction_keys) + 2 == cache_size, "Unpexpected cache size: it should be equal to the transaction size + 2" sleep(2) assert cache.size() > 0, "Cache should not be empty" logger.debug(cache.keys()) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 68c20eca0..217557484 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -29,9 +29,9 @@ def test_download_url_404(): with tempfile.TemporaryDirectory() as d: - with pytest.raises(lm_exceptions.DownloadException) as exec_info: - _ = utils.download_url('http://httpbin.org/status/404', os.path.join(d, 'get_404')) - assert exec_info.value.status == 404 + with pytest.raises(lm_exceptions.DownloadException) as excinfo: + _ = utils.download_url('https://github.com/crs4/life_monitor/fake_path', os.path.join(d, 'fake_path')) + assert excinfo.value.status == 404 def test_datetime_to_isoformat(): diff --git a/tests/utils.py b/tests/utils.py index 4b58a70d3..8ee66fda2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -23,6 +23,7 @@ import lifemonitor.db as lm_db from lifemonitor.api import models +from lifemonitor.api.models.workflows import WorkflowVersion from lifemonitor.api.services import LifeMonitor logger = logging.getLogger(__name__) @@ -60,7 +61,10 @@ def _get_attr(obj, name, default=None): def build_workflow_path(workflow=None, version_as_subpath=False, subpath=None, include_version=True): if workflow: - w = f"{_WORKFLOWS_ENDPOINT}/{_get_attr(workflow, 'uuid')}" + if isinstance(workflow, WorkflowVersion): + w = f"{_WORKFLOWS_ENDPOINT}/{_get_attr(workflow.workflow, 'uuid')}" + else: + w = f"{_WORKFLOWS_ENDPOINT}/{_get_attr(workflow, 'uuid')}" if include_version and version_as_subpath: w = f"{w}/versions/{_get_attr(workflow, 'version')}" if subpath: diff --git a/ws.py b/ws.py index 95e90d018..944854dc3 100644 --- a/ws.py +++ b/ws.py @@ -29,6 +29,6 @@ # create an app instance -application = create_app(init_app=True, load_jobs=False) +application = create_app(init_app=True, load_jobs=False, init_integrations=False) socketIO = initialise_ws(application) start_brodcaster(application, max_age=5) # use default ws_channel