From 1563547724fbab2741af860ed1c6b0cc28073477 Mon Sep 17 00:00:00 2001 From: vadim-zabolotniy <34000376+vadim-zabolotniy@users.noreply.github.com> Date: Tue, 7 Feb 2023 18:40:16 +0200 Subject: [PATCH] Event logging on invalid template variable. (#272) * Replaced pystache by chevron. Error rising on unknown template variable. * Warning event on invalid template variable during application installation. --- application/api/v1/endpoints/applications.py | 2 +- application/managers/applications.py | 5 +- application/requirements.txt | 2 +- .../tasks/application/install_flow.py | 19 +++++ application/utils/template.py | 78 +++++++++++++++++-- 5 files changed, 95 insertions(+), 11 deletions(-) diff --git a/application/api/v1/endpoints/applications.py b/application/api/v1/endpoints/applications.py index cfd611e..261a1b9 100644 --- a/application/api/v1/endpoints/applications.py +++ b/application/api/v1/endpoints/applications.py @@ -119,7 +119,7 @@ async def get_application_health_status( @router.get( '/{application_id}/outputs', - response_model=Outputs, + response_model=Outputs | None, dependencies=[Depends(AuthorizedUser(OperatorRolePermission))] ) async def get_application_outputs( diff --git a/application/managers/applications.py b/application/managers/applications.py index 07647d1..51ec3fe 100644 --- a/application/managers/applications.py +++ b/application/managers/applications.py @@ -570,13 +570,14 @@ async def get_components_manifests(self, application: Application, skip_absent: return manifests def render_manifest(self, template: TemplateRevision, *, application: Application | None = None, - user_inputs: dict | None = None, components_manifests: dict[str, list] | None = None) -> str: + user_inputs: dict | None = None, components_manifests: dict[str, list] | None = None, + skip_context_error: bool = True) -> str: """ Renders new application's manifest using existing user input. """ inputs = self.get_inputs(template, application=application, user_inputs=user_inputs) - manifest = render_template(template.template, inputs, components_manifests) + manifest = render_template(template.template, inputs, components_manifests, skip_context_error=skip_context_error) return manifest diff --git a/application/requirements.txt b/application/requirements.txt index 976aca6..9b5118f 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -1,6 +1,7 @@ aiosmtplib alembic asyncpg +chevron deepdiff email-validator Faker # This dependency temporary must be in production requirements until Organization title change will be implemented. @@ -13,7 +14,6 @@ kubernetes_asyncio==22.6.5 # Version locked because of issue https://github.com mergedeep procrastinate pydantic[dotenv] -pystache pyyaml sqlalchemy[asyncio]==1.4.46 uvicorn[standard] diff --git a/application/services/procrastinate/tasks/application/install_flow.py b/application/services/procrastinate/tasks/application/install_flow.py index f521307..0a42dd7 100644 --- a/application/services/procrastinate/tasks/application/install_flow.py +++ b/application/services/procrastinate/tasks/application/install_flow.py @@ -11,6 +11,7 @@ from exceptions.application import ApplicationException from exceptions.application import ApplicationHookLaunchException from exceptions.application import ApplicationHookTimeoutException +from exceptions.templates import TemlateVariableNotFoundException from schemas.events import EventSchema from schemas.templates import TemplateSchema from services.procrastinate.application import procrastinate @@ -129,6 +130,24 @@ async def install_applicatoin_components(application_id: int): raise await application_manager.set_health_status(application, ApplicationHealthStatuses.healthy) + components_manifests = await application_manager.get_components_manifests(application) + try: + raw_manifest = application_manager.render_manifest( + application.template, application=application, components_manifests=components_manifests, + skip_context_error=False + ) + except TemlateVariableNotFoundException as error: + await application_manager.event_manager.create(EventSchema( + title='Application deployment', + message=f'Application deployment failed. {error.message.strip(".")}.', + organization_id=application.organization.id, + category=EventCategory.application, + severity=EventSeverityLevel.warning, + data={'application_id': application.id} + )) + raw_manifest = application_manager.render_manifest( + application.template, application=application, components_manifests=components_manifests + ) application.manifest = raw_manifest await application_manager.db.save(application) diff --git a/application/utils/template.py b/application/utils/template.py index 2e24a08..52a317a 100644 --- a/application/utils/template.py +++ b/application/utils/template.py @@ -3,8 +3,9 @@ """ import logging import re +from typing import Any -import pystache +import chevron import yaml from pydantic import ValidationError from pydantic.error_wrappers import display_errors @@ -12,6 +13,7 @@ from exceptions.templates import InvalidTemplateException from exceptions.templates import InvalidUserInputsException +from exceptions.templates import TemlateVariableNotFoundException from schemas.templates import TemplateSchema @@ -25,6 +27,62 @@ PLACEHOLDERS = re.compile(r'.*{{(.*)}}.*') +class TemplateContextDictionaryProxy(dict): + """ + Wrapper to track absent keys in context. + """ + + def __init__(self, object, skip_error: bool = True) -> None: + self.skip_error = skip_error + super().__init__(object) + + def __getitem__(self, key: str) -> Any: + try: + next_scope = super().__getitem__(key) + + return context_factory(next_scope, skip_error=self.skip_error) + except KeyError: + if self.skip_error: + raise + raise TemlateVariableNotFoundException( + f'Unable to find key "{key}" in context provided for template rendering', variable=key + ) + + +class TemplateContextListProxy(list): + """ + Wrapper to track absent indexes in context. + """ + + def __init__(self, object, skip_error: bool = True) -> None: + self.skip_error = skip_error + super().__init__(object) + + def __getitem__(self, index: str) -> Any: + try: + next_scope = super().__getitem__(index) + + return context_factory(next_scope, skip_error=self.skip_error) + except IndexError: + if self.skip_error: + raise + raise TemlateVariableNotFoundException( + f'Unable to find index "{index}" in context provided for template rendering', variable=index + ) + + +def context_factory(object: Any, skip_error: bool = True): + """ + Create and returns template context proxy. + """ + if isinstance(object, dict): + return TemplateContextDictionaryProxy(object, skip_error=skip_error) + elif isinstance(object, list): + return TemplateContextListProxy(object, skip_error=skip_error) + else: + return object + + def make_template_yaml_safe(raw_template: str) -> str: """ Transforms template YAML into valid, by replacing `{{` to `"{{{` and `}}` to @@ -74,7 +132,9 @@ def load_template(raw_template: str) -> TemplateSchema: return validate_template(parsed_template_data) -def render_template(template: str, inputs: dict, components_manifests: dict[str, list] | None = None) -> str: +def render_template( + template: str, inputs: dict, components_manifests: dict[str, list] | None = None, skip_context_error: bool = True +) -> str: """ Renders template with provided context. """ @@ -86,11 +146,15 @@ def render_template(template: str, inputs: dict, components_manifests: dict[str, for entity in entities: grouped_entites.setdefault(entity.kind, {})[entity.metadata['name']] = entity.raw_representation components_context[component_name] = {'manifest': grouped_entites} - context = { - 'inputs': inputs, - 'components': components_context - } - return pystache.render(template, context) + context = context_factory( + { + 'inputs': inputs, + 'components': components_context + }, + skip_error=skip_context_error + ) + + return chevron.render(template, context) def validate_inputs(template: str, inputs: dict) -> None: