From d03273c22773c51442898eda39154c8256a47020 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Tue, 29 Oct 2024 12:59:52 +0000 Subject: [PATCH 01/16] migration to add MF enforcement permissions --- .../services/metadata_form_permissions.py | 7 +- ...9ab_backfill_mf_enforcement_permissions.py | 122 ++++++++++++++++++ 2 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 backend/migrations/versions/ba2da94739ab_backfill_mf_enforcement_permissions.py diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py b/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py index 883977a16..bf1a47027 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py @@ -17,13 +17,16 @@ # these permissions are attached to Organizations, Environments, Datasets etc. ATTACH_METADATA_FORM = 'ATTACH_METADATA_FORM' CREATE_METADATA_FORM = 'CREATE_METADATA_FORM' -ALL_METADATA_FORMS_ENTITY_PERMISSIONS = [ATTACH_METADATA_FORM, CREATE_METADATA_FORM] +ENFORCE_METADATA_FORM = 'ENFORCE_METADATA_FORM' +ALL_METADATA_FORMS_ENTITY_PERMISSIONS = [ENFORCE_METADATA_FORM, ATTACH_METADATA_FORM, CREATE_METADATA_FORM] RESOURCES_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS) RESOURCES_ALL_WITH_DESC[CREATE_METADATA_FORM] = 'Create metadata form within this visibility scope' +RESOURCES_ALL_WITH_DESC[ENFORCE_METADATA_FORM] = 'Enforce metadata form within this visibility scope' RESOURCES_ALL_WITH_DESC[ATTACH_METADATA_FORM] = 'Attach metadata form' ORGANIZATION_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS) -ORGANIZATION_INVITED_DESCRIPTIONS[CREATE_METADATA_FORM] = 'Create metadata form within this visibility scope' +ORGANIZATION_INVITED_DESCRIPTIONS[CREATE_METADATA_FORM] = 'Create metadata form within this organization' +ORGANIZATION_INVITED_DESCRIPTIONS[ENFORCE_METADATA_FORM] = 'Enforce metadata form within this organization' ORGANIZATION_INVITED_DESCRIPTIONS[ATTACH_METADATA_FORM] = 'Attach metadata form' ENVIRONMENT_INVITED.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS) diff --git a/backend/migrations/versions/ba2da94739ab_backfill_mf_enforcement_permissions.py b/backend/migrations/versions/ba2da94739ab_backfill_mf_enforcement_permissions.py new file mode 100644 index 000000000..cd3cade39 --- /dev/null +++ b/backend/migrations/versions/ba2da94739ab_backfill_mf_enforcement_permissions.py @@ -0,0 +1,122 @@ +"""backfill_mf_enforcement_permissions + +Revision ID: ba2da94739ab +Revises: b21f86882012 +Create Date: 2024-10-29 12:56:06.523524 + +""" + +from alembic import op +from sqlalchemy import orm + +from dataall.core.environment.db.environment_models import Environment +from dataall.core.organizations.db.organization_models import Organization +from dataall.core.permissions.api.enums import PermissionType +from dataall.core.permissions.services.permission_service import PermissionService +from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService +from dataall.core.permissions.services.resources_permissions import RESOURCES_ALL_WITH_DESC +from dataall.modules.datasets_base.db.dataset_models import DatasetBase +from dataall.modules.metadata_forms.services.metadata_form_permissions import ENFORCE_METADATA_FORM + +# revision identifiers, used by Alembic. +revision = 'ba2da94739ab' +down_revision = 'b21f86882012' +branch_labels = None +depends_on = None + + +def get_session(): + bind = op.get_bind() + session = orm.Session(bind=bind) + return session + + +def upgrade(): + session = get_session() + + PermissionService.save_permission( + session, + name=ENFORCE_METADATA_FORM, + description=RESOURCES_ALL_WITH_DESC.get(ENFORCE_METADATA_FORM, ENFORCE_METADATA_FORM), + permission_type=PermissionType.RESOURCE.name, + ) + print('Adding organization resource permissions...') + orgs = session.query(Organization).all() + for org in orgs: + ResourcePolicyService.attach_resource_policy( + session=session, + group=org.SamlGroupName, + resource_uri=org.organizationUri, + permissions=[ENFORCE_METADATA_FORM], + resource_type=Organization.__name__, + ) + print('Adding environment resource permissions...') + envs = session.query(Environment).all() + for env in envs: + ResourcePolicyService.attach_resource_policy( + session=session, + group=env.SamlGroupName, + resource_uri=env.environmentUri, + permissions=[ENFORCE_METADATA_FORM], + resource_type=Environment.__name__, + ) + print('Adding dataset resource permissions...') + datasets = session.query(DatasetBase).all() + for dataset in datasets: + ResourcePolicyService.attach_resource_policy( + session=session, + group=dataset.SamlAdminGroupName, + resource_uri=dataset.datasetUri, + permissions=[ENFORCE_METADATA_FORM], + resource_type=DatasetBase.__name__, + ) + + +def downgrade(): + bind = op.get_bind() + session = orm.Session(bind=bind) + all_environments = session.query(Environment).all() + for env in all_environments: + policies = ResourcePolicyService.find_resource_policies( + session=session, + group=env.SamlGroupName, + resource_uri=env.environmentUri, + resource_type=Environment.__name__, + permissions=[ENFORCE_METADATA_FORM], + ) + for policy in policies: + for resource_pol_permission in policy.permissions: + if resource_pol_permission.permission.name in [ENFORCE_METADATA_FORM]: + session.delete(resource_pol_permission) + session.commit() + + all_organizations = session.query(Organization).all() + for org in all_organizations: + policies = ResourcePolicyService.find_resource_policies( + session=session, + group=org.SamlGroupName, + resource_uri=org.organizationUri, + permissions=[ENFORCE_METADATA_FORM], + resource_type=Organization.__name__, + ) + for policy in policies: + for resource_pol_permission in policy.permissions: + if resource_pol_permission.permission.name in [ENFORCE_METADATA_FORM]: + session.delete(resource_pol_permission) + session.commit() + + datasets = session.query(DatasetBase).all() + for dataset in datasets: + policies = ResourcePolicyService.find_resource_policies( + session=session, + group=dataset.SamlAdminGroupName, + resource_uri=dataset.datasetUri, + permissions=[ENFORCE_METADATA_FORM], + resource_type=DatasetBase.__name__, + ) + + for policy in policies: + for resource_pol_permission in policy.permissions: + if resource_pol_permission.permission.name in [ENFORCE_METADATA_FORM]: + session.delete(resource_pol_permission) + session.commit() From 3d717a9f09564128154536540f29e0312e2f88ff Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Wed, 30 Oct 2024 12:54:26 +0000 Subject: [PATCH 02/16] create enforcement rule method + list affected entities --- .../db/environment_repositories.py | 4 + .../db/organization_repositories.py | 4 + .../datasets_base/db/dataset_repositories.py | 16 +- .../modules/metadata_forms/db/enums.py | 96 +++++++++- .../metadata_forms/db/metadata_form_models.py | 1 + .../db/metadata_form_repository.py | 38 +++- .../services/metadata_form_access_service.py | 51 ++++- .../metadata_form_enforcement_service.py | 174 ++++++++++++++++++ 8 files changed, 369 insertions(+), 15 deletions(-) create mode 100644 backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py diff --git a/backend/dataall/core/environment/db/environment_repositories.py b/backend/dataall/core/environment/db/environment_repositories.py index d5836203f..bf889fc25 100644 --- a/backend/dataall/core/environment/db/environment_repositories.py +++ b/backend/dataall/core/environment/db/environment_repositories.py @@ -323,3 +323,7 @@ def get_environment_consumption_role_by_name(session, uri, IAMRoleName): ) .first() ) + + @staticmethod + def get_all_envs_by_organization(session, uri): + return session.query(Environment).filter(Environment.organizationUri == uri).all() diff --git a/backend/dataall/core/organizations/db/organization_repositories.py b/backend/dataall/core/organizations/db/organization_repositories.py index 0fbf23935..044f9d248 100644 --- a/backend/dataall/core/organizations/db/organization_repositories.py +++ b/backend/dataall/core/organizations/db/organization_repositories.py @@ -127,3 +127,7 @@ def find_group_membership(session, groups, organization): .first() ) return membership + + @staticmethod + def query_all_active_organizations(session): + return session.query(models.Organization).filter(models.Organization.deleted.is_(None)).all() diff --git a/backend/dataall/modules/datasets_base/db/dataset_repositories.py b/backend/dataall/modules/datasets_base/db/dataset_repositories.py index 28281052e..478bed8f0 100644 --- a/backend/dataall/modules/datasets_base/db/dataset_repositories.py +++ b/backend/dataall/modules/datasets_base/db/dataset_repositories.py @@ -107,19 +107,19 @@ def paginated_environment_datasets( data=None, ) -> dict: return paginate( - query=DatasetListRepository._query_environment_datasets(session, uri, data), + query=DatasetListRepository.query_datasets(session, data, environmentUri=uri), page=data.get('page', 1), page_size=data.get('pageSize', 10), ).to_dict() @staticmethod - def _query_environment_datasets(session, uri, filter) -> Query: - query = session.query(DatasetBase).filter( - and_( - DatasetBase.environmentUri == uri, - DatasetBase.deleted.is_(None), - ) - ) + def query_datasets(session, filter=None, organizationUri=None, environmentUri=None) -> Query: + query = session.query(DatasetBase).filter(DatasetBase.deleted.is_(None)) + if organizationUri: + query = query.filter(DatasetBase.organizationUri == organizationUri) + if environmentUri: + query = query.filter(DatasetBase.environmentUri == environmentUri) + if filter and filter.get('term'): term = filter['term'] query = query.filter( diff --git a/backend/dataall/modules/metadata_forms/db/enums.py b/backend/dataall/modules/metadata_forms/db/enums.py index 0d573e862..c459a5c74 100644 --- a/backend/dataall/modules/metadata_forms/db/enums.py +++ b/backend/dataall/modules/metadata_forms/db/enums.py @@ -1,4 +1,14 @@ from dataall.base.api.constants import GraphQLEnumMapper +from dataall.core.environment.db.environment_models import Environment, EnvironmentGroup, ConsumptionRole +from dataall.core.organizations.db.organization_models import Organization, OrganizationGroup +from dataall.modules.dashboards.db.dashboard_models import Dashboard +from dataall.modules.datapipelines.db.datapipelines_models import DataPipeline +from dataall.modules.mlstudio.db.mlstudio_models import SagemakerStudioDomain +from dataall.modules.notebooks.db.notebook_models import SagemakerNotebook +from dataall.modules.redshift_datasets.db.redshift_models import RedshiftDataset +from dataall.modules.s3_datasets.db.dataset_models import S3Dataset, DatasetTable, DatasetStorageLocation, DatasetBucket +from dataall.modules.shares_base.db.share_object_models import ShareObject +from dataall.modules.worksheets.db.worksheet_models import Worksheet class MetadataFormVisibility(GraphQLEnumMapper): @@ -34,6 +44,91 @@ class MetadataFormEntityTypes(GraphQLEnumMapper): Share = 'Share' ShareItem = 'Share Item' + @staticmethod + def get_entity_class(value: str): + classes = { + MetadataFormEntityTypes.Organizations.value: ( + Organization, + MetadataFormEnforcementScope.Global, + lambda o: (o.organizationUri, o.SamlGroupName), + ), + MetadataFormEntityTypes.OrganizationTeams.value: ( + OrganizationGroup, + MetadataFormEnforcementScope.Organization, + lambda o: (o.organizationUri + o.groupUri, o.invitedBy), + ), + MetadataFormEntityTypes.Environments.value: ( + Environment, + MetadataFormEnforcementScope.Organization, + lambda o: (o.environmentUri, o.SamlGroupName), + ), + MetadataFormEntityTypes.EnvironmentTeams.value: ( + EnvironmentGroup, + MetadataFormEnforcementScope.Environment, + lambda o: (o.environmentUri + o.groupUri, o.invitedBy), + ), + MetadataFormEntityTypes.S3Datasets.value: ( + S3Dataset, + MetadataFormEnforcementScope.Environment, + lambda o: (o.datasetUri, o.SamlAdminGroupName), + ), + MetadataFormEntityTypes.RDDatasets.value: ( + RedshiftDataset, + MetadataFormEnforcementScope.Environment, + lambda o: (o.datasetUri, o.SamlAdminGroupName), + ), + MetadataFormEntityTypes.Worksheets.value: ( + Worksheet, + MetadataFormEnforcementScope.Global, + lambda o: (o.worksheetUri, o.SamlAdminGroupName), + ), + MetadataFormEntityTypes.Dashboards.value: ( + Dashboard, + MetadataFormEnforcementScope.Environment, + lambda o: (o.dashboardUri, o.SamlGroupName), + ), + MetadataFormEntityTypes.ConsumptionRoles.value: ( + ConsumptionRole, + MetadataFormEnforcementScope.Environment, + lambda o: (o.consumptionRoleUri, o.groupUri), + ), + MetadataFormEntityTypes.Notebooks.value: ( + SagemakerNotebook, + MetadataFormEnforcementScope.Environment, + lambda o: (o.notebookUri, o.SamlAdminGroupName), + ), + MetadataFormEntityTypes.MLStudioEntities.value: ( + SagemakerStudioDomain, + MetadataFormEnforcementScope.Environment, + lambda o: (o.sagemakerStudioUri, o.SamlGroupName), + ), + MetadataFormEntityTypes.Pipelines.value: ( + DataPipeline, + MetadataFormEnforcementScope.Environment, + lambda o: (o.DataPipelineUri, o.SamlGroupName), + ), + MetadataFormEntityTypes.Tables.value: ( + DatasetTable, + MetadataFormEnforcementScope.Dataset, + lambda o: (o.tableUri, None), # ToDo: resolve owner + ), + MetadataFormEntityTypes.Folder.value: ( + DatasetStorageLocation, + MetadataFormEnforcementScope.Dataset, + lambda o: (o.locationUri, None), # ToDo: resolve owner + ), + MetadataFormEntityTypes.Bucket.value: ( + DatasetBucket, + MetadataFormEnforcementScope.Dataset, + lambda o: (o.bucketUri, None), # ToDo: resolve owner + ), + MetadataFormEntityTypes.Share.value: ( + ShareObject, + MetadataFormEnforcementScope.Dataset, + lambda o: (o.shareUri, o.groupUri), + ), + } + class MetadataFormEnforcementSeverity(GraphQLEnumMapper): Mandatory = 'Mandatory' @@ -41,7 +136,6 @@ class MetadataFormEnforcementSeverity(GraphQLEnumMapper): class MetadataFormEnforcementScope(GraphQLEnumMapper): - Item = 'Item Level' Dataset = 'Dataset Level' Environment = 'Environmental Level' Organization = 'Organizational Level' diff --git a/backend/dataall/modules/metadata_forms/db/metadata_form_models.py b/backend/dataall/modules/metadata_forms/db/metadata_form_models.py index 9d23aee6e..739dbe256 100644 --- a/backend/dataall/modules/metadata_forms/db/metadata_form_models.py +++ b/backend/dataall/modules/metadata_forms/db/metadata_form_models.py @@ -36,6 +36,7 @@ class MetadataFormEnforcementRule(Base): metadataFormUri = Column(String, ForeignKey('metadata_form.uri'), nullable=False) version = Column(Integer, nullable=False) level = Column(String, nullable=False) # enum MetadataFormEnforcementScope + homeEntity = Column(String, nullable=True) entityTypes = Column(ARRAY(String), nullable=False) # enum MetadataFormEntityTypes severity = Column(String, nullable=False) # enum MetadataFormEnforcementSeverity diff --git a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py index 366841e90..363bcf8c5 100644 --- a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py +++ b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py @@ -2,7 +2,11 @@ from sqlalchemy.orm import with_polymorphic from sqlalchemy import func -from dataall.modules.metadata_forms.db.enums import MetadataFormVisibility, MetadataFormFieldType +from dataall.modules.metadata_forms.db.enums import ( + MetadataFormVisibility, + MetadataFormFieldType, + MetadataFormEnforcementSeverity, +) from dataall.modules.metadata_forms.db.metadata_form_models import ( MetadataForm, MetadataFormField, @@ -13,6 +17,7 @@ IntegerAttachedMetadataFormField, GlossaryTermAttachedMetadataFormField, MetadataFormVersion, + MetadataFormEnforcementRule, ) import json @@ -300,10 +305,17 @@ def query_attached_metadata_forms(session, is_da_admin, groups, user_envs_uris, return query.order_by(all_mfs.c.name) @staticmethod - def query_all_attached_metadata_forms_for_entity(session, entityUri, entityType): - return session.query(AttachedMetadataForm).filter( - and_(AttachedMetadataForm.entityType == entityType, AttachedMetadataForm.entityUri == entityUri) - ) + def query_all_attached_metadata_forms_for_entity( + session, entityUri, entityType=None, metadataFormUri=None, version=None + ): + amfs = session.query(AttachedMetadataForm).filter(AttachedMetadataForm.entityUri == entityUri) + if entityType: + amfs = amfs.filter(AttachedMetadataForm.entityType == entityType) + if metadataFormUri: + amfs = amfs.filter(AttachedMetadataForm.metadataFormUri == metadataFormUri) + if version: + amfs = amfs.filter(AttachedMetadataForm.version == version) + return amfs @staticmethod def get_metadata_form_versions_numbers(session, uri): @@ -331,3 +343,19 @@ def get_all_attached_metadata_forms(session, mf_uri, version=None): if version: all_attached = all_attached.filter(AttachedMetadataForm.version == version) return all_attached.all() + + @staticmethod + def create_mf_enforcement_rule(session, uri, data, version): + rule = MetadataFormEnforcementRule( + metadataFormUri=uri, + version=version, + level=data.get('level'), + homeEntity=data.get('homeEntity'), + entityTypes=data.get('entityTypes'), + severity=data.get('severity', MetadataFormEnforcementSeverity.Recommended.value), + ) + return rule + + @staticmethod + def get_mf_enforcement_rule_by_uri(session, uri): + return session.query(MetadataFormEnforcementRule).get(uri) diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_access_service.py b/backend/dataall/modules/metadata_forms/services/metadata_form_access_service.py index b62976b97..7e07bdee7 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_access_service.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_access_service.py @@ -1,12 +1,20 @@ +import logging + from dataall.base.context import get_context from dataall.core.environment.db.environment_repositories import EnvironmentRepository from dataall.core.organizations.db.organization_repositories import OrganizationRepository +from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService from dataall.modules.datasets_base.db.dataset_repositories import DatasetBaseRepository -from dataall.modules.metadata_forms.db.enums import MetadataFormUserRoles, MetadataFormEntityTypes +from dataall.modules.metadata_forms.db.enums import ( + MetadataFormUserRoles, + MetadataFormEntityTypes, + MetadataFormEnforcementScope, +) from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository from functools import wraps from dataall.base.db import exceptions +from dataall.modules.metadata_forms.services.metadata_form_permissions import ENFORCE_METADATA_FORM class MetadataFormAccessService: @@ -96,3 +104,44 @@ def get_target_orgs_and_envs(username, groups, is_da_admin=False, filter={}): envs = MetadataFormAccessService._target_env_uri_getter(filter.get('entityType'), filter.get('entityUri')) return orgs, envs + + @staticmethod + def check_enforcement_access(entityUri, level): + context = get_context() + if TenantPolicyValidationService.is_tenant_admin(context.groups): + return True + + if level == MetadataFormEnforcementScope.Global.value: + raise exceptions.UnauthorizedOperation( + action=ENFORCE_METADATA_FORM, message='Only data.all admins can enforce metadata forms on global level' + ) + + with context.db_engine.scoped_session() as session: + entities_to_check = [entityUri] + if level == MetadataFormEnforcementScope.Environment.value: + env = EnvironmentRepository.get_environment_by_uri(session, entityUri) + entities_to_check.append(env.organizationUri) + if level == MetadataFormEnforcementScope.Dataset.value: + dataset = DatasetBaseRepository.get_dataset_by_uri(session, entityUri) + entities_to_check.append(dataset.organizationUri) + entities_to_check.append(dataset.environmentUri) + + failed_checks = [] + for entity in entities_to_check: + try: + ResourcePolicyService.check_user_resource_permission( + session=session, + username=context.username, + groups=context.groups, + resource_uri=entity, + permission_name=ENFORCE_METADATA_FORM, + ) + except exceptions.ResourceUnauthorized: + failed_checks.append(entity) + + if failed_checks: + raise exceptions.UnauthorizedOperation( + action=ENFORCE_METADATA_FORM, + message=f'User {context.username} is not allowed to enforce metadata forms on resource {entityUri}', + ) + return True diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py b/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py new file mode 100644 index 000000000..6bff3ca1e --- /dev/null +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py @@ -0,0 +1,174 @@ +from dataall.base.context import get_context +from dataall.base.db import exceptions +from dataall.core.environment.db.environment_repositories import EnvironmentRepository +from dataall.core.organizations.db.organization_repositories import OrganizationRepository +from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService +from dataall.modules.datasets_base.db.dataset_repositories import DatasetBaseRepository, DatasetListRepository +from dataall.modules.metadata_forms.db.enums import MetadataFormEnforcementScope, MetadataFormEntityTypes +from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository +from dataall.modules.metadata_forms.services.metadata_form_access_service import MetadataFormAccessService +from dataall.modules.metadata_forms.services.metadata_form_permissions import ( + MANAGE_METADATA_FORMS, + ENFORCE_METADATA_FORM, +) +from dataall.modules.notifications.db.notification_repositories import NotificationRepository + + +class MetadataFormEnforcementRequestValidationService: + @staticmethod + def validate_create_request(data): + if 'level' not in data: + raise exceptions.RequiredParameter('level') + + if data.get('level') != MetadataFormEnforcementScope.Global.value: + if 'homeEntity' not in data: + raise exceptions.RequiredParameter('homeEntity') + + if 'entityTypes' not in data: + raise exceptions.RequiredParameter('entityTypes') + + +class MetadataFormEnforcementService: + @staticmethod + def _get_entity_uri(session, data): + return data.get('homeEntity') + + @staticmethod + @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS) + @MetadataFormAccessService.can_perform(ENFORCE_METADATA_FORM) + def create_mf_enforcement_rule(uri, data): + MetadataFormEnforcementRequestValidationService.validate_create_request(data) + MetadataFormAccessService.check_enforcement_access(data.get('homeEntity'), data.get('level')) + with get_context().db_engine.scoped_session() as session: + mf = MetadataFormRepository.get_metadata_form(session, uri) + version = data.get('version') or MetadataFormRepository.get_metadata_form_version_number_latest( + session, uri + ) + rule = MetadataFormRepository.create_mf_enforcement_rule(session, uri, data, version) + + affected_entities = MetadataFormEnforcementService.get_affected_entities(rule.uri, rule=rule) + for entity in affected_entities: + if entity['owner']: + NotificationRepository.create_notification( + session, + recipient=entity['owner'], + target_uri=f'{entity["uri"]}|{entity["type"]}', + message=f'Usage of metadata form "{mf.name}" was enforced for {entity["uri"]} {entity["type"]}', + notification_type='METADATA_FORM_ENFORCED', + ) + + return rule + + @staticmethod + def get_affected_organizations(uri, rule=None): + with get_context().db_engine.scoped_session() as session: + if not rule: + rule = MetadataFormRepository.get_mf_enforcement_rule_by_uri(session, uri) + if rule.level == MetadataFormEnforcementScope.Global.value: + return OrganizationRepository.query_all_active_organizations(session).all() + if rule.level == MetadataFormEnforcementScope.Organization.value: + return [OrganizationRepository.get_organization_by_uri(session, rule.homeEntity)] + return [] + + @staticmethod + def get_affected_environments(uri, rule=None): + with get_context().db_engine.scoped_session() as session: + if not rule: + rule = MetadataFormRepository.get_mf_enforcement_rule_by_uri(session, uri) + if rule.level == MetadataFormEnforcementScope.Global.value: + return EnvironmentRepository.query_all_active_environments(session) + if rule.level == MetadataFormEnforcementScope.Organization.value: + return EnvironmentRepository.get_all_envs_by_organization(session, rule.homeEntity) + if rule.level == MetadataFormEnforcementScope.Environment.value: + return [EnvironmentRepository.get_environment_by_uri(session, rule.homeEntity)] + return [] + + @staticmethod + def get_affected_datasets(uri, rule=None): + with get_context().db_engine.scoped_session() as session: + if not rule: + rule = MetadataFormRepository.get_mf_enforcement_rule_by_uri(session, uri) + if rule.level == MetadataFormEnforcementScope.Global.value: + return DatasetListRepository.query_datasets(session).all() + if rule.level == MetadataFormEnforcementScope.Organization.value: + return DatasetListRepository.query_datasets(session, organizationUri=rule.homeEntity).all() + if rule.level == MetadataFormEnforcementScope.Environment.value: + return DatasetListRepository.query_datasets(session, environmentUri=rule.homeEntity).all() + if rule.level == MetadataFormEnforcementScope.Dataset.value: + return [DatasetBaseRepository.get_dataset_by_uri(session, rule.homeEntity)] + return [] + + @staticmethod + def form_affected_entity_object(uri, owner, type, rule): + with get_context().db_engine.scoped_session() as session: + attached = MetadataFormRepository.query_all_attached_metadata_forms_for_entity( + session, + entityUri=uri, + metadataFormUri=rule.metadataFormUri, + version=rule.version, + ) + return {'type': type, 'uri': uri, 'owner': owner, 'attached': attached.first()} + + @staticmethod + def get_affected_entities(uri, rule=None): + affected_entities = [] + with get_context().db_engine.scoped_session() as session: + if not rule: + rule = MetadataFormRepository.get_mf_enforcement_rule_by_uri(session, uri) + + orgs = MetadataFormEnforcementService.get_affected_organizations(uri, rule) + affected_entities.extend( + [ + MetadataFormEnforcementService.form_affected_entity_object( + o.organizationUri, o.SamlGroupName, MetadataFormEntityTypes.Organizations.value, rule + ) + for o in orgs + ] + ) + + envs = MetadataFormEnforcementService.get_affected_environments(uri, rule) + affected_entities.extend( + [ + MetadataFormEnforcementService.form_affected_entity_object( + e.environmentUri, e.SamlGroupName, MetadataFormEntityTypes.Environments.value, rule + ) + for e in envs + ] + ) + + datasets = MetadataFormEnforcementService.get_affected_datasets(uri, rule) + affected_entities.extend( + [ + MetadataFormEnforcementService.form_affected_entity_object( + ds.datasetUri, ds.SamlAdminGroupName, ds.datasetType.value + '-Dataset', rule + ) + for ds in datasets + ] + ) + + entity_types = set(rule.entityTypes[:]) - { + MetadataFormEntityTypes.Organizations.value, + MetadataFormEntityTypes.Environments.value, + MetadataFormEntityTypes.RDDatasets.value, + MetadataFormEntityTypes.S3Datasets.value, + } + + for entity_type in entity_types: + entity_class, level, get_uri_and_owner = MetadataFormEntityTypes.get_entity_class(entity_type) + all_entities = session.query(entity_class) + if level == MetadataFormEnforcementScope.Organization.value: + all_entities = all_entities.filter(entity_class.organizationUri.in_([org.uri for org in orgs])) + if level == MetadataFormEnforcementScope.Environment.value: + all_entities = all_entities.filter(entity_class.environmentUri.in_([env.uri for env in envs])) + if level == MetadataFormEnforcementScope.Dataset.value: + all_entities = all_entities.filter(entity_class.datasetUri.in_([ds.uri for ds in datasets])) + all_entities = all_entities.all() + affected_entities.extend( + [ + MetadataFormEnforcementService.form_affected_entity_object( + *get_uri_and_owner(e), entity_type, rule + ) + for e in all_entities + ] + ) + return affected_entities From 2d5638bcc2422f3ef0777b7a167ada35e48f53fd Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Wed, 30 Oct 2024 14:09:23 +0000 Subject: [PATCH 03/16] create enforcement rule backend --- .../modules/metadata_forms/api/input_types.py | 14 ++++++- .../modules/metadata_forms/api/mutations.py | 10 +++++ .../modules/metadata_forms/api/resolvers.py | 8 +++- .../modules/metadata_forms/api/types.py | 13 ++++++- .../services/metadata_form_access_service.py | 39 +++++-------------- .../metadata_form_enforcement_service.py | 16 +++++++- 6 files changed, 66 insertions(+), 34 deletions(-) diff --git a/backend/dataall/modules/metadata_forms/api/input_types.py b/backend/dataall/modules/metadata_forms/api/input_types.py index ce876a4fb..090f531e2 100644 --- a/backend/dataall/modules/metadata_forms/api/input_types.py +++ b/backend/dataall/modules/metadata_forms/api/input_types.py @@ -40,7 +40,6 @@ ], ) - MetadataFormFilter = gql.InputType( name='MetadataFormFilter', arguments=[ @@ -65,7 +64,6 @@ ], ) - NewAttachedMetadataFormInput = gql.InputType( name='NewAttachedMetadataFormInput', arguments=[ @@ -83,3 +81,15 @@ gql.Field(name='value', type=gql.String), ], ) + +NewMetadataFormEnforcementInput = gql.InputType( + name='NewMetadataFormEnforcementInput', + arguments=[ + gql.Field(name='metadataFormUri', type=gql.NonNullableType(gql.String)), + gql.Field(name='level', type=gql.NonNullableType(gql.String)), + gql.Field(name='homeEntity', type=gql.String), + gql.Field(name='version', type=gql.String), + gql.Field(name='severity', type=gql.String), + gql.Field(name='entityTypes', type=gql.ArrayType(gql.String)), + ], +) diff --git a/backend/dataall/modules/metadata_forms/api/mutations.py b/backend/dataall/modules/metadata_forms/api/mutations.py index 4473d0716..a795325ac 100644 --- a/backend/dataall/modules/metadata_forms/api/mutations.py +++ b/backend/dataall/modules/metadata_forms/api/mutations.py @@ -9,6 +9,7 @@ delete_attached_metadata_form, create_metadata_form_version, delete_metadata_form_version, + create_mf_enforcement_rule, ) createMetadataForm = gql.MutationField( @@ -105,3 +106,12 @@ resolver=batch_metadata_form_field_update, test_scope='MetadataForm', ) + + +createMetadataFormEnforcementRule = gql.MutationField( + name='createMetadataForm', + args=[gql.Argument(name='input', type=gql.NonNullableType(gql.Ref('NewMetadataFormEnforcementInput')))], + type=gql.Ref('MetadataFormEnforcementRule'), + resolver=create_mf_enforcement_rule, + test_scope='MetadataForm', +) diff --git a/backend/dataall/modules/metadata_forms/api/resolvers.py b/backend/dataall/modules/metadata_forms/api/resolvers.py index d1a83f204..774471e4b 100644 --- a/backend/dataall/modules/metadata_forms/api/resolvers.py +++ b/backend/dataall/modules/metadata_forms/api/resolvers.py @@ -6,8 +6,10 @@ MetadataFormField, AttachedMetadataForm, AttachedMetadataFormField, + MetadataFormEnforcementRule, ) from dataall.modules.metadata_forms.services.attached_metadata_form_service import AttachedMetadataFormService +from dataall.modules.metadata_forms.services.metadata_form_enforcement_service import MetadataFormEnforcementService from dataall.modules.metadata_forms.services.metadata_form_permissions import MANAGE_METADATA_FORMS from dataall.modules.metadata_forms.services.metadata_form_service import MetadataFormService, MetadataFormAccessService @@ -56,7 +58,7 @@ def get_metadata_form(context: Context, source, uri): return MetadataFormService.get_metadata_form_by_uri(uri=uri) -def resolve_metadata_form(context: Context, source: AttachedMetadataForm): +def resolve_metadata_form(context: Context, source: AttachedMetadataForm | MetadataFormEnforcementRule): return MetadataFormService.get_metadata_form_by_uri(source.metadataFormUri) @@ -114,3 +116,7 @@ def get_entity_metadata_form_permissions(context: Context, source, entityUri): def list_metadata_form_versions(context: Context, source, uri): return MetadataFormService.list_metadata_form_versions(uri=uri) + + +def create_mf_enforcement_rule(context: Context, source, input): + return MetadataFormEnforcementService.create_mf_enforcement_rule(uri=input.get('metadataFormUri'), data=input) diff --git a/backend/dataall/modules/metadata_forms/api/types.py b/backend/dataall/modules/metadata_forms/api/types.py index e4eb82e0a..aac23cb5e 100644 --- a/backend/dataall/modules/metadata_forms/api/types.py +++ b/backend/dataall/modules/metadata_forms/api/types.py @@ -35,7 +35,6 @@ ], ) - MetadataFormField = gql.ObjectType( name='MetadataFormField', fields=[ @@ -116,3 +115,15 @@ gql.Field(name='hasTenantPermissions', type=gql.Boolean, resolver=has_tenant_permissions_for_metadata_forms), ], ) + +MetadataFormEnforcementRule = gql.ObjectType( + name='MetadataFormEnforcementRule', + fields=[ + gql.Field(name='uri', type=gql.String), + gql.Field(name='level', type=gql.String), + gql.Field(name='homeEntity', type=gql.String), + gql.Field(name='homeEntityName', type=gql.String, resolver=get_home_entity_name), + gql.Field(name='metadataForm', type=gql.Ref('MetadataForm'), resolver=resolve_metadata_form), + gql.Field(name='entityTypes', type=gql.ArrayType(gql.String)), + ], +) diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_access_service.py b/backend/dataall/modules/metadata_forms/services/metadata_form_access_service.py index 7e07bdee7..1c1a66a96 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_access_service.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_access_service.py @@ -108,40 +108,21 @@ def get_target_orgs_and_envs(username, groups, is_da_admin=False, filter={}): @staticmethod def check_enforcement_access(entityUri, level): context = get_context() - if TenantPolicyValidationService.is_tenant_admin(context.groups): - return True + is_admin = TenantPolicyValidationService.is_tenant_admin(context.groups) if level == MetadataFormEnforcementScope.Global.value: + if is_admin: + return True raise exceptions.UnauthorizedOperation( action=ENFORCE_METADATA_FORM, message='Only data.all admins can enforce metadata forms on global level' ) with context.db_engine.scoped_session() as session: - entities_to_check = [entityUri] - if level == MetadataFormEnforcementScope.Environment.value: - env = EnvironmentRepository.get_environment_by_uri(session, entityUri) - entities_to_check.append(env.organizationUri) - if level == MetadataFormEnforcementScope.Dataset.value: - dataset = DatasetBaseRepository.get_dataset_by_uri(session, entityUri) - entities_to_check.append(dataset.organizationUri) - entities_to_check.append(dataset.environmentUri) - - failed_checks = [] - for entity in entities_to_check: - try: - ResourcePolicyService.check_user_resource_permission( - session=session, - username=context.username, - groups=context.groups, - resource_uri=entity, - permission_name=ENFORCE_METADATA_FORM, - ) - except exceptions.ResourceUnauthorized: - failed_checks.append(entity) - - if failed_checks: - raise exceptions.UnauthorizedOperation( - action=ENFORCE_METADATA_FORM, - message=f'User {context.username} is not allowed to enforce metadata forms on resource {entityUri}', - ) + ResourcePolicyService.check_user_resource_permission( + session=session, + username=context.username, + groups=context.groups, + resource_uri=entityUri, + permission_name=ENFORCE_METADATA_FORM, + ) return True diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py b/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py index 6bff3ca1e..df5890a09 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py @@ -4,7 +4,11 @@ from dataall.core.organizations.db.organization_repositories import OrganizationRepository from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService from dataall.modules.datasets_base.db.dataset_repositories import DatasetBaseRepository, DatasetListRepository -from dataall.modules.metadata_forms.db.enums import MetadataFormEnforcementScope, MetadataFormEntityTypes +from dataall.modules.metadata_forms.db.enums import ( + MetadataFormEnforcementScope, + MetadataFormEntityTypes, + MetadataFormEnforcementSeverity, +) from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository from dataall.modules.metadata_forms.services.metadata_form_access_service import MetadataFormAccessService from dataall.modules.metadata_forms.services.metadata_form_permissions import ( @@ -17,9 +21,15 @@ class MetadataFormEnforcementRequestValidationService: @staticmethod def validate_create_request(data): + if 'metadataFormUri' not in data: + raise exceptions.RequiredParameter('metadataFormUri') + if 'level' not in data: raise exceptions.RequiredParameter('level') + if 'severity' not in data: + raise exceptions.RequiredParameter('severity') + if data.get('level') != MetadataFormEnforcementScope.Global.value: if 'homeEntity' not in data: raise exceptions.RequiredParameter('homeEntity') @@ -27,6 +37,10 @@ def validate_create_request(data): if 'entityTypes' not in data: raise exceptions.RequiredParameter('entityTypes') + # check that values are valid for the enums + MetadataFormEnforcementScope(data.get('level')) + MetadataFormEnforcementSeverity(data.get('severity')) + class MetadataFormEnforcementService: @staticmethod From c21382f108cc861620530097585e2e4e754ba49b Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Thu, 31 Oct 2024 13:05:10 +0000 Subject: [PATCH 04/16] create enforcement rule frontend for fixed Org --- .../modules/metadata_forms/api/input_types.py | 2 +- .../modules/metadata_forms/api/mutations.py | 2 +- .../modules/metadata_forms/api/queries.py | 8 + .../modules/metadata_forms/api/resolvers.py | 13 +- .../modules/metadata_forms/api/types.py | 3 +- .../db/metadata_form_repository.py | 4 + .../metadata_form_enforcement_service.py | 5 + .../components/MetadataFormEnforcement.js | 350 +++++++++++++++++- .../components/metadataAttachment.js | 6 +- .../createMetadataFormEnforcementRule.js | 20 + .../modules/Metadata_Forms/services/index.js | 2 + .../listMetadataFormEnforcementRules.js | 20 + .../Metadata_Forms/views/MetadataFormView.js | 4 +- 13 files changed, 415 insertions(+), 24 deletions(-) create mode 100644 frontend/src/modules/Metadata_Forms/services/createMetadataFormEnforcementRule.js create mode 100644 frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRules.js diff --git a/backend/dataall/modules/metadata_forms/api/input_types.py b/backend/dataall/modules/metadata_forms/api/input_types.py index 090f531e2..b8181894f 100644 --- a/backend/dataall/modules/metadata_forms/api/input_types.py +++ b/backend/dataall/modules/metadata_forms/api/input_types.py @@ -88,7 +88,7 @@ gql.Field(name='metadataFormUri', type=gql.NonNullableType(gql.String)), gql.Field(name='level', type=gql.NonNullableType(gql.String)), gql.Field(name='homeEntity', type=gql.String), - gql.Field(name='version', type=gql.String), + gql.Field(name='version', type=gql.Integer), gql.Field(name='severity', type=gql.String), gql.Field(name='entityTypes', type=gql.ArrayType(gql.String)), ], diff --git a/backend/dataall/modules/metadata_forms/api/mutations.py b/backend/dataall/modules/metadata_forms/api/mutations.py index a795325ac..9aa3696a2 100644 --- a/backend/dataall/modules/metadata_forms/api/mutations.py +++ b/backend/dataall/modules/metadata_forms/api/mutations.py @@ -109,7 +109,7 @@ createMetadataFormEnforcementRule = gql.MutationField( - name='createMetadataForm', + name='createMetadataFormEnforcementRule', args=[gql.Argument(name='input', type=gql.NonNullableType(gql.Ref('NewMetadataFormEnforcementInput')))], type=gql.Ref('MetadataFormEnforcementRule'), resolver=create_mf_enforcement_rule, diff --git a/backend/dataall/modules/metadata_forms/api/queries.py b/backend/dataall/modules/metadata_forms/api/queries.py index 072859b36..4ff69b20f 100644 --- a/backend/dataall/modules/metadata_forms/api/queries.py +++ b/backend/dataall/modules/metadata_forms/api/queries.py @@ -7,6 +7,7 @@ list_attached_forms, get_entity_metadata_form_permissions, list_metadata_form_versions, + list_mf_enforcement_rules ) listUserMetadataForms = gql.QueryField( @@ -49,6 +50,13 @@ test_scope='MetadataForm', ) +listMetadataFormEnforcementRules = gql.QueryField( + name='listMetadataFormEnforcementRules', + args=[gql.Argument('uri', gql.NonNullableType(gql.String))], + type=gql.Ref('MetadataFormEnforcementRule'), + resolver=list_mf_enforcement_rules, + test_scope='MetadataForm', +) getAttachedMetadataForm = gql.QueryField( name='getAttachedMetadataForm', diff --git a/backend/dataall/modules/metadata_forms/api/resolvers.py b/backend/dataall/modules/metadata_forms/api/resolvers.py index 774471e4b..0a6e2735c 100644 --- a/backend/dataall/modules/metadata_forms/api/resolvers.py +++ b/backend/dataall/modules/metadata_forms/api/resolvers.py @@ -6,13 +6,14 @@ MetadataFormField, AttachedMetadataForm, AttachedMetadataFormField, - MetadataFormEnforcementRule, ) from dataall.modules.metadata_forms.services.attached_metadata_form_service import AttachedMetadataFormService from dataall.modules.metadata_forms.services.metadata_form_enforcement_service import MetadataFormEnforcementService from dataall.modules.metadata_forms.services.metadata_form_permissions import MANAGE_METADATA_FORMS from dataall.modules.metadata_forms.services.metadata_form_service import MetadataFormService, MetadataFormAccessService +from typing import Protocol + def create_metadata_form(context: Context, source, input): return MetadataFormService.create_metadata_form(data=input) @@ -58,7 +59,11 @@ def get_metadata_form(context: Context, source, uri): return MetadataFormService.get_metadata_form_by_uri(uri=uri) -def resolve_metadata_form(context: Context, source: AttachedMetadataForm | MetadataFormEnforcementRule): +class HasMetadataFormUri(Protocol): + metadataFormUri: str + + +def resolve_metadata_form(context: Context, source: HasMetadataFormUri): return MetadataFormService.get_metadata_form_by_uri(source.metadataFormUri) @@ -120,3 +125,7 @@ def list_metadata_form_versions(context: Context, source, uri): def create_mf_enforcement_rule(context: Context, source, input): return MetadataFormEnforcementService.create_mf_enforcement_rule(uri=input.get('metadataFormUri'), data=input) + + +def list_mf_enforcement_rules(context: Context, source, uri): + return MetadataFormEnforcementService.list_mf_enforcement_rules(uri=uri) \ No newline at end of file diff --git a/backend/dataall/modules/metadata_forms/api/types.py b/backend/dataall/modules/metadata_forms/api/types.py index aac23cb5e..8aef053d5 100644 --- a/backend/dataall/modules/metadata_forms/api/types.py +++ b/backend/dataall/modules/metadata_forms/api/types.py @@ -123,7 +123,8 @@ gql.Field(name='level', type=gql.String), gql.Field(name='homeEntity', type=gql.String), gql.Field(name='homeEntityName', type=gql.String, resolver=get_home_entity_name), - gql.Field(name='metadataForm', type=gql.Ref('MetadataForm'), resolver=resolve_metadata_form), + gql.Field(name='version', type=gql.Integer,), + gql.Field(name='metadataFormUri', type=gql.String,), gql.Field(name='entityTypes', type=gql.ArrayType(gql.String)), ], ) diff --git a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py index 363bcf8c5..72ff9664c 100644 --- a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py +++ b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py @@ -359,3 +359,7 @@ def create_mf_enforcement_rule(session, uri, data, version): @staticmethod def get_mf_enforcement_rule_by_uri(session, uri): return session.query(MetadataFormEnforcementRule).get(uri) + + @staticmethod + def list_mf_enforcement_rules(session, uri): + return session.query(MetadataFormEnforcementRule).filter(MetadataFormEnforcementRule.metadataFormUri == uri).all() diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py b/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py index df5890a09..8e8d3cde6 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py @@ -186,3 +186,8 @@ def get_affected_entities(uri, rule=None): ] ) return affected_entities + + @staticmethod + def list_mf_enforcement_rules(uri): + with get_context().db_engine.scoped_session() as session: + return MetadataFormRepository.list_mf_enforcement_rules(session, uri) diff --git a/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js b/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js index 783806503..3c1bbde6a 100644 --- a/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js +++ b/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js @@ -1,21 +1,343 @@ import PropTypes from 'prop-types'; -import EngineeringOutlinedIcon from '@mui/icons-material/EngineeringOutlined'; -import { Box, Typography } from '@mui/material'; +import { + Autocomplete, + Box, + Button, + Card, + CardContent, + CardHeader, Checkbox, + Dialog, FormControlLabel, + Grid, + TextField, + Typography +} from '@mui/material'; +import { PlusIcon } from 'design'; +import React, { useEffect, useState } from 'react'; +import DeleteIcon from '@mui/icons-material/DeleteOutlined'; +import DoNotDisturbAltOutlinedIcon from '@mui/icons-material/DoNotDisturbAltOutlined'; +import { fetchEnums, useClient } from 'services'; +import { useDispatch } from 'react-redux'; +import { SET_ERROR } from 'globalErrors'; +import { createMetadataFormEnforcementRule, listMetadataFormEnforcementRules } from '../services'; +import * as Yup from 'yup'; +import { Formik } from 'formik'; +import FormControl from '@mui/material/FormControl'; +import { LoadingButton } from '@mui/lab'; +import SendIcon from '@mui/icons-material/Send'; + + +const CreateEnforcementRuleModal = (props) => { + const { onClose, open, metadataForm,severityOptions,entityTypesOptions,enforcementScopeOptions, ...other } = props; + + const client = useClient(); + + const enforcementScopeDict = {} + for (const option of enforcementScopeOptions) { + enforcementScopeDict[option.name] = option.value; + } + + async function submit(values, setStatus, setSubmitting, setErrors) { + const input = { + metadataFormUri: metadataForm.uri, + version: values.version, + level: values.scope, + severity: values.severity, + homeEntity: '9hj226qv', + entityTypes: values.entityTypes + } + const response = await client.mutate(createMetadataFormEnforcementRule(input)); + if (response.errors) { + setStatus({ success: false }); + setErrors({ submit: response.errors[0].message }); + setSubmitting(false); + } else { + setStatus({ success: true }); + setSubmitting(false); + props.refetch(); + onClose(); + } + } + + return ( + + + + Enforce {metadataForm.name} + + { + await submit(values, setStatus, setSubmitting, setErrors); + }}> + {({ + errors, + handleBlur, + handleChange, + handleSubmit, + isSubmitting, + setFieldValue, + touched, + values + }) => ( +
+ + + { + return { + label: 'version ' + option, + value: option + }; + })} + onChange={(event, value) => { + setFieldValue('version', value.value); + }} + defaultValue={'version ' + metadataForm.versions[0]} + renderInput={(params) => ( + + )} + /> + + + { + return { + label: option.name, + value: option.value, + }; + })} + defaultValue={enforcementScopeDict['Global']} + onChange={(event, value) => { + setFieldValue('scope', value.value); + }} + renderInput={(params) => ( + + )} + /> + + + { + return { + label: option.name, + value: option.value + }; + })} + onChange={(event, value) => { + setFieldValue('severity', value.value); + }} + defaultValue={severityOptions[0].value} + renderInput={(params) => ( + + )} + /> + + + + + {entityTypesOptions.map((entityType) => ( + + { + if(value){ + + setFieldValue('entityTypes', [...values.entityTypes, entityType.value]); + } + else{ + setFieldValue('entityTypes', values.entityTypes.filter((item) => item !== entityType.value)); + } + }}/>} label={entityType.value} /> + + ))} + + + + } + color="primary" + disabled={isSubmitting} + type="submit" + variant="contained" + > + Create + + + + + +
+ )} + + + + ); +}; export const MetadataFormEnforcement = (props) => { + const {canEdit, metadataForm} = props; + const client = useClient(); + const dispatch = useDispatch(); + const [showCreateRuleModal, setShowCreateRuleModal] = useState(false); + const [rules, setRules] = useState([]); + const [severityOptions, setSeverityOptions] = useState({}); + const [entityTypesOptions, setEntityTypesOptions] = useState({}) + const [enforcementScopeOptions, setEnforcementScopeOptions] = useState({}) + + + const fetchEnforcementRules = async () => { + const response = await client.query(listMetadataFormEnforcementRules(metadataForm.uri)); + if (!response.errors && response.data && response.data.listMetadataFormEnforcementRules) { + setRules(response.data.listMetadataFormEnforcementRules); + } + }; + + const fetchEnforcementEnums = async () => { + const enums = await fetchEnums(client, [ + 'MetadataFormEntityTypes', + 'MetadataFormEnforcementSeverity', + 'MetadataFormEnforcementScope' + ]); + if (enums['MetadataFormEntityTypes'].length > 0) { + setEntityTypesOptions(enums['MetadataFormEntityTypes']); + } else { + const error = 'Could not fetch entity type options'; + dispatch({ type: SET_ERROR, error }); + } + if (enums['MetadataFormEnforcementSeverity'].length > 0) { + setSeverityOptions(enums['MetadataFormEnforcementSeverity']); + } else { + const error = 'Could not fetch enforcement severity options'; + dispatch({ type: SET_ERROR, error }); + } + if (enums['MetadataFormEnforcementScope'].length > 0) { + setEnforcementScopeOptions(enums['MetadataFormEnforcementScope']); + } else { + const error = 'Could not fetch enforcement scope options'; + dispatch({ type: SET_ERROR, error }); + } + }; + + useEffect(() => { + if (client) { + fetchEnforcementRules().catch((e) => dispatch({ type: SET_ERROR, error: e.message })); + fetchEnforcementEnums().catch((e) => dispatch({ type: SET_ERROR, error: e.message })); + } + }, [client]); + return ( - - - - This tab is under construction. - + + + + + + + + + + {canEdit && ( + + + + )} + + + {rules.length > 0 ? ( + rules.map((rule) => ( + + + rule.uri + + + )) + ) : ( + + + + Metadata Form is not enforced + + + )} + + + + {showCreateRuleModal && ( + setShowCreateRuleModal(false)} + /> + )} ); }; diff --git a/frontend/src/modules/Metadata_Forms/components/metadataAttachment.js b/frontend/src/modules/Metadata_Forms/components/metadataAttachment.js index 3e06911e1..5f563b4dc 100644 --- a/frontend/src/modules/Metadata_Forms/components/metadataAttachment.js +++ b/frontend/src/modules/Metadata_Forms/components/metadataAttachment.js @@ -21,11 +21,11 @@ import { listAttachedMetadataForms, listEntityMetadataForms } from '../services'; -import { Defaults, PlusIcon } from '../../../design'; +import { Defaults, PlusIcon } from 'design'; import CircularProgress from '@mui/material/CircularProgress'; -import { useClient } from '../../../services'; +import { useClient } from 'services'; import { RenderedMetadataForm } from './renderedMetadataForm'; -import { SET_ERROR } from '../../../globalErrors'; +import { SET_ERROR } from 'globalErrors'; import { AttachedFormCard } from './AttachedFormCard'; import DoNotDisturbAltOutlinedIcon from '@mui/icons-material/DoNotDisturbAltOutlined'; import DeleteIcon from '@mui/icons-material/DeleteOutlined'; diff --git a/frontend/src/modules/Metadata_Forms/services/createMetadataFormEnforcementRule.js b/frontend/src/modules/Metadata_Forms/services/createMetadataFormEnforcementRule.js new file mode 100644 index 000000000..62b679b91 --- /dev/null +++ b/frontend/src/modules/Metadata_Forms/services/createMetadataFormEnforcementRule.js @@ -0,0 +1,20 @@ +import { gql } from 'apollo-boost'; + +export const createMetadataFormEnforcementRule = (input) => ({ + variables: { + input + }, + mutation: gql` + mutation createMetadataFormEnforcementRule($input: NewMetadataFormEnforcementInput!) { + createMetadataFormEnforcementRule(input: $input) { + uri + level + homeEntity + homeEntityName + entityTypes + metadataFormUri + version + } + } + ` +}); diff --git a/frontend/src/modules/Metadata_Forms/services/index.js b/frontend/src/modules/Metadata_Forms/services/index.js index 772ab4727..904310db7 100644 --- a/frontend/src/modules/Metadata_Forms/services/index.js +++ b/frontend/src/modules/Metadata_Forms/services/index.js @@ -12,3 +12,5 @@ export * from './getMetadataFormEntityPermissions'; export * from './createMetadataFormVersion'; export * from './deleteMetadataFormVersion'; export * from './listMetadataFormVersions'; +export * from './createMetadataFormEnforcementRule'; +export * from './listMetadataFormEnforcementRules'; diff --git a/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRules.js b/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRules.js new file mode 100644 index 000000000..b86a1cd05 --- /dev/null +++ b/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRules.js @@ -0,0 +1,20 @@ +import { gql } from 'apollo-boost'; + +export const listMetadataFormEnforcementRules = (uri) => ({ + variables: { + uri + }, + query: gql` + query listMetadataFormEnforcementRules($uri: String!) { + listMetadataFormEnforcementRules(uri: $uri) { + uri + level + homeEntity + homeEntityName + entityTypes + metadataFormUri + version + } + } + ` +}); diff --git a/frontend/src/modules/Metadata_Forms/views/MetadataFormView.js b/frontend/src/modules/Metadata_Forms/views/MetadataFormView.js index cdea81bc7..31651f1fd 100644 --- a/frontend/src/modules/Metadata_Forms/views/MetadataFormView.js +++ b/frontend/src/modules/Metadata_Forms/views/MetadataFormView.js @@ -38,7 +38,7 @@ const MetadataFormView = () => { const tabs = [ { label: 'Form Info', value: 'info' }, { label: 'Fields', value: 'fields' }, - // { label: 'Enforcement', value: 'enforcement' }, + { label: 'Enforcement', value: 'enforcement' }, { label: 'Preview', value: 'preview' } ]; const [metadataForm, setMetadataForm] = useState(null); @@ -251,7 +251,7 @@ const MetadataFormView = () => { /> )} {currentTab === 'enforcement' && ( - + )} {currentTab === 'preview' && ( From 93660346159b56da42940865dcc85739a2474664 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Thu, 31 Oct 2024 14:10:21 +0000 Subject: [PATCH 05/16] create MF enforcement rule. Notify owners. Fe + Be --- .../modules/metadata_forms/api/queries.py | 4 +- .../modules/metadata_forms/api/resolvers.py | 2 +- .../modules/metadata_forms/api/types.py | 11 +- .../db/metadata_form_repository.py | 6 +- ...9ab_backfill_mf_enforcement_permissions.py | 8 + .../components/MetadataFormEnforcement.js | 358 ++++++++++++++---- .../createMetadataFormEnforcementRule.js | 4 +- .../listMetadataFormEnforcementRules.js | 1 - .../Metadata_Forms/views/MetadataFormView.js | 5 +- 9 files changed, 311 insertions(+), 88 deletions(-) diff --git a/backend/dataall/modules/metadata_forms/api/queries.py b/backend/dataall/modules/metadata_forms/api/queries.py index 4ff69b20f..34836e41a 100644 --- a/backend/dataall/modules/metadata_forms/api/queries.py +++ b/backend/dataall/modules/metadata_forms/api/queries.py @@ -7,7 +7,7 @@ list_attached_forms, get_entity_metadata_form_permissions, list_metadata_form_versions, - list_mf_enforcement_rules + list_mf_enforcement_rules, ) listUserMetadataForms = gql.QueryField( @@ -53,7 +53,7 @@ listMetadataFormEnforcementRules = gql.QueryField( name='listMetadataFormEnforcementRules', args=[gql.Argument('uri', gql.NonNullableType(gql.String))], - type=gql.Ref('MetadataFormEnforcementRule'), + type=gql.ArrayType(gql.Ref('MetadataFormEnforcementRule')), resolver=list_mf_enforcement_rules, test_scope='MetadataForm', ) diff --git a/backend/dataall/modules/metadata_forms/api/resolvers.py b/backend/dataall/modules/metadata_forms/api/resolvers.py index 0a6e2735c..9bf291b10 100644 --- a/backend/dataall/modules/metadata_forms/api/resolvers.py +++ b/backend/dataall/modules/metadata_forms/api/resolvers.py @@ -128,4 +128,4 @@ def create_mf_enforcement_rule(context: Context, source, input): def list_mf_enforcement_rules(context: Context, source, uri): - return MetadataFormEnforcementService.list_mf_enforcement_rules(uri=uri) \ No newline at end of file + return MetadataFormEnforcementService.list_mf_enforcement_rules(uri=uri) diff --git a/backend/dataall/modules/metadata_forms/api/types.py b/backend/dataall/modules/metadata_forms/api/types.py index 8aef053d5..edb52ca15 100644 --- a/backend/dataall/modules/metadata_forms/api/types.py +++ b/backend/dataall/modules/metadata_forms/api/types.py @@ -122,9 +122,14 @@ gql.Field(name='uri', type=gql.String), gql.Field(name='level', type=gql.String), gql.Field(name='homeEntity', type=gql.String), - gql.Field(name='homeEntityName', type=gql.String, resolver=get_home_entity_name), - gql.Field(name='version', type=gql.Integer,), - gql.Field(name='metadataFormUri', type=gql.String,), + gql.Field( + name='version', + type=gql.Integer, + ), + gql.Field( + name='metadataFormUri', + type=gql.String, + ), gql.Field(name='entityTypes', type=gql.ArrayType(gql.String)), ], ) diff --git a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py index 72ff9664c..a0b5c208e 100644 --- a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py +++ b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py @@ -354,6 +354,8 @@ def create_mf_enforcement_rule(session, uri, data, version): entityTypes=data.get('entityTypes'), severity=data.get('severity', MetadataFormEnforcementSeverity.Recommended.value), ) + session.add(rule) + session.commit() return rule @staticmethod @@ -362,4 +364,6 @@ def get_mf_enforcement_rule_by_uri(session, uri): @staticmethod def list_mf_enforcement_rules(session, uri): - return session.query(MetadataFormEnforcementRule).filter(MetadataFormEnforcementRule.metadataFormUri == uri).all() + return ( + session.query(MetadataFormEnforcementRule).filter(MetadataFormEnforcementRule.metadataFormUri == uri).all() + ) diff --git a/backend/migrations/versions/ba2da94739ab_backfill_mf_enforcement_permissions.py b/backend/migrations/versions/ba2da94739ab_backfill_mf_enforcement_permissions.py index cd3cade39..521292b1c 100644 --- a/backend/migrations/versions/ba2da94739ab_backfill_mf_enforcement_permissions.py +++ b/backend/migrations/versions/ba2da94739ab_backfill_mf_enforcement_permissions.py @@ -7,6 +7,7 @@ """ from alembic import op +import sqlalchemy as sa from sqlalchemy import orm from dataall.core.environment.db.environment_models import Environment @@ -32,6 +33,11 @@ def get_session(): def upgrade(): + op.add_column('metadata_form_enforcement_rule', sa.Column('homeEntity', sa.String(), nullable=True)) + op.create_foreign_key( + 'fk_enforcement_version', 'metadata_form_version', 'metadata_form', ['metadataFormUri'], ['uri'] + ) + session = get_session() PermissionService.save_permission( @@ -73,6 +79,8 @@ def upgrade(): def downgrade(): + op.drop_constraint('fk_enforcement_version', 'metadata_form_version', type_='foreignkey') + op.drop_column('metadata_form_enforcement_rule', 'homeEntity') bind = op.get_bind() session = orm.Session(bind=bind) all_environments = session.query(Environment).all() diff --git a/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js b/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js index 3c1bbde6a..906fa6857 100644 --- a/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js +++ b/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js @@ -5,47 +5,152 @@ import { Button, Card, CardContent, - CardHeader, Checkbox, - Dialog, FormControlLabel, + CardHeader, + Checkbox, + Dialog, + FormControlLabel, Grid, TextField, Typography } from '@mui/material'; -import { PlusIcon } from 'design'; +import { Defaults, PlusIcon } from 'design'; import React, { useEffect, useState } from 'react'; -import DeleteIcon from '@mui/icons-material/DeleteOutlined'; import DoNotDisturbAltOutlinedIcon from '@mui/icons-material/DoNotDisturbAltOutlined'; -import { fetchEnums, useClient } from 'services'; +import { fetchEnums, listValidEnvironments, useClient } from 'services'; import { useDispatch } from 'react-redux'; import { SET_ERROR } from 'globalErrors'; -import { createMetadataFormEnforcementRule, listMetadataFormEnforcementRules } from '../services'; -import * as Yup from 'yup'; +import { + createMetadataFormEnforcementRule, + listMetadataFormEnforcementRules +} from '../services'; import { Formik } from 'formik'; -import FormControl from '@mui/material/FormControl'; import { LoadingButton } from '@mui/lab'; import SendIcon from '@mui/icons-material/Send'; - +import { listOrganizations } from '../../Organizations/services'; +import { listDatasets } from '../../DatasetsBase/services'; const CreateEnforcementRuleModal = (props) => { - const { onClose, open, metadataForm,severityOptions,entityTypesOptions,enforcementScopeOptions, ...other } = props; + const { + onClose, + open, + metadataForm, + severityOptions, + entityTypesOptions, + enforcementScopeOptions, + ...other + } = props; const client = useClient(); + const dispatch = useDispatch(); - const enforcementScopeDict = {} + const [environmentOptions, setEnvironmentOptions] = useState([]); + const [organizationOptions, setOrganizationOptions] = useState([]); + const [datasetOptions, setDatasetOptions] = useState([]); + + const enforcementScopeDict = {}; for (const option of enforcementScopeOptions) { enforcementScopeDict[option.name] = option.value; } + const fetchOrganizations = async () => { + try { + const response = await client.query( + listOrganizations({ + filter: Defaults.selectListFilter + }) + ); + if (!response.errors) { + setOrganizationOptions( + response.data.listOrganizations.nodes.map((e) => ({ + ...e, + value: e.organizationUri, + label: e.label + })) + ); + } else { + dispatch({ type: SET_ERROR, error: response.errors[0].message }); + } + } catch (e) { + dispatch({ type: SET_ERROR, error: e.message }); + } + }; + const fetchEnvironments = async () => { + try { + const response = await client.query( + listValidEnvironments({ + filter: Defaults.selectListFilter + }) + ); + if (!response.errors) { + setEnvironmentOptions( + response.data.listValidEnvironments.nodes.map((e) => ({ + ...e, + value: e.environmentUri, + label: e.label + })) + ); + } else { + dispatch({ type: SET_ERROR, error: response.errors[0].message }); + } + } catch (e) { + dispatch({ type: SET_ERROR, error: e.message }); + } + }; + + const fetchDatasets = async () => { + try { + const response = await client.query( + listDatasets({ + filter: Defaults.selectListFilter + }) + ); + if (!response.errors) { + setDatasetOptions( + response.data.listDatasets.nodes.map((e) => ({ + ...e, + value: e.datasetUri, + label: e.label + })) + ); + } else { + dispatch({ type: SET_ERROR, error: response.errors[0].message }); + } + } catch (e) { + dispatch({ type: SET_ERROR, error: e.message }); + } + }; + + useEffect(async () => { + if (client) { + await fetchEnvironments(); + await fetchOrganizations(); + await fetchDatasets(); + } + }, [client, open, dispatch]); + async function submit(values, setStatus, setSubmitting, setErrors) { + let homeEntity = null; + if (values.scope === enforcementScopeDict['Organization']) { + homeEntity = values.organization; + } + if (values.scope === enforcementScopeDict['Environment']) { + homeEntity = values.environment; + } + if (values.scope === enforcementScopeDict['Dataset']) { + homeEntity = values.dataset; + } + const input = { metadataFormUri: metadataForm.uri, version: values.version, level: values.scope, severity: values.severity, - homeEntity: '9hj226qv', + homeEntity: homeEntity, entityTypes: values.entityTypes - } - const response = await client.mutate(createMetadataFormEnforcementRule(input)); + }; + const response = await client.mutate( + createMetadataFormEnforcementRule(input) + ); if (response.errors) { setStatus({ success: false }); setErrors({ submit: response.errors[0].message }); @@ -70,25 +175,26 @@ const CreateEnforcementRuleModal = (props) => { Enforce {metadataForm.name} { - await submit(values, setStatus, setSubmitting, setErrors); - }}> + initialValues={{ + version: metadataForm.versions[0], + scope: enforcementScopeDict['Global'], + severity: severityOptions[0].value, + entityTypes: [] + }} + onSubmit={async (values, { setErrors, setStatus, setSubmitting }) => { + await submit(values, setStatus, setSubmitting, setErrors); + }} + > {({ - errors, - handleBlur, - handleChange, - handleSubmit, - isSubmitting, - setFieldValue, - touched, - values - }) => ( + errors, + handleBlur, + handleChange, + handleSubmit, + isSubmitting, + setFieldValue, + touched, + values + }) => (
@@ -125,7 +231,7 @@ const CreateEnforcementRuleModal = (props) => { options={enforcementScopeOptions.map((option) => { return { label: option.name, - value: option.value, + value: option.value }; })} defaultValue={enforcementScopeDict['Global']} @@ -145,6 +251,81 @@ const CreateEnforcementRuleModal = (props) => { )} /> + {values.scope === enforcementScopeDict['Organization'] && ( + + { + setFieldValue('organization', value.value); + }} + options={organizationOptions} + renderInput={(params) => ( + + )} + /> + + )} + {values.scope === enforcementScopeDict['Environment'] && ( + + { + setFieldValue('environment', value.value); + }} + renderInput={(params) => ( + + )} + /> + + )} + {values.scope === enforcementScopeDict['Dataset'] && ( + + { + setFieldValue('dataset', value.value); + }} + renderInput={(params) => ( + + )} + /> + + )} { /> - - - {entityTypesOptions.map((entityType) => ( - - { - if(value){ - - setFieldValue('entityTypes', [...values.entityTypes, entityType.value]); - } - else{ - setFieldValue('entityTypes', values.entityTypes.filter((item) => item !== entityType.value)); - } - }}/>} label={entityType.value} /> + + {entityTypesOptions.map((entityType) => ( + + { + if (value) { + setFieldValue('entityTypes', [ + ...values.entityTypes, + entityType.value + ]); + } else { + setFieldValue( + 'entityTypes', + values.entityTypes.filter( + (item) => item !== entityType.value + ) + ); + } + }} + /> + } + label={entityType.value} + /> + + ))} - ))} - - + { }; export const MetadataFormEnforcement = (props) => { - const {canEdit, metadataForm} = props; + const { canEdit, metadataForm } = props; const client = useClient(); const dispatch = useDispatch(); const [showCreateRuleModal, setShowCreateRuleModal] = useState(false); const [rules, setRules] = useState([]); const [severityOptions, setSeverityOptions] = useState({}); - const [entityTypesOptions, setEntityTypesOptions] = useState({}) - const [enforcementScopeOptions, setEnforcementScopeOptions] = useState({}) - + const [entityTypesOptions, setEntityTypesOptions] = useState({}); + const [enforcementScopeOptions, setEnforcementScopeOptions] = useState({}); const fetchEnforcementRules = async () => { - const response = await client.query(listMetadataFormEnforcementRules(metadataForm.uri)); - if (!response.errors && response.data && response.data.listMetadataFormEnforcementRules) { + const response = await client.query( + listMetadataFormEnforcementRules(metadataForm.uri) + ); + if ( + !response.errors && + response.data && + response.data.listMetadataFormEnforcementRules + ) { setRules(response.data.listMetadataFormEnforcementRules); } }; @@ -271,8 +467,12 @@ export const MetadataFormEnforcement = (props) => { useEffect(() => { if (client) { - fetchEnforcementRules().catch((e) => dispatch({ type: SET_ERROR, error: e.message })); - fetchEnforcementEnums().catch((e) => dispatch({ type: SET_ERROR, error: e.message })); + fetchEnforcementRules().catch((e) => + dispatch({ type: SET_ERROR, error: e.message }) + ); + fetchEnforcementEnums().catch((e) => + dispatch({ type: SET_ERROR, error: e.message }) + ); } }, [client]); @@ -285,27 +485,29 @@ export const MetadataFormEnforcement = (props) => { > - + - + - + {canEdit && ( - - - + + + )} diff --git a/frontend/src/modules/Metadata_Forms/services/createMetadataFormEnforcementRule.js b/frontend/src/modules/Metadata_Forms/services/createMetadataFormEnforcementRule.js index 62b679b91..718992b19 100644 --- a/frontend/src/modules/Metadata_Forms/services/createMetadataFormEnforcementRule.js +++ b/frontend/src/modules/Metadata_Forms/services/createMetadataFormEnforcementRule.js @@ -5,7 +5,9 @@ export const createMetadataFormEnforcementRule = (input) => ({ input }, mutation: gql` - mutation createMetadataFormEnforcementRule($input: NewMetadataFormEnforcementInput!) { + mutation createMetadataFormEnforcementRule( + $input: NewMetadataFormEnforcementInput! + ) { createMetadataFormEnforcementRule(input: $input) { uri level diff --git a/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRules.js b/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRules.js index b86a1cd05..bc70b770b 100644 --- a/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRules.js +++ b/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRules.js @@ -10,7 +10,6 @@ export const listMetadataFormEnforcementRules = (uri) => ({ uri level homeEntity - homeEntityName entityTypes metadataFormUri version diff --git a/frontend/src/modules/Metadata_Forms/views/MetadataFormView.js b/frontend/src/modules/Metadata_Forms/views/MetadataFormView.js index 31651f1fd..4b4137f74 100644 --- a/frontend/src/modules/Metadata_Forms/views/MetadataFormView.js +++ b/frontend/src/modules/Metadata_Forms/views/MetadataFormView.js @@ -251,7 +251,10 @@ const MetadataFormView = () => { /> )} {currentTab === 'enforcement' && ( - + )} {currentTab === 'preview' && ( From 6fe3f6dc26151347a25eaa60309f8f7039d97a6f Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Mon, 2 Dec 2024 11:21:13 +0000 Subject: [PATCH 06/16] show list of affected entities on Enforcement tab --- .../modules/metadata_forms/api/input_types.py | 8 + .../modules/metadata_forms/api/queries.py | 10 + .../modules/metadata_forms/api/resolvers.py | 10 +- .../modules/metadata_forms/api/types.py | 28 +++ .../modules/metadata_forms/db/enums.py | 32 +-- .../metadata_form_enforcement_service.py | 38 ++- .../components/MetadataFormEnforcement.js | 227 +++++++++++++++++- .../modules/Metadata_Forms/services/index.js | 1 + ...dataFormEnforcementRuleAffectedEntities.js | 31 +++ .../listMetadataFormEnforcementRules.js | 1 + 10 files changed, 352 insertions(+), 34 deletions(-) create mode 100644 frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRuleAffectedEntities.js diff --git a/backend/dataall/modules/metadata_forms/api/input_types.py b/backend/dataall/modules/metadata_forms/api/input_types.py index b8181894f..0525acf2e 100644 --- a/backend/dataall/modules/metadata_forms/api/input_types.py +++ b/backend/dataall/modules/metadata_forms/api/input_types.py @@ -93,3 +93,11 @@ gql.Field(name='entityTypes', type=gql.ArrayType(gql.String)), ], ) + +AffectedEntityFilter = gql.InputType( + name='AffectedEntityFilter', + arguments=[ + gql.Argument('page', gql.Integer), + gql.Argument('pageSize', gql.Integer), + ], +) \ No newline at end of file diff --git a/backend/dataall/modules/metadata_forms/api/queries.py b/backend/dataall/modules/metadata_forms/api/queries.py index 34836e41a..b54b45421 100644 --- a/backend/dataall/modules/metadata_forms/api/queries.py +++ b/backend/dataall/modules/metadata_forms/api/queries.py @@ -8,6 +8,7 @@ get_entity_metadata_form_permissions, list_metadata_form_versions, list_mf_enforcement_rules, + list_mf_affected_entities, ) listUserMetadataForms = gql.QueryField( @@ -58,6 +59,15 @@ test_scope='MetadataForm', ) +listEntityAffectedByEnforcementRule = gql.QueryField( + name='listEntityAffectedByEnforcementRules', + args=[gql.Argument('uri', gql.NonNullableType(gql.String)), + gql.Argument('filter', gql.Ref('AffectedEntityFilter'))], + type=gql.Ref('MFAffectedEntitiesSearchResult'), + resolver=list_mf_affected_entities, + test_scope='MetadataForm', +) + getAttachedMetadataForm = gql.QueryField( name='getAttachedMetadataForm', args=[gql.Argument('uri', gql.NonNullableType(gql.String))], diff --git a/backend/dataall/modules/metadata_forms/api/resolvers.py b/backend/dataall/modules/metadata_forms/api/resolvers.py index 9bf291b10..86756780a 100644 --- a/backend/dataall/modules/metadata_forms/api/resolvers.py +++ b/backend/dataall/modules/metadata_forms/api/resolvers.py @@ -5,7 +5,7 @@ MetadataForm, MetadataFormField, AttachedMetadataForm, - AttachedMetadataFormField, + AttachedMetadataFormField, MetadataFormEnforcementRule, ) from dataall.modules.metadata_forms.services.attached_metadata_form_service import AttachedMetadataFormService from dataall.modules.metadata_forms.services.metadata_form_enforcement_service import MetadataFormEnforcementService @@ -129,3 +129,11 @@ def create_mf_enforcement_rule(context: Context, source, input): def list_mf_enforcement_rules(context: Context, source, uri): return MetadataFormEnforcementService.list_mf_enforcement_rules(uri=uri) + + +def list_mf_affected_entities(context: Context, source, uri, filter): + return MetadataFormEnforcementService.paginate_mf_affected_entities(uri=uri, data = filter) + + +def get_mf_rule_home_entity_name(context: Context, source: MetadataFormEnforcementRule): + return MetadataFormEnforcementService.resolve_home_entity(source.uri, source) \ No newline at end of file diff --git a/backend/dataall/modules/metadata_forms/api/types.py b/backend/dataall/modules/metadata_forms/api/types.py index edb52ca15..b159c8a22 100644 --- a/backend/dataall/modules/metadata_forms/api/types.py +++ b/backend/dataall/modules/metadata_forms/api/types.py @@ -9,6 +9,7 @@ has_tenant_permissions_for_metadata_forms, resolve_metadata_form, resolve_metadata_form_field, + get_mf_rule_home_entity_name ) MetadataForm = gql.ObjectType( @@ -122,6 +123,7 @@ gql.Field(name='uri', type=gql.String), gql.Field(name='level', type=gql.String), gql.Field(name='homeEntity', type=gql.String), + gql.Field(name='homeEntityName', type=gql.String, resolver=get_mf_rule_home_entity_name), gql.Field( name='version', type=gql.Integer, @@ -133,3 +135,29 @@ gql.Field(name='entityTypes', type=gql.ArrayType(gql.String)), ], ) + +MFAffectedEntitiesSearchResult = gql.ObjectType( + name='MFAffectedEntitiesSearchResult', + fields=[ + gql.Field(name='count', type=gql.Integer), + gql.Field(name='nodes', type=gql.ArrayType(gql.Ref('MFAffectedEntity'))), + gql.Field(name='pageSize', type=gql.Integer), + gql.Field(name='nextPage', type=gql.Integer), + gql.Field(name='pages', type=gql.Integer), + gql.Field(name='page', type=gql.Integer), + gql.Field(name='previousPage', type=gql.Integer), + gql.Field(name='hasNext', type=gql.Boolean), + gql.Field(name='hasPrevious', type=gql.Boolean), + ], +) + +MFAffectedEntity = gql.ObjectType( + name='MFAffectedEntity', + fields=[ + gql.Field(name='type', type=gql.String), + gql.Field(name='uri', type=gql.String), + gql.Field(name='name', type=gql.String), + gql.Field(name='owner', type=gql.String), + gql.Field(name='attached', type=gql.Ref('AttachedMetadataForm')), + ], +) \ No newline at end of file diff --git a/backend/dataall/modules/metadata_forms/db/enums.py b/backend/dataall/modules/metadata_forms/db/enums.py index c459a5c74..73c767322 100644 --- a/backend/dataall/modules/metadata_forms/db/enums.py +++ b/backend/dataall/modules/metadata_forms/db/enums.py @@ -50,82 +50,82 @@ def get_entity_class(value: str): MetadataFormEntityTypes.Organizations.value: ( Organization, MetadataFormEnforcementScope.Global, - lambda o: (o.organizationUri, o.SamlGroupName), + lambda o: (o.organizationUri, o.SamlGroupName, o.lable), ), MetadataFormEntityTypes.OrganizationTeams.value: ( OrganizationGroup, MetadataFormEnforcementScope.Organization, - lambda o: (o.organizationUri + o.groupUri, o.invitedBy), + lambda o: (o.organizationUri + o.groupUri, o.invitedBy, o.groupUri), ), MetadataFormEntityTypes.Environments.value: ( Environment, MetadataFormEnforcementScope.Organization, - lambda o: (o.environmentUri, o.SamlGroupName), + lambda o: (o.environmentUri, o.SamlGroupName, o.lable), ), MetadataFormEntityTypes.EnvironmentTeams.value: ( EnvironmentGroup, MetadataFormEnforcementScope.Environment, - lambda o: (o.environmentUri + o.groupUri, o.invitedBy), + lambda o: (o.environmentUri + o.groupUri, o.invitedBy, o.groupUri), ), MetadataFormEntityTypes.S3Datasets.value: ( S3Dataset, MetadataFormEnforcementScope.Environment, - lambda o: (o.datasetUri, o.SamlAdminGroupName), + lambda o: (o.datasetUri, o.SamlAdminGroupName, o.groupUri), ), MetadataFormEntityTypes.RDDatasets.value: ( RedshiftDataset, MetadataFormEnforcementScope.Environment, - lambda o: (o.datasetUri, o.SamlAdminGroupName), + lambda o: (o.datasetUri, o.SamlAdminGroupName, o.groupUri), ), MetadataFormEntityTypes.Worksheets.value: ( Worksheet, MetadataFormEnforcementScope.Global, - lambda o: (o.worksheetUri, o.SamlAdminGroupName), + lambda o: (o.worksheetUri, o.SamlAdminGroupName, o.lable), ), MetadataFormEntityTypes.Dashboards.value: ( Dashboard, MetadataFormEnforcementScope.Environment, - lambda o: (o.dashboardUri, o.SamlGroupName), + lambda o: (o.dashboardUri, o.SamlGroupName, o.groupUri), ), MetadataFormEntityTypes.ConsumptionRoles.value: ( ConsumptionRole, MetadataFormEnforcementScope.Environment, - lambda o: (o.consumptionRoleUri, o.groupUri), + lambda o: (o.consumptionRoleUri, o.groupUri, o.consumptionRoleName), ), MetadataFormEntityTypes.Notebooks.value: ( SagemakerNotebook, MetadataFormEnforcementScope.Environment, - lambda o: (o.notebookUri, o.SamlAdminGroupName), + lambda o: (o.notebookUri, o.SamlAdminGroupName, o.lable), ), MetadataFormEntityTypes.MLStudioEntities.value: ( SagemakerStudioDomain, MetadataFormEnforcementScope.Environment, - lambda o: (o.sagemakerStudioUri, o.SamlGroupName), + lambda o: (o.sagemakerStudioUri, o.SamlGroupName, o.lable), ), MetadataFormEntityTypes.Pipelines.value: ( DataPipeline, MetadataFormEnforcementScope.Environment, - lambda o: (o.DataPipelineUri, o.SamlGroupName), + lambda o: (o.DataPipelineUri, o.SamlGroupName, o.lable), ), MetadataFormEntityTypes.Tables.value: ( DatasetTable, MetadataFormEnforcementScope.Dataset, - lambda o: (o.tableUri, None), # ToDo: resolve owner + lambda o: (o.tableUri, None, o.GlueTableName), # ToDo: resolve owner ), MetadataFormEntityTypes.Folder.value: ( DatasetStorageLocation, MetadataFormEnforcementScope.Dataset, - lambda o: (o.locationUri, None), # ToDo: resolve owner + lambda o: (o.locationUri, None, o.S3BucketName), # ToDo: resolve owner ), MetadataFormEntityTypes.Bucket.value: ( DatasetBucket, MetadataFormEnforcementScope.Dataset, - lambda o: (o.bucketUri, None), # ToDo: resolve owner + lambda o: (o.bucketUri, None, o.S3BucketName), # ToDo: resolve owner ), MetadataFormEntityTypes.Share.value: ( ShareObject, MetadataFormEnforcementScope.Dataset, - lambda o: (o.shareUri, o.groupUri), + lambda o: (o.shareUri, o.groupUri, o.shareUri), ), } diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py b/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py index 8e8d3cde6..1865e7e23 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py @@ -1,5 +1,6 @@ from dataall.base.context import get_context from dataall.base.db import exceptions +from dataall.base.db.paginator import paginate_list from dataall.core.environment.db.environment_repositories import EnvironmentRepository from dataall.core.organizations.db.organization_repositories import OrganizationRepository from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService @@ -113,7 +114,7 @@ def get_affected_datasets(uri, rule=None): return [] @staticmethod - def form_affected_entity_object(uri, owner, type, rule): + def form_affected_entity_object(uri, owner, label, type, rule): with get_context().db_engine.scoped_session() as session: attached = MetadataFormRepository.query_all_attached_metadata_forms_for_entity( session, @@ -121,7 +122,7 @@ def form_affected_entity_object(uri, owner, type, rule): metadataFormUri=rule.metadataFormUri, version=rule.version, ) - return {'type': type, 'uri': uri, 'owner': owner, 'attached': attached.first()} + return {'type': type, 'name': label, 'uri': uri, 'owner': owner, 'attached': attached.first()} @staticmethod def get_affected_entities(uri, rule=None): @@ -134,7 +135,7 @@ def get_affected_entities(uri, rule=None): affected_entities.extend( [ MetadataFormEnforcementService.form_affected_entity_object( - o.organizationUri, o.SamlGroupName, MetadataFormEntityTypes.Organizations.value, rule + o.organizationUri, o.SamlGroupName, o.label, MetadataFormEntityTypes.Organizations.value, rule ) for o in orgs ] @@ -144,7 +145,7 @@ def get_affected_entities(uri, rule=None): affected_entities.extend( [ MetadataFormEnforcementService.form_affected_entity_object( - e.environmentUri, e.SamlGroupName, MetadataFormEntityTypes.Environments.value, rule + e.environmentUri, e.SamlGroupName, e.label, MetadataFormEntityTypes.Environments.value, rule ) for e in envs ] @@ -154,7 +155,7 @@ def get_affected_entities(uri, rule=None): affected_entities.extend( [ MetadataFormEnforcementService.form_affected_entity_object( - ds.datasetUri, ds.SamlAdminGroupName, ds.datasetType.value + '-Dataset', rule + ds.datasetUri, ds.SamlAdminGroupName, ds.label, ds.datasetType.value + '-Dataset', rule ) for ds in datasets ] @@ -168,7 +169,7 @@ def get_affected_entities(uri, rule=None): } for entity_type in entity_types: - entity_class, level, get_uri_and_owner = MetadataFormEntityTypes.get_entity_class(entity_type) + entity_class, level, get_uri_owner_label = MetadataFormEntityTypes.get_entity_class(entity_type) all_entities = session.query(entity_class) if level == MetadataFormEnforcementScope.Organization.value: all_entities = all_entities.filter(entity_class.organizationUri.in_([org.uri for org in orgs])) @@ -180,7 +181,7 @@ def get_affected_entities(uri, rule=None): affected_entities.extend( [ MetadataFormEnforcementService.form_affected_entity_object( - *get_uri_and_owner(e), entity_type, rule + *get_uri_owner_label(e), entity_type, rule ) for e in all_entities ] @@ -191,3 +192,26 @@ def get_affected_entities(uri, rule=None): def list_mf_enforcement_rules(uri): with get_context().db_engine.scoped_session() as session: return MetadataFormRepository.list_mf_enforcement_rules(session, uri) + + @staticmethod + def paginate_mf_affected_entities(uri, data=None): + data = data or {} + return paginate_list( + items=MetadataFormEnforcementService.get_affected_entities(uri), + page=data.get('page', 1), + page_size=data.get('pageSize', 10), + ).to_dict() + + @staticmethod + def resolve_home_entity(uri, rule=None): + with get_context().db_engine.scoped_session() as session: + if not rule: + rule = MetadataFormRepository.get_mf_enforcement_rule_by_uri(session, uri) + if rule.level == MetadataFormEnforcementScope.Global.value: + return '' + if rule.level == MetadataFormEnforcementScope.Organization.value: + return OrganizationRepository.get_organization_by_uri(session, rule.homeEntity).label + if rule.level == MetadataFormEnforcementScope.Environment.value: + return EnvironmentRepository.get_environment_by_uri(session, rule.homeEntity).label + if rule.level == MetadataFormEnforcementScope.Dataset.value: + return DatasetBaseRepository.get_dataset_by_uri(session, rule.homeEntity).label diff --git a/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js b/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js index 906fa6857..24e97b1f3 100644 --- a/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js +++ b/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js @@ -7,13 +7,15 @@ import { CardContent, CardHeader, Checkbox, + Chip, + CircularProgress, Dialog, FormControlLabel, Grid, TextField, Typography } from '@mui/material'; -import { Defaults, PlusIcon } from 'design'; +import { Defaults, Label, PlusIcon } from 'design'; import React, { useEffect, useState } from 'react'; import DoNotDisturbAltOutlinedIcon from '@mui/icons-material/DoNotDisturbAltOutlined'; import { fetchEnums, listValidEnvironments, useClient } from 'services'; @@ -21,13 +23,17 @@ import { useDispatch } from 'react-redux'; import { SET_ERROR } from 'globalErrors'; import { createMetadataFormEnforcementRule, - listMetadataFormEnforcementRules + listMetadataFormEnforcementRules, + listEntityAffectedByEnforcementRules } from '../services'; import { Formik } from 'formik'; import { LoadingButton } from '@mui/lab'; import SendIcon from '@mui/icons-material/Send'; import { listOrganizations } from '../../Organizations/services'; import { listDatasets } from '../../DatasetsBase/services'; +import { useTheme } from '@mui/styles'; +import DeleteIcon from '@mui/icons-material/DeleteOutlined'; +import { DataGrid } from '@mui/x-data-grid'; const CreateEnforcementRuleModal = (props) => { const { @@ -419,14 +425,44 @@ const CreateEnforcementRuleModal = (props) => { export const MetadataFormEnforcement = (props) => { const { canEdit, metadataForm } = props; const client = useClient(); + const theme = useTheme(); const dispatch = useDispatch(); const [showCreateRuleModal, setShowCreateRuleModal] = useState(false); const [rules, setRules] = useState([]); + const [selectedRule, setSelectedRule] = useState(null); const [severityOptions, setSeverityOptions] = useState({}); const [entityTypesOptions, setEntityTypesOptions] = useState({}); const [enforcementScopeOptions, setEnforcementScopeOptions] = useState({}); + const [affectedEntities, setAffectedEntities] = useState([]); + const [paginationModel, setPaginationModel] = useState({ + pageSize: 5, + page: 0 + }); + const [selectedEntity, setSelectedEntity] = useState([]); + const [loading, setLoading] = useState(true); + const [loadingAffected, setLoadingAffected] = useState(true); + + const header = [ + { field: 'type', width: 200, headerName: 'Type', editable: false }, + { field: 'name', width: 350, headerName: 'Name', editable: false }, + { field: 'owner', width: 200, headerName: 'Owner', editable: false }, + { + field: 'attached', + width: 100, + headerName: 'Attached', + editable: false, + renderCell: (params) => { + return ( + + ); + } + } + ]; const fetchEnforcementRules = async () => { + setLoading(true); const response = await client.query( listMetadataFormEnforcementRules(metadataForm.uri) ); @@ -436,7 +472,49 @@ export const MetadataFormEnforcement = (props) => { response.data.listMetadataFormEnforcementRules ) { setRules(response.data.listMetadataFormEnforcementRules); + if (response.data.listMetadataFormEnforcementRules.length > 0) { + setSelectedRule(response.data.listMetadataFormEnforcementRules[0]); + await fetchAffectedEntities( + response.data.listMetadataFormEnforcementRules[0] + ); + } + } else { + const error = 'Could not fetch rules'; + dispatch({ type: SET_ERROR, error }); + } + setLoading(false); + }; + + const deleteRule = async (rule) => {}; + + const fetchAffectedEntities = async ( + rule, + page = paginationModel.page, + pageSize = paginationModel.pageSize + ) => { + setLoadingAffected(true); + const response = await client.query( + listEntityAffectedByEnforcementRules(rule.uri, { + pageSize: pageSize, + page: page + 1 + }) + ); + if ( + !response.errors && + response.data && + response.data.listEntityAffectedByEnforcementRules + ) { + response.data.listEntityAffectedByEnforcementRules.nodes.forEach( + (entity) => { + entity.id = entity.uri; + } + ); + setAffectedEntities(response.data.listEntityAffectedByEnforcementRules); + } else { + const error = 'Could not fetch affeceted entities'; + dispatch({ type: SET_ERROR, error }); } + setLoadingAffected(false); }; const fetchEnforcementEnums = async () => { @@ -467,9 +545,9 @@ export const MetadataFormEnforcement = (props) => { useEffect(() => { if (client) { - fetchEnforcementRules().catch((e) => - dispatch({ type: SET_ERROR, error: e.message }) - ); + fetchEnforcementRules() + .then() + .catch((e) => dispatch({ type: SET_ERROR, error: e.message })); fetchEnforcementEnums().catch((e) => dispatch({ type: SET_ERROR, error: e.message }) ); @@ -511,12 +589,83 @@ export const MetadataFormEnforcement = (props) => { )} - {rules.length > 0 ? ( + {rules.length > 0 && !loading ? ( rules.map((rule) => ( - - - rule.uri - + { + setSelectedRule(rule); + await fetchAffectedEntities(rule); + }} + sx={{ + backgroundColor: + selectedRule && + selectedRule.uri === rule.uri && + theme.palette.action.selected + }} + > + + + + {'v. ' + rule.version} + + + + + {rule.level} + + + + + {rule.homeEntityName} + + + + {rule.entityTypes.map((et) => ( + + ))} + + + {canEdit && ( + { + e.currentTarget.style.opacity = 1; + }} + onMouseOut={(e) => { + e.currentTarget.style.opacity = 0.5; + }} + onClick={() => deleteRule(rule.uri)} + /> + )} + + )) ) : ( @@ -527,6 +676,64 @@ export const MetadataFormEnforcement = (props) => { )} + {loading && ( + + + + )} + + + + + + + {!loadingAffected && + affectedEntities.nodes && + affectedEntities.nodes.length > 0 ? ( + { + setPaginationModel({ + ...paginationModel, + pageSize: newPageSize + }); + await fetchAffectedEntities( + selectedRule, + paginationModel.page, + newPageSize + ); + }} + page={paginationModel.page} + onPageChange={async (newPage) => { + setPaginationModel({ ...paginationModel, page: newPage }); + await fetchAffectedEntities( + selectedRule, + newPage, + paginationModel.pageSize + ); + }} + rowCount={affectedEntities.count} + autoHeight={true} + onSelectionModelChange={async (newSelection) => { + setSelectedEntity(newSelection); + }} + selectionModel={selectedEntity} + hideFooterSelectedRowCount={true} + /> + ) : ( + + {loadingAffected ? '' : 'No entities affected.'} + + )} + {loadingAffected && selectedRule && ( + + + + )} + diff --git a/frontend/src/modules/Metadata_Forms/services/index.js b/frontend/src/modules/Metadata_Forms/services/index.js index 904310db7..e231529b6 100644 --- a/frontend/src/modules/Metadata_Forms/services/index.js +++ b/frontend/src/modules/Metadata_Forms/services/index.js @@ -14,3 +14,4 @@ export * from './deleteMetadataFormVersion'; export * from './listMetadataFormVersions'; export * from './createMetadataFormEnforcementRule'; export * from './listMetadataFormEnforcementRules'; +export * from './listMetadataFormEnforcementRuleAffectedEntities'; diff --git a/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRuleAffectedEntities.js b/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRuleAffectedEntities.js new file mode 100644 index 000000000..59c5f18af --- /dev/null +++ b/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRuleAffectedEntities.js @@ -0,0 +1,31 @@ +import { gql } from 'apollo-boost'; + +export const listEntityAffectedByEnforcementRules = (uri, filter) => ({ + variables: { + uri, + filter + }, + query: gql` + query listEntityAffectedByEnforcementRules( + $uri: String! + $filter: AffectedEntityFilter + ) { + listEntityAffectedByEnforcementRules(uri: $uri, filter: $filter) { + count + page + pages + hasNext + hasPrevious + nodes { + uri + type + name + owner + attached { + uri + } + } + } + } + ` +}); diff --git a/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRules.js b/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRules.js index bc70b770b..b86a1cd05 100644 --- a/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRules.js +++ b/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRules.js @@ -10,6 +10,7 @@ export const listMetadataFormEnforcementRules = (uri) => ({ uri level homeEntity + homeEntityName entityTypes metadataFormUri version From db6d6dc9178ec40c89e1ced94b505bf3c6dc01f0 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Mon, 2 Dec 2024 16:06:12 +0000 Subject: [PATCH 07/16] Metadata Form Entity manager --- .../core/environment/db/environment_models.py | 36 +++++ .../dataall/core/metadata_manager/__init__.py | 5 + .../metadata_form_entity_manager.py | 53 +++++++ .../organizations/db/organization_models.py | 26 ++++ .../dataall/modules/dashboards/__init__.py | 3 +- .../modules/dashboards/db/dashboard_models.py | 8 + .../datasets_base/db/dataset_models.py | 11 ++ .../modules/metadata_forms/api/input_types.py | 2 +- .../modules/metadata_forms/api/queries.py | 6 +- .../modules/metadata_forms/api/resolvers.py | 7 +- .../modules/metadata_forms/api/types.py | 4 +- .../modules/metadata_forms/db/enums.py | 137 +++--------------- .../services/metadata_form_access_service.py | 2 +- .../metadata_form_enforcement_service.py | 39 +++-- .../services/metadata_form_service.py | 2 +- backend/dataall/modules/mlstudio/__init__.py | 7 +- .../modules/mlstudio/db/mlstudio_models.py | 11 ++ backend/dataall/modules/notebooks/__init__.py | 4 +- .../modules/notebooks/db/notebook_models.py | 12 ++ .../modules/redshift_datasets/__init__.py | 7 +- .../dataall/modules/s3_datasets/__init__.py | 9 ++ .../modules/s3_datasets/db/dataset_models.py | 31 ++++ .../dataall/modules/shares_base/__init__.py | 8 + .../shares_base/db/share_object_models.py | 21 +++ .../dataall/modules/worksheets/__init__.py | 7 + .../modules/worksheets/db/worksheet_models.py | 12 ++ 26 files changed, 327 insertions(+), 143 deletions(-) create mode 100644 backend/dataall/core/metadata_manager/__init__.py create mode 100644 backend/dataall/core/metadata_manager/metadata_form_entity_manager.py diff --git a/backend/dataall/core/environment/db/environment_models.py b/backend/dataall/core/environment/db/environment_models.py index c4890850a..70c91a78e 100644 --- a/backend/dataall/core/environment/db/environment_models.py +++ b/backend/dataall/core/environment/db/environment_models.py @@ -7,9 +7,11 @@ from dataall.base.db import Resource, Base, utils from dataall.core.environment.api.enums import EnvironmentPermission, EnvironmentType +from dataall.core.metadata_manager import MetadataFormEntityManager, MetadataFormEntity, MetadataFormEntityTypes class Environment(Resource, Base): + __metaclass__ = MetadataFormEntity __tablename__ = 'environment' organizationUri = Column(String, nullable=False) environmentUri = Column(String, primary_key=True, default=utils.uuid('environment')) @@ -40,8 +42,18 @@ class Environment(Resource, Base): subscriptionsConsumersTopicName = Column(String) subscriptionsConsumersTopicImported = Column(Boolean, default=False) + def get_uri(self): + return self.environmentUri + + def get_owner(self): + return self.SamlGroupName + + def get_entity_name(self): + return self.label + class EnvironmentGroup(Base): + __metaclass__ = MetadataFormEntity __tablename__ = 'environment_group_permission' groupUri = Column(String, primary_key=True) environmentUri = Column(String, primary_key=True) @@ -58,6 +70,15 @@ class EnvironmentGroup(Base): # environmentRole is the role of the entity (group or user) in the Environment groupRoleInEnvironment = Column(String, nullable=False, default=EnvironmentPermission.Invited.value) + def get_uri(self): + return f'{self.groupUri}-{self.environmentUri}' + + def get_owner(self): + return self.invitedBy + + def get_entity_name(self): + return f'{self.groupUri}-{self.environmentUri}' + class EnvironmentParameter(Base): """Represent the parameter of the environment""" @@ -78,6 +99,7 @@ def __repr__(self): class ConsumptionRole(Base): + __metaclass__ = MetadataFormEntity __tablename__ = 'consumptionrole' consumptionRoleUri = Column(String, primary_key=True, default=utils.uuid('group')) consumptionRoleName = Column(String, nullable=False) @@ -89,3 +111,17 @@ class ConsumptionRole(Base): created = Column(DateTime, default=datetime.datetime.now) updated = Column(DateTime, onupdate=datetime.datetime.now) deleted = Column(DateTime) + + def get_uri(self): + return self.consumptionRoleUri + + def get_owner(self): + return self.groupUri + + def get_entity_name(self): + return f'{self.consumptionRoleName}-{self.environmentUri}' + + +MetadataFormEntityManager.register(Environment, MetadataFormEntityTypes.Environments.value) +MetadataFormEntityManager.register(ConsumptionRole, MetadataFormEntityTypes.ConsumptionRoles.value) +MetadataFormEntityManager.register(EnvironmentGroup, MetadataFormEntityTypes.EnvironmentTeams.value) diff --git a/backend/dataall/core/metadata_manager/__init__.py b/backend/dataall/core/metadata_manager/__init__.py new file mode 100644 index 000000000..ed821e798 --- /dev/null +++ b/backend/dataall/core/metadata_manager/__init__.py @@ -0,0 +1,5 @@ +from dataall.core.metadata_manager.metadata_form_entity_manager import ( + MetadataFormEntityManager, + MetadataFormEntity, + MetadataFormEntityTypes, +) diff --git a/backend/dataall/core/metadata_manager/metadata_form_entity_manager.py b/backend/dataall/core/metadata_manager/metadata_form_entity_manager.py new file mode 100644 index 000000000..6de499b18 --- /dev/null +++ b/backend/dataall/core/metadata_manager/metadata_form_entity_manager.py @@ -0,0 +1,53 @@ +from abc import ABC +from typing import List + +from dataall.base.api import GraphQLEnumMapper + + +class MetadataFormEntityTypes(GraphQLEnumMapper): + Organizations = 'Organization' + OrganizationTeams = 'Organization Team' + Environments = 'Environment' + EnvironmentTeams = 'Environment Team' + S3Datasets = 'S3-Dataset' + RDDatasets = 'Redshift-Dataset' + Worksheets = 'Worksheets' + Dashboards = 'Dashboard' + ConsumptionRoles = 'Consumption Role' + Notebooks = 'Notebook' + MLStudioEntities = 'ML Studio Entity' + Pipelines = 'Pipeline' + Tables = 'Table' + Folder = 'Folder' + Bucket = 'Bucket' + Share = 'Share' + ShareItem = 'Share Item' + + +class MetadataFormEntity(ABC): + def get_owner(self): + pass + + def get_entity_name(self): + pass + + def get_uri(self): + pass + + +class MetadataFormEntityManager: + """ + API for managing entities, to which MF can be attached. + """ + + _resources: List[MetadataFormEntity] = {} + + @classmethod + def register(cls, resource: MetadataFormEntity, resource_key): + cls._resources[resource_key] = resource + + @classmethod + def get_resource(cls, resource_key): + if resource_key not in cls._resources: + raise NotImplementedError(f'Entity {resource_key} is not registered') + return cls._resources[resource_key] diff --git a/backend/dataall/core/organizations/db/organization_models.py b/backend/dataall/core/organizations/db/organization_models.py index 5d58e7d3f..a0421494d 100644 --- a/backend/dataall/core/organizations/db/organization_models.py +++ b/backend/dataall/core/organizations/db/organization_models.py @@ -6,8 +6,11 @@ from dataall.base.db import Base from dataall.base.db import Resource, utils +from dataall.core.metadata_manager import MetadataFormEntityManager, MetadataFormEntity, MetadataFormEntityTypes + class Organization(Resource, Base): + __metaclass__ = MetadataFormEntity __tablename__ = 'organization' organizationUri = Column(String, primary_key=True, default=utils.uuid('organization')) @@ -16,8 +19,18 @@ class Organization(Resource, Base): userRoleInOrganization = query_expression() SamlGroupName = Column(String, nullable=True) + def get_uri(self): + return self.organizationUri + + def get_owner(self): + return self.SamlGroupName + + def get_entity_name(self): + return self.label + class OrganizationGroup(Base): + __metaclass__ = MetadataFormEntity __tablename__ = 'organization_group' groupUri = Column(String, primary_key=True) organizationUri = Column(String, primary_key=True) @@ -26,3 +39,16 @@ class OrganizationGroup(Base): created = Column(DateTime, default=datetime.datetime.now) updated = Column(DateTime, onupdate=datetime.datetime.now) deleted = Column(DateTime) + + def get_uri(self): + return f'{self.groupUri}-{self.organizationUri}' + + def get_owner(self): + return self.invitedBy + + def get_entity_name(self): + return f'{self.groupUri}-{self.organizationUri}' + + +MetadataFormEntityManager.register(Organization, MetadataFormEntityTypes.Organizations.value) +MetadataFormEntityManager.register(OrganizationGroup, MetadataFormEntityTypes.OrganizationTeams.value) diff --git a/backend/dataall/modules/dashboards/__init__.py b/backend/dataall/modules/dashboards/__init__.py index ffbc8e92d..59b3ee05a 100644 --- a/backend/dataall/modules/dashboards/__init__.py +++ b/backend/dataall/modules/dashboards/__init__.py @@ -5,7 +5,6 @@ from dataall.base.loader import ImportMode, ModuleInterface - log = logging.getLogger(__name__) @@ -26,6 +25,7 @@ def depends_on() -> List[Type['ModuleInterface']]: def __init__(self): from dataall.core.environment.services.environment_resource_manager import EnvironmentResourceManager + from dataall.core.metadata_manager import MetadataFormEntityManager, MetadataFormEntityTypes from dataall.modules.dashboards.db.dashboard_repositories import DashboardRepository from dataall.modules.dashboards.db.dashboard_models import Dashboard import dataall.modules.dashboards.api @@ -45,6 +45,7 @@ def __init__(self): add_vote_type('dashboard', DashboardIndexer) EnvironmentResourceManager.register(DashboardRepository()) + MetadataFormEntityManager.register(Dashboard, MetadataFormEntityTypes.Dashboards.value) log.info('Dashboard API has been loaded') diff --git a/backend/dataall/modules/dashboards/db/dashboard_models.py b/backend/dataall/modules/dashboards/db/dashboard_models.py index 1f2fa95cd..3cd5e0f13 100644 --- a/backend/dataall/modules/dashboards/db/dashboard_models.py +++ b/backend/dataall/modules/dashboards/db/dashboard_models.py @@ -4,6 +4,7 @@ from sqlalchemy.orm import query_expression from dataall.base.db import Base, Resource, utils +from dataall.core.metadata_manager import MetadataFormEntity class DashboardShareStatus(Enum): @@ -22,6 +23,7 @@ class DashboardShare(Base): class Dashboard(Resource, Base): + __metaclass__ = MetadataFormEntity __tablename__ = 'dashboard' environmentUri = Column(String, ForeignKey('environment.environmentUri'), nullable=False) organizationUri = Column(String, nullable=False) @@ -37,3 +39,9 @@ class Dashboard(Resource, Base): @classmethod def uri(cls): return cls.dashboardUri + + def get_owner(self): + return self.SamlGroupName + + def get_entity_name(self): + return self.label diff --git a/backend/dataall/modules/datasets_base/db/dataset_models.py b/backend/dataall/modules/datasets_base/db/dataset_models.py index 6bf0840b5..8073d4e39 100644 --- a/backend/dataall/modules/datasets_base/db/dataset_models.py +++ b/backend/dataall/modules/datasets_base/db/dataset_models.py @@ -3,9 +3,11 @@ from sqlalchemy.orm import query_expression from dataall.base.db import Base, Resource, utils from dataall.modules.datasets_base.services.datasets_enums import ConfidentialityClassification, Language, DatasetTypes +from dataall.core.metadata_manager.metadata_form_entity_manager import MetadataFormEntity class DatasetBase(Resource, Base): + __metaclass__ = MetadataFormEntity __tablename__ = 'dataset' environmentUri = Column(String, ForeignKey('environment.environmentUri'), nullable=False) organizationUri = Column(String, nullable=False) @@ -38,5 +40,14 @@ class DatasetBase(Resource, Base): def uri(cls): return cls.datasetUri + def get_owner(self): + return self.SamlAdminGroupName + + def get_entity_name(self): + return self.label + + def get_uri(self): + return self.datasetUri + DatasetBase.__name__ = 'Dataset' diff --git a/backend/dataall/modules/metadata_forms/api/input_types.py b/backend/dataall/modules/metadata_forms/api/input_types.py index 0525acf2e..760e1a9ef 100644 --- a/backend/dataall/modules/metadata_forms/api/input_types.py +++ b/backend/dataall/modules/metadata_forms/api/input_types.py @@ -100,4 +100,4 @@ gql.Argument('page', gql.Integer), gql.Argument('pageSize', gql.Integer), ], -) \ No newline at end of file +) diff --git a/backend/dataall/modules/metadata_forms/api/queries.py b/backend/dataall/modules/metadata_forms/api/queries.py index b54b45421..d39142e1c 100644 --- a/backend/dataall/modules/metadata_forms/api/queries.py +++ b/backend/dataall/modules/metadata_forms/api/queries.py @@ -61,8 +61,10 @@ listEntityAffectedByEnforcementRule = gql.QueryField( name='listEntityAffectedByEnforcementRules', - args=[gql.Argument('uri', gql.NonNullableType(gql.String)), - gql.Argument('filter', gql.Ref('AffectedEntityFilter'))], + args=[ + gql.Argument('uri', gql.NonNullableType(gql.String)), + gql.Argument('filter', gql.Ref('AffectedEntityFilter')), + ], type=gql.Ref('MFAffectedEntitiesSearchResult'), resolver=list_mf_affected_entities, test_scope='MetadataForm', diff --git a/backend/dataall/modules/metadata_forms/api/resolvers.py b/backend/dataall/modules/metadata_forms/api/resolvers.py index 86756780a..7b108100b 100644 --- a/backend/dataall/modules/metadata_forms/api/resolvers.py +++ b/backend/dataall/modules/metadata_forms/api/resolvers.py @@ -5,7 +5,8 @@ MetadataForm, MetadataFormField, AttachedMetadataForm, - AttachedMetadataFormField, MetadataFormEnforcementRule, + AttachedMetadataFormField, + MetadataFormEnforcementRule, ) from dataall.modules.metadata_forms.services.attached_metadata_form_service import AttachedMetadataFormService from dataall.modules.metadata_forms.services.metadata_form_enforcement_service import MetadataFormEnforcementService @@ -132,8 +133,8 @@ def list_mf_enforcement_rules(context: Context, source, uri): def list_mf_affected_entities(context: Context, source, uri, filter): - return MetadataFormEnforcementService.paginate_mf_affected_entities(uri=uri, data = filter) + return MetadataFormEnforcementService.paginate_mf_affected_entities(uri=uri, data=filter) def get_mf_rule_home_entity_name(context: Context, source: MetadataFormEnforcementRule): - return MetadataFormEnforcementService.resolve_home_entity(source.uri, source) \ No newline at end of file + return MetadataFormEnforcementService.resolve_home_entity(source.uri, source) diff --git a/backend/dataall/modules/metadata_forms/api/types.py b/backend/dataall/modules/metadata_forms/api/types.py index b159c8a22..f88ae67ca 100644 --- a/backend/dataall/modules/metadata_forms/api/types.py +++ b/backend/dataall/modules/metadata_forms/api/types.py @@ -9,7 +9,7 @@ has_tenant_permissions_for_metadata_forms, resolve_metadata_form, resolve_metadata_form_field, - get_mf_rule_home_entity_name + get_mf_rule_home_entity_name, ) MetadataForm = gql.ObjectType( @@ -160,4 +160,4 @@ gql.Field(name='owner', type=gql.String), gql.Field(name='attached', type=gql.Ref('AttachedMetadataForm')), ], -) \ No newline at end of file +) diff --git a/backend/dataall/modules/metadata_forms/db/enums.py b/backend/dataall/modules/metadata_forms/db/enums.py index 73c767322..7e74b26e4 100644 --- a/backend/dataall/modules/metadata_forms/db/enums.py +++ b/backend/dataall/modules/metadata_forms/db/enums.py @@ -1,14 +1,5 @@ from dataall.base.api.constants import GraphQLEnumMapper -from dataall.core.environment.db.environment_models import Environment, EnvironmentGroup, ConsumptionRole -from dataall.core.organizations.db.organization_models import Organization, OrganizationGroup -from dataall.modules.dashboards.db.dashboard_models import Dashboard -from dataall.modules.datapipelines.db.datapipelines_models import DataPipeline -from dataall.modules.mlstudio.db.mlstudio_models import SagemakerStudioDomain -from dataall.modules.notebooks.db.notebook_models import SagemakerNotebook -from dataall.modules.redshift_datasets.db.redshift_models import RedshiftDataset -from dataall.modules.s3_datasets.db.dataset_models import S3Dataset, DatasetTable, DatasetStorageLocation, DatasetBucket -from dataall.modules.shares_base.db.share_object_models import ShareObject -from dataall.modules.worksheets.db.worksheet_models import Worksheet +from dataall.core.metadata_manager.metadata_form_entity_manager import MetadataFormEntityTypes class MetadataFormVisibility(GraphQLEnumMapper): @@ -25,111 +16,6 @@ class MetadataFormFieldType(GraphQLEnumMapper): GlossaryTerm = 'Glossary Term' -class MetadataFormEntityTypes(GraphQLEnumMapper): - Organizations = 'Organization' - OrganizationTeams = 'Organization Team' - Environments = 'Environment' - EnvironmentTeams = 'Environment Team' - S3Datasets = 'S3-Dataset' - RDDatasets = 'Redshift-Dataset' - Worksheets = 'Worksheets' - Dashboards = 'Dashboard' - ConsumptionRoles = 'Consumption Role' - Notebooks = 'Notebook' - MLStudioEntities = 'ML Studio Entity' - Pipelines = 'Pipeline' - Tables = 'Table' - Folder = 'Folder' - Bucket = 'Bucket' - Share = 'Share' - ShareItem = 'Share Item' - - @staticmethod - def get_entity_class(value: str): - classes = { - MetadataFormEntityTypes.Organizations.value: ( - Organization, - MetadataFormEnforcementScope.Global, - lambda o: (o.organizationUri, o.SamlGroupName, o.lable), - ), - MetadataFormEntityTypes.OrganizationTeams.value: ( - OrganizationGroup, - MetadataFormEnforcementScope.Organization, - lambda o: (o.organizationUri + o.groupUri, o.invitedBy, o.groupUri), - ), - MetadataFormEntityTypes.Environments.value: ( - Environment, - MetadataFormEnforcementScope.Organization, - lambda o: (o.environmentUri, o.SamlGroupName, o.lable), - ), - MetadataFormEntityTypes.EnvironmentTeams.value: ( - EnvironmentGroup, - MetadataFormEnforcementScope.Environment, - lambda o: (o.environmentUri + o.groupUri, o.invitedBy, o.groupUri), - ), - MetadataFormEntityTypes.S3Datasets.value: ( - S3Dataset, - MetadataFormEnforcementScope.Environment, - lambda o: (o.datasetUri, o.SamlAdminGroupName, o.groupUri), - ), - MetadataFormEntityTypes.RDDatasets.value: ( - RedshiftDataset, - MetadataFormEnforcementScope.Environment, - lambda o: (o.datasetUri, o.SamlAdminGroupName, o.groupUri), - ), - MetadataFormEntityTypes.Worksheets.value: ( - Worksheet, - MetadataFormEnforcementScope.Global, - lambda o: (o.worksheetUri, o.SamlAdminGroupName, o.lable), - ), - MetadataFormEntityTypes.Dashboards.value: ( - Dashboard, - MetadataFormEnforcementScope.Environment, - lambda o: (o.dashboardUri, o.SamlGroupName, o.groupUri), - ), - MetadataFormEntityTypes.ConsumptionRoles.value: ( - ConsumptionRole, - MetadataFormEnforcementScope.Environment, - lambda o: (o.consumptionRoleUri, o.groupUri, o.consumptionRoleName), - ), - MetadataFormEntityTypes.Notebooks.value: ( - SagemakerNotebook, - MetadataFormEnforcementScope.Environment, - lambda o: (o.notebookUri, o.SamlAdminGroupName, o.lable), - ), - MetadataFormEntityTypes.MLStudioEntities.value: ( - SagemakerStudioDomain, - MetadataFormEnforcementScope.Environment, - lambda o: (o.sagemakerStudioUri, o.SamlGroupName, o.lable), - ), - MetadataFormEntityTypes.Pipelines.value: ( - DataPipeline, - MetadataFormEnforcementScope.Environment, - lambda o: (o.DataPipelineUri, o.SamlGroupName, o.lable), - ), - MetadataFormEntityTypes.Tables.value: ( - DatasetTable, - MetadataFormEnforcementScope.Dataset, - lambda o: (o.tableUri, None, o.GlueTableName), # ToDo: resolve owner - ), - MetadataFormEntityTypes.Folder.value: ( - DatasetStorageLocation, - MetadataFormEnforcementScope.Dataset, - lambda o: (o.locationUri, None, o.S3BucketName), # ToDo: resolve owner - ), - MetadataFormEntityTypes.Bucket.value: ( - DatasetBucket, - MetadataFormEnforcementScope.Dataset, - lambda o: (o.bucketUri, None, o.S3BucketName), # ToDo: resolve owner - ), - MetadataFormEntityTypes.Share.value: ( - ShareObject, - MetadataFormEnforcementScope.Dataset, - lambda o: (o.shareUri, o.groupUri, o.shareUri), - ), - } - - class MetadataFormEnforcementSeverity(GraphQLEnumMapper): Mandatory = 'Mandatory' Recommended = 'Recommended' @@ -145,3 +31,24 @@ class MetadataFormEnforcementScope(GraphQLEnumMapper): class MetadataFormUserRoles(GraphQLEnumMapper): Owner = 'Owner' User = 'User' + + +ENTITY_SCOPE_BY_TYPE = { + MetadataFormEntityTypes.Organizations.value: MetadataFormEnforcementScope.Global, + MetadataFormEntityTypes.OrganizationTeams.value: MetadataFormEnforcementScope.Organization, + MetadataFormEntityTypes.Environments.value: MetadataFormEnforcementScope.Organization, + MetadataFormEntityTypes.EnvironmentTeams.value: MetadataFormEnforcementScope.Environment, + MetadataFormEntityTypes.S3Datasets.value: MetadataFormEnforcementScope.Environment, + MetadataFormEntityTypes.RDDatasets.value: MetadataFormEnforcementScope.Environment, + MetadataFormEntityTypes.Worksheets.value: MetadataFormEnforcementScope.Global, + MetadataFormEntityTypes.Dashboards.value: MetadataFormEnforcementScope.Environment, + MetadataFormEntityTypes.ConsumptionRoles.value: MetadataFormEnforcementScope.Environment, + MetadataFormEntityTypes.Notebooks.value: MetadataFormEnforcementScope.Environment, + MetadataFormEntityTypes.MLStudioEntities.value: MetadataFormEnforcementScope.Environment, + MetadataFormEntityTypes.Pipelines.value: MetadataFormEnforcementScope.Environment, + MetadataFormEntityTypes.Tables.value: MetadataFormEnforcementScope.Dataset, + MetadataFormEntityTypes.Folder.value: MetadataFormEnforcementScope.Dataset, + MetadataFormEntityTypes.Bucket.value: MetadataFormEnforcementScope.Dataset, + MetadataFormEntityTypes.Share.value: MetadataFormEnforcementScope.Dataset, + MetadataFormEntityTypes.ShareItem.value: MetadataFormEnforcementScope.Dataset, +} diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_access_service.py b/backend/dataall/modules/metadata_forms/services/metadata_form_access_service.py index 1c1a66a96..edf819a31 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_access_service.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_access_service.py @@ -8,9 +8,9 @@ from dataall.modules.datasets_base.db.dataset_repositories import DatasetBaseRepository from dataall.modules.metadata_forms.db.enums import ( MetadataFormUserRoles, - MetadataFormEntityTypes, MetadataFormEnforcementScope, ) +from dataall.core.metadata_manager.metadata_form_entity_manager import MetadataFormEntityTypes from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository from functools import wraps from dataall.base.db import exceptions diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py b/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py index 1865e7e23..a44b60a89 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py @@ -7,9 +7,15 @@ from dataall.modules.datasets_base.db.dataset_repositories import DatasetBaseRepository, DatasetListRepository from dataall.modules.metadata_forms.db.enums import ( MetadataFormEnforcementScope, - MetadataFormEntityTypes, MetadataFormEnforcementSeverity, + ENTITY_SCOPE_BY_TYPE, +) +from dataall.core.metadata_manager.metadata_form_entity_manager import ( + MetadataFormEntityTypes, + MetadataFormEntityManager, + MetadataFormEntity, ) + from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository from dataall.modules.metadata_forms.services.metadata_form_access_service import MetadataFormAccessService from dataall.modules.metadata_forms.services.metadata_form_permissions import ( @@ -114,15 +120,21 @@ def get_affected_datasets(uri, rule=None): return [] @staticmethod - def form_affected_entity_object(uri, owner, label, type, rule): + def form_affected_entity_object(type, entity: MetadataFormEntity, rule): with get_context().db_engine.scoped_session() as session: attached = MetadataFormRepository.query_all_attached_metadata_forms_for_entity( session, - entityUri=uri, + entityUri=entity.get_uri(), metadataFormUri=rule.metadataFormUri, version=rule.version, ) - return {'type': type, 'name': label, 'uri': uri, 'owner': owner, 'attached': attached.first()} + return { + 'type': type, + 'name': entity.get_entity_name(), + 'uri': entity.get_uri(), + 'owner': entity.get_owner(), + 'attached': attached.first(), + } @staticmethod def get_affected_entities(uri, rule=None): @@ -135,7 +147,7 @@ def get_affected_entities(uri, rule=None): affected_entities.extend( [ MetadataFormEnforcementService.form_affected_entity_object( - o.organizationUri, o.SamlGroupName, o.label, MetadataFormEntityTypes.Organizations.value, rule + MetadataFormEntityTypes.Organizations.value, o, rule ) for o in orgs ] @@ -145,7 +157,7 @@ def get_affected_entities(uri, rule=None): affected_entities.extend( [ MetadataFormEnforcementService.form_affected_entity_object( - e.environmentUri, e.SamlGroupName, e.label, MetadataFormEntityTypes.Environments.value, rule + MetadataFormEntityTypes.Environments.value, e, rule ) for e in envs ] @@ -155,7 +167,7 @@ def get_affected_entities(uri, rule=None): affected_entities.extend( [ MetadataFormEnforcementService.form_affected_entity_object( - ds.datasetUri, ds.SamlAdminGroupName, ds.label, ds.datasetType.value + '-Dataset', rule + ds.datasetType.value + '-Dataset', ds, rule ) for ds in datasets ] @@ -169,20 +181,19 @@ def get_affected_entities(uri, rule=None): } for entity_type in entity_types: - entity_class, level, get_uri_owner_label = MetadataFormEntityTypes.get_entity_class(entity_type) + entity_class = MetadataFormEntityManager.get_resource(entity_type) + level = ENTITY_SCOPE_BY_TYPE[entity_type] all_entities = session.query(entity_class) - if level == MetadataFormEnforcementScope.Organization.value: + if level == MetadataFormEnforcementScope.Organization: all_entities = all_entities.filter(entity_class.organizationUri.in_([org.uri for org in orgs])) - if level == MetadataFormEnforcementScope.Environment.value: + if level == MetadataFormEnforcementScope.Environment: all_entities = all_entities.filter(entity_class.environmentUri.in_([env.uri for env in envs])) - if level == MetadataFormEnforcementScope.Dataset.value: + if level == MetadataFormEnforcementScope.Dataset: all_entities = all_entities.filter(entity_class.datasetUri.in_([ds.uri for ds in datasets])) all_entities = all_entities.all() affected_entities.extend( [ - MetadataFormEnforcementService.form_affected_entity_object( - *get_uri_owner_label(e), entity_type, rule - ) + MetadataFormEnforcementService.form_affected_entity_object(entity_type, e, rule) for e in all_entities ] ) diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_service.py b/backend/dataall/modules/metadata_forms/services/metadata_form_service.py index ee81743d6..34f95e4d6 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_service.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_service.py @@ -10,8 +10,8 @@ from dataall.modules.metadata_forms.db.enums import ( MetadataFormVisibility, MetadataFormFieldType, - MetadataFormEntityTypes, ) +from dataall.core.metadata_manager.metadata_form_entity_manager import MetadataFormEntityTypes from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository from dataall.modules.metadata_forms.services.metadata_form_access_service import MetadataFormAccessService diff --git a/backend/dataall/modules/mlstudio/__init__.py b/backend/dataall/modules/mlstudio/__init__.py index 2e50fb64e..9bc34be5d 100644 --- a/backend/dataall/modules/mlstudio/__init__.py +++ b/backend/dataall/modules/mlstudio/__init__.py @@ -3,7 +3,7 @@ import logging from dataall.base.loader import ImportMode, ModuleInterface - +from dataall.modules.mlstudio.db.mlstudio_models import SagemakerStudioDomain log = logging.getLogger(__name__) @@ -17,6 +17,10 @@ def is_supported(cls, modes): def __init__(self): from dataall.core.environment.services.environment_resource_manager import EnvironmentResourceManager + from dataall.core.metadata_manager.metadata_form_entity_manager import ( + MetadataFormEntityManager, + MetadataFormEntityTypes, + ) from dataall.core.stacks.db.target_type_repositories import TargetType import dataall.modules.mlstudio.api from dataall.modules.mlstudio.services.mlstudio_service import SagemakerStudioEnvironmentResource @@ -29,6 +33,7 @@ def __init__(self): TargetType('mlstudio', GET_SGMSTUDIO_USER, UPDATE_SGMSTUDIO_USER, MANAGE_SGMSTUDIO_USERS) EnvironmentResourceManager.register(SagemakerStudioEnvironmentResource()) + MetadataFormEntityManager.register(SagemakerStudioDomain, MetadataFormEntityTypes.MLStudioEntities) log.info('API of sagemaker mlstudio has been imported') diff --git a/backend/dataall/modules/mlstudio/db/mlstudio_models.py b/backend/dataall/modules/mlstudio/db/mlstudio_models.py index c56695222..4821b8914 100644 --- a/backend/dataall/modules/mlstudio/db/mlstudio_models.py +++ b/backend/dataall/modules/mlstudio/db/mlstudio_models.py @@ -6,11 +6,13 @@ from dataall.base.db import Base from dataall.base.db import Resource, utils +from dataall.core.metadata_manager.metadata_form_entity_manager import MetadataFormEntity class SagemakerStudioDomain(Resource, Base): """Describes ORM model for sagemaker ML Studio domain""" + __metaclass__ = MetadataFormEntity __tablename__ = 'sagemaker_studio_domain' environmentUri = Column(String, ForeignKey('environment.environmentUri'), nullable=False) sagemakerStudioUri = Column(String, primary_key=True, default=utils.uuid('sagemakerstudio')) @@ -25,6 +27,15 @@ class SagemakerStudioDomain(Resource, Base): vpcId = Column(String, nullable=True) subnetIds = Column(ARRAY(String), nullable=True) + def get_owner(self): + return self.SamlGroupName + + def get_entity_name(self): + return self.sagemakerStudioDomainName + + def get_uri(self): + return self.sagemakerStudioUri + class SagemakerStudioUser(Resource, Base): """Describes ORM model for sagemaker ML Studio user""" diff --git a/backend/dataall/modules/notebooks/__init__.py b/backend/dataall/modules/notebooks/__init__.py index 0fc22ea07..e76acfa01 100644 --- a/backend/dataall/modules/notebooks/__init__.py +++ b/backend/dataall/modules/notebooks/__init__.py @@ -3,6 +3,7 @@ import logging from dataall.base.loader import ImportMode, ModuleInterface +from dataall.modules.notebooks.db.notebook_models import SagemakerNotebook log = logging.getLogger(__name__) @@ -17,6 +18,7 @@ def is_supported(modes): def __init__(self): import dataall.modules.notebooks.api from dataall.core.stacks.db.target_type_repositories import TargetType + from dataall.core.metadata_manager import MetadataFormEntityManager, MetadataFormEntityTypes from dataall.modules.notebooks.services.notebook_permissions import ( GET_NOTEBOOK, UPDATE_NOTEBOOK, @@ -24,7 +26,7 @@ def __init__(self): ) TargetType('notebook', GET_NOTEBOOK, UPDATE_NOTEBOOK, MANAGE_NOTEBOOKS) - + MetadataFormEntityManager.register(SagemakerNotebook, MetadataFormEntityTypes.Notebooks.value) log.info('API of sagemaker notebooks has been imported') diff --git a/backend/dataall/modules/notebooks/db/notebook_models.py b/backend/dataall/modules/notebooks/db/notebook_models.py index e6970476c..5f0341856 100644 --- a/backend/dataall/modules/notebooks/db/notebook_models.py +++ b/backend/dataall/modules/notebooks/db/notebook_models.py @@ -5,10 +5,13 @@ from dataall.base.db import Base from dataall.base.db import Resource, utils +from dataall.core.metadata_manager.metadata_form_entity_manager import MetadataFormEntity + class SagemakerNotebook(Resource, Base): """Describes ORM model for sagemaker notebooks""" + __metaclass__ = MetadataFormEntity __tablename__ = 'sagemaker_notebook' environmentUri = Column(String, ForeignKey('environment.environmentUri'), nullable=False) notebookUri = Column(String, primary_key=True, default=utils.uuid('notebook')) @@ -22,3 +25,12 @@ class SagemakerNotebook(Resource, Base): SubnetId = Column(String, nullable=True) VolumeSizeInGB = Column(Integer, nullable=True) InstanceType = Column(String, nullable=True) + + def get_owner(self): + return self.SamlAdminGroupName + + def get_entity_name(self): + return self.NotebookInstanceName + + def get_uri(self): + return self.notebookUri diff --git a/backend/dataall/modules/redshift_datasets/__init__.py b/backend/dataall/modules/redshift_datasets/__init__.py index cd9e73f68..3e97e62a6 100644 --- a/backend/dataall/modules/redshift_datasets/__init__.py +++ b/backend/dataall/modules/redshift_datasets/__init__.py @@ -32,6 +32,10 @@ def depends_on() -> List[Type['ModuleInterface']]: def __init__(self): from dataall.modules.vote.services.vote_service import add_vote_type from dataall.modules.feed.api.registry import FeedRegistry, FeedDefinition + from dataall.core.metadata_manager.metadata_form_entity_manager import ( + MetadataFormEntityTypes, + MetadataFormEntityManager, + ) from dataall.modules.catalog.indexers.registry import GlossaryRegistry, GlossaryDefinition from dataall.core.environment.services.environment_resource_manager import EnvironmentResourceManager @@ -79,8 +83,9 @@ def __init__(self): EnvironmentResourceManager.register(RedshiftDatasetEnvironmentResource()) EnvironmentResourceManager.register(RedshiftConnectionEnvironmentResource()) + MetadataFormEntityManager.register(RedshiftDataset, MetadataFormEntityTypes.RDDatasets.value) - log.info('API of Redshift datasets has been imported') + log.info('API of Redshift datasets has been imported') class RedshiftDatasetCdkModuleInterface(ModuleInterface): diff --git a/backend/dataall/modules/s3_datasets/__init__.py b/backend/dataall/modules/s3_datasets/__init__.py index dbd4f458c..c5149a998 100644 --- a/backend/dataall/modules/s3_datasets/__init__.py +++ b/backend/dataall/modules/s3_datasets/__init__.py @@ -4,6 +4,7 @@ from typing import List, Type, Set from dataall.base.loader import ModuleInterface, ImportMode +from dataall.modules.s3_datasets.db.dataset_models import DatasetBucket log = logging.getLogger(__name__) @@ -32,6 +33,10 @@ def depends_on() -> List[Type['ModuleInterface']]: def __init__(self): # these imports are placed inside the method because they are only related to GraphQL api. from dataall.core.stacks.db.target_type_repositories import TargetType + from dataall.core.metadata_manager.metadata_form_entity_manager import ( + MetadataFormEntityTypes, + MetadataFormEntityManager, + ) from dataall.modules.vote.services.vote_service import add_vote_type from dataall.modules.feed.api.registry import FeedRegistry, FeedDefinition from dataall.modules.catalog.indexers.registry import GlossaryRegistry, GlossaryDefinition @@ -80,6 +85,10 @@ def __init__(self): TargetType('dataset', GET_DATASET, UPDATE_DATASET, MANAGE_DATASETS) EnvironmentResourceManager.register(DatasetRepository()) + MetadataFormEntityManager.register(S3Dataset, MetadataFormEntityTypes.S3Datasets.value) + MetadataFormEntityManager.register(DatasetTable, MetadataFormEntityTypes.Tables.value) + MetadataFormEntityManager.register(DatasetStorageLocation, MetadataFormEntityTypes.Folder.value) + MetadataFormEntityManager.register(DatasetBucket, MetadataFormEntityTypes.Bucket.value) log.info('API of S3 datasets has been imported') diff --git a/backend/dataall/modules/s3_datasets/db/dataset_models.py b/backend/dataall/modules/s3_datasets/db/dataset_models.py index 3e9291485..3e38f0693 100644 --- a/backend/dataall/modules/s3_datasets/db/dataset_models.py +++ b/backend/dataall/modules/s3_datasets/db/dataset_models.py @@ -4,6 +4,7 @@ from dataall.base.db import Base, Resource, utils from dataall.modules.datasets_base.db.dataset_models import DatasetBase from dataall.modules.datasets_base.services.datasets_enums import DatasetTypes +from dataall.core.metadata_manager.metadata_form_entity_manager import MetadataFormEntity class DatasetTableColumn(Resource, Base): @@ -39,6 +40,7 @@ class DatasetProfilingRun(Resource, Base): class DatasetStorageLocation(Resource, Base): + __metaclass__ = MetadataFormEntity __tablename__ = 'dataset_storage_location' datasetUri = Column(String, nullable=False) locationUri = Column(String, primary_key=True, default=utils.uuid('location')) @@ -56,8 +58,18 @@ class DatasetStorageLocation(Resource, Base): def uri(cls): return cls.locationUri + def get_owner(self): + return '' + + def get_entity_name(self): + return f'{self.S3BucketName}/{self.S3Prefix}' + + def get_uri(self): + return self.locationUri + class DatasetTable(Resource, Base): + __metaclass__ = MetadataFormEntity __tablename__ = 'dataset_table' datasetUri = Column(String, nullable=False) tableUri = Column(String, primary_key=True, default=utils.uuid('table')) @@ -82,6 +94,15 @@ class DatasetTable(Resource, Base): def uri(cls): return cls.tableUri + def get_owner(self): + return '' + + def get_entity_name(self): + return f'{self.GlueDatabaseName}.{self.GlueTableName}' + + def get_uri(self): + return self.tableUri + class S3Dataset(DatasetBase): __tablename__ = 's3_dataset' @@ -119,6 +140,7 @@ class S3Dataset(DatasetBase): class DatasetBucket(Resource, Base): + __metaclass__ = MetadataFormEntity __tablename__ = 'dataset_bucket' datasetUri = Column(String, ForeignKey('s3_dataset.datasetUri', ondelete='CASCADE'), nullable=False) bucketUri = Column(String, primary_key=True, default=utils.uuid('bucket')) @@ -137,6 +159,15 @@ class DatasetBucket(Resource, Base): def uri(cls): return cls.bucketUri + def get_owner(self): + return '' + + def get_entity_name(self): + return self.S3BucketName + + def get_uri(self): + return self.bucketUri + class DatasetTableDataFilter(Resource, Base): __tablename__ = 'data_filter' diff --git a/backend/dataall/modules/shares_base/__init__.py b/backend/dataall/modules/shares_base/__init__.py index 6967d56dc..8d9672cd3 100644 --- a/backend/dataall/modules/shares_base/__init__.py +++ b/backend/dataall/modules/shares_base/__init__.py @@ -1,6 +1,7 @@ import logging from typing import Set, List, Type from dataall.base.loader import ModuleInterface, ImportMode +from dataall.modules.shares_base.db.share_object_models import ShareObject, ShareObjectItem log = logging.getLogger(__name__) @@ -18,6 +19,13 @@ def depends_on() -> List[Type['ModuleInterface']]: def __init__(self): import dataall.modules.shares_base.api + from dataall.core.metadata_manager.metadata_form_entity_manager import ( + MetadataFormEntityManager, + MetadataFormEntityTypes, + ) + + MetadataFormEntityManager.register(ShareObject, MetadataFormEntityTypes.Share.value) + MetadataFormEntityManager.register(ShareObjectItem, MetadataFormEntityTypes.ShareItem.value) class SharesBaseAsyncHandlerModuleInterface(ModuleInterface): diff --git a/backend/dataall/modules/shares_base/db/share_object_models.py b/backend/dataall/modules/shares_base/db/share_object_models.py index b6423026b..c7a79d8d1 100644 --- a/backend/dataall/modules/shares_base/db/share_object_models.py +++ b/backend/dataall/modules/shares_base/db/share_object_models.py @@ -10,6 +10,7 @@ ShareObjectStatus, ShareItemStatus, ) +from dataall.core.metadata_manager.metadata_form_entity_manager import MetadataFormEntity def in_one_month(): @@ -21,6 +22,7 @@ def _uuid4(): class ShareObject(Base): + __metaclass__ = MetadataFormEntity __tablename__ = 'share_object' shareUri = Column(String, nullable=False, primary_key=True, default=utils.uuid('share')) datasetUri = Column(String, nullable=False) @@ -48,8 +50,18 @@ class ShareObject(Base): nonExpirable = Column(Boolean, default=False, nullable=False) shareExpirationPeriod = Column(Integer, nullable=True) + def get_owner(self): + return self.owner + + def get_entity_name(self): + return self.shareUri + + def get_uri(self): + return self.shareUri + class ShareObjectItem(Base): + __metaclass__ = MetadataFormEntity __tablename__ = 'share_object_item' shareUri = Column(String, nullable=False) shareItemUri = Column(String, default=utils.uuid('shareitem'), nullable=False, primary_key=True) @@ -70,6 +82,15 @@ class ShareObjectItem(Base): String, ForeignKey('share_object_item_data_filter.attachedDataFilterUri'), nullable=True ) + def get_owner(self): + return self.owner + + def get_entity_name(self): + return self.itemName + + def get_uri(self): + return self.shareItemUri + class ShareObjectItemDataFilter(Base): __tablename__ = 'share_object_item_data_filter' diff --git a/backend/dataall/modules/worksheets/__init__.py b/backend/dataall/modules/worksheets/__init__.py index cc22f4400..096e84b21 100644 --- a/backend/dataall/modules/worksheets/__init__.py +++ b/backend/dataall/modules/worksheets/__init__.py @@ -4,6 +4,7 @@ from typing import Type, List from dataall.base.loader import ImportMode, ModuleInterface +from dataall.modules.worksheets.db.worksheet_models import Worksheet log = logging.getLogger(__name__) @@ -17,10 +18,16 @@ def is_supported(modes): def __init__(self): from dataall.core.environment.services.environment_resource_manager import EnvironmentResourceManager + from dataall.core.metadata_manager.metadata_form_entity_manager import ( + MetadataFormEntityManager, + MetadataFormEntityTypes, + ) + from dataall.modules.worksheets.db.worksheet_repositories import WorksheetRepository import dataall.modules.worksheets.api EnvironmentResourceManager.register(WorksheetRepository()) + MetadataFormEntityManager.register(Worksheet, MetadataFormEntityTypes.Worksheets.value) log.info('API of worksheets has been imported') @staticmethod diff --git a/backend/dataall/modules/worksheets/db/worksheet_models.py b/backend/dataall/modules/worksheets/db/worksheet_models.py index 6549cb96c..1bd22f27d 100644 --- a/backend/dataall/modules/worksheets/db/worksheet_models.py +++ b/backend/dataall/modules/worksheets/db/worksheet_models.py @@ -8,6 +8,8 @@ from dataall.base.db import Base from dataall.base.db import Resource, utils +from dataall.core.metadata_manager.metadata_form_entity_manager import MetadataFormEntity + class QueryType(enum.Enum): chart = 'chart' @@ -15,6 +17,7 @@ class QueryType(enum.Enum): class Worksheet(Resource, Base): + __metaclass__ = MetadataFormEntity __tablename__ = 'worksheet' worksheetUri = Column(String, primary_key=True, default=utils.uuid('_')) SamlAdminGroupName = Column(String, nullable=False) @@ -24,6 +27,15 @@ class Worksheet(Resource, Base): lastSavedAthenaQueryIdForQuery = Column(String, nullable=True) lastSavedAthenaQueryIdForChart = Column(String, nullable=True) + def get_owner(self): + return self.SamlAdminGroupName + + def get_entity_name(self): + return self.label + + def get_uri(self): + return self.worksheetUri + class WorksheetQueryResult(Base): __tablename__ = 'worksheet_query_result' From bea88605f527ed6eedba9f9d19eb3df3c639fe7a Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Mon, 2 Dec 2024 17:23:08 +0000 Subject: [PATCH 08/16] Delete Enforcement rule Check affected entities types --- .../modules/metadata_forms/api/mutations.py | 12 +++++ .../modules/metadata_forms/api/resolvers.py | 4 ++ .../metadata_form_enforcement_service.py | 45 ++++++++++++------- .../components/MetadataFormEnforcement.js | 24 ++++++++-- .../deleteMetadataFormEnforcementRule.js | 16 +++++++ .../modules/Metadata_Forms/services/index.js | 1 + 6 files changed, 83 insertions(+), 19 deletions(-) create mode 100644 frontend/src/modules/Metadata_Forms/services/deleteMetadataFormEnforcementRule.js diff --git a/backend/dataall/modules/metadata_forms/api/mutations.py b/backend/dataall/modules/metadata_forms/api/mutations.py index 9aa3696a2..dcd9fb452 100644 --- a/backend/dataall/modules/metadata_forms/api/mutations.py +++ b/backend/dataall/modules/metadata_forms/api/mutations.py @@ -10,6 +10,7 @@ create_metadata_form_version, delete_metadata_form_version, create_mf_enforcement_rule, + delete_mf_enforcement_rule, ) createMetadataForm = gql.MutationField( @@ -115,3 +116,14 @@ resolver=create_mf_enforcement_rule, test_scope='MetadataForm', ) + +deleteMetadataFormEnforcementRule = gql.MutationField( + name='deleteMetadataFormEnforcementRule', + args=[ + gql.Argument(name='uri', type=gql.NonNullableType(gql.String)), + gql.Argument(name='rule_uri', type=gql.NonNullableType(gql.String)), + ], + type=gql.Boolean, + resolver=delete_mf_enforcement_rule, + test_scope='MetadataForm', +) diff --git a/backend/dataall/modules/metadata_forms/api/resolvers.py b/backend/dataall/modules/metadata_forms/api/resolvers.py index 7b108100b..07a15d458 100644 --- a/backend/dataall/modules/metadata_forms/api/resolvers.py +++ b/backend/dataall/modules/metadata_forms/api/resolvers.py @@ -138,3 +138,7 @@ def list_mf_affected_entities(context: Context, source, uri, filter): def get_mf_rule_home_entity_name(context: Context, source: MetadataFormEnforcementRule): return MetadataFormEnforcementService.resolve_home_entity(source.uri, source) + + +def delete_mf_enforcement_rule(context: Context, source, uri, rule_uri): + return MetadataFormEnforcementService.delete_mf_enforcement_rule(uri=uri, rule_uri=rule_uri) diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py b/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py index a44b60a89..5e0c699fb 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py @@ -144,24 +144,26 @@ def get_affected_entities(uri, rule=None): rule = MetadataFormRepository.get_mf_enforcement_rule_by_uri(session, uri) orgs = MetadataFormEnforcementService.get_affected_organizations(uri, rule) - affected_entities.extend( - [ - MetadataFormEnforcementService.form_affected_entity_object( - MetadataFormEntityTypes.Organizations.value, o, rule - ) - for o in orgs - ] - ) + if MetadataFormEntityTypes.Organizations.value in rule.entityTypes: + affected_entities.extend( + [ + MetadataFormEnforcementService.form_affected_entity_object( + MetadataFormEntityTypes.Organizations.value, o, rule + ) + for o in orgs + ] + ) envs = MetadataFormEnforcementService.get_affected_environments(uri, rule) - affected_entities.extend( - [ - MetadataFormEnforcementService.form_affected_entity_object( - MetadataFormEntityTypes.Environments.value, e, rule - ) - for e in envs - ] - ) + if MetadataFormEntityTypes.Environments.value in rule.entityTypes: + affected_entities.extend( + [ + MetadataFormEnforcementService.form_affected_entity_object( + MetadataFormEntityTypes.Environments.value, e, rule + ) + for e in envs + ] + ) datasets = MetadataFormEnforcementService.get_affected_datasets(uri, rule) affected_entities.extend( @@ -170,6 +172,7 @@ def get_affected_entities(uri, rule=None): ds.datasetType.value + '-Dataset', ds, rule ) for ds in datasets + if ds.datasetType.value + '-Dataset' in rule.entityTypes ] ) @@ -226,3 +229,13 @@ def resolve_home_entity(uri, rule=None): return EnvironmentRepository.get_environment_by_uri(session, rule.homeEntity).label if rule.level == MetadataFormEnforcementScope.Dataset.value: return DatasetBaseRepository.get_dataset_by_uri(session, rule.homeEntity).label + + @staticmethod + @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS) + @MetadataFormAccessService.can_perform(ENFORCE_METADATA_FORM) + def delete_mf_enforcement_rule(uri, rule_uri): + with get_context().db_engine.scoped_session() as session: + rule = MetadataFormRepository.get_mf_enforcement_rule_by_uri(session, rule_uri) + session.delete(rule) + session.commit() + return True diff --git a/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js b/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js index 24e97b1f3..3f867ee0b 100644 --- a/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js +++ b/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js @@ -24,7 +24,8 @@ import { SET_ERROR } from 'globalErrors'; import { createMetadataFormEnforcementRule, listMetadataFormEnforcementRules, - listEntityAffectedByEnforcementRules + listEntityAffectedByEnforcementRules, + deleteMetadataFormEnforcementRule } from '../services'; import { Formik } from 'formik'; import { LoadingButton } from '@mui/lab'; @@ -485,7 +486,21 @@ export const MetadataFormEnforcement = (props) => { setLoading(false); }; - const deleteRule = async (rule) => {}; + const deleteRule = async (rule_uri) => { + const response = await client.mutate( + deleteMetadataFormEnforcementRule(metadataForm.uri, rule_uri) + ); + if (!response.errors) { + if (selectedRule.uri === rule_uri) { + setSelectedRule(null); + setAffectedEntities([]); + } + await fetchEnforcementRules(); + } else { + const error = 'Could not delete rule'; + dispatch({ type: SET_ERROR, error }); + } + }; const fetchAffectedEntities = async ( rule, @@ -661,7 +676,10 @@ export const MetadataFormEnforcement = (props) => { onMouseOut={(e) => { e.currentTarget.style.opacity = 0.5; }} - onClick={() => deleteRule(rule.uri)} + onClick={(e) => { + e.stopPropagation(); + deleteRule(rule.uri); + }} /> )} diff --git a/frontend/src/modules/Metadata_Forms/services/deleteMetadataFormEnforcementRule.js b/frontend/src/modules/Metadata_Forms/services/deleteMetadataFormEnforcementRule.js new file mode 100644 index 000000000..431f5e5ae --- /dev/null +++ b/frontend/src/modules/Metadata_Forms/services/deleteMetadataFormEnforcementRule.js @@ -0,0 +1,16 @@ +import { gql } from 'apollo-boost'; + +export const deleteMetadataFormEnforcementRule = (uri, rule_uri) => ({ + variables: { + uri: uri, + rule_uri: rule_uri + }, + mutation: gql` + mutation deleteMetadataFormEnforcementRule( + $uri: String! + $rule_uri: String! + ) { + deleteMetadataFormEnforcementRule(uri: $uri, rule_uri: $rule_uri) + } + ` +}); diff --git a/frontend/src/modules/Metadata_Forms/services/index.js b/frontend/src/modules/Metadata_Forms/services/index.js index e231529b6..6c1a1d415 100644 --- a/frontend/src/modules/Metadata_Forms/services/index.js +++ b/frontend/src/modules/Metadata_Forms/services/index.js @@ -15,3 +15,4 @@ export * from './listMetadataFormVersions'; export * from './createMetadataFormEnforcementRule'; export * from './listMetadataFormEnforcementRules'; export * from './listMetadataFormEnforcementRuleAffectedEntities'; +export * from './deleteMetadataFormEnforcementRule'; From 859c28c130e56c6281e24bcbfbd75c5f8a452022 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Tue, 3 Dec 2024 17:12:59 +0000 Subject: [PATCH 09/16] Show warning sign if enforced MFs are missing --- .../metadata_form_entity_manager.py | 12 +- .../modules/metadata_forms/api/queries.py | 21 +++ .../modules/metadata_forms/api/resolvers.py | 8 + .../modules/metadata_forms/api/types.py | 17 +++ .../modules/metadata_forms/db/enums.py | 14 ++ .../db/metadata_form_repository.py | 14 ++ .../metadata_form_enforcement_service.py | 127 ++++++++++++++-- backend/dataall/modules/mlstudio/__init__.py | 2 +- .../Environments/views/EnvironmentView.js | 137 ++++++++++++------ .../components/MetadataFormEnforcement.js | 41 ++++-- .../modules/Metadata_Forms/services/index.js | 2 + .../services/listAffectingRulesForEntity.js | 26 ++++ .../services/listEntityTypesWithScope.js | 13 ++ .../listMetadataFormEnforcementRules.js | 1 + .../Organizations/views/OrganizationView.js | 85 ++++++++--- .../modules/S3_Datasets/views/DatasetView.js | 50 ++++++- 16 files changed, 470 insertions(+), 100 deletions(-) create mode 100644 frontend/src/modules/Metadata_Forms/services/listAffectingRulesForEntity.js create mode 100644 frontend/src/modules/Metadata_Forms/services/listEntityTypesWithScope.js diff --git a/backend/dataall/core/metadata_manager/metadata_form_entity_manager.py b/backend/dataall/core/metadata_manager/metadata_form_entity_manager.py index 6de499b18..b88346238 100644 --- a/backend/dataall/core/metadata_manager/metadata_form_entity_manager.py +++ b/backend/dataall/core/metadata_manager/metadata_form_entity_manager.py @@ -1,5 +1,5 @@ from abc import ABC -from typing import List +from typing import Dict from dataall.base.api import GraphQLEnumMapper @@ -40,7 +40,7 @@ class MetadataFormEntityManager: API for managing entities, to which MF can be attached. """ - _resources: List[MetadataFormEntity] = {} + _resources: Dict[str, MetadataFormEntity] = {} @classmethod def register(cls, resource: MetadataFormEntity, resource_key): @@ -51,3 +51,11 @@ def get_resource(cls, resource_key): if resource_key not in cls._resources: raise NotImplementedError(f'Entity {resource_key} is not registered') return cls._resources[resource_key] + + @classmethod + def is_registered(cls, resource_key): + return resource_key in cls._resources + + @classmethod + def all_registered_keys(cls): + return cls._resources.keys() diff --git a/backend/dataall/modules/metadata_forms/api/queries.py b/backend/dataall/modules/metadata_forms/api/queries.py index d39142e1c..fc82cfdbf 100644 --- a/backend/dataall/modules/metadata_forms/api/queries.py +++ b/backend/dataall/modules/metadata_forms/api/queries.py @@ -9,6 +9,8 @@ list_metadata_form_versions, list_mf_enforcement_rules, list_mf_affected_entities, + list_entity_types_with_scope, + list_affecting_rules, ) listUserMetadataForms = gql.QueryField( @@ -85,3 +87,22 @@ resolver=get_entity_metadata_form_permissions, test_scope='MetadataForm', ) + + +listEntityTypesWithScope = gql.QueryField( + name='listEntityTypesWithScope', + type=gql.ArrayType(gql.Ref('EntityTypeWithScope')), + resolver=list_entity_types_with_scope, + test_scope='MetadataForm', +) + +listRulesThatAffectEntity = gql.QueryField( + name='listRulesThatAffectEntity', + args=[ + gql.Argument('entityUri', gql.NonNullableType(gql.String)), + gql.Argument('entityType', gql.NonNullableType(gql.String)), + ], + type=gql.ArrayType(gql.Ref('AffectingRules')), + resolver=list_affecting_rules, + test_scope='MetadataForm', +) diff --git a/backend/dataall/modules/metadata_forms/api/resolvers.py b/backend/dataall/modules/metadata_forms/api/resolvers.py index 07a15d458..f411036ae 100644 --- a/backend/dataall/modules/metadata_forms/api/resolvers.py +++ b/backend/dataall/modules/metadata_forms/api/resolvers.py @@ -142,3 +142,11 @@ def get_mf_rule_home_entity_name(context: Context, source: MetadataFormEnforceme def delete_mf_enforcement_rule(context: Context, source, uri, rule_uri): return MetadataFormEnforcementService.delete_mf_enforcement_rule(uri=uri, rule_uri=rule_uri) + + +def list_entity_types_with_scope(context: Context, source): + return MetadataFormEnforcementService.list_supported_entity_types() + + +def list_affecting_rules(context: Context, source, entityUri, entityType): + return MetadataFormEnforcementService.get_rules_that_affect_entity(entity_type=entityType, entity_uri=entityUri) diff --git a/backend/dataall/modules/metadata_forms/api/types.py b/backend/dataall/modules/metadata_forms/api/types.py index f88ae67ca..13ad3319d 100644 --- a/backend/dataall/modules/metadata_forms/api/types.py +++ b/backend/dataall/modules/metadata_forms/api/types.py @@ -122,6 +122,7 @@ fields=[ gql.Field(name='uri', type=gql.String), gql.Field(name='level', type=gql.String), + gql.Field(name='severity', type=gql.String), gql.Field(name='homeEntity', type=gql.String), gql.Field(name='homeEntityName', type=gql.String, resolver=get_mf_rule_home_entity_name), gql.Field( @@ -161,3 +162,19 @@ gql.Field(name='attached', type=gql.Ref('AttachedMetadataForm')), ], ) + +EntityTypeWithScope = gql.ObjectType( + name='EntityTypeWithScope', + fields=[ + gql.Field(name='name', type=gql.String), + gql.Field(name='levels', type=gql.ArrayType(gql.String)), + ], +) + +AffectingRules = gql.ObjectType( + name='AffectingRules', + fields=MetadataFormEnforcementRule.fields[:] + + [ + gql.Field(name='attached', type=gql.String), + ], +) diff --git a/backend/dataall/modules/metadata_forms/db/enums.py b/backend/dataall/modules/metadata_forms/db/enums.py index 7e74b26e4..6fd29d6b3 100644 --- a/backend/dataall/modules/metadata_forms/db/enums.py +++ b/backend/dataall/modules/metadata_forms/db/enums.py @@ -27,6 +27,20 @@ class MetadataFormEnforcementScope(GraphQLEnumMapper): Organization = 'Organizational Level' Global = 'Global' + @classmethod + def _ordering(cls): + return ['Global', 'Organization', 'Environment', 'Dataset'] + + def __lt__(self, other): + if self.__class__ is other.__class__: + return self.__class__._ordering().index(self._name_) > self.__class__._ordering().index(other._name_) + return NotImplemented + + def __gt__(self, other): + if self.__class__ is other.__class__: + return self.__class__._ordering().index(self._name_) < self.__class__._ordering().index(other._name_) + return NotImplemented + class MetadataFormUserRoles(GraphQLEnumMapper): Owner = 'Owner' diff --git a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py index a0b5c208e..e977ecd94 100644 --- a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py +++ b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py @@ -367,3 +367,17 @@ def list_mf_enforcement_rules(session, uri): return ( session.query(MetadataFormEnforcementRule).filter(MetadataFormEnforcementRule.metadataFormUri == uri).all() ) + + @staticmethod + def query_all_enforcement_rules(session, filter): + all_rules = session.query(MetadataFormEnforcementRule) + if filter: + if filter.get('entity_types'): + for etype in filter.get('entity_types'): + all_rules = all_rules.filter(MetadataFormEnforcementRule.entityTypes.any(etype)) + if filter.get('level'): + all_rules = all_rules.filter(MetadataFormEnforcementRule.level == filter.get('level')) + if filter.get('home_entity'): + all_rules = all_rules.filter(MetadataFormEnforcementRule.homeEntity == filter.get('home_entity')) + + return all_rules.all() diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py b/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py index 5e0c699fb..415aeaa37 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_enforcement_service.py @@ -44,6 +44,15 @@ def validate_create_request(data): if 'entityTypes' not in data: raise exceptions.RequiredParameter('entityTypes') + else: + for entity_type in data.get('entityTypes'): + if not MetadataFormEntityManager.is_registered(entity_type): + raise exceptions.InvalidInput( + param_name='entityType', + param_value=entity_type, + constraint='must be registered in MetadataFormEntityManager', + ) + # check that values are valid for the enums MetadataFormEnforcementScope(data.get('level')) MetadataFormEnforcementSeverity(data.get('severity')) @@ -120,20 +129,23 @@ def get_affected_datasets(uri, rule=None): return [] @staticmethod - def form_affected_entity_object(type, entity: MetadataFormEntity, rule): + def get_attachement_for_rule(rule, entityUri): with get_context().db_engine.scoped_session() as session: - attached = MetadataFormRepository.query_all_attached_metadata_forms_for_entity( + return MetadataFormRepository.query_all_attached_metadata_forms_for_entity( session, - entityUri=entity.get_uri(), + entityUri=entityUri, metadataFormUri=rule.metadataFormUri, version=rule.version, - ) + ).first() + + @staticmethod + def form_affected_entity_object(type, entity: MetadataFormEntity, rule): return { 'type': type, 'name': entity.get_entity_name(), 'uri': entity.get_uri(), 'owner': entity.get_owner(), - 'attached': attached.first(), + 'attached': MetadataFormEnforcementService.get_attachement_for_rule(rule, entity.get_uri()), } @staticmethod @@ -165,16 +177,21 @@ def get_affected_entities(uri, rule=None): ] ) - datasets = MetadataFormEnforcementService.get_affected_datasets(uri, rule) - affected_entities.extend( - [ - MetadataFormEnforcementService.form_affected_entity_object( - ds.datasetType.value + '-Dataset', ds, rule - ) - for ds in datasets - if ds.datasetType.value + '-Dataset' in rule.entityTypes - ] - ) + datasets = [] + if MetadataFormEntityManager.is_registered( + MetadataFormEntityTypes.S3Datasets.value + ) or MetadataFormEntityManager.is_registered(MetadataFormEntityTypes.RDDatasets.value): + datasets = MetadataFormEnforcementService.get_affected_datasets(uri, rule) + affected_entities.extend( + [ + MetadataFormEnforcementService.form_affected_entity_object( + ds.datasetType.value + '-Dataset', ds, rule + ) + for ds in datasets + if ds.datasetType.value + '-Dataset' in rule.entityTypes + and MetadataFormEntityManager.is_registered(ds.datasetType.value + '-Dataset') + ] + ) entity_types = set(rule.entityTypes[:]) - { MetadataFormEntityTypes.Organizations.value, @@ -239,3 +256,83 @@ def delete_mf_enforcement_rule(uri, rule_uri): session.delete(rule) session.commit() return True + + @staticmethod + def list_supported_entity_types(): + supported_entity_types = [] + all_levels_scope = [level for level in MetadataFormEnforcementScope] + for entity_type in MetadataFormEntityManager.all_registered_keys(): + entity_scope = ENTITY_SCOPE_BY_TYPE[entity_type] + levels = [level.value for level in all_levels_scope if level > entity_scope or level == entity_scope] + supported_entity_types.append({'name': entity_type, 'levels': levels}) + return supported_entity_types + + @staticmethod + def get_rules_that_affect_entity(entity_type, entity_uri): + if not MetadataFormEntityManager.is_registered(entity_type): + return [] + all_rules = [] + entity_class = MetadataFormEntityManager.get_resource(entity_type) + entity_scope = ENTITY_SCOPE_BY_TYPE[entity_type] + with get_context().db_engine.scoped_session() as session: + entity = session.query(entity_class).get(entity_uri) + parent_dataset_uri, parent_env_uri, parent_org_uri = None, None, None + + if entity_scope == MetadataFormEnforcementScope.Dataset: + parent_dataset_uri = entity.datasetUri + ds = DatasetBaseRepository.get_dataset_by_uri(session, parent_dataset_uri) + parent_env_uri = ds.environmentUri + parent_org_uri = ds.organizationUri + if entity_scope == MetadataFormEnforcementScope.Environment: + parent_env_uri = entity.environmentUri + env = EnvironmentRepository.get_environment_by_uri(session, parent_env_uri) + parent_org_uri = env.organizationUri + if entity_scope == MetadataFormEnforcementScope.Organization: + parent_org_uri = entity.organizationUri + + all_rules.extend( + MetadataFormRepository.query_all_enforcement_rules( + session=session, + filter={'entity_types': [entity_type], 'level': MetadataFormEnforcementScope.Global.value}, + ) + ) + if entity_scope < MetadataFormEnforcementScope.Global: + all_rules.extend( + MetadataFormRepository.query_all_enforcement_rules( + session=session, + filter={ + 'entity_types': [entity_type], + 'level': MetadataFormEnforcementScope.Organization.value, + 'home_entity': parent_org_uri, + }, + ) + ) + + if entity_scope < MetadataFormEnforcementScope.Organization: + all_rules.extend( + MetadataFormRepository.query_all_enforcement_rules( + session=session, + filter={ + 'entity_types': [entity_type], + 'level': MetadataFormEnforcementScope.Environment.value, + 'home_entity': parent_env_uri, + }, + ) + ) + if entity_scope < MetadataFormEnforcementScope.Environment: + all_rules.extend( + MetadataFormRepository.query_all_enforcement_rules( + session=session, + filter={ + 'entity_types': [entity_type], + 'level': MetadataFormEnforcementScope.Dataset.value, + 'home_entity': parent_dataset_uri, + }, + ) + ) + + for r in all_rules: + attached = MetadataFormEnforcementService.get_attachement_for_rule(r, entity_uri) + r.attached = attached.uri if attached else None + + return all_rules diff --git a/backend/dataall/modules/mlstudio/__init__.py b/backend/dataall/modules/mlstudio/__init__.py index 9bc34be5d..c3cde629c 100644 --- a/backend/dataall/modules/mlstudio/__init__.py +++ b/backend/dataall/modules/mlstudio/__init__.py @@ -33,7 +33,7 @@ def __init__(self): TargetType('mlstudio', GET_SGMSTUDIO_USER, UPDATE_SGMSTUDIO_USER, MANAGE_SGMSTUDIO_USERS) EnvironmentResourceManager.register(SagemakerStudioEnvironmentResource()) - MetadataFormEntityManager.register(SagemakerStudioDomain, MetadataFormEntityTypes.MLStudioEntities) + MetadataFormEntityManager.register(SagemakerStudioDomain, MetadataFormEntityTypes.MLStudioEntities.value) log.info('API of sagemaker mlstudio has been imported') diff --git a/frontend/src/modules/Environments/views/EnvironmentView.js b/frontend/src/modules/Environments/views/EnvironmentView.js index eccf64e2d..bf8272cd2 100644 --- a/frontend/src/modules/Environments/views/EnvironmentView.js +++ b/frontend/src/modules/Environments/views/EnvironmentView.js @@ -1,10 +1,12 @@ import { + BallotOutlined, FolderOpen, Info, LocalOffer, NotificationsActive, SupervisedUserCircleRounded, - Warning + Warning, + WarningAmber } from '@mui/icons-material'; import { Box, @@ -50,50 +52,7 @@ import { } from '../components'; import { ModuleNames, isModuleEnabled } from 'utils'; import { MetadataAttachment } from '../../Metadata_Forms/components'; - -const tabs = [ - { label: 'Overview', value: 'overview', icon: }, - { - label: 'Teams', - value: 'teams', - icon: - }, - { - label: 'Metadata', - value: 'metadata', - active: isModuleEnabled(ModuleNames.METADATA_FORMS) - }, - { - label: 'Datasets', - value: 'datasets', - icon: , - active: isModuleEnabled( - ModuleNames.S3_DATASETS || ModuleNames.REDSHIFT_DATASETS - ) - }, - { - label: 'Connections', - value: 'connections', - icon: , - active: isModuleEnabled(ModuleNames.REDSHIFT_DATASETS) - }, - { - label: 'ML Studio Domain', - value: 'mlstudio', - icon: , - active: isModuleEnabled(ModuleNames.MLSTUDIO) - }, - { label: 'Networks', value: 'networks', icon: }, - { - label: 'Subscriptions', - value: 'subscriptions', - icon: - }, - { label: 'Tags', value: 'tags', icon: }, - { label: 'Stack', value: 'stack', icon: } -]; - -const activeTabs = tabs.filter((tab) => tab.active !== false); +import { listRulesThatAffectEntity } from '../../Metadata_Forms/services'; const EnvironmentView = () => { const dispatch = useDispatch(); @@ -108,6 +67,75 @@ const EnvironmentView = () => { const [isAdmin, setIsAdmin] = useState(false); const [isArchiveObjectModalOpen, setIsArchiveObjectModalOpen] = useState(false); + const [affectingMFRules, setAffectingMFRules] = useState([]); + + const getTabs = () => { + const tabs = [ + { label: 'Overview', value: 'overview', icon: }, + { + label: 'Teams', + value: 'teams', + icon: + }, + { + label: ( + <> + Metadata{' '} + {affectingMFRules.filter( + (r) => r.severity === 'Mandatory' && !r.attached + ).length > 0 ? ( + + ) : null} + {affectingMFRules.filter( + (r) => r.severity === 'Mandatory' && !r.attached + ).length === 0 && + affectingMFRules.filter( + (r) => r.severity === 'Recommended' && !r.attached + ).length > 0 ? ( + + ) : null} + + ), + value: 'metadata', + icon: , + active: isModuleEnabled(ModuleNames.METADATA_FORMS) + }, + { + label: 'Datasets', + value: 'datasets', + icon: , + active: isModuleEnabled( + ModuleNames.S3_DATASETS || ModuleNames.REDSHIFT_DATASETS + ) + }, + { + label: 'Connections', + value: 'connections', + icon: , + active: isModuleEnabled(ModuleNames.REDSHIFT_DATASETS) + }, + { + label: 'ML Studio Domain', + value: 'mlstudio', + icon: , + active: isModuleEnabled(ModuleNames.MLSTUDIO) + }, + { + label: 'Networks', + value: 'networks', + icon: + }, + { + label: 'Subscriptions', + value: 'subscriptions', + icon: + }, + { label: 'Tags', value: 'tags', icon: }, + { label: 'Stack', value: 'stack', icon: } + ]; + + return tabs.filter((tab) => tab.active !== false); + }; const handleArchiveObjectModalOpen = () => { setIsArchiveObjectModalOpen(true); }; @@ -119,6 +147,19 @@ const EnvironmentView = () => { setCurrentTab(value); }; + const fetchAffectingMFRules = async () => { + if (isModuleEnabled(ModuleNames.METADATA_FORMS)) { + const response = await client.query( + listRulesThatAffectEntity(params.uri, 'Environment') + ); + if ( + !response.errors && + response.data.listRulesThatAffectEntity !== null + ) { + setAffectingMFRules(response.data.listRulesThatAffectEntity); + } + } + }; const archiveEnv = async () => { const response = await client.mutate( archiveEnvironment({ @@ -165,6 +206,9 @@ const EnvironmentView = () => { useEffect(() => { if (client) { fetchItem().catch((e) => dispatch({ type: SET_ERROR, error: e.message })); + fetchAffectingMFRules().catch((e) => + dispatch({ type: SET_ERROR, error: e.message }) + ); } }, [client, dispatch, fetchItem]); @@ -257,7 +301,7 @@ const EnvironmentView = () => { value={currentTab} variant="fullWidth" > - {activeTabs.map((tab) => ( + {getTabs().map((tab) => ( { )} {currentTab === 'teams' && } diff --git a/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js b/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js index 3f867ee0b..4de2e2d01 100644 --- a/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js +++ b/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js @@ -25,7 +25,8 @@ import { createMetadataFormEnforcementRule, listMetadataFormEnforcementRules, listEntityAffectedByEnforcementRules, - deleteMetadataFormEnforcementRule + deleteMetadataFormEnforcementRule, + listEntityTypesWithScope } from '../services'; import { Formik } from 'formik'; import { LoadingButton } from '@mui/lab'; @@ -53,6 +54,7 @@ const CreateEnforcementRuleModal = (props) => { const [environmentOptions, setEnvironmentOptions] = useState([]); const [organizationOptions, setOrganizationOptions] = useState([]); const [datasetOptions, setDatasetOptions] = useState([]); + const [entityTypes, setEntityTypes] = useState([...entityTypesOptions]); const enforcementScopeDict = {}; for (const option of enforcementScopeOptions) { @@ -244,6 +246,11 @@ const CreateEnforcementRuleModal = (props) => { defaultValue={enforcementScopeDict['Global']} onChange={(event, value) => { setFieldValue('scope', value.value); + setEntityTypes( + entityTypesOptions.filter((entityType) => { + return entityType.levels.includes(value.value); + }) + ); }} renderInput={(params) => ( { - {entityTypesOptions.map((entityType) => ( + {entityTypes.map((entityType) => ( { if (value) { setFieldValue('entityTypes', [ ...values.entityTypes, - entityType.value + entityType.name ]); } else { setFieldValue( 'entityTypes', values.entityTypes.filter( - (item) => item !== entityType.value + (item) => item !== entityType.name ) ); } }} /> } - label={entityType.value} + label={entityType.name} /> ))} @@ -462,6 +469,20 @@ export const MetadataFormEnforcement = (props) => { } ]; + const fetchEntityTypesWithScope = async () => { + const response = await client.query(listEntityTypesWithScope()); + if ( + !response.errors && + response.data && + response.data.listEntityTypesWithScope + ) { + setEntityTypesOptions(response.data.listEntityTypesWithScope); + } else { + const error = 'Could not fetch entity types'; + dispatch({ type: SET_ERROR, error }); + } + }; + const fetchEnforcementRules = async () => { setLoading(true); const response = await client.query( @@ -534,16 +555,9 @@ export const MetadataFormEnforcement = (props) => { const fetchEnforcementEnums = async () => { const enums = await fetchEnums(client, [ - 'MetadataFormEntityTypes', 'MetadataFormEnforcementSeverity', 'MetadataFormEnforcementScope' ]); - if (enums['MetadataFormEntityTypes'].length > 0) { - setEntityTypesOptions(enums['MetadataFormEntityTypes']); - } else { - const error = 'Could not fetch entity type options'; - dispatch({ type: SET_ERROR, error }); - } if (enums['MetadataFormEnforcementSeverity'].length > 0) { setSeverityOptions(enums['MetadataFormEnforcementSeverity']); } else { @@ -566,6 +580,9 @@ export const MetadataFormEnforcement = (props) => { fetchEnforcementEnums().catch((e) => dispatch({ type: SET_ERROR, error: e.message }) ); + fetchEntityTypesWithScope().catch((e) => + dispatch({ type: SET_ERROR, error: e.message }) + ); } }, [client]); diff --git a/frontend/src/modules/Metadata_Forms/services/index.js b/frontend/src/modules/Metadata_Forms/services/index.js index 6c1a1d415..65533dd53 100644 --- a/frontend/src/modules/Metadata_Forms/services/index.js +++ b/frontend/src/modules/Metadata_Forms/services/index.js @@ -16,3 +16,5 @@ export * from './createMetadataFormEnforcementRule'; export * from './listMetadataFormEnforcementRules'; export * from './listMetadataFormEnforcementRuleAffectedEntities'; export * from './deleteMetadataFormEnforcementRule'; +export * from './listEntityTypesWithScope'; +export * from './listAffectingRulesForEntity'; diff --git a/frontend/src/modules/Metadata_Forms/services/listAffectingRulesForEntity.js b/frontend/src/modules/Metadata_Forms/services/listAffectingRulesForEntity.js new file mode 100644 index 000000000..56d7f1b60 --- /dev/null +++ b/frontend/src/modules/Metadata_Forms/services/listAffectingRulesForEntity.js @@ -0,0 +1,26 @@ +import { gql } from 'apollo-boost'; + +export const listRulesThatAffectEntity = (uri, type) => ({ + variables: { + entityUri: uri, + entityType: type + }, + query: gql` + query listRulesThatAffectEntity($entityUri: String!, $entityType: String!) { + listRulesThatAffectEntity( + entityUri: $entityUri + entityType: $entityType + ) { + uri + level + homeEntity + homeEntityName + entityTypes + metadataFormUri + version + severity + attached + } + } + ` +}); diff --git a/frontend/src/modules/Metadata_Forms/services/listEntityTypesWithScope.js b/frontend/src/modules/Metadata_Forms/services/listEntityTypesWithScope.js new file mode 100644 index 000000000..558482ac4 --- /dev/null +++ b/frontend/src/modules/Metadata_Forms/services/listEntityTypesWithScope.js @@ -0,0 +1,13 @@ +import { gql } from 'apollo-boost'; + +export const listEntityTypesWithScope = () => ({ + variables: {}, + query: gql` + query listEntityTypesWithScope { + listEntityTypesWithScope { + name + levels + } + } + ` +}); diff --git a/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRules.js b/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRules.js index b86a1cd05..d8c719b79 100644 --- a/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRules.js +++ b/frontend/src/modules/Metadata_Forms/services/listMetadataFormEnforcementRules.js @@ -9,6 +9,7 @@ export const listMetadataFormEnforcementRules = (uri) => ({ listMetadataFormEnforcementRules(uri: $uri) { uri level + severity homeEntity homeEntityName entityTypes diff --git a/frontend/src/modules/Organizations/views/OrganizationView.js b/frontend/src/modules/Organizations/views/OrganizationView.js index 758409c46..59b1aa4b8 100644 --- a/frontend/src/modules/Organizations/views/OrganizationView.js +++ b/frontend/src/modules/Organizations/views/OrganizationView.js @@ -1,8 +1,10 @@ import { ArchiveOutlined, + BallotOutlined, Info, SupervisedUserCircleRounded, - Warning + Warning, + WarningAmber } from '@mui/icons-material'; import { Box, @@ -40,24 +42,7 @@ import { } from '../components'; import { MetadataAttachment } from '../../Metadata_Forms/components'; import { isModuleEnabled, ModuleNames } from '../../../utils'; - -const tabs = [ - { label: 'Overview', value: 'overview', icon: }, - { label: 'Environments', value: 'environments', icon: }, - { - label: 'Metadata', - value: 'metadata', - icon: , - active: isModuleEnabled(ModuleNames.METADATA_FORMS) - }, - { - label: 'Teams', - value: 'teams', - icon: - } -]; - -const activeTabs = tabs.filter((tab) => tab.active !== false); +import { listRulesThatAffectEntity } from '../../Metadata_Forms/services'; const OrganizationView = () => { const { settings } = useSettings(); @@ -72,10 +57,66 @@ const OrganizationView = () => { const [loading, setLoading] = useState(true); const [isArchiveObjectModalOpen, setIsArchiveObjectModalOpen] = useState(false); + + const [affectingMFRules, setAffectingMFRules] = useState([]); + + const getTabs = () => { + const tabs = [ + { label: 'Overview', value: 'overview', icon: }, + { + label: 'Environments', + value: 'environments', + icon: + }, + { + label: ( + <> + Metadata{' '} + {affectingMFRules.filter( + (r) => r.severity === 'Mandatory' && !r.attached + ).length > 0 ? ( + + ) : null} + {affectingMFRules.filter( + (r) => r.severity === 'Mandatory' && !r.attached + ).length === 0 && + affectingMFRules.filter( + (r) => r.severity === 'Recommended' && !r.attached + ).length > 0 ? ( + + ) : null} + + ), + value: 'metadata', + icon: , + active: isModuleEnabled(ModuleNames.METADATA_FORMS) + }, + { + label: 'Teams', + value: 'teams', + icon: + } + ]; + + return tabs.filter((tab) => tab.active !== false); + }; const handleArchiveObjectModalOpen = () => { setIsArchiveObjectModalOpen(true); }; + const fetchAffectingMFRules = async () => { + if (isModuleEnabled(ModuleNames.METADATA_FORMS)) { + const response = await client.query( + listRulesThatAffectEntity(params.uri, 'Environment') + ); + if ( + !response.errors && + response.data.listRulesThatAffectEntity !== null + ) { + setAffectingMFRules(response.data.listRulesThatAffectEntity); + } + } + }; const handleArchiveObjectModalClose = () => { setIsArchiveObjectModalOpen(false); }; @@ -120,6 +161,9 @@ const OrganizationView = () => { useEffect(() => { if (client) { fetchItem().catch((e) => dispatch({ type: SET_ERROR, error: e.message })); + fetchAffectingMFRules().catch((e) => + dispatch({ type: SET_ERROR, error: e.message }) + ); } }, [client, dispatch, fetchItem]); @@ -211,7 +255,7 @@ const OrganizationView = () => { value={currentTab} variant="fullWidth" > - {activeTabs.map((tab) => ( + {getTabs().map((tab) => ( { )} diff --git a/frontend/src/modules/S3_Datasets/views/DatasetView.js b/frontend/src/modules/S3_Datasets/views/DatasetView.js index f47b376c1..843dd1303 100644 --- a/frontend/src/modules/S3_Datasets/views/DatasetView.js +++ b/frontend/src/modules/S3_Datasets/views/DatasetView.js @@ -1,11 +1,13 @@ import { + BallotOutlined, ForumOutlined, Info, LocalOffer, LockOpen, ShareOutlined, Upload, - ViewArrayOutlined + ViewArrayOutlined, + WarningAmber } from '@mui/icons-material'; import { Box, @@ -47,6 +49,7 @@ import { import { isFeatureEnabled, isModuleEnabled, ModuleNames } from 'utils'; import { RequestAccessModal } from 'modules/Catalog/components'; import { MetadataAttachment } from '../../Metadata_Forms/components'; +import { listRulesThatAffectEntity } from '../../Metadata_Forms/services'; const DatasetView = () => { const dispatch = useDispatch(); @@ -63,6 +66,7 @@ const DatasetView = () => { const [isUpVoted, setIsUpVoted] = useState(false); const [upVotes, setUpvotes] = useState(null); const [openFeed, setOpenFeed] = useState(false); + const [affectingMFRules, setAffectingMFRules] = useState([]); const getTabs = () => { const tabs = [ { @@ -72,9 +76,26 @@ const DatasetView = () => { }, { label: 'Overview', value: 'overview', icon: }, { - label: 'Metadata', + label: ( + <> + Metadata{' '} + {affectingMFRules.filter( + (r) => r.severity === 'Mandatory' && !r.attached + ).length > 0 ? ( + + ) : null} + {affectingMFRules.filter( + (r) => r.severity === 'Mandatory' && !r.attached + ).length === 0 && + affectingMFRules.filter( + (r) => r.severity === 'Recommended' && !r.attached + ).length > 0 ? ( + + ) : null} + + ), value: 'metadata', - icon: , + icon: , active: isModuleEnabled(ModuleNames.METADATA_FORMS) } ]; @@ -107,6 +128,20 @@ const DatasetView = () => { return tabs.filter((tab) => tab.active !== false); }; + const fetchAffectingMFRules = async () => { + if (isModuleEnabled(ModuleNames.METADATA_FORMS)) { + const response = await client.query( + listRulesThatAffectEntity(params.uri, 'S3-Dataset') + ); + if ( + !response.errors && + response.data.listRulesThatAffectEntity !== null + ) { + setAffectingMFRules(response.data.listRulesThatAffectEntity); + } + } + }; + const handleDeleteObjectModalOpen = () => { setIsDeleteObjectModalOpen(true); }; @@ -184,6 +219,9 @@ const DatasetView = () => { dispatch({ type: SET_ERROR, error: e.message }) ); fetchItem().catch((e) => dispatch({ type: SET_ERROR, error: e.message })); + fetchAffectingMFRules().catch((e) => + dispatch({ type: SET_ERROR, error: e.message }) + ); } }, [client, fetchItem, getUserDatasetVote, dispatch, params.uri]); @@ -362,7 +400,11 @@ const DatasetView = () => { )} {currentTab === 'metadata' && ( - + )} {currentTab === 'overview' && ( From e1bb28d661b00ae8c90da516e65d0ce9b095f49b Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Wed, 4 Dec 2024 12:42:18 +0000 Subject: [PATCH 10/16] Show missing MFs --- .../components/metadataAttachment.js | 117 ++++++++++++++++-- 1 file changed, 110 insertions(+), 7 deletions(-) diff --git a/frontend/src/modules/Metadata_Forms/components/metadataAttachment.js b/frontend/src/modules/Metadata_Forms/components/metadataAttachment.js index 5f563b4dc..cc9ba02f6 100644 --- a/frontend/src/modules/Metadata_Forms/components/metadataAttachment.js +++ b/frontend/src/modules/Metadata_Forms/components/metadataAttachment.js @@ -31,7 +31,7 @@ import DoNotDisturbAltOutlinedIcon from '@mui/icons-material/DoNotDisturbAltOutl import DeleteIcon from '@mui/icons-material/DeleteOutlined'; export const MetadataAttachment = (props) => { - const { entityType, entityUri } = props; + const { entityType, entityUri, affectingRules } = props; const client = useClient(); const dispatch = useDispatch(); const [selectedForm, setSelectedForm] = useState(null); @@ -52,6 +52,7 @@ export const MetadataAttachment = (props) => { }); const [addNewForm, setAddNewForm] = useState(false); const [availableForms, setAvailableForms] = useState([]); + const [missingRules, setMissingRules] = useState([]); const fetchAvailableForms = async () => { const response = await client.query( @@ -79,6 +80,23 @@ export const MetadataAttachment = (props) => { setLoading(true); const response = await client.query(listAttachedMetadataForms(filter)); if (!response.errors) { + response.data.listAttachedMetadataForms.nodes.forEach((form) => { + const r = affectingRules.find( + (r) => r.metadataFormUri === form.metadataForm.uri + ); + if (r) { + form.required = r.severity; + form.required_version = r.version; + } + }); + const missing = affectingRules.filter( + (r) => + !response.data.listAttachedMetadataForms.nodes.find( + (form) => r.metadataFormUri === form.metadataForm.uri + ) + ); + + setMissingRules([...missing]); setFormsList(response.data.listAttachedMetadataForms.nodes); if ( response.data.listAttachedMetadataForms.nodes.length > 0 && @@ -191,7 +209,7 @@ export const MetadataAttachment = (props) => { return ( - + {canEdit && ( <> @@ -217,6 +235,7 @@ export const MetadataAttachment = (props) => { { if (value) { setSelectedForm(value.form); @@ -236,6 +255,69 @@ export const MetadataAttachment = (props) => { /> )} + {missingRules.length > 0 && + missingRules.map((rule) => ( + + + + + {rule.metadataFormUri + ' v.' + rule.version} + + + + + {'Missing ' + rule.severity} + + + + {canEdit && ( + { + e.currentTarget.style.opacity = 1; + }} + onMouseOut={(e) => { + e.currentTarget.style.opacity = 0.5; + }} + onClick={async () => { + { + setSelectedForm( + availableForms.find( + (form) => form.value === rule.metadataFormUri + ).form + ); + setEditMode(false); + setAddNewForm(true); + setValues({}); + await fetchFields(rule.metadataFormUri); + } + }} + /> + )} + + + + ))} + {formsList.length > 0 ? ( formsList.map((attachedForm) => ( { { setSelectedAttachedForm(attachedForm); setEditMode(false); @@ -275,8 +357,29 @@ export const MetadataAttachment = (props) => { attachedForm.version} - - + + {attachedForm.required && ( + + {attachedForm.required}{' '} + {attachedForm.version < attachedForm.required_version + ? 'v. ' + attachedForm.required_version + : ''} + + )} + + {canEdit && ( { )} - + {loadingFields && ( Date: Wed, 4 Dec 2024 13:56:19 +0000 Subject: [PATCH 11/16] Refetch rules after create --- .../components/MetadataFormEnforcement.js | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js b/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js index 4de2e2d01..4fccb56d2 100644 --- a/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js +++ b/frontend/src/modules/Metadata_Forms/components/MetadataFormEnforcement.js @@ -39,7 +39,8 @@ import { DataGrid } from '@mui/x-data-grid'; const CreateEnforcementRuleModal = (props) => { const { - onClose, + onCancel, + onSubmit, open, metadataForm, severityOptions, @@ -163,17 +164,18 @@ const CreateEnforcementRuleModal = (props) => { if (response.errors) { setStatus({ success: false }); setErrors({ submit: response.errors[0].message }); + const error = response.errors[0].message; + dispatch({ type: SET_ERROR, error }); setSubmitting(false); } else { setStatus({ success: true }); setSubmitting(false); - props.refetch(); - onClose(); + onSubmit(); } } return ( - + {