Metadata form enforcement

backend/dataall/core/environment/db/environment_models.py

@@ -7,9 +7,11 @@
 from dataall.base.db import Resource, Base, utils
 
 from dataall.core.environment.api.enums import EnvironmentPermission, EnvironmentType
+from dataall.core.metadata_manager import MetadataFormEntityManager, MetadataFormEntity, MetadataFormEntityTypes
 
 
 class Environment(Resource, Base):
+    __metaclass__ = MetadataFormEntity
     __tablename__ = 'environment'
     organizationUri = Column(String, nullable=False)
     environmentUri = Column(String, primary_key=True, default=utils.uuid('environment'))
@@ -40,8 +42,18 @@ class Environment(Resource, Base):
     subscriptionsConsumersTopicName = Column(String)
     subscriptionsConsumersTopicImported = Column(Boolean, default=False)
 
+    def get_uri(self):
+        return self.environmentUri
+
+    def get_owner(self):
+        return self.SamlGroupName
+
+    def get_entity_name(self):
+        return self.label
+
 
 class EnvironmentGroup(Base):
+    __metaclass__ = MetadataFormEntity
     __tablename__ = 'environment_group_permission'
     groupUri = Column(String, primary_key=True)
     environmentUri = Column(String, primary_key=True)
@@ -58,6 +70,15 @@ class EnvironmentGroup(Base):
     # environmentRole is the role of the entity (group or user) in the Environment
     groupRoleInEnvironment = Column(String, nullable=False, default=EnvironmentPermission.Invited.value)
 
+    def get_uri(self):
+        return f'{self.groupUri}-{self.environmentUri}'
+
+    def get_owner(self):
+        return self.invitedBy
+
+    def get_entity_name(self):
+        return f'{self.groupUri}-{self.environmentUri}'
+
 
 class EnvironmentParameter(Base):
     """Represent the parameter of the environment"""
@@ -78,6 +99,7 @@ def __repr__(self):
 
 
 class ConsumptionRole(Base):
+    __metaclass__ = MetadataFormEntity
     __tablename__ = 'consumptionrole'
     consumptionRoleUri = Column(String, primary_key=True, default=utils.uuid('group'))
     consumptionRoleName = Column(String, nullable=False)
@@ -89,3 +111,17 @@ class ConsumptionRole(Base):
     created = Column(DateTime, default=datetime.datetime.now)
     updated = Column(DateTime, onupdate=datetime.datetime.now)
     deleted = Column(DateTime)
+
+    def get_uri(self):
+        return self.consumptionRoleUri
+
+    def get_owner(self):
+        return self.groupUri
+
+    def get_entity_name(self):
+        return f'{self.consumptionRoleName}-{self.environmentUri}'
+
+
+MetadataFormEntityManager.register(Environment, MetadataFormEntityTypes.Environments.value)
+MetadataFormEntityManager.register(ConsumptionRole, MetadataFormEntityTypes.ConsumptionRoles.value)
+MetadataFormEntityManager.register(EnvironmentGroup, MetadataFormEntityTypes.EnvironmentTeams.value)

backend/dataall/core/environment/db/environment_repositories.py

@@ -326,3 +326,7 @@ def get_environment_consumption_role_by_name(session, uri, IAMRoleName):
             )
             .first()
         )
+
+    @staticmethod
+    def get_all_envs_by_organization(session, uri):
+        return session.query(Environment).filter(Environment.organizationUri == uri).all()

backend/dataall/core/metadata_manager/__init__.py

@@ -0,0 +1,5 @@
+from dataall.core.metadata_manager.metadata_form_entity_manager import (
+    MetadataFormEntityManager,
+    MetadataFormEntity,
+    MetadataFormEntityTypes,
+)

backend/dataall/core/metadata_manager/metadata_form_entity_manager.py

@@ -0,0 +1,60 @@
+from abc import ABC
+from typing import Dict
+
+from dataall.base.api import GraphQLEnumMapper
+
+
+class MetadataFormEntityTypes(GraphQLEnumMapper):
+    Organizations = 'Organization'
+    OrganizationTeams = 'Organization Team'
+    Environments = 'Environment'
+    EnvironmentTeams = 'Environment Team'
+    S3Datasets = 'S3-Dataset'
+    RDDatasets = 'Redshift-Dataset'
+    Worksheets = 'Worksheets'
+    Dashboards = 'Dashboard'
+    ConsumptionRoles = 'Consumption Role'
+    Notebooks = 'Notebook'
+    MLStudioEntities = 'ML Studio Entity'
+    Pipelines = 'Pipeline'
+    Tables = 'Table'
+    Folder = 'Folder'
+    Bucket = 'Bucket'
+    Share = 'Share'
+
+
+class MetadataFormEntity(ABC):
+    def get_owner(self):
+        pass
+
+    def get_entity_name(self):
+        pass
+
+    def get_uri(self):
+        pass
+
+
+class MetadataFormEntityManager:
+    """
+    API for managing entities, to which MF can be attached.
+    """
+
+    _resources: Dict[str, MetadataFormEntity] = {}
+
+    @classmethod
+    def register(cls, resource: MetadataFormEntity, resource_key):
+        cls._resources[resource_key] = resource
+
+    @classmethod
+    def get_resource(cls, resource_key):
+        if resource_key not in cls._resources:
+            raise NotImplementedError(f'Entity {resource_key} is not registered')
+        return cls._resources[resource_key]
+
+    @classmethod
+    def is_registered(cls, resource_key):
+        return resource_key in cls._resources
+
+    @classmethod
+    def all_registered_keys(cls):
+        return cls._resources.keys()

backend/dataall/core/organizations/db/organization_models.py

@@ -6,8 +6,11 @@
 from dataall.base.db import Base
 from dataall.base.db import Resource, utils
 
+from dataall.core.metadata_manager import MetadataFormEntityManager, MetadataFormEntity, MetadataFormEntityTypes
+
 
 class Organization(Resource, Base):
+    __metaclass__ = MetadataFormEntity
     __tablename__ = 'organization'
     organizationUri = Column(String, primary_key=True, default=utils.uuid('organization'))
 
@@ -16,8 +19,18 @@ class Organization(Resource, Base):
     userRoleInOrganization = query_expression()
     SamlGroupName = Column(String, nullable=True)
 
+    def get_uri(self):
+        return self.organizationUri
+
+    def get_owner(self):
+        return self.SamlGroupName
+
+    def get_entity_name(self):
+        return self.label
+
 
 class OrganizationGroup(Base):
+    __metaclass__ = MetadataFormEntity
     __tablename__ = 'organization_group'
     groupUri = Column(String, primary_key=True)
     organizationUri = Column(String, primary_key=True)
@@ -26,3 +39,16 @@ class OrganizationGroup(Base):
     created = Column(DateTime, default=datetime.datetime.now)
     updated = Column(DateTime, onupdate=datetime.datetime.now)
     deleted = Column(DateTime)
+
+    def get_uri(self):
+        return f'{self.groupUri}-{self.organizationUri}'
+
+    def get_owner(self):
+        return self.invitedBy
+
+    def get_entity_name(self):
+        return f'{self.groupUri}-{self.organizationUri}'
+
+
+MetadataFormEntityManager.register(Organization, MetadataFormEntityTypes.Organizations.value)
+MetadataFormEntityManager.register(OrganizationGroup, MetadataFormEntityTypes.OrganizationTeams.value)

backend/dataall/core/organizations/db/organization_repositories.py

@@ -131,3 +131,7 @@ def find_group_membership(session, groups, organization):
             .first()
         )
         return membership
+
+    @staticmethod
+    def query_all_active_organizations(session):
+        return session.query(models.Organization).filter(models.Organization.deleted.is_(None)).all()

backend/dataall/modules/dashboards/__init__.py

@@ -25,6 +25,7 @@ def depends_on() -> List[Type['ModuleInterface']]:
 
     def __init__(self):
         from dataall.core.environment.services.environment_resource_manager import EnvironmentResourceManager
+        from dataall.core.metadata_manager import MetadataFormEntityManager, MetadataFormEntityTypes
         from dataall.modules.dashboards.db.dashboard_repositories import DashboardRepository
         from dataall.modules.dashboards.db.dashboard_models import Dashboard
         import dataall.modules.dashboards.api
@@ -45,6 +46,7 @@ def __init__(self):
         add_vote_type('dashboard', DashboardIndexer)
 
         EnvironmentResourceManager.register(DashboardRepository())
+        MetadataFormEntityManager.register(Dashboard, MetadataFormEntityTypes.Dashboards.value)
         log.info('Dashboard API has been loaded')
 
 

backend/dataall/modules/dashboards/db/dashboard_models.py

@@ -4,6 +4,7 @@
 from sqlalchemy.orm import query_expression
 
 from dataall.base.db import Base, Resource, utils
+from dataall.core.metadata_manager import MetadataFormEntity
 
 
 class DashboardShareStatus(Enum):
@@ -22,6 +23,7 @@ class DashboardShare(Base):
 
 
 class Dashboard(Resource, Base):
+    __metaclass__ = MetadataFormEntity
     __tablename__ = 'dashboard'
     environmentUri = Column(String, ForeignKey('environment.environmentUri'), nullable=False)
     organizationUri = Column(String, nullable=False)
@@ -37,3 +39,9 @@ class Dashboard(Resource, Base):
     @classmethod
     def uri(cls):
         return cls.dashboardUri
+
+    def get_owner(self):
+        return self.SamlGroupName
+
+    def get_entity_name(self):
+        return self.label

backend/dataall/modules/datasets_base/db/dataset_models.py

@@ -3,9 +3,11 @@
 from sqlalchemy.orm import query_expression
 from dataall.base.db import Base, Resource, utils
 from dataall.modules.datasets_base.services.datasets_enums import ConfidentialityClassification, Language, DatasetTypes
+from dataall.core.metadata_manager.metadata_form_entity_manager import MetadataFormEntity
 
 
 class DatasetBase(Resource, Base):
+    __metaclass__ = MetadataFormEntity
     __tablename__ = 'dataset'
     environmentUri = Column(String, ForeignKey('environment.environmentUri'), nullable=False)
     organizationUri = Column(String, nullable=False)
@@ -38,5 +40,14 @@ class DatasetBase(Resource, Base):
     def uri(cls):
         return cls.datasetUri
 
+    def get_owner(self):
+        return self.SamlAdminGroupName
+
+    def get_entity_name(self):
+        return self.label
+
+    def get_uri(self):
+        return self.datasetUri
+
 
 DatasetBase.__name__ = 'Dataset'

backend/dataall/modules/datasets_base/db/dataset_repositories.py

@@ -115,19 +115,19 @@ def paginated_environment_datasets(
         data=None,
     ) -> dict:
         return paginate(
-            query=DatasetListRepository._query_environment_datasets(session, uri, data),
+            query=DatasetListRepository.query_datasets(session, data, environmentUri=uri),
             page=data.get('page', 1),
             page_size=data.get('pageSize', 10),
         ).to_dict()
 
     @staticmethod
-    def _query_environment_datasets(session, uri, filter) -> Query:
-        query = session.query(DatasetBase).filter(
-            and_(
-                DatasetBase.environmentUri == uri,
-                DatasetBase.deleted.is_(None),
-            )
-        )
+    def query_datasets(session, filter=None, organizationUri=None, environmentUri=None) -> Query:
+        query = session.query(DatasetBase).filter(DatasetBase.deleted.is_(None))
+        if organizationUri:
+            query = query.filter(DatasetBase.organizationUri == organizationUri)
+        if environmentUri:
+            query = query.filter(DatasetBase.environmentUri == environmentUri)
+
         if filter and filter.get('term'):
             term = filter['term']
             query = query.filter(

backend/dataall/modules/metadata_forms/__init__.py

@@ -1,3 +1,5 @@
+from typing import List, Type
+
 from dataall.base.loader import ModuleInterface, ImportMode
 
 
@@ -11,3 +13,9 @@ def is_supported(cls, modes):
     def __init__(self):
         import dataall.modules.metadata_forms.api
         import dataall.modules.metadata_forms.db.enums
+
+    @staticmethod
+    def depends_on() -> List[Type['ModuleInterface']]:
+        from dataall.modules.datasets_base import DatasetBaseApiModuleInterface
+
+        return [DatasetBaseApiModuleInterface]

backend/dataall/modules/metadata_forms/api/input_types.py

@@ -40,7 +40,6 @@
     ],
 )
 
-
 MetadataFormFilter = gql.InputType(
     name='MetadataFormFilter',
     arguments=[
@@ -65,7 +64,6 @@
     ],
 )
 
-
 NewAttachedMetadataFormInput = gql.InputType(
     name='NewAttachedMetadataFormInput',
     arguments=[
@@ -83,3 +81,22 @@
         gql.Field(name='value', type=gql.String),
     ],
 )
+
+NewMetadataFormEnforcementInput = gql.InputType(
+    name='NewMetadataFormEnforcementInput',
+    arguments=[
+        gql.Field(name='metadataFormUri', type=gql.NonNullableType(gql.String)),
+        gql.Field(name='level', type=gql.NonNullableType(gql.String)),
+        gql.Field(name='homeEntity', type=gql.String),
+        gql.Field(name='severity', type=gql.String),
+        gql.Field(name='entityTypes', type=gql.ArrayType(gql.String)),
+    ],
+)
+
+AffectedEntityFilter = gql.InputType(
+    name='AffectedEntityFilter',
+    arguments=[
+        gql.Argument('page', gql.Integer),
+        gql.Argument('pageSize', gql.Integer),
+    ],
+)

backend/dataall/modules/metadata_forms/api/mutations.py

@@ -9,6 +9,8 @@
     delete_attached_metadata_form,
     create_metadata_form_version,
     delete_metadata_form_version,
+    create_mf_enforcement_rule,
+    delete_mf_enforcement_rule,
 )
 
 createMetadataForm = gql.MutationField(
@@ -105,3 +107,23 @@
     resolver=batch_metadata_form_field_update,
     test_scope='MetadataForm',
 )
+
+
+createMetadataFormEnforcementRule = gql.MutationField(
+    name='createMetadataFormEnforcementRule',
+    args=[gql.Argument(name='input', type=gql.NonNullableType(gql.Ref('NewMetadataFormEnforcementInput')))],
+    type=gql.Ref('MetadataFormEnforcementRule'),
+    resolver=create_mf_enforcement_rule,
+    test_scope='MetadataForm',
+)
+
+deleteMetadataFormEnforcementRule = gql.MutationField(
+    name='deleteMetadataFormEnforcementRule',
+    args=[
+        gql.Argument(name='uri', type=gql.NonNullableType(gql.String)),
+        gql.Argument(name='rule_uri', type=gql.NonNullableType(gql.String)),
+    ],
+    type=gql.Boolean,
+    resolver=delete_mf_enforcement_rule,
+    test_scope='MetadataForm',
+)