diff --git a/metadata_service/__init__.py b/metadata_service/__init__.py index b70b3e78..45ebb290 100644 --- a/metadata_service/__init__.py +++ b/metadata_service/__init__.py @@ -17,7 +17,7 @@ from metadata_service.api.popular_tables import PopularTablesAPI from metadata_service.api.system import Neo4jDetailAPI from metadata_service.api.table \ - import TableDetailAPI, TableOwnerAPI, TableTagAPI, TableDescriptionAPI + import TableDetailAPI, TableOwnerAPI, TableTagAPI, TableDescriptionAPI, TableDashboardAPI from metadata_service.api.tag import TagAPI from metadata_service.api.user import (UserDetailAPI, UserFollowAPI, UserFollowsAPI, UserOwnsAPI, @@ -91,6 +91,8 @@ def create_app(*, config_module_class: str) -> Flask: '/table//tag/') api.add_resource(TableOwnerAPI, '/table//owner/') + api.add_resource(TableDashboardAPI, + '/table//dashboard/') api.add_resource(ColumnDescriptionAPI, '/table//column//description') api.add_resource(Neo4jDetailAPI, diff --git a/metadata_service/api/__init__.py b/metadata_service/api/__init__.py index 54862d5a..db6da73d 100644 --- a/metadata_service/api/__init__.py +++ b/metadata_service/api/__init__.py @@ -20,11 +20,15 @@ def get(self, *, id: Optional[str] = None) -> Iterable[Union[Mapping, int, None] """ Gets a single or multiple objects """ + return self.get_with_kwargs(id=id) + + def get_with_kwargs(self, *, id: Optional[str] = None, **kwargs: Optional[Any]) \ + -> Iterable[Union[Mapping, int, None]]: if id is not None: get_object = getattr(self.client, f'get_{self.str_type}') try: actual_id: Union[str, int] = int(id) if id.isdigit() else id - object = get_object(id=actual_id) + object = get_object(id=actual_id, **kwargs) if object is not None: return self.schema().dump(object).data, HTTPStatus.OK return None, HTTPStatus.NOT_FOUND diff --git a/metadata_service/api/swagger_doc/table/dashboards_using_table_get.yml b/metadata_service/api/swagger_doc/table/dashboards_using_table_get.yml new file mode 100644 index 00000000..3a32d279 --- /dev/null +++ b/metadata_service/api/swagger_doc/table/dashboards_using_table_get.yml @@ -0,0 +1,30 @@ +Gets Dashboards that is using this table +--- +tags: + - 'table' +parameters: + - name: id + in: path + type: string + schema: + type: string + required: true + example: 'hive://gold.test_schema/test_table2' +responses: + 200: + description: 'List of dashboards that table is used' + content: + application/json: + schema: + type: object + properties: + dashboards: + type: array + items: + $ref: '#/components/schemas/DashboardSummary' + 404: + description: 'Table not found' + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' diff --git a/metadata_service/api/table.py b/metadata_service/api/table.py index c6ed250d..eccc5b50 100644 --- a/metadata_service/api/table.py +++ b/metadata_service/api/table.py @@ -1,14 +1,16 @@ import json from http import HTTPStatus -from typing import Any, Iterable, Mapping, Union +from typing import Any, Iterable, Mapping, Union, Optional from amundsen_common.models.table import TableSchema from flasgger import swag_from from flask import request from flask_restful import Resource, reqparse +from metadata_service.api import BaseAPI from metadata_service.api.tag import TagCommon from metadata_service.entity.resource_type import ResourceType +from metadata_service.entity.dashboard_summary import DashboardSummarySchema from metadata_service.exception import NotFoundException from metadata_service.proxy import get_proxy_client @@ -152,3 +154,25 @@ def delete(self, id: str, tag: str) -> Iterable[Union[Mapping, int, None]]: resource_type=ResourceType.Table, tag=tag, tag_type=tag_type) + + +class TableDashboardAPI(BaseAPI): + """ + TableDashboard API that supports GET operation providing list of Dashboards using a table. + """ + + def __init__(self) -> None: + self.client = get_proxy_client() + super().__init__(DashboardSummarySchema, 'resources_using_table', self.client) + + @swag_from('swagger_doc/table/dashboards_using_table_get.yml') + def get(self, *, id: Optional[str] = None) -> Iterable[Union[Mapping, int, None]]: + """ + Supports GET operation providing list of Dashboards using a table. + :param id: Table URI + :return: See Swagger doc for the schema. swagger_doc/table/dashboards_using_table_get.yml + """ + try: + return super().get_with_kwargs(id=id, resource_type=ResourceType.Dashboard) + except NotFoundException: + return {'message': 'table_id {} does not exist'.format(id)}, HTTPStatus.NOT_FOUND diff --git a/metadata_service/entity/dashboard_summary.py b/metadata_service/entity/dashboard_summary.py new file mode 100644 index 00000000..0d65c359 --- /dev/null +++ b/metadata_service/entity/dashboard_summary.py @@ -0,0 +1,16 @@ +from typing import List + +import attr +from amundsen_common.models.dashboard import DashboardSummary as Summary +from marshmallow_annotations.ext.attrs import AttrsSchema + + +@attr.s(auto_attribs=True, kw_only=True) +class DashboardSummary: + dashboards: List[Summary] = attr.ib(factory=list) + + +class DashboardSummarySchema(AttrsSchema): + class Meta: + target = DashboardSummary + register_as_scheme = True diff --git a/metadata_service/proxy/atlas_proxy.py b/metadata_service/proxy/atlas_proxy.py index a981e485..ce273c50 100644 --- a/metadata_service/proxy/atlas_proxy.py +++ b/metadata_service/proxy/atlas_proxy.py @@ -657,3 +657,8 @@ def put_dashboard_description(self, *, id: str, description: str) -> None: pass + + def get_resources_using_table(self, *, + id: str, + resource_type: ResourceType) -> Dict[str, List[DashboardSummary]]: + pass diff --git a/metadata_service/proxy/base_proxy.py b/metadata_service/proxy/base_proxy.py index 231cd6ce..7137876c 100644 --- a/metadata_service/proxy/base_proxy.py +++ b/metadata_service/proxy/base_proxy.py @@ -128,3 +128,9 @@ def put_dashboard_description(self, *, id: str, description: str) -> None: pass + + @abstractmethod + def get_resources_using_table(self, *, + id: str, + resource_type: ResourceType) -> Dict[str, List[DashboardSummary]]: + pass diff --git a/metadata_service/proxy/gremlin_proxy.py b/metadata_service/proxy/gremlin_proxy.py index ea3821f4..dc06c75b 100644 --- a/metadata_service/proxy/gremlin_proxy.py +++ b/metadata_service/proxy/gremlin_proxy.py @@ -180,6 +180,11 @@ def put_dashboard_description(self, *, description: str) -> None: pass + def get_resources_using_table(self, *, + id: str, + resource_type: ResourceType) -> Dict[str, List[DashboardSummary]]: + pass + class GenericGremlinProxy(AbstractGremlinProxy): """ diff --git a/metadata_service/proxy/neo4j_proxy.py b/metadata_service/proxy/neo4j_proxy.py index 99ef17fc..1b1bb3f8 100644 --- a/metadata_service/proxy/neo4j_proxy.py +++ b/metadata_service/proxy/neo4j_proxy.py @@ -1177,3 +1177,46 @@ def put_dashboard_description(self, *, self._put_resource_description(resource_type=ResourceType.Dashboard, uri=id, description=description) + + @timer_with_counter + def get_resources_using_table(self, *, + id: str, + resource_type: ResourceType) -> Dict[str, List[DashboardSummary]]: + """ + + :param id: + :param resource_type: + :return: + """ + if resource_type != ResourceType.Dashboard: + raise NotImplementedError('{} is not supported'.format(resource_type)) + + get_dashboards_using_table_query = textwrap.dedent(u""" + MATCH (d:Dashboard)-[:DASHBOARD_WITH_TABLE]->(table:Table {key: $query_key}), + (d)-[:DASHBOARD_OF]->(dg:Dashboardgroup)-[:DASHBOARD_GROUP_OF]->(c:Cluster) + OPTIONAL MATCH (d)-[:DESCRIPTION]->(description:Description) + OPTIONAL MATCH (d)-[:EXECUTED]->(last_success_exec:Execution) + WHERE split(last_success_exec.key, '/')[5] = '_last_successful_execution' + OPTIONAL MATCH (d)-[read:READ_BY]->(:User) + WITH c, dg, d, description, last_success_exec, sum(read.read_count) as recent_view_count + RETURN + d.key as uri, + c.name as cluster, + dg.name as group_name, + dg.dashboard_group_url as group_url, + d.name as name, + d.dashboard_url as url, + description.description as description, + split(d.key, '_')[0] as product, + toInteger(last_success_exec.timestamp) as last_successful_run_timestamp + ORDER BY recent_view_count DESC; + """) + + records = self._execute_cypher_query(statement=get_dashboards_using_table_query, + param_dict={'query_key': id}) + + results = [] + + for record in records: + results.append(DashboardSummary(**record)) + return {'dashboards': results} diff --git a/setup.py b/setup.py index 166b8594..35a27bb6 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -__version__ = '2.4.6' +__version__ = '2.5.0' requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements.txt') diff --git a/tests/unit/api/table/test_dashboards_using_table_api.py b/tests/unit/api/table/test_dashboards_using_table_api.py new file mode 100644 index 00000000..a1464e58 --- /dev/null +++ b/tests/unit/api/table/test_dashboards_using_table_api.py @@ -0,0 +1,68 @@ +import unittest +from http import HTTPStatus + +from metadata_service.entity.resource_type import ResourceType +from tests.unit.api.table.table_test_case import TableTestCase + +TABLE_URI = 'wizards' + +QUERY_RESPONSE = { + 'dashboards': [ + { + 'uri': 'foo_dashboard://gold.foo/bar1', + 'cluster': 'gold', + 'group_name': 'foo', + 'group_url': 'https://foo', + 'product': 'foo', + 'name': 'test dashboard 1', + 'url': 'https://foo.bar', + 'description': 'test dashboard description 1', + 'last_successful_run_timestamp': 1234567890 + }, + { + 'uri': 'foo_dashboard://gold.foo/bar1', + 'cluster': 'gold', + 'group_name': 'foo', + 'group_url': 'https://foo', + 'product': 'foo', + 'name': 'test dashboard 1', + 'url': 'https://foo.bar', + 'description': None, + 'last_successful_run_timestamp': None + } + ] +} + +API_RESPONSE = { + 'dashboards': + [ + { + 'group_url': 'https://foo', 'uri': 'foo_dashboard://gold.foo/bar1', + 'last_successful_run_timestamp': 1234567890, 'group_name': 'foo', 'name': 'test dashboard 1', + 'url': 'https://foo.bar', 'description': 'test dashboard description 1', 'cluster': 'gold', + 'product': 'foo' + }, + { + 'group_url': 'https://foo', 'uri': 'foo_dashboard://gold.foo/bar1', + 'last_successful_run_timestamp': None, + 'group_name': 'foo', 'name': 'test dashboard 1', 'url': 'https://foo.bar', 'description': None, + 'cluster': 'gold', 'product': 'foo' + } + ] +} + + +class TestTableDashboardAPI(TableTestCase): + + def test_get_dashboards_using_table(self) -> None: + self.mock_proxy.get_resources_using_table.return_value = QUERY_RESPONSE + + response = self.app.test_client().get(f'/table/{TABLE_URI}/dashboard/') + self.assertEqual(response.json, API_RESPONSE) + self.assertEqual(response.status_code, HTTPStatus.OK) + self.mock_proxy.get_resources_using_table.assert_called_with(id=TABLE_URI, + resource_type=ResourceType.Dashboard) + + +if __name__ == '__main__': + unittest.main()