From 6a1c68442e1a24854b59b1707b329bc8fd89eddf Mon Sep 17 00:00:00 2001 From: Irwan Fathurrahman Date: Mon, 2 Sep 2024 17:54:40 +0700 Subject: [PATCH] Fix tests --- Makefile | 1 + .../core/models/materialized_view.py | 11 ++++- django_project/core/utils.py | 41 +++++++++++++++++++ .../commands/refresh_materialized_views.py | 28 +++++++++++++ .../data/migrations/0107_views_initiation.py | 4 ++ .../data/models/indicator/indicator_value.py | 2 + .../geosight/data/tests/api/v1/dataset.py | 2 +- .../data/tests/api/v1/dataset_grouped.py | 2 +- .../geosight/importer/importers/base/_base.py | 11 ++++- 9 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 django_project/geosight/data/management/commands/refresh_materialized_views.py diff --git a/Makefile b/Makefile index 3afee3de5..72335cb7c 100644 --- a/Makefile +++ b/Makefile @@ -220,6 +220,7 @@ devweb-load-demo-data: @docker compose $(ARGS) exec -T dev bash -c "python manage.py loaddata core/fixtures/demo/1.core.json" @docker compose $(ARGS) exec -T dev bash -c "python manage.py loaddata core/fixtures/demo/2.geosight_georepo.json" @docker compose $(ARGS) exec -T dev bash -c "python manage.py loaddata core/fixtures/demo/3.geosight_data.json" + @docker compose $(ARGS) exec -T dev bash -c "python manage.py refresh_materialized_views" devweb-test: @echo diff --git a/django_project/core/models/materialized_view.py b/django_project/core/models/materialized_view.py index 3252a7917..5dd69e433 100644 --- a/django_project/core/models/materialized_view.py +++ b/django_project/core/models/materialized_view.py @@ -16,6 +16,8 @@ from django.db import connection +from core.utils import child_classes + class MaterializeViewModel: """Materialized view model.""" @@ -24,4 +26,11 @@ class MaterializeViewModel: def refresh_materialized_views(cls): """Refresh materialized views.""" with connection.cursor() as cursor: - cursor.execute(f'REFRESH MATERIALIZED VIEW {cls._meta.db_table}') + query = f'REFRESH MATERIALIZED VIEW {cls._meta.db_table}' + print(query) + cursor.execute(query) + + @staticmethod + def child_classes(): + """Return child classes.""" + return child_classes(MaterializeViewModel) diff --git a/django_project/core/utils.py b/django_project/core/utils.py index df27e4b78..70e179cbb 100644 --- a/django_project/core/utils.py +++ b/django_project/core/utils.py @@ -49,3 +49,44 @@ def set_query_parameter(url, params): url_new_query = urlencode(url_dict) url_parse = url_parse._replace(query=url_new_query) return urlunparse(url_parse) + + +def child_classes(Class): + """Return child classes.""" + # If has subclasses + if not len(Class.__subclasses__()): + return [Class] + + # If has subclasses + classes = [] + for _class in Class.__subclasses__(): + if len(_class.__subclasses__()): + for child in _class.__subclasses__(): + classes += child_classes(child) + else: + classes.append(_class) + return classes + + +class temp_disconnect_signal(object): + """Temporarily disconnect a model from a signal.""" + + def __init__(self, signal, receiver, sender): + """Initialise the temporary disconnect signal.""" + self.signal = signal + self.receiver = receiver + self.sender = sender + + def __enter__(self): + """Enter the temporary disconnect signal.""" + self.signal.disconnect( + receiver=self.receiver, + sender=self.sender + ) + + def __exit__(self, type, value, traceback): + """Exit the temporary disconnect signal.""" + self.signal.connect( + receiver=self.receiver, + sender=self.sender + ) diff --git a/django_project/geosight/data/management/commands/refresh_materialized_views.py b/django_project/geosight/data/management/commands/refresh_materialized_views.py new file mode 100644 index 000000000..ad18b25d0 --- /dev/null +++ b/django_project/geosight/data/management/commands/refresh_materialized_views.py @@ -0,0 +1,28 @@ +# coding=utf-8 +""" +GeoSight is UNICEF's geospatial web-based business intelligence platform. + +Contact : geosight-no-reply@unicef.org + +.. note:: This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +""" +__author__ = 'zakki@kartoza.com' +__date__ = '05/07/2023' +__copyright__ = ('Copyright 2023, Unicef') + +from django.core.management.base import BaseCommand + +from core.models.materialized_view import MaterializeViewModel + + +class Command(BaseCommand): + """Refresh materialized views.""" + + def handle(self, *args, **options): + """Command handler.""" + for _class in MaterializeViewModel.child_classes(): + _class.refresh_materialized_views() diff --git a/django_project/geosight/data/migrations/0107_views_initiation.py b/django_project/geosight/data/migrations/0107_views_initiation.py index 56de6b8ce..c521eadf9 100644 --- a/django_project/geosight/data/migrations/0107_views_initiation.py +++ b/django_project/geosight/data/migrations/0107_views_initiation.py @@ -16,4 +16,8 @@ class Migration(migrations.Migration): operations = [ migrations.RunSQL(defaults, defaults), migrations.RunSQL(views, defaults), + migrations.AlterModelTable( + name='indicatorvaluewithgeo', + table='mv_indicator_value_geo', + ), ] diff --git a/django_project/geosight/data/models/indicator/indicator_value.py b/django_project/geosight/data/models/indicator/indicator_value.py index e26e4657d..bf6f8f8ca 100644 --- a/django_project/geosight/data/models/indicator/indicator_value.py +++ b/django_project/geosight/data/models/indicator/indicator_value.py @@ -263,3 +263,5 @@ def attributes(self): def increase_version(sender, instance, **kwargs): """Increase verison of indicator signal.""" instance.indicator.increase_version() + print('REFRESH FROM SIGNALS') + IndicatorValueWithGeo.refresh_materialized_views() diff --git a/django_project/geosight/data/tests/api/v1/dataset.py b/django_project/geosight/data/tests/api/v1/dataset.py index 54551ddd7..3b45daf94 100644 --- a/django_project/geosight/data/tests/api/v1/dataset.py +++ b/django_project/geosight/data/tests/api/v1/dataset.py @@ -264,7 +264,7 @@ def test_delete_api(self): # admin response = self.assertRequestGetView( - f'{url}?admin_level__in=1', 200, user=user + f'{url}?detail=true&admin_level__in=1', 200, user=user ) self.assertEqual(self.data_count(response), 20) diff --git a/django_project/geosight/data/tests/api/v1/dataset_grouped.py b/django_project/geosight/data/tests/api/v1/dataset_grouped.py index d0db6cf72..a24172fd0 100644 --- a/django_project/geosight/data/tests/api/v1/dataset_grouped.py +++ b/django_project/geosight/data/tests/api/v1/dataset_grouped.py @@ -259,7 +259,7 @@ def test_list_api_by_creator(self): def test_delete_api(self): """Test List API.""" user = self.creator_in_group - url = reverse('dataset-api') + '?group_admin_level=true' + url = reverse('dataset-api') + '?detail=true&group_admin_level=true' # admin response = self.assertRequestGetView( diff --git a/django_project/geosight/importer/importers/base/_base.py b/django_project/geosight/importer/importers/base/_base.py index ca22b76b8..d05855076 100644 --- a/django_project/geosight/importer/importers/base/_base.py +++ b/django_project/geosight/importer/importers/base/_base.py @@ -23,11 +23,13 @@ from django.conf import settings from django.contrib.auth import get_user_model from django.db import transaction +from django.db.models.signals import post_save from django.utils import timezone from django.utils.timezone import now +from core.utils import temp_disconnect_signal from geosight.data.models.indicator.indicator_value import ( - IndicatorValueWithGeo + IndicatorValueWithGeo, IndicatorValue, increase_version ) from geosight.importer.attribute import ImporterAttribute from geosight.importer.exception import ImporterError @@ -85,6 +87,7 @@ def after_import(self, success, note): def run(self): """To run the process.""" from geosight.data.models.context_layer import ContextLayerRequestError + try: self.log.status = LogStatus.RUNNING self.log.save() @@ -97,7 +100,11 @@ def run(self): # Use transaction atomic when indicator value if self.importer.import_type != ImportType.RELATED_TABLE: with transaction.atomic(): - self.after_import(success, note) + with temp_disconnect_signal( + signal=post_save, receiver=increase_version, + sender=IndicatorValue + ): + self.after_import(success, note) self._done(note) except (ImporterError, ContextLayerRequestError) as e: