From 167ca256c336db2a63cea628afb2447029a60b37 Mon Sep 17 00:00:00 2001 From: Irwan Fathurrahman Date: Wed, 11 Sep 2024 08:30:36 +0700 Subject: [PATCH] Improve data browser (#300) * Revert to using view * Update query * Fix flake --- Makefile | 1 - django_project/_version.txt | 2 +- .../core/api/refresh_materialized_view.py | 6 ----- .../core/models/materialized_view.py | 1 - .../geosight/data/admin/indicator.py | 1 - .../data/api/v1/data_browser/data_browser.py | 5 ---- .../data/api/v1/data_browser/dataset.py | 1 - .../data/migrations/0108_views_initiation.py | 23 +++++++++++++++++++ .../data/migrations/sql/views/views.sql | 12 ++++------ .../data/models/indicator/indicator_value.py | 7 ++---- .../geosight/importer/importers/base/_base.py | 3 +-- .../geosight/importer/models/log.py | 4 ---- 12 files changed, 31 insertions(+), 35 deletions(-) create mode 100644 django_project/geosight/data/migrations/0108_views_initiation.py diff --git a/Makefile b/Makefile index 72335cb7c..3afee3de5 100644 --- a/Makefile +++ b/Makefile @@ -220,7 +220,6 @@ devweb-load-demo-data: @docker compose $(ARGS) exec -T dev bash -c "python manage.py loaddata core/fixtures/demo/1.core.json" @docker compose $(ARGS) exec -T dev bash -c "python manage.py loaddata core/fixtures/demo/2.geosight_georepo.json" @docker compose $(ARGS) exec -T dev bash -c "python manage.py loaddata core/fixtures/demo/3.geosight_data.json" - @docker compose $(ARGS) exec -T dev bash -c "python manage.py refresh_materialized_views" devweb-test: @echo diff --git a/django_project/_version.txt b/django_project/_version.txt index d8841ecfe..d10d7aeee 100644 --- a/django_project/_version.txt +++ b/django_project/_version.txt @@ -1 +1 @@ -1.42.10 \ No newline at end of file +1.42.11 \ No newline at end of file diff --git a/django_project/core/api/refresh_materialized_view.py b/django_project/core/api/refresh_materialized_view.py index 318d7eae1..ef5cf62c3 100644 --- a/django_project/core/api/refresh_materialized_view.py +++ b/django_project/core/api/refresh_materialized_view.py @@ -18,9 +18,6 @@ from rest_framework.views import APIView from core.permissions import AdminAuthenticationPermission -from geosight.data.models.indicator.indicator_value import ( - IndicatorValueWithGeo -) class RefreshMaterializedViewApi(APIView): @@ -30,7 +27,4 @@ class RefreshMaterializedViewApi(APIView): def post(self, request, *args, **kwargs): """Get access request list.""" - view = request.data['view'] - if view == IndicatorValueWithGeo._meta.db_table: - IndicatorValueWithGeo.refresh_materialized_views() return Response(status=201) diff --git a/django_project/core/models/materialized_view.py b/django_project/core/models/materialized_view.py index 5dd69e433..dd5151338 100644 --- a/django_project/core/models/materialized_view.py +++ b/django_project/core/models/materialized_view.py @@ -27,7 +27,6 @@ def refresh_materialized_views(cls): """Refresh materialized views.""" with connection.cursor() as cursor: query = f'REFRESH MATERIALIZED VIEW {cls._meta.db_table}' - print(query) cursor.execute(query) @staticmethod diff --git a/django_project/geosight/data/admin/indicator.py b/django_project/geosight/data/admin/indicator.py index 4c0c66b35..7b00ff3bb 100644 --- a/django_project/geosight/data/admin/indicator.py +++ b/django_project/geosight/data/admin/indicator.py @@ -74,7 +74,6 @@ class IndicatorGroupAdmin(admin.ModelAdmin): class IndicatorValueWithGeoAdmin(admin.ModelAdmin): """Admin for checking indicator values with geometry.""" - change_list_template = 'admin/indicator_value_with_geo_admin.html' list_display = ( 'reference_layer_name', 'reference_layer_uuid', 'indicator_name', 'date', 'value' diff --git a/django_project/geosight/data/api/v1/data_browser/data_browser.py b/django_project/geosight/data/api/v1/data_browser/data_browser.py index f4392f816..7339e868e 100644 --- a/django_project/geosight/data/api/v1/data_browser/data_browser.py +++ b/django_project/geosight/data/api/v1/data_browser/data_browser.py @@ -29,9 +29,6 @@ from geosight.data.models.indicator import ( Indicator, IndicatorValue, IndicatorValueRejectedError ) -from geosight.data.models.indicator.indicator_value import ( - IndicatorValueWithGeo -) from geosight.data.serializer.indicator import ( IndicatorValueSerializer, IndicatorValueWithPermissionSerializer ) @@ -145,7 +142,6 @@ def post(self, request): value=data['value'], extras=data.get('attributes', {}) ) - IndicatorValueWithGeo.refresh_materialized_views() except KeyError as e: return HttpResponseBadRequest(f'{e} is required on payload') except Exception as e: @@ -176,7 +172,6 @@ def put(self, request): return HttpResponseBadRequest( f'Indicator {value.indicator} : {e}' ) - IndicatorValueWithGeo.refresh_materialized_views() return Response('OK') except KeyError: return HttpResponseBadRequest('`data` is required on payload') diff --git a/django_project/geosight/data/api/v1/data_browser/dataset.py b/django_project/geosight/data/api/v1/data_browser/dataset.py index 04f2cc180..ad6b3141a 100644 --- a/django_project/geosight/data/api/v1/data_browser/dataset.py +++ b/django_project/geosight/data/api/v1/data_browser/dataset.py @@ -216,7 +216,6 @@ def delete(self, request): identifier_with_level__in=to_be_deleted ).values_list('id', flat=True) IndicatorValue.objects.filter(id__in=list(ids)).delete() - IndicatorValueWithGeo.refresh_materialized_views() return Response(status=status.HTTP_204_NO_CONTENT) diff --git a/django_project/geosight/data/migrations/0108_views_initiation.py b/django_project/geosight/data/migrations/0108_views_initiation.py new file mode 100644 index 000000000..ffd54eb89 --- /dev/null +++ b/django_project/geosight/data/migrations/0108_views_initiation.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.12 on 2020-07-03 09:03 + +from django.db import migrations + +from geosight.data.migrations.sql.utils import load_sql + + +class Migration(migrations.Migration): + dependencies = [ + ('geosight_data', '0107_views_initiation'), + ] + + defaults = load_sql('views', 'default.sql') + views = load_sql('views', 'views.sql') + + operations = [ + migrations.RunSQL(defaults, defaults), + migrations.RunSQL(views, defaults), + migrations.AlterModelTable( + name='indicatorvaluewithgeo', + table='v_indicator_value_geo', + ), + ] diff --git a/django_project/geosight/data/migrations/sql/views/views.sql b/django_project/geosight/data/migrations/sql/views/views.sql index 52902fd50..f99891979 100644 --- a/django_project/geosight/data/migrations/sql/views/views.sql +++ b/django_project/geosight/data/migrations/sql/views/views.sql @@ -1,9 +1,9 @@ -- Indicator Value x Geometry CREATE VIEW v_indicator_value_geo as SELECT value.*, - date_part('day', value.date) as day, - date_part('month', value.date) as month, - date_part('year', value.date) as year, + date_trunc('day', value.date) as day, + date_trunc('month', value.date) as month, + date_trunc('year', value.date) as year, entity.concept_uuid as concept_uuid, entity.reference_layer_id as reference_layer_id, ref_view.name as reference_layer_name, @@ -17,8 +17,4 @@ SELECT value.*, FROM geosight_data_indicatorvalue as value LEFT JOIN geosight_georepo_entity as entity ON value.geom_id = entity.geom_id LEFT JOIN geosight_georepo_referencelayerview as ref_view ON ref_view.id = entity.reference_layer_id - LEFT JOIN geosight_data_indicator as indicator ON value.indicator_id = indicator.id; - - -CREATE MATERIALIZED VIEW mv_indicator_value_geo as -SELECT * from v_indicator_value_geo; \ No newline at end of file + LEFT JOIN geosight_data_indicator as indicator ON value.indicator_id = indicator.id; \ No newline at end of file diff --git a/django_project/geosight/data/models/indicator/indicator_value.py b/django_project/geosight/data/models/indicator/indicator_value.py index bf6f8f8ca..7d5e07009 100644 --- a/django_project/geosight/data/models/indicator/indicator_value.py +++ b/django_project/geosight/data/models/indicator/indicator_value.py @@ -19,7 +19,6 @@ from django.dispatch import receiver from django.utils.translation import ugettext_lazy as _ -from core.models.materialized_view import MaterializeViewModel from geosight.data.models.indicator.indicator import Indicator, IndicatorType @@ -154,7 +153,7 @@ def key(self): return self.name.replace(' ', '_').replace(':', '').lower() -class IndicatorValueWithGeo(MaterializeViewModel, models.Model): +class IndicatorValueWithGeo(models.Model): """Indicator value x entity view.""" # This is geom id for the value @@ -240,7 +239,7 @@ def reference_layer(self): class Meta: # noqa: D106 managed = False ordering = ('-date',) - db_table = 'mv_indicator_value_geo' + db_table = 'v_indicator_value_geo' @property def val(self): @@ -263,5 +262,3 @@ def attributes(self): def increase_version(sender, instance, **kwargs): """Increase verison of indicator signal.""" instance.indicator.increase_version() - print('REFRESH FROM SIGNALS') - IndicatorValueWithGeo.refresh_materialized_views() diff --git a/django_project/geosight/importer/importers/base/_base.py b/django_project/geosight/importer/importers/base/_base.py index d05855076..283d078b7 100644 --- a/django_project/geosight/importer/importers/base/_base.py +++ b/django_project/geosight/importer/importers/base/_base.py @@ -29,7 +29,7 @@ from core.utils import temp_disconnect_signal from geosight.data.models.indicator.indicator_value import ( - IndicatorValueWithGeo, IndicatorValue, increase_version + IndicatorValue, increase_version ) from geosight.importer.attribute import ImporterAttribute from geosight.importer.exception import ImporterError @@ -74,7 +74,6 @@ def after_import(self, success, note): logs = self.log.importerlogdata_set.order_by('id') for line_idx, log in enumerate(logs): self._save_log_data_to_model(log) - IndicatorValueWithGeo.refresh_materialized_views() elif not success: error = ( 'Importing is failed. No data saved. ' diff --git a/django_project/geosight/importer/models/log.py b/django_project/geosight/importer/models/log.py index ba291cfa4..d4645feee 100644 --- a/django_project/geosight/importer/models/log.py +++ b/django_project/geosight/importer/models/log.py @@ -22,9 +22,6 @@ from django.utils.translation import ugettext_lazy as _ from core.models.preferences import SitePreferences -from geosight.data.models.indicator.indicator_value import ( - IndicatorValueWithGeo -) from geosight.importer.models.importer import Importer @@ -177,4 +174,3 @@ def run(self): self.saved_ids.append(log_data.id) self.save() self.delete() - IndicatorValueWithGeo.refresh_materialized_views()