Improve data browser (#300)

* Revert to using view

* Update query

* Fix flake

Makefile

@@ -220,7 +220,6 @@ devweb-load-demo-data:
 	@docker compose $(ARGS) exec -T dev bash -c "python manage.py loaddata core/fixtures/demo/1.core.json"
 	@docker compose $(ARGS) exec -T dev bash -c "python manage.py loaddata core/fixtures/demo/2.geosight_georepo.json"
 	@docker compose $(ARGS) exec -T dev bash -c "python manage.py loaddata core/fixtures/demo/3.geosight_data.json"
-	@docker compose $(ARGS) exec -T dev bash -c "python manage.py refresh_materialized_views"
 
 devweb-test:
 	@echo

django_project/_version.txt

@@ -1 +1 @@
-1.42.10
+1.42.11

django_project/core/api/refresh_materialized_view.py

@@ -18,9 +18,6 @@
 from rest_framework.views import APIView
 
 from core.permissions import AdminAuthenticationPermission
-from geosight.data.models.indicator.indicator_value import (
-    IndicatorValueWithGeo
-)
 
 
 class RefreshMaterializedViewApi(APIView):
@@ -30,7 +27,4 @@ class RefreshMaterializedViewApi(APIView):
 
     def post(self, request, *args, **kwargs):
         """Get access request list."""
-        view = request.data['view']
-        if view == IndicatorValueWithGeo._meta.db_table:
-            IndicatorValueWithGeo.refresh_materialized_views()
         return Response(status=201)

django_project/core/models/materialized_view.py

@@ -27,7 +27,6 @@ def refresh_materialized_views(cls):
         """Refresh materialized views."""
         with connection.cursor() as cursor:
             query = f'REFRESH MATERIALIZED VIEW {cls._meta.db_table}'
-            print(query)
             cursor.execute(query)
 
     @staticmethod

django_project/geosight/data/admin/indicator.py

@@ -74,7 +74,6 @@ class IndicatorGroupAdmin(admin.ModelAdmin):
 class IndicatorValueWithGeoAdmin(admin.ModelAdmin):
     """Admin for checking indicator values with geometry."""
 
-    change_list_template = 'admin/indicator_value_with_geo_admin.html'
     list_display = (
         'reference_layer_name', 'reference_layer_uuid', 'indicator_name',
         'date', 'value'

django_project/geosight/data/api/v1/data_browser/data_browser.py

@@ -29,9 +29,6 @@
 from geosight.data.models.indicator import (
     Indicator, IndicatorValue, IndicatorValueRejectedError
 )
-from geosight.data.models.indicator.indicator_value import (
-    IndicatorValueWithGeo
-)
 from geosight.data.serializer.indicator import (
     IndicatorValueSerializer, IndicatorValueWithPermissionSerializer
 )
@@ -145,7 +142,6 @@ def post(self, request):
                 value=data['value'],
                 extras=data.get('attributes', {})
             )
-            IndicatorValueWithGeo.refresh_materialized_views()
         except KeyError as e:
             return HttpResponseBadRequest(f'{e} is required on payload')
         except Exception as e:
@@ -176,7 +172,6 @@ def put(self, request):
                     return HttpResponseBadRequest(
                         f'Indicator {value.indicator} : {e}'
                     )
-            IndicatorValueWithGeo.refresh_materialized_views()
             return Response('OK')
         except KeyError:
             return HttpResponseBadRequest('`data` is required on payload')

django_project/geosight/data/api/v1/data_browser/dataset.py

@@ -216,7 +216,6 @@ def delete(self, request):
             identifier_with_level__in=to_be_deleted
         ).values_list('id', flat=True)
         IndicatorValue.objects.filter(id__in=list(ids)).delete()
-        IndicatorValueWithGeo.refresh_materialized_views()
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 

django_project/geosight/data/migrations/0108_views_initiation.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.12 on 2020-07-03 09:03
+
+from django.db import migrations
+
+from geosight.data.migrations.sql.utils import load_sql
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('geosight_data', '0107_views_initiation'),
+    ]
+
+    defaults = load_sql('views', 'default.sql')
+    views = load_sql('views', 'views.sql')
+
+    operations = [
+        migrations.RunSQL(defaults, defaults),
+        migrations.RunSQL(views, defaults),
+        migrations.AlterModelTable(
+            name='indicatorvaluewithgeo',
+            table='v_indicator_value_geo',
+        ),
+    ]

django_project/geosight/data/migrations/sql/views/views.sql

@@ -1,9 +1,9 @@
 -- Indicator Value x Geometry
 CREATE VIEW v_indicator_value_geo as
 SELECT value.*,
-    date_part('day', value.date)    as day,
-    date_part('month', value.date)  as month,
-    date_part('year', value.date)   as year,
+    date_trunc('day', value.date)    as day,
+    date_trunc('month', value.date)  as month,
+    date_trunc('year', value.date)   as year,
     entity.concept_uuid             as concept_uuid,
     entity.reference_layer_id       as reference_layer_id,
     ref_view.name                   as reference_layer_name,
@@ -17,8 +17,4 @@ SELECT value.*,
 FROM geosight_data_indicatorvalue as value
      LEFT JOIN geosight_georepo_entity as entity ON value.geom_id = entity.geom_id
      LEFT JOIN geosight_georepo_referencelayerview as ref_view ON ref_view.id = entity.reference_layer_id
-     LEFT JOIN geosight_data_indicator as indicator ON value.indicator_id = indicator.id;
-
-
-CREATE MATERIALIZED VIEW mv_indicator_value_geo as
-SELECT * from v_indicator_value_geo;
+     LEFT JOIN geosight_data_indicator as indicator ON value.indicator_id = indicator.id;

django_project/geosight/data/models/indicator/indicator_value.py

@@ -19,7 +19,6 @@
 from django.dispatch import receiver
 from django.utils.translation import ugettext_lazy as _
 
-from core.models.materialized_view import MaterializeViewModel
 from geosight.data.models.indicator.indicator import Indicator, IndicatorType
 
 
@@ -154,7 +153,7 @@ def key(self):
         return self.name.replace(' ', '_').replace(':', '').lower()
 
 
-class IndicatorValueWithGeo(MaterializeViewModel, models.Model):
+class IndicatorValueWithGeo(models.Model):
     """Indicator value x entity view."""
 
     # This is geom id for the value
@@ -240,7 +239,7 @@ def reference_layer(self):
     class Meta:  # noqa: D106
         managed = False
         ordering = ('-date',)
-        db_table = 'mv_indicator_value_geo'
+        db_table = 'v_indicator_value_geo'
 
     @property
     def val(self):
@@ -263,5 +262,3 @@ def attributes(self):
 def increase_version(sender, instance, **kwargs):
     """Increase verison of indicator signal."""
     instance.indicator.increase_version()
-    print('REFRESH FROM SIGNALS')
-    IndicatorValueWithGeo.refresh_materialized_views()

django_project/geosight/importer/importers/base/_base.py

@@ -29,7 +29,7 @@
 
 from core.utils import temp_disconnect_signal
 from geosight.data.models.indicator.indicator_value import (
-    IndicatorValueWithGeo, IndicatorValue, increase_version
+    IndicatorValue, increase_version
 )
 from geosight.importer.attribute import ImporterAttribute
 from geosight.importer.exception import ImporterError
@@ -74,7 +74,6 @@ def after_import(self, success, note):
                 logs = self.log.importerlogdata_set.order_by('id')
                 for line_idx, log in enumerate(logs):
                     self._save_log_data_to_model(log)
-                IndicatorValueWithGeo.refresh_materialized_views()
         elif not success:
             error = (
                 'Importing is failed. No data saved. '

django_project/geosight/importer/models/log.py

@@ -22,9 +22,6 @@
 from django.utils.translation import ugettext_lazy as _
 
 from core.models.preferences import SitePreferences
-from geosight.data.models.indicator.indicator_value import (
-    IndicatorValueWithGeo
-)
 from geosight.importer.models.importer import Importer
 
 
@@ -177,4 +174,3 @@ def run(self):
                 self.saved_ids.append(log_data.id)
                 self.save()
         self.delete()
-        IndicatorValueWithGeo.refresh_materialized_views()