feat: change ref update process for admin_bdys

buildings/reference_data/admin_bdys.py

@@ -4,7 +4,7 @@
 
 from buildings.sql import buildings_reference_select_statements as reference_select
 from buildings.utilities import database as db
-from qgis.core import QgsVectorLayer
+from qgis.core import QgsGeometry, QgsVectorLayer
 
 # TODO: review if the filter works
 LAYERS = {
@@ -55,7 +55,7 @@ def current_date():
 
 # todo: add kx_api_key in config
 # todo: combine suburb_locality- and town city
-def update_admin_bdys(kx_api_key, dataset, dbconn):
+def update_admin_bdys(kx_api_key, dataset, dbconn: db):
 
     # get last update of layer date from log
     from_var = last_update(dataset)
@@ -83,21 +83,19 @@ def update_admin_bdys(kx_api_key, dataset, dbconn):
 
     external_id = LAYERS[dataset]["primary_id"]
 
-    ids_updates = []
     ids_attr_updates = []
-
+    geoms_diff = []
     for feature in layer.getFeatures():
         if feature.attribute("__change__") == "DELETE":
+            geoms_diff.append(feature.geometry())
             sql = "SELECT buildings_reference.{}_delete_by_external_id(%s)".format(
                 dataset
             )
-            result = dbconn.execute_no_commit(sql, (feature[external_id],))
-            result = result.fetchone()
-            if result is not None:
-                ids_updates.append(result[0])
+            dbconn.execute_no_commit(sql, (feature[external_id],))
 
         elif feature.attribute("__change__") == "UPDATE":
             if dataset == "suburb_locality":
+                # get attribute differences
                 result = dbconn.execute_return(
                     reference_select.suburb_locality_attribute_updates,
                     (
@@ -109,8 +107,21 @@ def update_admin_bdys(kx_api_key, dataset, dbconn):
                 result = result.fetchone()
                 if result is not None:
                     ids_attr_updates.append(result[0])
+
+                # get geometry differences
+                result = dbconn.execute_return(
+                    reference_select.suburb_locality_shape_updates,
+                    (
+                        feature.geometry().asWkt(),
+                        feature[external_id],
+                    ),
+                )
+                result = result.fetchone()
+                if result is not None:
+                    geoms_diff.append(QgsGeometry.fromWkt(result[0]))
+
                 sql = "SELECT buildings_reference.suburb_locality_update_by_external_id(%s, %s, %s, %s)"
-                result = dbconn.execute_no_commit(
+                dbconn.execute_no_commit(
                     sql,
                     (
                         feature[external_id],
@@ -119,10 +130,8 @@ def update_admin_bdys(kx_api_key, dataset, dbconn):
                         feature.geometry().asWkt(),
                     ),
                 )
-                result = result.fetchone()
-                if result is not None:
-                    ids_updates.append(result[0])
             else:
+                # get attribute differences
                 result = dbconn.execute_return(
                     reference_select.territorial_authority_attribute_updates,
                     (
@@ -133,18 +142,28 @@ def update_admin_bdys(kx_api_key, dataset, dbconn):
                 result = result.fetchone()
                 if result is not None:
                     ids_attr_updates.append(result[0])
+
+                # get geometry differences
+                result = dbconn.execute_return(
+                    reference_select.territorial_authority_shape_updates,
+                    (
+                        feature.geometry().asWkt(),
+                        feature[external_id],
+                    ),
+                )
+                result = result.fetchone()
+                if result is not None:
+                    geoms_diff.append(QgsGeometry.fromWkt(result[0]))
+
                 sql = "SELECT buildings_reference.territorial_authority_update_by_external_id(%s, %s, %s)"
-                result = dbconn.execute_no_commit(
+                dbconn.execute_no_commit(
                     sql,
                     (
                         feature[external_id],
                         correct_name_format(feature["name"]),
                         feature.geometry().asWkt(),
                     ),
                 )
-                result = result.fetchone()
-                if result is not None:
-                    ids_updates.append(result[0])
 
         elif feature.attribute("__change__") == "INSERT":
             # Check if feature is already in reference table
@@ -153,42 +172,53 @@ def update_admin_bdys(kx_api_key, dataset, dbconn):
                 (feature[external_id],),
             )
             result = result.fetchone()
-            if result is None:
-                if dataset == "suburb_locality":
-                    sql = "SELECT buildings_reference.suburb_locality_insert(%s, %s, %s, %s)"
-                    dbconn.execute_no_commit(
-                        sql,
-                        (
-                            feature[external_id],
-                            correct_name_format(feature["name"]),
-                            correct_name_format(feature["major_name"]),
-                            feature.geometry().asWkt(),
-                        ),
-                    )
-                else:
-                    sql = "SELECT buildings_reference.territorial_authority_insert(%s, %s, %s)"
-                    dbconn.execute_no_commit(
-                        sql,
-                        (
-                            feature[external_id],
-                            correct_name_format(feature["name"]),
-                            feature.geometry().asWkt(),
-                        ),
-                    )
-    print("updated_id {}".format(ids_updates))
-    print("updated_attrs {}".format(ids_attr_updates))
-    sql = "SELECT buildings_reference.{}_update_building_outlines(%s, %s)".format(
+            if result is not None:
+                dbconn.rollback_open_cursor()
+                raise Exception(
+                    "INSERT type feature exists in buildings db. Please contact devs."
+                )
+            geoms_diff.append(feature.geometry())
+            if dataset == "suburb_locality":
+                sql = (
+                    "SELECT buildings_reference.suburb_locality_insert(%s, %s, %s, %s)"
+                )
+                dbconn.execute_no_commit(
+                    sql,
+                    (
+                        feature[external_id],
+                        correct_name_format(feature["name"]),
+                        correct_name_format(feature["major_name"]),
+                        feature.geometry().asWkt(),
+                    ),
+                )
+            else:
+                sql = "SELECT buildings_reference.territorial_authority_insert(%s, %s, %s)"
+                dbconn.execute_no_commit(
+                    sql,
+                    (
+                        feature[external_id],
+                        correct_name_format(feature["name"]),
+                        feature.geometry().asWkt(),
+                    ),
+                )
+    print("updated_attrs: {}".format(ids_attr_updates))
+    print("updated_geom: {}".format(len(geoms_diff)))
+    geom_union = QgsGeometry.unaryUnion(geoms_diff).asWkt()
+    print(geom_union)
+    sql = "SELECT buildings_reference.{}_attribute_update_building_outlines(%s)".format(
+        dataset
+    )
+    dbconn.execute_no_commit(sql, (ids_attr_updates,))
+
+    sql = "SELECT buildings_reference.{}_geometry_update_building_outlines(%s)".format(
         dataset
     )
-    dbconn.execute_no_commit(sql, (ids_updates, ids_attr_updates))
+    dbconn.execute_no_commit(sql, (geom_union,))
 
     return "updated"
 
 
 def correct_name_format(name):
-    if name:
-        if "'" in name:
-            name = "{}".format(name.replace("'", "''"))
-    else:
+    if not name:
         name = ""
     return str(name)

buildings/reference_data/topo50.py

@@ -166,9 +166,6 @@ def update_topo50(kx_api_key, dataset, dbconn):
 
 
 def correct_name_format(name):
-    if name:
-        if "'" in name:
-            name = "{}".format(name.replace("'", "''"))
-    else:
+    if not name:
         name = ""
     return str(name)

buildings/sql/buildings_reference_select_statements.py

@@ -133,6 +133,14 @@
 AND NOT (suburb_locality = %s AND town_city = %s)
 """
 
+suburb_locality_shape_updates = """
+SELECT ST_AsText(
+    ST_SymDifference(shape, ST_SetSRID(ST_GeometryFromText(%s), 2193), 0.001)
+    ) AS diff
+FROM buildings_reference.suburb_locality
+WHERE external_suburb_locality_id = %s
+"""
+
 # territorial Authority
 
 territorial_authority_intersect_geom = """
@@ -170,6 +178,14 @@
 AND NOT name = %s
 """
 
+territorial_authority_shape_updates = """
+SELECT ST_AsText(
+    ST_SymDifference(shape, ST_SetSRID(ST_GeometryFromText(%s), 2193), 0.001)
+    ) AS diff
+FROM buildings_reference.territorial_authority
+WHERE external_territorial_authority_id = %s
+"""
+
 # territorial authority grid
 
 refresh_ta_grid_view = """

buildings/utilities/database.py

@@ -34,7 +34,9 @@
 _open_cursor = None
 
 try:
-    _conn = psycopg2.connect(host=_host, port=_port, database=_dbname, user=_user, password=_pw)
+    _conn = psycopg2.connect(
+        host=_host, port=_port, database=_dbname, user=_user, password=_pw
+    )
 except psycopg2.DatabaseError as error:
     _conn = None
     buildings_warning("Database Error", str(error), "critical")
@@ -64,7 +66,9 @@ def connect():
     """Connect to DB"""
     global _conn
     try:
-        _conn = psycopg2.connect(host=_host, port=_port, database=_dbname, user=_user, password=_pw)
+        _conn = psycopg2.connect(
+            host=_host, port=_port, database=_dbname, user=_user, password=_pw
+        )
     except psycopg2.DatabaseError as error:
         _conn = None
         buildings_warning("Database Error", str(error), "critical")
@@ -118,7 +122,7 @@ def execute_return(sql, data=None):
 
 
 def execute(sql, data=None):
-    """ Execute an update or insert statement with no return
+    """Execute an update or insert statement with no return
 
     @param  sql:    sql statement
     @type   sql:    string
@@ -144,13 +148,13 @@ def execute_no_commit(sql, data=None):
     except psycopg2.DatabaseError as error:
         _conn.rollback()
         buildings_warning("Database Error", str(error), "critical")
-        return None
+        raise error
     except psycopg2.InterfaceError as error:
         # Raise the error
         cursor.close()
         _conn.rollback()
         buildings_warning("Interface Error", str(error), "critical")
-        return None
+        raise error
 
     return cursor
 
@@ -171,7 +175,7 @@ def close_cursor():
 
 
 def set_uri():
-    """ Creates a QgsDataSourceUri with connection
+    """Creates a QgsDataSourceUri with connection
 
     @return:    QGIS URI object
     @rtype:     qgis.core.QgsDataSourceUri

db/sql/deploy/buildings_reference/functions/suburb_locality.sql

@@ -23,8 +23,12 @@ BEGIN;
     -- params: integer external_suburb_locality_id, varchar suburb_locality, varchar town_city, varchar geometry
     -- return: integer suburb_locality_id
 
--- suburb_locality_update_building_outlines
-    -- params: integer[] suburb_locality_id, integer[] suburb_locality_id
+-- suburb_locality_attribute_update_building_outlines
+    -- params: integer[] suburb_locality_id
+    -- return: integer building_outline_id
+
+-- suburb_locality_geometry_update_building_outlines
+    -- params: varchar shape
     -- return: integer building_outline_id
 
 --------------------------------------------
@@ -113,26 +117,39 @@ COMMENT ON FUNCTION buildings_reference.suburb_locality_update_by_external_id(in
 'Update suburb_locality table by external id';
 
 
--- suburb_locality_update_building_outlines
-    -- params: integer[] suburb_locality_id, integer[] suburb_locality_id
+-- suburb_locality_attribute_update_building_outlines
+    -- params: integer[] suburb_locality_id
     -- return: integer building_outline_id
-CREATE OR REPLACE FUNCTION buildings_reference.suburb_locality_update_building_outlines(integer[], integer[])
+CREATE OR REPLACE FUNCTION buildings_reference.suburb_locality_attribute_update_building_outlines(integer[])
+RETURNS integer AS
+$$
+    UPDATE buildings.building_outlines
+    SET last_modified = NOW()
+    WHERE suburb_locality_id = ANY($1)
+    RETURNING building_outline_id;
+$$
+LANGUAGE sql VOLATILE;
+
+COMMENT ON FUNCTION buildings_reference.suburb_locality_attribute_update_building_outlines(integer[]) IS
+'Update building_outlines last_modified value as suburb_locality/town_city attributes were updated';
+
+
+-- suburb_locality_geometry_update_building_outlines
+    -- params: varchar shape
+    -- return: integer building_outline_id
+CREATE OR REPLACE FUNCTION buildings_reference.suburb_locality_geometry_update_building_outlines(varchar)
 RETURNS integer AS
 $$
     UPDATE buildings.building_outlines
     SET suburb_locality_id = buildings_reference.suburb_locality_intersect_polygon(shape),
         last_modified = NOW()
-    WHERE (
-            suburb_locality_id = ANY($1)
-        AND suburb_locality_id != buildings_reference.suburb_locality_intersect_polygon(shape)
-    )
-    OR suburb_locality_id = ANY($2)
+    WHERE ST_Intersects(shape, ST_SetSRID(ST_GeometryFromText($1), 2193))
+    AND suburb_locality_id != buildings_reference.suburb_locality_intersect_polygon(shape)
     RETURNING building_outline_id;
 $$
 LANGUAGE sql VOLATILE;
 
-COMMENT ON FUNCTION buildings_reference.suburb_locality_update_building_outlines(integer[], integer[]) IS
-'Update building_outlines suburb_locality_id value using suburb_locality table';
-
+COMMENT ON FUNCTION buildings_reference.suburb_locality_geometry_update_building_outlines(varchar) IS
+'Update building_outlines suburb_locality_id value where building_outlines intersects with updated suburb_locality';
 
 COMMIT;