Merge pull request #141 from QGEP/2024-05-11-import-improve-progress-…

…messages

Improve user feedback and speed-up interlis import qgep

qgepqwat2ili/gui/__init__.py

@@ -45,6 +45,8 @@
 # 19.4.2023 / 25.4.2023 ohne Bindestrich / neu aus gui_import - Gui
 from .gui_importc import GuiImportc
 
+
+
 # 12.7.2022 for testing import time
 import time
 
@@ -354,12 +356,14 @@ def action_import(plugin):
                  log_path,
             )
 
-    progress_dialog.setLabelText("Set main_cover manually after import if vw_qgep_wastewater_structure does not display correctly!")
+    # 31.5.2024 should not be needed anymore
+    # progress_dialog.setLabelText("Set main_cover manually after import if vw_qgep_wastewater_structure does not display correctly!")
 
     time.sleep(2)
-    # to add option to run main_cover.sql manually
-    
+    # to add option to run main_cover.sql manually - see postimport.py
+
     # 24.7.2022 / moved to end
+    # shows progress of import to import wizard, not final commit
     progress_dialog.setValue(100)
 
 

qgepqwat2ili/gui/gui_import.py

@@ -19,6 +19,9 @@
 import logging
 from qgis.PyQt.QtCore import pyqtSlot
 
+# 31.5.2024 pfad angepasst, neu in gui_import.py statt _init_.py
+from ..postimport import qgep_postimport
+
 # Required for loadUi to find the custom widget
 sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
 
@@ -213,9 +216,23 @@ def commit_session(self):
         # TODO : rollback to pre-commit state, allowing user to try to fix issues
         # probably a matter of creating a savepoint before saving with
         # session.begin_nested() and one additionnal self.session.commit()
-        self.session.commit()
-        self.session.close()
+
+        # add info in message bar
+        iface.messageBar().pushMessage("Please be patient!", "Importing data in qgep - working ...", level=Qgis.Warning)
+
+        try:
+            self.session.commit()
+            self.session.close()
+        except:
+            rollback_session()
+
         iface.messageBar().pushMessage("Sucess", "Data successfully imported", level=Qgis.Success)
+
+        # add post session - in postimport.py
+        iface.messageBar().pushMessage("Info", "Start postimport", level=Qgis.Info)
+        qgep_postimport()
+
+        iface.messageBar().pushMessage("Sucess", "Finished postimport", level=Qgis.Success)
 
     def rollback_session(self):
         self.session.rollback()

qgepqwat2ili/postimport.py

@@ -0,0 +1,48 @@
+# Needs delta_1.6.3_functions_update_fk_main_cover_main_wastewater_node.sql to work properly
+
+from functools import lru_cache
+
+from geoalchemy2.functions import ST_Force3D
+from sqlalchemy.orm import Session
+from sqlalchemy.orm.attributes import flag_dirty
+
+#31.5.2024 pfade anpassen
+#from .. import utils
+#from ..utils.various import logger
+from . import utils
+from .utils.various import logger
+#from .model_abwasser import get_abwasser_model
+#from .model_qgep import get_qgep_model
+
+
+def qgep_postimport():
+    """
+    Additional queries run after qgep_import
+    """
+
+
+    # move in extra file and function postimport
+    # TODO : put this in an "finally" block (or context handler) to make sure it's executed
+    # even if there's an exception
+
+
+    post_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
+    logger.info("re-enabling symbology triggers (postimport.py)")
+    post_session.execute("SELECT qgep_sys.create_symbology_triggers();")
+    logger.info("symbology triggers successfully created! (postimport.py)")
+
+    # add queries for main_cover and main_node as in TEKSI, add to symbology functions
+    # see teksi ww https://github.com/teksi/wastewater/blob/3acfba249866d299f8a22e249d9f1e475fe7b88d/datamodel/app/symbology_functions.sql#L290
+    # needs delta_1.6.3_functions_update_fk_main_cover_main_wastewater_node.sql
+    logger.info("Update wastewater structure fk_main_cover")
+    post_session.execute("SELECT qgep_od.wastewater_structure_update_fk_main_cover('', True);")
+
+    logger.info("Update wastewater structure fk_main_wastewater_node")
+    post_session.execute("SELECT qgep_od.wastewater_structure_update_fk_main_wastewater_node('', True);"
+    )
+
+    logger.info("Refresh materialized views")
+    post_session.execute("SELECT qgep_network.refresh_network_simple();")
+
+    post_session.commit()
+    post_session.close()

qgepqwat2ili/qgep/import_.py

@@ -37,7 +37,8 @@ def qgep_import(precommit_callback=None):
     abwasser_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
     qgep_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
 
-    # Allow to insert rows with cyclic dependencies at once
+    # Allow to insert rows with cyclic dependencies at once, needs data modell version 1.6.2 https://github.com/QGEP/datamodel/pull/235 to work properly
+    logger.info("SET CONSTRAINTS ALL DEFERRED;")
     qgep_session.execute("SET CONSTRAINTS ALL DEFERRED;")
 
     def get_vl_instance(vl_table, value):
@@ -1091,20 +1092,19 @@ def structure_part_common(row):
         print(".", end="")
     logger.info("done")
 
-    # Recreate the triggers
-    # qgep_session.execute('SELECT qgep_sys.create_symbology_triggers();')
 
     # Calling the precommit callback if provided, allowing to filter before final import
     if precommit_callback:
         precommit_callback(qgep_session)
+        logger.info("precommit_callback(qgep_session)")
+        # improve user feedback
+        logger.info("Comitting qgep_session - please be patient ...")
     else:
+        # improve user feedback
+        logger.info("Comitting qgep_session - please be patient ...")
         qgep_session.commit()
+        logger.info("qgep_session sucessfully committed")
         qgep_session.close()
+        logger.info("qgep_session closed")
     abwasser_session.close()
-
-    # TODO : put this in an "finally" block (or context handler) to make sure it's executed
-    # even if there's an exception
-    post_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
-    post_session.execute("SELECT qgep_sys.create_symbology_triggers();")
-    post_session.commit()
-    post_session.close()
+    logger.info("abwasser_session closed")

qgepqwat2ili/qgepdss/import_.py

@@ -29,6 +29,7 @@ def qgep_import(precommit_callback=None):
 
     # We also drop symbology triggers as they badly affect performance. This must be done in a separate session as it
     # would deadlock other sessions.
+    logger.info("drop symbology triggers")
     pre_session.execute("SELECT qgep_sys.drop_symbology_triggers();")
     pre_session.commit()
     pre_session.close()
@@ -40,25 +41,9 @@ def qgep_import(precommit_callback=None):
     qgep_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
 
 
-    # 29.3.2023 Add additional DEREFERABLE constraints
-    # ALTER TABLE qgep_od.overflow ADD CONSTRAINT rel_overflow_overflow_to FOREIGN KEY (fk_overflow_to) REFERENCES qgep_od.wastewater_node(obj_id) ON UPDATE CASCADE ON DELETE set null;
 
-    # qgep_session.execute("ALTER TABLE qgep_od.overflow ALTER CONSTRAINT rel_overflow_overflow_to DEFERRABLE;")
-    # logger.info("ALTER TABLE qgep_od.overflow ALTER CONSTRAINT rel_overflow_overflow_to DEFERRABLE;")
-
-    # qgep_session.execute("ALTER TABLE qgep_od.throttle_shut_off_unit ALTER CONSTRAINT rel_throttle_shut_off_unit_wastewater_node DEFERRABLE;")
-    # logger.info("ALTER TABLE qgep_od.throttle_shut_off_unit ALTER CONSTRAINT rel_throttle_shut_off_unit_wastewater_node DEFERRABLE;")
-
-    # qgep_session.execute("ALTER TABLE qgep_od.tank_emptying ALTER CONSTRAINT rel_tank_emptying_overflow DEFERRABLE;")
-    # logger.info("ALTER TABLE qgep_od.tank_emptying ALTER CONSTRAINT rel_tank_emptying_overflow DEFERRABLE;")
-
-    # qgep_session.execute("ALTER TABLE qgep_od.wastewater_networkelement ALTER CONSTRAINT rel_wastewater_networkelement_wastewater_structure DEFERRABLE;")
-    # logger.info("ALTER TABLE qgep_od.wastewater_networkelement ALTER CONSTRAINT rel_wastewater_networkelement_wastewater_structure DEFERRABLE;")
-
-    # qgep_session.execute("ALTER TABLE qgep_od.re_maintenance_event_wastewater_structure ALTER CONSTRAINT rel_maintenance_event_wastewater_structure_maintenance_event DEFERRABLE;")
-    # logger.info("ALTER TABLE qgep_od.re_maintenance_event_wastewater_structure ALTER CONSTRAINT rel_maintenance_event_wastewater_structure_maintenance_event DEFERRABLE;")
-
-    # Allow to insert rows with cyclic dependencies at once
+    # Allow to insert rows with cyclic dependencies at once, needs data modell version 1.6.2 https://github.com/QGEP/datamodel/pull/235 to work properly
+    logger.info("SET CONSTRAINTS ALL DEFERRED;")
     qgep_session.execute("SET CONSTRAINTS ALL DEFERRED;")
 
 
@@ -71,9 +56,11 @@ def get_vl_instance(vl_table, value):
         # TODO : return "other" (or other applicable value) rather than None, or even throwing an exception, would probably be better
         row = qgep_session.query(vl_table).filter(vl_table.value_de == value).first()
         if row is None:
-            logger.warning(
-                f'Could not find value `{value}` in value list "{vl_table.__table__.schema}.{vl_table.__name__}". Setting to None instead.'
-            )
+            # write logger.warning only if value is not None
+            if value != None:
+                logger.warning(
+                    f'Could not find value `{value}` in value list "{vl_table.__table__.schema}.{vl_table.__name__}". Setting to None instead.'
+                )
             return None
         return row
 
@@ -3743,7 +3730,7 @@ def overflow_common(row):
         print(".", end="")
     logger.info("done")
 
-#neu 19.4.2023
+# added logger info
     logger.info("Importing ABWASSER.erhaltungsereignis, ABWASSER.metaattribute -> QGEP.maintenance_event")
     for row, metaattribute in abwasser_session.query(ABWASSER.erhaltungsereignis, ABWASSER.metaattribute).join(
         ABWASSER.metaattribute
@@ -3793,24 +3780,19 @@ def overflow_common(row):
     logger.info("done")
 
 
-    # Recreate the triggers
-    # qgep_session.execute('SELECT qgep_sys.create_symbology_triggers();')
 
     # Calling the precommit callback if provided, allowing to filter before final import
     if precommit_callback:
         precommit_callback(qgep_session)
         logger.info("precommit_callback(qgep_session)")
+        # improve user feedback
+        logger.info("Comitting qgep_session (precommit_callback) - please be patient ...")
     else:
+        # improve user feedback
+        logger.info("Comitting qgep_session - please be patient (else) ...")
         qgep_session.commit()
+        logger.info("qgep_session sucessfully committed")
         qgep_session.close()
+        logger.info("qgep_session closed")
     abwasser_session.close()
     logger.info("abwasser_session closed")
-
-    # TODO : put this in an "finally" block (or context handler) to make sure it's executed
-    # even if there's an exception
-    post_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
-    logger.info("re-enabling symbology triggers")
-    post_session.execute("SELECT qgep_sys.create_symbology_triggers();")
-    logger.info("symbology triggers successfully created!")
-    post_session.commit()
-    post_session.close()

qgepqwat2ili/qgepsia405/import_.py

@@ -37,7 +37,8 @@ def qgep_import(precommit_callback=None):
     abwasser_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
     qgep_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
 
-    # Allow to insert rows with cyclic dependencies at once
+    # Allow to insert rows with cyclic dependencies at once, needs data modell version 1.6.2 https://github.com/QGEP/datamodel/pull/235 to work properly
+    logger.info("SET CONSTRAINTS ALL DEFERRED;")
     qgep_session.execute("SET CONSTRAINTS ALL DEFERRED;")
 
     def get_vl_instance(vl_table, value):
@@ -1090,20 +1091,19 @@ def structure_part_common(row):
     # logger.info("done")
 
 
-    # Recreate the triggers
-    # qgep_session.execute('SELECT qgep_sys.create_symbology_triggers();')
 
     # Calling the precommit callback if provided, allowing to filter before final import
     if precommit_callback:
         precommit_callback(qgep_session)
+        logger.info("precommit_callback(qgep_session)")
+        # improve user feedback
+        logger.info("Comitting qgep_session (precommit_callback) - please be patient ...")
     else:
+        # improve user feedback
+        logger.info("Comitting qgep_session - please be patient ...")
         qgep_session.commit()
+        logger.info("qgep_session sucessfully committed")
         qgep_session.close()
+        logger.info("qgep_session closed")
     abwasser_session.close()
-
-    # TODO : put this in an "finally" block (or context handler) to make sure it's executed
-    # even if there's an exception
-    post_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
-    post_session.execute("SELECT qgep_sys.create_symbology_triggers();")
-    post_session.commit()
-    post_session.close()
+    logger.info("abwasser_session closed")

qgepqwat2ili/utils/various.py

@@ -27,7 +27,7 @@ def _log(self, level, msg, args, exc_info=None, extra=None):
         this_message = (level, msg)
         if self._last_message is None or self._last_message != this_message:
             if self._repeated > 0:
-                super()._log(self._last_message[0], f"[repeted {self._repeated} times]", args, exc_info, extra)
+                super()._log(self._last_message[0], f"[repeated {self._repeated} times]", args, exc_info, extra)
 
             super()._log(level, msg, args, exc_info, extra)
             self._repeated = 0