Improve user feedback and speed-up interlis import qgep

qgepqwat2ili/gui/__init__.py

@@ -45,6 +45,8 @@
 # 19.4.2023 / 25.4.2023 ohne Bindestrich / neu aus gui_import - Gui
 from .gui_importc import GuiImportc
 
+
+
 # 12.7.2022 for testing import time
 import time
 
@@ -354,12 +356,14 @@ def action_import(plugin):
                  log_path,
             )
 
-    progress_dialog.setLabelText("Set main_cover manually after import if vw_qgep_wastewater_structure does not display correctly!")
+    # 31.5.2024 should not be needed anymore
+    # progress_dialog.setLabelText("Set main_cover manually after import if vw_qgep_wastewater_structure does not display correctly!")
 
     time.sleep(2)
-    # to add option to run main_cover.sql manually
-    
+    # to add option to run main_cover.sql manually - see postimport.py
+
     # 24.7.2022 / moved to end
+    # shows progress of import to import wizard, not final commit
     progress_dialog.setValue(100)
 
 

qgepqwat2ili/gui/gui_import.py

@@ -19,6 +19,9 @@
 import logging
 from qgis.PyQt.QtCore import pyqtSlot
 
+# 31.5.2024 pfad angepasst, neu in gui_import.py statt _init_.py
+from ..postimport import qgep_postimport
+
 # Required for loadUi to find the custom widget
 sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
 
@@ -213,9 +216,24 @@ def commit_session(self):
         # TODO : rollback to pre-commit state, allowing user to try to fix issues
         # probably a matter of creating a savepoint before saving with
         # session.begin_nested() and one additionnal self.session.commit()
+
+        # 11.5.2024 try to add info in message bar
+        iface.messageBar().pushMessage("Please be patient!", "Importing data in qgep - working ...", level=Qgis.Warning)
+        time.sleep(5)
+        iface.messageBar().pushMessage("Please be patient!", "Importing data in qgep - still working ...", level=Qgis.Warning)
+
         self.session.commit()
+
         self.session.close()
         iface.messageBar().pushMessage("Sucess", "Data successfully imported", level=Qgis.Success)
+
+        # 31.5.2024 add post session - in postimport.py machen stattdessen
+        # init qgep_postimport
+        iface.messageBar().pushMessage("Info", "Start postimport", level=Qgis.Info)
+
+        qgep_postimport()
+
+        iface.messageBar().pushMessage("Sucess", "Finished postimport", level=Qgis.Success)
 
     def rollback_session(self):
         self.session.rollback()

qgepqwat2ili/postimport.py

@@ -0,0 +1,55 @@
+# Needs delta_1.6.3_functions_update_fk_main_cover_main_wastewater_node.sql to work properly
+
+from functools import lru_cache
+
+from geoalchemy2.functions import ST_Force3D
+from sqlalchemy.orm import Session
+from sqlalchemy.orm.attributes import flag_dirty
+
+#31.5.2024 pfade anpassen
+#from .. import utils
+#from ..utils.various import logger
+from . import utils
+from .utils.various import logger
+#from .model_abwasser import get_abwasser_model
+#from .model_qgep import get_qgep_model
+
+
+def qgep_postimport():
+    """
+    Additional queries run after qgep_import
+    """
+
+
+    # 31.5.2024 move in extra file and function postimport
+
+    # TODO : put this in an "finally" block (or context handler) to make sure it's executed
+    # even if there's an exception
+
+
+    post_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
+    logger.info("re-enabling symbology triggers (postimport.py)")
+    post_session.execute("SELECT qgep_sys.create_symbology_triggers();")
+    logger.info("symbology triggers successfully created! (postimport.py)")
+    post_session.commit()
+    post_session.close()
+
+    #11.5.2024 add post_session2 - to do add queries for main_cover and main_node as in TEKSI, add to symbology functions
+    # see teksi ww https://github.com/teksi/wastewater/blob/3acfba249866d299f8a22e249d9f1e475fe7b88d/datamodel/app/symbology_functions.sql#L290
+    # needs delta_1.6.3_functions_update_fk_main_cover_main_wastewater_node.sql
+
+
+    post_session2 = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
+
+    logger.info("Update wastewater structure fk_main_cover")
+    post_session2.execute("SELECT qgep_od.wastewater_structure_update_fk_main_cover('', True);")
+
+    logger.info("Update wastewater structure fk_main_wastewater_node")
+    post_session2.execute("SELECT qgep_od.wastewater_structure_update_fk_main_wastewater_node('', True);"
+    )
+
+    logger.info("Refresh materialized views")
+    post_session2.execute("SELECT qgep_network.refresh_network_simple();")
+
+    post_session2.commit()
+    post_session2.close()

qgepqwat2ili/qgep/import_.py

@@ -1097,14 +1097,22 @@ def structure_part_common(row):
     # Calling the precommit callback if provided, allowing to filter before final import
     if precommit_callback:
         precommit_callback(qgep_session)
+        logger.info("precommit_callback(qgep_session)")
+        # 11.5.2024 improve user feedback
+        logger.info("Comitting qgep_session - please be patient ...")
     else:
+        # 11.5.2024 improve user feedback
+        logger.info("Comitting qgep_session - please be patient ...")
         qgep_session.commit()
         qgep_session.close()
     abwasser_session.close()
 
+# 31.5.2024 seems to be at wrong place here - needs to be added to gui/gui_import.py - else it is executed too early.
+
     # TODO : put this in an "finally" block (or context handler) to make sure it's executed
     # even if there's an exception
-    post_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
-    post_session.execute("SELECT qgep_sys.create_symbology_triggers();")
-    post_session.commit()
-    post_session.close()
+
+    # post_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
+    # post_session.execute("SELECT qgep_sys.create_symbology_triggers();")
+    # post_session.commit()
+    # post_session.close()

qgepqwat2ili/qgepdss/import_.py

@@ -29,6 +29,7 @@ def qgep_import(precommit_callback=None):
 
     # We also drop symbology triggers as they badly affect performance. This must be done in a separate session as it
     # would deadlock other sessions.
+    logger.info("drop symbology triggers")
     pre_session.execute("SELECT qgep_sys.drop_symbology_triggers();")
     pre_session.commit()
     pre_session.close()
@@ -59,6 +60,7 @@ def qgep_import(precommit_callback=None):
     # logger.info("ALTER TABLE qgep_od.re_maintenance_event_wastewater_structure ALTER CONSTRAINT rel_maintenance_event_wastewater_structure_maintenance_event DEFERRABLE;")
 
     # Allow to insert rows with cyclic dependencies at once
+    logger.info("SET CONSTRAINTS ALL DEFERRED;")
     qgep_session.execute("SET CONSTRAINTS ALL DEFERRED;")
 
 
@@ -71,9 +73,11 @@ def get_vl_instance(vl_table, value):
         # TODO : return "other" (or other applicable value) rather than None, or even throwing an exception, would probably be better
         row = qgep_session.query(vl_table).filter(vl_table.value_de == value).first()
         if row is None:
-            logger.warning(
-                f'Could not find value `{value}` in value list "{vl_table.__table__.schema}.{vl_table.__name__}". Setting to None instead.'
-            )
+            # 12.5.2024 write logger.warning only if value is not None
+            if value != None:
+                logger.warning(
+                    f'Could not find value `{value}` in value list "{vl_table.__table__.schema}.{vl_table.__name__}". Setting to None instead.'
+                )
             return None
         return row
 
@@ -3800,17 +3804,48 @@ def overflow_common(row):
     if precommit_callback:
         precommit_callback(qgep_session)
         logger.info("precommit_callback(qgep_session)")
+        # 11.5.2024 improve user feedback
+        logger.info("Comitting qgep_session (precommit_callback) - please be patient ...")
     else:
+        # 11.5.2024 improve user feedback
+        logger.info("Comitting qgep_session - please be patient (else) ...")
         qgep_session.commit()
+        logger.info("qgep_session sucessfully committed")
         qgep_session.close()
+
     abwasser_session.close()
     logger.info("abwasser_session closed")
+
+# 31.5.2024 seems to be at wrong place here - needs to be added to gui/gui_import.py - else it is executed too early.
 
     # TODO : put this in an "finally" block (or context handler) to make sure it's executed
     # even if there's an exception
-    post_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
-    logger.info("re-enabling symbology triggers")
-    post_session.execute("SELECT qgep_sys.create_symbology_triggers();")
-    logger.info("symbology triggers successfully created!")
-    post_session.commit()
-    post_session.close()
+
+#31.5.2024 commented out and moved to postimport.py
+    # post_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
+    # logger.info("re-enabling symbology triggers")
+    # post_session.execute("SELECT qgep_sys.create_symbology_triggers();")
+    # logger.info("symbology triggers successfully created!")
+    # post_session.commit()
+    # post_session.close()
+
+    #11.5.2024 add post_session2 - to do add queries for main_cover and main_node as in TEKSI, add to symbology functions
+    # see teksi ww https://github.com/teksi/wastewater/blob/3acfba249866d299f8a22e249d9f1e475fe7b88d/datamodel/app/symbology_functions.sql#L290
+    # needs also delta_1.6.3_functions_update_fk_main_cover_main_wastewater_node.sql
+
+#31.5.2024 commented out and moved to postimport.py
+    # post_session2 = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
+
+    # logger.info("Update wastewater structure fk_main_cover")
+    # post_session2.execute("SELECT qgep_od.wastewater_structure_update_fk_main_cover('', True);")
+
+    # logger.info("Update wastewater structure fk_main_wastewater_node")
+    # post_session2.execute(
+            # "SELECT qgep_od.wastewater_structure_update_fk_main_wastewater_node('', True);"
+    # )
+
+    # logger.info("Refresh materialized views")
+    # post_session2.execute("SELECT qgep_network.refresh_network_simple();")
+
+    # post_session2.commit()
+    # post_session2.close()

qgepqwat2ili/qgepsia405/import_.py

@@ -1096,14 +1096,22 @@ def structure_part_common(row):
     # Calling the precommit callback if provided, allowing to filter before final import
     if precommit_callback:
         precommit_callback(qgep_session)
+        logger.info("precommit_callback(qgep_session)")
+        # 11.5.2024 improve user feedback
+        logger.info("Comitting qgep_session (precommit_callback) - please be patient ...")
     else:
+        # 11.5.2024 improve user feedback
+        logger.info("Comitting qgep_session - please be patient ...")
         qgep_session.commit()
         qgep_session.close()
     abwasser_session.close()
 
+# 31.5.2024 seems to be at wrong place here - needs to be added to gui/gui_import.py - else it is executed too early.
+
     # TODO : put this in an "finally" block (or context handler) to make sure it's executed
     # even if there's an exception
-    post_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
-    post_session.execute("SELECT qgep_sys.create_symbology_triggers();")
-    post_session.commit()
-    post_session.close()
+
+    # post_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False)
+    # post_session.execute("SELECT qgep_sys.create_symbology_triggers();")
+    # post_session.commit()
+    # post_session.close()

qgepqwat2ili/utils/various.py

@@ -27,7 +27,7 @@ def _log(self, level, msg, args, exc_info=None, extra=None):
         this_message = (level, msg)
         if self._last_message is None or self._last_message != this_message:
             if self._repeated > 0:
-                super()._log(self._last_message[0], f"[repeted {self._repeated} times]", args, exc_info, extra)
+                super()._log(self._last_message[0], f"[repeated {self._repeated} times]", args, exc_info, extra)
 
             super()._log(level, msg, args, exc_info, extra)
             self._repeated = 0