diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index 0d4fe130..2bc2f723 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -45,6 +45,8 @@ # 19.4.2023 / 25.4.2023 ohne Bindestrich / neu aus gui_import - Gui from .gui_importc import GuiImportc + + # 12.7.2022 for testing import time import time @@ -354,12 +356,14 @@ def action_import(plugin): log_path, ) - progress_dialog.setLabelText("Set main_cover manually after import if vw_qgep_wastewater_structure does not display correctly!") + # 31.5.2024 should not be needed anymore + # progress_dialog.setLabelText("Set main_cover manually after import if vw_qgep_wastewater_structure does not display correctly!") time.sleep(2) - # to add option to run main_cover.sql manually - + # to add option to run main_cover.sql manually - see postimport.py + # 24.7.2022 / moved to end + # shows progress of import to import wizard, not final commit progress_dialog.setValue(100) diff --git a/qgepqwat2ili/gui/gui_import.py b/qgepqwat2ili/gui/gui_import.py index a5574952..2c681451 100644 --- a/qgepqwat2ili/gui/gui_import.py +++ b/qgepqwat2ili/gui/gui_import.py @@ -19,6 +19,9 @@ import logging from qgis.PyQt.QtCore import pyqtSlot +# 31.5.2024 pfad angepasst, neu in gui_import.py statt _init_.py +from ..postimport import qgep_postimport + # Required for loadUi to find the custom widget sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) @@ -213,9 +216,23 @@ def commit_session(self): # TODO : rollback to pre-commit state, allowing user to try to fix issues # probably a matter of creating a savepoint before saving with # session.begin_nested() and one additionnal self.session.commit() - self.session.commit() - self.session.close() + + # add info in message bar + iface.messageBar().pushMessage("Please be patient!", "Importing data in qgep - working ...", level=Qgis.Warning) + + try: + self.session.commit() + self.session.close() + except: + rollback_session() + iface.messageBar().pushMessage("Sucess", "Data successfully imported", level=Qgis.Success) + + # add post session - in postimport.py + iface.messageBar().pushMessage("Info", "Start postimport", level=Qgis.Info) + qgep_postimport() + + iface.messageBar().pushMessage("Sucess", "Finished postimport", level=Qgis.Success) def rollback_session(self): self.session.rollback() diff --git a/qgepqwat2ili/postimport.py b/qgepqwat2ili/postimport.py new file mode 100644 index 00000000..2c8f994c --- /dev/null +++ b/qgepqwat2ili/postimport.py @@ -0,0 +1,48 @@ +# Needs delta_1.6.3_functions_update_fk_main_cover_main_wastewater_node.sql to work properly + +from functools import lru_cache + +from geoalchemy2.functions import ST_Force3D +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_dirty + +#31.5.2024 pfade anpassen +#from .. import utils +#from ..utils.various import logger +from . import utils +from .utils.various import logger +#from .model_abwasser import get_abwasser_model +#from .model_qgep import get_qgep_model + + +def qgep_postimport(): + """ + Additional queries run after qgep_import + """ + + + # move in extra file and function postimport + # TODO : put this in an "finally" block (or context handler) to make sure it's executed + # even if there's an exception + + + post_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False) + logger.info("re-enabling symbology triggers (postimport.py)") + post_session.execute("SELECT qgep_sys.create_symbology_triggers();") + logger.info("symbology triggers successfully created! (postimport.py)") + + # add queries for main_cover and main_node as in TEKSI, add to symbology functions + # see teksi ww https://github.com/teksi/wastewater/blob/3acfba249866d299f8a22e249d9f1e475fe7b88d/datamodel/app/symbology_functions.sql#L290 + # needs delta_1.6.3_functions_update_fk_main_cover_main_wastewater_node.sql + logger.info("Update wastewater structure fk_main_cover") + post_session.execute("SELECT qgep_od.wastewater_structure_update_fk_main_cover('', True);") + + logger.info("Update wastewater structure fk_main_wastewater_node") + post_session.execute("SELECT qgep_od.wastewater_structure_update_fk_main_wastewater_node('', True);" + ) + + logger.info("Refresh materialized views") + post_session.execute("SELECT qgep_network.refresh_network_simple();") + + post_session.commit() + post_session.close() \ No newline at end of file diff --git a/qgepqwat2ili/qgep/import_.py b/qgepqwat2ili/qgep/import_.py index db7bdbc9..71963e98 100644 --- a/qgepqwat2ili/qgep/import_.py +++ b/qgepqwat2ili/qgep/import_.py @@ -37,7 +37,8 @@ def qgep_import(precommit_callback=None): abwasser_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False) qgep_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False) - # Allow to insert rows with cyclic dependencies at once + # Allow to insert rows with cyclic dependencies at once, needs data modell version 1.6.2 https://github.com/QGEP/datamodel/pull/235 to work properly + logger.info("SET CONSTRAINTS ALL DEFERRED;") qgep_session.execute("SET CONSTRAINTS ALL DEFERRED;") def get_vl_instance(vl_table, value): @@ -1091,20 +1092,19 @@ def structure_part_common(row): print(".", end="") logger.info("done") - # Recreate the triggers - # qgep_session.execute('SELECT qgep_sys.create_symbology_triggers();') # Calling the precommit callback if provided, allowing to filter before final import if precommit_callback: precommit_callback(qgep_session) + logger.info("precommit_callback(qgep_session)") + # improve user feedback + logger.info("Comitting qgep_session - please be patient ...") else: + # improve user feedback + logger.info("Comitting qgep_session - please be patient ...") qgep_session.commit() + logger.info("qgep_session sucessfully committed") qgep_session.close() + logger.info("qgep_session closed") abwasser_session.close() - - # TODO : put this in an "finally" block (or context handler) to make sure it's executed - # even if there's an exception - post_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False) - post_session.execute("SELECT qgep_sys.create_symbology_triggers();") - post_session.commit() - post_session.close() + logger.info("abwasser_session closed") diff --git a/qgepqwat2ili/qgepdss/import_.py b/qgepqwat2ili/qgepdss/import_.py index 073b1c62..14be4bf8 100644 --- a/qgepqwat2ili/qgepdss/import_.py +++ b/qgepqwat2ili/qgepdss/import_.py @@ -29,6 +29,7 @@ def qgep_import(precommit_callback=None): # We also drop symbology triggers as they badly affect performance. This must be done in a separate session as it # would deadlock other sessions. + logger.info("drop symbology triggers") pre_session.execute("SELECT qgep_sys.drop_symbology_triggers();") pre_session.commit() pre_session.close() @@ -40,25 +41,9 @@ def qgep_import(precommit_callback=None): qgep_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False) - # 29.3.2023 Add additional DEREFERABLE constraints - # ALTER TABLE qgep_od.overflow ADD CONSTRAINT rel_overflow_overflow_to FOREIGN KEY (fk_overflow_to) REFERENCES qgep_od.wastewater_node(obj_id) ON UPDATE CASCADE ON DELETE set null; - # qgep_session.execute("ALTER TABLE qgep_od.overflow ALTER CONSTRAINT rel_overflow_overflow_to DEFERRABLE;") - # logger.info("ALTER TABLE qgep_od.overflow ALTER CONSTRAINT rel_overflow_overflow_to DEFERRABLE;") - - # qgep_session.execute("ALTER TABLE qgep_od.throttle_shut_off_unit ALTER CONSTRAINT rel_throttle_shut_off_unit_wastewater_node DEFERRABLE;") - # logger.info("ALTER TABLE qgep_od.throttle_shut_off_unit ALTER CONSTRAINT rel_throttle_shut_off_unit_wastewater_node DEFERRABLE;") - - # qgep_session.execute("ALTER TABLE qgep_od.tank_emptying ALTER CONSTRAINT rel_tank_emptying_overflow DEFERRABLE;") - # logger.info("ALTER TABLE qgep_od.tank_emptying ALTER CONSTRAINT rel_tank_emptying_overflow DEFERRABLE;") - - # qgep_session.execute("ALTER TABLE qgep_od.wastewater_networkelement ALTER CONSTRAINT rel_wastewater_networkelement_wastewater_structure DEFERRABLE;") - # logger.info("ALTER TABLE qgep_od.wastewater_networkelement ALTER CONSTRAINT rel_wastewater_networkelement_wastewater_structure DEFERRABLE;") - - # qgep_session.execute("ALTER TABLE qgep_od.re_maintenance_event_wastewater_structure ALTER CONSTRAINT rel_maintenance_event_wastewater_structure_maintenance_event DEFERRABLE;") - # logger.info("ALTER TABLE qgep_od.re_maintenance_event_wastewater_structure ALTER CONSTRAINT rel_maintenance_event_wastewater_structure_maintenance_event DEFERRABLE;") - - # Allow to insert rows with cyclic dependencies at once + # Allow to insert rows with cyclic dependencies at once, needs data modell version 1.6.2 https://github.com/QGEP/datamodel/pull/235 to work properly + logger.info("SET CONSTRAINTS ALL DEFERRED;") qgep_session.execute("SET CONSTRAINTS ALL DEFERRED;") @@ -71,9 +56,11 @@ def get_vl_instance(vl_table, value): # TODO : return "other" (or other applicable value) rather than None, or even throwing an exception, would probably be better row = qgep_session.query(vl_table).filter(vl_table.value_de == value).first() if row is None: - logger.warning( - f'Could not find value `{value}` in value list "{vl_table.__table__.schema}.{vl_table.__name__}". Setting to None instead.' - ) + # write logger.warning only if value is not None + if value != None: + logger.warning( + f'Could not find value `{value}` in value list "{vl_table.__table__.schema}.{vl_table.__name__}". Setting to None instead.' + ) return None return row @@ -3743,7 +3730,7 @@ def overflow_common(row): print(".", end="") logger.info("done") -#neu 19.4.2023 +# added logger info logger.info("Importing ABWASSER.erhaltungsereignis, ABWASSER.metaattribute -> QGEP.maintenance_event") for row, metaattribute in abwasser_session.query(ABWASSER.erhaltungsereignis, ABWASSER.metaattribute).join( ABWASSER.metaattribute @@ -3793,24 +3780,19 @@ def overflow_common(row): logger.info("done") - # Recreate the triggers - # qgep_session.execute('SELECT qgep_sys.create_symbology_triggers();') # Calling the precommit callback if provided, allowing to filter before final import if precommit_callback: precommit_callback(qgep_session) logger.info("precommit_callback(qgep_session)") + # improve user feedback + logger.info("Comitting qgep_session (precommit_callback) - please be patient ...") else: + # improve user feedback + logger.info("Comitting qgep_session - please be patient (else) ...") qgep_session.commit() + logger.info("qgep_session sucessfully committed") qgep_session.close() + logger.info("qgep_session closed") abwasser_session.close() logger.info("abwasser_session closed") - - # TODO : put this in an "finally" block (or context handler) to make sure it's executed - # even if there's an exception - post_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False) - logger.info("re-enabling symbology triggers") - post_session.execute("SELECT qgep_sys.create_symbology_triggers();") - logger.info("symbology triggers successfully created!") - post_session.commit() - post_session.close() diff --git a/qgepqwat2ili/qgepsia405/import_.py b/qgepqwat2ili/qgepsia405/import_.py index b24d1cff..46900360 100644 --- a/qgepqwat2ili/qgepsia405/import_.py +++ b/qgepqwat2ili/qgepsia405/import_.py @@ -37,7 +37,8 @@ def qgep_import(precommit_callback=None): abwasser_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False) qgep_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False) - # Allow to insert rows with cyclic dependencies at once + # Allow to insert rows with cyclic dependencies at once, needs data modell version 1.6.2 https://github.com/QGEP/datamodel/pull/235 to work properly + logger.info("SET CONSTRAINTS ALL DEFERRED;") qgep_session.execute("SET CONSTRAINTS ALL DEFERRED;") def get_vl_instance(vl_table, value): @@ -1090,20 +1091,19 @@ def structure_part_common(row): # logger.info("done") - # Recreate the triggers - # qgep_session.execute('SELECT qgep_sys.create_symbology_triggers();') # Calling the precommit callback if provided, allowing to filter before final import if precommit_callback: precommit_callback(qgep_session) + logger.info("precommit_callback(qgep_session)") + # improve user feedback + logger.info("Comitting qgep_session (precommit_callback) - please be patient ...") else: + # improve user feedback + logger.info("Comitting qgep_session - please be patient ...") qgep_session.commit() + logger.info("qgep_session sucessfully committed") qgep_session.close() + logger.info("qgep_session closed") abwasser_session.close() - - # TODO : put this in an "finally" block (or context handler) to make sure it's executed - # even if there's an exception - post_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False) - post_session.execute("SELECT qgep_sys.create_symbology_triggers();") - post_session.commit() - post_session.close() + logger.info("abwasser_session closed") diff --git a/qgepqwat2ili/utils/various.py b/qgepqwat2ili/utils/various.py index b03e6444..40a815df 100644 --- a/qgepqwat2ili/utils/various.py +++ b/qgepqwat2ili/utils/various.py @@ -27,7 +27,7 @@ def _log(self, level, msg, args, exc_info=None, extra=None): this_message = (level, msg) if self._last_message is None or self._last_message != this_message: if self._repeated > 0: - super()._log(self._last_message[0], f"[repeted {self._repeated} times]", args, exc_info, extra) + super()._log(self._last_message[0], f"[repeated {self._repeated} times]", args, exc_info, extra) super()._log(level, msg, args, exc_info, extra) self._repeated = 0