add sia405 export for QGEP (#84)

Co-authored-by: Olivier Dalang <[email protected]>

README.md

@@ -51,6 +51,7 @@ optional arguments:
   --pgservice PGSERVICE
                         name of the pgservice to use to connect to the database (default: pg_qgep)
   --log                 saves a log file next to the input/output file (default: False)
+  --export_sia405       export the model SIA405_ABWASSER_2015_LV95 (instead of default VSA_KEK_2019_LV95) (default: False)
 ```
 
 ### Import/export QWAT

qgepqwat2ili/__init__.py

@@ -61,6 +61,11 @@ def main(args):
         action="store_true",
         help="saves the log files next to the input/output file",
     )
+    parser_qgep.add_argument(
+        "--export_sia405",
+        action="store_true",
+        help="export the model SIA405_ABWASSER_2015_LV95 (instead of default VSA_KEK_2019_LV95)",
+    )
 
     parser_qwat = subparsers.add_parser(
         "qwat",
@@ -129,13 +134,23 @@ def main(args):
         config.PGSERVICE = args.pgservice
         SCHEMA = config.ABWASSER_SCHEMA
         ILI_MODEL = config.ABWASSER_ILI_MODEL
-        ILI_MODEL_NAME = config.ABWASSER_ILI_MODEL_NAME
+        if args.export_sia405:
+            ILI_MODEL_NAME = config.ABWASSER_ILI_MODEL_NAME_SIA405
+            ILI_EXPORT_MODEL_NAME = config.ABWASSER_ILI_MODEL_NAME_SIA405
+        else:
+            ILI_MODEL_NAME = config.ABWASSER_ILI_MODEL_NAME
+            ILI_EXPORT_MODEL_NAME = None
+
         if args.direction == "export":
             utils.ili2db.create_ili_schema(
                 SCHEMA, ILI_MODEL, make_log_path(log_path, "ilicreate"), recreate_schema=args.recreate_schema
             )
             qgep_export(selection=args.selection.split(",") if args.selection else None, labels_file=args.labels_file)
-            utils.ili2db.export_xtf_data(SCHEMA, ILI_MODEL_NAME, args.path, make_log_path(log_path, "iliexport"))
+
+            utils.ili2db.export_xtf_data(
+                SCHEMA, ILI_MODEL_NAME, ILI_EXPORT_MODEL_NAME, args.path, make_log_path(log_path, "iliexport")
+            )
+
             if not args.skip_validation:
                 try:
                     utils.ili2db.validate_xtf_data(args.path, make_log_path(log_path, "ilivalidate"))
@@ -164,12 +179,16 @@ def main(args):
         SCHEMA = config.WASSER_SCHEMA
         ILI_MODEL = config.WASSER_ILI_MODEL
         ILI_MODEL_NAME = config.WASSER_ILI_MODEL_NAME
+        ILI_EXPORT_MODEL_NAME = None
+
         if args.direction == "export":
             utils.ili2db.create_ili_schema(
                 SCHEMA, ILI_MODEL, make_log_path(log_path, "ilicreate"), recreate_schema=args.recreate_schema
             )
             qwat_export(include_hydraulics=args.include_hydraulics)
-            utils.ili2db.export_xtf_data(SCHEMA, ILI_MODEL_NAME, args.path, make_log_path(log_path, "iliexport"))
+            utils.ili2db.export_xtf_data(
+                SCHEMA, ILI_MODEL_NAME, ILI_EXPORT_MODEL_NAME, args.path, make_log_path(log_path, "iliexport")
+            )
             if not args.skip_validation:
                 try:
                     utils.ili2db.validate_xtf_data(args.path, make_log_path(log_path, "ilivalidate"))

qgepqwat2ili/config.py

@@ -19,6 +19,8 @@
 ABWASSER_SCHEMA = "pg2ili_abwasser"
 ABWASSER_ILI_MODEL = os.path.join(ILI_FOLDER, "VSA_KEK_2019_2_d_LV95-20210120.ili")
 ABWASSER_ILI_MODEL_NAME = "VSA_KEK_2019_LV95"
+ABWASSER_ILI_MODEL_NAME_SIA405 = "SIA405_ABWASSER_2015_LV95"
+ABWASSER_ILI_EXPORT_MODEL = os.path.join(ILI_FOLDER, "SIA405_Abwasser_2015_2_d-20180417.ili")
 
 QWAT_DEFAULT_PGSERVICE = "qwat"
 QWAT_SCHEMA = "qwat_od"

qgepqwat2ili/gui/__init__.py

@@ -175,6 +175,9 @@ def action_do_export():
             return
         QgsSettings().setValue("qgep_pluging/last_interlis_path", os.path.dirname(file_name))
 
+        # File name without extension (used later for export)
+        file_name_base, _ = os.path.splitext(file_name)
+
         # Configure logging
         if export_dialog.logs_next_to_file:
             base_log_path = file_name
@@ -254,48 +257,59 @@ def action_do_export():
         # Cleanup
         tempdir.cleanup()
 
-        # Export from ili2pg model to file
-        progress_dialog.setLabelText("Saving XTF file...")
-        QApplication.processEvents()
-        log_path = make_log_path(base_log_path, "ili2pg-export")
-        try:
-            export_xtf_data(
-                config.ABWASSER_SCHEMA,
-                config.ABWASSER_ILI_MODEL_NAME,
-                file_name,
-                log_path,
-            )
-        except CmdException:
-            progress_dialog.close()
-            show_failure(
-                "Could not export the ili2pg schema",
-                "Open the logs for more details on the error.",
-                log_path,
-            )
-            return
-        progress_dialog.setValue(75)
+        for model_name, export_model_name, progress in [
+            (config.ABWASSER_ILI_MODEL_NAME, None, 50),
+            (config.ABWASSER_ILI_MODEL_NAME_SIA405, config.ABWASSER_ILI_MODEL_NAME_SIA405, 70),
+        ]:
+
+            export_file_name = f"{file_name_base}_{model_name}.xtf"
+
+            # Export from ili2pg model to file
+            progress_dialog.setLabelText(f"Saving XTF file [{model_name}]...")
+            QApplication.processEvents()
+            log_path = make_log_path(base_log_path, f"ili2pg-export-{model_name}")
+            try:
+                export_xtf_data(
+                    config.ABWASSER_SCHEMA,
+                    model_name,
+                    export_model_name,
+                    export_file_name,
+                    log_path,
+                )
+            except CmdException:
+                progress_dialog.close()
+                show_failure(
+                    "Could not export the ili2pg schema",
+                    "Open the logs for more details on the error.",
+                    log_path,
+                )
+                continue
+            progress_dialog.setValue(progress + 10)
+
+            progress_dialog.setLabelText(f"Validating the network output file [{model_name}]...")
+            QApplication.processEvents()
+            log_path = make_log_path(base_log_path, f"ilivalidator-{model_name}")
+            try:
+                validate_xtf_data(
+                    export_file_name,
+                    log_path,
+                )
+            except CmdException:
+                progress_dialog.close()
+                show_failure(
+                    "Invalid file",
+                    f"The created file is not a valid {model_name} XTF file.",
+                    log_path,
+                )
+                continue
+
+            progress_dialog.setValue(progress + 20)
 
-        progress_dialog.setLabelText("Validating the output file...")
-        QApplication.processEvents()
-        log_path = make_log_path(base_log_path, "ilivalidator")
-        try:
-            validate_xtf_data(
-                file_name,
-                log_path,
-            )
-        except CmdException:
-            progress_dialog.close()
-            show_failure(
-                "Invalid file",
-                "The created file is not a valid XTF file.",
-                log_path,
-            )
-            return
         progress_dialog.setValue(100)
 
         show_success(
             "Sucess",
-            f"Data successfully exported to {file_name}",
+            f"Data successfully exported to {file_name_base}",
             os.path.dirname(log_path),
         )
 

qgepqwat2ili/tests/test_qgep.py

@@ -18,7 +18,12 @@
 logger.addHandler(handler)
 
 
-def findall_in_xml(root, tag, basket="VSA_KEK_2019_LV95.KEK"):
+def findall_in_xml_kek_2019(root, tag, basket="VSA_KEK_2019_LV95.KEK"):
+    ns = {"ili": "http://www.interlis.ch/INTERLIS2.3"}
+    return root.findall(f"ili:DATASECTION/ili:{basket}/ili:{tag}", ns)
+
+
+def findall_in_xml_sia_abwasser_2015(root, tag, basket="SIA405_ABWASSER_2015_LV95.SIA405_Abwasser"):
     ns = {"ili": "http://www.interlis.ch/INTERLIS2.3"}
     return root.findall(f"ili:DATASECTION/ili:{basket}/ili:{tag}", ns)
 
@@ -86,15 +91,16 @@ def test_case_b_export_complete_qgep_to_xtf(self):
         path = os.path.join(tempfile.mkdtemp(), "export.xtf")
         main(["qgep", "export", path, "--recreate_schema"])
 
-
     def test_case_d_import_complete_xtf_to_qgep(self):
         """
         # D. import a whole valid interlis transfer file into QGEP
         """
 
         # Incomming XTF case_c_import_all_without_errors.xtf
         # THIS INPUT FILE IS VALID !
-        path = os.path.join(os.path.dirname(__file__), "..", "data", "test_data", "case_d_import_all_without_errors.xtf")
+        path = os.path.join(
+            os.path.dirname(__file__), "..", "data", "test_data", "case_d_import_all_without_errors.xtf"
+        )
 
         # Prepare subset db (we import in an empty schema)
         main(["setupdb", "empty"])
@@ -116,7 +122,7 @@ def test_case_d_import_complete_xtf_to_qgep(self):
         # checking some properties  # TODO : add some more...
         self.assertEqual(session.query(QGEP.manhole).get("ch080qwzNS000113").year_of_construction, 1950)
         session.close()
-        
+
     def test_case_e_export_selection(self):
         """
         # E. export a selection
@@ -125,7 +131,52 @@ def test_case_e_export_selection(self):
         # Prepare db
         main(["setupdb", "full"])
 
-        path = os.path.join(tempfile.mkdtemp(), "export.xtf")
+        # 1. Export VSA_KEK_2019_LV95
+        path = os.path.join(tempfile.mkdtemp(), "export_VSA_KEK_2019_LV95.xtf")
+        selection = [
+            # reach_id
+            "ch13p7mzRE001221",
+            # node_a_id
+            "ch13p7mzWN003445",
+            # node_b_id
+            "ch13p7mzWN008122",
+        ]
+        labels_file = os.path.join(os.path.dirname(__file__), "data", "labels.geojson")
+        main(
+            [
+                "qgep",
+                "export",
+                path,
+                "--recreate_schema",
+                "--selection",
+                ",".join(selection),
+                "--labels_file",
+                labels_file,
+            ]
+        )
+
+        # Perform various checks
+        logger.warning("Perform various checks VSA_KEK_2019_LV95 ...")
+
+        root = ET.parse(path)
+
+        self.assertEqual(len(findall_in_xml_kek_2019(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Kanal")), 1)
+        self.assertEqual(
+            len(findall_in_xml_kek_2019(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Normschacht")), 2
+        )
+        self.assertEqual(
+            len(findall_in_xml_kek_2019(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Haltung_Text")), 3
+        )
+        self.assertEqual(
+            len(findall_in_xml_kek_2019(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Abwasserbauwerk_Text")), 6
+        )
+
+    def test_case_f_export_selection_sia405(self):
+        """
+        # F. export a selection
+        """
+
+        path = os.path.join(tempfile.mkdtemp(), "export_VSA_KEK_2019_LV95.xtf")
         selection = [
             # reach_id
             "ch13p7mzRE001221",
@@ -140,20 +191,36 @@ def test_case_e_export_selection(self):
                 "qgep",
                 "export",
                 path,
+                "--export_sia405",
                 "--recreate_schema",
                 "--selection",
                 ",".join(selection),
                 "--labels_file",
                 labels_file,
             ]
         )
+
         # Perform various checks
+        logger.warning("Perform various checks SIA405_ABWASSER_2015_LV95 ...")
+
         root = ET.parse(path)
-        self.assertEquals(len(findall_in_xml(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Kanal")), 1)
-        self.assertEquals(len(findall_in_xml(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Normschacht")), 2)
-        self.assertEquals(len(findall_in_xml(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Haltung_Text")), 3)
-        self.assertEquals(
-            len(findall_in_xml(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Abwasserbauwerk_Text")), 6
+
+        self.assertEqual(
+            len(findall_in_xml_sia_abwasser_2015(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Kanal")), 1
+        )
+        self.assertEqual(
+            len(findall_in_xml_sia_abwasser_2015(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Normschacht")), 2
+        )
+        self.assertEqual(
+            len(findall_in_xml_sia_abwasser_2015(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Haltung_Text")), 3
+        )
+        self.assertEqual(
+            len(
+                findall_in_xml_sia_abwasser_2015(
+                    root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Abwasserbauwerk_Text"
+                )
+            ),
+            6,
         )
 
 

qgepqwat2ili/utils/ili2db.py

@@ -97,8 +97,15 @@ def import_xtf_data(schema, xtf_file, log_path):
     )
 
 
-def export_xtf_data(schema, model_name, xtf_file, log_path):
+def export_xtf_data(schema, model_name, export_model_name, xtf_file, log_path):
     logger.info("EXPORT ILIDB...")
+
+    # if optional export_model_name is set, add it to the args
+    if export_model_name:
+        export_model_name_args = ["--exportModels", export_model_name]
+    else:
+        export_model_name_args = []
+
     exec_(
         " ".join(
             [
@@ -108,6 +115,7 @@ def export_xtf_data(schema, model_name, xtf_file, log_path):
                 "--export",
                 "--models",
                 f"{model_name}",
+                *export_model_name_args,
                 *get_pgconf_as_ili_args(),
                 "--dbschema",
                 f"{schema}",