OSM importer speedup

aequilibrae/parameters.yml

@@ -42,30 +42,6 @@ network:
           description: name
           osm_source: name
           type: text
-      - cycleway:
-          description: cycleway, both way
-          osm_source: cycleway
-          type: text
-      - cycleway_right:
-          description: cycleway, right
-          osm_source: cycleway:right
-          type: text
-      - cycleway_left:
-          description: cycleway, left
-          osm_source: cycleway:left
-          type: text
-      - busway:
-          description: busway
-          osm_source: busway
-          type: text
-      - busway_right:
-          description: busway, right
-          osm_source: busway:right
-          type: text
-      - busway_left:
-          description: busway, left
-          osm_source: busway:left
-          type: text            
       two-way:
       - lanes:
           description: lanes

aequilibrae/paths/graph.py

@@ -237,9 +237,9 @@ def _build_directed_graph(self, network: pd.DataFrame, centroids: np.ndarray):
         if nans:
             self.logger.warning(f"Field(s) {nans} has(ve) at least one NaN value. Check your computations")
 
-        df.loc[:, "b_node"] = df.b_node.values.astype(self.__integer_type)
-        df.loc[:, "id"] = df.id.values.astype(self.__integer_type)
-        df.loc[:, "link_id"] = df.link_id.values.astype(self.__integer_type)
+        df["link_id"] = df["link_id"].astype(self.__integer_type)
+        df["b_node"] = df.b_node.values.astype(self.__integer_type)
+        df["id"] = df.id.values.astype(self.__integer_type)
         df["direction"] = df.direction.values.astype(np.int8)
 
         return all_nodes, num_nodes, nodes_to_indices, fs, df

aequilibrae/project/network/link_types.py

@@ -1,6 +1,7 @@
-from sqlite3 import IntegrityError, Connection
-from aequilibrae.project.network.link_type import LinkType
+from sqlite3 import IntegrityError
+
 from aequilibrae.project.field_editor import FieldEditor
+from aequilibrae.project.network.link_type import LinkType
 from aequilibrae.project.table_loader import TableLoader
 from aequilibrae.utils.db_utils import commit_and_close
 from aequilibrae.utils.spatialite_utils import connect_spatialite
@@ -84,7 +85,6 @@ def new(self, link_type_id: str) -> LinkType:
         tp["link_type_id"] = link_type_id
         lt = LinkType(tp, self.project)
         self.__items[link_type_id] = lt
-        self.logger.warning("Link type has not yet been saved to the database. Do so explicitly")
         return lt
 
     def delete(self, link_type_id: str) -> None:

aequilibrae/project/network/network.py

@@ -126,6 +126,7 @@ def create_from_osm(
         model_area: Optional[Polygon] = None,
         place_name: Optional[str] = None,
         modes=("car", "transit", "bicycle", "walk"),
+        clean=True,
     ) -> None:
         """
         Downloads the network from Open-Street Maps
@@ -141,6 +142,9 @@ def create_from_osm(
             **modes** (:obj:`tuple`, Optional): List of all modes to be downloaded. Defaults to the modes in the parameter
             file
 
+            **clean** (:obj:`bool`, Optional): Keeps only the links that intersects the model area polygon. Defaults to
+            True. Does not apply to networks downloaded with a place name
+
         .. code-block:: python
 
             >>> from aequilibrae import Project
@@ -191,6 +195,7 @@ def create_from_osm(
                 raise ValueError("Coordinates out of bounds. Polygon must be in WGS84")
             west, south, east, north = model_area.bounds
         else:
+            clean = False
             bbox, report = placegetter(place_name)
             if bbox is None:
                 msg = f'We could not find a reference for place name "{place_name}"'
@@ -229,14 +234,14 @@ def create_from_osm(
                     if subarea.intersects(model_area):
                         polygons.append(subarea)
         self.logger.info("Downloading data")
-        self.downloader = OSMDownloader(polygons, modes, logger=self.logger)
+        dwnloader = OSMDownloader(polygons, modes, logger=self.logger)
         if pyqt:
-            self.downloader.downloading.connect(self.signal_handler)
+            dwnloader.downloading.connect(self.signal_handler)
 
-        self.downloader.doWork()
+        dwnloader.doWork()
 
         self.logger.info("Building Network")
-        self.builder = OSMBuilder(self.downloader.json, project=self.project, model_area=model_area)
+        self.builder = OSMBuilder(dwnloader.data, project=self.project, model_area=model_area, clean=clean)
 
         if pyqt:
             self.builder.building.connect(self.signal_handler)

aequilibrae/project/network/osm/model_area_gridding.py

@@ -0,0 +1,26 @@
+# Inspired by https://www.matecdev.com/posts/shapely-polygon-gridding.html
+from math import ceil
+
+import geopandas as gpd
+import numpy as np
+from shapely.geometry import Polygon
+
+
+def geometry_grid(model_area, srid) -> gpd.GeoDataFrame:
+    minx, miny, maxx, maxy = model_area.bounds
+    # Some rough heuristic to get the number of points per sub-polygon in the 2 digits range
+    subd = ceil((len(model_area.exterior.coords) / 32) ** 0.5)
+    dx = (maxx - minx) / subd
+    dy = (maxy - miny) / subd
+    elements = []
+    x1 = minx
+    for i in range(subd):
+        j1 = miny
+        for j in range(subd):
+            elements.append(Polygon([[x1, j1], [x1, j1 + dy], [x1 + dx, j1 + dy], [x1 + dx, j1]]))
+            j1 += dy
+        x1 += dx
+
+    gdf = gpd.GeoDataFrame({"id": np.arange(len(elements))}, geometry=elements, crs=srid)
+
+    return gdf.clip(model_area)