update Lasso based on the latest CUBE logfile

examples/settings/log_to_net.csv

@@ -1,5 +1,5 @@
 log,net
-LINK_ID,model_link_id
+MODEL_LINK_ID,model_link_id
 SHSTGEOMETRYID,shstGeometryId
 DISTANCE,distance
 ROADWAY,roadway

lasso/parameters.py

@@ -675,4 +675,15 @@ def __init__(self, **kwargs):
             "Y"
         ]
 
+        self.string_col = [
+            "osm_node_id",
+            "name",
+            "roadway",
+            "shstGeometryId",
+            "access_AM",
+            "access_MD",
+            "access_PM",
+            "access_NT"
+        ]
+
         self.__dict__.update(kwargs)

lasso/project.py

@@ -280,17 +280,21 @@ def read_logfile(logfilename: str) -> DataFrame:
         nodecol_names = ["OBJECT", "OPERATION", "GROUP"] + NodeLines[0].split(",")[1:]
 
         link_df = DataFrame(
-            data=[re.split(",|;", x) for x in LinkLines[1:]], columns=linkcol_names
+            data=[re.split(",|;", x.replace("\"", "")) for x in LinkLines[1:]], columns=linkcol_names
         )
 
         node_df = DataFrame(
-            data=[re.split(",|;", x) for x in NodeLines[1:]], columns=nodecol_names
+            data=[re.split(",|;", x.replace("\"", "")) for x in NodeLines[1:]], columns=nodecol_names
         )
 
         log_df = pd.concat([link_df, node_df], ignore_index=True, sort=False)
+
+        # CUBE logfile headers for string fields: NAME[111] instead of NAME, need to shorten that
+        log_df.columns = [c.split("[")[0] for c in log_df.columns]
+
         WranglerLogger.info(
             "Processed {} Node lines and {} Link lines".format(
-                link_df.shape[0], node_df.shape[0]
+                node_df.shape[0], link_df.shape[0]
             )
         )
 
@@ -398,6 +402,10 @@ def add_highway_changes(self, limit_variables_to_existing_network=False):
             limit_variables_to_existing_network (bool): True if no ad-hoc variables.  Default to False.
         """
 
+        for c in self.parameters.string_col:
+            if c in self.roadway_changes.columns:
+                self.roadway_changes[c] = self.roadway_changes[c].str.lstrip(" ")
+
         ## if worth it, could also add some functionality  to network wrangler itself.
         node_changes_df = self.roadway_changes[
             self.roadway_changes.OBJECT == "N"
@@ -557,7 +565,7 @@ def _consolidate_actions(log, base, key_list):
                 out_col = []
                 for col in changeable_col:
                 # if it is the same as before, or a static value, don't process as a change
-                    if (str(change_row[col]) == base_row[col].astype(str)) | (
+                    if (str(change_row[col]) == str(base_row[col])) | (
                         col in Project.STATIC_VALUES
                     ):
                         continue
@@ -567,7 +575,7 @@ def _consolidate_actions(log, base, key_list):
                     if col == "distance":
                         if (
                             abs(
-                                (change_row[col] - base_row[col].astype(float))
+                                (change_row[col] - float(base_row[col]))
                                 / base_row[col].astype(float)
                             )
                                 > 0.01
@@ -657,7 +665,7 @@ def _consolidate_actions(log, base, key_list):
 
             # WranglerLogger.debug('change_link_dict_df 3: {}'.format(change_link_dict_df))
             change_link_dict_df["properties"] = change_link_dict_df["properties"].apply(
-                lambda x: json.loads(x.replace("'", '"'))
+                lambda x: json.loads(x.replace("'\"", "'").replace("\"'", "'").replace("'", '"'))
             )
 
             change_link_dict_list = change_link_dict_df[["facility", "properties"]].to_dict(