From 472be96b4e0c91c7ba5ef015fa456adde0b66337 Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Wed, 30 Oct 2024 14:21:42 -0700 Subject: [PATCH 01/17] Added TAZ-level active skims to travel time reporter --- src/main/python/TravelTimeReporter.py | 46 +++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/main/python/TravelTimeReporter.py b/src/main/python/TravelTimeReporter.py index efaebb78b..f9db00c5c 100644 --- a/src/main/python/TravelTimeReporter.py +++ b/src/main/python/TravelTimeReporter.py @@ -102,6 +102,29 @@ def read_skims(self, mode = "transit"): skims.close() + # Read bike and walk times from AM traffic skim + am_traffic_skim_file = os.path.join( + self.model_run, + "output", + "skims", + "traffic_skims_AM.omx" + ) + skims = omx.open_file(am_traffic_skim_file, "r") + for core in ["BIKE_TIME", "walkTime"]: + skim_values = np.array(skims[core.format(self.settings["time_period"])]) + skim_values = np.where( + skim_values == 0, + self.ssettings["infinity"], + skim_values + ) + self.skims[core] = self.expand_skim( + pd.DataFrame( + skim_values, + zones, + zones + ) + ) + def read_active_skims(self): """ Reads active skims into memory as data frames @@ -411,6 +434,29 @@ def get_drive_alone_time(self): dest_terminal_time = self.field2matrix("terminal_time", origin = False) self.skims["drive_alone_time"] = orig_terminal_time + self.expand_skim(self.skims["SOV_NT_L_TIME__" + self.settings["time_period"]]) + dest_terminal_time + def add_active_taz_time(self, bike = True): + """ + Adds the skim values for OD pairs not in the MAZ skim files and reads in the TAZ-level skim values. + + Parameters + ---------- + bike (bool): + If set to `True`, obtain the bike skim values. Otherwise obtain the walk skim values. + """ + if bike: + mode = "bike" + skim = "BIKE_TIME" + else: + mode = "walk" + skim = "walkTIME" + + self.skims["taz_time"] = self.unpivot_skim(skim) + self.skims[mode + "_time"] = np.where( + self.skims[mode + "_time"] == self.settings["infinity"], + self.skims["taz_time"], + self.skims[mode + "_time"] + ) + # # # # # # # # # # # # OUTPUT FUNCTIONS # # # # # # # # # # # # #==============================================================# def coalesce_results(self): From b7b713195d776ec08cdec81b003fa2b2deb5e8ba Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Wed, 30 Oct 2024 15:09:21 -0700 Subject: [PATCH 02/17] Corrected spelling of settings --- src/main/python/TravelTimeReporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/python/TravelTimeReporter.py b/src/main/python/TravelTimeReporter.py index f9db00c5c..4628899e3 100644 --- a/src/main/python/TravelTimeReporter.py +++ b/src/main/python/TravelTimeReporter.py @@ -114,7 +114,7 @@ def read_skims(self, mode = "transit"): skim_values = np.array(skims[core.format(self.settings["time_period"])]) skim_values = np.where( skim_values == 0, - self.ssettings["infinity"], + self.settings["infinity"], skim_values ) self.skims[core] = self.expand_skim( From db7ca6114127a9c6036c26f6d5f27a625eb9159f Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Thu, 31 Oct 2024 13:36:14 -0700 Subject: [PATCH 03/17] Separated maz and taz skims and added call of function to add taz skims (which was made to edit the correct data frame) --- .../python/AMTravelTimeReporterConfigs.yaml | 4 +-- .../python/MDTravelTimeReporterConfigs.yaml | 4 +-- src/main/python/TravelTimeReporter.py | 29 ++++++++++++------- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/main/python/AMTravelTimeReporterConfigs.yaml b/src/main/python/AMTravelTimeReporterConfigs.yaml index f4bc1dc75..6eea1377f 100644 --- a/src/main/python/AMTravelTimeReporterConfigs.yaml +++ b/src/main/python/AMTravelTimeReporterConfigs.yaml @@ -20,5 +20,5 @@ traffic_skim_matrices: # If set to True, replace values of zero with the number SOV_NT_L_TIME__{}: False active_skim_files: - walk_time: maz_maz_walk.csv - bike_time: maz_maz_bike.csv \ No newline at end of file + maz_walk_time: maz_maz_walk.csv + maz_bike_time: maz_maz_bike.csv \ No newline at end of file diff --git a/src/main/python/MDTravelTimeReporterConfigs.yaml b/src/main/python/MDTravelTimeReporterConfigs.yaml index 981bde183..6d002598e 100644 --- a/src/main/python/MDTravelTimeReporterConfigs.yaml +++ b/src/main/python/MDTravelTimeReporterConfigs.yaml @@ -20,5 +20,5 @@ traffic_skim_matrices: # If set to True, replace values of zero with the number SOV_NT_L_TIME__{}: False active_skim_files: - walk_time: maz_maz_walk.csv - bike_time: maz_maz_bike.csv \ No newline at end of file + maz_walk_time: maz_maz_walk.csv + maz_bike_time: maz_maz_bike.csv \ No newline at end of file diff --git a/src/main/python/TravelTimeReporter.py b/src/main/python/TravelTimeReporter.py index 4628899e3..1d7308b54 100644 --- a/src/main/python/TravelTimeReporter.py +++ b/src/main/python/TravelTimeReporter.py @@ -110,14 +110,18 @@ def read_skims(self, mode = "transit"): "traffic_skims_AM.omx" ) skims = omx.open_file(am_traffic_skim_file, "r") - for core in ["BIKE_TIME", "walkTime"]: + active_taz_skims = { + "BIKE_TIME": "taz_bike_time", + "walkTime": "taz_walk_time" + } + for core in active_taz_skims: skim_values = np.array(skims[core.format(self.settings["time_period"])]) skim_values = np.where( skim_values == 0, self.settings["infinity"], skim_values ) - self.skims[core] = self.expand_skim( + self.skims[active_taz_skims[core]] = self.expand_skim( pd.DataFrame( skim_values, zones, @@ -445,16 +449,16 @@ def add_active_taz_time(self, bike = True): """ if bike: mode = "bike" - skim = "BIKE_TIME" + skim = "taz_bike_time" else: mode = "walk" - skim = "walkTIME" + skim = "taz_walk_time" - self.skims["taz_time"] = self.unpivot_skim(skim) - self.skims[mode + "_time"] = np.where( - self.skims[mode + "_time"] == self.settings["infinity"], - self.skims["taz_time"], - self.skims[mode + "_time"] + self.results["taz_time"] = self.unpivot_skim(skim) + self.results[mode] = np.where( + self.results[mode] == self.settings["infinity"], + self.results["taz_time"], + self.results[mode] ) # # # # # # # # # # # # OUTPUT FUNCTIONS # # # # # # # # # # # # @@ -470,8 +474,8 @@ def coalesce_results(self): self.results = pd.DataFrame( { "transit": self.unpivot_skim("total_transit_time"), - "walk": self.skims["walk_time"].query("walkTime <= @time_threshold")["walkTime"], - "bike": self.skims["bike_time"].query("BIKE_TIME <= @time_threshold")["BIKE_TIME"], + "walk": self.skims["maz_walk_time"].query("walkTime <= @time_threshold")["walkTime"], + "bike": self.skims["maz_bike_time"].query("BIKE_TIME <= @time_threshold")["BIKE_TIME"], "microtransit": self.unpivot_skim("microtransit_time"), "nev": self.unpivot_skim("nev_time"), # "drive_alone": self.unpivot_skim("drive_alone_time") @@ -480,6 +484,9 @@ def coalesce_results(self): ["i", "j"] ) + self.add_active_taz_time(bike = False) + self.add_active_taz_time(bike = True) + _ebikeMaxTime = self.constants["ebikeMaxDist"] / self.constants["ebikeSpeed"] * 60 _escooterMaxTime = self.constants["escooterMaxDist"] / self.constants["escooterSpeed"] * 60 From a7f5ed4339494a5f29696bd1f7dd6515e880b2bd Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Fri, 1 Nov 2024 08:45:51 -0700 Subject: [PATCH 04/17] Removed microtransit access to transit from travel time reporter --- src/main/python/TravelTimeReporter.py | 56 ++++++++++----------------- 1 file changed, 21 insertions(+), 35 deletions(-) diff --git a/src/main/python/TravelTimeReporter.py b/src/main/python/TravelTimeReporter.py index 1d7308b54..d58860a8c 100644 --- a/src/main/python/TravelTimeReporter.py +++ b/src/main/python/TravelTimeReporter.py @@ -301,47 +301,31 @@ def get_accegr_times(self): """ Returns transit access and egress times from every MGRA to every other MGRA """ - print("Calculating direct flexible fleet access and egress times") - microtransit_direct_access_time = self.get_microtransit_direct_accegr_time(access = True, nev = False) - microtransit_direct_egress_time = self.get_microtransit_direct_accegr_time(access = False, nev = False) - nev_direct_access_time = self.get_microtransit_direct_accegr_time(access = True, nev = True) - nev_direct_egress_time = self.get_microtransit_direct_accegr_time(access = False, nev = True) - - print("Calculating flexible fleet diverted access and egress times") - microtransit_access_time = self.get_total_microtransit_accegr_times(microtransit_direct_access_time, nev = False) - microtransit_egress_time = self.get_total_microtransit_accegr_times(microtransit_direct_egress_time, nev = False) - nev_access_time = self.get_total_microtransit_accegr_times(nev_direct_access_time, nev = True) - nev_egress_time = self.get_total_microtransit_accegr_times(nev_direct_egress_time, nev = True) - - print("Getting flexible fleet availability") - microtransit_access_available = self.field2matrix("microtransit_access_available", origin = True) - microtransit_egress_available = self.field2matrix("microtransit_egress_available", origin = False) - nev_access_available = self.field2matrix("nev_access_available", origin = True) - nev_egress_available = self.field2matrix("nev_egress_available", origin = False) + # print("Calculating direct flexible fleet access and egress times") + # microtransit_direct_access_time = self.get_microtransit_direct_accegr_time(access = True, nev = False) + # microtransit_direct_egress_time = self.get_microtransit_direct_accegr_time(access = False, nev = False) + # nev_direct_access_time = self.get_microtransit_direct_accegr_time(access = True, nev = True) + # nev_direct_egress_time = self.get_microtransit_direct_accegr_time(access = False, nev = True) + + # print("Calculating flexible fleet diverted access and egress times") + # microtransit_access_time = self.get_total_microtransit_accegr_times(microtransit_direct_access_time, nev = False) + # microtransit_egress_time = self.get_total_microtransit_accegr_times(microtransit_direct_egress_time, nev = False) + # nev_access_time = self.get_total_microtransit_accegr_times(nev_direct_access_time, nev = True) + # nev_egress_time = self.get_total_microtransit_accegr_times(nev_direct_egress_time, nev = True) + + # print("Getting flexible fleet availability") + # microtransit_access_available = self.field2matrix("microtransit_access_available", origin = True) + # microtransit_egress_available = self.field2matrix("microtransit_egress_available", origin = False) + # nev_access_available = self.field2matrix("nev_access_available", origin = True) + # nev_egress_available = self.field2matrix("nev_egress_available", origin = False) print("Getting walk access and egress times") walk_access_time = self.field2matrix("walk_accegr_time", origin = True) walk_egress_time = self.field2matrix("walk_accegr_time", origin = False) print("Calculating access and egress times") - self.skims["access_time"] = np.where( - nev_access_available, - nev_access_time, - np.where( - microtransit_access_available, - microtransit_access_time, - walk_access_time - ) - ) - self.skims["egress_time"] = np.where( - nev_egress_available, - nev_egress_time, - np.where( - microtransit_egress_available, - microtransit_egress_time, - walk_egress_time - ) - ) + self.skims["access_time"] = walk_access_time + self.skims["egress_time"] = walk_egress_time def get_transit_time(self): """ @@ -461,6 +445,8 @@ def add_active_taz_time(self, bike = True): self.results[mode] ) + del self.results["taz_time"] + # # # # # # # # # # # # OUTPUT FUNCTIONS # # # # # # # # # # # # #==============================================================# def coalesce_results(self): From 2c77dda0cd4ac9a4aa58a4559e0c8c05d5ec730b Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Fri, 1 Nov 2024 09:24:53 -0700 Subject: [PATCH 05/17] Moved addition of TAZ-level active skims to function reading active skims and changed how it was done --- src/main/python/TravelTimeReporter.py | 91 +++++++++++---------------- 1 file changed, 38 insertions(+), 53 deletions(-) diff --git a/src/main/python/TravelTimeReporter.py b/src/main/python/TravelTimeReporter.py index d58860a8c..7675d868e 100644 --- a/src/main/python/TravelTimeReporter.py +++ b/src/main/python/TravelTimeReporter.py @@ -102,6 +102,31 @@ def read_skims(self, mode = "transit"): skims.close() + def read_active_skims(self): + """ + Reads active skims into memory as data frames + """ + # Read MAZ-level skims + for skim_name in self.settings["active_skim_files"]: + active_skims = pd.read_csv( + os.path.join( + self.model_run, + "output", + "skims", + self.settings["active_skim_files"][skim_name] + ) + ) + if "i" in active_skims.columns: + active_skims = active_skims.drop(["i", "j"],axis=1) + self.skims[skim_name] = active_skims.rename( + columns = { + "OMAZ": "i", + "DMAZ": "j", + } + ).set_index( + ["i", "j"] + ) + # Read bike and walk times from AM traffic skim am_traffic_skim_file = os.path.join( self.model_run, @@ -110,6 +135,7 @@ def read_skims(self, mode = "transit"): "traffic_skims_AM.omx" ) skims = omx.open_file(am_traffic_skim_file, "r") + zones = skims.mapping("zone_number").keys() active_taz_skims = { "BIKE_TIME": "taz_bike_time", "walkTime": "taz_walk_time" @@ -129,29 +155,16 @@ def read_skims(self, mode = "transit"): ) ) - def read_active_skims(self): - """ - Reads active skims into memory as data frames - """ - for skim_name in self.settings["active_skim_files"]: - active_skims = pd.read_csv( - os.path.join( - self.model_run, - "output", - "skims", - self.settings["active_skim_files"][skim_name] - ) - ) - if "i" in active_skims.columns: - active_skims = active_skims.drop(["i", "j"],axis=1) - self.skims[skim_name] = active_skims.rename( - columns = { - "OMAZ": "i", - "DMAZ": "j", - } - ).set_index( - ["i", "j"] - ) + # Replace TAZ-skim level values with MGRA-skim values if they are present + self.unpivot_skim("taz_bike_time") + self.unpivot_skim("taz_walk_time") + self.unpivot_skim("maz_bike_time") + self.unpivot_skim("maz_walk_time") + + self.skims["bike_time"] = self.skims["taz_bike_time"].copy() + self.skims["walk_time"] = self.skims["taz_walk_time"].copy() + self.skims["bike_time"].loc[self.skims["maz_bike_time"].index] = self.skims["maz_bike_time"] + self.skims["walk_time"].loc[self.skims["maz_walk_time"].index] = self.skims["maz_walk_time"] def init_land_use(self): """ @@ -422,31 +435,6 @@ def get_drive_alone_time(self): dest_terminal_time = self.field2matrix("terminal_time", origin = False) self.skims["drive_alone_time"] = orig_terminal_time + self.expand_skim(self.skims["SOV_NT_L_TIME__" + self.settings["time_period"]]) + dest_terminal_time - def add_active_taz_time(self, bike = True): - """ - Adds the skim values for OD pairs not in the MAZ skim files and reads in the TAZ-level skim values. - - Parameters - ---------- - bike (bool): - If set to `True`, obtain the bike skim values. Otherwise obtain the walk skim values. - """ - if bike: - mode = "bike" - skim = "taz_bike_time" - else: - mode = "walk" - skim = "taz_walk_time" - - self.results["taz_time"] = self.unpivot_skim(skim) - self.results[mode] = np.where( - self.results[mode] == self.settings["infinity"], - self.results["taz_time"], - self.results[mode] - ) - - del self.results["taz_time"] - # # # # # # # # # # # # OUTPUT FUNCTIONS # # # # # # # # # # # # #==============================================================# def coalesce_results(self): @@ -460,8 +448,8 @@ def coalesce_results(self): self.results = pd.DataFrame( { "transit": self.unpivot_skim("total_transit_time"), - "walk": self.skims["maz_walk_time"].query("walkTime <= @time_threshold")["walkTime"], - "bike": self.skims["maz_bike_time"].query("BIKE_TIME <= @time_threshold")["BIKE_TIME"], + "walk": self.skims["walk_time"], + "bike": self.skims["bike_time"], "microtransit": self.unpivot_skim("microtransit_time"), "nev": self.unpivot_skim("nev_time"), # "drive_alone": self.unpivot_skim("drive_alone_time") @@ -470,9 +458,6 @@ def coalesce_results(self): ["i", "j"] ) - self.add_active_taz_time(bike = False) - self.add_active_taz_time(bike = True) - _ebikeMaxTime = self.constants["ebikeMaxDist"] / self.constants["ebikeSpeed"] * 60 _escooterMaxTime = self.constants["escooterMaxDist"] / self.constants["escooterSpeed"] * 60 From 8e8bb31580bbffc7a9b3bcf440ce5ac0f59e7953 Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Fri, 1 Nov 2024 10:13:49 -0700 Subject: [PATCH 06/17] Moved query for values under time threshold out of unpivot_skims() and into coelesce_results() --- src/main/python/TravelTimeReporter.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/python/TravelTimeReporter.py b/src/main/python/TravelTimeReporter.py index 7675d868e..8a4100842 100644 --- a/src/main/python/TravelTimeReporter.py +++ b/src/main/python/TravelTimeReporter.py @@ -251,13 +251,11 @@ def unpivot_skim(self, skim_name): id_vars = ["i"], var_name = "j", value_name = "time" - ).query( - "time <= @time_threshold" - ).sort_values( - ["i", "j"] + ).sort_values( + ["i", "j"] ).set_index( ["i", "j"] - )["time"] + )["time"] # # # # # # # # # # # # CALCULATOR FUNCTIONS # # # # # # # # # # # # #==================================================================# @@ -454,9 +452,11 @@ def coalesce_results(self): "nev": self.unpivot_skim("nev_time"), # "drive_alone": self.unpivot_skim("drive_alone_time") } - ).reset_index().fillna(self.settings["infinity"]).sort_values( + ).query( + "time <= @time_threshold" + ).reset_index().fillna(self.settings["infinity"]).sort_values( ["i", "j"] - ) + ) _ebikeMaxTime = self.constants["ebikeMaxDist"] / self.constants["ebikeSpeed"] * 60 _escooterMaxTime = self.constants["escooterMaxDist"] / self.constants["escooterSpeed"] * 60 From a462dd24fd09b2241962a6ceb04380447514fb2e Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Fri, 1 Nov 2024 11:20:03 -0700 Subject: [PATCH 07/17] Made expansion_matrix an attribute of the TravelTimeReporter class so pd.get_dummies() isn't called every time expand_skim() is --- src/main/python/TravelTimeReporter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/python/TravelTimeReporter.py b/src/main/python/TravelTimeReporter.py index 8a4100842..acdd431a4 100644 --- a/src/main/python/TravelTimeReporter.py +++ b/src/main/python/TravelTimeReporter.py @@ -179,6 +179,8 @@ def init_land_use(self): self.land_use["nev_egress_available"] = (self.land_use["nev"] > 0) & (self.land_use["micro_accegr_dist"] <= self.constants["nevMaxDist"]) & (self.land_use["micro_accegr_dist"] >= self.constants["maxWalkIfMTAccessAvailable"]) self.land_use["nev_accegr_time"] = 60 * self.land_use["micro_accegr_dist"] / self.constants["nevSpeed"] + self.expansion_matrix = pd.get_dummies(self.land_use["TAZ"]) # Indicates which MGRAs belong to which TAZ + # # # # # # # # # # # # UTILITY FUNCTIONS # # # # # # # # # # # # #===============================================================# def field2matrix(self, field, origin = True): @@ -222,9 +224,8 @@ def expand_skim(self, skim): expanded_skim (pandas.DataFrame): Skim matrix where the index and columns are the origins and destinations MGRAs, respectively, and the values are the impedance from each origin MGRA to the destination MGRA """ - expansion_matrix = pd.get_dummies(self.land_use["TAZ"]) # Indicates which MGRAs belong to which TAZ return pd.DataFrame( - expansion_matrix.values.dot(skim.values).dot(expansion_matrix.T.values), + self.expansion_matrix.values.dot(skim.values).dot(self.expansion_matrix.T.values), self.land_use.index, self.land_use.index ) From 305050e0e20c04c9d5101be54ce3181d84b50cd5 Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Fri, 1 Nov 2024 11:21:42 -0700 Subject: [PATCH 08/17] Removed unpivoting of taz skims as they are already in that format --- src/main/python/TravelTimeReporter.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/python/TravelTimeReporter.py b/src/main/python/TravelTimeReporter.py index acdd431a4..6b54f3402 100644 --- a/src/main/python/TravelTimeReporter.py +++ b/src/main/python/TravelTimeReporter.py @@ -158,8 +158,6 @@ def read_active_skims(self): # Replace TAZ-skim level values with MGRA-skim values if they are present self.unpivot_skim("taz_bike_time") self.unpivot_skim("taz_walk_time") - self.unpivot_skim("maz_bike_time") - self.unpivot_skim("maz_walk_time") self.skims["bike_time"] = self.skims["taz_bike_time"].copy() self.skims["walk_time"] = self.skims["taz_walk_time"].copy() From 95bbefb6286dbb32d3c9ee01956d19ffa8f3f87e Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Fri, 1 Nov 2024 14:04:46 -0700 Subject: [PATCH 09/17] Moved time under time_threshold query back to unpivot_skims --- src/main/python/TravelTimeReporter.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/python/TravelTimeReporter.py b/src/main/python/TravelTimeReporter.py index 6b54f3402..09b48dcb7 100644 --- a/src/main/python/TravelTimeReporter.py +++ b/src/main/python/TravelTimeReporter.py @@ -250,11 +250,13 @@ def unpivot_skim(self, skim_name): id_vars = ["i"], var_name = "j", value_name = "time" - ).sort_values( - ["i", "j"] - ).set_index( + ).query( + "time <= @time_threshold" + ).sort_values( ["i", "j"] - )["time"] + ).set_index( + ["i", "j"] + )["time"] # # # # # # # # # # # # CALCULATOR FUNCTIONS # # # # # # # # # # # # #==================================================================# @@ -451,9 +453,7 @@ def coalesce_results(self): "nev": self.unpivot_skim("nev_time"), # "drive_alone": self.unpivot_skim("drive_alone_time") } - ).query( - "time <= @time_threshold" - ).reset_index().fillna(self.settings["infinity"]).sort_values( + ).reset_index().fillna(self.settings["infinity"]).sort_values( ["i", "j"] ) From f2405f86f3cd94bd534d7689f8298074106c87ab Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Fri, 1 Nov 2024 16:56:45 -0700 Subject: [PATCH 10/17] unpivot_skim returns unpivoted skims; it is not done in place --- src/main/python/TravelTimeReporter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/python/TravelTimeReporter.py b/src/main/python/TravelTimeReporter.py index 09b48dcb7..02a5717ee 100644 --- a/src/main/python/TravelTimeReporter.py +++ b/src/main/python/TravelTimeReporter.py @@ -156,8 +156,8 @@ def read_active_skims(self): ) # Replace TAZ-skim level values with MGRA-skim values if they are present - self.unpivot_skim("taz_bike_time") - self.unpivot_skim("taz_walk_time") + self.skims["taz_bike_time"] = self.unpivot_skim("taz_bike_time") + self.skims["taz_walk_time"] = self.unpivot_skim("taz_walk_time") self.skims["bike_time"] = self.skims["taz_bike_time"].copy() self.skims["walk_time"] = self.skims["taz_walk_time"].copy() From ee62c90f30285c4f11a661d1df212a64ea37cf9d Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Mon, 4 Nov 2024 08:36:06 -0800 Subject: [PATCH 11/17] Changed from assignment of maz times to taz times from vectorized to for loops --- src/main/python/TravelTimeReporter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/python/TravelTimeReporter.py b/src/main/python/TravelTimeReporter.py index 02a5717ee..b5af28344 100644 --- a/src/main/python/TravelTimeReporter.py +++ b/src/main/python/TravelTimeReporter.py @@ -161,8 +161,10 @@ def read_active_skims(self): self.skims["bike_time"] = self.skims["taz_bike_time"].copy() self.skims["walk_time"] = self.skims["taz_walk_time"].copy() - self.skims["bike_time"].loc[self.skims["maz_bike_time"].index] = self.skims["maz_bike_time"] - self.skims["walk_time"].loc[self.skims["maz_walk_time"].index] = self.skims["maz_walk_time"] + for ix, row in self.skims["maz_bike_time"].iterrows(): + self.skims["bike_time"].loc[ix, "time"] = self.skims["maz_bike_time"].loc[ix, "time"] + for ix, row in self.skims["maz_walk_time"].iterrows(): + self.skims["walk_time"].loc[ix, "time"] = self.skims["maz_walk_time"].loc[ix, "time"] def init_land_use(self): """ From 0851bf5b079faccf9129d463ff13f96ea93f0fbe Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Mon, 4 Nov 2024 09:20:51 -0800 Subject: [PATCH 12/17] self.skims['bike_time'] and self.skims['walk_time'] are Series, not DataFrames --- src/main/python/TravelTimeReporter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/python/TravelTimeReporter.py b/src/main/python/TravelTimeReporter.py index b5af28344..308d71dd3 100644 --- a/src/main/python/TravelTimeReporter.py +++ b/src/main/python/TravelTimeReporter.py @@ -162,9 +162,9 @@ def read_active_skims(self): self.skims["bike_time"] = self.skims["taz_bike_time"].copy() self.skims["walk_time"] = self.skims["taz_walk_time"].copy() for ix, row in self.skims["maz_bike_time"].iterrows(): - self.skims["bike_time"].loc[ix, "time"] = self.skims["maz_bike_time"].loc[ix, "time"] + self.skims["bike_time"].loc[ix] = self.skims["maz_bike_time"].loc[ix, "time"] for ix, row in self.skims["maz_walk_time"].iterrows(): - self.skims["walk_time"].loc[ix, "time"] = self.skims["maz_walk_time"].loc[ix, "time"] + self.skims["walk_time"].loc[ix] = self.skims["maz_walk_time"].loc[ix, "time"] def init_land_use(self): """ From 223a9f93268cd9c67cf07e0614c3965f470d4cf9 Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Mon, 4 Nov 2024 10:07:25 -0800 Subject: [PATCH 13/17] Updated with correct column names for MAZ-level active skims --- src/main/python/TravelTimeReporter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/python/TravelTimeReporter.py b/src/main/python/TravelTimeReporter.py index 308d71dd3..88c5d6b5c 100644 --- a/src/main/python/TravelTimeReporter.py +++ b/src/main/python/TravelTimeReporter.py @@ -162,9 +162,9 @@ def read_active_skims(self): self.skims["bike_time"] = self.skims["taz_bike_time"].copy() self.skims["walk_time"] = self.skims["taz_walk_time"].copy() for ix, row in self.skims["maz_bike_time"].iterrows(): - self.skims["bike_time"].loc[ix] = self.skims["maz_bike_time"].loc[ix, "time"] + self.skims["bike_time"].loc[ix] = self.skims["maz_bike_time"].loc[ix, "BIKE_TIME"] for ix, row in self.skims["maz_walk_time"].iterrows(): - self.skims["walk_time"].loc[ix] = self.skims["maz_walk_time"].loc[ix, "time"] + self.skims["walk_time"].loc[ix] = self.skims["maz_walk_time"].loc[ix, "walkTime"] def init_land_use(self): """ From 545e84cf4788aa6342be1534fcaa4a2877a348bf Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Mon, 4 Nov 2024 11:49:39 -0800 Subject: [PATCH 14/17] Replaced for loop in read_active_skims() with vectorized assignment and updated unpivot_skim() to give option to not filter for time to prevent KeyError --- src/main/python/TravelTimeReporter.py | 50 +++++++++++++++++++-------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/src/main/python/TravelTimeReporter.py b/src/main/python/TravelTimeReporter.py index 88c5d6b5c..d0261fb1d 100644 --- a/src/main/python/TravelTimeReporter.py +++ b/src/main/python/TravelTimeReporter.py @@ -156,15 +156,22 @@ def read_active_skims(self): ) # Replace TAZ-skim level values with MGRA-skim values if they are present - self.skims["taz_bike_time"] = self.unpivot_skim("taz_bike_time") - self.skims["taz_walk_time"] = self.unpivot_skim("taz_walk_time") + self.skims["taz_bike_time"] = self.unpivot_skim("taz_bike_time", False) + self.skims["taz_walk_time"] = self.unpivot_skim("taz_walk_time", False) self.skims["bike_time"] = self.skims["taz_bike_time"].copy() self.skims["walk_time"] = self.skims["taz_walk_time"].copy() - for ix, row in self.skims["maz_bike_time"].iterrows(): - self.skims["bike_time"].loc[ix] = self.skims["maz_bike_time"].loc[ix, "BIKE_TIME"] - for ix, row in self.skims["maz_walk_time"].iterrows(): - self.skims["walk_time"].loc[ix] = self.skims["maz_walk_time"].loc[ix, "walkTime"] + self.skims["bike_time"].loc[self.skims["maz_bike_time"].index] = self.skims["maz_bike_time"]["BIKE_TIME"] + self.skims["walk_time"].loc[self.skims["maz_walk_time"].index] = self.skims["maz_walk_time"]["walkTime"] + + # Remove OD-pairs above time threshold + time_threshold = self.settings["time_threshold"] + self.skims["bike_time"] = self.skims["bike_time"].loc[ + self.skims["bike_time"] <= time_threshold + ] + self.skims["walk_time"] = self.skims["walk_time"].loc[ + self.skims["walk_time"] <= time_threshold + ] def init_land_use(self): """ @@ -230,7 +237,7 @@ def expand_skim(self, skim): self.land_use.index ) - def unpivot_skim(self, skim_name): + def unpivot_skim(self, skim_name, filter_for_time = True): """ Unpivots a skim into a series with the origin and destination as the index @@ -238,6 +245,8 @@ def unpivot_skim(self, skim_name): ---------- skim_name (str): Name of skim to unpivot + filter_for_time (bool): + If true, values will above the time threshold will be removed Returns ------- @@ -247,13 +256,26 @@ def unpivot_skim(self, skim_name): time_threshold = self.settings["time_threshold"] self.skims[skim_name].index.name = "i" - return pd.melt( - self.skims[skim_name].reset_index(), - id_vars = ["i"], - var_name = "j", - value_name = "time" - ).query( - "time <= @time_threshold" + if filter_for_time: + return pd.melt( + self.skims[skim_name].reset_index(), + id_vars = ["i"], + var_name = "j", + value_name = "time" + ).query( + "time <= @time_threshold" + ).sort_values( + ["i", "j"] + ).set_index( + ["i", "j"] + )["time"] + + else: + return pd.melt( + self.skims[skim_name].reset_index(), + id_vars = ["i"], + var_name = "j", + value_name = "time" ).sort_values( ["i", "j"] ).set_index( From 4869a4538f57c691504c20b77759deee14587013 Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Mon, 4 Nov 2024 15:22:14 -0800 Subject: [PATCH 15/17] Lowered threshold in travel time reporter to 30 minutes --- src/main/python/AMTravelTimeReporterConfigs.yaml | 4 ++-- src/main/python/MDTravelTimeReporterConfigs.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/python/AMTravelTimeReporterConfigs.yaml b/src/main/python/AMTravelTimeReporterConfigs.yaml index 6eea1377f..9ce62d0c2 100644 --- a/src/main/python/AMTravelTimeReporterConfigs.yaml +++ b/src/main/python/AMTravelTimeReporterConfigs.yaml @@ -1,7 +1,7 @@ time_period: AM -time_threshold: 45 +time_threshold: 30 infinity: 999 -outfile: report\walkMgrasWithin45Min_AM.csv +outfile: report\walkMgrasWithin30Min_AM.csv transit_skim_matrices: # If set to True, replace values of zero with the number set as infinity WALK_LOC_XFERWALK__{}: False diff --git a/src/main/python/MDTravelTimeReporterConfigs.yaml b/src/main/python/MDTravelTimeReporterConfigs.yaml index 6d002598e..3cefbe986 100644 --- a/src/main/python/MDTravelTimeReporterConfigs.yaml +++ b/src/main/python/MDTravelTimeReporterConfigs.yaml @@ -1,7 +1,7 @@ time_period: MD -time_threshold: 45 +time_threshold: 30 infinity: 999 -outfile: report\walkMgrasWithin45Min_MD.csv +outfile: report\walkMgrasWithin30Min_MD.csv transit_skim_matrices: # If set to True, replace values of zero with the number set as infinity WALK_LOC_XFERWALK__{}: False From a046f1974909d15f94b5750efe6ae6e714f02a28 Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Mon, 4 Nov 2024 15:24:24 -0800 Subject: [PATCH 16/17] Updated filenames in datalake exporter to reflect change in travel time reporter outputs --- src/main/python/datalake_exporter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/python/datalake_exporter.py b/src/main/python/datalake_exporter.py index 851b7fc14..436faf256 100644 --- a/src/main/python/datalake_exporter.py +++ b/src/main/python/datalake_exporter.py @@ -235,8 +235,8 @@ def write_to_datalake(output_path, models, exclude, env): os.path.abspath(os.path.join(output_path, '..', 'input', 'zone_term.csv')), os.path.abspath(os.path.join(output_path, '..', 'input', 'trlink.csv')), os.path.join(output_path, 'bikeMgraLogsum.csv'), - os.path.join(report_path, 'walkMgrasWithin45Min_AM.csv'), - os.path.join(report_path, 'walkMgrasWithin45Min_MD.csv') + os.path.join(report_path, 'walkMgrasWithin30Min_AM.csv'), + os.path.join(report_path, 'walkMgrasWithin30Min_MD.csv') ] for file in other_files: try: From 6e8e4663fc10a0ca6bf85d1b363b97be8c1422db Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Tue, 5 Nov 2024 10:07:32 -0800 Subject: [PATCH 17/17] Removed str.format() call during read_active_skims() as it's not necessary --- src/main/python/TravelTimeReporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/python/TravelTimeReporter.py b/src/main/python/TravelTimeReporter.py index d0261fb1d..9760c32f4 100644 --- a/src/main/python/TravelTimeReporter.py +++ b/src/main/python/TravelTimeReporter.py @@ -141,7 +141,7 @@ def read_active_skims(self): "walkTime": "taz_walk_time" } for core in active_taz_skims: - skim_values = np.array(skims[core.format(self.settings["time_period"])]) + skim_values = np.array(skims[core]) skim_values = np.where( skim_values == 0, self.settings["infinity"],