Merge pull request #224 from SANDAG/144-remove-walk-logsum-step-in-abm3

144 remove walk logsum step in abm3

src/asim/configs/airport.CBX/preprocessing.yaml

@@ -33,25 +33,6 @@ stop_frequency_expressions_output_formattable_fname: 'stop_frequency_{purpose}.c
 trip_purpose_probs_output_fname: trip_purpose_probs.csv
 trip_scheduling_probs_output_fname: trip_scheduling_probs.csv
 
-# skims
-skims:
-  maz_to_maz:
-    walk:
-      input_fname: microMgraEquivMinutes.csv
-      output_fname: maz_maz_walk.csv
-      rename_columns:
-        i: OMAZ
-        j: DMAZ
-  taz_to_taz:
-    periods:
-      - EA
-      - AM
-      - MD
-      - PM
-      - EV
-    input_base_fname: traffic_skims
-    output_base_fname: traffic_skims_processed
-
 # Tours
 tours:
   num_enplanements: 4186500

src/asim/configs/airport.SAN/preprocessing.yaml

@@ -33,25 +33,6 @@ stop_frequency_expressions_output_formattable_fname: 'stop_frequency_{purpose}.c
 trip_purpose_probs_output_fname: trip_purpose_probs.csv
 trip_scheduling_probs_output_fname: trip_scheduling_probs.csv
 
-# skims
-skims:
-  maz_to_maz:
-    walk:
-      input_fname: microMgraEquivMinutes.csv
-      output_fname: maz_maz_walk.csv
-      rename_columns:
-        i: OMAZ
-        j: DMAZ
-  taz_to_taz:
-    periods:
-      - EA
-      - AM
-      - MD
-      - PM
-      - EV
-    input_base_fname: traffic_skims
-    output_base_fname: traffic_skims_processed
-
 # Tours
 tours:
   num_enplanements: 14536000

src/asim/scripts/resident/2zoneSkim.py

@@ -17,6 +17,34 @@
 with open(yml_path, 'r') as f:
     parms = yaml.safe_load(f)
 
+def add_missing_mazs_to_skim_table(centroids, maz_to_maz_walk_cost_out, maz_to_maz_cost):
+    """
+    Considering that we are using a maximum walk threshold for our skims here, it is possible that some MGRAs, 
+    especially in non-urban areas, might not be within threshold distance of other MGRAs. This means that the skim table 
+    may not have skims for all MGRAs. 
+    
+    This function allows identifying those MGRAs that may be missing from 
+    the skim table, finds the shortest distance 
+    for them (not to themselves), and therefore creating a missig_maz df that now contains  skims for those MGRAs that 
+    otherwise did not have skim at all.
+
+    """
+    # Identify missing MAZs
+    missing_maz = centroids[~centroids['MAZ'].isin(maz_to_maz_walk_cost_out['OMAZ'])][['MAZ']]
+    missing_maz = missing_maz.rename(columns={'MAZ': 'OMAZ'})
+
+    # Filter and sort maz_to_maz_cost DataFrame so we have the shortest distance for each OMAZ first
+    filtered_maz_to_maz_cost = maz_to_maz_cost[maz_to_maz_cost['OMAZ'] != maz_to_maz_cost['DMAZ']]
+    sorted_maz_to_maz_cost = filtered_maz_to_maz_cost.sort_values('DISTWALK')
+
+    # Group by 'OMAZ' and select the first (shortest dist) 'DMAZ' and 'DISTWALK' for each group
+    grouped_maz_to_maz_cost = sorted_maz_to_maz_cost.groupby('OMAZ').agg({'DMAZ': 'first', 'DISTWALK': 'first'}).reset_index()
+
+    # Merge the missing MAZs with the grouped maz_to_maz_cost DataFrame
+    result = missing_maz.merge(grouped_maz_to_maz_cost, on='OMAZ', how='left')
+
+    return result
+
 print(f"{datetime.now().strftime('%H:%M:%S')} Preparing MAZ-MAZ and MAZ-Stop Connectors...")
 startTime = time.time()
 #asim_inputs = os.path.join(path, "ASIM_INPUTS")
@@ -111,20 +139,41 @@
 print(f"{datetime.now().strftime('%H:%M:%S')} Get Shortest Path Length...")
 maz_to_maz_walk_cost["DISTWALK"] = net.shortest_path_lengths(maz_to_maz_walk_cost["OMAZ_NODE"], maz_to_maz_walk_cost["DMAZ_NODE"])
 maz_to_maz_walk_cost_out = maz_to_maz_walk_cost[maz_to_maz_walk_cost["DISTWALK"] <= max_maz_maz_walk_dist_feet / 5280.0]
-missing_maz = pd.DataFrame(centroids[~centroids['MAZ'].isin(maz_to_maz_walk_cost_out['OMAZ'])]['MAZ']).rename(columns = {'MAZ': 'OMAZ'}).merge(maz_to_maz_cost[maz_to_maz_cost['OMAZ'] != maz_to_maz_cost['DMAZ']].sort_values('DISTWALK').groupby('OMAZ').agg({'DMAZ': 'first', 'DISTWALK': 'first'}).reset_index(), on = 'OMAZ', how = 'left')
+missing_maz = add_missing_mazs_to_skim_table(centroids, maz_to_maz_walk_cost_out, maz_to_maz_cost)
+maz_maz_walk_output = maz_to_maz_walk_cost_out[["OMAZ","DMAZ","DISTWALK"]].append(missing_maz).sort_values(['OMAZ', 'DMAZ'])
+#creating fields as required by the TNC routing Java model. "actual" is walk time in minutes
+maz_maz_walk_output[['i', 'j']] = maz_maz_walk_output[['OMAZ', 'DMAZ']]
+maz_maz_walk_output['actual'] = maz_maz_walk_output['DISTWALK'] / walk_speed_mph * 60.0
+
+# find intrazonal distance by averaging the closest 3 zones and then half it
+maz_maz_walk_output = maz_maz_walk_output.sort_values(['OMAZ', 'DISTWALK'])
+maz_maz_walk_output.set_index(['OMAZ', 'DMAZ'], inplace=True)
+unique_omaz = maz_maz_walk_output.index.get_level_values(0).unique()
+# find the average of the closest 3 zones
+means = maz_maz_walk_output.loc[(unique_omaz, slice(None)), 'DISTWALK'].groupby(level=0).head(3).groupby(level=0).mean()
+intra_skims = pd.DataFrame({
+    'OMAZ': unique_omaz,
+    'DMAZ': unique_omaz,
+    'DISTWALK': means.values/2,
+    'i': unique_omaz,
+    'j': unique_omaz,
+    'actual': (means.values/walk_speed_mph * 60.0) / 2
+}).set_index(['OMAZ', 'DMAZ'])
+maz_maz_walk_output = pd.concat([maz_maz_walk_output, intra_skims], axis=0)
+# write output
 print(f"{datetime.now().strftime('%H:%M:%S')} Write Results...")
-maz_to_maz_walk_cost_out[["OMAZ","DMAZ","DISTWALK"]].append(missing_maz).sort_values(['OMAZ', 'DMAZ']).to_csv(path + '/output/skims/' + parms['mmms']["maz_maz_walk_output"], index=False)
+maz_maz_walk_output.to_csv(path + '/output/skims/' + parms['mmms']["maz_maz_walk_output"])
 del(missing_maz)
 
-
 # %%
 # MAZ-to-MAZ Bike
 print(f"{datetime.now().strftime('%H:%M:%S')} Build Maz To Maz Bike Table...") # same table above
 maz_to_maz_bike_cost = maz_to_maz_cost[maz_to_maz_cost["DISTWALK"] <= max_maz_maz_bike_dist_feet / 5280.0].copy()
 print(f"{datetime.now().strftime('%H:%M:%S')} Get Shortest Path Length...")
 maz_to_maz_bike_cost["DISTBIKE"] = net.shortest_path_lengths(maz_to_maz_bike_cost["OMAZ_NODE"], maz_to_maz_bike_cost["DMAZ_NODE"])
 maz_to_maz_bike_cost_out = maz_to_maz_bike_cost[maz_to_maz_bike_cost["DISTBIKE"] <= max_maz_maz_bike_dist_feet / 5280.0]
-missing_maz = pd.DataFrame(centroids[~centroids['MAZ'].isin(maz_to_maz_bike_cost_out['OMAZ'])]['MAZ']).rename(columns = {'MAZ': 'OMAZ'}).merge(maz_to_maz_cost[maz_to_maz_cost['OMAZ'] != maz_to_maz_cost['DMAZ']].sort_values('DISTWALK').groupby('OMAZ').agg({'DMAZ': 'first', 'DISTWALK': 'first'}).reset_index().rename(columns = {'DISTWALK': 'DISTBIKE'}), on = 'OMAZ', how = 'left')
+_missing_maz = add_missing_mazs_to_skim_table(centroids, maz_to_maz_bike_cost_out, maz_to_maz_cost)
+missing_maz = _missing_maz.rename(columns = {'DISTWALK': 'DISTBIKE'})
 print(f"{datetime.now().strftime('%H:%M:%S')} Write Results...")
 maz_to_maz_bike_cost_out[["OMAZ","DMAZ","DISTBIKE"]].append(missing_maz).sort_values(['OMAZ', 'DMAZ']).to_csv(path + '/output/skims/' + parms['mmms']["maz_maz_bike_output"], index=False)
 del(missing_maz)
@@ -174,7 +223,17 @@
     maz_to_stop_walk_cost_out_mode.loc[:, 'MODE'] = mode
     # in case straight line distance is less than max and actual distance is greater than max (e.g., street net), set actual distance to max
     maz_to_stop_walk_cost_out_mode['DISTWALK'] = maz_to_stop_walk_cost_out_mode['DISTWALK'].clip(upper=max_walk_dist)
-    missing_maz = pd.DataFrame(centroids[~centroids['MAZ'].isin(maz_to_stop_walk_cost_out_mode['MAZ'])]['MAZ']).merge(maz_to_stop_cost.sort_values('DISTANCE').groupby(['MAZ', 'MODE']).agg({'stop': 'first', 'DISTANCE': 'first'}).reset_index(), on = 'MAZ', how = 'left')
+    # peforms a similar operation as the add_missing_mazs_to_skim_table() function
+    missing_maz = pd.DataFrame(
+    centroids[~centroids["MAZ"].isin(maz_to_stop_walk_cost_out_mode["MAZ"])]["MAZ"]
+).merge(
+    maz_to_stop_cost.sort_values("DISTANCE")
+    .groupby(["MAZ", "MODE"])
+    .agg({"stop": "first", "DISTANCE": "first"})
+    .reset_index(),
+    on="MAZ",
+    how="left",
+)
     maz_to_stop_walk_cost = maz_to_stop_walk_cost_out_mode.append(missing_maz.rename(columns = {'DISTANCE': 'DISTWALK'})).sort_values(['MAZ', 'stop'])
     del(maz_to_stop_walk_cost_out_mode)
     del(missing_maz)

src/main/emme/toolbox/import/run4Ds.py

@@ -258,13 +258,12 @@ def get_density(self):
         mgra_landuse = mgra_landuse.merge(syn_pop, how = 'left', on = 'mgra')
         #all street distance  
         equiv_min = pd.read_csv(_join(self.path, "output", self.equivmins_file))
-        equiv_min['dist'] = equiv_min['actual']/60*3
         print("MGRA input landuse: " + self.mgradata_file)
 
         def density_function(mgra_in):
             eqmn = equiv_min[equiv_min['i'] == mgra_in]
-            mgra_circa_int = eqmn[eqmn['dist'] < self.int_radius]['j'].unique()
-            mgra_circa_oth = eqmn[eqmn['dist'] < self.oth_radius]['j'].unique()
+            mgra_circa_int = eqmn[eqmn['DISTWALK'] < self.int_radius]['j'].unique()
+            mgra_circa_oth = eqmn[eqmn['DISTWALK'] < self.oth_radius]['j'].unique()
             totEmp = mgra_landuse[mgra_landuse.mgra.isin(mgra_circa_oth)]['emp_total'].sum()
             totRet = mgra_landuse[mgra_landuse.mgra.isin(mgra_circa_oth)]['emp_ret'].sum()
             totHH = mgra_landuse[mgra_landuse.mgra.isin(mgra_circa_oth)]['hh'].sum()

src/main/emme/toolbox/master_run.py

@@ -285,14 +285,13 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title,
 
         useLocalDrive = props["RunModel.useLocalDrive"]
 
+        skipMGRASkims = props["RunModel.skipMGRASkims"]
         skip4Ds = props["RunModel.skip4Ds"]
         skipInputChecker = props["RunModel.skipInputChecker"]
         skipInitialization = props["RunModel.skipInitialization"]
         deleteAllMatrices = props["RunModel.deleteAllMatrices"]
         skipCopyWarmupTripTables = props["RunModel.skipCopyWarmupTripTables"]
         skipCopyBikeLogsum = props["RunModel.skipCopyBikeLogsum"]
-        skipCopyWalkImpedance = props["RunModel.skipCopyWalkImpedance"]
-        skipWalkLogsums = props["RunModel.skipWalkLogsums"]
         skipBikeLogsums = props["RunModel.skipBikeLogsums"]
         skipBuildNetwork = props["RunModel.skipBuildNetwork"]
         skipHighwayAssignment = props["RunModel.skipHighwayAssignment"]
@@ -447,18 +446,15 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title,
             del(householdFile)
 
             if startFromIteration == 1:  # only run the setup / init steps if starting from iteration 1
-                if not skipWalkLogsums:
-                    self.run_proc("runSandagWalkLogsums.cmd", [drive, path_forward_slash],
-                                  "Walk - create AT logsums and impedances", capture_output=True)
-                if not skipCopyWalkImpedance:
-                    self.copy_files(["walkMgraEquivMinutes.csv", "microMgraEquivMinutes.csv"],
-                                    input_dir, output_dir)
+                if not skipMGRASkims:
+                    self.run_proc("runSandagMGRASkims.cmd", [drive, path_forward_slash],
+                                  "Create MGRA-level skims", capture_output=True)
 
                 if not skip4Ds:
                     run4Ds(path=self._path, int_radius=0.65, ref_path='visualizer_reference_path')
 
                 mgraFile = 'mgra15_based_input' + str(scenarioYear) + '.csv'  # Should be read in from properties? -JJF
-                self.complete_work(scenarioYear, input_dir, output_dir, mgraFile, "walkMgraEquivMinutes.csv")
+                self.complete_work(scenarioYear, input_dir, output_dir, mgraFile, "maz_maz_walk.csv")
 
                 # Update rapid dwell time before importing network
                 mode5tod = pd.read_csv(_join(input_dir,'MODE5TOD.csv'))
@@ -1026,7 +1022,7 @@ def copy_files(self, file_names, from_dir, to_dir):
     def complete_work(self, scenarioYear, input_dir, output_dir, input_file, output_file):
 
         fullList = np.array(pd.read_csv(_join(input_dir, input_file))['mgra'])
-        workList = np.array(pd.read_csv(_join(output_dir, output_file))['i'])
+        workList = np.array(pd.read_csv(_join(output_dir, "skims", output_file))['i'])
 
         list_set = set(workList)
         unique_list = (list(list_set))

src/main/emme/toolbox/utilities/properties.py

@@ -30,14 +30,13 @@ class PropertiesSetter(object):
     sample_rates = _m.Attribute(str)
 
     useLocalDrive = _m.Attribute(bool)
+    skipMGRASkims = _m.Attribute(bool)
     skip4Ds = _m.Attribute(bool)
     skipBuildNetwork = _m.Attribute(bool)
     skipInputChecker = _m.Attribute(bool)
     skipInitialization = _m.Attribute(bool)
     deleteAllMatrices = _m.Attribute(bool)
     skipCopyWarmupTripTables = _m.Attribute(bool)
-    skipWalkLogsums = _m.Attribute(bool)
-    skipCopyWalkImpedance = _m.Attribute(bool)
     skipBikeLogsums = _m.Attribute(bool)
     skipCopyBikeLogsum = _m.Attribute(bool)
     skipTransitConnector = _m.Attribute(bool)
@@ -155,9 +154,9 @@ def _set_list_prop(self, name, value):
 
     def __init__(self):
         self._run_model_names = (
-            "env", "useLocalDrive", "skip4Ds", "skipInputChecker",
+            "env", "useLocalDrive", "skipMGRASkims", "skip4Ds", "skipInputChecker",
             "startFromIteration", "skipInitialization", "deleteAllMatrices", "skipCopyWarmupTripTables",
-            "skipCopyBikeLogsum", "skipCopyWalkImpedance", "skipWalkLogsums", "skipBikeLogsums", "skipBuildNetwork",
+            "skipCopyBikeLogsum", "skipBikeLogsums", "skipBuildNetwork",
             "skipHighwayAssignment", "skipTransitSkimming", "skipTransitConnector", "skipTransponderExport", "skipScenManagement", "skipABMPreprocessing", "skipABMResident", "skipABMAirport", "skipABMXborderWait", "skipABMXborder", "skipABMVisitor", "skipMAASModel",
             "skipCVMEstablishmentSyn", "skipCTM", "skipTruck", "skipEI", "skipExternal", "skipTripTableCreation", "skipFinalHighwayAssignment",
             "skipFinalTransitAssignment", "skipVisualizer", "skipDataExport", "skipDatalake", "skipDataLoadRequest",
@@ -209,14 +208,13 @@ def add_properties_interface(self, pb, disclosure=False):
 
         skip_startup_items = [
             ("useLocalDrive",           "Use the local drive during the model run"),
+            ("skipMGRASkims",           "Skip MGRA skims"),
             ("skip4Ds",                 "Skip running 4Ds"),
             ("skipBuildNetwork",        "Skip build of highway and transit network"),
             ("skipInputChecker",		"Skip running input checker"),
             ("skipInitialization",      "Skip matrix and transit database initialization"),
             ("deleteAllMatrices",       "    Delete all matrices"),
             ("skipCopyWarmupTripTables","Skip import of warmup trip tables"),
-            ("skipWalkLogsums",         "Skip walk logsums"),
-            ("skipCopyWalkImpedance",   "Skip copy of walk impedance"),
             ("skipBikeLogsums",         "Skip bike logsums"),
             ("skipCopyBikeLogsum",      "Skip copy of bike logsum"),
         ]
@@ -355,14 +353,13 @@ def load_properties(self):
         self.sample_rates = ",".join(str(x) for x in props.get("sample_rates"))
 
         self.useLocalDrive = props.get("RunModel.useLocalDrive", True)
+        self.skipMGRASkims = props.get("RunModel.skipMGRASkims", False)
         self.skip4Ds = props.get("RunModel.skip4Ds", False)
         self.skipBuildNetwork = props.get("RunModel.skipBuildNetwork", False)
         self.skipInputChecker = props.get("RunModel.skipInputChecker", False)
         self.skipInitialization = props.get("RunModel.skipInitialization", False)
         self.deleteAllMatrices = props.get("RunModel.deleteAllMatrices", False)
         self.skipCopyWarmupTripTables = props.get("RunModel.skipCopyWarmupTripTables", False)
-        self.skipWalkLogsums = props.get("RunModel.skipWalkLogsums", False)
-        self.skipCopyWalkImpedance = props.get("RunModel.skipCopyWalkImpedance", False)
         self.skipBikeLogsums = props.get("RunModel.skipBikeLogsums", False)
         self.skipCopyBikeLogsum = props.get("RunModel.skipCopyBikeLogsum", False)
 
@@ -401,17 +398,16 @@ def save_properties(self):
         props["sample_rates"] = [float(x) for x in self.sample_rates.split(",")]
 
         props["RunModel.useLocalDrive"] = self.useLocalDrive
+        props["RunModel.skipMGRASkims"] = self.skipMGRASkims
         props["RunModel.skip4Ds"] = self.skip4Ds
         props["RunModel.skipBuildNetwork"] = self.skipBuildNetwork
         props["RunModel.skipInputChecker"] = self.skipInputChecker
         props["RunModel.skipInitialization"] = self.skipInitialization
         props["RunModel.deleteAllMatrices"] = self.deleteAllMatrices
         props["RunModel.skipCopyWarmupTripTables"] = self.skipCopyWarmupTripTables
-        props["RunModel.skipWalkLogsums"] = self.skipWalkLogsums
-        props["RunModel.skipCopyWalkImpedance"] = self.skipCopyWalkImpedance
+
         props["RunModel.skipBikeLogsums"] = self.skipBikeLogsums
         props["RunModel.skipCopyBikeLogsum"] = self.skipCopyBikeLogsum
-
         props["RunModel.skipHighwayAssignment"] = self.skipHighwayAssignment
         props["RunModel.skipTransitSkimming"] = self.skipTransitSkimming
         props["RunModel.skipTransitConnector"] = self.skipTransitConnector

src/main/resources/runSandagAbm_Preprocessing.cmd

@@ -61,8 +61,6 @@ MD assignment
 CD ..
 
 if %ITERATION% equ 1 (
-    ECHO Running resident model pre-processing   
-    %PYTHON3% src/asim/scripts/resident/2zoneSkim.py %PROJECT_DIRECTORY% || goto error
 
     %PYTHON3% src/asim/scripts/resident/resident_preprocessing.py input output %SCENYEAR% %PROJECT_DIRECTORY% || goto error
 

src/main/resources/runSandagMGRASkims.cmd

@@ -0,0 +1,23 @@
+rem @echo off
+
+set PROJECT_DRIVE=%1
+set PROJECT_DIRECTORY=%2
+
+%PROJECT_DRIVE%
+cd %PROJECT_DRIVE%%PROJECT_DIRECTORY%
+:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+:: SET UP PATHS
+:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+SET ANACONDA3_DIR=%CONDA_PREFIX%
+SET CONDA3_ACT=%ANACONDA3_DIR%\Scripts\activate.bat
+
+CALL %CONDA3_ACT% asim_baydag
+SET PYTHON3=%ANACONDA3_DIR%\envs\asim_baydag\python.exe
+
+
+%PROJECT_DRIVE%
+cd %PROJECT_DRIVE%%PROJECT_DIRECTORY%
+
+rem   rem build walk skims
+ECHO Running 2-zone skimming procedure... 
+%PYTHON3% src/asim/scripts/resident/2zoneSkim.py %PROJECT_DIRECTORY% || goto error