From 330480c811f8161f9b9a686abaaaddbd89abcf6c Mon Sep 17 00:00:00 2001 From: Ali Etezady <58451076+aletzdy@users.noreply.github.com> Date: Tue, 24 Sep 2024 08:15:46 -0700 Subject: [PATCH 1/8] removing java based walk skims and replacing with already existing python-based script --- .../configs/airport.CBX/preprocessing.yaml | 19 -------- .../configs/airport.SAN/preprocessing.yaml | 19 -------- src/asim/scripts/resident/2zoneSkim.py | 6 ++- src/main/emme/toolbox/import/run4Ds.py | 5 +-- src/main/emme/toolbox/master_run.py | 18 +++----- src/main/python/datalake_exporter.py | 2 +- src/main/resources/runSandagMGRASkims.cmd | 30 +++++++++++++ src/main/resources/runSandagWalkLogsums.cmd | 45 ------------------- src/main/resources/sandag_abm.properties | 4 +- 9 files changed, 47 insertions(+), 101 deletions(-) create mode 100644 src/main/resources/runSandagMGRASkims.cmd delete mode 100644 src/main/resources/runSandagWalkLogsums.cmd diff --git a/src/asim/configs/airport.CBX/preprocessing.yaml b/src/asim/configs/airport.CBX/preprocessing.yaml index 4ef8c5abe..b596817f6 100644 --- a/src/asim/configs/airport.CBX/preprocessing.yaml +++ b/src/asim/configs/airport.CBX/preprocessing.yaml @@ -33,25 +33,6 @@ stop_frequency_expressions_output_formattable_fname: 'stop_frequency_{purpose}.c trip_purpose_probs_output_fname: trip_purpose_probs.csv trip_scheduling_probs_output_fname: trip_scheduling_probs.csv -# skims -skims: - maz_to_maz: - walk: - input_fname: microMgraEquivMinutes.csv - output_fname: maz_maz_walk.csv - rename_columns: - i: OMAZ - j: DMAZ - taz_to_taz: - periods: - - EA - - AM - - MD - - PM - - EV - input_base_fname: traffic_skims - output_base_fname: traffic_skims_processed - # Tours tours: num_enplanements: 4186500 diff --git a/src/asim/configs/airport.SAN/preprocessing.yaml b/src/asim/configs/airport.SAN/preprocessing.yaml index 4a7819ecf..cec03ed22 100644 --- a/src/asim/configs/airport.SAN/preprocessing.yaml +++ b/src/asim/configs/airport.SAN/preprocessing.yaml @@ -33,25 +33,6 @@ stop_frequency_expressions_output_formattable_fname: 'stop_frequency_{purpose}.c trip_purpose_probs_output_fname: trip_purpose_probs.csv trip_scheduling_probs_output_fname: trip_scheduling_probs.csv -# skims -skims: - maz_to_maz: - walk: - input_fname: microMgraEquivMinutes.csv - output_fname: maz_maz_walk.csv - rename_columns: - i: OMAZ - j: DMAZ - taz_to_taz: - periods: - - EA - - AM - - MD - - PM - - EV - input_base_fname: traffic_skims - output_base_fname: traffic_skims_processed - # Tours tours: num_enplanements: 14536000 diff --git a/src/asim/scripts/resident/2zoneSkim.py b/src/asim/scripts/resident/2zoneSkim.py index 6a0e1bc7a..6f16e292d 100644 --- a/src/asim/scripts/resident/2zoneSkim.py +++ b/src/asim/scripts/resident/2zoneSkim.py @@ -112,8 +112,12 @@ maz_to_maz_walk_cost["DISTWALK"] = net.shortest_path_lengths(maz_to_maz_walk_cost["OMAZ_NODE"], maz_to_maz_walk_cost["DMAZ_NODE"]) maz_to_maz_walk_cost_out = maz_to_maz_walk_cost[maz_to_maz_walk_cost["DISTWALK"] <= max_maz_maz_walk_dist_feet / 5280.0] missing_maz = pd.DataFrame(centroids[~centroids['MAZ'].isin(maz_to_maz_walk_cost_out['OMAZ'])]['MAZ']).rename(columns = {'MAZ': 'OMAZ'}).merge(maz_to_maz_cost[maz_to_maz_cost['OMAZ'] != maz_to_maz_cost['DMAZ']].sort_values('DISTWALK').groupby('OMAZ').agg({'DMAZ': 'first', 'DISTWALK': 'first'}).reset_index(), on = 'OMAZ', how = 'left') +maz_maz_walk_output = maz_to_maz_walk_cost_out[["OMAZ","DMAZ","DISTWALK"]].append(missing_maz).sort_values(['OMAZ', 'DMAZ']) +#creating fields as required by the TNC routing Java model. "actual" is walk time in minutes +maz_maz_walk_output[['i', 'j']] = maz_maz_walk_output[['OMAZ', 'DMAZ']] +maz_maz_walk_output['actual'] = maz_maz_walk_output['DISTWALK'] / walk_speed_mph * 60.0 print(f"{datetime.now().strftime('%H:%M:%S')} Write Results...") -maz_to_maz_walk_cost_out[["OMAZ","DMAZ","DISTWALK"]].append(missing_maz).sort_values(['OMAZ', 'DMAZ']).to_csv(path + '/output/skims/' + parms['mmms']["maz_maz_walk_output"], index=False) +maz_maz_walk_output.to_csv(path + '/output/skims/' + parms['mmms']["maz_maz_walk_output"], index=False) del(missing_maz) diff --git a/src/main/emme/toolbox/import/run4Ds.py b/src/main/emme/toolbox/import/run4Ds.py index c77fb68c6..fdb141f20 100644 --- a/src/main/emme/toolbox/import/run4Ds.py +++ b/src/main/emme/toolbox/import/run4Ds.py @@ -258,13 +258,12 @@ def get_density(self): mgra_landuse = mgra_landuse.merge(syn_pop, how = 'left', on = 'mgra') #all street distance equiv_min = pd.read_csv(_join(self.path, "output", self.equivmins_file)) - equiv_min['dist'] = equiv_min['actual']/60*3 print("MGRA input landuse: " + self.mgradata_file) def density_function(mgra_in): eqmn = equiv_min[equiv_min['i'] == mgra_in] - mgra_circa_int = eqmn[eqmn['dist'] < self.int_radius]['j'].unique() - mgra_circa_oth = eqmn[eqmn['dist'] < self.oth_radius]['j'].unique() + mgra_circa_int = eqmn[eqmn['DISTWALK'] < self.int_radius]['j'].unique() + mgra_circa_oth = eqmn[eqmn['DISTWALK'] < self.oth_radius]['j'].unique() totEmp = mgra_landuse[mgra_landuse.mgra.isin(mgra_circa_oth)]['emp_total'].sum() totRet = mgra_landuse[mgra_landuse.mgra.isin(mgra_circa_oth)]['emp_ret'].sum() totHH = mgra_landuse[mgra_landuse.mgra.isin(mgra_circa_oth)]['hh'].sum() diff --git a/src/main/emme/toolbox/master_run.py b/src/main/emme/toolbox/master_run.py index 976d03dac..eda8b1107 100644 --- a/src/main/emme/toolbox/master_run.py +++ b/src/main/emme/toolbox/master_run.py @@ -450,17 +450,17 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title, if startFromIteration == 1: # only run the setup / init steps if starting from iteration 1 if not skipWalkLogsums: - self.run_proc("runSandagWalkLogsums.cmd", [drive, path_forward_slash], - "Walk - create AT logsums and impedances", capture_output=True) - if not skipCopyWalkImpedance: - self.copy_files(["walkMgraEquivMinutes.csv", "microMgraEquivMinutes.csv"], - input_dir, output_dir) + self.run_proc("runSandagMGRASkims.cmd", [drive, path_forward_slash], + "Create MGRA-level skims", capture_output=True) + # if not skipCopyWalkImpedance: + # self.copy_files(["walkMgraEquivMinutes.csv", "microMgraEquivMinutes.csv"], + # input_dir, output_dir) if not skip4Ds: run4Ds(path=self._path, int_radius=0.65, ref_path='visualizer_reference_path') mgraFile = 'mgra15_based_input' + str(scenarioYear) + '.csv' # Should be read in from properties? -JJF - self.complete_work(scenarioYear, input_dir, output_dir, mgraFile, "walkMgraEquivMinutes.csv") + self.complete_work(scenarioYear, input_dir, output_dir, mgraFile, "maz_maz_walk.csv") # Update rapid dwell time before importing network mode5tod = pd.read_csv(_join(input_dir,'MODE5TOD.csv')) @@ -861,10 +861,6 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title, # UPLOAD DATA AND SWITCH PATHS if useLocalDrive: - # # Uncomment to get disk usage at end of run - # # Note that max disk usage occurs in resident model, not at end of run - # disk_usage = win32.Dispatch('Scripting.FileSystemObject').GetFolder(self._path).Size - # _m.logbook_write("Disk space usage: %f GB" % (disk_usage / (1024 ** 3))) file_manager("UPLOAD", main_directory, username, scenario_id, delete_local_files=not skipDeleteIntermediateFiles) self._path = main_directory @@ -1072,7 +1068,7 @@ def copy_files(self, file_names, from_dir, to_dir): def complete_work(self, scenarioYear, input_dir, output_dir, input_file, output_file): fullList = np.array(pd.read_csv(_join(input_dir, input_file))['mgra']) - workList = np.array(pd.read_csv(_join(output_dir, output_file))['i']) + workList = np.array(pd.read_csv(_join(output_dir, "skims", output_file))['i']) list_set = set(workList) unique_list = (list(list_set)) diff --git a/src/main/python/datalake_exporter.py b/src/main/python/datalake_exporter.py index 0ba4dbcba..c540eecae 100644 --- a/src/main/python/datalake_exporter.py +++ b/src/main/python/datalake_exporter.py @@ -232,7 +232,7 @@ def write_to_datalake(output_path, models, exclude, env): os.path.abspath(os.path.join(output_path, '..', 'input', 'zone_term.csv')), os.path.abspath(os.path.join(output_path, '..', 'input', 'trlink.csv')), os.path.join(output_path, 'bikeMgraLogsum.csv'), - os.path.join(output_path, 'microMgraEquivMinutes.csv'), + # os.path.join(output_path, 'microMgraEquivMinutes.csv'), os.path.join(report_path, 'walkMgrasWithin45Min_AM.csv'), os.path.join(report_path, 'walkMgrasWithin45Min_MD.csv') ] diff --git a/src/main/resources/runSandagMGRASkims.cmd b/src/main/resources/runSandagMGRASkims.cmd new file mode 100644 index 000000000..a064118b7 --- /dev/null +++ b/src/main/resources/runSandagMGRASkims.cmd @@ -0,0 +1,30 @@ +rem @echo off + +set PROJECT_DRIVE=%1 +set PROJECT_DIRECTORY=%2 + +%PROJECT_DRIVE% +cd %PROJECT_DRIVE%%PROJECT_DIRECTORY% +::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:: SET UP PATHS +::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +SET ANACONDA3_DIR=%CONDA_PREFIX% +SET CONDA3_ACT=%ANACONDA3_DIR%\Scripts\activate.bat + +CALL %CONDA3_ACT% asim_baydag +SET PYTHON3=%ANACONDA3_DIR%\envs\asim_baydag\python.exe + + +%PROJECT_DRIVE% +cd %PROJECT_DRIVE%%PROJECT_DIRECTORY% + +rem rem build walk skims +ECHO Running 2-zone skimming procedure... +%PYTHON3% src/asim/scripts/resident/2zoneSkim.py %PROJECT_DIRECTORY% || goto error + +@REM python %PROJECT_DRIVE%%PROJECT_DIRECTORY%\python\calculate_micromobility.py --properties_file %PROJECT_DRIVE%%PROJECT_DIRECTORY%\conf\sandag_abm.properties --outputs_directory %PROJECT_DRIVE%%PROJECT_DIRECTORY%\output --inputs_parent_directory %PROJECT_DRIVE%%PROJECT_DIRECTORY% + +:done +rem ### restore saved environment variable values, and change back to original current directory +set JAVA_PATH=%OLDJAVAPATH% +set PATH=%OLDPATH% diff --git a/src/main/resources/runSandagWalkLogsums.cmd b/src/main/resources/runSandagWalkLogsums.cmd deleted file mode 100644 index a2c9f95ac..000000000 --- a/src/main/resources/runSandagWalkLogsums.cmd +++ /dev/null @@ -1,45 +0,0 @@ -rem @echo off - -set PROJECT_DRIVE=%1 -set PROJECT_DIRECTORY=%2 - -%PROJECT_DRIVE% -cd %PROJECT_DRIVE%%PROJECT_DIRECTORY% -call %PROJECT_DIRECTORY%\bin\CTRampEnv.bat - -rem ### First save the JAVA_PATH environment variable so it s value can be restored at the end. -set OLDJAVAPATH=%JAVA_PATH% - -rem ### Set the directory of the jdk version desired for this model run -set JAVA_PATH=%JAVA_64_PATH% - -rem ### Name the project directory. This directory will hava data and runtime subdirectories -set RUNTIME=%PROJECT_DIRECTORY% -set CONFIG=%RUNTIME%/conf - -set JAR_LOCATION=%PROJECT_DIRECTORY%/application -set LIB_JAR_PATH=%JAR_LOCATION%\* - -rem ### Define the CLASSPATH environment variable for the classpath needed in this model run. -set CLASSPATH=%CONFIG%;%RUNTIME%;%LIB_JAR_PATH%; - -rem ### Save the name of the PATH environment variable, so it can be restored at the end of the model run. -set OLDPATH=%PATH% - -rem ### Change the PATH environment variable so that JAVA_HOME is listed first in the PATH. -rem ### Doing this ensures that the JAVA_HOME path we defined above is the on that gets used in case other java paths are in PATH. -set PATH=%JAVA_PATH%\bin;%OLDPATH% - -%PROJECT_DRIVE% -cd %PROJECT_DRIVE%%PROJECT_DIRECTORY% - -rem rem build walk skims -%JAVA_64_PATH%\bin\java -showversion -server -Xmx%MEMORY_WALKLOGSUM_MAX% -cp "%CLASSPATH%" -Dlog4j.configuration=log4j.xml -Dproject.folder=%PROJECT_DIRECTORY% org.sandag.abm.active.sandag.SandagWalkPathChoiceLogsumMatrixApplication %PROPERTIES_NAME% -if ERRORLEVEL 1 goto DONE - -python %PROJECT_DRIVE%%PROJECT_DIRECTORY%\python\calculate_micromobility.py --properties_file %PROJECT_DRIVE%%PROJECT_DIRECTORY%\conf\sandag_abm.properties --outputs_directory %PROJECT_DRIVE%%PROJECT_DIRECTORY%\output --inputs_parent_directory %PROJECT_DRIVE%%PROJECT_DIRECTORY% - -:done -rem ### restore saved environment variable values, and change back to original current directory -set JAVA_PATH=%OLDJAVAPATH% -set PATH=%OLDPATH% diff --git a/src/main/resources/sandag_abm.properties b/src/main/resources/sandag_abm.properties index 241a505f5..a9b27514e 100644 --- a/src/main/resources/sandag_abm.properties +++ b/src/main/resources/sandag_abm.properties @@ -396,7 +396,7 @@ path.choice.max.path.count = 200 btpc.alts.file = bike_path_alts.csv active.logsum.matrix.file.bike.taz = bikeTazLogsum.csv active.logsum.matrix.file.bike.mgra = bikeMgraLogsum.csv -active.logsum.matrix.file.walk.mgra = walkMgraEquivMinutes.csv +active.logsum.matrix.file.walk.mgra = skims/maz_maz_walk.csv #active.logsum.matrix.file.walk.mgratap = walkMgraTapEquivMinutes.csv active.bike.write.derived.network = true @@ -405,7 +405,7 @@ active.bike.derived.network.nodes = derivedBikeNodes.csv active.bike.derived.network.traversals = derivedBikeTraversals.csv active.assignment.file.bike = bikeAssignmentResults.csv -active.micromobility.file.walk.mgra = microMgraEquivMinutes.csv +# active.micromobility.file.walk.mgra = microMgraEquivMinutes.csv #active.micromobility.file.walk.mgratap = microMgraTapEquivMinutes.csv AtTransitConsistency.xThreshold = 1.0 From 1a2e08cccfb561aab555c1db5414ef94e6d5ffb5 Mon Sep 17 00:00:00 2001 From: Ali Etezady <58451076+aletzdy@users.noreply.github.com> Date: Fri, 4 Oct 2024 14:01:38 -0700 Subject: [PATCH 2/8] create intrazonal walk distances --- src/asim/scripts/resident/2zoneSkim.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/asim/scripts/resident/2zoneSkim.py b/src/asim/scripts/resident/2zoneSkim.py index 6f16e292d..6020b01de 100644 --- a/src/asim/scripts/resident/2zoneSkim.py +++ b/src/asim/scripts/resident/2zoneSkim.py @@ -116,11 +116,27 @@ #creating fields as required by the TNC routing Java model. "actual" is walk time in minutes maz_maz_walk_output[['i', 'j']] = maz_maz_walk_output[['OMAZ', 'DMAZ']] maz_maz_walk_output['actual'] = maz_maz_walk_output['DISTWALK'] / walk_speed_mph * 60.0 + +# find intrazonal distance by averaging the closest 3 zones and then half it +maz_maz_walk_output = maz_maz_walk_output.sort_values(['OMAZ', 'DISTWALK']) +maz_maz_walk_output.set_index(['OMAZ', 'DMAZ'], inplace=True) +unique_omaz = maz_maz_walk_output.index.get_level_values(0).unique() +# find the average of the closest 3 zones +means = maz_maz_walk_output.loc[(unique_omaz, slice(None)), 'DISTWALK'].groupby(level=0).head(3).groupby(level=0).mean() +intra_skims = pd.DataFrame({ + 'OMAZ': unique_omaz, + 'DMAZ': unique_omaz, + 'DISTWALK': means.values/2, + 'i': unique_omaz, + 'j': unique_omaz, + 'actual': means.values/2 +}).set_index(['OMAZ', 'DMAZ']) +maz_maz_walk_output = pd.concat([maz_maz_walk_output, intra_skims], axis=0) +# write output print(f"{datetime.now().strftime('%H:%M:%S')} Write Results...") -maz_maz_walk_output.to_csv(path + '/output/skims/' + parms['mmms']["maz_maz_walk_output"], index=False) +maz_maz_walk_output.to_csv(path + '/output/skims/' + parms['mmms']["maz_maz_walk_output"]) del(missing_maz) - # %% # MAZ-to-MAZ Bike print(f"{datetime.now().strftime('%H:%M:%S')} Build Maz To Maz Bike Table...") # same table above From 8b10e480a023dd262aba0ae569b0298e7c206de7 Mon Sep 17 00:00:00 2001 From: Ali Etezady <58451076+aletzdy@users.noreply.github.com> Date: Fri, 4 Oct 2024 14:02:19 -0700 Subject: [PATCH 3/8] moved 2zone skimming to the top of the model run --- src/main/resources/runSandagAbm_Preprocessing.cmd | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/resources/runSandagAbm_Preprocessing.cmd b/src/main/resources/runSandagAbm_Preprocessing.cmd index c37b9e9e0..50be4520b 100644 --- a/src/main/resources/runSandagAbm_Preprocessing.cmd +++ b/src/main/resources/runSandagAbm_Preprocessing.cmd @@ -61,8 +61,6 @@ MD assignment CD .. if %ITERATION% equ 1 ( - ECHO Running resident model pre-processing - %PYTHON3% src/asim/scripts/resident/2zoneSkim.py %PROJECT_DIRECTORY% || goto error %PYTHON3% src/asim/scripts/resident/resident_preprocessing.py input output %SCENYEAR% %PROJECT_DIRECTORY% || goto error From 4c32e45b6e952c9b218bf8eaf980777529413baf Mon Sep 17 00:00:00 2001 From: Ali Etezady <58451076+aletzdy@users.noreply.github.com> Date: Mon, 7 Oct 2024 07:13:53 -0700 Subject: [PATCH 4/8] changing walklogsum step naming and ordering in master run --- src/main/emme/toolbox/master_run.py | 8 ++------ src/main/emme/toolbox/utilities/properties.py | 18 +++++++----------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/src/main/emme/toolbox/master_run.py b/src/main/emme/toolbox/master_run.py index eda8b1107..e15a5b00d 100644 --- a/src/main/emme/toolbox/master_run.py +++ b/src/main/emme/toolbox/master_run.py @@ -286,14 +286,13 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title, useLocalDrive = props["RunModel.useLocalDrive"] + skipMGRASkims = props["RunModel.skipMGRASkims"] skip4Ds = props["RunModel.skip4Ds"] skipInputChecker = props["RunModel.skipInputChecker"] skipInitialization = props["RunModel.skipInitialization"] deleteAllMatrices = props["RunModel.deleteAllMatrices"] skipCopyWarmupTripTables = props["RunModel.skipCopyWarmupTripTables"] skipCopyBikeLogsum = props["RunModel.skipCopyBikeLogsum"] - skipCopyWalkImpedance = props["RunModel.skipCopyWalkImpedance"] - skipWalkLogsums = props["RunModel.skipWalkLogsums"] skipBikeLogsums = props["RunModel.skipBikeLogsums"] skipBuildNetwork = props["RunModel.skipBuildNetwork"] skipHighwayAssignment = props["RunModel.skipHighwayAssignment"] @@ -449,12 +448,9 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title, del(householdFile) if startFromIteration == 1: # only run the setup / init steps if starting from iteration 1 - if not skipWalkLogsums: + if not skipMGRASkims: self.run_proc("runSandagMGRASkims.cmd", [drive, path_forward_slash], "Create MGRA-level skims", capture_output=True) - # if not skipCopyWalkImpedance: - # self.copy_files(["walkMgraEquivMinutes.csv", "microMgraEquivMinutes.csv"], - # input_dir, output_dir) if not skip4Ds: run4Ds(path=self._path, int_radius=0.65, ref_path='visualizer_reference_path') diff --git a/src/main/emme/toolbox/utilities/properties.py b/src/main/emme/toolbox/utilities/properties.py index 209df83ee..2a59e1bdf 100644 --- a/src/main/emme/toolbox/utilities/properties.py +++ b/src/main/emme/toolbox/utilities/properties.py @@ -30,14 +30,13 @@ class PropertiesSetter(object): sample_rates = _m.Attribute(str) useLocalDrive = _m.Attribute(bool) + skipMGRASkims = _m.Attribute(bool) skip4Ds = _m.Attribute(bool) skipBuildNetwork = _m.Attribute(bool) skipInputChecker = _m.Attribute(bool) skipInitialization = _m.Attribute(bool) deleteAllMatrices = _m.Attribute(bool) skipCopyWarmupTripTables = _m.Attribute(bool) - skipWalkLogsums = _m.Attribute(bool) - skipCopyWalkImpedance = _m.Attribute(bool) skipBikeLogsums = _m.Attribute(bool) skipCopyBikeLogsum = _m.Attribute(bool) skipTransitConnector = _m.Attribute(bool) @@ -155,9 +154,9 @@ def _set_list_prop(self, name, value): def __init__(self): self._run_model_names = ( - "env", "useLocalDrive", "skip4Ds", "skipInputChecker", + "env", "useLocalDrive", "skipMGRASkims", "skip4Ds", "skipInputChecker", "startFromIteration", "skipInitialization", "deleteAllMatrices", "skipCopyWarmupTripTables", - "skipCopyBikeLogsum", "skipCopyWalkImpedance", "skipWalkLogsums", "skipBikeLogsums", "skipBuildNetwork", + "skipCopyBikeLogsum", "skipBikeLogsums", "skipBuildNetwork", "skipHighwayAssignment", "skipTransitSkimming", "skipTransitConnector", "skipTransponderExport", "skipScenManagement", "skipABMPreprocessing", "skipABMResident", "skipABMAirport", "skipABMXborderWait", "skipABMXborder", "skipABMVisitor", "skipMAASModel", "skipCVMEstablishmentSyn", "skipCTM", "skipTruck", "skipEI", "skipExternal", "skipTripTableCreation", "skipFinalHighwayAssignment", "skipFinalTransitAssignment", "skipVisualizer", "skipDataExport", "skipDatalake", "skipDataLoadRequest", @@ -209,14 +208,13 @@ def add_properties_interface(self, pb, disclosure=False): skip_startup_items = [ ("useLocalDrive", "Use the local drive during the model run"), + ("skipMGRASkims", "Skip MGRA skims"), ("skip4Ds", "Skip running 4Ds"), ("skipBuildNetwork", "Skip build of highway and transit network"), ("skipInputChecker", "Skip running input checker"), ("skipInitialization", "Skip matrix and transit database initialization"), ("deleteAllMatrices", "    Delete all matrices"), ("skipCopyWarmupTripTables","Skip import of warmup trip tables"), - ("skipWalkLogsums", "Skip walk logsums"), - ("skipCopyWalkImpedance", "Skip copy of walk impedance"), ("skipBikeLogsums", "Skip bike logsums"), ("skipCopyBikeLogsum", "Skip copy of bike logsum"), ] @@ -355,14 +353,13 @@ def load_properties(self): self.sample_rates = ",".join(str(x) for x in props.get("sample_rates")) self.useLocalDrive = props.get("RunModel.useLocalDrive", True) + self.skipMGRASkims = props.get("RunModel.skipMGRASkims", False) self.skip4Ds = props.get("RunModel.skip4Ds", False) self.skipBuildNetwork = props.get("RunModel.skipBuildNetwork", False) self.skipInputChecker = props.get("RunModel.skipInputChecker", False) self.skipInitialization = props.get("RunModel.skipInitialization", False) self.deleteAllMatrices = props.get("RunModel.deleteAllMatrices", False) self.skipCopyWarmupTripTables = props.get("RunModel.skipCopyWarmupTripTables", False) - self.skipWalkLogsums = props.get("RunModel.skipWalkLogsums", False) - self.skipCopyWalkImpedance = props.get("RunModel.skipCopyWalkImpedance", False) self.skipBikeLogsums = props.get("RunModel.skipBikeLogsums", False) self.skipCopyBikeLogsum = props.get("RunModel.skipCopyBikeLogsum", False) @@ -401,17 +398,16 @@ def save_properties(self): props["sample_rates"] = [float(x) for x in self.sample_rates.split(",")] props["RunModel.useLocalDrive"] = self.useLocalDrive + props["RunModel.skipMGRASkims"] = self.skipMGRASkims props["RunModel.skip4Ds"] = self.skip4Ds props["RunModel.skipBuildNetwork"] = self.skipBuildNetwork props["RunModel.skipInputChecker"] = self.skipInputChecker props["RunModel.skipInitialization"] = self.skipInitialization props["RunModel.deleteAllMatrices"] = self.deleteAllMatrices props["RunModel.skipCopyWarmupTripTables"] = self.skipCopyWarmupTripTables - props["RunModel.skipWalkLogsums"] = self.skipWalkLogsums - props["RunModel.skipCopyWalkImpedance"] = self.skipCopyWalkImpedance + props["RunModel.skipBikeLogsums"] = self.skipBikeLogsums props["RunModel.skipCopyBikeLogsum"] = self.skipCopyBikeLogsum - props["RunModel.skipHighwayAssignment"] = self.skipHighwayAssignment props["RunModel.skipTransitSkimming"] = self.skipTransitSkimming props["RunModel.skipTransitConnector"] = self.skipTransitConnector From 8406ccda9b635783c082550bb20c01269cc69fc0 Mon Sep 17 00:00:00 2001 From: Ali Etezady <58451076+aletzdy@users.noreply.github.com> Date: Mon, 7 Oct 2024 07:14:53 -0700 Subject: [PATCH 5/8] removing old java related lines from runSandagMGRASkims.cmd --- src/main/resources/runSandagMGRASkims.cmd | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/main/resources/runSandagMGRASkims.cmd b/src/main/resources/runSandagMGRASkims.cmd index a064118b7..206eba629 100644 --- a/src/main/resources/runSandagMGRASkims.cmd +++ b/src/main/resources/runSandagMGRASkims.cmd @@ -20,11 +20,4 @@ cd %PROJECT_DRIVE%%PROJECT_DIRECTORY% rem rem build walk skims ECHO Running 2-zone skimming procedure... -%PYTHON3% src/asim/scripts/resident/2zoneSkim.py %PROJECT_DIRECTORY% || goto error - -@REM python %PROJECT_DRIVE%%PROJECT_DIRECTORY%\python\calculate_micromobility.py --properties_file %PROJECT_DRIVE%%PROJECT_DIRECTORY%\conf\sandag_abm.properties --outputs_directory %PROJECT_DRIVE%%PROJECT_DIRECTORY%\output --inputs_parent_directory %PROJECT_DRIVE%%PROJECT_DIRECTORY% - -:done -rem ### restore saved environment variable values, and change back to original current directory -set JAVA_PATH=%OLDJAVAPATH% -set PATH=%OLDPATH% +%PYTHON3% src/asim/scripts/resident/2zoneSkim.py %PROJECT_DIRECTORY% || goto error \ No newline at end of file From 99e971cf86789208d95ac27aa3dc5b1bdd88a1e9 Mon Sep 17 00:00:00 2001 From: Ali Etezady <58451076+aletzdy@users.noreply.github.com> Date: Fri, 11 Oct 2024 13:57:50 -0700 Subject: [PATCH 6/8] fix bug by setting 'actual' to be time --- src/asim/scripts/resident/2zoneSkim.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/asim/scripts/resident/2zoneSkim.py b/src/asim/scripts/resident/2zoneSkim.py index 6020b01de..6902b0828 100644 --- a/src/asim/scripts/resident/2zoneSkim.py +++ b/src/asim/scripts/resident/2zoneSkim.py @@ -129,7 +129,7 @@ 'DISTWALK': means.values/2, 'i': unique_omaz, 'j': unique_omaz, - 'actual': means.values/2 + 'actual': (means.values/walk_speed_mph * 60.0) / 2 }).set_index(['OMAZ', 'DMAZ']) maz_maz_walk_output = pd.concat([maz_maz_walk_output, intra_skims], axis=0) # write output From 0cc951293708379a3f2bde9ac2c81296636e7bc1 Mon Sep 17 00:00:00 2001 From: Ali Etezady <58451076+aletzdy@users.noreply.github.com> Date: Thu, 24 Oct 2024 18:26:24 -0700 Subject: [PATCH 7/8] Add disk usage logging back to master run --- src/main/emme/toolbox/master_run.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/emme/toolbox/master_run.py b/src/main/emme/toolbox/master_run.py index 1a285a62e..9771ec91f 100644 --- a/src/main/emme/toolbox/master_run.py +++ b/src/main/emme/toolbox/master_run.py @@ -857,6 +857,10 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title, # UPLOAD DATA AND SWITCH PATHS if useLocalDrive: + # # Uncomment to get disk usage at end of run + # # Note that max disk usage occurs in resident model, not at end of run + # disk_usage = win32.Dispatch('Scripting.FileSystemObject').GetFolder(self._path).Size + # _m.logbook_write("Disk space usage: %f GB" % (disk_usage / (1024 ** 3))) file_manager("UPLOAD", main_directory, username, scenario_id, delete_local_files=not skipDeleteIntermediateFiles) self._path = main_directory From 4755bbf29205bd372aeae78c6915c80758733a32 Mon Sep 17 00:00:00 2001 From: Ali Etezady <58451076+aletzdy@users.noreply.github.com> Date: Fri, 25 Oct 2024 08:48:53 -0700 Subject: [PATCH 8/8] updated 2zone skimming to add a function for adding missing mazs --- src/asim/scripts/resident/2zoneSkim.py | 45 ++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/src/asim/scripts/resident/2zoneSkim.py b/src/asim/scripts/resident/2zoneSkim.py index 6902b0828..7f940d205 100644 --- a/src/asim/scripts/resident/2zoneSkim.py +++ b/src/asim/scripts/resident/2zoneSkim.py @@ -17,6 +17,34 @@ with open(yml_path, 'r') as f: parms = yaml.safe_load(f) +def add_missing_mazs_to_skim_table(centroids, maz_to_maz_walk_cost_out, maz_to_maz_cost): + """ + Considering that we are using a maximum walk threshold for our skims here, it is possible that some MGRAs, + especially in non-urban areas, might not be within threshold distance of other MGRAs. This means that the skim table + may not have skims for all MGRAs. + + This function allows identifying those MGRAs that may be missing from + the skim table, finds the shortest distance + for them (not to themselves), and therefore creating a missig_maz df that now contains skims for those MGRAs that + otherwise did not have skim at all. + + """ + # Identify missing MAZs + missing_maz = centroids[~centroids['MAZ'].isin(maz_to_maz_walk_cost_out['OMAZ'])][['MAZ']] + missing_maz = missing_maz.rename(columns={'MAZ': 'OMAZ'}) + + # Filter and sort maz_to_maz_cost DataFrame so we have the shortest distance for each OMAZ first + filtered_maz_to_maz_cost = maz_to_maz_cost[maz_to_maz_cost['OMAZ'] != maz_to_maz_cost['DMAZ']] + sorted_maz_to_maz_cost = filtered_maz_to_maz_cost.sort_values('DISTWALK') + + # Group by 'OMAZ' and select the first (shortest dist) 'DMAZ' and 'DISTWALK' for each group + grouped_maz_to_maz_cost = sorted_maz_to_maz_cost.groupby('OMAZ').agg({'DMAZ': 'first', 'DISTWALK': 'first'}).reset_index() + + # Merge the missing MAZs with the grouped maz_to_maz_cost DataFrame + result = missing_maz.merge(grouped_maz_to_maz_cost, on='OMAZ', how='left') + + return result + print(f"{datetime.now().strftime('%H:%M:%S')} Preparing MAZ-MAZ and MAZ-Stop Connectors...") startTime = time.time() #asim_inputs = os.path.join(path, "ASIM_INPUTS") @@ -111,7 +139,7 @@ print(f"{datetime.now().strftime('%H:%M:%S')} Get Shortest Path Length...") maz_to_maz_walk_cost["DISTWALK"] = net.shortest_path_lengths(maz_to_maz_walk_cost["OMAZ_NODE"], maz_to_maz_walk_cost["DMAZ_NODE"]) maz_to_maz_walk_cost_out = maz_to_maz_walk_cost[maz_to_maz_walk_cost["DISTWALK"] <= max_maz_maz_walk_dist_feet / 5280.0] -missing_maz = pd.DataFrame(centroids[~centroids['MAZ'].isin(maz_to_maz_walk_cost_out['OMAZ'])]['MAZ']).rename(columns = {'MAZ': 'OMAZ'}).merge(maz_to_maz_cost[maz_to_maz_cost['OMAZ'] != maz_to_maz_cost['DMAZ']].sort_values('DISTWALK').groupby('OMAZ').agg({'DMAZ': 'first', 'DISTWALK': 'first'}).reset_index(), on = 'OMAZ', how = 'left') +missing_maz = add_missing_mazs_to_skim_table(centroids, maz_to_maz_walk_cost_out, maz_to_maz_cost) maz_maz_walk_output = maz_to_maz_walk_cost_out[["OMAZ","DMAZ","DISTWALK"]].append(missing_maz).sort_values(['OMAZ', 'DMAZ']) #creating fields as required by the TNC routing Java model. "actual" is walk time in minutes maz_maz_walk_output[['i', 'j']] = maz_maz_walk_output[['OMAZ', 'DMAZ']] @@ -144,7 +172,8 @@ print(f"{datetime.now().strftime('%H:%M:%S')} Get Shortest Path Length...") maz_to_maz_bike_cost["DISTBIKE"] = net.shortest_path_lengths(maz_to_maz_bike_cost["OMAZ_NODE"], maz_to_maz_bike_cost["DMAZ_NODE"]) maz_to_maz_bike_cost_out = maz_to_maz_bike_cost[maz_to_maz_bike_cost["DISTBIKE"] <= max_maz_maz_bike_dist_feet / 5280.0] -missing_maz = pd.DataFrame(centroids[~centroids['MAZ'].isin(maz_to_maz_bike_cost_out['OMAZ'])]['MAZ']).rename(columns = {'MAZ': 'OMAZ'}).merge(maz_to_maz_cost[maz_to_maz_cost['OMAZ'] != maz_to_maz_cost['DMAZ']].sort_values('DISTWALK').groupby('OMAZ').agg({'DMAZ': 'first', 'DISTWALK': 'first'}).reset_index().rename(columns = {'DISTWALK': 'DISTBIKE'}), on = 'OMAZ', how = 'left') +_missing_maz = add_missing_mazs_to_skim_table(centroids, maz_to_maz_bike_cost_out, maz_to_maz_cost) +missing_maz = _missing_maz.rename(columns = {'DISTWALK': 'DISTBIKE'}) print(f"{datetime.now().strftime('%H:%M:%S')} Write Results...") maz_to_maz_bike_cost_out[["OMAZ","DMAZ","DISTBIKE"]].append(missing_maz).sort_values(['OMAZ', 'DMAZ']).to_csv(path + '/output/skims/' + parms['mmms']["maz_maz_bike_output"], index=False) del(missing_maz) @@ -194,7 +223,17 @@ maz_to_stop_walk_cost_out_mode.loc[:, 'MODE'] = mode # in case straight line distance is less than max and actual distance is greater than max (e.g., street net), set actual distance to max maz_to_stop_walk_cost_out_mode['DISTWALK'] = maz_to_stop_walk_cost_out_mode['DISTWALK'].clip(upper=max_walk_dist) - missing_maz = pd.DataFrame(centroids[~centroids['MAZ'].isin(maz_to_stop_walk_cost_out_mode['MAZ'])]['MAZ']).merge(maz_to_stop_cost.sort_values('DISTANCE').groupby(['MAZ', 'MODE']).agg({'stop': 'first', 'DISTANCE': 'first'}).reset_index(), on = 'MAZ', how = 'left') + # peforms a similar operation as the add_missing_mazs_to_skim_table() function + missing_maz = pd.DataFrame( + centroids[~centroids["MAZ"].isin(maz_to_stop_walk_cost_out_mode["MAZ"])]["MAZ"] +).merge( + maz_to_stop_cost.sort_values("DISTANCE") + .groupby(["MAZ", "MODE"]) + .agg({"stop": "first", "DISTANCE": "first"}) + .reset_index(), + on="MAZ", + how="left", +) maz_to_stop_walk_cost = maz_to_stop_walk_cost_out_mode.append(missing_maz.rename(columns = {'DISTANCE': 'DISTWALK'})).sort_values(['MAZ', 'stop']) del(maz_to_stop_walk_cost_out_mode) del(missing_maz)