diff --git a/input/input_checker/config/inputs_checks.csv b/input/input_checker/config/inputs_checks.csv deleted file mode 100644 index d0a9c4629..000000000 --- a/input/input_checker/config/inputs_checks.csv +++ /dev/null @@ -1,80 +0,0 @@ -Test,Input_Table,Input_ID_Column,Severity,Type,Expression,Test_Vals,Report_Statistic,Test_Description -scenarioYear,scenario,Year,NA,Calculation,scenario['Year'].iloc[0],,,Obtain scenario year -zoneCount,scenario,zoneCount,NA,Calculation,scenario['zoneCount'].iloc[0],,,Obtain number of zones in scenario. Number of zones may change as a result of zone splitting. -PERSON_CTRAMP_Fields_Availability,persons,perid,Fatal,Test,test_val in persons.columns,"hhid,perid,household_serial_no,pnum,age,sex,miltary,pemploy,pstudent,ptype,educ,grade,occen5,occsoc5,indcen,weeks,hours,rac1p,hisp,version",,Check if person file contains all the required fields for ABM -PERSON_military_valueCheck_agg,persons,perid,Warning,Test,"set(persons.miltary)=={0,1}",,,"Check if Person data matches the pre-defined military categories [0 = N/A Less than 17 years old, 1 = Yes, Now on Active Duty]" -PERSON_military_valueCheck_disagg,persons,perid,Fatal,Test,"persons.miltary.apply(lambda x: True if x in [0,1,2,3,4] else False)",,,"Check if Person data matches the pre-defined military categories [0 = N/A Less than 17 years old, 1 = Yes, Now on Active Duty, 2 = Yes, on Active Duty in Past, but not Now, 3 = No, Training for Reserves/National Guard Only, 4 = No, Never Served in the Military]" -PERSON_pemploy_valueCheck_agg,persons,perid,Warning,Test,"set(persons.pemploy)=={1,2,3,4}",,,"Check if Person data matches the pre-defined pemploy categories [1 = Employed Full-Time, 2 = Employed Part-Time, 3 = Unemployed or Not in Labor Force, 4 = Less than 16 Years Old]" -PERSON_pemploy_valueCheck_disagg,persons,perid,Fatal,Test,"persons.pemploy.apply(lambda x: True if x in [1,2,3,4] else False)",,,"Check if Person data matches the pre-defined pemploy categories [1 = Employed Full-Time, 2 = Employed Part-Time, 3 = Unemployed or Not in Labor Force, 4 = Less than 16 Years Old]" -PERSON_pstudent_valueCheck_agg,persons,perid,Warning,Test,"set(persons.pstudent)=={1,2,3}",,,"Check if Person data matches the pre-defined pstudent categories [1 = Pre K-12, 2 = College Undergrad + Grad and Prof. School, 3 = Not Attending School]" -PERSON_pstudent_valueCheck_disagg,persons,perid,Fatal,Test,"persons.pstudent.apply(lambda x: True if x in [1,2,3] else False)",,,"Check if Person data matches the pre-defined pstudent categories [1 = Pre K-12, 2 = College Undergrad + Grad and Prof. School, 3 = Not Attending School]" -PERSON_ptype_valueCheck_agg,persons,perid,Warning,Test,"set(persons.ptype)=={1,2,3,4,5,6,7,8}",,,"Check if Person data matches the pre-defined ptype categories [1 = Full-time Worker, 2 = Part-time Worker, 3 = College Student, 4 = Non-working Adult, 5 = Non-working Senior, 6 = Driving Age Student, 7 = Non-driving Student, 8 = Pre-school]" -PERSON_ptype_valueCheck_disagg,persons,perid,Fatal,Test,"persons.ptype.apply(lambda x: True if x in [1,2,3,4,5,6,7,8] else False)",,,"Check if Person data matches the pre-defined ptype categories [1 = Full-time Worker, 2 = Part-time Worker, 3 = College Student, 4 = Non-working Adult, 5 = Non-working Senior, 6 = Driving Age Student, 7 = Non-driving Student, 8 = Pre-school]" -PERSON_educ_valueCheck_agg,persons,perid,Warning,Test,"set(persons.educ)=={1,9,13}",,,"Check if Person data matches the pre-defined educ categories [1 = No schooling completed, 9 = High school graduate, 13 = Bachelor's degree]" -PERSON_educ_valueCheck_disagg,persons,perid,Fatal,Test,"persons.educ.apply(lambda x: True if x in [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16] else False)",,,"Check if Person data matches the pre-defined educ categories [0 = N/A (Under 3 years), 1 = No schooling completed, 2 = Nursery school to 4th grade, 3 = 5th grade or 6th grade, 4 = 7th grade or 8th grade, 5 = 9th grade, 6 = 10th grade, 7 = 11th grade, 8 = High school graduate, 9 = High school graduate, 10 = Some college but less than 1 year, 11 = One or more years of college no degree, 12 = Associate degree, 13 = Bachelor's degree, 14 = Master's degree, 15 = Professional degree, 16 = Doctorate degree]" -PERSON_grade_valueCheck_agg,persons,perid,Warning,Test,"set(persons.grade)=={0,2,5,6}",,,"Check if Person data mathces the pre-defined grade categories [0 = N/A (not attending school), 2 = Kindergarten, 5 = Grade 9 to grade 12, 6 = College undergraduate]" -PERSON_grade_valueCheck_disagg,persons,perid,Fatal,Test,"persons.grade.apply(lambda x: True if x in [0,1,2,3,4,5,6,7] else False)",,,"Check if Person data mathces the pre-defined grade categories [0 = N/A (not attending school), 1 = Nursery school/preschool, 2 = Kindergarten, 3 = Grade 1 to grade 4, 4 = Grade 5 to grade 8, 5 = Grade 9 to grade 12, 6 = College undergraduate, 7 = Graduate or professional school]" -#PERSON_occsoc5_valueCheck_agg,persons,perid,Warning,Test,set(persons.occsoc5)==set(occsoc.occsoc5),,,Check if Person data matches the pre-defined detailed occupation codes defined by the Standard Occupational Classification system -occsoc_codes,occsoc,occsoc5,NA,Calculation,set(occsoc.occsoc5.values),,,Get unique occupational codes from occsoc -PERSON_occsoc5_valueCheck_disagg,persons,perid,Fatal,Test,persons.occsoc5.apply(lambda x: True if x in occsoc_codes else False),,,Check if Person data matches the pre-defined detailed occupation codes defined by the Standard Occupational Classification system -PERSON_weeks_valueCheck_agg,persons,perid,Warning,Test,"set(persons.weeks)=={1,5}",,,"Check if Person data matches the pre-defined weeks categories [1 = 50 to 52 weeks, 5 = 14 to 26 weeks]" -PERSON_weeks_valueCheck_disagg,persons,perid,Fatal,Test,"persons.weeks.apply(lambda x: True if x in [0,1,2,3,4,5,6] else False)",,,"Check if Person data matches the pre-defined weeks categories [0 = N/A (less than 16 years old/did not work in past 12 months), 1 = 50 to 52 weeks, 2 = 48 to 49 weeks, 3 = 40 to 47 weeks, 4 = 27 to 39 weeks, 5 = 14 to 26 weeks, 6 = 13 weeks or less]" -PERSON_race1p_valueCheck_agg,persons,perid,Warning,Test,"set(persons.rac1p)=={1,2,5,6,7,8,9}",,,"Check if Person data matches the pre-defined race1p categories [1 = White Alone, 2 = Black or African American Alone, 5 = American Indian and Alaska Native Tribes specified; or American Indian or Alaska Native, not specified and no other races, 6 = Asian Alone, 7 = Native Hawaiian and Other Pacific Islander Alone, 8 = Some Other Race Alone, 9 = Two or More Major Race Groups]" -PERSON_race1p_valueCheck_disagg,persons,perid,Fatal,Test,"persons.rac1p.apply(lambda x: True if x in [1,2,3,4,5,6,7,8,9] else False)",,,"Check if Person data matches the pre-defined race1p categories [1 = White Alone, 2 = Black or African American Alone, 3 = American Indian Alone, 4 = Alaska Native Alone, 5 = American Indian and Alaska Native Tribes specified; or American Indian or Alaska Native, not specified and no other races, 6 = Asian Alone, 7 = Native Hawaiian and Other Pacific Islander Alone, 8 = Some Other Race Alone, 9 = Two or More Major Race Groups]" -PERSON_hisp_valueCheck_agg,persons,perid,Warning,Test,"set(persons.hisp)=={1,2}",,,"Check if Person data matches the pre-defined hisp categories [1 = Non-Hispanic, 2 = Hispanic]" -PERSON_hisp_valueCheck_disagg,persons,perid,Fatal,Test,"persons.hisp.apply(lambda x: True if x in [1,2] else False)",,,"Check if Person data matches the pre-defined hisp categories [1 = Non-Hispanic, 2 = Hispanic]" -PERSON_gender_valueCheck_agg,persons,perid,Fatal,Test,"set(persons.sex)=={1,2}",,,"Check if Person data matches the pre-defined gender categories [1=male, 2=female]" -PERSON_gender_valueCheck_disagg,persons,perid,Fatal,Test,"persons.sex.apply(lambda x: True if x in [1,2] else False)",,,"Check if Person data matches the pre-defined gender categories [1=male, 2=female]" -PERSON_age_valueCheck,persons,perid,Fatal,Test,(persons.age>=0),,,Check Person's age to be a positive number -PERSON_hhid_sequential_check,persons,perid,Fatal,Test,(min(persons.hhid)==1) & (max(persons.hhid)==len(set(households.hhid))),,,Ensure hhid in Person data is sequential -PERSON_perid_sequential_check,persons,perid,Fatal,Test,(min(persons.perid)==1) & (max(persons.perid)==len(set(persons.perid))),,,Ensure perid in Person's data is sequential -HH_hinccat1_valueCheck_agg,households,hhid,Warning,Test,"set(households.hinccat1)=={1,2,3,4,5}",,,"Check if Household data matches the pre-defined hinccat1 categories [1 = <$30k, 2 = $30-60k, 3 = $60-100k, 4 = $100-150k, 5 = $150+]" -HH_hinccat1_valueCheck_disagg,households,hhid,Fatal,Test,"households.hinccat1.apply(lambda x: True if x in [1,2,3,4,5] else False)",,,"Check if Household data matches the pre-defined hinccat1 categories [1 = <$30k, 2 = $30-60k, 3 = $60-100k, 4 = $100-150k, 5 = $150+]" -HH_hht_valueCheck_agg,households,hhid,Warning,Test,"set(households.hht)=={0,1,2,3,4,5,6,7}",,,"Check if Household data matches the pre-defined hht categories [0 = Not in universe (vacant or GQ), 1 = Family household: married-couple, 2 = Family household: male householder, no wife present, 3 = Family household, female householder, no husband present, 4 = Nonfamily household: male householder, living along, 5 = Nonfamily household: male householder, not living alone, 6 = Nonfamily household: female householder, living alone, 7 = Nonfamily household: female householder, not living alone]" -HH_hht_valueCheck_disagg,households,hhid,Fatal,Test,"households.hht.apply(lambda x: True if x in [0,1,2,3,4,5,6,7] else False)",,,"Check if Household data matches the pre-defined hht categories [0 = Not in universe (vacant or GQ), 1 = Family household: married-couple, 2 = Family household: male householder, no wife present, 3 = Family household, female householder, no husband present, 4 = Nonfamily household: male householder, living along, 5 = Nonfamily household: male householder, not living alone, 6 = Nonfamily household: female householder, living alone, 7 = Nonfamily household: female householder, not living alone]" -HH_bldgsz_valueCheck_agg,households,hhid,Warning,Test,"set(households.bldgsz)=={1,2,3,8,9}",,,"Check if Household data matches the pre-defined bldgsz categories [1 = Mobile home or trailer, 2 = One-family house detached, 3 = One-family house attached, 8 = 20-49 Apartments, 9 = 50 or more apartments]" -HH_bldgsz_valueCheck_disagg,households,hhid,Warning,Test,"households.bldgsz.apply(lambda x: True if x in [1,2,3,4,5,6,7,8,9,10] else False)",,,"Check if Household data matches the pre-defined bldgsz categories [1 = Mobile home or trailer, 2 = One-family house detached, 3 = One-family house attached, 4 = 2 Apartments, 5 = 3-4 Apartments, 6 = 5-9 Apartments, 7 = 10-19 Apartments, 8 = 20-49 Apartments, 9 = 50 or more apartments, 10 = Boat, RV, van, etc.]" -HH_persons_valueCheck,households,hhid,Fatal,Test,(households.persons>0),,,Check if number of persons in household is a positive number -HH_hworkers_persons_valueCheck,households,hhid,Fatal,Test,(households.hworkers >= 0) & (households.hworkers <= households.persons),,,Check if number of workers in household are within the valid range -HH_hhid_sequential_check,households,hhid,Fatal,Test,(min(households.hhid)==1) & (max(households.hhid)==len(set(households.hhid))),,,Ensure hhid in Household data is sequential -mgra_id_list,mgra_data,mgra,NA,Calculation,set(mgra_data.mgra.values),,,Get unique mgra IDs from mgra_data -HH_mgra_range_check,households,hhid,Fatal,Test,households.mgra.apply(lambda x: True if x in mgra_id_list else False),,,Check if mgra IDs in the Household data are in the allowable range -taz_id_list,mgra_data,mgra,NA,Calculation,set(mgra_data.taz.values),,,Get unique taz IDs from mgra_data -HH_TAZ_range_check,households,hhid,Fatal,Test,households.taz.apply(lambda x: True if x in taz_id_list else False),,,Check if taz IDs in the Household data are in the allowable range -#zero_hh_mgras,mgra_data,mgra,NA,Calculation,set(mgra_data.mgra[mgra_data.hh==0]),,,Get mgras with zero households from mgra_data -#HH_household_in_zeroHH_mgra,households,hhid,Fatal,Test,households.mgra.apply(lambda x: True if x not in zero_hh_mgras else False),,,Check if any household is assigned to an mgra with zero households -NODE_CTRAMP_Fields_Availability,network_nodes,id,Fatal,Test,test_val in network_nodes.columns,"id,@tap_id",,Check if Node data contains all the required fields for ABM. The CSV exporter should throw an error if any field is missing in the Emme object -LINK_POSITIVE_LENGTH,network_links,id,Logical,Test,network_links.length>0,,,All link length should be greater than zero -#TRANSITSEGMENT_Unique_Stop_ID_Check,transit_segments,id,Fatal,Test,len(set(transit_segments['@stop_id'])) == len(transit_segments['@stop_id']),,,All transit segment stop IDs must be unique -mgra_total_employment,mgra_data,mgra,NA,Calculation,mgra_data[[col for col in mgra_data if (col.startswith('emp_')) and not (col.endswith('total'))]].sum(axis=1),,, -MGRA_TotalEmployment_CalcCheck,mgra_data,mgra,Logical,Test,mgra_data.emp_total==mgra_total_employment,,,Check if employment by industry adds up to total employment -ZONE_CHECK,centroids,id,Fatal,Test,len(centroids) == zoneCount,,len(centroids),Check that there are 4996 zones present in network -zone_connector_table,centroids,id,NA,Calculation,centroids.loc[(centroids['numInLinks']>0) & (centroids['numOutLinks']>0)],,,Obtain table of centroids (i.e. zones) that have more than 1 incoming and outgoing links (i.e. centroid connectors) -zone_connector_list,zone_connector_table,id,NA,Calculation,set(zone_connector_table.id),,,Obtain list of unique centroid ids that have more than 1 incoming and outgoing links -NO_ZONE_CONNECTOR_CHECK,centroids,id,Fatal,Test,centroids.id.apply(lambda x: True if x in zone_connector_list else False),,,Check that each zone has a zone connector -emme_trstops,transit_segments,id,NA,Calculation,set(transit_segments['@stop_id']),,,Obtain list of unique transit stop IDs from Emme network -TRANSIT_STOP_CHECK,transit_stops,Stop_ID,Fatal,Test,transit_stops.Stop_ID.apply(lambda x: True if x in emme_trstops else False),,,Check that the Emme network is pulling in all transit stops located in the trstop.csv input file -veh_class_toll_by_year,veh_class_toll,Facility_name,NA,Calculation,veh_class_toll[veh_class_toll['Year'] == scenarioYear],,,Scenario year-specific toll facilities from vehicle_class_toll_factors.csv file -veh_class_toll_by_year_names,veh_class_toll_by_year,Facility_name,NA,Calculation,list(veh_class_toll_by_year.Facility_name),,,Obtain list of unique scenario year-specific toll facility names from vehicle_class_toll_factors.csv file -veh_class_toll_by_year_names_concat,veh_class_toll_by_year_names,Facility_name,NA,Calculation,"""|"".join(veh_class_toll_by_year_names)",,,Concatenate list of unique scenario year-specific toll facility names from vehicle_class_toll_factors.csv file -toll_column_names,network_links,id,NA,Calculation,[col for col in network_links if (col.startswith('@toll'))],,,Create list containing column names of toll cost columns -no_toll_rows,network_links,id,NA,Calculation,(network_links[toll_column_names] == 0).all(axis=1),,,Create boolean array of whether or not each row's total toll cost exceeds 0 -emme_toll,network_links,id,NA,Calculation,network_links.loc[(network_links['@lane_restriction'] >1) & (~no_toll_rows)],,,Table of links that have an IHOV (or @lane_restriction) value greater than 1 and a toll cost -emme_toll_names_matching,emme_toll,id,NA,Calculation,emme_toll[emme_toll['linkNames'].str.contains(veh_class_toll_by_year_names_concat)],,,Obtain list of emme network toll links that contain facility toll names from the vehicle_class_toll_factors.csv file -emme_toll_names_matching_ids,emme_toll_names_matching,id,NA,Calculation,set(emme_toll_names_matching.id),,,Obtain unique ids of matching toll names -IHOV_FACILITY_CHECK,emme_toll,id,Logical,Test,emme_toll.id.apply(lambda x: True if x in emme_toll_names_matching_ids else False),,,Check that scenario year-specific toll facilities (from vehicle_class_toll_factors.csv) are present in Emme network toll facilities -TRANSIT_LINE_TAP_CHECK,transit_lines,id,Logical,Test,transit_lines.hasTAP.apply(lambda x: True if x else False),,,Check that all transit lines begin and end at a TAP -ZONE_NON_TOLL_CONN_CHECK,centroids,id,Fatal,Test,centroids.hasLocalConnection.apply(lambda x: True if x else False),,,Check that each zone has a non-toll connection -ifc_speed_dict,network_links,id,NA,Calculation,"{1:85, 2:70, 3:60, 4:50, 5:50, 6:50, 7:50, 8:85, 9:50, 10:50}",,,"Create IFC-Speed dictionary (IFC=key, Speed=value)" -#IFC_SPEED_CHECK,network_links,id,Warning,Test,"network_links.apply(lambda x: not (ifc_speed_dict[x['type']]==x['@speed_posted']) if (x['type'] in ifc_speed_dict.keys()) else False, axis=1) ",,,"Check if an IFC's assigned speed is correct (1 = 85mph, 2 = 70mph, 3 = 60mph, 4 = 50mph, 5 = 50mph, 6 = 50mph, 7 = 50mph, 8 = 85mph, 9 = 50mph, 10 = 50mph)" -#toll_column_names,network_links,id,NA,Calculation,[col for col in network_links if (col.startswith('@toll'))],,,Create list containing column names of toll cost columns -#no_toll_rows,network_links,id,NA,Calculation,(network_links[toll_column_names] == 0).all(axis=1),,,Create boolean array of whether or not each row's total toll cost exceeds 0 -#IHOV2_0_TOLL_CHECK,network_links,id,Warning,Test,"network_links[(network_links['@lane_restriction'] == 2) & (no_toll_rows)].apply(lambda x: set(list('shitmv')).difference(list(x['modes'])) == {'h', 'i'}, axis=1)",,,Check that HOV2 and HOV3+ modes are only available for IHOV=2 with no toll cost -#IHOV3_0_TOLL_CHECK,network_links,id,Warning,Test,"network_links[(network_links['@lane_restriction'] == 3) & (no_toll_rows)].apply(lambda x: set(list('shitmv')).intersection(list(x['modes'])) == {'i'}, axis=1)",,,Check that HOV3+ modes are only available for IHOV=3 with no toll cost -#IHOV1_TOLL_0_TOLL_CHECK,network_links,id,Warning,Test,"network_links[(network_links['@lane_restriction'].isin([1,4])) & (no_toll_rows)].apply(lambda x: len(set(list('shitmv')).difference(list(x['modes']))) == 0, axis=1)",,,Check that all modes are available for General Purpose and no cost toll lanes -#IHOV1_NO_TOLL_CHECK,network_links,id,Fatal,Test,network_links[network_links['@lane_restriction'] == 1][toll_column_names].sum().sum() == 0,,,Check if toll cost is 0 for General Purpose lanes (IHOV=1) -#NON_ZERO_TOLL_CHECK,network_links,id,Warning,Test,"network_links[not(no_toll_rows)].apply(lambda x: len(set(list('shitmv')).difference((list(x['modes'])))) == 0, axis=1)",,,Check that all operation type lanes are available at a nonzero toll cost -#TRANSIT_ON_ZONE_CONNECTOR,network_links,id,Fatal,Test,len(network_links[(network_links['isTransit'] == 1) & (network_links['@lane_restriction'] == 10)]) == 0,,,Check if any transit links are on zone connectors -net_links_lanes,network_links,id,NA,Calculation,network_links.loc[network_links['num_lanes']>0],,,Subset of Emme network links that contain at least 1 lane -HWYCOV_ID_CHECK_TCOV_ID,net_links_lanes,@tcov_id,Fatal,Test,"~net_links_lanes.duplicated(subset=['@tcov_id'], keep = False)",,,Check if there is more than one network link associated for each unique HWYCOV_ID (i.e. @tcov_id). Reports @tcov_id -HWYCOV_ID_CHECK_LINK_ID,net_links_lanes,id,Fatal,Test,"~net_links_lanes.duplicated(subset=['@tcov_id'], keep = False)",,,Check if there is more than one network link associated for each unique HWYCOV_ID (i.e. @tcov_id). Reports link id diff --git a/input/input_checker/config/inputs_list.csv b/input/input_checker/config/inputs_list.csv deleted file mode 100644 index a9c729bca..000000000 --- a/input/input_checker/config/inputs_list.csv +++ /dev/null @@ -1,13 +0,0 @@ -Input_Table,Property_Token,Emme_Object,Fields,Column_Map,Input_Description -persons,PopulationSynthesizer.InputToCTRAMP.PersonFile,NA,All,,Synthetic population person file -households,PopulationSynthesizer.InputToCTRAMP.HouseholdFile,NA,All,,Synthetic population household file -mgra_data,mgra.socec.file,NA,All,,MGRA data -network_links,NA,LINK,All,,Link data from Emme network. File name not needed -network_nodes,NA,NODE,All,,Node data from Emme network -transit_segments,NA,TRANSIT_SEGMENT,All,,Transit segment data from Emme network -transit_lines,NA,TRANSIT_LINE,All,,Transit line data from Emme network -centroids,NA,CENTROID,All,,Centroid data from Emme network -occsoc,PopulationSynthesizer.OccupCodes,NA,All,,Occupation codes defined by the Standard Occupational Classification system -bike_node,active.node.file,NA,All,,Bike network nodes -transit_stops,transit.stop.file,NA,All,,Transit stop attribute file -veh_class_toll,vehicle.class.toll.factor.path,NA,All,,Vehicle class toll factors file diff --git a/src/main/emme/toolbox/import/input_checker.py b/src/main/emme/toolbox/import/input_checker.py deleted file mode 100644 index 3dc42ec6e..000000000 --- a/src/main/emme/toolbox/import/input_checker.py +++ /dev/null @@ -1,508 +0,0 @@ -#////////////////////////////////////////////////////////////////////////////// -#//// /// -#//// Copyright RSG, 2019-2020. /// -#//// Rights to use and modify are granted to the /// -#//// San Diego Association of Governments and partner agencies. /// -#//// This copyright notice must be preserved. /// -#//// /// -#//// import/input_checker.py /// -#//// /// -#//// /// -#//// /// -#//// /// -#////////////////////////////////////////////////////////////////////////////// -# -# Reviews all inputs to SANDAG ABM for possible issues that will result in model errors -# -# -# Files referenced: -# input_checker\config\inputs_checks.csv -# input_checker\config\inputs_list.csv -# -# Script example: -# python C:\ABM_runs\maint_2020_RSG\Tasks\input_checker\emme_toolbox\emme\toolbox\import\input_checker.py - - -import os, shutil, sys, time, csv, logging -import win32com.client as com -import numpy as np -import pandas as pd -import traceback as _traceback -import datetime -import warnings -from simpledbf import Dbf5 -import inro.modeller as _m - -warnings.filterwarnings("ignore") - -_join = os.path.join -_dir = os.path.dirname - -#gen_utils = _m.Modeller().module("sandag.utilities.general") - -class input_checker(_m.Tool()): - - path = _m.Attribute(unicode) - - tool_run_msg = "" - - @_m.method(return_type=_m.UnicodeType) - def tool_run_msg_status(self): - return self.tool_run_msg - - def __init__(self): - project_dir = _dir(_m.Modeller().desktop.project.path) - self.path = _dir(project_dir) - self.input_checker_path = '' - self.inputs_list_path = '' - self.inputs_checks_path = '' - self.prop_input_paths = {} - self.inputs_list = pd.DataFrame() - self.inputs_checks = pd.DataFrame() - self.inputs = {} - self.results = {} - self.result_list = {} - self.problem_ids = {} - self.report_stat = {} - self.num_fatal = int() - self.num_warning = int() - self.num_logical = int() - - def page(self): - pb = _m.ToolPageBuilder(self) - pb.title = "Input Checker" - pb.description = """ - Reviews all inputs to SANDAG ABM for possible issues that could result - in model errors. List of inputs and checks are read from two CSV files: -
-
- -
- The input checker goes through the list of checks and evaluates each - one as True or False. A log file is produced at the end with results - for each check and additionally, a summary of all checks: -
-
- -
- """ - pb.branding_text = "SANDAG - Input Checker" - - if self.tool_run_msg != "": - pb.tool_run_status(self.tool_run_msg_status) - - return pb.render() - - def run(self): - self.tool_run_msg = "" - try: - self() - run_msg = "Input Checker Complete" - self.tool_run_msg = _m.PageBuilder.format_info(run_msg, escape=False) - except Exception as error: - self.tool_run_msg = _m.PageBuilder.format_exception( - error, _traceback.format_exc(error)) - raise - - def __call__(self): - _m.logbook_write("Started running input checker...") - - self.input_checker_path = _join(self.path, 'input_checker') - self.inputs_list_path = _join(self.input_checker_path, 'config', 'inputs_list.csv') - self.inputs_checks_path = _join(self.input_checker_path, 'config', 'inputs_checks.csv') - - #attributes = {"path": self.path} - #gen_utils.log_snapshot("Run Input Checker", str(self), attributes) - - file_paths = [self.inputs_list_path, self.inputs_checks_path] - for path in file_paths: - if not os.path.exists(path): - raise Exception("missing file '%s'" % (path)) - - _m.logbook_write("Reading inputs...") - self.read_inputs() - - _m.logbook_write("Conducting checks...") - self.checks() - - _m.logbook_write("Writing logs...") - self.write_log() - - _m.logbook_write("Checking for fatal errors...") - self.check_num_fatal() - - _m.logbook_write("Finisehd running input checker") - - def read_inputs(self): - # read list of inputs from CSV file - self.inputs_list = pd.read_csv(self.inputs_list_path) - - # remove all commented inputs from the inputs list - self.inputs_list = self.inputs_list.loc[[not i for i in (self.inputs_list['Input_Table'].str.startswith('#'))]] - - # obtain file paths from the sandag_abm.properties - self.prop_file_paths() - - # load emme network - network = _m.Modeller().emmebank.scenario(100).get_network() - - def get_emme_object(emme_network, emme_network_object, fields_to_export): - # Emme network attribute and object names - net_attr = { - 'NODE':'nodes', - 'LINK':'links', - 'TRANSIT_SEGMENT':'transit_segments', - 'TRANSIT_LINE':'transit_lines' - } - - # read-in entire emme network object as a list - get_objs = 'list(emme_network.' + net_attr[emme_network_object] + '())' - uda = eval(get_objs) - - # get list of network object attributes - obj_attr = [] - if fields_to_export[0] in ['all','All','ALL']: - obj_attr = emme_network.attributes(emme_network_object) - else: - obj_attr = fields_to_export - - # instantiate list of network objects - net_objs = [] - for i in range(len(uda)): - obj_fields = [] - get_id = 'uda[i].id' - obj_fields.append(eval(get_id)) - for attr in obj_attr: - get_field = 'uda[i]["' + attr + '"]' - obj_fields.append(eval(get_field)) - net_objs.append(obj_fields) - net_obj_df = pd.DataFrame(net_objs, columns = ['id'] + obj_attr) - - return(net_obj_df) - - for item, row in self.inputs_list.iterrows(): - - print('Adding Input: ' + row['Input_Table']) - - table_name = row['Input_Table'] - emme_network_object = row['Emme_Object'] - column_map = row['Column_Map'] - fields_to_export = row['Fields'].split(',') - - # obtain emme network object, csv or dbf input - if not (pd.isnull(emme_network_object)): - df = get_emme_object(network, emme_network_object, fields_to_export) - self.inputs[table_name] = df - print(' - ' + table_name + ' added') - else: - input_path = self.prop_input_paths[table_name] - input_ext = os.path.splitext(input_path)[1] - if input_ext == '.csv': - df = pd.read_csv(_join(self.path, input_path)) - self.inputs[table_name] = df - print(' - ' + table_name + ' added') - else: - dbf = Dbf5(_join(_dir(self.path), input_path)) - df = dbf.to_dataframe() - self.inputs[table_name] = df - print(' - ' + table_name + ' added') - - def checks(self): - # read all input DFs into memory - for key, df in self.inputs.items(): - expr = key + ' = df' - exec(expr) - - # copy of locals(), a dictionary of all local variables - calc_dict = locals() - - # read list of checks from CSV file - self.inputs_checks = pd.read_csv(self.inputs_checks_path) - - # remove all commented checks from the checks list - self.inputs_checks = self.inputs_checks.loc[[not i for i in (self.inputs_checks['Test'].str.startswith('#'))]] - - # loop through list of checks and conduct all checks - # checks must evaluate to True if inputs are correct - for item, row in self.inputs_checks.iterrows(): - - test = row['Test'] - table = row['Input_Table'] - id_col = row['Input_ID_Column'] - expr = row['Expression'] - test_vals = row['Test_Vals'] - if not (pd.isnull(row['Test_Vals'])): - test_vals = test_vals.split(',') - test_vals = [txt.strip() for txt in test_vals] - test_type = row['Type'] - Severity = row['Severity'] - stat_expr = row['Report_Statistic'] - - if test_type == 'Test': - - print ('Performing Check: ' + row['Test']) - - if (pd.isnull(row['Test_Vals'])): - - # perform test - out = eval(expr, calc_dict) - - # check if test result is a series - if str(type(out)) == "": - # for series, the test must be evaluated across all items - # result is False if a single False is found - self.results[test] = not (False in out.values) - - # reverse results list since we need all False IDs - reverse_results = [not i for i in out.values] - error_expr = table + '.' + id_col + '[reverse_results]' - error_id_list = eval(error_expr) - - # report first 25 problem IDs in the log - if error_id_list.size > 25: - self.problem_ids[test] = error_id_list.iloc[range(25)] - else: - self.problem_ids[test] = error_id_list if error_id_list.size > 0 else [] - - # compute report statistics - if (pd.isnull(stat_expr)): - self.report_stat[test] = '' - else: - stat_list = eval(stat_expr) - self.report_stat[test] = stat_list[reverse_results] - else: - self.results[test] = out - self.problem_ids[test] = [] - if (pd.isnull(stat_expr)): - self.report_stat[test] = '' - else: - self.report_stat[test] = eval(stat_expr) - else: - # loop through test_vals and perform test for each item - self.result_list[test] = [] - for test_val in test_vals: - # perform test (test result must not be of type Series) - out = eval(expr) - - # compute report statistic - if (pd.isnull(stat_expr)): - self.report_stat[test] = '' - else: - self.report_stat[test] = eval(stat_expr) - - # append to list - self.result_list[test].append(out) - self.results[test] = not (False in self.result_list[test]) - self.problem_ids[test] = [] - - print (' - Check Complete') - - else: - # perform calculation - print ('Performing Calculation: ' + row['Test']) - calc_expr = test + ' = ' + expr - exec(calc_expr, {}, calc_dict) - print (' - Calculation Complete') - - def prop_file_paths(self): - prop_files = self.inputs_list[['Input_Table','Property_Token']].dropna() - - load_properties = _m.Modeller().tool('sandag.utilities.properties') - props = load_properties(_join(self.path, 'conf', 'sandag_abm.properties')) - - for item, row in prop_files.iterrows(): - input_table = row['Input_Table'] - input_path = props[row['Property_Token']] - self.prop_input_paths[input_table] = input_path - - def write_log(self): - # function to write out the input checker log file - # there are three blocks: - # - Introduction - # - Action Required: FATAL, LOGICAL, WARNINGS - # - List of passed checks - - # create log file - now = datetime.datetime.now() - - # create log directory if it doesn't already exist - log_path = _join(self.input_checker_path,'logs') - if not os.path.exists(log_path): - os.makedirs(log_path) - - f = open(_join(self.input_checker_path,'logs', ('inputCheckerLog ' + now.strftime("[%Y-%m-%d]") + '.LOG')), 'wb') - - # define re-usable elements - seperator1 = '###########################################################' - seperator2 = '***********************************************************' - - # write out Header - f.write(seperator1 + seperator1 + "\r\n") - f.write(seperator1 + seperator1 + "\r\n\r\n") - f.write("\t SANDAG ABM Input Checker Log File \r\n") - f.write("\t ____________________________ \r\n\r\n\r\n") - f.write("\t Log created on: " + now.strftime("%Y-%m-%d %H:%M") + "\r\n\r\n") - f.write("\t Notes:-\r\n") - f.write("\t The SANDAG ABM Input Checker performs various QA/QC checks on SANDAG ABM inputs as specified by the user.\r\n") - f.write("\t The Input Checker allows the user to specify three severity levels for each QA/QC check:\r\n\r\n") - f.write("\t 1) FATAL 2) LOGICAL 3) WARNING\r\n\r\n") - f.write("\t FATAL Checks: The failure of these checks would result in a FATAL errors in the SANDAG ABM run.\r\n") - f.write("\t In case of FATAL failure, the Input Checker returns a return code of 1 to the\r\n") - f.write("\t main SANDAG ABM model, cauing the model run to halt.\r\n") - f.write("\t LOGICAL Checks: The failure of these checks indicate logical inconsistencies in the inputs.\r\n") - f.write("\t With logical errors in inputs, the SANDAG ABM outputs may not be meaningful.\r\n") - f.write("\t WARNING Checks: The failure of Warning checks would indicate problems in data that would not.\r\n") - f.write("\t halt the run or affect model outputs but might indicate an issue with inputs.\r\n\r\n\r\n") - f.write("\t The results of all the checks are organized as follows: \r\n\r\n") - f.write("\t IMMEDIATE ACTION REQUIRED:\r\n") - f.write("\t -------------------------\r\n") - f.write("\t A log under this heading will be generated in case of failure of a FATAL check\r\n\r\n") - f.write("\t ACTION REQUIRED:\r\n") - f.write("\t ---------------\r\n") - f.write("\t A log under this heading will be generated in case of failure of a LOGICAL check\r\n\r\n") - f.write("\t WARNINGS:\r\n") - f.write("\t ---------------\r\n") - f.write("\t A log under this heading will be generated in case of failure of a WARNING check\r\n\r\n") - f.write("\t LOG OF ALL PASSED CHECKS:\r\n") - f.write("\t -----------\r\n") - f.write("\t A complete listing of results of all passed checks\r\n\r\n") - f.write(seperator1 + seperator1 + "\r\n") - f.write(seperator1 + seperator1 + "\r\n\r\n\r\n\r\n") - - # combine results, inputs_checks and inputs_list - self.inputs_checks['result'] = self.inputs_checks['Test'].map(self.results) - checks_df = pd.merge(self.inputs_checks, self.inputs_list, on='Input_Table') - checks_df = checks_df[checks_df.Type=='Test'] - checks_df['reverse_result'] = [not i for i in checks_df.result] - - # get all FATAL failures - self.num_fatal = checks_df.result[(checks_df.Severity=='Fatal') & (checks_df.reverse_result)].count() - - # get all LOGICAL failures - self.num_logical = checks_df.result[(checks_df.Severity=='Logical') & (checks_df.reverse_result)].count() - - # get all WARNING failures - self.num_warning = checks_df.result[(checks_df.Severity=='Warning') & (checks_df.reverse_result)].count() - - def write_check_log(self, fh, row): - # define constants - seperator2 = '-----------------------------------------------------------' - - # integerize problem ID list - problem_ids = self.problem_ids[row['Test']] - problem_ids = [int(x) for x in problem_ids] - - # write check summary - fh.write('\r\n\r\n' + seperator2 + seperator2) - fh.write("\r\n\t Input File Name: " + ('NA' if not pd.isnull(row['Emme_Object']) else - (self.prop_input_paths[row['Input_Table']].rsplit('/', 1)[-1]))) - fh.write("\r\n\t Input File Location: " + ('NA' if not pd.isnull(row['Emme_Object']) else - (_join(self.input_checker_path, self.prop_input_paths[row['Input_Table']].replace('/','\\'))))) - fh.write("\r\n\t Emme Object: " + (row['Emme_Object'] if not pd.isnull(row['Emme_Object']) else 'NA')) - fh.write("\r\n\t Input Description: " + (row['Input_Description'] if not pd.isnull(row['Input_Description']) else "")) - fh.write("\r\n\t Test Name: " + row['Test']) - fh.write("\r\n\t Test_Description: " + (row['Test_Description'] if not pd.isnull(row['Test_Description']) else "")) - fh.write("\r\n\t Test Severity: " + row['Severity']) - fh.write("\r\n\r\n\t TEST RESULT: " + ('PASSED' if row['result'] else 'FAILED')) - - # display problem IDs for failed column checks - if (not row['result']) & (len(problem_ids)>0) : - fh.write("\r\n\t TEST failed for following values of ID Column: " + row['Input_ID_Column'] + " (only upto 25 IDs displayed)") - fh.write("\r\n\t " + row['Input_ID_Column'] + ": " + ','.join(map(str, problem_ids[0:25]))) - if not (pd.isnull(row['Report_Statistic'])): - this_report_stat = self.report_stat[row['Test']] - fh.write("\r\n\t Test Statistics: " + ','.join(map(str, this_report_stat[0:25]))) - fh.write("\r\n\t Total number of failures: " + str(len(problem_ids))) - else: - if not (pd.isnull(row['Report_Statistic'])): - fh.write("\r\n\t Test Statistic: " + str(self.report_stat[row['Test']])) - - # display result for each test val if it was specified - if not (pd.isnull(row['Test_Vals'])): - fh.write("\r\n\t TEST results for each test val") - result_tuples = zip(row['Test_Vals'].split(","), self.result_list[row['Test']]) - fh.write("\r\n\t ") - fh.write(','.join('[{} - {}]'.format(x[0],x[1]) for x in result_tuples)) - - fh.write("\r\n" + seperator2 + seperator2 + "\r\n\r\n") - - # write out IMMEDIATE ACTION REQUIRED section if needed - if self.num_fatal > 0: - fatal_checks = checks_df[(checks_df.Severity=='Fatal') & (checks_df.reverse_result)] - f.write('\r\n\r\n' + seperator2 + seperator2 + "\r\n") - f.write(seperator2 + seperator2 + "\r\n\r\n") - f.write('\t' + "IMMEDIATE ACTION REQUIRED \r\n") - f.write('\t' + "------------------------- \r\n\r\n") - f.write(seperator2 + seperator2 + "\r\n") - f.write(seperator2 + seperator2 + "\r\n") - - # write out log for each check - for item, row in fatal_checks.iterrows(): - #self.write_check_log(f, row, self.problem_ids[row['Test']]) - #write_check_log(self, f, row, self.problem_ids[row['Test']]) - write_check_log(self, f, row) - - # write out ACTION REQUIRED section if needed - if self.num_logical > 0: - logical_checks = checks_df[(checks_df.Severity=='Logical') & (checks_df.reverse_result)] - f.write('\r\n\r\n' + seperator2 + seperator2 + "\r\n") - f.write(seperator2 + seperator2 + "\r\n\r\n") - f.write('\t' + "ACTION REQUIRED \r\n") - f.write('\t' + "--------------- \r\n\r\n") - f.write(seperator2 + seperator2 + "\r\n") - f.write(seperator2 + seperator2 + "\r\n") - - #write out log for each check - for item, row in logical_checks.iterrows(): - write_check_log(self, f, row) - - # write out WARNINGS section if needed - if self.num_warning > 0: - warning_checks = checks_df[(checks_df.Severity=='Warning') & (checks_df.reverse_result)] - f.write('\r\n\r\n' + seperator2 + seperator2 + "\r\n") - f.write(seperator2 + seperator2 + "\r\n\r\n") - f.write('\t' + "WARNINGS \r\n") - f.write('\t' + "-------- \r\n") - f.write(seperator2 + seperator2 + "\r\n") - f.write(seperator2 + seperator2 + "\r\n") - - # write out log for each check - for item, row in warning_checks.iterrows(): - write_check_log(self, f, row) - - # write out the complete listing of all checks that passed - passed_checks = checks_df[(checks_df.result)] - f.write('\r\n\r\n' + seperator2 + seperator2 + "\r\n") - f.write(seperator2 + seperator2 + "\r\n\r\n") - f.write('\t' + "LOG OF ALL PASSED CHECKS \r\n") - f.write('\t' + "------------------------ \r\n") - f.write(seperator2 + seperator2 + "\r\n") - f.write(seperator2 + seperator2 + "\r\n") - - # write out log for each check - for item, row in passed_checks.iterrows(): - write_check_log(self, f, row) - - f.close() - # write out a summary of results from input checker for main model - f = open(_join(self.input_checker_path,'logs', ('inputCheckerSummary' + '.txt')), 'wb') - f.write('\r\n' + seperator2 + '\r\n') - f.write('\t Summary of Input Checker Fails \r\n') - f.write(seperator2 + '\r\n\r\n') - f.write(' Number of Fatal Errors: ' + str(self.num_fatal)) - f.write('\r\n\r\n Number of Logical Errors: ' + str(self.num_logical)) - f.write('\r\n\r\n Number of Warnings: ' + str(self.num_warning) + '\r\n\r\n') - f.close() - - def check_num_fatal(self): - # return code to the main model based on input checks and results - if self.num_fatal > 0: - _m.logbook_write('At least one fatal error in the inputs.') - _m.logbook_write('Input Checker Failed') - sys.exit(2) \ No newline at end of file diff --git a/src/main/emme/toolbox/master_run.py b/src/main/emme/toolbox/master_run.py index cbb4cd6e6..23567192c 100644 --- a/src/main/emme/toolbox/master_run.py +++ b/src/main/emme/toolbox/master_run.py @@ -225,7 +225,6 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title, copy_scenario = modeller.tool("inro.emme.data.scenario.copy_scenario") run4Ds = modeller.tool("sandag.import.run4Ds") import_network = modeller.tool("sandag.import.import_network") - input_checker = modeller.tool("sandag.import.input_checker") init_transit_db = modeller.tool("sandag.initialize.initialize_transit_database") init_matrices = modeller.tool("sandag.initialize.initialize_matrices") import_demand = modeller.tool("sandag.import.import_seed_demand") @@ -287,7 +286,6 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title, skipMGRASkims = props["RunModel.skipMGRASkims"] skip4Ds = props["RunModel.skip4Ds"] - skipInputChecker = props["RunModel.skipInputChecker"] skipInitialization = props["RunModel.skipInitialization"] deleteAllMatrices = props["RunModel.deleteAllMatrices"] skipCopyWarmupTripTables = props["RunModel.skipCopyWarmupTripTables"] @@ -475,8 +473,6 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title, overwrite=True, emmebank=main_emmebank) - if not skipInputChecker: - input_checker(path=self._path) # parse vehicle availablility file by time-of-day availability_file = "vehicle_class_availability.csv" diff --git a/src/main/emme/toolbox/utilities/properties.py b/src/main/emme/toolbox/utilities/properties.py index d3c4efc50..c70fc77e1 100644 --- a/src/main/emme/toolbox/utilities/properties.py +++ b/src/main/emme/toolbox/utilities/properties.py @@ -33,7 +33,6 @@ class PropertiesSetter(object): skipMGRASkims = _m.Attribute(bool) skip4Ds = _m.Attribute(bool) skipBuildNetwork = _m.Attribute(bool) - skipInputChecker = _m.Attribute(bool) skipInitialization = _m.Attribute(bool) deleteAllMatrices = _m.Attribute(bool) skipCopyWarmupTripTables = _m.Attribute(bool) @@ -154,7 +153,7 @@ def _set_list_prop(self, name, value): def __init__(self): self._run_model_names = ( - "env", "useLocalDrive", "skipMGRASkims", "skip4Ds", "skipInputChecker", + "env", "useLocalDrive", "skipMGRASkims", "skip4Ds", "startFromIteration", "skipInitialization", "deleteAllMatrices", "skipCopyWarmupTripTables", "skipBikeLogsums", "skipBuildNetwork", "skipHighwayAssignment", "skipTransitSkimming", "skipTransitConnector", "skipTransponderExport", "skipScenManagement", "skipABMPreprocessing", "skipABMResident", "skipABMAirport", "skipABMXborderWait", "skipABMXborder", "skipABMVisitor", "skipMAASModel", @@ -211,7 +210,6 @@ def add_properties_interface(self, pb, disclosure=False): ("skipMGRASkims", "Skip MGRA skims"), ("skip4Ds", "Skip running 4Ds"), ("skipBuildNetwork", "Skip build of highway and transit network"), - ("skipInputChecker", "Skip running input checker"), ("skipInitialization", "Skip matrix and transit database initialization"), ("deleteAllMatrices", "    Delete all matrices"), ("skipCopyWarmupTripTables","Skip import of warmup trip tables"), @@ -356,7 +354,6 @@ def load_properties(self): self.skipMGRASkims = props.get("RunModel.skipMGRASkims", False) self.skip4Ds = props.get("RunModel.skip4Ds", False) self.skipBuildNetwork = props.get("RunModel.skipBuildNetwork", False) - self.skipInputChecker = props.get("RunModel.skipInputChecker", False) self.skipInitialization = props.get("RunModel.skipInitialization", False) self.deleteAllMatrices = props.get("RunModel.deleteAllMatrices", False) self.skipCopyWarmupTripTables = props.get("RunModel.skipCopyWarmupTripTables", False) @@ -401,7 +398,6 @@ def save_properties(self): props["RunModel.skipMGRASkims"] = self.skipMGRASkims props["RunModel.skip4Ds"] = self.skip4Ds props["RunModel.skipBuildNetwork"] = self.skipBuildNetwork - props["RunModel.skipInputChecker"] = self.skipInputChecker props["RunModel.skipInitialization"] = self.skipInitialization props["RunModel.deleteAllMatrices"] = self.deleteAllMatrices props["RunModel.skipCopyWarmupTripTables"] = self.skipCopyWarmupTripTables diff --git a/src/main/resources/sandag_abm.properties b/src/main/resources/sandag_abm.properties index c5146be8b..3cb3ba577 100644 --- a/src/main/resources/sandag_abm.properties +++ b/src/main/resources/sandag_abm.properties @@ -29,7 +29,6 @@ RunModel.useLocalDrive = true RunModel.skipInitialization = false RunModel.deleteAllMatrices = false RunModel.skip4Ds = false -RunModel.skipInputChecker = true RunModel.skipCopyWarmupTripTables = false RunModel.skipShadowPricing = false RunModel.skipBikeLogsums = true