Merge pull request #47 from RSGInc/ABM2_TRUNK

Merge SANDAG AB2+ enhancements changes

input/input_checker/config/inputs_checks.csv

@@ -1,5 +1,6 @@
 Test,Input_Table,Input_ID_Column,Severity,Type,Expression,Test_Vals,Report_Statistic,Test_Description
-scenarioYear,scenario,Year,NA,Calculation,scenario['Year'][0],,,Obtain scenario year
+scenarioYear,scenario,Year,NA,Calculation,scenario['Year'].iloc[0],,,Obtain scenario year
+zoneCount,scenario,zoneCount,NA,Calculation,scenario['zoneCount'].iloc[0],,,Obtain number of zones in scenario. Number of zones may change as a result of zone splitting.
 PERSON_CTRAMP_Fields_Availability,persons,perid,Fatal,Test,test_val in persons.columns,"hhid,perid,household_serial_no,pnum,age,sex,miltary,pemploy,pstudent,ptype,educ,grade,occen5,occsoc5,indcen,weeks,hours,rac1p,hisp,version",,Check if person file contains all the required fields for ABM
 PERSON_military_valueCheck_agg,persons,perid,Warning,Test,"set(persons.miltary)=={0,1}",,,"Check if Person data matches the pre-defined military categories [0 = N/A Less than 17 years old, 1 = Yes, Now on Active Duty]"
 PERSON_military_valueCheck_disagg,persons,perid,Fatal,Test,"persons.miltary.apply(lambda x: True if x in [0,1,2,3,4] else False)",,,"Check if Person data matches the pre-defined military categories [0 = N/A Less than 17 years old, 1 = Yes, Now on Active Duty, 2 = Yes, on Active Duty in Past, but not Now, 3 = No, Training for Reserves/National Guard Only, 4 = No, Never Served in the Military]"
@@ -47,7 +48,7 @@ LINK_POSITIVE_LENGTH,network_links,id,Logical,Test,network_links.length>0,,,All
 #TRANSITSEGMENT_Unique_Stop_ID_Check,transit_segments,id,Fatal,Test,len(set(transit_segments['@stop_id'])) == len(transit_segments['@stop_id']),,,All transit segment stop IDs must be unique
 mgra_total_employment,mgra_data,mgra,NA,Calculation,mgra_data[[col for col in mgra_data if (col.startswith('emp_')) and not (col.endswith('total'))]].sum(axis=1),,,
 MGRA_TotalEmployment_CalcCheck,mgra_data,mgra,Logical,Test,mgra_data.emp_total==mgra_total_employment,,,Check if employment by industry adds up to total employment
-ZONE_CHECK,centroids,id,Fatal,Test,len(centroids) == 4996,,len(centroids),Check that there are 4996 zones present in network
+ZONE_CHECK,centroids,id,Fatal,Test,len(centroids) == zoneCount,,len(centroids),Check that there are 4996 zones present in network
 zone_connector_table,centroids,id,NA,Calculation,centroids.loc[(centroids['numInLinks']>0) & (centroids['numOutLinks']>0)],,,Obtain table of centroids (i.e. zones) that have more than 1 incoming and outgoing links (i.e. centroid connectors)
 zone_connector_list,zone_connector_table,id,NA,Calculation,set(zone_connector_table.id),,,Obtain list of unique centroid ids that have more than 1 incoming and outgoing links
 NO_ZONE_CONNECTOR_CHECK,centroids,id,Fatal,Test,centroids.id.apply(lambda x: True if x in zone_connector_list else False),,,Check that each zone has a zone connector

src/main/emme/init_emme_project.py

@@ -28,35 +28,50 @@
 import argparse
 import os
 
+from collections import OrderedDict
+
 WKT_PROJECTION = 'PROJCS["NAD_1983_NSRS2007_StatePlane_California_VI_FIPS_0406_Ft_US",GEOGCS["GCS_NAD_1983_NSRS2007",DATUM["D_NAD_1983_NSRS2007",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Lambert_Conformal_Conic"],PARAMETER["False_Easting",6561666.666666666],PARAMETER["False_Northing",1640416.666666667],PARAMETER["Central_Meridian",-116.25],PARAMETER["Standard_Parallel_1",32.78333333333333],PARAMETER["Standard_Parallel_2",33.88333333333333],PARAMETER["Latitude_Of_Origin",32.16666666666666],UNIT["Foot_US",0.3048006096012192]];-118608900 -91259500 3048.00609601219;-100000 10000;-100000 10000;3.28083333333333E-03;0.001;0.001;IsHighPrecision'
 
 def init_emme_project(root, title, emmeversion):
     project_path = _app.create_project(root, "emme_project")
-    desktop = _app.start_dedicated(
-        project=project_path, user_initials="WS", visible=False)
+    project_root = os.path.dirname(project_path)
+    desktop = _app.start_dedicated(project=project_path, user_initials="WS", visible=False)
     project = desktop.project
     project.name = "SANDAG Emme project"
-    prj_file_path = os.path.join(os.path.dirname(project_path), 'NAD 1983 NSRS2007 StatePlane California VI FIPS 0406 (US Feet).prj')
+    prj_file_path = os.path.join(project_root, 'NAD 1983 NSRS2007 StatePlane California VI FIPS 0406 (US Feet).prj')
     with open(prj_file_path, 'w') as f:
         f.write(WKT_PROJECTION)
     project.spatial_reference_file = prj_file_path
     project.initial_view = _ws_types.Box(6.18187e+06, 1.75917e+06, 6.42519e+06, 1.89371e+06)
-    project_root = os.path.dirname(project_path)
+
+    property_path = os.path.join(root, "conf", "sandag_abm.properties")
+    properties = load_properties(property_path)
+    sla_limit = properties.get('traffic.sla_limit', 3)
+
+    num_links = 90000
+    num_turn_entries = 13000
+    num_traffic_classes = 15
+    if sla_limit > 3:
+        # extra_attribute_values = 18000000 + 90000 * (sla_limit - 3)
+        extra_attribute_values = 18000000 + (sla_limit - 3) * ((num_links + 1) * (num_traffic_classes + 1) + (num_turn_entries + 1) * (num_traffic_classes))
+    else:
+        extra_attribute_values = 18000000
+
     dimensions = {
         'scalar_matrices': 9999,
         'destination_matrices': 999,
         'origin_matrices': 999,
-        'full_matrices': 1600,
+        'full_matrices': 9999,
 
         'scenarios': 10,
         'centroids': 5000,
         'regular_nodes': 29999,
-        'links': 90000,
-        'turn_entries': 13000,
+        'links': num_links,
+        'turn_entries': num_turn_entries,
         'transit_vehicles': 200,
         'transit_lines': 450,
         'transit_segments': 40000,
-        'extra_attribute_values': 28000000,
+        'extra_attribute_values': extra_attribute_values,
 
         'functions': 99,
         'operators': 5000
@@ -84,6 +99,45 @@ def init_emme_project(root, title, emmeversion):
     desktop.add_modeller_toolbox("%<$ProjectPath>%/scripts/solutions.mtbx")
     project.save()
 
+def load_properties(path):
+    prop = OrderedDict()
+    comments = {}
+    with open(path, 'r') as properties:
+        comment = []
+        for line in properties:
+            line = line.strip()
+            if not line or line.startswith('#'):
+                comment.append(line)
+                continue
+            key, value = line.split('=')
+            key = key.strip()
+            tokens = value.split(',')
+            if len(tokens) > 1:
+                value = _parse_list(tokens)
+            else:
+                value = _parse(value)
+            prop[key] = value
+            comments[key], comment = comment, []
+    return prop
+
+def _parse_list(values):
+    converted_values = []
+    for v in values:
+        converted_values.append(_parse(v))
+    return converted_values
+
+def _parse(value):
+    value = str(value).strip()
+    if value == 'true':
+        return True
+    elif value == 'false':
+        return False
+    for caster in int, float:
+        try:
+            return caster(value)
+        except ValueError:
+            pass
+    return value
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Create a new empty Emme project and database with Sandag defaults.")

src/main/emme/toolbox/import/import_auto_demand.py

@@ -422,7 +422,7 @@ def import_commercial_vehicle_demand(self, props):
             path = os.path.join(self.output_dir, "TripMatrices.csv")
             table = _pandas.read_csv(path)
             for key, value in mapping.iteritems():
-                cvm_array = table[key].values.reshape((4996, 4996))     # reshape method deprecated since v 0.19.0, yma, 2/12/2019
+                cvm_array = table[key].values.reshape((props["zones.count"], props["zones.count"]))     # reshape method deprecated since v 0.19.0, yma, 2/12/2019
                 #factor in cvm demand by the scale factor used in trip generation
                 cvm_array = cvm_array/scale_factor
                 #scale trips to take care of underestimation
@@ -440,7 +440,7 @@ def import_commercial_vehicle_demand(self, props):
                         key_new = "CVM_%s:%sNT" % (period, veh)
                         value_new = mapping[key_new]
                         if value_new["share"] != 0.0:
-                            cvm_array = table[key_new].values.reshape((4996, 4996))
+                            cvm_array = table[key_new].values.reshape((props["zones.count"], props["zones.count"]))
                             cvm_array = cvm_array/scale_factor
                             cvm_array = cvm_array * value_new["scale"]
                             value["array"] = value["array"] + (cvm_array * value_new["share"])

src/main/emme/toolbox/import/input_checker.py

@@ -297,7 +297,7 @@ def get_emme_object(emme_network, emme_network_object, fields_to_export):
 					df = dbf.to_dataframe()
 					self.inputs[table_name] = df
 
-		# add scenario year
+		# add scenario table to input dictionary
 		self.inputs['scenario'] = self.scenario_df
 
 	def checks(self):
@@ -444,8 +444,9 @@ def prop_file_paths(self):
 			input_path = props[row['Property_Token']]
 			self.prop_input_paths[input_table] = input_path
 
-		# obtain scenario year
+		# obtain scenario year and number of zones
 		self.scenario_df['Year'] = [props['scenarioYear']]
+		self.scenario_df['zoneCount'] = [props['zones.count']]
 
 	def write_log(self):
 		# function to write out the input checker log file