Merge branch 'develop' into fill_zone_forecast_zeros

Author: theocharides

.gitignore

@@ -16,6 +16,8 @@ venv
 ual_baus_install.sh
 configs/hlcm_renter_no_unplaced.yaml
 configs/hlcm_owner_no_unplaced.yaml
+configs/hlcm_renter_lowincome_no_unplaced.yaml
+configs/hlcm_owner_lowincome_no_unplaced.yaml
 data/2015_06_01_zoning_parcels.csv
 data/2015_08_13_parcels_geography.csv
 data/2015_08_13_zoning_parcels.csv
@@ -49,3 +51,4 @@ data/2020_06_15_zoning_lookup_hybrid_pba50.csv
 data/2020_06_15_zoning_parcels_hybrid_pba50.csv
 data/2020_07_10_parcels_geography.csv
 .DS_Store
+data/2020_09_21_parcels_geography.csv

baus.py

@@ -24,7 +24,7 @@
 
 SLACK = MAPS = "URBANSIM_SLACK" in os.environ
 LOGS = True
-RANDOM_SEED = False
+RANDOM_SEED = True
 INTERACT = False
 SCENARIO = None
 MODE = "simulation"
@@ -129,7 +129,7 @@
     host = socket.gethostname()
 
 if MAPS:
-    from baus.utils import ue_config, ue_files 
+    from baus.utils import ue_config, ue_files
 
 
 def get_simulation_models(SCENARIO):
@@ -194,6 +194,15 @@ def get_simulation_models(SCENARIO):
         # allocate renters to vacant rental units
         "hlcm_renter_simulate",
 
+        # we first put Q1/Q2/Q3 in market-rate units only, then allow Q1 into
+        # either deed-restricted or market-rate units
+        # this leaves deed-restricted units and the remaining market-rate
+        # units for Q1, whereas placing Q1 first could leave deed-restricted
+        # units vacant-- since deed-restricted units are not explicitly
+        # tied to price, there is not a greater probability Q1 will choose them
+        "hlcm_owner_lowincome_simulate",
+        "hlcm_renter_lowincome_simulate",
+
         # we have to run the hlcm above before this one - we first want to
         # try and put unplaced households into their appropraite tenured
         # units and then when that fails, force them to place using the
@@ -203,10 +212,14 @@ def get_simulation_models(SCENARIO):
 
         # force placement of any unplaced households, in terms of rent/own
         # is a noop except in the final simulation year
+        # 09 11 2020 ET: enabled for all simulation years
         "hlcm_owner_simulate_no_unplaced",
+        "hlcm_owner_lowincome_simulate_no_unplaced",
         # this one crashes right no because there are no unplaced, so
         # need to fix the crash in urbansim
+        # 09 11 2020 ET: appears to be working
         "hlcm_renter_simulate_no_unplaced",
+        "hlcm_renter_lowincome_simulate_no_unplaced",
 
         # update building/unit/hh correspondence
         "reconcile_placed_households",
@@ -225,7 +238,7 @@ def get_simulation_models(SCENARIO):
         "travel_model_output",
         # "travel_model_2_output",
         "hazards_slr_summary",
-        "hazards_eq_summary", 
+        "hazards_eq_summary",
         "slack_report"
 
     ]
@@ -313,13 +326,40 @@ def run_models(MODE, SCENARIO):
                 "hlcm_owner_simulate",
                 # allocate renters to vacant rental units
                 "hlcm_renter_simulate",
+
+                # we first put Q1/Q2/Q3 in market-rate units only, then
+                # allow Q1 into either deed-restricted or market-rate units
+                # this leaves deed-restricted units and the remaining
+                # market-rate units for Q1, whereas placing Q1 first could
+                # leave deed-restricted units vacant-- since deed-restricted
+                # units are not explicitly tied to price, there is not a
+                # greater probability Q1 will choose them
+                "hlcm_owner_lowincome_simulate",
+                "hlcm_renter_lowincome_simulate",
+
+                # we have to run the hlcm above before this one - we first want
+                # to try and put unplaced households into their appropraite
+                # tenured units and then when that fails, force them to place
+                # using the code below.
+
+                # force placement of any unplaced households, in terms of
+                # rent/own, is a noop except in the final simulation year
+                # 09 11 2020 ET: enabled for all simulation years
+                "hlcm_owner_simulate_no_unplaced",
+                "hlcm_owner_lowincome_simulate_no_unplaced",
+                # this one crashes right no because there are no unplaced, so
+                # need to fix the crash in urbansim
+                # 09 11 2020 ET: appears to be working
+                "hlcm_renter_simulate_no_unplaced",
+                "hlcm_renter_lowincome_simulate_no_unplaced",
+
                 # update building/unit/hh correspondence
                 "reconcile_placed_households",
 
                 "elcm_simulate",
 
                 "price_vars",
-#                "scheduled_development_events",
+                # "scheduled_development_events",
 
                 "topsheet",
                 "simulation_validation",
@@ -490,7 +530,9 @@ def run_models(MODE, SCENARIO):
             as_user=True)
     else:
         slack.chat.post_message(
-            '#urbansim_sim_update', "No differences with reference run.", as_user=True)
+            '#urbansim_sim_update',
+            "No differences with reference run.",
+            as_user=True)
 
 if S3:
     os.system('ls runs/run%d_* ' % run_num +

baus/datasources.py

@@ -182,6 +182,11 @@ def hlcm_owner_lowincome_config():
     return get_config_file('hlcm_owner_lowincome')
 
 
+@orca.injectable(cache=True)
+def hlcm_owner_lowincome_no_unplaced_config():
+    return get_config_file('hlcm_owner_lowincome_no_unplaced')
+
+
 @orca.injectable(cache=True)
 def hlcm_renter_config():
     return get_config_file('hlcm_renter')
@@ -197,6 +202,11 @@ def hlcm_renter_lowincome_config():
     return get_config_file('hlcm_renter_lowincome')
 
 
+@orca.injectable(cache=True)
+def hlcm_renter_lowincome_no_unplaced_config():
+    return get_config_file('hlcm_renter_lowincome_no_unplaced')
+
+
 @orca.injectable(cache=True)
 def rsh_config():
     fname = get_config_file('rsh')
@@ -445,7 +455,9 @@ def add_drop_helper(col, val):
     add_drop_helper("add_bldg", 1)
     add_drop_helper("drop_bldg", 0)
 
-    if 'pba50zoningmodcat' in scenario_zoning.columns:
+    if scenario in policy['geographies_fb_enable']:
+        join_col = 'fbpzoningm'
+    elif scenario in policy['geographies_db_enable']:
         join_col = 'pba50zoningmodcat'
     elif 'zoninghzcat' in scenario_zoning.columns:
         join_col = 'zoninghzcat'
@@ -490,9 +502,9 @@ def parcel_rejections():
 
 
 @orca.table(cache=True)
-def parcels_geography(parcels, scenario, settings):
+def parcels_geography(parcels, scenario, settings, policy):
     df = pd.read_csv(
-        os.path.join(misc.data_dir(), "2020_07_10_parcels_geography.csv"),
+        os.path.join(misc.data_dir(), "2020_09_21_parcels_geography.csv"),
         index_col="geom_id")
     df = geom_id_to_parcel_id(df, parcels)
 
@@ -517,13 +529,23 @@ def parcels_geography(parcels, scenario, settings):
     df["pda_id_pba40"] = df.pda_id_pba40.replace("dan1", np.nan)
 
     # Add Draft Blueprint geographies: PDA, TRA, PPA, sesit
-    df["pda_id_pba50"] = df.pda_id_pba50.str.lower()
-    df["tra_id"] = df.tra_id.str.lower()
-    df['juris_tra'] = df.juris + '-' + df.tra_id
-    df["ppa_id"] = df.ppa_id.str.lower()
-    df['juris_ppa'] = df.juris + '-' + df.ppa_id
-    df["sesit_id"] = df.sesit_id.str.lower()
-    df['juris_sesit'] = df.juris + '-' + df.sesit_id
+    if scenario in policy['geographies_db_enable']:
+        df["pda_id_pba50"] = df.pda_id_pba50.str.lower()
+        df["tra_id"] = df.tra_id.str.lower()
+        df['juris_tra'] = df.juris + '-' + df.tra_id
+        df["ppa_id"] = df.ppa_id.str.lower()
+        df['juris_ppa'] = df.juris + '-' + df.ppa_id
+        df["sesit_id"] = df.sesit_id.str.lower()
+        df['juris_sesit'] = df.juris + '-' + df.sesit_id
+    # Use Final Blueprint geographies: PDA, TRA, PPA, sesit
+    elif scenario in policy['geographies_fb_enable']:
+        df["pda_id_pba50"] = df.pda_id_pba50_fb.str.lower()
+        df["tra_id"] = df.fbp_tra_id.str.lower()
+        df['juris_tra'] = df.juris + '-' + df.tra_id
+        df["ppa_id"] = df.fbp_ppa_id.str.lower()
+        df['juris_ppa'] = df.juris + '-' + df.ppa_id
+        df["sesit_id"] = df.fbp_sesit_id.str.lower()
+        df['juris_sesit'] = df.juris + '-' + df.sesit_id
 
     return df
 
@@ -656,7 +678,7 @@ def get_dev_projects_table(scenario, parcels):
     # requires the user has MTC's urban_data_internal
     # repository alongside bayarea_urbansim
     urban_data_repo = ("../urban_data_internal/development_projects/")
-    current_dev_proj = ("2020_0731_1607_development_projects.csv")
+    current_dev_proj = ("2020_0914_1529_development_projects.csv")
     orca.add_injectable("dev_proj_file", current_dev_proj)
     df = pd.read_csv(os.path.join(urban_data_repo, current_dev_proj))
     df = reprocess_dev_projects(df)

baus/summaries.py

@@ -326,6 +326,8 @@ def policy_activated(policy_loc, policy_nm, scenario):
             if key != "jobs_housing_com_for_res_scenarios":
                 counter += 1
         write("Jobs-housing fees are activated for %d counties" % counter)
+    else:
+        write("Jobs-housing fees are not activated")
     write("")
 
     # affordable housing bonds
@@ -350,12 +352,13 @@ def policy_activated(policy_loc, policy_nm, scenario):
             amount = float(policy_loc["total_amount_db"])
         elif scenario in policy_loc["alternate_amount_scenarios_db"]:
             amount = float(policy_loc["alternate_total_amount_db"])
-        else:
+        elif scenario in (policy["acct_settings"]["lump_sum_accounts"]
+                          [county+"_bond_settings"]["enable_in_scenarios"]):
             amount = float(policy_loc["total_amount"])
-        # sum annual ammount over the simulation period
-        regional_funding += amount*5*7
-    write("Total funding for deed-restricted housing is $%d"
-          % regional_funding)
+        # sum annual amount over the simulation period
+        if 'amount' in locals():
+            regional_funding += amount*5*7
+    write("Total funding is $%d" % regional_funding)
 
     f.close()
 
@@ -407,8 +410,9 @@ def topsheet(households, jobs, buildings, parcels, zones, year,
         # round to nearest 100s
         hhincome_by_intrich = (hhincome_by_intrich/100).round()*100
 
-    # Summaries for Draft Blueprint geographies
-    if scenario in policy["geographies_db_enable"]:
+    # Summaries for Draft/Final Blueprint geographies
+    if scenario in policy["geographies_db_enable"] or \
+            scenario in policy["geographies_fb_enable"]:
         hh_by_inpda_pba50 = households_df.pda_id_pba50.notnull().value_counts()
 
         hhincome_by_inpda_pba50 = households_df.income.groupby(
@@ -449,7 +453,8 @@ def topsheet(households, jobs, buildings, parcels, zones, year,
     if scenario in policy["geographies_fr2_enable"]:
         jobs_by_intrich = jobs_df.trich_id.notnull().value_counts()
 
-    if scenario in policy["geographies_db_enable"]:
+    if scenario in policy["geographies_db_enable"] or \
+            scenario in policy["geographies_fb_enable"]:
         jobs_by_inpda_pba50 = jobs_df.pda_pba50.notnull().value_counts()
         jobs_by_intra = jobs_df.tra_id.notnull().value_counts()
 
@@ -478,7 +483,8 @@ def topsheet(households, jobs, buildings, parcels, zones, year,
                 "hhincome_by_intrich": hhincome_by_intrich,
                 "capacity": capacity
             })
-        if scenario in policy["geographies_db_enable"]:
+        if scenario in policy["geographies_db_enable"] or \
+                scenario in policy["geographies_fb_enable"]:
             orca.add_injectable("base_year_measures", {
                 "hh_by_subregion": hh_by_subregion,
                 "jobs_by_subregion": jobs_by_subregion,
@@ -562,6 +568,16 @@ def norm_and_round(s):
         write("Draft Blueprint year mean income by whether household\
               is in hra/dr:\n%s" % hhincome_by_insesit)
 
+    if scenario in policy["geographies_fb_enable"]:
+        write("Base year mean income by whether household is in tra:\n%s" %
+              base_year_measures["hhincome_by_intra"])
+        write("Final Blueprint year mean income by whether household\
+              is in tra:\n%s" % hhincome_by_intra)
+        write("Base year mean income by whether household is in hra/dr:\n%s" %
+              base_year_measures["hhincome_by_insesit"])
+        write("Final Blueprint year mean income by whether household\
+              is in hra/dr:\n%s" % hhincome_by_insesit)
+
     jsp = buildings.job_spaces.sum()
     write("Number of job spaces = %d" % jsp)
     write("Non-residential vacancy rate = %.2f" % (1-0 - float(nj)/jsp))
@@ -658,8 +674,9 @@ def norm_and_round(s):
         write("Jobs pct of regional growth in trichs:\n%s" %
               norm_and_round(diff))
 
-    # write Draft Blueprint additional summaries: pda, tra, sesit(hra/dr)
-    if scenario in policy["geographies_db_enable"]:
+    # write Draft/Final Blueprint additional summaries: pda, tra, sesit(hra/dr)
+    if scenario in policy["geographies_db_enable"] or \
+            scenario in policy["geographies_fb_enable"]:
         tmp = base_year_measures["hh_by_inpda_pba50"]
         write("Households base year share in pdas:\n%s" %
               norm_and_round(tmp))
@@ -952,10 +969,12 @@ def geographic_summary(parcels, households, jobs, buildings, taz_geography,
        (scenario in policy["geographies_fr2_enable"]):
         geographies.append('juris_trich')
 
-    # append Draft Blueprint strategy geographis
-    if scenario in policy["geographies_db_enable"]:
-        geographies.extend(['pda_pba50', 'juris_tra',
-                            'juris_sesit', 'juris_ppa'])
+# disable final blueprint summaries being handled in post processing summaries
+#    # append Draft/Final Blueprint strategy geographis
+#    if scenario in policy["geographies_db_enable"] or \
+#            scenario in policy["geographies_fb_enable"]:
+#        geographies.extend(['pda_pba50', 'juris_tra',
+#                            'juris_sesit', 'juris_ppa'])
 
     if year in [2010, 2015, 2020, 2025, 2030, 2035, 2040, 2045, 2050]:
 
@@ -1168,10 +1187,11 @@ def building_summary(parcels, run_number, year,
     df = orca.merge_tables(
         'buildings',
         [parcels, buildings],
-        columns=['performance_zone', 'year_built', 'residential_units',
-                 'unit_price', 'zone_id', 'non_residential_sqft',
-                 'deed_restricted_units', 'job_spaces', 'x', 'y', 'geom_id',
-                 'source'])
+        columns=['performance_zone', 'year_built', 'building_type',
+                 'residential_units', 'unit_price', 'zone_id', 
+                 'non_residential_sqft', 'vacant_res_units', 
+                 'deed_restricted_units', 'job_spaces', 
+                 'x', 'y', 'geom_id', 'source'])
 
     df.to_csv(
         os.path.join("runs", "run%d_building_data_%d.csv" %
@@ -1191,7 +1211,6 @@ def parcel_summary(parcels, buildings, households, jobs,
     df = parcels.to_frame([
         "geom_id",
         "x", "y",
-        "total_residential_units",
         "total_job_spaces",
         "first_building_type"
     ])
@@ -1225,6 +1244,15 @@ def parcel_summary(parcels, buildings, households, jobs,
             parcel_id.value_counts()
     df["tothh"] = households_df.groupby('parcel_id').size()
 
+    building_df = orca.merge_tables(
+        'buildings',
+        [parcels, buildings],
+        columns=['parcel_id', 'residential_units', 'deed_restricted_units'])
+    df['residential_units'] = \
+        building_df.groupby('parcel_id')['residential_units'].sum()
+    df['deed_restricted_units'] = \
+        building_df.groupby('parcel_id')['deed_restricted_units'].sum()
+
     jobs_df = orca.merge_tables(
         'jobs',
         [buildings, jobs],