Feasibility Staleness hotfix

baus-env-2023.yml

@@ -20,4 +20,6 @@ dependencies:
   - urbansim=3.2
   - urbansim_defaults=0.2
   - pyyaml<6.0
-  - slack_sdk
+  - slack_sdk
+  - asana=3.2.1  # not tested on later versions
+  - holidays

baus.py

@@ -8,6 +8,7 @@
     datasources, variables, models, subsidies, ual, slr, earthquake, 
     utils, preprocessing)
 from baus.tests import validation
+
 from baus.summaries import \
     core_summaries, geographic_summaries, affordable_housing_summaries, \
     hazards_summaries, metrics, travel_model_summaries
@@ -29,6 +30,8 @@
 EVERY_NTH_YEAR = 5
 IN_YEAR, OUT_YEAR = 2010, 2050
 
+CURRENT_BRANCH = os.popen('git rev-parse --abbrev-ref HEAD').read().rstrip()
+CURRENT_COMMIT = os.popen('git rev-parse HEAD').read().rstrip()
 
 SLACK = "URBANSIM_SLACK" in os.environ
 if SLACK:
@@ -38,9 +41,6 @@
     client = WebClient(token=os.environ["SLACK_TOKEN"])
     slack_channel = "#urbansim_sim_update"
 
-SET_RANDOM_SEED = True
-if SET_RANDOM_SEED:
-    np.random.seed(42)
 
 
 parser = argparse.ArgumentParser(description='Run UrbanSim models.')
@@ -49,6 +49,8 @@
 parser.add_argument('-i', action='store_true', dest='interactive', help='enter interactive mode after imports')
 parser.add_argument('--set-random-seed', action='store_true', dest='set_random_seed', help='set a random seed for consistent stochastic output')
 parser.add_argument('--disable-slack', action='store_true', dest='no_slack', help='disable slack outputs')
+parser.add_argument('--enable-asana', action='store_true', dest='use_asana', default=False, help='disable Asana task creation')
+
 
 options = parser.parse_args()
 
@@ -60,10 +62,24 @@
 
 if options.set_random_seed:
     SET_RANDOM_SEED = True
+    np.random.seed(42)
+else:
+    SET_RANDOM_SEED = False
 
 if options.no_slack:
     SLACK = False
 
+if options.use_asana:
+    ASANA = True
+
+    from scripts.meta.asana_utils import (
+    create_asana_task_from_yaml,
+    add_comment_to_task, 
+    mark_task_as_complete)
+    ASANA_SECTION_NAME = 'Final Blueprint Runs'
+else:
+    ASANA = False
+
 orca.add_injectable("years_per_iter", EVERY_NTH_YEAR)
 orca.add_injectable("base_year", IN_YEAR)
 orca.add_injectable("final_year", OUT_YEAR)
@@ -73,7 +89,6 @@
 outputs_dir = pathlib.Path(orca.get_injectable("outputs_dir"))
 outputs_dir.mkdir(parents=True, exist_ok=True)
 
-
 def run_models(MODE):
 
     if MODE == "estimation":
@@ -239,8 +254,13 @@ def get_simulation_models():
 
                 "residential_developer",
                 "developer_reprocess",
+
+                "alt_feasibility",
+
                 "retail_developer",
 
+                "alt_feasibility",
+
                 "office_developer",
                 "subsidized_office_developer_vmt",
 
@@ -425,14 +445,10 @@ def get_simulation_visualization_models():
 print('***The Standard stream is being written to {}.log***'.format(run_name))
 sys.stdout = sys.stderr = open(os.path.join(orca.get_injectable("outputs_dir"), "%s.log") % run_name, 'w')
 
-# Memorialize the run config with the outputs - goes by run name attribute
-
-print('***Copying run_setup.yaml to output directory')
-shutil.copyfile("run_setup.yaml", os.path.join(orca.get_injectable("outputs_dir"), f'run_setup_{run_name}.yaml'))
 
 print("Started", time.ctime())
-print("Current Branch : ", os.popen('git rev-parse --abbrev-ref HEAD').read().rstrip())
-print("Current Commit : ", os.popen('git rev-parse HEAD').read().rstrip())
+print("Current Branch : ", CURRENT_BRANCH)
+print("Current Commit : ",  CURRENT_COMMIT)
 print("Set Random Seed : ", SET_RANDOM_SEED)
 print("python version: %s" % sys.version.split('|')[0])
 print("urbansim version: %s" % urbansim.__version__)
@@ -446,6 +462,21 @@ def get_simulation_visualization_models():
 print("SLACK: {}".format(SLACK))
 print("MODE: {}".format(MODE))
 
+if ASANA:
+    # We can do this before the shutil copy step and just use the native run_setup.yaml in the same dir as baus.py
+    task_handle = create_asana_task_from_yaml('run_setup.yaml', run_name, ASANA_SECTION_NAME)
+
+    # Get task identifer for later comment posting 
+    task_gid = task_handle['gid']
+
+    print(f"Creating asana run task with URL: {task_handle['permalink_url']}")
+
+# Memorialize the run config with the outputs - goes by run name attribute
+
+print('***Copying run_setup.yaml to output directory')
+shutil.copyfile("run_setup.yaml", os.path.join(orca.get_injectable("outputs_dir"), f'run_setup_{run_name}.yaml'))
+
+
 if SLACK and MODE == "estimation":
     slack_start_message = f'Starting estimation {run_name} on host {host}'
     try:
@@ -464,6 +495,14 @@ def get_simulation_visualization_models():
         # For first slack channel posting of a run, catch any auth errors
         init_response = client.chat_postMessage(channel=slack_channel,
                                            text=slack_start_message)
+
+        if ASANA:
+
+            asana_msg = f"Creating asana run task with URL: {task_handle['permalink_url']}"
+            asana_response = client.chat_postMessage(channel=slack_channel,
+                                    thread_ts=init_response.data['ts'],
+                                    text=asana_msg)
+
     except SlackApiError as e:
         assert e.response["ok"] is False
         assert e.response["error"]  
@@ -496,6 +535,11 @@ def get_simulation_visualization_models():
                                            thread_ts=init_response.data['ts'],
                                            text=slack_fail_message)
 
+        if ASANA:
+            # Add a fail comment
+            add_comment_to_task(task_gid, slack_fail_message)
+
+
     else:
         raise e
     sys.exit(0)
@@ -506,5 +550,21 @@ def get_simulation_visualization_models():
                                        thread_ts=init_response.data['ts'],
                                        text=slack_completion_message)
 
+
+    if ASANA:
+        # Add a comment
+        add_comment_to_task(task_gid, "Simulation completed successfully.")
+
+        # Mark the task as completed
+        mark_task_as_complete(task_gid)
+
+        response = client.chat_postMessage(channel=slack_channel,
+                                       thread_ts=init_response.data['ts'],
+                                       text='Check asana for details.')
+
+
+
+
+
 
 print("Finished", time.ctime())         

baus/summaries/core_summaries.py

@@ -101,7 +101,7 @@ def new_buildings_summary(run_name, parcels, parcels_zoning_calculations, buildi
 
     df = df[['parcel_id', 'building_type', 'building_sqft', 'deed_restricted_units', 'year_built',
              'preserved_units', 'inclusionary_units', 'subsidized_units',
-             'non_residential_sqft', 'residential_price', 'residential_units', 'source',	
+             'non_residential_sqft','total_non_residential_sqft', 'residential_price', 'residential_units', 'source',	
              'vacant_residential_units', 'vacant_job_spaces', 'vacant_res_units', 'price_per_sqft',	'unit_price',	
              'land_value',	'acres', 'x', 'y', 'parcel_acres', 'total_residential_units',	'total_job_spaces',	
              'zoned_du', 'zoned_du_underbuild', 'zoned_du_build_ratio', 'zoned_far', 'zoned_far_underbuild', 

baus/variables.py

@@ -459,6 +459,17 @@ def stories(buildings):
     return buildings.stories.groupby(buildings.parcel_id).max()
 
 
+
+# get whether a parcel contains a building from the developer model
+@orca.column('parcels', cache=True)
+def bldg_source(buildings):
+    return buildings.source.groupby(buildings.parcel_id).first()
+
+@orca.column('parcels')
+def first_building_source(buildings):
+    df = buildings.to_frame(columns=['source', 'parcel_id'])
+    return df.groupby('parcel_id').source.first()
+
 @orca.column('parcels', cache=True)
 def height(parcels):
     return parcels.stories * 12  # hard coded 12 feet per story

configs/developer/developer_settings.yaml

@@ -11,7 +11,8 @@ profit_vs_return_on_cost_combination_factor: 0.5
 # optionally we don't look at small lot single family
 # pass_through are columns not used by the developer but which go in the debug output
 feasibility:
-  parcel_filter: (nodev != 1 and manual_nodev != 1 and sdem != 1 and oldest_building > 1906
+  parcel_filter: (nodev != 1 and manual_nodev != 1 and first_building_source != 'developer_model' 
+    and sdem != 1 and oldest_building > 1906
     and oldest_building_age > 20 and (total_residential_units != 1 or parcel_acres > 1.0)
     and first_building_type != 'HO' and first_building_type != 'SC')
   residential_to_yearly: True

scripts/meta/asana_caller.py

@@ -0,0 +1,61 @@
+import argparse
+import logging
+import pathlib
+from asana_utils import create_asana_task_from_yaml
+from logging_config import setup_logging
+
+
+def main():
+
+
+    # Set up logging configuration
+    setup_logging()
+
+    # Set up argument parsing
+    parser = argparse.ArgumentParser(description='Create an Asana task from a YAML file.')
+
+    # Adding arguments with defaults
+    parser.add_argument(
+        '-p','--yaml_path', 
+        default='/Volumes/Data/Models/urban_modeling/baus/PBA50Plus/PBA50Plus_FinalBlueprint/PBA50Plus_Final_Blueprint_v00/run_setup_PBA50Plus_Final_Blueprint_v00.yaml',
+        help='Path to the YAML file with simulation setup details'
+    )
+    # TODO: the use case is limited for having this set when we use the yaml - but it could still be useful
+    parser.add_argument(
+        '-t','--task_name', 
+        default='Example Task X',
+        help='Name of the Asana task to create'
+    )
+    parser.add_argument(
+        '-s','--section_name', 
+        default='Final Blueprint Runs',
+        help='Name of the section in the Asana project'
+    )
+
+    args = parser.parse_args()
+
+    # Set up logging
+    setup_logging()
+    logger = logging.getLogger(__name__)
+
+    # Log the received arguments
+    logger.info(f"YAML Path: {args.yaml_path}")
+    logger.info(f"Task Name: {args.task_name}")
+    logger.info(f"Section Name: {args.section_name}")
+
+    if not args.task_name:
+        # Grab run_name from the yaml directly
+        run_name = f"BAUS Run: {pathlib.Path(args.yaml_path).name.replace('run_setup_', '').split('.')[0]}"
+    else:
+        run_name = args.task_name
+
+    # Call the function to create the Asana task
+    try:
+        # currently just overriding the --task_name arg and placing run_name from the yaml directly
+        task_handle = create_asana_task_from_yaml(args.yaml_path, run_name, args.section_name)
+        logger.info(f"Task created successfully with ID: {task_handle['gid']}")
+    except Exception as e:
+        logger.error(f"Failed to create task: {e}")
+
+if __name__ == "__main__":
+    main()