From 878d7cfedf1c900998ca7fcefdb0e26435ca88f9 Mon Sep 17 00:00:00 2001 From: aayad Date: Thu, 2 Jul 2026 12:54:02 -0600 Subject: [PATCH 1/4] refactor reading the system_availability_data files once per scenario, year --- .../simulation_structs/investor_creator.jl | 42 ++++++++++++++++++- .../simulation_structs/project_creator.jl | 7 ++-- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/struct_creators/simulation_structs/investor_creator.jl b/src/struct_creators/simulation_structs/investor_creator.jl index 770bd2e..1633edf 100644 --- a/src/struct_creators/simulation_structs/investor_creator.jl +++ b/src/struct_creators/simulation_structs/investor_creator.jl @@ -7,6 +7,10 @@ function create_investors(simulation_data::AgentSimulationData) dir_name = joinpath(get_data_dir(get_case(simulation_data)), "investors") investor_names = readdir(dir_name) investors = Vector{Investor}(undef, length(investor_names)) + + system_availability_dict = Dict{String, DataFrames.DataFrame}() + system_availability_rt_dict = Dict{String, DataFrames.DataFrame}() + for i = 1:length(investor_names) investor_dir = joinpath(dir_name, "$(investor_names[i])") @@ -145,7 +149,43 @@ function create_investors(simulation_data::AgentSimulationData) for sim_year in collect(1:horizon) println(scenario) println(sim_year) - add_investor_project_availability!(simulation_data_dir, scenario, sim_year, projects, sys_UC) + @info "Adding availability data for investor $(investor_names[i]) for scenario $(scenario) and simulation year $(sim_year)" + sys_name = "$(sim_year)_$(scenario)" + + if !haskey(system_availability_dict, sys_name) + system_availability_dict[sys_name] = DataFrames.DataFrame( + CSV.File( + joinpath( + simulation_data_dir, + "timeseries_data_files", + scenario, + "sim_year_$(sim_year)", + "Availability", + "DAY_AHEAD_availability.csv", + ), + )) + system_availability_rt_dict[sys_name] = DataFrames.DataFrame( + CSV.File( + joinpath( + simulation_data_dir, + "timeseries_data_files", + scenario, + "sim_year_$(sim_year)", + "Availability", + "REAL_TIME_availability.csv", + ), + )) + end + + add_investor_project_availability!( + simulation_data_dir, + scenario, + sim_year, + projects, + sys_UC, + system_availability_dict[sys_name], + system_availability_rt_dict[sys_name], + ) end end diff --git a/src/struct_creators/simulation_structs/project_creator.jl b/src/struct_creators/simulation_structs/project_creator.jl index 2d04733..1aa6504 100644 --- a/src/struct_creators/simulation_structs/project_creator.jl +++ b/src/struct_creators/simulation_structs/project_creator.jl @@ -78,9 +78,10 @@ function add_investor_project_availability!(simulation_dir::String, scenario::String, sim_year::Int64, projects::Vector{Project}, - sys_UC::Union{Nothing, PSY.System}) - system_availability_data = DataFrames.DataFrame(CSV.File(joinpath(simulation_dir, "timeseries_data_files", scenario, "sim_year_$(sim_year)", "Availability", "DAY_AHEAD_availability.csv"))) - system_availability_data_rt = DataFrames.DataFrame(CSV.File(joinpath(simulation_dir, "timeseries_data_files", scenario, "sim_year_$(sim_year)", "Availability", "REAL_TIME_availability.csv"))) + sys_UC::Union{Nothing, PSY.System}, + system_availability_data::DataFrames.DataFrame, + system_availability_data_rt::DataFrames.DataFrame) + gennames = names(system_availability_data)[5:length(names(system_availability_data))] ################# for project in projects From c86a022fae72aee11c8c9a16162f18c5d4ba163c Mon Sep 17 00:00:00 2001 From: aayad Date: Thu, 2 Jul 2026 12:56:28 -0600 Subject: [PATCH 2/4] Revert "refactor reading the system_availability_data files once per scenario, year" This reverts commit 878d7cfedf1c900998ca7fcefdb0e26435ca88f9. --- .../simulation_structs/investor_creator.jl | 42 +------------------ .../simulation_structs/project_creator.jl | 7 ++-- 2 files changed, 4 insertions(+), 45 deletions(-) diff --git a/src/struct_creators/simulation_structs/investor_creator.jl b/src/struct_creators/simulation_structs/investor_creator.jl index 1633edf..770bd2e 100644 --- a/src/struct_creators/simulation_structs/investor_creator.jl +++ b/src/struct_creators/simulation_structs/investor_creator.jl @@ -7,10 +7,6 @@ function create_investors(simulation_data::AgentSimulationData) dir_name = joinpath(get_data_dir(get_case(simulation_data)), "investors") investor_names = readdir(dir_name) investors = Vector{Investor}(undef, length(investor_names)) - - system_availability_dict = Dict{String, DataFrames.DataFrame}() - system_availability_rt_dict = Dict{String, DataFrames.DataFrame}() - for i = 1:length(investor_names) investor_dir = joinpath(dir_name, "$(investor_names[i])") @@ -149,43 +145,7 @@ function create_investors(simulation_data::AgentSimulationData) for sim_year in collect(1:horizon) println(scenario) println(sim_year) - @info "Adding availability data for investor $(investor_names[i]) for scenario $(scenario) and simulation year $(sim_year)" - sys_name = "$(sim_year)_$(scenario)" - - if !haskey(system_availability_dict, sys_name) - system_availability_dict[sys_name] = DataFrames.DataFrame( - CSV.File( - joinpath( - simulation_data_dir, - "timeseries_data_files", - scenario, - "sim_year_$(sim_year)", - "Availability", - "DAY_AHEAD_availability.csv", - ), - )) - system_availability_rt_dict[sys_name] = DataFrames.DataFrame( - CSV.File( - joinpath( - simulation_data_dir, - "timeseries_data_files", - scenario, - "sim_year_$(sim_year)", - "Availability", - "REAL_TIME_availability.csv", - ), - )) - end - - add_investor_project_availability!( - simulation_data_dir, - scenario, - sim_year, - projects, - sys_UC, - system_availability_dict[sys_name], - system_availability_rt_dict[sys_name], - ) + add_investor_project_availability!(simulation_data_dir, scenario, sim_year, projects, sys_UC) end end diff --git a/src/struct_creators/simulation_structs/project_creator.jl b/src/struct_creators/simulation_structs/project_creator.jl index 1aa6504..2d04733 100644 --- a/src/struct_creators/simulation_structs/project_creator.jl +++ b/src/struct_creators/simulation_structs/project_creator.jl @@ -78,10 +78,9 @@ function add_investor_project_availability!(simulation_dir::String, scenario::String, sim_year::Int64, projects::Vector{Project}, - sys_UC::Union{Nothing, PSY.System}, - system_availability_data::DataFrames.DataFrame, - system_availability_data_rt::DataFrames.DataFrame) - + sys_UC::Union{Nothing, PSY.System}) + system_availability_data = DataFrames.DataFrame(CSV.File(joinpath(simulation_dir, "timeseries_data_files", scenario, "sim_year_$(sim_year)", "Availability", "DAY_AHEAD_availability.csv"))) + system_availability_data_rt = DataFrames.DataFrame(CSV.File(joinpath(simulation_dir, "timeseries_data_files", scenario, "sim_year_$(sim_year)", "Availability", "REAL_TIME_availability.csv"))) gennames = names(system_availability_data)[5:length(names(system_availability_data))] ################# for project in projects From b9a64989d74f7a4be4c351748cc79c9919f2238e Mon Sep 17 00:00:00 2001 From: aayad Date: Thu, 2 Jul 2026 13:22:25 -0600 Subject: [PATCH 3/4] refactor reading files once per investor --- .../simulation_structs/investor_creator.jl | 42 +++++++++++++++++-- .../simulation_structs/project_creator.jl | 7 ++-- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/struct_creators/simulation_structs/investor_creator.jl b/src/struct_creators/simulation_structs/investor_creator.jl index 770bd2e..3ebf473 100644 --- a/src/struct_creators/simulation_structs/investor_creator.jl +++ b/src/struct_creators/simulation_structs/investor_creator.jl @@ -7,6 +7,11 @@ function create_investors(simulation_data::AgentSimulationData) dir_name = joinpath(get_data_dir(get_case(simulation_data)), "investors") investor_names = readdir(dir_name) investors = Vector{Investor}(undef, length(investor_names)) + + # Build once and use for all investors + system_availability_dict = Dict{String, DataFrames.DataFrame}() + system_availability_rt_dict = Dict{String, DataFrames.DataFrame}() + for i = 1:length(investor_names) investor_dir = joinpath(dir_name, "$(investor_names[i])") @@ -143,9 +148,40 @@ function create_investors(simulation_data::AgentSimulationData) for scenario in scenario_names for sim_year in collect(1:horizon) - println(scenario) - println(sim_year) - add_investor_project_availability!(simulation_data_dir, scenario, sim_year, projects, sys_UC) + @info "Adding availability data for investor $(investor_names[i]) for scenario $(scenario) and simulation year $(sim_year)" + sys_name = "$(sim_year)_$(scenario)" + if !haskey(system_availability_dict, sys_name) + system_availability_dict[sys_name] = DataFrames.DataFrame( + CSV.File( + joinpath( + simulation_data_dir, + "timeseries_data_dir", + scenario, + "sim_year_$(sim_year)", + "Availability", + "DAY_AHEAD_availability.csv", + ), + )) + system_availability_rt_dict[sys_name] = DataFrames.DataFrame( + CSV.File( + joinpath( + simulation_data_dir, + "timeseries_data_dir", + scenario, + "sim_year_$(sim_year)", + "Availability", + "REAL_TIME_availability.csv", + ), + )) + end + add_investor_project_availability!( + simulation_data_dir, + scenario, + sim_year, + projects, + sys_UC, + system_availability_dict[sys_name], + system_availability_rt_dict[sys_name]) end end diff --git a/src/struct_creators/simulation_structs/project_creator.jl b/src/struct_creators/simulation_structs/project_creator.jl index 2d04733..1aa6504 100644 --- a/src/struct_creators/simulation_structs/project_creator.jl +++ b/src/struct_creators/simulation_structs/project_creator.jl @@ -78,9 +78,10 @@ function add_investor_project_availability!(simulation_dir::String, scenario::String, sim_year::Int64, projects::Vector{Project}, - sys_UC::Union{Nothing, PSY.System}) - system_availability_data = DataFrames.DataFrame(CSV.File(joinpath(simulation_dir, "timeseries_data_files", scenario, "sim_year_$(sim_year)", "Availability", "DAY_AHEAD_availability.csv"))) - system_availability_data_rt = DataFrames.DataFrame(CSV.File(joinpath(simulation_dir, "timeseries_data_files", scenario, "sim_year_$(sim_year)", "Availability", "REAL_TIME_availability.csv"))) + sys_UC::Union{Nothing, PSY.System}, + system_availability_data::DataFrames.DataFrame, + system_availability_data_rt::DataFrames.DataFrame) + gennames = names(system_availability_data)[5:length(names(system_availability_data))] ################# for project in projects From 8829b9cb5cd2b29af2b3fb20e51c6413dc81fae5 Mon Sep 17 00:00:00 2001 From: aayad Date: Thu, 2 Jul 2026 13:28:24 -0600 Subject: [PATCH 4/4] fix typo --- src/struct_creators/simulation_structs/investor_creator.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/struct_creators/simulation_structs/investor_creator.jl b/src/struct_creators/simulation_structs/investor_creator.jl index 3ebf473..b44f787 100644 --- a/src/struct_creators/simulation_structs/investor_creator.jl +++ b/src/struct_creators/simulation_structs/investor_creator.jl @@ -155,7 +155,7 @@ function create_investors(simulation_data::AgentSimulationData) CSV.File( joinpath( simulation_data_dir, - "timeseries_data_dir", + "timeseries_data_files", scenario, "sim_year_$(sim_year)", "Availability", @@ -166,7 +166,7 @@ function create_investors(simulation_data::AgentSimulationData) CSV.File( joinpath( simulation_data_dir, - "timeseries_data_dir", + "timeseries_data_files", scenario, "sim_year_$(sim_year)", "Availability",