Skip to content

Commit

Permalink
Update snakefile with new file locations, dp file
Browse files Browse the repository at this point in the history
- Moved all input data to `input_data` folder removing intermediate
  folder structure
- Created `working_directory` to contain all processed files
- Only final results are placed in `results` directory
- Use shell from snakemake to create LP file using glpsol and remove
  equivalent Python script
- Added parameters for Gurobi, which fixes some of KTH-dESA#23
- Added as input to rule which converts datapackage list
  of all dp files. Closes KTH-dESA#10
  • Loading branch information
willu47 committed Mar 18, 2022
1 parent 01e5aec commit 84481ab
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 57 deletions.
31 changes: 0 additions & 31 deletions gen_lp.py

This file was deleted.

58 changes: 32 additions & 26 deletions snakefile
Original file line number Diff line number Diff line change
@@ -1,17 +1,22 @@
import os
import pandas as pd

scenario_path = os.path.join("input_data", "diagnostic_scen")
SCENARIOS = [x.path for x in os.scandir(scenario_path) if x.is_dir()]
dp_files = pd.read_csv('config/dp_files.txt')

scenario_path = os.path.join("input_data")
SCENARIOS = [x.name for x in os.scandir(scenario_path) if x.is_dir()]

rule all:
input:
expand("results/{scen}/{scen}.xlsx", scen=SCENARIOS) #for testing, to be changed during development
expand("results/{scen}.xlsx", scen=SCENARIOS) #for testing, to be changed during development

rule convert_dp:
message: "Coverting datapackage for {wildcards.scen}"
input:
dp_path = "{scen}/datapackage.json"
other = expand("input_data/{{scen}}/{files}", files=dp_files),
dp_path = "input_data/{scen}/datapackage.json"
output:
df_path = "{scen}/{scen}.txt"
df_path = "working_directory/{scen}.txt"
conda:
"envs/otoole_env.yaml"
shell:
Expand All @@ -21,55 +26,56 @@ rule pre_process:
input:
"working_directory/{scen}.txt"
output:
temporary("working_directory/{scen}.pre")
temp("working_directory/{scen}.pre")
conda:
"envs/otoole_env.yaml"
shell:
"python pre_process.py otoole {input} {output}"

rule build_lp:
input:
df_path = "{scen}/{scen}.txt"
df_path = "working_directory/{scen}.pre"
params:
model_path = "model/osemosys.txt",
lp_path = "{scen}/{scen}.lp"
model_path = "model/osemosys.txt"
output:
lp_complete_path = "{scen}/lp_done.txt"
temp("working_directory/{scen}.lp")
log:
"working_directory/{scen}.log"
shell:
"python gen_lp.py {input.df_path} {params.lp_path}"
"glpsol -m {params.model_path} -d {input.df_path} --wlp {output} --check > {log}"

rule run_model:
message: "Solving the LP for '{input}'"
input:
lp_complete_path = "{scen}/lp_done.txt"
params:
lp_path = "{scen}/{scen}.lp",
path = "results/{scen}/{scen}"
"working_directory/{scen}.lp",
output:
done_path = "results/{scen}/{scen}-sol_done.txt"
temp("working_directory/{scen}.sol")
conda:
"envs/gurobi_env.yaml"
shell:
"python run.py {params.lp_path} {params.path}"
log:
"working_directory/gurobi/{scen}.log",
threads: 2
script:
"run.py"

rule convert_sol:
input:
sol_complete_path = "results/{scen}/{scen}-sol_done.txt",
dp_path = "{scen}/datapackage.json"
sol_path = "working_directory/{scen}.sol",
dp_path = "input_data/{scen}/datapackage.json"
params:
sol_path = "results/{scen}/{scen}.sol",
res_folder = "results/{scen}/results_csv"
output:
res_path = "results/{scen}/res-csv_done.txt"
conda:
"envs/otoole_env.yaml"
shell:
"python convert.py {params.sol_path} {params.res_folder} {input.dp_path}"
"python convert.py {input.sol_path} {params.res_folder} {input.dp_path}"

rule create_configs:
input:
config_tmpl = "config.yaml"
output:
config_scen = "config_{scen}.yaml"
config_scen = "working_directory/config_{scen}.yaml"
conda:
"envs/yaml_env.yaml"
shell:
Expand All @@ -78,12 +84,12 @@ rule create_configs:
rule res_to_iamc:
input:
res_path = "results/{scen}/res-csv_done.txt",
config_file = "config_{scen}.yaml"
config_file = "working_directory/config_{scen}.yaml"
params:
inputs_folder = "{scen}/data",
inputs_folder = "input_data/{scen}/data",
res_folder = "results/{scen}/results_csv"
output:
output_file = "results/{scen}/{scen}.xlsx"
output_file = "results/{scen}.xlsx"
conda:
"envs/openentrance_env.yaml"
shell:
Expand Down

0 comments on commit 84481ab

Please sign in to comment.