Merge pull request #203 from ktehranchi/master

Update Environment Versioning. New Demand Module.
PyPSA · Feb 28, 2024 · 02afa3c · 02afa3c
2 parents 946bb7b + f09e309
commit 02afa3c
Show file tree

Hide file tree

Showing 22 changed files with 380 additions and 389 deletions.
diff --git a/docs/source/configtables/clustering.csv b/docs/source/configtables/clustering.csv
@@ -2,10 +2,10 @@
 simplify_network:,,,
 to_substations,bool,"{true, false}",Implementation curerntly overrides to true. Network is simplified to substation nodes with positive or negative power injection.
 algorithm,str,{'kmeans'},
-feature,str," {'solar+onwind-time', 'solar+onwind-cap', 'solar-time', 'solar-cap', 'solar+offwind-cap'}",For HAC clustering. Currenntly unused in pypsa-usa.
+feature,str," {'solar+onwind-time', 'solar+onwind-cap', 'solar-time', 'solar-cap', 'solar+offwind-cap'}",For HAC clustering.
 cluster_network:,,,
 algorithm,str,{'kmeans'},
-feature,str," {'solar+onwind-time', 'solar+onwind-cap', 'solar-time', 'solar-cap', 'solar+offwind-cap'}",For HAC clustering. Currenntly unused in pypsa-usa.
-aggregation_zones,str,"{'balancing_area', 'state', 'country'}",Boundaries of GIS shapes that are to be respected in clustering. Retain if you would like to analyze expansion within a given zone.
+feature,str," {'solar+onwind-time', 'solar+onwind-cap', 'solar-time', 'solar-cap', 'solar+offwind-cap'}",For HAC clustering.
+aggregation_zones,str,"{'balancing_area', 'state'}",Boundaries of GIS shapes that are to be respected in clustering. Retain if you would like to analyze expansion within a given zone.
 aggregation_strategies:,,,
 table --> {key},str,"{'mean','max','min',etc}","Specifiy the method of aggregating fields within the generators, buses tables. "
diff --git a/docs/source/configtables/lines.csv b/docs/source/configtables/lines.csv
@@ -1,13 +1,13 @@
-,Unit,Values,Description
-types,--,"Values should specify a `line type in PyPSA <https://pypsa.readthedocs.io/en/latest/components.html#line-types>`_. Keys should specify the corresponding voltage level (e.g. 220., 300. and 380. kV)","Specifies line types to assume for the different voltage levels of the ENTSO-E grid extraction. Should normally handle voltage levels 220, 300, and 380 kV"
-s_max_pu,--,"Value in [0.,1.]","Correction factor for line capacities (`s_nom`) to approximate :math:`N-1` security and reserve capacity for reactive power flows"
-s_nom_max,MW,"float","Global upper limit for the maximum capacity of each extendable line."
-max_extension,MW,"float","Upper limit for the extended capacity of each extendable line."
-length_factor,--,float,"Correction factor to account for the fact that buses are *not* connected by lines through air-line distance."
-under_construction,--,"One of {'zero': set capacity to zero, 'remove': remove completely, 'keep': keep with full capacity}","Specifies how to handle lines which are currently under construction."
-dynamic_line_rating,,,
--- activate,bool,"true or false","Whether to take dynamic line rating into account"
--- cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
--- correction_factor,--,"float","Factor to compensate for overestimation of wind speeds in hourly averaged wind data"
--- max_voltage_difference,deg,"float","Maximum voltage angle difference in degrees or 'false' to disable"
--- max_line_rating,--,"float","Maximum line rating relative to nominal capacity without DLR, e.g. 1.3 or 'false' to disable"
+,Unit,Values,Description
+types,--,"Values should specify a `line type in PyPSA <https://pypsa.readthedocs.io/en/latest/components.html#line-types>`_. Keys should specify the corresponding voltage level (e.g. 220., 300. and 380. kV)",Specifies line types to assume for the different voltage levels of the TAMU Network.
+s_max_pu,--,"Value in [0.,1.]",Correction factor for line capacities (`s_nom`) to approximate :math:`N-1` security and reserve capacity for reactive power flows
+s_nom_max,MW,float,Global upper limit for the maximum capacity of each extendable line.
+max_extension,MW,float,Upper limit for the extended capacity of each extendable line.
+length_factor,--,float,Correction factor to account for the fact that buses are *not* connected by lines through air-line distance.
+under_construction,--,"One of {'zero': set capacity to zero, 'remove': remove completely, 'keep': keep with full capacity}",Specifies how to handle lines which are currently under construction.
+dynamic_line_rating,,,
+#NAME?,bool,true or false,Whether to take dynamic line rating into account
+#NAME?,--,Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.,Specifies the directory where the relevant weather data is stored.
+#NAME?,--,float,Factor to compensate for overestimation of wind speeds in hourly averaged wind data
+#NAME?,deg,float,Maximum voltage angle difference in degrees or 'false' to disable
+#NAME?,--,float,"Maximum line rating relative to nominal capacity without DLR, e.g. 1.3 or 'false' to disable"
diff --git a/docs/source/configtables/opts.csv b/docs/source/configtables/opts.csv
@@ -1,12 +1,12 @@
-Trigger, Description, Definition, Status
-``nH``; i.e. ``2H``-``6H``, Resample the time-resolution by averaging over every ``n`` snapshots, ``prepare_network``: `average_every_nhours() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L110>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L146>`__), In active use
-``nSEG``; e.g. ``4380SEG``, "Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load.", ``prepare_network``: apply_time_segmentation(), In active use
-``Co2L``, Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L19>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L154>`__, In active use
-``Ep``, Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well), ``prepare_network``: `add_emission_prices() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L24>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L158>`__, In active use
-``CCL``, Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``., ``solve_network``, In active use
-``EQ``, "Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption.", ``solve_network``, In active use
-``ATK``, "Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links.", ``prepare_network``, In active use
-``BAU``, Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``, ``solve_network``: `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L66>`__, Untested
-``SAFE``, Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network., ``solve_network`` `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L73>`__, Untested
-``carrier+{c|p|m}factor``,"Alter the capital cost (``c``), installable potential (``p``) or marginal costs (``m``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use
-``CH4L``,"Add an overall absolute gas limit. If configured in ``electricity: gaslimit`` it is given in MWh thermal, if a float is appended, the overall gaslimit is assumed to be given in TWh thermal (e.g. ``CH4L200`` limits gas dispatch to 200 TWh termal)", ``prepare_network``: ``add_gaslimit()``, In active use
+Trigger, Description, Definition, Status,
+``nH``; i.e. ``2H``-``6H``, Resample the time-resolution by averaging over every ``n`` snapshots, ``prepare_network``: `average_every_nhours() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L110>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L146>`__), In active use,
+``nSEG``; e.g. ``4380SEG``," ""Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables"," hydro inflow and load.""", ``prepare_network``: apply_time_segmentation(), In active use
+``Co2L``, Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L19>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L154>`__, In active use,
+``Ep``, Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well), ``prepare_network``: `add_emission_prices() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L24>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L158>`__, In active use,
+``CCL``, Add minimum and maximum levels of generator nominal capacity per carrier for individual regions. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``., ``solve_network``, In active use,
+``EQ``," ""Require each region or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each region to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption.""", ``solve_network``, In active use,
+``BAU``, Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in a given Interconnect; configured in ``electricity: BAU_mincapacities``, ``solve_network``: `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L66>`__, Untested,
+``SAFE``, Add a capacity reserve margin (a.k.a Planning Reserve Margin) of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network constraints., ``solve_network`` `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L73>`__, Untested,
+``carrier+{c|p|m}factor``,"Alter the capital cost (``c``), installable potential (``p``) or marginal costs (``m``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use,
+``CH4L``,"Add an overall absolute gas limit. If configured in ``electricity: gaslimit`` it is given in MWh thermal, if a float is appended, the overall gaslimit is assumed to be given in TWh thermal (e.g. ``CH4L200`` limits gas dispatch to 200 TWh termal)", ``prepare_network``: ``add_gaslimit()``, In active use,
+``ATK``," ""Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links.""", ``prepare_network``, In active use,
diff --git a/workflow/Snakefile b/workflow/Snakefile
@@ -22,7 +22,6 @@ FIGURES_SINGLE = [
     "production_bar",
     "production_area",
     "emissions_area",
-    "emissions_accumulated",
     "emissions_accumulated_tech",
     "emissions_map",
     "renewable_potential_map",
@@ -38,7 +37,6 @@ FIGURES_VALIDATE = [
 FIGURES_SINGLE_HTML = [
     "production_area_html",
     "emissions_area_html",
-    # "emissions_node_html",
     "emissions_region_html",
     "emissions_accumulated_tech_html",
 ]
@@ -66,8 +64,8 @@ wildcard_constraints:
 
 # Merge subworkflow configs and main config
 # configfile: "config/tests/config.validation.yaml"
-# configfile: "config/tests/config.test_simple.yaml"
-configfile: "config/config.default.yaml"
+configfile: "config/tests/config.test_simple.yaml"
+# configfile: "config/config.default.yaml"
 configfile: "config/config.cluster.yaml"
 configfile: "config/config.osw.yaml"
 configfile: "config/config.plotting.yaml"

diff --git a/workflow/config/config.cluster.yaml b/workflow/config/config.cluster.yaml
@@ -5,8 +5,8 @@ __default__:
   walltime: 00:30:00   # time limit for each job
   cpus_per_task: 1   # number of cores per job
   chdir: $GROUP_HOME/kamran/pypsa-usa/workflow
-  output: logs/slurm/{rule}-%j.out
-  error: logs/slurm/err_{rule}-%j.err
+  output: logs/{rule}/log-%j.out
+  error: logs/{rule}/errlog-%j.err
 
 build_renewable_profiles:
   walltime: 02:00:00

diff --git a/workflow/config/config.default.yaml b/workflow/config/config.default.yaml
@@ -17,7 +17,7 @@ scenario:
   scope: "total" # "urban", "rural", or "total"
   sector: "" # G
   planning_horizons:
-  - 2030   #  # Uncomment to use NREL EFS Demand Forecasts (2030, 2040, 2050)
+  - 2030   #(2030, 2040, 2050)
 
 foresight:  # Only Single Stage Currently
 
@@ -73,6 +73,7 @@ electricity:
   co2base: 226.86e+6 #base_from_2020 Locations of the 250 MMmt of CO2 emissions from the WECC 2021.
   gaslimit: false # global gas usage limit of X MWh_th
   retirement: economic # "economic" or "technical"
+  SAFE_reservemargin: 0.15
 
   operational_reserve:
     activate: false
@@ -90,7 +91,7 @@ electricity:
     Store: [] #[H2]
     Link: [] #[H2 pipeline]
 
-  demand: #EFS used for given planning_horizons year
+  demand: #EFS used for given planning_horizons year (only ref/mod implemented)
     EFS_case: reference # reference, medium, high
     EFS_speed: moderate # slow, moderate, rapid
 
@@ -259,19 +260,14 @@ clustering:
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time
-    aggregation_zones: 'balancing_area' # [balancing_area, state]
+    aggregation_zones: 'state' # [balancing_area, state]
+    exclude_carriers: []
+    consider_efficiency_classes: false
   aggregation_strategies:
     generators:
-      p_nom_max: sum # use "min" for more conservative assumptions
-      p_nom_min: sum
-      p_min_pu: mean
-      marginal_cost: mean
       committable: any
-      ramp_limit_up: mean
-      ramp_limit_down: mean
-      efficiency: mean
-    buses:
-      state: max # temp fix. When fixing state aggregation- change add electricity such that region info not is use is removed.
+      ramp_limit_up: max
+      ramp_limit_down: max
 
 focus_weights:
   # California: 0.5

diff --git a/workflow/config/tests/config.test.yaml b/workflow/config/tests/config.test.yaml
@@ -12,13 +12,13 @@ run:
 # docs :
 scenario:
   interconnect: western #"usa|texas|western|eastern"
-  clusters: [30, 100]
-  opts: [Co2L1.0-4H, Co2L0.75-4H, Co2L0.5-4H, Co2L0.25-4H, Co2L0.0-4H]
-  ll: [vopt, v1.15]
+  clusters: [40]
+  opts: [Co2L0.30-4H-Ep-SAFE]
+  ll: [v1.05]
   scope: "total" # "urban", "rural", or "total"
   sector: "" # G
   planning_horizons:
-  - 2030   #  # Uncomment to use NREL EFS Demand Forecasts (2030, 2040, 2050)
+  - 2030   #(2030, 2040, 2050)
 
 foresight:  # Only Single Stage Currently
 
@@ -76,9 +76,10 @@ electricity:
   co2base: 226.86e+6 #base_from_2020 Locations of the 250 MMmt of CO2 emissions from the WECC 2021.
   gaslimit: false # global gas usage limit of X MWh_th
   retirement: economic # "economic" or "technical"
+  SAFE_reservemargin: 0.15
 
   operational_reserve:
-    activate: true
+    activate: false
     epsilon_load: 0.02
     epsilon_vres: 0.02
     contingency: 4000
@@ -242,7 +243,7 @@ costs:  # based on the potentials, assuming  (0.1 kW/m2 and 10 m2/person)
     battery: 0.
     battery inverter: 0.
   emission_prices: # in currency per tonne emission, only used with the option Ep
-    co2: 0.
+    co2: 20.0
 
 # docs :
 sector:
@@ -263,19 +264,14 @@ clustering:
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time
-    aggregation_zones: 'balancing_area' # [balancing_area, state]
+    aggregation_zones: 'state' # [balancing_area, state]
+    exclude_carriers: []
+    consider_efficiency_classes: false
   aggregation_strategies:
     generators:
-      p_nom_max: sum # use "min" for more conservative assumptions
-      p_nom_min: sum
-      p_min_pu: mean
-      marginal_cost: mean
       committable: any
-      ramp_limit_up: mean
-      ramp_limit_down: mean
-      efficiency: mean
-    buses:
-      state: max # temp fix. When fixing state aggregation- change add electricity such that region info not is use is removed.
+      ramp_limit_up: max
+      ramp_limit_down: max
 
 focus_weights:
   # California: 0.5

diff --git a/workflow/config/tests/config.test_simple.yaml b/workflow/config/tests/config.test_simple.yaml
@@ -12,12 +12,12 @@ run:
 scenario:
   interconnect: [western] #"usa|texas|western|eastern"
   clusters: [40]
-  opts: [Co2L0.1, Co2L0.2]
+  opts: [Co2L0.30-3H-Ep-SAFE]
   ll: [v1.0]
   scope: "total" # "urban", "rural", or "total"
   sector: "" # G
   planning_horizons:
-  - 2030   #  # Uncomment to use NREL EFS Demand Forecasts (2030, 2040, 2050)
+  - 2030   #(2030, 2040, 2050)
 
 foresight:  # Only Single Stage Currently
 
@@ -29,7 +29,7 @@ countries: [US]
 
 snapshots:
   start: "2019-01-01"
-  end: "2020-01-01"
+  end: "2019-02-01"
   inclusive: 'left'
 
 # docs :
@@ -72,6 +72,7 @@ electricity:
   co2base: 225.0e+6 #base_from_2020 Locations of the 250 MMmt of CO2 emissions from the WECC 2021.
   gaslimit: false # global gas usage limit of X MWh_th
   retirement: economic # "economic" or "technical"
+  SAFE_reservemargin: 0.15
 
   operational_reserve:
     activate: false
@@ -254,24 +255,19 @@ sector:
 clustering:
   simplify_network:
     to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections)
-    algorithm: kmeans # choose from: [hac, kmeans]
+    algorithm: hac # choose from: [hac, kmeans]
     feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc.
   cluster_network:
-    algorithm: kmeans
+    algorithm: hac
     feature: solar+onwind-time
-    aggregation_zones: 'balancing_area' # [balancing_area, state]
+    aggregation_zones: 'state' # [balancing_area, state]
+    exclude_carriers: []
+    consider_efficiency_classes: false
   aggregation_strategies:
     generators:
-      p_nom_max: sum # use "min" for more conservative assumptions
-      p_nom_min: sum
-      p_min_pu: mean
-      marginal_cost: mean
       committable: any
-      ramp_limit_up: mean
-      ramp_limit_down: mean
-      efficiency: mean
-    buses:
-      state: max # temp fix. When fixing state aggregation- change add electricity such that region info not is use is removed.
+      ramp_limit_up: max
+      ramp_limit_down: max
 
 focus_weights:
   # California: 0.5