feat: Split flex-context settings by commodity for dynamic capacity scheduling

documentation/features/scheduling.rst

@@ -5,8 +5,8 @@ Scheduling
 
 Scheduling is the main value-drive of FlexMeasures. We have two major types of schedulers built-in, for storage devices (usually batteries or hot water storage) and processes (usually in industry).
 
-FlexMeasures computes schedules for energy systems that consist of multiple devices that consume and/or produce electricity.
-We model a device as an asset with a power sensor, and compute schedules only for flexible devices, while taking into account inflexible devices.
+FlexMeasures computes schedules for energy systems that consist of multiple devices that consume and/or produce a commodity (e.g. electricity or gas).
+We model a device as an asset with a consumption/production sensor recording power values, and compute schedules only for flexible devices, while taking into account inflexible devices.
 
 .. contents::
     :local:
@@ -39,14 +39,15 @@ The flex-context
 
 The ``flex-context`` is independent of the type of flexible device that is optimized, or which scheduler is used.
 With the flexibility context, we aim to describe the system in which the flexible assets operate, such as its physical and contractual limitations.
+For multi-commodity scheduling problems, the flex-context can be defined separately per commodity (e.g. electricity and gas), using the ``commodities`` field.
 
 Fields can have fixed values, but some fields can also point to sensors, so they will always represent the dynamics of the asset's environment (as long as that sensor has current data).
 The full list of flex-context fields follows below.
 For more details on the possible formats for field values, see :ref:`variable_quantities`.
 
 Where should you set these fields?
 Within requests to the API or by editing the relevant asset in the UI.
-If they are not sent in via the API (one of the endpoints triggering schedule computation), the scheduler will look them up on the `flex-context` field of the asset.
+If they are not sent in via the API (one of the endpoints triggering schedule computation), the scheduler will look them up on the flex-context field of the asset.
 And if the asset belongs to a larger system (a hierarchy of assets), the scheduler will also search if parent assets have them set.
 
 
@@ -58,6 +59,9 @@ And if the asset belongs to a larger system (a hierarchy of assets), the schedul
    * - Field
      - Example value
      - Description
+   * - ``commodity``
+     - |COMMODITY_FLEX_CONTEXT.example|
+     - .. include:: ../_autodoc/COMMODITY_FLEX_CONTEXT.rst
    * - ``inflexible-device-sensors``
      - |INFLEXIBLE_DEVICE_SENSORS.example|
      - .. include:: ../_autodoc/INFLEXIBLE_DEVICE_SENSORS.rst
@@ -70,9 +74,6 @@ And if the asset belongs to a larger system (a hierarchy of assets), the schedul
    * - ``production-price``
      - |PRODUCTION_PRICE.example|
      - .. include:: ../_autodoc/PRODUCTION_PRICE.rst
-   * - ``gas-price``
-     - |GAS_PRICE.example|
-     - .. include:: ../_autodoc/GAS_PRICE.rst
    * - ``site-power-capacity``
      - |SITE_POWER_CAPACITY.example|
      - .. include:: ../_autodoc/SITE_POWER_CAPACITY.rst
@@ -187,8 +188,8 @@ For more details on the possible formats for field values, see :ref:`variable_qu
      - Example value
      - Description
    * - ``commodity``
-     - |COMMODITY.example|
-     - .. include:: ../_autodoc/COMMODITY.rst
+     - |COMMODITY_FLEX_MODEL.example|
+     - .. include:: ../_autodoc/COMMODITY_FLEX_MODEL.rst
    * - ``consumption``
      - |CONSUMPTION.example|
      - .. include:: ../_autodoc/CONSUMPTION.rst

flexmeasures/api/common/schemas/utils.py

@@ -32,7 +32,11 @@ def make_openapi_compatible(  # noqa: C901
             sensor_only_validators.append(validator[-1])
 
     new_fields = {}
-    tobeadded_fields = schema_cls._declared_fields
+    try:
+        # in case `schema_cls.__init__` reordered the fields, preserve their order
+        tobeadded_fields = schema_cls().fields
+    except TypeError:
+        tobeadded_fields = schema_cls._declared_fields
     if include:
         for item in include:
             tobeadded_fields.update(item)

flexmeasures/api/v3_0/assets.py

@@ -98,7 +98,7 @@ def __init__(self, *args, **kwargs):
 
     flex_context = fields.Nested(
         flex_context_schema_openAPI,
-        required=True,
+        load_default={},
         data_key="flex-context",
         metadata=dict(
             description="The flex-context is validated according to the scheduler's `FlexContextSchema`.",
@@ -107,11 +107,11 @@ def __init__(self, *args, **kwargs):
     flex_model = fields.List(
         fields.Nested(
             storage_flex_model_schema_openAPI(exclude=["asset"]),
-            required=True,
-            data_key="flex-model",
-            metadata=dict(
-                description="Flex-model per device (identified by `sensor`). The flex-model validation is handled by the scheduler. What follows is the schema used by the `StorageScheduler`.",
-            ),
+        ),
+        load_default=[],
+        data_key="flex-model",
+        metadata=dict(
+            description="Flex-model per device (identified by `sensor`). The flex-model validation is handled by the scheduler. What follows is the schema used by the `StorageScheduler`.",
         ),
     )
 

flexmeasures/data/models/planning/linear_optimization.py

@@ -35,13 +35,14 @@
 
 def device_scheduler(  # noqa C901
     device_constraints: list[pd.DataFrame],
-    ems_constraints: pd.DataFrame,
+    ems_constraints: pd.DataFrame | list[pd.DataFrame],
     commitment_quantities: list[pd.Series] | None = None,
     commitment_downwards_deviation_price: list[pd.Series] | list[float] | None = None,
     commitment_upwards_deviation_price: list[pd.Series] | list[float] | None = None,
     commitments: list[pd.DataFrame] | list[Commitment] | None = None,
     initial_stock: float | list[float] = 0,
     stock_groups: dict[int, list[int]] | None = None,
+    ems_constraint_groups: list[list[int]] | None = None,
 ) -> tuple[list[pd.Series], float, SolverResults, ConcreteModel]:
     """This generic device scheduler is able to handle an EMS with multiple devices,
     with various types of constraints on the EMS level and on the device level,
@@ -64,6 +65,13 @@ def device_scheduler(  # noqa C901
     :param ems_constraints:     EMS constraints are on an EMS level. Handled constraints (listed by column name):
                                     derivative max: maximum flow
                                     derivative min: minimum flow
+                                May be a single DataFrame (the constraint is applied to the summed flow of all devices),
+                                or a list of DataFrames (one per device group). In the latter case, ``ems_constraint_groups``
+                                lists the device indices each DataFrame applies to. The StorageScheduler uses one device
+                                group per commodity, so each commodity gets its own EMS-level capacity constraint.
+    :param ems_constraint_groups: For each EMS constraint DataFrame, the list of device indices it applies to. When omitted,
+                                each EMS constraint is applied to the summed flow of all devices (legacy behaviour). A device
+                                may appear in more than one group.
     :param commitments:         Commitments are on an EMS level by default. Handled parameters (listed by column name):
                                     quantity:                   for example, 5.5
                                     downwards deviation price:  10.1
@@ -101,7 +109,27 @@ def device_scheduler(  # noqa C901
     resolution = pd.to_timedelta(device_constraints[0].index.freq).to_pytimedelta()
     end = device_constraints[0].index.to_pydatetime()[-1] + resolution
 
-    # map device → stock group
+    # Normalise EMS constraints to a list of (DataFrame, device-group) pairs.
+    # A single DataFrame (legacy behaviour) applies to the summed flow of all devices;
+    # a list of DataFrames applies one EMS-level constraint per device group, as set up
+    # per commodity by the StorageScheduler.
+    all_devices = list(range(len(device_constraints)))
+    if isinstance(ems_constraints, pd.DataFrame):
+        ems_constraints_list = [ems_constraints]
+        ems_constraint_device_groups = [all_devices]
+    else:
+        ems_constraints_list = ems_constraints
+        if ems_constraint_groups is None:
+            if len(ems_constraints_list) > 1:
+                raise ValueError(
+                    "When passing multiple EMS constraint DataFrames, you must also specify ems_constraint_groups."
+                )
+            ems_constraint_device_groups = [all_devices for _ in ems_constraints_list]
+        else:
+            ems_constraint_device_groups = ems_constraint_groups
+
+    # map device -> primary stock group (used for per-device stock bounds)
+    # and map stock group -> all member devices (used for stock accumulation).
     device_to_group = {}
 
     if stock_groups:
@@ -389,15 +417,15 @@ def device_derivative_min_select(m, d, j):
         else:
             return np.nanmax([min_v, equal_v])
 
-    def ems_derivative_max_select(m, j):
-        v = ems_constraints["derivative max"].iloc[j]
+    def ems_derivative_max_select(m, g, j):
+        v = ems_constraints_list[g]["derivative max"].iloc[j]
         if np.isnan(v):
             return infinity
         else:
             return v
 
-    def ems_derivative_min_select(m, j):
-        v = ems_constraints["derivative min"].iloc[j]
+    def ems_derivative_min_select(m, g, j):
+        v = ems_constraints_list[g]["derivative min"].iloc[j]
         if np.isnan(v):
             return -infinity
         else:
@@ -483,8 +511,15 @@ def grouped_commitment_equalities(m, c, j, g):
     model.device_derivative_min = Param(
         model.d, model.j, initialize=device_derivative_min_select
     )
-    model.ems_derivative_max = Param(model.j, initialize=ems_derivative_max_select)
-    model.ems_derivative_min = Param(model.j, initialize=ems_derivative_min_select)
+    model.eg = RangeSet(
+        0, len(ems_constraints_list) - 1, doc="Set of EMS constraint (device) groups"
+    )
+    model.ems_derivative_max = Param(
+        model.eg, model.j, initialize=ems_derivative_max_select
+    )
+    model.ems_derivative_min = Param(
+        model.eg, model.j, initialize=ems_derivative_min_select
+    )
     model.device_efficiency = Param(model.d, model.j, initialize=device_efficiency)
     model.device_derivative_down_efficiency = Param(
         model.d, model.j, initialize=device_derivative_down_efficiency
@@ -628,8 +663,15 @@ def device_down_derivative_sign(m, d, j):
         """Derivative down if sign points down, derivative not down if sign points up."""
         return -m.device_power_down[d, j] <= Md * (1 - m.device_power_sign[d, j])
 
-    def ems_derivative_bounds(m, j):
-        return m.ems_derivative_min[j], sum(m.ems_power[:, j]), m.ems_derivative_max[j]
+    def ems_derivative_bounds(m, g, j):
+        devices = ems_constraint_device_groups[g]
+        if not devices:
+            return Constraint.Skip
+        return (
+            m.ems_derivative_min[g, j],
+            sum(m.ems_power[d, j] for d in devices),
+            m.ems_derivative_max[g, j],
+        )
 
     def commitment_up_derivative_sign(m, c):
         """Up deviation active only if sign points up."""
@@ -722,7 +764,7 @@ def device_derivative_equalities(m, d, j):
     model.device_power_down_sign = Constraint(
         model.d, model.j, rule=device_down_derivative_sign
     )
-    model.ems_power_bounds = Constraint(model.j, rule=ems_derivative_bounds)
+    model.ems_power_bounds = Constraint(model.eg, model.j, rule=ems_derivative_bounds)
     if not convex_cost_curve:
         model.commitment_up_derivative_sign_con = Constraint(
             model.c, rule=commitment_up_derivative_sign