Advance integration of MESSAGE.

Fix LCIs for electrolyzers.
Fix translation of outdated biosphere flows.
Update docs.

docs/extract.rst

@@ -577,7 +577,10 @@ The new datasets introduced are listed in the table below (only production datas
   hydrogen production, gaseous, 25 bar, from gasification of woody biomass in entrained flow gasifier, with CCS, at gasification plant    CH
   hydrogen production, gaseous, 25 bar, from gasification of woody biomass in entrained flow gasifier, at gasification plant              CH
   hydrogen production, gaseous, 30 bar, from hard coal gasification and reforming, at coal gasification plant                             RER
-  hydrogen production, gaseous, 200 bar, from PEM electrolysis, from grid electricity                                                     RER
+  hydrogen production, gaseous, 30 bar, from PEM electrolysis, from grid electricity                                                      RER
+  hydrogen production, gaseous, 20 bar, from AEC electrolysis, from grid electricity                                                      RER
+  hydrogen production, gaseous, 1 bar, from SOEC electrolysis, from grid electricity                                                      RER
+  hydrogen production, gaseous, 1 bar, from SOEC electrolysis, with steam input, from grid electricity                                    RER
   hydrogen production, gaseous, 25 bar, from thermochemical water splitting, at solar tower                                               RER
   hydrogen production, gaseous, 100 bar, from methane pyrolysis                                                                           RER
  ======================================================================================================================================= ===========

premise/activity_maps.py

@@ -42,6 +42,60 @@ def get_mapping(filepath: Path, var: str) -> dict:
     return mapping
 
 
+def act_fltr(
+    database: List[dict],
+    fltr: Union[str, List[str]] = None,
+    mask: Union[str, List[str]] = None,
+) -> List[dict]:
+    """Filter `database` for activities matching field contents given by `fltr` excluding strings in `mask`.
+    `fltr`: string, list of strings or dictionary.
+    If a string is provided, it is used to match the name field from the start (*startswith*).
+    If a list is provided, all strings in the lists are used and results are joined (*or*).
+    A dict can be given in the form <fieldname>: <str> to filter for <str> in <fieldname>.
+    `mask`: used in the same way as `fltr`, but filters add up with each other (*and*).
+    `filter_exact` and `mask_exact`: boolean, set `True` to only allow for exact matches.
+
+    :param database: A lice cycle inventory database
+    :type database: brightway2 database object
+    :param fltr: value(s) to filter with.
+    :type fltr: Union[str, lst, dict]
+    :param mask: value(s) to filter with.
+    :type mask: Union[str, lst, dict]
+    :return: list of activity data set names
+    :rtype: list
+
+    """
+    if fltr is None:
+        fltr = {}
+    if mask is None:
+        mask = {}
+
+    # default field is name
+    if isinstance(fltr, (list, str)):
+        fltr = {"name": fltr}
+    if isinstance(mask, (list, str)):
+        mask = {"name": mask}
+
+    assert len(fltr) > 0, "Filter dict must not be empty."
+
+    # find `act` in `database` that match `fltr`
+    # and do not match `mask`
+    filters = []
+    for field, value in fltr.items():
+        if isinstance(value, list):
+            filters.extend([ws.either(*[ws.contains(field, v) for v in value])])
+        else:
+            filters.append(ws.contains(field, value))
+
+    for field, value in mask.items():
+        if isinstance(value, list):
+            filters.extend([ws.exclude(ws.contains(field, v)) for v in value])
+        else:
+            filters.append(ws.exclude(ws.contains(field, value)))
+
+    return list(ws.get_many(database, *filters))
+
+
 class InventorySet:
     """
     Hosts different filter sets to find equivalencies
@@ -181,90 +235,19 @@ def generate_material_map(self) -> dict:
         """
         return self.generate_sets_from_filters(self.materials_filters)
 
-    @staticmethod
-    def act_fltr(
-        database: List[dict],
-        fltr: Union[str, List[str]] = None,
-        mask: Union[str, List[str]] = None,
-        filter_exact: bool = False,
-        mask_exact: bool = False,
-    ) -> List[dict]:
-        """Filter `database` for activities matching field contents given by `fltr` excluding strings in `mask`.
-        `fltr`: string, list of strings or dictionary.
-        If a string is provided, it is used to match the name field from the start (*startswith*).
-        If a list is provided, all strings in the lists are used and results are joined (*or*).
-        A dict can be given in the form <fieldname>: <str> to filter for <str> in <fieldname>.
-        `mask`: used in the same way as `fltr`, but filters add up with each other (*and*).
-        `filter_exact` and `mask_exact`: boolean, set `True` to only allow for exact matches.
-
-        :param database: A lice cycle inventory database
-        :type database: brightway2 database object
-        :param fltr: value(s) to filter with.
-        :type fltr: Union[str, lst, dict]
-        :param mask: value(s) to filter with.
-        :type mask: Union[str, lst, dict]
-        :param filter_exact: requires exact match when true.
-        :type filter_exact: bool
-        :param mask_exact: requires exact match when true.
-        :type mask_exact: bool
-        :return: list of activity data set names
-        :rtype: list
-
-        """
-        if fltr is None:
-            fltr = {}
-        if mask is None:
-            mask = {}
-        result = []
-
-        # default field is name
-        if isinstance(fltr, (list, str)):
-            fltr = {"name": fltr}
-        if isinstance(mask, (list, str)):
-            mask = {"name": mask}
-
-        def like(item_a, item_b):
-            if filter_exact:
-                return item_a.lower() == item_b.lower()
-            return item_a.lower().startswith(item_b.lower())
-
-        def notlike(item_a, item_b):
-            if mask_exact:
-                return item_a.lower() != item_b.lower()
-            return item_b.lower() not in item_a.lower()
-
-        assert len(fltr) > 0, "Filter dict must not be empty."
-
-        # find `act` in `database` that match `fltr`
-        # and do not match `mask`
-        filters = []
-        for field, value in fltr.items():
-            if isinstance(value, list):
-                filters.extend([ws.either(*[ws.contains(field, v) for v in value])])
-            else:
-                filters.append(ws.contains(field, value))
-
-        for field, value in mask.items():
-            if isinstance(value, list):
-                filters.extend([ws.exclude(ws.contains(field, v)) for v in value])
-            else:
-                filters.append(ws.exclude(ws.contains(field, value)))
-
-        return list(ws.get_many(database, *filters))
-
     def generate_sets_from_filters(self, filtr: dict, database=None) -> dict:
         """
         Generate a dictionary with sets of activity names for
         technologies from the filter specifications.
 
-            :param filtr:
-            :func:`activity_maps.InventorySet.act_fltr`.
+        :param filtr:
+        :func:`activity_maps.InventorySet.act_fltr`.
         :return: dictionary with the same keys as provided in filter
             and a set of activity data set names as values.
         :rtype: dict
         """
 
         database = database or self.database
 
-        techs = {tech: self.act_fltr(database, **fltr) for tech, fltr in filtr.items()}
+        techs = {tech: act_fltr(database, fltr.get("fltr"), fltr.get("mask")) for tech, fltr in filtr.items()}
         return {tech: {act["name"] for act in actlst} for tech, actlst in techs.items()}

premise/data/additional_inventories/migration_map.csv

@@ -340,6 +340,8 @@ from;to;name_from;ref_prod_from;location_from;name_to;ref_prod_to;location_to
 35;39;market for potassium fertiliser, as K2O;potassium fertiliser, as K2O;GLO;market group for inorganic potassium fertiliser, as K2O;inorganic potassium fertiliser, as K2O;RER
 36;39;market for potassium fertiliser, as K2O;potassium fertiliser, as K2O;GLO;market group for inorganic potassium fertiliser, as K2O;inorganic potassium fertiliser, as K2O;RER
 37;39;market for chemicals, inorganic;chemical, inorganic;GLO;market for chemical, inorganic;;
+39;37;market for chemical, inorganic;chemical, inorganic;GLO;market for chemicals, inorganic;;
+38;37;market for chemical, inorganic;chemical, inorganic;GLO;market for chemicals, inorganic;;
 35;39;heat pump production, for heat and power co-generation unit, 160kW electrical;heat pump, for heat and power co-generation unit, 160kW electrical;RER;heat pump production, heat and power co-generation unit, 160kW electrical;heat pump, heat and power co-generation unit, 160kW electrical;
 36;39;heat pump production, for heat and power co-generation unit, 160kW electrical;heat pump, for heat and power co-generation unit, 160kW electrical;RER;heat pump production, heat and power co-generation unit, 160kW electrical;heat pump, heat and power co-generation unit, 160kW electrical;
 37;39;heat pump production, for heat and power co-generation unit, 160kW electrical;heat pump, for heat and power co-generation unit, 160kW electrical;RER;heat pump production, heat and power co-generation unit, 160kW electrical;heat pump, heat and power co-generation unit, 160kW electrical;

premise/data/fuels/hydrogen_activities.yml

@@ -1,5 +1,5 @@
 from electrolysis:
-  name: hydrogen production, gaseous, 200 bar, from PEM electrolysis, from grid electricity
+  name: hydrogen production, gaseous, 30 bar, from PEM electrolysis, from grid electricity
   var: hydrogen, electrolysis
   feedstock name: electricity, low voltage
   feedstock unit: kilowatt hour

premise/data_collection.py

@@ -187,11 +187,13 @@ def get_vehicle_fleet_composition(model, vehicle_type) -> Union[xr.DataArray, No
         dataframe = pd.read_csv(
             FILEPATH_FLEET_COMP, sep=get_delimiter(filepath=FILEPATH_FLEET_COMP)
         )
-    else:
+    elif model == "image":
         dataframe = pd.read_csv(
             FILEPATH_IMAGE_TRUCKS_FLEET_COMP,
             sep=get_delimiter(filepath=FILEPATH_FLEET_COMP),
         )
+    else:
+        return None
 
     dataframe = dataframe.loc[~dataframe["region"].isnull()]
 
@@ -215,7 +217,7 @@ def get_vehicle_fleet_composition(model, vehicle_type) -> Union[xr.DataArray, No
     return None
 
 
-def fix_efficiencies(data: xr.DataArray) -> xr.DataArray:
+def fix_efficiencies(data: xr.DataArray, min_year: int) -> xr.DataArray:
     """
     Fix the efficiency data to ensure plausibility.
 
@@ -244,7 +246,7 @@ def fix_efficiencies(data: xr.DataArray) -> xr.DataArray:
     # ensure that efficiency can not decrease over time
     while data.diff(dim="year").min().values < 0:
         diff = data.diff(dim="year")
-        diff = xr.concat([data.sel(year=2005), diff], dim="year")
+        diff = xr.concat([data.sel(year=min_year), diff], dim="year")
         diff.values[diff.values > 0] = 0
         diff *= -1
         data += diff
@@ -301,6 +303,8 @@ def __init__(
         self.external_scenarios = external_scenarios
         self.system_model_args = system_model_args
         self.use_absolute_efficiency = use_absolute_efficiency
+        self.min_year = 2005
+        self.max_year = 2100
         key = key or None
 
         electricity_prod_vars = self.__get_iam_variable_labels(
@@ -488,7 +492,11 @@ def __init__(
             data=data,
         )
 
-        self.other_vars = self.__fetch_market_data(data=data, input_vars=other_vars)
+        self.other_vars = self.__fetch_market_data(
+            data=data,
+            input_vars=other_vars,
+            normalize=False,
+        )
 
         self.electricity_efficiencies = self.get_iam_efficiencies(
             data=data, efficiency_labels=electricity_eff_vars
@@ -661,20 +669,27 @@ def __get_iam_data(
         # remove any column that is a string
         # and that is not any of "Region", "Variable", "Unit"
         for col in dataframe.columns:
-            if isinstance(col, str) and col not in ["Region", "Variable", "Unit"]:
-                dataframe = dataframe.drop(col, axis=1)
+            if isinstance(col, str):
+                if col.lower() not in ["region", "variable", "unit"]:
+                    dataframe = dataframe.drop(col, axis=1)
+
+        # identify the lowest and highest column name that is numeric
+        # and consider it the minimum year
+        self.min_year = min([x for x in dataframe.columns if isinstance(x, int)])
+        self.max_year = max([x for x in dataframe.columns if isinstance(x, int)])
 
         dataframe = dataframe.reset_index()
 
         # remove "index" column
         if "index" in dataframe.columns:
             dataframe = dataframe.drop("index", axis=1)
 
-        dataframe = dataframe.loc[dataframe["Variable"].isin(variables)]
+        # convert all column names that are string to lower case
+        dataframe.columns = [x.lower() if isinstance(x, str) else x for x in dataframe.columns]
 
-        dataframe = dataframe.rename(
-            columns={"Region": "region", "Variable": "variables", "Unit": "unit"}
-        )
+        dataframe = dataframe.loc[dataframe["variable"].isin(variables)]
+
+        dataframe = dataframe.rename(columns={"variable": "variables"})
 
         array = (
             dataframe.melt(
@@ -690,7 +705,7 @@ def __get_iam_data(
         return array
 
     def __fetch_market_data(
-        self, data: xr.DataArray, input_vars: dict
+        self, data: xr.DataArray, input_vars: dict, normalize: bool = True
     ) -> [xr.DataArray, None]:
         """
         This method retrieves the market share for each technology,
@@ -716,23 +731,26 @@ def __fetch_market_data(
 
         if available_vars:
             market_data = data.loc[
-                :, [v for v in input_vars.values() if v in available_vars], :
+                :, available_vars, :
             ]
         else:
             return None
 
+        rev_input_vars = {v: k for k, v in input_vars.items()}
+
         market_data.coords["variables"] = [
-            k for k, v in input_vars.items() if v in available_vars
+            rev_input_vars[v] for v in market_data.variables.values
         ]
 
         if self.system_model == "consequential":
             market_data = consequential_method(
                 market_data, self.year, self.system_model_args
             )
         else:
-            market_data /= (
-                data.loc[:, available_vars, :].groupby("region").sum(dim="variables")
-            )
+            if normalize is True:
+                market_data /= (
+                    data.loc[:, available_vars, :].groupby("region").sum(dim="variables")
+                )
 
         # back-fill nans
         market_data = market_data.bfill(dim="year")
@@ -815,7 +833,7 @@ def get_iam_efficiencies(
         if not self.use_absolute_efficiency:
             eff_data /= eff_data.sel(year=2020)
             # fix efficiencies
-            eff_data = fix_efficiencies(eff_data)
+            eff_data = fix_efficiencies(eff_data, self.min_year)
         else:
             # if absolute efficiencies are used, we need to make sure that
             # the efficiency is not greater than 1
@@ -855,7 +873,7 @@ def __get_carbon_capture_rate(
 
         # if variable is missing, we assume that the rate is 0
         # and that none of the  CO2 emissions are captured
-        if dict_vars["cement - cco2"] not in data.variables.values.tolist():
+        if dict_vars.get("cement - cco2") not in data.variables.values.tolist():
             cement_rate = xr.DataArray(
                 np.zeros((len(data.region), len(data.year))),
                 coords=[data.region, data.year],
@@ -868,7 +886,7 @@ def __get_carbon_capture_rate(
 
         cement_rate.coords["variables"] = "cement"
 
-        if dict_vars["steel - cco2"] not in data.variables.values.tolist():
+        if dict_vars.get("steel - cco2") not in data.variables.values.tolist():
             steel_rate = xr.DataArray(
                 np.zeros((len(data.region), len(data.year))),
                 coords=[data.region, data.year],

premise/ecoinvent_modification.py

@@ -133,6 +133,7 @@
 FILEPATH_NUCLEAR_SMR = INVENTORY_DIR / "lci-nuclear_SMR.xlsx"
 FILEPATH_WAVE = INVENTORY_DIR / "lci-wave_energy.xlsx"
 FILEPATH_FUEL_CELL = INVENTORY_DIR / "lci-fuel_cell.xlsx"
+FILEPATH_CSP = INVENTORY_DIR / "lci-concentrating-solar-power.xlsx"
 
 config = load_constants()
 
@@ -673,6 +674,7 @@ def __import_inventories(self, keep_uncertainty_data: bool = False) -> List[dict
             (FILEPATH_NUCLEAR_SMR, "3.8"),
             (FILEPATH_WAVE, "3.8"),
             (FILEPATH_FUEL_CELL, "3.9"),
+            (FILEPATH_CSP, "3.9"),
         ]
         for filepath in filepaths:
             # make an exception for FILEPATH_OIL_GAS_INVENTORIES
@@ -765,28 +767,29 @@ def update_electricity(self) -> None:
                     use_absolute_efficiency=self.use_absolute_efficiency,
                 )
 
+                electricity.create_missing_power_plant_datasets()
                 electricity.adjust_coal_power_plant_emissions()
 
                 # datasets in 3.9 have been updated
                 if self.version not in ["3.9", "3.9.1"]:
-                    electricity.update_ng_production_ds()
+                   electricity.update_ng_production_ds()
 
                 electricity.update_efficiency_of_solar_pv()
 
                 if scenario["iam data"].biomass_markets is not None:
-                    electricity.create_biomass_markets()
+                   electricity.create_biomass_markets()
 
                 electricity.create_region_specific_power_plants()
 
                 if scenario["iam data"].electricity_markets is not None:
-                    electricity.update_electricity_markets()
+                   electricity.update_electricity_markets()
                 else:
-                    print("No electricity markets found in IAM data. Skipping.")
+                   print("No electricity markets found in IAM data. Skipping.")
 
                 if scenario["iam data"].electricity_efficiencies is not None:
-                    electricity.update_electricity_efficiency()
+                   electricity.update_electricity_efficiency()
                 else:
-                    print("No electricity efficiencies found in IAM data. Skipping.")
+                   print("No electricity efficiencies found in IAM data. Skipping.")
 
                 scenario["database"] = electricity.database
                 self.modified_datasets = electricity.modified_datasets