Align with master.

conda/meta.yaml

@@ -34,6 +34,7 @@ requirements:
     - requests
     - bottleneck
     - constructive_geometries>=0.8.2
+    - pyarrow
 
 test:
   imports:

dev/Untitled.ipynb

@@ -8,8 +8,8 @@
    "outputs": [],
    "source": [
     "from premise import *\n",
-    "import brightway2 as bw\n",
-    "bw.projects.set_current(\"ei39\")"
+    "import bw2data\n",
+    "bw2data.projects.set_current(\"ei39\")"
    ]
   },
   {
@@ -20,23 +20,46 @@
    "outputs": [],
    "source": [
     "scenarios = [\n",
-    "    #{\"model\": \"image\", \"pathway\":\"SSP1-RCP19\", \"year\": 2050, \"filepath\":\"/Users/romain/Documents\"},\n",
-    "    {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2050,},\n",
+    "    {\"model\": \"image\", \"pathway\":\"SSP1-Base\", \"year\": 2050,},\n",
+    "    #{\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2050,},\n",
     "    #{\"model\": \"remind\", \"pathway\":\"SSP1-PkBudg500\", \"year\": 2050,},\n",
     "]"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 3,
+   "id": "084a4bb2-f34a-49c2-ab93-6286c4d60613",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Databases dictionary with 2 object(s):\n",
+       "\tbiosphere3\n",
+       "\tecoinvent cutoff 3.9.1"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "bw2data.databases"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
    "id": "bc61f1aa-81d4-4a83-b7c5-0b10ad1a5b65",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "premise v.(1, 5, 8)\n",
+      "premise v.(1, 5, 9)\n",
       "+------------------------------------------------------------------+\n",
       "| Warning                                                          |\n",
       "+------------------------------------------------------------------+\n",
@@ -85,7 +108,7 @@
    "source": [
     "ndb = NewDatabase(\n",
     "        scenarios = scenarios,\n",
-    "        source_db= \"ecoinvent 3.9.1 cutoff\",\n",
+    "        source_db= \"ecoinvent cutoff 3.9.1\",\n",
     "        source_version=\"3.9.1\",\n",
     "        #source_type=source,\n",
     "        #source_file_path=f\"/Users/romain/Documents/ecoinvent {ei}_cutoff_ecoSpold02/datasets\" if source == \"ecospold\" else None, # <-- this is NEW\n",
@@ -98,7 +121,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "c7996c75-e3e7-42ba-9ecb-c016b6462109",
    "metadata": {},
    "outputs": [
@@ -107,67 +130,13 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "////////////////////////////// METALS ///////////////////////////////\n",
-      "Creating metal markets\n",
-      "... for Bauxite.\n",
-      "... for Beryl.\n",
-      "... for Cadmium.\n",
-      "... for Chromium ores and concentrate.\n",
-      "... for Cobalt, mine.\n",
-      "... for Cobalt, refined.\n",
-      "... for Copper, mine.\n",
-      "... for Copper, smelter.\n",
-      "... for Copper, refined.\n",
-      "... for Gallium.\n",
-      "... for Germanium metal.\n",
-      "... for Gold, mine.\n",
-      "... for graphite, purified.\n",
-      "... for graphite, spherical.\n",
-      "... for graphite ore, concentrated.\n",
-      "... for Graphite, natural.\n",
-      "... for Indium.\n",
-      "... for Indium, from zinc.\n",
-      "... for Iron, ore.\n",
-      "... for Iron, pig.\n",
-      "... for Lead, mine.\n",
-      "... for Lead, refined.\n",
-      "... for Lithium.\n",
-      "... for spodumene.\n",
-      "... for Magnesite.\n",
-      "... for Magnesium metal.\n",
-      "... for Manganese, ore.\n",
-      "... for Molybdenum, mine.\n",
-      "... for Molybdenum, refining.\n",
-      "... for Nickel, mine.\n",
-      "... for Nickel, smelter-refiner.\n",
-      "... for Phosphate rock.\n",
-      "New location CX for Christmas Island not found\n",
-      "... for Palladium.\n",
-      "... for Platinum.\n",
-      "... for REE oxides.\n",
-      "... for Rhenium.\n",
-      "... for Selenium, refined.\n",
-      "... for Silver, mine.\n",
-      "... for Strontium, mineral.\n",
-      "... for pyrochlore.\n",
-      "... for Niobium.\n",
-      "... for Tantalum.\n",
-      "... for Tellurium, refined.\n",
-      "... for Tin, mine.\n",
-      "... for Tin, smelter.\n",
-      "... for Ilmenite.\n",
-      "... for Rutile.\n",
-      "... for Tungsten, mine.\n",
-      "... for Uranium.\n",
-      "... for Vanadium, mine.\n",
-      "... for Zinc, mine.\n",
-      "... for Zinc, slab.\n",
-      "Creating additional mining processes\n"
+      "////////////////////////////// METALS ///////////////////////////////\n"
      ]
     }
    ],
    "source": [
     "#ndb.update_electricity()\n",
+    "#ndb.update_all()\n",
     "ndb.update_metals()"
    ]
   },
@@ -1206,7 +1175,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.15"
+   "version": "3.10.0"
   }
  },
  "nbformat": 4,

docs/extract.rst

@@ -577,7 +577,10 @@ The new datasets introduced are listed in the table below (only production datas
   hydrogen production, gaseous, 25 bar, from gasification of woody biomass in entrained flow gasifier, with CCS, at gasification plant    CH
   hydrogen production, gaseous, 25 bar, from gasification of woody biomass in entrained flow gasifier, at gasification plant              CH
   hydrogen production, gaseous, 30 bar, from hard coal gasification and reforming, at coal gasification plant                             RER
-  hydrogen production, gaseous, 200 bar, from PEM electrolysis, from grid electricity                                                     RER
+  hydrogen production, gaseous, 30 bar, from PEM electrolysis, from grid electricity                                                      RER
+  hydrogen production, gaseous, 20 bar, from AEC electrolysis, from grid electricity                                                      RER
+  hydrogen production, gaseous, 1 bar, from SOEC electrolysis, from grid electricity                                                      RER
+  hydrogen production, gaseous, 1 bar, from SOEC electrolysis, with steam input, from grid electricity                                    RER
   hydrogen production, gaseous, 25 bar, from thermochemical water splitting, at solar tower                                               RER
   hydrogen production, gaseous, 100 bar, from methane pyrolysis                                                                           RER
  ======================================================================================================================================= ===========

premise/__init__.py

@@ -1,5 +1,5 @@
 __all__ = ("NewDatabase", "clear_cache", "get_regions_definition")
-__version__ = (1, 5, 8)
+__version__ = (1, 5, 9)
 
 from pathlib import Path
 

premise/activity_maps.py

@@ -48,31 +48,58 @@ def get_mapping(filepath: Path, var: str) -> dict:
     return mapping
 
 
-@lru_cache
-def biosphere_flows_dictionary(version):
-    """
-    Create a dictionary with biosphere flows
-    (name, category, sub-category, unit) -> code
+def act_fltr(
+    database: List[dict],
+    fltr: Union[str, List[str]] = None,
+    mask: Union[str, List[str]] = None,
+) -> List[dict]:
+    """Filter `database` for activities matching field contents given by `fltr` excluding strings in `mask`.
+    `fltr`: string, list of strings or dictionary.
+    If a string is provided, it is used to match the name field from the start (*startswith*).
+    If a list is provided, all strings in the lists are used and results are joined (*or*).
+    A dict can be given in the form <fieldname>: <str> to filter for <str> in <fieldname>.
+    `mask`: used in the same way as `fltr`, but filters add up with each other (*and*).
+    `filter_exact` and `mask_exact`: boolean, set `True` to only allow for exact matches.
+
+    :param database: A lice cycle inventory database
+    :type database: brightway2 database object
+    :param fltr: value(s) to filter with.
+    :type fltr: Union[str, lst, dict]
+    :param mask: value(s) to filter with.
+    :type mask: Union[str, lst, dict]
+    :return: list of activity data set names
+    :rtype: list
+
     """
-    if version == "3.9":
-        fp = DATA_DIR / "utils" / "export" / "flows_biosphere_39.csv"
-    else:
-        fp = DATA_DIR / "utils" / "export" / "flows_biosphere_38.csv"
+    if fltr is None:
+        fltr = {}
+    if mask is None:
+        mask = {}
 
-    if not Path(fp).is_file():
-        raise FileNotFoundError("The dictionary of biosphere flows could not be found.")
+    # default field is name
+    if isinstance(fltr, (list, str)):
+        fltr = {"name": fltr}
+    if isinstance(mask, (list, str)):
+        mask = {"name": mask}
 
-    csv_dict = {}
+    assert len(fltr) > 0, "Filter dict must not be empty."
 
-    with open(fp, encoding="utf-8") as file:
-        input_dict = csv.reader(
-            file,
-            delimiter=get_delimiter(filepath=fp),
-        )
-        for row in input_dict:
-            csv_dict[(row[0], row[1], row[2], row[3])] = row[-1]
+    # find `act` in `database` that match `fltr`
+    # and do not match `mask`
+    filters = []
+    for field, value in fltr.items():
+        if isinstance(value, list):
+            filters.extend([ws.either(*[ws.contains(field, v) for v in value])])
+        else:
+            filters.append(ws.contains(field, value))
 
-    return csv_dict
+    for field, value in mask.items():
+        if isinstance(value, list):
+            filters.extend([ws.exclude(ws.contains(field, v)) for v in value])
+        else:
+            filters.append(ws.exclude(ws.contains(field, value)))
+
+    return list(ws.get_many(database, *filters))
 
 
 class InventorySet:
@@ -229,20 +256,6 @@ def generate_activities_using_metals_map(self) -> dict:
         """
         return self.generate_sets_from_filters(self.activity_metals_filters)
 
-    def generate_metals_map(self) -> dict:
-        """
-        Filter ecoinvent processes related to metals.
-        Returns a dictionary with metal names as keys (see below) and
-        a set of related ecoinvent activities' names as values.
-        """
-
-        return self.generate_sets_from_filters(
-            self.metals_filters,
-            database=[
-                {"name": k[0], "categories": k[1]}
-                for k in biosphere_flows_dictionary(version=self.version)
-            ],
-        )
 
     @staticmethod
     def act_fltr(
@@ -318,5 +331,8 @@ def generate_sets_from_filters(self, filtr: dict, database=None) -> dict:
 
         database = database or self.database
 
-        techs = {tech: self.act_fltr(database, **fltr) for tech, fltr in filtr.items()}
+        techs = {
+            tech: act_fltr(database, fltr.get("fltr"), fltr.get("mask"))
+            for tech, fltr in filtr.items()
+        }
         return {tech: {act["name"] for act in actlst} for tech, actlst in techs.items()}

premise/cement.py

@@ -14,22 +14,33 @@
 
 import yaml
 
+from .logger import create_logger
 from .transformation import BaseTransformation, Dict, IAMDataCollection, List, np, ws
 from .utils import DATA_DIR
 
-LOG_CONFIG = DATA_DIR / "utils" / "logging" / "logconfig.yaml"
-# directory for log files
-DIR_LOG_REPORT = Path.cwd() / "export" / "logs"
-# if DIR_LOG_REPORT folder does not exist
-# we create it
-if not Path(DIR_LOG_REPORT).exists():
-    Path(DIR_LOG_REPORT).mkdir(parents=True, exist_ok=True)
+logger = create_logger("cement")
 
-with open(LOG_CONFIG, "r") as f:
-    config = yaml.safe_load(f.read())
-    logging.config.dictConfig(config)
 
-logger = logging.getLogger("cement")
+def _update_cement(scenario, version, system_model, modified_datasets):
+    cement = Cement(
+        database=scenario["database"],
+        model=scenario["model"],
+        pathway=scenario["pathway"],
+        iam_data=scenario["iam data"],
+        year=scenario["year"],
+        version=version,
+        system_model=system_model,
+        modified_datasets=modified_datasets,
+    )
+
+    if scenario["iam data"].cement_markets is not None:
+        cement.add_datasets_to_database()
+        scenario["database"] = cement.database
+        modified_datasets = cement.modified_datasets
+    else:
+        print("No cement markets found in IAM data. Skipping.")
+
+    return scenario, modified_datasets
 
 
 class Cement(BaseTransformation):
@@ -453,9 +464,9 @@ def add_datasets_to_database(self) -> None:
         :return: Does not return anything. Modifies in place.
         """
 
-        print("Start integration of cement data...")
+        # print("Start integration of cement data...")
 
-        print("Create new clinker production datasets and delete old datasets")
+        # print("Create new clinker production datasets and delete old datasets")
 
         clinker_prod_datasets = list(self.build_clinker_production_datasets().values())
         self.database.extend(clinker_prod_datasets)
@@ -475,7 +486,7 @@ def add_datasets_to_database(self) -> None:
                 )
             )
 
-        print("Create new clinker market datasets and delete old datasets")
+        # print("Create new clinker market datasets and delete old datasets")
         clinker_market_datasets = list(
             self.fetch_proxies(
                 name="market for clinker",
@@ -501,7 +512,7 @@ def add_datasets_to_database(self) -> None:
                 )
             )
 
-        print("Create new cement market datasets")
+        # print("Create new cement market datasets")
 
         # cement markets
         markets = ws.get_many(
@@ -542,10 +553,10 @@ def add_datasets_to_database(self) -> None:
 
         self.database.extend(new_datasets)
 
-        print(
-            "Create new cement production datasets and "
-            "adjust electricity consumption"
-        )
+        # print(
+        #    "Create new cement production datasets and "
+        #    "adjust electricity consumption"
+        # )
         # cement production
         production = ws.get_many(
             self.database,

premise/data/additional_inventories/migration_map.csv

@@ -340,6 +340,8 @@ from;to;name_from;ref_prod_from;location_from;name_to;ref_prod_to;location_to
 35;39;market for potassium fertiliser, as K2O;potassium fertiliser, as K2O;GLO;market group for inorganic potassium fertiliser, as K2O;inorganic potassium fertiliser, as K2O;RER
 36;39;market for potassium fertiliser, as K2O;potassium fertiliser, as K2O;GLO;market group for inorganic potassium fertiliser, as K2O;inorganic potassium fertiliser, as K2O;RER
 37;39;market for chemicals, inorganic;chemical, inorganic;GLO;market for chemical, inorganic;;
+39;37;market for chemical, inorganic;chemical, inorganic;GLO;market for chemicals, inorganic;;
+38;37;market for chemical, inorganic;chemical, inorganic;GLO;market for chemicals, inorganic;;
 35;39;heat pump production, for heat and power co-generation unit, 160kW electrical;heat pump, for heat and power co-generation unit, 160kW electrical;RER;heat pump production, heat and power co-generation unit, 160kW electrical;heat pump, heat and power co-generation unit, 160kW electrical;
 36;39;heat pump production, for heat and power co-generation unit, 160kW electrical;heat pump, for heat and power co-generation unit, 160kW electrical;RER;heat pump production, heat and power co-generation unit, 160kW electrical;heat pump, heat and power co-generation unit, 160kW electrical;
 37;39;heat pump production, for heat and power co-generation unit, 160kW electrical;heat pump, for heat and power co-generation unit, 160kW electrical;RER;heat pump production, heat and power co-generation unit, 160kW electrical;heat pump, heat and power co-generation unit, 160kW electrical;