Implement caching of results from relink_technosphere_exchanges(). + …

…version bump.
polca · Aug 5, 2023 · 4ebff08 · 4ebff08
1 parent 867f42c
commit 4ebff08
Show file tree

Hide file tree

Showing 11 changed files with 86 additions and 33 deletions.
diff --git a/dev/profile.prof b/dev/profile.prof
diff --git a/dev/test.py b/dev/test.py
@@ -5,17 +5,16 @@
 bw2data.projects.set_current("ei39")
 
 scenarios = [
-    {"model": "remind", "pathway": "SSP1-Base", "year": 2005},
-    {"model": "remind", "pathway": "SSP2-Base", "year": 2010},
-    {"model": "remind", "pathway": "SSP5-Base", "year": 2015},
+    #{"model": "remind", "pathway": "SSP1-Base", "year": 2005},
+    #{"model": "remind", "pathway": "SSP2-Base", "year": 2010},
+    {"model": "remind", "pathway": "SSP5-Base", "year": 2050},
 ]
 
 ndb = NewDatabase(
     scenarios=scenarios,
-    source_db="ecoinvent cutoff 3.9.1",
-    source_version="3.9",
+    source_db="ecoinvent 3.9.1 cutoff",
     key=b"tUePmX_S5B8ieZkkM7WUU2CnO8SmShwmAeWK9x2rTFo=",
 )
 
 ndb.update_all()
-ndb.write_datapackage(name="my_dp")
+ndb.write_db_to_brightway()
diff --git a/premise/__init__.py b/premise/__init__.py
@@ -1,5 +1,5 @@
 __all__ = ("NewDatabase", "clear_cache", "get_regions_definition")
-__version__ = (1, 5, 9)
+__version__ = (1, 6, 0)
 
 from pathlib import Path
 

diff --git a/premise/cement.py b/premise/cement.py
@@ -21,7 +21,7 @@
 logger = create_logger("cement")
 
 
-def _update_cement(scenario, version, system_model, modified_datasets):
+def _update_cement(scenario, version, system_model, modified_datasets, cache):
     cement = Cement(
         database=scenario["database"],
         model=scenario["model"],
@@ -31,16 +31,18 @@ def _update_cement(scenario, version, system_model, modified_datasets):
         version=version,
         system_model=system_model,
         modified_datasets=modified_datasets,
+        cache=cache
     )
 
     if scenario["iam data"].cement_markets is not None:
         cement.add_datasets_to_database()
         scenario["database"] = cement.database
         modified_datasets = cement.modified_datasets
+        cache = cement.cache
     else:
         print("No cement markets found in IAM data. Skipping.")
 
-    return scenario, modified_datasets
+    return scenario, modified_datasets, cache
 
 
 class Cement(BaseTransformation):
@@ -74,6 +76,7 @@ def __init__(
         version: str,
         system_model: str,
         modified_datasets: dict,
+        cache: dict = None
     ):
         super().__init__(
             database,
@@ -84,6 +87,7 @@ def __init__(
             version,
             system_model,
             modified_datasets,
+            cache
         )
         self.version = version
 

diff --git a/premise/direct_air_capture.py b/premise/direct_air_capture.py
@@ -37,7 +37,7 @@ def fetch_mapping(filepath: str) -> dict:
     return mapping
 
 
-def _update_dac(scenario, version, system_model, modified_datasets):
+def _update_dac(scenario, version, system_model, modified_datasets, cache=None):
     dac = DirectAirCapture(
         database=scenario["database"],
         iam_data=scenario["iam data"],
@@ -47,16 +47,18 @@ def _update_dac(scenario, version, system_model, modified_datasets):
         version=version,
         system_model=system_model,
         modified_datasets=modified_datasets,
+        cache=cache
     )
 
     if scenario["iam data"].dac_markets is not None:
         dac.generate_dac_activities()
         scenario["database"] = dac.database
         modified_datasets = dac.modified_datasets
+        cache = dac.cache
     else:
         print("No DAC markets found in IAM data. Skipping.")
 
-    return scenario, modified_datasets
+    return scenario, modified_datasets, cache
 
 
 class DirectAirCapture(BaseTransformation):
@@ -75,6 +77,7 @@ def __init__(
         version: str,
         system_model: str,
         modified_datasets: dict,
+        cache: dict = None,
     ):
         super().__init__(
             database,
@@ -85,6 +88,7 @@ def __init__(
             version,
             system_model,
             modified_datasets,
+            cache
         )
         self.database = database
         self.iam_data = iam_data

diff --git a/premise/ecoinvent_modification.py b/premise/ecoinvent_modification.py
@@ -410,23 +410,48 @@ def _update_all(
     vehicle_type,
     gains_scenario,
 ):
-    scenario, modified_datasets = _update_vehicles(
-        scenario, vehicle_type, version, system_model, modified_datasets
+    scenario, modified_datasets, cache = _update_vehicles(
+        scenario=scenario,
+        vehicle_type=vehicle_type,
+        version=version,
+        system_model=system_model,
+        modified_datasets=modified_datasets
     )
-    scenario, modified_datasets = _update_electricity(
-        scenario, version, system_model, modified_datasets, use_absolute_efficiency
+    scenario, modified_datasets, cache = _update_electricity(
+        scenario=scenario,
+        version=version,
+        system_model=system_model,
+        modified_datasets=modified_datasets,
+        use_absolute_efficiency=use_absolute_efficiency,
+        cache=cache
     )
-    scenario, modified_datasets = _update_dac(
-        scenario, version, system_model, modified_datasets
+    scenario, modified_datasets, cache = _update_dac(
+        scenario=scenario,
+        version=version,
+        system_model=system_model,
+        modified_datasets=modified_datasets,
+        cache=cache
     )
-    scenario, modified_datasets = _update_cement(
-        scenario, version, system_model, modified_datasets
+    scenario, modified_datasets, cache = _update_cement(
+        scenario=scenario,
+        version=version,
+        system_model=system_model,
+        modified_datasets=modified_datasets,
+        cache=cache
     )
-    scenario, modified_datasets = _update_steel(
-        scenario, version, system_model, modified_datasets
+    scenario, modified_datasets, cache = _update_steel(
+        scenario=scenario,
+        version=version,
+        system_model=system_model,
+        modified_datasets=modified_datasets,
+        cache=cache
     )
-    scenario, modified_datasets = _update_fuels(
-        scenario, version, system_model, modified_datasets
+    scenario, modified_datasets, cache = _update_fuels(
+        scenario=scenario,
+        version=version,
+        system_model=system_model,
+        modified_datasets=modified_datasets,
+        cache=cache
     )
     scenario, modified_datasets = _update_emissions(
         scenario, version, system_model, gains_scenario, modified_datasets
@@ -1112,7 +1137,8 @@ def update_all(self) -> None:
         print("`update_all()` will skip the following steps:")
         print("update_two_wheelers(), update_cars(), and update_buses()")
         print(
-            "If you want to update these steps, please run them separately afterwards."
+            "If you want to update these steps, "
+            "please run them separately afterwards."
         )
 
         # use multiprocessing to speed up the process

diff --git a/premise/electricity.py b/premise/electricity.py
@@ -163,7 +163,7 @@ def get_production_weighted_losses(
 
 
 def _update_electricity(
-    scenario, version, system_model, modified_datasets, use_absolute_efficiency
+    scenario, version, system_model, modified_datasets, use_absolute_efficiency, cache=None
 ):
     electricity = Electricity(
         database=scenario["database"],
@@ -175,6 +175,7 @@ def _update_electricity(
         system_model=system_model,
         modified_datasets=modified_datasets,
         use_absolute_efficiency=use_absolute_efficiency,
+        cache=cache,
     )
 
     electricity.create_missing_power_plant_datasets()
@@ -203,8 +204,9 @@ def _update_electricity(
 
     scenario["database"] = electricity.database
     modified_datasets = electricity.modified_datasets
+    cache = electricity.cache
 
-    return scenario, modified_datasets
+    return scenario, modified_datasets, cache
 
 
 class Electricity(BaseTransformation):
@@ -235,6 +237,7 @@ def __init__(
         system_model: str,
         modified_datasets: dict,
         use_absolute_efficiency: bool = False,
+        cache: dict = None,
     ) -> None:
         super().__init__(
             database,
@@ -245,6 +248,7 @@ def __init__(
             version,
             system_model,
             modified_datasets,
+            cache
         )
         mapping = InventorySet(self.database)
         self.powerplant_map = mapping.generate_powerplant_map()

diff --git a/premise/fuels.py b/premise/fuels.py
@@ -274,7 +274,7 @@ def update_dataset(dataset, supplier_key, amount):
     return dataset
 
 
-def _update_fuels(scenario, version, system_model, modified_datasets):
+def _update_fuels(scenario, version, system_model, modified_datasets, cache):
     fuels = Fuels(
         database=scenario["database"],
         iam_data=scenario["iam data"],
@@ -284,6 +284,7 @@ def _update_fuels(scenario, version, system_model, modified_datasets):
         version=version,
         system_model=system_model,
         modified_datasets=modified_datasets,
+        cache=cache
     )
 
     if any(
@@ -298,10 +299,11 @@ def _update_fuels(scenario, version, system_model, modified_datasets):
         fuels.generate_fuel_markets()
         scenario["database"] = fuels.database
         modified_datasets = fuels.modified_datasets
+        cache = fuels.cache
     else:
         print("No fuel markets found in IAM data. Skipping.")
 
-    return scenario, modified_datasets
+    return scenario, modified_datasets, cache
 
 
 class Fuels(BaseTransformation):
@@ -319,6 +321,7 @@ def __init__(
         version: str,
         system_model: str,
         modified_datasets: dict,
+        cache: dict = None
     ):
         super().__init__(
             database,
@@ -329,6 +332,7 @@ def __init__(
             version,
             system_model,
             modified_datasets,
+            cache
         )
         # ecoinvent version
         self.version = version

diff --git a/premise/steel.py b/premise/steel.py
@@ -16,7 +16,7 @@
 logger = create_logger("steel")
 
 
-def _update_steel(scenario, version, system_model, modified_datasets):
+def _update_steel(scenario, version, system_model, modified_datasets, cache):
     steel = Steel(
         database=scenario["database"],
         model=scenario["model"],
@@ -26,16 +26,18 @@ def _update_steel(scenario, version, system_model, modified_datasets):
         version=version,
         system_model=system_model,
         modified_datasets=modified_datasets,
+        cache=cache,
     )
 
     if scenario["iam data"].steel_markets is not None:
         steel.generate_activities()
         scenario["database"] = steel.database
         modified_datasets = steel.modified_datasets
+        cache = steel.cache
     else:
         print("No steel markets found in IAM data. Skipping.")
 
-    return scenario, modified_datasets
+    return scenario, modified_datasets, cache
 
 
 class Steel(BaseTransformation):
@@ -59,6 +61,7 @@ def __init__(
         version: str,
         system_model: str,
         modified_datasets: dict,
+        cache: dict = None
     ) -> None:
         super().__init__(
             database,
@@ -69,6 +72,7 @@ def __init__(
             version,
             system_model,
             modified_datasets,
+            cache
         )
         self.version = version
 

diff --git a/premise/transport.py b/premise/transport.py
@@ -33,7 +33,14 @@
 FILEPATH_VEHICLES_MAP = DATA_DIR / "transport" / "vehicles_map.yaml"
 
 
-def _update_vehicles(scenario, vehicle_type, version, system_model, modified_datasets):
+def _update_vehicles(
+        scenario,
+        vehicle_type,
+        version,
+        system_model,
+        modified_datasets,
+        cache=None,
+):
     trspt = Transport(
         database=scenario["database"],
         year=scenario["year"],
@@ -63,10 +70,11 @@ def _update_vehicles(scenario, vehicle_type, version, system_model, modified_dat
         trspt.create_vehicle_markets()
         scenario["database"] = trspt.database
         modified_datasets = trspt.modified_datasets
+        cache = trspt.cache
     else:
         print(f"No markets found for {vehicle_type} in IAM data. Skipping.")
 
-    return scenario, modified_datasets
+    return scenario, modified_datasets, {} or cache
 
 
 def get_average_truck_load_factors() -> Dict[str, Dict[str, Dict[str, float]]]:

diff --git a/setup.py b/setup.py
@@ -32,7 +32,7 @@ def package_files(directory):
 
 setup(
     name="premise",
-    version="1.5.9",
+    version="1.6.0",
     python_requires=">=3.9,<3.11",
     packages=packages,
     author="Romain Sacchi <[email protected]>, Alois Dirnaichner <[email protected]>, Chris Mutel "