EODC

polca · Aug 23, 2023 · 0b0d3e8 · 0b0d3e8
1 parent d0b97c6
commit 0b0d3e8
Show file tree

Hide file tree

Showing 9 changed files with 798 additions and 220 deletions.
diff --git a/dev/Untitled.ipynb b/dev/Untitled.ipynb
diff --git a/premise/data/iam_output_files/image_SSP1-Base.csv b/premise/data/iam_output_files/image_SSP1-Base.csv
diff --git a/premise/data/iam_output_files/image_SSP2-Base.csv b/premise/data/iam_output_files/image_SSP2-Base.csv
diff --git a/premise/data/iam_output_files/image_SSP2-RCP19.csv b/premise/data/iam_output_files/image_SSP2-RCP19.csv
diff --git a/premise/data/iam_output_files/image_SSP2-RCP26.csv b/premise/data/iam_output_files/image_SSP2-RCP26.csv
diff --git a/premise/data_collection.py b/premise/data_collection.py
@@ -364,6 +364,14 @@ def __init__(
             IAM_CROPS_VARS, variable="land_use_change"
         )
 
+        fertilizer_vars = self.__get_iam_variable_labels(
+            IAM_CROPS_VARS, variable="fertilizer use"
+        )
+
+        crops_production = self.__get_iam_variable_labels(
+            IAM_CROPS_VARS, variable="production"
+        )
+
         carbon_capture_vars = self.__get_iam_variable_labels(
             IAM_CARBON_CAPTURE_VARS, variable="iam_aliases"
         )
@@ -389,6 +397,8 @@ def __init__(
             + list(biomass_eff_vars.values())
             + list(land_use_vars.values())
             + list(land_use_change_vars.values())
+            + list(fertilizer_vars.values())
+            + list(crops_production.values())
             + list(carbon_capture_vars.values())
             + list(other_vars.values())
         )
@@ -569,6 +579,15 @@ def __init__(
             data=data, input_vars=land_use_change_vars, fill=True
         )
 
+        self.fertilizer_use = self.__get_iam_production_volumes(
+            data=data, input_vars=fertilizer_vars, fill=True
+        )
+
+        self.crops_production = self.__get_iam_production_volumes(
+            data=data, input_vars=crops_production, fill=True
+        )
+
+
         self.trsp_cars = get_vehicle_fleet_composition(self.model, vehicle_type="car")
         self.trsp_trucks = get_vehicle_fleet_composition(
             self.model, vehicle_type="truck"
@@ -732,21 +751,29 @@ def __fetch_market_data(
             data.year.values.min() <= self.year <= data.year.values.max()
         ), f"{self.year} is outside of the boundaries of the IAM file: {data.year.values.min()}-{data.year.values.max()}"
 
-        missing_vars = set(input_vars.values()) - set(data.variables.values)
+        missing_vars = set(flatten(input_vars.values())) - set(data.variables.values)
 
         if missing_vars:
             print(
                 f"The following variables are missing from the IAM file: {list(missing_vars)}"
             )
 
-        available_vars = list(set(input_vars.values()) - missing_vars)
+        available_vars = list(set(flatten(input_vars.values())) - missing_vars)
 
         if available_vars:
             market_data = data.loc[:, available_vars, :]
         else:
             return None
 
-        rev_input_vars = {v: k for k, v in input_vars.items()}
+        # create a reverse dictionary
+        # considering that values may contain a list
+        rev_input_vars = {}
+        for k, v in input_vars.items():
+            if isinstance(v, list):
+                for x in v:
+                    rev_input_vars[x] = k
+            else:
+                rev_input_vars[v] = k
 
         market_data.coords["variables"] = [
             rev_input_vars[v] for v in market_data.variables.values
@@ -828,7 +855,7 @@ def get_iam_efficiencies(
                 # and that each element of prod.values() is in data.variables.values
                 if (
                     all(var in data.variables.values for var in energy_labels[k])
-                    and v in data.variables.values
+                    and v in data.variables.values.tolist()
                 ):
                     d = 1 / (
                         data.loc[:, energy_labels[k], :].sum(dim="variables")
@@ -1025,19 +1052,21 @@ def __get_iam_production_volumes(
                 f"of the IAM file: {data.year.values.min()}-{data.year.values.max()}"
             )
 
-        missing_vars = set(input_vars.values()) - set(data.variables.values)
+        missing_vars = set(flatten(input_vars.values())) - set(data.variables.values)
 
         if missing_vars:
             print(
                 f"The following variables are missing from the IAM file: {list(missing_vars)}"
             )
 
-        available_vars = list(set(input_vars.values()) - missing_vars)
+        available_vars = list(set(flatten(input_vars.values())) - missing_vars)
 
         if available_vars:
             data_to_return = data.loc[
-                :, [v for v in input_vars.values() if v in available_vars], :
+                :, [v for v in flatten(input_vars.values())
+                    if v in available_vars], :
             ]
+
         else:
             return None
 

diff --git a/premise/ecoinvent_modification.py b/premise/ecoinvent_modification.py
@@ -605,8 +605,29 @@ def _fetch_iam_data(scenario):
             scenario["database"] = copy.deepcopy(self.database)
 
         # use multiprocessing to speed up the process
-        with Pool(processes=multiprocessing.cpu_count()) as pool:
-            pool.map(_fetch_iam_data, self.scenarios)
+        #with Pool(processes=multiprocessing.cpu_count()) as pool:
+        #    pool.map(_fetch_iam_data, self.scenarios)
+
+        for scenario in self.scenarios:
+            data = IAMDataCollection(
+                model=scenario["model"],
+                pathway=scenario["pathway"],
+                year=scenario["year"],
+                external_scenarios=scenario.get("external scenarios"),
+                filepath_iam_files=scenario["filepath"],
+                key=key,
+                system_model=self.system_model,
+                system_model_args=self.system_model_args,
+                gains_scenario=self.gains_scenario,
+                use_absolute_efficiency=self.use_absolute_efficiency,
+            )
+
+            scenario["iam data"] = data
+
+            if self.datapackages:
+                scenario["external data"] = data.get_external_data(self.datapackages)
+
+            scenario["database"] = copy.deepcopy(self.database)
 
         print("Done!")
 

diff --git a/premise/iam_variables_mapping/crops_variables.yaml b/premise/iam_variables_mapping/crops_variables.yaml
@@ -17,6 +17,10 @@ sugar:
     image: Land Use|Marginal|Biomass|Sugar
   land_use_change:
     image: Emission Factor|CO2|Energy|Supply|Biomass|Marginal|Sugar
+  production:
+    image: Agricultural Production|Crops|Energy|Sugarcrops
+  fertilizer use:
+    image: Fertilizer Use|Nitrogen|Energy-Crops|Sugarcrops
 oil:
   crop_type:
     image:
@@ -32,6 +36,10 @@ oil:
     image: Land Use|Marginal|Biomass|OilCrop
   land_use_change:
     image: Emission Factor|CO2|Energy|Supply|Biomass|Marginal|Oilcrops
+  production:
+    image: Agricultural Production|Crops|Energy|Oilcrops
+  fertilizer use:
+    image: Fertilizer Use|Nitrogen|Energy-Crops|Oil-crops
 wood:
   crop_type:
     image:
@@ -47,6 +55,10 @@ wood:
     image: Land Use|Marginal|Biomass|Woody
   land_use_change:
     image: Emission Factor|CO2|Energy|Supply|Biomass|Marginal|Woody
+  production:
+    image: Agricultural Production|Crops|Energy|Woody
+  fertilizer use:
+    image: Fertilizer Use|Nitrogen|Energy-Crops|Woody crops
 grass:
   crop_type:
     image:
@@ -62,6 +74,10 @@ grass:
     image: Land Use|Marginal|Biomass|Grassy
   land_use_change:
     image: Emission Factor|CO2|Energy|Supply|Biomass|Marginal|Grassy
+  production:
+    image: Agricultural Production|Crops|Energy|Non-Woody
+  fertilizer use:
+    image: Fertilizer Use|Nitrogen|Energy-Crops|Non-woody crops
 grain:
   crop_type:
     image:
@@ -76,4 +92,8 @@ grain:
   land_use:
     image: Land Use|Marginal|Biomass|Maize
   land_use_change:
-    image: Emission Factor|CO2|Energy|Supply|Biomass|Marginal|Maize
+    image: Emission Factor|CO2|Energy|Supply|Biomass|Marginal|Maize
+  production:
+    image: Agricultural Production|Crops|Energy|Maize
+  fertilizer use:
+    image: Fertilizer Use|Nitrogen|Energy-Crops|Non-woody crops
diff --git a/premise/iam_variables_mapping/steel_variables.yaml b/premise/iam_variables_mapping/steel_variables.yaml
@@ -4,7 +4,20 @@
 steel - primary:
   iam_aliases:
     remind: Production|Industry|Steel|Primary
-    image: Production|Steel|Primary
+    image:
+      - Production|Industry|Iron and Steel|BF/BOF Standard
+      - Production|Industry|Iron and Steel|BF/ BOF Standard + CCS
+      - Production|Industry|Iron and Steel|BF/BOF Modernized
+      - Production|Industry|Iron and Steel|BF/BOF Standard
+      - Production|Industry|Iron and Steel|DRI EAF + CCS
+      - Production|Industry|Iron and Steel|DRI EAF
+      - Production|Industry|Iron and Steel|Electrowinning/EAF
+      - Production|Industry|Iron and Steel|H-DRI/EAF
+      - Production|Industry|Iron and Steel|SR/BOF + CCS
+      - Production|Industry|Iron and Steel|SR/BOF
+      - Production|Industry|Iron and Steel|TGR BF/BOF BAT + CCS
+      - Production|Industry|Iron and Steel|TGR BF/BOF BAT
+      - Production|Industry|Iron and Steel|TGR BF/BOF
     message: Production|Primary|Steel
   energy_use_aliases:
     remind:
@@ -36,7 +49,7 @@ steel - primary:
 steel - secondary:
   iam_aliases:
     remind: Production|Industry|Steel|Secondary
-    image: Production|Steel|Secondary
+    image: Production|Industry|Iron and Steel|EAF scrap
     message: Production|Secondary|Steel
   energy_use_aliases:
     remind: