Update emissions dataset base branch

dag/archive/emissions.yml

@@ -56,3 +56,35 @@ steps:
     - data://garden/energy/2022-07-29/primary_energy_consumption
   data://grapher/gcp/2022-11-11/global_carbon_budget:
     - data://garden/gcp/2022-11-11/global_carbon_budget
+  #
+  # GCP - Global Carbon Budget (2023-04-28).
+  #
+  data://meadow/gcp/2023-04-28/global_carbon_budget:
+    - snapshot://gcp/2023-04-28/global_carbon_budget_fossil_co2_emissions.csv
+    - snapshot://gcp/2023-04-28/global_carbon_budget_global_emissions.xlsx
+    - snapshot://gcp/2023-04-28/global_carbon_budget_national_emissions.xlsx
+    - snapshot://gcp/2023-04-28/global_carbon_budget_land_use_change_emissions.xlsx
+  data://garden/gcp/2023-04-28/global_carbon_budget:
+    - data://meadow/gcp/2023-04-28/global_carbon_budget
+    # Loaded to calculate emissions per unit energy.
+    - data://garden/energy/2023-06-01/primary_energy_consumption
+    # Loaded to calculate emissions per GDP.
+    - data://garden/ggdc/2020-10-01/ggdc_maddison
+    # Loaded to create per-capita variables.
+    - data://garden/demography/2022-12-08/population
+    # Loaded to create region aggregates (including income groups).
+    - data://garden/regions/2023-01-01/regions
+    - data://garden/wb/2021-07-01/wb_income
+  data://grapher/gcp/2023-04-28/global_carbon_budget:
+    - data://garden/gcp/2023-04-28/global_carbon_budget
+  #
+  # Emissions - CO2 dataset (2022).
+  #
+  data://garden/emissions/2023-05-03/owid_co2:
+    - data://garden/emissions/2023-05-02/national_contributions
+    - data://garden/gcp/2023-04-28/global_carbon_budget
+    - data://garden/cait/2022-08-10/ghg_emissions_by_sector
+    - data://garden/energy/2023-06-01/primary_energy_consumption
+    - data://garden/demography/2022-12-08/population
+    - data://garden/ggdc/2020-10-01/ggdc_maddison
+    - data://garden/regions/2023-01-01/regions

dag/archive/energy.yml

@@ -265,3 +265,164 @@ steps:
     - data://garden/demography/2023-03-31/population
     - data://garden/ggdc/2020-10-01/ggdc_maddison
     - data://garden/regions/2023-01-01/regions
+  #
+  # IRENA - Renewable power generation costs (2022).
+  #
+  data://meadow/irena/2023-01-04/renewable_power_generation_costs:
+    - walden://irena/2022-10-07/renewable_power_generation_costs
+  data://garden/irena/2023-01-04/renewable_power_generation_costs:
+    - data://meadow/irena/2023-01-04/renewable_power_generation_costs
+  data://grapher/irena/2023-01-04/renewable_power_generation_costs:
+    - data://garden/irena/2023-01-04/renewable_power_generation_costs
+  #
+  # IRENA - Renewable electricity capacity (and generation, although the generation part is ignored for now, 2022).
+  #
+  data://meadow/irena/2023-01-04/renewable_electricity_capacity_and_generation:
+    - walden://irena/2022-10-07/renewable_electricity_capacity_and_generation
+  data://garden/irena/2023-01-04/renewable_electricity_capacity:
+    - data://meadow/irena/2023-01-04/renewable_electricity_capacity_and_generation
+  data://grapher/irena/2023-01-04/renewable_electricity_capacity_by_technology:
+    - data://garden/irena/2023-01-04/renewable_electricity_capacity
+  #
+  # Energy - Photovoltaic cost and capacity.
+  #
+  data://garden/energy/2023-01-04/photovoltaic_cost_and_capacity:
+    - data://garden/papers/2023-01-04/nemet_2009
+    - data://garden/papers/2023-01-04/farmer_lafond_2016
+    - data://garden/irena/2023-01-04/renewable_electricity_capacity
+    - data://garden/irena/2023-01-04/renewable_power_generation_costs
+  data://grapher/energy/2023-01-04/photovoltaic_cost_and_capacity:
+    - data://garden/energy/2023-01-04/photovoltaic_cost_and_capacity
+  #
+  # BP - Statistical review 2021.
+  #
+  # NOTE: This dataset is not used in grapher. It exists only to fill gaps in the 2022 version.
+  data://garden/bp/2022-07-11/statistical_review:
+    - backport://backport/owid/latest/dataset_5347_statistical_review_of_world_energy__bp__2021
+    - data://garden/owid/latest/key_indicators
+    - data://garden/wb/2021-07-01/wb_income
+    - data://garden/regions/2023-01-01/regions
+  #
+  # BP - Statistical review 2022.
+  #
+  # NOTE: For the moment this is not the full processing (which is still done in importers).
+  #  This garden step loads the dataset and adds region aggregates properly, plus some other minor improvements.
+  #  Here, we also remove some regions that had misleading data (BP regions like "Other *").
+  data://garden/bp/2022-12-28/statistical_review:
+    # The backported 2022 release is the main source of data of this step.
+    - backport://backport/owid/latest/dataset_5650_statistical_review_of_world_energy__bp__2022
+    # The 2021 release is loaded just to fill missing data in the current version (and to get a missing column).
+    - data://garden/bp/2022-07-11/statistical_review
+    - data://garden/owid/latest/key_indicators
+    - data://garden/wb/2021-07-01/wb_income
+    - data://garden/regions/2023-01-01/regions
+  data://grapher/bp/2022-12-28/statistical_review:
+    - data://garden/bp/2022-12-28/statistical_review
+  #
+  # BP - Fossil fuel reserves/production ratio (2022).
+  #
+  data://garden/bp/2022-12-28/fossil_fuel_reserves_production_ratio:
+    - data://garden/bp/2022-12-28/statistical_review
+  data://grapher/bp/2022-12-28/fossil_fuel_reserves_production_ratio:
+    - data://garden/bp/2022-12-28/fossil_fuel_reserves_production_ratio
+  #
+  # Energy - Primary energy consumption 2023.
+  #
+  data://garden/energy/2023-06-01/primary_energy_consumption:
+    - data://garden/bp/2022-12-28/statistical_review
+    - data://garden/eia/2022-07-27/energy_consumption
+    - data://garden/ggdc/2020-10-01/ggdc_maddison
+    - data://garden/demography/2023-03-31/population
+  data://grapher/energy/2023-06-01/primary_energy_consumption:
+    - data://garden/energy/2023-06-01/primary_energy_consumption
+  #
+  # EIA - Energy consumption 2022.
+  #
+  data://meadow/eia/2022-07-27/energy_consumption:
+    - walden://eia/2022-07-27/international_energy_data
+  data://garden/eia/2022-07-27/energy_consumption:
+    - data://meadow/eia/2022-07-27/energy_consumption
+    - data://garden/owid/latest/key_indicators
+    - data://garden/regions/2023-01-01/regions
+  #
+  # Shift - Fossil fuel production 2022.
+  #
+  data://meadow/shift/2022-07-18/fossil_fuel_production:
+    - walden://shift/2022-07-18/fossil_fuel_production
+  data://garden/shift/2022-07-18/fossil_fuel_production:
+    - data://meadow/shift/2022-07-18/fossil_fuel_production
+    - data://garden/regions/2023-01-01/regions
+  #
+  # Energy - Fossil Fuel Production 2023.
+  #
+  data://garden/energy/2023-06-01/fossil_fuel_production:
+    - data://garden/bp/2022-12-28/statistical_review
+    - data://garden/shift/2022-07-18/fossil_fuel_production
+    - data://garden/demography/2023-03-31/population
+  data://grapher/energy/2023-06-01/fossil_fuel_production:
+    - data://garden/energy/2023-06-01/fossil_fuel_production
+  #
+  # Smil - Global primary energy (2017).
+  #
+  data://meadow/smil/2017-01-01/global_primary_energy:
+    - walden://smil/2017-01-01/global_primary_energy
+  data://garden/smil/2017-01-01/global_primary_energy:
+    - data://meadow/smil/2017-01-01/global_primary_energy
+  #
+  # Energy - Global primary energy (2023).
+  #
+  data://garden/energy/2023-06-01/global_primary_energy:
+    - data://garden/smil/2017-01-01/global_primary_energy
+    - data://garden/bp/2022-12-28/statistical_review
+  data://grapher/energy/2023-06-01/global_primary_energy:
+    - data://garden/energy/2023-06-01/global_primary_energy
+  #
+  # Ember - Yearly electricity data 2023.
+  #
+  data://meadow/ember/2023-06-01/yearly_electricity:
+    - snapshot://ember/2023-06-01/yearly_electricity.csv
+  data://garden/ember/2023-06-01/yearly_electricity:
+    - data://meadow/ember/2023-06-01/yearly_electricity
+    - data://garden/demography/2023-03-31/population
+    - data://garden/wb/2023-04-30/income_groups
+    - data://garden/regions/2023-01-01/regions
+  #
+  # Ember - Combined electricity 2023.
+  #
+  # We still use EER 2022 for data from 1990-1999, which is not included in the 2023 version.
+  data://garden/ember/2023-06-01/combined_electricity:
+    - data://garden/ember/2022-08-01/european_electricity_review
+    - data://garden/ember/2023-06-01/yearly_electricity
+  #
+  # Energy - Electricity mix (BP & Ember, 2023).
+  #
+  data://garden/energy/2023-06-01/electricity_mix:
+    - data://garden/bp/2022-12-28/statistical_review
+    - data://garden/ember/2023-06-01/combined_electricity
+    - data://garden/demography/2023-03-31/population
+  data://grapher/energy/2023-06-01/electricity_mix:
+    - data://garden/energy/2023-06-01/electricity_mix
+  #
+  # UK BEIS - UK historical electricity (2022).
+  #
+  data://meadow/uk_beis/2022-07-28/uk_historical_electricity:
+    - walden://uk_beis/2022-07-28/uk_historical_electricity
+  data://garden/uk_beis/2022-07-28/uk_historical_electricity:
+    - data://meadow/uk_beis/2022-07-28/uk_historical_electricity
+  #
+  # Energy - UK historical electricity (2023).
+  #
+  data://garden/energy/2023-06-01/uk_historical_electricity:
+    - data://garden/uk_beis/2022-07-28/uk_historical_electricity
+    - data://garden/energy/2023-06-01/electricity_mix
+  data://grapher/energy/2023-06-01/uk_historical_electricity:
+    - data://garden/energy/2023-06-01/uk_historical_electricity
+  #
+  # BP - Energy mix 2023.
+  #
+  data://garden/bp/2023-02-20/energy_mix:
+    - data://garden/bp/2022-12-28/statistical_review
+    - data://garden/demography/2022-12-08/population
+    - data://garden/wb/2021-07-01/wb_income
+  data://grapher/bp/2023-02-20/energy_mix:
+    - data://garden/bp/2023-02-20/energy_mix

dag/archive/main.yml

@@ -77,8 +77,8 @@ include:
   - dag/archive/energy.yml
   # - dag/archive/walkthrough.yml
   # - dag/archive/examples.yml
-  # - dag/archive/emissions.yml
-  # - dag/archive/papers.yml
+  - dag/archive/emissions.yml
+  - dag/archive/papers.yml
   - dag/archive/demography.yml
   # - dag/archive/war.yml
   - dag/archive/fasttrack.yml

dag/archive/papers.yml

@@ -0,0 +1,17 @@
+steps:
+  #
+  # Farmer & Lafond (2016).
+  #
+  data://meadow/papers/2023-01-04/farmer_lafond_2016:
+    - snapshot://papers/2023-01-04/farmer_lafond_2016.csv
+  data://garden/papers/2023-01-04/farmer_lafond_2016:
+    - data://meadow/papers/2023-01-04/farmer_lafond_2016
+  data://grapher/papers/2023-01-04/farmer_lafond_2016:
+    - data://garden/papers/2023-01-04/farmer_lafond_2016
+  #
+  # Nemet (2009).
+  #
+  data://meadow/papers/2023-01-04/nemet_2009:
+    - snapshot://papers/2023-01-04/nemet_2009.csv
+  data://garden/papers/2023-01-04/nemet_2009:
+    - data://meadow/papers/2023-01-04/nemet_2009

dag/emissions.yml

@@ -17,17 +17,6 @@ steps:
   data://grapher/cait/2022-08-10/n2o_emissions:
     - data://garden/cait/2022-08-10/ghg_emissions_by_sector
   #
-  # Emissions - CO2 dataset (2022).
-  #
-  data://garden/emissions/2023-05-03/owid_co2:
-    - data://garden/emissions/2023-05-02/national_contributions
-    - data://garden/gcp/2023-04-28/global_carbon_budget
-    - data://garden/cait/2022-08-10/ghg_emissions_by_sector
-    - data://garden/energy/2023-06-01/primary_energy_consumption
-    - data://garden/demography/2022-12-08/population
-    - data://garden/ggdc/2020-10-01/ggdc_maddison
-    - data://garden/regions/2023-01-01/regions
-  #
   # Andrew - CO2 mitigation curves (2019).
   #
   data://meadow/andrew/2019-12-03/co2_mitigation_curves:
@@ -40,26 +29,37 @@ steps:
   data://grapher/andrew/2019-12-03/co2_mitigation_curves_2celsius:
     - data://garden/andrew/2019-12-03/co2_mitigation_curves
   #
-  # GCP - Global Carbon Budget (2023-04-28).
+  # GCP - Global Carbon Budget (2023-07-10).
   #
-  data://meadow/gcp/2023-04-28/global_carbon_budget:
+  data://meadow/gcp/2023-07-10/global_carbon_budget:
     - snapshot://gcp/2023-04-28/global_carbon_budget_fossil_co2_emissions.csv
     - snapshot://gcp/2023-04-28/global_carbon_budget_global_emissions.xlsx
     - snapshot://gcp/2023-04-28/global_carbon_budget_national_emissions.xlsx
     - snapshot://gcp/2023-04-28/global_carbon_budget_land_use_change_emissions.xlsx
-  data://garden/gcp/2023-04-28/global_carbon_budget:
-    - data://meadow/gcp/2023-04-28/global_carbon_budget
+  data://garden/gcp/2023-07-10/global_carbon_budget:
+    - data://meadow/gcp/2023-07-10/global_carbon_budget
     # Loaded to calculate emissions per unit energy.
-    - data://garden/energy/2023-06-01/primary_energy_consumption
+    - data://garden/energy/2023-07-10/primary_energy_consumption
     # Loaded to calculate emissions per GDP.
     - data://garden/ggdc/2020-10-01/ggdc_maddison
     # Loaded to create per-capita variables.
-    - data://garden/demography/2022-12-08/population
+    - data://garden/demography/2023-03-31/population
     # Loaded to create region aggregates (including income groups).
     - data://garden/regions/2023-01-01/regions
-    - data://garden/wb/2021-07-01/wb_income
-  data://grapher/gcp/2023-04-28/global_carbon_budget:
-    - data://garden/gcp/2023-04-28/global_carbon_budget
+    - data://garden/wb/2023-04-30/income_groups
+  data://grapher/gcp/2023-07-10/global_carbon_budget:
+    - data://garden/gcp/2023-07-10/global_carbon_budget
+  #
+  # Emissions - CO2 dataset (2023).
+  #
+  data://garden/emissions/2023-07-10/owid_co2:
+    - data://garden/emissions/2023-05-02/national_contributions
+    - data://garden/gcp/2023-07-10/global_carbon_budget
+    - data://garden/cait/2022-08-10/ghg_emissions_by_sector
+    - data://garden/energy/2023-07-10/primary_energy_consumption
+    - data://garden/demography/2023-03-31/population
+    - data://garden/ggdc/2020-10-01/ggdc_maddison
+    - data://garden/regions/2023-01-01/regions
   #
   # RFF - World Carbon Pricing (2022-10-11).
   #
@@ -97,3 +97,7 @@ steps:
     - data://garden/demography/2022-12-08/population
   data://grapher/emissions/2023-05-02/national_contributions:
     - data://garden/emissions/2023-05-02/national_contributions
+  ######################################################################################################################
+  # Older versions to be archived once they are not used by any other steps.
+
+  ######################################################################################################################