📊 one: Official development assistance data

dag/main.yml

@@ -797,10 +797,9 @@ steps:
   data://meadow/oecd/2024-08-21/official_development_assistance:
     - snapshot://oecd/2024-08-21/official_development_assistance_dac1.zip
     - snapshot://oecd/2024-08-21/official_development_assistance_dac2a.zip
-    - snapshot://oecd/2024-08-21/official_development_assistance_dac5.zip
-    - snapshot://oecd/2024-08-21/official_development_assistance_crs.zip
   data://garden/oecd/2024-08-21/official_development_assistance:
     - data://meadow/oecd/2024-08-21/official_development_assistance
+    - data://garden/one/2024-10-02/official_development_assistance_one
     - data://garden/demography/2024-07-15/population
   data://grapher/oecd/2024-08-21/official_development_assistance:
     - data://garden/oecd/2024-08-21/official_development_assistance
@@ -812,6 +811,13 @@ steps:
     - data://meadow/itopf/2024-10-16/oil_spills
   data://grapher/itopf/2024-10-16/oil_spills:
     - data://garden/itopf/2024-10-16/oil_spills
+
+# OECD Official Development Assistance (ODA) via ONE
+  data://meadow/one/2024-10-02/official_development_assistance_one:
+    - snapshot://one/2024-10-02/official_development_assistance_one.feather
+  data://garden/one/2024-10-02/official_development_assistance_one:
+    - data://meadow/one/2024-10-02/official_development_assistance_one
+
 include:
   - dag/open_numbers.yml
   - dag/faostat.yml

etl/steps/data/garden/oecd/2024-08-21/official_development_assistance.meta.yml

@@ -39,7 +39,7 @@ definitions:
 
   short_grant_equivalents: Monetary aid is estimated as [grant equivalents](#dod:grant-equivalents).
   short_net_disbursements: Monetary aid is estimated as net disbursements.
-  short_commitments_sectors: Monetary aid is estimated using commitment or gross disbursement data.
+  short_gross_sectors: Monetary aid is estimated using gross disbursement data.
 
   short_oof: Other official flows (OOF) are official sector transactions that do not meet [official development assistance (ODA)](#dod:oda) criteria.
   short_officially_supported_export_credits: Officially supported export credits are credits that governments provide to support national exporters competing for overseas sales.
@@ -93,19 +93,19 @@ definitions:
     Official development assistance does not include military aid, except for the cost of using armed forces to deliver humanitarian aid. It also excludes most peacekeeping expenditures, except for those closely related to development. Nuclear energy is included only if it is for civilian purposes, and cultural programs are eligible only if they build the cultural capacities of recipient countries.
 
   key_sector: |-
-    <% if sector == "Humanitarian aid" %>
+    <% if sector == "VIII. Humanitarian aid" %>
     <<sector>> is assistance designed to save lives, alleviate suffering and maintain and protect human dignity during and in the aftermath of emergencies. To be classified as humanitarian, aid should be consistent with the humanitarian principles of humanity, impartiality, neutrality and independence. It broadly includes aid given for emergencies, such as natural disasters and wars, reconstruction in their aftermath, but also prevention and preparation for future emergencies.
     <% elif sector == "Non-humanitarian aid" %>
     <<sector>> broadly is aid for longer-term development, such as social and economic infrastructure.
-    <% elif sector == "Social infrastructure and services" %>
+    <% elif sector == "I. Social infrastructure and services" %>
     <<sector>> aid relates essentially to efforts to develop the human resource potential of developing countries. Includes education, health, population and reproductive health policies, water supply and sanitation, government and sanitation and other social services.
-    <% elif sector == "Economic infrastructure and services" %>
+    <% elif sector == "II. Economic infrastructure and services" %>
     <<sector>> aid groups assistance for networks, utilities and services that facilitate economic activity. Includes transport and storage, communications, energy, banking and financial services, and business and other services.
-    <% elif sector == "Production sectors" %>
+    <% elif sector == "III. Production sectors" %>
     <<sector>> aid groups contributions to all directly productive sectors. Includes agriculture, forestry, fishing, industry, mineral resources and mining, construction, trade and tourism.
-    <% elif sector == "Multi-sector / Cross-cutting" %>
+    <% elif sector == "IV. Multisector/cross-cutting" %>
     <<sector>> aid includes aid that is not sector-specific, such as aid for general environmental protection not allocable by sector.
-    <% elif sector == "Commodity aid / General programme assistance" %>
+    <% elif sector == "VI. Commodity aid / General programme assistance" %>
     <<sector>> aid includes general budget support, development food assistance and other commodity assistance.
     <%- endif -%>
 
@@ -167,16 +167,10 @@ definitions:
     We have combined net disbursements aid data from the [DAC1: Flows by donor (ODA+OOF+Private) dataset](https://data-explorer.oecd.org/vis?fs[0]=Topic%2C1%7CDevelopment%23DEV%23%7COfficial%20Development%20Assistance%20%28ODA%29%23DEV_ODA%23&pg=0&fc=Topic&bp=true&snb=20&df[ds]=dsDisseminateFinalDMZ&df[id]=DSD_DAC1%40DF_DAC1&df[ag]=OECD.DCD.FSD&df[vs]=1.2&dq=DAC...1140%2B1160..Q.&lom=LASTNPERIODS&lo=10&to[TIME_PERIOD]=false) with the [DAC2A: Aid (ODA) disbursements to countries and regions dataset](https://data-explorer.oecd.org/vis?fs[0]=Topic%2C1%7CDevelopment%23DEV%23%7COfficial%20Development%20Assistance%20%28ODA%29%23DEV_ODA%23&pg=0&fc=Topic&bp=true&snb=20&df[ds]=dsDisseminateFinalDMZ&df[id]=DSD_DAC2%40DF_DAC2A&df[ag]=OECD.DCD.FSD&df[vs]=1.1&dq=.DPGC.206.USD.Q&lom=LASTNPERIODS&lo=5&to[TIME_PERIOD]=false) to add aid given by multilateral organizations and grants given by private foundations.
 
   processing_oda_by_sector: |-
-    <% if sector == "Non-humanitarian aid" %>
-    We calculate non-humanitarian aid as the difference between total aid and humanitarian aid.
-    <%- endif -%>
+    We calculated this indicator by aggregating the gross bilateral and imputed multilateral flows processed by the [ONE Campaign](https://github.com/ONEcampaign/oda_data_package/) from the OECD data. We also aggregate sector categories to provide a more comprehensive view of aid given and received.
 
-  processing_level_by_sector: &processing_level_by_sector |-
-    <% if sector == "Non-humanitarian aid" %>
-    major
-    <% else %>
-    minor
-    <%- endif -%>
+  processing_oda_by_channel: |-
+    We calculated this indicator by aggregating the gross bilateral and imputed multilateral flows processed by the [ONE Campaign](https://github.com/ONEcampaign/oda_data_package/) from the OECD data. We also aggregate channel categories to provide a more comprehensive view of aid given and received.
 
 # Learn more about the available fields:
 # http://docs.owid.io/projects/etl/architecture/metadata/reference/
@@ -956,20 +950,20 @@ tables:
           numDecimalPlaces: 2
           <<: *common_display
 
-      # From DAC5 dataset
-      oda_by_sector:
+      # From ONE dataset, which processes CRS data from OECD
+      oda_by_sector_donor:
         title: ODA by donor and sector (<<sector>>)
         unit: "constant {definitions.inflation_year} US$"
         short_unit: "$"
-        description_short: "[Official development assistance](#dod:oda) given that is <<sector.lower()>>. {definitions.short_commitments_sectors} {definitions.short_constant_usd}"
+        description_short: "[Official development assistance](#dod:oda) given that is <<sector.lower()>>. {definitions.short_gross_sectors} {definitions.short_constant_usd}"
         description_key:
           - "{definitions.key_oda}"
           - "{definitions.key_sector}"
           - "{definitions.key_commitments_sectors}"
           - "{definitions.key_oda_coverage_limits}"
           - "{definitions.key_constant_usd}"
           - "{definitions.key_oda_list_of_countries}"
-        processing_level: *processing_level_by_sector
+        processing_level: major
         description_processing: "{definitions.processing_oda_by_sector}"
         presentation:
           title_public: ODA by donor and sector (<<sector>>)

etl/steps/data/garden/oecd/2024-08-21/official_development_assistance.py

@@ -66,18 +66,12 @@
             "Technical Cooperation": {"new_name": "technical_cooperation"},
         },
     },
-    "dac5": {
-        "aid_type": {
-            "Total ODA": {"new_name": "oda_by_sector"},
-        },
-    },
 }
 
 # Define indices for pivot tables.
 INDICES = {
     "dac1": ["country", "year"],
     "dac2a": ["country", "year", "donor"],
-    "dac5": ["country", "year", "sector"],
 }
 
 # Define type of donors to include from the recipient dataset.
@@ -91,58 +85,24 @@
 # Define official donors aggregation
 OFFICIAL_DONORS = {"Official donors (OECD)": "Official donors"}
 
-# Define sectors to include from the DAC5 dataset.
-SECTORS_DAC5 = {
-    "I. Social Infrastructure & Services": "Social infrastructure and services",
-    "II. Economic Infrastructure & Services": "Economic infrastructure and services",
-    "III. Production Sectors": "Production sectors",
-    "IV. Multi-Sector / Cross-Cutting": "Multi-sector / Cross-cutting",
-    "V. Total Sector Allocable (I+II+III+IV)": "Total sector allocable",
-    "VI. Commodity Aid / General Programme Assistance": "Commodity aid / General programme assistance",
-    "VII. Action Relating to Debt": "Action relating to debt",
-    "VIII. Humanitarian Aid": "Humanitarian aid",
-    "IX. Unallocated / Unspecified": "Unallocated / Unspecified",
-    "Total (V+VI+VII+VIII+IX)": "Total",
-    "I.1. Education": "Education",
-    "I.2. Health": "Health",
-    "I.3. Population Policies/Programmes & Reproductive Health": "Population policies/programmes and reproductive health",
-    "I.4. Water Supply & Sanitation": "Water supply and sanitation",
-    "I.5. Government & Civil Society": "Government and civil society",
-    "I.5.a. Government & Civil Society-general": "Government and civil society (subcategory)",
-    "I.5.b. Conflict, Peace & Security": "Conflict, peace and security",
-    "I.6. Other Social Infrastructure & Services": "Other social infrastructure and services",
-    "II.1. Transport & Storage": "Transport and storage",
-    "II.2. Communications": "Communications",
-    "II.3. Energy": "Energy",
-    "II.4. Banking & Financial Services": "Banking and financial services",
-    "II.5. Business & Other Services": "Business and other services",
-    "III.1. Agriculture, Forestry, Fishing": "Agriculture, forestry, fishing",
-    "III.2. Industry, Mining, Construction": "Industry, mining, construction",
-    "III.3.a. Trade Policies & Regulations": "Trade policies and regulations",
-    "III.3.b. Tourism": "Tourism",
-    "IV.1. General Environment Protection": "General environment protection",
-    "IV.2. Other Multisector": "Other multisector",
-    "VI.1. General Budget Support": "General budget support",
-    "VI.2. Development Food Assistance": "Development food assistance",
-    "VI.3. Other Commodity Assistance": "Other commodity assistance",
-    "VIII.1. Emergency Response": "Emergency response",
-    "VIII.2. Reconstruction Relief & Rehabilitation": "Reconstruction relief and rehabilitation",
-    "VIII.3. Disaster Prevention & Preparedness": "Disaster prevention and preparedness",
-}
-
 
 def run(dest_dir: str) -> None:
     #
     # Load inputs.
     #
     # Load meadow dataset.
     ds_meadow = paths.load_dataset("official_development_assistance")
+    ds_one = paths.load_dataset("official_development_assistance_one")
     ds_population = paths.load_dataset("population")
 
     # Read table from meadow dataset.
     tb_dac1 = ds_meadow["dac1"].reset_index()
     tb_dac2a = ds_meadow["dac2a"].reset_index()
-    tb_dac5 = ds_meadow["dac5"].reset_index()
+
+    tb_sectors_donor = ds_one["sectors_donor"].reset_index()
+    tb_sectors_recipient = ds_one["sectors_recipient"].reset_index()
+    tb_channels_donor = ds_one["channels_donor"].reset_index()
+    tb_channels_recipient = ds_one["channels_recipient"].reset_index()
 
     #
     # Process data.
@@ -163,14 +123,6 @@ def run(dest_dir: str) -> None:
         warn_on_unused_countries=False,
         warn_on_unknown_excluded_countries=False,
     )
-    tb_dac5 = geo.harmonize_countries(
-        df=tb_dac5,
-        country_col="donor",
-        countries_file=paths.country_mapping_path,
-        excluded_countries_file=paths.excluded_countries_path,
-        warn_on_unused_countries=False,
-        warn_on_unknown_excluded_countries=False,
-    )
 
     # Harmonize names of donors in tb_dac2a
     tb_dac2a = geo.harmonize_countries(
@@ -194,12 +146,6 @@ def run(dest_dir: str) -> None:
         columns_in_current_prices=["oda_share_gni"],
         recipient_or_donor="recipient",
     )
-    tb_dac5 = reformat_table_and_make_it_wide(
-        tb=tb_dac5,
-        short_name="dac5",
-        columns_in_current_prices=[],
-        recipient_or_donor="donor",
-    )
 
     tb_dac1 = create_indicators_per_capita(
         tb=tb_dac1, indicator_list=["i_oda_net_disbursements", "oda_grant_equivalents"]
@@ -219,7 +165,9 @@ def run(dest_dir: str) -> None:
 
     tb = add_recipient_dataset(tb=tb, tb_recipient=tb_dac2a)
 
-    tb = add_aid_by_sector_donor_dataset(tb=tb, tb_sector=tb_dac5)
+    tb = add_aid_by_sector(tb=tb, tb_sectors_donor=tb_sectors_donor, tb_sectors_recipient=tb_sectors_recipient)
+
+    tb = add_aid_by_channel(tb=tb, tb_channels_donor=tb_channels_donor, tb_channels_recipient=tb_channels_recipient)
 
     tb = create_indicators_per_capita_owid_population(
         tb=tb,
@@ -230,12 +178,15 @@ def run(dest_dir: str) -> None:
             "technical_cooperation_recipient",
             "development_food_aid_recipient",
             "humanitarian_aid_recipient",
-            "oda_by_sector",
+            "oda_by_sector_donor",
+            "oda_by_sector_recipient",
+            "oda_by_channel_donor",
+            "oda_by_channel_recipient",
         ],
         ds_population=ds_population,
     )
 
-    tb = tb.format(["country", "year", "donor", "sector"], short_name=paths.short_name)
+    tb = tb.format(["country", "year", "donor", "sector", "channel"], short_name=paths.short_name)
     tb_dac2a = tb_dac2a.format(["country", "year", "donor"])
 
     #
@@ -472,56 +423,58 @@ def create_indicators_per_capita_owid_population(tb: Table, indicator_list: List
     return tb
 
 
-def add_aid_by_sector_donor_dataset(tb: Table, tb_sector: Table) -> Table:
+def add_aid_by_sector(tb: Table, tb_sectors_donor: Table, tb_sectors_recipient: Table) -> Table:
     """
-    Add sector data to the main dataset.
-    This data comes from donor data in the DAC5 dataset.
+    Add sector data to the main dataset (both for donors and recipients)
+    This data comes from the ONE data package that processes OECD data.
     """
 
-    tb_sector = tb_sector.copy()
+    # For tb_sectors_donor, keep only the columns we need
+    tb_sectors_donor = tb_sectors_donor[["donor_name", "sector_name", "year", "value"]]
+    tb_sectors_donor = tb_sectors_donor.rename(
+        columns={"donor_name": "country", "sector_name": "sector", "value": "oda_by_sector_donor"}
+    )
 
-    # Assert if the sectors are in the sector dataset
-    assert set(SECTORS_DAC5.keys()).issubset(
-        set(tb_sector["sector"].unique())
-    ), f"There are missing sectors in the sector dataset: {set(SECTORS_DAC5) - set(tb_sector['sector'].unique())}"
+    # For tb_sectors_recipient, keep only the columns we need
+    tb_sectors_recipient = tb_sectors_recipient[["recipient_name", "sector_name", "year", "value"]]
+    tb_sectors_recipient = tb_sectors_recipient.rename(
+        columns={"recipient_name": "country", "sector_name": "sector", "value": "oda_by_sector_recipient"}
+    )
 
-    # Filter categories
-    tb_sector = tb_sector[tb_sector["sector"].isin(SECTORS_DAC5.keys())].reset_index(drop=True)
+    # Create an empty sector column in tb
+    tb["sector"] = None
 
-    # Rename sectors set in SECTORS_DAC5
-    tb_sector["sector"] = tb_sector["sector"].cat.rename_categories(SECTORS_DAC5)
+    # Merge tables
+    tb = pr.merge(tb, tb_sectors_donor, on=["country", "year", "sector"], how="outer")
+    tb = pr.merge(tb, tb_sectors_recipient, on=["country", "year", "sector"], how="outer")
 
-    # Create a new table with "Total", "Emergency response", "Reconstruction relief and rehabilitation"
-    tb_sector_humanitarian_aid = (
-        tb_sector[tb_sector["sector"].isin(["Total", "Humanitarian aid"])].reset_index(drop=True).copy()
-    )
+    return tb
 
-    # Make the table wide
-    tb_sector_humanitarian_aid = tb_sector_humanitarian_aid.pivot(
-        index=["country", "year"], columns="sector", values="oda_by_sector", join_column_levels_with="_"
-    ).reset_index(drop=True)
 
-    # Create the column "Non-humanitarian aid" as the difference between "Total" and "Humanitarian aid"
-    tb_sector_humanitarian_aid["Non-humanitarian aid"] = (
-        tb_sector_humanitarian_aid["Total"] - tb_sector_humanitarian_aid["Humanitarian aid"]
-    )
+def add_aid_by_channel(tb: Table, tb_channels_donor: Table, tb_channels_recipient: Table) -> Table:
+    """
+    Add channel data to the main dataset (both for donors and recipients)
+    This data comes from the ONE data package that processes OECD data.
+    """
 
-    # Make the table long again only using the columns "country", "year", "Emergency humanitarian aid" and "Non-emergency aid"
-    tb_sector_humanitarian_aid = tb_sector_humanitarian_aid.melt(
-        id_vars=["country", "year"],
-        value_vars=["Non-humanitarian aid"],
-        var_name="sector",
-        value_name="oda_by_sector",
+    # For tb_channels_donor, keep only the columns we need
+    tb_channels_donor = tb_channels_donor[["donor_name", "channel_name", "year", "value"]]
+    tb_channels_donor = tb_channels_donor.rename(
+        columns={"donor_name": "country", "channel_name": "channel", "value": "oda_by_channel_donor"}
     )
 
-    # Concatenate the sector tables
-    tb_sector = pr.concat([tb_sector, tb_sector_humanitarian_aid], ignore_index=True)
+    # For tb_channels_recipient, keep only the columns we need
+    tb_channels_recipient = tb_channels_recipient[["recipient_name", "channel_name", "year", "value"]]
+    tb_channels_recipient = tb_channels_recipient.rename(
+        columns={"recipient_name": "country", "channel_name": "channel", "value": "oda_by_channel_recipient"}
+    )
 
-    # Create an empty sector column in tb
-    tb["sector"] = None
+    # Create an empty channel column in tb
+    tb["channel"] = None
 
     # Merge tables
-    tb = pr.merge(tb, tb_sector, on=["country", "year", "sector"], how="outer")
+    tb = pr.merge(tb, tb_channels_donor, on=["country", "year", "channel"], how="outer")
+    tb = pr.merge(tb, tb_channels_recipient, on=["country", "year", "channel"], how="outer")
 
     return tb