notebooks/statements_preview.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('./../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from src.input_data.table import Table\n",
    "from src.io import read_jsonl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['structured_data', 'source', 'cell_annotations', 'unstructured_data', 'ud_type', 'augmentation_status', 'current_hash'])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read data (download data to appropriate directory before running the following cells)\n",
    "dfile = Path(\"./../training_data/se_direct/test.jsonl\",)\n",
    "data = read_jsonl(dfile.parent, dfile.name, lines_to_read=100)\n",
    "# let's see the keys inside one item\n",
    "data[2].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'augmented'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# is this table augmented or original\n",
    "data[2]['augmentation_status']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Australia - Scope 1</td>\n",
       "      <td>2021</td>\n",
       "      <td>2020</td>\n",
       "      <td>2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Emissions from fuel consumption (tCO$_{2}$e)</td>\n",
       "      <td>17,957</td>\n",
       "      <td>14,574</td>\n",
       "      <td>19,561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Emissions from Killara Feedlot cattle (tCO$_{2...</td>\n",
       "      <td>37,462</td>\n",
       "      <td>-</td>\n",
       "      <td>44,826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Fuel consumption (GJ)</td>\n",
       "      <td>255,873</td>\n",
       "      <td>207,569</td>\n",
       "      <td>278,969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Australia - Scope 2</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Electricity consumption from the grid (GJ)</td>\n",
       "      <td>24,201</td>\n",
       "      <td>41,051</td>\n",
       "      <td>29,104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Total renewable electricity (MWh)</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>8,084 (100%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Voluntary LGCs</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>6,581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Electricity consumption from the grid (MWh)</td>\n",
       "      <td>6,722</td>\n",
       "      <td>11,403</td>\n",
       "      <td>8,084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Mandatory LGCs 1</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>1,503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Emissions from purchased electricity (tCO$_{2}...</td>\n",
       "      <td>4,982</td>\n",
       "      <td>8,946</td>\n",
       "      <td>5,801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Total energy (Australia)</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Total energy consumption (GJ)</td>\n",
       "      <td>280,074</td>\n",
       "      <td></td>\n",
       "      <td>308,073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>China - Scope 2</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Electricity consumption from the grid (GJ)</td>\n",
       "      <td>2,466</td>\n",
       "      <td>-</td>\n",
       "      <td>2,580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Scope 2 GHG emissions from electricity consump...</td>\n",
       "      <td>427</td>\n",
       "      <td>-</td>\n",
       "      <td>385</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    0         1         2  \\\n",
       "0                                Australia - Scope 1      2021      2020    \n",
       "1       Emissions from fuel consumption (tCO$_{2}$e)    17,957    14,574    \n",
       "2   Emissions from Killara Feedlot cattle (tCO$_{2...   37,462         -    \n",
       "3                              Fuel consumption (GJ)   255,873   207,569    \n",
       "4                                Australia - Scope 2                        \n",
       "5         Electricity consumption from the grid (GJ)    24,201    41,051    \n",
       "6                  Total renewable electricity (MWh)         -         -    \n",
       "7                                     Voluntary LGCs         -         -    \n",
       "8        Electricity consumption from the grid (MWh)     6,722    11,403    \n",
       "9                                   Mandatory LGCs 1         -         -    \n",
       "10  Emissions from purchased electricity (tCO$_{2}...    4,982     8,946    \n",
       "11                          Total energy (Australia)                        \n",
       "12                     Total energy consumption (GJ)   280,074              \n",
       "13                                   China - Scope 2                        \n",
       "14        Electricity consumption from the grid (GJ)     2,466         -    \n",
       "15  Scope 2 GHG emissions from electricity consump...      427         -    \n",
       "\n",
       "                3  \n",
       "0           2022   \n",
       "1         19,561   \n",
       "2         44,826   \n",
       "3        278,969   \n",
       "4                  \n",
       "5         29,104   \n",
       "6   8,084 (100%)   \n",
       "7          6,581   \n",
       "8          8,084   \n",
       "9          1,503   \n",
       "10         5,801   \n",
       "11                 \n",
       "12       308,073   \n",
       "13                 \n",
       "14         2,580   \n",
       "15           385   "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# original table\n",
    "t = Table(table=pd.DataFrame(data[2]['unstructured_data']), annotations=data[2]['structured_data'],)\n",
    "t.to_dataframe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'property': ['Australia - Scope 1 : Emissions from fuel consumption (tCO$_{2}$e) ',\n",
       "   'time'],\n",
       "  'property_value': ['17,957 ', '2021 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['595552ba529257f919c4da9696e07990',\n",
       "   '595552ba529257f919c4da9696e07990']},\n",
       " {'property': ['Australia - Scope 1 : Emissions from fuel consumption (tCO$_{2}$e) ',\n",
       "   'time'],\n",
       "  'property_value': ['14,574 ', '2020 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['3c47364a596f1ca1d3850cd41d87568c',\n",
       "   '3c47364a596f1ca1d3850cd41d87568c']},\n",
       " {'property': ['Australia - Scope 1 : Emissions from fuel consumption (tCO$_{2}$e) ',\n",
       "   'time'],\n",
       "  'property_value': ['19,561 ', '2022 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['de51e3a418aa32420ca753dbc6726e74',\n",
       "   'de51e3a418aa32420ca753dbc6726e74']},\n",
       " {'property': ['Australia - Scope 1 : Emissions from Killara Feedlot cattle (tCO$_{2}$e) ',\n",
       "   'time'],\n",
       "  'property_value': ['37,462 ', '2021 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['b895e214c874a3c3d6296d88a74e7549',\n",
       "   'b895e214c874a3c3d6296d88a74e7549']},\n",
       " {'property': ['Australia - Scope 1 : Emissions from Killara Feedlot cattle (tCO$_{2}$e) ',\n",
       "   'time'],\n",
       "  'property_value': ['- ', '2020 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['afb6df06a5b4329a9c28b3c8bf805542',\n",
       "   'afb6df06a5b4329a9c28b3c8bf805542']},\n",
       " {'property': ['Australia - Scope 1 : Emissions from Killara Feedlot cattle (tCO$_{2}$e) ',\n",
       "   'time'],\n",
       "  'property_value': ['44,826 ', '2022 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['63577205cba0c9d1e4599e11d37fad73',\n",
       "   '63577205cba0c9d1e4599e11d37fad73']},\n",
       " {'property': ['Australia - Scope 1 : Fuel consumption (GJ) ', 'time'],\n",
       "  'property_value': ['255,873 ', '2021 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['b46822f70c50460fcd62a54e2d212069',\n",
       "   'b46822f70c50460fcd62a54e2d212069']},\n",
       " {'property': ['Australia - Scope 1 : Fuel consumption (GJ) ', 'time'],\n",
       "  'property_value': ['207,569 ', '2020 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['41ac91d96e4c2f26b69d38e5e8b0a4a7',\n",
       "   '41ac91d96e4c2f26b69d38e5e8b0a4a7']},\n",
       " {'property': ['Australia - Scope 1 : Fuel consumption (GJ) ', 'time'],\n",
       "  'property_value': ['278,969 ', '2022 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['28b5d98e13521337f8273f89099bb5d7',\n",
       "   '28b5d98e13521337f8273f89099bb5d7']},\n",
       " {'property': ['Australia - Scope 2 : Electricity consumption from the grid (GJ) ',\n",
       "   'time'],\n",
       "  'property_value': ['24,201 ', '2021 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['619dde49745267f9b837e5ff66f731d9',\n",
       "   '619dde49745267f9b837e5ff66f731d9']},\n",
       " {'property': ['Australia - Scope 2 : Electricity consumption from the grid (GJ) ',\n",
       "   'time'],\n",
       "  'property_value': ['41,051 ', '2020 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['ee25e8198cf154dc673671f544d7241d',\n",
       "   'ee25e8198cf154dc673671f544d7241d']},\n",
       " {'property': ['Australia - Scope 2 : Electricity consumption from the grid (GJ) ',\n",
       "   'time'],\n",
       "  'property_value': ['29,104 ', '2022 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['83b1fb5b29e835f7ef6640525fb9eb09',\n",
       "   '83b1fb5b29e835f7ef6640525fb9eb09']},\n",
       " {'property': ['Australia - Scope 2 : Total renewable electricity (MWh) ',\n",
       "   'time'],\n",
       "  'property_value': ['- ', '2021 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['91411c193cb3c0d0dd05fdfd90467f6a',\n",
       "   '91411c193cb3c0d0dd05fdfd90467f6a']},\n",
       " {'property': ['Australia - Scope 2 : Total renewable electricity (MWh) ',\n",
       "   'time'],\n",
       "  'property_value': ['- ', '2020 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['f41d02fe8e8b108dae0fd77115e277bd',\n",
       "   'f41d02fe8e8b108dae0fd77115e277bd']},\n",
       " {'property': ['Australia - Scope 2 : Total renewable electricity (MWh) ',\n",
       "   'time'],\n",
       "  'property_value': ['8,084 (100%) ', '2022 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['fb390821e754554b1630a9754c5bcb30',\n",
       "   'fb390821e754554b1630a9754c5bcb30']},\n",
       " {'property': ['Australia - Scope 2 : Voluntary LGCs ', 'time'],\n",
       "  'property_value': ['- ', '2021 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['774201908a6afe20286ab6f7295696a5',\n",
       "   '774201908a6afe20286ab6f7295696a5']},\n",
       " {'property': ['Australia - Scope 2 : Voluntary LGCs ', 'time'],\n",
       "  'property_value': ['- ', '2020 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['d3afdc94957ce537e309228d90b12e12',\n",
       "   'd3afdc94957ce537e309228d90b12e12']},\n",
       " {'property': ['Australia - Scope 2 : Voluntary LGCs ', 'time'],\n",
       "  'property_value': ['6,581 ', '2022 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['4f9b99108f01a47294355a02093ab814',\n",
       "   '4f9b99108f01a47294355a02093ab814']},\n",
       " {'property': ['Australia - Scope 2 : Electricity consumption from the grid (MWh) ',\n",
       "   'time'],\n",
       "  'property_value': ['6,722 ', '2021 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['4bf623d3353cae0e6089de2347617ce0',\n",
       "   '4bf623d3353cae0e6089de2347617ce0']},\n",
       " {'property': ['Australia - Scope 2 : Electricity consumption from the grid (MWh) ',\n",
       "   'time'],\n",
       "  'property_value': ['11,403 ', '2020 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['892ff26f222477e5af1669761046cb6e',\n",
       "   '892ff26f222477e5af1669761046cb6e']},\n",
       " {'property': ['Australia - Scope 2 : Electricity consumption from the grid (MWh) ',\n",
       "   'time'],\n",
       "  'property_value': ['8,084 ', '2022 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['ef5bf235c912d23df1c98cb7dfbcb361',\n",
       "   'ef5bf235c912d23df1c98cb7dfbcb361']},\n",
       " {'property': ['Australia - Scope 2 : Mandatory LGCs 1 ', 'time'],\n",
       "  'property_value': ['- ', '2021 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['8d964d8a0cab1432a239d40007cabcf2',\n",
       "   '8d964d8a0cab1432a239d40007cabcf2']},\n",
       " {'property': ['Australia - Scope 2 : Mandatory LGCs 1 ', 'time'],\n",
       "  'property_value': ['- ', '2020 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['42ee4762be45ef2ead31307d5dd4f06f',\n",
       "   '42ee4762be45ef2ead31307d5dd4f06f']},\n",
       " {'property': ['Australia - Scope 2 : Mandatory LGCs 1 ', 'time'],\n",
       "  'property_value': ['1,503 ', '2022 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['69b63402ca7872c464929fc29eb7b2c6',\n",
       "   '69b63402ca7872c464929fc29eb7b2c6']},\n",
       " {'property': ['Australia - Scope 2 : Emissions from purchased electricity (tCO$_{2}$e) 2 ',\n",
       "   'time'],\n",
       "  'property_value': ['4,982 ', '2021 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['102a90a9d0aff698db93118a33d52909',\n",
       "   '102a90a9d0aff698db93118a33d52909']},\n",
       " {'property': ['Australia - Scope 2 : Emissions from purchased electricity (tCO$_{2}$e) 2 ',\n",
       "   'time'],\n",
       "  'property_value': ['8,946 ', '2020 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['86ec01bacc422eea2f6ed2234ee1dffa',\n",
       "   '86ec01bacc422eea2f6ed2234ee1dffa']},\n",
       " {'property': ['Australia - Scope 2 : Emissions from purchased electricity (tCO$_{2}$e) 2 ',\n",
       "   'time'],\n",
       "  'property_value': ['5,801 ', '2022 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['1e3a47201eb1cbc58013558a72cd8664',\n",
       "   '1e3a47201eb1cbc58013558a72cd8664']},\n",
       " {'property': ['Total energy (Australia) : Total energy consumption (GJ) ',\n",
       "   'time'],\n",
       "  'property_value': ['280,074 ', '2021 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['04a785e807e256e31731719570bfe591',\n",
       "   '04a785e807e256e31731719570bfe591']},\n",
       " {'property': ['Total energy (Australia) : Total energy consumption (GJ) ',\n",
       "   'time'],\n",
       "  'property_value': ['308,073 ', '2022 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['28f782167551282313cf0ffd75b9c3f0',\n",
       "   '28f782167551282313cf0ffd75b9c3f0']},\n",
       " {'property': ['China - Scope 2 : Electricity consumption from the grid (GJ) ',\n",
       "   'time'],\n",
       "  'property_value': ['2,466 ', '2021 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['8e9a6ca887beca4352ebe30d755e5a94',\n",
       "   '8e9a6ca887beca4352ebe30d755e5a94']},\n",
       " {'property': ['China - Scope 2 : Electricity consumption from the grid (GJ) ',\n",
       "   'time'],\n",
       "  'property_value': ['- ', '2020 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['d713c3183164c5b308db62f30f7c07d7',\n",
       "   'd713c3183164c5b308db62f30f7c07d7']},\n",
       " {'property': ['China - Scope 2 : Electricity consumption from the grid (GJ) ',\n",
       "   'time'],\n",
       "  'property_value': ['2,580 ', '2022 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['a3079838da2ab44e6b71e0b473e89397',\n",
       "   'a3079838da2ab44e6b71e0b473e89397']},\n",
       " {'property': ['China - Scope 2 : Scope 2 GHG emissions from electricity consumption (tCO$_{2}$e) ',\n",
       "   'time'],\n",
       "  'property_value': ['427 ', '2021 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['29929d87e2efef8989b207f471272249',\n",
       "   '29929d87e2efef8989b207f471272249']},\n",
       " {'property': ['China - Scope 2 : Scope 2 GHG emissions from electricity consumption (tCO$_{2}$e) ',\n",
       "   'time'],\n",
       "  'property_value': ['- ', '2020 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['2e70360ccc972c77814f699b39362a07',\n",
       "   '2e70360ccc972c77814f699b39362a07']},\n",
       " {'property': ['China - Scope 2 : Scope 2 GHG emissions from electricity consumption (tCO$_{2}$e) ',\n",
       "   'time'],\n",
       "  'property_value': ['385 ', '2022 '],\n",
       "  'unit': ['', ''],\n",
       "  'subject': ['', ''],\n",
       "  'subject_value': ['', ''],\n",
       "  'predicate_hash': ['704e945ff24896da015ea13c4b3f2ff4',\n",
       "   '704e945ff24896da015ea13c4b3f2ff4']}]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.annotations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example of converting labels table to statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['table', 'collection', 'annotations', 'verifications', 'classification', 'cell_annotations', 'ds_table', 'source', 'verified', 'augmentation_status'])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read data (download data to appropriate directory before running the following cells)\n",
    "dfile = Path(\"./../training_data/se_indirect_2d/test.jsonl\",)\n",
    "data = read_jsonl(dfile.parent, dfile.name, lines_to_read=100)\n",
    "data[0].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'original'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# is this table augmented or original\n",
    "data[0]['augmentation_status']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Emissions in scope</td>\n",
       "      <td>Base year emissions (kiloton CO$_{2}$e)</td>\n",
       "      <td>Reduction  target (%)</td>\n",
       "      <td>Base year</td>\n",
       "      <td>Target year</td>\n",
       "      <td>Scope</td>\n",
       "      <td>Use of offsets  or removals</td>\n",
       "      <td>SBTi status</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Scope 1</td>\n",
       "      <td>75</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Scope 2 (market-based)</td>\n",
       "      <td>185</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Scope 3:</td>\n",
       "      <td></td>\n",
       "      <td>35 (all scopes)</td>\n",
       "      <td>2016</td>\n",
       "      <td>2022</td>\n",
       "      <td>Company-wide</td>\n",
       "      <td>None</td>\n",
       "      <td>Verified (2017)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Business travel</td>\n",
       "      <td>154</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Downstream transportation</td>\n",
       "      <td>146</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Total</td>\n",
       "      <td>560</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            0                                         1  \\\n",
       "0         Emissions in scope   Base year emissions (kiloton CO$_{2}$e)    \n",
       "1                    Scope 1                                        75    \n",
       "2     Scope 2 (market-based)                                       185    \n",
       "3                  Scope 3:                                               \n",
       "4            Business travel                                       154    \n",
       "5  Downstream transportation                                       146    \n",
       "6                      Total                                       560    \n",
       "\n",
       "                        2           3             4              5  \\\n",
       "0  Reduction  target (%)   Base year   Target year          Scope    \n",
       "1                                                                    \n",
       "2                                                                    \n",
       "3        35 (all scopes)        2016          2022   Company-wide    \n",
       "4                                                                    \n",
       "5                                                                    \n",
       "6                                                                    \n",
       "\n",
       "                               6                 7  \n",
       "0  Use of offsets  or removals        SBTi status   \n",
       "1                                                   \n",
       "2                                                   \n",
       "3                          None   Verified (2017)   \n",
       "4                                                   \n",
       "5                                                   \n",
       "6                                                   "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# original table\n",
    "t = Table(table=pd.DataFrame(data[0]['table']), annotations=data[0]['annotations'],cell_annotations=pd.DataFrame(data[0]['cell_annotations']))\n",
    "t.to_dataframe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# currently the statements are empty\n",
    "t.annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>header_1</td>\n",
       "      <td>unit_value</td>\n",
       "      <td>key</td>\n",
       "      <td>time_property</td>\n",
       "      <td>key</td>\n",
       "      <td>key</td>\n",
       "      <td>key</td>\n",
       "      <td>key</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>property</td>\n",
       "      <td>property_value</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>property</td>\n",
       "      <td>property_value</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>property</td>\n",
       "      <td>empty</td>\n",
       "      <td>key_value</td>\n",
       "      <td>time_value</td>\n",
       "      <td>key_value</td>\n",
       "      <td>key_value</td>\n",
       "      <td>key_value</td>\n",
       "      <td>key_value</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>property</td>\n",
       "      <td>property_value</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>property</td>\n",
       "      <td>property_value</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>property</td>\n",
       "      <td>property_value</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "      <td>empty</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0               1          2              3          4          5  \\\n",
       "0  header_1      unit_value        key  time_property        key        key   \n",
       "1  property  property_value      empty          empty      empty      empty   \n",
       "2  property  property_value      empty          empty      empty      empty   \n",
       "3  property           empty  key_value     time_value  key_value  key_value   \n",
       "4  property  property_value      empty          empty      empty      empty   \n",
       "5  property  property_value      empty          empty      empty      empty   \n",
       "6  property  property_value      empty          empty      empty      empty   \n",
       "\n",
       "           6          7  \n",
       "0        key        key  \n",
       "1      empty      empty  \n",
       "2      empty      empty  \n",
       "3  key_value  key_value  \n",
       "4      empty      empty  \n",
       "5      empty      empty  \n",
       "6      empty      empty  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# labels table\n",
    "t.cell_annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'property': ['Emissions in scope : Scope 1 '],\n",
       "  'property_value': ['75 '],\n",
       "  'unit': ['Base year emissions (kiloton CO$_{2}$e) '],\n",
       "  'subject': [''],\n",
       "  'subject_value': ['']},\n",
       " {'property': ['Emissions in scope : Scope 2 (market-based) '],\n",
       "  'property_value': ['185 '],\n",
       "  'unit': ['Base year emissions (kiloton CO$_{2}$e) '],\n",
       "  'subject': [''],\n",
       "  'subject_value': ['']},\n",
       " {'property': ['Emissions in scope : Business travel '],\n",
       "  'property_value': ['154 '],\n",
       "  'unit': ['Base year emissions (kiloton CO$_{2}$e) '],\n",
       "  'subject': [''],\n",
       "  'subject_value': ['']},\n",
       " {'property': ['Emissions in scope : Downstream transportation '],\n",
       "  'property_value': ['146 '],\n",
       "  'unit': ['Base year emissions (kiloton CO$_{2}$e) '],\n",
       "  'subject': [''],\n",
       "  'subject_value': ['']},\n",
       " {'property': ['Emissions in scope : Total '],\n",
       "  'property_value': ['560 '],\n",
       "  'unit': ['Base year emissions (kiloton CO$_{2}$e) '],\n",
       "  'subject': [''],\n",
       "  'subject_value': ['']}]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert labels table to statements:\n",
    "t.convert_to_structured_data()\n",
    "t.annotations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}