nlpda2ques1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "nlpda2ques1",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1fe9uZjHl8mzmkDlgeEkEiDSO5z7rZLa2",
      "authorship_tag": "ABX9TyNVmUoN+F/TdjPJs1c7exWG",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/zhackerx/Nlp/blob/master/nlpda2ques1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ExNOot6oZwJR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 547
        },
        "outputId": "aaabec8d-fc34-4293-8552-b0eef6fa61e5"
      },
      "source": [
        "!pip install -U spacy \n",
        "!pip install sklearn_crfsuite"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already up-to-date: spacy in /usr/local/lib/python3.6/dist-packages (2.2.4)\n",
            "Requirement already satisfied, skipping upgrade: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.0.2)\n",
            "Requirement already satisfied, skipping upgrade: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (0.6.0)\n",
            "Requirement already satisfied, skipping upgrade: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (0.4.1)\n",
            "Requirement already satisfied, skipping upgrade: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (4.41.1)\n",
            "Requirement already satisfied, skipping upgrade: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.23.0)\n",
            "Requirement already satisfied, skipping upgrade: setuptools in /usr/local/lib/python3.6/dist-packages (from spacy) (46.3.0)\n",
            "Requirement already satisfied, skipping upgrade: thinc==7.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (7.4.0)\n",
            "Requirement already satisfied, skipping upgrade: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.1.3)\n",
            "Requirement already satisfied, skipping upgrade: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.0.0)\n",
            "Requirement already satisfied, skipping upgrade: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.0.3)\n",
            "Requirement already satisfied, skipping upgrade: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy) (3.0.2)\n",
            "Requirement already satisfied, skipping upgrade: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.0.2)\n",
            "Requirement already satisfied, skipping upgrade: numpy>=1.15.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.18.4)\n",
            "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2020.4.5.1)\n",
            "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.9)\n",
            "Requirement already satisfied, skipping upgrade: importlib-metadata>=0.20; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy) (1.6.0)\n",
            "Requirement already satisfied, skipping upgrade: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy) (3.1.0)\n",
            "Collecting sklearn_crfsuite\n",
            "  Downloading https://files.pythonhosted.org/packages/25/74/5b7befa513482e6dee1f3dd68171a6c9dfc14c0eaa00f885ffeba54fe9b0/sklearn_crfsuite-0.3.6-py2.py3-none-any.whl\n",
            "Collecting python-crfsuite>=0.8.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/95/99/869dde6dbf3e0d07a013c8eebfb0a3d30776334e0097f8432b631a9a3a19/python_crfsuite-0.9.7-cp36-cp36m-manylinux1_x86_64.whl (743kB)\n",
            "\u001b[K     |████████████████████████████████| 747kB 4.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sklearn_crfsuite) (1.12.0)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.6/dist-packages (from sklearn_crfsuite) (0.8.7)\n",
            "Requirement already satisfied: tqdm>=2.0 in /usr/local/lib/python3.6/dist-packages (from sklearn_crfsuite) (4.41.1)\n",
            "Installing collected packages: python-crfsuite, sklearn-crfsuite\n",
            "Successfully installed python-crfsuite-0.9.7 sklearn-crfsuite-0.3.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y3a4zTjvZ1J7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn_crfsuite import CRF\n",
        "from sklearn_crfsuite.metrics import flat_f1_score\n",
        "from sklearn_crfsuite.metrics import flat_classification_report"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TWEq7jpTavNO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.read_csv('/content/ner_dataset.csv', encoding = \"ISO-8859-1\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PQlNILSMcTTz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        },
        "outputId": "124ccd29-751c-4462-ab16-ec4ca4efe04b"
      },
      "source": [
        "df.head(10)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence #</th>\n",
              "      <th>Word</th>\n",
              "      <th>POS</th>\n",
              "      <th>Tag</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>Thousands</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>of</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>demonstrators</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>have</td>\n",
              "      <td>VBP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NaN</td>\n",
              "      <td>marched</td>\n",
              "      <td>VBN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>NaN</td>\n",
              "      <td>through</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>NaN</td>\n",
              "      <td>London</td>\n",
              "      <td>NNP</td>\n",
              "      <td>B-geo</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>NaN</td>\n",
              "      <td>to</td>\n",
              "      <td>TO</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>NaN</td>\n",
              "      <td>protest</td>\n",
              "      <td>VB</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>NaN</td>\n",
              "      <td>the</td>\n",
              "      <td>DT</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Sentence #           Word  POS    Tag\n",
              "0  Sentence: 1      Thousands  NNS      O\n",
              "1          NaN             of   IN      O\n",
              "2          NaN  demonstrators  NNS      O\n",
              "3          NaN           have  VBP      O\n",
              "4          NaN        marched  VBN      O\n",
              "5          NaN        through   IN      O\n",
              "6          NaN         London  NNP  B-geo\n",
              "7          NaN             to   TO      O\n",
              "8          NaN        protest   VB      O\n",
              "9          NaN            the   DT      O"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Mcm422CcVuz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "e7d96115-fcac-485e-a117-63e7ddb45538"
      },
      "source": [
        "df.describe()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence #</th>\n",
              "      <th>Word</th>\n",
              "      <th>POS</th>\n",
              "      <th>Tag</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>47959</td>\n",
              "      <td>1048575</td>\n",
              "      <td>1048575</td>\n",
              "      <td>1048575</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>unique</th>\n",
              "      <td>47959</td>\n",
              "      <td>35178</td>\n",
              "      <td>42</td>\n",
              "      <td>17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>top</th>\n",
              "      <td>Sentence: 221</td>\n",
              "      <td>the</td>\n",
              "      <td>NN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>freq</th>\n",
              "      <td>1</td>\n",
              "      <td>52573</td>\n",
              "      <td>145807</td>\n",
              "      <td>887908</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           Sentence #     Word      POS      Tag\n",
              "count           47959  1048575  1048575  1048575\n",
              "unique          47959    35178       42       17\n",
              "top     Sentence: 221      the       NN        O\n",
              "freq                1    52573   145807   887908"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0UgcRO_icYgX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df['Tag'].unique()\n",
        "df.isnull().sum()\n",
        "df = df.fillna(method = 'ffill')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "54n0V8PCcejU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class sentence(object):\n",
        "    def __init__(self, df):\n",
        "        self.n_sent = 1\n",
        "        self.df = df\n",
        "        self.empty = False\n",
        "        agg = lambda s : [(w, p, t) for w, p, t in zip(s['Word'].values.tolist(),\n",
        "                                                       s['POS'].values.tolist(),\n",
        "                                                       s['Tag'].values.tolist())]\n",
        "        self.grouped = self.df.groupby(\"Sentence #\").apply(agg)\n",
        "        self.sentences = [s for s in self.grouped]\n",
        "        \n",
        "    def get_text(self):\n",
        "        try:\n",
        "            s = self.grouped['Sentence: {}'.format(self.n_sent)]\n",
        "            self.n_sent +=1\n",
        "            return s\n",
        "        except:\n",
        "            return None"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "szPmSdIVchnM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "010fe094-bd94-4392-8c11-1da031cbf0f6"
      },
      "source": [
        "getter = sentence(df)\n",
        "sentences = [\" \".join([s[0] for s in sent]) for sent in getter.sentences]\n",
        "sentences[0]\n",
        "sent[0][1]"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'NNS'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IlBRuiKVcjoS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "60069d50-e4d2-4724-d9c5-c38e05f5f93b"
      },
      "source": [
        "sent = getter.get_text()\n",
        "print(sent)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('Thousands', 'NNS', 'O'), ('of', 'IN', 'O'), ('demonstrators', 'NNS', 'O'), ('have', 'VBP', 'O'), ('marched', 'VBN', 'O'), ('through', 'IN', 'O'), ('London', 'NNP', 'B-geo'), ('to', 'TO', 'O'), ('protest', 'VB', 'O'), ('the', 'DT', 'O'), ('war', 'NN', 'O'), ('in', 'IN', 'O'), ('Iraq', 'NNP', 'B-geo'), ('and', 'CC', 'O'), ('demand', 'VB', 'O'), ('the', 'DT', 'O'), ('withdrawal', 'NN', 'O'), ('of', 'IN', 'O'), ('British', 'JJ', 'B-gpe'), ('troops', 'NNS', 'O'), ('from', 'IN', 'O'), ('that', 'DT', 'O'), ('country', 'NN', 'O'), ('.', '.', 'O')]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vHBehP8ecm5b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def word2features(sent, i):\n",
        "    print(sent[0][1])\n",
        "    word = sent[i][0]\n",
        "    postag = sent[i][1]\n",
        "\n",
        "    features = {\n",
        "        'bias': 1.0,\n",
        "        'word.lower()': word.lower(),\n",
        "        'word[-3:]': word[-3:],\n",
        "        'word[-2:]': word[-2:],\n",
        "        'word.isupper()': word.isupper(),\n",
        "        'word.istitle()': word.istitle(),\n",
        "        'word.isdigit()': word.isdigit(),\n",
        "        'postag': postag,\n",
        "        'postag[:2]': postag[:2],\n",
        "    }\n",
        "    if i > 0:\n",
        "        word1 = sent[i-1][0]\n",
        "        postag1 = sent[i-1][1]\n",
        "        features.update({\n",
        "            '-1:word.lower()': word1.lower(),\n",
        "            '-1:word.istitle()': word1.istitle(),\n",
        "            '-1:word.isupper()': word1.isupper(),\n",
        "            '-1:postag': postag1,\n",
        "            '-1:postag[:2]': postag1[:2],\n",
        "        })\n",
        "    else:\n",
        "        features['BOS'] = True\n",
        "\n",
        "    if i < len(sent)-1:\n",
        "        word1 = sent[i+1][0]\n",
        "        postag1 = sent[i+1][1]\n",
        "        features.update({\n",
        "            '+1:word.lower()': word1.lower(),\n",
        "            '+1:word.istitle()': word1.istitle(),\n",
        "            '+1:word.isupper()': word1.isupper(),\n",
        "            '+1:postag': postag1,\n",
        "            '+1:postag[:2]': postag1[:2],\n",
        "        })\n",
        "    else:\n",
        "        features['EOS'] = True\n",
        "\n",
        "    return features\n",
        "\n",
        "\n",
        "def sent2features(sent):\n",
        "    return [word2features(sent, i) for i in range(len(sent))]\n",
        "\n",
        "def sent2labels(sent):\n",
        "    return [label for token, postag, label in sent]\n",
        "\n",
        "def sent2tokens(sent):\n",
        "    return [token for token, postag, label in sent]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n9JxSzKYgG48",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sentences = getter.sentences"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LqmicgMqhLAw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "batch_size = 64 \n",
        "# Passes through entire dataset\n",
        "epochs = 8\n",
        "# Maximum length of review\n",
        "max_len = 75 \n",
        "# Dimension of embedding vector\n",
        "embedding = 40"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G0Rk-04XnyUz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "words = list(df['Word'].unique())\n",
        "tags = list(df['Tag'].unique())\n",
        "# Dictionary word:index pair\n",
        "# word is key and its value is corresponding index\n",
        "word_to_index = {w : i + 2 for i, w in enumerate(words)}\n",
        "word_to_index[\"UNK\"] = 1\n",
        "word_to_index[\"PAD\"] = 0\n",
        "\n",
        "# Dictionary lable:index pair\n",
        "# label is key and value is index.\n",
        "tag_to_index = {t : i + 1 for i, t in enumerate(tags)}\n",
        "tag_to_index[\"PAD\"] = 0\n",
        "\n",
        "idx2word = {i: w for w, i in word_to_index.items()}\n",
        "idx2tag = {i: w for w, i in tag_to_index.items()}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QIYrCkcZn2oX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "c6a73b8d-0fd3-4f7a-9e93-f92da0f91ed8"
      },
      "source": [
        "print(\"The word India is identified by the index: {}\".format(word_to_index[\"India\"]))\n",
        "print(\"The label B-org for the organization is identified by the index: {}\".format(tag_to_index[\"B-org\"]))"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The word India is identified by the index: 2570\n",
            "The label B-org for the organization is identified by the index: 6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BuYHXJOzoN0_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 343
        },
        "outputId": "b25f4538-a864-4c25-cca7-a59b18f9131f"
      },
      "source": [
        "!pip install git+https://www.github.com/keras-team/keras-contrib.git"
      ],
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting git+https://www.github.com/keras-team/keras-contrib.git\n",
            "  Cloning https://www.github.com/keras-team/keras-contrib.git to /tmp/pip-req-build-jk3n6fzp\n",
            "  Running command git clone -q https://www.github.com/keras-team/keras-contrib.git /tmp/pip-req-build-jk3n6fzp\n",
            "Requirement already satisfied: keras in /usr/local/lib/python3.6/dist-packages (from keras-contrib==2.0.8) (2.3.1)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from keras->keras-contrib==2.0.8) (1.12.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from keras->keras-contrib==2.0.8) (1.1.2)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from keras->keras-contrib==2.0.8) (1.0.8)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from keras->keras-contrib==2.0.8) (3.13)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras->keras-contrib==2.0.8) (2.10.0)\n",
            "Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.6/dist-packages (from keras->keras-contrib==2.0.8) (1.18.4)\n",
            "Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.6/dist-packages (from keras->keras-contrib==2.0.8) (1.4.1)\n",
            "Building wheels for collected packages: keras-contrib\n",
            "  Building wheel for keras-contrib (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-contrib: filename=keras_contrib-2.0.8-cp36-none-any.whl size=101064 sha256=f0334dc3a044d3b29cc24abe589a35bdabd8b0c6307dcb45bdca2387b81b1385\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-bsd7qs36/wheels/11/27/c8/4ed56de7b55f4f61244e2dc6ef3cdbaff2692527a2ce6502ba\n",
            "Successfully built keras-contrib\n",
            "Installing collected packages: keras-contrib\n",
            "Successfully installed keras-contrib-2.0.8\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dyOllNjnof_-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 972
        },
        "outputId": "4648044d-75af-4e00-888e-051f4d7110a8"
      },
      "source": [
        "!pip install simpletransformers"
      ],
      "execution_count": 79,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting simpletransformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fc/a6/555246c0e1f3aee73a722f153d25ad05f7b886558cc268a7aa824400f7da/simpletransformers-0.28.10-py3-none-any.whl (185kB)\n",
            "\u001b[K     |████████████████████████████████| 194kB 2.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (2019.12.20)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (1.4.1)\n",
            "Collecting tensorboardx\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/35/f1/5843425495765c8c2dd0784a851a93ef204d314fc87bcc2bbb9f662a3ad1/tensorboardX-2.0-py2.py3-none-any.whl (195kB)\n",
            "\u001b[K     |████████████████████████████████| 204kB 9.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (1.0.3)\n",
            "Collecting transformers>=2.9.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/12/b5/ac41e3e95205ebf53439e4dd087c58e9fd371fd8e3724f2b9b4cdb8282e5/transformers-2.10.0-py3-none-any.whl (660kB)\n",
            "\u001b[K     |████████████████████████████████| 665kB 7.9MB/s \n",
            "\u001b[?25hCollecting tokenizers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/14/e5/a26eb4716523808bb0a799fcfdceb6ebf77a18169d9591b2f46a9adb87d9/tokenizers-0.7.0-cp36-cp36m-manylinux1_x86_64.whl (3.8MB)\n",
            "\u001b[K     |████████████████████████████████| 3.8MB 18.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (2.23.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (4.41.1)\n",
            "Collecting seqeval\n",
            "  Downloading https://files.pythonhosted.org/packages/34/91/068aca8d60ce56dd9ba4506850e876aba5e66a6f2f29aa223224b50df0de/seqeval-0.0.12.tar.gz\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (0.22.2.post1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (1.18.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from tensorboardx->simpletransformers) (1.12.0)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from tensorboardx->simpletransformers) (3.10.0)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas->simpletransformers) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->simpletransformers) (2018.9)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers>=2.9.1->simpletransformers) (3.0.12)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 43.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers>=2.9.1->simpletransformers) (0.7)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 44.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->simpletransformers) (2.9)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->simpletransformers) (2020.4.5.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->simpletransformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->simpletransformers) (1.24.3)\n",
            "Requirement already satisfied: Keras>=2.2.4 in /usr/local/lib/python3.6/dist-packages (from seqeval->simpletransformers) (2.3.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->simpletransformers) (0.15.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->tensorboardx->simpletransformers) (46.3.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers>=2.9.1->simpletransformers) (7.1.2)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from Keras>=2.2.4->seqeval->simpletransformers) (2.10.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from Keras>=2.2.4->seqeval->simpletransformers) (1.1.2)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from Keras>=2.2.4->seqeval->simpletransformers) (1.0.8)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from Keras>=2.2.4->seqeval->simpletransformers) (3.13)\n",
            "Building wheels for collected packages: seqeval, sacremoses\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for seqeval: filename=seqeval-0.0.12-cp36-none-any.whl size=7424 sha256=7f67756fe55ec9653087719044383fb197b6fc0103779110e1f12b81150c60c3\n",
            "  Stored in directory: /root/.cache/pip/wheels/4f/32/0a/df3b340a82583566975377d65e724895b3fad101a3fb729f68\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=416eeee90dfb3674dd4397255af11e4f5145f169d865f4f2769ec23c9d323e5e\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built seqeval sacremoses\n",
            "Installing collected packages: tensorboardx, tokenizers, sacremoses, sentencepiece, transformers, seqeval, simpletransformers\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 seqeval-0.0.12 simpletransformers-0.28.10 tensorboardx-2.0 tokenizers-0.7.0 transformers-2.10.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KWQiWYrgoFeX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import to_categorical\n",
        "from keras.layers import LSTM, Dense, TimeDistributed, Embedding, Bidirectional\n",
        "from keras.models import Model, Input\n",
        "from keras_contrib.layers import CRF\n",
        "from keras.callbacks import ModelCheckpoint\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "from sklearn_crfsuite.metrics import flat_classification_report\n",
        "from sklearn.metrics import f1_score\n",
        "from seqeval.metrics import precision_score, recall_score, f1_score, classification_report\n",
        "from keras.preprocessing.text import text_to_word_sequence\n",
        "import pickle"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cB3qsJmAn5q6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Converting each sentence into list of index from list of tokens\n",
        "X = [[word_to_index[w[0]] for w in s] for s in sentences]\n",
        "\n",
        "# Padding each sequence to have same length  of each word\n",
        "X = pad_sequences(maxlen = max_len, sequences = X, padding = \"post\", value = word_to_index[\"PAD\"])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mw6fDvWQpeSI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = [[tag_to_index[w[2]] for w in s] for s in sentences]\n",
        "\n",
        "# padding\n",
        "y = pad_sequences(maxlen = max_len, sequences = y, padding = \"post\", value = tag_to_index[\"PAD\"])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XYERNPmVn7WH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "num_tag = df['Tag'].nunique()\n",
        "# One hot encoded labels\n",
        "y = [to_categorical(i, num_classes = num_tag + 1) for i in y]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tvRE5HYSpcwS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.15)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VcshbT37pla7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "e0cf3edc-45f4-4e31-e9de-63de37ca525b"
      },
      "source": [
        "print(\"Size of training input data : \", X_train.shape)\n",
        "print(\"Size of training output data : \", np.array(y_train).shape)\n",
        "print(\"Size of testing input data : \", X_test.shape)\n",
        "print(\"Size of testing output data : \", np.array(y_test).shape)"
      ],
      "execution_count": 86,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Size of training input data :  (40765, 75)\n",
            "Size of training output data :  (40765, 75, 18)\n",
            "Size of testing input data :  (7194, 75)\n",
            "Size of testing output data :  (7194, 75, 18)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "139gP7XWpniD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        },
        "outputId": "c5a590ce-6688-478a-ac60-4ff508ba97f1"
      },
      "source": [
        "num_tags = df['Tag'].nunique()\n",
        "# Model architecture\n",
        "input = Input(shape = (max_len,))\n",
        "model = Embedding(input_dim = len(words) + 2, output_dim = embedding, input_length = max_len)(input)\n",
        "model = Bidirectional(LSTM(units = 50, return_sequences=True, recurrent_dropout=0.1))(model)\n",
        "model = TimeDistributed(Dense(50, activation=\"relu\"))(model)\n",
        "crf = CRF(num_tags+1)  # CRF layer\n",
        "out = crf(model)  # output\n",
        "\n",
        "model = Model(input, out)\n",
        "model.compile(optimizer=\"rmsprop\", loss=crf.loss_function, metrics=[crf.accuracy])\n",
        "\n",
        "model.summary()"
      ],
      "execution_count": 88,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_2 (InputLayer)         (None, 75)                0         \n",
            "_________________________________________________________________\n",
            "embedding_2 (Embedding)      (None, 75, 40)            1407200   \n",
            "_________________________________________________________________\n",
            "bidirectional_2 (Bidirection (None, 75, 100)           36400     \n",
            "_________________________________________________________________\n",
            "time_distributed_2 (TimeDist (None, 75, 50)            5050      \n",
            "_________________________________________________________________\n",
            "crf_2 (CRF)                  (None, 75, 18)            1278      \n",
            "=================================================================\n",
            "Total params: 1,449,928\n",
            "Trainable params: 1,449,928\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CxtMWKnVpsle",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "checkpointer = ModelCheckpoint(filepath = 'model.h5',\n",
        "                       verbose = 0,\n",
        "                       mode = 'auto',\n",
        "                       save_best_only = True,\n",
        "                       monitor='val_loss')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2NCYpDIOp8SB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 326
        },
        "outputId": "5c00313e-9cd4-47c7-ed02-dd67b109bcb0"
      },
      "source": [
        "history = model.fit(X_train, np.array(y_train), batch_size=batch_size, epochs=epochs,\n",
        "                    validation_split=0.1, callbacks=[checkpointer])"
      ],
      "execution_count": 90,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 36688 samples, validate on 4077 samples\n",
            "Epoch 1/8\n",
            "36688/36688 [==============================] - 105s 3ms/step - loss: 0.1574 - crf_viterbi_accuracy: 0.9588 - val_loss: 0.0447 - val_crf_viterbi_accuracy: 0.9845\n",
            "Epoch 2/8\n",
            "36688/36688 [==============================] - 102s 3ms/step - loss: 0.0243 - crf_viterbi_accuracy: 0.9883 - val_loss: 0.0155 - val_crf_viterbi_accuracy: 0.9892\n",
            "Epoch 3/8\n",
            "36688/36688 [==============================] - 109s 3ms/step - loss: 0.0060 - crf_viterbi_accuracy: 0.9909 - val_loss: 0.0036 - val_crf_viterbi_accuracy: 0.9901\n",
            "Epoch 4/8\n",
            "36688/36688 [==============================] - 103s 3ms/step - loss: -0.0059 - crf_viterbi_accuracy: 0.9919 - val_loss: -0.0065 - val_crf_viterbi_accuracy: 0.9903\n",
            "Epoch 5/8\n",
            "36688/36688 [==============================] - 102s 3ms/step - loss: -0.0160 - crf_viterbi_accuracy: 0.9925 - val_loss: -0.0145 - val_crf_viterbi_accuracy: 0.9901\n",
            "Epoch 6/8\n",
            "36688/36688 [==============================] - 102s 3ms/step - loss: -0.0253 - crf_viterbi_accuracy: 0.9929 - val_loss: -0.0237 - val_crf_viterbi_accuracy: 0.9905\n",
            "Epoch 7/8\n",
            "36688/36688 [==============================] - 102s 3ms/step - loss: -0.0341 - crf_viterbi_accuracy: 0.9933 - val_loss: -0.0308 - val_crf_viterbi_accuracy: 0.9898\n",
            "Epoch 8/8\n",
            "36688/36688 [==============================] - 102s 3ms/step - loss: -0.0427 - crf_viterbi_accuracy: 0.9936 - val_loss: -0.0399 - val_crf_viterbi_accuracy: 0.9909\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eO7g9EXyp-yl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Evaluation\n",
        "y_pred = model.predict(X_test)\n",
        "y_pred = np.argmax(y_pred, axis=-1)\n",
        "y_test_true = np.argmax(y_test, -1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NNfPggZMqO5O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_pred = [[idx2tag[i] for i in row] for row in y_pred]\n",
        "y_test_true = [[idx2tag[i] for i in row] for row in y_test_true]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5rE4-caUqRDm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "2cbae304-d746-488e-8db7-67233b6e50c7"
      },
      "source": [
        "print(\"F1-score is : {:.1%}\".format(f1_score(y_test_true, y_pred)))"
      ],
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "F1-score is : 88.5%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9ej-qpbWqT-V",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 442
        },
        "outputId": "fba6f521-fc71-4400-c9eb-d08f7bd98c9c"
      },
      "source": [
        "report = flat_classification_report(y_pred=y_pred, y_true=y_test_true)\n",
        "print(report)"
      ],
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "       B-art       0.25      0.02      0.04        48\n",
            "       B-eve       0.45      0.20      0.28        50\n",
            "       B-geo       0.86      0.89      0.87      5674\n",
            "       B-gpe       0.95      0.94      0.95      2380\n",
            "       B-nat       0.54      0.24      0.33        29\n",
            "       B-org       0.80      0.70      0.75      3051\n",
            "       B-per       0.82      0.84      0.83      2530\n",
            "       B-tim       0.92      0.89      0.90      3102\n",
            "       I-art       0.25      0.03      0.05        35\n",
            "       I-eve       0.08      0.02      0.03        51\n",
            "       I-geo       0.79      0.79      0.79      1088\n",
            "       I-gpe       0.74      0.50      0.60        28\n",
            "       I-nat       0.00      0.00      0.00        10\n",
            "       I-org       0.78      0.77      0.78      2469\n",
            "       I-per       0.87      0.85      0.86      2566\n",
            "       I-tim       0.85      0.74      0.79      1011\n",
            "           O       0.99      0.99      0.99    133505\n",
            "         PAD       1.00      1.00      1.00    381923\n",
            "\n",
            "    accuracy                           0.99    539550\n",
            "   macro avg       0.66      0.58      0.60    539550\n",
            "weighted avg       0.99      0.99      0.99    539550\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qT1hvCDzqWqi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        },
        "outputId": "6c19ec04-8467-430f-9955-c88d2b720c34"
      },
      "source": [
        "i = np.random.randint(0,X_test.shape[0]) # choose a random number between 0 and len(X_te)b\n",
        "p = model.predict(np.array([X_test[i]]))\n",
        "p = np.argmax(p, axis=-1)\n",
        "true = np.argmax(y_test[i], -1)\n",
        "\n",
        "print(\"Sample number {} of {} (Test Set)\".format(i, X_test.shape[0]))\n",
        "# Visualization\n",
        "print(\"{:15}||{:5}||{}\".format(\"Word\", \"True\", \"Pred\"))\n",
        "print(30 * \"=\")\n",
        "for w, t, pred in zip(X_test[i], true, p[0]):\n",
        "    if w != 0:\n",
        "        print(\"{:15}: {:5} {}\".format(words[w-2], idx2tag[t], idx2tag[pred]))"
      ],
      "execution_count": 95,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sample number 4256 of 7194 (Test Set)\n",
            "Word           ||True ||Pred\n",
            "==============================\n",
            "The            : O     O\n",
            "attack         : O     O\n",
            "follows        : O     O\n",
            "the            : O     O\n",
            "assassination  : O     O\n",
            "of             : O     O\n",
            "four           : O     O\n",
            "French         : B-gpe B-gpe\n",
            "tourists       : O     O\n",
            "on             : O     O\n",
            "Monday         : B-tim B-tim\n",
            "in             : O     O\n",
            "eastern        : B-geo B-geo\n",
            "Mauritania     : I-geo I-geo\n",
            ".              : O     O\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FuyP4zHqtXFA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}