stanford_corenlp.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "stanford_corenlp.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "CtztGcf6i-or",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 316
        },
        "outputId": "c1e82df0-2ae1-4330-8924-b352671d9036"
      },
      "source": [
        "!pip install stanfordnlp"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting stanfordnlp\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/41/bf/5d2898febb6e993fcccd90484cba3c46353658511a41430012e901824e94/stanfordnlp-0.2.0-py3-none-any.whl (158kB)\n",
            "\r\u001b[K     |██                              | 10kB 15.7MB/s eta 0:00:01\r\u001b[K     |████▏                           | 20kB 2.9MB/s eta 0:00:01\r\u001b[K     |██████▏                         | 30kB 3.5MB/s eta 0:00:01\r\u001b[K     |████████▎                       | 40kB 3.9MB/s eta 0:00:01\r\u001b[K     |██████████▎                     | 51kB 3.3MB/s eta 0:00:01\r\u001b[K     |████████████▍                   | 61kB 3.7MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 71kB 3.9MB/s eta 0:00:01\r\u001b[K     |████████████████▌               | 81kB 4.3MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 92kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████████████▋           | 102kB 4.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 112kB 4.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 122kB 4.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 133kB 4.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 143kB 4.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 153kB 4.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 163kB 4.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from stanfordnlp) (2.23.0)\n",
            "Requirement already satisfied: torch>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from stanfordnlp) (1.5.1+cu101)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from stanfordnlp) (4.41.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from stanfordnlp) (1.18.5)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.6/dist-packages (from stanfordnlp) (3.12.2)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->stanfordnlp) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->stanfordnlp) (2020.6.20)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->stanfordnlp) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->stanfordnlp) (2.10)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=1.0.0->stanfordnlp) (0.16.0)\n",
            "Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.6/dist-packages (from protobuf->stanfordnlp) (1.12.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->stanfordnlp) (49.1.0)\n",
            "Installing collected packages: stanfordnlp\n",
            "Successfully installed stanfordnlp-0.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OhAAvbdaFMEP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import stanfordnlp"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QwgxpTFdizdl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        },
        "outputId": "7ba346a9-0ba5-4fa6-f9b8-feba105f6978"
      },
      "source": [
        "stanfordnlp.download('en')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using the default treebank \"en_ewt\" for language \"en\".\n",
            "Would you like to download the models for: en_ewt now? (Y/n)\n",
            "Y\n",
            "\n",
            "Default download directory: /root/stanfordnlp_resources\n",
            "Hit enter to continue or type an alternate directory.\n",
            "\n",
            "\n",
            "Downloading models for: en_ewt\n",
            "Download location: /root/stanfordnlp_resources/en_ewt_models.zip\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 235M/235M [00:32<00:00, 7.18MB/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Download complete.  Models saved to: /root/stanfordnlp_resources/en_ewt_models.zip\n",
            "Extracting models file for: en_ewt\n",
            "Cleaning up...Done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ydu3GfkOjHUi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "bc2932a5-8adb-4342-91c0-2f89180b7d73"
      },
      "source": [
        "pip freeze | grep torch"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch==1.5.1+cu101\n",
            "torchsummary==1.5.1\n",
            "torchtext==0.3.1\n",
            "torchvision==0.6.1+cu101\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0IyiAIOPjbdd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 389
        },
        "outputId": "3fc8221b-37e3-4863-9244-42b98a4f9590"
      },
      "source": [
        "nlp = stanfordnlp.Pipeline(processors = \"tokenize,mwt,lemma,pos\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Use device: cpu\n",
            "---\n",
            "Loading: tokenize\n",
            "With settings: \n",
            "{'model_path': '/root/stanfordnlp_resources/en_ewt_models/en_ewt_tokenizer.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
            "---\n",
            "Loading: lemma\n",
            "With settings: \n",
            "{'model_path': '/root/stanfordnlp_resources/en_ewt_models/en_ewt_lemmatizer.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
            "Building an attentional Seq2Seq model...\n",
            "Using a Bi-LSTM encoder\n",
            "Using soft attention for LSTM.\n",
            "Finetune all embeddings.\n",
            "[Running seq2seq lemmatizer with edit classifier]\n",
            "---\n",
            "Loading: pos\n",
            "With settings: \n",
            "{'model_path': '/root/stanfordnlp_resources/en_ewt_models/en_ewt_tagger.pt', 'pretrain_path': '/root/stanfordnlp_resources/en_ewt_models/en_ewt.pretrain.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
            "Done loading processors!\n",
            "---\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QQElC-h3jhD6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "doc = nlp(\"\"\"The prospects for Britain’s orderly withdrawal from the European Union on March 29 have receded further, even as MPs rallied to stop a no-deal scenario. An amendment to the draft bill on the termination of London’s membership of the bloc obliges Prime Minister Theresa May to renegotiate her withdrawal agreement with Brussels. A Tory backbencher’s proposal calls on the government to come up with alternatives to the Irish backstop, a central tenet of the deal Britain agreed with the rest of the EU.\"\"\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g88D_qBujj2G",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 510
        },
        "outputId": "0a92c1e8-ffb9-494b-b0cc-dd37e7c5b7e2"
      },
      "source": [
        "doc.sentences[0].print_tokens()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<Token index=1;words=[<Word index=1;text=The;lemma=the;upos=DET;xpos=DT;feats=Definite=Def|PronType=Art>]>\n",
            "<Token index=2;words=[<Word index=2;text=prospects;lemma=prospect;upos=NOUN;xpos=NNS;feats=Number=Plur>]>\n",
            "<Token index=3;words=[<Word index=3;text=for;lemma=for;upos=ADP;xpos=IN;feats=_>]>\n",
            "<Token index=4;words=[<Word index=4;text=Britain;lemma=Britain;upos=PROPN;xpos=NNP;feats=Number=Sing>]>\n",
            "<Token index=5;words=[<Word index=5;text=’s;lemma='s;upos=PART;xpos=POS;feats=_>]>\n",
            "<Token index=6;words=[<Word index=6;text=orderly;lemma=orderly;upos=ADJ;xpos=JJ;feats=Degree=Pos>]>\n",
            "<Token index=7;words=[<Word index=7;text=withdrawal;lemma=withdrawal;upos=NOUN;xpos=NN;feats=Number=Sing>]>\n",
            "<Token index=8;words=[<Word index=8;text=from;lemma=from;upos=ADP;xpos=IN;feats=_>]>\n",
            "<Token index=9;words=[<Word index=9;text=the;lemma=the;upos=DET;xpos=DT;feats=Definite=Def|PronType=Art>]>\n",
            "<Token index=10;words=[<Word index=10;text=European;lemma=european;upos=ADJ;xpos=JJ;feats=Degree=Pos>]>\n",
            "<Token index=11;words=[<Word index=11;text=Union;lemma=union;upos=PROPN;xpos=NNP;feats=Number=Sing>]>\n",
            "<Token index=12;words=[<Word index=12;text=on;lemma=on;upos=ADP;xpos=IN;feats=_>]>\n",
            "<Token index=13;words=[<Word index=13;text=March;lemma=March;upos=PROPN;xpos=NNP;feats=Number=Sing>]>\n",
            "<Token index=14;words=[<Word index=14;text=29;lemma=29;upos=NUM;xpos=CD;feats=NumType=Card>]>\n",
            "<Token index=15;words=[<Word index=15;text=have;lemma=have;upos=AUX;xpos=VBP;feats=Mood=Ind|Tense=Pres|VerbForm=Fin>]>\n",
            "<Token index=16;words=[<Word index=16;text=receded;lemma=recede;upos=VERB;xpos=VBN;feats=Tense=Past|VerbForm=Part>]>\n",
            "<Token index=17;words=[<Word index=17;text=further;lemma=further;upos=ADV;xpos=RB;feats=_>]>\n",
            "<Token index=18;words=[<Word index=18;text=,;lemma=,;upos=PUNCT;xpos=,;feats=_>]>\n",
            "<Token index=19;words=[<Word index=19;text=even;lemma=even;upos=ADV;xpos=RB;feats=_>]>\n",
            "<Token index=20;words=[<Word index=20;text=as;lemma=as;upos=SCONJ;xpos=IN;feats=_>]>\n",
            "<Token index=21;words=[<Word index=21;text=MPs;lemma=mps;upos=PROPN;xpos=NNPS;feats=Number=Plur>]>\n",
            "<Token index=22;words=[<Word index=22;text=rallied;lemma=rally;upos=VERB;xpos=VBD;feats=Mood=Ind|Tense=Past|VerbForm=Fin>]>\n",
            "<Token index=23;words=[<Word index=23;text=to;lemma=to;upos=PART;xpos=TO;feats=_>]>\n",
            "<Token index=24;words=[<Word index=24;text=stop;lemma=stop;upos=VERB;xpos=VB;feats=VerbForm=Inf>]>\n",
            "<Token index=25;words=[<Word index=25;text=a;lemma=a;upos=DET;xpos=DT;feats=Definite=Ind|PronType=Art>]>\n",
            "<Token index=26;words=[<Word index=26;text=no-deal;lemma=no-deal;upos=ADJ;xpos=JJ;feats=Degree=Pos>]>\n",
            "<Token index=27;words=[<Word index=27;text=scenario;lemma=scenario;upos=NOUN;xpos=NN;feats=Number=Sing>]>\n",
            "<Token index=28;words=[<Word index=28;text=.;lemma=.;upos=PUNCT;xpos=.;feats=_>]>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QZWy0GCxjro8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "3fd36ec0-80fb-40d0-97b4-d27d8ef1af09"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "#extract lemma\n",
        "def extract_lemma(doc):\n",
        "    parsed_text = {'word':[], 'lemma':[]}\n",
        "    for sent in doc.sentences:\n",
        "        for wrd in sent.words:\n",
        "            #extract text and lemma\n",
        "            parsed_text['word'].append(wrd.text)\n",
        "            parsed_text['lemma'].append(wrd.lemma)\n",
        "    #return a dataframe\n",
        "    return pd.DataFrame(parsed_text)\n",
        "\n",
        "#call the function on doc\n",
        "extract_lemma(doc)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>word</th>\n",
              "      <th>lemma</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>The</td>\n",
              "      <td>the</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>prospects</td>\n",
              "      <td>prospect</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>for</td>\n",
              "      <td>for</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Britain</td>\n",
              "      <td>Britain</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>’s</td>\n",
              "      <td>'s</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86</th>\n",
              "      <td>rest</td>\n",
              "      <td>rest</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>87</th>\n",
              "      <td>of</td>\n",
              "      <td>of</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>88</th>\n",
              "      <td>the</td>\n",
              "      <td>the</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>89</th>\n",
              "      <td>EU</td>\n",
              "      <td>EU</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>90</th>\n",
              "      <td>.</td>\n",
              "      <td>.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>91 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "         word     lemma\n",
              "0         The       the\n",
              "1   prospects  prospect\n",
              "2         for       for\n",
              "3     Britain   Britain\n",
              "4          ’s        's\n",
              "..        ...       ...\n",
              "86       rest      rest\n",
              "87         of        of\n",
              "88        the       the\n",
              "89         EU        EU\n",
              "90          .         .\n",
              "\n",
              "[91 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zBr4M7ZAj2w5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "56d52f17-b01c-40cc-e71b-70f3971cd859"
      },
      "source": [
        "#dictionary that contains pos tags and their explanations\n",
        "pos_dict = {\n",
        "'CC': 'coordinating conjunction','CD': 'cardinal digit','DT': 'determiner',\n",
        "'EX': 'existential there (like: \\\"there is\\\" ... think of it like \\\"there exists\\\")',\n",
        "'FW': 'foreign word','IN':  'preposition/subordinating conjunction','JJ': 'adjective \\'big\\'',\n",
        "'JJR': 'adjective, comparative \\'bigger\\'','JJS': 'adjective, superlative \\'biggest\\'',\n",
        "'LS': 'list marker 1)','MD': 'modal could, will','NN': 'noun, singular \\'desk\\'',\n",
        "'NNS': 'noun plural \\'desks\\'','NNP': 'proper noun, singular \\'Harrison\\'',\n",
        "'NNPS': 'proper noun, plural \\'Americans\\'','PDT': 'predeterminer \\'all the kids\\'',\n",
        "'POS': 'possessive ending parent\\'s','PRP': 'personal pronoun I, he, she',\n",
        "'PRP$': 'possessive pronoun my, his, hers','RB': 'adverb very, silently,',\n",
        "'RBR': 'adverb, comparative better','RBS': 'adverb, superlative best',\n",
        "'RP': 'particle give up','TO': 'to go \\'to\\' the store.','UH': 'interjection errrrrrrrm',\n",
        "'VB': 'verb, base form take','VBD': 'verb, past tense took',\n",
        "'VBG': 'verb, gerund/present participle taking','VBN': 'verb, past participle taken',\n",
        "'VBP': 'verb, sing. present, non-3d take','VBZ': 'verb, 3rd person sing. present takes',\n",
        "'WDT': 'wh-determiner which','WP': 'wh-pronoun who, what','WP$': 'possessive wh-pronoun whose',\n",
        "'WRB': 'wh-abverb where, when','QF' : 'quantifier, bahut, thoda, kam (Hindi)','VM' : 'main verb',\n",
        "'PSP' : 'postposition, common in indian langs','DEM' : 'demonstrative, common in indian langs'\n",
        "}\n",
        "\n",
        "#extract parts of speech\n",
        "def extract_pos(doc):\n",
        "    parsed_text = {'word':[], 'pos':[], 'exp':[]}\n",
        "    for sent in doc.sentences:\n",
        "        for wrd in sent.words:\n",
        "            if wrd.pos in pos_dict.keys():\n",
        "                pos_exp = pos_dict[wrd.pos]\n",
        "            else:\n",
        "                pos_exp = 'NA'\n",
        "            parsed_text['word'].append(wrd.text)\n",
        "            parsed_text['pos'].append(wrd.pos)\n",
        "            parsed_text['exp'].append(pos_exp)\n",
        "    #return a dataframe of pos and text\n",
        "    return pd.DataFrame(parsed_text)\n",
        "\n",
        "#extract pos\n",
        "extract_pos(doc)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>word</th>\n",
              "      <th>pos</th>\n",
              "      <th>exp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>The</td>\n",
              "      <td>DT</td>\n",
              "      <td>determiner</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>prospects</td>\n",
              "      <td>NNS</td>\n",
              "      <td>noun plural 'desks'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>for</td>\n",
              "      <td>IN</td>\n",
              "      <td>preposition/subordinating conjunction</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Britain</td>\n",
              "      <td>NNP</td>\n",
              "      <td>proper noun, singular 'Harrison'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>’s</td>\n",
              "      <td>POS</td>\n",
              "      <td>possessive ending parent's</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86</th>\n",
              "      <td>rest</td>\n",
              "      <td>NN</td>\n",
              "      <td>noun, singular 'desk'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>87</th>\n",
              "      <td>of</td>\n",
              "      <td>IN</td>\n",
              "      <td>preposition/subordinating conjunction</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>88</th>\n",
              "      <td>the</td>\n",
              "      <td>DT</td>\n",
              "      <td>determiner</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>89</th>\n",
              "      <td>EU</td>\n",
              "      <td>NNP</td>\n",
              "      <td>proper noun, singular 'Harrison'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>90</th>\n",
              "      <td>.</td>\n",
              "      <td>.</td>\n",
              "      <td>NA</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>91 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "         word  pos                                    exp\n",
              "0         The   DT                             determiner\n",
              "1   prospects  NNS                    noun plural 'desks'\n",
              "2         for   IN  preposition/subordinating conjunction\n",
              "3     Britain  NNP       proper noun, singular 'Harrison'\n",
              "4          ’s  POS             possessive ending parent's\n",
              "..        ...  ...                                    ...\n",
              "86       rest   NN                  noun, singular 'desk'\n",
              "87         of   IN  preposition/subordinating conjunction\n",
              "88        the   DT                             determiner\n",
              "89         EU  NNP       proper noun, singular 'Harrison'\n",
              "90          .    .                                     NA\n",
              "\n",
              "[91 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qnkDWDXhPQCz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "nlp = stanfordnlp.Pipeline(processors = \"sentiment\")\n",
        "stanfordnlp.Pipeline()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q5OqcoDJOaes",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HJMb7IKdOatW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        },
        "outputId": "e45c1b9a-b9db-4b35-8ced-bc4d02341f47"
      },
      "source": [
        "stanfordnlp.download('hi')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using the default treebank \"hi_hdtb\" for language \"hi\".\n",
            "Would you like to download the models for: hi_hdtb now? (Y/n)\n",
            "Y\n",
            "\n",
            "Default download directory: /root/stanfordnlp_resources\n",
            "Hit enter to continue or type an alternate directory.\n",
            "\n",
            "\n",
            "Downloading models for: hi_hdtb\n",
            "Download location: /root/stanfordnlp_resources/hi_hdtb_models.zip\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 208M/208M [00:45<00:00, 4.57MB/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Download complete.  Models saved to: /root/stanfordnlp_resources/hi_hdtb_models.zip\n",
            "Extracting models file for: hi_hdtb\n",
            "Cleaning up...Done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BpIdUBNKObI7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 389
        },
        "outputId": "35069a97-bd6a-4ee4-98e2-a7bc7df07abc"
      },
      "source": [
        "nlp = stanfordnlp.Pipeline(processors = \"tokenize,mwt,lemma,pos\", lang='hi')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Use device: cpu\n",
            "---\n",
            "Loading: tokenize\n",
            "With settings: \n",
            "{'model_path': '/root/stanfordnlp_resources/hi_hdtb_models/hi_hdtb_tokenizer.pt', 'lang': 'hi', 'shorthand': 'hi_hdtb', 'mode': 'predict'}\n",
            "---\n",
            "Loading: lemma\n",
            "With settings: \n",
            "{'model_path': '/root/stanfordnlp_resources/hi_hdtb_models/hi_hdtb_lemmatizer.pt', 'lang': 'hi', 'shorthand': 'hi_hdtb', 'mode': 'predict'}\n",
            "Building an attentional Seq2Seq model...\n",
            "Using a Bi-LSTM encoder\n",
            "Using soft attention for LSTM.\n",
            "Finetune all embeddings.\n",
            "[Running seq2seq lemmatizer with edit classifier]\n",
            "---\n",
            "Loading: pos\n",
            "With settings: \n",
            "{'model_path': '/root/stanfordnlp_resources/hi_hdtb_models/hi_hdtb_tagger.pt', 'pretrain_path': '/root/stanfordnlp_resources/hi_hdtb_models/hi_hdtb.pretrain.pt', 'lang': 'hi', 'shorthand': 'hi_hdtb', 'mode': 'predict'}\n",
            "Done loading processors!\n",
            "---\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7fZWtyDVOgUV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "hindi_doc = nlp(\"\"\"केंद्र की मोदी सरकार ने शुक्रवार को अपना अंतरिम बजट पेश किया. कार्यवाहक वित्त मंत्री पीयूष गोयल ने अपने बजट में किसान, मजदूर, करदाता, महिला वर्ग समेत हर किसी के लिए बंपर ऐलान किए. हालांकि, बजट के बाद भी टैक्स को लेकर काफी कन्फ्यूजन बना रहा. केंद्र सरकार के इस अंतरिम बजट क्या खास रहा और किसको क्या मिला, आसान भाषा में यहां समझें\"\"\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gAmmoRE8Ol_l",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 246
        },
        "outputId": "750901b2-a3ea-424a-c9e8-ef40bdcb8d54"
      },
      "source": [
        "hindi_doc.sentences[0].print_tokens()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<Token index=1;words=[<Word index=1;text=केंद्र;lemma=केंद्र;upos=PROPN;xpos=NNP;feats=Case=Acc|Gender=Masc|Number=Sing|Person=3>]>\n",
            "<Token index=2;words=[<Word index=2;text=की;lemma=का;upos=ADP;xpos=PSP;feats=AdpType=Post|Case=Acc|Gender=Fem|Number=Sing>]>\n",
            "<Token index=3;words=[<Word index=3;text=मोदी;lemma=मोदी;upos=PROPN;xpos=NNP;feats=Case=Nom|Number=Sing|Person=3>]>\n",
            "<Token index=4;words=[<Word index=4;text=सरकार;lemma=सरकार;upos=NOUN;xpos=NN;feats=Case=Acc|Gender=Fem|Number=Sing|Person=3>]>\n",
            "<Token index=5;words=[<Word index=5;text=ने;lemma=ने;upos=ADP;xpos=PSP;feats=AdpType=Post>]>\n",
            "<Token index=6;words=[<Word index=6;text=शुक्रवार;lemma=शुक्रवार;upos=PROPN;xpos=NNP;feats=Case=Acc|Gender=Masc|Number=Sing|Person=3>]>\n",
            "<Token index=7;words=[<Word index=7;text=को;lemma=को;upos=ADP;xpos=PSP;feats=AdpType=Post>]>\n",
            "<Token index=8;words=[<Word index=8;text=अपना;lemma=अपना;upos=PRON;xpos=PRP;feats=Case=Nom|Gender=Masc|Number=Sing|PronType=Prs>]>\n",
            "<Token index=9;words=[<Word index=9;text=अंतरिम;lemma=अंतरिम;upos=ADJ;xpos=JJ;feats=Case=Nom>]>\n",
            "<Token index=10;words=[<Word index=10;text=बजट;lemma=बजट;upos=NOUN;xpos=NN;feats=Case=Nom|Gender=Masc|Number=Sing|Person=3>]>\n",
            "<Token index=11;words=[<Word index=11;text=पेश;lemma=पेश;upos=ADJ;xpos=JJ;feats=_>]>\n",
            "<Token index=12;words=[<Word index=12;text=किया;lemma=कर;upos=VERB;xpos=VM;feats=Aspect=Perf|Gender=Masc|Number=Sing|VerbForm=Part|Voice=Act>]>\n",
            "<Token index=13;words=[<Word index=13;text=.;lemma=.;upos=PUNCT;xpos=SYM;feats=_>]>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oPps8FvrOnRr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "2efb12ca-edb9-445e-d150-e8e3026e3592"
      },
      "source": [
        "extract_lemma(hindi_doc)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>word</th>\n",
              "      <th>lemma</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>केंद्र</td>\n",
              "      <td>केंद्र</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>की</td>\n",
              "      <td>का</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>मोदी</td>\n",
              "      <td>मोदी</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>सरकार</td>\n",
              "      <td>सरकार</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ने</td>\n",
              "      <td>ने</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>67</th>\n",
              "      <td>आसान</td>\n",
              "      <td>आसान</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>68</th>\n",
              "      <td>भाषा</td>\n",
              "      <td>भाषा</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69</th>\n",
              "      <td>में</td>\n",
              "      <td>में</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70</th>\n",
              "      <td>यहां</td>\n",
              "      <td>यहाँ</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>71</th>\n",
              "      <td>समझें</td>\n",
              "      <td>समझ</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>72 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      word   lemma\n",
              "0   केंद्र  केंद्र\n",
              "1       की      का\n",
              "2     मोदी    मोदी\n",
              "3    सरकार   सरकार\n",
              "4       ने      ने\n",
              "..     ...     ...\n",
              "67    आसान    आसान\n",
              "68    भाषा    भाषा\n",
              "69     में     में\n",
              "70    यहां    यहाँ\n",
              "71   समझें     समझ\n",
              "\n",
              "[72 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tpSj3lX-Os30",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "f38b0318-585a-41b4-a783-1a31c67d4dc9"
      },
      "source": [
        "extract_pos(hindi_doc)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>word</th>\n",
              "      <th>pos</th>\n",
              "      <th>exp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>केंद्र</td>\n",
              "      <td>NNP</td>\n",
              "      <td>proper noun, singular 'Harrison'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>की</td>\n",
              "      <td>PSP</td>\n",
              "      <td>postposition, common in indian langs</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>मोदी</td>\n",
              "      <td>NNP</td>\n",
              "      <td>proper noun, singular 'Harrison'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>सरकार</td>\n",
              "      <td>NN</td>\n",
              "      <td>noun, singular 'desk'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ने</td>\n",
              "      <td>PSP</td>\n",
              "      <td>postposition, common in indian langs</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>67</th>\n",
              "      <td>आसान</td>\n",
              "      <td>JJ</td>\n",
              "      <td>adjective 'big'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>68</th>\n",
              "      <td>भाषा</td>\n",
              "      <td>NN</td>\n",
              "      <td>noun, singular 'desk'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69</th>\n",
              "      <td>में</td>\n",
              "      <td>PSP</td>\n",
              "      <td>postposition, common in indian langs</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70</th>\n",
              "      <td>यहां</td>\n",
              "      <td>PRP</td>\n",
              "      <td>personal pronoun I, he, she</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>71</th>\n",
              "      <td>समझें</td>\n",
              "      <td>VM</td>\n",
              "      <td>main verb</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>72 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      word  pos                                   exp\n",
              "0   केंद्र  NNP      proper noun, singular 'Harrison'\n",
              "1       की  PSP  postposition, common in indian langs\n",
              "2     मोदी  NNP      proper noun, singular 'Harrison'\n",
              "3    सरकार   NN                 noun, singular 'desk'\n",
              "4       ने  PSP  postposition, common in indian langs\n",
              "..     ...  ...                                   ...\n",
              "67    आसान   JJ                       adjective 'big'\n",
              "68    भाषा   NN                 noun, singular 'desk'\n",
              "69     में  PSP  postposition, common in indian langs\n",
              "70    यहां  PRP           personal pronoun I, he, she\n",
              "71   समझें   VM                             main verb\n",
              "\n",
              "[72 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wJpcRxNaOtvH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X21YljdBkr79",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}