Polyglot.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.5"
    },
    "colab": {
      "name": "Polyglot.ipynb",
      "provenance": [],
      "toc_visible": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tWwTc-m5ggLy",
        "colab_type": "text"
      },
      "source": [
        "### Polyglot"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V0wvl460ggL2",
        "colab_type": "text"
      },
      "source": [
        "Polyglot is a natural language pipeline that supports massive multilingual applications."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8zrVQ4OzggL5",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "Features\n",
        "\n",
        "- Tokenization (165 Languages)\n",
        "- Language detection (196 Languages)\n",
        "- Named Entity Recognition (40 Languages)\n",
        "- Part of Speech Tagging (16 Languages)\n",
        "- Sentiment Analysis (136 Languages)\n",
        "- Word Embeddings (137 Languages)\n",
        "- Morphological analysis (135 Languages)\n",
        "- Transliteration (69 Languages)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zx6Zk87PggL8",
        "colab_type": "text"
      },
      "source": [
        "### Installation / Dependencies"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PoTsNZTpggMO",
        "colab_type": "text"
      },
      "source": [
        "polyglot depends on numpy and libicu-dev, on ubuntu/debian linux distribution you can install such packages by executing the following command:\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qzvdhHz4hHPH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 801
        },
        "outputId": "e7676889-055d-4715-a964-8279d095507a"
      },
      "source": [
        "!sudo apt-get install python-numpy libicu-dev\n",
        "!pip install polyglot\n",
        "!pip3 install pyicu\n",
        "!pip3 install pycld2\n",
        "!pip3 install morfessor"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "python-numpy is already the newest version (1:1.13.3-2ubuntu1).\n",
            "python-numpy set to manually installed.\n",
            "libicu-dev is already the newest version (60.2-3ubuntu3.1).\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-440\n",
            "Use 'sudo apt autoremove' to remove it.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 35 not upgraded.\n",
            "Collecting polyglot\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e7/98/e24e2489114c5112b083714277204d92d372f5bbe00d5507acf40370edb9/polyglot-16.7.4.tar.gz (126kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 2.8MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: polyglot\n",
            "  Building wheel for polyglot (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for polyglot: filename=polyglot-16.7.4-py2.py3-none-any.whl size=52557 sha256=4c2a87a95a8cc9e43b7a0131cbbb370e7ec547cae2b20a46fab7a72d25b84ff7\n",
            "  Stored in directory: /root/.cache/pip/wheels/5e/91/ef/f1369fdc1203b0a9347d4b24f149b83a305f39ab047986d9da\n",
            "Successfully built polyglot\n",
            "Installing collected packages: polyglot\n",
            "Successfully installed polyglot-16.7.4\n",
            "Collecting pyicu\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/5a/99/c48c816095208bf3f4936ff67e571621fbddef461303a35a076f234e31f6/PyICU-2.5.tar.gz (225kB)\n",
            "\u001b[K     |████████████████████████████████| 235kB 2.6MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyicu\n",
            "  Building wheel for pyicu (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyicu: filename=PyICU-2.5-cp36-cp36m-linux_x86_64.whl size=1252519 sha256=3da7ab2fffa15b22debb7133ad8c9036fa6423ac7009613dd37392369a4c1e95\n",
            "  Stored in directory: /root/.cache/pip/wheels/07/3a/28/09f90c38785945ddf9af61b7add1aa62a740f40e259626ef3a\n",
            "Successfully built pyicu\n",
            "Installing collected packages: pyicu\n",
            "Successfully installed pyicu-2.5\n",
            "Collecting pycld2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/21/d2/8b0def84a53c88d0eb27c67b05269fbd16ad68df8c78849e7b5d65e6aec3/pycld2-0.41.tar.gz (41.4MB)\n",
            "\u001b[K     |████████████████████████████████| 41.4MB 116kB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pycld2\n",
            "  Building wheel for pycld2 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pycld2: filename=pycld2-0.41-cp36-cp36m-linux_x86_64.whl size=9833551 sha256=8f990d3e6d4cb5b8b9a39fe8b508f27bec9e26b54fc0971da9a6d3ab767474dd\n",
            "  Stored in directory: /root/.cache/pip/wheels/c6/8f/e9/08a1a8932a490175bd140206cd86a3dbcfc70498100de11079\n",
            "Successfully built pycld2\n",
            "Installing collected packages: pycld2\n",
            "Successfully installed pycld2-0.41\n",
            "Collecting morfessor\n",
            "  Downloading https://files.pythonhosted.org/packages/39/e6/7afea30be2ee4d29ce9de0fa53acbb033163615f849515c0b1956ad074ee/Morfessor-2.0.6-py3-none-any.whl\n",
            "Installing collected packages: morfessor\n",
            "Successfully installed morfessor-2.0.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g3ETOiaUggMQ",
        "colab_type": "text"
      },
      "source": [
        "### Language Detection"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9LEeyQx9ggMR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "66a8ffcc-6491-49e0-fe44-b69349058511"
      },
      "source": [
        "from polyglot.detect import Detector\n",
        "arabic_text = u\"\"\"\n",
        "أفاد مصدر امني في قيادة عمليات صلاح الدين في العراق بأن \" القوات الامنية تتوقف لليوم\n",
        "الثالث على التوالي عن التقدم الى داخل مدينة تكريت بسبب\n",
        "انتشار قناصي التنظيم الذي يطلق على نفسه اسم \"الدولة الاسلامية\" والعبوات الناسفة\n",
        "والمنازل المفخخة والانتحاريين، فضلا عن ان القوات الامنية تنتظر وصول تعزيزات اضافية \".\n",
        "\"\"\"\n",
        "detector = Detector(arabic_text)\n",
        "print(detector.language)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "name: Arabic      code: ar       confidence:  99.0 read bytes:   907\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F3Gh5c1Lvdd3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "ed85b211-ff16-4c9c-c5f2-5504d343cf29"
      },
      "source": [
        "en_text = \"He is a student \"\n",
        "fr_text = \"Il est un étudiant\"\n",
        "ru_text = \"Он студент\"\n",
        "hn_text= \"अंदर आ जाओ।\"\n",
        "detect_en = Detector(en_text)\n",
        "detect_fr = Detector(fr_text)\n",
        "detect_ru = Detector(ru_text)\n",
        "detect_hn = Detector(hn_text)\n",
        "print(detect_en.language)\n",
        "\n",
        "print(detect_fr.language)\n",
        "print(detect_ru.language)\n",
        "print(detect_hn.language)"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Detector is not able to detect the language reliably.\n",
            "Detector is not able to detect the language reliably.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "name: English     code: en       confidence:  94.0 read bytes:   704\n",
            "name: French      code: fr       confidence:  95.0 read bytes:   870\n",
            "name: Serbian     code: sr       confidence:  95.0 read bytes:   614\n",
            "name: Hindi       code: hi       confidence:  96.0 read bytes:   530\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_SjV-ewSvdpQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "krTHlJSriDhi",
        "colab_type": "text"
      },
      "source": [
        "#### Mixed Text"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SMEexImEiKwA",
        "colab_type": "text"
      },
      "source": [
        "If the text contains snippets from different languages, the detector is able to find the most probable langauges used in the text. For each language, we can query the model confidence level:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Im6wD82WiTnP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mixed_text = u\"\"\"\n",
        "China (simplified Chinese: 中国; traditional Chinese: 中國),\n",
        "officially the People's Republic of China (PRC), is a sovereign state located in East Asia.\n",
        "\"\"\"\n"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8XqiwfOwggMa",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "788405cd-12c9-4a06-f9e0-fbd1133d3427"
      },
      "source": [
        "for language in Detector(mixed_text).languages:\n",
        "  print(language)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "name: English     code: en       confidence:  87.0 read bytes:  1154\n",
            "name: Chinese     code: zh_Hant  confidence:   5.0 read bytes:  1755\n",
            "name: un          code: un       confidence:   0.0 read bytes:     0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FAjWI_phiyZh",
        "colab_type": "text"
      },
      "source": [
        "Sometimes, there is no enough text to make a decision, like detecting a language from one word. This forces the detector to switch to a best effort strategy, a warning will be thrown and the attribute reliable will be set to False.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Wkl12ATggMo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "outputId": "fc511749-f31e-4a67-fa8d-425ab053a6c8"
      },
      "source": [
        "detector = Detector(\"pizza\")\n",
        "print(detector)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Detector is not able to detect the language reliably.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Prediction is reliable: False\n",
            "Language 1: name: English     code: en       confidence:  85.0 read bytes:  1194\n",
            "Language 2: name: un          code: un       confidence:   0.0 read bytes:     0\n",
            "Language 3: name: un          code: un       confidence:   0.0 read bytes:     0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "67-cGMNDjG0W",
        "colab_type": "text"
      },
      "source": [
        "#### Supported Languages\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8FRp_XWNjQLa",
        "colab_type": "text"
      },
      "source": [
        "cld2 can detect up to 165 languages."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "elkoabYcjPf9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "d175d92b-13b4-4ba3-a0d1-e83c60c6336c"
      },
      "source": [
        "from polyglot.utils import pretty_list\n",
        "print(pretty_list(Detector.supported_languages()))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  1. Abkhazian                  2. Afar                       3. Afrikaans                \n",
            "  4. Akan                       5. Albanian                   6. Amharic                  \n",
            "  7. Arabic                     8. Armenian                   9. Assamese                 \n",
            " 10. Aymara                    11. Azerbaijani               12. Bashkir                  \n",
            " 13. Basque                    14. Belarusian                15. Bengali                  \n",
            " 16. Bihari                    17. Bislama                   18. Bosnian                  \n",
            " 19. Breton                    20. Bulgarian                 21. Burmese                  \n",
            " 22. Catalan                   23. Cebuano                   24. Cherokee                 \n",
            " 25. Nyanja                    26. Corsican                  27. Croatian                 \n",
            " 28. Croatian                  29. Czech                     30. Chinese                  \n",
            " 31. Chinese                   32. Chinese                   33. Chinese                  \n",
            " 34. Chineset                  35. Chineset                  36. Chineset                 \n",
            " 37. Chineset                  38. Chineset                  39. Chineset                 \n",
            " 40. Danish                    41. Dhivehi                   42. Dutch                    \n",
            " 43. Dzongkha                  44. English                   45. Esperanto                \n",
            " 46. Estonian                  47. Ewe                       48. Faroese                  \n",
            " 49. Fijian                    50. Finnish                   51. French                   \n",
            " 52. Frisian                   53. Ga                        54. Galician                 \n",
            " 55. Ganda                     56. Georgian                  57. German                   \n",
            " 58. Greek                     59. Greenlandic               60. Guarani                  \n",
            " 61. Gujarati                  62. Haitian_creole            63. Hausa                    \n",
            " 64. Hawaiian                  65. Hebrew                    66. Hebrew                   \n",
            " 67. Hindi                     68. Hmong                     69. Hungarian                \n",
            " 70. Icelandic                 71. Igbo                      72. Indonesian               \n",
            " 73. Interlingua               74. Interlingue               75. Inuktitut                \n",
            " 76. Inupiak                   77. Irish                     78. Italian                  \n",
            " 79. Ignore                    80. Javanese                  81. Javanese                 \n",
            " 82. Japanese                  83. Kannada                   84. Kashmiri                 \n",
            " 85. Kazakh                    86. Khasi                     87. Khmer                    \n",
            " 88. Kinyarwanda               89. Krio                      90. Kurdish                  \n",
            " 91. Kyrgyz                    92. Korean                    93. Laothian                 \n",
            " 94. Latin                     95. Latvian                   96. Limbu                    \n",
            " 97. Limbu                     98. Limbu                     99. Lingala                  \n",
            "100. Lithuanian               101. Lozi                     102. Luba_lulua               \n",
            "103. Luo_kenya_and_tanzania   104. Luxembourgish            105. Macedonian               \n",
            "106. Malagasy                 107. Malay                    108. Malayalam                \n",
            "109. Maltese                  110. Manx                     111. Maori                    \n",
            "112. Marathi                  113. Mauritian_creole         114. Romanian                 \n",
            "115. Mongolian                116. Montenegrin              117. Montenegrin              \n",
            "118. Montenegrin              119. Montenegrin              120. Nauru                    \n",
            "121. Ndebele                  122. Nepali                   123. Newari                   \n",
            "124. Norwegian                125. Norwegian                126. Norwegian_n              \n",
            "127. Nyanja                   128. Occitan                  129. Oriya                    \n",
            "130. Oromo                    131. Ossetian                 132. Pampanga                 \n",
            "133. Pashto                   134. Pedi                     135. Persian                  \n",
            "136. Polish                   137. Portuguese               138. Punjabi                  \n",
            "139. Quechua                  140. Rajasthani               141. Rhaeto_romance           \n",
            "142. Romanian                 143. Rundi                    144. Russian                  \n",
            "145. Samoan                   146. Sango                    147. Sanskrit                 \n",
            "148. Scots                    149. Scots_gaelic             150. Serbian                  \n",
            "151. Serbian                  152. Seselwa                  153. Seselwa                  \n",
            "154. Sesotho                  155. Shona                    156. Sindhi                   \n",
            "157. Sinhalese                158. Siswant                  159. Slovak                   \n",
            "160. Slovenian                161. Somali                   162. Spanish                  \n",
            "163. Sundanese                164. Swahili                  165. Swedish                  \n",
            "166. Syriac                   167. Tagalog                  168. Tajik                    \n",
            "169. Tamil                    170. Tatar                    171. Telugu                   \n",
            "172. Thai                     173. Tibetan                  174. Tigrinya                 \n",
            "175. Tonga                    176. Tsonga                   177. Tswana                   \n",
            "178. Tumbuka                  179. Turkish                  180. Turkmen                  \n",
            "181. Twi                      182. Uighur                   183. Ukrainian                \n",
            "184. Urdu                     185. Uzbek                    186. Venda                    \n",
            "187. Vietnamese               188. Volapuk                  189. Waray_philippines        \n",
            "190. Welsh                    191. Wolof                    192. Xhosa                    \n",
            "193. Yiddish                  194. Yoruba                   195. Zhuang                   \n",
            "196. Zulu                     \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aadC0GlGjNAK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4MzTHfnJjy6K",
        "colab_type": "text"
      },
      "source": [
        "### **Tokenization**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xds7ave3kBcx",
        "colab_type": "text"
      },
      "source": [
        "Tokenization is the process that identifies the text boundaries of words and sentences. We can identify the boundaries of sentences first then tokenize each sentence to identify the words that compose the sentence. Of course, we can do word tokenization first and then segment the token sequence into sentneces. Tokenization in polyglot relies on the Unicode Text Segmentation algorithm as implemented by the ICU Project."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TVy80ml6j25T",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Load packages\n",
        "import polyglot\n",
        "from polyglot.text import Text,Word"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C3v9CdgMlPTj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Word Tokens\n",
        "docx = Text(u\"He likes reading and painting\")"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t8qiTbjilRTY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "cf0c2086-23ef-4bb3-b99a-6eebc98fcff4"
      },
      "source": [
        "docx.words"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "WordList(['He', 'likes', 'reading', 'and', 'painting'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cy4J3T6JlTZn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "docx2 = Text(u\"He exclaimed, 'what're you doing? Reading?'.\")"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n_uMGupRlVeu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "3f62410d-301b-4272-e0ab-738d70604e74"
      },
      "source": [
        "docx2.words"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "WordList(['He', 'exclaimed', ',', \"'\", \"what're\", 'you', 'doing', '?', 'Reading', '?', \"'\", '.'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RTOwXR70lX7z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Sentence tokens\n",
        "docx3 = Text(u\"He likes reading and painting.He exclaimed, 'what're you doing? Reading?'.\")"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u4g5t8aVlaPO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "1741b5bc-de55-4260-edda-c0350ec21d7e"
      },
      "source": [
        "docx3.sentences"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Sentence(\"He likes reading and painting.He exclaimed, 'what're you doing?\"),\n",
              " Sentence(\"Reading?'.\")]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "89JR0xLpldvs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zMyQkpF-maYN",
        "colab_type": "text"
      },
      "source": [
        "### Part of Speech Tagging"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qBzmsjG3t7ee",
        "colab_type": "text"
      },
      "source": [
        "Part of speech tagging task aims to assign every word/token in plain text a category that identifies the syntactic functionality of the word occurrence.\n",
        "\n",
        "Polyglot recognizes 17 parts of speech, this set is called the universal part of speech tag set:\n",
        "\n",
        "\n",
        "\n",
        "-  ADJ: adjective\n",
        "- ADP: adposition\n",
        "- ADV: adverb\n",
        "- AUX: auxiliary verb\n",
        "- CONJ: coordinating conjunction\n",
        "- DET: determiner\n",
        "- INTJ: interjection\n",
        "- NOUN: noun\n",
        "- NUM: numeral\n",
        "- PART: particle\n",
        "- PRON: pronoun\n",
        "- PROPN: proper noun\n",
        "- PUNCT: punctuation\n",
        "- SCONJ: subordinating conjunction\n",
        "- SYM: symbol\n",
        "- VERB: verb\n",
        "- X: other\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BWiMjga1merE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "66ebfabc-e3d3-453b-d8e5-1f0fa6a84ab7"
      },
      "source": [
        "from polyglot.downloader import downloader\n",
        "print(downloader.supported_languages_table(\"pos2\"))"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  1. Italian                    2. French                     3. Spanish; Castilian       \n",
            "  4. Bulgarian                  5. Slovene                    6. Irish                    \n",
            "  7. Finnish                    8. Dutch                      9. Swedish                  \n",
            " 10. Danish                    11. Portuguese                12. English                  \n",
            " 13. German                    14. Indonesian                15. Czech                    \n",
            " 16. Hungarian                \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t8YZtjzDus-Z",
        "colab_type": "text"
      },
      "source": [
        "Download **Necessary** ***Models***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6HgR6s4Rujeh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "81cf8fad-6906-4797-ff61-e04613a3ab70"
      },
      "source": [
        "!polyglot download embeddings2.en pos2.en"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[polyglot_data] Downloading package embeddings2.en to\n",
            "[polyglot_data]     /root/polyglot_data...\n",
            "[polyglot_data] Downloading package pos2.en to /root/polyglot_data...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gofrABd8uqAJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "outputId": "c001dd9d-0052-444d-dfc7-a8974f176531"
      },
      "source": [
        "docx.pos_tags\n"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('He', 'PRON'),\n",
              " ('likes', 'VERB'),\n",
              " ('reading', 'VERB'),\n",
              " ('and', 'CONJ'),\n",
              " ('painting', 'NOUN')]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sncdDbuYu8BF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 191
        },
        "outputId": "b34e55ee-3755-4dd0-ccc6-ba0bf712b3a5"
      },
      "source": [
        "blob = \"\"\"We will meet at eight o'clock on Thursday morning.\"\"\"\n",
        "text = Text(blob)\n",
        "\n",
        "# We can also specify language of that text by using\n",
        "# text = Text(blob, hint_language_code='en')\n",
        "text.pos_tags"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('We', 'PRON'),\n",
              " ('will', 'AUX'),\n",
              " ('meet', 'VERB'),\n",
              " ('at', 'ADP'),\n",
              " ('eight', 'NUM'),\n",
              " (\"o'clock\", 'NOUN'),\n",
              " ('on', 'ADP'),\n",
              " ('Thursday', 'PROPN'),\n",
              " ('morning', 'NOUN'),\n",
              " ('.', 'PUNCT')]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "orrWlJu0vFQr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gb0HzLwkvuxv",
        "colab_type": "text"
      },
      "source": [
        "### Named Entity Extraction"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VbnATTJdv8gg",
        "colab_type": "text"
      },
      "source": [
        "Named entity extraction task aims to extract phrases from plain text that correpond to entities. Polyglot recognizes 3 categories of entities:\n",
        "\n",
        "- Locations (Tag: I-LOC): cities, countries, regions, continents, neighborhoods, administrative divisions …\n",
        "- Organizations (Tag: I-ORG): sports teams, newspapers, banks, universities, schools, non-profits, companies, …\n",
        "- Persons (Tag: I-PER): politicians, scientists, artists, atheletes …\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4NI3xON3wD24",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xtqI7MBAwEZ0",
        "colab_type": "text"
      },
      "source": [
        "#### Languages Coverage"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1cZXFXtowIYz",
        "colab_type": "text"
      },
      "source": [
        "The models were trained on datasets extracted automatically from Wikipedia. Polyglot currently supports 40 major languages."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pEa1-xT_v25s",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 260
        },
        "outputId": "431a2e4e-e8a2-40b0-e4ba-43a37536c28b"
      },
      "source": [
        "from polyglot.downloader import downloader\n",
        "print(downloader.supported_languages_table(\"ner2\", 3))"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  1. Italian                    2. Hindi                      3. French                   \n",
            "  4. Spanish; Castilian         5. Vietnamese                 6. Arabic                   \n",
            "  7. Bulgarian                  8. Norwegian                  9. Estonian                 \n",
            " 10. Japanese                  11. Greek, Modern             12. Slovene                  \n",
            " 13. Korean                    14. Serbian                   15. Finnish                  \n",
            " 16. Catalan; Valencian        17. Croatian                  18. Dutch                    \n",
            " 19. Swedish                   20. Tagalog                   21. Danish                   \n",
            " 22. Latvian                   23. Ukrainian                 24. Romanian, Moldavian, ... \n",
            " 25. Persian                   26. Slovak                    27. Portuguese               \n",
            " 28. English                   29. Malay                     30. Polish                   \n",
            " 31. German                    32. Indonesian                33. Chinese                  \n",
            " 34. Czech                     35. Hebrew (modern)           36. Lithuanian               \n",
            " 37. Turkish                   38. Hungarian                 39. Thai                     \n",
            " 40. Russian                  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aVa6jHsuwb03",
        "colab_type": "text"
      },
      "source": [
        "Download Necessary Models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CrjbLkpPwfeS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "outputId": "7a719aec-f2bc-47ad-8ae9-a001495ff2fe"
      },
      "source": [
        "!polyglot download embeddings2.en ner2.en"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[polyglot_data] Downloading package embeddings2.en to\n",
            "[polyglot_data]     /root/polyglot_data...\n",
            "[polyglot_data]   Package embeddings2.en is already up-to-date!\n",
            "[polyglot_data] Downloading package ner2.en to /root/polyglot_data...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4JLFKzN1wLWE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "713ea929-d187-4fd6-b97b-afa0b3b3d2b2"
      },
      "source": [
        "blob = \"\"\"The Israeli Prime Minister Benjamin Netanyahu has warned that Iran poses a \"threat to the entire world\".\"\"\"\n",
        "text = Text(blob)\n",
        "\n",
        "# We can also specify language of that text by using\n",
        "# text = Text(blob, hint_language_code='en')\n",
        "\n",
        "text.entities"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[I-ORG(['Israeli']), I-PER(['Benjamin', 'Netanyahu']), I-LOC(['Iran'])]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aKyBSh25xVZ0",
        "colab_type": "text"
      },
      "source": [
        "Or, we can query entites per sentence"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aob_0EKWxTLP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "outputId": "b8ee8800-eca5-491d-a372-8a7740b334e1"
      },
      "source": [
        "for sent in text.sentences:\n",
        "  print(sent, \"\\n\")\n",
        "  for entity in sent.entities:\n",
        "    print(entity.tag, entity)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The Israeli Prime Minister Benjamin Netanyahu has warned that Iran poses a \"threat to the entire world\". \n",
            "\n",
            "I-ORG ['Israeli']\n",
            "I-PER ['Benjamin', 'Netanyahu']\n",
            "I-LOC ['Iran']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FldSkoRVw0XA",
        "colab_type": "text"
      },
      "source": [
        "By doing more careful inspection of the second entity Benjamin Netanyahu, we can locate the position of the entity within the sentence."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PvJDlocgwYBU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "dc5c47b6-6a6c-46a9-bf42-af6e8c9b0405"
      },
      "source": [
        "benjamin = sent.entities[1]\n",
        "sent.words[benjamin.start: benjamin.end]"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "WordList(['Benjamin', 'Netanyahu'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pPLoSbitw_Kg",
        "colab_type": "text"
      },
      "source": [
        "### Morphological Analysis\n",
        " "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zLpVNoC9xpel",
        "colab_type": "text"
      },
      "source": [
        "#### Morphology\n",
        "+  morpheme is the smallest grammatical unit in a language. \n",
        "+ morpheme may or may not stand alone, word, by definition, is freestanding. \n",
        "\n",
        "\n",
        "Morphies which are the primitive units of syntax, the smallest individually meaningful elements in the utterances of a language. Morphemes are important in automatic generation and recognition of a language, especially in languages in which words may have many different inflected forms.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XHpgbExHygc5",
        "colab_type": "text"
      },
      "source": [
        "#### Languages Coverage"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OSPejRFiylrG",
        "colab_type": "text"
      },
      "source": [
        "Using polyglot vocabulary dictionaries, morfessor models is trained  on the most frequent words 50,000 words of each language."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MFqrTcjkyYbO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 816
        },
        "outputId": "e69225c4-84af-4698-ef46-32b6024972d9"
      },
      "source": [
        "from polyglot.downloader import downloader\n",
        "print(downloader.supported_languages_table(\"morph2\"))"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  1. Kapampangan                2. Italian                    3. Upper Sorbian            \n",
            "  4. Sakha                      5. Hindi                      6. French                   \n",
            "  7. Spanish; Castilian         8. Vietnamese                 9. Arabic                   \n",
            " 10. Macedonian                11. Pashto, Pushto            12. Bosnian-Croatian-Serbian \n",
            " 13. Egyptian Arabic           14. Norwegian Nynorsk         15. Sundanese                \n",
            " 16. Sicilian                  17. Azerbaijani               18. Bulgarian                \n",
            " 19. Yoruba                    20. Tajik                     21. Georgian                 \n",
            " 22. Tatar                     23. Galician                  24. Malagasy                 \n",
            " 25. Uighur, Uyghur            26. Amharic                   27. Venetian                 \n",
            " 28. Yiddish                   29. Norwegian                 30. Alemannic                \n",
            " 31. Estonian                  32. West Flemish              33. Divehi; Dhivehi; Mald... \n",
            " 34. Japanese                  35. Ilokano                   36. Haitian; Haitian Creole  \n",
            " 37. Belarusian                38. Greek, Modern             39. Ossetian, Ossetic        \n",
            " 40. Welsh                     41. Malayalam                 42. Albanian                 \n",
            " 43. Marathi (Marāṭhī)         44. Armenian                  45. Slovene                  \n",
            " 46. Korean                    47. Breton                    48. Irish                    \n",
            " 49. Luxembourgish, Letzeb...  50. Bengali                   51. Serbian                  \n",
            " 52. Fiji Hindi                53. Javanese                  54. Finnish                  \n",
            " 55. Gan Chinese               56. Kirghiz, Kyrgyz           57. Catalan; Valencian       \n",
            " 58. Quechua                   59. Croatian                  60. Dutch                    \n",
            " 61. Swedish                   62. Ido                       63. Tagalog                  \n",
            " 64. Sanskrit (Saṁskṛta)       65. Piedmontese language      66. Asturian                 \n",
            " 67. Danish                    68. Cebuano                   69. Western Frisian          \n",
            " 70. Kannada                   71. Scots                     72. Maltese                  \n",
            " 73. Swahili                   74. Limburgish, Limburgan...  75. Waray-Waray              \n",
            " 76. Lombard language          77. Uzbek                     78. Kurdish                  \n",
            " 79. Latvian                   80. Burmese                   81. Aragonese                \n",
            " 82. Volapük                   83. Northern Sami             84. Faroese                  \n",
            " 85. Kazakh                    86. Telugu                    87. Ukrainian                \n",
            " 88. Assamese                  89. Chuvash                   90. Silesian                 \n",
            " 91. Turkmen                   92. Romanian, Moldavian, ...  93. Persian                  \n",
            " 94. Tibetan Standard, Tib...  95. Latin                     96. Slovak                   \n",
            " 97. Sinhala, Sinhalese        98. Bavarian                  99. Icelandic                \n",
            "100. Mongolian                101. Walloon                  102. Portuguese               \n",
            "103. Urdu                     104. Gujarati                 105. Manx                     \n",
            "106. Tamil                    107. Khmer                    108. English                  \n",
            "109. Malay                    110. Chechen                  111. Bishnupriya Manipuri     \n",
            "112. Afrikaans                113. Basque                   114. Polish                   \n",
            "115. German                   116. Esperanto                117. Indonesian               \n",
            "118. Occitan                  119. Chinese                  120. Czech                    \n",
            "121. Hebrew (modern)          122. Romansh                  123. Lithuanian               \n",
            "124. Turkish                  125. Nepali                   126. Bosnian                  \n",
            "127. Interlingua              128. Zazaki                   129. Oriya                    \n",
            "130. Hungarian                131. Scottish Gaelic; Gaelic  132. Bashkir                  \n",
            "133. Thai                     134. Panjabi, Punjabi         135. Russian                  \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DG3_xif-y0VA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BpopaWhzy2Ja",
        "colab_type": "text"
      },
      "source": [
        "#### **Download Necessary Models**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LAFVwbt5y4BB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "outputId": "1004387e-a69b-467f-904d-e8afc4835f5a"
      },
      "source": [
        "!polyglot download morph2.en morph2.ar"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[polyglot_data] Downloading package morph2.en to\n",
            "[polyglot_data]     /root/polyglot_data...\n",
            "[polyglot_data] Downloading package morph2.ar to\n",
            "[polyglot_data]     /root/polyglot_data...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tfPL2GSlzX2s",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IrRDnjR7y7Rk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "outputId": "2ba812f5-a8a5-40db-be66-cdcf473ca5ec"
      },
      "source": [
        "words = [\"preprocessing\", \"processor\", \"invaluable\", \"thankful\", \"crossed\"]\n",
        "for w in words:\n",
        "  w = Word(w, language=\"en\")\n",
        "  print(\"{:<20}{}\".format(w, w.morphemes))\n"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "preprocessing       ['pre', 'process', 'ing']\n",
            "processor           ['process', 'or']\n",
            "invaluable          ['in', 'valuable']\n",
            "thankful            ['thank', 'ful']\n",
            "crossed             ['cross', 'ed']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7bHbcAfAzI1w",
        "colab_type": "text"
      },
      "source": [
        "If the text is not tokenized properly, morphological analysis could offer a smart of way of splitting the text into its original units. Here, is an example:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "caMM39dBzEjd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "2af7001c-121e-4e05-cee2-6346045bb4da"
      },
      "source": [
        "blob = \"Wewillmeettoday.\"\n",
        "text = Text(blob)\n",
        "text.language = \"en\"\n",
        "text.morphemes"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "WordList(['We', 'will', 'meet', 'to', 'day', '.'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w0jHZReGzNP7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f0o3oBOhzYnG",
        "colab_type": "text"
      },
      "source": [
        "### Transliteration"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J-k0LyZQzeyR",
        "colab_type": "text"
      },
      "source": [
        "Transliteration is the conversion of a text from one script to another. For instance, a Latin transliteration of the Greek phrase “Ελληνική Δημοκρατία”, usually translated as ‘Hellenic Republic’, is “Ellēnikḗ Dēmokratía”."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qNN8TekdzaT9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from polyglot.transliteration import Transliterator"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4WiGBlubzkLw",
        "colab_type": "text"
      },
      "source": [
        "#### **Languages Coverage**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ts41oNUNzccc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 434
        },
        "outputId": "8af6b08c-ffff-4efd-d00f-b25ef2b5c8d9"
      },
      "source": [
        "from polyglot.downloader import downloader\n",
        "print(downloader.supported_languages_table(\"transliteration2\"))"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  1. Italian                    2. Hindi                      3. French                   \n",
            "  4. Spanish; Castilian         5. Vietnamese                 6. Arabic                   \n",
            "  7. Macedonian                 8. Bosnian-Croatian-Serbian   9. Norwegian Nynorsk        \n",
            " 10. Azerbaijani               11. Bulgarian                 12. Georgian                 \n",
            " 13. Galician                  14. Amharic                   15. Yiddish                  \n",
            " 16. Norwegian                 17. Estonian                  18. Japanese                 \n",
            " 19. Haitian; Haitian Creole   20. Belarusian                21. Greek, Modern            \n",
            " 22. Welsh                     23. Albanian                  24. Marathi (Marāṭhī)        \n",
            " 25. Armenian                  26. Slovene                   27. Korean                   \n",
            " 28. Irish                     29. Bengali                   30. Serbian                  \n",
            " 31. Finnish                   32. Catalan; Valencian        33. Croatian                 \n",
            " 34. Dutch                     35. Swedish                   36. Tagalog                  \n",
            " 37. Danish                    38. Kannada                   39. Maltese                  \n",
            " 40. Swahili                   41. Latvian                   42. Telugu                   \n",
            " 43. Ukrainian                 44. Romanian, Moldavian, ...  45. Persian                  \n",
            " 46. Latin                     47. Slovak                    48. Icelandic                \n",
            " 49. Portuguese                50. Urdu                      51. Gujarati                 \n",
            " 52. Tamil                     53. Khmer                     54. Malay                    \n",
            " 55. Afrikaans                 56. Basque                    57. Polish                   \n",
            " 58. German                    59. Esperanto                 60. Indonesian               \n",
            " 61. Chinese                   62. Czech                     63. Hebrew (modern)          \n",
            " 64. Lithuanian                65. Turkish                   66. Bosnian                  \n",
            " 67. Hungarian                 68. Thai                      69. Russian                  \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gA_tX22dzrvK",
        "colab_type": "text"
      },
      "source": [
        "#### Downloading Necessary Models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_j3XX9KZzpXI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "outputId": "b05c999e-849a-42b6-972c-0233763f8394"
      },
      "source": [
        "!polyglot download embeddings2.en transliteration2.ar"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[polyglot_data] Downloading package embeddings2.en to\n",
            "[polyglot_data]     /root/polyglot_data...\n",
            "[polyglot_data]   Package embeddings2.en is already up-to-date!\n",
            "[polyglot_data] Downloading package transliteration2.ar to\n",
            "[polyglot_data]     /root/polyglot_data...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y6uyaqMvzxNT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "blob = \"\"\"We will meet at eight on Thursday morning.\"\"\"\n",
        "text = Text(blob)"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DjMSxL2Jz454",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "c1283197-c557-4ea4-8eb2-bf13d2e9d8b0"
      },
      "source": [
        "for x in text.transliterate('ar'):\n",
        "  print(x)"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "وي\n",
            "ويل\n",
            "ميت\n",
            "ات\n",
            "ييايت\n",
            "ون\n",
            "ثورسداي\n",
            "مورنينغ\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BlyLscx0z6i0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "af6a0fe6-92d2-4bcf-da7e-f488475216f9"
      },
      "source": [
        "for x in text.transliterate('hi'):\n",
        "  print(x)"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "वे\n",
            "विलल\n",
            "मीत\n",
            "ात\n",
            "ेिगहत\n",
            "ोन\n",
            "थूरसदे\n",
            "मोरनिंग\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mpeAz9Rw0NDO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "9839fd38-c268-48c6-c4cc-f833cd04b20c"
      },
      "source": [
        "!polyglot download transliteration2.hi"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[polyglot_data] Downloading package transliteration2.hi to\n",
            "[polyglot_data]     /root/polyglot_data...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qFvyccBW02nn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T-evZRoj1jLf",
        "colab_type": "text"
      },
      "source": [
        "### Sentiment Analysis"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9FQ_exWJ1ryC",
        "colab_type": "text"
      },
      "source": [
        "Polyglot has polarity lexicons for 136 languages. The scale of the words’ polarity consisted of three degrees: +1 for positive words, and -1 for negatives words. Neutral words will have a score of 0."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x01yJ6dk1uxv",
        "colab_type": "text"
      },
      "source": [
        "#### Languages Coverage"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rVuQRcMZ1mMm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 816
        },
        "outputId": "5656f953-9987-4807-ce73-1e99fc214c96"
      },
      "source": [
        "from polyglot.downloader import downloader\n",
        "print(downloader.supported_languages_table(\"sentiment2\", 3))"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  1. Kapampangan                2. Italian                    3. Upper Sorbian            \n",
            "  4. Sakha                      5. Hindi                      6. French                   \n",
            "  7. Spanish; Castilian         8. Vietnamese                 9. Arabic                   \n",
            " 10. Macedonian                11. Pashto, Pushto            12. Bosnian-Croatian-Serbian \n",
            " 13. Egyptian Arabic           14. Norwegian Nynorsk         15. Sundanese                \n",
            " 16. Sicilian                  17. Azerbaijani               18. Bulgarian                \n",
            " 19. Yoruba                    20. Tajik                     21. Georgian                 \n",
            " 22. Tatar                     23. Galician                  24. Malagasy                 \n",
            " 25. Uighur, Uyghur            26. Amharic                   27. Venetian                 \n",
            " 28. Yiddish                   29. Norwegian                 30. Alemannic                \n",
            " 31. Estonian                  32. West Flemish              33. Divehi; Dhivehi; Mald... \n",
            " 34. Japanese                  35. Ilokano                   36. Haitian; Haitian Creole  \n",
            " 37. Belarusian                38. Greek, Modern             39. Ossetian, Ossetic        \n",
            " 40. Welsh                     41. Malayalam                 42. Albanian                 \n",
            " 43. Marathi (Marāṭhī)         44. Armenian                  45. Slovene                  \n",
            " 46. Korean                    47. Breton                    48. Irish                    \n",
            " 49. Luxembourgish, Letzeb...  50. Bengali                   51. Serbian                  \n",
            " 52. Fiji Hindi                53. Javanese                  54. Finnish                  \n",
            " 55. Gan Chinese               56. Kirghiz, Kyrgyz           57. Catalan; Valencian       \n",
            " 58. Quechua                   59. Croatian                  60. Dutch                    \n",
            " 61. Swedish                   62. Ido                       63. Tagalog                  \n",
            " 64. Sanskrit (Saṁskṛta)       65. Piedmontese language      66. Asturian                 \n",
            " 67. Danish                    68. Cebuano                   69. Western Frisian          \n",
            " 70. Kannada                   71. Scots                     72. Maltese                  \n",
            " 73. Swahili                   74. Limburgish, Limburgan...  75. Waray-Waray              \n",
            " 76. Lombard language          77. Uzbek                     78. Kurdish                  \n",
            " 79. Latvian                   80. Burmese                   81. Aragonese                \n",
            " 82. Volapük                   83. Northern Sami             84. Faroese                  \n",
            " 85. Kazakh                    86. Telugu                    87. Ukrainian                \n",
            " 88. Assamese                  89. Chuvash                   90. Silesian                 \n",
            " 91. Turkmen                   92. Romanian, Moldavian, ...  93. Persian                  \n",
            " 94. Tibetan Standard, Tib...  95. Latin                     96. Slovak                   \n",
            " 97. Sinhala, Sinhalese        98. Bavarian                  99. Icelandic                \n",
            "100. Mongolian                101. Walloon                  102. Portuguese               \n",
            "103. Urdu                     104. Gujarati                 105. Manx                     \n",
            "106. Tamil                    107. Chinese Word             108. Khmer                    \n",
            "109. English                  110. Malay                    111. Chechen                  \n",
            "112. Bishnupriya Manipuri     113. Afrikaans                114. Basque                   \n",
            "115. Polish                   116. German                   117. Esperanto                \n",
            "118. Indonesian               119. Occitan                  120. Chinese                  \n",
            "121. Czech                    122. Hebrew (modern)          123. Romansh                  \n",
            "124. Lithuanian               125. Turkish                  126. Nepali                   \n",
            "127. Bosnian                  128. Interlingua              129. Zazaki                   \n",
            "130. Oriya                    131. Hungarian                132. Scottish Gaelic; Gaelic  \n",
            "133. Bashkir                  134. Thai                     135. Panjabi, Punjabi         \n",
            "136. Russian                  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JJwy_GRy2RUN",
        "colab_type": "text"
      },
      "source": [
        "#### Download Package"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B7q44Z3r1pHj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "92f43c44-eb3f-44a9-a7a8-c1cd7d9da56f"
      },
      "source": [
        "!polyglot download sentiment2.en"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[polyglot_data] Downloading package sentiment2.en to\n",
            "[polyglot_data]     /root/polyglot_data...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UYJyn8lV15hS",
        "colab_type": "text"
      },
      "source": [
        "Polarity\n",
        "\n",
        "To inquiry the polarity of a word, we can just call its own attribute polarity"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O6ktmZMo17gd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text = Text(\"The movie was really good.\")"
      ],
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KoTIZCyf19Xp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        },
        "outputId": "ceda3198-6842-4993-d003-f399ac6f2937"
      },
      "source": [
        "print(\"{:<16}{}\".format(\"Word\", \"Polarity\")+\"\\n\"+\"-\"*30)\n",
        "for w in text.words:\n",
        "    print(\"{:<16}{:>2}\".format(w, w.polarity))"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Word            Polarity\n",
            "------------------------------\n",
            "The              0\n",
            "movie            0\n",
            "was              0\n",
            "really           0\n",
            "good             1\n",
            ".                0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ojz48Iqm2EbC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "docx = Text(u\"He hates reading and playing\")\n",
        "docy= Text(\"He likes reading and painting\")"
      ],
      "execution_count": 59,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "buPZVsmQ2fyE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0387d280-79ca-4e7a-8682-50a1a0ce5334"
      },
      "source": [
        "docx.polarity"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-1.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zJCsMiPh21Rp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7c68a967-3e02-44cb-9d63-664d35052c11"
      },
      "source": [
        "docy.polarity"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NvARcK_Q23Ly",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "blob = (\"Barack Obama gave a fantastic speech last night. \"\n",
        "        \"Reports indicate he will move next to New Hampshire.\")\n",
        "text = Text(blob)"
      ],
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hW1mf0712-zK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "outputId": "1f4332f9-52f6-41d9-bf2a-0e5fa6eff5e8"
      },
      "source": [
        "first_sentence = text.sentences[0]\n",
        "print(first_sentence)\n",
        "print(first_sentence.polarity)\n",
        "first_entity = first_sentence.entities[0]\n",
        "print(first_entity)\n",
        "print(first_entity.positive_sentiment)"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Barack Obama gave a fantastic speech last night.\n",
            "1.0\n",
            "['Barack', 'Obama']\n",
            "0.9444444444444444\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dpwh4Jpd3Ejg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}