NLP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NLP",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "R076K7xFy9sy"
      },
      "outputs": [],
      "source": [
        "import nltk"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tokenization in NLP"
      ],
      "metadata": {
        "id": "n1JD67-O0ju5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "nltk.download('stopwords')\n",
        "nltk.download(\"punkt\")\n",
        "nltk.download('wordnet')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3PVnQUUWzHQ2",
        "outputId": "147858fd-de11-46ce-aba1-612078f87360"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "paragraph = \"\"\"India, officially the Republic of India (Hindi: Bhārat Gaṇarājya),[26] is a country in South Asia. It is the seventh-largest country by area, \n",
        "            the second-most populous country, and the most populous democracy in the world. Bounded by the Indian Ocean on the south, the Arabian Sea on the southwest, \n",
        "            and the Bay of Bengal on the southeast, it shares land borders with Pakistan to the west;[f] China, Nepal, and Bhutan to the north; and Bangladesh and,\n",
        "            Myanmar to the east. In the Indian Ocean, India is in the vicinity of Sri Lanka and the Maldives; \n",
        "            its Andaman and Nicobar Islands share a maritime border with Thailand, Myanmar and Indonesia,\n",
        "            Modern humans arrived on the Indian subcontinent from Africa no later than 55,000 years ago.[27][28][29] Their long occupation, \n",
        "            initially in varying forms of isolation as hunter-gatherers, has made the region highly diverse, second only to Africa in human \n",
        "            genetic diversity.[30] Settled life emerged on the subcontinent in the western margins of the Indus river basin 9,000 years ago, \n",
        "            evolving gradually into the Indus Valley Civilisation of the third millennium BCE.[31] By 1200 BCE, an archaic form of Sanskrit, \n",
        "            an Indo-European language, had diffused into India from the northwest,[32][33] unfolding as the language of the Rigveda, \n",
        "            and recording the dawning of Hinduism in India.[34] The Dravidian languages of India were supplanted in the northern and \n",
        "            western regions.[35] By 400 BCE, stratification and exclusion by caste had emerged within Hinduism,[36] and Buddhism and \n",
        "            Jainism had arisen, proclaiming social orders unlinked to heredity.[37] Early political consolidations gave rise to the \n",
        "            loose-knit Maurya and Gupta Empires based in the Ganges Basin.[38] Their collective era was suffused with wide-ranging \n",
        "            creativity,[39] but also marked by the declining status of women,[40] and the incorporation of untouchability into an organised system of belief.\n",
        "            [g][41] In South India, the Middle kingdoms exported Dravidian-languages scripts and religious cultures to the kingdoms of Southeast Asia.[42]\"\"\""
      ],
      "metadata": {
        "id": "cJJ2g20JzWiS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# converting paragraph into sentences\n",
        "\n",
        "sentences = nltk.sent_tokenize(paragraph)\n",
        "print(len(sentences))\n",
        "sentences"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TlEKZ_4Fzx0G",
        "outputId": "f6d9ac4f-25b0-4e75-f7ef-b93f05d333d4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['India, officially the Republic of India (Hindi: Bhārat Gaṇarājya),[26] is a country in South Asia.',\n",
              " 'It is the seventh-largest country by area, \\n            the second-most populous country, and the most populous democracy in the world.',\n",
              " 'Bounded by the Indian Ocean on the south, the Arabian Sea on the southwest, \\n            and the Bay of Bengal on the southeast, it shares land borders with Pakistan to the west;[f] China, Nepal, and Bhutan to the north; and Bangladesh and,\\n            Myanmar to the east.',\n",
              " 'In the Indian Ocean, India is in the vicinity of Sri Lanka and the Maldives; \\n            its Andaman and Nicobar Islands share a maritime border with Thailand, Myanmar and Indonesia,\\n            Modern humans arrived on the Indian subcontinent from Africa no later than 55,000 years ago.',\n",
              " '[27][28][29] Their long occupation, \\n            initially in varying forms of isolation as hunter-gatherers, has made the region highly diverse, second only to Africa in human \\n            genetic diversity.',\n",
              " '[30] Settled life emerged on the subcontinent in the western margins of the Indus river basin 9,000 years ago, \\n            evolving gradually into the Indus Valley Civilisation of the third millennium BCE.',\n",
              " '[31] By 1200 BCE, an archaic form of Sanskrit, \\n            an Indo-European language, had diffused into India from the northwest,[32][33] unfolding as the language of the Rigveda, \\n            and recording the dawning of Hinduism in India.',\n",
              " '[34] The Dravidian languages of India were supplanted in the northern and \\n            western regions.',\n",
              " '[35] By 400 BCE, stratification and exclusion by caste had emerged within Hinduism,[36] and Buddhism and \\n            Jainism had arisen, proclaiming social orders unlinked to heredity.',\n",
              " '[37] Early political consolidations gave rise to the \\n            loose-knit Maurya and Gupta Empires based in the Ganges Basin.',\n",
              " '[38] Their collective era was suffused with wide-ranging \\n            creativity,[39] but also marked by the declining status of women,[40] and the incorporation of untouchability into an organised system of belief.',\n",
              " '[g][41] In South India, the Middle kingdoms exported Dravidian-languages scripts and religious cultures to the kingdoms of Southeast Asia.',\n",
              " '[42]']"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# converting paragraph into words\n",
        "\n",
        "words = nltk.word_tokenize(paragraph)\n",
        "print(len(words))\n",
        "words"
      ],
      "metadata": {
        "id": "5vm2P4WUz9W2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Stemming and lemmatization "
      ],
      "metadata": {
        "id": "tNuQBmgB8KZA"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        ""
      ],
      "metadata": {
        "id": "sz8OGC-J8KR2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# stemming ( keep the common for words, used in sentiment analysis models)\n",
        "from nltk.stem import PorterStemmer\n",
        "from nltk.corpus import stopwords"
      ],
      "metadata": {
        "id": "kWjXBZ0g0SPb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "stemmer = PorterStemmer()"
      ],
      "metadata": {
        "id": "HH-EGc_w2NAn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(len(sentences)):\n",
        "  tmp_word = nltk.word_tokenize(sentences[i])\n",
        "\n",
        "  tmp_word = [stemmer.stem(word) for word in tmp_word if word not in set(stopwords.words('english'))]\n",
        "  sentences[i] = \" \".join(tmp_word)"
      ],
      "metadata": {
        "id": "DeA_C3_32Rct"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentences"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JKAMq6fV24rm",
        "outputId": "13237c19-df68-4fb4-cb5e-f639c462074d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['india , offici republ india ( hindi : bhārat gaṇarājya ) , [ 26 ] countri south asia .',\n",
              " 'It seventh-largest countri area , second-most popul countri , popul democraci world .',\n",
              " 'bound indian ocean south , arabian sea southwest , bay bengal southeast , share land border pakistan west ; [ f ] china , nepal , bhutan north ; bangladesh , myanmar east .',\n",
              " 'In indian ocean , india vicin sri lanka maldiv ; andaman nicobar island share maritim border thailand , myanmar indonesia , modern human arriv indian subcontin africa later 55,000 year ago .',\n",
              " '[ 27 ] [ 28 ] [ 29 ] their long occup , initi vari form isol hunter-gather , made region highli divers , second africa human genet divers .',\n",
              " '[ 30 ] settl life emerg subcontin western margin indu river basin 9,000 year ago , evolv gradual indu valley civilis third millennium bce .',\n",
              " '[ 31 ] By 1200 bce , archaic form sanskrit , indo-european languag , diffus india northwest , [ 32 ] [ 33 ] unfold languag rigveda , record dawn hinduism india .',\n",
              " '[ 34 ] the dravidian languag india supplant northern western region .',\n",
              " '[ 35 ] By 400 bce , stratif exclus cast emerg within hinduism , [ 36 ] buddhism jainism arisen , proclaim social order unlink hered .',\n",
              " '[ 37 ] earli polit consolid gave rise loose-knit maurya gupta empir base gang basin .',\n",
              " '[ 38 ] their collect era suffus wide-rang creativ , [ 39 ] also mark declin statu women , [ 40 ] incorpor untouch organis system belief .',\n",
              " '[ g ] [ 41 ] In south india , middl kingdom export dravidian-languag script religi cultur kingdom southeast asia .',\n",
              " '[ 42 ]']"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Lemetizer ( keep the meaning of word, used in chatbot)\n",
        "from nltk.stem import WordNetLemmatizer"
      ],
      "metadata": {
        "id": "D75CTYbj3LhE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lemma = WordNetLemmatizer()"
      ],
      "metadata": {
        "id": "ftYr69-S47dB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(len(sentences)):\n",
        "  tmp_word = nltk.word_tokenize(sentences[i])\n",
        "  tmp_word = [lemma.lemmatize(word) for word in tmp_word if word not in set(stopwords.words('english'))]\n",
        "  sentences[i] = \" \".join(tmp_word)"
      ],
      "metadata": {
        "id": "d1iO-2wS4-dr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentences"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IqQRbbya5MLw",
        "outputId": "6a4f1d1c-3e62-4a69-e746-8803a7ca3dea"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['India , officially Republic India ( Hindi : Bhārat Gaṇarājya ) , [ 26 ] country South Asia .',\n",
              " 'It seventh-largest country area , second-most populous country , populous democracy world .',\n",
              " 'Bounded Indian Ocean south , Arabian Sea southwest , Bay Bengal southeast , share land border Pakistan west ; [ f ] China , Nepal , Bhutan north ; Bangladesh , Myanmar east .',\n",
              " 'In Indian Ocean , India vicinity Sri Lanka Maldives ; Andaman Nicobar Islands share maritime border Thailand , Myanmar Indonesia , Modern human arrived Indian subcontinent Africa later 55,000 year ago .',\n",
              " '[ 27 ] [ 28 ] [ 29 ] Their long occupation , initially varying form isolation hunter-gatherer , made region highly diverse , second Africa human genetic diversity .',\n",
              " '[ 30 ] Settled life emerged subcontinent western margin Indus river basin 9,000 year ago , evolving gradually Indus Valley Civilisation third millennium BCE .',\n",
              " '[ 31 ] By 1200 BCE , archaic form Sanskrit , Indo-European language , diffused India northwest , [ 32 ] [ 33 ] unfolding language Rigveda , recording dawning Hinduism India .',\n",
              " '[ 34 ] The Dravidian language India supplanted northern western region .',\n",
              " '[ 35 ] By 400 BCE , stratification exclusion caste emerged within Hinduism , [ 36 ] Buddhism Jainism arisen , proclaiming social order unlinked heredity .',\n",
              " '[ 37 ] Early political consolidation gave rise loose-knit Maurya Gupta Empires based Ganges Basin .',\n",
              " '[ 38 ] Their collective era suffused wide-ranging creativity , [ 39 ] also marked declining status woman , [ 40 ] incorporation untouchability organised system belief .',\n",
              " '[ g ] [ 41 ] In South India , Middle kingdom exported Dravidian-languages script religious culture kingdom Southeast Asia .',\n",
              " '[ 42 ]']"
            ]
          },
          "metadata": {},
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Bag of words with lemmitazation"
      ],
      "metadata": {
        "id": "nvtaUoUm8Xrf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.internals import read_int\n",
        "import re\n",
        "\n",
        "corpus = []\n",
        "for i in range(len(sentences)):\n",
        "  # taking only alphabets\n",
        "  review = re.sub('[^a-zA-Z]',' ', sentences[i])\n",
        "  review = review.lower()\n",
        "  review = review.split()\n",
        "  review = [lemma.lemmatize(word) for word in review if word not in set(stopwords.words('english'))]\n",
        "  review = \" \".join(review)\n",
        "  corpus.append(review)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 356
        },
        "id": "JvewSCwE5fjc",
        "outputId": "3b410292-51b0-41f4-f145-6bad149b00df"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-8-6ddcc021b8a8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m   \u001b[0mreview\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreview\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlower\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m   \u001b[0mreview\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreview\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m   \u001b[0mreview\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mlemma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlemmatize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mword\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mreview\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstopwords\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'english'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m   \u001b[0mreview\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\" \"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreview\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m   \u001b[0mcorpus\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreview\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-8-6ddcc021b8a8>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m      8\u001b[0m   \u001b[0mreview\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreview\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlower\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m   \u001b[0mreview\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreview\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m   \u001b[0mreview\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mlemma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlemmatize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mword\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mreview\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstopwords\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'english'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m   \u001b[0mreview\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\" \"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreview\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m   \u001b[0mcorpus\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreview\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'lemma' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "corpus"
      ],
      "metadata": {
        "id": "XQqQt8C99cQQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Creating bag of words\n",
        "\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "\n",
        "cv = CountVectorizer(max_features=1500)\n",
        "x = cv.fit_transform(corpus).toarray()\n",
        "print(len(x))\n",
        "x"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aQJbLtUh9dV8",
        "outputId": "a1b54714-206d-4511-be11-e96b335053a3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0, 0, 0, ..., 0, 0, 0],\n",
              "       [0, 0, 0, ..., 0, 1, 0],\n",
              "       [0, 0, 0, ..., 0, 0, 0],\n",
              "       ...,\n",
              "       [0, 0, 1, ..., 1, 0, 0],\n",
              "       [0, 0, 0, ..., 0, 0, 0],\n",
              "       [0, 0, 0, ..., 0, 0, 0]])"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# TF IDF"
      ],
      "metadata": {
        "id": "_m_Y-hpRAtKF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "\n",
        "cv = TfidfVectorizer()\n",
        "x = cv.fit_transform(corpus).toarray()\n",
        "print(len(x))\n",
        "x"
      ],
      "metadata": {
        "id": "h1G9-al796uF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Word2Vec"
      ],
      "metadata": {
        "id": "KAQJDyQsKjoN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from gensim.models import Word2Vec"
      ],
      "metadata": {
        "id": "5v05YJR8Kak9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "text = re.sub(r\"\\[[0-9]*\\]\", \" \", paragraph)\n",
        "# removing special character\n",
        "text = re.sub(r\"\\s+\", \" \", text)\n",
        "# converting all in lower\n",
        "text = text.lower()\n",
        "text = re.sub(r\"\\d\", \" \", text)\n",
        "text = re.sub(r\"\\s+\", \" \", text)\n",
        "text"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 174
        },
        "id": "5J1_H2g1Kanc",
        "outputId": "b0e0acde-b8d8-40fe-92fa-1bfdf7a40202"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'india, officially the republic of india (hindi: bhārat gaṇarājya), is a country in south asia. it is the seventh-largest country by area, the second-most populous country, and the most populous democracy in the world. bounded by the indian ocean on the south, the arabian sea on the southwest, and the bay of bengal on the southeast, it shares land borders with pakistan to the west;[f] china, nepal, and bhutan to the north; and bangladesh and, myanmar to the east. in the indian ocean, india is in the vicinity of sri lanka and the maldives; its andaman and nicobar islands share a maritime border with thailand, myanmar and indonesia, modern humans arrived on the indian subcontinent from africa no later than , years ago. their long occupation, initially in varying forms of isolation as hunter-gatherers, has made the region highly diverse, second only to africa in human genetic diversity. settled life emerged on the subcontinent in the western margins of the indus river basin , years ago, evolving gradually into the indus valley civilisation of the third millennium bce. by bce, an archaic form of sanskrit, an indo-european language, had diffused into india from the northwest, unfolding as the language of the rigveda, and recording the dawning of hinduism in india. the dravidian languages of india were supplanted in the northern and western regions. by bce, stratification and exclusion by caste had emerged within hinduism, and buddhism and jainism had arisen, proclaiming social orders unlinked to heredity. early political consolidations gave rise to the loose-knit maurya and gupta empires based in the ganges basin. their collective era was suffused with wide-ranging creativity, but also marked by the declining status of women, and the incorporation of untouchability into an organised system of belief. [g] in south india, the middle kingdoms exported dravidian-languages scripts and religious cultures to the kingdoms of southeast asia. '"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentences = nltk.sent_tokenize(text)\n",
        "sentences = [nltk.word_tokenize(sentence) for sentence in sentences ]"
      ],
      "metadata": {
        "id": "_g40QO24Kap8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(len(sentences)):\n",
        "  sentences[i] = [word for word in sentences[i] if word not in set(stopwords.words('english'))]"
      ],
      "metadata": {
        "id": "aydyWouOKasu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentences"
      ],
      "metadata": {
        "id": "DqWUYov7Kaut"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = Word2Vec(sentences, min_count=1)"
      ],
      "metadata": {
        "id": "QhzDncXpKaxR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "words = model.wv.vocab\n",
        "words"
      ],
      "metadata": {
        "id": "rjoxzH5qKaz9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vector = model.wv['india']\n",
        "len(vector)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ubs-IygFKa12",
        "outputId": "d897bfc7-b558-4cb5-cf92-b872bcac3256"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "100"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "similar = model.wv.most_similar('asia')\n",
        "similar"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0VhSWOwhKa4U",
        "outputId": "032758af-ed70-41fe-ab20-667cb416ec57"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('country', 0.24626584351062775),\n",
              " ('populous', 0.2228507101535797),\n",
              " ('g', 0.22104939818382263),\n",
              " ('consolidations', 0.21067361533641815),\n",
              " ('southwest', 0.1958770900964737),\n",
              " ('dravidian-languages', 0.18834611773490906),\n",
              " ('within', 0.18729442358016968),\n",
              " (':', 0.18541505932807922),\n",
              " ('dravidian', 0.16651472449302673),\n",
              " ('hindi', 0.16215012967586517)]"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "OcqFkDXiKa6s"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "uy5THoloKa-R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Spam Detection"
      ],
      "metadata": {
        "id": "7pSueC5YDeyZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd"
      ],
      "metadata": {
        "id": "8fuBk2psA712"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "message = pd.read_csv('SMSSpamCollection', sep='\\t', names=[\"labels\", \"message\"])"
      ],
      "metadata": {
        "id": "LIO5-u6sEQa9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "message"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "OL0XC4StEfkb",
        "outputId": "fc130743-ebbb-4d88-b439-66175a63eec5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     labels                                            message\n",
              "0       ham  Go until jurong point, crazy.. Available only ...\n",
              "1       ham                      Ok lar... Joking wif u oni...\n",
              "2      spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
              "3       ham  U dun say so early hor... U c already then say...\n",
              "4       ham  Nah I don't think he goes to usf, he lives aro...\n",
              "...     ...                                                ...\n",
              "5567   spam  This is the 2nd time we have tried 2 contact u...\n",
              "5568    ham               Will ü b going to esplanade fr home?\n",
              "5569    ham  Pity, * was in mood for that. So...any other s...\n",
              "5570    ham  The guy did some bitching but I acted like i'd...\n",
              "5571    ham                         Rofl. Its true to its name\n",
              "\n",
              "[5572 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-cf269332-e861-4fc7-abb8-224695af012e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>labels</th>\n",
              "      <th>message</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ham</td>\n",
              "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ham</td>\n",
              "      <td>Ok lar... Joking wif u oni...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>spam</td>\n",
              "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ham</td>\n",
              "      <td>U dun say so early hor... U c already then say...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ham</td>\n",
              "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5567</th>\n",
              "      <td>spam</td>\n",
              "      <td>This is the 2nd time we have tried 2 contact u...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5568</th>\n",
              "      <td>ham</td>\n",
              "      <td>Will ü b going to esplanade fr home?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5569</th>\n",
              "      <td>ham</td>\n",
              "      <td>Pity, * was in mood for that. So...any other s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5570</th>\n",
              "      <td>ham</td>\n",
              "      <td>The guy did some bitching but I acted like i'd...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5571</th>\n",
              "      <td>ham</td>\n",
              "      <td>Rofl. Its true to its name</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5572 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-cf269332-e861-4fc7-abb8-224695af012e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-cf269332-e861-4fc7-abb8-224695af012e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-cf269332-e861-4fc7-abb8-224695af012e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ps = PorterStemmer()\n",
        "corpus = []\n",
        "for i in range(len(message)):\n",
        "  review = re.sub(\"[^a-zA-z]\", \" \", message.iloc[i][1])\n",
        "  review = review.lower()\n",
        "  review = review.split()\n",
        "  review = [ps.stem(word) for word in review if word not in stopwords.words('english')]\n",
        "  review = \" \".join(review)\n",
        "  corpus.append(review)"
      ],
      "metadata": {
        "id": "god7D8nKEkS5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cv = CountVectorizer(max_features=1500)\n",
        "X = cv.fit_transform(corpus).toarray()\n",
        "\n",
        "y = pd.get_dummies(message.iloc[:,0])"
      ],
      "metadata": {
        "id": "nGd2ZlJqFLBA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "61VaLPRjF5GR",
        "outputId": "02c11c6f-70ee-4093-c48f-a01a7ce21700"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0, 0, 0, ..., 0, 0, 0],\n",
              "       [0, 0, 0, ..., 0, 0, 0],\n",
              "       [0, 0, 0, ..., 0, 0, 0],\n",
              "       ...,\n",
              "       [0, 0, 0, ..., 0, 0, 0],\n",
              "       [0, 0, 0, ..., 0, 0, 0],\n",
              "       [0, 0, 0, ..., 0, 0, 0]])"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y = y.iloc[:,1]\n",
        "y"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vhd9kHB3F_LE",
        "outputId": "7077c43b-ff08-41a0-ef2a-63add3d45a06"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0       0\n",
              "1       0\n",
              "2       1\n",
              "3       0\n",
              "4       0\n",
              "       ..\n",
              "5567    1\n",
              "5568    0\n",
              "5569    0\n",
              "5570    0\n",
              "5571    0\n",
              "Name: spam, Length: 5572, dtype: uint8"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state = 42)"
      ],
      "metadata": {
        "id": "SNtjQXR2GAtj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# train model using Naive bayer claassifier\n",
        "\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "\n",
        "model = MultinomialNB().fit(x_train, y_train)"
      ],
      "metadata": {
        "id": "oeXCYFvQHL2c"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pre = model.predict(x_test)"
      ],
      "metadata": {
        "id": "nvp4OagwHwW5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import confusion_matrix, accuracy_score\n",
        "\n",
        "cm = confusion_matrix(y_test, pre)"
      ],
      "metadata": {
        "id": "feuW9B0LH34O"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cm"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OUhyMWCLICXc",
        "outputId": "9c5c7b48-00f1-401d-e5fd-950913a9a0ab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[956,  10],\n",
              "       [  7, 142]])"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_score(y_test, pre)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "330FwSydIFS3",
        "outputId": "2e8ae19a-3d63-4ebe-aa64-09761929709b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9847533632286996"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "bS6BNrVIIQTG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Stock market sentiment analysis"
      ],
      "metadata": {
        "id": "Tg-6LEumRJLw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "data = pd.read_csv(\"Data.csv\", encoding = \"ISO-8859-1\")\n",
        "data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "myQ30RVQRLok",
        "outputId": "0e245f24-09ad-4305-f364-98769dc4ab75"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            Date  Label                                               Top1  \\\n",
              "0     2000-01-03      0  A 'hindrance to operations': extracts from the...   \n",
              "1     2000-01-04      0                                          Scorecard   \n",
              "2     2000-01-05      0                  Coventry caught on counter by Flo   \n",
              "3     2000-01-06      1                      Pilgrim knows how to progress   \n",
              "4     2000-01-07      1                               Hitches and Horlocks   \n",
              "...          ...    ...                                                ...   \n",
              "4096  2016-06-27      0  Barclays and RBS shares suspended from trading...   \n",
              "4097  2016-06-28      1  2,500 Scientists To Australia: If You Want To ...   \n",
              "4098  2016-06-29      1                   Explosion At Airport In Istanbul   \n",
              "4099  2016-06-30      1  Jamaica proposes marijuana dispensers for tour...   \n",
              "4100  2016-07-01      1  A 117-year-old woman in Mexico City finally re...   \n",
              "\n",
              "                                                   Top2  \\\n",
              "0                                             Scorecard   \n",
              "1                                   The best lake scene   \n",
              "2                    United's rivals on the road to Rio   \n",
              "3                                   Thatcher facing ban   \n",
              "4                        Beckham off but United survive   \n",
              "...                                                 ...   \n",
              "4096  Pope says Church should ask forgiveness from g...   \n",
              "4097  The personal details of 112,000 French police ...   \n",
              "4098  Yemeni former president: Terrorism is the offs...   \n",
              "4099  Stephen Hawking says pollution and 'stupidity'...   \n",
              "4100   IMF chief backs Athens as permanent Olympic host   \n",
              "\n",
              "                                                   Top3  \\\n",
              "0                       Hughes' instant hit buoys Blues   \n",
              "1                         Leader: German sleaze inquiry   \n",
              "2         Thatcher issues defence before trial by video   \n",
              "3               McIlroy calls for Irish fighting spirit   \n",
              "4                               Breast cancer screening   \n",
              "...                                                 ...   \n",
              "4096  Poland 'shocked' by xenophobic abuse of Poles ...   \n",
              "4097  S&amp;P cuts United Kingdom sovereign credit r...   \n",
              "4098  UK must accept freedom of movement to access E...   \n",
              "4099  Boris Johnson says he will not run for Tory pa...   \n",
              "4100  The president of France says if Brexit won, so...   \n",
              "\n",
              "                                                   Top4  \\\n",
              "0              Jack gets his skates on at ice-cold Alex   \n",
              "1                                         Cheerio, boyo   \n",
              "2         Police help Smith lay down the law at Everton   \n",
              "3                       Leicester bin stadium blueprint   \n",
              "4                                           Alan Parker   \n",
              "...                                                 ...   \n",
              "4096  There will be no second referendum, cabinet ag...   \n",
              "4097                Huge helium deposit found in Africa   \n",
              "4098  Devastated: scientists too late to captive bre...   \n",
              "4099  Six gay men in Ivory Coast were abused and for...   \n",
              "4100  British Man Who Must Give Police 24 Hours' Not...   \n",
              "\n",
              "                                                   Top5  \\\n",
              "0                Chaos as Maracana builds up for United   \n",
              "1                              The main recommendations   \n",
              "2           Tale of Trautmann bears two more retellings   \n",
              "3                        United braced for Mexican wave   \n",
              "4               Guardian readers: are you all whingers?   \n",
              "...                                                 ...   \n",
              "4096      Scotland welcome to join EU, Merkel ally says   \n",
              "4097  CEO of the South African state broadcaster qui...   \n",
              "4098  British Labor Party leader Jeremy Corbyn loses...   \n",
              "4099  Switzerland denies citizenship to Muslim immig...   \n",
              "4100  100+ Nobel laureates urge Greenpeace to stop o...   \n",
              "\n",
              "                                                   Top6  \\\n",
              "0     Depleted Leicester prevail as Elliott spoils E...   \n",
              "1                                Has Cubie killed fees?   \n",
              "2                                   England on the rack   \n",
              "3     Auntie back in fashion, even if the dress look...   \n",
              "4                                      Hollywood Beyond   \n",
              "...                                                 ...   \n",
              "4096  Sterling dips below Friday's 31-year low amid ...   \n",
              "4097  Brexit cost investors $2 trillion, the worst o...   \n",
              "4098  A Muslim Shop in the UK Was Just Firebombed Wh...   \n",
              "4099  Palestinian terrorist stabs israeli teen girl ...   \n",
              "4100  Brazil: Huge spike in number of police killing...   \n",
              "\n",
              "                                                   Top7  \\\n",
              "0                      Hungry Spurs sense rich pickings   \n",
              "1                                Has Cubie killed fees?   \n",
              "2       Pakistan retaliate with call for video of Walsh   \n",
              "3                         Shoaib appeal goes to the top   \n",
              "4                                    Ashes and diamonds   \n",
              "...                                                 ...   \n",
              "4096  No negative news about South African President...   \n",
              "4097  Hong Kong democracy activists call for return ...   \n",
              "4098  Mexican Authorities Sexually Torture Women in ...   \n",
              "4099  Puerto Rico will default on $1 billion of debt...   \n",
              "4100  Austria's highest court annuls presidential el...   \n",
              "\n",
              "                                                   Top8  ...  \\\n",
              "0                     Gunners so wide of an easy target  ...   \n",
              "1                                Has Cubie killed fees?  ...   \n",
              "2                  Cullinan continues his Cape monopoly  ...   \n",
              "3     Hussain hurt by 'shambles' but lays blame on e...  ...   \n",
              "4                      Whingers - a formidable minority  ...   \n",
              "...                                                 ...  ...   \n",
              "4096  Surge in Hate Crimes in the U.K. Following U.K...  ...   \n",
              "4097  Brexit: Iceland president says UK can join 'tr...  ...   \n",
              "4098            UK shares and pound continue to recover  ...   \n",
              "4099  Republic of Ireland fans to be awarded medal f...  ...   \n",
              "4100  Facebook wins privacy case, can track any Belg...  ...   \n",
              "\n",
              "                                                  Top16  \\\n",
              "0              Flintoff injury piles on woe for England   \n",
              "1                                  On the critical list   \n",
              "2                           South Melbourne (Australia)   \n",
              "3     Putin admits Yeltsin quit to give him a head s...   \n",
              "4                                Most everywhere:  UDIs   \n",
              "...                                                 ...   \n",
              "4096  German lawyers to probe Erdogan over alleged w...   \n",
              "4097  US, Canada and Mexico pledge 50% of power from...   \n",
              "4098  Escape Tunnel, Dug by Hand, Is Found at Holoca...   \n",
              "4099  Googles free wifi at Indian railway stations i...   \n",
              "4100  The United States has placed Myanmar, Uzbekist...   \n",
              "\n",
              "                                                  Top17  \\\n",
              "0     Hunters threaten Jospin with new battle of the...   \n",
              "1                             The timing of their lives   \n",
              "2                                       Necaxa (Mexico)   \n",
              "3            BBC worst hit as digital TV begins to bite   \n",
              "4                          Most wanted:  Chloe lunettes   \n",
              "...                                                 ...   \n",
              "4096  Boris Johnson says the UK will continue to \"in...   \n",
              "4097  There is increasing evidence that Australia is...   \n",
              "4098  The land under Beijing is sinking by as much a...   \n",
              "4099  Mounting evidence suggests 'hobbits' were wipe...   \n",
              "4100  S&amp;P revises European Union credit rating t...   \n",
              "\n",
              "                                                  Top18  \\\n",
              "0                   Kohl's successor drawn into scandal   \n",
              "1                                           Dear doctor   \n",
              "2                                   Real Madrid (Spain)   \n",
              "3                           How much can you pay for...   \n",
              "4        Return of the cane 'completely off the agenda'   \n",
              "...                                                 ...   \n",
              "4096  Richard Branson is calling on the UK governmen...   \n",
              "4097  Richard Branson, the founder of Virgin Group, ...   \n",
              "4098  Car bomb and Anti-Islamic attack on Mosque in ...   \n",
              "4099  The men who carried out Tuesday's terror attac...   \n",
              "4100  India gets $1 billion loan from World Bank for...   \n",
              "\n",
              "                                                  Top19  \\\n",
              "0                  The difference between men and women   \n",
              "1     Irish court halts IRA man's extradition to Nor...   \n",
              "2                             Raja Casablanca (Morocco)   \n",
              "3                                    Christmas glitches   \n",
              "4                      From Sleepy Hollow to Greeneland   \n",
              "...                                                 ...   \n",
              "4096             Turkey 'sorry for downing Russian jet'   \n",
              "4097  37,000-yr-old skull from Borneo reveals surpri...   \n",
              "4098  Emaciated lions in Taiz Zoo are trapped in blo...   \n",
              "4099  Calls to suspend Saudi Arabia from UN Human Ri...   \n",
              "4100  U.S. sailors detained by Iran spoke too much u...   \n",
              "\n",
              "                                                  Top20  \\\n",
              "0                   Sara Denver, nurse turned solicitor   \n",
              "1     Burundi peace initiative fades after rebels re...   \n",
              "2                                  Corinthians (Brazil)   \n",
              "3     Upending a table, Chopping a line and Scoring ...   \n",
              "4                 Blunkett outlines vision for over 11s   \n",
              "...                                                 ...   \n",
              "4096  Edward Snowden lawyer vows new push for pardon...   \n",
              "4097  Palestinians stone Western Wall worshipers; po...   \n",
              "4098  Rupert Murdoch describes Brexit as 'wonderful'...   \n",
              "4099  More Than 100 Nobel Laureates Call Out Greenpe...   \n",
              "4100  Mass fish kill in Vietnam solved as Taiwan ste...   \n",
              "\n",
              "                                                  Top21  \\\n",
              "0        Diana's landmine crusade put Tories in a panic   \n",
              "1                  PE points the way forward to the ECB   \n",
              "2                                    Tony's pet project   \n",
              "3      Scientific evidence 'unreliable', defence claims   \n",
              "4     Embattled Dobson attacks 'play now, pay later'...   \n",
              "...                                                 ...   \n",
              "4096  Brexit opinion poll reveals majority don't wan...   \n",
              "4097  Jean-Claude Juncker asks Farage: Why are you h...   \n",
              "4098       More than 40 killed in Yemen suicide attacks   \n",
              "4099  British pedophile sentenced to 85 years in US ...   \n",
              "4100  Philippines president Rodrigo Duterte urges pe...   \n",
              "\n",
              "                                                  Top22  \\\n",
              "0     Yeltsin's resignation caught opposition flat-f...   \n",
              "1     Campaigners keep up pressure on Nazi war crime...   \n",
              "2                               Al Nassr (Saudi Arabia)   \n",
              "3        Fusco wins judicial review in extradition case   \n",
              "4                                     Doom and the Dome   \n",
              "...                                                 ...   \n",
              "4096  Conservative MP Leave Campaigner: \"The leave c...   \n",
              "4097  \"Romanians for Remainians\" offering a new home...   \n",
              "4098  Google Found Disastrous Symantec and Norton Vu...   \n",
              "4099  US permitted 1,200 offshore fracks in Gulf of ...   \n",
              "4100  Spain arrests three Pakistanis accused of prom...   \n",
              "\n",
              "                                                  Top23  \\\n",
              "0                                      Russian roulette   \n",
              "1                                        Jane Ratcliffe   \n",
              "2                                     Ideal Holmes show   \n",
              "3                         Rebels thwart Russian advance   \n",
              "4                       What is the north-south divide?   \n",
              "...                                                 ...   \n",
              "4096  Economists predict UK recession, further weake...   \n",
              "4097  Brexit: Gibraltar in talks with Scotland to st...   \n",
              "4098  Extremist violence on the rise in Germany: Dom...   \n",
              "4099  We will be swimming in ridicule - French beach...   \n",
              "4100  Venezuela, where anger over food shortages is ...   \n",
              "\n",
              "                                                  Top24  \\\n",
              "0                                              Sold out   \n",
              "1     Yet more things you wouldn't know without the ...   \n",
              "2                  Pinochet leaves hospital after tests   \n",
              "3                  Blair orders shake-up of failing NHS   \n",
              "4                             Aitken released from jail   \n",
              "...                                                 ...   \n",
              "4096  New EU 'superstate plan by France, Germany: Cr...   \n",
              "4097                   8 Suicide Bombers Strike Lebanon   \n",
              "4098  BBC News: Labour MPs pass Corbyn no-confidence...   \n",
              "4099  UEFA says no minutes of silence for Istanbul v...   \n",
              "4100  A Hindu temple worker has been killed by three...   \n",
              "\n",
              "                                                  Top25  \n",
              "0                                    Recovering a title  \n",
              "1                          Millennium bug fails to bite  \n",
              "2                                          Useful links  \n",
              "3                           Lessons of law's hard heart  \n",
              "4                                            Gone aloft  \n",
              "...                                                 ...  \n",
              "4096  Pakistani clerics declare transgender marriage...  \n",
              "4097  Mexico's security forces routinely use 'sexual...  \n",
              "4098  Tiny New Zealand town with 'too many jobs' lau...  \n",
              "4099  Law Enforcement Sources: Gun Used in Paris Ter...  \n",
              "4100  Ozone layer hole seems to be healing - US &amp...  \n",
              "\n",
              "[4101 rows x 27 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-143d8772-5a39-469f-93da-fed8e71007ea\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Date</th>\n",
              "      <th>Label</th>\n",
              "      <th>Top1</th>\n",
              "      <th>Top2</th>\n",
              "      <th>Top3</th>\n",
              "      <th>Top4</th>\n",
              "      <th>Top5</th>\n",
              "      <th>Top6</th>\n",
              "      <th>Top7</th>\n",
              "      <th>Top8</th>\n",
              "      <th>...</th>\n",
              "      <th>Top16</th>\n",
              "      <th>Top17</th>\n",
              "      <th>Top18</th>\n",
              "      <th>Top19</th>\n",
              "      <th>Top20</th>\n",
              "      <th>Top21</th>\n",
              "      <th>Top22</th>\n",
              "      <th>Top23</th>\n",
              "      <th>Top24</th>\n",
              "      <th>Top25</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2000-01-03</td>\n",
              "      <td>0</td>\n",
              "      <td>A 'hindrance to operations': extracts from the...</td>\n",
              "      <td>Scorecard</td>\n",
              "      <td>Hughes' instant hit buoys Blues</td>\n",
              "      <td>Jack gets his skates on at ice-cold Alex</td>\n",
              "      <td>Chaos as Maracana builds up for United</td>\n",
              "      <td>Depleted Leicester prevail as Elliott spoils E...</td>\n",
              "      <td>Hungry Spurs sense rich pickings</td>\n",
              "      <td>Gunners so wide of an easy target</td>\n",
              "      <td>...</td>\n",
              "      <td>Flintoff injury piles on woe for England</td>\n",
              "      <td>Hunters threaten Jospin with new battle of the...</td>\n",
              "      <td>Kohl's successor drawn into scandal</td>\n",
              "      <td>The difference between men and women</td>\n",
              "      <td>Sara Denver, nurse turned solicitor</td>\n",
              "      <td>Diana's landmine crusade put Tories in a panic</td>\n",
              "      <td>Yeltsin's resignation caught opposition flat-f...</td>\n",
              "      <td>Russian roulette</td>\n",
              "      <td>Sold out</td>\n",
              "      <td>Recovering a title</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2000-01-04</td>\n",
              "      <td>0</td>\n",
              "      <td>Scorecard</td>\n",
              "      <td>The best lake scene</td>\n",
              "      <td>Leader: German sleaze inquiry</td>\n",
              "      <td>Cheerio, boyo</td>\n",
              "      <td>The main recommendations</td>\n",
              "      <td>Has Cubie killed fees?</td>\n",
              "      <td>Has Cubie killed fees?</td>\n",
              "      <td>Has Cubie killed fees?</td>\n",
              "      <td>...</td>\n",
              "      <td>On the critical list</td>\n",
              "      <td>The timing of their lives</td>\n",
              "      <td>Dear doctor</td>\n",
              "      <td>Irish court halts IRA man's extradition to Nor...</td>\n",
              "      <td>Burundi peace initiative fades after rebels re...</td>\n",
              "      <td>PE points the way forward to the ECB</td>\n",
              "      <td>Campaigners keep up pressure on Nazi war crime...</td>\n",
              "      <td>Jane Ratcliffe</td>\n",
              "      <td>Yet more things you wouldn't know without the ...</td>\n",
              "      <td>Millennium bug fails to bite</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2000-01-05</td>\n",
              "      <td>0</td>\n",
              "      <td>Coventry caught on counter by Flo</td>\n",
              "      <td>United's rivals on the road to Rio</td>\n",
              "      <td>Thatcher issues defence before trial by video</td>\n",
              "      <td>Police help Smith lay down the law at Everton</td>\n",
              "      <td>Tale of Trautmann bears two more retellings</td>\n",
              "      <td>England on the rack</td>\n",
              "      <td>Pakistan retaliate with call for video of Walsh</td>\n",
              "      <td>Cullinan continues his Cape monopoly</td>\n",
              "      <td>...</td>\n",
              "      <td>South Melbourne (Australia)</td>\n",
              "      <td>Necaxa (Mexico)</td>\n",
              "      <td>Real Madrid (Spain)</td>\n",
              "      <td>Raja Casablanca (Morocco)</td>\n",
              "      <td>Corinthians (Brazil)</td>\n",
              "      <td>Tony's pet project</td>\n",
              "      <td>Al Nassr (Saudi Arabia)</td>\n",
              "      <td>Ideal Holmes show</td>\n",
              "      <td>Pinochet leaves hospital after tests</td>\n",
              "      <td>Useful links</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2000-01-06</td>\n",
              "      <td>1</td>\n",
              "      <td>Pilgrim knows how to progress</td>\n",
              "      <td>Thatcher facing ban</td>\n",
              "      <td>McIlroy calls for Irish fighting spirit</td>\n",
              "      <td>Leicester bin stadium blueprint</td>\n",
              "      <td>United braced for Mexican wave</td>\n",
              "      <td>Auntie back in fashion, even if the dress look...</td>\n",
              "      <td>Shoaib appeal goes to the top</td>\n",
              "      <td>Hussain hurt by 'shambles' but lays blame on e...</td>\n",
              "      <td>...</td>\n",
              "      <td>Putin admits Yeltsin quit to give him a head s...</td>\n",
              "      <td>BBC worst hit as digital TV begins to bite</td>\n",
              "      <td>How much can you pay for...</td>\n",
              "      <td>Christmas glitches</td>\n",
              "      <td>Upending a table, Chopping a line and Scoring ...</td>\n",
              "      <td>Scientific evidence 'unreliable', defence claims</td>\n",
              "      <td>Fusco wins judicial review in extradition case</td>\n",
              "      <td>Rebels thwart Russian advance</td>\n",
              "      <td>Blair orders shake-up of failing NHS</td>\n",
              "      <td>Lessons of law's hard heart</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2000-01-07</td>\n",
              "      <td>1</td>\n",
              "      <td>Hitches and Horlocks</td>\n",
              "      <td>Beckham off but United survive</td>\n",
              "      <td>Breast cancer screening</td>\n",
              "      <td>Alan Parker</td>\n",
              "      <td>Guardian readers: are you all whingers?</td>\n",
              "      <td>Hollywood Beyond</td>\n",
              "      <td>Ashes and diamonds</td>\n",
              "      <td>Whingers - a formidable minority</td>\n",
              "      <td>...</td>\n",
              "      <td>Most everywhere:  UDIs</td>\n",
              "      <td>Most wanted:  Chloe lunettes</td>\n",
              "      <td>Return of the cane 'completely off the agenda'</td>\n",
              "      <td>From Sleepy Hollow to Greeneland</td>\n",
              "      <td>Blunkett outlines vision for over 11s</td>\n",
              "      <td>Embattled Dobson attacks 'play now, pay later'...</td>\n",
              "      <td>Doom and the Dome</td>\n",
              "      <td>What is the north-south divide?</td>\n",
              "      <td>Aitken released from jail</td>\n",
              "      <td>Gone aloft</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4096</th>\n",
              "      <td>2016-06-27</td>\n",
              "      <td>0</td>\n",
              "      <td>Barclays and RBS shares suspended from trading...</td>\n",
              "      <td>Pope says Church should ask forgiveness from g...</td>\n",
              "      <td>Poland 'shocked' by xenophobic abuse of Poles ...</td>\n",
              "      <td>There will be no second referendum, cabinet ag...</td>\n",
              "      <td>Scotland welcome to join EU, Merkel ally says</td>\n",
              "      <td>Sterling dips below Friday's 31-year low amid ...</td>\n",
              "      <td>No negative news about South African President...</td>\n",
              "      <td>Surge in Hate Crimes in the U.K. Following U.K...</td>\n",
              "      <td>...</td>\n",
              "      <td>German lawyers to probe Erdogan over alleged w...</td>\n",
              "      <td>Boris Johnson says the UK will continue to \"in...</td>\n",
              "      <td>Richard Branson is calling on the UK governmen...</td>\n",
              "      <td>Turkey 'sorry for downing Russian jet'</td>\n",
              "      <td>Edward Snowden lawyer vows new push for pardon...</td>\n",
              "      <td>Brexit opinion poll reveals majority don't wan...</td>\n",
              "      <td>Conservative MP Leave Campaigner: \"The leave c...</td>\n",
              "      <td>Economists predict UK recession, further weake...</td>\n",
              "      <td>New EU 'superstate plan by France, Germany: Cr...</td>\n",
              "      <td>Pakistani clerics declare transgender marriage...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4097</th>\n",
              "      <td>2016-06-28</td>\n",
              "      <td>1</td>\n",
              "      <td>2,500 Scientists To Australia: If You Want To ...</td>\n",
              "      <td>The personal details of 112,000 French police ...</td>\n",
              "      <td>S&amp;amp;P cuts United Kingdom sovereign credit r...</td>\n",
              "      <td>Huge helium deposit found in Africa</td>\n",
              "      <td>CEO of the South African state broadcaster qui...</td>\n",
              "      <td>Brexit cost investors $2 trillion, the worst o...</td>\n",
              "      <td>Hong Kong democracy activists call for return ...</td>\n",
              "      <td>Brexit: Iceland president says UK can join 'tr...</td>\n",
              "      <td>...</td>\n",
              "      <td>US, Canada and Mexico pledge 50% of power from...</td>\n",
              "      <td>There is increasing evidence that Australia is...</td>\n",
              "      <td>Richard Branson, the founder of Virgin Group, ...</td>\n",
              "      <td>37,000-yr-old skull from Borneo reveals surpri...</td>\n",
              "      <td>Palestinians stone Western Wall worshipers; po...</td>\n",
              "      <td>Jean-Claude Juncker asks Farage: Why are you h...</td>\n",
              "      <td>\"Romanians for Remainians\" offering a new home...</td>\n",
              "      <td>Brexit: Gibraltar in talks with Scotland to st...</td>\n",
              "      <td>8 Suicide Bombers Strike Lebanon</td>\n",
              "      <td>Mexico's security forces routinely use 'sexual...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4098</th>\n",
              "      <td>2016-06-29</td>\n",
              "      <td>1</td>\n",
              "      <td>Explosion At Airport In Istanbul</td>\n",
              "      <td>Yemeni former president: Terrorism is the offs...</td>\n",
              "      <td>UK must accept freedom of movement to access E...</td>\n",
              "      <td>Devastated: scientists too late to captive bre...</td>\n",
              "      <td>British Labor Party leader Jeremy Corbyn loses...</td>\n",
              "      <td>A Muslim Shop in the UK Was Just Firebombed Wh...</td>\n",
              "      <td>Mexican Authorities Sexually Torture Women in ...</td>\n",
              "      <td>UK shares and pound continue to recover</td>\n",
              "      <td>...</td>\n",
              "      <td>Escape Tunnel, Dug by Hand, Is Found at Holoca...</td>\n",
              "      <td>The land under Beijing is sinking by as much a...</td>\n",
              "      <td>Car bomb and Anti-Islamic attack on Mosque in ...</td>\n",
              "      <td>Emaciated lions in Taiz Zoo are trapped in blo...</td>\n",
              "      <td>Rupert Murdoch describes Brexit as 'wonderful'...</td>\n",
              "      <td>More than 40 killed in Yemen suicide attacks</td>\n",
              "      <td>Google Found Disastrous Symantec and Norton Vu...</td>\n",
              "      <td>Extremist violence on the rise in Germany: Dom...</td>\n",
              "      <td>BBC News: Labour MPs pass Corbyn no-confidence...</td>\n",
              "      <td>Tiny New Zealand town with 'too many jobs' lau...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4099</th>\n",
              "      <td>2016-06-30</td>\n",
              "      <td>1</td>\n",
              "      <td>Jamaica proposes marijuana dispensers for tour...</td>\n",
              "      <td>Stephen Hawking says pollution and 'stupidity'...</td>\n",
              "      <td>Boris Johnson says he will not run for Tory pa...</td>\n",
              "      <td>Six gay men in Ivory Coast were abused and for...</td>\n",
              "      <td>Switzerland denies citizenship to Muslim immig...</td>\n",
              "      <td>Palestinian terrorist stabs israeli teen girl ...</td>\n",
              "      <td>Puerto Rico will default on $1 billion of debt...</td>\n",
              "      <td>Republic of Ireland fans to be awarded medal f...</td>\n",
              "      <td>...</td>\n",
              "      <td>Googles free wifi at Indian railway stations i...</td>\n",
              "      <td>Mounting evidence suggests 'hobbits' were wipe...</td>\n",
              "      <td>The men who carried out Tuesday's terror attac...</td>\n",
              "      <td>Calls to suspend Saudi Arabia from UN Human Ri...</td>\n",
              "      <td>More Than 100 Nobel Laureates Call Out Greenpe...</td>\n",
              "      <td>British pedophile sentenced to 85 years in US ...</td>\n",
              "      <td>US permitted 1,200 offshore fracks in Gulf of ...</td>\n",
              "      <td>We will be swimming in ridicule - French beach...</td>\n",
              "      <td>UEFA says no minutes of silence for Istanbul v...</td>\n",
              "      <td>Law Enforcement Sources: Gun Used in Paris Ter...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4100</th>\n",
              "      <td>2016-07-01</td>\n",
              "      <td>1</td>\n",
              "      <td>A 117-year-old woman in Mexico City finally re...</td>\n",
              "      <td>IMF chief backs Athens as permanent Olympic host</td>\n",
              "      <td>The president of France says if Brexit won, so...</td>\n",
              "      <td>British Man Who Must Give Police 24 Hours' Not...</td>\n",
              "      <td>100+ Nobel laureates urge Greenpeace to stop o...</td>\n",
              "      <td>Brazil: Huge spike in number of police killing...</td>\n",
              "      <td>Austria's highest court annuls presidential el...</td>\n",
              "      <td>Facebook wins privacy case, can track any Belg...</td>\n",
              "      <td>...</td>\n",
              "      <td>The United States has placed Myanmar, Uzbekist...</td>\n",
              "      <td>S&amp;amp;P revises European Union credit rating t...</td>\n",
              "      <td>India gets $1 billion loan from World Bank for...</td>\n",
              "      <td>U.S. sailors detained by Iran spoke too much u...</td>\n",
              "      <td>Mass fish kill in Vietnam solved as Taiwan ste...</td>\n",
              "      <td>Philippines president Rodrigo Duterte urges pe...</td>\n",
              "      <td>Spain arrests three Pakistanis accused of prom...</td>\n",
              "      <td>Venezuela, where anger over food shortages is ...</td>\n",
              "      <td>A Hindu temple worker has been killed by three...</td>\n",
              "      <td>Ozone layer hole seems to be healing - US &amp;amp...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>4101 rows × 27 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-143d8772-5a39-469f-93da-fed8e71007ea')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-143d8772-5a39-469f-93da-fed8e71007ea button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-143d8772-5a39-469f-93da-fed8e71007ea');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train = data[data['Date'] < '20150101']\n",
        "test = data[data['Date'] >= '20150101']"
      ],
      "metadata": {
        "id": "f5VJ4WeYSrlj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "td = train.iloc[:,2:27]\n",
        "td.replace(\"[^a-zA-Z]\", \" \", regex=True, inplace = True)\n",
        "\n",
        "list1 = [i for i in range(25)]\n",
        "new_index = [str(i) for i in list1]\n",
        "td.columns = new_index\n",
        "td.head(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 431
        },
        "id": "JFdNSkxCVsvU",
        "outputId": "626e48e1-30dd-4f89-8b91-03d18c63df1a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                   0                    1  \\\n",
              "0  A  hindrance to operations   extracts from the...            Scorecard   \n",
              "1                                          Scorecard  The best lake scene   \n",
              "\n",
              "                                 2                                         3  \\\n",
              "0  Hughes  instant hit buoys Blues  Jack gets his skates on at ice cold Alex   \n",
              "1    Leader  German sleaze inquiry                             Cheerio  boyo   \n",
              "\n",
              "                                        4  \\\n",
              "0  Chaos as Maracana builds up for United   \n",
              "1                The main recommendations   \n",
              "\n",
              "                                                   5  \\\n",
              "0  Depleted Leicester prevail as Elliott spoils E...   \n",
              "1                             Has Cubie killed fees    \n",
              "\n",
              "                                  6                                  7  \\\n",
              "0  Hungry Spurs sense rich pickings  Gunners so wide of an easy target   \n",
              "1            Has Cubie killed fees              Has Cubie killed fees    \n",
              "\n",
              "                                                   8  \\\n",
              "0      Derby raise a glass to Strupar s debut double   \n",
              "1  Hopkins  furious  at Foster s lack of Hannibal...   \n",
              "\n",
              "                                          9  ...  \\\n",
              "0  Southgate strikes  Leeds pay the penalty  ...   \n",
              "1                    Has Cubie killed fees   ...   \n",
              "\n",
              "                                         15  \\\n",
              "0  Flintoff injury piles on woe for England   \n",
              "1                      On the critical list   \n",
              "\n",
              "                                                  16  \\\n",
              "0  Hunters threaten Jospin with new battle of the...   \n",
              "1                          The timing of their lives   \n",
              "\n",
              "                                    17  \\\n",
              "0  Kohl s successor drawn into scandal   \n",
              "1                          Dear doctor   \n",
              "\n",
              "                                                  18  \\\n",
              "0               The difference between men and women   \n",
              "1  Irish court halts IRA man s extradition to Nor...   \n",
              "\n",
              "                                                  19  \\\n",
              "0                Sara Denver  nurse turned solicitor   \n",
              "1  Burundi peace initiative fades after rebels re...   \n",
              "\n",
              "                                               20  \\\n",
              "0  Diana s landmine crusade put Tories in a panic   \n",
              "1            PE points the way forward to the ECB   \n",
              "\n",
              "                                                  21                22  \\\n",
              "0  Yeltsin s resignation caught opposition flat f...  Russian roulette   \n",
              "1  Campaigners keep up pressure on Nazi war crime...    Jane Ratcliffe   \n",
              "\n",
              "                                                  23  \\\n",
              "0                                           Sold out   \n",
              "1  Yet more things you wouldn t know without the ...   \n",
              "\n",
              "                             24  \n",
              "0            Recovering a title  \n",
              "1  Millennium bug fails to bite  \n",
              "\n",
              "[2 rows x 25 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-760fb0df-b89c-4bd9-a1e8-391bd8bb03d1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>...</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>A  hindrance to operations   extracts from the...</td>\n",
              "      <td>Scorecard</td>\n",
              "      <td>Hughes  instant hit buoys Blues</td>\n",
              "      <td>Jack gets his skates on at ice cold Alex</td>\n",
              "      <td>Chaos as Maracana builds up for United</td>\n",
              "      <td>Depleted Leicester prevail as Elliott spoils E...</td>\n",
              "      <td>Hungry Spurs sense rich pickings</td>\n",
              "      <td>Gunners so wide of an easy target</td>\n",
              "      <td>Derby raise a glass to Strupar s debut double</td>\n",
              "      <td>Southgate strikes  Leeds pay the penalty</td>\n",
              "      <td>...</td>\n",
              "      <td>Flintoff injury piles on woe for England</td>\n",
              "      <td>Hunters threaten Jospin with new battle of the...</td>\n",
              "      <td>Kohl s successor drawn into scandal</td>\n",
              "      <td>The difference between men and women</td>\n",
              "      <td>Sara Denver  nurse turned solicitor</td>\n",
              "      <td>Diana s landmine crusade put Tories in a panic</td>\n",
              "      <td>Yeltsin s resignation caught opposition flat f...</td>\n",
              "      <td>Russian roulette</td>\n",
              "      <td>Sold out</td>\n",
              "      <td>Recovering a title</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Scorecard</td>\n",
              "      <td>The best lake scene</td>\n",
              "      <td>Leader  German sleaze inquiry</td>\n",
              "      <td>Cheerio  boyo</td>\n",
              "      <td>The main recommendations</td>\n",
              "      <td>Has Cubie killed fees</td>\n",
              "      <td>Has Cubie killed fees</td>\n",
              "      <td>Has Cubie killed fees</td>\n",
              "      <td>Hopkins  furious  at Foster s lack of Hannibal...</td>\n",
              "      <td>Has Cubie killed fees</td>\n",
              "      <td>...</td>\n",
              "      <td>On the critical list</td>\n",
              "      <td>The timing of their lives</td>\n",
              "      <td>Dear doctor</td>\n",
              "      <td>Irish court halts IRA man s extradition to Nor...</td>\n",
              "      <td>Burundi peace initiative fades after rebels re...</td>\n",
              "      <td>PE points the way forward to the ECB</td>\n",
              "      <td>Campaigners keep up pressure on Nazi war crime...</td>\n",
              "      <td>Jane Ratcliffe</td>\n",
              "      <td>Yet more things you wouldn t know without the ...</td>\n",
              "      <td>Millennium bug fails to bite</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2 rows × 25 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-760fb0df-b89c-4bd9-a1e8-391bd8bb03d1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-760fb0df-b89c-4bd9-a1e8-391bd8bb03d1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-760fb0df-b89c-4bd9-a1e8-391bd8bb03d1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 86
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for ind in new_index:\n",
        "  td[ind] = td[ind].str.lower()\n",
        "td.head(1)  "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        },
        "id": "LXT8DBtgWUiK",
        "outputId": "4594ee2d-50de-4d8e-d956-f2e4270ab825"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                   0          1  \\\n",
              "0  a  hindrance to operations   extracts from the...  scorecard   \n",
              "\n",
              "                                 2                                         3  \\\n",
              "0  hughes  instant hit buoys blues  jack gets his skates on at ice cold alex   \n",
              "\n",
              "                                        4  \\\n",
              "0  chaos as maracana builds up for united   \n",
              "\n",
              "                                                   5  \\\n",
              "0  depleted leicester prevail as elliott spoils e...   \n",
              "\n",
              "                                  6                                  7  \\\n",
              "0  hungry spurs sense rich pickings  gunners so wide of an easy target   \n",
              "\n",
              "                                               8  \\\n",
              "0  derby raise a glass to strupar s debut double   \n",
              "\n",
              "                                          9  ...  \\\n",
              "0  southgate strikes  leeds pay the penalty  ...   \n",
              "\n",
              "                                         15  \\\n",
              "0  flintoff injury piles on woe for england   \n",
              "\n",
              "                                                  16  \\\n",
              "0  hunters threaten jospin with new battle of the...   \n",
              "\n",
              "                                    17                                    18  \\\n",
              "0  kohl s successor drawn into scandal  the difference between men and women   \n",
              "\n",
              "                                    19  \\\n",
              "0  sara denver  nurse turned solicitor   \n",
              "\n",
              "                                               20  \\\n",
              "0  diana s landmine crusade put tories in a panic   \n",
              "\n",
              "                                                  21                22  \\\n",
              "0  yeltsin s resignation caught opposition flat f...  russian roulette   \n",
              "\n",
              "         23                  24  \n",
              "0  sold out  recovering a title  \n",
              "\n",
              "[1 rows x 25 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d913310d-8e2c-4b85-bb49-a16556481279\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>...</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>a  hindrance to operations   extracts from the...</td>\n",
              "      <td>scorecard</td>\n",
              "      <td>hughes  instant hit buoys blues</td>\n",
              "      <td>jack gets his skates on at ice cold alex</td>\n",
              "      <td>chaos as maracana builds up for united</td>\n",
              "      <td>depleted leicester prevail as elliott spoils e...</td>\n",
              "      <td>hungry spurs sense rich pickings</td>\n",
              "      <td>gunners so wide of an easy target</td>\n",
              "      <td>derby raise a glass to strupar s debut double</td>\n",
              "      <td>southgate strikes  leeds pay the penalty</td>\n",
              "      <td>...</td>\n",
              "      <td>flintoff injury piles on woe for england</td>\n",
              "      <td>hunters threaten jospin with new battle of the...</td>\n",
              "      <td>kohl s successor drawn into scandal</td>\n",
              "      <td>the difference between men and women</td>\n",
              "      <td>sara denver  nurse turned solicitor</td>\n",
              "      <td>diana s landmine crusade put tories in a panic</td>\n",
              "      <td>yeltsin s resignation caught opposition flat f...</td>\n",
              "      <td>russian roulette</td>\n",
              "      <td>sold out</td>\n",
              "      <td>recovering a title</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1 rows × 25 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d913310d-8e2c-4b85-bb49-a16556481279')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d913310d-8e2c-4b85-bb49-a16556481279 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d913310d-8e2c-4b85-bb49-a16556481279');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 87
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "td.tail(1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 313
        },
        "id": "59Vo80o8Zh0y",
        "outputId": "f274315a-fb2e-4cb7-892f-9a7f12d76245"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                      0  \\\n",
              "3974  microsoft corp said on wednesday it will begin...   \n",
              "\n",
              "                                                      1  \\\n",
              "3974  greek orthodox church  homosexuals are god s c...   \n",
              "\n",
              "                                                      2  \\\n",
              "3974  vladmir putin officially dissolves roscosmos  ...   \n",
              "\n",
              "                                                      3  \\\n",
              "3974  donald trump could be refused entry to the uk ...   \n",
              "\n",
              "                                                      4  \\\n",
              "3974  saudi arabia says its ready to meet any additi...   \n",
              "\n",
              "                                                      5  \\\n",
              "3974  new year fireworks and festivities have been c...   \n",
              "\n",
              "                                                      6  \\\n",
              "3974  new species of human may have shared our caves...   \n",
              "\n",
              "                                                      7  \\\n",
              "3974  egypt becomes second nation to shut down faceb...   \n",
              "\n",
              "                                                      8  \\\n",
              "3974  polish government takes control of public medi...   \n",
              "\n",
              "                                                      9  ...  \\\n",
              "3974  from jan    uk to stop financial aid to india ...  ...   \n",
              "\n",
              "                                                     15  \\\n",
              "3974  ramadi residents fleeing isis   they wanted to...   \n",
              "\n",
              "                                                     16  \\\n",
              "3974  israeli gov t approves nis    billion plan to ...   \n",
              "\n",
              "                                                     17  \\\n",
              "3974  russia denies  absurd  claims that air strikes...   \n",
              "\n",
              "                                          18  \\\n",
              "3974  web attack knocks bbc websites offline   \n",
              "\n",
              "                                                    19  \\\n",
              "3974  iran says any u s  sanctions on missiles illegal   \n",
              "\n",
              "                                                     20  \\\n",
              "3974  maternity benefits  indian government plans to...   \n",
              "\n",
              "                                                     21  \\\n",
              "3974   rat on a plane  forces air india flight to tu...   \n",
              "\n",
              "                                                     22  \\\n",
              "3974  philippines to join china backed aiib infrastr...   \n",
              "\n",
              "                                                     23  \\\n",
              "3974  espn invites five fifa presidential candidates...   \n",
              "\n",
              "                                                     24  \n",
              "3974  hamas to keep palestinians from ringing in new...  \n",
              "\n",
              "[1 rows x 25 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b35a8da2-a070-4cb2-84a1-633531af036b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>...</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3974</th>\n",
              "      <td>microsoft corp said on wednesday it will begin...</td>\n",
              "      <td>greek orthodox church  homosexuals are god s c...</td>\n",
              "      <td>vladmir putin officially dissolves roscosmos  ...</td>\n",
              "      <td>donald trump could be refused entry to the uk ...</td>\n",
              "      <td>saudi arabia says its ready to meet any additi...</td>\n",
              "      <td>new year fireworks and festivities have been c...</td>\n",
              "      <td>new species of human may have shared our caves...</td>\n",
              "      <td>egypt becomes second nation to shut down faceb...</td>\n",
              "      <td>polish government takes control of public medi...</td>\n",
              "      <td>from jan    uk to stop financial aid to india ...</td>\n",
              "      <td>...</td>\n",
              "      <td>ramadi residents fleeing isis   they wanted to...</td>\n",
              "      <td>israeli gov t approves nis    billion plan to ...</td>\n",
              "      <td>russia denies  absurd  claims that air strikes...</td>\n",
              "      <td>web attack knocks bbc websites offline</td>\n",
              "      <td>iran says any u s  sanctions on missiles illegal</td>\n",
              "      <td>maternity benefits  indian government plans to...</td>\n",
              "      <td>rat on a plane  forces air india flight to tu...</td>\n",
              "      <td>philippines to join china backed aiib infrastr...</td>\n",
              "      <td>espn invites five fifa presidential candidates...</td>\n",
              "      <td>hamas to keep palestinians from ringing in new...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1 rows × 25 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b35a8da2-a070-4cb2-84a1-633531af036b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b35a8da2-a070-4cb2-84a1-633531af036b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b35a8da2-a070-4cb2-84a1-633531af036b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 88
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "headline = []\n",
        "for row in range(len(td)):    \n",
        "  headline.append(\" \".join(str(x) for x in td.iloc[row,0:25]))"
      ],
      "metadata": {
        "id": "5xlg1SH7WlrS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "headline"
      ],
      "metadata": {
        "id": "YpRK8_U3ZFHU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.ensemble import RandomForestClassifier\n"
      ],
      "metadata": {
        "id": "-dg9KM2dZUpX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cv = CountVectorizer(ngram_range=(2,2))\n",
        "train_d = cv.fit_transform(headline)"
      ],
      "metadata": {
        "id": "E7j9PB_vacxQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = RandomForestClassifier(n_estimators=200, criterion='entropy')\n",
        "model.fit(train_d, train['Label'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d9b6S_Prap-m",
        "outputId": "30fbe92b-c116-4803-fabb-d73650362897"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomForestClassifier(criterion='entropy', n_estimators=200)"
            ]
          },
          "metadata": {},
          "execution_count": 93
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_headline = []\n",
        "for row in range(len(test)):    \n",
        "  test_headline.append(\" \".join(str(x) for x in test.iloc[row,2:27]))\n",
        "\n",
        "test_d = cv.transform(test_headline)  "
      ],
      "metadata": {
        "id": "e_WmyH3Ca97x"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(test_headline)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eGstfLQobdYt",
        "outputId": "6749360d-3270-4f9e-8a7b-3f7acd874736"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "126"
            ]
          },
          "metadata": {},
          "execution_count": 100
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pred = model.predict(test_d)"
      ],
      "metadata": {
        "id": "-CIbcYgnbQcm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(pred)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n8e_qAH4bY0N",
        "outputId": "4961d9f4-dd27-4fae-fc26-7147a701c07d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "126"
            ]
          },
          "metadata": {},
          "execution_count": 102
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import confusion_matrix, accuracy_score\n",
        "\n",
        "cm = confusion_matrix(test['Label'], pred)"
      ],
      "metadata": {
        "id": "kGQ-fAdHb4mB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cm"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b_q_Y4BacPJ_",
        "outputId": "e08a6e3b-83ef-45ab-eb7f-126b6bdcd5d5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[10, 46],\n",
              "       [13, 57]])"
            ]
          },
          "metadata": {},
          "execution_count": 104
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "NwEbymtGeIkx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        " # Fake news classifier"
      ],
      "metadata": {
        "id": "E9W3liOYAdcd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd"
      ],
      "metadata": {
        "id": "2O4q08OmAfqs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv(\"/content/train.csv\", encoding='latin-1')\n",
        "df.head(1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        },
        "id": "wdGaVBYMAlCB",
        "outputId": "17937b75-eed9-421d-a010-b3b17c0e3623"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   id                                              title         author  \\\n",
              "0   0  House Dem Aide: We Didnât Even See Comeyâs...  Darrell Lucus   \n",
              "\n",
              "                                                text  label  \n",
              "0  House Dem Aide: We Didnât Even See Comeyâs...      1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d96b3980-4432-4b09-811e-d6837a30fbe1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>title</th>\n",
              "      <th>author</th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>House Dem Aide: We Didnât Even See Comeyâs...</td>\n",
              "      <td>Darrell Lucus</td>\n",
              "      <td>House Dem Aide: We Didnât Even See Comeyâs...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d96b3980-4432-4b09-811e-d6837a30fbe1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d96b3980-4432-4b09-811e-d6837a30fbe1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d96b3980-4432-4b09-811e-d6837a30fbe1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x = df.drop('label', axis=1)\n",
        "x.head(1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        },
        "id": "V70ONpKnAn0b",
        "outputId": "b9d6c7a2-53aa-40f0-af95-b1ea474e7370"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   id                                              title         author  \\\n",
              "0   0  House Dem Aide: We Didnât Even See Comeyâs...  Darrell Lucus   \n",
              "\n",
              "                                                text  \n",
              "0  House Dem Aide: We Didnât Even See Comeyâs...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b9af1302-5ba8-4d1f-a4b3-343c6cadb310\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>title</th>\n",
              "      <th>author</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>House Dem Aide: We Didnât Even See Comeyâs...</td>\n",
              "      <td>Darrell Lucus</td>\n",
              "      <td>House Dem Aide: We Didnât Even See Comeyâs...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b9af1302-5ba8-4d1f-a4b3-343c6cadb310')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b9af1302-5ba8-4d1f-a4b3-343c6cadb310 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b9af1302-5ba8-4d1f-a4b3-343c6cadb310');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y = df['label']"
      ],
      "metadata": {
        "id": "M7R0uCxdA35n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n"
      ],
      "metadata": {
        "id": "4IixvKg-BcXl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = df.dropna()"
      ],
      "metadata": {
        "id": "8-sWIfE5BkPh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "message = df.copy()\n",
        "message.reset_index(inplace=True)"
      ],
      "metadata": {
        "id": "jjGPEIKzBmVL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.corpus import stopwords\n",
        "from nltk.stem.porter import PorterStemmer\n",
        "import re\n",
        "\n",
        "ps = PorterStemmer()\n",
        "\n",
        "corpus = []\n",
        "\n",
        "for i in range(len(message)):\n",
        "  review = re.sub(\"[^a-zA-Z]\",\" \", message['title'][i])\n",
        "  review = review.lower()\n",
        "  review = review.split()\n",
        "  review = [ps.stem(word) for word in review if word not in stopwords.words('english')]\n",
        "  review = \" \".join(review)\n",
        "  corpus.append(review)"
      ],
      "metadata": {
        "id": "23-csz4yBoRg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "\n",
        "cv = CountVectorizer(max_features=5000, ngram_range=(2,2))\n",
        "X = cv.fit_transform(corpus).toarray()"
      ],
      "metadata": {
        "id": "34nv8BYyBoTw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y=message['label']"
      ],
      "metadata": {
        "id": "aGWM-W7fBoWI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn import model_selection\n",
        "from sklearn.model_selection import train_test_split\n",
        "x_train, x_test, y_train, y_test = train_test_split(X,y, random_state=42, test_size=0.2)"
      ],
      "metadata": {
        "id": "fXWzmtcCBoYl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cv.get_feature_names()"
      ],
      "metadata": {
        "id": "24GP8Bd1Boa9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cv.get_params()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dOxcGcutBodg",
        "outputId": "ccff0a29-44d9-4442-c0fd-3d4e35d1579a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'analyzer': 'word',\n",
              " 'binary': False,\n",
              " 'decode_error': 'strict',\n",
              " 'dtype': numpy.int64,\n",
              " 'encoding': 'utf-8',\n",
              " 'input': 'content',\n",
              " 'lowercase': True,\n",
              " 'max_df': 1.0,\n",
              " 'max_features': 5000,\n",
              " 'min_df': 1,\n",
              " 'ngram_range': (2, 2),\n",
              " 'preprocessor': None,\n",
              " 'stop_words': None,\n",
              " 'strip_accents': None,\n",
              " 'token_pattern': '(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
              " 'tokenizer': None,\n",
              " 'vocabulary': None}"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "count_df = pd.DataFrame(x_train)\n",
        "count_df"
      ],
      "metadata": {
        "id": "0n2OE3JsBofS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn import naive_bayes\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "model = MultinomialNB()\n",
        "\n",
        "model.fit(x_train, y_train)\n",
        "pred = model.predict(x_test)"
      ],
      "metadata": {
        "id": "67S2OaFcBohZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk import metrics\n",
        "from sklearn.metrics import confusion_matrix\n",
        "cm = confusion_matrix(y_test, pred)\n",
        "cm"
      ],
      "metadata": {
        "id": "oyqj2a83Bol9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ed4a5f1d-2dae-4a22-b172-330b2c7dcff9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[1907,  175],\n",
              "       [ 677,  898]])"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# passive aggressive classifier"
      ],
      "metadata": {
        "id": "YM-CREHoj9ag"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.linear_model import PassiveAggressiveClassifier\n",
        "lin_model = PassiveAggressiveClassifier(max_iter=50)"
      ],
      "metadata": {
        "id": "9kYL946OBnJS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lin_model.fit(x_train, y_train)\n",
        "pred = lin_model.predict(x_test)\n",
        "cm = confusion_matrix(y_test, pred)\n",
        "cm"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q7E0ya76jNPu",
        "outputId": "3de877a2-64fa-41ec-bc3e-ca3679bd3378"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[1643,  439],\n",
              "       [  56, 1519]])"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# multinomial classifier with hyperparameter"
      ],
      "metadata": {
        "id": "lVNlFgx6kIUI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "classifier = MultinomialNB(alpha=0.1)"
      ],
      "metadata": {
        "id": "uZNzbCWPjyJ8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import sklearn\n",
        "import numpy as np\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "prev_score = 0\n",
        "for alpha in np.arange(0,1,0.1):\n",
        "  \n",
        "  sub_model = MultinomialNB(alpha=alpha)\n",
        "  sub_model.fit(x_train, y_train)\n",
        "  y_pred = sub_model.predict(x_test)\n",
        "  score = accuracy_score(y_test, y_pred)\n",
        "  print(score)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m_jrC08ykQrR",
        "outputId": "f277b344-b807-4794-9e8d-9ce8dd937ba9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/naive_bayes.py:557: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  % _ALPHA_MIN\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.7571780147662018\n",
            "0.7686628383921247\n",
            "0.7697566311184031\n",
            "0.7700300792999727\n",
            "0.7700300792999727\n",
            "0.7681159420289855\n",
            "0.7667487011211376\n",
            "0.7670221493027072\n",
            "0.7670221493027072\n",
            "0.7667487011211376\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "B925flDgkywR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "wzGP0ix0wWgx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Word embedding with keras"
      ],
      "metadata": {
        "id": "H4CYxUrcwXRj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from keras.preprocessing.text import one_hot"
      ],
      "metadata": {
        "id": "io8MhOmNwhSN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = [\"the glass of milk\",\n",
        "            \"the glass of juice\",\n",
        "            \"i am a good boy\",\n",
        "            \"understand the meaning\",\n",
        "            \"your videos are good\"]"
      ],
      "metadata": {
        "id": "BXf4-JCcxYs1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "voc_size = 10000"
      ],
      "metadata": {
        "id": "mRext4EkxoGh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "onehot_sent = [one_hot(words, voc_size) for words in sentence]\n",
        "onehot_sent"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sVtWEsFExtyf",
        "outputId": "0e5d4a57-394d-47c3-b4a1-718de2aa140c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[324, 5833, 8413, 4948],\n",
              " [324, 5833, 8413, 6840],\n",
              " [9625, 7563, 1193, 4809, 1045],\n",
              " [6768, 324, 7122],\n",
              " [5378, 2176, 1927, 4809]]"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Word embedding"
      ],
      "metadata": {
        "id": "H8ElUMBDx9AN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from keras.layers import Embedding, LSTM, Dense, Dropout\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.models import Sequential\n",
        "import numpy as np"
      ],
      "metadata": {
        "id": "ThVzq5qNx4cN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from gensim.matutils import pad\n",
        "sent_l = 8\n",
        "embedded = pad_sequences(onehot_sent, padding='pre', maxlen=sent_l)\n",
        "embedded"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6b2dzzZzyJAu",
        "outputId": "dc8c93ef-027a-4d7e-ec84-a8214ebafbcd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[   0,    0,    0,    0,  324, 5833, 8413, 4948],\n",
              "       [   0,    0,    0,    0,  324, 5833, 8413, 6840],\n",
              "       [   0,    0,    0, 9625, 7563, 1193, 4809, 1045],\n",
              "       [   0,    0,    0,    0,    0, 6768,  324, 7122],\n",
              "       [   0,    0,    0,    0, 5378, 2176, 1927, 4809]], dtype=int32)"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dim = 15"
      ],
      "metadata": {
        "id": "UuV5CRZEyvRQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = Sequential()\n",
        "model.add(Embedding(voc_size, 15, input_length=sent_l))\n",
        "model.compile('adam', 'mse')\n",
        "model.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ALYM5G_hyz5P",
        "outputId": "617c9b17-04be-45e2-bb6b-02d8ff89bcdc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding (Embedding)       (None, 8, 15)             150000    \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 150,000\n",
            "Trainable params: 150,000\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.predict(embedded)"
      ],
      "metadata": {
        "id": "MtYdBwC3zG1w"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.predict(embedded[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QZUlqEfBzNwE",
        "outputId": "d7573fad-4e99-4f4c-a46f-4dd1cffb3f16"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:Model was constructed with shape (None, 8) for input KerasTensor(type_spec=TensorSpec(shape=(None, 8), dtype=tf.float32, name='embedding_input'), name='embedding_input', description=\"created by layer 'embedding_input'\"), but it was called on an input with incompatible shape (None,).\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-0.03015925, -0.03700181,  0.0446083 ,  0.04006027, -0.03383664,\n",
              "         0.02871597, -0.03615773, -0.04672622,  0.03441199, -0.02341638,\n",
              "         0.03590367, -0.03995843, -0.00548173, -0.01363111, -0.03005326],\n",
              "       [-0.03015925, -0.03700181,  0.0446083 ,  0.04006027, -0.03383664,\n",
              "         0.02871597, -0.03615773, -0.04672622,  0.03441199, -0.02341638,\n",
              "         0.03590367, -0.03995843, -0.00548173, -0.01363111, -0.03005326],\n",
              "       [-0.03015925, -0.03700181,  0.0446083 ,  0.04006027, -0.03383664,\n",
              "         0.02871597, -0.03615773, -0.04672622,  0.03441199, -0.02341638,\n",
              "         0.03590367, -0.03995843, -0.00548173, -0.01363111, -0.03005326],\n",
              "       [-0.03015925, -0.03700181,  0.0446083 ,  0.04006027, -0.03383664,\n",
              "         0.02871597, -0.03615773, -0.04672622,  0.03441199, -0.02341638,\n",
              "         0.03590367, -0.03995843, -0.00548173, -0.01363111, -0.03005326],\n",
              "       [ 0.01255033, -0.01388117,  0.00227948, -0.01808159, -0.03711464,\n",
              "        -0.03131342,  0.04491737, -0.00208696, -0.02890465,  0.02046547,\n",
              "        -0.02513938,  0.03411451, -0.00112016, -0.01028615, -0.00091506],\n",
              "       [-0.02577587, -0.03604943, -0.02903976,  0.032153  , -0.03567165,\n",
              "        -0.00518602, -0.03453815, -0.02646154,  0.0282811 ,  0.00723888,\n",
              "        -0.00047276, -0.04359999, -0.04538256, -0.01429213, -0.04213375],\n",
              "       [-0.00988605,  0.02514717, -0.00479758,  0.04211589,  0.03256801,\n",
              "         0.04540269, -0.00716728, -0.01589401,  0.00876049, -0.0075802 ,\n",
              "         0.00144345,  0.00139886,  0.03232599,  0.0088603 , -0.04023913],\n",
              "       [-0.03706817, -0.03764652,  0.02225676,  0.04727373, -0.00723799,\n",
              "         0.04235804, -0.01728465,  0.03270807, -0.01303517,  0.04239817,\n",
              "        -0.04824593,  0.03970737,  0.04612174,  0.02935525, -0.04795585]],\n",
              "      dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 2nd model"
      ],
      "metadata": {
        "id": "BJ0TaHJb1SKx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# adding a LSTM layer\n",
        "embedding_vector_feature = 40\n",
        "model = Sequential()\n",
        "model.add(Embedding(voc_size, embedding_vector_feature, input_length=sent_l))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(LSTM(100))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Dense(1, activation='sigmoid'))\n",
        "model.compile(loss = 'binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "model.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u46zHYypzt_8",
        "outputId": "d188ad9c-9668-4aa5-ce0f-f2f785bd8aa0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_7\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_7 (Embedding)     (None, 8, 40)             400000    \n",
            "                                                                 \n",
            " dropout (Dropout)           (None, 8, 40)             0         \n",
            "                                                                 \n",
            " lstm_4 (LSTM)               (None, 100)               56400     \n",
            "                                                                 \n",
            " dropout_1 (Dropout)         (None, 100)               0         \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 1)                 101       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 456,501\n",
            "Trainable params: 456,501\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "j6-xN1od1mBt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# scale down the values\n",
        "from sklearn.preprocessing import MinMaxScaler \n",
        "\n",
        "# scale up ( reverse )\n",
        "scaler.inverse_transform"
      ],
      "metadata": {
        "id": "pJHKYYVg71Z4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "nJrihtxp71cY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "Blugzths71e8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "X79naINt71hO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "Xy4jfZUP71j9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "U3qsW1on71mp"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}