Language_model_lstm.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Language_model_lstm.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HePjv9BiYTa8",
        "outputId": "d2e03418-0a4e-4750-b1a4-4247fd53b75e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount(\"/content/drive\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import sys\n",
        "sys.path.append('/content/drive/MyDrive/data-sets/Masnavi.csv')"
      ],
      "metadata": {
        "id": "2DwPbW_OYaCz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.layers import Embedding, LSTM, Dense, Dropout\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.callbacks import EarlyStopping\n",
        "from keras.models import Sequential\n",
        "from keras.utils import np_utils\n",
        "import numpy as np\n",
        "import pandas as pd"
      ],
      "metadata": {
        "id": "jPNh1LuBY9lw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "f = open(\"/content/drive/MyDrive/data-sets/Masnavi1.txt\", \"r\")\n",
        "data = f.read()"
      ],
      "metadata": {
        "id": "Yz3MH1e7gmUn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = Tokenizer()\n",
        "def dataset_preparation(data):\n",
        "    corpus = data.lower().split(\"\\n\")    \n",
        "    tokenizer.fit_on_texts(corpus)\n",
        "    total_words = len(tokenizer.word_index) + 1\n",
        "\n",
        "    input_sequences = []\n",
        "    for line in corpus:\n",
        "        token_list = tokenizer.texts_to_sequences([line])[0]\n",
        "        for i in range(1, len(token_list)):\n",
        "            n_gram_sequence = token_list[:i+1]\n",
        "            input_sequences.append(n_gram_sequence)\n",
        "\n",
        "    max_sequence_len = max([len(x) for x in input_sequences])\n",
        "    input_sequences = np.array(pad_sequences(input_sequences, maxlen=max_sequence_len, padding='pre'))\n",
        "\n",
        "    predictors, label = input_sequences[:,:-1],input_sequences[:,-1]\n",
        "    label = np_utils.to_categorical(label, num_classes=total_words)\n",
        "\n",
        "    return predictors, label, max_sequence_len, total_words"
      ],
      "metadata": {
        "id": "sV1-LPRlZIMC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\"\"\"\n",
        "Sentence: \"they are learning data science\"\n",
        "PREDICTORS             | LABEL\n",
        "they                   | are\n",
        "they are               | learning\n",
        "they are learning      | data\n",
        "they are learning data | science\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "1FsY6MrtZOq2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "ecf97add-4e1c-40f2-d9e1-fa654cecd780"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'\\nSentence: \"they are learning data science\"\\nPREDICTORS             | LABEL\\nthey                   | are\\nthey are               | learning\\nthey are learning      | data\\nthey are learning data | science\\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def create_model(predictors, label, max_sequence_len, total_words):\n",
        "    input_len = max_sequence_len - 1\n",
        "    model = Sequential()\n",
        "    model.add(Embedding(total_words, 10, input_length=input_len))\n",
        "    model.add(LSTM(150))\n",
        "    model.add(Dropout(0.1))\n",
        "    model.add(Dense(total_words, activation='softmax'))\n",
        "    model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
        "    model.fit(predictors, label, epochs=100, verbose=1)\n",
        "\n",
        "    return model"
      ],
      "metadata": {
        "id": "J0Zz1f2yZURi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X, Y, max_len, total_words = dataset_preparation(data)\n",
        "model = create_model(X, Y, max_len, total_words)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "il0HPOHiZYTW",
        "outputId": "28a8f3a0-4a7f-43d9-871b-b34447ead698"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/100\n",
            "1553/1553 [==============================] - 13s 8ms/step - loss: 7.4515\n",
            "Epoch 2/100\n",
            "1553/1553 [==============================] - 12s 8ms/step - loss: 7.1319\n",
            "Epoch 3/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 7.0150\n",
            "Epoch 4/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 6.8564\n",
            "Epoch 5/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 6.6832\n",
            "Epoch 6/100\n",
            "1553/1553 [==============================] - 12s 8ms/step - loss: 6.5018\n",
            "Epoch 7/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 6.2967\n",
            "Epoch 8/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 6.0699\n",
            "Epoch 9/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 5.8342\n",
            "Epoch 10/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 5.5899\n",
            "Epoch 11/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 5.3467\n",
            "Epoch 12/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 5.1162\n",
            "Epoch 13/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 4.8969\n",
            "Epoch 14/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 4.6937\n",
            "Epoch 15/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 4.4955\n",
            "Epoch 16/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 4.3210\n",
            "Epoch 17/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 4.1616\n",
            "Epoch 18/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 4.0056\n",
            "Epoch 19/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 3.8667\n",
            "Epoch 20/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 3.7353\n",
            "Epoch 21/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 3.6189\n",
            "Epoch 22/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 3.5050\n",
            "Epoch 23/100\n",
            "1553/1553 [==============================] - 13s 8ms/step - loss: 3.4021\n",
            "Epoch 24/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 3.3075\n",
            "Epoch 25/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 3.2178\n",
            "Epoch 26/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 3.1293\n",
            "Epoch 27/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 3.0563\n",
            "Epoch 28/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.9810\n",
            "Epoch 29/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.9131\n",
            "Epoch 30/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.8491\n",
            "Epoch 31/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.7881\n",
            "Epoch 32/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 2.7315\n",
            "Epoch 33/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 2.6720\n",
            "Epoch 34/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.6200\n",
            "Epoch 35/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.5697\n",
            "Epoch 36/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.5244\n",
            "Epoch 37/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.4720\n",
            "Epoch 38/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 2.4323\n",
            "Epoch 39/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.3935\n",
            "Epoch 40/100\n",
            "1553/1553 [==============================] - 12s 8ms/step - loss: 2.3533\n",
            "Epoch 41/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.3174\n",
            "Epoch 42/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.2761\n",
            "Epoch 43/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 2.2478\n",
            "Epoch 44/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.2086\n",
            "Epoch 45/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.1871\n",
            "Epoch 46/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.1474\n",
            "Epoch 47/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.1265\n",
            "Epoch 48/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.1005\n",
            "Epoch 49/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.0756\n",
            "Epoch 50/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.0398\n",
            "Epoch 51/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 2.0230\n",
            "Epoch 52/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.9992\n",
            "Epoch 53/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.9683\n",
            "Epoch 54/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.9554\n",
            "Epoch 55/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.9317\n",
            "Epoch 56/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.9120\n",
            "Epoch 57/100\n",
            "1553/1553 [==============================] - 12s 8ms/step - loss: 1.8952\n",
            "Epoch 58/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.8767\n",
            "Epoch 59/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 1.8666\n",
            "Epoch 60/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 1.8318\n",
            "Epoch 61/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 1.8203\n",
            "Epoch 62/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.8107\n",
            "Epoch 63/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.7950\n",
            "Epoch 64/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 1.7788\n",
            "Epoch 65/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.7662\n",
            "Epoch 66/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.7557\n",
            "Epoch 67/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.7443\n",
            "Epoch 68/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.7262\n",
            "Epoch 69/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.7034\n",
            "Epoch 70/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.7095\n",
            "Epoch 71/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.6938\n",
            "Epoch 72/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.6739\n",
            "Epoch 73/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.6694\n",
            "Epoch 74/100\n",
            "1553/1553 [==============================] - 12s 8ms/step - loss: 1.6547\n",
            "Epoch 75/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 1.6502\n",
            "Epoch 76/100\n",
            "1553/1553 [==============================] - 12s 7ms/step - loss: 1.6456\n",
            "Epoch 77/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.7624\n",
            "Epoch 78/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.6503\n",
            "Epoch 79/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.6158\n",
            "Epoch 80/100\n",
            "1553/1553 [==============================] - 13s 8ms/step - loss: 1.6069\n",
            "Epoch 81/100\n",
            "1553/1553 [==============================] - 14s 9ms/step - loss: 1.6072\n",
            "Epoch 82/100\n",
            "1553/1553 [==============================] - 14s 9ms/step - loss: 1.5944\n",
            "Epoch 83/100\n",
            "1553/1553 [==============================] - 14s 9ms/step - loss: 1.5988\n",
            "Epoch 84/100\n",
            "1553/1553 [==============================] - 14s 9ms/step - loss: 1.6047\n",
            "Epoch 85/100\n",
            "1553/1553 [==============================] - 14s 9ms/step - loss: 1.5790\n",
            "Epoch 86/100\n",
            "1553/1553 [==============================] - 14s 9ms/step - loss: 1.5601\n",
            "Epoch 87/100\n",
            "1553/1553 [==============================] - 14s 9ms/step - loss: 1.5513\n",
            "Epoch 88/100\n",
            "1553/1553 [==============================] - 14s 9ms/step - loss: 1.5431\n",
            "Epoch 89/100\n",
            "1553/1553 [==============================] - 13s 9ms/step - loss: 1.5382\n",
            "Epoch 90/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.5372\n",
            "Epoch 91/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.5485\n",
            "Epoch 92/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.5061\n",
            "Epoch 93/100\n",
            "1553/1553 [==============================] - 11s 7ms/step - loss: 1.5118\n",
            "Epoch 94/100\n",
            "1553/1553 [==============================] - 13s 8ms/step - loss: 1.5077\n",
            "Epoch 95/100\n",
            "1553/1553 [==============================] - 14s 9ms/step - loss: 1.5148\n",
            "Epoch 96/100\n",
            "1553/1553 [==============================] - 14s 9ms/step - loss: 1.5296\n",
            "Epoch 97/100\n",
            "1553/1553 [==============================] - 14s 9ms/step - loss: 1.5348\n",
            "Epoch 98/100\n",
            "1553/1553 [==============================] - 14s 9ms/step - loss: 1.4968\n",
            "Epoch 99/100\n",
            "1553/1553 [==============================] - 14s 9ms/step - loss: 1.4762\n",
            "Epoch 100/100\n",
            "1553/1553 [==============================] - 14s 9ms/step - loss: 1.4744\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_text(seed_text, next_words, max_sequence_len, model):\n",
        "    for j in range(next_words):\n",
        "        token_list = tokenizer.texts_to_sequences([seed_text])[0]\n",
        "        token_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')\n",
        "        predict_x = model.predict(token_list, verbose=0)\n",
        "        predicted = np.argmax(predict_x,axis=1)\n",
        "        output_word = \"\"\n",
        "        for word, index in tokenizer.word_index.items():\n",
        "            if index == predicted:\n",
        "                output_word = word\n",
        "                break\n",
        "        seed_text += \" \" + output_word\n",
        "    return seed_text"
      ],
      "metadata": {
        "id": "4u6pcnzZZWa2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# !mkdir -p saved_model\n",
        "# model.save('masnavi_language_model.h5') \n",
        "\n",
        "print(model.summary())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5F8xvToElmHp",
        "outputId": "e00a3eb7-6e4c-4465-b78f-a741cbf554e5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_1 (Embedding)     (None, 62, 10)            92910     \n",
            "                                                                 \n",
            " lstm_1 (LSTM)               (None, 150)               96600     \n",
            "                                                                 \n",
            " dropout_1 (Dropout)         (None, 150)               0         \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 9291)              1402941   \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,592,451\n",
            "Trainable params: 1,592,451\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "seed_text = \"گر بريزى بحر\"\n",
        "text = generate_text(seed_text, 10, 50, model)\n",
        "print(text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gDgX4kdlZbns",
        "outputId": "2c4fd557-a438-4ce6-ab53-8be832b694d6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "گر بريزى بحر را در كوزه‏اى چند گنجد قسمت يك روزه‏اى‏ يك خلق\n"
          ]
        }
      ]
    }
  ]
}