LSTM Text Generation using Tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1cc68e96",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import string\n",
    "import requests\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cb5f7499",
   "metadata": {},
   "outputs": [],
   "source": [
    "response = requests.get('https://raw.githubusercontent.com/laxmimerit/poetry-data/master/adele.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "2a5b0134",
   "metadata": {},
   "outputs": [],
   "source": [
    "# response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "99d893cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# response.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "5e7b345c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Looking for some education', 'Made my way into the night', 'All that bullshit conversation', \"Baby, can't you read the signs? I won't bore you with the details, baby\", \"I don't even wanna waste your time\", \"Let's just say that maybe\", 'You could help me ease my mind', \"I ain't Mr. Right But if you're looking for fast love\", \"If that's love in your eyes\", \"It's more than enough\"]\n"
     ]
    }
   ],
   "source": [
    "data=response.text.splitlines()\n",
    "print(data[:10]) # 10 sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "74499609",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2400"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)\n",
    "#there is 2400 lines in this poetry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "a16e4751",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "91330"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Total word number is:\n",
    "len(\" \".join(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f3e7cb42",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, LSTM, Embedding, Dropout, Conv1D, MaxPooling1D\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "911137dd",
   "metadata": {},
   "source": [
    "#### LSTM model and train test preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b1babd07",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer=Tokenizer()\n",
    "tokenizer.fit_on_texts(data) #it's going to fit on the data in the forms of lines."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "2e15ce1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# help(tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "de90b86d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[254, 21, 219, 725],\n",
       " [117, 8, 80, 153, 3, 133],\n",
       " [14, 10, 726, 727],\n",
       " [41, 56, 2, 603, 3, 728, 1, 68, 517, 2, 40, 3, 518, 41],\n",
       " [1, 23, 107, 189, 300, 9, 57],\n",
       " [286, 35, 46, 10, 230],\n",
       " [2, 83, 134, 4, 519, 8, 120],\n",
       " [1, 37, 520, 102, 19, 27, 25, 254, 21, 328, 11],\n",
       " [27, 209, 11, 13, 9, 124],\n",
       " [42, 67, 210, 125]]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_text=tokenizer.texts_to_sequences(data)\n",
    "encoded_text[:10]\n",
    "# These number actually comes from vocabulary\n",
    "# it assigns number to each words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "4e8aaec2",
   "metadata": {},
   "outputs": [],
   "source": [
    "wc=tokenizer.word_counts\n",
    "# word frequency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "1609d8e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "wi=tokenizer.word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "63af34c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique words and total vocab size: 1396\n"
     ]
    }
   ],
   "source": [
    "print(f\"Number of unique words and total vocab size: {len(tokenizer.word_counts)+1}\")\n",
    "vocab_size=len(tokenizer.word_counts)+1 # always adding plus one for tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d9035af5",
   "metadata": {},
   "outputs": [],
   "source": [
    "x=[\"play this song\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a9efe9a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[241, 44, 409]]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.texts_to_sequences(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33262d23",
   "metadata": {},
   "source": [
    "### Prepare data for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "022b5d17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[254, 21, 219, 725],\n",
       " [117, 8, 80, 153, 3, 133],\n",
       " [14, 10, 726, 727],\n",
       " [41, 56, 2, 603, 3, 728, 1, 68, 517, 2, 40, 3, 518, 41],\n",
       " [1, 23, 107, 189, 300, 9, 57],\n",
       " [286, 35, 46, 10, 230],\n",
       " [2, 83, 134, 4, 519, 8, 120],\n",
       " [1, 37, 520, 102, 19, 27, 25, 254, 21, 328, 11],\n",
       " [27, 209, 11, 13, 9, 124],\n",
       " [42, 67, 210, 125]]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " encoded_text[:10] # 10 sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "c32ae96f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_list=[]\n",
    "for i in encoded_text:\n",
    "    if len(i)>1:\n",
    "        for j in range(2,len(i)):\n",
    "            data_list.append(i[:j])\n",
    "#             print(i[:j]) # if you want to check data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f592d01d",
   "metadata": {},
   "source": [
    "#### Paddding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "905f8572",
   "metadata": {},
   "outputs": [],
   "source": [
    "max_length=20\n",
    "#max length of line is 20 token per line in our poetry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "6a7c4079",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0, 254,  21],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0, 254,  21, 219],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0, 117,   8],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0, 117,   8,  80],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0, 117,   8,  80, 153]])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequences=pad_sequences(data_list,maxlen=max_length,padding=\"pre\") # we set the lenght size equal to 20\n",
    "sequences[:5] # 5 sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "e621c3ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14231, 20)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequences.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "3f1a3a8b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X values\n",
      "[[  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "  254]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0 254\n",
      "   21]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "  117]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0 117\n",
      "    8]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0 117   8\n",
      "   80]]\n",
      "------------------------------\n",
      "X values\n",
      "[ 21 219   8  80 153]\n"
     ]
    }
   ],
   "source": [
    "X=sequences[:,:-1]\n",
    "y=sequences[:,-1]\n",
    "print(\"X values\")\n",
    "print(X[:5]) # 5 sample\n",
    "print(\"-\"*30)\n",
    "print(\"X values\")\n",
    "print(y[:5]) # 5 sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e144e0d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((14231, 19), (14231,))"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape,y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "18894fe2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]]\n",
      "Shape of y (14231, 1396)\n"
     ]
    }
   ],
   "source": [
    "y=to_categorical(y,num_classes=vocab_size)\n",
    "#since unique word number is vocab_size, thus there is vocab_size classes\n",
    "print(y[:5]) # 5 sample\n",
    "print(\"Shape of y\",y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "bd466740",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of X (14231, 19)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Shape of X\",X.shape)\n",
    "seq_length=X.shape[1]\n",
    "seq_length"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "66e6e819",
   "metadata": {},
   "source": [
    "#### Build Model\n",
    "- We will build a simple LSTM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "fb22d3a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "model=Sequential()\n",
    "model.add(Embedding(vocab_size,50,input_length=seq_length)) \n",
    "#The first layer is the Embedded layer that uses 50-length vectors\n",
    "#return_sequences=True because we add another LSTM\n",
    "model.add(LSTM(100,return_sequences=True))\n",
    "model.add(LSTM(100))\n",
    "model.add(Dense(100,activation=\"relu\"))\n",
    "model.add(Dense(vocab_size,activation=\"softmax\")) # we use softmax because there is multiclasses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "0478f894",
   "metadata": {},
   "outputs": [],
   "source": [
    "# help(Embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "125714e4",
   "metadata": {},
   "source": [
    "#### You can use dropout() , You will see that Training will be slightly slower trend in convergence, maybe lower final accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "95ef4906",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model=Sequential()\n",
    "# model.add(Embedding(vocab_size,50,input_length=seq_length)) \n",
    "# #The first layer is the Embedded layer that uses 50-length vectors\n",
    "# #return_sequences=True because we add another LSTM\n",
    "# model.add(Dropout(0.2))\n",
    "# model.add(LSTM(100,return_sequences=True))\n",
    "# model.add(Dropout(0.2))\n",
    "# model.add(LSTM(100))\n",
    "# model.add(Dropout(0.2))\n",
    "# model.add(Dense(100,activation=\"relu\"))\n",
    "# model.add(Dense(vocab_size,activation=\"softmax\")) # we use softmax because there is multiclasses"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f5c1fc9",
   "metadata": {},
   "source": [
    "####  Dropout can be applied to the input and recurrent connections of the memory units with the LSTM precisely and separately.\n",
    "#### LSTM-specific dropout may have more pronounced effect on the convergence of the network than the layer-wise dropout."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "3c095832",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model=Sequential()\n",
    "# model.add(Embedding(vocab_size,50,input_length=seq_length)) \n",
    "# #The first layer is the Embedded layer that uses 50-length vectors\n",
    "# #return_sequences=True because we add another LSTM\n",
    "# model.add(LSTM(100,return_sequences=True,dropout=0.2,recurrent_dropout=0.2))\n",
    "# model.add(LSTM(100,dropout=0.2,recurrent_dropout=0.2))\n",
    "# model.add(Dense(100,activation=\"relu\"))\n",
    "# model.add(Dense(vocab_size,activation=\"softmax\")) # we use softmax because there is multiclasses"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6ce2a8b",
   "metadata": {},
   "source": [
    "#### LSTM and CNN can be used together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "fead4a76",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model=Sequential()\n",
    "# model.add(Embedding(vocab_size,50,input_length=seq_length)) \n",
    "# #The first layer is the Embedded layer that uses 50-length vectors\n",
    "# #return_sequences=True because we add another LSTM\n",
    "# model.add(Conv1D(filters=32,kernel_size=3, padding=\"same\",activation=\"relu\"))\n",
    "# model.add(MaxPooling1D(pool_size=2))\n",
    "# model.add(LSTM(100,return_sequences=True))\n",
    "# model.add(LSTM(100))\n",
    "# model.add(Dense(100,activation=\"relu\"))\n",
    "# model.add(Dense(vocab_size,activation=\"softmax\")) # we use softmax because there is multiclasses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "5d6f5963",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding (Embedding)       (None, 19, 50)            69800     \n",
      "                                                                 \n",
      " lstm (LSTM)                 (None, 19, 100)           60400     \n",
      "                                                                 \n",
      " lstm_1 (LSTM)               (None, 100)               80400     \n",
      "                                                                 \n",
      " dense (Dense)               (None, 100)               10100     \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 1396)              140996    \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 361,696\n",
      "Trainable params: 361,696\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "62c42ef3",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"categorical_crossentropy\",optimizer=\"adam\",metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "f06cbeaf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/150\n",
      "445/445 [==============================] - 17s 29ms/step - loss: 5.6915 - accuracy: 0.0400\n",
      "Epoch 2/150\n",
      "445/445 [==============================] - 14s 31ms/step - loss: 5.2304 - accuracy: 0.0522\n",
      "Epoch 3/150\n",
      "445/445 [==============================] - 13s 30ms/step - loss: 5.0358 - accuracy: 0.0628\n",
      "Epoch 4/150\n",
      "445/445 [==============================] - 14s 32ms/step - loss: 4.8863 - accuracy: 0.0734\n",
      "Epoch 5/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 4.7719 - accuracy: 0.0828\n",
      "Epoch 6/150\n",
      "445/445 [==============================] - 12s 27ms/step - loss: 4.6617 - accuracy: 0.0948\n",
      "Epoch 7/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 4.5568 - accuracy: 0.0993\n",
      "Epoch 8/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 4.4429 - accuracy: 0.1119\n",
      "Epoch 9/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 4.3251 - accuracy: 0.1254\n",
      "Epoch 10/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 4.2096 - accuracy: 0.1332\n",
      "Epoch 11/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 4.0892 - accuracy: 0.1477\n",
      "Epoch 12/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 3.9683 - accuracy: 0.1667\n",
      "Epoch 13/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 3.8440 - accuracy: 0.1867\n",
      "Epoch 14/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 3.7150 - accuracy: 0.2048\n",
      "Epoch 15/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 3.5987 - accuracy: 0.2213\n",
      "Epoch 16/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 3.4838 - accuracy: 0.2400\n",
      "Epoch 17/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 3.3711 - accuracy: 0.2532\n",
      "Epoch 18/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 3.2615 - accuracy: 0.2738\n",
      "Epoch 19/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 3.1571 - accuracy: 0.2932\n",
      "Epoch 20/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 3.0598 - accuracy: 0.3093\n",
      "Epoch 21/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 2.9672 - accuracy: 0.3246\n",
      "Epoch 22/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 2.8794 - accuracy: 0.3400\n",
      "Epoch 23/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 2.8083 - accuracy: 0.3532\n",
      "Epoch 24/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 2.7300 - accuracy: 0.3695\n",
      "Epoch 25/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 2.6536 - accuracy: 0.3864\n",
      "Epoch 26/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 2.5893 - accuracy: 0.3944\n",
      "Epoch 27/150\n",
      "445/445 [==============================] - 13s 30ms/step - loss: 2.5248 - accuracy: 0.4066\n",
      "Epoch 28/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 2.4621 - accuracy: 0.4179\n",
      "Epoch 29/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 2.4009 - accuracy: 0.4326\n",
      "Epoch 30/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 2.3451 - accuracy: 0.4466\n",
      "Epoch 31/150\n",
      "445/445 [==============================] - 13s 30ms/step - loss: 2.2948 - accuracy: 0.4552\n",
      "Epoch 32/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 2.2427 - accuracy: 0.4655\n",
      "Epoch 33/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 2.1874 - accuracy: 0.4775\n",
      "Epoch 34/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 2.1461 - accuracy: 0.4829\n",
      "Epoch 35/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 2.1003 - accuracy: 0.4960\n",
      "Epoch 36/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 2.0546 - accuracy: 0.5041\n",
      "Epoch 37/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 2.0164 - accuracy: 0.5124\n",
      "Epoch 38/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.9705 - accuracy: 0.5254\n",
      "Epoch 39/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.9327 - accuracy: 0.5312\n",
      "Epoch 40/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.8950 - accuracy: 0.5409\n",
      "Epoch 41/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.8564 - accuracy: 0.5486\n",
      "Epoch 42/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.8222 - accuracy: 0.5547\n",
      "Epoch 43/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.7804 - accuracy: 0.5673\n",
      "Epoch 44/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.7529 - accuracy: 0.5737\n",
      "Epoch 45/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.7161 - accuracy: 0.5773\n",
      "Epoch 46/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.6911 - accuracy: 0.5839\n",
      "Epoch 47/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.6525 - accuracy: 0.5956\n",
      "Epoch 48/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.6227 - accuracy: 0.5963\n",
      "Epoch 49/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.5862 - accuracy: 0.6075\n",
      "Epoch 50/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.5619 - accuracy: 0.6144\n",
      "Epoch 51/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.5369 - accuracy: 0.6186\n",
      "Epoch 52/150\n",
      "445/445 [==============================] - 13s 30ms/step - loss: 1.5131 - accuracy: 0.6254\n",
      "Epoch 53/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.4810 - accuracy: 0.6334\n",
      "Epoch 54/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.4659 - accuracy: 0.6361\n",
      "Epoch 55/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.4332 - accuracy: 0.6431\n",
      "Epoch 56/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.4133 - accuracy: 0.6475\n",
      "Epoch 57/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.3864 - accuracy: 0.6534\n",
      "Epoch 58/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.3573 - accuracy: 0.6610\n",
      "Epoch 59/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.3433 - accuracy: 0.6615\n",
      "Epoch 60/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.3182 - accuracy: 0.6671\n",
      "Epoch 61/150\n",
      "445/445 [==============================] - 13s 30ms/step - loss: 1.2969 - accuracy: 0.6704\n",
      "Epoch 62/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.2864 - accuracy: 0.6756\n",
      "Epoch 63/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.2580 - accuracy: 0.6833\n",
      "Epoch 64/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.2325 - accuracy: 0.6888\n",
      "Epoch 65/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.2138 - accuracy: 0.6917\n",
      "Epoch 66/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.1955 - accuracy: 0.6940\n",
      "Epoch 67/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.1788 - accuracy: 0.7019\n",
      "Epoch 68/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.1567 - accuracy: 0.7040\n",
      "Epoch 69/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.1382 - accuracy: 0.7109\n",
      "Epoch 70/150\n",
      "445/445 [==============================] - 13s 30ms/step - loss: 1.1224 - accuracy: 0.7146\n",
      "Epoch 71/150\n",
      "445/445 [==============================] - 13s 30ms/step - loss: 1.1066 - accuracy: 0.7137\n",
      "Epoch 72/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.0913 - accuracy: 0.7177\n",
      "Epoch 73/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.0729 - accuracy: 0.7231\n",
      "Epoch 74/150\n",
      "445/445 [==============================] - 13s 30ms/step - loss: 1.0571 - accuracy: 0.7284\n",
      "Epoch 75/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.0460 - accuracy: 0.7325\n",
      "Epoch 76/150\n",
      "445/445 [==============================] - 13s 30ms/step - loss: 1.0285 - accuracy: 0.7340\n",
      "Epoch 77/150\n",
      "445/445 [==============================] - 13s 30ms/step - loss: 1.0128 - accuracy: 0.7381\n",
      "Epoch 78/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 1.0013 - accuracy: 0.7385\n",
      "Epoch 79/150\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "445/445 [==============================] - 12s 28ms/step - loss: 0.9798 - accuracy: 0.7465\n",
      "Epoch 80/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.9668 - accuracy: 0.7496\n",
      "Epoch 81/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.9541 - accuracy: 0.7516\n",
      "Epoch 82/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.9408 - accuracy: 0.7531\n",
      "Epoch 83/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.9292 - accuracy: 0.7557\n",
      "Epoch 84/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.9143 - accuracy: 0.7576\n",
      "Epoch 85/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.9059 - accuracy: 0.7600\n",
      "Epoch 86/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.8895 - accuracy: 0.7635\n",
      "Epoch 87/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.8974 - accuracy: 0.7623\n",
      "Epoch 88/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.8747 - accuracy: 0.7670\n",
      "Epoch 89/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.8633 - accuracy: 0.7668\n",
      "Epoch 90/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.8483 - accuracy: 0.7726\n",
      "Epoch 91/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.8295 - accuracy: 0.7776\n",
      "Epoch 92/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.8270 - accuracy: 0.7782\n",
      "Epoch 93/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.8184 - accuracy: 0.7779\n",
      "Epoch 94/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.8002 - accuracy: 0.7830\n",
      "Epoch 95/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.8071 - accuracy: 0.7824\n",
      "Epoch 96/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.7829 - accuracy: 0.7836\n",
      "Epoch 97/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.7819 - accuracy: 0.7865\n",
      "Epoch 98/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.7709 - accuracy: 0.7896\n",
      "Epoch 99/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.7721 - accuracy: 0.7893\n",
      "Epoch 100/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.7522 - accuracy: 0.7958\n",
      "Epoch 101/150\n",
      "445/445 [==============================] - 12s 28ms/step - loss: 0.7421 - accuracy: 0.7992\n",
      "Epoch 102/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.7381 - accuracy: 0.7973\n",
      "Epoch 103/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.7345 - accuracy: 0.7964\n",
      "Epoch 104/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.7234 - accuracy: 0.8016\n",
      "Epoch 105/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.7202 - accuracy: 0.8011\n",
      "Epoch 106/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.7085 - accuracy: 0.8042\n",
      "Epoch 107/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.7002 - accuracy: 0.8036\n",
      "Epoch 108/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.6968 - accuracy: 0.8078\n",
      "Epoch 109/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.6896 - accuracy: 0.8079\n",
      "Epoch 110/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.6994 - accuracy: 0.8056\n",
      "Epoch 111/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.6765 - accuracy: 0.8136\n",
      "Epoch 112/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.6839 - accuracy: 0.8084\n",
      "Epoch 113/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.6659 - accuracy: 0.8141\n",
      "Epoch 114/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.6620 - accuracy: 0.8149\n",
      "Epoch 115/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.6545 - accuracy: 0.8155\n",
      "Epoch 116/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.6549 - accuracy: 0.8157\n",
      "Epoch 117/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.6553 - accuracy: 0.8145\n",
      "Epoch 118/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.6448 - accuracy: 0.8175\n",
      "Epoch 119/150\n",
      "445/445 [==============================] - 13s 30ms/step - loss: 0.6414 - accuracy: 0.8183\n",
      "Epoch 120/150\n",
      "445/445 [==============================] - 14s 31ms/step - loss: 0.6357 - accuracy: 0.8203\n",
      "Epoch 121/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.6357 - accuracy: 0.8192\n",
      "Epoch 122/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.6271 - accuracy: 0.8224\n",
      "Epoch 123/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.6288 - accuracy: 0.8229\n",
      "Epoch 124/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.6277 - accuracy: 0.8193\n",
      "Epoch 125/150\n",
      "445/445 [==============================] - 13s 28ms/step - loss: 0.6125 - accuracy: 0.8245\n",
      "Epoch 126/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.6108 - accuracy: 0.8259\n",
      "Epoch 127/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.6183 - accuracy: 0.8245\n",
      "Epoch 128/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.6029 - accuracy: 0.8270\n",
      "Epoch 129/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.6020 - accuracy: 0.8271\n",
      "Epoch 130/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.5991 - accuracy: 0.8292\n",
      "Epoch 131/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.5995 - accuracy: 0.8262\n",
      "Epoch 132/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.6082 - accuracy: 0.8240\n",
      "Epoch 133/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.5972 - accuracy: 0.8273\n",
      "Epoch 134/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.5913 - accuracy: 0.8299\n",
      "Epoch 135/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.5879 - accuracy: 0.8290\n",
      "Epoch 136/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.5978 - accuracy: 0.8261\n",
      "Epoch 137/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.5783 - accuracy: 0.8310\n",
      "Epoch 138/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.5834 - accuracy: 0.8283\n",
      "Epoch 139/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.5724 - accuracy: 0.8349\n",
      "Epoch 140/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.5726 - accuracy: 0.8327\n",
      "Epoch 141/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.5800 - accuracy: 0.8293\n",
      "Epoch 142/150\n",
      "445/445 [==============================] - 13s 30ms/step - loss: 0.5701 - accuracy: 0.8336\n",
      "Epoch 143/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.5778 - accuracy: 0.8297\n",
      "Epoch 144/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.5712 - accuracy: 0.8303\n",
      "Epoch 145/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.5673 - accuracy: 0.8319\n",
      "Epoch 146/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.5689 - accuracy: 0.8312\n",
      "Epoch 147/150\n",
      "445/445 [==============================] - 14s 32ms/step - loss: 0.5642 - accuracy: 0.8336\n",
      "Epoch 148/150\n",
      "445/445 [==============================] - 13s 29ms/step - loss: 0.5626 - accuracy: 0.8341\n",
      "Epoch 149/150\n",
      "445/445 [==============================] - 13s 30ms/step - loss: 0.5574 - accuracy: 0.8351\n",
      "Epoch 150/150\n",
      "445/445 [==============================] - 13s 30ms/step - loss: 0.5559 - accuracy: 0.8357\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x21c5cc04100>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X,y,batch_size=32,epochs=150)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "66e41f61",
   "metadata": {},
   "source": [
    "### Text Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "56f2d4f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_lenght= 15 # 15 words per line\n",
    "\n",
    "def generate_text(input_text, no_lines):\n",
    "    general_text=[]\n",
    "    for i in range(no_lines):\n",
    "        text=[]\n",
    "        for _ in range(text_lenght):\n",
    "            encoded=tokenizer.texts_to_sequences([input_text])\n",
    "            encoded=pad_sequences(encoded,maxlen=seq_length,padding=\"pre\")\n",
    "            y_pred=np.argmax(model.predict(encoded),axis=-1) # it will generate a word index, loop up into dictionary containing word index\n",
    "            \n",
    "            predicted_word=\"\"\n",
    "            for word,index in tokenizer.word_index.items():\n",
    "                if index==y_pred:\n",
    "                    predicted_word=word\n",
    "                    break\n",
    "                    \n",
    "            input_text=input_text +' '+ predicted_word\n",
    "            text.append(predicted_word)\n",
    "        \n",
    "        input_text=text[-1]\n",
    "        text=\" \".join(text) # input text will be the last word of first created line\n",
    "        general_text.append(text)\n",
    "    \n",
    "    return general_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "5c9bada2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 1s 689ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 22ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 23ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 17ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 22ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 17ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 22ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 22ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 22ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 22ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 22ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[\"by the seams no bribing me to doubt and i'll do do i never knew\",\n",
       " 'believe is a memory i can tell by the way you walk that the air',\n",
       " \"she gave you things i didn't give to you old friend why are you so\",\n",
       " 'i spend my whole life hiding my heart away away woke up feeling heavy the',\n",
       " \"last time let it lose hold me in won't out can you prefer the morning\",\n",
       " \"t show emotions don t know just a place in you and you're in my\"]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_text=\"me\"\n",
    "text_produced=generate_text(input_text,6)\n",
    "text_produced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "28b0d05e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 25ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 22ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 22ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 22ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 23ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 22ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 23ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['were away its hole you go at the world town steady as she goes so',\n",
       " 'i spend my whole life hiding my heart away away woke up feeling heavy the',\n",
       " \"last time let it lose hold me in won't out can you prefer the morning\",\n",
       " \"t show emotions don t know just a place in you and you're in my\",\n",
       " 'god this reminds so give me till that i went on home to my been',\n",
       " \"that scares me the more eager i will always still but i know that's where\"]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_text=\"i want to see you\"\n",
    "text_produced=generate_text(input_text,6)\n",
    "text_produced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cf2ba28",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_text=\"i want to see you\"\n",
    "text_produced=generate_text(input_text,6)\n",
    "text_produced"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}