RNNTextGeneration (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7824e112",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fefd263a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Text Size : 99984 | Vocab Size : 62\n"
     ]
    }
   ],
   "source": [
    "text=open('shakespeare.txt','r').read()\n",
    "chars=list(set(text))\n",
    "text_size, vocab_size = len(text), len(chars)\n",
    "print(f\"\\n Text Size : {text_size} | Vocab Size : {vocab_size}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b6d4f988",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['p', 'o', 'R', 'M', 'r', 'g', ',', 'u', 'b', 'P', 'X', 'B', 't', 'j', 'J', 'W', ';', 'v', 'q', 'G', '.', 'Z', 'F', 'C', 'f', 'w', 'L', 'N', 'm', 'Q', \"'\", 'x', 'T', 'h', 'y', ' ', 'U', 'z', 'H', 's', 'd', 'i', 'O', 'S', 'Y', 'a', 'k', 'E', 'K', 'V', '?', ':', 'e', 'D', '\\n', 'I', '!', 'l', '-', 'c', 'A', 'n']\n"
     ]
    }
   ],
   "source": [
    "print(chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e343e32d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Tokenize\n",
    "char_to_int = {ch:i for i,ch in enumerate(chars)}\n",
    "int_to_char = {i:ch for i,ch in enumerate(chars)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "3de43632",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Hyperparameters\n",
    "hidden_size = 72\n",
    "seq_length = 24\n",
    "learning_rate = 0.05\n",
    "beta1 = 0.999\n",
    "beta2 = 0.9999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "539c9f61",
   "metadata": {},
   "outputs": [],
   "source": [
    "Wi2h = np.random.randn(hidden_size, vocab_size)*0.01 #input to hidden embedding\n",
    "Wh2h = np.random.randn(hidden_size, hidden_size)*0.01 #hidden to hidden\n",
    "Wh2o = np.random.randn(vocab_size, hidden_size)*0.01 #hidden to output\n",
    "hb = np.zeros((hidden_size,1)) #hidden bias\n",
    "ob = np.zeros((vocab_size,1)) #output bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a4a102c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((72, 62), (72, 72), (62, 72), (72, 1), (62, 1))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Wi2h.shape, Wh2h.shape, Wh2o.shape, hb.shape, ob.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "61771f3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent_single(inputs, targets, hactive):\n",
    "    xenc,henc,yprobs,probs={},{},{},{}\n",
    "    henc[-1]=np.copy(hactive)\n",
    "    loss=0\n",
    "    #forward pass a single example\n",
    "    for i in range(len(inputs)):\n",
    "        xenc[i] = np.zeros((vocab_size,1))\n",
    "        xenc[i][inputs[i]] = 1 #one hot encoding of input sequence\n",
    "        henc[i] = np.tanh((Wi2h @ xenc[i]) + (Wh2h @ henc[i-1]) + hb) #hidden state update\n",
    "        yprobs[i] = (Wh2o @ henc[i]) + ob\n",
    "        #yprobs[i] -= np.max(yprobs[i])#unnormalized logprobs for next character\n",
    "        probs[i] = np.exp(yprobs[i]) / np.sum(np.exp(yprobs[i])) #softmax normalization\n",
    "        loss += -np.log(probs[i][targets[i],0]) #-log liklihood (cross entropy)\n",
    "    \n",
    "    #backward pass for current example\n",
    "    dWi2h,dWh2h,dWh2o,dhb,dob = np.zeros_like(Wi2h),np.zeros_like(Wh2h),np.zeros_like(Wh2o),np.zeros_like(hb),np.zeros_like(ob)\n",
    "    dhactive = np.zeros_like(henc[0])\n",
    "    for i in reversed(range(len(inputs))):\n",
    "        dyps = np.copy(probs[i]) #backward into logits\n",
    "        dyps[targets[i]] -= 1\n",
    "        dWh2o += dyps @ henc[i].T\n",
    "        dob += dyps\n",
    "        dhb += (1-henc[i]**2)*(Wh2o.T @ dyps + dhactive)\n",
    "        dWi2h += ((1-henc[i]**2)*(Wh2o.T @ dyps + dhactive)) @ xenc[i].T\n",
    "        dWh2h += ((1-henc[i]**2)*(Wh2o.T @ dyps + dhactive)) @ henc[i-1].T\n",
    "        dhactive = Wh2h.T @ ((1-henc[i]**2)*(Wh2o.T @ dyps + dhactive))\n",
    "    for grads in [dWi2h,dWh2h,dWh2o,dhb,dob]:\n",
    "        np.clip(grads, -5,5, out=grads) #clip gradients from exploding or vanishing\n",
    "    return loss,dWi2h,dWh2h,dWh2o,dhb,dob,henc[len(inputs)-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1a854bfb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample(hidden, input_seed, n):\n",
    "    ip = np.zeros((vocab_size,1))\n",
    "    ip[input_seed] = 1\n",
    "    ops = []\n",
    "    for i in range(n):\n",
    "        hidden = np.tanh(Wi2h @ ip + Wh2h @ hidden + hb) #forward pass\n",
    "        y = ((Wh2o @ hidden) + ob) #outputs\n",
    "        prob = np.exp(y)/np.sum(np.exp(y)) #softmax logits\n",
    "        ix = np.random.choice(range(vocab_size),p=prob.ravel()) #choose random from range\n",
    "        ip = np.zeros((vocab_size,1))\n",
    "        ip[ix] = 1\n",
    "        ops.append(ix)\n",
    "    return ops  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "eb9c5fbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "n, offset = 0, 0\n",
    "#Memory variables for Adam optimizer\n",
    "#Momentum variables\n",
    "vWi2h, vWh2h, vWh2o = np.zeros_like(Wi2h), np.zeros_like(Wh2h), np.zeros_like(Wh2o)\n",
    "vhb, vob = np.zeros_like(hb), np.zeros_like(ob)\n",
    "#RMSProp variables\n",
    "sWi2h, sWh2h, sWh2o = np.zeros_like(Wi2h), np.zeros_like(Wh2h), np.zeros_like(Wh2o)\n",
    "shb, sob = np.zeros_like(hb), np.zeros_like(ob)\n",
    "\n",
    "smooth_loss = -np.log(1.0/vocab_size)*seq_length #smoothen loss by a constant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "6a93a8a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration : 454600 --> loss : 40.475732330547125\n",
      "iteration : 454700 --> loss : 40.91389950079509\n",
      "iteration : 454800 --> loss : 41.31420043939537\n",
      "iteration : 454900 --> loss : 41.77212723928842\n",
      "iteration : 455000 --> loss : 42.098306323336246\n",
      "iteration : 455100 --> loss : 42.52059847508115\n",
      "iteration : 455200 --> loss : 42.53418620196879\n",
      "iteration : 455300 --> loss : 42.652596270595616\n",
      "iteration : 455400 --> loss : 42.767096275375565\n",
      "iteration : 455500 --> loss : 42.88864495609789\n",
      "iteration : 455600 --> loss : 42.957855576733515\n",
      "iteration : 455700 --> loss : 42.89880276994455\n",
      "iteration : 455800 --> loss : 42.990953849893366\n",
      "iteration : 455900 --> loss : 43.07389689893254\n",
      "iteration : 456000 --> loss : 43.28638225732019\n",
      "iteration : 456100 --> loss : 43.202453307473746\n",
      "iteration : 456200 --> loss : 43.1651045637224\n",
      "iteration : 456300 --> loss : 43.12445002667969\n",
      "iteration : 456400 --> loss : 43.18978693431721\n",
      "iteration : 456500 --> loss : 43.044030123553114\n",
      "iteration : 456600 --> loss : 42.99885549737695\n",
      "iteration : 456700 --> loss : 43.03090368229707\n",
      "iteration : 456800 --> loss : 42.844117083572286\n",
      "iteration : 456900 --> loss : 42.68513782661458\n",
      "iteration : 457000 --> loss : 42.51742799343653\n",
      "iteration : 457100 --> loss : 42.4773813711249\n",
      "iteration : 457200 --> loss : 42.34477886491127\n",
      "iteration : 457300 --> loss : 42.25891403636665\n",
      "iteration : 457400 --> loss : 42.037271912438804\n",
      "iteration : 457500 --> loss : 41.875738875300414\n",
      "iteration : 457600 --> loss : 41.77051180769412\n",
      "iteration : 457700 --> loss : 41.593380306553705\n",
      "iteration : 457800 --> loss : 41.707781332835026\n",
      "iteration : 457900 --> loss : 41.76309268748809\n",
      "iteration : 458000 --> loss : 41.75887715823075\n",
      "iteration : 458100 --> loss : 41.56722353037116\n",
      "iteration : 458200 --> loss : 41.465272377659815\n",
      "iteration : 458300 --> loss : 41.33599301026544\n",
      "iteration : 458400 --> loss : 41.126284069107626\n",
      "iteration : 458500 --> loss : 41.03967870093729\n",
      "iteration : 458600 --> loss : 40.92692437943681\n",
      "iteration : 458700 --> loss : 40.54391486165899\n",
      "iteration : 458800 --> loss : 40.51718488324839\n",
      "iteration : 458900 --> loss : 40.4811905586096\n",
      "iteration : 459000 --> loss : 40.41259319498723\n",
      "iteration : 459100 --> loss : 40.313735512389414\n",
      "iteration : 459200 --> loss : 40.310809652679666\n",
      "iteration : 459300 --> loss : 40.275806659446005\n",
      "iteration : 459400 --> loss : 40.13426639017459\n",
      "iteration : 459500 --> loss : 40.02770526215976\n",
      "iteration : 459600 --> loss : 39.99665206987316\n",
      "iteration : 459700 --> loss : 39.86859500897685\n",
      "iteration : 459800 --> loss : 39.824744549100075\n",
      "iteration : 459900 --> loss : 39.78400492644593\n",
      "iteration : 460000 --> loss : 39.80785660553898\n",
      "iteration : 460100 --> loss : 39.86197697167735\n",
      "iteration : 460200 --> loss : 39.880413651909194\n",
      "iteration : 460300 --> loss : 39.93416309357295\n",
      "iteration : 460400 --> loss : 39.83243106582729\n",
      "iteration : 460500 --> loss : 39.88400420756485\n",
      "iteration : 460600 --> loss : 39.95867858242961\n",
      "iteration : 460700 --> loss : 40.04861483273652\n",
      "iteration : 460800 --> loss : 40.13692919109441\n",
      "iteration : 460900 --> loss : 40.1421529702521\n",
      "iteration : 461000 --> loss : 40.07998307575407\n",
      "iteration : 461100 --> loss : 39.922603382755284\n",
      "iteration : 461200 --> loss : 39.94919969390554\n",
      "iteration : 461300 --> loss : 39.90677099981675\n",
      "iteration : 461400 --> loss : 40.024675363596906\n",
      "iteration : 461500 --> loss : 39.90658540557299\n",
      "iteration : 461600 --> loss : 39.802285478141215\n",
      "iteration : 461700 --> loss : 39.65208490595467\n",
      "iteration : 461800 --> loss : 39.70322987010066\n",
      "iteration : 461900 --> loss : 39.74720366217154\n",
      "iteration : 462000 --> loss : 39.930472503236054\n",
      "iteration : 462100 --> loss : 40.01278349202754\n",
      "iteration : 462200 --> loss : 39.99042514107075\n",
      "iteration : 462300 --> loss : 39.92091387528091\n",
      "iteration : 462400 --> loss : 39.84554953649749\n",
      "iteration : 462500 --> loss : 39.80272707686774\n",
      "iteration : 462600 --> loss : 39.6386979806637\n",
      "iteration : 462700 --> loss : 39.67897699496407\n",
      "iteration : 462800 --> loss : 39.69848579360233\n",
      "iteration : 462900 --> loss : 39.458437455492195\n",
      "iteration : 463000 --> loss : 39.512501757077246\n",
      "iteration : 463100 --> loss : 39.48100114902724\n",
      "iteration : 463200 --> loss : 39.51037761016338\n",
      "iteration : 463300 --> loss : 39.466853254967596\n",
      "iteration : 463400 --> loss : 39.580141068464954\n",
      "iteration : 463500 --> loss : 39.51188593910483\n",
      "iteration : 463600 --> loss : 39.38213570644196\n",
      "iteration : 463700 --> loss : 39.312354512860644\n",
      "iteration : 463800 --> loss : 39.36054901687104\n",
      "iteration : 463900 --> loss : 39.34988332986652\n",
      "iteration : 464000 --> loss : 39.3171171843878\n",
      "iteration : 464100 --> loss : 39.24638618565167\n",
      "iteration : 464200 --> loss : 39.359266514565604\n",
      "iteration : 464300 --> loss : 39.507850539090526\n",
      "iteration : 464400 --> loss : 39.5066274742563\n",
      "iteration : 464500 --> loss : 39.586300183631884\n",
      "iteration : 464600 --> loss : 39.59082639242159\n",
      "iteration : 464700 --> loss : 39.697354112552894\n",
      "iteration : 464800 --> loss : 39.70314127700431\n",
      "iteration : 464900 --> loss : 39.76073448429553\n",
      "iteration : 465000 --> loss : 39.90440396218041\n",
      "iteration : 465100 --> loss : 39.91068487779936\n",
      "iteration : 465200 --> loss : 39.830279964579375\n",
      "iteration : 465300 --> loss : 39.699033769560316\n",
      "iteration : 465400 --> loss : 39.81565778142367\n",
      "iteration : 465500 --> loss : 39.799069488878594\n",
      "iteration : 465600 --> loss : 39.819217338395816\n",
      "iteration : 465700 --> loss : 39.733740711048874\n",
      "iteration : 465800 --> loss : 39.65231305208522\n",
      "iteration : 465900 --> loss : 39.60878408678703\n",
      "iteration : 466000 --> loss : 39.59538277763843\n",
      "iteration : 466100 --> loss : 39.62486854181199\n",
      "iteration : 466200 --> loss : 39.77884607909052\n",
      "iteration : 466300 --> loss : 39.901284652015974\n",
      "iteration : 466400 --> loss : 39.89990568014471\n",
      "iteration : 466500 --> loss : 39.80321386328762\n",
      "iteration : 466600 --> loss : 39.70500617952649\n",
      "iteration : 466700 --> loss : 39.6780176681731\n",
      "iteration : 466800 --> loss : 39.54687796202654\n",
      "iteration : 466900 --> loss : 39.56984660261758\n",
      "iteration : 467000 --> loss : 39.53208484724942\n",
      "iteration : 467100 --> loss : 39.47145523350791\n",
      "iteration : 467200 --> loss : 39.49002534898005\n",
      "iteration : 467300 --> loss : 39.41333528087571\n",
      "iteration : 467400 --> loss : 39.47932829185296\n",
      "iteration : 467500 --> loss : 39.43482621148026\n",
      "iteration : 467600 --> loss : 39.529455325548604\n",
      "iteration : 467700 --> loss : 39.356080590073354\n",
      "iteration : 467800 --> loss : 39.29004067709268\n",
      "iteration : 467900 --> loss : 39.29163378554187\n",
      "iteration : 468000 --> loss : 39.29523789722305\n",
      "iteration : 468100 --> loss : 39.26176524207436\n",
      "iteration : 468200 --> loss : 39.20752258824308\n",
      "iteration : 468300 --> loss : 39.26870680227376\n",
      "iteration : 468400 --> loss : 39.29133766452961\n",
      "iteration : 468500 --> loss : 39.51897652973106\n",
      "iteration : 468600 --> loss : 39.47394215304654\n",
      "iteration : 468700 --> loss : 39.48272882879233\n",
      "iteration : 468800 --> loss : 39.499856941943776\n",
      "iteration : 468900 --> loss : 39.64548212944641\n",
      "iteration : 469000 --> loss : 39.65528496935151\n",
      "iteration : 469100 --> loss : 39.747907105843694\n",
      "iteration : 469200 --> loss : 39.91838592221827\n",
      "iteration : 469300 --> loss : 39.91889641974887\n",
      "iteration : 469400 --> loss : 39.845943403376324\n",
      "iteration : 469500 --> loss : 39.81921164539176\n",
      "iteration : 469600 --> loss : 39.886265567537876\n",
      "iteration : 469700 --> loss : 39.90840182509881\n",
      "iteration : 469800 --> loss : 39.94440455184613\n",
      "iteration : 469900 --> loss : 39.81159729435558\n",
      "iteration : 470000 --> loss : 39.743918449593124\n",
      "iteration : 470100 --> loss : 39.745727404457334\n",
      "iteration : 470200 --> loss : 39.670008776881474\n",
      "iteration : 470300 --> loss : 39.83187383982954\n",
      "iteration : 470400 --> loss : 40.01234180131701\n",
      "iteration : 470500 --> loss : 40.067064394530185\n",
      "iteration : 470600 --> loss : 39.95339565954589\n",
      "iteration : 470700 --> loss : 39.96674583516255\n",
      "iteration : 470800 --> loss : 39.92002075324368\n",
      "iteration : 470900 --> loss : 39.8087238724551\n",
      "iteration : 471000 --> loss : 39.81399304381508\n",
      "iteration : 471100 --> loss : 39.810732163421505\n",
      "iteration : 471200 --> loss : 39.69070430966454\n",
      "iteration : 471300 --> loss : 39.757130875692894\n",
      "iteration : 471400 --> loss : 39.751847687536106\n",
      "iteration : 471500 --> loss : 39.722490199458456\n",
      "iteration : 471600 --> loss : 39.63484879466359\n",
      "iteration : 471700 --> loss : 39.71687506107804\n",
      "iteration : 471800 --> loss : 39.73267576995542\n",
      "iteration : 471900 --> loss : 39.54476534336544\n",
      "iteration : 472000 --> loss : 39.48718370075525\n",
      "iteration : 472100 --> loss : 39.551326212027384\n",
      "iteration : 472200 --> loss : 39.40328457630332\n",
      "iteration : 472300 --> loss : 39.45374897225778\n",
      "iteration : 472400 --> loss : 39.41842556802125\n",
      "iteration : 472500 --> loss : 39.51942419821118\n",
      "iteration : 472600 --> loss : 39.606055326809795\n",
      "iteration : 472700 --> loss : 39.65329291103253\n",
      "iteration : 472800 --> loss : 39.70381513168246\n",
      "iteration : 472900 --> loss : 39.644477600621904\n",
      "iteration : 473000 --> loss : 39.72971566947991\n",
      "iteration : 473100 --> loss : 39.80356404780813\n",
      "iteration : 473200 --> loss : 39.94291625481526\n",
      "iteration : 473300 --> loss : 40.01353133920527\n",
      "iteration : 473400 --> loss : 40.08693285092918\n",
      "iteration : 473500 --> loss : 40.070914583748916\n",
      "iteration : 473600 --> loss : 39.91811253260092\n",
      "iteration : 473700 --> loss : 39.96524427967243\n",
      "iteration : 473800 --> loss : 40.0052615310408\n",
      "iteration : 473900 --> loss : 40.17462570621799\n",
      "iteration : 474000 --> loss : 40.07329794295839\n",
      "iteration : 474100 --> loss : 39.93945613925339\n",
      "iteration : 474200 --> loss : 39.82103984842891\n",
      "iteration : 474300 --> loss : 39.881690539099004\n",
      "iteration : 474400 --> loss : 39.87275735158147\n",
      "iteration : 474500 --> loss : 40.05567419483603\n",
      "iteration : 474600 --> loss : 40.12172728178469\n",
      "iteration : 474700 --> loss : 40.133826545909045\n",
      "iteration : 474800 --> loss : 40.08656826496005\n",
      "iteration : 474900 --> loss : 40.05129844167776\n",
      "iteration : 475000 --> loss : 40.050304662153216\n",
      "iteration : 475100 --> loss : 39.92507640694232\n",
      "iteration : 475200 --> loss : 39.96362314950346\n",
      "iteration : 475300 --> loss : 39.98297311809797\n",
      "iteration : 475400 --> loss : 39.848317103844906\n",
      "iteration : 475500 --> loss : 39.90185191896823\n",
      "iteration : 475600 --> loss : 39.84177829653251\n",
      "iteration : 475700 --> loss : 39.86114252620886\n",
      "iteration : 475800 --> loss : 39.82377077604442\n",
      "iteration : 475900 --> loss : 39.96710511742071\n",
      "iteration : 476000 --> loss : 39.83787402893755\n",
      "iteration : 476100 --> loss : 39.75507305418116\n",
      "iteration : 476200 --> loss : 39.7116865189209\n",
      "iteration : 476300 --> loss : 39.73114076359347\n",
      "iteration : 476400 --> loss : 39.733659658439514\n",
      "iteration : 476500 --> loss : 39.713632165222\n",
      "iteration : 476600 --> loss : 39.65073756312047\n",
      "iteration : 476700 --> loss : 39.732435839907914\n",
      "iteration : 476800 --> loss : 39.93555269159735\n",
      "iteration : 476900 --> loss : 39.89506617015705\n",
      "iteration : 477000 --> loss : 39.975191349585934\n",
      "iteration : 477100 --> loss : 39.99534700737846\n",
      "iteration : 477200 --> loss : 40.10097452968821\n",
      "iteration : 477300 --> loss : 40.08837537636846\n",
      "iteration : 477400 --> loss : 40.157403337028796\n",
      "iteration : 477500 --> loss : 40.30636802763207\n",
      "iteration : 477600 --> loss : 40.380607129954335\n",
      "iteration : 477700 --> loss : 40.257487209179935\n",
      "iteration : 477800 --> loss : 40.14478736093685\n",
      "iteration : 477900 --> loss : 40.28714861984881\n",
      "iteration : 478000 --> loss : 40.23579311380092\n",
      "iteration : 478100 --> loss : 40.26479892806208\n",
      "iteration : 478200 --> loss : 40.171274526589876\n",
      "iteration : 478300 --> loss : 40.11375935053547\n",
      "iteration : 478400 --> loss : 40.06086335630493\n",
      "iteration : 478500 --> loss : 39.9920195374806\n",
      "iteration : 478600 --> loss : 40.01249517502898\n",
      "iteration : 478700 --> loss : 40.20356116974792\n",
      "iteration : 478800 --> loss : 40.3010898081341\n",
      "iteration : 478900 --> loss : 40.30809233880544\n",
      "iteration : 479000 --> loss : 40.241532396312024\n",
      "iteration : 479100 --> loss : 40.18758950232214\n",
      "iteration : 479200 --> loss : 40.10797896536997\n",
      "iteration : 479300 --> loss : 40.02592738406865\n",
      "iteration : 479400 --> loss : 40.03468115226327\n",
      "iteration : 479500 --> loss : 40.01348931914416\n",
      "iteration : 479600 --> loss : 39.94877932880076\n",
      "iteration : 479700 --> loss : 40.00896913907374\n",
      "iteration : 479800 --> loss : 39.929005853242394\n",
      "iteration : 479900 --> loss : 39.89611345504637\n",
      "iteration : 480000 --> loss : 39.88238128963469\n",
      "iteration : 480100 --> loss : 39.957863314334084\n",
      "iteration : 480200 --> loss : 39.83722275173367\n",
      "iteration : 480300 --> loss : 39.75341183264746\n",
      "iteration : 480400 --> loss : 39.75965059188358\n",
      "iteration : 480500 --> loss : 39.80082595797517\n",
      "iteration : 480600 --> loss : 39.76181022437761\n",
      "iteration : 480700 --> loss : 39.70134429899084\n",
      "iteration : 480800 --> loss : 39.746472362876254\n",
      "iteration : 480900 --> loss : 39.79724213273907\n",
      "iteration : 481000 --> loss : 39.94128359464762\n",
      "iteration : 481100 --> loss : 39.88821362711091\n",
      "iteration : 481200 --> loss : 39.8814763165478\n",
      "iteration : 481300 --> loss : 39.9175428412074\n",
      "iteration : 481400 --> loss : 40.00027218394048\n",
      "iteration : 481500 --> loss : 40.079158757357554\n",
      "iteration : 481600 --> loss : 40.127191374943024\n",
      "iteration : 481700 --> loss : 40.33453553454395\n",
      "iteration : 481800 --> loss : 40.266450328804865\n",
      "iteration : 481900 --> loss : 40.23187418118606\n",
      "iteration : 482000 --> loss : 40.21087241127558\n",
      "iteration : 482100 --> loss : 40.26912444305389\n",
      "iteration : 482200 --> loss : 40.28317021685826\n",
      "iteration : 482300 --> loss : 40.28816387209024\n",
      "iteration : 482400 --> loss : 40.09703737454624\n",
      "iteration : 482500 --> loss : 40.0246465007824\n",
      "iteration : 482600 --> loss : 40.048082312142675\n",
      "iteration : 482700 --> loss : 40.03167648995827\n",
      "iteration : 482800 --> loss : 40.209460246784886\n",
      "iteration : 482900 --> loss : 40.285137094226364\n",
      "iteration : 483000 --> loss : 40.31372552825958\n",
      "iteration : 483100 --> loss : 40.18474949173033\n",
      "iteration : 483200 --> loss : 40.20352194911604\n",
      "iteration : 483300 --> loss : 40.15936860508208\n",
      "iteration : 483400 --> loss : 40.04111206578595\n",
      "iteration : 483500 --> loss : 40.04718868648499\n",
      "iteration : 483600 --> loss : 40.04815642483015\n",
      "iteration : 483700 --> loss : 39.910982199545224\n",
      "iteration : 483800 --> loss : 39.993101517302364\n",
      "iteration : 483900 --> loss : 40.01321825986076\n",
      "iteration : 484000 --> loss : 39.97266861950893\n",
      "iteration : 484100 --> loss : 39.877682848788346\n",
      "iteration : 484200 --> loss : 39.96966021428056\n",
      "iteration : 484300 --> loss : 40.00037470946313\n",
      "iteration : 484400 --> loss : 39.802880061810455\n",
      "iteration : 484500 --> loss : 39.77172253239052\n",
      "iteration : 484600 --> loss : 39.84289937001394\n",
      "iteration : 484700 --> loss : 39.707900766843096\n",
      "iteration : 484800 --> loss : 39.73373448770936\n",
      "iteration : 484900 --> loss : 39.604419874321735\n",
      "iteration : 485000 --> loss : 39.715489961930174\n",
      "iteration : 485100 --> loss : 39.79014415494372\n",
      "iteration : 485200 --> loss : 39.84747247156824\n",
      "iteration : 485300 --> loss : 39.93749515307844\n",
      "iteration : 485400 --> loss : 39.88272207989516\n",
      "iteration : 485500 --> loss : 39.980971236188765\n",
      "iteration : 485600 --> loss : 40.01636677038432\n",
      "iteration : 485700 --> loss : 40.146407353913744\n",
      "iteration : 485800 --> loss : 40.23133398014328\n",
      "iteration : 485900 --> loss : 40.236681475710924\n",
      "iteration : 486000 --> loss : 40.242240368533565\n",
      "iteration : 486100 --> loss : 40.13541994876731\n",
      "iteration : 486200 --> loss : 40.19611234488105\n",
      "iteration : 486300 --> loss : 40.18966833871987\n",
      "iteration : 486400 --> loss : 40.33262783833434\n",
      "iteration : 486500 --> loss : 40.195963457806194\n",
      "iteration : 486600 --> loss : 40.07216378025303\n",
      "iteration : 486700 --> loss : 39.963441681662715\n",
      "iteration : 486800 --> loss : 40.02886909262808\n",
      "iteration : 486900 --> loss : 40.01376742824746\n",
      "iteration : 487000 --> loss : 40.153160603912454\n",
      "iteration : 487100 --> loss : 40.271421872378696\n",
      "iteration : 487200 --> loss : 40.26994536711475\n",
      "iteration : 487300 --> loss : 40.20825920646605\n",
      "iteration : 487400 --> loss : 40.13936421308946\n",
      "iteration : 487500 --> loss : 40.115476471229705\n",
      "iteration : 487600 --> loss : 40.01022856787653\n",
      "iteration : 487700 --> loss : 40.02921899983152\n",
      "iteration : 487800 --> loss : 40.04134342625929\n",
      "iteration : 487900 --> loss : 39.923653759034025\n",
      "iteration : 488000 --> loss : 39.95951231664439\n",
      "iteration : 488100 --> loss : 39.902646813393844\n",
      "iteration : 488200 --> loss : 39.925124002251906\n",
      "iteration : 488300 --> loss : 39.81692022233836\n",
      "iteration : 488400 --> loss : 40.01891730608525\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[26], line 10\u001b[0m\n\u001b[0;32m      8\u001b[0m targets \u001b[38;5;241m=\u001b[39m [char_to_int[ch] \u001b[38;5;28;01mfor\u001b[39;00m ch \u001b[38;5;129;01min\u001b[39;00m text[offset\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m1\u001b[39m:offset\u001b[38;5;241m+\u001b[39mseq_length\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m1\u001b[39m]]\n\u001b[0;32m      9\u001b[0m \u001b[38;5;66;03m#forward pass + get grads for a single sequence\u001b[39;00m\n\u001b[1;32m---> 10\u001b[0m loss,dWi2h,dWh2h,dWh2o,dhb,dob,hactive \u001b[38;5;241m=\u001b[39m \u001b[43mgradient_descent_single\u001b[49m\u001b[43m(\u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtargets\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhactive\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     11\u001b[0m smooth_loss \u001b[38;5;241m=\u001b[39m smooth_loss \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m0.999\u001b[39m \u001b[38;5;241m+\u001b[39m loss \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m0.001\u001b[39m \u001b[38;5;66;03m#weighted smoothing\u001b[39;00m\n\u001b[0;32m     12\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m n\u001b[38;5;241m%\u001b[39m\u001b[38;5;241m100\u001b[39m:\n",
      "Cell \u001b[1;32mIn[8], line 24\u001b[0m, in \u001b[0;36mgradient_descent_single\u001b[1;34m(inputs, targets, hactive)\u001b[0m\n\u001b[0;32m     22\u001b[0m dob \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m dyps\n\u001b[0;32m     23\u001b[0m dhb \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m (\u001b[38;5;241m1\u001b[39m\u001b[38;5;241m-\u001b[39mhenc[i]\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m2\u001b[39m)\u001b[38;5;241m*\u001b[39m(Wh2o\u001b[38;5;241m.\u001b[39mT \u001b[38;5;241m@\u001b[39m dyps \u001b[38;5;241m+\u001b[39m dhactive)\n\u001b[1;32m---> 24\u001b[0m dWi2h \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[43m(\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[43mhenc\u001b[49m\u001b[43m[\u001b[49m\u001b[43mi\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mWh2o\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mT\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m@\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mdyps\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mdhactive\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m@\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mxenc\u001b[49m\u001b[43m[\u001b[49m\u001b[43mi\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mT\u001b[49m\n\u001b[0;32m     25\u001b[0m dWh2h \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m ((\u001b[38;5;241m1\u001b[39m\u001b[38;5;241m-\u001b[39mhenc[i]\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m2\u001b[39m)\u001b[38;5;241m*\u001b[39m(Wh2o\u001b[38;5;241m.\u001b[39mT \u001b[38;5;241m@\u001b[39m dyps \u001b[38;5;241m+\u001b[39m dhactive)) \u001b[38;5;241m@\u001b[39m henc[i\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]\u001b[38;5;241m.\u001b[39mT\n\u001b[0;32m     26\u001b[0m dhactive \u001b[38;5;241m=\u001b[39m Wh2h\u001b[38;5;241m.\u001b[39mT \u001b[38;5;241m@\u001b[39m ((\u001b[38;5;241m1\u001b[39m\u001b[38;5;241m-\u001b[39mhenc[i]\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m2\u001b[39m)\u001b[38;5;241m*\u001b[39m(Wh2o\u001b[38;5;241m.\u001b[39mT \u001b[38;5;241m@\u001b[39m dyps \u001b[38;5;241m+\u001b[39m dhactive))\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for epoch in range(1,200001):\n",
    "    #grab chunks from inputs\n",
    "    if offset+seq_length+1>=len(text) or not n:\n",
    "        #print(\"came here\")\n",
    "        hactive = np.zeros((hidden_size,1)) #reset rnn preactivations from memory\n",
    "        offset=0\n",
    "    inputs = [char_to_int[ch] for ch in text[offset:offset+seq_length]]\n",
    "    targets = [char_to_int[ch] for ch in text[offset+1:offset+seq_length+1]]\n",
    "    #forward pass + get grads for a single sequence\n",
    "    loss,dWi2h,dWh2h,dWh2o,dhb,dob,hactive = gradient_descent_single(inputs, targets, hactive)\n",
    "    smooth_loss = smooth_loss * 0.999 + loss *0.001 #weighted smoothing\n",
    "    if not n%100:\n",
    "        print(f'iteration : {n} --> loss : {smooth_loss}')\n",
    "    \n",
    "    #Adam updation\n",
    "    #Momentum variables\n",
    "    vWi2h = (beta1 * vWi2h + (1-beta1) * dWi2h)\n",
    "    vWh2h = (beta1 * vWh2h + (1-beta1) * dWh2h)\n",
    "    vWh2o = (beta1 * vWh2o + (1-beta1) * dWh2o)\n",
    "    vhb = (beta1 * vhb + (1-beta1) * dhb)\n",
    "    vob = (beta1 * vob + (1-beta1) * dob)\n",
    "    \n",
    "    #RMSprop variables\n",
    "    sWi2h = (beta2 * sWi2h + (1-beta2) * np.square(dWi2h))\n",
    "    sWh2h = (beta2 * sWh2h + (1-beta2) * np.square(dWh2h))\n",
    "    sWh2o = (beta2 * sWh2o + (1-beta2) * np.square(dWh2o))\n",
    "    shb = (beta2 * shb + (1-beta2) * np.square(dhb))\n",
    "    sob = (beta2 * sob + (1-beta2) * np.square(dob))\n",
    "    \n",
    "    #Parameter updation. Momentum bias correction -> 1-beta1**epoch | RMS bias correction -> 1-beta2**epoch\n",
    "    for param,momentum,rms in zip([Wi2h, Wh2h, Wh2o, hb, ob],[vWi2h, vWh2h, vWh2o, vhb, vob],[sWi2h, sWh2h, sWh2o, shb, sob]):\n",
    "        param -= learning_rate * ((momentum / (1-beta1**epoch))/ np.sqrt((rms / (1-beta2**epoch)) + 1e-8)) / len(inputs)\n",
    "    offset += seq_length\n",
    "    n += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "787d7977",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dirs, Fore orstiom the know not me thou aran thee me like in a sispomian a stort. Come ever and an it know\n",
      "Live no commore,\n",
      "If peresemes, now it canfesing with my in that wife;\n",
      "What my putices of lostly you, you shichosiging shouth it,\n",
      "To in the hath joviss not with your safe to troze\n",
      "The nobleting came, I'll make we son, were could like\n",
      "Thou man spear.\n",
      "\n",
      "BORTIV:\n",
      "He!\n",
      "\n",
      "Tis:\n",
      "As thruch me when thrinct's day\n",
      "Tas,\n",
      "That me.\n",
      "\n",
      "ARWELUS:\n",
      "So the hath a last?\n",
      "\n",
      "SIAGTAR:\n",
      "Sir; geleing':\n",
      "Which of suigs of me.\n",
      "\n",
      "C\n"
     ]
    }
   ],
   "source": [
    "#sample random batches from trained model\n",
    "sample_ix = sample(hactive, inputs[0], 500)\n",
    "txt = ''.join(int_to_char[ix] for ix in sample_ix)\n",
    "print(txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c31a4e02",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}