makemore-mlp.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Multi Layer Perceptron @ Bengio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reading the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['emma', 'olivia', 'ava', 'isabella', 'sophia']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = open(file=\"names.txt\", mode=\"r\").read().splitlines()\n",
    "words[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32033"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(words)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Building character to index and vice-versa mappings\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "chars = sorted(list(set(\"\".join(words))))\n",
    "stoi = {s: i+1 for i, s in enumerate(chars)}\n",
    "stoi[\".\"] = 0\n",
    "itos = {i:s for s, i in stoi.items()}"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating the dataset with context window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "window = 3\n",
    "X, Y = [], []\n",
    "\n",
    "for w in words:\n",
    "\n",
    "  # print(w)\n",
    "  context = [0] * window\n",
    "  for ch in w + \".\":\n",
    "    ix = stoi[ch]\n",
    "    X.append(context)\n",
    "    Y.append(ix)\n",
    "    # print(\"\".join(itos[i] for i in context), \"==>\", itos[ix])\n",
    "    context = context[1:] + [ix]\n",
    "\n",
    "X = torch.tensor(X)\n",
    "Y = torch.tensor(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([228146, 3]), torch.int64, torch.Size([228146]), torch.int64)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape, X.dtype, Y.shape, Y.dtype"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lookup table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "C = torch.randn(size=(27, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([228146, 3, 2])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embed = C[X]\n",
    "embed.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "W1 = torch.randn(size=(6, 100))\n",
    "b1 = torch.randn(size=(100,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([228146, 100])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h = torch.tanh(embed.view(-1, 6) @ W1 + b1)\n",
    "h.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "W2 = torch.randn(size=(100, 27))\n",
    "b2 = torch.randn(size=(27,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "logits = h @ W2 + b2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([228146, 27])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "counts = logits.exp()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([228146, 27])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prob = counts / counts.sum(1, keepdims=True)\n",
    "prob.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([228146, 3]), torch.Size([228146]))"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape, Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "gen = torch.Generator().manual_seed(2147483647)\n",
    "\n",
    "C = torch.randn(size=(27, 10), generator=gen)\n",
    "W1 = torch.randn(size=(30, 200), generator=gen)\n",
    "b1 = torch.randn(size=(200,), generator=gen)\n",
    "W2 = torch.randn(size=(200, 27), generator=gen)\n",
    "b2 = torch.randn(size=(27,), generator=gen)\n",
    "\n",
    "params = [C, W1, b1, W2, b2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11897"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(p.nelement() for p in params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "for p in params:\n",
    "\n",
    "  p.requires_grad = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_list = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "lre = torch.linspace(-3, 0, 1000)\n",
    "lrs = 10**lre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "lri = []\n",
    "lr_loss = []\n",
    "stepi = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.9459733963012695"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(50000):\n",
    "\n",
    "  # mini-batch\n",
    "  ix = torch.randint(0, X_train.shape[0], (32,))\n",
    "\n",
    "  # forward pass\n",
    "  embed = C[X_train[ix]]\n",
    "  h = torch.tanh(embed.view(-1, 30) @ W1 + b1)\n",
    "  logits = h @ W2 + b2\n",
    "  loss = F.cross_entropy(logits, Y_train[ix])\n",
    "  # loss_list.append(loss.item())\n",
    "  # print(loss.item())\n",
    "\n",
    "  # backward pass\n",
    "  # lr = lrs[i]\n",
    "  for p in params:\n",
    "    p.grad = None\n",
    "\n",
    "  loss.backward()\n",
    "\n",
    "  # update\n",
    "  for p in params:\n",
    "    p.data += -0.01 * p.grad\n",
    "\n",
    "  # stats\n",
    "  # lri.append(lre[i])\n",
    "  stepi.append(i)\n",
    "  lr_loss.append(loss.item())\n",
    "\n",
    "loss.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.172678232192993"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embed = C[X_dev]\n",
    "h = torch.tanh(embed.view(-1, 30) @ W1 + b1)\n",
    "logits = h @ W2 + b2\n",
    "loss = F.cross_entropy(logits, Y_dev)\n",
    "loss.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeMAAAHSCAYAAADfUaMwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAxvElEQVR4nO3de3zU1Z3/8ffJzCSEBAiChDRBYxGpl4AoAmrVWLoqbFuh0hYf1Yr4ky3bbrWXbd3F33bdXXbburZuf7V0sWWl7bbY6tKyFpZWthGt5aYCAaxcRA3IRZEACbnM5fz+SCZkbskkTObM5fV8PHiY+X6/E45nhnl/v+d853yMtVYAAMCdAtcNAAAg3xHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA45nX1F48cOdJWV1dHbGtublZJSYmbBmUx+q3v6LP+od/6jj7rn1ztt5deeulda+250dudhXF1dbW2bNkSsa2urk61tbVuGpTF6Le+o8/6h37rO/qsf3K134wxb8bbzjA1AACOEcYAADhGGAMA4BhhDACAY4QxAACOEcYAADhGGAMA4BhhDACAY4QxAACOEcYAADhGGAMAECUQDOlkq1/BkE3L3+dsbWoAADJJWyCo1fWHtKRun/YcbZK3wCgQsrpoVKk+WztWM2sqVOT1DMjfTRgDAPLe1oZGzVu2Sf5gSM3tQUmSP9hxVfzakSY9uHKHHlq1S8vnT9HEMWUp//sZpgYA5LVtDY26fekGNbb4u4I4WnN7UI0tfs1dukHbGhpT3gbCGACQt9oCQd21bJNa/PFDOFqLv+P4tkByxyeLYWoAQN5aXX9I/mAoZvtffehCzZpUqfea23WosUX1B0/q8edflyT5gyGtqT+sWZMqU9YOrowBAHlrSd2+mKHpCVXDNOOy0Zr5b89r3rJNqqkqi9jf3B7Ukrq9KW0HV8YAgLwUDFntOdoUs33y+cP1u11H1BYIqU3SulePxByz+2iTgiErT4FJSVu4MgYA5KXm9oC8/QxTb4FRc3sgZW0hjAEAeamk0KtAnEU9trx5XNMvLleRt0CDCz360AdGxRwTCFmVFKZucJlhagBAXvIUGI0bVardRyKHqrcfOKFnXz2iNfddp3eb2vXakVM61eqPOOaiUaUpG6KWuDIGAOSxhbVjVVIYu6rW0vWv60OPPKfPLNuoyrJi1R880bWvpNCjhbUXprQdXBkDAPLWzJoKPbRql6TIO6r/5eM1GjeqVEVej55++YB2vn2ya5/PU6AZNaNT2g7CGACQt4q8Hi2fP0Vzl26IWPjjvhVb4x5f7Os4PtVrVDNMDQDIaxPHlGnFgmkqK/bFHbKWOoamy4p9WrFg2oCsTc2VMQAg700cU6aNi6ZrTf1hLanbq90RVZuGaGHtWM2oGU3VJgAABlKR16NZkyo1a1KlgiGr5vaASgq9Kb1rOhHCGACAKJ4Co6GDfGn7+5gzBgDAMcIYAADHCGMAABwjjAEAcIwwBgDAMcIYAADHCGMAABwjjAEAcIwwBgDAMcIYAADHCGMAABwjjAEAcIwwBgDAMcIYAADHCGMAABwjjAEAcIwwBgDAMcIYAADHCGMAABwjjAEAcIwwBgDAMcIYAADHCGMAABwjjAEAcIwwBgDAMcIYAADHCGMAABwjjAEAcIwwBgDAMcIYAADHCGMAABwjjAEAcIwwBgDAsV7D2BizzBhz1BizI8H+WmPMCWPM1s4/f5f6ZgIAkLu8SRzzhKTvSfpxD8c8b639SEpaBABAnun1ythau17Se2loCwAAeSlVc8ZXG2O2GWPWGGMuTdHvBAAgLxhrbe8HGVMt6Rlr7WVx9g2VFLLWNhljZkr6N2vtuAS/Z4GkBZJUXl5+5YoVKyL2NzU1qbS0tM//E/mOfus7+qx/6Le+o8/6J1f77cYbb3zJWjs5evtZh3GcY9+QNNla+25Px02ePNlu2bIlYltdXZ1qa2t7bQ8i0W99R5/1D/3Wd/RZ/+Rqvxlj4obxWQ9TG2NGG2NM589TOn/nsbP9vQAA5Ite76Y2xvxcUq2kkcaYA5K+LsknSdbaH0iaI2mhMSYgqUXSXJvM5TYAAJCURBhba2/vZf/31PHVJwAA0A+swAUAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjhHGAAA4RhgDAOAYYQwAgGOEMQAAjvUaxsaYZcaYo8aYHQn2G2PMd40xe40x240xV6S+mQAA5K5kroyfkHRLD/tnSBrX+WeBpCVn3ywAAPJHr2FsrV0v6b0eDrlV0o9thw2SyowxFalqIAAAuc6bgt9RKamh2+MDndsORR9ojFmgjqtnlZeXq66uLmJ/U1NTzDb0jn7rO/qsf+i3vqPP+iff+i0VYZw0a+1SSUslafLkyba2tjZif11dnaK3oXf0W9/RZ/1Dv/UdfdY/+dZvqbib+qCkMd0eV3VuAwAASUhFGK+S9JnOu6qnSTphrY0ZogYAAPH1OkxtjPm5pFpJI40xByR9XZJPkqy1P5C0WtJMSXslnZZ090A1FgCAXNRrGFtrb+9lv5X0uZS1CACAPMMKXAAAOEYYAwDgGGEMAIBjhDEAAI4RxgAAOEYYAwDgGGEMAIBjhDEAAI4RxgAAOEYYAwDgGGEMAIBjhDEAAI4RxgAAOEYYAwDgGGEMAIBjhDEAAI4RxgAAOEYYAwDgGGEMAIBjhDEAAI4RxgAAOEYYAwDgGGEMAIBjhDEAAI4RxgAAOEYYAwDgGGEMAIBjhDEAAI4RxgAAOEYYAwDgGGEMAIBjhDEAAI4RxgAAOEYYAwDgGGEMAIBjhDEAAI4RxgAAOEYYAwDgGGEMAIBjhDEAAI4RxgAAOEYYAwDgGGEMAIBjhDEAAI4RxgAAOEYYAwDgGGEMAIBjhDEAAI4RxgAAOEYYAwDgGGEMAIBjhDEAAI4RxgAAOEYYAwDgGGEMAIBjhDEAAI4RxgAAOEYYAwDgGGEMAIBjhDEAAI4RxgAAOEYYAwDgGGEMAIBjhDEAAI7lRBgHgiGdbPUrGLKumwIAQJ95XTegv9oCQa2uP6Qldfu052iTvAVGgZDVRaNK9dnasZpZU6Eir8d1M4GcFAiGdNofVEmhV54C47o5QNbLyjDe2tCoecs2yR8Mqbk9KEnyBzuuil870qQHV+7QQ6t2afn8KZo4psxhS4Hc0RYIqrHFr5u+8xwnwECKZd0w9baGRt2+dIMaW/xdQRytub3jQ2Pu0g3a1tCY3gYCOWhrQ6OmLl6ng8dbtPtIk6ztOAG29swJ8NTF6/j3BvRTVoVxWyCou5ZtUos/fghHa/F3HN8WSO54ALG6nwCHbPz7MjgBBs5OVg1Tr64/JH8wFLP9czdeqNuuqNSx5nYdamxR/cGTevz51yVJ/mBIa+oPa9akynQ3F8h6/T0B3rhoOkPWQB9k1ZXxkrp9MUPTl1UO1UcnVmjmd5/X3f+xWROqyiL2N7cHtaRubxpbCeSO6BPgay6/RHdOO7/r8f0fHqd7r3t/xHPCJ8AAkpc1YRwMWe052hSzfUr1OVq784ha/SE1tQX07KtHYo7ZfbSJrz0B/RB9Arz7zYP6yISKrsd/XlOhZ7a/HfEcToCBvsuaMG5uD8jbz69QeAuMmtsDKW4RkNvinQC/e/yERpQWadSQIl1cMUQnWvw6dKI15rmcAAN9kzVhXFLoVSDOP+6N+9/TTZeUq8hboJJCj6ZfXB5zTCBkVVKYVdPjgHOJToB/U39IM2sq9JEJ79Mz2w/FfS4nwEDfZE1CeQqMxo0q1e4jkWfqO98+qWe2H9Ka+67TseZ2bT/QGPPci0aVsjAB0EeJToCf2fa2vnHbBA0f7NOnlm6I+1xOgIG+yZorY0laWDtWJYWxd2g+9vu9+tAjz+kTP/ij9r/bHLGvpNCjhbUXpquJQM4InwBH23O0SSVFHh052aZ3TrXFfS4nwEDfZFUYz6ypkM/Ttyb7PAWaUTN6gFoE5LZEJ8C3PPq8bn88/lUxJ8BA32VVGBd5PVo+f4qKfYm/v/jos3u6vmNc7Os4nu87Av3DCTCQHlkVxpI0cUyZViyYprJiX9wzdqnjzLys2KcVC6axNjVwFpI5Ae6OE2Cgf7IujKWOQN64aLoWz67R+PJSGSP5PEbGSOPLh2jx7BptXDSdIAZSoPsJcIGJPw/MCTBwdrL2dscir0ezJlVq1qRKBUNWze0ByrkBAyR8Arzmd/+r8eWF2h1RtWmIFtaO1Yya0VwRA/2UtWHcnafAaOggn+tmADmtyNtx9bv2izdwAgykWFYOUyO7BYIhnWz1s0JTljrZ6pckDR3kI4iBFMmJK2NkvrZAUKvrD2lJ3T4K02eZ7q/dR0ed0L3/+DteOyDFCGMMuK0NjZq3bJP8wVBX0QF/sOOqOFyY/qFVu7R8/hRu/skwMa/dKF47YCAkNUxtjLnFGPOaMWavMeaBOPvnGWPeMcZs7fzzf1LfVGSj7oXpo8tfhlGYPjPx2gHp02sYG2M8kh6TNEPSJZJuN8ZcEufQJ621l3f++WGK24ks1FNh+p0P3RyzLVyYvi2QXCF7DJyeXrt4eO2As5PMMPUUSXutta9LkjFmhaRbJe0ayIYh+0UXpk9GuDD9rEmVA9QqJKP7a1c1vFjL756iVxqOa/q44RpzyQn98qUGffHDF2lEaZHuX/GKth04wWsHnIVkhqkrJTV0e3ygc1u024wx240xTxljxqSkdchq0YXpk0Fh+swQ/dqdP2KwHl+/Xz/973Uae26pbr28UnN+8Ef98+pX9bkbO9ah5rUD+i9VN3D9t6SfW2vbjDF/IWm5pA9FH2SMWSBpgSSVl5errq4uYn9TU1PMNvQuU/vto6NOSKPi7/MVSF+uSVTvtnHA/38ytc8yRffXbkhJQE3Np/WRUcdVXiz52k9qWNtRfbkmoKGlxzWpYly313LgX7tsw3utf/Kt35IJ44OSul/pVnVu62KtPdbt4Q8lfSveL7LWLpW0VJImT55sa2trI/bX1dUpeht6l4n9drLVr3v/8Xddd95Gmz9HeqQ+/tvP5zF66bZrB3Qhl0zss0wR/dpVDffq6mlWj9R79eWagHYcl9btl9bs8KpquEfTphZ0vZbpeO2yDe+1/sm3fktmmHqzpHHGmAuMMYWS5kpa1f0AY0xFt4cfk/Rq6pqIbJSoMH0yKEzvFq8dkH69hrG1NiDp85LWqiNkf2Gt3WmM+QdjzMc6D/uCMWanMWabpC9ImjdQDUZ2SFSYPhkUpneL1w5Iv6S+Z2ytXW2tvchaO9Zau7hz299Za1d1/vw31tpLrbUTrbU3Wmv/NJCNRnZIVJi+JxSmzwzdX7sDx1t086Pru/Z95ZfbtWbH4Zh9vHZA/7E2NQZMT4XpL/362rjbKUyfGXp67RLhtQP6jzDGgKEwffbitQPSizDGgOpemD7RkDWF6TMTrx2QPtz2iAHXVZi+/rCW1O2lMH0WiX7tpEb5PLx2QKoRxkiLIq9HsyZVatakSgrTZ5nur11dXZ1euu1aXjsgxRimRtp5CgyF6XsQCIZ0stWvYD+/6zvQeO2A1OPKGMgAbYGgVtcf0pK6fdoTMYxfqs/WjtXMmgqGgoEcRhgDjm1taNS8ZZvkD4a6ijOEl6J87UiTHly5Qw+t2qXl86dwkxSQoximBhza1tCo25duUGOLP2GFq+b2oBpb/Jq7dIO2NTSmt4EA0oIwBhxpCwR117JNavEnV2ayxd9xfFugb2UpAWQ+whhwZHX9IfmDoT49xx8MaU394QFqEQBXmDMGHFlSty9maPrTU8/Tp6eeJ0kaMsinA8dbdPvjG7r2N7cHtaRur2ZNqkxrWwEMLMIYcCAYstpztClm+39ufEv/ufEteQuMfnbvNP3whddjjtl9tEnBkOXrRUAOYZgacKC5PSBvD2H69Y9eqj/ue1frXj0as89bYNTcHhjI5gFIM8IYcKCk0KtAgkU95lxZpcrhxXp03Z64+wMhq5JCBrWAXEIYAw54CozGjSqN2X5Z5VDde937df+Tr8gmWIDLSNpx8MTANhBAWhHGgCMLa8fGVEO66+pqlQ32acW907T6Cx/UN26rkSQ9vfCarmNCVnznGMgxjHUhowSCIZ32B/OiEMHMmgo9tGqXpDN3VP/1U9vjHnvbkhcjHoe/c7xx0XSWyQRyAGEM5/J1XeYir0fL50/R3KUbel34Y+dDN+vSr6+N2Bb+zjFfcwKyH8PUcGprQ6OmLl6nB1fu0O4jTbK2Y11ma8+syzx18bqcHZKdOKZMKxZMU38GAcLfOQaQ/QhjOMO6zB0uqxym/lZLDH/nGEB2I4zhBOsyn9HcHpDP07/5cb5zDOQGwhhOJLMus4nKp1xdl7mn7xz3hu8cA7mBMIYT8dZlrhperHVfvkGPfHKifvvF6/W+YcUR+3N1jjTRd467SxTVF40qzfm7zoF8QBgj7RKtyyxJF4wo0U//+KZu+s56HWxsidmfq3Ok8b5zHFY22KfG0+0x20sKPVpYe+FANw1AGhDGSLue1mU+2NiiV3q4UStX50hn1lTI54n95zhqSJH+a+E1evz5/TH7fJ4CzagZnY7mARhghDHSrqc50tMJ7qoOy9U50vB3jot9kVfHR0+16UOPPKflL74Rsb3Y13F8Ln7/GshHhDHSLpk50kRyeY40/J3jsmJfwiHrkkKPyop9WrFgmiaOKUtvA/uhtT2gg42n1R7o+WY9IN/l3iUGssLC2rF6cOWOhN8vjicf5kgnjinTxkXTtab+sJbU7dXuiBXJhmhh7VjNqBmd0VfEp1r9+saaP+mXWw6ovdsd84WeAn1icpUemPEBDRnkc9hCIPMQxnAi3rrMB4636OZH1yd8Tr7MkRZ5PZo1qVKzJlUqGLJqbg9kzVrdv9jSoK89tT3u3d/twZD+c+Nb+tnGt/TwnAmaM3lM2tsHZCrCGE70ZV1mKX/nSD0FRkOz5CryqS0N+mpUoYt511Trjmnna8fBE3rjWLOa24J6/PnX9ZXO4whkoANzxnAmF+dI89WpVn/cilN3Tjtfd/xwo+5/cmvMvr9+artOtfrT0Dog8xHGcCo8R7p4do3Gl5fKGMnnMTJGGl8+RItn12jjoukZGcSBYEgnW/1Ov/ecCW2QpG+s+VPM0PTiWZepemSJ1n+1Vuu/Wqv3jyyJ2G8lfXPNn9LWRiCTMUwN57JpjjQTyj1mQhui/XLLgZhtP9/8lj551Rhd//Dv1dQa1DN/9UHVHzwZccwvthzQP82uSVczgYxFGCOjZPIc6daGRs1btkn+YKjrLnB/sON6MFzu8aFVu7R8/pQBu5LPhDZEaw+EIu6aDptSfY5a2oNq84fU1BbQs68eiX1uMKT2QEiFXgbpkN/4FwAkIbrcY9XwYq29//qIYwa63GOmlpx8p6k1bc/PlGF5INW4MgZ60d9yjxsXTU/ZcHEmtCGRc0sHxd2+cf97GuTzqMhboJJCj6ZfXK6fbXwr6eeHZeKwPJBqXBkDvUhU7tFTYPQvH6/Rb794vX48f4qKug21prrcY/c2LLj+/Zp3TbUk6f9+5GL97N6pkqSrx47Qo5+6fMDakEiht0CFcdbV3vn2SbX4g1qxYJqemD9F2w80xj7XU9DjEPXWhkZNXbxOD67cod1HmmRtx7C8tWeG5acuXpe2UQBgoBDGQC/ilXuUpOoRg/WTzgpTJ1v9mnFZRde+VJd77N6Gzfvf01XV50iSairLNLjQK2+B0ZTqc7Rp/3sD1oaefGJyVdztEx/6rWr/9Tl94gd/1H0rturx51+P2P/JBM+TYoflhw7y6o5p50cc42JYHhgIhDHQg57KPTYcb9GuQx13B+84eEJV50TWX05VucfoNtQfPKGaymEqLfKqPRDSK28d14SqYbqq+hxteuO9iOfuPpKekpMPzPiA+nrvu5H0tRkfiLsv3rD80GKf7owK47DwsHxbIPnlVYFMQhgDPeip3GP34gfBkGKOS1W5x+g2BEJWDcdPa86VVXrprePa906T/v3OyaoeOVh7o04crKSNrx876zb0Zsggnx6eM6FPz3l4zoSEa1THmxr42i0f0PkjBmv1Fz6ov4kT4ukalgcGAmEM9KCnco+9SVW5x3ht2PzGe7r3+vdr0/5j2tbQqLLBPu18+2Tc5y/86UtpuWKcM3mM/nXOhF6vkI2kf+1lbep4UwPf/J8/6c1jpzXzuy/oX+IsFpLOYXkg1QhjoAeZUO4xXhs27X9Po4YU6eU3G3X8tF/Wdmwbc06xfvOFD2pC1bCuYwMhm7YrxjmTx2j739+kO6aeF3NTV6GnQHdMPU/b//6mHoO4p6mB3qRqagBIN77aBPQiXrnH6ApT0TcmpbrcY3QbXtx3TOMWrenav//dZv3+T0f1gzuu1Fd+uU2vHjrVtS98xThrUmXK2tOTIYN8+qfZNfqn2TVqD4T0TlOrzi0dlPTCHuFh+fBiJn0RnhrI1IVjgES4MgZ6MbOmQr44X93pSarLPfbWhnNKCvX4ZybrvhVbI4I4zNUVY6G3QJVlg/u0wlaiqYGmtoBKinr+PnGqpgaAdCOMgV6Eyz0W+5JbWGIgyj0WeT36/h1XJNx/qtWvg40tuqp6eNz9qbqZLB0STQ00nvbrpTePa+3918e9gUtK3dQAkG6EMZCETCj3OPWCEQn3+YNWf/GTl/TxK6r0sYnvi9mfbVeMC2vHxu3n+1Zs1c2Pro97A1eqpwaAdCKMgSS5LvfoKTC6qDzxzWQt/qDueWKz7vngBfrwxaMi9rm+Yoy3pnRP60xnwtQAkE7Zc6oMZADX5R57u5nsZGtAtz72h4jnuLpijLemtD9oNXpokSSjI6daE64zHZ4amLt0Q1LrcQ/E1EBvAsGQTvuDGVvuE9mFMM5RfFAMPBflHmfWVOihVbskJf+9YRdXjD2Vejx8sq3ruJ7KP4anBu6K+j3dlRR65PMUpK1kJEUrMFAI4xzCB0Xuy4YrxvCa0slWmArrCNug5i7d0DXvHp4aWFN/WEvq9mp3xPt6iBbWjtWMmtFp+f/LxFrSyB2EcY7ggyJ/ZOIVY1hfSz3GE13+0fXUgJTcCUa8kwkgWdzAlQNa/MGMLDqPgeP6ZrJEEpWb/PgVlVpz33Vac991+vYnJ6rY59GyeVdpzX3Xae391+sjEyoijk+0znR4aqA/QdzTDWM96W8taYpWoC+4Ms5ybYGg9r/brBZ/csN06Sw6j4GVCVeM0eKtKT1uVKk+f+OFum3Jizp+2q9hxT7dMP5cHTnZqvlPbJYkDSmK/ChK1aphqZi6iXeCUTW8WMvvnqL6gyd0WeUw7T5ySl/6xVa1+juOC59MpGvVM2Q/royz3Or6Q7LdTvSrhhfr2S/dkLDovUR1m1x0NleMqZJoTelrLhyp1fWHdfy0X5J0osWv1w6f0nXjRuqBWz6gq6qH61Rb7IIkZ7tq2NaGRk1dvE4Prtyh3UeaZG3H1I21Z6Zupi5e1+tIUaJ61mNHleonG97Uh7/9nJraArpzWnXXPopWoK8I4yy3pG6fQjbyA6unovcSHxQYGD2Vm4y2/91m/fl3X9CfDp/SV24ary9Mj/3q1dmsGhae4w1P3VQNL9ba+6+Pam/vUzc9Fa042Niil948Lkla+crBmNXPKFqBviCMs1iiD4reit5LfFAg9RKtKf3i3nc1s2a0ygZ3fA1sWLFPo4YUqdUf1K+2HtS/r39dl71vWMzz+rtqWCrneHs6wbBRJ8HR/+fZtAQp3GPOOIsl+qCILno/yBd7DNVtkGrhNaV3H4k8QdxztEmP/X6vnlxwtULWaufbJ7Rq69v6m5kXy1orf9DqwV/tiPl9/V01LNFNZN4Co0c/dXmf5nh7qmddNXywrjivTC+/1ahbL3+fNr/xXsT+bFuCFG7xTslimVD4Pp+wkErv4q0QJklPv3xQT798MGLb+n97PuHvOZtVw3qa4/3q09v10pvH9a05E3TntOqu0peJbhhLdIIhSfuONunOq6v1rTnDtOfoKf10w5sR+10vQYrswqdxFjtT3aaxz8/lgyI5LKTSN/1ZISye/q4a1pc53ruvqdbj3c4HwlM30f8uEp1gBEJWX3xya9y/i6IV6CvmjLPcwtqxKjBnPjziFb1/9Nk9Ec/hgyI5qbobN5/0tdxkPGezathAzPFStALpQBhnuZk1FTJ9vMDlg6J30XfjxsNCKvElU24ynlSUoExmjldSn+Z4451gRJ/0dudiCVJkP8I4yxV5PbpgZInTwve5hhWXzl6iFcIkqWLoIFUMGzQgq4admbqJFZ7jffZLN2hYsa9Pc7yZUM8auY054xxQ7PNoxYIrM3Kt4myU6G7cnrDiUqzeVggbqFXDEpWZnP7t5xI+J5mpm0wqWoHcQxjnCD4oUif6btwJVcP0zdsmaNZjf1CBMfr156/V53/2csQdtqlavjFXxSs3OVAlKAeyzGQmLkGK3EAY5xA+KM5evLtxtx84oWdfPaIv3zReg3wF+tUrB+N+1SXR3bhIr3SVmXRRzxq5iznjHJUJaxVno0R343533R5dN26kJlQO0w+e2xf3uay4lDmY40W24coY6CbR3bhlgws1uNAjb4FRkdcT94qLhVQyC1M3yCZ8cgDdJFpx6Z9n1+iR3+7WmHMG64EZH9DXV+2MeS4LqWQepm6QLRimBqIsrB0bMbT58SsqFQiFtGrb21pSt1cTq4bp6rEjIp7DQiqZj6kbZDLCGIgSveLSf718UAt/+rIkKWSlWd9/UX/cdyziOSykAuBsEMZAlL4u6chCKgDOFmEMxMHduADSiRu4gAS4GxdAuhDGQA+4GxdAOhDGQJJYcQnAQGHOGAAAxwhjAAAcI4wBAHCMMAYAwDHCGAAAxwhjAAAcI4wBAHCMMAYAwDHCGAAAxwhjAAAcSyqMjTG3GGNeM8bsNcY8EGd/kTHmyc79G40x1SlvKQAAOarXMDbGeCQ9JmmGpEsk3W6MuSTqsHskHbfWXijpO5K+meqGAgCQq5K5Mp4iaa+19nVrbbukFZJujTrmVknLO39+StJ0YwxlbQAASEIyVZsqJTV0e3xA0tREx1hrA8aYE5JGSHq3+0HGmAWSFkhSeXm56urqIn5JU1NTzDb0jn7rO/qsf+i3vqPP+iff+i2tJRSttUslLZWkyZMn29ra2oj9dXV1it6G3tFvfUef9Q/91nf0Wf/kW78lM0x9UNKYbo+rOrfFPcYY45U0TNKxVDQQAIBcl0wYb5Y0zhhzgTGmUNJcSauijlkl6a7On+dI+l9rrU1dMwEAyF29DlN3zgF/XtJaSR5Jy6y1O40x/yBpi7V2laQfSfqJMWavpPfUEdgAACAJSc0ZW2tXS1odte3vuv3cKukTqW0aAAD5gRW4AABwjDAGAMAxwhh5IxAM6WSrX8EQ9xYCyCxp/Z4xkG5tgaBW1x/Skrp92nO0Sd4Co0DI6m8nWR1/5YBm1lSoyOtx3UwAeY4wRs7a2tCoecs2yR8Mqbk9KEnyBzuuilv9QT24coceWrVLy+dP0cQxZQ5bCiDfMUzdDwx3Zr5tDY26fekGNbb4u4I4WnN7UI0tfs1dukHbGhrT20AA6IYr4yQlGu68aFSpPls7Nq+GOwPBkE77gyop9MpTkHn1QNoCQd21bJNa/PFDOFqLv+P4jYum581rCCCzEMZJ6Gm487UjTXkx3JlNJyOr6w/JHwxFbKsaXqwf3XWVbn50vSRp0sVjdX95oR59do8kyR8MaU39Yc2aVJn29gIAw9S9YLiz42Rk6uJ1enDlDu0+0iRrO05GrD1zMjJ18bqM+X9fUrcv4WuVSHN7UEvq9g5QiwCgZ4RxD/o73NkW6FsQZLJEJyNPL7ym6+dMOhkJhqz2HG3q13N3H23iPgAAThDGPYge7vzin12k+ddWdz3+yk3jdXe3x9KZ4c5c0NPJyG1LXozZlgknI83tAXnjzGMHglbdN3s8sUPq3gKj5vbAQDYPAOIijHsQPdz5yy0N+vgVVZIkY6SPTqzQylciq0nm0nBnvLnXsJ0P3Rx3u+uTkZJCrwJxrm7fbWrTiNIilQ32qdBToAsqy2OOCYSsSgq5jQJA+vHJk0C84c4Dx1t0/HS7Ln3fUI0sLdLOt0+q8bQ/5rnh4c5MvNO4L85m7tXVjVCeAqNxo0q1+0jkaxcIWX133R79+nPX6vDJVh0/ETuUfdGo0qx/zQBkJ8I4gfBwZ/iu6bAnNzdozpVVOre0SL/Y0hD3ueHhzqGDfOlo6oBIxdyrq2BbWDtWD67cEXMi8cSLb+iJF9+QJH25JqBH68+8/UsKPVpYe2E6mwkAXRimTiDRcOfanYd1/UXnakJVmdbvfifuc3NhuDPR3GsyXM+9zqypkM/Tt7e2z1OgGTWjB6hFANAzwjiB8HBnNH/QasO+Y/pN/dtKdONtLgx3JjoZSYbrk5Eir0fL509RsS+57z0X+zqOz5TvSQPIP4RxDxbWjlVJYeQHtDHSpPPK9OTm+EPUuTLcmehkJBmZcDIycUyZViyYprJiX8xrGFZS6FFZsU8rFkzL2cVaAGQHwrgH0cOdF44q1XNfuVF/2HdMbxw7Hfc5uTTcGe9kJOzSr6+Nuz2TTkYmjinTxkXTtXh2jcaXl8oYyecxMkYa5PNo8ewabVw0nSAG4Fx2T2wOsPBw59ylG9TiD2rv0SZd//DvEx6fa8OdM2sq9NCqXZKSv6M6005GirwezZpUqVmTKhUMWTW3B1RS6NXz659TLUtfAsgQXBn3Ip+HO3Nt7tVTYDR0kM/5EDoARCOMk9DTcOf48iE5PdyZzycjAJAuDFMnKdFwZz5cZYVPRtbUH9aSur3aHVG1aYgW1o7VjJrRGXtFDACZjjDuh/BwZz7J55ORfNe9fjXgWqbXU+8v/nWhz/LxZCTfJKpf/beTrI6/ciCj6lcj92VTPfX+IowBRNja0Kh5yzbJHwx1LSkaXha21R/Ugyt36KFVu7R8/hTuEcCA6+n9GK6nngvvR27gAjJAIBjSyVa/83rKiepXd5dJ9auR2/Lp/ciVMeBIpg299VS/Op5w/eqNi6Zn/RAhMo+V8ur9SBgDDmTi0Fui+tVL77xSFcOKVeQr0IE39kn1Z2p4h+tXuyqZidx1osWfsJ56Itn8fmSYGkizTB16S1S/+q+f2q6Pfu8FffT/vaCJ49+vssFnbt4L168GUu2dU239rqeejbgyBtIoU4eCe6pfffe11br50o4lTksHF+uCESV65XRj137X9auRe4Ihq1Z/UP2JqGx9PxLGQBolGgqedXml5l1brUKP0daGRj34qx1dJTrTMfQWrl8dHioPm/b+c3TthSM1+/t/UKs/pBe+OFVFvsgBtXD9ar7uhlRpbg/ImLOrp55t70eGqYE0ijcUPPbcUn1kYoXmLHlRM7/7goIhRQRvOobeEtWvHjLIpxMtfrX6Qxp7bolGjxwec4zr+tXIPSWFXlmbnfXU+yv7WgxkqURDwddeOEI1lcO06vPXSpKKfB4da26LOGagh97C9at3H4ls33OvvaNPTz1Pz37pBr3+TpMOv3s85rmZUL8aucVTYDQoQYGa/5h3lb729HYdPdUWd3+2vh8JYyBNEg0FG2P09EsH9K21ryV8bjqG3hbWjtWDK3dEXLm3B0Oa9x+bux5/uSagDa+f+djIpPrVyC3nDilSSWEwZiTp7ic2J3hGdr8fGaYG0iTRUPAf9r6rGTUVGlFSKEkaVuxTZVlxxDHpGHqbWVMhn6dvHwmZVr8auWNYsS+v3o+EMZAm4aHgaHuPNumR376mn9wzRWvuu04/vWeqRg0pijgmHUNvuVa/GtnNSHn1fmSYGkijeEPBkvTM9kN6ZvuhuM9J59BbuH71XVELkkS3x+cpyPq1gJH58un9yJUxkEbZMBQcrl+9eHaNxpeXyhjJ5zEyRhrk82jx7BptXDQ9qz/4kD16ej+OLx+SM+9HroyBNAoPBc9duiGphT9cDb0lql/9/PrnVJuFSw0iu+VDPXWujIE0Cw+9lRX7VFIYP2RLCj0qK/ZpxYJpzs/4w/Wrc+mDD9krV9+PXBkDDoSH3tbUH9aSur3aHVG1aYgW1o7VjJrRWXszCoC+IYwBR/Jh6A1AcghjIAOEh94A5CfmjAEAcIwwBgDAMcIYAADHCGMAABwjjAEAcIwwBgDAMcIYAADHCGMAABwjjAHEFQiGdLLVr2DIum4KkPNYgQtAl7ZAUKvrD2lJ3T7tiVgvu1SfrR2rMtcNBHIUYQxAkrS1oVHzooq4+4MdV8WvHWnSgyt36HMXt+uchkbnlaSAXMMwNQBta2jU7Us3qLHF3xXE0ZrbgwqGrOYu3aBtDY3pbSCQ4whjIM+1BYK6a9kmtfjjh3C0Fn/H8W2B5I4H0DvCGMhzq+sPyR8MxWy/54MXaO3912vt/ddr/rXVEfv8wZDW1B9OUwuB3EcYA3luSd2+mKHpyyqH6hOTqzTrsT9o9vf/oLlTztOl7xvatb+5PagldXvT3VQgZxHGQB4Lhqz2HG2K2X5V9Tlau/OIWvxBnW4P6n92HNZV1edEHLP7aBNfewJShDAG8lhze0DeAtOv53oLjJrbAyluEZCfCGMgj5UUehWIc3W7af97uumScg3yFajY59HNl47W5jfeizgmELIqKeTbkUAq8C8JyGOeAqNxo0q1+0jkUPXOt0/qqZcO6Nef+6Ak6cnNb2nn2yd1y4gzx1w0qlSefl5VA4hEGAN5bmHtWD24ckfMTVw/emG/fvTC/rjPKSn0aGHtheloHpAXGKYG8tzMmgr5PH37KPB5CjSjZvQAtQjIP4QxkOeKvB4tnz9FxT5PUscX+zqOL/ImdzyA3hHGADRxTJlWLJimsmKfSgrjh2xJoUeeAqMVC6axNjWQYoQxAEkdgbxx0XQtnl2j8eWlMkbyeYyMkcaXD9Hi2TW6uGIoQQwMAG7gAtClyOvRrEmVmjWpUsGQVXN7QCWF3q67puvq9jhuIZCbCGMAcXkKjIYO8rluBpAXGKYGAMAxwhgAAMcIYwAAHCOMAQBwjDAGAMAxwhgAAMcIYwAAHCOMAQBwjDAGAMAxwhgAAMeMtdbNX2zMO5LejNo8UtK7DpqT7ei3vqPP+od+6zv6rH9ytd/Ot9aeG73RWRjHY4zZYq2d7Lod2YZ+6zv6rH/ot76jz/on3/qNYWoAABwjjAEAcCzTwnip6wZkKfqt7+iz/qHf+o4+65+86reMmjMGACAfZdqVMQAAecdpGBtjzjHG/M4Ys6fzv8MTHHeeMea3xphXjTG7jDHVaW5qRkm23zqPHWqMOWCM+V4625hpkukzY8zlxpg/GmN2GmO2G2M+5aKtrhljbjHGvGaM2WuMeSDO/iJjzJOd+zfm+7/HsCT67Uudn1/bjTHrjDHnu2hnJumtz7odd5sxxhpjcvbuatdXxg9IWmetHSdpXefjeH4s6WFr7cWSpkg6mqb2Zapk+02S/lHS+rS0KrMl02enJX3GWnuppFskPWqMKUtfE90zxngkPSZphqRLJN1ujLkk6rB7JB231l4o6TuSvpneVmaeJPvtFUmTrbUTJD0l6VvpbWVmSbLPZIwZIuk+SRvT28L0ch3Gt0pa3vnzckmzog/ofHG81trfSZK1tslaezptLcxMvfabJBljrpRULum36WlWRuu1z6y1u621ezp/flsdJ30xX87PcVMk7bXWvm6tbZe0Qh191133vnxK0nRjjEljGzNRr/1mrf19t8+uDZKq0tzGTJPMe03quKD4pqTWdDYu3VyHcbm19lDnz4fVERzRLpLUaIz5L2PMK8aYhzvPqPJZr/1mjCmQ9Iikr6SzYRksmfdaF2PMFEmFkvYNdMMyTKWkhm6PD3Rui3uMtTYg6YSkEWlpXeZKpt+6u0fSmgFtUebrtc+MMVdIGmOt/U06G+aCd6D/AmPMs5JGx9m1qPsDa601xsS7tdsr6TpJkyS9JelJSfMk/Si1Lc0sKei3v5S02lp7IF8uWlLQZ+HfUyHpJ5LustaGUttK5DtjzB2SJku6wXVbMlnnBcW31fF5n/MGPIyttR9OtM8Yc8QYU2GtPdT5ARhvLviApK3W2tc7n/MrSdOU42Gcgn67WtJ1xpi/lFQqqdAY02St7Wl+OauloM9kjBkq6TeSFllrNwxQUzPZQUljuj2u6twW75gDxhivpGGSjqWneRkrmX6TMebD6jg5vMFa25amtmWq3vpsiKTLJNV1XlCMlrTKGPMxa+2WtLUyTVwPU6+SdFfnz3dJ+nWcYzZLKjPGhOfuPiRpVxralsl67Tdr7aettedZa6vVMVT941wO4iT02mfGmEJJK9XRV0+lsW2ZZLOkccaYCzr7Y646+q677n05R9L/WhYs6LXfjDGTJP27pI9Za/P9JlSplz6z1p6w1o601lZ3fo5tUEff5VwQS+7D+BuS/swYs0fShzsfyxgz2RjzQ0my1gbVESbrjDH1koykxx21N1P02m+IkUyffVLS9ZLmGWO2dv653ElrHemcA/68pLWSXpX0C2vtTmPMPxhjPtZ52I8kjTDG7JX0JfV8N39eSLLfHlbHKNUvO99b0Sc5eSXJPssbrMAFAIBjrq+MAQDIe4QxAACOEcYAADhGGAMA4BhhDACAY4QxAACOEcYAADhGGAMA4Nj/B723y7R2SCUcAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 8))\n",
    "plt.scatter(C[:, 0].data, C[:,1].data, s=200)\n",
    "for i in range(C.shape[0]):\n",
    "  plt.text(C[i, 0].item(), C[i, 1].item(), itos[i], ha=\"center\", va=\"center\", color=\"white\")\n",
    "plt.grid(\"minor\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data splitting into train, dev, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# building the dataset\n",
    "\n",
    "def build_dataset(words):\n",
    "\n",
    "  window = 3\n",
    "  X, Y = [], []\n",
    "\n",
    "  for w in words:\n",
    "\n",
    "    # print(w)\n",
    "    context = [0] * window\n",
    "    for ch in w + \".\":\n",
    "      ix = stoi[ch]\n",
    "      X.append(context)\n",
    "      Y.append(ix)\n",
    "      # print(\"\".join(itos[i] for i in context), \"==>\", itos[ix])\n",
    "      context = context[1:] + [ix]\n",
    "\n",
    "  X = torch.tensor(X)\n",
    "  Y = torch.tensor(Y)\n",
    "  print(X.shape, Y.shape)\n",
    "  return X, Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([182625, 3]) torch.Size([182625])\n",
      "torch.Size([22655, 3]) torch.Size([22655])\n",
      "torch.Size([22866, 3]) torch.Size([22866])\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "random.seed(42)\n",
    "random.shuffle(words)\n",
    "\n",
    "n1 = int(0.8 * len(words))\n",
    "n2 = int(0.9 * len(words))\n",
    "\n",
    "X_train, Y_train = build_dataset(words[:n1])\n",
    "X_dev, Y_dev = build_dataset(words[n1:n2])\n",
    "X_test, Y_test = build_dataset(words[n2:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25626, 28829, 32033, 3203, 6407)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n1, n2, len(words), n2 - n1, len(words) - n1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2 (tags/v3.9.2:1a79785, Feb 19 2021, 13:44:55) [MSC v.1928 64 bit (AMD64)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "2b9dfff7b51cc1688c7cfa732c957a22f0e1ad605493f822f59c3fa5410fc251"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}