adder.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GPT adder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This repo was inspired by karpathy's suggested exercice in his video [Let's build GPT: from scratch, in code, spelled out.](https://youtu.be/kCc8FmEb1nY?feature=shared)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "from gpt.model import GPT, GPTConfig, Trainer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## generate samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following function will generate random samples for training the GPT adder.\n",
    "\n",
    "It generates the digits of the answer in reverse order, as the typical addition algorithm would proceed right to left too.\n",
    "\n",
    "i.e. `79+11=09` because `79+11=90`\n",
    "\n",
    "If `DataConfig.chain_of_thought` is `True`, the answer will be broken down into atomic additions, with the carry in brackets.\n",
    "\n",
    "i.e. `55+96;5+6=11(1);5+9+1=15;151` because to compute 55+96 you do, in this order:\n",
    "\n",
    "- 5 + 6 = 11 carry 1\n",
    "- 5 + 9 + _carry_ = 5 + 9 + 1 = 15\n",
    "- result is 151"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dataclasses import dataclass\n",
    "\n",
    "@dataclass\n",
    "class DataConfig:\n",
    "    train_low: int = 0\n",
    "    train_high: int = 1000\n",
    "    test_low: int = 0\n",
    "    test_high: int = 1000\n",
    "    chain_of_thought: bool = True\n",
    "\n",
    "data_config = DataConfig()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('748+714;8+4=12(1);4+1+1=6(0);7+7=14;1462', 7)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def generate_sample(low, high, detailed=data_config.chain_of_thought):\n",
    "    a = random.randint(low, high)\n",
    "    b = random.randint(low, high)\n",
    "\n",
    "    if not detailed:\n",
    "        sample = f\"{a}+{b}=\"\n",
    "        prompt_end_idx = len(sample) - 1\n",
    "        res = a + b\n",
    "\n",
    "        # add in reverse order\n",
    "        sample += str(res)[::-1]\n",
    "\n",
    "        return sample, prompt_end_idx\n",
    "    \n",
    "    sample = f\"{a}+{b};\"\n",
    "\n",
    "    # give the index of the last character of the prompt (useful for ignore_index of cross entropy loss, see later...)\n",
    "    prompt_end_idx = len(sample) - 1\n",
    "\n",
    "    array_of_numbers_a = [int(x) for x in str(a)]\n",
    "    array_of_numbers_b = [int(x) for x in str(b)]\n",
    "\n",
    "    diff_in_num = abs(len(array_of_numbers_a) - len(array_of_numbers_b))\n",
    "\n",
    "    carry = 0\n",
    "\n",
    "    while len(array_of_numbers_a) and len(array_of_numbers_b):\n",
    "        prefix_carry = f\"+{carry}\" if carry else \"\"\n",
    "\n",
    "        d1 = array_of_numbers_a.pop()\n",
    "        d2 = array_of_numbers_b.pop()\n",
    "\n",
    "        res = d1 + d2 + carry\n",
    "\n",
    "        carry = res//10\n",
    "\n",
    "        if len(array_of_numbers_a) == 0 and len(array_of_numbers_b) == 0:\n",
    "            sample += f\"{d1}+{d2}{prefix_carry}={res};\"\n",
    "        else:\n",
    "            sample += f\"{d1}+{d2}{prefix_carry}={res}({carry});\"\n",
    "\n",
    "    while len(array_of_numbers_a):\n",
    "        d1 = array_of_numbers_a.pop()\n",
    "        d2 = carry\n",
    "        res = d1 + d2\n",
    "        carry = res//10\n",
    "        if len(array_of_numbers_a) > 0:\n",
    "            sample += f\"{d1}+{d2}={res}({carry});\"\n",
    "        else:\n",
    "            sample += f\"{d1}+{d2}={res};\"\n",
    "\n",
    "    while len(array_of_numbers_b):\n",
    "        d1 = array_of_numbers_b.pop()\n",
    "        d2 = carry\n",
    "        res = d1 + d2\n",
    "        carry = res//10\n",
    "        if len(array_of_numbers_b) > 0:\n",
    "            sample += f\"{d1}+{d2}={res}({carry});\"\n",
    "        else:\n",
    "            sample += f\"{d1}+{d2}={res};\"\n",
    "\n",
    "    sample += f\"{a+b}\"\n",
    "\n",
    "    return sample, prompt_end_idx\n",
    "\n",
    "# get a sample from the training set\n",
    "sample, prompt_idx = generate_sample(low=data_config.train_low, high=data_config.train_high)\n",
    "sample, prompt_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('868+658;8+8=16(1);6+5+1=12(1);8+6+1=15;1526', 7)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a sample from the test set\n",
    "sample, prompt_idx = generate_sample(low=data_config.test_low, high=data_config.test_high)\n",
    "sample, prompt_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'868+658;'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def extract_prompt_from_sample(sample, prompt_idx):\n",
    "    \"\"\"extract only the prompt from a full addition sample\"\"\"\n",
    "    return sample[:prompt_idx+1]\n",
    "\n",
    "extract_prompt_from_sample(sample, prompt_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def correct_answer(sample, model_output, only_result=False):\n",
    "    \"\"\"check if the model calculated the correct answer by comparing against the ground truth\"\"\"\n",
    "    if only_result:\n",
    "        response = f\";{sample.split(';')[-1]}<|endoftext|>\"\n",
    "        return response in model_output\n",
    "    else:\n",
    "        return model_output == (sample + \"<|endoftext|>\")\n",
    "\n",
    "correct_answer(sample, sample + \"<|endoftext|>\", only_result=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## build the vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'(',\n",
       " ')',\n",
       " '+',\n",
       " '0',\n",
       " '1',\n",
       " '2',\n",
       " '3',\n",
       " '4',\n",
       " '5',\n",
       " '6',\n",
       " '7',\n",
       " '8',\n",
       " '9',\n",
       " ';',\n",
       " '<|endoftext|>',\n",
       " '='}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens = set()\n",
    "for i in range(10):\n",
    "    tokens.add((str(i)))\n",
    "\n",
    "if data_config.chain_of_thought:\n",
    "    tokens.add(\"+\")\n",
    "    tokens.add(\"=\")\n",
    "    tokens.add(\";\")\n",
    "    tokens.add(\"(\")\n",
    "    tokens.add(\")\")\n",
    "else:\n",
    "    tokens.add(\"+\")\n",
    "    tokens.add(\"=\")\n",
    "tokens.add(\"<|endoftext|>\")\n",
    "tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## create config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GPTConfig(vocab_size=16, batch_size=32, block_size=64, max_iters=5000, eval_iters=100, eval_interval=1000, learning_rate=0.001, weight_decay=0.1, n_embd=128, n_head=4, n_layer=1, dropout=0.2, device='cpu', eos_token_id=-1)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = GPTConfig(\n",
    "    vocab_size=len(tokens),\n",
    "    block_size=64,\n",
    "    batch_size=32,\n",
    "    max_iters=5000,\n",
    "    eval_iters=100,\n",
    "    eval_interval=1000,\n",
    "    learning_rate=1e-3,\n",
    "    weight_decay=1e-1,\n",
    "    n_embd=128,\n",
    "    n_head=4,\n",
    "    n_layer=1,\n",
    "    dropout=0.2\n",
    ")\n",
    "config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "stoi = { s:i for i,s in enumerate(tokens) }\n",
    "itos = { i:s for i,s in enumerate(tokens) }\n",
    "\n",
    "def encode(s):\n",
    "    return [stoi[ch] for ch in s]\n",
    "\n",
    "def decode(l):\n",
    "    return ''.join([itos[i] for i in l])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "EOS_ID = stoi['<|endoftext|>']\n",
    "config.eos_token_id = EOS_ID\n",
    "EOS_ID"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## model definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "210448 parameters\n"
     ]
    }
   ],
   "source": [
    "model = GPT(config=config)\n",
    "\n",
    "# put the model on the device\n",
    "model.to(config.device)\n",
    "\n",
    "# print the number of parameters in the model\n",
    "print(sum(p.numel() for p in model.parameters()), 'parameters')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## data loader and trainer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The tokens that belongs to the prompt are converted into `-1` in the label vector, as the loss will not be calculated for these tokens (see `ignore_index` of the [cross entropy loss](https://pytorch.org/docs/stable/generated/torch.nn.functional.cross_entropy.html))."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([32, 43]), torch.Size([32, 43]))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class AdderTrainer(Trainer):\n",
    "    def __init__(self, model, config):\n",
    "        super().__init__(model=model, config=config)\n",
    "\n",
    "    def get_batch(self, split: str = \"train\"):\n",
    "        X, Y = [], []\n",
    "        for k in range(self.config.batch_size):\n",
    "            sample, prompt_end_idx = generate_sample(low=data_config.train_low, high=data_config.train_high)\n",
    "            encoded_sample = torch.tensor(encode(sample))\n",
    "\n",
    "            # add EOS token\n",
    "            encoded_sample = torch.cat((encoded_sample, torch.tensor([EOS_ID])))\n",
    "\n",
    "            x = encoded_sample[:-1].clone()\n",
    "            y = encoded_sample[1:].clone()\n",
    "\n",
    "            y[:prompt_end_idx] = -1\n",
    "            X.append(x)\n",
    "            Y.append(y)\n",
    "\n",
    "        X = torch.nn.utils.rnn.pad_sequence(X, batch_first=True, padding_value=EOS_ID)\n",
    "        Y = torch.nn.utils.rnn.pad_sequence(Y, batch_first=True, padding_value=-1)\n",
    "            \n",
    "        X, Y = X.to(self.config.device), Y.to(self.config.device)\n",
    "        return X, Y\n",
    "\n",
    "    @torch.no_grad()\n",
    "    def estimate_loss(self):\n",
    "        out = {}\n",
    "        self.model.eval()\n",
    "        losses = torch.zeros(self.config.eval_iters)\n",
    "        for k in tqdm(range(self.config.eval_iters)):\n",
    "            X, Y = self.get_batch()\n",
    "            logits, loss = self.model(X, Y)\n",
    "            losses[k] = loss.item()\n",
    "        out[\"val\"] = losses.mean()\n",
    "        out[\"train\"] = losses.mean()\n",
    "        self.model.train()\n",
    "        return out\n",
    "\n",
    "trainer = AdderTrainer(model, config)\n",
    "\n",
    "x, y = trainer.get_batch()\n",
    "x.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y=tensor([-1, -1, -1, -1, -1, -1, -1,  7, 15,  7,  5,  4, 10,  2, 11,  9, 12, 15,\n",
      "         7,  5,  8, 10,  2, 11,  9,  7, 15,  1,  5, 13, 13,  9, 13, 13,  8,  4,\n",
      "         0, -1, -1, -1, -1, -1, -1])\n",
      "x=tensor([ 7, 12,  7, 15,  1,  7,  7,  9,  7, 15,  7,  5,  4, 10,  2, 11,  9, 12,\n",
      "        15,  7,  5,  8, 10,  2, 11,  9,  7, 15,  1,  5, 13, 13,  9, 13, 13,  8,\n",
      "         4,  0,  0,  0,  0,  0,  0])\n",
      "y=2+2=4(0);6+2=8(0);2+9=11;1184<|endoftext|>\n",
      "x=262+922;2+2=4(0);6+2=8(0);2+9=11;1184<|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|>\n"
     ]
    }
   ],
   "source": [
    "idx = 2\n",
    "print(f\"y={y[idx]}\")\n",
    "print(f\"x={x[idx]}\")\n",
    "\n",
    "without_ignore_index = y[idx].tolist()\n",
    "without_ignore_index = [i for i in without_ignore_index if i != -1]\n",
    "\n",
    "print(f\"y={decode(without_ignore_index)}\")\n",
    "print(f\"x={decode(x[idx].tolist())}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.210448 M parameters\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training:   0%|          | 0/5000 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:00<00:00, 164.84it/s]\n",
      "training:   0%|          | 8/5000 [00:00<06:03, 13.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0: train loss 2.8060, val loss 2.8060\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:00<00:00, 183.96it/s]61.14it/s]\n",
      "training:  20%|██        | 1010/5000 [00:18<02:17, 29.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 1000: train loss 0.0165, val loss 0.0165\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:00<00:00, 161.24it/s] 60.27it/s]\n",
      "training:  40%|████      | 2010/5000 [00:35<01:45, 28.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 2000: train loss 0.0081, val loss 0.0081\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:00<00:00, 183.50it/s] 59.44it/s]\n",
      "training:  60%|██████    | 3008/5000 [00:52<01:08, 28.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 3000: train loss 0.0056, val loss 0.0056\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:00<00:00, 187.62it/s] 62.15it/s]\n",
      "training:  80%|████████  | 4007/5000 [01:09<00:38, 25.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 4000: train loss 0.0053, val loss 0.0053\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:00<00:00, 171.73it/s] 58.84it/s]\n",
      "training: 100%|██████████| 5000/5000 [01:27<00:00, 57.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 4999: train loss 0.0048, val loss 0.0048\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train_losses, val_losses = trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+HElEQVR4nO3deXxU5d3///eZhGwkmZBAFiSQsBQEISwCJgRBRbkpLtSfikiL8lNbK1Qptb1L795AbS1qb1vqCrZVUEtdsMJ9UysiKrIE2YoCKoWwBSGBJGSyQBYy5/sHZCQkQJaZOTMnr+fjMQ/JNdc55zOjJe+e6zrXZZimaQoAAMAmHFYXAAAA4E2EGwAAYCuEGwAAYCuEGwAAYCuEGwAAYCuEGwAAYCuEGwAAYCuEGwAAYCuEGwAAYCuEGwBNduDAARmGoUWLFnna5s6dK8MwmnS8YRiaO3euV2saPXq0Ro8e7dVzAghuhBvApm6++WZFRUWprKzsgn0mT56ssLAwFRUV+bGy5vviiy80d+5cHThwwOpSPD7++GMZhqGlS5daXQqA8xBuAJuaPHmyTp06pXfeeafR90+ePKnly5frP/7jP5SQkNDi6/zyl7/UqVOnWnx8U3zxxRf61a9+1Wi4ef/99/X+++/79PoAggvhBrCpm2++WTExMVqyZEmj7y9fvlwVFRWaPHlyq64TGhqqiIiIVp2jNcLCwhQWFmbZ9QEEHsINYFORkZG69dZbtXr1ah07dqzB+0uWLFFMTIxuvvlmFRcX65FHHlH//v0VHR2t2NhYjRs3Tp999tklr9PYnJuqqir9+Mc/VqdOnTzXOHz4cINjDx48qAcffFC9e/dWZGSkEhISdPvtt9e7Q7No0SLdfvvtkqRrrrlGhmHIMAx9/PHHkhqfc3Ps2DHde++9SkpKUkREhDIyMrR48eJ6fermD/3P//yPXnzxRfXo0UPh4eEaOnSoNm/efMnP3VT79u3T7bffrvj4eEVFRemqq67SP/7xjwb9nnnmGfXr109RUVHq0KGDrrzyynrBtKysTDNmzFBaWprCw8OVmJio66+/Xtu2bfNarYBdhFpdAADfmTx5shYvXqw333xT06dP97QXFxdr5cqVmjRpkiIjI7Vr1y4tW7ZMt99+u9LT01VQUKCFCxdq1KhR+uKLL9S5c+dmXfe+++7Ta6+9prvuuktZWVn68MMPNX78+Ab9Nm/erA0bNujOO+9Uly5ddODAAb3wwgsaPXq0vvjiC0VFRenqq6/WQw89pKefflq/+MUvdPnll0uS55/nO3XqlEaPHq29e/dq+vTpSk9P11tvvaV77rlHJSUlevjhh+v1X7JkicrKyvSDH/xAhmHoySef1K233qp9+/apXbt2zfrc5ysoKFBWVpZOnjyphx56SAkJCVq8eLFuvvlmLV26VN/5znckSX/605/00EMP6bbbbtPDDz+syspKff755/r000911113SZIeeOABLV26VNOnT1ffvn1VVFSkdevW6csvv9TgwYNbVSdgOyYA2zp9+rSZkpJiZmZm1mtfsGCBKclcuXKlaZqmWVlZadbW1tbrs3//fjM8PNx89NFH67VJMl9++WVP25w5c8xz/yrZvn27Kcl88MEH653vrrvuMiWZc+bM8bSdPHmyQc05OTmmJPOVV17xtL311lumJPOjjz5q0H/UqFHmqFGjPD/Pnz/flGS+9tprnrbq6mozMzPTjI6ONktLS+t9loSEBLO4uNjTd/ny5aYk8//+7/8aXOtcH330kSnJfOutty7YZ8aMGaYkc+3atZ62srIyMz093UxLS/N857fccovZr1+/i17P6XSa06ZNu2gfAGcwLAXYWEhIiO68807l5OTUG+pZsmSJkpKSdN1110mSwsPD5XCc+eugtrZWRUVFio6OVu/evZs97PHuu+9Kkh566KF67TNmzGjQNzIy0vPnmpoaFRUVqWfPnoqLi2vxcMu7776r5ORkTZo0ydPWrl07PfTQQyovL9eaNWvq9Z84caI6dOjg+XnkyJGSzgwntda7776rYcOGKTs729MWHR2t73//+zpw4IC++OILSVJcXJwOHz580eGwuLg4ffrppzpy5Eir6wLsjnAD2FzdhOG6+RuHDx/W2rVrdeeddyokJESS5Ha79Yc//EG9evVSeHi4OnbsqE6dOunzzz+Xy+Vq1vUOHjwoh8OhHj161Gvv3bt3g76nTp3S7NmzlZqaWu+6JSUlzb7uudfv1auXJ6zVqRvGOnjwYL32rl271vu5LuicOHGiRdc/v5bGPvf5tfznf/6noqOjNWzYMPXq1UvTpk3T+vXr6x3z5JNPaufOnUpNTdWwYcM0d+5crwQwwI4IN4DNDRkyRH369NHf/vY3SdLf/vY3maZZ7ymp3/72t5o5c6auvvpqvfbaa1q5cqVWrVqlfv36ye12+6y2H/3oR3rsscd0xx136M0339T777+vVatWKSEhwafXPVddwDufaZp+ub50Juzs3r1br7/+urKzs/X2228rOztbc+bM8fS54447tG/fPj3zzDPq3Lmzfve736lfv3765z//6bc6gWDBhGKgDZg8ebL++7//W59//rmWLFmiXr16aejQoZ73ly5dqmuuuUZ/+ctf6h1XUlKijh07Nuta3bp1k9vtVm5ubr27Frt3727Qd+nSpbr77rv11FNPedoqKytVUlJSr19TV0Cuu/7nn38ut9td7+7NV1995XnfX7p169bo526slvbt22vixImaOHGiqqurdeutt+qxxx7TrFmzPI/ap6Sk6MEHH9SDDz6oY8eOafDgwXrsscc0btw4/3wgIEhw5wZoA+ru0syePVvbt29vsLZNSEhIgzsVb731lr7++utmX6vuF+3TTz9dr33+/PkN+jZ23WeeeUa1tbX12tq3by9JDUJPY7797W8rPz9fb7zxhqft9OnTeuaZZxQdHa1Ro0Y15WN4xbe//W1t2rRJOTk5nraKigq9+OKLSktLU9++fSWpwQrRYWFh6tu3r0zTVE1NjWpraxsM0yUmJqpz586qqqry/QcBggx3boA2ID09XVlZWVq+fLkkNQg3N954ox599FFNnTpVWVlZ2rFjh/7617+qe/fuzb7WwIEDNWnSJD3//PNyuVzKysrS6tWrtXfv3gZ9b7zxRr366qtyOp3q27evcnJy9MEHHzRYMXngwIEKCQnRE088IZfLpfDwcF177bVKTExscM7vf//7Wrhwoe655x5t3bpVaWlpWrp0qdavX6/58+crJiam2Z/pYt5++23PnZhz3X333fr5z3+uv/3tbxo3bpweeughxcfHa/Hixdq/f7/efvttz52lG264QcnJyRoxYoSSkpL05Zdf6tlnn9X48eMVExOjkpISdenSRbfddpsyMjIUHR2tDz74QJs3b6531wvAWZY+qwXAb5577jlTkjls2LAG71VWVpo/+clPzJSUFDMyMtIcMWKEmZOT0+Ax66Y8Cm6apnnq1CnzoYceMhMSEsz27dubN910k5mXl9fgUfATJ06YU6dONTt27GhGR0ebY8eONb/66iuzW7du5t13313vnH/605/M7t27myEhIfUeCz+/RtM0zYKCAs95w8LCzP79+9er+dzP8rvf/a7B93F+nY2pexT8Qq+6x79zc3PN2267zYyLizMjIiLMYcOGmStWrKh3roULF5pXX321mZCQYIaHh5s9evQwf/rTn5oul8s0TdOsqqoyf/rTn5oZGRlmTEyM2b59ezMjI8N8/vnnL1oj0FYZpunHWXMAAAA+xpwbAABgK4QbAABgK4QbAABgK4QbAABgK4QbAABgK4QbAABgK21uET+3260jR44oJiamWUu6AwAA65imqbKyMnXu3LnBxrjna3Ph5siRI0pNTbW6DAAA0AJ5eXnq0qXLRfu0uXBTt/R6Xl6eYmNjLa4GAAA0RWlpqVJTU5u0hUqbCzd1Q1GxsbGEGwAAgkxTppQwoRgAANgK4QYAANgK4QYAANgK4QYAANgK4QYAANgK4QYAANgK4QYAANgK4QYAANgK4QYAANgK4QYAANgK4QYAANgK4QYAANgK4caLisqr9FV+qdVlAADQphFuvOT9Xfka8psP9J9LP7e6FAAA2jTCjZcM6BInSfr8a5dKTlZbWwwAAG0Y4cZLkp0R6pUYLdOUNuQWWV0OAABtFuHGi7J7dZQkrd1TaHElAAC0XYQbLxp5Ntys23vc4koAAGi7CDdeNDw9Qe1CDOUVn9LBogqrywEAoE0i3HhR+/BQDeraQRJDUwAAWIVw42Uje54dmiLcAABgCcKNl9VNKt6QW6hat2lxNQAAtD2EGy8b0CVOsRGhKq08rc8Pl1hdDgAAbQ7hxstCHIayejA0BQCAVQg3PuBZ72Yv4QYAAH8j3PhA3Xo3/zp0QhVVpy2uBgCAtoVw4wPdEtorNT5SNbWmPt3PVgwAAPgT4cZHsnt2ksR6NwAA+Bvhxkc8WzEQbgAA8CvCjY9k9UiQYUh7jpUr31VpdTkAALQZhBsfiYsK04DLnJKkdTw1BQCA3xBufCjbMzTFLuEAAPgL4caH6iYVr9tbJNNkKwYAAPyBcONDg7vFKbJdiArLq/RVfpnV5QAA0CYQbnwoPDREw7vHS+KpKQAA/IVw42PZPdmKAQAAfyLc+NjIXmfm3WzaX6TKmlqLqwEAwP4INz72raRoJcaEq7LGrW0HT1hdDgAAtke48THDMBiaAgDAjwg3fpDNVgwAAPgN4cYP6u7c7Dzi0omKaourAQDA3gg3fpAYG6HeSTEyTWl9LndvAADwJcKNnzA0BQCAfxBu/KQu3KzdU8hWDAAA+BDhxk+Gp8crLMShr0tO6UDRSavLAQDAtgg3fhIVFqrB3eIksUs4AAC+ZGm4mTdvnoYOHaqYmBglJiZqwoQJ2r1790WPWbRokQzDqPeKiIjwU8WtU7da8Vrm3QAA4DOWhps1a9Zo2rRp2rhxo1atWqWamhrdcMMNqqiouOhxsbGxOnr0qOd18OBBP1XcOnWPhOfkFul0rdviagAAsKdQKy/+3nvv1ft50aJFSkxM1NatW3X11Vdf8DjDMJScnOzr8rzuisuccka2k+tUjT477NKQbh2sLgkAANsJqDk3LpdLkhQfH3/RfuXl5erWrZtSU1N1yy23aNeuXf4or9VCHIZG9EyQxCPhAAD4SsCEG7fbrRkzZmjEiBG64oorLtivd+/eeumll7R8+XK99tprcrvdysrK0uHDhxvtX1VVpdLS0novK2X3PDPvZt1eJhUDAOALARNupk2bpp07d+r111+/aL/MzExNmTJFAwcO1KhRo/T3v/9dnTp10sKFCxvtP2/ePDmdTs8rNTXVF+U32ciz693861CJyqtOW1oLAAB2FBDhZvr06VqxYoU++ugjdenSpVnHtmvXToMGDdLevXsbfX/WrFlyuVyeV15enjdKbrHU+Ch1S4jSabepjblFltYCAIAdWRpuTNPU9OnT9c477+jDDz9Uenp6s89RW1urHTt2KCUlpdH3w8PDFRsbW+9ltbqnptbtZd4NAADeZmm4mTZtml577TUtWbJEMTExys/PV35+vk6dOuXpM2XKFM2aNcvz86OPPqr3339f+/bt07Zt2/Td735XBw8e1H333WfFR2iRkZ6tGJh3AwCAt1n6KPgLL7wgSRo9enS99pdffln33HOPJOnQoUNyOL7JYCdOnND999+v/Px8dejQQUOGDNGGDRvUt29ff5XdapndO8phSLnHK3TUdUopzkirSwIAwDYMs43t4lhaWiqn0ymXy2XpENUtz63XZ3klevK2AbrjSmsnOQMAEOia8/s7ICYUt0Uj6+bdsN4NAABeRbixSPbZeTfr9xbK7W5TN88AAPApwo1FBnftoKiwEBVVVOvLfGsXFgQAwE4INxYJC3VoePqZbSYYmgIAwHsINxbK7lW3FQPhBgAAbyHcWKhuvZtN+4tVWVNrcTUAANgD4cZCvRKjlRQbrqrTbm05cMLqcgAAsAXCjYUMw9CIs4+Er2WXcAAAvIJwY7G6oSkmFQMA4B2EG4vV3bnZdaRUReVVFlcDAEDwI9xYLDEmQn2SYyRJ63OLLK4GAIDgR7gJANmerRiYdwMAQGsRbgJA9jnzbtrYPqYAAHgd4SYADE9PUFiIQ0dcldpXWGF1OQAABDXCTQCIDAvRkG4dJPHUFAAArUW4CRB1Q1NrCTcAALQK4SZA1K13s3FfkWpq3RZXAwBA8CLcBIh+nZ2Ki2qn8qrT+iyvxOpyAAAIWoSbABHiMDSiB0NTAAC0FuEmgHgeCd9LuAEAoKUINwGkbjG/7XklKq2ssbgaAACCE+EmgKTGRyktIUq1blMb2YoBAIAWIdwEGIamAABoHcJNgMnu2UkSi/kBANBShJsAk9kjQQ5D2ldYoa9LTlldDgAAQYdwE2Ccke2UkRoniV3CAQBoCcJNABrZk/VuAABoKcJNAMrudWbezYbcIrndpsXVAAAQXAg3AWhQ1zi1DwtRcUW1vjhaanU5AAAEFcJNAGoX4tBV3RMkMTQFAEBzEW4C1Dfr3TCpGACA5iDcBKiRZ8PN5gMnVFlTa3E1AAAED8JNgOrRKVrJsRGqPu3Wpv3FVpcDAEDQINwEKMMw2IoBAIAWINwEsLqhKSYVAwDQdISbADbi7GJ+Xx4t1fGyKourAQAgOBBuAljH6HBdnhIrSdqQy90bAACagnAT4BiaAgCgeQg3AS777NDUuj2FMk22YgAA4FIINwFuWHq8wkIdyi+tVO7xcqvLAQAg4BFuAlxEuxANTesgiaEpAACagnATBLJ7ntklfB3hBgCASyLcBIG6ScUb9xWpptZtcTUAAAQ2wk0Q6JsSq/j2YaqortW/DpVYXQ4AAAGNcBMEHA5DWT0SJEnr9rBLOAAAF0O4CRKe9W7YZwoAgIsi3ASJ7F5nJhV/llci16kai6sBACBwEW6CxGVxkeresb3cppSTW2R1OQAABCzCTRDJPjs0tW4v824AALgQS8PNvHnzNHToUMXExCgxMVETJkzQ7t27L3ncW2+9pT59+igiIkL9+/fXu+++64dqrXfuVgwAAKBxloabNWvWaNq0adq4caNWrVqlmpoa3XDDDaqoqLjgMRs2bNCkSZN077336l//+pcmTJigCRMmaOfOnX6s3BpX9UhQiMPQgaKTyis+aXU5AAAEJMMMoN0Yjx8/rsTERK1Zs0ZXX311o30mTpyoiooKrVixwtN21VVXaeDAgVqwYMElr1FaWiqn0ymXy6XY2Fiv1e4v/98LG7T14AnNu7W/Jg3ranU5AAD4RXN+fwfUnBuXyyVJio+Pv2CfnJwcjRkzpl7b2LFjlZOT49PaAgVDUwAAXFzAhBu3260ZM2ZoxIgRuuKKKy7YLz8/X0lJSfXakpKSlJ+f32j/qqoqlZaW1nsFs7r1btbnFqrWHTA33QAACBgBE26mTZumnTt36vXXX/fqeefNmyen0+l5paamevX8/paRGqfo8FCVnKzRriMuq8sBACDgBES4mT59ulasWKGPPvpIXbp0uWjf5ORkFRQU1GsrKChQcnJyo/1nzZoll8vleeXl5Xmtbiu0C3Hoqu5ntmJYy9AUAAANWBpuTNPU9OnT9c477+jDDz9Uenr6JY/JzMzU6tWr67WtWrVKmZmZjfYPDw9XbGxsvVewqxuaYt4NAAANhVp58WnTpmnJkiVavny5YmJiPPNmnE6nIiMjJUlTpkzRZZddpnnz5kmSHn74YY0aNUpPPfWUxo8fr9dff11btmzRiy++aNnn8Le6xfy2HjyhU9W1igwLsbgiAAACh6V3bl544QW5XC6NHj1aKSkpntcbb7zh6XPo0CEdPXrU83NWVpaWLFmiF198URkZGVq6dKmWLVt20UnIdtO9Y3t1dkaoutatT/ezFQMAAOcKqHVu/CHY17mp87Oln+nNLYd1X3a6fnljX6vLAQDAp4J2nRs0Xd0u4ev2Mu8GAIBzEW6C1IgeZ56Y+iq/TMfKKi2uBgCAwEG4CVIJ0eHq1/nMbbn13L0BAMCDcBPE6p6aYr0bAAC+QbgJYiN7np13s6dQbWxeOAAAF0S4CWJXpnVQeKhDx8qqtOdYudXlAAAQEAg3QSyiXYiGpZ/ZQZ2hKQAAziDcBLnsnnVbMRy3uBIAAAID4SbI1U0q/nR/sapPuy2uBgAA6xFugtzlybFKaB+mk9W12nbohNXlAABgOcJNkHM4DI3oyS7hAADUIdzYgGe9GxbzAwCAcGMHI8+Gmx2HS+Q6WWNxNQAAWItwYwMpzkj16NReblPakMvdGwBA20a4sYmRZ3cJZ2gKANDWEW5sIptJxQAASCLc2MZVPRIU6jB0qPikDhWdtLocAAAsQ7ixiejwUA3qGidJWruX1YoBAG0X4cZGss/ZJRwAgLaKcGMjdevdbMgtUq3btLgaAACsQbixkYwuTsVEhMp1qkY7vnZZXQ4AAJYg3NhIaIhDmd0TJLFLOACg7SLc2EzdasVrmXcDAGijCDc2k312Mb9th06oouq0xdUAAOB/hBubSUuI0mVxkaqpNbVpf7HV5QAA4HeEG5sxDIOhKQBAm0a4saG6R8LXsZgfAKANItzY0IgeHWUY0r8LylVQWml1OQAA+BXhxoY6tA/TFZ2dklitGADQ9hBubOqboSnCDQCgbSHc2NTInt+EG9NkKwYAQNtBuLGpIWkdFNHOoeNlVdpdUGZ1OQAA+A3hxqbCQ0M0LL1uKwaGpgAAbQfhxsbqhqZY7wYA0JYQbmysblLxp/uLVHW61uJqAADwD8KNjfVJjlHH6HBV1ri19eAJq8sBAMAvCDc2ZhiGsnsy7wYA0LYQbmyubpdw1rsBALQVhBubyz47qXjH1y6dqKi2uBoAAHyPcGNzyc4I9UqMlmlKG3KLrC4HAACfI9y0AWzFAABoSwg3bcBIT7g5bnElAAD4HuGmDRienqB2IYbyik/pYFGF1eUAAOBThJs2oH14qAZ17SCJ1YoBAPZHuGkjPLuEE24AADZHuGkj6iYVb8gtVK3btLgaAAB8h3DTRgzoEqfYiFCVVp7W54dLrC4HAACfIdy0ESEOQ1k9GJoCANgf4aYNqRuaWst6NwAAG7M03HzyySe66aab1LlzZxmGoWXLll20/8cffyzDMBq88vPz/VNwkKtb7+Zfh06oouq0xdUAAOAbloabiooKZWRk6LnnnmvWcbt379bRo0c9r8TERB9VaC/dEtorNT5SNbWmPt3PVgwAAHsKtfLi48aN07hx45p9XGJiouLi4rxfUBuQ3bOT/rbpkNbuKdS1fZKsLgcAAK8Lyjk3AwcOVEpKiq6//nqtX7/e6nKCimcrBiYVAwBsqkXhJi8vT4cPH/b8vGnTJs2YMUMvvvii1wprTEpKihYsWKC3335bb7/9tlJTUzV69Ght27btgsdUVVWptLS03qsty+qRIMOQ9hwrV76r0upyAADwuhaFm7vuuksfffSRJCk/P1/XX3+9Nm3apP/6r//So48+6tUCz9W7d2/94Ac/0JAhQ5SVlaWXXnpJWVlZ+sMf/nDBY+bNmyen0+l5paam+qy+YBAXFaYBlzklsUs4AMCeWhRudu7cqWHDhkmS3nzzTV1xxRXasGGD/vrXv2rRokXerO+Shg0bpr17917w/VmzZsnlcnleeXl5fqwuMGV7hqbYJRwAYD8tCjc1NTUKDw+XJH3wwQe6+eabJUl9+vTR0aNHvVddE2zfvl0pKSkXfD88PFyxsbH1Xm1dds9OkqR1e4tkmmzFAACwlxY9LdWvXz8tWLBA48eP16pVq/TrX/9aknTkyBElJCQ0+Tzl5eX17rrs379f27dvV3x8vLp27apZs2bp66+/1iuvvCJJmj9/vtLT09WvXz9VVlbqz3/+sz788EO9//77LfkYbdbgbnGKbBeiwvIqfZVfpstTCHwAAPto0Z2bJ554QgsXLtTo0aM1adIkZWRkSJL+93//1zNc1RRbtmzRoEGDNGjQIEnSzJkzNWjQIM2ePVuSdPToUR06dMjTv7q6Wj/5yU/Uv39/jRo1Sp999pk++OADXXfddS35GG1WeGiIhnePl8RTUwAA+zHMFo5L1NbWqrS0VB06dPC0HThwQFFRUQG9qF5paamcTqdcLlebHqL689p9+s0/vtTV3+qkV/7/pgdSAACs0Jzf3y26c3Pq1ClVVVV5gs3Bgwc1f/587d69O6CDDb4xsteZeTeb9hepsqbW4moAAPCeFoWbW265xTMPpqSkRMOHD9dTTz2lCRMm6IUXXvBqgfCNbyVFKzEmXJU1bm07eMLqcgAA8JoWhZtt27Zp5MiRkqSlS5cqKSlJBw8e1CuvvKKnn37aqwXCNwzDUHZPdgkHANhPi8LNyZMnFRMTI0l6//33deutt8rhcOiqq67SwYMHvVogfCebrRgAADbUonDTs2dPLVu2THl5eVq5cqVuuOEGSdKxY8fa9CTdYFN352bnEZdOVFRbXA0AAN7RonAze/ZsPfLII0pLS9OwYcOUmZkp6cxdnLrHuhH4EmMj1DspRqYprc/l7g0AwB5aFG5uu+02HTp0SFu2bNHKlSs97dddd91F93lC4GFoCgBgNy0KN5KUnJysQYMG6ciRI54dwocNG6Y+ffp4rTj4Xl24WbunkK0YAAC20KJw43a79eijj8rpdKpbt27q1q2b4uLi9Otf/1put9vbNcKHhqfHKyzEoa9LTulA0UmrywEAoNVatLfUf/3Xf+kvf/mLHn/8cY0YMUKStG7dOs2dO1eVlZV67LHHvFokfCcqLFSDu8Vp475irdtzXOkd21tdEgAArdKicLN48WL9+c9/9uwGLkkDBgzQZZddpgcffJBwE2RG9uqkjfuKtXZPob6XmWZ1OQAAtEqLhqWKi4sbnVvTp08fFRcXt7oo+FfdI+E5uUU6XcuwIgAguLUo3GRkZOjZZ59t0P7ss89qwIABrS4K/nXFZU45I9uprOq0PjvssrocAABapUXDUk8++aTGjx+vDz74wLPGTU5OjvLy8vTuu+96tUD4XojD0IieCXp3R77W7SnUkG4dLn0QAAABqkV3bkaNGqV///vf+s53vqOSkhKVlJTo1ltv1a5du/Tqq696u0b4QXbPM7uEr9t73OJKAABoHcP04uImn332mQYPHqza2lpvndLrSktL5XQ65XK52CriHHnFJzXyyY8U6jC0fc4Nig5v0U09AAB8ojm/v1u8iB/sJTU+St0SonTabWpjbpHV5QAA0GKEG3jUPTW1bi9bMQAAghfhBh4jPVsxMO8GABC8mjWx4tZbb73o+yUlJa2pBRbL7NFRDkPKPV6ho65TSnFGWl0SAADN1qxw43Q6L/n+lClTWlUQrOOMbKcBXeK0Pa9Ea/cU6o4rU60uCQCAZmtWuHn55Zd9VQcCxMheHbU9r0TrCDcAgCDFnBvUUzepeP3eQrndXlslAAAAvyHcoJ5BXTsoKixERRXV+jK/1OpyAABoNsIN6gkLdeiq7gmSpHV7eCQcABB8CDdogPVuAADBjHCDBurWu9m0v1iVNYG7lQYAAI0h3KCBnonRSooNV9Vpt7YcOGF1OQAANAvhBg0YhuHZJXwtu4QDAIIM4QaNqhuaYlIxACDYEG7QqBFnJxXvOlKqovIqi6sBAKDpCDdoVKeYcPVJjpEkrc8tsrgaAACajnCDC/pmaIp5NwCA4EG4wQVl9zozqXjdnkKZJlsxAACCA+EGFzQsLV5hIQ4dcVVqX2GF1eUAANAkhBtcUGRYiK5M6yCJp6YAAMGDcIOLyj4772Yt4QYAECQIN7iokWcX89u4r0g1tW6LqwEA4NIIN7iofp1j1SGqncqrTuuzvBKrywEA4JIIN7goh8NQVk+GpgAAwYNwg0saeTbcrNtLuAEABD7CDS6pblLx9rwSlVbWWFwNAAAXR7jBJXXpEKX0ju1V6za1ka0YAAABjnCDJslmaAoAECQIN2iSbM8+U4QbAEBgI9ygSTJ7JCjEYWhfYYW+LjlldTkAAFwQ4QZNEhvRThldnJLYJRwAENgIN2iyul3CWe8GABDICDdospFn591syC2S221aXA0AAI2zNNx88sknuummm9S5c2cZhqFly5Zd8piPP/5YgwcPVnh4uHr27KlFixb5vE6cMTA1TtHhoSquqNYXR0utLgcAgEZZGm4qKiqUkZGh5557rkn99+/fr/Hjx+uaa67R9u3bNWPGDN13331auXKljyuFJLULceiq7vGSGJoCAASuUCsvPm7cOI0bN67J/RcsWKD09HQ99dRTkqTLL79c69at0x/+8AeNHTvWV2XiHNk9O+qDL49p3d7j+uHoHlaXAwBAA0E15yYnJ0djxoyp1zZ27Fjl5ORYVFHbUzepePOBE6qsqbW4GgAAGgqqcJOfn6+kpKR6bUlJSSotLdWpU42vvVJVVaXS0tJ6L7Rcj07tleKMUPVptzbtL7a6HAAAGgiqcNMS8+bNk9Pp9LxSU1OtLimoGYbBVgwAgIAWVOEmOTlZBQUF9doKCgoUGxuryMjIRo+ZNWuWXC6X55WXl+ePUm2tbisGJhUDAAKRpROKmyszM1PvvvtuvbZVq1YpMzPzgseEh4crPDzc16W1KSPO3rn58mipjpdVqVMM3y8AIHBYeuemvLxc27dv1/bt2yWdedR7+/btOnTokKQzd12mTJni6f/AAw9o3759+tnPfqavvvpKzz//vN588039+Mc/tqL8NqtjdLj6psRKkjbkcvcGABBYLA03W7Zs0aBBgzRo0CBJ0syZMzVo0CDNnj1bknT06FFP0JGk9PR0/eMf/9CqVauUkZGhp556Sn/+8595DNwCIxmaAgAEKMM0zTa1jn5paamcTqdcLpdiY2OtLidord1zXN/7yyYlx0YoZ9a1MgzD6pIAADbWnN/fQTWhGIFjaFq8wkIdyi+tVO7xcqvLAQDAg3CDFoloF6JhaWzFAAAIPIQbtFjdI+HrCDcAgABCuEGL1S3mt3FfkWpq3RZXAwDAGYQbtFjflFgltA9TRXWt/nWoxOpyAACQRLhBKzgchrLqtmLYc9ziagAAOINwg1YZeTbcrGWfKQBAgCDcoFXqJhV/llci16kai6sBAIBwg1bqHBep7p3ay21KOblFVpcDAADhBq1XNzS1bi/zbgAA1iPcoNWye3WSxHo3AIDAQLhBq13VPV4hDkMHik4qr/ik1eUAANo4wg1aLSainQalxkmS1vHUFADAYoQbeAVbMQAAAgXhBl4x8my4WZ9bqFq3aXE1AIC2jHADr8joEqeY8FCVnKzRriMuq8sBALRhhBt4RWiIQ1f1SJAkrWVoCgBgIcINvGYk824AAAGAcAOvyT67mN/Wgyd0qrrW4moAAG0V4QZek96xvS6Li1R1rVuf7mcrBgCANQg38BrDMDx3bxiaAgBYhXADr/Ksd8NifgAAixBu4FUjenaUYUhf5ZfpWFml1eUAANogwg28Kr59mPp1jpUkrefuDQDAAoQbeF12zzO7hLPeDQDACoQbeN25692YJlsxAAD8i3ADrxvSrYPCQx06VlalPcfKrS4HANDGEG7gdRHtQjQsPV4SQ1MAAP8j3MAnvhmaOm5xJQCAtoZwA5+om1T86f5iVZ92W1wNAKAtIdzAJ/okx6hjdJhOVtdq26ETVpcDAGhDCDfwCYfD0Ai2YgAAWIBwA5+p22dqLYv5AQD8iHADnxnZ68y8mx2HS+Q6WWNxNQCAtoJwA59JdkaoZ2K03Ka0IZe7NwAA/yDcwKcYmgIA+BvhBj517lYMAAD4A+EGPjW8e4JCHYYOFZ/UoaKTVpcDAGgDCDfwqejwUA3u2kGStHYvqxUDAHyPcAOfy2ZoCgDgR4Qb+FxduNmQW6Rat2lxNQAAuyPcwOcGXOZUTESoXKdqtONrl9XlAABsjnADnwsNcSirR4IkdgkHAPge4QZ+kX12teK1zLsBAPgY4QZ+MfLsYn7bDp1QRdVpi6sBANgZ4QZ+0S0hSl06RKqm1tSm/cVWlwMAsDHCDfzCMAzPasUMTQEAfIlwA7/J7nlm3s06FvMDAPgQ4QZ+k9UjQYYh/bugXAWllVaXAwCwqYAIN88995zS0tIUERGh4cOHa9OmTRfsu2jRIhmGUe8VERHhx2rRUh3ah6n/ZU5JrFYMAPAdy8PNG2+8oZkzZ2rOnDnatm2bMjIyNHbsWB07duyCx8TGxuro0aOe18GDB/1YMVoj++xTU+v2Em4AAL5hebj5/e9/r/vvv19Tp05V3759tWDBAkVFRemll1664DGGYSg5OdnzSkpK8mPFaA3PPlN7C2WabMUAAPA+S8NNdXW1tm7dqjFjxnjaHA6HxowZo5ycnAseV15erm7duik1NVW33HKLdu3a5Y9y4QVDunVQZLsQHS+r0u6CMqvLAQDYkKXhprCwULW1tQ3uvCQlJSk/P7/RY3r37q2XXnpJy5cv12uvvSa3262srCwdPny40f5VVVUqLS2t94J1wkNDNCw9XhLzbgAAvmH5sFRzZWZmasqUKRo4cKBGjRqlv//97+rUqZMWLlzYaP958+bJ6XR6XqmpqX6uGOdjvRsAgC9ZGm46duyokJAQFRQU1GsvKChQcnJyk87Rrl07DRo0SHv37m30/VmzZsnlcnleeXl5ra4brVM37+bT/UWqOl1rcTUAALuxNNyEhYVpyJAhWr16tafN7XZr9erVyszMbNI5amtrtWPHDqWkpDT6fnh4uGJjY+u9YK3eSTHqFBOuyhq3th48YXU5AACbsXxYaubMmfrTn/6kxYsX68svv9QPf/hDVVRUaOrUqZKkKVOmaNasWZ7+jz76qN5//33t27dP27Zt03e/+10dPHhQ9913n1UfAc1kGMY3j4QzNAUA8LJQqwuYOHGijh8/rtmzZys/P18DBw7Ue++955lkfOjQITkc32SwEydO6P7771d+fr46dOigIUOGaMOGDerbt69VHwEtkN2zo97519dat7dQP7O6GACArRhmG1tspLS0VE6nUy6XiyEqCxWUVmr4b1fLMKRtv7xeHdqHWV0SACCANef3t+XDUmibkmIj9K2kaJmmtCG3yOpyAAA2QriBZdglHADgC4QbWObc9W7a2OgoAMCHCDewzPDu8WoXYujwiVM6WHTS6nIAADZBuIFlosJCNbhrB0nSWnYJBwB4CeEGlqobmlq3h3k3AADvINzAUtm9zkwq3pBbpNO1bourAQDYAeEGlup/mVPOyHYqqzytz792WV0OAMAGCDewVIjDUFaPBElsxQAA8A7CDSyX3Yt9pgAA3kO4geVGnl3Mb9uhEyqvOm1xNQCAYEe4geW6JkSpa3yUTrtNfbqPrRgAAK1DuEFAyD5ntWIAAFqDcIOAMLLn2Xk3LOYHAGglwg0CQlaPjnIY0t5j5TrqOmV1OQCAIEa4QUBwRrVT/y5xknhqCgDQOoQbBAyGpgAA3kC4QcCom1S8fm+h3G7T4moAAMGKcIOAMbhrB0WFhaiwvFpf5ZdZXQ4AIEgRbhAwwkIdGp4eL0lat5ddwgEALUO4QUCp2yWc9W4AAC1FuEFAGXl23s2m/cWqrKm1uBoAQDAi3CCg9EqMVlJsuKpOu7X14AmrywEABCHCDQKKYRga0ZOtGAAALUe4QcCpG5piUjEAoCUINwg4dXdudh0pVXFFtcXVAACCDeEGAScxJkJ9kmNkmmcW9AMAoDkINwhI2XVbMTDvBgDQTIQbBKTsXt/sM2WabMUAAGg6wg0C0vD0BIWFOPR1ySntL6ywuhwAQBAh3CAgRYaFaEi3DpLYJRwA0DyEGwSsuqEp1rsBADQH4QYBq269m425RTpd67a4GgBAsCDcIGD16+xUXFQ7lVWd1meHS6wuBwAQJAg3CFghDkMjejA0BQBoHsINAprnkXDCDQCgiQg3CGh1i/n9K69EZZU1FlcDAAgGhBsEtNT4KKUlRKnWbWrjvmKrywEABAHCDQLeN0NT7BIOALg0wg0CXnbPTpKktSzmBwBoAsINAl5mjwQ5DGnf8QodKTlldTkAgABHuEHAc0a2U0ZqnCSemgIAXBrhBkFh5NmnphiaAgBcCuEGQSG715l5N+v3FsrtNi2uBgAQyAg3CAqDusapfViIiiuq9cXRUqvLAQAEMMINgkK7EIeu6p4gSVrH0BQA4CIINwgabMUAAGgKwg2Cxsiz4WbTgWJV1tRaXA0AIFARbhA0enSKVnJshKpPu7X5AFsxAAAaFxDh5rnnnlNaWpoiIiI0fPhwbdq06aL933rrLfXp00cRERHq37+/3n33XT9VCisZhsHQFADgkkKtLuCNN97QzJkztWDBAg0fPlzz58/X2LFjtXv3biUmJjbov2HDBk2aNEnz5s3TjTfeqCVLlmjChAnatm2brrjiCgs+AfxpZK+OWrr1sFZ8flThoQ45HIZCDEMOhyGHYSjEobP/PPNyGE1v95zDMORwSCEXaj//mk1o91zDMOQwzgQ1AIBvGKZpWrpoyPDhwzV06FA9++yzkiS3263U1FT96Ec/0s9//vMG/SdOnKiKigqtWLHC03bVVVdp4MCBWrBgwSWvV1paKqfTKZfLpdjYWO99EPhFYXmVhj32gYJ9qRuH0XjoOTd8edrPBq26IBfiMGScDWbntp8JVfXPG2Kc07cJ7fVC39mfDUmGIRkyzv7zbIPU6HvGeeGt0fdl6JzTyKh3nW+OP/d651/r/OvrnGONeuc+75jz3z+vNp37Oc+v7bxrNaz93O+osfOff62G17uUpgbjpp+vif2aeEZv53Z/1ddY+4XOecFzNOuarT/3hf5baE4dzf2MF3J+/7BQhxJjIpp3kktozu9vS+/cVFdXa+vWrZo1a5anzeFwaMyYMcrJyWn0mJycHM2cObNe29ixY7Vs2bJG+1dVVamqqsrzc2kpa6QEs47R4Xr2rsHatL9YbtNUrdv0/LPWLZmmqdomtrvdOvNn05TbXfe+5K7rc067260z5zLNM+dy152rYXtTgpfblNy1pqQgT2kA0IjBXeP09wdHWHZ9S8NNYWGhamtrlZSUVK89KSlJX331VaPH5OfnN9o/Pz+/0f7z5s3Tr371K+8UjIDw7f4p+nb/FKvLuCDT/Cb0uM36YercgNXcds/7Z9vdZwPWN8Gqae3fXEPnBMDG292mqbp7u6ZpypS++Vln3vum7ezPF3jf1JkfTDU8n3nOzzr/eLPxa6mRY8/9+Zt/H42fy3OWBuc3z6mxro9Z73N8853Uv15j5z/3czV+/m/6N+2/r6b3ba7m3MhvThnNrdlsxtmb9d016xyN977Q9S587obvNK+O5p3jQm801nyhf9/Nrq+RI8JCrZ3Sa/mcG1+bNWtWvTs9paWlSk1NtbAi2J1hGAo5OzwEAPA/S8NNx44dFRISooKCgnrtBQUFSk5ObvSY5OTkZvUPDw9XeHi4dwoGAAABz9L7RmFhYRoyZIhWr17taXO73Vq9erUyMzMbPSYzM7Nef0latWrVBfsDAIC2xfJhqZkzZ+ruu+/WlVdeqWHDhmn+/PmqqKjQ1KlTJUlTpkzRZZddpnnz5kmSHn74YY0aNUpPPfWUxo8fr9dff11btmzRiy++aOXHAAAAAcLycDNx4kQdP35cs2fPVn5+vgYOHKj33nvPM2n40KFDcji+ucGUlZWlJUuW6Je//KV+8YtfqFevXlq2bBlr3AAAAEkBsM6Nv7HODQAAwac5v78DYvsFAAAAbyHcAAAAWyHcAAAAWyHcAAAAWyHcAAAAWyHcAAAAWyHcAAAAWyHcAAAAWyHcAAAAW7F8+wV/q1uQubS01OJKAABAU9X93m7KxgptLtyUlZVJklJTUy2uBAAANFdZWZmcTudF+7S5vaXcbreOHDmimJgYGYbh1XOXlpYqNTVVeXl57FvlQ3zP/sH37B98z/7Dd+0fvvqeTdNUWVmZOnfuXG9D7ca0uTs3DodDXbp08ek1YmNj+R+OH/A9+wffs3/wPfsP37V/+OJ7vtQdmzpMKAYAALZCuAEAALZCuPGi8PBwzZkzR+Hh4VaXYmt8z/7B9+wffM/+w3ftH4HwPbe5CcUAAMDeuHMDAABshXADAABshXADAABshXADAABshXDjJc8995zS0tIUERGh4cOHa9OmTVaXZDuffPKJbrrpJnXu3FmGYWjZsmVWl2RL8+bN09ChQxUTE6PExERNmDBBu3fvtros23nhhRc0YMAAz0JnmZmZ+uc//2l1Wbb3+OOPyzAMzZgxw+pSbGXu3LkyDKPeq0+fPpbVQ7jxgjfeeEMzZ87UnDlztG3bNmVkZGjs2LE6duyY1aXZSkVFhTIyMvTcc89ZXYqtrVmzRtOmTdPGjRu1atUq1dTU6IYbblBFRYXVpdlKly5d9Pjjj2vr1q3asmWLrr32Wt1yyy3atWuX1aXZ1ubNm7Vw4UINGDDA6lJsqV+/fjp69KjntW7dOstq4VFwLxg+fLiGDh2qZ599VtKZ/atSU1P1ox/9SD//+c8trs6eDMPQO++8owkTJlhdiu0dP35ciYmJWrNmja6++mqry7G1+Ph4/e53v9O9995rdSm2U15ersGDB+v555/Xb37zGw0cOFDz58+3uizbmDt3rpYtW6bt27dbXYok7ty0WnV1tbZu3aoxY8Z42hwOh8aMGaOcnBwLKwO8w+VySTrzixe+UVtbq9dff10VFRXKzMy0uhxbmjZtmsaPH1/v72p41549e9S5c2d1795dkydP1qFDhyyrpc1tnOlthYWFqq2tVVJSUr32pKQkffXVVxZVBXiH2+3WjBkzNGLECF1xxRVWl2M7O3bsUGZmpiorKxUdHa133nlHffv2tbos23n99de1bds2bd682epSbGv48OFatGiRevfuraNHj+pXv/qVRo4cqZ07dyomJsbv9RBuAFzQtGnTtHPnTkvHzu2sd+/e2r59u1wul5YuXaq7775ba9asIeB4UV5enh5++GGtWrVKERERVpdjW+PGjfP8ecCAARo+fLi6deumN99805JhVsJNK3Xs2FEhISEqKCio115QUKDk5GSLqgJab/r06VqxYoU++eQTdenSxepybCksLEw9e/aUJA0ZMkSbN2/WH//4Ry1cuNDiyuxj69atOnbsmAYPHuxpq62t1SeffKJnn31WVVVVCgkJsbBCe4qLi9O3vvUt7d2715LrM+emlcLCwjRkyBCtXr3a0+Z2u7V69WrGzhGUTNPU9OnT9c477+jDDz9Uenq61SW1GW63W1VVVVaXYSvXXXedduzYoe3bt3teV155pSZPnqzt27cTbHykvLxcubm5SklJseT63LnxgpkzZ+ruu+/WlVdeqWHDhmn+/PmqqKjQ1KlTrS7NVsrLy+v9v4D9+/dr+/btio+PV9euXS2szF6mTZumJUuWaPny5YqJiVF+fr4kyel0KjIy0uLq7GPWrFkaN26cunbtqrKyMi1ZskQff/yxVq5caXVpthITE9Ngvlj79u2VkJDAPDIveuSRR3TTTTepW7duOnLkiObMmaOQkBBNmjTJknoIN14wceJEHT9+XLNnz1Z+fr4GDhyo9957r8EkY7TOli1bdM0113h+njlzpiTp7rvv1qJFiyyqyn5eeOEFSdLo0aPrtb/88su65557/F+QTR07dkxTpkzR0aNH5XQ6NWDAAK1cuVLXX3+91aUBzXb48GFNmjRJRUVF6tSpk7Kzs7Vx40Z16tTJknpY5wYAANgKc24AAICtEG4AAICtEG4AAICtEG4AAICtEG4AAICtEG4AAICtEG4AAICtEG4AtDlpaWmaP3++1WUA8BHCDQCfuueeezRhwgRJZ1Y9njFjht+uvWjRIsXFxTVo37x5s77//e/7rQ4A/sX2CwCCTnV1tcLCwlp8vFVLwgPwD+7cAPCLe+65R2vWrNEf//hHGYYhwzB04MABSdLOnTs1btw4RUdHKykpSd/73vdUWFjoOXb06NGaPn26ZsyYoY4dO2rs2LGSpN///vfq37+/2rdvr9TUVD344IMqLy+XJH388ceaOnWqXC6X53pz586V1HBY6tChQ7rlllsUHR2t2NhY3XHHHSooKPC8P3fuXA0cOFCvvvqq0tLS5HQ6deedd6qsrMy3XxqAFiHcAPCLP/7xj8rMzNT999+vo0eP6ujRo0pNTVVJSYmuvfZaDRo0SFu2bNF7772ngoIC3XHHHfWOX7x4scLCwrR+/XotWLBAkuRwOPT0009r165dWrx4sT788EP97Gc/kyRlZWVp/vz5io2N9VzvkUceaVCX2+3WLbfcouLiYq1Zs0arVq3Svn37NHHixHr9cnNztWzZMq1YsUIrVqzQmjVr9Pjjj/vo2wLQGgxLAfALp9OpsLAwRUVFKTk52dP+7LPPatCgQfrtb3/raXvppZeUmpqqf//73/rWt74lSerVq5eefPLJeuc8d/5OWlqafvOb3+iBBx7Q888/r7CwMDmdThmGUe9651u9erV27Nih/fv3KzU1VZL0yiuvqF+/ftq8ebOGDh0q6UwIWrRokWJiYiRJ3/ve97R69Wo99thjrftiAHgdd24AWOqzzz7TRx99pOjoaM+rT58+ks7cLakzZMiQBsd+8MEHuu6663TZZZcpJiZG3/ve91RUVKSTJ082+fpffvmlUlNTPcFGkvr27au4uDh9+eWXnra0tDRPsJGklJQUHTt2rFmfFYB/cOcGgKXKy8t100036YknnmjwXkpKiufP7du3r/fegQMHdOONN+qHP/yhHnvsMcXHx2vdunW69957VV1draioKK/W2a5du3o/G4Yht9vt1WsA8A7CDQC/CQsLU21tbb22wYMH6+2331ZaWppCQ5v+V9LWrVvldrv11FNPyeE4cxP6zTffvOT1znf55ZcrLy9PeXl5nrs3X3zxhUpKStS3b98m1wMgcDAsBcBv0tLS9Omnn+rAgQMqLCyU2+3WtGnTVFxcrEmTJmnz5s3Kzc3VypUrNXXq1IsGk549e6qmpkbPPPOM9u3bp1dffdUz0fjc65WXl2v16tUqLCxsdLhqzJgx6t+/vyZPnqxt27Zp06ZNmjJlikaNGqUrr7zS698BAN8j3ADwm0ceeUQhISHq27evOnXqpEOHDqlz585av369amtrdcMNN6h///6aMWOG4uLiPHdkGpORkaHf//73euKJJ3TFFVfor3/9q+bNm1evT1ZWlh544AFNnDhRnTp1ajAhWTozvLR8+XJ16NBBV199tcaMGaPu3bvrjTfe8PrnB+AfhmmaptVFAAAAeAt3bgAAgK0QbgAAgK0QbgAAgK0QbgAAgK0QbgAAgK0QbgAAgK0QbgAAgK0QbgAAgK0QbgAAgK0QbgAAgK0QbgAAgK0QbgAAgK38P8P57L08aaEnAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(val_losses)\n",
    "plt.xlabel(xlabel=\"Iteration\")\n",
    "plt.ylabel(ylabel=\"Loss\")\n",
    "plt.title(\"Validation Loss\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model_response(prompt):\n",
    "    context = torch.tensor(encode(prompt), device=config.device)\n",
    "    context = context.unsqueeze(0)\n",
    "\n",
    "    generated = model.generate(context, max_new_tokens=70)[0].tolist()\n",
    "    decoded = decode(generated)\n",
    "    return decoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'6+9;6+7+6=13(1);6+1=7(0);6+0=6;673<|endoftext|>'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_model_response(\"6+9;\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(model: GPT, num_to_run=1000, verbose=False):\n",
    "    score = 0\n",
    "\n",
    "    goods, wrongs = [], []\n",
    "\n",
    "    for _ in tqdm(range(num_to_run)):\n",
    "        result = {}\n",
    "\n",
    "        sample, prompt_idx = generate_sample(low=data_config.test_low, high=data_config.test_high)\n",
    "\n",
    "        prompt = extract_prompt_from_sample(sample, prompt_idx)\n",
    "\n",
    "        result['prompt'] = prompt\n",
    "\n",
    "        # generate from the model\n",
    "        context = torch.tensor(encode(prompt), device=config.device)\n",
    "        context = context.unsqueeze(0)\n",
    "\n",
    "        generated = model.generate(context, max_new_tokens=70)[0].tolist()\n",
    "        decoded = decode(generated)\n",
    "\n",
    "        if verbose:\n",
    "            print(f\"prompt: {prompt}\")\n",
    "            print(f\"decoded: {decoded}\")\n",
    "            print(f\"sample: {sample}\")\n",
    "            print(f\"correct: {correct_answer(sample, decoded)}\")\n",
    "            print()\n",
    "\n",
    "        result['decoded'] = decoded\n",
    "\n",
    "        correct = correct_answer(sample, decoded)\n",
    "\n",
    "        result['correct'] = correct\n",
    "\n",
    "        if correct:\n",
    "            score += 1\n",
    "            goods.append(result)\n",
    "        else:\n",
    "            wrongs.append(result)\n",
    "\n",
    "    return score / num_to_run, goods, wrongs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:23<00:00, 43.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 80.0%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "score, goods, wrongs = accuracy(model, num_to_run=1000, verbose=False)\n",
    "print(f\"Accuracy: {score*100}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800\n",
      "[\n",
      "  {\n",
      "    \"prompt\": \"139+685;\",\n",
      "    \"decoded\": \"139+685;9+5=14(1);3+8+1=12(1);1+6+1=8;824<|endoftext|>\",\n",
      "    \"correct\": true\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"275+954;\",\n",
      "    \"decoded\": \"275+954;5+4=9(0);7+5=12(1);2+9+1=12;1229<|endoftext|>\",\n",
      "    \"correct\": true\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"499+889;\",\n",
      "    \"decoded\": \"499+889;9+9=18(1);9+8+1=18(1);4+8+1=13;1388<|endoftext|>\",\n",
      "    \"correct\": true\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"815+990;\",\n",
      "    \"decoded\": \"815+990;5+0=5(0);1+9=10(1);8+9+1=18;1805<|endoftext|>\",\n",
      "    \"correct\": true\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"527+229;\",\n",
      "    \"decoded\": \"527+229;7+9=16(1);2+2+1=5(0);5+2=7;756<|endoftext|>\",\n",
      "    \"correct\": true\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"182+21;\",\n",
      "    \"decoded\": \"182+21;2+1=3(0);8+2=10(1);1+1=2;203<|endoftext|>\",\n",
      "    \"correct\": true\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"514+916;\",\n",
      "    \"decoded\": \"514+916;4+6=10(1);1+1+1=3(0);5+9=14;1430<|endoftext|>\",\n",
      "    \"correct\": true\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"297+645;\",\n",
      "    \"decoded\": \"297+645;7+5=12(1);9+4+1=14(1);2+6+1=9;942<|endoftext|>\",\n",
      "    \"correct\": true\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"790+528;\",\n",
      "    \"decoded\": \"790+528;0+8=8(0);9+2=11(1);7+5+1=13;1318<|endoftext|>\",\n",
      "    \"correct\": true\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"343+102;\",\n",
      "    \"decoded\": \"343+102;3+2=5(0);4+0=4(0);3+1=4;445<|endoftext|>\",\n",
      "    \"correct\": true\n",
      "  }\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "print(len(goods))\n",
    "print(json.dumps(goods[:10], indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n",
      "[\n",
      "  {\n",
      "    \"prompt\": \"260+456;\",\n",
      "    \"decoded\": \"260+456;0+6=6(0);6+5=12(1);2+4+1=7;726<|endoftext|>\",\n",
      "    \"correct\": false\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"48+621;\",\n",
      "    \"decoded\": \"48+621;8+1=9(0);4+2=4(0);6+0=6;649<|endoftext|>\",\n",
      "    \"correct\": false\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"82+230;\",\n",
      "    \"decoded\": \"82+230;2+0=2(0);2+3=5(0);2+0=2;252<|endoftext|>\",\n",
      "    \"correct\": false\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"280+465;\",\n",
      "    \"decoded\": \"280+465;0+5=5(0);8+6=14(1);6+4+1=11;1145<|endoftext|>\",\n",
      "    \"correct\": false\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"13+189;\",\n",
      "    \"decoded\": \"13+189;3+9=12(1);1+6+1=8(0);1+0=1;182<|endoftext|>\",\n",
      "    \"correct\": false\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"977+91;\",\n",
      "    \"decoded\": \"977+91;7+1=8(0);7+9=16(1);9+1=10;1868<|endoftext|>\",\n",
      "    \"correct\": false\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"916+10;\",\n",
      "    \"decoded\": \"916+10;6+0=6(0);9+1=10(1);9+1=10;1006<|endoftext|>\",\n",
      "    \"correct\": false\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"561+564;\",\n",
      "    \"decoded\": \"561+564;1+4=5(0);6+6=12(1);5+5+1=15;1525<|endoftext|>\",\n",
      "    \"correct\": false\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"109+405;\",\n",
      "    \"decoded\": \"109+405;9+5=14(1);0+0+1=1(0);1+4=5;511<|endoftext|>\",\n",
      "    \"correct\": false\n",
      "  },\n",
      "  {\n",
      "    \"prompt\": \"48+205;\",\n",
      "    \"decoded\": \"48+205;8+5=13(1);5+0+1=6(0);2+0=2;263<|endoftext|>\",\n",
      "    \"correct\": false\n",
      "  }\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "print(len(wrongs))\n",
    "print(json.dumps(wrongs[:10], indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}