sort_interp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "import einops\n",
    "import tqdm.notebook as tqdm\n",
    "\n",
    "import random\n",
    "import time\n",
    "\n",
    "# from google.colab import drive\n",
    "from pathlib import Path\n",
    "import pickle\n",
    "import os\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from torch.utils.data import DataLoader\n",
    "from functools import *\n",
    "import pandas as pd\n",
    "import gc\n",
    "\n",
    "# import comet_ml\n",
    "import itertools\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# A helper class to get access to intermediate activations (inspired by Garcon)\n",
    "# It's a dummy module that is the identity function by default\n",
    "# I can wrap any intermediate activation in a HookPoint and get a convenient\n",
    "# way to add PyTorch hooks\n",
    "class HookPoint(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.fwd_hooks = []\n",
    "        self.bwd_hooks = []\n",
    "\n",
    "    def give_name(self, name):\n",
    "        # Called by the model at initialisation\n",
    "        self.name = name\n",
    "\n",
    "    def add_hook(self, hook, dir='fwd'):\n",
    "        # Hook format is fn(activation, hook_name)\n",
    "        # Change it into PyTorch hook format (this includes input and output,\n",
    "        # which are the same for a HookPoint)\n",
    "        def full_hook(module, module_input, module_output):\n",
    "            return hook(module_output, name=self.name)\n",
    "        if dir=='fwd':\n",
    "            handle = self.register_forward_hook(full_hook)\n",
    "            self.fwd_hooks.append(handle)\n",
    "        elif dir=='bwd':\n",
    "            handle = self.register_backward_hook(full_hook)\n",
    "            self.bwd_hooks.append(handle)\n",
    "        else:\n",
    "            raise ValueError(f\"Invalid direction {dir}\")\n",
    "\n",
    "    def remove_hooks(self, dir='fwd'):\n",
    "        if (dir=='fwd') or (dir=='both'):\n",
    "            for hook in self.fwd_hooks:\n",
    "                hook.remove()\n",
    "            self.fwd_hooks = []\n",
    "        if (dir=='bwd') or (dir=='both'):\n",
    "            for hook in self.bwd_hooks:\n",
    "                hook.remove()\n",
    "            self.bwd_hooks = []\n",
    "        if dir not in ['fwd', 'bwd', 'both']:\n",
    "            raise ValueError(f\"Invalid direction {dir}\")\n",
    "\n",
    "    def forward(self, x):\n",
    "        return x\n",
    "\n",
    "# Define network architecture\n",
    "# I defined my own transformer from scratch so I'd fully understand each component\n",
    "# - I expect this wasn't necessary or particularly important, and a bunch of this\n",
    "# replicates existing PyTorch functionality\n",
    "\n",
    "# Embed & Unembed\n",
    "class Embed(nn.Module):\n",
    "    def __init__(self, d_vocab, d_model):\n",
    "        super().__init__()\n",
    "        self.W_E = nn.Parameter(torch.randn(d_model, d_vocab)/np.sqrt(d_model))\n",
    "\n",
    "    def forward(self, x):\n",
    "        return torch.einsum('dbp -> bpd', self.W_E[:, x])\n",
    "\n",
    "class Unembed(nn.Module):\n",
    "    def __init__(self, d_vocab, d_model):\n",
    "        super().__init__()\n",
    "        self.W_U = nn.Parameter(torch.randn(d_model, d_vocab)/np.sqrt(d_vocab))\n",
    "\n",
    "    def forward(self, x):\n",
    "        return (x @ self.W_U)\n",
    "\n",
    "# Positional Embeddings\n",
    "class PosEmbed(nn.Module):\n",
    "    def __init__(self, max_ctx, d_model):\n",
    "        super().__init__()\n",
    "        self.W_pos = nn.Parameter(torch.randn(max_ctx, d_model)/np.sqrt(d_model))\n",
    "\n",
    "    def forward(self, x):\n",
    "        return x+self.W_pos[:x.shape[-2]]\n",
    "\n",
    "# LayerNorm\n",
    "class LayerNorm(nn.Module):\n",
    "    def __init__(self, d_model, epsilon = 1e-4, model=[None]):\n",
    "        super().__init__()\n",
    "        self.model = model\n",
    "        self.w_ln = nn.Parameter(torch.ones(d_model))\n",
    "        self.b_ln = nn.Parameter(torch.zeros(d_model))\n",
    "        self.epsilon = epsilon\n",
    "\n",
    "    def forward(self, x):\n",
    "        if self.model[0].use_ln:\n",
    "            x = x - x.mean(axis=-1)[..., None]\n",
    "            x = x / (x.std(axis=-1)[..., None] + self.epsilon)\n",
    "            x = x * self.w_ln\n",
    "            x = x + self.b_ln\n",
    "            return x\n",
    "        else:\n",
    "            return x\n",
    "\n",
    "# Attention\n",
    "class Attention(nn.Module):\n",
    "    def __init__(self, d_model, num_heads, d_head, n_ctx, model):\n",
    "        super().__init__()\n",
    "        self.model = model\n",
    "        self.W_K = nn.Parameter(torch.randn(num_heads, d_head, d_model)/np.sqrt(d_model))\n",
    "        self.W_Q = nn.Parameter(torch.randn(num_heads, d_head, d_model)/np.sqrt(d_model))\n",
    "        self.W_V = nn.Parameter(torch.randn(num_heads, d_head, d_model)/np.sqrt(d_model))\n",
    "        self.W_O = nn.Parameter(torch.randn(d_model, d_head * num_heads)/np.sqrt(d_model))\n",
    "        self.register_buffer('mask', torch.tril(torch.ones((n_ctx, n_ctx))))\n",
    "        self.d_head = d_head\n",
    "        self.hook_k = HookPoint()\n",
    "        self.hook_q = HookPoint()\n",
    "        self.hook_v = HookPoint()\n",
    "        self.hook_z = HookPoint()\n",
    "        self.hook_attn = HookPoint()\n",
    "        self.hook_attn_pre = HookPoint()\n",
    "\n",
    "    def forward(self, x):\n",
    "        k = self.hook_k(torch.einsum('ihd,bpd->biph', self.W_K, x))\n",
    "        q = self.hook_q(torch.einsum('ihd,bpd->biph', self.W_Q, x))\n",
    "        v = self.hook_v(torch.einsum('ihd,bpd->biph', self.W_V, x))\n",
    "        attn_scores_pre = torch.einsum('biph,biqh->biqp', k, q)\n",
    "        attn_scores_masked = torch.tril(attn_scores_pre) - 1e10 * (1 - self.mask[:x.shape[-2], :x.shape[-2]])\n",
    "        attn_matrix = self.hook_attn(F.softmax(self.hook_attn_pre(attn_scores_masked/np.sqrt(self.d_head)), dim=-1))\n",
    "        z = self.hook_z(torch.einsum('biph,biqp->biqh', v, attn_matrix))\n",
    "        z_flat = einops.rearrange(z, 'b i q h -> b q (i h)')\n",
    "        out = torch.einsum('df,bqf->bqd', self.W_O, z_flat)\n",
    "        return out\n",
    "\n",
    "# MLP Layers\n",
    "class MLP(nn.Module):\n",
    "    def __init__(self, d_model, d_mlp, act_type, model):\n",
    "        super().__init__()\n",
    "        self.model = model\n",
    "        self.W_in = nn.Parameter(torch.randn(d_mlp, d_model)/np.sqrt(d_model))\n",
    "        self.b_in = nn.Parameter(torch.zeros(d_mlp))\n",
    "        self.W_out = nn.Parameter(torch.randn(d_model, d_mlp)/np.sqrt(d_model))\n",
    "        self.b_out = nn.Parameter(torch.zeros(d_model))\n",
    "        self.act_type = act_type\n",
    "        # self.ln = LayerNorm(d_mlp, model=self.model)\n",
    "        self.hook_pre = HookPoint()\n",
    "        self.hook_post = HookPoint()\n",
    "        assert act_type in ['ReLU', 'GeLU']\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.hook_pre(torch.einsum('md,bpd->bpm', self.W_in, x) + self.b_in)\n",
    "        if self.act_type=='ReLU':\n",
    "            x = F.relu(x)\n",
    "        elif self.act_type=='GeLU':\n",
    "            x = F.gelu(x)\n",
    "        x = self.hook_post(x)\n",
    "        x = torch.einsum('dm,bpm->bpd', self.W_out, x) + self.b_out\n",
    "        return x\n",
    "\n",
    "# Transformer Block\n",
    "class TransformerBlock(nn.Module):\n",
    "    def __init__(self, d_model, d_mlp, d_head, num_heads, n_ctx, act_type, model):\n",
    "        super().__init__()\n",
    "        self.model = model\n",
    "        # self.ln1 = LayerNorm(d_model, model=self.model)\n",
    "        self.attn = Attention(d_model, num_heads, d_head, n_ctx, model=self.model)\n",
    "        # self.ln2 = LayerNorm(d_model, model=self.model)\n",
    "        self.mlp = MLP(d_model, d_mlp, act_type, model=self.model)\n",
    "        self.hook_attn_out = HookPoint()\n",
    "        self.hook_mlp_out = HookPoint()\n",
    "        self.hook_resid_pre = HookPoint()\n",
    "        self.hook_resid_mid = HookPoint()\n",
    "        self.hook_resid_post = HookPoint()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.hook_resid_mid(x + self.hook_attn_out(self.attn((self.hook_resid_pre(x)))))\n",
    "        x = self.hook_resid_post(x + self.hook_mlp_out(self.mlp((x))))\n",
    "        return x\n",
    "\n",
    "# Full transformer\n",
    "class Transformer(nn.Module):\n",
    "    def __init__(self, num_layers, d_vocab, d_model, d_mlp, d_head, num_heads, n_ctx, act_type, use_cache=False, use_ln=True):\n",
    "        super().__init__()\n",
    "        self.cache = {}\n",
    "        self.use_cache = use_cache\n",
    "\n",
    "        self.embed = Embed(d_vocab, d_model)\n",
    "        self.pos_embed = PosEmbed(n_ctx, d_model)\n",
    "        self.blocks = nn.ModuleList([TransformerBlock(d_model, d_mlp, d_head, num_heads, n_ctx, act_type, model=[self]) for i in range(num_layers)])\n",
    "        # self.ln = LayerNorm(d_model, model=[self])\n",
    "        self.unembed = Unembed(d_vocab, d_model)\n",
    "        self.use_ln = use_ln\n",
    "\n",
    "        for name, module in self.named_modules():\n",
    "            if type(module)==HookPoint:\n",
    "                module.give_name(name)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.embed(x)\n",
    "        x = self.pos_embed(x)\n",
    "        for block in self.blocks:\n",
    "            x = block(x)\n",
    "        # x = self.ln(x)\n",
    "        x = self.unembed(x)\n",
    "        return x\n",
    "\n",
    "    def set_use_cache(self, use_cache):\n",
    "        self.use_cache = use_cache\n",
    "\n",
    "    def hook_points(self):\n",
    "        return [module for name, module in self.named_modules() if 'hook' in name]\n",
    "\n",
    "    def remove_all_hooks(self):\n",
    "        for hp in self.hook_points():\n",
    "            hp.remove_hooks('fwd')\n",
    "            hp.remove_hooks('bwd')\n",
    "\n",
    "    def cache_all(self, cache, incl_bwd=False):\n",
    "        # Caches all activations wrapped in a HookPoint\n",
    "        def save_hook(tensor, name):\n",
    "            cache[name] = tensor.detach()\n",
    "        def save_hook_back(tensor, name):\n",
    "            cache[name+'_grad'] = tensor[0].detach()\n",
    "        for hp in self.hook_points():\n",
    "            hp.add_hook(save_hook, 'fwd')\n",
    "            if incl_bwd:\n",
    "                hp.add_hook(save_hook_back, 'bwd')\n",
    "\n",
    "# Helper functions\n",
    "def cuda_memory():\n",
    "    print(torch.cuda.memory_allocated()/1e9)\n",
    "\n",
    "def cross_entropy_high_precision(logits, labels):\n",
    "    # Shapes: batch x seq x vocab, batch x seq\n",
    "    # Cast logits to float64 because log_softmax has a float32 underflow on overly\n",
    "    # confident data and can only return multiples of 1.2e-7 (the smallest float x\n",
    "    # such that 1+x is different from 1 in float32). This leads to loss spikes\n",
    "    # and dodgy gradients\n",
    "\n",
    "    logprobs = F.log_softmax(logits.to(torch.float64), dim=-1)\n",
    "    # print(logprobs.shape, labels.shape)\n",
    "    prediction_logprobs = torch.gather(logprobs, index=labels[:, :, None], dim=-1)\n",
    "    # print(prediction_logprobs.shape)\n",
    "    loss = -torch.mean(prediction_logprobs)\n",
    "    return loss\n",
    "\n",
    "def full_loss(model, data, arr_len, device='cuda'):\n",
    "    \"\"\"\n",
    "    Calculate the full loss and accuracy of the model.\n",
    "\n",
    "    Parameters:\n",
    "    model (nn.Module): The PyTorch model.\n",
    "    data (Tensor): The input data.\n",
    "    arr_len (int): The length of the array.\n",
    "\n",
    "    Returns:\n",
    "    tuple: A tuple containing the loss and accuracy.\n",
    "    \"\"\"\n",
    "    # Take the final position only\n",
    "    logits = model(data)[:, arr_len:-1]\n",
    "\n",
    "    # labels = torch.tensor([fn(i, j) for i, j, _ in data]).to('cuda')\n",
    "    # labels = torch.tensor([np.sort(x) for x in data])\n",
    "    labels = torch.tensor(data[:, arr_len + 1:]).to(device)\n",
    "\n",
    "    # Calculate loss\n",
    "    loss = cross_entropy_high_precision(logits, labels)\n",
    "\n",
    "    # Calculate accuracy\n",
    "    predictions = torch.argmax(logits, dim=2)\n",
    "    # print(predictions.shape, labels.shape, len(labels))\n",
    "    accuracy = torch.sum(predictions == labels) / (arr_len * len(labels))\n",
    "    # Calculate exact match accuracy\n",
    "    exact_match_accuracy = torch.sum(torch.all(predictions == labels, dim=-1)) / labels.shape[0]\n",
    "    return loss, accuracy,exact_match_accuracy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=1e-4 #@param\n",
    "weight_decay = 1.0 #@param\n",
    "p=113 #@param\n",
    "d_model = 128 #@param\n",
    "fn_name = 'add' #@param ['add', 'subtract', 'x2xyy2','rand']\n",
    "frac_train = 0.3 #@param\n",
    "dataset_size = 10000 #@param\n",
    "num_epochs = 20000 #@param\n",
    "save_models = True #@param\n",
    "save_every = 1000 #@param\n",
    "# Stop training when test loss is <stopping_thresh\n",
    "stopping_thresh = -1 #@param\n",
    "seed = 0 #@param\n",
    "\n",
    "arr_len = 5 #@param\n",
    "\n",
    "start = 1 #@param\n",
    "end = 100 #@param\n",
    "\n",
    "num_layers = 1\n",
    "\n",
    "batch_style = 'full'\n",
    "d_vocab = p+1\n",
    "n_ctx = 3\n",
    "d_mlp = 4*d_model\n",
    "num_heads = 4\n",
    "assert d_model % num_heads == 0\n",
    "d_head = d_model//num_heads\n",
    "act_type = 'ReLU' #@param ['ReLU', 'GeLU']\n",
    "# batch_size = 512\n",
    "use_ln = False\n",
    "random_answers = np.random.randint(low=0, high=p, size=(p, p))\n",
    "fns_dict = {'add': lambda x,y:(x+y)%p, 'subtract': lambda x,y:(x-y)%p, 'x2xyy2':lambda x,y:(x**2+x*y+y**2)%p, 'rand':lambda x,y:random_answers[x][y]}\n",
    "fn = fns_dict[fn_name]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Transformer(num_layers=num_layers, d_vocab=(end - start + 1 + 1), d_model=d_model, d_mlp=d_mlp, d_head=d_head, num_heads=num_heads, n_ctx=2 * arr_len + 1, act_type=act_type, use_cache=True, use_ln=use_ln)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_60018/1582388634.py:1: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  state_dict = torch.load('./max5_unbalanced/3000.pth')\n"
     ]
    }
   ],
   "source": [
    "state_dict = torch.load('./max5_unbalanced/3000.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate(model, inp, it=5):\n",
    "    for _ in range(it):\n",
    "        logits = model(torch.tensor([inp]))[0, -1]\n",
    "        inp += [torch.argmax(logits).item()]\n",
    "    return inp\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[100, 100, 100, 100, 100, 0, 99, 100, 100, 100, 100]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate(model, [100, 100, 100, 100, 100 ,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "cache = {}\n",
    "model.load_state_dict(state_dict=state_dict['model'])\n",
    "model.cache_all(cache)\n",
    "logits = model(torch.tensor([[9, 9, 9, 9, 9, 0]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1.2133e-14, 2.8188e-19, 5.3974e-19, 6.3419e-16, 1.3957e-14, 1.8202e-16,\n",
       "        6.2969e-20, 6.4036e-18, 4.4715e-16, 2.0711e-02, 1.1392e-15, 2.2810e-15,\n",
       "        5.4302e-18, 2.6683e-14, 4.9924e-19, 1.3238e-16, 1.5617e-16, 8.6689e-17,\n",
       "        3.5091e-06, 1.3803e-06, 7.2491e-05, 4.4337e-08, 3.6206e-04, 5.4173e-06,\n",
       "        1.1185e-05, 9.7691e-01, 3.0481e-06, 5.4826e-05, 1.9819e-06, 2.9280e-07,\n",
       "        5.9603e-05, 9.8793e-04, 1.5419e-04, 4.9706e-04, 1.2741e-07, 5.5706e-05,\n",
       "        5.7589e-12, 9.7193e-09, 1.6797e-07, 7.0151e-09, 2.2807e-11, 3.4504e-05,\n",
       "        6.4121e-07, 1.7443e-08, 1.1321e-09, 4.0765e-09, 1.0089e-09, 8.2814e-10,\n",
       "        5.8914e-09, 6.9164e-05, 9.3713e-11, 2.4103e-14, 3.2673e-11, 2.2452e-07,\n",
       "        1.0016e-16, 1.3445e-08, 8.9484e-08, 2.3284e-12, 2.2820e-15, 3.8174e-13,\n",
       "        5.0677e-10, 8.5496e-16, 2.7942e-20, 1.5381e-13, 1.0450e-10, 1.8587e-17,\n",
       "        2.9633e-17, 1.4887e-18, 6.9185e-21, 1.4474e-24, 5.6406e-14, 1.8453e-14,\n",
       "        1.0618e-13, 1.4654e-16, 5.9711e-14, 2.5506e-14, 1.4004e-17, 2.1165e-21,\n",
       "        1.4638e-16, 3.2994e-21, 3.0150e-20, 8.6942e-17, 9.2510e-17, 1.0867e-16,\n",
       "        8.5505e-24, 1.8104e-23, 9.9648e-18, 5.4880e-21, 4.8424e-25, 4.4132e-25,\n",
       "        9.9375e-22, 5.2180e-22, 1.9023e-28, 1.0315e-25, 3.0499e-23, 6.6400e-28,\n",
       "        8.3703e-25, 4.3091e-23, 1.0997e-31, 3.7078e-25, 3.6959e-25],\n",
       "       grad_fn=<SoftmaxBackward0>)"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(logits[0,-1].softmax(dim=-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dfXhU5Z3/8c/M5BlJrKCByINx1/5Eaa1N9FpQ6kPbcAF1d/skra2gQrcsPmFWW5BeLXKpcbsuS20FtVVZf9rK+tTVNpeari1icX1AaFH41baiAUxMQ9sEEvIwM+f3x2RmziSTkDmc4Zz78H5d11w9mZwh3/s03nz5nu99n5BlWZYAAAA8EvY6AAAAcGwjGQEAAJ4iGQEAAJ4iGQEAAJ4iGQEAAJ4iGQEAAJ4iGQEAAJ4iGQEAAJ4q8DqA0YjH43r//fc1duxYhUIhr8MBAACjYFmWDhw4oKqqKoXDw9c/jEhG3n//fU2ePNnrMAAAgAN79uzRpEmThv2+EcnI2LFjJSUGU15enr8f1NUlVVUljt9/XxozJn8/y8R4nArKOAC4gznhmNHZ2anJkyen/h4fjhHJSPLWTHl5eX6TkUgkfVxe7v1/IH6Lx6mgjAOAO5gTjjmHa7GggRUAAHgq52TkxRdf1CWXXKKqqiqFQiH99Kc/PexnNm3apJqaGpWUlOjUU0/VPffc4yjYvCsokBYuTLwKfFA08ls8TgVlHADcwZyAQXL+Lejq6tJZZ52lK6+8Up///OcPe/7u3bs1d+5cfe1rX9PDDz+sX//611q6dKlOPPHEUX3+qCouljZs8DqKNL/F41RQxgHAHcwJGCTnZGTOnDmaM2fOqM+/5557NGXKFK1du1aSNG3aNL3++uu68847/ZeMAACAoy7vPSMvv/yy6urqMt6bPXu2Xn/9dfX39+f7x+fGshJd3l1diWOv+S0ep4IyDgDuYE7AIHm/Wdfa2qrKysqM9yorKxWNRtXe3q6JEycO+Uxvb696e3tTX3d2duY7zITubum44xLHBw963+Htt3icCso4ALiDOQGDHJXVNIOX9FgDmfBwS30aGhpUUVGRerHhGQAAwZX3ZGTChAlqbW3NeK+trU0FBQUaN25c1s+sWLFCHR0dqdeePXvyHSYAAPBI3m/TzJgxQ88880zGe88//7xqa2tVWFiY9TPFxcUqLi7Od2gAAMAHcq6MHDx4UNu3b9f27dslJZbubt++Xc3NzZISVY0FCxakzl+yZInee+891dfXa9euXXrggQd0//3368Ybb3RpCAAAwGQ5V0Zef/11XXTRRamv6+vrJUkLFy7Uhg0b1NLSkkpMJKm6ulqNjY264YYbdPfdd6uqqkp33XUXy3oBAIAkKWRZ/l9X1dnZqYqKCnV0dOT/QXl+6vD2WzxOBWUcANzBnHDMGO3f3+zDaxeJSF/4QvrYa36Lx6k8jmPLH9q19y+HdOk5rLgCjBGUuQ2uoTICo513xwva99dD+vXyi3Xy8aVehwMAsBnt3988tRdG6+xJ7OJ7sCfqcSQAAKdIRmC0WDxR2IvG4x5HAgBwimTErqtLCoUSr64ur6PxXzxO5XEcyWSEXAQwSFDmNriGZARGozICAOYjGYHRYgP913H/92EDAIZBMgJjxeNW6unj0RjJCACYimQExorZqiExKiMAYCySERgr2S8i0cAKACYjGYGx7MkIDawAYC62g7eLRKS5c9PHXvNbPE7laRz2WzM0sAIGCcrcBteQjNiVlEg//7nXUaT5LR6n8jSOmK1plQZWwCBBmdvgGm7TwFhURgAgGEhGYKzMnhGSEQAwFcmIXVeXNGZM4uWHLYr9Fo9TeRqHPRmJkYwA5gjK3AbX0DMyWHe31xFk8ls8TuVhHBlLe7lNA5glKHMbXEFlBMbKuE1DAysAGItkBMaigRUAgoFkBMaigRUAgoFkBMbK3A6eZAQATEUyAmNRGQGAYGA1jV04LF1wQfrYa36Lx6k8jYOlvYChgjK3wTUkI3alpdKvfuV1FGl+i8epPI2DBlbAUEGZ2+AaUlIYi9s0ABAMJCMwFg2sABAMJCN2XV3SiScmXn7Yothv8TiVp3FQGQEMFZS5Da6hZ2Sw9navI8jkt3icysM4qIwABgvK3AZXUBmBsaiMAEAwkIzAWBlLe1lNAwDGIhmBsezVkBgPygMAY5GMwFj2vUWojACAuUhGYCwaWAEgGFhNYxcOS7W16WOv+S0ep/I0DhpYAUMFZW6Da0hG7EpLpdde8zqKNL/F41SexpFRGeE2DWCOoMxtcA0pKYyVURmhgRUAjEUyAmPFaGAFgEAgGbHr7pZOOSXx6u72Ohr/xeNUnsaRsbSXnhHAHEGZ2+AaekbsLEt67730sdf8Fo9TeRpHnGQEMFNQ5ja4hsoIjEUDKwAEA8kIjEUDKwAEA8kIjGVvWqUyAgDmIhmBsdj0DACCgWQExorRwAoAgcBqGrtQSDrjjPSx1/wWj1N5GgdLewFDBWVug2tIRuzKyqS33vI6ijS/xeNUnsbB0l7AUEGZ2+AabtPAWFRGACAYSEZgrDjbwQNAIJCM2HV3S2eemXj5YYtiv8XjVJ7GkbHpGZURwBxBmdvgGnpG7CxL2rkzfew1v8XjVJ7GwdJewFBBmdvgGiojMBZLewEgGEhGYCwaWAEgGEhGYKyMpb2UegHAWCQjMBaVEQAIBpIRGCtjaS/JCAAYi9U0dqGQNHVq+thrfovHqTyNg6W9gKGCMrfBNSQjdmVl0rvveh1Fmt/icSpP42BpL2CooMxtcA23aWCsjMoIDawAYCxHyci6detUXV2tkpIS1dTUaPPmzSOe/8gjj+iss85SWVmZJk6cqCuvvFL79+93FDCQFKUyAgCBkHMysnHjRi1btkwrV67Utm3bNGvWLM2ZM0fNzc1Zz3/ppZe0YMECLVq0SG+99ZYee+wxvfbaa1q8ePERB++6Q4ekc85JvA4d8joa/8XjVJ7GQQMrYKigzG1wTc49I2vWrNGiRYtSycTatWv13HPPaf369WpoaBhy/v/+7//qlFNO0XXXXSdJqq6u1te//nV997vfPcLQ8yAel15/PX3sNb/F41SexsHSXsBQQZnb4JqcKiN9fX3aunWr6urqMt6vq6vTli1bsn5m5syZ2rt3rxobG2VZlj744AM9/vjjmjdv3rA/p7e3V52dnRkvYLA4yQgABEJOyUh7e7tisZgqKysz3q+srFRra2vWz8ycOVOPPPKI5s+fr6KiIk2YMEHHH3+8vv/97w/7cxoaGlRRUZF6TZ48OZcwcYyI2v5FRTICAOZy1MAaGrQu3LKsIe8l7dy5U9ddd52+/e1va+vWrXr22We1e/duLVmyZNg/f8WKFero6Ei99uzZ4yRMBJy9ust28ABgrpx6RsaPH69IJDKkCtLW1jakWpLU0NCg8847TzfddJMk6aMf/ajGjBmjWbNm6dZbb9XEiROHfKa4uFjFxcW5hIZjkD0BsazEbZtwmA2UAMA0OVVGioqKVFNTo6ampoz3m5qaNHPmzKyf6e7uVjic+WMikYikREUFcGrwcl6qIwBgppxX09TX1+vyyy9XbW2tZsyYofvuu0/Nzc2p2y4rVqzQvn379NBDD0mSLrnkEn3ta1/T+vXrNXv2bLW0tGjZsmU699xzVVVV5e5o3DB+vNcRZPJbPE7lYRyDt4CPxS0VRlz/MQDyIShzG1yRczIyf/587d+/X6tXr1ZLS4umT5+uxsZGTR14zkBLS0vGniNXXHGFDhw4oB/84Af6l3/5Fx1//PG6+OKL9a//+q/ujcItY8ZIf/qT11Gk+S0ep/I0jiGVEZpYATMEZW6Da0KWAfdKOjs7VVFRoY6ODpWXl3sdDnxi9n+8qN99cCD19W9X1am8pNDDiAAAdqP9+5tn08BY0UGbJcVivs+rAQBZkIzYHTokXXhh4uWHLYr9Fo9TeRrH4LsyNLAChgjK3AbX5NwzEmjxuLRpU/rYa36Lx6k8jWNwj8jghlYAPhWUuQ2uoTICYw1ORnhyLwCYiWQExhqcjLCaBgDMRDICY7G0FwCCgWQExopb7MAKAEFAMgJjRWODlvZSGQEAI7GaZrCyMq8jyOS3eJzKwziGLO0lGQHMEZS5Da4gGbEbM0bq6vI6ijS/xeNUnsZBAytgqKDMbXANt2lgrGTyEQ5lfg0AMAvJCIyVbFgtKghnfA0AMAvJiF1PjzRvXuLV0+N1NP6Lx6k8jMOyrFQlpDAykIxQGQHMEJS5Da6hZ8QuFpMaG9PHXvNbPE7lYRz2vKO4IKwDIhkBjBGUuQ2uoTICI9mf2FtEZQQAjEYyAiPZn62V6hkhGQEAI5GMwEgZlRGSEQAwGskIjERlBACCg2QERrIv4031jLC0FwCMRDICIyVv04RCUkGYyggAmIylvXZjxkh++te13+JxKg/jSN6miYRCigxswUoyAhgiKHMbXENlBEZKVkbCYZIRADAdyQiMlKyMFJCMAIDxSEbsenqkL34x8fLDFsV+i8epPIwjWRnhNg1goKDMbXANyYhdLCY9/nji5Yctiv0Wj1N5GEd84H5zJGJLRrgHDZghKHMbXEMyAiPF7A2sISojAGAykhEYKXWbhp4RADAeyQiMlFraSzICAMYjGYGRUkt7aWAFAOORjMBIyQbWAhpYAcB4JCMwUjQ2sJqGBlYAMB7bwduVlUkHD6aPvea3eJzKwziSVZBIOKRIhGQEMEpQ5ja4hmTELhRKPDPBL/wWj1N5GEcy8YiE05WRKMkIYIagzG1wDbdpYKSMZGSgZyROMgIARiIZsevtla64IvHq7fU6Gv/F41QexhG3hiYjNLAChgjK3AbXkIzYRaPSf/5n4hWNeh2N/+JxKg/jSDawsrQXMFBQ5ja4hmQERkot7WXTMwAwHskIjJRsVg2HWdoLAKYjGYGRkokHlREAMB/JCIyUbTUNS3sBwEwkIzASS3sBIDhIRmCk1NLeEEt7AcB07MBqV1YmtbWlj73mt3icysM4oll2YKVnBDBEUOY2uIZkxC4Ukk480eso0vwWj1N5GEc8y20akhHAEEGZ2+AabtPASBlLe0lGAMBoJCN2vb3S1VcnXn7Yothv8TiVh3GwtBcwWFDmNriGZMQuGpXWrUu8/LBFsd/icSoP40itpgmxtBcwTlDmNriGZARGimV5UF6c1TQAYCSSERgpzmoaAAgMkhEYKcpqGgAIDJIRGImlvQAQHCQjMFJqaW+IZAQATEcyAiMlG1gzlvbSwAoARmIHVrvSUmn37vSx1/wWj1N5GEcsNrSBlaW9gCGCMrfBNSQjduGwdMopXkeR5rd4nMrDODKW9kZ4ai9glKDMbXANt2lgpFiWpb1URgDATCQjdn190k03JV59fV5H4794nMrDOGJZVtNQGQEMEZS5Da4hGbHr75fuvDPx6u/3Ohr/xeNUHsaR3G3Vvh08DayAIYIyt8E1jpKRdevWqbq6WiUlJaqpqdHmzZtHPL+3t1crV67U1KlTVVxcrL/5m7/RAw884ChgQJKiyQbWCEt7AcB0OTewbty4UcuWLdO6det03nnn6d5779WcOXO0c+dOTZkyJetnLr30Un3wwQe6//779bd/+7dqa2tTlIcj4QjEslVGSEYAwEg5JyNr1qzRokWLtHjxYknS2rVr9dxzz2n9+vVqaGgYcv6zzz6rTZs26Z133tEJJ5wgSTqFLmocoWwNrCQjAGCmnG7T9PX1aevWraqrq8t4v66uTlu2bMn6maefflq1tbX67ne/q5NPPlkf/vCHdeONN+rQoUPD/pze3l51dnZmvAC7bA2sJCMAYKacKiPt7e2KxWKqrKzMeL+yslKtra1ZP/POO+/opZdeUklJiZ566im1t7dr6dKl+vOf/zxs30hDQ4NuueWWXELDMSZbMsLSXgAwk6MG1tBAWTzJsqwh7yXF43GFQiE98sgjOvfcczV37lytWbNGGzZsGLY6smLFCnV0dKRee/bscRImAsyejBQkl/aymgYAjJRTZWT8+PGKRCJDqiBtbW1DqiVJEydO1Mknn6yKiorUe9OmTZNlWdq7d69OO+20IZ8pLi5WcXFxLqG5o7RUevPN9LHX/BaPU3kYh31pb5jbNIBZgjK3wTU5VUaKiopUU1OjpqamjPebmpo0c+bMrJ8577zz9P777+vgwYOp995++22Fw2FNmjTJQch5FA5LZ56ZeIV9sAWL3+JxKg/jiNLACpgrKHMbXJPzb0F9fb1+9KMf6YEHHtCuXbt0ww03qLm5WUuWLJGUuMWyYMGC1PmXXXaZxo0bpyuvvFI7d+7Uiy++qJtuuklXXXWVSsmI4RANrAAQHDkv7Z0/f77279+v1atXq6WlRdOnT1djY6OmTp0qSWppaVFzc3Pq/OOOO05NTU269tprVVtbq3HjxunSSy/Vrbfe6t4o3NLXJ91+e+L45puloiLicUMexkEyAhgsKHMbXBOyLP93/XV2dqqiokIdHR0qLy/P3w/q6pKOOy5xfPCgNGZM/n6WifE4lYdxzL/3Zb2y+8/6wWVn69xTTtC5t/+PIuGQ/nj73CP+swHkWVDmNhzWaP/+5mYdjJSqjAxqYDUgtwYADEIyAiOltoO3Le2VJO7UAIB5SEZgJHvPSNiWjETjca9CAgA4RDICI2V7No0kkYsAgHlIRmCkbKtppPTtGwCAOUhGYKRhk5EYyQgAmCbnfUYCraREevXV9LHX/BaPU3kYR8y2Hbz9Ng2VEcAAQZnb4BqSEbtIRDrnHK+jSPNbPE7lYRyDG1hDIcmyaGAFjBCUuQ2u4TYNjGRPRiSln9xLLgIAxqEyYtfXJ33ve4nj66/3fotiv8XjVB7GMTgZCYdCkiwqI4AJgjK3wTVsB2/nty2K/RaPU3kYx7m3/UJtB3r18+vO15lVFTrj28+quy+mF2+6SFPGlR3xnw8gj4Iyt+Gw2A4egRa3MisjqYfl+T+3BgAMQjICI0UHbtMUDE5GuE0DAMYhGYGRkj0j4VBmA2uMXAQAjEMyAiNlb2BlaS8AmIhkBEZiaS8ABAfJCIw0pDISpjICAKZinxG7khLpl79MH3vNb/E4lYdxxIZZTRNnNQ3gf0GZ2+AakhG7SES68EKvo0jzWzxOuTyOeNxSMudIPpcmmYxEeVAe4H9BmdvgGm7TwDj2vUQKwolf4WRSwj4jAGAeKiN2/f3Sffcljv/pn6TCQuJxg8vjSPaLSNJALpK+TUPLCOB/QZnb4BqSEbu+PumaaxLHV1zh/X8gfovHKZfHYU9GUpURGlgBcwRlboNruE0D49hvxSQrIwU0sAKAsUhGYJyYrUk12SsSpoEVAIxFMgLj2CsjqaW9ISojAGAqkhEYJ/1cGik0eGlvnGQEAExDMgLjxFJP7E3/+qaf2ksyAgCmIRmBcVKVEdtvLzuwAoC5WNprV1ws/exn6WOv+S0ep1wex0iVERpYAQMEZW6Da0hG7AoKpHnzvI4izW/xOOXyOJINrAP5R+JHUBkBzBGUuQ2u4TYNjDP4ib2SFA7RwAoApqIyYtffLz3ySOL4K1/xfldAv8XjlMvjSCcj6Vy6IJLcDp5kBPC9oMxtcA3JiF1fn3TllYnjL37R+/9A/BaPUy6PI52MpN+jMgIYJChzG1zDbRoYh6W9ABAsJCMwTqqBlaW9ABAIJCMwTtbKCLdpAMBYJCMwjn07+CQaWAHAXCQjME62yggNrABgLpIRGCe9HXy6NJLa9IxkBACMw9Jeu+Ji6b/+K33sNb/F45TL48i6tJen9gLmCMrcBteQjNgVFCTWvPuF3+JxyuVxZNv0LNnAGmM1DeB/QZnb4Bpu08A4yepHxNbAGhn4IsaD8gDAOFRG7KJR6amnEsef/WwieyeeI+fyOJJ7iWRb2ktlBDBAUOY2uIbfALveXunSSxPHBw96/x+I3+JxyuVxpBtY0+/RwAoYJChzG1zDbRoYJ+vSXhpYAcBYJCMwzohLe7lNAwDGIRmBcWJZGlhTlREaWAHAOCQjME6ySZWlvQAQDCQjME40y6Znyaf2xugZAQDjkIzAOPFsT+0lGQEAY7Geyq6oSHrwwfSx1/wWj1Muj4MGVsBwQZnb4BqSEbvCQumKK7yOIs1v8Tjl8jjSS3vTyQgNrIBBgjK3wTXcpoFxkk2q4RCVEQAIAiojdtGo9NxziePZs73fFdBv8Tjl8jiyPrU3xKZngDGCMrfBNfwG2PX2Sp/5TOLYD1sU+y0ep1weR9an9tLACpgjKHMbXMNtGhiHpb0AECwkIzAOS3sBIFhIRmCcZGUkWwMryQgAmMdRMrJu3TpVV1erpKRENTU12rx586g+9+tf/1oFBQX62Mc+5uTHApLSK2YKbA+nCbMdPAAYK+dkZOPGjVq2bJlWrlypbdu2adasWZozZ46am5tH/FxHR4cWLFigT37yk46DBSTbpmf2yshAYhKnMgIAxsk5GVmzZo0WLVqkxYsXa9q0aVq7dq0mT56s9evXj/i5r3/967rssss0Y8YMx8EC0jCbnrG0FwCMldN6qr6+Pm3dulXLly/PeL+urk5btmwZ9nMPPvig/vjHP+rhhx/Wrbfeetif09vbq97e3tTXnZ2duYTpXFGR9IMfpI+95rd4nHJ5HNm3gw9nfA+AjwVlboNrckpG2tvbFYvFVFlZmfF+ZWWlWltbs37m97//vZYvX67NmzerYJRryRsaGnTLLbfkEpo7Cgulq68++j93OH6LxymXx5Fa2huybwef+F+SEcAAQZnb4BpHDawh218CkmRZ1pD3JCkWi+myyy7TLbfcog9/+MOj/vNXrFihjo6O1GvPnj1OwkRApZb22hpYIzSwAoCxcqqMjB8/XpFIZEgVpK2tbUi1RJIOHDig119/Xdu2bdM111wjSYrH47IsSwUFBXr++ed18cUXD/lccXGxiouLcwnNHbGYlFwZNGuWFIkc/Rj8HI9TLo8j69LeCEt7AWMEZW6Da3JKRoqKilRTU6OmpiZ99rOfTb3f1NSkf/iHfxhyfnl5uXbs2JHx3rp16/TCCy/o8ccfV3V1tcOw86SnR7roosTxwYPSmDHE4waXx5Fa2pulgZVkBDBAUOY2uCbnBwLU19fr8ssvV21trWbMmKH77rtPzc3NWrJkiaTELZZ9+/bpoYceUjgc1vTp0zM+f9JJJ6mkpGTI+8BojdTAytJeADBPzsnI/PnztX//fq1evVotLS2aPn26GhsbNXXqVElSS0vLYfccAY5E1qW9A91PLO0FAPM4elTi0qVLtXTp0qzf27Bhw4ifXbVqlVatWuXkxwKSDlMZoYEVAIzDs2lgnGxLeyNURgDAWCQjMA4NrAAQLCQjME6UHVgBIFAc9YwEVmGh9N3vpo+95rd4nHJ5HPERGlhJRgADBGVug2tIRuyKiqSbbvI6ijS/xeOUy+Pg2TSA4YIyt8E13KaBcUZa2st28ABgHiojdrGY9MYbieOPf9z7LYr9Fo9TLo8jmXBkbAc/kI1YVuI2jr1qAsBngjK3wTUkI3Y9PdK55yaO/bBFsd/iccrlcUSzVEbsy3xjlqWwSEYA3wrK3AbXcJsGxkk2sEay3KaR6BsBANOQjMA4Iy3tlUhGAMA0JCMwzkhLeyWaWAHANCQjME40HpeUvYFVkmIxkhEAMAnJCIyTvAtTELFvB5/+PpURADALyQiMk9r0zFYZCYVCqYbWOD0jAGAUlvbaFRZK3/lO+thrfovHKZfHkW3TMymxvDcmiyf3An4XlLkNriEZsSsqklat8jqKNL/F45TL44hlWdorDTSxxlhNA/heUOY2uIbbNDBONMttGonn0wCAqaiM2MXj0q5dieNp0zLXixKPcy6PIz7QoGpvYJXSTaw0sAI+F5S5Da4hGbE7dEiaPj1x7Ictiv0Wj1MujyMaG7q0V5IKIlRGACMEZW6Da0hHYZzU0t7BPSMDyQnJCACYhWQExhmugTWZnJCMAIBZSEZgnOGSkQjJCAAYiWQExkk2qGZd2isaWAHANCQjMIplWSPcpqGBFQBMRDICo9jzjEhomKW9JCMAYBSW9toVFko33pg+9prf4nHKxXEkn9grSWEqI4CZgjK3wTUkI3ZFRdK//ZvXUaT5LR6nXByHLRcZurSXBlbADEGZ2+AabtPAKPbKCEt7ASAYqIzYxeNSc3PieMoU77co9ls8Trk4DntlZOhqGpIRwAhBmdvgGpIRu0OHpOrqxLEftij2WzxOuTgO+7LdwQ2sqcoIS3sBfwvK3AbXkI7CKMnbNKHQ0AbWCNvBA4CRSEZglORtmsFVEcm26RnJCAAYhWQERklWRgZXRSSW9gKAqUhGYJRkZWTwsl6JBlYAMBXJCIySrIxku03D0l4AMBPJCIwSTz4kL5KlMhJiNQ0AmIilvXYFBdLSpeljr/ktHqdcHEdshAZWKiOAIYIyt8E1/BbYFRdLd9/tdRRpfovHKRfHkbpNk6VnJEIyApghKHMbXMNtGhgltbSXBlYACAwqI3aWJbW3J47Hj0/srEU8R87FcYxUGeE2DWCIoMxtcA3JiF13t3TSSYljP2xR7Ld4nHJxHKkG1myVERpYATMEZW6Da7hNA6NEYwPJCA2sABAYJCMwSmykygjJCAAYiWQERkkmGiP1jERJRgDAKCQjMMpIyUjyvTjJCAAYhWQERqGBFQCCh2QERkk1sGa7TROhZwQATMTSXruCAmnhwvSx1/wWj1MujiNVGcmymiZVGSEZAfwtKHMbXMNvgV1xsbRhg9dRpPktHqdcHEeyOTXMpmeAuYIyt8E13KaBUZKJRgFLewEgMKiM2FlWYmdASSor836LYr/F45SL42BpLxAAQZnb4BoqI3bd3dJxxyVeyf9QiOfIuTgOlvYCARCUuQ2uIRmBUUbVwMrSXgAwCskIjBIdxW0aekYAwCwkIzBKfIRkhAZWADATyQiMQmUEAIKHZARGGamBlcoIAJiJZARGSSUjWRpYWdoLAGZylIysW7dO1dXVKikpUU1NjTZv3jzsuU8++aQ+/elP68QTT1R5eblmzJih5557znHAeRWJSF/4QuIViXgdjf/iccrFccRGeFBeMkGJs5oG8LegzG1wTc6bnm3cuFHLli3TuhFrMQoAABXeSURBVHXrdN555+nee+/VnDlztHPnTk2ZMmXI+S+++KI+/elP6/bbb9fxxx+vBx98UJdccoleeeUVnX322a4MwjUlJdJjj3kdRZrf4nHKxXGM1MAa4TYNYIagzG1wTc6VkTVr1mjRokVavHixpk2bprVr12ry5Mlav3591vPXrl2rb3zjGzrnnHN02mmn6fbbb9dpp52mZ5555oiDx7FnpAZWkhEAMFNOyUhfX5+2bt2qurq6jPfr6uq0ZcuWUf0Z8XhcBw4c0AknnDDsOb29vers7Mx4ARJLewEgiHJKRtrb2xWLxVRZWZnxfmVlpVpbW0f1Z/z7v/+7urq6dOmllw57TkNDgyoqKlKvyZMn5xKmc11diWckhEKJY6/5LR6nXBwHS3uBAAjK3AbXOGpgDQ1ayWBZ1pD3svnJT36iVatWaePGjTrppJOGPW/FihXq6OhIvfbs2eMkTARQjO3gASBwcmpgHT9+vCKRyJAqSFtb25BqyWAbN27UokWL9Nhjj+lTn/rUiOcWFxeruLg4l9BwjIjFeGovAARNTpWRoqIi1dTUqKmpKeP9pqYmzZw5c9jP/eQnP9EVV1yhH//4x5o3b56zSAEdZmkvT+0FACPlvLS3vr5el19+uWprazVjxgzdd999am5u1pIlSyQlbrHs27dPDz30kKREIrJgwQJ973vf09/93d+lqiqlpaWqqKhwcSg4Foy0A2uEyggAGCnnZGT+/Pnav3+/Vq9erZaWFk2fPl2NjY2aOnWqJKmlpUXNzc2p8++9915Fo1FdffXVuvrqq1PvL1y4UBs2bDjyEeCYMppkhMoIAJgl52REkpYuXaqlS5dm/d7gBONXv/qVkx8BZBWngRUAAsdRMhJYkYg0d2762Gt+i8cpF8cRTTawRrI0sEZY2gsYIShzG1xDMmJXUiL9/OdeR5Hmt3iccnEco1raSzIC+FtQ5ja4hqf2wigj9Yyw6RkAmIlkBEYZTQMryQgAmIVkxK6rSxozJvHywxbFfovHKRfHwdJeIACCMrfBNfSMDNbd7XUEmfwWj1MujWNUS3tZTQP4X1DmNriCygiMMqqlvVRGAMAoJCMwCk/tBYDgIRmBUWhgBYDgIRmBUUhGACB4SEZglFElIzSwAoBRWE1jFw5LF1yQPvaa3+JxysVxpJKRLA2s9sqIZVkKZTkHgA8EZW6Da0hG7EpLJT892M9v8Tjl4jhS28Fnq4zYko+4JWV5fA0APwjK3AbXkJLCKCPdpgnb3ovG40ctJgDAkSEZgVFG82waSSIXAQBzkIzYdXVJJ56YePlhi2K/xeOUi+MYTQOrRBMr4GtBmdvgGnpGBmtv9zqCTH6LxymXxjHqZCRGMgL4WlDmNriCygiMEhthO3j7e1RGAMAcJCMwyuEaWJP5CA2sAGAOkhEYZaRkREpXR8hFAMAcJCMwyuGSkeTyXiojAGAOkhEY5XDJSHJ5L7kIAJiD1TR24bBUW5s+9prf4nHKxXHER9iBVUrfpqGBFfCxoMxtcA3JiF1pqfTaa15Hkea3eJxycRzRgcpIwXDJSCT5fBpKI4BvBWVug2tISWGU5G2a8DAPwUtVRshFAMAYJCMwSixVGcn+qxuhgRUAjEMyYtfdLZ1ySuLV3e11NP6LxykXx5GqjAzzmxuhgRXwv6DMbXANPSN2liW991762Gt+i8cpF8dx2H1GqIwA/heUuQ2uoTICo8QOt5omWRlhggMAY5CMwBjxuJX6R1S2Z9PY34/yoDwAMAbJCIxh3zvkcA2s7DMCAOYgGYExkv0iEg2sABAkJCMwhj0ZYWkvAAQHq2nsQiHpjDPSx17zWzxOuTQO+62Xw1ZGuE0D+FdQ5ja4hmTErqxMeustr6NI81s8Trk0jpitKXXYBtYwDayA7wVlboNruE0DY9grI4d7UB6VEQAwB8kIjJF+Lo0UGqYyEk71jJCMAIApSEbsurulM89MvPywRbHf4nHKpXEc7rk0ie+FMs4F4ENBmdvgGnpG7CxL2rkzfew1v8XjlEvjONxzaSQaWAEjBGVug2uojMAYo6mM0MAKAOYhGYExkg2sw/SuSqKBFQBMRDICY6QqI5FRVEboGQEAY5CMwBjp1TTDl0bS28GTjACAKUhGYIxkMjJCYYSlvQBgIFbT2IVC0tSp6WOv+S0ep1waB0t7gYAIytwG15CM2JWVSe++63UUaX6LxymXxpFqYB1paS8NrID/BWVug2u4TQNj5LS0l8oIABiDZATGsG8HPxwaWAHAPCQjdocOSeeck3gdOuR1NP6LxymXxkFlBAiIoMxtcA09I3bxuPT66+ljr/ktHqdcGkd6O3iW9gJGC8rcBtdQGYExRrW0N0RlBABMQzICY6STkVEs7WU1DQAYg2QExkhWOyKjaGCN8aA8ADAGyYiPWfzrPkNy75DRNLBSGQEAc5CM+NjPd7Skjl/Y9YGHkfhDuoF1+HNoYAUA85CMDDZ+fOLlsb909amhcZf2l5Zrf2m5bv35Th3sjXodlnMuXFeW9gIB4pO5Fv7A0l67MWOkP/3J6ygkSbc17tK+/oi+fNvTOtQf054/H9Kdz/1Oq/7+TK9Dy51L13VUS3vZDh7wPx/NtfAHR5WRdevWqbq6WiUlJaqpqdHmzZtHPH/Tpk2qqalRSUmJTj31VN1zzz2Ogj1WbPlDux7fulehkNTwuY/qtn/8iCTpP19+V9v3/NXb4DyUroyMkIwMdLdGaWAFAGPknIxs3LhRy5Yt08qVK7Vt2zbNmjVLc+bMUXNzc9bzd+/erblz52rWrFnatm2bbr75Zl133XV64oknjjj4IOrpj+nmp3ZIki7/u6mqmfohfeLDJ+qzZ58sy5KWP/Fb9ceOzU2CUg/KG+Epn8nKCA2sAGCOnJORNWvWaNGiRVq8eLGmTZumtWvXavLkyVq/fn3W8++55x5NmTJFa9eu1bRp07R48WJdddVVuvPOO484eNcdOiRdeGHi5dEWxd9/4fd6d3+3JpSX6KZPTEnF862LT9HxZYX6f60H9KPNuz2JzTGXrmt0FJuepVbT0DMC+JcP5lr4S049I319fdq6dauWL1+e8X5dXZ22bNmS9TMvv/yy6urqMt6bPXu27r//fvX396uwsHDIZ3p7e9Xb25v6urOzM5cwR+2JrXv15vsdqa8Le7p186ZNkqTbf/am+kvK8vJzhxOPW3rklUSF6ZZ/OFNjiyLSQDzjygr1rXln6MbHfqO1v3hbH3T2aIQCQV6NtuhgWZbillTQ06XvDIxj9X/vUKy0TKFQSHHLUl80rr5oXL2xuPqjcRVEQiqKhFVUkHjZm1V3tSR+D0bTwPrbvR369n+/OSTWUEgKSQoNc/Esy5I1MMaYZelgT1R/PdSvv3b36a/d/eqPxVVeUqjy0gJVlBZqbEmhLMtSXyyuvmjify3LUklhRKUDr5LCsEKhUMafnRQOpWNx4/9P+589eGm4fcxH63cnGUJy7MlenpBCGf9fePW7DG94Pdciu89/fJKmn1zhyc/OKRlpb29XLBZTZWVlxvuVlZVqbW3N+pnW1tas50ejUbW3t2vixIlDPtPQ0KBbbrkll9Ac2fT2n/T0b95PfV3a16ObB47/78vNOlRUkvcYspl9ZqVmnzlB6urKeP/zHz9ZT76xV1v+uF8btrzrSWxOlPb16DsDxz95dc8RX9fy0qEJbFLFwPd2t3dpd3vXsOcdiZaOnrz8ucCxwi9zLTKdPeVDZiQjSYP/VWlZ1rD/0hzu/GzvJ61YsUL19fWprzs7OzV58mQnoY7o02dUavIJpamvC3u6pf9IHH/9gmpPsvWSgogunzE16/dCoZC+96Wz9eNXmtUXix3lyHIXDoUUklTYcyh1Xf/5wr9Rf0mpLCtRFUhWQIoiYRUWhBWLW+rtj6svFldvNK5YPK7En5JQXBDWF2onDfsz535kov7S3a+/dPWl/uWtgTgSVQlLliUl/p0+fMzJz4wtKdDxZUU6vrRQx5cVqjAS1oGeqDp7+tV5qF+dPf0Kh0KpMRQVhBUKST39cR3qi+lQf0w9/TFZlrLGo4Hq0XDxOJG8Xqmfl/xZiR/n6s8abTyhgQpQaCAuyxpaLcGxww9zLYY67aTjPPvZOSUj48ePVyQSGVIFaWtrG1L9SJowYULW8wsKCjRu3LisnykuLlZxcXEuoTlyyVlVuuSsqvQbtkrEsk/9n8TyM585cWyxrv/UaV6HkRvbdb3uk6fl9bqWFEa06PzqvP35AFxgwFyLoyunBtaioiLV1NSoqakp4/2mpibNnDkz62dmzJgx5Pznn39etbW1WftFAADAsSXn1TT19fX60Y9+pAceeEC7du3SDTfcoObmZi1ZskRS4hbLggULUucvWbJE7733nurr67Vr1y498MADuv/++3XjjTe6NwoAAGCsnHtG5s+fr/3792v16tVqaWnR9OnT1djYqKlTE30OLS0tGXuOVFdXq7GxUTfccIPuvvtuVVVV6a677tLnP/9590bhpjKf3bv0WzxOBWUcANzBnACbkGXAo2E7OztVUVGhjo4OlZeXex0OAAAYhdH+/c2D8gAAgKdIRgAAgKdIRux6eqR58xKvHh9sbOW3eJwKyjgAuIM5AYM42vQssGIxqbExfew1v8XjVFDGAcAdzAkYhMoIAADwFMkIAADwFMkIAADwFMkIAADwFMkIAADwlBGraZKbxHZ2dub3B9meJKnOTu+7vP0Wj1NBGQcAdzAnHDOSf28fbrN3I7aD37t3ryZPnux1GAAAwIE9e/Zo0qRJw37fiGQkHo/r/fff19ixYxUKhVz7czs7OzV58mTt2bOHZ97kGdf66OA6Hx1c56OD63x05PM6W5alAwcOqKqqSuHw8J0hRtymCYfDI2ZUR6q8vJxf9KOEa310cJ2PDq7z0cF1PjrydZ0rKioOew4NrAAAwFMkIwAAwFORVatWrfI6CC9FIhFdeOGFKigw4o6V0bjWRwfX+ejgOh8dXOejw+vrbEQDKwAACC5u0wAAAE+RjAAAAE+RjAAAAE+RjAAAAE8d08nIunXrVF1drZKSEtXU1Gjz5s1eh2S0hoYGnXPOORo7dqxOOukk/eM//qN+97vfZZxjWZZWrVqlqqoqlZaW6sILL9Rbb73lUcTB0NDQoFAopGXLlqXe4zq7Y9++ffrqV7+qcePGqaysTB/72Me0devW1Pe5zkcuGo3qW9/6lqqrq1VaWqpTTz1Vq1evVjweT53DdXbmxRdf1CWXXKKqqiqFQiH99Kc/zfj+aK5rb2+vrr32Wo0fP15jxozR3//932vv3r3uB2sdox599FGrsLDQ+uEPf2jt3LnTuv76660xY8ZY7733ntehGWv27NnWgw8+aL355pvW9u3brXnz5llTpkyxDh48mDrnjjvusMaOHWs98cQT1o4dO6z58+dbEydOtDo7Oz2M3Fyvvvqqdcopp1gf/ehHreuvvz71Ptf5yP35z3+2pk6dal1xxRXWK6+8Yu3evdv6xS9+Yf3hD39IncN1PnK33nqrNW7cOOtnP/uZtXv3buuxxx6zjjvuOGvt2rWpc7jOzjQ2NlorV660nnjiCUuS9dRTT2V8fzTXdcmSJdbJJ59sNTU1WW+88YZ10UUXWWeddZYVjUZdjfWYTUbOPfdca8mSJRnvnX766dby5cs9iih42traLEnWpk2bLMuyrHg8bk2YMMG64447Uuf09PRYFRUV1j333ONVmMY6cOCAddppp1lNTU3WBRdckEpGuM7u+OY3v2mdf/75w36f6+yOefPmWVdddVXGe5/73Oesr371q5ZlcZ3dMjgZGc11/etf/2oVFhZajz76aOqcffv2WeFw2Hr22Wddje+YvE3T19enrVu3qq6uLuP9uro6bdmyxaOogqejo0OSdMIJJ0iSdu/erdbW1ozrXlxcrAsuuIDr7sDVV1+tefPm6VOf+lTG+1xndzz99NOqra3VF7/4RZ100kk6++yz9cMf/jD1fa6zO84//3z9z//8j95++21J0m9+8xu99NJLmjt3riSuc76M5rpu3bpV/f39GedUVVVp+vTprl/7Y3JLu/b2dsViMVVWVma8X1lZqdbWVo+iChbLslRfX6/zzz9f06dPl6TUtc123d97772jHqPJHn30Ub3xxht67bXXhnyP6+yOd955R+vXr1d9fb1uvvlmvfrqq7ruuutUXFysBQsWcJ1d8s1vflMdHR06/fTTFYlEFIvFdNttt+nLX/6yJH6f82U017W1tVVFRUX60Ic+NOQct/+uPCaTkaRQKJTxtWVZQ96DM9dcc41++9vf6qWXXhryPa77kdmzZ4+uv/56Pf/88yopKRn2PK7zkYnH46qtrdXtt98uSTr77LP11ltvaf369VqwYEHqPK7zkdm4caMefvhh/fjHP9aZZ56p7du3a9myZaqqqtLChQtT53Gd88PJdc3HtT8mb9OMHz9ekUhkSGbX1tY2JEtE7q699lo9/fTT+uUvf6lJkyal3p8wYYIkcd2P0NatW9XW1qaamhoVFBSooKBAmzZt0l133aWCgoLUteQ6H5mJEyfqjDPOyHhv2rRpam5ulsTvs1tuuukmLV++XF/60pf0kY98RJdffrluuOEGNTQ0SOI658toruuECRPU19env/zlL8Oe45ZjMhkpKipSTU2NmpqaMt5vamrSzJkzPYrKfJZl6ZprrtGTTz6pF154QdXV1Rnfr66u1oQJEzKue19fnzZt2sR1z8EnP/lJ7dixQ9u3b0+9amtr9ZWvfEXbt2/XqaeeynV2wXnnnTdkafrbb7+tqVOnSuL32S3d3d0KhzP/KopEIqmlvVzn/BjNda2pqVFhYWHGOS0tLXrzzTfdv/autsMaJLm09/7777d27txpLVu2zBozZoz17rvveh2asf75n//ZqqiosH71q19ZLS0tqVd3d3fqnDvuuMOqqKiwnnzySWvHjh3Wl7/8ZZboucC+msayuM5uePXVV62CggLrtttus37/+99bjzzyiFVWVmY9/PDDqXO4zkdu4cKF1sknn5xa2vvkk09a48ePt77xjW+kzuE6O3PgwAFr27Zt1rZt2yxJ1po1a6xt27altrAYzXVdsmSJNWnSJOsXv/iF9cYbb1gXX3wxS3vddvfdd1tTp061ioqKrI9//OOpJahwRlLW14MPPpg6Jx6PW9/5znesCRMmWMXFxdYnPvEJa8eOHd4FHRCDkxGuszueeeYZa/r06VZxcbF1+umnW/fdd1/G97nOR66zs9O6/vrrrSlTplglJSXWqaeeaq1cudLq7e1NncN1duaXv/xl1jl54cKFlmWN7roeOnTIuuaaa6wTTjjBKi0ttT7zmc9Yzc3NrscasizLcrfWAgAAMHrHZM8IAADwD5IRAADgKZIRAADgKZIRAADgKZIRAADgKZIRAADgKZIRAADgKZIRAADgKZIRAADgKZIRAADgKZIRAADgKZIRAADgqf8PDJOJufnhq0wAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "logits[0, -1].shape\n",
    "plt.plot((logits[0, -1].softmax(-1)).detach().numpy())\n",
    "plt.axvline(x=1, color='r', linestyle='--', label='Vertical Line 1')\n",
    "plt.axvline(x=20, color='r', linestyle='--', label='Vertical Line 1')\n",
    "plt.axvline(x=78, color='r', linestyle='--', label='Vertical Line 2')\n",
    "plt.axvline(x=9, color='r', linestyle='--', label='Vertical Line 2')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(25)"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.argmax(logits[0, -1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_60018/2073116116.py:1: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  init_dict = torch.load('./sort5/init.pth')\n"
     ]
    }
   ],
   "source": [
    "init_dict = torch.load('./sort5/init.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20000, 11)"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tr = init_dict['train_data']\n",
    "te = init_dict['test_data']\n",
    "tr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "all() received an invalid combination of arguments - got (numpy.ndarray, dim=int), but expected one of:\n * (Tensor input, *, Tensor out = None)\n * (Tensor input, tuple of ints dim = None, bool keepdim = False, *, Tensor out = None)\n * (Tensor input, int dim, bool keepdim = False, *, Tensor out = None)\n * (Tensor input, name dim, bool keepdim = False, *, Tensor out = None)\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[12], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m common_rows \u001b[38;5;241m=\u001b[39m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mall\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtr\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m]\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m==\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mte\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      2\u001b[0m num_common_rows \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39msum(torch\u001b[38;5;241m.\u001b[39many(common_rows, dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m))\n",
      "\u001b[0;31mTypeError\u001b[0m: all() received an invalid combination of arguments - got (numpy.ndarray, dim=int), but expected one of:\n * (Tensor input, *, Tensor out = None)\n * (Tensor input, tuple of ints dim = None, bool keepdim = False, *, Tensor out = None)\n * (Tensor input, int dim, bool keepdim = False, *, Tensor out = None)\n * (Tensor input, name dim, bool keepdim = False, *, Tensor out = None)\n"
     ]
    }
   ],
   "source": [
    "common_rows = torch.all(tr[:100][:, None] == te[:100], dim=2)\n",
    "num_common_rows = torch.sum(torch.any(common_rows, dim=0))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_common_rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0 18 23 78 91 91 91]\n",
      "[  0  13  39  40  69 100 100]\n",
      "[ 0 14 15 16 59 86 86]\n",
      "[ 0 13 35 43 64 97 97]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tr_s = tr.sort(axis=-1)\n",
    "te_s = te.sort(axis=-1)\n",
    "\n",
    "cnt = 0\n",
    "for i, x in enumerate(te):\n",
    "    # print(x.shape, )\n",
    "    if torch.tensor(tr == te[i]).all(dim=1).any():\n",
    "        # print(x)\n",
    "        cnt+=1\n",
    "\n",
    "        print(x)\n",
    "cnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnt = 0\n",
    "for i, x in enumerate(te):\n",
    "    # print(x.shape, )\n",
    "    if torch.tensor(tr == te[i]).all(dim=1).any():\n",
    "        # print(x)\n",
    "        cnt+=1\n",
    "        print(cnt)\n",
    "cnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(False)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([  9,  18,  19,  20,  21,  22,  23,  24,  25,  26,  27,  28,  29,\n",
       "         30,  31,  32,  33,  34,  35,  36,  37,  38,  39,  40,  41,  42,\n",
       "         43,  44,  45,  46,  47,  48,  49,  50,  51,  52,  53,  54,  55,\n",
       "         56,  57,  58,  59,  60,  61,  62,  63,  64,  65,  66,  67,  68,\n",
       "         69,  70,  71,  72,  73,  74,  75,  76,  77,  78,  79,  80,  81,\n",
       "         82,  83,  84,  85,  86,  87,  88,  89,  90,  91,  92,  93,  94,\n",
       "         95,  96,  97,  98,  99, 100]),\n",
       " array([  1,   1,   1,   1,   3,   1,   3,   1,   2,   4,  10,   4,   7,\n",
       "          4,   8,   6,  13,  18,   6,   9,  12,  20,  25,  21,  26,  22,\n",
       "         34,  24,  32,  41,  52,  56,  62,  55,  66,  70,  79,  78,  89,\n",
       "        102,  96, 106, 139, 108, 136, 145, 145, 163, 186, 192, 202, 221,\n",
       "        240, 225, 250, 247, 269, 283, 303, 346, 332, 358, 390, 377, 404,\n",
       "        431, 441, 453, 515, 509, 551, 638, 585, 664, 654, 711, 746, 817,\n",
       "        842, 837, 836, 901, 975, 962]))"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(tr[:, 10].flatten(), return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([  0,   1,   2,   3,   4,   5,   6,   7,   8,   9,  10,  11,  12,\n",
       "         13,  14,  15,  16,  17,  18,  19,  20,  21,  22,  23,  24,  25,\n",
       "         26,  27,  28,  29,  30,  31,  32,  33,  34,  35,  36,  37,  38,\n",
       "         39,  40,  41,  42,  43,  44,  45,  46,  47,  48,  49,  50,  51,\n",
       "         52,  53,  54,  55,  56,  57,  58,  59,  60,  61,  62,  63,  64,\n",
       "         65,  66,  67,  68,  69,  70,  71,  72,  73,  74,  75,  76,  77,\n",
       "         78,  79,  80,  81,  82,  83,  84,  85,  86,  87,  88,  89,  90,\n",
       "         91,  92,  93,  94,  95,  96,  97,  98,  99, 100]),\n",
       " array([20000,   983,  1032,  1018,   997,  1003,  1084,  1044,   962,\n",
       "         1004,  1012,   996,   991,  1013,   986,  1011,  1020,  1018,\n",
       "          965,   924,  1036,   953,   999,   986,   934,  1025,  1057,\n",
       "         1056,   964,   964,  1021,   989,   994,   913,   977,   971,\n",
       "         1000,  1042,  1033,   995,  1008,   977,   987,   982,  1010,\n",
       "         1027,  1081,   956,  1003,  1011,   990,  1039,  1015,  1019,\n",
       "          969,  1004,   999,   969,   981,  1019,   939,   994,   973,\n",
       "          976,   979,   974,   998,  1009,  1079,  1060,   942,  1030,\n",
       "         1024,   985,  1004,   950,  1025,   983,  1002,  1037,   975,\n",
       "         1017,   971,   996,   942,  1028,   969,   966,  1055,   998,\n",
       "         1058,   967,  1022,  1016,  1032,  1045,   980,   983,   992,\n",
       "         1030,   977]))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(tr[:,:-1].flatten(), return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "embed.W_E\n",
      "pos_embed.W_pos\n",
      "blocks.0.attn.W_K\n",
      "blocks.0.attn.W_Q\n",
      "blocks.0.attn.W_V\n",
      "blocks.0.attn.W_O\n",
      "blocks.0.mlp.W_in\n",
      "blocks.0.mlp.b_in\n",
      "blocks.0.mlp.W_out\n",
      "blocks.0.mlp.b_out\n",
      "unembed.W_U\n"
     ]
    }
   ],
   "source": [
    "for name, parma in model.named_parameters():\n",
    "    print(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.7014, device='cuda:0')"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "posemb = state_dict['model']['pos_embed.W_pos']\n",
    "torch.nn.functional.cosine_similarity(posemb[2], posemb[3], dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 0.1259,  0.2533,  0.0897,  0.0441,  0.1471,  0.1510,  0.0424,  0.1423,\n",
       "         0.1608,  0.0490,  0.1179,  0.1008,  0.1049,  0.0392,  0.1429,  0.1048,\n",
       "         0.0712,  0.1164,  0.1196,  0.1489,  0.1084,  0.0857,  0.0750,  0.0248,\n",
       "         0.0666,  0.1847,  0.1384,  0.1600,  0.0889,  0.0506,  0.1518,  0.1631,\n",
       "         0.0708,  0.1518,  0.0415,  0.0947,  0.0629,  0.1294,  0.0331,  0.0285,\n",
       "         0.1084,  0.1484,  0.0865,  0.1551,  0.1805,  0.0916,  0.0895,  0.1082,\n",
       "         0.1252,  0.0488,  0.0379,  0.0220,  0.1183,  0.1610,  0.1272,  0.0788,\n",
       "         0.2069,  0.0876,  0.2081, -0.0110,  0.0467,  0.1589,  0.0831,  0.0567,\n",
       "         0.0935,  0.0597,  0.1389,  0.0991,  0.0974,  0.1015,  0.1642,  0.0279,\n",
       "         0.1283,  0.0854,  0.1110,  0.1279,  0.1414,  0.1395,  0.1015,  0.0367,\n",
       "         0.1073,  0.0848,  0.0635,  0.1744,  0.0748,  0.1349,  0.0308,  0.1319,\n",
       "         0.0593,  0.1497,  0.1194,  0.2095,  0.0658,  0.1432,  0.1131,  0.0313,\n",
       "         0.1124,  0.0627,  0.1211,  0.1587,  0.1326,  0.2092,  0.0797,  0.0246,\n",
       "         0.1262,  0.0202,  0.1089,  0.0152,  0.0963,  0.1010,  0.0937,  0.0883,\n",
       "         0.0452,  0.0849,  0.1027,  0.0672,  0.1258,  0.0747,  0.1371,  0.0705,\n",
       "         0.0986,  0.0527,  0.1498,  0.0526,  0.0620,  0.1153,  0.0651,  0.0792],\n",
       "       device='cuda:0')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state_dict['model']['embed.W_E'].norm(dim=-1) - init_dict['model']['embed.W_E'].norm(dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 4, 128])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cache['blocks.0.hook_resid_pre']"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}