diff --git a/.gitignore b/.gitignore index 65fd0fc78..2c1ef1364 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ __pycache__/ dist/ output* .DS_Store +env/ +build/ +.eggs/ diff --git a/data/mnist/raw/t10k-images-idx3-ubyte b/data/mnist/raw/t10k-images-idx3-ubyte new file mode 100644 index 000000000..1170b2cae Binary files /dev/null and b/data/mnist/raw/t10k-images-idx3-ubyte differ diff --git a/data/mnist/raw/t10k-images-idx3-ubyte.gz b/data/mnist/raw/t10k-images-idx3-ubyte.gz new file mode 100644 index 000000000..5ace8ea93 Binary files /dev/null and b/data/mnist/raw/t10k-images-idx3-ubyte.gz differ diff --git a/data/mnist/raw/t10k-labels-idx1-ubyte b/data/mnist/raw/t10k-labels-idx1-ubyte new file mode 100644 index 000000000..d1c3a9706 Binary files /dev/null and b/data/mnist/raw/t10k-labels-idx1-ubyte differ diff --git a/data/mnist/raw/t10k-labels-idx1-ubyte.gz b/data/mnist/raw/t10k-labels-idx1-ubyte.gz new file mode 100644 index 000000000..a7e141541 Binary files /dev/null and b/data/mnist/raw/t10k-labels-idx1-ubyte.gz differ diff --git a/data/mnist/raw/train-images-idx3-ubyte b/data/mnist/raw/train-images-idx3-ubyte new file mode 100644 index 000000000..bbce27659 Binary files /dev/null and b/data/mnist/raw/train-images-idx3-ubyte differ diff --git a/data/mnist/raw/train-images-idx3-ubyte.gz b/data/mnist/raw/train-images-idx3-ubyte.gz new file mode 100644 index 000000000..b50e4b6bc Binary files /dev/null and b/data/mnist/raw/train-images-idx3-ubyte.gz differ diff --git a/data/mnist/raw/train-labels-idx1-ubyte b/data/mnist/raw/train-labels-idx1-ubyte new file mode 100644 index 000000000..d6b4c5db3 Binary files /dev/null and b/data/mnist/raw/train-labels-idx1-ubyte differ diff --git a/data/mnist/raw/train-labels-idx1-ubyte.gz b/data/mnist/raw/train-labels-idx1-ubyte.gz new file mode 100644 index 000000000..707a576bb Binary files /dev/null and b/data/mnist/raw/train-labels-idx1-ubyte.gz differ diff --git a/example/language_model/lm.py b/example/language_model/lm.py deleted file mode 100644 index ac0f72e6b..000000000 --- a/example/language_model/lm.py +++ /dev/null @@ -1,88 +0,0 @@ -import torch -import torch.nn as nn -import numpy as np - -# Define the characters in the vocabulary -text = open('./names.txt').read().splitlines() -characters = sorted(list(set(text))) # 'text' is the training data -sequence_length = 10 -learning_rate = 0.1 -num_epochs = 1000 -max_length = 10 - -# Create character-to-index and index-to-character mappings -char_to_idx = {char: idx for idx, char in enumerate(characters)} -idx_to_char = {idx: char for idx, char in enumerate(characters)} - -# Convert the text into sequences of indices -sequences = [] -sequence_length = 3 -for i in range(len(text) - sequence_length): - sequence = text[i:i+sequence_length] - target = text[i+sequence_length] - sequences.append((sequence, target)) - -# Generate input and target data -X = np.zeros((len(sequences), sequence_length, len(characters)), dtype=np.float32) -y = np.zeros((len(sequences), len(characters)), dtype=np.float32) -for i, (sequence, target) in enumerate(sequences): - for t, char in enumerate(sequence): - X[i, t, char_to_idx[char]] = 1.0 - y[i, char_to_idx[target]] = 1.0 - -# Convert numpy arrays to PyTorch tensors -X = torch.from_numpy(X) -y = torch.from_numpy(y) - -# Define the model architecture -class CharFFN(nn.Module): - def __init__(self, input_size, hidden_size, output_size): - super(CharFFN, self).__init__() - self.flatten = nn.Flatten() - self.fc1 = nn.Linear(input_size, hidden_size) - self.relu = nn.ReLU() - self.fc2 = nn.Linear(hidden_size, output_size) - - def forward(self, x): - x = self.flatten(x) - x = self.fc1(x) - x = self.relu(x) - x = self.fc2(x) - return x - -input_size = sequence_length * len(characters) -hidden_size = 128 -output_size = len(characters) -model = CharFFN(input_size, hidden_size, output_size) - -# Define the loss function and optimizer -criterion = nn.CrossEntropyLoss() -optimizer = torch.optim.Adam(model.parameters(), lr=0.01) - -# Training loop -num_epochs = 100 -for epoch in range(num_epochs): - # Forward pass - outputs = model(X) - loss = criterion(outputs, torch.argmax(y, dim=1)) - - # Backward and optimize - optimizer.zero_grad() - loss.backward() - optimizer.step() - - if (epoch + 1): - print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {loss.item():.4f}') - -# Generate sample text using the trained model -start_sequence = 'The ' -generated_text = start_sequence -with torch.no_grad(): - input_seq = torch.from_numpy(np.array([[char_to_idx[ch] for ch in start_sequence]])) - for _ in range(max_length): - output = model(input_seq) - predicted_idx = torch.argmax(output, dim=1) - generated_text += idx_to_char[predicted_idx.item()] - input_seq = torch.cat((input_seq[:, 1:], predicted_idx.unsqueeze(0)), dim=1) - -print(generated_text) diff --git a/example/language_model/makemore_part2_mlp.ipynb b/example/language_model/makemore_part2_mlp.ipynb new file mode 100644 index 000000000..74696a1ae --- /dev/null +++ b/example/language_model/makemore_part2_mlp.ipynb @@ -0,0 +1,482 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 62, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn.functional as F\n", + "import matplotlib.pyplot as plt # for making figures\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": {}, + "outputs": [], + "source": [ + "# read in all the words\n", + "words = open('names.txt', 'r').read().splitlines()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z', 0: '.'}\n" + ] + } + ], + "source": [ + "# build the vocabulary of characters and mappings to/from integers\n", + "chars = sorted(list(set(''.join(words))))\n", + "stoi = {s:i+1 for i,s in enumerate(chars)}\n", + "stoi['.'] = 0\n", + "itos = {i:s for s,i in stoi.items()}\n", + "print(itos)" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [], + "source": [ + "# build the dataset\n", + "\n", + "block_size = 3 # context length: how many characters do we take to predict the next one?\n", + "X, Y = [], []\n", + "for w in words:\n", + " \n", + " #print(w)\n", + " context = [0] * block_size\n", + " for ch in w + '.':\n", + " ix = stoi[ch]\n", + " X.append(context)\n", + " Y.append(ix)\n", + " #print(''.join(itos[i] for i in context), '--->', itos[ix])\n", + " context = context[1:] + [ix] # crop and append\n", + " \n", + "X = torch.tensor(X)\n", + "Y = torch.tensor(Y)" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(torch.Size([228146, 3]), torch.int64, torch.Size([228146]), torch.int64)" + ] + }, + "execution_count": 66, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X.shape, X.dtype, Y.shape, Y.dtype" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([182625, 3]) torch.Size([182625])\n", + "torch.Size([22655, 3]) torch.Size([22655])\n", + "torch.Size([22866, 3]) torch.Size([22866])\n" + ] + } + ], + "source": [ + "# build the dataset\n", + "block_size = 3 # context length: how many characters do we take to predict the next one?\n", + "\n", + "def build_dataset(words): \n", + " X, Y = [], []\n", + " for w in words:\n", + "\n", + " #print(w)\n", + " context = [0] * block_size\n", + " for ch in w + '.':\n", + " ix = stoi[ch]\n", + " X.append(context)\n", + " Y.append(ix)\n", + " #print(''.join(itos[i] for i in context), '--->', itos[ix])\n", + " context = context[1:] + [ix] # crop and append\n", + "\n", + " X = torch.tensor(X)\n", + " Y = torch.tensor(Y)\n", + " print(X.shape, Y.shape)\n", + " return X, Y\n", + "\n", + "import random\n", + "random.seed(42)\n", + "random.shuffle(words)\n", + "n1 = int(0.8*len(words))\n", + "n2 = int(0.9*len(words))\n", + "\n", + "Xtr, Ytr = build_dataset(words[:n1])\n", + "Xdev, Ydev = build_dataset(words[n1:n2])\n", + "Xte, Yte = build_dataset(words[n2:])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [], + "source": [ + "C = torch.randn((27, 2))" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([228146, 3, 2])" + ] + }, + "execution_count": 69, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "emb = C[X]\n", + "emb.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [], + "source": [ + "g = torch.Generator().manual_seed(2147483647) # for reproducibility\n", + "embdedding_size = 10\n", + "C = torch.randn((27, embdedding_size), generator=g)\n", + "W1 = torch.randn((block_size * embdedding_size, 200), generator=g)\n", + "b1 = torch.randn(200, generator=g)\n", + "W2 = torch.randn((200, 27), generator=g)\n", + "b2 = torch.randn(27, generator=g)\n", + "parameters = [C, W1, b1, W2, b2]" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "11897" + ] + }, + "execution_count": 71, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sum(p.nelement() for p in parameters) # number of parameters in total" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": {}, + "outputs": [], + "source": [ + "for p in parameters:\n", + " p.requires_grad = True" + ] + }, + { + "cell_type": "code", + "execution_count": 83, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.971953272819519\n" + ] + } + ], + "source": [ + "epochs = 200000\n", + "# learning_rates = 10**(torch.linspace(-3, 0, epochs))\n", + "used_lrs = []\n", + "lossi = []\n", + "# stepi = []\n", + "for i in range(epochs):\n", + " \n", + " # minibatch construct\n", + " ix = torch.randint(0, Xtr.shape[0], (32,))\n", + " \n", + " # forward pass\n", + " emb = C[Xtr[ix]] # (32, 3, 10)\n", + " h = torch.tanh(emb.view(-1, embdedding_size * block_size) @ W1 + b1) # (32, 200)\n", + " logits = h @ W2 + b2 # (32, 27)\n", + " loss = F.cross_entropy(logits, Ytr[ix])\n", + " # print(loss.item())\n", + " \n", + " # backward pass\n", + " for p in parameters:\n", + " p.grad = None\n", + " loss.backward()\n", + " lr = 0.01 if i < epochs / 2 else 0.001\n", + " for p in parameters:\n", + " p.data -= lr * p.grad\n", + "\n", + " # track stats\n", + " used_lrs.append(i)\n", + " # stepi.append(i)\n", + " lossi.append(loss.item())\n", + "\n", + "print(loss.item())" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 82, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAGdCAYAAADJ6dNTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABSUklEQVR4nO3dd3hUVfoH8O8kpAEpQEgjoUOoCU0gAamhySKoi4i4gCIqgsKqyOJaEFfDT1ZZbChKURFBlKKIIC3SQkkgdCKhhZKCQAol/fz+CBlmkin3ztyZuTP5fp4nz5PcuTP33JnJ3HfOec97NEIIASIiIiKVcHN0A4iIiIh0MTghIiIiVWFwQkRERKrC4ISIiIhUhcEJERERqQqDEyIiIlIVBidERESkKgxOiIiISFVqOLoBUpSVleHKlSvw9fWFRqNxdHOIiIhIAiEE8vPzERYWBjc36f0hThGcXLlyBREREY5uBhEREVng4sWLCA8Pl7y/UwQnvr6+AMpPzs/Pz8GtISIiIiny8vIQERGhvY5L5RTBScVQjp+fH4MTIiIiJyM3JYMJsURERKQqDE6IiIhIVRicEBERkaowOCEiIiJVYXBCREREqsLghIiIiFSFwQkRERGpCoMTIiIiUhWrgpM5c+ZAo9Fg2rRpJvdbtWoVWrVqBW9vb7Rv3x4bNmyw5rBERETkwiwOTg4cOIAvvvgCUVFRJvfbs2cPRo8ejQkTJuDQoUMYMWIERowYgWPHjll6aCIiInJhFgUnN2/exJgxY/Dll1+iTp06JvedP38+Bg8ejOnTp6N169Z455130KlTJ3zyyScWNZiIiIhcm0XByeTJkzF06FDExcWZ3TcxMbHKfoMGDUJiYqLR+xQWFiIvL0/vh4iIiKoH2Qv/rVixAgcPHsSBAwck7Z+ZmYng4GC9bcHBwcjMzDR6n/j4eLz99ttym+YStp7Mwu2iUgyLDnN0U4iIiBxCVs/JxYsXMXXqVHz33Xfw9va2VZswc+ZM5Obman8uXrxos2OpSVmZwISvk/DC94dwNb/Q0c0hIiJyCFk9J8nJycjOzkanTp2020pLS7Fjxw588sknKCwshLu7u959QkJCkJWVpbctKysLISEhRo/j5eUFLy8vOU1zCULn97yCYtT3rX7PARERkayek/79++Po0aNISUnR/nTp0gVjxoxBSkpKlcAEAGJiYrB161a9bZs3b0ZMTIx1LSciIiKXJKvnxNfXF+3atdPbVqtWLdSrV0+7fezYsWjQoAHi4+MBAFOnTkXv3r3xwQcfYOjQoVixYgWSkpKwcOFChU6BiIiIXIniFWLT09ORkZGh/Ts2NhbLly/HwoULER0djR9//BFr166tEuQQERERARbM1qksISHB5N8AMHLkSIwcOdLaQxEREVE1wLV1iIiISFUYnBAREZGqMDghIiIiVWFwQkRERKrC4ISIiIhUhcEJERERqQqDEyIiIlIVBidERESkKgxOiIiISFUYnBAREZGqMDghIiIiVWFwQkRERKrC4ISIiIhUhcEJERERqQqDEyIiIlIVBidERESkKgxOiIiISFUYnKiIEMLRTSAiInI4BicqpXF0A4iIiByEwQkRERGpCoMTIiIiUhUGJ0RERKQqDE6IiIhIVRicEBERkaowOCEiIiJVYXBCREREqsLghIiIiFSFwQkRERGpCoMTIiIiUhUGJ0RERKQqDE6IiIhIVRicEBERkaowOCEiIiJVYXBCREREqsLghIiIiFSFwQnZnBACQghHN4OIiJwEgxOyuQlfJ+HBT3ajtIwBChERmVfD0Q0g17ftVDYA4FRmHtqG+Tu4NUREpHbsOSG70UDj6CYQEZETYHBCREREqsLgREWYkUFERMTgRLU0Gg6BEBFR9cTgpBq5U1SKD39PxdFLuY5uChERkVEMTqqRT7afxkfb0jDsk12ObgoREZFRDE6qkZMZ+Y5uAhERkVkMToiIiEhVGJwQERGRqjA4ISIiIlVhcEJERESqwuCEiIiIVIXBiRElpWUoKC51dDMUJQRr0JJyTmbkYcrygzj31y1HN4WIXAxXJTai99wEZOcX4OisQfD2cHd0c4hUZ8Snu1FYUoajl3Pxx/S+jm4OEbkQWT0nCxYsQFRUFPz8/ODn54eYmBj89ttvRvdfunQpNBqN3o+3t7fVjbaHyzl3UFwqkJZ909FNIVKlwpIyAMCFa7cd3BIicjWygpPw8HDMmTMHycnJSEpKQr9+/TB8+HAcP37c6H38/PyQkZGh/blw4YLVjXa0Q+k38LePd2Lf2WuObgoREZHLkTWsM2zYML2/3333XSxYsAB79+5F27ZtDd5Ho9EgJCTE8haq0GML96KwpAyjFu7F+TlDHd0cIiLVen3tUdSt5YWXBrR0dFPIiVicEFtaWooVK1bg1q1biImJMbrfzZs30ahRI0RERJjtZalQWFiIvLw8vR81qejOtqUyJq8SkZNLy87Hsr3p+GjraUc3hZyM7ODk6NGjqF27Nry8vPDcc89hzZo1aNOmjcF9IyMjsXjxYqxbtw7Lli1DWVkZYmNjcenSJZPHiI+Ph7+/v/YnIiJCbjMdSolZMT8cuKhAS/SpPdy5ml+ItGyu/0PlTmbkcSaQkysotv0XOXJNsoOTyMhIpKSkYN++fZg0aRLGjRuHEydOGNw3JiYGY8eORYcOHdC7d2+sXr0a9evXxxdffGHyGDNnzkRubq725+JF5S/UtlJQXIq4D//ASytTrHqc9UcylGmQE7nv3S2I+3AHLt1ggmV1l3O7CEPm70Tf/yY4uilE5ACygxNPT080b94cnTt3Rnx8PKKjozF//nxJ9/Xw8EDHjh2RlpZmcj8vLy/tjKCKH2eRkJqNM1dvYfWhy45uitM6dlldw3hkf5dz7ji6CUTkQFYXYSsrK0NhYaGkfUtLS3H06FGEhoZae1iHEELgdlGJo5tBRETk0mTN1pk5cyaGDBmChg0bIj8/H8uXL0dCQgI2bdoEABg7diwaNGiA+Ph4AMDs2bPRvXt3NG/eHDk5OZg7dy4uXLiAp59+WvkzsYOXfzisSI/I1pNZSL5wA68MjISbm0aBlhEREbkOWcFJdnY2xo4di4yMDPj7+yMqKgqbNm3CgAEDAADp6elwc7vXGXPjxg1MnDgRmZmZqFOnDjp37ow9e/YYTaBVO6WGaiZ8nQQAaBXqhwejwxR5TCIiIlchKzhZtGiRydsTEhL0/p43bx7mzZsnu1HO5MatItSp5WnRfTNz1T2uLoTA+iMZaBXiixbBvo5uDhERVRNc+M9K/T5IUOyxbF3apPLjZ+YW4OOtp/HXTcM5QwmpV/HC94cwYN4O2zaMqBKW+SGq3hicmHH8Sq7J22/cLrbJcTV2SEV5YtE+fLD5Tzz/3UGDtx+5ZPrciYiIbIHBiRkzfjqK0Qv3orTM9b7KVSxquP/cdQe3hIiI6B4GJxIknr2GpPPOeQG/drMQ2XkFjm4GOdCVnDvYciJLkcrFRHLwLUeWYnAikb07TjSa8nUpXv7hMC5cs6yEd1mZQOf/bEHX97biTlGpwi0kZxE7Zxue/iYJvx3LdHRTiIgkYXCi49vE8/gm8bzs+209mYWHP9ut+DogjyxIxE8HL2Hc4v0W3b+o9N66FlfzpRXKI9eVeOaao5tARCSJrKnEriy/oBhvrCtfMXlExway7ltRt2TayhRM6t1MsTbl3ilPtj1/zTFrzQiFlwq0R5IvERE5P/ac3FVceu9CXFRSdSXN/ALzs3IOX8xRskmK4/AvERE5AwYnKO+h+OXwFZP7zNl4yk6tKaeB8t0Mtk6I3J32FyZ/d1DyEFLO7SJt7xAREVEFDusAePrrAzhw/obJfc5eVTafxBWN+WofgPLhoM/GdDa5b2FJKTrM3gwAOPPeA3DnGkNEspzKzMN/1p/EywNbomPDOo5uDpGi2HMCVAlMrOlgkJNXYeo46deVzTOxJH/E0ufhco75qct/3SzS/s6Vnonke+Kr/diV9hce+myPo5tilNJ5a1R9MDhxYdYmoPJjhUi9jC07QeQKGJy4gO2p2Zi0LBnXbxWZ35nICbB4V1XXbxXh58NXUFDMmkXk+hicuIAnlxzAb8cy8d6Gk45uCpEiqvNwwNFLuXhkwR4kX9CvSj164V68+P0h/J+dk/OJHIHBiQFrDl1ydBMskl3NC61l5xdg7OL9+P04K6EawjozzmH0l3uRfOEGHlmQqLc9NSsfAPDrkQxHNMsi7AEjSzE4MeC9DfxmYk9KfX69/csJ7PjzKp75NlmhR6wevt5zHo9+kYibhUxMVgO+DkQMThTnyC+nZ+6uMmyI2r7B2OJ5umYkQTDndhH+veYoDqXfwL6z13Dur1soLi3Dt3svKL7kgK0cvpiDM1eNv77WeOvn49h/7joW7Txnk8cnIpKLdU5cyOWcO8o+oImIJvdOMfx9PJQ9no3M/uUEVh+6jO/2pWu3/WtIK8z5rbyH7PycoY5qmiTZeQUY/uluALZt6x0VJVqqLZgmIvtiz4kLs0WVWaB8zDv67d9tmJin7JXJUI9D0vnrBvZUp4s3HLO2EhGRozA4IaOMhQizfilfIHFBwhnzj8GvwEREJBOHdRysrExAowE0ZqZSFJeWoaikDLW8nOAlEwIbj2XiuWXJqFvL09Gtqbbe/fWE3oKWRETOgj0nDlRcWoZ+HyTgqaUHzO7b6/3taPvWJkmrIxtjz06M55aVz5gxVhhONxaT266ikjI8+kUi4jecZJVMI24WluDLneewdM95RzeFyOldvH4bz3yT5FTDwc6OwYkD7Tt3Heev3cb21Ks4a2YmRkZu+Xo1Ry7lWnQsS+IStY7IbDiagf3nruOLHWfR5T9bkGZilhIAly7wsXDHGYN1eUrLqr54rvssENnWiysO4fcTWfj754nmdyZFMDixk5zbRcgz0esh9U2/O+0vXFR4UcAK124WYsX+dNXXWSgqLdP725mKUikpLTsf7204hX+uPKzI46kphlNpXEzV1KUbCs+EJLMYnNhBQXEpOszejKhZv6NM5xutbs/E9VtFkkp2f5ZwBve/v13ScfWHTsw/9tjF+/Gv1Ufx2uqjkh7fVqYsP2S+N8RKZ66qp75JWZnADwcu4vTdCqBS3bh9L9g1FfgSETkbBicK001sFULg/zaewmc6s1oqf+tXk+NX8gAAG485tvx7SZnA41/urbJdyZk/aiq+tjblMl796QgGzNth8WN8ui1Nf4MDux52p/2FR79ItHmASeqnxh6wEhV/BtM9DE5sKOnCDSxIOIOPtp52dFNgIAXBofRqsBhoW+V1gkpKyzD809148ftDVfZ19kXiDl/Msfoxcu9Y1nPy0soUq49d2Ziv9mH/uevapGiiCskXbmDC0gM476AvB9tTsxH5xkasSrrokOOTdAxOFKY7bH/DyEwVe9Fty87TV62+iC/ZfQ495mzDVQcsMJh84QaOXMrFz4evyL6vrVIp8guK8cqqw9jx51UbHcG28gqKsfrQZZs9PmdSUWWPLNiDraeyHRa4Pv11EkrLBKb/eMQhxyfpGJyoyF83bRfM3C6SX5q8cjDz9i8nlC+RL7kt6vPR1tP4MfkSxi7e7+imWESwd5tszNhQ7GUmmJIZDE4cqPLFv8ecbYo87ppDl7D/nPXz8dU6lVhp64/I740pKC7FlzIXysu5XaSXEC2FEMKiITlLesnsNVnnm8TzeGD+TpM9K85SWTjndhH2nr3mNO0lchYMThQmjPxuL8cu5+KfKw/j0S8SzVadNUbOhe3VHw8jI9ex34KsvS5MWX5IdnGltTKHQ45fyUWH2Zvx1Nf3Cu5dzrmDrSezTD7b45YcwMhK08xvF5XglpHp3t/vT8dPyZdUHVi+ue44TmTkYf4W47lYKm6+ngHzduCxhXstGm60lLM8N44ghMCz3yZh1s/HHd0UshKDE4Vl5xdof88vMF0vZHfaNauOdTnnDj7/Q399G1vVQDHmh6RLmPzdQb1txj48dS+Yt4rM11KRO7XWGqd1ZpYkX7iO+N9OosDEKr2mbjNk2d4LAICE1Hv5KT3mbMOEr5Pw+/Eso/ernM9SWibQ5s1NaPvWJhSX6I/LXLtZiJmrj+LlVYdRbGBGgqXBqq3IfQ7VqCL/auqKFNm9YqS841fysOl4FisjuwAnWKjFuehegA1dCmavP6HIcd799QRW7L+I/ErfoKUmIVo6u8OQP7PkTRk9c/Um+n/wh/ZvYz01A+btwPk5Q61qGyC/uNgjC8p7Knw83DEtrqXVxzcnM6/A/E536RbIu1rptdbNKypVc9eJizp/7Raa1q/t6GY4BVu9O0sYILoM9pzY0BUDyaPL96Ur8thf7jxXJTABgDfWSevOjH77d0XaAZRfMP+35U/J+3+beEGxYwO2+6D735bTePXHw7K+4c/+5QQe/mw3ikpsn236c4r9hhJswdUuI7bomTI2fEe2d0cn2Nd9Za/k3EHiGet6vck8BicKe33tMe3vH2yWfsG2tbmbUmUPI8m5ePzPRP5AhVm/HMexy5atDWSSDXsJfki6hEW7pCe+Lt59DgfTc7DtVLbN2lRhqx2OIZWhJQ/MvSw/Jl/CkUs5tmmQk6pcfuDdDScd1BLX9/TXB3DASK7ZtlNZaP3mRoM1qmLnbMPoL/dq71tQXIpD6Tc4rKewah2clJUJh02NrY72n7uOv328y67HPHD+usWLJVawpGBUmYOHVaQcvvJwmjVf/N/91bLhygc/2W1we3UclSooLkXHdzbrbduT9peDWmPeztNXkZZtv7wwpW05mV0l2bzCzLtLeHxo4gtmRXAybvF+PPTZHnydeF7xNlZn1To4eeXHw4pN33UlxaUCaw9dxm0VdCn3mbsdqZn5Fk1zzb1djJGfJxpcoddajrp2nsrIc9CRTTtw/oajm+D0KlYeV4uTGXl499cTyLldtf7SyYw8/GPRfsR9aHrJBVv9nxSXliHTwc/X+xtTkZqZj313yzZ8v1+ZIXsqV62Dk9UHbVcd09lNW5mCrxXODTHG1Lfk89duY+qKSiXrK+2/5WS2djaMrusGPlRtRU4A9FPyJYuPM2ph1TWHXN1XO886ugl6HF3T5Gp+oV0uhEPm78SXO88ZnJb7p5Uz6ax9Dv/+eSK6x2/FwXTHBsUjP98jaT9Hv2ecUbUOTkgdzP3bmlss8URGHl5fewypmfofmPb8QPin2TVq7vX9vLzqsMk9hRA4eilXLyFPLkvP/GZhCfaevYayMgEhhFVtUMp/flVP3sU3iefR9b2tdp3mbshMO64cXrEgqJpUrEf1o4RAv7DEdu/hPDPlIoDypObecxPw7zXKvGbFpWXYnfYXbksox+DMGJy4sMUyEjnN+fVIhmKPZYkrEgq93bh9b3p0aZkwms9gC0oW4Vpz6DKGfbILoxYaHg83ZNPxTL3aJpYGZqMX7sVjC/fi68TzmPhNElq/uRGXbti3do5Svtp5Fgt3nDF6e+KZaxjz1V6cvSp9Kvyb647jan4hXlPoQgMAM1cfwaB5O2x6EVWabikCNQSwFSoP/77642FEvr5R9irkSo4Er025jPTrt/GdQjM1P9z8J8Z8tQ/PfuvaC2syOHFhSmb6T15+0PxONvTPlaZ7Gyo7cSXP4AwStSsoLtUOUclJ5L1xuxifJRi/EEvxV34Rjt6dTfVj8iVsOVk+G2hVkvlvp/a8QN0pKsWB89dNDqXdLCzBf349ifc2nDKYMwEAo7/ci91p1/D8d/Lf20p2yn2//yJSs/Kx5YTxYny2cOnGbYvzNnRr6pSUlQfFuXeKsS7lsqq+0f9w9737pcyhQSUXN1XivXLpxm0M+3gX1h66jGV3h9t3nlZvsrQSGJyQS7J2BWb9x7KfmauP4mB6jkX3/e2odb1bK61YRv5yzp0qFyXrhtWM33fckv0Y+XmiyVwU3eq55mrOZDtglW1DjD1dmbkFig9R3ikqRc//247u8VsVSxif+E0Spq5Iwetrjpnd1xVn3WoMpO2XlQm98hKWenPdcRy9nItpZoePXQeDE3I6aRK64U/KnNUi97Pf3P5CAIfSbxj9FmmsQu8amWv22IPUp+aEidyEtOybmLL8YJW8IEvsNzI74trNQkUv4kIInLl6U/Jj6l6a7hSVIuVijtXtOX/tNrrHb8U765XNu9HtGTC01IElKl6XNSnm38N3bLR0gRJ18KRW2Tbl8MUcHL+Si3PXqg4nZeTewfJ96bKKO9qqF/jCtVuqrc/C4IQUJ/9Cb+5Kr//nwh3mu2hn/CQ/JyAztwAv/ZBSZfuvRzJwMiMPm09kSU6E/CHpIh76bA8eMzK7RskKvRVuyRhaWbr7HL6SuKqyoUJUcv1j0T6sP5KBRxZIm90g17qUy+j8ny2KJs/O23Ia/T/4A+9ZMDz6+Fd7MeLT3Vh5wPLeKF2LdyuXP6Y0S9cIy5axbIM9WTpEWREY5RUUY/inuzH0o10oKa362fa3j3bhtTVH8d9NqdY002rL9l5A77kJmP7jEYe2wxgGJ6R6Zy0ogmaJl35IMTi9/E5xKYbM34mJ3yRhwDzTdR0q/HF3wT5rC8BZ6uxV48/ZnaJSzPrlBD7Znqb4cT9LuPeYut/2Kmp4KPkN8Py129h3tvzCWNGzIKearzkVQdmXEoM4oDxI2ngsE4fuDs1ZM1TmLJ5bZlliZtf3tlZZ2FIuqV+ErO1Qk9Mjk3PrXq+ooV6pa3erAO84bd25W2v+3ff3TwctL21gSwxOyOHWpVxxeB2Ayzm3JQ8FWbKarr0XBB67eL/R24rLbLfuz/sb730btEdvsaV1X5IvGC5bLpWh1/ParSJMXZEi62J9XadcfVa+dT0JRy/l4qWVKQbX9FJSSWmZYmtHyU1UdTaWfqwlnb+OIfN3aofKqiOuSkwAlBt3BuSXbp+9/gSa1q+l2PEt8el26TNdqmM9peu3ijB+yX483LEB+kQGIaCmh+T7WjKGb8vnOL+gWLvytNKPK8fZqzfRT2d17mV7rZtqOuyT8qUhLt64jVXPxVr1WKb0en87rkic5WPuddx5+i+UlJahhrsy35MNJaU6o7/fLav/6BeJslZmv1VYglpernFZd42zIKvN/sWytVEMsaRo0ykFEiXV6satIiQZWWDM3kpKy7DvrPy2fLItDUcu5ZYPUxl5r1y4ZrgeSpf/bJF9PFvGfzm35QURtqJkbRxdZ0wM6Vmici+RucBEf3fzr+Ti3efwTK9msttlLzsdPPwiVeKZaxj95V6Mj22MWQ+2dXRzrMZhHQIAfGug/DsZlyyjbHaf/ybgzyzphb5sRaMpz6OY+E2S7PtKmV1hrvKto+SbyXNx9JCiIyg51d5aG45mytr/f1t0F+MrP4/1R65gd6VFEi9eV6Z44AyJCaMFxaUOrfHy/qZTAICle84b3eeWE9V+YnBCdmfqn8cZ7Er7S1bFXGPThh3hGwuD0EMOXsNErjydIZb+H/yh6tV9lfDh75bP/LB3PpS1/rdFf/ZY+rXbmLL8EMZ8tU9v+/3vb9f5y/bB2Plrt9HmzU16OV1KBIHrJEzNluKrnWfR9q1NWK3SBNjKGJwQybQ9NdvRTbCINR0EzjTstnjXOfSem6C37b8mLt4Vyx7cKSqVPI1UbbkNH21TfuaVpcoEtLOobE0IIFtGIrEtViiv7NpNZRccnboiRX+Dzlvv9bVHJS+/UDHN/qUf1NnDWRmDE1IFRy9/LsdyhdbIsLele85bNNPI1krLBGauPippETcpZq+Xnz917HIuWr+5Ea3f3GhxcrjGTBfEtZuF2HQ8EyV3H99WAY4tHvVOUamsYRJDs6g+NlMvJzuvAI3/9Ssa/+tXq1buNmavnQIme1q2Nx2PfqF8crcayApOFixYgKioKPj5+cHPzw8xMTH47bffTN5n1apVaNWqFby9vdG+fXts2LDBqgaTa3L2oR5nUVBsu2nElvrtWAa+35+OV3RyVir38nyWkKbojLLKdEvhbz8lrWdM7nTaBz/ZjWe/TcZXu86hrExg0S7HTaPdeExenkfvudtx//vbTVYBNueDzX+avD12zjbt75bmLxmLD4WA5KKD1tINguwxnf4vC3pqhBCKrh9kC7KCk/DwcMyZMwfJyclISkpCv379MHz4cBw/ftzg/nv27MHo0aMxYcIEHDp0CCNGjMCIESNw7Jj1aw0Qke2YKuKmNCkfku9vTMV3JvNllLsKPPNtMnZVWlTtP+tP6NUkAYARn8pb9fry3fojG49lYv3RDOQVOC45Mf63U7L2r1h/aMtJ2y1OWGLBlVxu2QJLyTnMhzpBmKlk6z+zbuLwxRwrWmU5NeXBGSMrOBk2bBgeeOABtGjRAi1btsS7776L2rVrY+9ew4WQ5s+fj8GDB2P69Olo3bo13nnnHXTq1AmffPKJIo0nItuw1donhsRvkHah/DPb+Ni60qsiP7FIP7nyq13nMHO1/qyNE5WK9smZ9WPLIQZLE1wzcwvskpNRmbEjbpawSvMPElbMBqzLt5Ja08USUld71y3Glu/AoNaeLM45KS0txYoVK3Dr1i3ExMQY3CcxMRFxcXF62wYNGoTERNccIyMi+YoUGK6Rs66QpY5d1glGrEjsSLmYo7q8pT1pf6F7/FY0e20DPt56Wq+CbrqEXJPEM8oHWxO/SYIQAt/uveDwSqlKrVz9S6XaNrrVfLPzC3BNgUUHXYXsImxHjx5FTEwMCgoKULt2baxZswZt2rQxuG9mZiaCg4P1tgUHByMz0/R4Z2FhIQoL771IeXmWj3MSkeOUyAw8ks5fR5fGdQ3etnxfOmpbWP3S3BdnJfoL3jZQnM4ZSqhooME3ifeGzD7Y/Cc+2Hzvdil5SqO/tGwZAXN2pf2FN9aWpwHIqZRa2Y3bxvMyhBBmk5ktVfnlf+H7Q3p/V3RU3SkqRdd3twIAzr73gE3a4mxk95xERkYiJSUF+/btw6RJkzBu3DicOKFcdVEAiI+Ph7+/v/YnIiJC0ccnIvuQO3PG3CKPUlakdpRzBtruiGGSCrl3ig0msJqbNaOUPWesry1zXqFFP38/kYXCEsO9a0Pm78TfF+xBrg0qBy9IkLYshu50aCXWvnKFwoKygxNPT080b94cnTt3Rnx8PKKjozF//nyD+4aEhCArS3/cMCsrCyEhISaPMXPmTOTm5mp/Ll50/ZU9iVyR7jdyOeR8uDpiaQCpwwxHL+eitEwonhNjWnkvQI852/DARzuRfOFeAb0rOXcMzprZeFza7B0517yfki/bpedoxQFp1wdj5QpOZeYj6cINRM/+HYfSbxhdhsESUvJmbKHre1v1EnOdkdV1TsrKyvSGYHTFxMRg69atets2b95sNEelgpeXl3a6csUPkdqpsYaIs3pq6QHJ+1YskmbOR1tP46UfUuw+1DLs411o/eZGux7zTlEpbt4tVZ6gUzTQ2kTnNIkFv5Rys9B8e5WqjwMAD322p0oytL0p8f68ml+Ij+72kBWVlOHPLOcpolhB1gDuzJkzMWTIEDRs2BD5+flYvnw5EhISsGnTJgDA2LFj0aBBA8THxwMApk6dit69e+ODDz7A0KFDsWLFCiQlJWHhwoXKnwmRgw2ct8PRTXAZtkhwrfgm2btlfYvuf1kneVGOyrN67OGNdbYp11A5oVMRQuDY5VyDN/3fRnlTnp2tFH8FS99bugwFNZtPZFm0lpYayApOsrOzMXbsWGRkZMDf3x9RUVHYtGkTBgwYAABIT0+Hm9u9zpjY2FgsX74cr7/+Ol577TW0aNECa9euRbt27ZQ9CyIVkDKrgUx7a91xDO8QZtNjFBpI8FyXYpsVgh3FWdZPqXD8iuHgRNf0VYftUtTMEXaeti4/5531J5BioGaKscDktl2HGS0jKzhZtGiRydsTEhKqbBs5ciRGjhwpq1FEVD3dKS7FzJ+OOroZTs3Zeg/Sr99GUan5qGOVlcM3f7nwNN1Fu+RVv12xX11T2Q3h2jpEpCqrDymzCquumzqFq5RYKdaVODqYuXG7WDtd2Fqm1iu6YYPZOEqx90tQaMOlIJTC4ISIXN5pE9VlXc2twhLVDH/YuxlK5G6QOjA4IaJqxQVKQJhUOZ/A3PkmpF61YWvsyxmTP6/k3MFnEuuhKCXhlPpfcwYnRFStuHhsUsWiXecwaVmy0Wq9znhBdyW6qzED9gmeU51gajGDEyIiF3anuBS/HcuUXGiNHMveNXHUisEJEVUrzliQSgnOMH2UqAKDEyKqVpbsPu/oJjjE/C2nkXfHvjNWXD2/h2yHwQkRUTVwOecOZv183NHNIBWK33DS0U2ogsEJEVE1cfiS+UqsVP18seMski/YfwFNUxicEBGR4n46eAmfbE9zdDNIokcWJCJNRfWAGJwQEZFN7PhT/fU06J7UTPUkizM4ISIiIlVhcEJERESqwuCEiIiIVIXBCREREakKgxMiIiKCUNHKUwxOiIiISFUYnBAREZGqMDghIiIiVWFwQkRERLh8446jm6DF4ISIiIhwq6jU0U3QYnBCREREqsLghIiIiFSFwQkRERGpCoMTIiIiAgSLsBEREREZxOCEiIiIVFS8nsEJERERQVWjOgxOiIiISF0YnBARERFXJSYiIiIyhsEJERERMeeEiIiIyBgGJ0RERKQqDE6IiIhIRemwDE6IiIgIzDkhIiIiMorBCREREbHOCREREZExDE6IiIgIeXeKHd0ELQYnREREhOJSDusQERERGcTghIiIiDiVmIiIiNSFs3WIiIhIXdQTmzA4ISIiInVhcEJERESqwuCEiIiI1DSqw+CEiIiIAKGi6ToMToiIiEhVGJwQERERh3WIiIiIjGFwQkRERM5bITY+Ph733XcffH19ERQUhBEjRiA1NdXkfZYuXQqNRqP34+3tbVWjiYiISFkqik3kBSd//PEHJk+ejL1792Lz5s0oLi7GwIEDcevWLZP38/PzQ0ZGhvbnwoULVjWaiIiIXFcNOTtv3LhR7++lS5ciKCgIycnJ6NWrl9H7aTQahISEWNZCIiIisjmXmUqcm5sLAKhbt67J/W7evIlGjRohIiICw4cPx/Hjx03uX1hYiLy8PL0fIiIish31hCZWBCdlZWWYNm0aevTogXbt2hndLzIyEosXL8a6deuwbNkylJWVITY2FpcuXTJ6n/j4ePj7+2t/IiIiLG0mERERORmNsLAfZ9KkSfjtt9+wa9cuhIeHS75fcXExWrdujdGjR+Odd94xuE9hYSEKCwu1f+fl5SEiIgK5ubnw8/OzpLkGNf7Xr4o9FhERkTMb2j4Un47ppOhj5uXlwd/fX/b1W1bOSYUpU6Zg/fr12LFjh6zABAA8PDzQsWNHpKWlGd3Hy8sLXl5eljSNiIiInJysYR0hBKZMmYI1a9Zg27ZtaNKkiewDlpaW4ujRowgNDZV9XyIiIrKNMhUlxMrqOZk8eTKWL1+OdevWwdfXF5mZmQAAf39/+Pj4AADGjh2LBg0aID4+HgAwe/ZsdO/eHc2bN0dOTg7mzp2LCxcu4Omnn1b4VIiIiMhSKopN5AUnCxYsAAD06dNHb/uSJUswfvx4AEB6ejrc3O51yNy4cQMTJ05EZmYm6tSpg86dO2PPnj1o06aNdS0nIiIilyQrOJGSO5uQkKD397x58zBv3jxZjSIiIqLqi2vrEBEREYSKKp0wOCEiIiJV5ZwwOCEiIiJVYXBCREREqsLghIiIiFSUccLghIiIiMCcEyIiIlId9UQnDE6IiIiIPSdERERExjA4ISIiIhUN6jA4ISIiIpVhcEJERESS1s+zFwYnRERExGEdIiIiUpcyFUUnDE6IiIhIVRicEBERETSOboAOBidEREQErxrqCQnU0xIiIiJymNahfo5ughaDEyIiIuJsHSIiIiJjGJwQERGRqlb+Y3BCREREqsLghIiIiJhzQkREROqiolEdBidERESkLgxOiIiISFUYnBARERGEirJOGJwQERGRqjA4ISIiIlVhcEJEREScrUNERETqoqLYhMEJERERqQuDEyIiIuKwDhEREZExDE6IiIhIVRicEBEREYuwERERERnD4ISIiIhUNZeYwQkRERGpKTZhcEJERESAUNFcYgYnREREpCoMToiIiAgajcbRTdBicEJERESqwuCEiIiImHNCRERE6qKi2ITBCREREXEqMREREZFRDE6IiIhIVRicEBEREXNOiIiISF24KjERERGREQxOiIiISFVkBSfx8fG477774Ovri6CgIIwYMQKpqalm77dq1Sq0atUK3t7eaN++PTZs2GBxg4mIiEh5Tptz8scff2Dy5MnYu3cvNm/ejOLiYgwcOBC3bt0yep89e/Zg9OjRmDBhAg4dOoQRI0ZgxIgROHbsmNWNJyIiImWE+Hs7uglaGmFFvdqrV68iKCgIf/zxB3r16mVwn1GjRuHWrVtYv369dlv37t3RoUMHfP7555KOk5eXB39/f+Tm5sLPz8/S5lbR+F+/KvZYREREzmz5090Q2zxQ0ce09PptVc5Jbm4uAKBu3bpG90lMTERcXJzetkGDBiExMdHofQoLC5GXl6f3Q0RERNWDxcFJWVkZpk2bhh49eqBdu3ZG98vMzERwcLDetuDgYGRmZhq9T3x8PPz9/bU/ERERljaTiIiIJFBRyonlwcnkyZNx7NgxrFixQsn2AABmzpyJ3Nxc7c/FixcVPwYRERGpUw1L7jRlyhSsX78eO3bsQHh4uMl9Q0JCkJWVpbctKysLISEhRu/j5eUFLy8vS5pGRERETk5Wz4kQAlOmTMGaNWuwbds2NGnSxOx9YmJisHXrVr1tmzdvRkxMjLyWEhERUbUgq+dk8uTJWL58OdatWwdfX19t3oi/vz98fHwAAGPHjkWDBg0QHx8PAJg6dSp69+6NDz74AEOHDsWKFSuQlJSEhQsXKnwqREREZCmnrXOyYMEC5Obmok+fPggNDdX+rFy5UrtPeno6MjIytH/HxsZi+fLlWLhwIaKjo/Hjjz9i7dq1JpNoiYiIyL7UtLaOrJ4TKSVREhISqmwbOXIkRo4cKedQREREVE1xbR0iIiJSFQYnREREpCoMToiIiMh5E2KJiIiIbI3BCREREUGjcXQL7mFwQkRERKrC4ISIiIiYc0JERETqoqLYhMEJERERqQuDEyIiIlIVBidEREQkaYkae2FwQkRERKrC4ISIiIhUhcEJERERqQqDEyIiIlIVBidERETEOidERESkMiqKThicEBEREcCF/4iIiIgMY3BCTqFNqJ+jm0BE5No4rOMc/hnX0tFNICIiqnYYnBhxcvZgTI1r4ehm0F3ubioaDCUiIpticGKEj6e7o5tQxQv9mju6CQ7z/t+jHN0EIiKyEwYnTiSiTk1HN8FhWjPnhIjIpur7ejm6CVoMTszoG1nfZo/do3k9fDamk80en4iISCo1TTxgcGKGu5vtnqLvnu6OB9qHSt4/MsTXZm0x5uGODex+TCIiD3fmmVVnDE7s6Junuhrcvub5WLP3/fqprqjlpUwezMOdGHAQkbppNAxOqjMGJ3bUq6XhIaKODeuYvW9vI/eVq0ujOhjYJliRx3owOkyRx7GXuNbB8POu4ehmEJFKhNfxcXQTyAgGJ9XMj5NiUUPiUNX8xzqYvP2VgZEY1SUCAPBMr6ba7bW9pAUA5+IfkLTf+NjGkvYzx9e7hmpqDHVrUheT+zbD9EGRjm4KkSrZut+keVBt7JrRz8ZHIUsxODHg3w+01v5uTfby8bcH4UUnnv47vIPp4Z+Iuj5496F2WDe5B2YMbqXdLrUkiUajwQ/Pxpjd742/tZH2gFaKCvfHNDvVtvH19sD0Qa0wua/zvj/I9pz580PtRndt6OgmkAkMTgwIC7jX1VenpofFj1PLqwa8VVAv5fFulv8Tjrvba9GpYYDe9nmjoqHRaFDD3Q3REQEWF0nr2qSu2X2UKsDWqJ7pqdjP9mqGaTKqAtsjkNn5al+bH8NWmgbWMrg9oi670qV6tnezalnjR+1DxjVV8LluC2pK82FwIpOnu/qfsn2v9cfi8V3QIqg2fp7SA/0ig/RulzO0ER0RgJQ3B+DLsV0Ua9/org2xfGI3xR7PkLeGtakynvxc72aKfuhNvL+p+Z10NAiQf1GOqFs1oLLkcSrc19h8fpMSXh/aGp41pP2vBNb2tHFrnJeHE3ze2IK5IWVHerhjA0WXNglSUW0RName73wFTYtrgXmjorFpWi98/kRn7fb/jGgHABB2TnII8vVCsJ83+rUKxuaXeiMqPKDKPkJmowJqelbJnO/apJ7J+7z3UHujt8U/3B6xzQJltUGuJ3s0qfIB5+3hbrchIl1HZw3EqXcGY9eMe70gIzpaHiTNGNLK/E5GaOy0JvrT9zeFn7fhXsfKb79xMY2x/oWeePNvbfAvK87NHqb0bY7+rYJM7qPb0+clMUAzZPH4LpIDPFfSs3kgNBqNavLDKvvnAMOBSeWyCyM7h+PZ3lW/wHytM2vzwegw1KvN4MSQ6vfOV1iInzce6hiOyBBfDG4Xot1ubNpvxQVzyfj77NE8xVQeWjH07f3JHo0BAK890BqPd2uIyGD9uiz+Ph7Y+nJvm7VRCm8PdxydNdDmx/HxuPf6+3p7wNvDXS/Aq1vT8t4CtXR5mwv0DA1HGApaPWq4oV0DfzzVswm8VX4xfmVQpKRhxmd6NcWD0WGYPbytxcfq16p8Vp2/j+VDy7bw9VNdcfY9acnslniks7pLHRjqzQRQJYP3/x6JwvN9quYM6c68VNMwitqo+5PAjlY9Zz4xUwnDOzTA2fceQF8z377URsoH5Jt/a4P9r/XHY0YSzXbO6Itm9WtLOt7LBr6dGOpKHSMxn2bFM921v/sa+UavFs4yzFGvlul2Ng6shVo6Y/NP9mhsVf6TrZga6tr3Wn/t77++2BNA+XmZ89oDrfHR6I6SygSY466yK1jvlvXhZoeFOIdFqSMIB1Dli5ZBlbp67PEcmWKq99oZMDi5677G9xIzTb2nlPickPumbddAHSWFn+1lOsdCo9EgyM/b4G2N69U02s1vSPOgqkGMoYTWLiZzKO49z92bmh6GUpqwUaf0vFHRAIAv/tHZzJ62FervjYZmEowB4LepvbS/G5tibs+egSMGes10//crC/bzxp//GYKDbwxA2zB/AMCL/VvgH90bGb2Pbi5SSykXNTPq1DL//CiRZCwlOd1alRPrDakYepz1oLQhWKmlC3T5+3iYrX4d5n/vsyw6wl/2MZQmt9aVGr8IyMHgRMe4mEaIDvdH/9aGi5S9+1C7Ktu8PGz7FP40KRYrn5Heq2PLcdpJfZohKtxfct6G7gVa7jdIa4LAFgYCm8p6Nq+a8yI397CGncpr7/lXeS2Gfq2C8FDHcADAoLYh+Oaprvj1xZ74fmJ3g/eTOvzTu2V9rHymO1ZLqFSsS8qFVzeA6RARYHAfpYoCSmEoQDaXdOpZww11dXqJanvVwDsjqn4WAMDE+5vglYHGkyVjmtbTnu/QKGlLV3RqWAdTjEw5n/v3KPwzriVWyPiMMOaHZ2Pw+RO2XeurSaC0nlOg/HmWMoRmag9DU7F/mhSLg28MQB0zPX/bp/fR/j6gTQjmP9YBW14qD7a7Na0ayD3Zowk2Trsf7Rr4YcmTyg/bazTltaHO2HBITU0YnOh4e3g7rJvSUy8JTfciOaab/relns0DMbS9bbseOzeqg1p3vxk0DayNaCMf8HJYGsAE1PTEz1N6YkLPJrLvO+tBeWPv1iSJfTvB/Ewg3cBpfGxjdIgI0I7xS+VVw93okgRKCgvwwal3BmPROP0ZU71a1kfbMH/ENKtXpZhbqxBffDS6o6SZPRoN0K1pPXSyYgiibdi93r3Ks6QSXumDz5/ohH5GhjLdZEaivpWq/B58Y4DB/f4m8eL/lAXvZ2O6NamHGiaCne+f6Y55ozrg8yc6Ya7EKcIajQavDIrE0KhQ+N79LPD2cMPZ9x7AyC4RmBrXwqoZXJYy15NqLY1Gg2OzBuHY2+U/xowzUaTRy6Nq7l/nRnXMBj3REQHwqnHvvhqUD8k3DyoPyA1NNAio6YFWIX5Y/8L96Bspbdi+fm0vvHr3f/eJ7qZ7OjQof06UKq2gdgxOrLDs6W5Gs+nr1y7vElRyCWo3Nw3WPh9rcMhDKVHh5d2XSo9Xyum63zWjr15CqSmVZ/3c3yIQIf6Gh5aMmfVgW6yd3EP7WhrqVTHG2JIEAT7K5o1UTqg1p2LfLS/1tqpWjy5T1Wxf1SnCV7knp3FgLQxuF2q0/XKD5RGVigMaqznxyePSegH8fTzQSoFFNbs1qYs+ElYxr+VVA4PbhaKmp7zhiE8f74QjswYi7d0hOPXOEIfnNMx8oDUSZ9q2wqqPpztqe9UwOnSz8pnuNqk3tGaSvF5Eub4c2wV/iwrFi3Et0LdVEA6/ORCzHzTcG2cJtSVRW4LBicIWjOmEqf1boEfz8hyHhzs2wD+6N8JnY6R3l5r6Nq7RVJ0MqsQHa4XxsY1x6p3BDh2vDK9jOJfB0LUt2M8bya/HKXr8j0Z3NLjd0LCeMUOjQjGyczjef0T/23FFsmu7cMNj2Muf7gYlC3f7eLrj5YH3gorY5vJyb2J0cnV0q9mamo1u7lt8hJHXVwmWTr1VIiF+5bMxJntNDE0r3fZyb6PDcoZUFD6UKzoiABPvl9JDJO+9F+rvIzvfRalgeXxsY3RrarqnylK2DvwGtAnGJ4930g4z+tf0UHTmTv/W5ntuwvy9EaDQa2ELDE4UNqR9KP45oKX2W2INdze8M6IdHmgvrYsZAOrJnK2hxOqdTevXwkMdG+DB6DB4S+y1UAul6wTUNTIWXa+W9OPUcNNg7shoPHpfhN72Pf/qj6OzBhrMfXhlYEvEyui1kWp014b45PGO2P2vfpjUpxn+75H2mCmhnsibf2sDY8swVU74bRpYC61CfNG1SV2zH7IfjopGd50xe926O9bOYvu/Ryzr8bPHe76bgYTTpvVrI6aZ7ZO1fTzcMMTEZ1DnRsaH9Kz5Fv5xpUC/lpe73nCJIVI/znSHin+bej/eGmb/Gkbm2KNoZ5/I+pjzsP77XspQ6dKnukqe7egIDE6cULsGymeOj76vIeaN6qDot5CKtXmU7Nlxdp413Ow+ldndTYO/RYWhQYAPvGq4Y9R9DdFAJy/E0MfY/0Z1MJuLoRtUuLtpsOHF+7Hyme5mg+VQfx98P7E7+rUKQt/I+nrBYKN6tbT1cowet1JgpNuLY2mROQ93Nyyb0A2Lx1tWCdnDTsnRtlIRLFryPcfYVOdHOoVjWKWk7Kn9lRmC+anSsEvrUD882aPq+9WSInhvy8yPM8XH011Wj2tl5T2pptVw0xgt3/D60PJ14oxVtH2hXwvVrjHE4ESmOlYUz5KqobEiP3fNGtZWr4u4QYC8HAt7ebZXUyx58j6slLC4n6sJc0CCopLkdmtrNOX3kdqLp9FosHj8fVjyZNcq9wk2Mh1divbh/gis7YVoI8NmpvRsESg7KbqCud4AezCW/wTA4IVbqfVhAmR8Jtar7WX1NPvWoX4me3oqdIgIwOPdGsrK+5vct5nBBFtrWlx5IoVU5+If0OtJNfa/ZWqB1qfvb4r9/+6PqUbycrw93DGhZ2Pt3/auaG4KgxOZFo+/D9Hh/lgmYUaIpXy9PbB3Zn+jsxD8a3pg5pDWWP50NwxuG4J3ZSav2qMbGSgf0uobGWS2W7hy168aDG4bUmWblETk4R3C8HTPJrJydiryJO5vIa+OgS30iawPX+8a6Hs3sbO5xKJ5thLi5437GteRHGx4urth78x+WPN8D4uPWbGOVA+Z+TmO9vkTnfDq4EiMi9G/GO6a0ReDDLyf1cpczpKp8Lfivm1C/bB2cg/U9KyB7a/0wc9TemBEhzB88rjpz5onTNSvMUfpC7vUQN9cDacgX8PBfsXQT8UsUHsV15NKfvWaaq5NmB/WTelp8+NImXES2zxQUo5C5SRBP28PnHpnML5NvIB3N5wEYPt6LaYMMFDnQm73cnS4Pw5fysXILvdyPKxZUGvBE53w/f6LWHkgHTMGtwI0QLP69yqDGprS/f4jUVVyTKTY/1p/XMkpQBvtdFzbf32JahCg/V0332LJ+PtQUia0tT9eGRQJjUaDBzs4plrn+hd7IrC2Fx76bLd2W10zuT/mhib3/KsfYudsM3r7gDbBODl7MLzt+D/h7eGGguIyqx6jpmcNPN+nOb7ec15ve0WCuZ+3bT7udd+t4XV8cOnGHQDy/4ffGd4W0GjQxURRPHO+n9gdXyee1yt3UNurBqLCA/C/x0wHJs2DaiPU3/l6PC3piXqgfYj286xiFqjaMDipBno0D0Rc6yBE6uR+eHu4Y2KvpigqLcMfqVfxaBf5F1WleHu4I651MLaczLL4MVY+G4OzV2+hdei9c4yoWxPzH+tgcChuUu/m2J12zWgtDI1Gg8e7NazSA3LojQE4c/WmIvVmKgTU9JTVNS5FezNVhRvWq4nxsY2xLuUyXtcpqqfRaPTyJ3y9PQzWqLFl92+oTmAeaCDZeWxMI3y09bT2b932Spl9EBbggxpuGpSUGT8JH4WGPKRa+UwMnlp6ANduFQHQr04q14PRYXjr5+NVtjcP8sX0QZGYuym1ym2RIeXvF93Ceh0bBqBHs0B8u/eCdpu553fJ+PswYN4Oi9od0yzQ6jIJDevVtHhxzxoO7jXQaDRoElgL5/66ZfVjjYtpbPS21c/HVqlppMSkCqUxOKkG3N00+Gqc4YqFk/s215si6ihS8nCbmqgu6e3hrtPzcI+x8dieLQKR/Hqc0Zk5xtSp5YkutQx/swv0VceaOJP7NsOzvZuZ3W/Wg21lF8fTpZtnIWdpAnOGRYXhz6x8dGlk+HmuPLOmhrsbfn2xJ0pKharXTTKVyB4dEYDVz8ei99wEAOXF2ixVp5Yn1r/QE7PXn8BLldaomty3uV5wsv6Fnjh0MQfD7gbpTQJr4YdnY1Cvtqd2HaxvEs9r9/9jel9UoROptpBZrr++rxeu5hfKuo8ri21Wz2hwIjVgjm1WD+0NDIPu/lc/XLx+26pii/bE4IRUyVDQ0D7cH58+3gllQmDjsUyDdSPkUGoK8sejO+LYlVzJVSFtbfog89OEleBZww3rX+iJkjKhrWKsBDc3TZVzMPe9rmLdG0cRJrqSDr81ELcKS4yO/RtiTVIwUB4I/SAhEb1dA/8qQZOpNXbkTCuW8l18yz97I3r273f/cmw2php6D14a0BKnMvPxaJdw7bb3H4nC0j3ntTNvzDH2GjUI8HFIJWFLVevgpCJPgdQn1N8H8x/rUOUbecV6JJWnKDrSsOgwVbXHnmwxrd3V+Pt4uETFTmOsCSkqL0XgzIL8rP+yU6+2V5Vp0o/eF2Eyl03u8g/OolrP1hnpwDwLpVWstxDlQheL4R0aWF2Ui8hZ6A6T2eOC46yXNFs9NZ4matUY6hmL06nCmvR6nF2LV1Ys0DikXUiVvCxXiVVcJ2y1wGP3ReBWYYndptba0qZp92PlgYt4TkKugRqpaX49GTc0KhS/Hsmw+/tMDV3uthbi740pfZvDx9Pd4jL8jmDsf9dZeov+OzIa87f+if+OjJZ1vxmDW2HLyWwA5s+1bi1PXL+b7KyEwe1CcX7OUMUeT42qdXBSw93NbOKgLdcBUVLzIF/8e6j6yjeTa5k/qgOm9m+BFjZcfNKQ9x5qj1ELE/FCP8srjP5ihxIA1nrFxOKKalV5KusHI6OxNuUyXjBSDVZtX0T+3jkcf+8cbn7HSnRrgpgLnZdN6IYHPtop+xjVmezwfMeOHRg2bBjCwsKg0Wiwdu1ak/snJCSUL1ZX6SczM9PSNtvV3zuH4/k+zfDtBOOL8RFVFzXc3dAy2NfuPRmRIb44+PoAvfoVcgxpF2JwBoNSOjQMsNljO5tHOofj2wndtL0J74xQbrVdZ9UmzA+Lx3fBxmn3O7opTkN2z8mtW7cQHR2Np556Cg8//LDk+6WmpsLP795Uz6Ag58glqOHuprccPBE5RsU3Vd1yFC/2Mz0N/pcpPbHiQHqVKbVNAmvhdPZNxdr2v1Hqq3JsL+Z6QupLXMhUbT0qSrN0aYTqSnZwMmTIEAwZMkT2gYKCghAQECD7fkTVTWyzQPx8+AoCZaxOXQ1SMrRqetbA+NjGKCwpw0sDTQ+DtA/3R/vwqss7fDm2C97fdAqTeltf4+f5Ps1krd9S3cS1Dkb/VkEGCxdWp/ctyWO3nJMOHTqgsLAQ7dq1w6xZs9Cjh/G1LwoLC1FYeK8wT15enj2aSA40vEMD/H4iC011SsRXV/95qB3ahPkZrV5LsKp4HAA0DqyFz8Z0tuoxNJryb/tqWBPJkcz1eNRwd8Oi8YaLQBIZY/OU8NDQUHz++ef46aef8NNPPyEiIgJ9+vTBwYMHjd4nPj4e/v7+2p+ICNeZ8mtMRfe0s862sdYD7UPw85Qe+NkJkhZtzc/bA8/1bqZdE4XU6cC/4/DTpBiXmO1nynsPl/c8TTOysq29qaW3RW3DUK1Dy9MmHu4oP7lXjWzecxIZGYnIyHtdr7GxsThz5gzmzZuHb7/91uB9Zs6ciZdeekn7d15enssHKP8c0BIPdQpH43rV84Kk0WgQFR7g6GYQSRZY28vg2j9qNzamEb5JvIDpEmcG/S0qDL1b1je6NEBMs3o4kZHnVNOf5WpQR/2VVdc8H4v067f11kdyZg6ZSty1a1fs2rXL6O1eXl7w8nK+f3prVCz6RERkS28/2BYT72+KiLrSvwiZWrPolYGRCK/jg/5WJnyqrCMCALDime64fOOO2eUR1FCHx9vD3WUCE8BBwUlKSgpCQzmeTkRkbxqNRlZgYo6Ppzue7GHZFG81XNRN6d7UtYfs1Ex2cHLz5k2kpaVp/z537hxSUlJQt25dNGzYEDNnzsTly5fxzTffAAD+97//oUmTJmjbti0KCgrw1VdfYdu2bfj999+NHYKIiFyUl4frDv+QcmQHJ0lJSejb996y2RW5IePGjcPSpUuRkZGB9PR07e1FRUV4+eWXcfnyZdSsWRNRUVHYsmWL3mMQEVH1sGBMZ0z6LhkzLKwf5UpDF2Sc7OCkT58+JpcHX7p0qd7fr776Kl599VXZDSMiItfTroE/dr7aT/b91r/QEz8dvIRp/Vua35mcXrVeW4eIiJxDuwb+aOdCq66TaRz8IyIih3OWVYzvUeP8ItfBnhMiInKYReO64GZhCYL9vB3dFIupe86Rc2JwQuQCNPx4JCfVvzUXxKOqOKxDREREqsLghIiIiFSFwQkRERGpCoMTIiIiUhUGJ0RERDK5qXxdIGfH2TpEREQyNQmshSHtQuDv4wE3NwYqSmNwQkREJJNGo8GCJzo7uhkui8M6REREpCoMToiIiEhVGJwQERGRqjA4IXIBrUJ9Hd0EIiLFMCGWyAX0aVkf/x0ZjVYhDFKIyPkxOCFyARqNBn/vHO7oZhARKYLDOkRERKQqDE6IiIhIVRicEBERkaowOCEiIiJVYXBCREREqsLghIiIiFSFwQkRERGpCoMTIiIiUhUGJ0RERKQqDE6IiIhIVRicEBERkaowOCEiIiJVYXBCREREquIUqxILIQAAeXl5Dm4JERERSVVx3a64jkvlFMFJfn4+ACAiIsLBLSEiIiK58vPz4e/vL3l/jZAbzjhAWVkZrly5Al9fX2g0GsUeNy8vDxEREbh48SL8/PwUe1w1cfVz5Pk5P1c/R56f83P1c7Tl+QkhkJ+fj7CwMLi5Sc8kcYqeEzc3N4SHh9vs8f38/FzyDafL1c+R5+f8XP0ceX7Oz9XP0VbnJ6fHpAITYomIiEhVGJwQERGRqlTr4MTLywtvvfUWvLy8HN0Um3H1c+T5OT9XP0een/Nz9XNU4/k5RUIsERERVR/VuueEiIiI1IfBCREREakKgxMiIiJSFQYnREREpCrVOjj59NNP0bhxY3h7e6Nbt27Yv3+/o5uE+Ph43HffffD19UVQUBBGjBiB1NRUvX369OkDjUaj9/Pcc8/p7ZOeno6hQ4eiZs2aCAoKwvTp01FSUqK3T0JCAjp16gQvLy80b94cS5curdIepZ+jWbNmVWl7q1attLcXFBRg8uTJqFevHmrXro1HHnkEWVlZTnFuFRo3blzlHDUaDSZPngzA+V6/HTt2YNiwYQgLC4NGo8HatWv1bhdC4M0330RoaCh8fHwQFxeH06dP6+1z/fp1jBkzBn5+fggICMCECRNw8+ZNvX2OHDmC+++/H97e3oiIiMD7779fpS2rVq1Cq1at4O3tjfbt22PDhg2y2yLn/IqLizFjxgy0b98etWrVQlhYGMaOHYsrV67oPYah13zOnDmqOD9z5wgA48ePr9L+wYMH6+3jrK8hAIP/jxqNBnPnztXuo+bXUMp1QU2fnVLaYpaoplasWCE8PT3F4sWLxfHjx8XEiRNFQECAyMrKcmi7Bg0aJJYsWSKOHTsmUlJSxAMPPCAaNmwobt68qd2nd+/eYuLEiSIjI0P7k5ubq729pKREtGvXTsTFxYlDhw6JDRs2iMDAQDFz5kztPmfPnhU1a9YUL730kjhx4oT4+OOPhbu7u9i4caN2H1s8R2+99ZZo27atXtuvXr2qvf25554TERERYuvWrSIpKUl0795dxMbGOsW5VcjOztY7v82bNwsAYvv27UII53v9NmzYIP7973+L1atXCwBizZo1erfPmTNH+Pv7i7Vr14rDhw+LBx98UDRp0kTcuXNHu8/gwYNFdHS02Lt3r9i5c6do3ry5GD16tPb23NxcERwcLMaMGSOOHTsmvv/+e+Hj4yO++OIL7T67d+8W7u7u4v333xcnTpwQr7/+uvDw8BBHjx6V1RY555eTkyPi4uLEypUrxalTp0RiYqLo2rWr6Ny5s95jNGrUSMyePVvvNdX9n3Xk+Zk7RyGEGDdunBg8eLBe+69fv663j7O+hkIIvfPKyMgQixcvFhqNRpw5c0a7j5pfQynXBTV9dpprixTVNjjp2rWrmDx5svbv0tJSERYWJuLj4x3Yqqqys7MFAPHHH39ot/Xu3VtMnTrV6H02bNgg3NzcRGZmpnbbggULhJ+fnygsLBRCCPHqq6+Ktm3b6t1v1KhRYtCgQdq/bfEcvfXWWyI6OtrgbTk5OcLDw0OsWrVKu+3kyZMCgEhMTFT9uRkzdepU0axZM1FWViaEcO7Xr/IHf1lZmQgJCRFz587VbsvJyRFeXl7i+++/F0IIceLECQFAHDhwQLvPb7/9JjQajbh8+bIQQojPPvtM1KlTR3t+QggxY8YMERkZqf370UcfFUOHDtVrT7du3cSzzz4ruS1yz8+Q/fv3CwDiwoUL2m2NGjUS8+bNM3oftZyfEIbPcdy4cWL48OFG7+Nqr+Hw4cNFv3799LY502tY+bqgps9OKW2RoloO6xQVFSE5ORlxcXHabW5uboiLi0NiYqIDW1ZVbm4uAKBu3bp627/77jsEBgaiXbt2mDlzJm7fvq29LTExEe3bt0dwcLB226BBg5CXl4fjx49r99E9/4p9Ks7fls/R6dOnERYWhqZNm2LMmDFIT08HACQnJ6O4uFjvmK1atULDhg21x1T7uVVWVFSEZcuW4amnntJbtNKZXz9d586dQ2Zmpt5x/P390a1bN73XLCAgAF26dNHuExcXBzc3N+zbt0+7T69eveDp6al3Pqmpqbhx44akc5bSFiXk5uZCo9EgICBAb/ucOXNQr149dOzYEXPnztXrLneG80tISEBQUBAiIyMxadIkXLt2Ta/9rvIaZmVl4ddff8WECROq3OYsr2Hl64KaPjultEUKp1j4T2l//fUXSktL9V4kAAgODsapU6cc1KqqysrKMG3aNPTo0QPt2rXTbn/88cfRqFEjhIWF4ciRI5gxYwZSU1OxevVqAEBmZqbBc6u4zdQ+eXl5uHPnDm7cuGGT56hbt25YunQpIiMjkZGRgbfffhv3338/jh07hszMTHh6elb50A8ODjbbbjWcmyFr165FTk4Oxo8fr93mzK9fZRXtMXQc3bYGBQXp3V6jRg3UrVtXb58mTZpUeYyK2+rUqWP0nHUfw1xbrFVQUIAZM2Zg9OjRegukvfjii+jUqRPq1q2LPXv2YObMmcjIyMCHH37oFOc3ePBgPPzww2jSpAnOnDmD1157DUOGDEFiYiLc3d1d6jX8+uuv4evri4cfflhvu7O8hoauC2r67JTSFimqZXDiLCZPnoxjx45h165detufeeYZ7e/t27dHaGgo+vfvjzNnzqBZs2b2bqYsQ4YM0f4eFRWFbt26oVGjRvjhhx/g4+PjwJbZxqJFizBkyBCEhYVptznz61edFRcX49FHH4UQAgsWLNC77aWXXtL+HhUVBU9PTzz77LOIj49XVUlwYx577DHt7+3bt0dUVBSaNWuGhIQE9O/f34EtU97ixYsxZswYeHt76213ltfQ2HXB1VTLYZ3AwEC4u7tXyR7OyspCSEiIg1qlb8qUKVi/fj22b9+O8PBwk/t269YNAJCWlgYACAkJMXhuFbeZ2sfPzw8+Pj52e44CAgLQsmVLpKWlISQkBEVFRcjJyTF6TGc6twsXLmDLli14+umnTe7nzK9fxWOZOk5ISAiys7P1bi8pKcH169cVeV11bzfXFktVBCYXLlzA5s2bzS4r361bN5SUlOD8+fMm267bbkeeX2VNmzZFYGCg3nvS2V9DANi5cydSU1PN/k8C6nwNjV0X1PTZKaUtUlTL4MTT0xOdO3fG1q1btdvKysqwdetWxMTEOLBl5dPMpkyZgjVr1mDbtm1VuhENSUlJAQCEhoYCAGJiYnD06FG9D5OKD9Q2bdpo99E9/4p9Ks7fXs/RzZs3cebMGYSGhqJz587w8PDQO2ZqairS09O1x3Smc1uyZAmCgoIwdOhQk/s58+vXpEkThISE6B0nLy8P+/bt03vNcnJykJycrN1n27ZtKCsr0wZmMTEx2LFjB4qLi/XOJzIyEnXq1JF0zlLaYomKwOT06dPYsmUL6tWrZ/Y+KSkpcHNz0w6FqPn8DLl06RKuXbum95505tewwqJFi9C5c2dER0eb3VdNr6G564KaPjultEUSyamzLmbFihXCy8tLLF26VJw4cUI888wzIiAgQC+T2REmTZok/P39RUJCgt6Uttu3bwshhEhLSxOzZ88WSUlJ4ty5c2LdunWiadOmolevXtrHqJgyNnDgQJGSkiI2btwo6tevb3DK2PTp08XJkyfFp59+anDKmNLP0csvvywSEhLEuXPnxO7du0VcXJwIDAwU2dnZQojyKWgNGzYU27ZtE0lJSSImJkbExMQ4xbnpKi0tFQ0bNhQzZszQ2+6Mr19+fr44dOiQOHTokAAgPvzwQ3Ho0CHtbJU5c+aIgIAAsW7dOnHkyBExfPhwg1OJO3bsKPbt2yd27dolWrRooTcNNScnRwQHB4t//OMf4tixY2LFihWiZs2aVaZp1qhRQ/z3v/8VJ0+eFG+99ZbBaZrm2iLn/IqKisSDDz4owsPDRUpKit7/ZMUMhz179oh58+aJlJQUcebMGbFs2TJRv359MXbsWFWcn7lzzM/PF6+88opITEwU586dE1u2bBGdOnUSLVq0EAUFBU7/GlbIzc0VNWvWFAsWLKhyf7W/huauC0Ko67PTXFukqLbBiRBCfPzxx6Jhw4bC09NTdO3aVezdu9fRTRIADP4sWbJECCFEenq66NWrl6hbt67w8vISzZs3F9OnT9erkyGEEOfPnxdDhgwRPj4+IjAwULz88suiuLhYb5/t27eLDh06CE9PT9G0aVPtMXQp/RyNGjVKhIaGCk9PT9GgQQMxatQokZaWpr39zp074vnnnxd16tQRNWvWFA899JDIyMhwinPTtWnTJgFApKam6m13xtdv+/btBt+T48aNE0KUT4984403RHBwsPDy8hL9+/evct7Xrl0To0ePFrVr1xZ+fn7iySefFPn5+Xr7HD58WPTs2VN4eXmJBg0aiDlz5lRpyw8//CBatmwpPD09Rdu2bcWvv/6qd7uUtsg5v3Pnzhn9n6yoW5OcnCy6desm/P39hbe3t2jdurV477339C7sjjw/c+d4+/ZtMXDgQFG/fn3h4eEhGjVqJCZOnFgliHXW17DCF198IXx8fEROTk6V+6v9NTR3XRBCXZ+dUtpijubuiRMRERGpQrXMOSEiIiL1YnBCREREqsLghIiIiFSFwQkRERGpCoMTIiIiUhUGJ0RERKQqDE6IiIhIVRicEBERkaowOCEiIiJVYXBCREREqsLghIiIiFSFwQkRERGpyv8DJoZUBUW5RZgAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(used_lrs, lossi)" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(2.3539, grad_fn=)" + ] + }, + "execution_count": 75, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "emb = C[Xtr] # (32, 3, 2)\n", + "h = torch.tanh(emb.view(-1, 30) @ W1 + b1) # (32, 100)\n", + "logits = h @ W2 + b2 # (32, 27)\n", + "loss = F.cross_entropy(logits, Ytr)\n", + "loss" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(2.3602, grad_fn=)" + ] + }, + "execution_count": 76, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "emb = C[Xdev] # (32, 3, 2)\n", + "h = torch.tanh(emb.view(-1, 30) @ W1 + b1) # (32, 100)\n", + "logits = h @ W2 + b2 # (32, 27)\n", + "loss = F.cross_entropy(logits, Ydev)\n", + "loss" + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAp8AAAKTCAYAAABfKmNzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABVFklEQVR4nO3de3xU9Z3/8ffM5IJAAgkxXEIkKCgihlhCEG/ddhFvtdjSeK1a16pbpVXYomJVtLZq0VW2LS2r1db+WjeCVLFKKRSl3pBAMAYEBSQJEEggJCQhaDKZOb8/6MSEzExmMiffzCSv5+PB42HOnHPmOx8jvud7vheHZVmWAAAAAAOcPd0AAAAA9B2ETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDFxPd2AUHi9Xu3bt09JSUlyOBw93RwAAAAcx7IsNTQ0aMSIEXI6A/dvxkT43LdvnzIzM3u6GQAAAOjEnj17NHLkyICvx0T4TEpKknTswyQnJ/dwa6KH2+3WqlWrNH36dMXHx/d0c2IWdbQHdYwcNbQHdbQHdYxcX6thfX29MjMzW3NbIDERPn2P2pOTkwmfbbjdbvXv31/Jycl94pe6u1BHe1DHyFFDe1BHe1DHyPXVGnY2RJIJRwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAC9lNdr9XQTOojr6QYAAADAHlsq6rR04x4VltVo54EjcnssxbscGpM+UHlZqcrPzdSEjEE92kbCJwAAQIwrq27U3ctKVFhaI5fTIU+bHk+3x9K2/Q3aXnVEL6wrV97oVC2Yma2stAE90lYeuwMAAMSw5cUVmv702yoqr5WkdsGzLd/xovJaTX/6bS0vrjDWxrbo+QQAAIhRy4srdFdBscIZ2enxWvLI0l0FxZKkGTkZ3dK2QOj5BAAAiEGl1Y2au7QkrODZliVp7tISlVU32tmsThE+AQAAYtA9y0rksSKbze6xLN29rMSmFoWG8AkAABBjNu+tU2FpTcDxnaHyeC0VltZoS0WdTS3rHGM+AQAAYszLRXsU53SoxU/4PCHepZ99a4IuPmOYGpta9Mw7uzTt9KHauq9eP319a4fzXU6Hlm7cY2wJJsInAABAjCksq/EbPCXpvktP15TRqbrljxt16Eiz5l58ms4Ykayt++r9nu/xWtpQVtudzW2Hx+4AAAAxZueBI36P909w6crJI/Xoim16/7ND+rSqQf+15CPFOYNHvh0HGrqjmX4RPgEAAGKI12vJ7fHf6zlqSH8lxrlUvPtw67G6z93aVe0/rPq4PZaxrTgJnwAAADHE6XQo3uWw9Z7xLoecTnvvGQjhEwAAIMaMSR/o93j5oaNqbvEq56TBrceST4jT6E620hybnmRn84JiwhEAAECMyctK1faqIx2WWjra7NGSjXt036Wnq/aoW4eONGnuRacp2BN1l9OhyVkp3dziLxE+AQAAYkx+bqZeWFfu97VHV2xT/wSXnrsxV41NLXr2nVIl9YsPeC+P11J+bmZ3NbUDwicAAECMmZAxSHmjU1VUXuu393POko80Z8lHrce+Pi7d731cTocmjUoxtsanxJhPAACAmLRgZrZcjsgmCbkcDi2YmW1Ti0JD+AQAAIhBWWkD9ER+troaPx2SnsjPVlYnk5HsxmN3AACAGDUjJ0OSNHdpiTyWFXCv96uf+aD1n11Oh1wOh57Iz2693iR6PgEAAGLYjJwMrZp9gSaNOjZj3RVgvU7f8dxRKVo1+4IeCZ4SPZ8AAAAxLyttgJbcNlVbKuq0dOMebSir1Y4DDXJ7LMW7HBqbnqTJWSnKz800OrnIH8InAABALzEhY1C7cOn1WsZ2LgoVj90BAAB6qWgLnhLhEwAAAAYRPgEAAGAM4RMAAADGED4BAABgDOETAAAAxhA+AQAAYAzhEwAAAMYQPgEAAGAM4RMAAADGdCl8Llq0SFlZWerXr5+mTJmiwsLCoOcvXLhQp512mk444QRlZmZq9uzZ+uKLL7rUYAAAAMSusMPnSy+9pDlz5mj+/PnatGmTJk6cqIsuukgHDhzwe/6LL76oe++9V/Pnz9e2bdv03HPP6aWXXtJ9990XceMBAAAQW8IOn0899ZRuueUW3XTTTRo/frwWL16s/v376/nnn/d7/vvvv69zzz1X1157rbKysjR9+nRdc801nfaWAgAAoPeJC+fk5uZmFRUVad68ea3HnE6npk2bpnXr1vm95pxzztGf/vQnFRYWKi8vT7t27dKKFSt0/fXXB3yfpqYmNTU1tf5cX18vSXK73XK73eE0uVfz1YKaRIY62oM6Ro4a2oM62oM6Rq6v1TDUz+mwLMsK9ab79u1TRkaG3n//fU2dOrX1+N13361//vOfWr9+vd/rfvnLX+rHP/6xLMtSS0uL/vM//1O//e1vA77PQw89pIcffrjD8RdffFH9+/cPtbkAAAAw5OjRo7r22mtVV1en5OTkgOeF1fPZFWvXrtWjjz6q3/zmN5oyZYp27typO++8U4888ogeeOABv9fMmzdPc+bMaf25vr5emZmZmj59etAP09e43W6tXr1aF154oeLj43u6OTGLOtqDOkaOGtqDOtqDOkaur9XQ96S6M2GFz7S0NLlcLlVVVbU7XlVVpWHDhvm95oEHHtD111+v73//+5KkM888U42Njbr11lv1k5/8RE5nx2GniYmJSkxM7HA8Pj6+T/zLCxd1sQd1tAd1jBw1tAd1tAd1jFxfqWGonzGsCUcJCQmaNGmS1qxZ03rM6/VqzZo17R7Dt3X06NEOAdPlckmSwnjiDwAAgF4g7Mfuc+bM0Y033qjc3Fzl5eVp4cKFamxs1E033SRJuuGGG5SRkaHHHntMknT55Zfrqaee0llnndX62P2BBx7Q5Zdf3hpCAQAA0DeEHT6vuuoqHTx4UA8++KAqKyuVk5OjlStXaujQoZKk3bt3t+vpvP/+++VwOHT//feroqJCJ554oi6//HL9/Oc/t+9TAAAAICZ0acLRrFmzNGvWLL+vrV27tv0bxMVp/vz5mj9/flfeCgAAAL0Ie7sDAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInuszrtXq6CQAAIMbE9XQDEDu2VNRp6cY9Kiyr0c4DR+T2WIp3OTQmfaDyslKVn5upCRmDerqZAAAgihE+0amy6kbdvaxEhaU1cjkd8rTp8XR7LG3b36DtVUf0wrpy5Y1O1YKZ2cpKG9CDLQYAANGKx+4IanlxhaY//baKymslqV3wbMt3vKi8VtOfflvLiyuMtREAAMQOej4R0PLiCt1VUKxwRnZ6vJY8snRXQbEkaUZORre0DQAAxCZ6PuFXaXWj5i4tCSt4tmVJmru0RGXVjXY2CwAAxDjCJ/y6Z1mJPFZks9k9lqW7l5XY1CIAANAb8NgdHWzeW6fC0hq/rxXcerY+rWyQJH3rKxlq8Vj60wflemr19g7neryWCktrtKWijlnwAABAEj2f8OPloj2KczoCvj5z0kh5vJau+PV7evivH+v754/W1ZMz/Z7rcjq0dOOe7moqAACIMfR8ooPCshq1BFlAfv/hz/XT17dKknZVN2rcsCTdfN5oFWzoGDI9Xksbymq7ra0AACC20POJDnYeOBL09Q/3HG7386bdh5WVNkCBOkt3HGiwqWUAACDWET7Rjtdrye2xd9tMt8diK04AACCJ8InjOJ0OxbsCj/eUpJzMwe1+PitzsMqqGxUoX8a7HHIGGUMKAAD6DsInOhiTPjDo6yMGn6D7LztdJ6cN0DcnjtCN52Tp9++VBTx/bHqSzS0EAACxiglH6CAvK1Xbq44E3ErzL5v2ql+8S6/OOlder6Xfv1emFwt3+z3X5XRoclZKdzYXAADEEMInOsjPzdQL68oDvt7isfTT1z/W/a9u6fReHq+l/Fz/yzABAIC+h8fu6GBCxiDljU6VK8Jxmi6nQ3mjU1lgHgAAtCJ8wq8FM7PlckQYPh0OLZiZbVOLAABAb0D4hF9ZaQP0RH62jo+fVz/zQesC88E4JD2Rn62stAHd0j4AABCbGPOJgGbkZEiS5i4tkceyAk5AasvldMjlcOiJ/OzW6wEAAHzo+URQM3IytGr2BZo06tiM9UDjQH3Hc0elaNXsCwieAADAL3o+0amstAFacttUbamo09KNe7ShrFY7DjTI7bEU73JobHqSJmelKD83k8lFAAAgKMInQjYhY1C7cOn1WuxcBAAAwsJjd3QZwRMAAISL8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGO6FD4XLVqkrKws9evXT1OmTFFhYWHQ8w8fPqw77rhDw4cPV2Jiok499VStWLGiSw0GAABA7IoL94KXXnpJc+bM0eLFizVlyhQtXLhQF110kT799FOlp6d3OL+5uVkXXnih0tPT9fLLLysjI0Pl5eUaPHiwHe0HAABADAk7fD711FO65ZZbdNNNN0mSFi9erDfeeEPPP/+87r333g7nP//886qpqdH777+v+Ph4SVJWVlZkrQYAAEBMCit8Njc3q6ioSPPmzWs95nQ6NW3aNK1bt87vNa+99pqmTp2qO+64Q8uXL9eJJ56oa6+9Vvfcc49cLpffa5qamtTU1NT6c319vSTJ7XbL7XaH0+RezVcLahIZ6mgP6hg5amgP6mgP6hi5vlbDUD9nWOGzurpaHo9HQ4cObXd86NCh+uSTT/xes2vXLr355pu67rrrtGLFCu3cuVO333673G635s+f7/eaxx57TA8//HCH46tWrVL//v3DaXKfsHr16p5uQq9AHe1BHSNHDe1BHe1BHSPXV2p49OjRkM4L+7F7uLxer9LT0/XMM8/I5XJp0qRJqqio0BNPPBEwfM6bN09z5sxp/bm+vl6ZmZmaPn26kpOTu7vJMcPtdmv16tW68MILW4c0IHzU0R7UMXLU0B7U0R7UMXJ9rYa+J9WdCSt8pqWlyeVyqaqqqt3xqqoqDRs2zO81w4cPV3x8fLtH7KeffroqKyvV3NyshISEDtckJiYqMTGxw/H4+Pg+8S8vXNTFHtTRHtQxctTQHtTRHtQxcn2lhqF+xrCWWkpISNCkSZO0Zs2a1mNer1dr1qzR1KlT/V5z7rnnaufOnfJ6va3Htm/fruHDh/sNngAAAOi9wl7nc86cOXr22Wf1wgsvaNu2bfrBD36gxsbG1tnvN9xwQ7sJST/4wQ9UU1OjO++8U9u3b9cbb7yhRx99VHfccYd9nwIAAAAxIewxn1dddZUOHjyoBx98UJWVlcrJydHKlStbJyHt3r1bTueXmTYzM1N///vfNXv2bGVnZysjI0N33nmn7rnnHvs+BQAAAGJClyYczZo1S7NmzfL72tq1azscmzp1qj744IOuvBUAAAB6EfZ2BwAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDFdCp+LFi1SVlaW+vXrpylTpqiwsDCk6woKCuRwOHTFFVd05W0BAAAQ48IOny+99JLmzJmj+fPna9OmTZo4caIuuugiHThwIOh1ZWVl+vGPf6zzzz+/y40FAABAbIsL94KnnnpKt9xyi2666SZJ0uLFi/XGG2/o+eef17333uv3Go/Ho+uuu04PP/yw3nnnHR0+fDjoezQ1Nampqan15/r6ekmS2+2W2+0Ot8m9lq8W1CQy1NEe1DFy1NAe1NEe1DFyfa2GoX5Oh2VZVqg3bW5uVv/+/fXyyy+3e3R+44036vDhw1q+fLnf6+bPn6+SkhK98sor+t73vqfDhw/r1VdfDfg+Dz30kB5++OEOx1988UX1798/1OYCAADAkKNHj+raa69VXV2dkpOTA54XVs9ndXW1PB6Phg4d2u740KFD9cknn/i95t1339Vzzz2n4uLikN9n3rx5mjNnTuvP9fX1yszM1PTp04N+mL7G7XZr9erVuvDCCxUfH9/TzYlZ1NEe1DFy1NAe1NEe1DFyfa2GvifVnQn7sXs4GhoadP311+vZZ59VWlpayNclJiYqMTGxw/H4+Pg+8S8vXNTFHtTRHtQxctTQHtTRHtQxcn2lhqF+xrDCZ1pamlwul6qqqtodr6qq0rBhwzqc/9lnn6msrEyXX3556zGv13vsjePi9Omnn+qUU04JpwkAAACIYWHNdk9ISNCkSZO0Zs2a1mNer1dr1qzR1KlTO5w/btw4bd68WcXFxa1/vvnNb+prX/uaiouLlZmZGfknAAAAQMwI+7H7nDlzdOONNyo3N1d5eXlauHChGhsbW2e/33DDDcrIyNBjjz2mfv36acKECe2uHzx4sCR1OA4AAIDeL+zwedVVV+ngwYN68MEHVVlZqZycHK1cubJ1EtLu3bvldLJxEgAAADrq0oSjWbNmadasWX5fW7t2bdBr//CHP3TlLQEAANAL0EUJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMIn0A38Xqtnm4CAABRJ66nGwD0Flsq6rR04x4VltVo54EjcnssxbscGpM+UHlZqcrPzdSEjEE93UwAAHoU4ROIUFl1o+5eVqLC0hq5nA552vR4uj2Wtu1v0PaqI3phXbnyRqdqwcxsZaUN6MEWAwDQc3jsDkRgeXGFpj/9torKayWpXfBsy3e8qLxW059+W8uLK4y1EQCAaELPJ9BFy4srdFdBscIZ2enxWvLI0l0FxZKkGTkZ3dI2AACiFT2fQBeUVjdq7tKSsIJnW5akuUtLVFbdaGezAACIeoRPoAvuWVYijxXZbHaPZenuZSU2tQgAgNhA+ATCtHlvnQpLawKO7wyVx2upsLRGWyrqbGoZAADRjzGfQJheLtqjOKdDLQHC51dPPVGzvj5Gpw1NksdradPuWj38163aXXO0w7kup0NLN+5hCSYAQJ9BzycQpsKymoDBU5JOSHDpd++U6vJfv6vrfrdeXkv63+snyeHoeK7Ha2lDWW03thYAgOhCzycQpp0HjgR9feWWynY/3/3yR/rwwekamz5Q26s6XrvjQIOt7QMAIJoRPoEweL2W3J7gYz2zhvTXnAtPVU5milIGxMv5ry7PEYNP8Bs+3R5LXq8lp9NP1ygAAL0M4RMIg9PpULzLETSAPnfjZFUc/lz3/qVEVfVNcjqk1XO+qgSX/1Eu8S4HwRMA0Gcw5hMI05j0gQFfG9w/XqekD9Sv3tyh9z87pM8OHtGgE+KD3m9sepLdTQQAIGrR8wmEKS8rVdurjvhdaqnuc7dqGpt1Td5JOtDQpBGDT9A9F48LeC+X06HJWSnd2VwAAKIKPZ9AmPJzMwOu8WlZ0g//b5POzBikVXddoAe/MV6PrdgW8F4er6X83MzuaioAAFGHnk8gTBMyBilvdKqKymv9htD3dh7ShU+/3e5Y1r1vdDjP5XRo0qgU1vgEAPQp9HwCXbBgZrZc/hbuDIPL4dCCmdk2tQgAgNhA+AS6ICttgJ7Iz1ZX46dD0hP52cpKG2BnswAAiHo8dge6aEZOhiRp7tISeSwrpL3eXU6HXA6HnsjPbr0eAIC+hJ5PIAIzcjK0avYFmjTq2Ix1V4D1On3Hc0elaNXsCwieAIA+i55PIEJZaQO05Lap2lJRp6Ub92hDWa12HGiQ22Mp3uXQ2PQkTc5KUX5uJpOLAAB9HuETsMmEjEHtwiVbZgIA0BGP3YFuQvAEAKAjwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjOlS+Fy0aJGysrLUr18/TZkyRYWFhQHPffbZZ3X++ecrJSVFKSkpmjZtWtDzAQAA0HuFHT5feuklzZkzR/Pnz9emTZs0ceJEXXTRRTpw4IDf89euXatrrrlGb731ltatW6fMzExNnz5dFRUVETceAAAAsSXs8PnUU0/plltu0U033aTx48dr8eLF6t+/v55//nm/5//5z3/W7bffrpycHI0bN06/+93v5PV6tWbNmogbDwAAgNgSF87Jzc3NKioq0rx581qPOZ1OTZs2TevWrQvpHkePHpXb7VZqamrAc5qamtTU1NT6c319vSTJ7XbL7XaH0+RezVcLahIZ6mgP6hg5amgP6mgP6hi5vlbDUD+nw7IsK9Sb7tu3TxkZGXr//fc1derU1uN33323/vnPf2r9+vWd3uP222/X3//+d3388cfq16+f33MeeughPfzwwx2Ov/jii+rfv3+ozQUAAIAhR48e1bXXXqu6ujolJycHPC+sns9IPf744yooKNDatWsDBk9JmjdvnubMmdP6c319fetY0WAfpq9xu91avXq1LrzwQsXHx/d0c2JWb6vjtv31euXDChWV12rXwSNyey3FOx06+cSBmjQqRd86K0OnD7f/v6PeVseeQA3tQR3tQR0j19dq6HtS3ZmwwmdaWppcLpeqqqraHa+qqtKwYcOCXvvkk0/q8ccf1z/+8Q9lZ2cHPTcxMVGJiYkdjsfHx/eJf3nhoi72iPU6llU36u5lJSosrZHL6ZDH63uo4VCTRyrZd0QfVzbq9+v2KG90qhbMzFZW2gDb2xHrdYwG1NAe1NEe1DFyfaWGoX7GsCYcJSQkaNKkSe0mC/kmD7V9DH+8BQsW6JFHHtHKlSuVm5sbzlsCCMHy4gpNf/ptFZXXSlKb4Nme73hRea2mP/22lhez6gQAwKywH7vPmTNHN954o3Jzc5WXl6eFCxeqsbFRN910kyTphhtuUEZGhh577DFJ0i9+8Qs9+OCDevHFF5WVlaXKykpJ0sCBAzVw4EAbPwrQNy0vrtBdBcUKefC2joVQjyzdVVAsSZqRk9EtbQMA4Hhhh8+rrrpKBw8e1IMPPqjKykrl5ORo5cqVGjp0qCRp9+7dcjq/7FD97W9/q+bmZn3nO99pd5/58+froYceiqz1QB9XWt2ouUtLwgqebVmS5i4t0cSRg7vlETwAAMfr0oSjWbNmadasWX5fW7t2bbufy8rKuvIWAEJwz7ISeUJfsMIvj2Xp7mUlWnJb4KEzJnm9lpxOR083AwDQTYzOdgdgn81761RYWhPxfTxeS4WlNdpSUacJGYNsaFl4tlTUaenGPSosq9HOA0fk9liKdzk0Jn2g8rJSlZ+b2SPtAgB0D8InEKNeLtqjOKdDLX4mF717z9f0/Lulev69stZjK350nlZtrdLCf+zocL7L6dDSjXuMhrzAs/Mlt8fStv0N2l51RC+sK+/W2fkAALPC3l4TQHQoLKvxGzy7wuO1tKGs1pZ7hYLZ+QDQd9HzCcSonQeO2Hq/HQcabL1fIMzOB4C+jZ5PIAZ5vZbcHnt6PX3cHktem3pSA7Frdn5ZdaOdzQIAGET4BGKQ0+lQvCvwjHCvV3I42r8e5wr+n3u8y9FhlrndYdTO2fkAgNjEY3cgRo1JH6ht+/0/Kq9pbNKJSV9uUTswMU6ZKf2D3m9selK3zjzvbHa+wyHdev7JuibvJA0f3E/VR5r14vrdWvTWznbn9fTsfABAZAifQIzKy0rV9qojfifrvP/ZIX1n0kit2Val+s9bNGf6qUF7HF0Oh6obm/SNX73bbTPPg83Ol6R7Lhqnq/My9cjrW7WhrFbpSYk6Jd3/Lmg9MTsfAGAPHrsDMSo/NzPgLPHfrP1M60tr9Nz3Juv5myZr1ceV2n0o8DhJj2WpuqHp2D9308zzYLPzByS4dNO5WXrsb59o2aYK7a45qo3ltXppw56AbTE5Ox8AYB96PoEYNSFjkPJGp6qovLZDYDzS1KIf/t+H7Y4t2xQ8LIY6vLOrM8+Dzc4fkz5QifEuvbezOrRGyNzsfACAvej5BGLYgpnZcjl6ZivKcGaedzY7/wu3N+z3NzE7HwBgP8InEMOy0gboifxs9dRO6KHOPO9sdn7ZoUZ93uzRuWPSQn5vf7PzAQDRj/AJ2KgneuJm5GRo4dU5SnA55QoxjNmV2drOPO/MmACThySpqcWrxf/8TPMuGadvfyVDJ6X211mZg3VlbmbAa8amJ3WpzQCAnsWYTyAC3bk0UThm5GRo4sjBAfdK9/EdPzEpUdUNTfL3JDzB5dS8S8fp8okjlJQYp5KKOj3y+laV7PUfMH0zz++/9LSgbQw2O1+SfvnmDrV4Lc258FSlJ/XTgYYv9OL63QHfc3JWStD3AwBEJ8In0AVl1Y0Bg55dSxOFKyttgJbcNrU1EG8oq9WOAw2tgXhsepImZ6UoPzdTc1/+SFX1TX7vM+/ScbpkwnD9eMlH2nv4c/3nV0/WH/8jT199Yq3qPnd3OD/Umef5uZl6YV15wNctS1r01s4O63r64/Fayg/SKwoAiF6ETyBMy4srNHfplzv1hLo00RP52Ub2JJ+QMahdb6vXa3UYGxlo5vkJ8S5dN2WUfrz0I63dflCSdO+yzXr3nhN11eRMPfP2Lr/XhTLzPNjs/HC4nA5NGpXCGp8AEKMY8wmEYXlxhe4qKFazxxtygPJ4LTV7vLqroDjstTH9CXdcqb8tMwPNPB81pL8S4pwqKv+yJ7PFa+mjvYeDjtkMdea5HbPzXQ6HFszMjugeAICeQ88nEKLS6kbNXVqirvbZ+ZYmmjhycFiP4O0eV+qbeR5s6aNwhTrz3Dc7/66C4i7V0SHpifzuH8IAAOg+9HwCIbpnWUnQLSpDEerSRNKxcaVX/u86feNX7+pP63dr2/6G1sDoG1f6p/W79Y1fvasr/3ddSOtt+gTqxSw/dFRNLR5NGvXlZJ44p0PZIwdpR1XgReLDmXneldn5LqdDCS6nFl6dY2ToAgCg+9DzCYRg8946FZbW+H3N4ZB+8NVTdE3eSToxKVGl1Y365Zod+tuWyg7ntl2aKFhvZXePKw008/xzt0d//mC37rv0dNV97lbFvyYcnRDv0ksb7Zt57pudf8eLm/TxvvqA5zkdx3Zeyh2Vol8YmLQFAOh+hE8gBC8X7VGc0+F3b/Lb/22MvnVWhn7yymaVHmrUlNFDtPCqHNU0Fmq9n8DqW5ooUPj0jSv1FzcLbj1bW/fV66evb213PNwtL4PNPP/Fyk/kcEhPXTlRA/+11NINzxeq/vMWv+d3Zea5b7WAj/fVy+WQ3yWfpGPB84wRyQRPAOhFCJ9ACArLavwGzwSXU3d87RR993frtWn3YUnSnpq9ys1K0bVTTvIbPoMtTdTZuNLb/l+RWjyBt6IMdVxpsJnnTS1ePfzXrXr4r1sDXP2ltjPP3e6OyzD506FXt5ORDJ9UNhhdLQAA0L0In0AIAi1NNGpIf/VPiNP/u3lKu+PxLqe27gu860+gpYk6G1fqb53N4/nGlS65bWrQ8xbMzNb0p9+Wp8tTqMKfeR6sVzeQcHt1AQDRjfAJdCLY0kQDEo/9J/Qff9igyvov2r3W3BK4h9K3NFHbGeLBxpX6BHrs3lao40pNzzzvqdUCAADRhdnuQCd8SxP5s6OqQU1uj0YMPkHlh462+7O/7gu/10j+lybyjSu1g29caWdMzjzvrtUCwl33FADQs+j5BEIwJn2gtu3v+Ki8sdmjZ97ZpQe+MV5Oh7ShrFZJ/eKUm5WqI1+4tWyT/0Xl/S1NFGhcaVeEuuWlFP6+8F2ZeR5Kr24ofL26s/5cpM+qG21Z9xQAYBbhEwhBoKWJJOm/V21XTWOzbv+3McpM7a/6L9z6uKJOi9Z+5vdegZYmCjSutKtC2fLSJ5x94bsS7AKtFpA6IEF/v+t8/f69Mv3mX/X6ykkpKrj1bH3v94V6/7NDfu/3xubKdo/vfeuebq86ohfWlStvdKoWMEMeAKIS4RMIQbCliSTp9++V6ffvlYV0L39LEwUbV9pV/saVdiaUfeG7IlCvbk1js+a+XKJnrs/VOzuqtevgET191UT9cV1ZwOApKeC40a6uewoAMIfwCYQg2NJE4Wi7NFFbPbnlZTB2BE8peK/u2k8PqmDDbi28Okeb99bpaLNHC1Z+GtH7MUMeAKIXE46AEC2YmS2XI7IwFmxpokBbXnZVOFtedqdQenV//sY2xTkduvTM4bqroFjNQdYyDYdvhnw4W48CALoX4RMIkW9poq7Gz86WJsrLSg15xnlnurLlZXcJtlqAz6gh/TU0uZ+cDmlk6gm2vr+/GfIAgJ5D+ATC0J1LE+XnZnb6SP/qZz4IusanT1e2vOxOwXp1410OLbwqR6+X7NNTq7fr8W9na8iABNveu+26pwCAnkf4BMI0IydDq2ZfoEmjjvUsBgqhvuO5o1K0avYFnY479I0rjbT30+V0KG90alQtNxSsV/fH009TUr94PfTaVv32n5+ptLpRC77T+a5JAxJcWnhVjrb+9CIV3vfvuvm80Sq49Ww9+I3xHc4Ndd1TAED3I3wCXeBbmuj1H56n7045SeOHJ7c+Wo53OTR+eLK+O+Ukvf7D8/TSbVOVlTYgpMXQu3tcaU8J1Kt79smp+o/zRmv2S8U60tQiy5LmLCnW5NGp+u6Uk4Le8/5vjFduVoq+/8JGffe59ZqclaozRiT7PTecdU8BAN2L2e5ABIItTeRbM3Puyx+FvBi66S0vTQm0WsAHu2o09id/a3fu3trPlf3QqqD3G5Dg0syvjNSdBR+2Lsk0d+lHWv+Tfw94TTjrngIAug/hE7CR0+lQWXVjwN2CQlkM3fd4fu7SY9tRhrK0k8vpkMvhiOp1LRfMzNa0p/5py71OGtJfCXFOfbTncOuxhqYW7ToYeFZ7V9Y9BQDYj8fugI2WF1do+tNvq6j82CPeQMHx+MXQlxe334azu8aVmtZ2qIElyRvh3u6RsGPdUwBA5Oj5BGyyvLgi7MflwRZD7+4tL7vLoyu26YOywx2GGlTUfq5gnbgFt56trfvqQ5rNv/vQUTW3eJWdOVj76iolSUmJcRqdNiDgHvLRsu4pAPR1hE/ABqXVjZq7tKRD8Aw1UPkWQ584cnCH8ZrdteWlncqqG3XfX4p11VDppY17dNT95Wu+oQZ2amz2aNmmvbrvktNVd9St6iNNmn3hqfJaliw/8T+a1j0FgL6Ox+6ADe5Zdmx8ZiRCXQw92oKnb6hB8b/GX3Zl+9En87N19slD9B/njVbZ45ep7PHLNDIl+GLzP3t9qzbtrtVz38vVn78/RUXltfrswBE1uTvujhRt654CQF9GzycQoc176wI+6g1H28XQo+kxejBthxo4ujQ//5iHX9uq0WkD9Wllg55evV2SdKixKeg1jc0e3fVScevPJ8S7dOe/j9WLhe3X83Q5HZo0KiVmagoAvR3hE4jQy0V7FOd0qCVAj5/DId17yThdPTlTbo9Xf16/Wwv/scPvub7F0GMhKAUaahCuSyYM053TxurU9CSdMSJZJ6X21y1/3BhwfKhvKMOyTXt1yokDVbznsJL6xenOfx8rSVq9tbLd+dG47ikA9GWETyBChWU1AYOnJM2cNFLPvVOqKxa9p6+MStGT35mojWW1endndYdzY2kxdDuGGpyYlKhfXnOWHv/bJ/pG9nCVVTeqaPdhhbrO/i3nn6yTTxwgt8erzRV1yl+8TrVtBpxG67qnANCXET6BCO08cCTo65/sb9D/rDnW01l26KhumJqlc8cM8Rs+pdhYDN2uoQbpSYmKdzm1ckulLhw/VLVH3frTB+UhXfvxvnpd/ut3/b4WC+ueAkBfRfgEIuD1WnJ7gvf+fVJZ3+7ngw1faMjAxIDnx8Ji6P6GGvzbaem69NKzdO+Gv0uSxg9P1oo7z9dv1+7UL1Z+Kkl6fOaZSoxzafa/xmpu21+vd3dUa+Vd5+sLt1der6XkE+JU/3mLpGPjOH/2rQm6+Ixhamxq0TPv7AqpfbmjUvSLmfR4AkA0YrY7EAGn09G6p3sgLceFU8uSguXKWFgM3d9Qg41lNYqLi9P4EcfGq045OVWHjjTp7JOHtJ4zZfQQfbDrUOvPXkv67nPr9b3fb1Bl3ec666QUrf3xv+mMEclyOKT7Lj1dU0an6pY/btT1zxXq7JOHBNy/XTr2mP0bZw7TS7dNJXgCQJQifAIRGpM+0Nb7xcJi6P6GGhxpalFdXZ3yRh8Lm2efPETPvVuq8SOS1T/BpaHJiRqdNkDr24RPn6LyWv2ooFjbKuuV0j9Bb/zofJ1y4kBdOXmkHl2xTe9/dkifVjXov5Z8pDhn4L+2LEm7qo/6fc3bhSWgAAD247E7EKG8rFRtrzrSpfUtj2fHYujd/cg+2FCD6upq5Y0eot/+s1STs1K1YOWxiUSTs1I16IR4VdZ9obJDX4bDnMzBOueUIXpnR7UOHWnS797ZpaevytFtfyxSvMuhxDiXincfbj2/7nO3dlUHH2PrGzPr2xmqsKymw25LeVmpUbczFAD0FYRPIEL5uZl6YV1ok2Q605XF0E2HLN9QA38B9NChQ5qUPUrjhyerxePVZwcb9cGuGp198rHwub60fa9nwxctmjI6Vf9x3mglJcZp7+HP9fM3tmnt9oM6fXjXeoDdHkv5i9/XhrJauZyOdl8KfLstba86ohfWlStvdKoWMDYUAIwifAIRmpAxSHmjU1VUXtuh9/PqZz7ocP6t/6/I733CXQy9rLpRdy8rUWFpjfGQNSZ9oN8tMw8dOqQBiXG6+bzRWv+v2fAf7DqkH/zbKUo+IV6/O27C0GcHj+jG32/w+x7l/9q/Peekwdq3+djancknxP3r0X3wmfab/tVbGqg32ne8qLxW059+m1nxAGAQYz4BGyyYmS1XqItTBhDOYui+LS2Lyo+tCRpqyFpeXBFRG33yslLl8vNo3+1269PKes3IGdE6sWh9aY3OGDFIp5w4sNPQ2NbRZo+WbNyj+y49XVNPGaJThw7Uf+dPDLj4fFuhDoHweC01e7y6q6DYttoAAIIjfAI2yEoboCfys9XV+BnOYui+LS2bPd4eC1n5uZkB33tD6SHFuZyt4bPuc7d2HmjQgfovtKu6Maz3eXTFNhWW1ui5G4/t376hrFZbKuoibv/xLElzl5aoLMz2AQDCx2N3wCa+x7Zzlx7b+SeUYBjuYuiRbmnpC1kTRw6O6BF8sKEGj67Yqvl/3dbu2KW/9L8YfGdSByTo218ZKUnqnxCn+y49vWsNbiPQeFWPZenuZSVactvUiN8DABAY4ROw0YycDE0cOTjgWEwf3/FwF0O3Y0tLu0LWgpnZmv702/JEvLt7YPsOf67JP/tH688nJiXqT9+f0jqeNBQFt56tTysb5PFauuKsDH1a2aBrnu04FtfjtVRYWqMtFXXMggcQE6J9Q5JACJ+AzbLSBmjJbVNbZ6FvKKvVjgMNrbPQx6YnaXJWStiz0O3a0tKukOUbanBXQXGX4+f5Y9P0zg7/24xKxxahP3ikSZKUGOfUMzdM0qbdtVr4j+1hvc/MSSP1pw/K9Z3fvh/0PJfToaUb9xA+AUSl3rKEHOET6CYTMga1+0sg0m+o/ra09Pn2VzL0wGXjNeXRNWr2eFuPP3P9JB1patGcJR+1O9+ukNV2qIHL6e3k7C/f2zfU4PLsETr5vhUhXbfgO9kakBin7/5uvcLt/C2rbtTjf/uk0/M8XksbymrDuzkAdLOeXN2kOzDhCDAk0kcj/ra09HmjZL9cToemjU9vPTZkQIK+Ni5dSzfu7XC+nSFrRk6GVs2+QDmZgyXJ7yz4tsdzR6Vo1ewLNCMnI6TtSSVp1tfH6IKxJ+r7L2xUY7Mn7DZuDmOSkm+RegCIBj29ukl3IHwCMcLflpY+TS1eLS/ep/xJXy5Qf8VZGdp3+HOt87OdpWRvyMpKG6AXbsqTJF2Vm6nxw5NbQ2W8y6Hxw5P13Skn6fUfntdh3/XOtie9eMIw/ejrY3XHi5u0u8b/1pmd+TyMwOr2WGzFCSAqRMPqJt2Bx+5ADAi2paVPwYbdWn7HuRqanKiq+iZ9Z9JIvVzUsdfTxxey7B6sft+lpys+Pr613Z3dP9j2pKcOHainrpyoxf/8TDuqjujEgYmSpGaPV3Wfu21tt0+8yxGTA/gB9C7RsrpJd6DnE4gBoTye/nhfvbbtb9DMr4zUhIxknTo0KWj4NBGyQrl/sDVDs0cOVv+EOP3o38dqw/3TWv/87/WT7G5qq7HpXdvWEwDsZOfqJtGGnk8gRgTa0rKtlzbs1k3njdbQ5H56b2e19td9EfDcaAlZwdYMfblob9AAbTeX06HJWSnG3g8A/Im21U3sRs8nECMCbWnZ1vLifRo+qJ+uzsvUko17Ap4XbSHLju1JA7n6mQ/009e3hnSux2spPzez8xMBoBv5Vjexg291k2hC+ARiRLDH0z4NTS3625ZKHW3yaNXHVQHPi7aQFen2pJIUaXZ1OR3KG50aVb0DAPqmYKubhCsal5AjfALdoDtmS/seT3fW+zksuZ9eLa5ot95nW9EasmbkZGjh1TlKcDk7/Yw+LqdDCS6nHrjsdMU7I/vrzOVwaMHM7IjuAQB2CLa6SVdE2xJyjPkEbGBq14lgW1omnxCnqScP0dknD9EDr24JeI9oDlmRbE+alpTY5d2WHJKeyI/uRZkB9A2hrG4Sru5a3aSrCJ9ABEzvOhFsS8sVPzpfySfE6/G/faJd1Y1+r4+FkNV2e9IlG/ZoY3lo25O23W3JY1khrYnXdrcl3/UA0JN8q5vYGUCjbQk5wifQRcuLK1qDjhT6rhORBp1AIeu8X7wV8JpYCVnBepAnj0rRlZNPCtqDHEnPKQBEi1BWNwlHtKxu4kP4RK9h8pGCb9eJcL6XeryWPLJ0V0GxJEUcQHtTyAq1B/mPH+zutAe5bc/p0o17tKEstJ5TAIgWwTbfaOuGqaN00RnDdN3v1gc8J9pWN5EIn4hhpsZZHi/QrhMFt56trfvqO13Wx65dJ3pLyOquHuQJGYPafe5oGu8EAMHk52bqhXXlnZ6XOiBBo4b0D3pOtK1uIhE+EYNMj7M8np27Tiy5bWrE7YnlkGWyBzlWagIAwTbfaGvhP3Zo4T92BHzd5XRo0qiUqOuAYKklxJTlxRWa/vTbKio/tmZZqL1ky4srbHl/364ToUxmCabtrhN2i5WQZde+xWUBJlcBQCyzY/ONaF3dhPCJmOHrJWv2eEMOfx6vpWaPV3cVFNsSQMPZdeJrp6Wr5KHpmpEzwu/r0bjrhEm9ed9iAIhUpJtvRPPqJoRPxIRo6SULddeJb04coV9ek/Ov0LvP7znRuOuEKbHQgwwAPS2SzTcWXp0TtaubMOYTMSFaxlmGsuvE9WeP0tyLTtP3X9io9aU1Qc+Ntl0nTPH1IB8f5AtuPVvb9terqcWrqydnyu3x6s/rd3c6pmnpxj1RN6YJAOzQ21Y3kQifiAG+XjKfglvP1ieVDfJ6Lc2cNFLNLV7996pPtbx4n3464wxdcuZwVTc06aHXPtba7Qdbr2vbS9aVoBLKrhOXnDlMQwYk6juL31fJ3s5746Jt1wlTgvUgz5w0Us+9U6orFr2nr4xK0ZPfmaiNZbV6d2e13/P7cg8ygL6ht6xu4kP4RNTz10s28ysZ+t+3d2nGr9/VNyaO0M+umKCLzhimv39cqUVv7dTN552sp67K0TmPr9EX7i/3OI+klyyUXSc+3levCSMG6crczJDCZ7TtOmFKsB7kT/Y36H/WHOvpLDt0VDdMzdK5Y4YEDJ9S3+1BBtC3xPLqJm0x5hNRz18v2bb9Dfr1mztVduiofvPWTjW1eFVztFkFG/ao7NBR/XLNDqUOSNDpw5LbXRdpL9mY9IFBX9996KiuefYDXTh+qB7+5hmd3i/adp0wobMe5E8q69v9fLDhCw0ZmBj0nr4eZADoS2IxeEqET8QAf71kbQOK15Jqjzbr08ove78OHmmSJA0ZmNDh2kh6yfKyUjsd9F1a3ahrnvlAl0wYpge/MT7gedG464QJvh7kQFqOC6aWJXX292tf7UEGgFhE+ERUC9RLdnxAOXbM2+GY088aaZH0kuXnZoY0Q3tXdaOueXa9Lp84Qj+57HS/50TjrhOmdNaDHK6+2IMMALGKMZ+IaqGMswxXJL1kwXaduPqZD9r9/NnBI5r883/4vU+07jphSqj7Foeir/YgA0CsoucTUS/aesl6864TpoTagxyKvtyDDACxiJ5PRL1o6yXz7ToR7p7kPtG864QpgXqQj+89lqRb/19RwPv09R5kAIhFhE9EvfzcTL2wrrz1Z38B5bxfvNXhWNa9b3Q4ZlcvmW/XiB8v+UheBd5jvi2X0yGXw6En8rOjdtcJkxbMzNb0p9+Wp8v7VtGDDACxiPCJqBdsnGU47Ogl8y3wW1hWo50HjsjttVr33XVIfmNULO06YRI9yObE6lqAAHonwidiQk/3kpVVNwbc2uz4FvlCaCzuOmGarwd47tJj26fSg2yPDl+S/rULypj0gcrLSuX3EUCPInwiJvRkL9ny4orWcCQFfsTuO+okHIWlN+5b3FOCfUlyeyxt29+g7VVH9MK6cuWNTtUC6gigBxA+ETN6opdseXFF2IHX47XkkaW7CorbtRuB9bZ9i3tCqF+SfMeLyms1/em3+ZIEwDjCJ2KKyV6y0upGzV1aEjB4Ftx6trbuq9dPX9/q93VLx4LyxJGD6V0KUW/Zt9g0f1+SOvv95EsSgJ5C+ETMMdVLds+yL3uRuspjWbp7WYmW3DY1ovv0VQTPznX2JakzfEkCYBrhEzGrO3vJNu+tU2FpTcT38XgtFZbWaEtFHY+L0S34kgQg1rDDEXoNO3vJXi7aozib7udyOrR04x5b7gW05fuSFGh8p8vp0MPfPEMlD03Xpgcu1JwLT/V7XtsvSQDQ3QifgB+FZTVqsXH7xw1ltbbcC2irsy9JMyeNlMdr6Ypfv6eH//qxvn/+aF092f8mC3xJAmAKj90BP3YeOGLr/XYcaLD1foDU+Zek/Yc/b51wtKu6UeOGJenm80arYEPHkMmXJACm0PMJHMfrteT22NPr6eP2WPLa1JMK+HT2JenDPYfb/bxp92FlpQ1QoM5SviQBMIHwCRzH6XQo3mXvLOt4l4OZ27AVX5IAxCrCJ+DHmPSBtt5vbHqSrfcDQvmSlJM5uN3PZ2UOVll1owLlS74kATCB8An4kZeVKpeNs90nZ6XYci+grc6+JI0YfILuv+x0nZw2QN+cOEI3npOl379XFvB8viQBMIHwCfiRn5sZ0vadofB4LeXn+p9hDESisy9Jf9m0V/3iXXp11rn66Ywz9Pv3yvRi4W6/5/IlCYApzHYH/JiQMUh5o1NVVF4bMIRe/cwHnd7H5XRo0qgUFphHt8jPzdQL68r9vtb29/P+V7d0ei++JAEwhZ5PIIAFM7PlckT26N3lcGjBzGybWgS05/uSFOkQEZfTobzRqXxJAmAE4RMIICttgJ7Iz1ZX/7fukPREfjb7ZaNb8SUJQKwhfAJBzMjJ0MKrc5Tgcobcu+RyOpTgcmrh1TmakZPRzS1EX8eXJACxpkvhc9GiRcrKylK/fv00ZcoUFRYWBj1/6dKlGjdunPr166czzzxTK1as6FJjgZ4wIydDq2ZfoEmjjk3GCBRCfcdzR6Vo1ewLCJ4whi9JAGJJ2OHzpZde0pw5czR//nxt2rRJEydO1EUXXaQDBw74Pf/999/XNddco5tvvlkffvihrrjiCl1xxRXasqXzAfBAtMhKG6Alt03V6z88T9+dcpLGD09uXWMx3uXQ+OHJ+u6Uk/T6D8/TS7dNpRcJxvElCUCsCHu2+1NPPaVbbrlFN910kyRp8eLFeuONN/T888/r3nvv7XD+//zP/+jiiy/W3LlzJUmPPPKIVq9erV//+tdavHhxhM0HzJqQMajdpAyv12JRbkQN35ekLRV1WrpxjzaU1WrHgQa5PZbiXQ6NTU/S5KwU5edmMrkIQI8JK3w2NzerqKhI8+bNaz3mdDo1bdo0rVu3zu8169at05w5c9odu+iii/Tqq68GfJ+mpiY1NTW1/lxfXy9Jcrvdcrvd4TS5V/PVgppEJtI6ejx2tiZ28fsYObtqeFp6f91/6WmtP/v7ktSb/z3xu2gP6hi5vlbDUD9nWOGzurpaHo9HQ4cObXd86NCh+uSTT/xeU1lZ6ff8ysrKgO/z2GOP6eGHH+5wfNWqVerfv384Te4TVq9e3dNN6BWooz2oY+SooT2ooz2oY+T6Sg2PHj0a0nlRucj8vHnz2vWW1tfXKzMzU9OnT1dycnIPtiy6uN1urV69WhdeeKHi4+N7ujkxizragzpGjhragzragzpGrq/V0PekujNhhc+0tDS5XC5VVVW1O15VVaVhw4b5vWbYsGFhnS9JiYmJSkxM7HA8Pj6+T/zLCxd1sQd1tAd1jBw1tAd1tAd1jFxfqWGonzGs2e4JCQmaNGmS1qxZ03rM6/VqzZo1mjp1qt9rpk6d2u586Vj3c6DzAQAA0HuF/dh9zpw5uvHGG5Wbm6u8vDwtXLhQjY2NrbPfb7jhBmVkZOixxx6TJN1555366le/qv/+7//WZZddpoKCAm3cuFHPPPOMvZ8EAKIcqyMAQBfC51VXXaWDBw/qwQcfVGVlpXJycrRy5crWSUW7d++W0/llh+o555yjF198Uffff7/uu+8+jR07Vq+++qomTJhg36cAgCjkW/KosKxGOw8caV3yaEz6QOVlpfbaJY8I2QCC6dKEo1mzZmnWrFl+X1u7dm2HY/n5+crPz+/KWwFAzCmrbtTdy0pUWFojl9Mhj9dqfc3tsbRtf4O2Vx3RC+vKlTc6VQtmxvb2ln01ZAPomqic7Q4AsWp5cYXmLi2RxzoWONsGz7Z8x4vKazX96bf1RH62Lj0j3Vg77dDXQjYAe3Rpb3cAQEfLiyt0V0Gxmj3egKHzeB6vpWaPV3cVFGvF5v3d3EL7LC+u0PSn31ZRea2k0EP28uIKY20EEJ0InwBgg9LqRs1dWqLQImdHlqT7X9liZ5O6TaQhmwAK9G2ETwCwwT3LvnzU3lWeLkdXc0IJ2Y9+60wVP3ihyh6/TOOHt98YxJI0d2mJyqobu7WdAKIX4RMAIrR5b50KS2tC7gUMxHf9tv2h7RLSEzoL2f926on6zqSR+o8/bNTkn/1Dn1Y1dDjHY1m6e1lJdzYTQBQjfAJAhF4u2qM4P0sLXZOXqfX3/bscx7307A2TtOA72QHv98qH0flYOpSQfdKQ/jrQ8IU27a7VwSNNfs/1eC0VltZoS0VddzYXQJQifAJAhArLatTiJ2S9sXm/BveP19STh7QeG3RCvC449US9GiRgbio/3B3NjFigkO3zZH62fjpjgkam9FfZ45fp3Xu+FvBcl9OhpRv3dEczAUQ5lloCgAjtPHDE7/H6z1v0z08PakZOht7/7JAk6dIzh6m20a11uw4FvN9nBzs+qo4GgUK2z8OvbVX5oaO6Ju8kzfj1e0Efz3u8ljaU1XZHMwFEOXo+ASACXq8ltydwyHq1uEKXTBimBNexv26vyMnQX0v2KdjcJLfXkjfC8aPdIVDI9mloalFjU4u8lqWDR5pU09gc9PwdB6IzZAPoXoRPAIiA0+lQvCvwo+g12w5IDulr49I1fFA/Tc5KDfrIXZLinY6o256ys5DdFW5PdIZsAN2Lx+4AEKEx6QO1bb//XrymFq/+vqVSV5w1QllD+mtXdaM+3hd8NvspJyZ1RzMj4gvZdgbQeFf0hWwA3Y+eTwCIUF5WqlxBQtSrxRX6+mnpujI3U6+GsMD6V0YNtrF19hmTPtDW+41Nj76QDaD7ET4BIEL5uZlBlx96/7NDOvy5W6ekDwxpd59vnZVhZ/Ns01nIDofL6dDkrBRb7gUgtvDYHQAiNCFjkPJGp6qovNZvCLUsacqjazq9jy/YnX7crkDRIj83Uy+sK7flXh6vpfzcTFvuBSC20PMJADZYMDNbruNXkw+TS9E9/tEXsoP1fj7/XpnO+8VbQe/jcjqUNzpVEzIG2d1EADGA8AkANshKG6An8rO7HB8dkn72rQl2Nqlb2BKyHQ4tmBl4hycAvRvhEwBsMiMnQwuvzlGCyxny2EiX06EEl1MLr87RpWcO7+YWRs6OkP1Efray0gbY2SwAMYTwCQA2mpGToVWzL9CkUccm0wQKob7juaNStGr2BZqRE52TjPyJNGTH0mcFYD8mHAGAzbLSBmjJbVO1paJOSzfu0YayWu040CC3x1K8y6Gx6UmanJWi/NzMmB33OCMnQxNHDtbdy0pUWFojl9Phd7KV73juqBT9YiY9ngAInwDQbSZkDGoXLr1eq1ctqt4XQjYA+xE+AcCQ3hQ82+rtIRuAvRjzCQCwFcETQDCETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4DMDrtXq6CQAAAL1OXE83IFpsqajT0o17VFhWo50HjsjtsRTvcmhM+kDlZaUqPzdTEzIG9XQzAQAAYlqfD59l1Y26e1mJCktr5HI65GnT4+n2WNq2v0Hbq47ohXXlyhudqgUzs5WVNqAHWwwAABC7+vRj9+XFFZr+9NsqKq+VpHbBsy3f8aLyWk1/+m0tL64w1kYAAIDepM/2fC4vrtBdBcUKZ2Snx2vJI0t3FRRLkmbkZHRL2wAAAHqrPtnzWVrdqLlLS4IGzyfzs/XM9ZP8vmZJmru0RGXVjd3SPgAAgN6qT4bPe5aVyGMF7/N8+LWt+vHSjwK+7rEs3b2sxO6mAQAA9Gp9Lnxu3lunwtKagOM7fRqaWlT/RUvA1z1eS4WlNdpSUWd3EwEAAHqtPhc+Xy7aozino9Pzgj1293E5HVq6cY9dTQMAAOj1+lz4LCyrUYtNC8h7vJY2lNXaci8AAIC+oM+Fz50Hjth6vx0HGmy9HwAAQG/Wp8Kn12vJ7bF320y3x2IrTgAAgBD1qfDpdDoU7+p8vGc44l0OOUMYQwoAAIA+Fj4laUz6QFvvNzY9ydb7AQAA9GZ9LnzmZaXKZVNPpcvp0OSsFFvuBQAA0Bf0ufCZn5vZ6RqfofJ4LeXnZtpyLwAAgL6gz4XPCRmDlDe6897PBJdTjc2egK+7nA7ljU7VhIxBdjcRAACg1+pz4VOSFszMlsvhP3y6nA6NSR+or4xK0Y6qwMsouRwOLZiZ3V1NBAAA6JX6ZPjMShugJ/Kz5S9+njY0SX+ddZ62Vx3Rn9aX+73eIemJ/GxlpQ3o1nYCAAD0NnE93YCeMiMnQ5I0d2mJPJbVOg506/56nf7gSr/XuJwOuRwOPZGf3Xo9AAAAQtcnez59ZuRkaNXsCzRp1LEZ64HGgfqO545K0arZFxA8AQAAuqjP9nz6ZKUN0JLbpmpLRZ2WbtyjDWW12nGgQW6PpXiXQ2PTkzQ5K0X5uZlMLgIAAIhQnw+fPhMyBrULl16vxc5FAAAANuvTj92DIXgCAADYj/AJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMCaupxsQCsuyJEn19fU93JLo4na7dfToUdXX1ys+Pr6nmxOzqKM9qGPkqKE9qKM9qGPk+loNfTnNl9sCiYnw2dDQIEnKzMzs4ZYAAAAgmIaGBg0aNCjg6w6rs3gaBbxer/bt26ekpCQ5HI6ebk7UqK+vV2Zmpvbs2aPk5OSebk7Moo72oI6Ro4b2oI72oI6R62s1tCxLDQ0NGjFihJzOwCM7Y6Ln0+l0auTIkT3djKiVnJzcJ36puxt1tAd1jBw1tAd1tAd1jFxfqmGwHk8fJhwBAADAGMInAAAAjCF8xrDExETNnz9fiYmJPd2UmEYd7UEdI0cN7UEd7UEdI0cN/YuJCUcAAADoHej5BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYTPGFNTU6PrrrtOycnJGjx4sG6++WYdOXIkpGsty9Ill1wih8OhV199tXsbGuXCrWNNTY1++MMf6rTTTtMJJ5ygk046ST/60Y9UV1dnsNU9b9GiRcrKylK/fv00ZcoUFRYWBj1/6dKlGjdunPr166czzzxTK1asMNTS6BVODZ999lmdf/75SklJUUpKiqZNm9ZpzfuKcH8XfQoKCuRwOHTFFVd0bwNjRLh1PHz4sO644w4NHz5ciYmJOvXUU/v8f9fh1nDhwoWt/y/JzMzU7Nmz9cUXXxhqbZSwEFMuvvhia+LEidYHH3xgvfPOO9aYMWOsa665JqRrn3rqKeuSSy6xJFmvvPJK9zY0yoVbx82bN1vf/va3rddee83auXOntWbNGmvs2LHWzJkzDba6ZxUUFFgJCQnW888/b3388cfWLbfcYg0ePNiqqqrye/57771nuVwua8GCBdbWrVut+++/34qPj7c2b95suOXRI9waXnvttdaiRYusDz/80Nq2bZv1ve99zxo0aJC1d+9ewy2PLuHW0ae0tNTKyMiwzj//fGvGjBlmGhvFwq1jU1OTlZuba1166aXWu+++a5WWllpr1661iouLDbc8eoRbwz//+c9WYmKi9ec//9kqLS21/v73v1vDhw+3Zs+ebbjlPYvwGUO2bt1qSbI2bNjQeuxvf/ub5XA4rIqKiqDXfvjhh1ZGRoa1f//+Ph8+I6ljW0uWLLESEhIst9vdHc2MOnl5edYdd9zR+rPH47FGjBhhPfbYY37Pv/LKK63LLrus3bEpU6ZYt912W7e2M5qFW8PjtbS0WElJSdYLL7zQXU2MCV2pY0tLi3XOOedYv/vd76wbb7yR8GmFX8ff/va31sknn2w1NzebamLUC7eGd9xxh/X1r3+93bE5c+ZY5557bre2M9rw2D2GrFu3ToMHD1Zubm7rsWnTpsnpdGr9+vUBrzt69KiuvfZaLVq0SMOGDTPR1KjW1Toer66uTsnJyYqLi+uOZkaV5uZmFRUVadq0aa3HnE6npk2bpnXr1vm9Zt26de3Ol6SLLroo4Pm9XVdqeLyjR4/K7XYrNTW1u5oZ9bpax5/+9KdKT0/XzTffbKKZUa8rdXzttdc0depU3XHHHRo6dKgmTJigRx99VB6Px1Szo0pXanjOOeeoqKio9dH8rl27tGLFCl166aVG2hwtev//NXuRyspKpaentzsWFxen1NRUVVZWBrxu9uzZOuecczRjxozubmJM6God26qurtYjjzyiW2+9tTuaGHWqq6vl8Xg0dOjQdseHDh2qTz75xO81lZWVfs8Ptca9TVdqeLx77rlHI0aM6BDq+5Ku1PHdd9/Vc889p+LiYgMtjA1dqeOuXbv05ptv6rrrrtOKFSu0c+dO3X777XK73Zo/f76JZkeVrtTw2muvVXV1tc477zxZlqWWlhb953/+p+677z4TTY4a9HxGgXvvvVcOhyPon1D/53S81157TW+++aYWLlxob6OjUHfWsa36+npddtllGj9+vB566KHIGw6E4PHHH1dBQYFeeeUV9evXr6ebEzMaGhp0/fXX69lnn1VaWlpPNyemeb1epaen65lnntGkSZN01VVX6Sc/+YkWL17c002LGWvXrtWjjz6q3/zmN9q0aZP+8pe/6I033tAjjzzS000zip7PKPBf//Vf+t73vhf0nJNPPlnDhg3TgQMH2h1vaWlRTU1NwMfpb775pj777DMNHjy43fGZM2fq/PPP19q1ayNoeXTpzjr6NDQ06OKLL1ZSUpJeeeUVxcfHR9rsmJCWliaXy6Wqqqp2x6uqqgLWbNiwYWGd39t1pYY+Tz75pB5//HH94x//UHZ2dnc2M+qFW8fPPvtMZWVluvzyy1uPeb1eSceeeHz66ac65ZRTurfRUagrv4/Dhw9XfHy8XC5X67HTTz9dlZWVam5uVkJCQre2Odp0pYYPPPCArr/+en3/+9+XJJ155plqbGzUrbfeqp/85CdyOvtGn2Df+JRR7sQTT9S4ceOC/klISNDUqVN1+PBhFRUVtV775ptvyuv1asqUKX7vfe+996qkpETFxcWtfyTp6aef1u9//3sTH8+Y7qyjdKzHc/r06UpISNBrr73Wp3qfEhISNGnSJK1Zs6b1mNfr1Zo1azR16lS/10ydOrXd+ZK0evXqgOf3dl2poSQtWLBAjzzyiFauXNlunHJfFW4dx40bp82bN7f7O/Cb3/ymvva1r6m4uFiZmZkmmx81uvL7eO6552rnzp2t4V2Stm/fruHDh/e54Cl1rYZHjx7tEDB9Yd6yrO5rbLTp6RlPCM/FF19snXXWWdb69eutd9991xo7dmy7JYL27t1rnXbaadb69esD3kN9fLa7ZYVfx7q6OmvKlCnWmWeeae3cudPav39/65+Wlpae+hhGFRQUWImJidYf/vAHa+vWrdatt95qDR482KqsrLQsy7Kuv/5669577209/7333rPi4uKsJ5980tq2bZs1f/58lloKs4aPP/64lZCQYL388svtfucaGhp66iNEhXDreDxmux8Tbh13795tJSUlWbNmzbI+/fRT6/XXX7fS09Otn/3sZz31EXpcuDWcP3++lZSUZP3f//2ftWvXLmvVqlXWKaecYl155ZU99RF6BOEzxhw6dMi65pprrIEDB1rJycnWTTfd1O5/RKWlpZYk66233gp4D8Jn+HV86623LEl+/5SWlvbMh+gBv/rVr6yTTjrJSkhIsPLy8qwPPvig9bWvfvWr1o033tju/CVLllinnnqqlZCQYJ1xxhnWG2+8YbjF0SecGo4aNcrv79z8+fPNNzzKhPu72Bbh80vh1vH999+3pkyZYiUmJlonn3yy9fOf/7zPfAEPJJwaut1u66GHHrJOOeUUq1+/flZmZqZ1++23W7W1teYb3oMcltWX+nkBAADQkxjzCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAY/4/EbmnwRC1Ae4AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# visualize dimensions 0 and 1 of the embedding matrix C for all characters\n", + "plt.figure(figsize=(8,8))\n", + "plt.scatter(C[:,0].data, C[:,1].data, s=200)\n", + "for i in range(C.shape[0]):\n", + " plt.text(C[i,0].item(), C[i,1].item(), itos[i], ha=\"center\", va=\"center\", color='white')\n", + "plt.grid('minor')" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "metadata": {}, + "outputs": [], + "source": [ + "# training split, dev/validation split, test split\n", + "# 80%, 10%, 10%" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 3, 10])" + ] + }, + "execution_count": 79, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "context = [0] * block_size\n", + "C[torch.tensor([context])].shape" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "carlah.\n", + "ambril.\n", + "khkimyli.\n", + "thiy.\n", + "salaysge.\n", + "mahnen.\n", + "deliy.\n", + "chigeni.\n", + "nelania.\n", + "chaiir.\n", + "kaleig.\n", + "dham.\n", + "jore.\n", + "quinn.\n", + "srojlir.\n", + "jamii.\n", + "wazelog.\n", + "jaryxi.\n", + "jaxeeni.\n", + "sayley.\n" + ] + } + ], + "source": [ + "\n", + "\n", + "# sample from the model\n", + "g = torch.Generator().manual_seed(2147483647 + 10)\n", + "\n", + "for _ in range(20):\n", + " \n", + " out = []\n", + " context = [0] * block_size # initialize with all ...\n", + " while True:\n", + " emb = C[torch.tensor([context])] # (1,block_size,d)\n", + " h = torch.tanh(emb.view(1, -1) @ W1 + b1)\n", + " logits = h @ W2 + b2\n", + " probs = F.softmax(logits, dim=1)\n", + " ix = torch.multinomial(probs, num_samples=1, generator=g).item()\n", + " context = context[1:] + [ix]\n", + " out.append(ix)\n", + " if ix == 0:\n", + " break\n", + " \n", + " print(''.join(itos[i] for i in out))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/example/language_model/rnn.py b/example/language_model/rnn.py new file mode 100644 index 000000000..c853ba72f --- /dev/null +++ b/example/language_model/rnn.py @@ -0,0 +1,105 @@ +import torch +import torch.nn.functional as F +import matplotlib.pyplot as plt +from math import sqrt + +def generate_mapping(data): + chars = sorted(list(set(''.join(data)))) + stoi = {char: index + 1 for index, char in enumerate(chars)} + # marks beginning or end of a word + stoi['.'] = 0 + return stoi + +def generate_learning_rates(size): + lre = torch.linspace(-6, 0, size) + return 10 ** lre # we want the learning rates to be spaced exponentially + +def load_data(context_size): + data, label = [], [] + words = open('./names.txt', 'r').read().splitlines() + stoi = generate_mapping(words) + # itos = {v: k for k, v in stoi.items()} + + for w in words: + context = [0] * context_size + for ch in w + '.': + ix = stoi[ch] + data.append(context) + label.append(ix) + context = context[1:] + [ix] # crop and append + + data = torch.tensor(data) + label = torch.tensor(label) + return data, label + +def main(): + # How much tokens to keep as context when making the prediction for the next one + CONTEXT_SIZE = 3 + # Size of the vector to represent a single token + EMBEDDING_SIZE = 10 + VOCAB_SIZE = 27 # There are 27 possible chars in our dataset + + data, label = load_data(CONTEXT_SIZE) + # Creating an embedding from our data with each token being embedding represented + # by a vector of length "EMBEDDING_SIZE" + C = torch.rand((VOCAB_SIZE, EMBEDDING_SIZE)) + + NUMBER_OF_NEURONS = 200 + + # Creating hidden layer + # Using Kaiming init https://pytorch.org/docs/stable/nn.init.html + w1 = torch.rand((CONTEXT_SIZE * EMBEDDING_SIZE, NUMBER_OF_NEURONS)) * ((5/3) / (CONTEXT_SIZE*EMBEDDING_SIZE)) + print("First ", ((5/3) / (CONTEXT_SIZE*EMBEDDING_SIZE))) + b1 = torch.rand(NUMBER_OF_NEURONS) * 0.01 + + # Creating the output layer + w2 = torch.rand((NUMBER_OF_NEURONS, 27)) * ((5/3) / (NUMBER_OF_NEURONS)) + print("second ", ((5/3) * sqrt(NUMBER_OF_NEURONS))) + b2 = torch.rand(27) * 0.01 + + parameters = [C, w1, b1, w2, b2] + print("Number of parameters:", sum(p.nelement() for p in parameters)) + + for p in parameters: + p.requires_grad = True + + used_lrs = [] + losses = [] + + EPOCHS = 200000 + MINIBATCH_SIZE = 32 + avgs = [] + for i in range(EPOCHS): + # Minibatching + minibatch_indexes = torch.randint(0, data.shape[0], (MINIBATCH_SIZE,)) + embedding = C[data[minibatch_indexes]] + + # Forward pass + h = torch.tanh(embedding.view(-1, EMBEDDING_SIZE * CONTEXT_SIZE) @ w1 + b1) + logits = h @ w2 + b2 + + loss = F.cross_entropy(logits, label[minibatch_indexes]) + for p in parameters: + p.grad = None + loss.backward() + + # track stats + if i % 1000 == 0: # print every once in a while + print(f'{i:7d}/{EPOCHS:7d}: {loss.item():.4f}') + if i > EPOCHS / 2: + avgs.append(loss.item()) + + used_lrs.append(i) + losses.append(loss.item()) + + lr = 0.1 if i < EPOCHS / 2 else 0.01 + for p in parameters: + p.data -= lr * p.grad + + print("Average loss", sum(avgs) / len(avgs)) + plt.plot(used_lrs, losses) + plt.legend() + plt.show() + +if __name__ == "__main__": + main() diff --git a/gigatorch/activation_fn.py b/gigatorch/activation_fn.py index 1d700dfe2..80ab94931 100644 --- a/gigatorch/activation_fn.py +++ b/gigatorch/activation_fn.py @@ -1,2 +1,2 @@ -def relu(x): +def relu(x: int) -> int: return max(0, x) diff --git a/gigatorch/cnn.py b/gigatorch/cnn.py index 0e6656155..d29dfb9bf 100644 --- a/gigatorch/cnn.py +++ b/gigatorch/cnn.py @@ -10,47 +10,56 @@ from abc import ABC, abstractmethod from os import listdir from os.path import join +import numpy as np class Compute(ABC): @abstractmethod - def compute(self, data) -> List[List[Tensor]]: + def compute(self, input: Tensor) -> Tensor: pass +""" +The MaxPool2D layer extracts the maximum value over the window defined by pool_size +for each dimension along the features axis. The window is shifted by strides in each dimension. + +MaxPool2D accepts a 4-dimensional tensor as input. The dimensions represent: +Batch size: The number of samples in a batch. We can do parallel processing if it's more than 1 batch. +Channels: The number of input channels. For example, an RGB image would have 3 channels. +Height: The height of the input. +Width: The width of the input. +""" class MaxPool2D(Compute): def __init__(self, kernel_size, stride=None): self.kernel_size = kernel_size self.stride = stride if stride is not None else kernel_size - def compute(self, data_list) -> List[List[Tensor]]: + + def compute(self, input: Tensor) -> Tensor: + assert len(input.shape) == 4, f"can't 2d pool {input.shape}" + (batch_size, channels, height, width) = input.shape + assert (height - self.kernel_size) % self.stride == 0, f"Height does not fit the kernel size {self.kernel_size} and stride {self.stride}" + assert (width - self.kernel_size) % self.stride == 0, f"Width does not fit the kernel size {self.kernel_size} and stride {self.stride}" + print("Computing maxpool") - print("Size of data", len(data_list[0])) - print("Number of input", len(data_list)) - output = [] - for data in data_list: - if len(data) < self.kernel_size or len(data[0]) < self.kernel_size: - raise Exception("Received data is smaller than the kernel_size") - - new_data = [] - for row_index in range(0, len(data) - self.kernel_size + 1, self.stride): - row = [] - for column_index in range( - 0, len(data[row_index]) - self.kernel_size + 1, self.stride - ): - current_max = 0 - for i in range(self.kernel_size): - for j in range(self.kernel_size): - current_max = max( - current_max, data[row_index + i][column_index + j] - ) - row.append(current_max) - new_data.append(row) - output.append(new_data) - print("Size of data", len(output[0])) - print("Number of output", len(output)) + print("Input shape: ", input.shape) + + pooled_height = (height - self.kernel_size) // self.stride + 1 + pooled_width = (width - self.kernel_size) // self.stride + 1 + output = np.zeros((batch_size, channels, pooled_height, pooled_width)) + + for b in range(batch_size): + for c in range(channels): + for i in range(pooled_height): + for j in range(pooled_width): + h_start = i * self.stride + h_end = h_start + self.kernel_size + w_start = j * self.stride + w_end = w_start + self.kernel_size + output[b, c, i, j] = np.max(input.data[b, c, h_start:h_end, w_start:w_end]) + print("\n") - return output + return Tensor(output) class Conv2D(Compute): @@ -88,45 +97,25 @@ def __init__(self, in_channels, out_channels, kernel_size, activation_fn, stride self.activation_fn = activation_fn self.stride = stride - def compute(self, data_list): - print("computing conv2d") - print("Size of data", data_list.shape) - output = Tensor([]) - print("Number of kernels", self.kernels.shape) - # Iterate for out_channels number of times - for i in range(self.kernels.shape[0]): - print("output", i) - for layer_index in range(data_list.shape[0]): - print("layer index", layer_index) - data = data_list[layer_index] - kernel = self.kernels[layer_index] - print("data", data.shape) - print("kernel", kernel.shape) - - if data.shape[0] < self.kernel_size or data.shape[1] < self.kernel_size: - raise Exception("Received data is smaller than the kernel_size") - - new_data = [] - for row_index in range( - 0, len(data) - self.kernel_size + 1, self.stride - ): - row = [] - for column_index in range(len(data[0]) - self.kernel_size + 1): - sum = Tensor(0) - for i in range(self.kernel_size): - for j in range(self.kernel_size): - sum += ( - data[row_index + i][column_index + j] * kernel[i][j] - ) - row.append(self.activation_fn(sum)) - new_data.append(row) - output.append(new_data) - - print("Size of data", len(output[0])) - print("Number of output", len(output)) - print("\n") - return output + def compute(self, input): + (batch_size, _, height, width) = input.shape + output_height = (height - self.kernel_size) // self.stride + 1 + output_width = (width - self.kernel_size) // self.stride + 1 + output = Tensor(np.zeros((batch_size, self.out_channels, output_height, output_width))) + + for b in range(batch_size): + for k in range(self.out_channels): + for i in range(output_height): + for j in range(output_width): + h_start = i * self.stride + h_end = h_start + self.kernel_size + w_start = j * self.stride + w_end = w_start + self.kernel_size + output[b, k, i, j] = self.activation_fn( + np.sum(input[b, :, h_start:h_end, w_start:w_end] * self.kernels[k]) + ) + return output class CNN: def __init__(self, train_data_dir, test_data_dir, categories): diff --git a/gigatorch/nn.py b/gigatorch/nn.py index 3d0153692..e83865a27 100644 --- a/gigatorch/nn.py +++ b/gigatorch/nn.py @@ -1,5 +1,4 @@ import random -from typing import List from gigatorch.tensor import Tensor diff --git a/gigatorch/tensor.py b/gigatorch/tensor.py index dc3ae6046..df82496e5 100644 --- a/gigatorch/tensor.py +++ b/gigatorch/tensor.py @@ -108,7 +108,6 @@ def _backprop(): self.grad += (out.data > 0) * out.grad out._backprop = _backprop - return out def to(self, new_type): diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..5293a5e0e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,21 @@ +black==24.2.0 +click==8.1.7 +filelock==3.13.1 +fsspec==2024.2.0 +iniconfig==2.0.0 +Jinja2==3.1.3 +MarkupSafe==2.1.5 +mpmath==1.3.0 +mypy-extensions==1.0.0 +networkx==3.2.1 +numpy==1.26.4 +packaging==23.2 +pathspec==0.12.1 +pillow==10.2.0 +platformdirs==4.2.0 +pluggy==1.4.0 +pytest==8.0.2 +setuptools-black==0.1.5 +sympy==1.12 +torch==2.2.1 +typing_extensions==4.10.0 diff --git a/temp/0/1.png b/temp/0/1.png new file mode 100644 index 000000000..bf99edfe4 Binary files /dev/null and b/temp/0/1.png differ diff --git a/temp/0/108.png b/temp/0/108.png new file mode 100644 index 000000000..f172f2511 Binary files /dev/null and b/temp/0/108.png differ diff --git a/temp/0/114.png b/temp/0/114.png new file mode 100644 index 000000000..8ede44950 Binary files /dev/null and b/temp/0/114.png differ diff --git a/temp/0/118.png b/temp/0/118.png new file mode 100644 index 000000000..a3ebfe235 Binary files /dev/null and b/temp/0/118.png differ diff --git a/temp/0/21.png b/temp/0/21.png new file mode 100644 index 000000000..ee5c090c4 Binary files /dev/null and b/temp/0/21.png differ diff --git a/temp/0/34.png b/temp/0/34.png new file mode 100644 index 000000000..55227c6c4 Binary files /dev/null and b/temp/0/34.png differ diff --git a/temp/0/37.png b/temp/0/37.png new file mode 100644 index 000000000..5f5ecdd16 Binary files /dev/null and b/temp/0/37.png differ diff --git a/temp/0/51.png b/temp/0/51.png new file mode 100644 index 000000000..e83a6b75b Binary files /dev/null and b/temp/0/51.png differ diff --git a/temp/0/56.png b/temp/0/56.png new file mode 100644 index 000000000..604317272 Binary files /dev/null and b/temp/0/56.png differ diff --git a/temp/0/63.png b/temp/0/63.png new file mode 100644 index 000000000..f4969d56b Binary files /dev/null and b/temp/0/63.png differ diff --git a/temp/0/68.png b/temp/0/68.png new file mode 100644 index 000000000..2f2a56b48 Binary files /dev/null and b/temp/0/68.png differ diff --git a/temp/0/69.png b/temp/0/69.png new file mode 100644 index 000000000..de8900aec Binary files /dev/null and b/temp/0/69.png differ diff --git a/temp/0/75.png b/temp/0/75.png new file mode 100644 index 000000000..a9a416a20 Binary files /dev/null and b/temp/0/75.png differ diff --git a/temp/0/81.png b/temp/0/81.png new file mode 100644 index 000000000..fe782a044 Binary files /dev/null and b/temp/0/81.png differ diff --git a/temp/0/88.png b/temp/0/88.png new file mode 100644 index 000000000..84e641b1a Binary files /dev/null and b/temp/0/88.png differ diff --git a/temp/0/95.png b/temp/0/95.png new file mode 100644 index 000000000..7d5c86fc1 Binary files /dev/null and b/temp/0/95.png differ diff --git a/tests/cnn_test.py b/tests/cnn_test.py index b4ea74ab7..cdb75a77e 100644 --- a/tests/cnn_test.py +++ b/tests/cnn_test.py @@ -15,8 +15,8 @@ def test_conv2d_success(): ] ) sample_data = Tensor( - [ - [ # input_channel 1 + [ # Batch 1 + [ # Channel 1 [1, 1, 1], [1, 1, 1], [1, 1, 1], @@ -25,14 +25,9 @@ def test_conv2d_success(): ) output = conv2d.compute(sample_data) - print(output) - # Output size is (m-n+1) - assert len(output[0][0]) == len(sample_data[0]) - len(conv2d.kernels[0][0]) + 1 - assert len(output[0][0]) == len(sample_data[0][0]) - len(conv2d.kernels[0][0]) + 1 - - expected = [[[10, 10], [10, 10]]] + expected = Tensor([[[10, 10], [10, 10]]]) # for layer 1 - assert output == expected + assert all(output.item() == expected) def test_conv2d_kernel_size_larger_than_input(): @@ -58,42 +53,44 @@ def test_conv2d_kernel_size_larger_than_input(): def test_maxpool2d_success(): - maxpool2d = MaxPool2D(2, 1) - sample_data = Tensor( - [ + maxpool2d = MaxPool2D(kernel_size=2, stride=1) + sample_data = Tensor([ + [ # Batch 1 [ # channel 1 [1, 2, 3, 4], [5, 6, 7, -1], [0, 2, 100, 9], [0, 0, 0, 0], ] - ] + ]] ) - expected = Tensor( - [ - [ + expected = Tensor([ + [ # Batch 1 + [ # Channel 1 [6, 7, 7], [6, 100, 100], [2, 100, 100], ] ] - ) + ]) output = maxpool2d.compute(sample_data) - print(output) - print(expected) assert (expected == output).all() - maxpool2d_with_default_stride = MaxPool2D(2) + maxpool2d_with_default_stride = MaxPool2D(kernel_size=2) - expected = [ - [6, 7], - [2, 100], - ] + expected = Tensor([ + [ # Batch 1 + [ # Channel 1 + [6, 7], + [2, 100] + ] + ] + ]) output = maxpool2d_with_default_stride.compute(sample_data) - assert expected == output + assert (expected == output).all() def test_maxpool2d_kernel_size_larger_than_input(): diff --git a/tests/nn_test.py b/tests/nn_test.py index 4c0995b24..df32e4210 100644 --- a/tests/nn_test.py +++ b/tests/nn_test.py @@ -1,4 +1,4 @@ -from gigatorch.loss import squared_loss +from gigatorch.loss import squared_loss, softmax from gigatorch.nn import Neuron, Layer, MLP from gigatorch.tensor import Tensor from torch import allclose, nn @@ -56,7 +56,7 @@ def test_mlp_forward_pass(): ] neuron_biases = [[-3, 2], [1, -2, -1]] - mlp = MLP(number_of_inputs, neurons_per_layer, squared_loss) + mlp = MLP(number_of_inputs, neurons_per_layer, squared_loss, softmax) # Setting the weights and biases in my NN for i in range(len(neurons_per_layer)): @@ -101,7 +101,7 @@ def test_mlp_with_loss_function(): ] neuron_biases = [[-3, 2], [1, -2, -1]] - mlp = MLP(number_of_inputs, neurons_per_layer, squared_loss) + mlp = MLP(number_of_inputs, neurons_per_layer, squared_loss, softmax) # Setting the weights and biases in my NN for i in range(len(neurons_per_layer)): @@ -129,7 +129,7 @@ def test_mlp_backward_pass(): ] neuron_biases = [[-3, 2], [1, -2, -1], [1]] - mlp = MLP(number_of_inputs, neurons_per_layer, squared_loss) + mlp = MLP(number_of_inputs, neurons_per_layer, squared_loss, softmax) # Setting the weights and biases in my NN for i in range(len(neurons_per_layer)):