examples/tutorials/transformers_advanced_training_guide.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transformer Models Advanced Training Guide\n",
    "This guide is showing advanced features of RecTools transformer models training.\n",
    "\n",
    "### Table of Contents\n",
    "\n",
    "* Prepare data\n",
    "* Advanced training guide\n",
    "    * Validation fold\n",
    "    * Validation loss\n",
    "    * Callback for Early Stopping\n",
    "    * Callbacks for Checkpoints\n",
    "        * Loading Checkpoints\n",
    "    * Callbacks for RecSys metrics\n",
    "        * RecSys metrics for Early Stopping anf Checkpoints\n",
    "* Advanced training full example\n",
    "    * Running full training with all of the described validation features on Kion dataset\n",
    "* More RecTools features for transformers\n",
    "    * Saving and loading models\n",
    "    * Configs for transformer models\n",
    "        * Classes and function in configs\n",
    "    * Multi-gpu training\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import itertools\n",
    "import typing as tp\n",
    "import warnings\n",
    "from collections import Counter\n",
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "from lightning_fabric import seed_everything\n",
    "from pytorch_lightning import Trainer, LightningModule\n",
    "from pytorch_lightning.loggers import CSVLogger\n",
    "from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, Callback\n",
    "\n",
    "from rectools import Columns, ExternalIds\n",
    "from rectools.dataset import Dataset\n",
    "from rectools.metrics import NDCG, Recall, Serendipity, calc_metrics\n",
    "from rectools.models import BERT4RecModel, SASRecModel, load_model\n",
    "from rectools.models.nn.item_net import IdEmbeddingsItemNet\n",
    "from rectools.models.nn.transformers.base import TransformerModelBase\n",
    "\n",
    "# Enable deterministic behaviour with CUDA >= 10.2\n",
    "os.environ[\"CUBLAS_WORKSPACE_CONFIG\"] = \":4096:8\"\n",
    "warnings.simplefilter(\"ignore\", UserWarning)\n",
    "warnings.simplefilter(\"ignore\", FutureWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "!wget -q https://github.com/irsafilo/KION_DATASET/raw/f69775be31fa5779907cf0a92ddedb70037fb5ae/data_en.zip -O data_en.zip\n",
    "!unzip -o data_en.zip\n",
    "!rm data_en.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5476251, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>datetime</th>\n",
       "      <th>total_dur</th>\n",
       "      <th>watched_pct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>176549</td>\n",
       "      <td>9506</td>\n",
       "      <td>2021-05-11</td>\n",
       "      <td>4250</td>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>699317</td>\n",
       "      <td>1659</td>\n",
       "      <td>2021-05-29</td>\n",
       "      <td>8317</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id   datetime  total_dur  watched_pct\n",
       "0   176549     9506 2021-05-11       4250         72.0\n",
       "1   699317     1659 2021-05-29       8317        100.0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Download dataset\n",
    "DATA_PATH = Path(\"./data_en\")\n",
    "items = pd.read_csv(DATA_PATH / 'items_en.csv', index_col=0)\n",
    "interactions = (\n",
    "    pd.read_csv(DATA_PATH / 'interactions.csv', parse_dates=[\"last_watch_dt\"])\n",
    "    .rename(columns={\"last_watch_dt\": Columns.Datetime})\n",
    ")\n",
    "\n",
    "print(interactions.shape)\n",
    "interactions.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(962179, 15706)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "interactions[Columns.User].nunique(), interactions[Columns.Item].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5476251, 4)\n"
     ]
    }
   ],
   "source": [
    "# Process interactions\n",
    "interactions[Columns.Weight] = np.where(interactions['watched_pct'] > 10, 3, 1)\n",
    "raw_interactions = interactions[[\"user_id\", \"item_id\", \"datetime\", \"weight\"]]\n",
    "print(raw_interactions.shape)\n",
    "raw_interactions.head(2)\n",
    "\n",
    "dataset = Dataset.construct(raw_interactions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Seed set to 60\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "60"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RANDOM_STATE=60\n",
    "torch.use_deterministic_algorithms(True)\n",
    "seed_everything(RANDOM_STATE, workers=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Advanced Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Validation fold\n",
    "\n",
    "Models do not create validation fold during `fit` by default. However, there is a simple way to force it.\n",
    "\n",
    "Let's assume that we want to use Leave-One-Out validation for specific set of users. To apply it we need to implement `get_val_mask_func` with required logic and pass it to model during initialization. \n",
    "\n",
    "This function should receive interactions with standard RecTools columns and return a binary mask which identifies interactions that should not be used during model training. But instrad should be used for validation loss calculation. They will also be available for Lightning Callbacks to allow RecSys metrics computations.\n",
    "\n",
    "*Please make sure you do not use `partial` while doing this. Partial functions cannot be by serialized using RecTools.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Implement `get_val_mask_func`\n",
    "\n",
    "N_VAL_USERS = 2048\n",
    "unique_users = raw_interactions[Columns.User].unique()\n",
    "VAL_USERS = unique_users[: N_VAL_USERS]\n",
    "\n",
    "def leave_one_out_mask_for_users(interactions: pd.DataFrame, val_users: ExternalIds) -> np.ndarray:\n",
    "    rank = (\n",
    "        interactions\n",
    "        .sort_values(Columns.Datetime, ascending=False, kind=\"stable\")\n",
    "        .groupby(Columns.User, sort=False)\n",
    "        .cumcount()\n",
    "    )\n",
    "    val_mask = (\n",
    "        (interactions[Columns.User].isin(val_users))\n",
    "        & (rank == 0)\n",
    "    )\n",
    "    return val_mask.values\n",
    "\n",
    "# We do not use `partial` for correct serialization of the model\n",
    "def get_val_mask_func(interactions: pd.DataFrame):\n",
    "    return leave_one_out_mask_for_users(interactions, val_users = VAL_USERS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this guide we are going to use custom Lighhning trainers. We need to implement function that return desired Lightining trainer and pass it to model during initialization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to get custom trainer\n",
    "\n",
    "def get_debug_trainer() -> Trainer:\n",
    "    return Trainer(\n",
    "        accelerator=\"gpu\",\n",
    "        devices=1,\n",
    "        min_epochs=2,\n",
    "        max_epochs=2,\n",
    "        deterministic=True,\n",
    "        enable_model_summary=False,\n",
    "        enable_progress_bar=False,\n",
    "        enable_checkpointing=False,\n",
    "        limit_train_batches=2,  # limit train batches for quick debug runs\n",
    "        logger = CSVLogger(\"test_logs\"),  # We use CSV logging for this guide but there are many other options\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "HPU available: False, using: 0 HPUs\n"
     ]
    }
   ],
   "source": [
    "model = SASRecModel(\n",
    "    n_factors=64,\n",
    "    n_blocks=2,\n",
    "    n_heads=2,\n",
    "    dropout_rate=0.2,\n",
    "    train_min_user_interactions=5,\n",
    "    session_max_len=50,\n",
    "    verbose=0,\n",
    "    deterministic=True,\n",
    "    item_net_block_types=(IdEmbeddingsItemNet,),\n",
    "    get_val_mask_func=get_val_mask_func,  # pass our custom `get_val_mask_func`\n",
    "    get_trainer_func=get_debug_trainer,  # pass our custom trainer func\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Validation loss\n",
    "\n",
    "Let's check how the validation loss is being logged."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "`Trainer.fit` stopped: `max_epochs=2` reached.\n"
     ]
    }
   ],
   "source": [
    "# Fit model. Validation fold and validation loss computation will be done under the hood.\n",
    "model.fit(dataset);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's look at model logs. We can access logs directory with `model.fit_trainer.log_dir`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hparams.yaml  metrics.csv\r\n"
     ]
    }
   ],
   "source": [
    "# What's inside the logs directory?\n",
    "!ls $model.fit_trainer.log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch,step,train_loss,val_loss\r",
      "\r\n",
      "0,1,,22.365339279174805\r",
      "\r\n",
      "0,1,22.38391876220703,\r",
      "\r\n",
      "1,3,,22.189851760864258\r",
      "\r\n",
      "1,3,22.898216247558594,\r",
      "\r\n"
     ]
    }
   ],
   "source": [
    "# Losses and metrics are in the `metrics.csv`\n",
    "# Let's look at logs\n",
    "\n",
    "!tail $model.fit_trainer.log_dir/metrics.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Callback for Early Stopping\n",
    "\n",
    "By default RecTools transfomers train for exact amount of epochs (specified in `epochs` argument).\n",
    "When `get_trainer_func` is provided, number of model training epochs depends on Lightning trainer arguments instead.\n",
    "\n",
    "Now that we have validation loss logged, let's use it for model Early Stopping. It will ensure that model will not resume training if validation loss (or any other custom metric) doesn't impove. We have Lightning Callbacks for that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "HPU available: False, using: 0 HPUs\n"
     ]
    }
   ],
   "source": [
    "early_stopping_callback = EarlyStopping(\n",
    "    monitor=SASRecModel.val_loss_name,   # or just pass \"val_loss\" here\n",
    "    mode=\"min\",\n",
    "    min_delta=1.  # just for a quick test of functionality\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    accelerator='gpu',\n",
    "    devices=1,\n",
    "    min_epochs=1,  # minimum number of epochs to train before early stopping\n",
    "    max_epochs=20,  # maximum number of epochs to train\n",
    "    deterministic=True,\n",
    "    limit_train_batches=2,  # use only 2 batches for each epoch for a test run\n",
    "    enable_checkpointing=False,\n",
    "    logger = CSVLogger(\"test_logs\"),\n",
    "    callbacks=early_stopping_callback,  # pass our callback\n",
    "    enable_progress_bar=False,\n",
    "    enable_model_summary=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We want to pass our new trainer to model.  \n",
    "We just want to quickly check functionality for now and we already have model initialized. So let's just assign new trainer to model `_trainer` attribute."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    }
   ],
   "source": [
    "# Replace trainer with our custom one\n",
    "model._trainer = trainer\n",
    "\n",
    "# Fit model. Everything will happen under the hood\n",
    "model.fit(dataset);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here model stopped training after 4 epochs because validation loss wasn't improving by our specified `min_delta`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch,step,train_loss,val_loss\r",
      "\r\n",
      "0,1,,22.343637466430664\r",
      "\r\n",
      "0,1,22.36273765563965,\r",
      "\r\n",
      "1,3,,22.159835815429688\r",
      "\r\n",
      "1,3,22.33755874633789,\r",
      "\r\n",
      "2,5,,21.94308853149414\r",
      "\r\n",
      "2,5,22.244243621826172,\r",
      "\r\n",
      "3,7,,21.702259063720703\r",
      "\r\n",
      "3,7,22.196012496948242,\r",
      "\r\n"
     ]
    }
   ],
   "source": [
    "# Let's check out logs\n",
    "!tail $model.fit_trainer.log_dir/metrics.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Callback for Checkpoints\n",
    "Checkpoints are model states that are saved periodically during training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checkpoint last epoch\n",
    "last_epoch_ckpt = ModelCheckpoint(filename=\"last_epoch\")\n",
    "\n",
    "# Checkpoints based on validation loss\n",
    "least_val_loss_ckpt = ModelCheckpoint(\n",
    "    monitor=SASRecModel.val_loss_name,   # or just pass \"val_loss\" here,\n",
    "    mode=\"min\",\n",
    "    filename=\"{epoch}-{val_loss:.2f}\",\n",
    "    save_top_k=2,  # Let's save top 2 checkpoints for validation loss\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "`Trainer.fit` stopped: `max_epochs=6` reached.\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "    accelerator=\"gpu\",\n",
    "    devices=1,\n",
    "    min_epochs=1,\n",
    "    max_epochs=6,\n",
    "    deterministic=True,\n",
    "    limit_train_batches=2,  # use only 2 batches for each epoch for a test run\n",
    "    logger = CSVLogger(\"test_logs\"),\n",
    "    callbacks=[last_epoch_ckpt, least_val_loss_ckpt],  # pass our callbacks for checkpoints\n",
    "    enable_progress_bar=False,\n",
    "    enable_model_summary=False,\n",
    ")\n",
    "\n",
    "# Replace trainer with our custom one\n",
    "model._trainer = trainer\n",
    "\n",
    "# Fit model. Everything will happen under the hood\n",
    "model.fit(dataset);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's look at model checkpoints that were saved. By default they are neing saved to `checkpoints` directory in  `model.fit_trainer.log_dir`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch=4-val_loss=21.52.ckpt  epoch=5-val_loss=21.24.ckpt  last_epoch.ckpt\r\n"
     ]
    }
   ],
   "source": [
    "# We have 2 checkpoints for 2 best validation loss values and one for last epoch\n",
    "!ls $model.fit_trainer.log_dir/checkpoints"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading checkpoints"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading checkpoints is very simple with `load_from_weights_from_checkpoint` method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>score</th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>176549</td>\n",
       "      <td>15297</td>\n",
       "      <td>0.675964</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>176549</td>\n",
       "      <td>2657</td>\n",
       "      <td>0.661444</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>176549</td>\n",
       "      <td>10440</td>\n",
       "      <td>0.562942</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>176549</td>\n",
       "      <td>4495</td>\n",
       "      <td>0.557208</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>176549</td>\n",
       "      <td>6443</td>\n",
       "      <td>0.546108</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id     score  rank\n",
       "0   176549    15297  0.675964     1\n",
       "1   176549     2657  0.661444     2\n",
       "2   176549    10440  0.562942     3\n",
       "3   176549     4495  0.557208     4\n",
       "4   176549     6443  0.546108     5"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ckpt_path = os.path.join(model.fit_trainer.log_dir, \"checkpoints\", \"last_epoch.ckpt\")\n",
    "model.load_weights_from_checkpoint(ckpt_path)\n",
    "model.recommend(users=VAL_USERS[:1], dataset=dataset, filter_viewed=True, k=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can also load both model and its weights from checkpoint using `load_from_checkpoint` class method.\n",
    "Note that there is an  important limitation: **loaded model will not have `fit_trainer` and can't be saved again. But it is fully ready for recommendations.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "HPU available: False, using: 0 HPUs\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>score</th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>176549</td>\n",
       "      <td>15297</td>\n",
       "      <td>0.675964</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>176549</td>\n",
       "      <td>2657</td>\n",
       "      <td>0.661444</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>176549</td>\n",
       "      <td>10440</td>\n",
       "      <td>0.562942</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>176549</td>\n",
       "      <td>4495</td>\n",
       "      <td>0.557208</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>176549</td>\n",
       "      <td>6443</td>\n",
       "      <td>0.546108</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id     score  rank\n",
       "0   176549    15297  0.675964     1\n",
       "1   176549     2657  0.661444     2\n",
       "2   176549    10440  0.562942     3\n",
       "3   176549     4495  0.557208     4\n",
       "4   176549     6443  0.546108     5"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ckpt_path = os.path.join(model.fit_trainer.log_dir, \"checkpoints\", \"last_epoch.ckpt\")\n",
    "loaded = SASRecModel.load_from_checkpoint(ckpt_path)\n",
    "loaded.recommend(users=VAL_USERS[:1], dataset=dataset, filter_viewed=True, k=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Callbacks for RecSys metrics during training\n",
    "\n",
    "Monitoring RecSys metrics (or any other custom things) on validation fold is not available out of the box, but we can create a custom Lightning Callback for that.\n",
    "\n",
    "Below is an example of calculating standard RecTools metrics on validation fold during training. We use it as an explicit example that any customization is possible. But it is recommend to implement metrics calculation using `torch` for faster computations.\n",
    "\n",
    "Please look at PyTorch Lightning documentation for more details on custom callbacks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Implement custom Callback for RecTools metrics computation within validation epochs during training.\n",
    "\n",
    "class ValidationMetrics(Callback):\n",
    "    \n",
    "    def __init__(self, top_k: int, val_metrics: tp.Dict, verbose: int = 0) -> None:\n",
    "        self.top_k = top_k\n",
    "        self.val_metrics = val_metrics\n",
    "        self.verbose = verbose\n",
    "\n",
    "        self.epoch_n_users: int = 0\n",
    "        self.batch_metrics: tp.List[tp.Dict[str, float]] = []\n",
    "\n",
    "    def on_validation_batch_end(\n",
    "        self, \n",
    "        trainer: Trainer, \n",
    "        pl_module: LightningModule, \n",
    "        outputs: tp.Dict[str, torch.Tensor], \n",
    "        batch: tp.Dict[str, torch.Tensor], \n",
    "        batch_idx: int, \n",
    "        dataloader_idx: int = 0\n",
    "    ) -> None:\n",
    "        logits = outputs[\"logits\"]\n",
    "        if logits is None:\n",
    "            logits = pl_module.torch_model.encode_sessions(batch[\"x\"], pl_module.item_embs)[:, -1, :]\n",
    "        _, sorted_batch_recos = logits.topk(k=self.top_k)\n",
    "\n",
    "        batch_recos = sorted_batch_recos.tolist()\n",
    "        targets = batch[\"y\"].tolist()\n",
    "\n",
    "        batch_val_users = list(\n",
    "            itertools.chain.from_iterable(\n",
    "                itertools.repeat(idx, len(recos)) for idx, recos in enumerate(batch_recos)\n",
    "            )\n",
    "        )\n",
    "\n",
    "        batch_target_users = list(\n",
    "            itertools.chain.from_iterable(\n",
    "                itertools.repeat(idx, len(targets)) for idx, targets in enumerate(targets)\n",
    "            )\n",
    "        )\n",
    "\n",
    "        batch_recos_df = pd.DataFrame(\n",
    "            {\n",
    "                Columns.User: batch_val_users,\n",
    "                Columns.Item: list(itertools.chain.from_iterable(batch_recos)),\n",
    "            }\n",
    "        )\n",
    "        batch_recos_df[Columns.Rank] = batch_recos_df.groupby(Columns.User, sort=False).cumcount() + 1\n",
    "\n",
    "        interactions = pd.DataFrame(\n",
    "            {\n",
    "                Columns.User: batch_target_users,\n",
    "                Columns.Item: list(itertools.chain.from_iterable(targets)),\n",
    "            }\n",
    "        )\n",
    "\n",
    "        prev_interactions = pl_module.data_preparator.train_dataset.interactions.df\n",
    "        catalog = prev_interactions[Columns.Item].unique()\n",
    "\n",
    "        batch_metrics = calc_metrics(\n",
    "            self.val_metrics, \n",
    "            batch_recos_df,\n",
    "            interactions, \n",
    "            prev_interactions,\n",
    "            catalog\n",
    "        )\n",
    "\n",
    "        batch_n_users = batch[\"x\"].shape[0]\n",
    "        self.batch_metrics.append({metric: value * batch_n_users for metric, value in batch_metrics.items()})\n",
    "        self.epoch_n_users += batch_n_users\n",
    "\n",
    "    def on_validation_epoch_end(self, trainer: Trainer, pl_module: LightningModule) -> None:\n",
    "        epoch_metrics = dict(sum(map(Counter, self.batch_metrics), Counter()))\n",
    "        epoch_metrics = {metric: value / self.epoch_n_users for metric, value in epoch_metrics.items()}\n",
    "\n",
    "        self.log_dict(epoch_metrics, on_step=False, on_epoch=True, prog_bar=self.verbose > 0)\n",
    "\n",
    "        self.batch_metrics.clear()\n",
    "        self.epoch_n_users = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### RecSys metrics for Early Stopping and Checkpoints\n",
    "When custom metrics callback is implemented, we can use the values of these metrics for both Early Stopping and Checkpoints."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize callbacks for metrics calculation and checkpoint based on NDCG value\n",
    "\n",
    "metrics = {\n",
    "    \"NDCG@10\": NDCG(k=10),\n",
    "    \"Recall@10\": Recall(k=10),\n",
    "    \"Serendipity@10\": Serendipity(k=10),\n",
    "}\n",
    "top_k = max([metric.k for metric in metrics.values()])\n",
    "\n",
    "# Callback for calculating RecSys metrics\n",
    "val_metrics_callback = ValidationMetrics(top_k=top_k, val_metrics=metrics, verbose=0)\n",
    "\n",
    "# Callback for checkpoint based on maximization of NDCG@10\n",
    "best_ndcg_ckpt = ModelCheckpoint(\n",
    "    monitor=\"NDCG@10\",\n",
    "    mode=\"max\",\n",
    "    filename=\"{epoch}-{NDCG@10:.2f}\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "`Trainer.fit` stopped: `max_epochs=6` reached.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<rectools.models.nn.transformers.sasrec.SASRecModel at 0x7f27b1e9a430>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "    accelerator=\"gpu\",\n",
    "    devices=1,\n",
    "    min_epochs=1,\n",
    "    max_epochs=6,\n",
    "    deterministic=True,\n",
    "    limit_train_batches=2,  # use only 2 batches for each epoch for a test run\n",
    "    logger = CSVLogger(\"test_logs\"),\n",
    "    callbacks=[val_metrics_callback, best_ndcg_ckpt],  # pass our callbacks\n",
    "    enable_progress_bar=False,\n",
    "    enable_model_summary=False,\n",
    ")\n",
    "\n",
    "# Replace trainer with our custom one\n",
    "model._trainer = trainer\n",
    "\n",
    "# Fit model. Everything will happen under the hood\n",
    "model.fit(dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have checkpoint for best NDCG@10 model in the usual directory for checkpoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch=5-NDCG@10=0.01.ckpt\r\n"
     ]
    }
   ],
   "source": [
    "!ls $model.fit_trainer.log_dir/checkpoints"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We also now have metrics in our logs. Let's load them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>val_loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>22.383919</td>\n",
       "      <td>22.365339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>22.898216</td>\n",
       "      <td>22.189852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>22.218102</td>\n",
       "      <td>21.964468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>22.875019</td>\n",
       "      <td>21.701391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>21.739164</td>\n",
       "      <td>21.417864</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epoch  train_loss   val_loss\n",
       "0      0   22.383919  22.365339\n",
       "1      1   22.898216  22.189852\n",
       "2      2   22.218102  21.964468\n",
       "3      3   22.875019  21.701391\n",
       "4      4   21.739164  21.417864"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_logs(model: TransformerModelBase) -> tp.Tuple[pd.DataFrame, ...]:\n",
    "    log_path = Path(model.fit_trainer.log_dir) / \"metrics.csv\"\n",
    "    epoch_metrics_df = pd.read_csv(log_path)\n",
    "    \n",
    "    loss_df = epoch_metrics_df[[\"epoch\", \"train_loss\"]].dropna()\n",
    "    val_loss_df = epoch_metrics_df[[\"epoch\", \"val_loss\"]].dropna()\n",
    "    loss_df = pd.merge(loss_df, val_loss_df, how=\"inner\", on=\"epoch\")\n",
    "    loss_df.reset_index(drop=True, inplace=True)\n",
    "    \n",
    "    metrics_df = epoch_metrics_df.drop(columns=[\"train_loss\", \"val_loss\"]).dropna()\n",
    "    metrics_df.reset_index(drop=True, inplace=True)\n",
    "\n",
    "    return loss_df, metrics_df\n",
    "\n",
    "loss_df, metrics_df = get_logs(model)\n",
    "\n",
    "loss_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NDCG@10</th>\n",
       "      <th>Recall@10</th>\n",
       "      <th>Serendipity@10</th>\n",
       "      <th>epoch</th>\n",
       "      <th>step</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000052</td>\n",
       "      <td>0.000657</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.002204</td>\n",
       "      <td>0.024984</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.006865</td>\n",
       "      <td>0.071006</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.009856</td>\n",
       "      <td>0.097304</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.010442</td>\n",
       "      <td>0.107824</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    NDCG@10  Recall@10  Serendipity@10  epoch  step\n",
       "0  0.000052   0.000657        0.000004      0     1\n",
       "1  0.002204   0.024984        0.000006      1     3\n",
       "2  0.006865   0.071006        0.000004      2     5\n",
       "3  0.009856   0.097304        0.000003      3     7\n",
       "4  0.010442   0.107824        0.000002      4     9"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "del model\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Advanced training full example\n",
    "Running full training with all of the described validation features on Kion dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Seed set to 60\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    }
   ],
   "source": [
    "# seed again for reproducibility of this piece of code\n",
    "seed_everything(RANDOM_STATE, workers=True)\n",
    "\n",
    "# Callbacks\n",
    "val_metrics_callback = ValidationMetrics(top_k=top_k, val_metrics=metrics, verbose=0)\n",
    "best_ndcg_ckpt = ModelCheckpoint(\n",
    "    monitor=\"NDCG@10\",\n",
    "    mode=\"max\",\n",
    "    filename=\"{epoch}-{NDCG@10:.2f}\",\n",
    ")\n",
    "last_epoch_ckpt = ModelCheckpoint(filename=\"{epoch}-last_epoch\")\n",
    "early_stopping_callback = EarlyStopping(\n",
    "    monitor=\"NDCG@10\",\n",
    "    patience=5,\n",
    "    mode=\"max\",\n",
    ")\n",
    "\n",
    "# Function to get custom trainer with desired callbacks\n",
    "def get_custom_trainer() -> Trainer:\n",
    "    return Trainer(\n",
    "        accelerator=\"gpu\",\n",
    "        devices=[1],\n",
    "        min_epochs=1,\n",
    "        max_epochs=100,\n",
    "        deterministic=True,\n",
    "        logger = CSVLogger(\"sasrec_logs\"),\n",
    "        enable_progress_bar=False,\n",
    "        enable_model_summary=False,\n",
    "        callbacks=[\n",
    "            val_metrics_callback,  # calculate RecSys metrics\n",
    "            best_ndcg_ckpt,  # save best NDCG model checkpoint\n",
    "            last_epoch_ckpt,  # save model checkpoint after last epoch\n",
    "            early_stopping_callback,  # early stopping on NDCG\n",
    "        ],\n",
    "    )\n",
    "\n",
    "# Model\n",
    "model = SASRecModel(\n",
    "    n_factors=256,\n",
    "    n_blocks=2,\n",
    "    n_heads=4,\n",
    "    dropout_rate=0.2,\n",
    "    train_min_user_interactions=5,\n",
    "    session_max_len=50,\n",
    "    verbose=1,\n",
    "    deterministic=True,\n",
    "    item_net_block_types=(IdEmbeddingsItemNet,),\n",
    "    get_val_mask_func=get_val_mask_func,  # pass our custom `get_val_mask_func`\n",
    "    get_trainer_func=get_custom_trainer,  # pass function to initialize our custom trainer\n",
    ")\n",
    "\n",
    "\n",
    "# Fit model. Everything will happen under the hood\n",
    "model.fit(dataset);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Early stopping was triggered. We have checkpoints for best NDCG model (on epoch 14) and on last epoch (19)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch=14-NDCG@10=0.03.ckpt  epoch=19-last_epoch.ckpt\r\n"
     ]
    }
   ],
   "source": [
    "!ls $model.fit_trainer.log_dir/checkpoints"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading best NDCG model from checkpoint and recommending"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "HPU available: False, using: 0 HPUs\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9ef25b79cb441bd9be5bd65667495b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Predicting: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>score</th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>176549</td>\n",
       "      <td>11749</td>\n",
       "      <td>2.610277</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>176549</td>\n",
       "      <td>2025</td>\n",
       "      <td>2.577398</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>176549</td>\n",
       "      <td>9342</td>\n",
       "      <td>2.394489</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>176549</td>\n",
       "      <td>14488</td>\n",
       "      <td>2.366664</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>176549</td>\n",
       "      <td>7571</td>\n",
       "      <td>2.289778</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id     score  rank\n",
       "0   176549    11749  2.610277     1\n",
       "1   176549     2025  2.577398     2\n",
       "2   176549     9342  2.394489     3\n",
       "3   176549    14488  2.366664     4\n",
       "4   176549     7571  2.289778     5"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ckpt_path = os.path.join(model.fit_trainer.log_dir, \"checkpoints\", \"epoch=14-NDCG@10=0.03.ckpt\")\n",
    "best_model = SASRecModel.load_from_checkpoint(ckpt_path)\n",
    "best_model.recommend(users=VAL_USERS[:1], dataset=dataset, filter_viewed=True, k=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's also look at our logs for losses and metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NDCG@10</th>\n",
       "      <th>Recall@10</th>\n",
       "      <th>Serendipity@10</th>\n",
       "      <th>epoch</th>\n",
       "      <th>step</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.023663</td>\n",
       "      <td>0.183432</td>\n",
       "      <td>0.000067</td>\n",
       "      <td>0</td>\n",
       "      <td>2362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.027919</td>\n",
       "      <td>0.209730</td>\n",
       "      <td>0.000122</td>\n",
       "      <td>1</td>\n",
       "      <td>4725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.029360</td>\n",
       "      <td>0.216305</td>\n",
       "      <td>0.000166</td>\n",
       "      <td>2</td>\n",
       "      <td>7088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.030170</td>\n",
       "      <td>0.226824</td>\n",
       "      <td>0.000203</td>\n",
       "      <td>3</td>\n",
       "      <td>9451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.030412</td>\n",
       "      <td>0.225510</td>\n",
       "      <td>0.000161</td>\n",
       "      <td>4</td>\n",
       "      <td>11814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.031640</td>\n",
       "      <td>0.226167</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>15</td>\n",
       "      <td>37807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.031333</td>\n",
       "      <td>0.230769</td>\n",
       "      <td>0.000203</td>\n",
       "      <td>16</td>\n",
       "      <td>40170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.031238</td>\n",
       "      <td>0.228139</td>\n",
       "      <td>0.000184</td>\n",
       "      <td>17</td>\n",
       "      <td>42533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.031893</td>\n",
       "      <td>0.232084</td>\n",
       "      <td>0.000195</td>\n",
       "      <td>18</td>\n",
       "      <td>44896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.031560</td>\n",
       "      <td>0.230112</td>\n",
       "      <td>0.000179</td>\n",
       "      <td>19</td>\n",
       "      <td>47259</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     NDCG@10  Recall@10  Serendipity@10  epoch   step\n",
       "0   0.023663   0.183432        0.000067      0   2362\n",
       "1   0.027919   0.209730        0.000122      1   4725\n",
       "2   0.029360   0.216305        0.000166      2   7088\n",
       "3   0.030170   0.226824        0.000203      3   9451\n",
       "4   0.030412   0.225510        0.000161      4  11814\n",
       "15  0.031640   0.226167        0.000186     15  37807\n",
       "16  0.031333   0.230769        0.000203     16  40170\n",
       "17  0.031238   0.228139        0.000184     17  42533\n",
       "18  0.031893   0.232084        0.000195     18  44896\n",
       "19  0.031560   0.230112        0.000179     19  47259"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss_df, metrics_df = get_logs(model)\n",
    "pd.concat([metrics_df.head(5), metrics_df.tail(5)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Don't be surprised by the fact that validation loss is less then train loss in the plot below. \n",
    "- First, this is data-specific, you may not see this in other datasets. \n",
    "- Second, validation loss is calculated after the full training epoch while train loss is computed for each batch during training when model still hasn't seen other batches and hasn't updated weights.\n",
    "- Validation loss is calculated only in the last item in validation users history. While train loss for SASRec is calculated for each item in user histor except the first one and the validation one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAHHCAYAAABHp6kXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8ekN5oAAAACXBIWXMAAA9hAAAPYQGoP6dpAABjtUlEQVR4nO3dd3wUZf4H8M9sTe8d0mmREgJiRJAWJESkW0AOQVB/eGChqXgnCngXwXIWFL1TwYZdsKBoQAIIQSEQEURKCISSQgLpySbZnd8fk2yypC7Zzewmn/frNa/dnXlm9jsZYj4+88yMIIqiCCIiIiI7p5C7ACIiIiJLYKghIiKiDoGhhoiIiDoEhhoiIiLqEBhqiIiIqENgqCEiIqIOgaGGiIiIOgSGGiIiIuoQGGqIiIioQ2CoISIiog6BoYaIrGrDhg0QBAEHDhyQuxQi6uAYaoiIiKhDYKghIiKiDoGhhohkd+jQISQkJMDNzQ0uLi6Ii4vDvn37TNpUVVVhxYoV6N69OxwcHODt7Y2hQ4ciKSnJ2CY7Oxv33nsvunbtCq1Wi8DAQEycOBFnzpwx2dYPP/yAm2++Gc7OznB1dcW4ceNw9OhRkzat3RYR2Q6V3AUQUed29OhR3HzzzXBzc8Njjz0GtVqNt956CyNGjMDOnTsRGxsLAHjmmWeQmJiI++67DzfccAOKiopw4MABHDx4ELfccgsAYOrUqTh69CgeeughhIWFITc3F0lJScjMzERYWBgA4IMPPsCsWbMQHx+P1atXo6ysDOvWrcPQoUNx6NAhY7vWbIuIbIxIRGRF69evFwGI+/fvb3T5pEmTRI1GI6anpxvnXbx4UXR1dRWHDRtmnBcdHS2OGzeuye+5cuWKCEB8/vnnm2xTXFwsenh4iPfff7/J/OzsbNHd3d04vzXbIiLbw9NPRCQbvV6Pn376CZMmTUJERIRxfmBgIO6++2788ssvKCoqAgB4eHjg6NGjOHnyZKPbcnR0hEajQXJyMq5cudJom6SkJBQUFGD69OnIy8szTkqlErGxsdixY0ert0VEtoehhohkc+nSJZSVlaFnz54NlkVFRcFgMODcuXMAgJUrV6KgoAA9evRA3759sXTpUhw+fNjYXqvVYvXq1fjhhx/g7++PYcOGYc2aNcjOzja2qQ1Eo0aNgq+vr8n0008/ITc3t9XbIiLbw1BDRHZh2LBhSE9Px7vvvos+ffrg7bffxoABA/D2228b2zz66KM4ceIEEhMT4eDggKeeegpRUVE4dOgQAMBgMACQxtUkJSU1mL7++utWb4uIbJDc57+IqGNrbkxNdXW16OTkJN55550Nls2bN09UKBRiYWFho9stLi4WY2JixC5dujT53SdOnBCdnJzEGTNmiKIoip999pkIQPzxxx/N3o+rt0VEtoc9NUQkG6VSiTFjxuDrr782uVQ6JycHGzduxNChQ+Hm5gYAyM/PN1nXxcUF3bp1g06nAwCUlZWhoqLCpE1kZCRcXV2NbeLj4+Hm5oZ///vfqKqqalDPpUuXWr0tIrI9vKSbiNrFu+++i61btzaY/8wzzyApKQlDhw7F3//+d6hUKrz11lvQ6XRYs2aNsd11112HESNGYODAgfDy8sKBAwfwxRdfYMGCBQCAEydOIC4uDnfeeSeuu+46qFQqbNq0CTk5OZg2bRoAwM3NDevWrcPMmTMxYMAATJs2Db6+vsjMzMSWLVswZMgQrF27tlXbIiIbJHdXERF1bLWnn5qazp07Jx48eFCMj48XXVxcRCcnJ3HkyJHi3r17Tbbz7LPPijfccIPo4eEhOjo6ir169RL/9a9/iZWVlaIoimJeXp44f/58sVevXqKzs7Po7u4uxsbGip999lmDmnbs2CHGx8eL7u7uooODgxgZGSnOnj1bPHDggNnbIiLbIYiiKMqYqYiIiIgsgmNqiIiIqENgqCEiIqIOgaGGiIiIOgSGGiIiIuoQGGqIiIioQ2CoISIiog6h09x8z2Aw4OLFi3B1dYUgCHKXQ0RERK0giiKKi4sRFBQEhaL5vphOE2ouXryI4OBgucsgIiKia3Du3Dl07dq12TadJtS4uroCkH4otc+SISIiIttWVFSE4OBg49/x5nSaUFN7ysnNzY2hhoiIyM60ZugIBwoTERFRh2B2qNm1axfGjx+PoKAgCIKAzZs3N2hz7NgxTJgwAe7u7nB2dsagQYOQmZnZ5Db/97//4eabb4anpyc8PT0xevRo/PbbbyZtZs+eDUEQTKaxY8eaWz4RERF1UGaHmtLSUkRHR+P1119vdHl6ejqGDh2KXr16ITk5GYcPH8ZTTz0FBweHJreZnJyM6dOnY8eOHUhJSUFwcDDGjBmDCxcumLQbO3YssrKyjNPHH39sbvlERETUQbXpKd2CIGDTpk2YNGmScd60adOgVqvxwQcfXHNRer0enp6eWLt2Le655x4AUk9NQUFBoz1DrVFUVAR3d3cUFhZyTA0RUQei1+tRVVUldxnUBhqNpsnLtc35+23RgcIGgwFbtmzBY489hvj4eBw6dAjh4eFYtmyZSfBpSVlZGaqqquDl5WUyPzk5GX5+fvD09MSoUaPw7LPPwtvb25K7QEREdkIURWRnZ6OgoEDuUqiNFAoFwsPDodFo2rQdi4aa3NxclJSU4LnnnsOzzz6L1atXY+vWrZgyZQp27NiB4cOHt2o7jz/+OIKCgjB69GjjvLFjx2LKlCkIDw9Heno6nnzySSQkJCAlJQVKpbLBNnQ6HXQ6nfFzUVFR23eQiIhsRm2g8fPzg5OTE2+saqdqb46blZWFkJCQNh1Hi/fUAMDEiROxcOFCAED//v2xd+9evPnmm60KNc899xw++eQTJCcnm4zDmTZtmvF937590a9fP0RGRiI5ORlxcXENtpOYmIgVK1a0dZeIiMgG6fV6Y6Bhj7398/X1xcWLF1FdXQ21Wn3N27HoJd0+Pj5QqVS47rrrTOZHRUU1e/VTrRdeeAHPPfccfvrpJ/Tr16/ZthEREfDx8cGpU6caXb5s2TIUFhYap3PnzrV+R4iIyKbVjqFxcnKSuRKyhNrTTnq9vk3bsWhPjUajwaBBg3D8+HGT+SdOnEBoaGiz665Zswb/+te/8OOPP+L6669v8bvOnz+P/Px8BAYGNrpcq9VCq9W2vngiIrI7POXUMVjqOJodakpKSkx6RzIyMpCWlgYvLy+EhIRg6dKluOuuuzBs2DCMHDkSW7duxbfffovk5GTjOvfccw+6dOmCxMREAMDq1auxfPlybNy4EWFhYcjOzgYAuLi4wMXFBSUlJVixYgWmTp2KgIAApKen47HHHkO3bt0QHx/fxh8BERERdQRmn346cOAAYmJiEBMTAwBYtGgRYmJisHz5cgDA5MmT8eabb2LNmjXo27cv3n77bXz55ZcYOnSocRuZmZnIysoyfl63bh0qKytx++23IzAw0Di98MILAAClUonDhw9jwoQJ6NGjB+bOnYuBAwdi9+7d7I0hIqJOKywsDC+//LJFtpWcnAxBEOz6arI23afGnvA+NUREHUdFRQUyMjIQHh7e7M1dbdGIESPQv39/i4SRS5cuwdnZ2SJji5KTkzFy5EhcuXIFHh4ebd6eOZo7nub8/eaznyzgSmklTuQUy10GERF1AKIoorq6ulVtfX19OVi6HoaaNko9ewUxq5Jw7/r9cpdCREQ2bvbs2di5cydeeeUV43MMN2zYAEEQ8MMPP2DgwIHQarX45ZdfkJ6ejokTJ8Lf3x8uLi4YNGgQtm3bZrK9q08/CYKAt99+G5MnT4aTkxO6d++Ob7755prr/fLLL9G7d29otVqEhYXhxRdfNFn+xhtvoHv37nBwcIC/vz9uv/1247IvvvgCffv2haOjI7y9vTF69GiUlpZecy2tYdGrnzqjCB9nAMCFgnKU6qrhrOWPlIhIDqIooryqbZcEXytHtbJVV/C88sorOHHiBPr06YOVK1cCAI4ePQoAeOKJJ/DCCy8gIiICnp6eOHfuHG699Vb861//glarxfvvv4/x48fj+PHjCAkJafI7VqxYgTVr1uD555/Ha6+9hhkzZuDs2bMN7tLfktTUVNx555145plncNddd2Hv3r34+9//Dm9vb8yePRsHDhzAww8/jA8++AA33XQTLl++jN27dwMAsrKyMH36dKxZswaTJ09GcXExdu/eDWuPeOFf4DbydNbA21mD/NJKnL5Uir5d3eUuiYioUyqv0uO65T/K8t1/royHk6blP6nu7u7QaDRwcnJCQEAAAOCvv/4CAKxcuRK33HKLsa2Xlxeio6ONn1etWoVNmzbhm2++wYIFC5r8jtmzZ2P69OkAgH//+9949dVX8dtvv2Hs2LFm7dNLL72EuLg4PPXUUwCAHj164M8//8Tzzz+P2bNnIzMzE87Ozrjtttvg6uqK0NBQ40VEWVlZqK6uxpQpU4y3dOnbt69Z338tePrJAiJ9XQAA6ZdKZK6EiIjs1dX3aCspKcGSJUsQFRUFDw8PuLi44NixYy3ezLb+zWudnZ3h5uaG3Nxcs+s5duwYhgwZYjJvyJAhOHnyJPR6PW655RaEhoYiIiICM2fOxEcffYSysjIAQHR0NOLi4tC3b1/ccccd+N///ocrV66YXYO52FNjAZF+LvjtzGWcymWoISKSi6NaiT9XynPvMkd1w2cQmsvZ2dnk85IlS5CUlIQXXngB3bp1g6OjI26//XZUVlY2u52rHzMgCILxMUaW5OrqioMHDyI5ORk//fQTli9fjmeeeQb79++Hh4cHkpKSsHfvXvz000947bXX8I9//AO//vorwsPDLV5LLYYaC+jmJ/XUMNQQEclHEIRWnQKSm0ajadXjAPbs2YPZs2dj8uTJAKSemzNnzli5ujpRUVHYs2dPg5p69OhhfJC0SqXC6NGjMXr0aDz99NPw8PDAzz//jClTpkAQBAwZMgRDhgzB8uXLERoaik2bNmHRokVWq9n2j74diPSV0jVPPxERUUvCwsLw66+/4syZM3BxcWmyF6V79+746quvMH78eAiCgKeeesoqPS5NWbx4MQYNGoRVq1bhrrvuQkpKCtauXYs33ngDAPDdd9/h9OnTGDZsGDw9PfH999/DYDCgZ8+e+PXXX7F9+3aMGTMGfn5++PXXX3Hp0iVERUVZtWaOqbGA2p6aM/mlqNa33z84IiKyP0uWLIFSqcR1110HX1/fJsfIvPTSS/D09MRNN92E8ePHIz4+HgMGDGi3OgcMGIDPPvsMn3zyCfr06YPly5dj5cqVmD17NgDAw8MDX331FUaNGoWoqCi8+eab+Pjjj9G7d2+4ublh165duPXWW9GjRw/885//xIsvvoiEhASr1sw7CluAwSCi99M/orxKj+2LhxsHDhMRkXXY8x2FqSHeUdiGKBQCImpPQXFcDRERkSwYaizEOFiY42qIiMgGzZs3Dy4uLo1O8+bNk7s8i+BAYQvp5ssroIiIyHatXLkSS5YsaXRZR3nQM0ONhdT21PD0ExER2SI/Pz/4+fnJXYZV8fSThUTWhppLpVZ/tgURERE1xFBjIWHezlAqBJToqpFTpJO7HCIiok6HocZCNCoFQr2cAHBcDRERkRwYaiwogg+2JCIikg1DjQXxGVBERETyYaixIIYaIiKytrCwMLz88sutaisIAjZv3mzVemwJQ40F8QZ8RERE8mGosaDaRyVcKtahsLxK5mqIiIg6F4YaC3JzUMPfTQuAg4WJiKih//73vwgKCoLBYDCZP3HiRMyZMwfp6emYOHEi/P394eLigkGDBmHbtm0W+/4//vgDo0aNgqOjI7y9vfHAAw+gpKTu71VycjJuuOEGODs7w8PDA0OGDMHZs2cBAL///jtGjhwJV1dXuLm5YeDAgThw4IDFarMEhhoL47gaIiKZiCJQWSrP1Mqbrt5xxx3Iz8/Hjh07jPMuX76MrVu3YsaMGSgpKcGtt96K7du349ChQxg7dizGjx+PzMzMNv94SktLER8fD09PT+zfvx+ff/45tm3bhgULFgAAqqurMWnSJAwfPhyHDx9GSkoKHnjgAQiCAACYMWMGunbtiv379yM1NRVPPPEE1Gp1m+uyJD4mwcIifV2w51Q+e2qIiNpbVRnw7yB5vvvJi4DGucVmnp6eSEhIwMaNGxEXFwcA+OKLL+Dj44ORI0dCoVAgOjra2H7VqlXYtGkTvvnmG2P4uFYbN25ERUUF3n//fTg7S7WuXbsW48ePx+rVq6FWq1FYWIjbbrsNkZGRAICoqCjj+pmZmVi6dCl69eoFAOjevXub6rEG9tRYGJ8BRUREzZkxYwa+/PJL6HTS3ec/+ugjTJs2DQqFAiUlJViyZAmioqLg4eEBFxcXHDt2zCI9NceOHUN0dLQx0ADAkCFDYDAYcPz4cXh5eWH27NmIj4/H+PHj8corryArK8vYdtGiRbjvvvswevRoPPfcc0hPT29zTZbGnhoL49O6iYhkonaSekzk+u5WGj9+PERRxJYtWzBo0CDs3r0b//nPfwAAS5YsQVJSEl544QV069YNjo6OuP3221FZWWmtyk2sX78eDz/8MLZu3YpPP/0U//znP5GUlIQbb7wRzzzzDO6++25s2bIFP/zwA55++ml88sknmDx5crvU1hoMNRZW21OTebkMumo9tCqlzBUREXUSgtCqU0Byc3BwwJQpU/DRRx/h1KlT6NmzJwYMGAAA2LNnD2bPnm0MCiUlJThz5oxFvjcqKgobNmxAaWmpsbdmz549UCgU6Nmzp7FdTEwMYmJisGzZMgwePBgbN27EjTfeCADo0aMHevTogYULF2L69OlYv369TYUann6yMF9XLVy1KhhE4ExemdzlEBGRDZoxYwa2bNmCd999FzNmzDDO7969O7766iukpaXh999/x913393gSqm2fKeDgwNmzZqFI0eOYMeOHXjooYcwc+ZM+Pv7IyMjA8uWLUNKSgrOnj2Ln376CSdPnkRUVBTKy8uxYMECJCcn4+zZs9izZw/2799vMubGFrCnxsIEQUCknwvSzhXgVG4Jega4yl0SERHZmFGjRsHLywvHjx/H3XffbZz/0ksvYc6cObjpppvg4+ODxx9/HEVFRRb5TicnJ/z444945JFHMGjQIDg5OWHq1Kl46aWXjMv/+usvvPfee8jPz0dgYCDmz5+P//u//0N1dTXy8/Nxzz33ICcnBz4+PpgyZQpWrFhhkdosRRDFVl6HZueKiorg7u6OwsJCuLm5WfW7lnz+O75IPY+Fo3vgkdG2NzqciMjeVVRUICMjA+Hh4XBwcJC7HGqj5o6nOX+/efrJCiL5tG4iIqJ2x1BjBbwBHxERWdtHH30EFxeXRqfevXvLXZ4sOKbGCmpDzem8EhgMIhQKQeaKiIioo5kwYQJiY2MbXWZrd/ptLww1VhDs6QiNUoGKKgMuFJQj2Kv19y8gIiJqDVdXV7i68mKU+nj6yQpUSgXCfKQgc4rjaoiIiNqF2aFm165dGD9+PIKCgiAIAjZv3tygzbFjxzBhwgS4u7vD2dkZgwYNavEWz59//jl69eoFBwcH9O3bF99//73JclEUsXz5cgQGBsLR0RGjR4/GyZMnzS2/3fBxCURE1mepe7iQvCx1IbbZp59KS0sRHR2NOXPmYMqUKQ2Wp6enY+jQoZg7dy5WrFgBNzc3HD16tNlL7vbu3Yvp06cjMTERt912GzZu3IhJkybh4MGD6NOnDwBgzZo1ePXVV/Hee+8hPDwcTz31FOLj4/Hnn3/a5OV83XgFFBGR1Wg0GigUCly8eBG+vr7QaDTGp0mTfRFFEZcuXYIgCG0eC9Sm+9QIgoBNmzZh0qRJxnnTpk2DWq3GBx980Ort3HXXXSgtLcV3331nnHfjjTeif//+ePPNNyGKIoKCgrB48WIsWbIEAFBYWAh/f39s2LAB06ZNa/E72vM+NQDwddoFPPJJGgaFeeLzeTdZ/fuIiDqbyspKZGVloayMd2+3d4IgoGvXrnBxcWmwzJy/3xYdKGwwGLBlyxY89thjiI+Px6FDhxAeHo5ly5aZBJ+rpaSkYNGiRSbz4uPjjae2MjIykJ2djdGjRxuXu7u7IzY2FikpKY2GGp1OZ3wCKgCL3ZGxtSL5YEsiIqvSaDQICQlBdXU19Hq93OVQG6jVaiiVbX9WokVDTW5uLkpKSvDcc8/h2WefxerVq7F161ZMmTIFO3bswPDhwxtdLzs7G/7+/ibz/P39kZ2dbVxeO6+pNldLTEyU9fbNtaHmSlkV8kt08HbRylYLEVFHVXvKorNewkymLHr1U+2ArYkTJ2LhwoXo378/nnjiCdx222148803LflVLVq2bBkKCwuN07lz59r1+x01SnTxcAQApF8qbdfvJiIi6owsGmp8fHygUqlw3XXXmcyPiopq9uqngIAA5OTkmMzLyclBQECAcXntvKbaXE2r1cLNzc1kam+8szAREVH7sWio0Wg0GDRoEI4fP24y/8SJEwgNDW1yvcGDB2P79u0m85KSkjB48GAAQHh4OAICAkzaFBUV4ddffzW2sUUMNURERO3H7DE1JSUlOHXqlPFzRkYG0tLS4OXlhZCQECxduhR33XUXhg0bhpEjR2Lr1q349ttvkZycbFznnnvuQZcuXZCYmAgAeOSRRzB8+HC8+OKLGDduHD755BMcOHAA//3vfwFI50wfffRRPPvss+jevbvxku6goKBmByDLzXivGl7WTUREZHVmh5oDBw5g5MiRxs+1Vy3NmjULGzZswOTJk/Hmm28iMTERDz/8MHr27Ikvv/wSQ4cONa6TmZkJhaKuk+imm27Cxo0b8c9//hNPPvkkunfvjs2bNxvvUQMAjz32GEpLS/HAAw+goKAAQ4cOxdatW23yHjW1eAUUERFR+2nTfWrsSXvfpwYALpdWYsCqJADAnyvj4aTho7aIiIjMYc7fbz77yYq8nDXwctYAAE7zCigiIiKrYqixskhfZwAcV0NERGRtDDVWxiugiIiI2gdDjZVxsDAREVH7YKixskhe1k1ERNQuGGqsrFtNT01GXimq9QaZqyEiIuq4GGqsrIuHIxzVSlTpRWReLpO7HCIiog6LocbKFAoBEcYroHhZNxERkbUw1LQDDhYmIiKyPoaadsDLuomIiKyPoaYdGEMNr4AiIiKyGoaadlB7+ul0bgk6yaO2iIiI2h1DTTsI83GCQgCKddXILdbJXQ4REVGHxFDTDrQqJUK9pSugOK6GiIjIOhhq2kntKSjeWZiIiMg6GGraSaQfe2qIiIisiaGmnXTjvWqIiIisiqGmnXTjgy2JiIisiqGmndQ+rTunSIeiiiqZqyEiIup4GGraiZuDGn6uWgBAOk9BERERWRxDTTvi4xKIiIish6GmHdVd1s2ndRMREVkaQ007Yk8NERGR9TDUtCNeAUVERGQ9DDXtqDbUZF4ug65aL3M1REREHQtDTTvyc9XCRauC3iDibH6Z3OUQERF1KAw17UgQBOP9ajiuhoiIyLIYatpZ7eMSeK8aIiIiy2KoaWfGB1tysDAREZFFMdS0Mz7YkoiIyDoYatpZ/cu6DQZR5mqIiIg6Doaadhbi5QS1UkBFlQEXC8vlLoeIiKjDYKhpZyqlAmHeNeNqeAqKiIjIYhhqZMDHJRAREVkeQ40M6sbV8MGWRERElsJQI4NI3quGiIjI4swONbt27cL48eMRFBQEQRCwefNmk+WzZ8+GIAgm09ixY5vdZlhYWIN1BEHA/PnzjW1GjBjRYPm8efPMLd8mGE8/8V41REREFqMyd4XS0lJER0djzpw5mDJlSqNtxo4di/Xr1xs/a7XaZre5f/9+6PV1D3g8cuQIbrnlFtxxxx0m7e6//36sXLnS+NnJycnc8m1ChK80UPhyaSUul1bCy1kjc0VERET2z+xQk5CQgISEhGbbaLVaBAQEtHqbvr6+Jp+fe+45REZGYvjw4SbznZyczNqurXLSqNDFwxEXCsqRfqkEXs5ecpdERERk96wypiY5ORl+fn7o2bMnHnzwQeTn57d63crKSnz44YeYM2cOBEEwWfbRRx/Bx8cHffr0wbJly1BW1vSTrnU6HYqKikwmW8IHWxIREVmW2T01LRk7diymTJmC8PBwpKen48knn0RCQgJSUlKgVCpbXH/z5s0oKCjA7NmzTebffffdCA0NRVBQEA4fPozHH38cx48fx1dffdXodhITE7FixQpL7JJVdPN1wa4TlxhqiIiILEQQRfGa79UvCAI2bdqESZMmNdnm9OnTiIyMxLZt2xAXF9fiNuPj46HRaPDtt9822+7nn39GXFwcTp06hcjIyAbLdToddDqd8XNRURGCg4NRWFgINze3Fuuwto2/ZuLJTX9gRE9fbLj3BrnLISIisklFRUVwd3dv1d9vq1/SHRERAR8fH5w6darFtmfPnsW2bdtw3333tdg2NjYWAJrcrlarhZubm8lkSyJ9eVdhIiIiS7J6qDl//jzy8/MRGBjYYtv169fDz88P48aNa7FtWloaALRqu7ao9rLuCwXlKK/Ut9CaiIiIWmJ2qCkpKUFaWpoxVGRkZCAtLQ2ZmZkoKSnB0qVLsW/fPpw5cwbbt2/HxIkT0a1bN8THxxu3ERcXh7Vr15ps12AwYP369Zg1axZUKtOhPunp6Vi1ahVSU1Nx5swZfPPNN7jnnnswbNgw9OvX7xp2W37eLlp4OqkhisDpPPbWEBERtZXZoebAgQOIiYlBTEwMAGDRokWIiYnB8uXLoVQqcfjwYUyYMAE9evTA3LlzMXDgQOzevdvkXjXp6enIy8sz2e62bduQmZmJOXPmNPhOjUaDbdu2YcyYMejVqxcWL16MqVOntjjuxtbV3lmYp6CIiIjark0Dhe2JOQON2ssTXx7GJ/vP4eFR3bBoTE+5yyEiIrI5NjVQmJrGB1sSERFZDkONjHgDPiIiIsthqJFRt5oxNRl5pajWG2SuhoiIyL4x1Mioi4cjHNQKVOoNOHelXO5yiIiI7BpDjYwUCgERPjXjangKioiIqE0YamRmHFdziaGGiIioLRhqZNaN96ohIiKyCIYamdVd1s1QQ0RE1BYMNTLrVu+y7k5yH0QiIiKrYKiRWZiPExQCUFxRjUvFOrnLISIislsMNTLTqpQI8XICwHE1REREbcFQYwM4roaIiKjtGGpsAJ/WTURE1HYMNTaA96ohIiJqO4YaG2A8/ZTLp3UTERFdK4YaG1B7+im7qALFFVUyV0NERGSfGGpsgLujGr6uWgBA+iX21hAREV0LhhobUfu4BD7YkoiI6Now1NiIbhwsTERE1CYMNTYi0tcZAC/rJiIiulYMNTaim58rAJ5+IiIiulYMNTai9vTT2ctlqKw2yFwNERGR/WGosRH+blq4aFXQG0SczecVUEREROZiqLERgiBwXA0REVEbMNTYkEg+2JKIiOiaMdTYEONl3eypISIiMhtDjQ0xPq2bPTVERERmY6ixIfUfbGkwiDJXQ0REZF8YamxIiJcT1EoB5VV6ZBVVyF0OERGRXWGosSFqpQKh3rwCioiI6Fow1NiY2gdbMtQQERGZh6HGxnTjZd1ERETXhKHGxvCybiIiomvDUGNjai/r5oMtiYiIzMNQY2Mi/aSBwvmllbhSWilzNURERPaDocbGOGlU6OLhCIDjaoiIiMxhdqjZtWsXxo8fj6CgIAiCgM2bN5ssnz17NgRBMJnGjh3b7DafeeaZBuv06tXLpE1FRQXmz58Pb29vuLi4YOrUqcjJyTG3fLsQwQdbEhERmc3sUFNaWoro6Gi8/vrrTbYZO3YssrKyjNPHH3/c4nZ79+5tss4vv/xisnzhwoX49ttv8fnnn2Pnzp24ePEipkyZYm75doFXQBEREZlPZe4KCQkJSEhIaLaNVqtFQECAeYWoVE2uU1hYiHfeeQcbN27EqFGjAADr169HVFQU9u3bhxtvvNGs77J1vAKKiIjIfFYZU5OcnAw/Pz/07NkTDz74IPLz81tc5+TJkwgKCkJERARmzJiBzMxM47LU1FRUVVVh9OjRxnm9evVCSEgIUlJSGt2eTqdDUVGRyWQv+GBLIiIi81k81IwdOxbvv/8+tm/fjtWrV2Pnzp1ISEiAXq9vcp3Y2Fhs2LABW7duxbp165CRkYGbb74ZxcXFAIDs7GxoNBp4eHiYrOfv74/s7OxGt5mYmAh3d3fjFBwcbLF9tLbanprzV8pRUdX0z42IiIjqmH36qSXTpk0zvu/bty/69euHyMhIJCcnIy4urtF16p/O6tevH2JjYxEaGorPPvsMc+fOvaY6li1bhkWLFhk/FxUV2U2w8XbWwMNJjYKyKpy+VIrrgtzkLomIiMjmWf2S7oiICPj4+ODUqVOtXsfDwwM9evQwrhMQEIDKykoUFBSYtMvJyWlyHI5Wq4Wbm5vJZC8EQah7BhRPQREREbWK1UPN+fPnkZ+fj8DAwFavU1JSgvT0dOM6AwcOhFqtxvbt241tjh8/jszMTAwePNjiNduCSD7YkoiIyCxmh5qSkhKkpaUhLS0NAJCRkYG0tDRkZmaipKQES5cuxb59+3DmzBls374dEydORLdu3RAfH2/cRlxcHNauXWv8vGTJEuzcuRNnzpzB3r17MXnyZCiVSkyfPh0A4O7ujrlz52LRokXYsWMHUlNTce+992Lw4MEd7sqnWrysm4iIyDxmj6k5cOAARo4cafxcO25l1qxZWLduHQ4fPoz33nsPBQUFCAoKwpgxY7Bq1SpotVrjOunp6cjLyzN+Pn/+PKZPn478/Hz4+vpi6NCh2LdvH3x9fY1t/vOf/0ChUGDq1KnQ6XSIj4/HG2+8cU07bQ+MoYY9NURERK0iiKIoyl1EeygqKoK7uzsKCwvtYnxNZn4Zhj2/AxqVAsdWjoVSIchdEhERUbsz5+83n/1ko7p4OkKrUqCy2oDzV8rkLoeIiMjmMdTYKKVCQAQHCxMREbUaQ40N4+MSiIiIWo+hxoZF8mndRERErcZQY8N4WTcREVHrMdTYsPqnnzrJRWpERETXjKHGhoV5O0MhAEUV1bhUopO7HCIiIpvGUGPDHNRKBHs5AQDSc0tlroaIiMi2MdTYOD7YkoiIqHUYamxcZM24mpM5xTJXQkREZNsYamxc/2APAMCmgxeQW1QhbzFEREQ2jKHGxsX3DkB0V3cU66rx7++PyV0OERGRzWKosXFKhYBVk/pAEIDNaRexNz2v5ZWIiIg6IYYaO9Cvqwf+FhsKAHhq8xFUVhtkroiIiMj2MNTYiSVjesLHRYP0S6V4+5fTcpdDRERkcxhq7IS7kxpP3hoFAHh1+0mcv1Imc0VERES2haHGjkyO6YIbwr1QUWXAim//lLscIiIim8JQY0cEQcCzk/pApRCQ9GcOtv2ZI3dJRERENoOhxs708HfF3JvDAQDPfHsU5ZV6mSsiIiKyDQw1dujhUd0R5O6A81fK8fqOU3KXQ0REZBMYauyQs1aF5eN7AwDe2pWOdD4XioiIiKHGXsX39sfInr6o0otY/vURiKIod0lERESyYqixU4IgYMWEPtCqFNhzKh/fHs6SuyQiIiJZMdTYsRBvJ8wf2Q0AsOq7P1FUUSVzRURERPJhqGmrqnLg5Dbg909l+foHhkUg3McZl4p1+E/SCVlqICIisgUMNW114SDw0VTgx2WAof2fyeSgVmLFBGnQ8Ht7z+DoxcJ2r4GIiMgWMNS0VddBgNoZKMsHcv6QpYRhPXwxrl8gDCLwz81HYDBw0DAREXU+DDVtpdIAYUOl9+k7ZCvjqXHXwVmjxKHMAnx24JxsdRAREcmFocYSIkdKr6flCzUB7g5YeEsPAMBzW//C5dJK2WohIiKSA0ONJUSOkl7PpkgDh2Uy+6Yw9ApwRUFZFVb/8JdsdRAREcmBocYSfHoArkGAXgdkpshWhkqpwLOT+gAAPj1wDqlnL8tWCxERUXtjqLEEQag7BSXjuBoAuD7MC3de3xUA8M/NR1Gtb/8rsoiIiOTAUGMpEfKPq6n1REIUPJzUOJZVhPdTzspdDhERUbtgqLGUiBHSa/YfQMklWUvxctbg8bG9AAAvJZ1ATlGFrPUQERG1B4YaS3HxBfz7Su8zdspbC4C7rg9G/2APlOiq8eyWY3KXQ0REZHUMNZYUOUJ6lXlcDQAoFAKendQHCgH49veL+OVkntwlERERWZXZoWbXrl0YP348goKCIAgCNm/ebLJ89uzZEATBZBo7dmyz20xMTMSgQYPg6uoKPz8/TJo0CcePHzdpM2LEiAbbnTdvnrnlW1f9cTWi/Hf17dPFHfcMDgMALP/6CHTVenkLIiIisiKzQ01paSmio6Px+uuvN9lm7NixyMrKMk4ff/xxs9vcuXMn5s+fj3379iEpKQlVVVUYM2YMSktLTdrdf//9Jttds2aNueVbV+hNgFILFF0A8mzj4ZKLxvSAr6sWp/NK8b9dp+Uuh4iIyGpU5q6QkJCAhISEZttotVoEBAS0eptbt241+bxhwwb4+fkhNTUVw4YNM853cnIya7vtTu0IhNwojalJ3wH49pS7Irg5qPHPcVF45JM0vPbzKUzs3wXBXk5yl0VERGRxVhlTk5ycDD8/P/Ts2RMPPvgg8vPzzVq/sFB60rSXl5fJ/I8++gg+Pj7o06cPli1bhrKysia3odPpUFRUZDK1Cxt4ZMLVJkQH4aZIb+iqDXj6m6MQbeDUGBERkaVZPNSMHTsW77//PrZv347Vq1dj586dSEhIgF7fuvEcBoMBjz76KIYMGYI+ffoY599999348MMPsWPHDixbtgwffPAB/va3vzW5ncTERLi7uxun4ODgNu9bq9SOqznzC6Cvap/vbIEgCFg5sQ/USgE//5WLpD9z5C6JiIjI4gSxDf/bLggCNm3ahEmTJjXZ5vTp04iMjMS2bdsQFxfX4jYffPBB/PDDD/jll1/QtWvXJtv9/PPPiIuLw6lTpxAZGdlguU6ng06nM34uKipCcHAwCgsL4ebm1mId18xgAF7oBpTlA/f+II2zsRHP//gXXt+Rji4ejkhaNAxOGrPPPhIREbWroqIiuLu7t+rvt9Uv6Y6IiICPjw9OnTrVYtsFCxbgu+++w44dO5oNNAAQGxsLAE1uV6vVws3NzWRqFwoFED5cem8Dl3bXt2Bkd3TxcMSFgnK89nPLx4OIiMieWD3UnD9/Hvn5+QgMDGyyjSiKWLBgATZt2oSff/4Z4eHhLW43LS0NAJrdrmxscFwNADhqlHhmQm8AwP92ncbJnGKZKyIiIrIcs0NNSUkJ0tLSjKEiIyMDaWlpyMzMRElJCZYuXYp9+/bhzJkz2L59OyZOnIhu3bohPj7euI24uDisXbvW+Hn+/Pn48MMPsXHjRri6uiI7OxvZ2dkoLy8HAKSnp2PVqlVITU3FmTNn8M033+Cee+7BsGHD0K9fvzb+CKygdlzNhVSgvEDWUq52y3X+GB3lh2qDiKe+PsJBw0RE1GGYHWoOHDiAmJgYxMTEAAAWLVqEmJgYLF++HEqlEocPH8aECRPQo0cPzJ07FwMHDsTu3buh1WqN20hPT0deXt0dbtetW4fCwkKMGDECgYGBxunTTz8FAGg0Gmzbtg1jxoxBr169sHjxYkydOhXffvttW/ffOjyCAe/ugGgAzuyWu5oGnh7fGw5qBfadvoyv0y7KXQ4REZFFtGmgsD0xZ6CRRXy/FPjtv8D1c4HbXrL+95np9R2n8PyPx+HjosX2xcPh7qiWuyQiIqIGbGqgcKcVYZvjamrdd3M4InydkVeiw0s/HW95BSIiIhvHUGMtYUMBQQlcPg1cOSt3NQ1oVUqsmijdB+iDfWfx2f5zHF9DRER2jaHGWhzcgK6DpPc22lszpJsPpgzoAoMIPPblYUz77z6cyuUVUUREZJ8Yaqyp9tJuG7tfTX2rp/bDEwm94KBW4NeMy0h4ZTde+PE4Kqr4RG8iIrIvDDXWVDuuJmMnYLDNkKBWKjBveCSSFg5HXC8/VOlFrN1xCvEv78KuE5fkLo+IiKjVGGqsqctAQOsGlF8BstLkrqZZwV5OeHvW9XjzbwMQ4OaAs/lluOfd37Bg40HkFlXIXR4REVGLGGqsSakCwm6W3tvwKahagiBgbJ9AbFs8HHOGhEMhAN8dzkLcizvxQcoZ6A0cSExERLaLocbajI9MSJa1DHO4aFVYPv46fLNgKPp1dUexrhpPfX0UU9btxZELhXKXR0RE1CiGGmurHVeTuQ+oLJW3FjP16eKOTX8fgpUTe8NVq8Lv5wowYe0vWPXdnyjRVctdHhERkQmGGmvzjgTcgwFDFXB2r9zVmE2pEHDP4DBsWzwct/ULhEEE3vklA7e8tBNbj2Tz3jZERGQzGGqsTRCAiBHSezsYV9MUfzcHrL17ADbcOwghXk7IKqzAvA9Tcf/7B3D+Spnc5RERETHUtItI235kgjlG9PTDTwuHYcHIblArBWw7lotbXtqFt3amo0pvkLs8IiLqxBhq2kP4CAACkPsnUJwtczFt56BWYkl8T3z/8M24IdwL5VV6JP7wF8a/9gtSz16WuzwiIuqkGGrag7M3EBgtvbejq6Ba0t3fFZ8+cCPW3N4Pnk5q/JVdjKnrUrDsqz9QUFYpd3lERNTJMNS0Fzt4ZMK1EAQBd14fjO2LR+COgV0BAB//lom4F3di06HzHEhMRETthqGmvUTUu19NB/xD7+WswfN3ROPTB25ENz8X5JdWYuGnv2PG278i/VKJ3OUREVEnwFDTXkJuBFSOQEk2kHtM7mqsJjbCG98/fDOWxveEVqXA3vR8jH5pJ6b9NwWf7s9EUUWV3CUSEVEHxVDTXlRaIPQm6X0HuAqqORqVAvNHdkPSwuEYHeUHUQT2nb6Mx7/8A9c/uw3zPzqIpD9zUFnNq6WIiMhyBLGTDHooKiqCu7s7CgsL4ebmJk8Re18Dfvon0O0W4G9fyFODDC4UlOPrtAvYdPACTubWnYrydFLjtn5BmBTTBQNCPCAIgoxVEhGRLTLn7zdDTXvKPgK8OUQ6DfXEWan3phMRRRFHLxZh86EL+Pr3i7hUrDMuC/V2wqT+XTAppgvCfZxlrJKIiGwJQ00jbCLUiCLwQg+gNBeY9S0QPkyeOmyA3iBiz6k8bD50AVuPZqOsUm9c1j/YA1MGdMG4voHwdulcwY+IiEwx1DTCJkINAHx5P/DHZ8DQRcDop+Wrw4aUVVbjp6M52HToAnafvARDzb9IlULA8B6+mDygC0ZH+cNBrZS3UCIiancMNY2wmVCTthHY/CAQFAM8kCxfHTYqt7gC3/6ehc2HLuCPC4XG+S5aFRL6BGDygC64MdwbCgXH3xARdQYMNY2wmVBTdBF4KQqAADx2GnDykq8WG3cqtxibDl3A5kMXcaGg3Dg/0N0BE/oHYUpMV/QMcJWxQiIisjaGmkbYTKgBgNdjgUt/AXdsAHpPlrcWO2AwiNh/5jI2p13Ad4ezUFxRbVwWFeiGW6L8EBPqiQHBnnB3UstYKRERWZo5f79V7VQT1RcxUgo16TsYalpBoRAQG+GN2AhvPD2+N3b8lYtNhy5gx/FcHMsqwrGsImPbSF9nDAjxxIBQTwwI8UR3PxeeqiIi6iTYUyOHEz8CG+8EPEKARw4DvD/LNSkoq8TWI9n4LeMyDmZewZn8sgZtXB1U6B/sYQw6/YM94O7I3hwiInvB00+NsKlQoysBVocBhirgoYOAd6S89XQQ+SU6HMoswMHMKziYeQW/nytEeZW+Qbvufi41IUcKO5G+7M0hIrJVDDWNsKlQAwDrxwFnfwHGvQgMuk/uajqkar0Bf2UX41DmFRysCTtnG+nNcXNQoX+IJwaESCGnf4gH3BzYm0NEZAsYahphc6Fm1/PAz88CvW4Dpn0kdzWdRl5Nb07qWak35/D5AlRUmT6DShDq9eaEeKJHgCsifJ0ZdIiIZMBQ0wibCzXnU4G3RwFad+nSbiXHbMuhSm/AX1nFxlNWBzOv4Nzl8kbb+rpqEeHjjEg/F+NrpI8Lung6QsnTV0REVsFQ0wibCzUGPbAmAqgoAOZuA4IHyV0R1cgtrjCOzfn9XAHSL5WaPKfqahqVAuHezoj0c0aEj4vxNcLXGa7s3SEiahNe0m0PFErp2U/HvgFO72CosSF+rg6I7x2A+N4BxnlFFVU4fakUpy+V4PSlUqTXvGbklaKy2oDjOcU4nlPcyLa0iPB1RqSvCyJ8XRBZ8z7Ig707RESWxlAjp8iRUqhJ3wEMf0zuaqgZbg5q9A/2QP9gD5P5eoOIC1fKkX6pRAo6eaVIz5VeLxXrkFsz7Tt92WQ9rUqBcB9nRPjW9epE+Lpw7A4RURsw1MgpYqT0ev43QFcMaHnLf3ujVAgI8XZCiLcTRvbyM1lW27sjhZy6Hp4zeWXQVUtXZv2V3bB3x8dFUy/oSKEn3NcZIV5OUCsV7bVrRER2x+xQs2vXLjz//PNITU1FVlYWNm3ahEmTJhmXz549G++9957JOvHx8di6dWuz23399dfx/PPPIzs7G9HR0Xjttddwww03GJdXVFRg8eLF+OSTT6DT6RAfH4833ngD/v7+5u6C7fAKBzzDgCtngDO/AD0T5K6ILKi53p3zV8rqTmPl1Z3Wyi3WIa+kEnkll/HbGdPeHZVCQIiXk7FXJ9zHGRE+0nsfFw0E3sSRiDo5s0NNaWkpoqOjMWfOHEyZMqXRNmPHjsX69euNn7VabbPb/PTTT7Fo0SK8+eabiI2Nxcsvv4z4+HgcP34cfn7S//0uXLgQW7Zsweeffw53d3csWLAAU6ZMwZ49e8zdBdsSMRJIXS+dgmKo6RSUCgGh3s4I9XZu0LtToqtGxqVSnM4rQXrNGJ6MPGnsTlmlXgpAeaXAsVyT9VwdVNLpq3pBJ8LXGaHeTnDSsEOWiDqHNl39JAhCoz01BQUF2Lx5c6u3Exsbi0GDBmHt2rUAAIPBgODgYDz00EN44oknUFhYCF9fX2zcuBG33347AOCvv/5CVFQUUlJScOONN7b4HTZ39VOtP78GPrsH8OkBLNgvdzVko0RRRHZRhTRYuV7Pzum8Epy/Uo7mfos9ndTo4umIIHdHBHk4oqun9Brk4YguHo7s5SEimyb71U/Jycnw8/ODp6cnRo0ahWeffRbe3t6Ntq2srERqaiqWLVtmnKdQKDB69GikpKQAAFJTU1FVVYXRo0cb2/Tq1QshISGtDjU2K3wYICiAvBNA4QXAvYvcFZENEgQBge6OCHR3xJBuPibLKqr0yLxchtOXant3SpGRJ53WKiirwpWa6ciFoka3rVEp0MXDEUEeDghyd5QCUE3g6eLhiAB3Bziole2xm0REbWLxUDN27FhMmTIF4eHhSE9Px5NPPomEhASkpKRAqWz4H8a8vDzo9foGY2P8/f3x119/AQCys7Oh0Wjg4eHRoE12dnajdeh0Ouh0dfcWKSpq/D/osnP0BIJigAup0qXdMX+TuyKyMw5qJXr4u6KHf8OB5kUVVbhYUI6LBeW4cKUcFwoqcKHm88WCcuQUVaCy2mA8xdUUHxctung6oouHQ00AkqZAdwcEuDnA20XLS9SJSHYWDzXTpk0zvu/bty/69euHyMhIJCcnIy4uztJf16TExESsWLGi3b6vTSJGSqEmnaGGLMvNQQ23ADV6BTTeZVulNyC70DToXCioCT9XynCxoALlVXrkleiQV6LD7+ca/x6lQoCvixb+7g4IcNPC380B/m5S4PF3c0CAuzSPNyMkImuy+gjCiIgI+Pj44NSpU42GGh8fHyiVSuTk5JjMz8nJQUCAdPOzgIAAVFZWoqCgwKS3pn6bqy1btgyLFi0yfi4qKkJwcLAF9sgKIkcCu18ATicDBgOg4GW71D7USgWCvZwQ7OXU6HJRFFFQVlUTdKTenosF5bhYKL3PLqrApWId9AZpzE92UQV+b+b7nDXKusDj7gA/Ny0CasKPX+08Vy0vXSeia2L1UHP+/Hnk5+cjMDCw0eUajQYDBw7E9u3bjQOODQYDtm/fjgULFgAABg4cCLVaje3bt2Pq1KkAgOPHjyMzMxODBw9udLtarbbFq65sRtcbALUzUJYH5BwBAvvJXRERAGksj6ezBp7OGvTp4t5om2q9AfmllcgulEJNbk24yS7UIaeoAjk1n4srqlFa/wquJr8T8HbWws9VCy9nDTyc1NLkWPteAw9Hdd17JzXcHdUMQkRkfqgpKSnBqVOnjJ8zMjKQlpYGLy8veHl5YcWKFZg6dSoCAgKQnp6Oxx57DN26dUN8fLxxnbi4OEyePNkYWhYtWoRZs2bh+uuvxw033ICXX34ZpaWluPfeewEA7u7umDt3LhYtWgQvLy+4ubnhoYcewuDBg+17kHAtlQYIGwqc/FEaV8NQQ3ZEpVQYe1+im2lXVlmNnCIdsgsrTMKO9F6an1tcgSq9aDzdZQ5XrQruDQJQwzDk6ayGu6MGnjXzOBaIqOMwO9QcOHAAI0eONH6uPcUza9YsrFu3DocPH8Z7772HgoICBAUFYcyYMVi1apVJr0l6ejry8vKMn++66y5cunQJy5cvR3Z2Nvr374+tW7eaDB7+z3/+A4VCgalTp5rcfK/DiBwphZr0HcCQR+SuhsjinDQqhPuoEO7j3GQbg0HElbLKmh4fHQrKK41XcBWWVaKg/Kr3pZUoqqgGABTrqlGsq8b5K40/Zb0xgoCaoKOBt7MGnk4aeLtIr17ODSdvZy0cNbwSjMhW8SndtiL3L+CNWEDlADx+FlA7yF0RkV3QG0QUlVfhSk3QKSyreV9WVfO5Eleufl9WF4bM5aBWwNtZC09nNbyctfByqnl1Nn110argoFbAQa2Eg1oJrUp6z54hIvPIfp8auga+PQHXQKA4C8hMkXpuiKhFSkXduB9zVOsNNffwqUR+SSUul1biclklLpdUSvNKK3GltO71cmklKvUGVFQZjAOnr4VaKcBBpYRWrYBWpYRDvderA1DtMq1aAQeV6TJnrRKuDiq4aNVw0apq3qvg4qDi+CLqtBhqbIUgSJd2/75RGlfDUENkVSqlAr6uWvi6aoFWPEJOFEWUVupxuaQm/JTqkH9VALpcbyrR6aGr0qOiWo8qfV2HeJVeRJW+GsXmDRkyi1alMAk5Llop/DScV/fZ9ar5GpVCmpQKqBiSyE4w1NiSyJpQk/4zcMtKuashonoEQTD+wQ/xbvwS+KboDSJ01XpUVBlQUaWHrlp6NX1vgK5aD12VARXVNcuM7xuuV6rTo0RXXTdVVKO8Sg8A0FUboCupRF5JpUX2XSHAGHA0Kqm3qPazWiXUzJeWaZQKk+XGcKRSQF2zTKtSQFuv16mpV4ererN46o5awlBjSyJGSK/ZfwAllwAXX1nLISLLUCoEOGlUcDLvDJnZqvUGlOr0KNZVGYNOcc1rw89Sm+J6y0prBluX6KpNnidmEFETrAwArm0skiWolYJJyKk9LdfUa+0pPW1NQKo9hVd3qk9x1alA09OADir2Utkbhhpb4uIH+PeR7lWTsRPoe7vcFRGRHVEpFXB3UsDdqW13bhZFEdUGEZXVBmnSX/V61Xtdvc9VV7XRVdf/rDe219X0TFU08VrbS9XYqTszr/ZvE5VCaNBr5FzTY1f3qqwZ26Q0zqtd7lwz3slZq4KLRmrLoGQ9DDW2JmKEFGrSdzDUEJEsBEGAWilArVTAWeZ7mNaeuqsNOfVPyUljlpp4rWrktF390371TvPVD1S17WtVG8SaU3yW2ycHteKqUFT/VQlnjQpOWhWcNEo4a5RwqglDJq8aFZy0SjhplHBUKyEIPDUHMNTYnsiRQMpaabCwKEoDiImIOqn2OnVXn8Eg1lzpVhd06o9tKquUTtuV6vTGU3aluuqG7yuqUVoptSupqEalXgpL0nYsN+ZJEAAntRJOWlWDEOTUyGcHtQKONaflHGtOtTlefeqtpk3tZ3sZz8RQY2tCbgKUGqDoApB3EvDtIXdFRESdikIhwEEh/TG3pMpqA0rrDe5u+F4KP2VV1SjT6VFaWfdaXqlHaaUUqEp10mtZpTQwXBSB0prllyxacR2NUmESeq4OQbWfAz0c8fjYXlaqomUMNbZG4wSE3Ahk7JJ6axhqiIg6BOkqMPPvqdQUg0FEeVW90FMTdkor9SjTVZuEoPLKus+1PU7lV52mq5tnQHmVNP6pVqVeGiPV0k0rI32dGWroKhEjpVCTvgOI/T+5qyEiIhukUAjGwcjWYDCIJqfd6kJQTfCplE7LSa/SWCYXK9XSWgw1tihyJLB9BXDmF0BfBSjbdiUDERGRuRQyjGdqK15XZosCogFHL6CyGDh/QO5qiIiI7AJDjS1SKOpuxHd6h6ylEBER2QuGGltV++yndIYaIiKi1mCosVURNaHmQipQUShvLURERHaAocZWeQQD3t0AUQ9k7Ja7GiIiIpvHUGPLantrOK6GiIioRQw1tsw4ruZneesgIiKyAww1tixsKCAogcungStn5a6GiIjIpjHU2DIHd6Dr9dJ7noIiIiJqFkONrasdV5P6HqBv/pkbREREnRlDja0bcA+gdQcuHgR+eUnuaoiIiGwWQ42tc+8CjHtBep/8nHTfGiIiImqAocYe9L0D6D1FumfNVw8AlWVyV0RERGRzGGrsgSAA414EXIOA/FNA0lNyV0RERGRzGGrshZMXMOkN6f3+t4GTSfLWQ0REZGMYauxJ5Egg9kHp/dfzgdJ8eeshIiKyIQw19mb004BvL6AkB/juEUAU5a6IiIjIJjDU2Bu1IzDlv4BCDRz7FkjbKHdFRERENoGhxh4FRgMjn5Te//A4cOWMrOUQERHZAoYaezXkESBkMFBZDGyaBxj0cldEREQkK4Yae6VQApPfBDSuQGYKsOcVuSsiIiKSFUONPfMMAxJWS+93/BvI+l3WcoiIiOTEUGPv+t8N9LoNMFRJdxuuKpe7IiIiIlkw1Ng7QQDGvwq4+AOX/gK2rZC7IiIiIlkw1HQEzt7AxNel97+uA9J3yFsPERGRDMwONbt27cL48eMRFBQEQRCwefPmJtvOmzcPgiDg5ZdfbnabYWFhEAShwTR//nxjmxEjRjRYPm/ePHPL77i63wJcP1d6v/nvQNlleeshIiJqZ2aHmtLSUkRHR+P1119vtt2mTZuwb98+BAUFtbjN/fv3IysryzglJUnPNbrjjjtM2t1///0m7dasWWNu+R3bmGcB725A8UXg+yVyV0NERNSuVOaukJCQgISEhGbbXLhwAQ899BB+/PFHjBs3rsVt+vr6mnx+7rnnEBkZieHDh5vMd3JyQkBAgLkldx4aJ+luw2/fAhz5EuiRAPS7o+X1iIiIOgCLj6kxGAyYOXMmli5dit69e5u9fmVlJT788EPMmTMHgiCYLPvoo4/g4+ODPn36YNmyZSgrK7NU2R1Hl4HA8Mel91sWAwXn5K2HiIionZjdU9OS1atXQ6VS4eGHH76m9Tdv3oyCggLMnj3bZP7dd9+N0NBQBAUF4fDhw3j88cdx/PhxfPXVV41uR6fTQafTGT8XFRVdUz126ebFwKkk4Px+YPODwD3fAAqOCScioo7NoqEmNTUVr7zyCg4ePNigl6W13nnnHSQkJDQYi/PAAw8Y3/ft2xeBgYGIi4tDeno6IiMjG2wnMTERK1Z00sublSpg8lvAm0OBM7uBfW8ANy2QuyoiIiKrsuj/vu/evRu5ubkICQmBSqWCSqXC2bNnsXjxYoSFhbW4/tmzZ7Ft2zbcd999LbaNjY0FAJw6darR5cuWLUNhYaFxOneuk52G8Y4E4v8tvd++Asg5Km89REREVmbRnpqZM2di9OjRJvPi4+Mxc+ZM3HvvvS2uv379evj5+bVqcHFaWhoAIDAwsNHlWq0WWq225aI7soGzgRNbpemrB4D7fwZUnfxnQkREHZbZoaakpMSkdyQjIwNpaWnw8vJCSEgIvL29Tdqr1WoEBASgZ8+exnlxcXGYPHkyFiyoOyViMBiwfv16zJo1CyqVaVnp6enYuHEjbr31Vnh7e+Pw4cNYuHAhhg0bhn79+pm7C52HIAATXgPeGAzkHAF+fhYYs0ruqoiIiKzC7NNPBw4cQExMDGJiYgAAixYtQkxMDJYvX97qbaSnpyMvL89k3rZt25CZmYk5c+Y0aK/RaLBt2zaMGTMGvXr1wuLFizF16lR8++235pbf+bj4ScEGAPa+Bpz5Rd56iIiIrEQQRVGUu4j2UFRUBHd3dxQWFsLNzU3uctrfNw8BB98H3IOBB/cADu5yV0RERNQic/5+8zrfziI+EfAMAwrPAd8/Jnc1REREFsdQ01loXYAp/wMEBXD4E+DoJrkrIiIisiiGms4k+AbpxnwA8O2jQNFFWcshIiKyJIaazmb440Bgf6CiAPh6PmAwyF0RERGRRTDUdDZKtXQaSuUIpP8M7H9b7oqIiIgsgqGmM/LtUXe/mqSngEvH5a2HiIjIAhhqOqtB9wGRcUB1BfDV/UDOn4BBL3dVRERE14z3qenMirKAdYOB8ivSZ7UzEBgNdBlQMw0EPEKlOxMTERHJwJy/3xZ99hPZGbdAYPqnwM+rgIuHgMoSIHOvNNVy8gaC6oWcoAGAi698NRMRETWBPTUkMeiBvJPAxYPAhVTgwkEg+w/AUNWwrXsI0CWmLuQE9Qe0ru1eMhERdXzm/P1mqKGmVeukB2FeOFgzpQJ5JwBc/U9GAHx71uvRGQD49+ETwYmIqM0YahrBUGMhFUVAVlpdyLl4SHr0wtWUGinYdBkARIwAeo4DFByXTkRE5mGoaQRDjRWV5Eohp/6pq/LLpm0C+gG3rAAiR8lTIxER2SWGmkYw1LQjUQSunJFCzrn9QNpHgK5IWhY+XAo3QTGylkhERPaBoaYRDDUyKs0Hdr8I7P8foK+U5vWeAsQ9BXhFyFsbERHZNHP+fnOQA1mfszcw9t/AggNAv2kABODoV8DaQcD3S4GSS3JXSEREHQBDDbUfz1BgylvAvN1At1sAQzXw23+BV/sDyc8BumK5KyQiIjvGUEPtL6Av8LcvgFnfSpeBV5YAyYnAqzHAb/8DqivlrpCIiOwQQw3JJ3wYcP/PwB3vAV6RQOkl4PslwOs3AEe+BAwGuSskIiI7wlBD8hIEoPckYP6vwLiXAGc/4EoG8MUc4H8jgdPJcldIRER2gqGGbINSDQyaCzx8CBj5D0DjIt3k7/2JwAeTgazf5a6QiIhsHEMN2RatCzD8MeCR34HYeYBCDaT/DLw1DPjyfun+N0RERI1gqCHb5OwDJKwGFuwH+t4hzfvjM+C164EfngBK8+Stj4iIbA5DDdk2r3Bg6tvA/+2SHrFgqAJ+XQe80h/Y+TxQWSp3hUREZCN4R2GyL+k7gG1P142xcfYDrr8XCL0J6DoI0DjLWx8REVkUH5PQCIaaDsRgAP7cBGxfaTrGRqECAqOBkMF1k7O3bGUSEVHbMdQ0gqGmA6quBI58IfXeZKYAhecatvHpCYQOBkJukl49Qtq/TiIiumYMNY1gqOkECs5J4ebsXun10l8N27h1rQk5g6VTVj49AQWHlhER2SqGmkYw1HRCpfnAuX11ISfrd+l5U/U5etadqgq9STp9pVTLUy8RETXAUNMIhhpCZSlwfj9wNgXI3AucPwBUlZm2UTkCXa+XAk7IYCD4Bg4+JiKSEUNNIxhqqAF9ldR7U9uTk5kClF8xbSMoAb/rgC4xQFCM9ABO/97szSEiaicMNY1gqKEWGQxA3vG6kHM2BSg637CdUgsE9JECTlAM0GUA4NMDUCjbv2Yiog6OoaYRDDV0TQrPAxcPARcOAhcPSu8rChu2UztL43G61ASdoBjAK0J6YCcREV0zhppGMNSQRYgicPm0FG6MUxpQ1cidjR3c6wJO0AAp8Lh1YdAhIjIDQ00jGGrIagx6IO+k1JNzoaY3J/sPQK9r2NbZ1/S0VVAM4OLX/jUTEdkJhppGMNRQu6quBC4dqws5Fw8COX8Cor5hW/fgmpAzUAo6gf0BB/4bJSICzPv7bfZdx3bt2oXx48cjKCgIgiBg8+bNTbadN28eBEHAyy+/3Ow2n3nmGQiCYDL16tXLpE1FRQXmz58Pb29vuLi4YOrUqcjJyTG3fKL2odJIY2yuvxeY8Cow7xfgyQvA3G1Awhogerp04z8I0p2Qj30jPdPqvfHAcyHA2huATfOA3/4HnE8Fqhvp9SEiIhMqc1coLS1FdHQ05syZgylTpjTZbtOmTdi3bx+CgoJatd3evXtj27ZtdYWpTEtbuHAhtmzZgs8//xzu7u5YsGABpkyZgj179pi7C0TyUDsCwYOkqVZFkXRZ+YXUmtNXh4DCTOkqrLzjwO8fS+0UaumKqy4D68bn8IorIiITZoeahIQEJCQkNNvmwoULeOihh/Djjz9i3LhxrStEpUJAQECjywoLC/HOO+9g48aNGDVqFABg/fr1iIqKwr59+3DjjTeatxNEtsLBDQi/WZpqlVyqCTip0umrC6lA+eW6gcm1NC7SqaouNSGny0DpVBYHIhNRJ2V2qGmJwWDAzJkzsXTpUvTu3bvV6508eRJBQUFwcHDA4MGDkZiYiJAQ6eGDqampqKqqwujRo43te/XqhZCQEKSkpDQaanQ6HXS6ui77oqKiNuwVUTty8QV6xEsTIF1xVXC2LuDUXnFVWQKc/UWaajn51AWcoAGAT3fA0QPQurFXh4g6PIuHmtWrV0OlUuHhhx9u9TqxsbHYsGEDevbsiaysLKxYsQI333wzjhw5AldXV2RnZ0Oj0cDDw8NkPX9/f2RnZze6zcTERKxYsaItu0JkGwQB8AyTpj41p3wNeuDS8XqnrQ4COUeAsjzg5E/SdDWtm3SZ+bVMDEVEZAcsGmpSU1Pxyiuv4ODBgxDM6AKvfzqrX79+iI2NRWhoKD777DPMnTv3mmpZtmwZFi1aZPxcVFSE4ODga9oWkc1RKAH/66RpwExpXlWFFGzqn7YqulD3fCtdkTQVnru272wsFLkHS71BPt0B7+6AWxBPfxGRbCwaanbv3o3c3FzjaSMA0Ov1WLx4MV5++WWcOXOmVdvx8PBAjx49cOrUKQBAQEAAKisrUVBQYNJbk5OT0+Q4HK1WC61We837QmR31A7Swzi7Xm86v7pSCjMVhUBFAVBeUPO+lVN1ubSd1oQitTPgHSkNYvbpDnh3q3vlg0GJyMosGmpmzpxpMu4FAOLj4zFz5kzce++9rd5OSUkJ0tPTMXOm9H+gAwcOhFqtxvbt2zF16lQAwPHjx5GZmYnBgwdbbgeIOiKVBlD5AM4+17Z+tU66SssYdAqk1/LLwJUzQN4pIP8kcDlDurNy9mFpuppbV8Cnm9SjYww8PaS7LCvMvrsEEVEDZoeakpISYw8KAGRkZCAtLQ1eXl4ICQmBt7e3SXu1Wo2AgAD07NnTOC8uLg6TJ0/GggULAABLlizB+PHjERoaiosXL+Lpp5+GUqnE9OnTAQDu7u6YO3cuFi1aBC8vL7i5ueGhhx7C4MGDeeUTkbWptNLgZRff5ttVV0ohJ/+kdIfl/JN1gacsX3o4aNF54HTyVdt3rAk49QKPT3fAKxLQuvJ0FhG1mtmh5sCBAxg5cqTxc+24lVmzZmHDhg2t2kZ6ejry8vKMn8+fP4/p06cjPz8fvr6+GDp0KPbt2wdf37r/iP7nP/+BQqHA1KlTodPpEB8fjzfeeMPc8onIWlQawLeHNF2t7HK9oHMSyD8lvV4+LZ3eyvlDmq4mKKVL17UuV726NjLftebVud77q5apeEqaqCPjYxKISD76auly9QaB5wRQesny36dQ14UcjbN0Q0S1I6ByMH1t8N5RGrNU+6p2an4dlQN7mIgsxJy/3xa/pJuIqNWUKmlgsXckgLGmyypLpbE8lSWArrjmtaSVn0uAymJpG7qSusHOhiqg/Io0WZujpzReyDVQuirMrUvNa71J69a5wo8oAllpQGke0HWQdA8lIgtiqCEi26RxttwVU/pqKezUDz6VJdJl8NXlQFXNVF1h+tpgXlm9deqvW/PeUF33nbXhKedIM/voUi/kNBaAugBOXvYffHKPAX98ARz5EriSIc0TFNIdsSOGA+HDgZAbpV4uojbg6SciIkvRV9cFnbJ86T5BRRcbmS5IV5G1hlILuAVe1dPTRXqye9AAqbfLFl05I4WYP74Eco/WzVc5Aq4BdeGmllILBN9QF3Jsed+oXZnz95uhhohIDpWlQFEWUFwv6Fwdfkpzm9+G1g0IGyqFgIgRgG9PeXt1inOAo5uAI18A5/fXzVeogW5xQJ/bgZ4J0rimwvNAxi7g9E4gYydQnGW6LY0rEDakZt+GA37X2X+PVVNEUQq5BZnSWDJBAShU0kB5hapmunqeUppMPqvq1q2dJygbv2WCQQ/oqwB9pfRqqPe+0fnNLas339EDGHSfRX88DDWNYKghIrtTXSn9sS+6WC/8XJR6Qc7ubdjb4xIghZva3g73LtavsfwKcOxb6fTSmd2AaKhZIEgPau1zOxA1XjqN1hRRlAaIn06WAk7G7ob75uwLhA+T9it8GOAVbqUdsgJRlMYRFWQChZnSa8E56UaWte8ri61YgFAXcgApfBiPk4X59AAW7G+5nRkYahrBUENEHYpBL93k8HSyNGXuk8b91OfTQwo54cOlHh1LDcytLAWO/yCdXjqZJP0fe60u1wN9bwd6T5ZOM10L477V9OKcTakb7F3LI6Suhyp8GODid82702YGPVCcXRNSzklX9BnfZ0q9UlfX3xgnH+lnJoqAqJfGaBn00mTyuVoKJSaf9W3bB0EJKNWAUiO9KmrfqxqZp26irVoaFzbqH22r5SoMNY1gqCGiDq2qAjj3a13IyUoz/b9xQSGNU6ntyQmONe++PdWVQPp24I/PpUBT+0wxQDo11GeqNFmjB6VaB5w/IAWc0zuBCwdMB2XX1lDbi+MZKu177WSo917UX7VMXxMiWrlcX9N7Vj+8FF4wDXaNEqTA4hEiPTPNI7jmfUjNa1dA43TtP6PaGuuHHEO1tO/1PwM1AaR+IFHb9ANrGWoawVBDRJ1K+RXgzC91ISf/lOlylSMQOriuJyegX8OxFwa9dErpjy+AY99Ij8eo5RlWE2Rulx6s2p50xVLvTUZNT052IzdubG+CUjrd5x5SL7DUCy9uXXjzx2vEUNMIhhoi6tQKz0u9HLUh5+pByI5eUi9HxAjAKwI4/r006Lckp66NSwDQZ4oUZLoMsJ2Bu6X5wJmaQcdnfpHCl6ComxQK08/GSVnzKtRrq2yibb3lLv6AR6gUWNxrQotrIK/WshKGmkYw1BAR1RBF6d4xGTUh58wv0n17GuPgAVw3URonEzrEpk9TUMfEOwoTEVHTBEE6ZeR/HXDjg9LluBdS63py8k9J42763A5EjpKe60VkB9hTQ0RERDbLnL/fjdyRh4iIiMj+MNQQERFRh8BQQ0RERB0CQw0RERF1CAw1RERE1CEw1BAREVGHwFBDREREHQJDDREREXUIDDVERETUITDUEBERUYfAUENEREQdAkMNERERdQgMNURERNQhMNQQERFRh6CSu4D2IooiAOkR5kRERGQfav9u1/4db06nCTXFxcUAgODgYJkrISIiInMVFxfD3d292TaC2Jro0wEYDAZcvHgRrq6uEATBotsuKipCcHAwzp07Bzc3N4tu29Z0pn0FOtf+cl87rs60v9zXjkcURRQXFyMoKAgKRfOjZjpNT41CoUDXrl2t+h1ubm4d+h9WfZ1pX4HOtb/c146rM+0v97VjaamHphYHChMREVGHwFBDREREHQJDjQVotVo8/fTT0Gq1cpdidZ1pX4HOtb/c146rM+0v97Vz6zQDhYmIiKhjY08NERERdQgMNURERNQhMNQQERFRh8BQQ0RERB0CQ00rvf766wgLC4ODgwNiY2Px22+/Ndv+888/R69eveDg4IC+ffvi+++/b6dKr11iYiIGDRoEV1dX+Pn5YdKkSTh+/Hiz62zYsAGCIJhMDg4O7VRx2zzzzDMNau/Vq1ez69jjcQWAsLCwBvsqCALmz5/faHt7Oq67du3C+PHjERQUBEEQsHnzZpPloihi+fLlCAwMhKOjI0aPHo2TJ0+2uF1zf+fbS3P7W1VVhccffxx9+/aFs7MzgoKCcM899+DixYvNbvNafhfaQ0vHdvbs2Q3qHjt2bIvbtcVj29K+Nvb7KwgCnn/++Sa3aavH1ZoYalrh008/xaJFi/D000/j4MGDiI6ORnx8PHJzcxttv3fvXkyfPh1z587FoUOHMGnSJEyaNAlHjhxp58rNs3PnTsyfPx/79u1DUlISqqqqMGbMGJSWlja7npubG7KysozT2bNn26nituvdu7dJ7b/88kuTbe31uALA/v37TfYzKSkJAHDHHXc0uY69HNfS0lJER0fj9ddfb3T5mjVr8Oqrr+LNN9/Er7/+CmdnZ8THx6OioqLJbZr7O9+emtvfsrIyHDx4EE899RQOHjyIr776CsePH8eECRNa3K45vwvtpaVjCwBjx441qfvjjz9udpu2emxb2tf6+5iVlYV3330XgiBg6tSpzW7XFo+rVYnUohtuuEGcP3++8bNerxeDgoLExMTERtvfeeed4rhx40zmxcbGiv/3f/9n1TotLTc3VwQg7ty5s8k269evF93d3duvKAt6+umnxejo6Fa37yjHVRRF8ZFHHhEjIyNFg8HQ6HJ7Pa4AxE2bNhk/GwwGMSAgQHz++eeN8woKCkStVit+/PHHTW7H3N95uVy9v4357bffRADi2bNnm2xj7u+CHBrb11mzZokTJ040azv2cGxbc1wnTpwojho1qtk29nBcLY09NS2orKxEamoqRo8ebZynUCgwevRopKSkNLpOSkqKSXsAiI+Pb7K9rSosLAQAeHl5NduupKQEoaGhCA4OxsSJE3H06NH2KM8iTp48iaCgIERERGDGjBnIzMxssm1HOa6VlZX48MMPMWfOnGYf7mrPx7VWRkYGsrOzTY6bu7s7YmNjmzxu1/I7b8sKCwshCAI8PDyabWfO74ItSU5Ohp+fH3r27IkHH3wQ+fn5TbbtKMc2JycHW7Zswdy5c1tsa6/H9Vox1LQgLy8Per0e/v7+JvP9/f2RnZ3d6DrZ2dlmtbdFBoMBjz76KIYMGYI+ffo02a5nz55499138fXXX+PDDz+EwWDATTfdhPPnz7djtdcmNjYWGzZswNatW7Fu3TpkZGTg5ptvRnFxcaPtO8JxBYDNmzejoKAAs2fPbrKNPR/X+mqPjTnH7Vp+521VRUUFHn/8cUyfPr3ZBx6a+7tgK8aOHYv3338f27dvx+rVq7Fz504kJCRAr9c32r6jHNv33nsPrq6umDJlSrPt7PW4tkWneUo3mWf+/Pk4cuRIi+dfBw8ejMGDBxs/33TTTYiKisJbb72FVatWWbvMNklISDC+79evH2JjYxEaGorPPvusVf8HZK/eeecdJCQkICgoqMk29nxcSVJVVYU777wToihi3bp1zba119+FadOmGd/37dsX/fr1Q2RkJJKTkxEXFydjZdb17rvvYsaMGS0O3rfX49oW7KlpgY+PD5RKJXJyckzm5+TkICAgoNF1AgICzGpvaxYsWIDvvvsOO3bsQNeuXc1aV61WIyYmBqdOnbJSddbj4eGBHj16NFm7vR9XADh79iy2bduG++67z6z17PW41h4bc47btfzO25raQHP27FkkJSU120vTmJZ+F2xVREQEfHx8mqy7Ixzb3bt34/jx42b/DgP2e1zNwVDTAo1Gg4EDB2L79u3GeQaDAdu3bzf5P9n6Bg8ebNIeAJKSkppsbytEUcSCBQuwadMm/PzzzwgPDzd7G3q9Hn/88QcCAwOtUKF1lZSUID09vcna7fW41rd+/Xr4+flh3LhxZq1nr8c1PDwcAQEBJsetqKgIv/76a5PH7Vp+521JbaA5efIktm3bBm9vb7O30dLvgq06f/488vPzm6zb3o8tIPW0Dhw4ENHR0Wava6/H1Sxyj1S2B5988omo1WrFDRs2iH/++af4wAMPiB4eHmJ2drYoiqI4c+ZM8YknnjC237Nnj6hSqcQXXnhBPHbsmPj000+LarVa/OOPP+TahVZ58MEHRXd3dzE5OVnMysoyTmVlZcY2V+/rihUrxB9//FFMT08XU1NTxWnTpokODg7i0aNH5dgFsyxevFhMTk4WMzIyxD179oijR48WfXx8xNzcXFEUO85xraXX68WQkBDx8ccfb7DMno9rcXGxeOjQIfHQoUMiAPGll14SDx06ZLza57nnnhM9PDzEr7/+Wjx8+LA4ceJEMTw8XCwvLzduY9SoUeJrr71m/NzS77ycmtvfyspKccKECWLXrl3FtLQ0k99jnU5n3MbV+9vS74JcmtvX4uJiccmSJWJKSoqYkZEhbtu2TRwwYIDYvXt3saKiwrgNezm2Lf07FkVRLCwsFJ2cnMR169Y1ug17Oa7WxFDTSq+99poYEhIiajQa8YYbbhD37dtnXDZ8+HBx1qxZJu0/++wzsUePHqJGoxF79+4tbtmypZ0rNh+ARqf169cb21y9r48++qjx5+Lv7y/eeuut4sGDB9u/+Gtw1113iYGBgaJGoxG7dOki3nXXXeKpU6eMyzvKca31448/igDE48ePN1hmz8d1x44djf67rd0fg8EgPvXUU6K/v7+o1WrFuLi4Bj+D0NBQ8emnnzaZ19zvvJya29+MjIwmf4937Nhh3MbV+9vS74JcmtvXsrIyccyYMaKvr6+oVqvF0NBQ8f77728QTuzl2Lb071gURfGtt94SHR0dxYKCgka3YS/H1ZoEURRFq3YFEREREbUDjqkhIiKiDoGhhoiIiDoEhhoiIiLqEBhqiIiIqENgqCEiIqIOgaGGiIiIOgSGGiIiIuoQGGqIqNNKTk6GIAgoKCiQuxQisgCGGiIiIuoQGGqIiIioQ2CoISLZGAwGJCYmIjw8HI6OjoiOjsYXX3wBoO7U0JYtW9CvXz84ODjgxhtvxJEjR0y28eWXX6J3797QarUICwvDiy++aLJcp9Ph8ccfR3BwMLRaLbp164Z33nnHpE1qaiquv/56ODk54aabbsLx48etu+NEZBUMNUQkm8TERLz//vt48803cfToUSxcuBB/+9vfsHPnTmObpUuX4sUXX8T+/fvh6+uL8ePHo6qqCoAURu68805MmzYNf/zxB5555hk89dRT2LBhg3H9e+65Bx9//DFeffVVHDt2DG+99RZcXFxM6vjHP/6BF198EQcOHIBKpcKcOXPaZf+JyLL4QEsikoVOp4OXlxe2bduGwYMHG+ffd999KCsrwwMPPICRI0fik08+wV133QUAuHz5Mrp27YoNGzbgzjvvxIwZM3Dp0iX89NNPxvUfe+wxbNmyBUePHsWJEyfQs2dPJCUlYfTo0Q1qSE5OxsiRI7Ft2zbExcUBAL7//nuMGzcO5eXlcHBwsPJPgYgsiT01RCSLU6dOoaysDLfccgtcXFyM0/vvv4/09HRju/qBx8vLCz179sSxY8cAAMeOHcOQIUNMtjtkyBCcPHkSer0eaWlpUCqVGD58eLO19OvXz/g+MDAQAJCbm9vmfSSi9qWSuwAi6pxKSkoAAFu2bEGXLl1Mlmm1WpNgc60cHR1b1U6tVhvfC4IAQBrvQ0T2hT01RCSL6667DlqtFpmZmejWrZvJFBwcbGy3b98+4/srV67gxIkTiIqKAgBERUVhz549Jtvds2cPevToAaVSib59+8JgMJiM0SGijos9NUQkC1dXVyxZsgQLFy6EwWDA0KFDUVhYiD179sDNzQ2hoaEAgJUrV8Lb2xv+/v74xz/+AR8fH0yaNAkAsHjxYgwaNAirVq3CXXfdhZSUFKxduxZvvPEGACAsLAyzZs3CnDlz8OqrryI6Ohpnz55Fbm4u7rzzTrl2nYishKGGiGSzatUq+Pr6IjExEadPn4aHhwcGDBiAJ5980nj657nnnsMjjzyCkydPon///vj222+h0WgAAAMGDMBnn32G5cuXY9WqVQgMDMTKlSsxe/Zs43esW7cOTz75JP7+978jPz8fISEhePLJJ+XYXSKyMl79REQ2qfbKpCtXrsDDw0PucojIDnBMDREREXUIDDVERETUIfD0ExEREXUI7KkhIiKiDoGhhoiIiDoEhhoiIiLqEBhqiIiIqENgqCEiIqIOgaGGiIiIOgSGGiIiIuoQGGqIiIioQ2CoISIiog7h/wEXecK//RgKowAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "loss_df.plot(kind=\"line\", x=\"epoch\", title=\"Losses\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAHHCAYAAABHp6kXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8ekN5oAAAACXBIWXMAAA9hAAAPYQGoP6dpAABhqElEQVR4nO3dd1zU9eMH8Ncd4449ZSnLiQNBUYaVIyk0UynLkTvbZZYNc6RW335aZvmtzFHmqByZpaZm4V44wYEDcQGyEdn77v37A7nkKygH3OT1fDzuUXzu/fnc+3Mfznvx/ryHRAghQERERGTgpLquABEREVFTYKghIiIio8BQQ0REREaBoYaIiIiMAkMNERERGQWGGiIiIjIKDDVERERkFBhqiIiIyCgw1BAREZFRYKghIiIio8BQQ0Ras2rVKkgkEsjlcqSkpNzzfN++fdGlSxfVzz4+PpBIJJBIJJBKpbC3t4e/vz9eeuklHDt2rM7XKS0txVdffYWQkBDY2dlBLpejffv2eOONN3D58uV7yp89exYTJ06Er68v5HI5rK2tERgYiPfffx/Xrl1rmpMnIo0z1XUFiKj5KSsrw/z58/HNN988sGxgYCDeeecdAEBBQQEuXryIjRs34vvvv8fbb7+NL7/8skb57OxsDBgwAKdOncKTTz6J5557DtbW1oiPj8f69euxfPlylJeXq8p///33ePXVV+Hs7IzRo0fDz88PlZWViIuLw5o1a7Bo0SKUlJTAxMSkad8EImp6gohIS1auXCkAiMDAQCGTyURKSkqN5/v06SM6d+6s+tnb21sMGjTonuMUFxeLyMhIAUB89913NZ4bNGiQkEql4rfffrtnv9LSUvHOO++ofj58+LAwMTERvXv3Fvn5+feULykpEbNmzRKVlZVqnysRaR9vPxGR1s2YMQMKhQLz589v0P4WFhb46aef4OjoiE8//RRCCADAsWPHsH37dkyaNAnDhg27Zz+ZTIYvvvhC9fNHH30EiUSCX375BTY2NveUl8vl+OSTT9hKQ2QgGGqISOt8fX0xbtw4fP/990hNTW3QMaytrfHUU08hJSUFFy5cAABs3boVADB27NgH7l9cXIw9e/agb9++aNWqVYPqQET6haGGiHRi5syZqKysxGeffdbgY1R3Kr569SoA4OLFiwAAf3//B+575coVVFZW1uiYXC0nJwfZ2dmqx919cIhIfzHUEJFOtG7dGmPHjsXy5cuRlpbWoGNYW1sDqOpADAD5+fkAUOutpP9VXbb6GP9btxYtWqge1S1ARKTfGGqISGdmzZqFysrKBvetKSwsBPBviLG1tQXwb8i5n+p9qo9xty1btiAqKqpG/xsi0n8MNUSkM61bt8aYMWMa3FoTFxcHAGjbti0AwM/PDwBw7ty5B+7btm1bmJqaqo5xtz59+iA8PBxBQUFq14mIdIehhoh0qrq1Rt2+NYWFhfjjjz/g6emJjh07AgAGDx4MAPj5558fuL+VlRX69u2L/fv31zoRIBEZHoYaItKpNm3aYMyYMVi2bBnS09PrtU9JSQnGjh2LnJwczJw5ExKJBAAQFhaGAQMG4IcffsDmzZvv2a+8vBzvvvuu6ufZs2dDoVBgzJgxtd6Gqh4qTkSGgTMKE5HOzZw5Ez/99BPi4+PRuXPnGs+lpKSoWl4KCwtx4cIFbNy4Eenp6XjnnXfw8ssv1yi/Zs0aPP7443j66acxePBg9O/fH1ZWVkhISMD69euRlpam6ivzyCOP4Ntvv8XkyZPRrl071YzC5eXluHz5Mn755ReYm5vDzc1NO28EETWKRPBPESLSklWrVmHixIk4ceIEevToUeO5CRMmYPXq1ejcubOqn4uPjw8SExMBABKJBDY2NvD09ESvXr3wwgsvIDg4uNbXKSkpwXfffYcNGzbg4sWLKC8vh7e3NwYMGIApU6agTZs2NcqfPn0aX331Ffbt24f09HSYmZmhTZs2eOyxx/Dqq6/eU56I9BNDDRERERkF9qkhIiIio8BQQ0REREaBoYaIiIiMAkMNERERGQWGGiIiIjIKDDVERERkFJrN5HtKpRKpqamwsbFRzT5KRERE+k0IgYKCAnh4eEAqvX9bTLMJNampqfD09NR1NYiIiKgBkpOT0apVq/uWaTahxsbGBkDVm2Jra6vj2hAREVF95Ofnw9PTU/U9fj/NJtRU33KytbVlqCEiIjIw9ek6wo7CREREZBQYaoiIiMgoMNQQERGRUWg2fWrqS6FQoKKiQtfVIA0xNzd/4JBAIiIyTAw1dwghkJ6ejtzcXF1XhTRIKpXC19cX5ubmuq4KERE1MYaaO6oDjYuLCywtLTlBnxGqnoAxLS0NXl5evMZEREaGoQZVt5yqA42Tk5Ouq0Ma1KJFC6SmpqKyshJmZma6rg4RETUhdi4AVH1oLC0tdVwT0rTq204KhULHNSEioqbGUHMX3o4wfrzGRETGq0GhZvHixfDx8YFcLkdISAiOHz9+3/IbN26En58f5HI5/P39sWPHjhrPz507F35+frCysoKDgwPCw8Nx7Ngx1fM3btzApEmT4OvrCwsLC7Rp0wZz5sxBeXl5Q6pPRERERkjtULNhwwZMnToVc+bMQUxMDAICAhAREYHMzMxayx85cgSjRo3CpEmTEBsbi8jISERGRiIuLk5Vpn379vj2229x7tw5HDp0CD4+Pnj88ceRlZUFALh06RKUSiWWLVuG8+fP46uvvsLSpUsxY8aMBp42ERERGR2hpuDgYPH666+rflYoFMLDw0PMmzev1vLDhw8XgwYNqrEtJCREvPzyy3W+Rl5engAgdu3aVWeZzz//XPj6+ta73tXHzMvLu+e5kpISceHCBVFSUlLv4+mL8ePHCwD3vP9//PGHqL68e/fuFQAEACGRSIStra0IDAwU7733nkhNTb3nmHl5eWLGjBmiQ4cOQiaTCVdXV9G/f3+xadMmoVQqVeUSEhLExIkThaenpzA3NxceHh7i0UcfFT///LOoqKi457iJiYninXfeEV27dhVOTk7C19dXDBs2TPz111+1ntvkyZNF9+7dhbm5uQgICKi1zJkzZ8TDDz8sZDKZaNWqlfjss8/u+34Z8rUmImqO7vf9/b/UaqkpLy/HqVOnEB4ertomlUoRHh6O6OjoWveJjo6uUR4AIiIi6ixfXl6O5cuXw87ODgEBAXXWJS8vD46OjnU+X1ZWhvz8/BoPYyWXy/HZZ5/h9u3b9y0XHx+P1NRUnDhxAtOmTcOuXbvQpUsXnDt3TlUmNzcXvXr1wpo1azB9+nTExMTgwIEDGDFiBN5//33k5eUBAI4fP47u3bvj4sWLWLx4MeLi4rBv3z688MILWLJkCc6fP1/jtX/66Sd06dIFKSkpmDt3Lnbv3o1169YhNDQUL730EsaNG1dr593nn38eI0aMqPV88vPz8fjjj8Pb2xunTp3CggULMHfuXCxfvlzdt5CItKRCoYRSKXRdDTJW6qSllJQUAUAcOXKkxvb33ntPBAcH17qPmZmZWLt2bY1tixcvFi4uLjW2/fnnn8LKykpIJBLh4eEhjh8/Xmc9EhIShK2trVi+fHmdZebMmaNqnbj7YYwtNU8++aTw8/MT7733nmp7bS01t2/frrFvcXGx6NChg3jooYdU21599VVhZWUlUlJS7nmtgoICUVFRIZRKpejYsaMICgoSCoWi1nrd3aKzdetW4erqKqKjo2stW1hYKCIiIsQbb7xR6/Nz5syptaXmu+++Ew4ODqKsrEy1bdq0aaJDhw61HkcIw77WRIautKJSPP7lfhHw0d9iTfQNUalQPngnavY01lKjSf369cPp06dx5MgRDBgwAMOHD6+1n05KSgoGDBiAZ599Fi+++GKdx5s+fTry8vJUj+TkZLXqI4RAcXmlTh5CqPdXjImJCf7v//4P33zzDW7evFnv/SwsLPDKK6/g8OHDyMzMhFKpxPr16zF69Gh4eHjcU97a2hqmpqY4ffo0Ll68iHfffbfOJQeqRxmVl5fjjTfewKpVqxAaGopDhw6hR48ecHV1xSuvvIJx48Zh8+bN+OWXX7B27VpcvXq13vWPjo5G7969a8wOHBERgfj4+Ae2WhGR9v15Jg3xGQXILa7Ah5vjMPibQzh5I0fX1SIjotbke87OzjAxMUFGRkaN7RkZGXBzc6t1Hzc3t3qVt7KyQtu2bdG2bVuEhoaiXbt2WLFiBaZPn64qk5qain79+qFXr14PvMUgk8kgk8nUOb0aSioU6DT77wbv3xgXPo6Apbl68yI+9dRTCAwMxJw5c7BixYp67+fn5wegaoQZANy+fVu1rS6XL18GAHTo0EG1LTMzE61bt1b9/Pnnn+O1117D/v370aJFCwwYMAC5ubkYOnQo3njjDTz11FP47bffMH/+fDz66KNwcnLCE088gaioKLRp06ZedU9PT4evr2+Nba6urqrnHBwc6nUcItI8IQRWHr4OAOjTvgVik27jQlo+nlkajae7tcQHA/3gYivXcS3J0KnVUmNubo6goCDs3r1btU2pVGL37t0ICwurdZ+wsLAa5QEgKiqqzvJ3H7esrEz1c0pKCvr27YugoCCsXLmSixLW4rPPPsPq1atx8eLFeu9T3SokkUjUbiG6m5OTE06fPo3Tp0/D3t5eNdz+3Llz6NWrF4CqkXBOTk746KOPEBgYiP/85z81Qom7uztbWIiM1KnE2zifmg+ZqRRfjQjE3nf7YmRPT0gkwO+xKXh04X78cPAaKhRKXVfVaKXllSCroOzBBQ2Y2sskTJ06FePHj0ePHj0QHByMRYsWoaioCBMnTgQAjBs3Di1btsS8efMAAFOmTEGfPn2wcOFCDBo0COvXr8fJkydVLS1FRUX49NNPMWTIELi7uyM7OxuLFy9GSkoKnn32WQD/Bhpvb2988cUXqqHeAOpsIWosCzMTXPg4QiPHrs9rN0Tv3r0RERGB6dOnY8KECfXapzoA+fj4wMnJCfb29rh06dJ992nXrh2Aqo7H3bp1A1B1C6xt27YAAFPTf3+tKisrYWFhAaDqVpSVlVWNY1lbW6v+PyYmBi+//HK96g3U3QpY/RwR6Y+VR24AACIDW8LRquqW8fxhXTEq2Auzt57HmeRc/Gf7Raw/kYy5gzvj4XbOOqyt8Tl+PQdjVxyDEMDEh33wer+2sJUb31Ixajd3jBgxAl988QVmz56NwMBAnD59Gjt37lQ1+yclJSEtLU1VvlevXli7di2WL1+OgIAA/Pbbb9i8eTO6dOkCoOrL8NKlSxg2bBjat2+PwYMH49atWzh48CA6d+4MoKpl58qVK9i9ezdatWoFd3d31UNTJBIJLM1NdfJozKy38+fPx59//lnn6LK7lZSUYPny5ejduzdatGgBqVSKkSNH4pdffkFqauo95QsLC1FZWYlu3brBz88PX3zxBZTK+/9V1bZtW9Xoqp49e+LSpUvYsmULlEoltmzZgjNnzqCkpAQLFixAcnIyhgwZUu9zDQsLw4EDB1TLXABVvysdOnTgrSciPZKWV4KdcekAgAkP+dR4LsDTHn+82gufP9MVTlbmuJJZiDErjuHVn0/h5u1iHdTW+MSnF+CF1SdQVqlEuUKJZfuvod+CffjlWCIqja1lTLN9lvWHMc9TM3To0Brbxo4dK+Ry+T2jn+Lj40VaWpq4fPmyWLdunejWrZtwcnIS58+fV+1769Yt4efnJ1q1aiVWr14tzp8/Ly5fvixWrFgh2rZtqxpBFR0dLaytrUVoaKjYsmWLuHz5sjh//rxYsmSJsLS0FF9//bUQoup9d3R0FPHx8UIIIVasWCEsLCyEiYmJCA0NFQMGDBBmZmZiyJAhIjk5ucZ5JCQkiNjYWPHyyy+L9u3bi9jYWBEbG6sa7ZSbmytcXV3F2LFjRVxcnFi/fr2wtLQUy5Ytq/P9MuRrTWSoPt95UXhP2yZGLDty33K5xeVizpY40Xr6duE9bZvoMGuH+HrXZVFSXqmlmhqflNvFIuTTXcJ72jbx9HeHxV/n0kS/L/YK72nbhPe0beLxL/eLA5czdV3N+1Jn9BNDjTDsL7raQs3169eFubl5nZPv2djYiICAAPHee++JtLS0e46Zm5srPvjgA9GuXTthbm4uXF1dRXh4uPjjjz9qDNWOj48X48ePF61atRKmpqbCzs5O9O7dWyxbtqzG5HufffaZCAgIENnZ2UIIIcrKylST/mVnZ4vi4uJaz61Pnz61Dsu/fv26qszdk++1bNlSzJ8//77vlyFfayJDVFJeKQI/+lt4T9sm/jp37783tbmYlieGLz2i+uJ95LM9Iup8eo1/f+jBbheVifCF+4T3tG2i/8J94nZR1R+E5ZUK8eOha6Lr3L9V7/HElcdFQkaBjmtcO3VCjUSIRvQONSD5+fmws7NDXl4ebG1tazxXWlqK69evw9fXF3I5e983NSEEXnvtNWzbtg2zZ89GZGQkWrRogaKiIuzcuROffPIJfvjhB/To0UPjdeG1JtKuX08k4/1NZ9HS3gL73+sLU5P69XoQQuDPs2n4dPsFZORXdW7t16EFZg/uDF9nqwfsTaUVCoxdcQwnbtyGm60cm17rhZb2FjXK5BaX47+7E/BTdCIqlQKmUgnGhHrjrfB2sLc0r+PI2ne/7+//xSFEpHESiQRLlizB4sWLsXr1ari5uUEmk8HW1haff/45Zs2apZVAQ0TaJYRQdRAeF+Zd70ADVP27MSTAA3ve6YtX+7aBmYkEe+OzEPHVAXy+8xKKyys1VGvDp1AKvLkuFidu3IaN3BSrnu95T6ABAHtLc8wZ3Bl/v90b4R1dUKkUWHXkBvos2IcfD103yJFobKkB/3rXtpKSEmRnZ8Pe3h42NjZafW1eayLtOXbtFkYsPwoLMxMcnd4fdpYNH21zLasQH/15AfsvV41+dbeTY8YTHfFkV/dGDa4wNkIIzNwch7XHkmBuKsWa54MR2tqpXvseSsjGf7ZfwKX0AgBAa2crzHiiI/p3dNHpe8yWGtJrFhYW8PT01HqgISLtWnWnleap7i0bFWgAoHULa6ya2BPfj+sBT0cLpOWVYvK6WIz6/iji73wJE/DNnitYeywJEgnw3xGB9Q40APBwO2dsf/MRzHvaH87W5riWXYQX1pzEmBXHcDHNMNZPZKghIqImd/N2Mf4+f2cYdy+fJjmmRCLBY51cEfV2H0x9rD1kplIcvZaDJ74+iI/+PI+8kooHH8SIrT+ehC+jqmZ8nzu4Mwb6qz/tiYlUglHBXtj7bl+80qcNzE2kOHzlFgZ9fRDTfz+r95P3MdTcpZnciWvWeI2JtOOno4lQCuChtk5o79q0rbJyMxO82b8ddk3tgwGd3aBQCqw8fAP9F+7DryeTm+Uq4LsuZGDGH1Vzgr3erw3GNzJI2sjN8MFAP+x+pw8G+btDKYB1x5PR74t9WLLvKkorFE1Q66bHUAPAzKyqWbS4mBM9Gbvq5RtMTBo2azMRPVhJuQLrj1ctIjyhl+8DSjecp6Mllo4Nwk+TgtGmhRWyC8vx/m9nMWTxIeyLz2w2f8TEJN3GG+tioBTAM0Gt8O7jHR68Uz15Olpi8eju2PhKGLq2skNhWSU+23kJ4V/ux/azaXr3HrOj8B1paWnIzc2Fi4sLLC0t2fHMCCmVSqSmpsLMzAxeXl68xmSQ/jyTilmb4zDjCT+M6Oml6+rUat3xJEz//Rw8HS2w791+MJFq/rNWXqnE6iM38N/dCSgsqxoZFezjiHcjOiDY11Hjr68rVzIL8czSI8gtrkDfDi3w/bgeMFNjlJk6lEqBP2JT8Pnfl1TD7Hv6OODDJzuhayt7jbwmoF5HYYaaO4QQSE9PR25urvYrR1ojlUrh6+sLc3P9mYOBqL5OJ+di+LJolFcqYS0zxZ53+ujdytZCCAxYdBDxGQWYNagjXniktVZf/1ZhGZbuv4rV0Ykor6wakty3Qwu8+3gHdGlpp9W6aFpGfime/u4IUnJLENDKDuteCoWludpLOqqtuLwSy/Zfw7IDV1FaUfUeP929Jd6P8IObXdP/PjLU1KK+b4pCoaixlhAZF3Nzc67wTgYpI78Ug785hMyCMphIJVAoBZ7u1hJfjgjUddVqOHIlG8/9cAyW5iaInt4fdha6WTQxLa8E3+y5gl9PJKPyTh+bJ/zdMPWx9mjrYvgjL/NLKzB8aTQupRfA19kKv70SBidrmVbrkJZXggU74/F7bAqAqsWYX+7TGpMfbdekrXMMNbVQ500hItInpRUKjFh+FGeSc9He1RpzBnfG6B+OAQB+eyUMPXz05/bKi2tOIupCBsaGeuOTyC66rg4SbxVh0a4EbD6dAiEAqQR4unsrTOnfDp6OlrquXoOUVSow4ccTiL52C87WMvz+ai94OenuXM4k5+KTbRdwMvE2HmnnjDXPBzfp7X2Gmlow1BCRIRJC4J1fz+D32BTYW5phy+sPwdvJCtN+O4sNJ5PRyd0Wf05+WCv9Vh4kOacYvRfshRDArqm99apFJD69AAv/icc/FzIAAGYmVUOX3+jXVu9u4d2PUikweX0stp9Ng7XMFOtfCtWL22pCCOw4l452rtZNPtqNk+8RERmJ7w9ew++xKTCRSvDdc93h7VS17tH7AzrAVm6KC2n5WHs8Sce1rLIm+gaEAB5p56xXgQYAOrjZYPm4Htj8+kN4pJ0zKhQCa6IT0XvBXsz/6xJyi8t1XcUHEkLgk+0XsP1sGsxMJFg6JkgvAg1QNYfQoK7uTR5o1MVQQ0Skp/bGZ2L+X5cAALOf7IRebZ1VzzlZy/DOnaG7X/wdj5wi3X4pF5VVYv2JqmHczz+kuWHcjRXoaY+fJoVg3YuhCPJ2QGmFEkv3X8Ujn+3F13eNnNJHyw9cw8rDNwAAXzwbgIfbOd9/h2aIoYaImp2kW8WIS8nTdTXu60pmId5cGwulAEYFe2JcmPc9ZUaHeMHPzQZ5JRVY8He8Dmr5r99jU1BQWgkfJ0v0ad9Cp3Wpj7A2TvjtlTD8OKEHOrrboqCsEl9GXUbvz/fih4PX9G5yud9jbmLenYA7a1BHDA1sqeMa6SeGGiJqVorLK/H0ksN48ptDWHHouq6rU6u8kgq8tOYkCsoq0dPHAR8N6VJrx0tTEyk+HlrVGXf9iSScu6mboCaEwKrDVe/l+F4+kOpB/576kEgkeNTPFdsnP4xvn+uG1s5WyCkqx3+2X0TfBfuw9liSXqxUvf9yFt7/7SwA4IWHfbU+TN6QMNQQUbOyKSYF2YVVt2o+2XYB83Zc1Ktp9RVKgcnrYnEtuwgednIsGRMEc9O6/6kO9nXE0EAPCAHM3hqnk3M5dCUbV7OKYGVugmeCWmn99RtLKpXgya4e+Oft3vh8WFd42MmRnl+KGX+cQ/iX+7HldIrOfkfO3szFqz+fQqVSYEiAB2Y80VEn9TAUDDVE1GwolQI/3mmdqZ5ldtmBa3hn4xnVRG26Nv+vizhwOQtyMym+H98DzvWYe2TGEx1hZW6C2KRcbIq5qYVa1rTqTj+PZ3t4wkaum3lpmoKpiRTDe3pi73t9MXdwJzhbmyPxVjGmrD+Ngf89iH/Op2t1WYAb2UWYuPIEissVeLitM754NsBgWsF0haGGiJqNvfGZuJ5dBBu5KVZO6IkFz3SFiVSCP2JTMGn1CZ13Et106ia+P1gVuhY+G4jOHvUb2eJqK8eb/dsBAD7beUmrq1XfyC7CnvhMAGj0Ior6QmZqggkP+eLA+/3wXkTVKLP4jAK89NMpRC4+jIX/xGPL6RScT83TWN+b7MIyjF95HLeKytHJ3RZLxnS/b4sdVdH8fMpERHrihzuB4bkQL1jJTPFsD08428jw2s8xOJiQjVHLj2LlxJ71ah1pajFJtzH996pVlt98tC0GdXVXa/+JD/liw8lkXMsqwqJdlzFncGdNVPMeq+8M4+7XoQV8na208praYmluitf7tcWYUG98f+Aafjx8HWdu5uHMXX2XJBLA08ES7Vys0fZ/Hg1ttSoqq8TElSeQeKsYno4WWPV8T4NuAdMmTr5HRM1CXEoenvzmEEykEhx8vx887C1Uz51OzsXzq04gp6gc3k6WWPN8sGo+GG1IzyvF4G8PIaugDI93csXSMUENus1w4HIWxv14HCZSCXa8+Qg6uGl2zpDCskqE/t9uFJZVYvXzwQYx6qkxsgrKsO1sKi5nFOJKZgESMguRW1x3q5ibrRztXP8NOe1cbNDWxRqOVnWvPVdeqcSk1SdwMCEbjlbm+O2VMLRuYa2J0zEY6nx/s6WGiJqF6r40g/zdawQaoGrukt9eCcO4H48j8VYxhi05gpUTguHfSvMTm5VWKPDyTyeRVVCGDq42+HJEYIP7TfRu3wIDOrth5/l0zNkah3Uvhmp0NfpNp26isKwSrVtY4ZG2xj9nSgsbGSbeNQePEAK3isqRkFGIK1mFuJJRgCtZhUjIKERmQRnS80uRnl+KgwnZNY7jZGWONi7WaHfn0dbFBu1crdHCWoYPNp3FwYRsWJiZ4McJPZt9oFEXQw0RGb2M/FL8eTYVAPDCI7VPDNe6hTV+f60XJvx4AhfS8jFyeTSWjg3CI+001/oghMAHm87izM082Fua4ftxPWAta9w/yzMHdcTe+EwcvZaDbWfTMDjAo4lqW5NSKbD6yA0AwAQDGsbdlCQSCZytZXC2liGsjVON5/JKKnAls6pF50pmIRIyC3ElsxA3b5fgVlE5bl3PwfHrOTX2sTAzQUmFomr26NHdEehpr8WzMQ4MNURk9NZE30CFQqCnjwO6trKvs5yLjRwbXg7FKz+fwuErtzBx5Ql88WwAIrtpZqKz5QeuYfPpVNWXWFMsSujpaInX+rbFV7su49PtF/GonwusGhmUanMgIQvXsotgIzPFsO6GN4xb0+wszBDk7YAgb4ca24vLK3E1swhXsgqqWnjuhJ3EnGKU3Ol0PP9pf/Tzc9FFtQ0eQw0RGbWScgV+OVa1NtKkhx88aZmN3AwrJwTj3Y1nsPVMKt7acBpZBWV4sXfTTni291Im5u+smiF27uBO6NWm6W7fvNynNX6LSUZyTgm+3XsF0wb4Ndmxq1VP1z+8p6dGQpOxsjQ3hX8ru3tubZZVKnAjuxgANN4XyphxfBgRGbVNMTeRW1wBL0dLPNbJtV77mJtKsWhEICY9XHWr6tMdF/HJtgtNNgHblcwCvLkuFkIAo4K9MCb03iUQGkNuZoLZT1aNfvrh4DVcyyps0uNfzSrE/stZkEhQ6/INpD6ZqQk6uNkw0DQSQw0RGa27J9ub+JAPTNTo9yGVSvDhk50w884MrisOXcdbG06jrLJx85LkFVfgxTWnUFBWiWAfR3w0pLNGOvOGd3RB3w4tUKEQ+OjPC006adyaO31p+vu5aHWUGNGDMNQQkdHaG59Z1e9DXjUnTUO82Ls1Fo0IhKlUgq1nUvH8qhMoKG3Y5HaVCiXeWBeD69lFaGlvge80OKGaRCLBnMGdYW4ixf7LWdh1MbNJjptfWoHfTlXNWjyhl/6uxk3NE0MNERmt6gUrnwv2atSooshuLfHjhJ6wMjfB4Su3MGLZUWQWlKp9nPl/XVIN110+Lkjjk/z5Olth0p3RXh9vO98ks9/+dvImisoVaOdijYfaOj14ByItYqghIqN0PjUPR67egolU0iTT9/du3wLrXwqDs7U5LqTlY9iSI2r1Vfnt1E38cCdkfTk8oN5LIDTWG/3aws1WjuScEizbf61Rx1IoBVZH3wAATHjIR6Nz4BA1BEMNkRERQiAzvxRHrmTj56OJ2BmXjpJyzaxNo++qW2meqGWyvYbyb2WHTa/2greTJZJzSvDM0micTs594H4xSbcxo3oJhP7tMNBfvSUQGsNKZoqZg6r6BX237wqSc4obfKx98ZlIvFUMW7kpntLQMHeixuA4PCIDVKlQIimnGFezilTzXFzNqnoUlNZclNHCzASP+rlgoL8b+nXQzJwl+iYzvxR/nqmabK96BFNT8XaywqZXe2HiyhM4l5KHUcuP4rsx3dGvQ+3ziqTlleDln06hXKFERGdXvHVn4UlterKrO345loij13Lw6faLWDo2qEHHWXWng/DIYC9Ymhv/7xEZHv5WEumx6om6rmb9G1yuZBbixq0iVChqH80ilQBejpbwdbbC5YxCpOSWYPu5NGw/lwaZqRR92rfAE/7ueLSjC2yNdJG8NdGJqsn2NDErq7O1DOtfqpqk72BCNl5YfRKfDeuKZ4JqTkJXWqHAS2tOIaugDH5uNvhyeMOXQGgMiUSCj4Z0wRNfH8TO8+k4cDkLvdVcpykhowAHE7IhlQBjm3gIOlFTYagh0jEhBLILy+8JLteyipCSW1LnfnIzKdq0qFoo7+7/+jhbQmZqojp2XEo+dsSl4a9zabhxqxj/XMjAPxcyYG4ixSPtnDGgixse6+QKe8u6F9kzJCXlCvx8LBFA07fS3M1KZooV43ti2qaz+CM2Be9uPIPMglK82qcNJBIJhBCYtukszqXkweHOEgi6bCXr4GaD8WE++PHwdcz98zx2Tumt1sir6r404R1d4enY+JmPiTSBoYZIR7aeScXqIzdwJbMQeSV1DxGuXvzu3+BihbYu1vCws3jgX/0SiUQ1e+n7ER1wMa0Af8WlYce5NFzNKsLuS5nYfSkTplIJerV1xhN3Ao6ThkflaNLvsVWT7Xk6WuCxTm4afS1zUykWPhsAF1sZlu2/hs93xiMzvwwfPtkJyw9cw5bTqTCVSvDd6CC9CAJvPdYOW8+k4FpWEVYevo6X+7Sp1355xRXYdCoFAGos6EikbySiKWdk0mPqLF1OpGlxKXkYuvgwFHdmqJVIAE8HyxqhpU2LqoeDlWZaUBIyCrDjXDr+ikvDpfQC1XapBAht7YSB/u6I6OwKFxu5Rl5fE5RKgfCv9uNaVhHmDO6k1S/gFYeu45NtFwAAwb6OOHEjB0IAn0R20avbNRtPJuO9387CytwEu9/pCze7B1/f7w9cw6c7LsLPzQZ/TXmEo55Iq9T5/maoIdIyhVLg6e8O48zNPIR3dME7j3eAr7MV5GYmOqvTtaxC/BVXFXDiUvJV2yUSoKe3Iwb6u2FAFze42zXNKCJN2XMpA8+vOgkbmSmiZ/Rv9IrX6tp6JhXv/Hpa1d9pdIgXPn3KX6t1eBClUmDY0iOITcrF0EAP/Hdkt/uWVygF+izYi5u3SzD/aX+MDPbSUk2JqjDU1IKhhvTFqsPXMffPC7CRmWL3O33gYqtfLSFJt4qx83wadpxLv2e4cncvewzs4o4BXdz04nbK/xr9w1EcvnILL/VujRl3ljfQtiNXsvHm+tMI9LTDd6ODNDZjcGOcu5mHIYsPQQhgw0uhCGld9yR6/5xPx0s/nYK9pRmiP+gPC3PdhW9qnhhqasFQQ/ogLa8Ej315AIVllXp3W6I2Kbkl2BmXjp1xaTiZeBt3/2vRtZUdZg3qhGBfR91V8C4XUvPxxNcHYSKV4MD7/dCyieamaQiFUkAqgV7fppnxxzmsPZYEPzcbbJv8MExNag9fz31/FEeu3sIrfdrgg4FNv9o30YOo8/2tf39CEBmxj7ZeQGFZJbp52WO0ATTjt7S3wKSHfbHxlV44Or0/Ph7aGaGtHSGVAGdv5mHS6hO4nl2k62oC+HeyvYFd3HQaaADARCrR60ADAO893gF2Fma4lF6AX44l1VrmUnq+albmsVyNmwwAQw2RlkRdyMDO8+kwlUow72l/ncxX0hiutnKMC/PB+pfCcHxmOIK8HVBQWomX1pxEYVnlgw+gQZn5pdh6pmp0zguPtNZpXQyFg5U53o3oAABY+E88sgvL7imz+s5kexGdXXUeFInqg6GGSAsKyyoxe0scgKovXT83w74F6mwtw5LR3eFiI0NCZiHe/+0MdHkn+6ejVZPt9fDWzGR7xuq5YC90crdFfmklFuyMr/Hc7aJy/BFbFRS5GjcZCoYaIi34Kuoy0vJK0crBAlN0ME2+JrjYyrFkTBDMTCTYcS4dSxu5WGJDlZQr8PNRzU+2Z4xMpBJ8PLQzAODXU8k1OoZvOJmM0golOrnboqePg45qSKQehhoiDYtLycPKw1X9Pf4T2cWoRo8EeTvgoyFdAACf/30J+y9nab0Ov8fexO07k+093lmzk+0Zox4+jni6e0sIAczZEgelUqBSocRP0VVBkatxkyFhqCHSoEqFEtN/PwelAAYHeKBvHYseGrLnQrwwKtgTQgBvrotF0q2GrwKtLqVS4Mc7HYQn9vKFiYH1U9IXHwz0g7XMFGdu5mHjqWREXchASm4JHK3MMSTAQ9fVI6o3hhoiDVoTnYhzKXmwkZviwyd1M2+KNswd0hmBnvbIK6nASz+dRHG5djoO77+chatZRbCRmWJ4T0+tvKYxcrGR463wqtuin+2Mx9L9VwFU9bnR5aSQROpiqCHSkNTcEiz8p6rz5QcD/QxquQF1yUxNsHRMEJytZbiUXoBpm85ppePwD4eq+vGMDPbU+uzBxmZ8Lx+0dbFGTlE5ztzMg4lUgjF6Po8S0f9iqCHSkDlbz6OoXIEgbweM6qn/c9I0lpudHEvGdIepVII/z6Tih4PXNfp6F9PycfhK1Rwq43v5aPS1mgMzEyk+GtJZ9fPALm71WheKSJ8w1BBpwN/n0xF1IQOmUgn+7ynDm5OmoXr6OGL24E4AgHl/XcShhGyNvVb1ZHsDurihlYP+LdlgiB5q64yRPT0hN5PilXqu4E2kTxhqiJpYYVkl5mw5DwB4qXdrdHCz0XGNtGtsqDeeCWoFpQAmr4tBck7TdxzOLCjF1tOpAIAXOIy7Sc172h9xcyPQpaWdrqtCpDaGGqImtvCfeKTnl8LL0RJvGsmcNOqQSCT4T2QXdG1lh9vFFXj5p1MoKVc06Wv8HJ2IcoUSQd4O6ObFOVSakkQiqXMdKCJ9x99coiZ09mauamr5/0R2abYjR+RmVR2HnazMcSEtH9N/P9tkHYdLKxT4iZPtEVEtGGqImsjdc9IMDfRA7/YtdF0lnfKwt8C3z3WHiVSCzadTsfLwjSY57u8xKbhdXIFWDhZ4vJNrkxyTiIwDQw1RE1l15AbOp+bDVm6KWYM66bo6eiGsjRNmPlE1P8+nOy4i+uqtRh1PqRT48c7szBMf8uVtEiKqgf8iEDWBlNwSfBl1GQAw44mOaGEj03GN9MfEh3zwVLeWUCgF3lgbg5TckgYfa39CFq5kFsJaZorhPVo1YS2JyBgw1JDByswvxU/RN3A5o0Cn9RBCYPbmOBSXK9DTxwHDe3Bm27tJJFXD2jt72OJWUTle+ekUSisa1nF4xZ25b0b29ISN3Kwpq0lERoChhgyKEALRV2/h9V9i0Gv+Hny45Tye/PoQfjh4DUql5mewrc3f59Ox+1ImzEya15w06rAwr+o47GBphnMpeZj5R5zaHYcvpuXj0JVsSCVViywSEf0vhhoyCPmlFVh1+Doe++oARn1/FNvPpaFSKdDS3gLlCiX+s/0ixqw4htRG3NpoiILSCszZWjUnzcu926Cda/Oak0Ydno6W+Pa57pBKgE0xN1UjmOqreuHKgf7unGyPiGrFUEN6LS4lD9N/P4uQT3dj7p8XcCWzEJbmJhgd4oW/pjyCQ9P64f+e8oeFmQmOXL2FAYsO4M8zqVqr3xd/xyMjvww+TpZ449G2WntdQ/VQW2dMH1jVcfjjPy/g+PWceu2XWVCKLXcm2+MwbiKqC1eAI71TWqHAjnNp+OloImKTclXb27taY0yoN57q1rJGf4rnQrwQ2toRb284jTM38zB5XSz2XMrER0M7w1aD/S5OJ+dizZ3Whv9E+jfbOWnU9cIjvjibkoc/z6TitV9isG3yww9cY6h6sr3uXvbozsn2iKgODDWkNxJvFWHtsST8ejIZt4srAABmJhIM6OKOMSFeCPZ1hERSe3+V1i2s8durvfDN7gR8u/cK/ohNwfHrOfhyeABCWjs1eV2r56QRAniqW0s83M65yV/DWEkkEnw2zB8JGQW4lF6AV34+hQ0vh0JmWnsoLK1Q4OdjSQCAFx5prc2qEpGBYaghnVIoBfZcysTPRxOx/3KWantLews8F+KF4T086z082sxEiqmPd0CfDi3w1obTSM4pwcjvj+KVPm3wdnh7mJs23d3WHw9fx8W0fNhbmmHmoI5NdtzmwtLcFMvH9sDgbw/hdHIu5m49j3lPd6217B+xKcgpKkdLe062R0T3x1BDOpFVUIYNJ5Kw7niyat4SiQTo3a4FxoZ6o5+fC0waOIooyNsRf03pjY+2nsfGUzexZN9VHLichf+ODERbl8Z35E3OKcZXUQkAgBkDO8LZmnPSNISXkyW+GdUNE1Yex7rjyfBvaY/nQrxqlBFCqFbjnviQDyfbI6L7YqghrRFC4Pj1HPx0NBF/n09HhaJqSK+DpRmG9/DEcyFe8HayapLXspaZYsGzAejf0QUf/H4O51PzMejrQ5g5qCPGhnrXeRurPucwe0scSioUCPZ1xLOcAK5Rerdvgfci/PDZzkuYszUOHdxsEOT9b5+Z/Zf/nWxvRE/O/0NE98dQQxpXUFqBP2JT8PPRRFzOKFRt7+5ljzGh3njC311jnWwHdHFHNy8HvLvxDA4mZGP2lvPYcykTnz/TFS429++cWpu/4tKxNz7rzpw0XRocjuhfr/RpjXMpudhxLh2v/nwK2yY/DBfbqmtT3UozgpPtEVE9SERTLZ2r5/Lz82FnZ4e8vDzY2trqujrNglIpsOCfeKw+cgPF5VUzyFqYmSCyW0uMCfVCZw87rdZlTfQN/N9fl1BeqYSjlTnmPe2PiM5u9T5GfmkFwhfuR2ZBGd7s3w5TH2uvwRo3L0VllXjqu8O4nFGIIG8HrHsxFNeyCzFg0UFIJcD+9/rB05Fz0xA1R+p8f7OlhjTmx8PXsWTfVQBAWxdrjAnxwtNBrTQ6zLouUqkEEx7yRa+2zpiy/jQupuXj5Z9OYWRPT3z4ZCdYyR78UViwMx6ZBWXwdbbCa33baKHWzYeV7N+Ow6cSb+PjbedRXqkEAAzs4s5AQ0T1wpYa0oi4lDw89d1hVCgEZg3qiEkP++rNrZqySgW+jLqM5QeuQQjA28kSX40IvO/8JzFJtzFsyREIAax9IQS92nIItybsuZSBSatPQghAKgGUAtj0aq8a/WyIqHlR5/ubQwmoyRWVVWLyulhUKAQe7+SqV4EGAGSmJpg+sCPWvhAKDzs5Em8V49ml0Vi06zIqFcp7ylcolJhxZ06ap7u3ZKDRoEf9XDE1vOq2nlIA3bzsGWiIqN4aFGoWL14MHx8fyOVyhISE4Pjx4/ctv3HjRvj5+UEul8Pf3x87duyo8fzcuXPh5+cHKysrODg4IDw8HMeOHatRJicnB6NHj4atrS3s7e0xadIkFBYWgvTPnK3ncT27CO52cnz+TFe9CjR3C2vjhL/e6o2hgR5QKAUW7UrAM0ujcSO7qEa5FYeu41J6ARwszTBrUCcd1bb5eL1fWwzydwcATObSE0SkBrVDzYYNGzB16lTMmTMHMTExCAgIQEREBDIzM2stf+TIEYwaNQqTJk1CbGwsIiMjERkZibi4OFWZ9u3b49tvv8W5c+dw6NAh+Pj44PHHH0dW1r+TsY0ePRrnz59HVFQUtm3bhgMHDuCll15qwCmTJm05nYLfTt2EVAJ8NSIQ9pbmuq7SfdlZmOG/I7vhvyMDYSM3xenkXDzx9UGsP54EIQSSc4qxaNdlAMCMJzrC0Uq/z8cYSKUSfDOqG47P6I9H/TjZHhHVn9p9akJCQtCzZ098++23AAClUglPT09MnjwZH3zwwT3lR4wYgaKiImzbtk21LTQ0FIGBgVi6dGmtr1F9/2zXrl3o378/Ll68iE6dOuHEiRPo0aMHAGDnzp144okncPPmTXh4eDyw3uxTo3lJt4rxxNcHUVhWaZCjg1JySzB1w2kcu7PI4mOdXFFSrsChK9kIbe2IdS+G6m2rExGRsdJYn5ry8nKcOnUK4eHh/x5AKkV4eDiio6Nr3Sc6OrpGeQCIiIios3x5eTmWL18OOzs7BAQEqI5hb2+vCjQAEB4eDqlUes9tqmplZWXIz8+v8SDNqVAoMXl9LArLKtHTxwFvGuBtg5b2Flj7YiimD/SDmYkEURcycOhKNsxNpPj0KX8GGiIiPadWqMnOzoZCoYCra80mYVdXV6Snp9e6T3p6er3Kb9u2DdbW1pDL5fjqq68QFRUFZ2dn1TFcXFxqlDc1NYWjo2Odrztv3jzY2dmpHp6enI1Uk76MuowzybmwlZti0chuBjudvYlUgpf7tMHm1x9COxdrAMCb/duiTQtrHdeMiIgeRG++efr164fTp0/jyJEjGDBgAIYPH15nP536mD59OvLy8lSP5OTkJqwt3e1QQjaW7q+aj+azYV3R0t5CxzVqvM4edtj25sPY+dYjeL2f4bU6ERE1R2qFGmdnZ5iYmCAjI6PG9oyMDLi51T4zq5ubW73KW1lZoW3btggNDcWKFStgamqKFStWqI7xvwGnsrISOTk5db6uTCaDra1tjQc1vVuFZXj719MQAnguxAsD74xaMQYyUxP4udnythMRkYFQK9SYm5sjKCgIu3fvVm1TKpXYvXs3wsLCat0nLCysRnkAiIqKqrP83cctKytTHSM3NxenTp1SPb9nzx4olUqEhISocwrUhIQQeHfjGWQVlKGdizU+5HBnIiLSIbWXSZg6dSrGjx+PHj16IDg4GIsWLUJRUREmTpwIABg3bhxatmyJefPmAQCmTJmCPn36YOHChRg0aBDWr1+PkydPYvny5QCAoqIifPrppxgyZAjc3d2RnZ2NxYsXIyUlBc8++ywAoGPHjhgwYABefPFFLF26FBUVFXjjjTcwcuTIeo18Is1YefgG9sZnwdxUim+e6wYLc80sSklERFQfaoeaESNGICsrC7Nnz0Z6ejoCAwOxc+dOVWfgpKQkSKX/NgD16tULa9euxaxZszBjxgy0a9cOmzdvRpcuXQAAJiYmuHTpElavXo3s7Gw4OTmhZ8+eOHjwIDp37qw6zi+//II33ngD/fv3h1QqxbBhw/D111839vypgeJS8jD/r0sAgA8HdYSfG2/vERGRbnHtJ1JbUVklBn9zCNeyi/BYJ1csHxvEfidERKQRXPuJNOqjP8/jWnYR3Gzl+HyY/i6DQEREzQtDDall65lU/HryJiQSYNHIQDhw2QAiItITDDVUb8k5xZj5+zkAwOR+bRHa2knHNSIiIvoXQw3VS4VCiTfXx6KgrBJB3g54s387XVeJiIioBoYaqpdFuy4jNikXNnJT/HdkoMEug0BERMaL30z0QEeuZOO7fVXLIMx/uitaOVjquEZERET3Yqih+8opKsdbG6qWQRgV7IlBXY1nGQQiIjIuDDVUJyEE3tt4BpkFZWjrYo3ZT3Z+8E5EREQ6wlBDdVp95AZ2X8qEuakUX4/kMghERKTfGGqoVhdS8/F/O6qWQZj5REd08uAszEREpN8YaugexeWVmLwuBuUKJcI7umJcmLeuq0RERPRADDV0j4//vICrWVXLICx4hssgEBGRYWCooRq2nU3F+hPJkEiAr0ZwGQQiIjIcDDWkkpxTjOl3lkF4o19bhLXhMghERGQ4GGoIAFCpUGLK+lgUlFaiu5c9pnAZBCIiMjAMNQQAWLQrATGqZRC6cRkEIiIyOPzmIhy5mo3F+64AqFoGwdORyyAQEZHhYahp5nKKyvH2nWUQRvbkMghERGS4GGqauTlbzyMjvwxtWlhh9uBOuq4OERFRgzHUNGMpuSXYfjYVAPDfkd1gaW6q4xoRERE1HENNM7b+eBKUAghr7YQuLe10XR0iIqJGYahppioUSqw/kQwAGB3qpePaEBERNR5DTTMVdSEDWQVlcLaW4fFObrquDhERUaMx1DRTvxxLBACM6NkK5qb8NSAiIsPHb7Nm6FpWIQ5fuQWJBBgVzFtPRERkHBhqmqG1x5IAAP06uKCVAyfaIyIi48BQ08yUVijwW8xNAMAYdhAmIiIjwlDTzGw/m4bc4gq0tLdAn/Yuuq4OERFRk2GoaWaqOwg/F+IFE6lEx7UhIiJqOgw1zciF1HzEJOXCVCrBsz1a6bo6RERETYqhphmpbqWJ6OwGFxu5jmtDRETUtBhqmonCskpsjk0BwBmEiYjIODHUNBObY1NQVK5A6xZWCGvtpOvqEBERNTmGmmZACIGfj1bdehod4g2JhB2EiYjI+DDUNAMxSbm4lF4AmakUw7q31HV1iIiINIKhphn45U4rzeAAD9hbmuu4NkRERJrBUGPkbheVY9u5NADA6BB2ECYiIuPFUGPkNsXcRHmlEp09bBHoaa/r6hAREWkMQ40RUyoFfrmzeCU7CBMRkbFjqDFi0ddu4Xp2Eaxlphga6KHr6hAREWkUQ40Rqx7G/VS3lrCSmeq4NkRERJrFUGOkMvJL8c+FDACcQZiIiJoHhhojteFEMhRKgR7eDvBzs9V1dYiIiDSOocYIVSqUWHf8TgdhttIQEVEzwVBjhPbGZyEtrxQOlmYY2MVd19UhIiLSCoYaI/TLsaoOws/28ITczETHtSEiItIOhhojk5xTjP2XswAAzwXz1hMRETUfDDVGZu3xJAgBPNLOGT7OVrquDhERkdYw1BiRskoFfj2RDKBqBmEiIqLmhKHGiPx9PgO3isrhaitDeEcXXVeHiIhIqxhqjMgvd2YQHtnTC6YmvLRERNS88JvPSCRkFODY9RyYSCUYGeyp6+oQERFpHUONkahejbu/nwvc7Sx0XBsiIiLtY6gxAsXlldgUcxMAMDqUHYSJiKh5YqgxAtvOpKGgtBJejpZ4pK2zrqtDRESkEww1RuDnOzMIPxfiBalUouPaEBER6QZDjYE7ezMXZ2/mwdxEimeDWum6OkRERDrDUGPgfjla1UF4oL8bnKxlOq4NERGR7jDUGLC8kgpsPZMKgDMIExERMdQYsD9ibqKkQoH2rtbo6eOg6+oQERHpFEONgRJCqOamGR3iDYmEHYSJiKh5Y6gxUCdu3EZCZiEszEzwVPeWuq4OERGRzjHUGKif76zzNDTQA7ZyMx3XhoiISPcYagxQdmEZ/opLA8AOwkRERNUYagzQxpM3UaEQCGhlB/9WdrquDhERkV5gqDEwSqXA2uNVt564zhMREdG/GGoMzIGELCTnlMBGborBXT10XR0iIiK9wVBjYKqHcQ/r3goW5iY6rg0REZH+YKgxIKm5Jdh9MQMAMCbUS8e1ISIi0i8MNQZk/YlkKAUQ4uuIti42uq4OERGRXmlQqFm8eDF8fHwgl8sREhKC48eP37f8xo0b4efnB7lcDn9/f+zYsUP1XEVFBaZNmwZ/f39YWVnBw8MD48aNQ2pqao1jXL58GUOHDoWzszNsbW3x8MMPY+/evQ2pvkGqUCix/njVracx7CBMRER0D7VDzYYNGzB16lTMmTMHMTExCAgIQEREBDIzM2stf+TIEYwaNQqTJk1CbGwsIiMjERkZibi4OABAcXExYmJi8OGHHyImJga///474uPjMWTIkBrHefLJJ1FZWYk9e/bg1KlTCAgIwJNPPon09PQGnLbh2X0xA5kFZXC2NkdEZzddV4eIiEjvSIQQQp0dQkJC0LNnT3z77bcAAKVSCU9PT0yePBkffPDBPeVHjBiBoqIibNu2TbUtNDQUgYGBWLp0aa2vceLECQQHByMxMRFeXl7Izs5GixYtcODAATzyyCMAgIKCAtja2iIqKgrh4eEPrHd+fj7s7OyQl5cHW1tbdU5ZL4z54RgOXcnGa33b4P0BfrquDhERkVao8/2tVktNeXk5Tp06VSNESKVShIeHIzo6utZ9oqOj7wkdERERdZYHgLy8PEgkEtjb2wMAnJyc0KFDB6xZswZFRUWorKzEsmXL4OLigqCgoFqPUVZWhvz8/BoPQ3U9uwiHrmRDIgFGBbODMBERUW3UCjXZ2dlQKBRwdXWtsd3V1bXO20Dp6elqlS8tLcW0adMwatQoVSKTSCTYtWsXYmNjYWNjA7lcji+//BI7d+6Eg4NDrceZN28e7OzsVA9PT091TlWvrLvTl6Zv+xbwdLTUcW2IiIj0k16NfqqoqMDw4cMhhMCSJUtU24UQeP311+Hi4oKDBw/i+PHjiIyMxODBg5GWllbrsaZPn468vDzVIzk5WVun0aRKKxTYeLKq7lzniYiIqG6m6hR2dnaGiYkJMjIyamzPyMiAm1vtnVfd3NzqVb460CQmJmLPnj017pvt2bMH27Ztw+3bt1Xbv/vuO0RFRWH16tW19uWRyWSQyWTqnJ5e2hmXjtvFFfCwk6Ofn4uuq0NERKS31GqpMTc3R1BQEHbv3q3aplQqsXv3boSFhdW6T1hYWI3yABAVFVWjfHWgSUhIwK5du+Dk5FSjfHFxcVVlpTWrK5VKoVQq1TkFg3Ps+i0AwJDAljCRSnRcGyIiIv2lVksNAEydOhXjx49Hjx49EBwcjEWLFqGoqAgTJ04EAIwbNw4tW7bEvHnzAABTpkxBnz59sHDhQgwaNAjr16/HyZMnsXz5cgBVgeaZZ55BTEwMtm3bBoVCoepv4+joCHNzc4SFhcHBwQHjx4/H7NmzYWFhge+//x7Xr1/HoEGDmuq90EuJt6oCXTsXax3XhIiISL+pHWpGjBiBrKwszJ49G+np6QgMDMTOnTtVnYGTkpJqtKj06tULa9euxaxZszBjxgy0a9cOmzdvRpcuXQAAKSkp2Lp1KwAgMDCwxmvt3bsXffv2hbOzM3bu3ImZM2fi0UcfRUVFBTp37owtW7YgICCgoeduEKpDjY8zOwgTERHdj9rz1BgqQ5ynpqxSAb8Pd0II4MTMcLSwMfw+QkREROrQ2Dw1pF3JOSUQArA0N4Gztbmuq0NERKTXGGr0WFJOEQDA28kKEgk7CRMREd0PQ40eq+5P480J94iIiB6IoUaPqUKNE0MNERHRgzDU6LHEW//efiIiIqL7Y6jRY4k5bKkhIiKqL4YaPaVQCiQz1BAREdUbQ42eSssrQYVCwMxEAnc7C11Xh4iISO8x1Oip6k7Cng6WXPOJiIioHhhq9BRHPhEREamHoUZPJeZw5BMREZE6GGr0VGJ2VUuNFyfeIyIiqheGGj1VPZybq3MTERHVD0ONHhJCIOnOxHtejrz9REREVB8MNXoou7AcReUKSCSApyOHcxMREdUHQ40eql6d28POAjJTEx3XhoiIyDAw1Oih6uHc7CRMRERUfww1eujGLXYSJiIiUhdDjR5iJ2EiIiL1MdTooRucTZiIiEhtDDV6KImrcxMREamNoUbP5JdWIKeoHACXSCAiIlIHQ42eSbpz68nZ2hzWMlMd14aIiMhwMNToGQ7nJiIiahiGGj3D1bmJiIgahqFGz1Svzs1OwkREROphqNEz/7bUMNQQERGpg6FGz/zbp4a3n4iIiNTBUKNHSisUSM8vBQD4sKWGiIhILQw1euTm7WIIAVjLTOFoZa7r6hARERkUhho9cuOuTsISiUTHtSEiIjIsDDV6JJHLIxARETUYQ40e4ercREREDcdQo0eqV+dmJ2EiIiL1MdTokerVub0YaoiIiNTGUKMnKhVKJKv61PD2ExERkboYavREWl4pKpUC5qZSuNvKdV0dIiIig8NQoyeqZxL2dLCAVMrh3EREROpiqNETN25xdW4iIqLGYKjRE0mco4aIiKhRGGr0RGJ1S40jQw0REVFDMNToieo+Nd7OvP1ERETUEAw1ekAI8e/tJ7bUEBERNQhDjR7IKixDcbkCUgnQyoGhhoiIqCEYavRA9a0nD3sLmJvykhARETUEv0H1gKo/DUc+ERERNRhDjR5I5OrcREREjcZQowcSuTo3ERFRozHU6IFETrxHRETUaAw1eiCRSyQQERE1GkONjuWVVCC3uAIA4MU5aoiIiBqMoUbHku70p3G2lsFKZqrj2hARERkuhhodq16dm52EiYiIGoehRseql0fwYqghIiJqFIYaHbuRXb06NzsJExERNQZDjY5VD+f2cWZLDRERUWMw1OhYdUdhjnwiIiJqHIYaHSqtUCA9vxQA4MM5aoiIiBqFoUaHqjsJ28hNYW9ppuPaEBERGTaGGh26e3VuiUSi49oQEREZNoYaHeLyCERERE2HoUaHVC017CRMRETUaAw1OnRD1VLDUENERNRYDDU6VN1RmLefiIiIGo+hRkcqFEqk3C4BwJYaIiKipsBQoyOpuSWoVArITKVwtZHrujpEREQGj6FGRxLvmklYKuVwbiIiosZiqNGRxJx/56ghIiKixmOo0ZHEbM5RQ0RE1JQaFGoWL14MHx8fyOVyhISE4Pjx4/ctv3HjRvj5+UEul8Pf3x87duxQPVdRUYFp06bB398fVlZW8PDwwLhx45CamnrPcbZv346QkBBYWFjAwcEBkZGRDam+XmBLDRERUdNSO9Rs2LABU6dOxZw5cxATE4OAgABEREQgMzOz1vJHjhzBqFGjMGnSJMTGxiIyMhKRkZGIi4sDABQXFyMmJgYffvghYmJi8PvvvyM+Ph5DhgypcZxNmzZh7NixmDhxIs6cOYPDhw/jueeea8Ap6weuzk1ERNS0JEIIoc4OISEh6NmzJ7799lsAgFKphKenJyZPnowPPvjgnvIjRoxAUVERtm3bptoWGhqKwMBALF26tNbXOHHiBIKDg5GYmAgvLy9UVlbCx8cHH330ESZNmqROdVXy8/NhZ2eHvLw82NraNugYTUUIgY6zd6K0Qol97/aFjzNvQREREdVGne9vtVpqysvLcerUKYSHh/97AKkU4eHhiI6OrnWf6OjoGuUBICIios7yAJCXlweJRAJ7e3sAQExMDFJSUiCVStGtWze4u7tj4MCBqtYeQ5NZUIbSCiVMpBK0dLDQdXWIiIiMglqhJjs7GwqFAq6urjW2u7q6Ij09vdZ90tPT1SpfWlqKadOmYdSoUapEdu3aNQDA3LlzMWvWLGzbtg0ODg7o27cvcnJyaj1OWVkZ8vPzazz0xY07nYRb2lvAzIR9tYmIiJqCXn2jVlRUYPjw4RBCYMmSJartSqUSADBz5kwMGzYMQUFBWLlyJSQSCTZu3FjrsebNmwc7OzvVw9PTUyvnUB/sJExERNT01Ao1zs7OMDExQUZGRo3tGRkZcHNzq3UfNze3epWvDjSJiYmIioqqcd/M3d0dANCpUyfVNplMhtatWyMpKanW150+fTry8vJUj+Tk5PqfqIaxkzAREVHTUyvUmJubIygoCLt371ZtUyqV2L17N8LCwmrdJywsrEZ5AIiKiqpRvjrQJCQkYNeuXXBycqpRPigoCDKZDPHx8TX2uXHjBry9vWt9XZlMBltb2xoPfVG9OrcP56ghIiJqMqbq7jB16lSMHz8ePXr0QHBwMBYtWoSioiJMnDgRADBu3Di0bNkS8+bNAwBMmTIFffr0wcKFCzFo0CCsX78eJ0+exPLlywFUhZNnnnkGMTEx2LZtGxQKhaq/jaOjI8zNzWFra4tXXnkFc+bMgaenJ7y9vbFgwQIAwLPPPtskb4Q2Va/O7cXbT0RERE1G7VAzYsQIZGVlYfbs2UhPT0dgYCB27typ6gyclJQEqfTfBqBevXph7dq1mDVrFmbMmIF27dph8+bN6NKlCwAgJSUFW7duBQAEBgbWeK29e/eib9++AIAFCxbA1NQUY8eORUlJCUJCQrBnzx44ODg05Lx1qnrdJ/apISIiajpqz1NjqPRlnprc4nIEfhwFALjwcQQszdXOlURERM2GxuapocarbqVxsZEx0BARETUhhhotYydhIiIizWCo0TLVcG72pyEiImpSDDVappp4j3PUEBERNSmGGi1LvHP7yZuLWBIRETUphhotUw3nZksNERFRk2Ko0aLi8kpkFpQB4Bw1RERETY2hRouqZxK2szCDvaW5jmtDRERkXBhqtIgzCRMREWkOQ40WqToJc44aIiKiJsdQo0XsJExERKQ5DDVaxNW5iYiINIehRou4RAIREZHmMNRoSYVCidTcUgDsKExERKQJDDVaknK7BAqlgNxMChcbma6rQ0REZHQYarSk+taTt6MVJBKJjmtDRERkfBhqtISdhImIiDSLoUZLqodz+zDUEBERaQRDjZZUT7znxZFPREREGsFQoyWceI+IiEizGGq0QKkUSMypvv3ElhoiIiJNYKjRgoyCUpRXKmEqlcDDXq7r6hARERklhhotqL711NLBAqYmfMuJiIg0gd+wWsDVuYmIiDSPoUYL2EmYiIhI8xhqtKC6kzDXfCIiItIchhot4O0nIiIizWOo0TAhxL+3n9hSQ0REpDEMNRp2u7gCBaWVAAAv9qkhIiLSGIYaDau+9eRmK4fczETHtSEiIjJeDDUaxtW5iYiItIOhRsNuZHN1biIiIm1gqNGwxByOfCIiItIGhhoNS+LIJyIiIq1gqNGwG6rZhNlSQ0REpEkMNRpUVFaJ7MIyAOwoTEREpGkMNRpUPemeg6UZ7CzMdFwbIiIi48ZQo0FJdzoJe7GTMBERkcYx1GgQV+cmIiLSHoYaDaruJMw5aoiIiDSPoUaDePuJiIhIexhqNIircxMREWkPQ42GlFcqkZpbAoChhoiISBsYajTk5u1iKAVgaW6CFtYyXVeHiIjI6DHUaEhi9ercjpaQSCQ6rg0REZHxY6jRkMTs6oUseeuJiIhIGxhqNKS6pYarcxMREWkHQ42GcOQTERGRdjHUaEjirTu3n7g6NxERkVYw1GiAQimQnMPh3ERERNrEUKMB6fmlKFcoYWYigbudXNfVISIiahYYajSg+tZTKwdLmJrwLSYiItIGfuNqQBI7CRMREWkdQ40GVK/O7e3IUENERKQtDDUawNW5iYiItI+hRgNuZFe11Pjw9hMREZHWMNQ0MSEEknLYp4aIiEjbGGqaWE5ROQrLKiGRVI1+IiIiIu1gqGli1Z2E3W3lkJuZ6Lg2REREzQdDTRP7t5MwW2mIiIi0iaGmiVUvZOnDkU9ERERaxVDTxKpDDVtqiIiItIuhpolxdW4iIiLdYKhpYolcIoGIiEgnGGqaUGFZJW4VlQNgqCEiItI2hpomVH3rydHKHDZyMx3XhoiIqHlhqGlCvPVERESkOww1TSiRq3MTERHpDENNE6qeeM+bc9QQERFpHUNNE6penZu3n4iIiLSvQaFm8eLF8PHxgVwuR0hICI4fP37f8hs3boSfnx/kcjn8/f2xY8cO1XMVFRWYNm0a/P39YWVlBQ8PD4wbNw6pqam1HqusrAyBgYGQSCQ4ffp0Q6qvMVydm4iISHfUDjUbNmzA1KlTMWfOHMTExCAgIAARERHIzMystfyRI0cwatQoTJo0CbGxsYiMjERkZCTi4uIAAMXFxYiJicGHH36ImJgY/P7774iPj8eQIUNqPd77778PDw8PdautcWWVCqTmlQDg7SciIiJdkAghhDo7hISEoGfPnvj2228BAEqlEp6enpg8eTI++OCDe8qPGDECRUVF2LZtm2pbaGgoAgMDsXTp0lpf48SJEwgODkZiYiK8vLxU2//66y9MnToVmzZtQufOnREbG4vAwMB61Ts/Px92dnbIy8uDra2tGmdcP1cyCxH+5X5YmZsg7qMISCSSJn8NIiKi5kad72+1WmrKy8tx6tQphIeH/3sAqRTh4eGIjo6udZ/o6Oga5QEgIiKizvIAkJeXB4lEAnt7e9W2jIwMvPjii/jpp59gafng2ztlZWXIz8+v8dCkf1fntmKgISIi0gG1Qk12djYUCgVcXV1rbHd1dUV6enqt+6Snp6tVvrS0FNOmTcOoUaNUiUwIgQkTJuCVV15Bjx496lXXefPmwc7OTvXw9PSs134NVd1J2If9aYiIiHRCr0Y/VVRUYPjw4RBCYMmSJart33zzDQoKCjB9+vR6H2v69OnIy8tTPZKTkzVRZZXqTsJcnZuIiEg3TNUp7OzsDBMTE2RkZNTYnpGRATc3t1r3cXNzq1f56kCTmJiIPXv21LhvtmfPHkRHR0Mmk9XYp0ePHhg9ejRWr159z+vKZLJ7ymtS9RIJPuwkTEREpBNqtdSYm5sjKCgIu3fvVm1TKpXYvXs3wsLCat0nLCysRnkAiIqKqlG+OtAkJCRg165dcHJyqlH+66+/xpkzZ3D69GmcPn1aNSR8w4YN+PTTT9U5BY3hbMJERES6pVZLDQBMnToV48ePR48ePRAcHIxFixahqKgIEydOBACMGzcOLVu2xLx58wAAU6ZMQZ8+fbBw4UIMGjQI69evx8mTJ7F8+XIAVYHmmWeeQUxMDLZt2waFQqHqb+Po6Ahzc/MaI6AAwNraGgDQpk0btGrVquFn30QUSoHk27z9REREpEtqh5oRI0YgKysLs2fPRnp6OgIDA7Fz505VZ+CkpCRIpf82APXq1Qtr167FrFmzMGPGDLRr1w6bN29Gly5dAAApKSnYunUrANwzPHvv3r3o27dvA09Ne9LySlChEDA3kcLdzkLX1SEiImqW1J6nxlBpcp6aw1eyMfqHY2jdwgp73unbpMcmIiJqzjQ2Tw3Vjv1piIiIdI+hpglUj3zi8ghERES6w1DTBFQtNewkTEREpDMMNU0gMad6NmG21BAREekKQ00jCSFUt584nJuIiEh3GGoaKbuwHMXlCkgkQCsHDucmIiLSFYaaRqpendvDzgIyUxMd14aIiKj5YqhppNziCtjITdlJmIiISMc4+V4TEEKgrFIJuRlbaoiIiJoSJ9/TMolEwkBDRESkYww1REREZBQYaoiIiMgoMNQQERGRUWCoISIiIqPAUENERERGgaGGiIiIjAJDDRERERkFhhoiIiIyCgw1REREZBQYaoiIiMgoMNQQERGRUWCoISIiIqPAUENERERGwVTXFdAWIQSAqiXMiYiIyDBUf29Xf4/fT7MJNQUFBQAAT09PHdeEiIiI1FVQUAA7O7v7lpGI+kQfI6BUKpGamgobGxtIJJImPXZ+fj48PT2RnJwMW1vbJj22vuG5Gq/mdL48V+PVnM63uZyrEAIFBQXw8PCAVHr/XjPNpqVGKpWiVatWGn0NW1tbo/7FuhvP1Xg1p/PluRqv5nS+zeFcH9RCU40dhYmIiMgoMNQQERGRUWCoaQIymQxz5syBTCbTdVU0judqvJrT+fJcjVdzOt/mdK711Ww6ChMREZFxY0sNERERGQWGGiIiIjIKDDVERERkFBhqiIiIyCgw1NTT4sWL4ePjA7lcjpCQEBw/fvy+5Tdu3Ag/Pz/I5XL4+/tjx44dWqppw82bNw89e/aEjY0NXFxcEBkZifj4+Pvus2rVKkgkkhoPuVyupRo3zty5c++pu5+f3333McTrCgA+Pj73nKtEIsHrr79ea3lDuq4HDhzA4MGD4eHhAYlEgs2bN9d4XgiB2bNnw93dHRYWFggPD0dCQsIDj6vuZ15b7ne+FRUVmDZtGvz9/WFlZQUPDw+MGzcOqamp9z1mQz4L2vCgazthwoR76j1gwIAHHlcfr+2DzrW2z69EIsGCBQvqPKa+XldNYqiphw0bNmDq1KmYM2cOYmJiEBAQgIiICGRmZtZa/siRIxg1ahQmTZqE2NhYREZGIjIyEnFxcVquuXr279+P119/HUePHkVUVBQqKirw+OOPo6io6L772draIi0tTfVITEzUUo0br3PnzjXqfujQoTrLGup1BYATJ07UOM+oqCgAwLPPPlvnPoZyXYuKihAQEIDFixfX+vznn3+Or7/+GkuXLsWxY8dgZWWFiIgIlJaW1nlMdT/z2nS/8y0uLkZMTAw+/PBDxMTE4Pfff0d8fDyGDBnywOOq81nQlgddWwAYMGBAjXqvW7fuvsfU12v7oHO9+xzT0tLw448/QiKRYNiwYfc9rj5eV40S9EDBwcHi9ddfV/2sUCiEh4eHmDdvXq3lhw8fLgYNGlRjW0hIiHj55Zc1Ws+mlpmZKQCI/fv311lm5cqVws7OTnuVakJz5swRAQEB9S5vLNdVCCGmTJki2rRpI5RKZa3PG+p1BSD++OMP1c9KpVK4ubmJBQsWqLbl5uYKmUwm1q1bV+dx1P3M68r/nm9tjh8/LgCIxMTEOsuo+1nQhdrOdfz48WLo0KFqHccQrm19ruvQoUPFo48+et8yhnBdmxpbah6gvLwcp06dQnh4uGqbVCpFeHg4oqOja90nOjq6RnkAiIiIqLO8vsrLywMAODo63rdcYWEhvL294enpiaFDh+L8+fPaqF6TSEhIgIeHB1q3bo3Ro0cjKSmpzrLGcl3Ly8vx888/4/nnn7/v4q6GfF2rXb9+Henp6TWum52dHUJCQuq8bg35zOuzvLw8SCQS2Nvb37ecOp8FfbJv3z64uLigQ4cOePXVV3Hr1q06yxrLtc3IyMD27dsxadKkB5Y11OvaUAw1D5CdnQ2FQgFXV9ca211dXZGenl7rPunp6WqV10dKpRJvvfUWHnroIXTp0qXOch06dMCPP/6ILVu24Oeff4ZSqUSvXr1w8+ZNLda2YUJCQrBq1Srs3LkTS5YswfXr1/HII4+goKCg1vLGcF0BYPPmzcjNzcWECRPqLGPI1/Vu1ddGnevWkM+8viotLcW0adMwatSo+y54qO5nQV8MGDAAa9aswe7du/HZZ59h//79GDhwIBQKRa3ljeXarl69GjY2Nnj66afvW85Qr2tjNJtVukk9r7/+OuLi4h54/zUsLAxhYWGqn3v16oWOHTti2bJl+OSTTzRdzUYZOHCg6v+7du2KkJAQeHt749dff63XX0CGasWKFRg4cCA8PDzqLGPI15WqVFRUYPjw4RBCYMmSJfcta6ifhZEjR6r+39/fH127dkWbNm2wb98+9O/fX4c106wff/wRo0ePfmDnfUO9ro3BlpoHcHZ2homJCTIyMmpsz8jIgJubW637uLm5qVVe37zxxhvYtm0b9u7di1atWqm1r5mZGbp164YrV65oqHaaY29vj/bt29dZd0O/rgCQmJiIXbt24YUXXlBrP0O9rtXXRp3r1pDPvL6pDjSJiYmIioq6bytNbR70WdBXrVu3hrOzc531NoZre/DgQcTHx6v9GQYM97qqg6HmAczNzREUFITdu3ertimVSuzevbvGX7J3CwsLq1EeAKKiouosry+EEHjjjTfwxx9/YM+ePfD19VX7GAqFAufOnYO7u7sGaqhZhYWFuHr1ap11N9TrereVK1fCxcUFgwYNUms/Q72uvr6+cHNzq3Hd8vPzcezYsTqvW0M+8/qkOtAkJCRg165dcHJyUvsYD/os6KubN2/i1q1bddbb0K8tUNXSGhQUhICAALX3NdTrqhZd91Q2BOvXrxcymUysWrVKXLhwQbz00kvC3t5epKenCyGEGDt2rPjggw9U5Q8fPixMTU3FF198IS5evCjmzJkjzMzMxLlz53R1CvXy6quvCjs7O7Fv3z6RlpamehQXF6vK/O+5fvTRR+Lvv/8WV69eFadOnRIjR44UcrlcnD9/XhenoJZ33nlH7Nu3T1y/fl0cPnxYhIeHC2dnZ5GZmSmEMJ7rWk2hUAgvLy8xbdq0e54z5OtaUFAgYmNjRWxsrAAgvvzySxEbG6sa7TN//nxhb28vtmzZIs6ePSuGDh0qfH19RUlJieoYjz76qPjmm29UPz/oM69L9zvf8vJyMWTIENGqVStx+vTpGp/jsrIy1TH+93wf9FnQlfuda0FBgXj33XdFdHS0uH79uti1a5fo3r27aNeunSgtLVUdw1Cu7YN+j4UQIi8vT1haWoolS5bUegxDua6axFBTT998843w8vIS5ubmIjg4WBw9elT1XJ8+fcT48eNrlP/1119F+/bthbm5uejcubPYvn27lmusPgC1PlauXKkq87/n+tZbb6neF1dXV/HEE0+ImJgY7Ve+AUaMGCHc3d2Fubm5aNmypRgxYoS4cuWK6nljua7V/v77bwFAxMfH3/OcIV/XvXv31vp7W30+SqVSfPjhh8LV1VXIZDLRv3//e94Db29vMWfOnBrb7veZ16X7ne/169fr/Bzv3btXdYz/Pd8HfRZ05X7nWlxcLB5//HHRokULYWZmJry9vcWLL754TzgxlGv7oN9jIYRYtmyZsLCwELm5ubUew1CuqyZJhBBCo01BRERERFrAPjVERERkFBhqiIiIyCgw1BAREZFRYKghIiIio8BQQ0REREaBoYaIiIiMAkMNERERGQWGGiJqtvbt2weJRILc3FxdV4WImgBDDRERERkFhhoiIiIyCgw1RKQzSqUS8+bNg6+vLywsLBAQEIDffvsNwL+3hrZv346uXbtCLpcjNDQUcXFxNY6xadMmdO7cGTKZDD4+Pli4cGGN58vKyjBt2jR4enpCJpOhbdu2WLFiRY0yp06dQo8ePWBpaYlevXohPj5esydORBrBUENEOjNv3jysWbMGS5cuxfnz5/H2229jzJgx2L9/v6rMe++9h4ULF+LEiRNo0aIFBg8ejIqKCgBVYWT48OEYOXIkzp07h7lz5+LDDz/EqlWrVPuPGzcO69atw9dff42LFy9i2bJlsLa2rlGPmTNnYuHChTh58iRMTU3x/PPPa+X8iahpcUFLItKJsrIyODo6YteuXQgLC1Ntf+GFF1BcXIyXXnoJ/fr1w/r16zFixAgAQE5ODlq1aoVVq1Zh+PDhGD16NLKysvDPP/+o9n///fexfft2nD9/HpcvX0aHDh0QFRWF8PDwe+qwb98+9OvXD7t27UL//v0BADt27MCgQYNQUlICuVyu4XeBiJoSW2qISCeuXLmC4uJiPPbYY7C2tlY91qxZg6tXr6rK3R14HB0d0aFDB1y8eBEAcPHiRTz00EM1jvvQQw8hISEBCoUCp0+fhomJCfr06XPfunTt2lX1/+7u7gCAzMzMRp8jEWmXqa4rQETNU2FhIQBg+/btaNmyZY3nZDJZjWDTUBYWFvUqZ2Zmpvp/iUQCoKq/DxEZFrbUEJFOdOrUCTKZDElJSWjbtm2Nh6enp6rc0aNHVf9/+/ZtXL58GR07dgQAdOzYEYcPH65x3MOHD6N9+/YwMTGBv78/lEpljT46RGS82FJDRDphY2ODd999F2+//TaUSiUefvhh5OXl4fDhw7C1tYW3tzcA4OOPP4aTkxNcXV0xc+ZMODs7IzIyEgDwzjvvoGfPnvjkk08wYsQIREdH49tvv8V3330HAPDx8cH48ePx/PPP4+uvv0ZAQAASExORmZmJ4cOH6+rUiUhDGGqISGc++eQTtGjRAvPmzcO1a9dgb2+P7t27Y8aMGarbP/Pnz8eUKVOQkJCAwMBA/PnnnzA3NwcAdO/eHb/++itmz56NTz75BO7u7vj4448xYcIE1WssWbIEM2bMwGuvvYZbt27By8sLM2bM0MXpEpGGcfQTEeml6pFJt2/fhr29va6rQ0QGgH1qiIiIyCgw1BAREZFR4O0nIiIiMgpsqSEiIiKjwFBDRERERoGhhoiIiIwCQw0REREZBYYaIiIiMgoMNURERGQUGGqIiIjIKDDUEBERkVFgqCEiIiKj8P/FjCtXCrq5QwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "metrics_df[[\"epoch\", \"NDCG@10\"]].plot(kind=\"line\", x=\"epoch\", title=\"NDCG\");"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## More RecTools features for transformers\n",
    "### Saving and loading models\n",
    "Transformer models can be saved and loaded just like any other RecTools models. \n",
    "\n",
    "*Note that you can't use these common functions for savings and loading lightning checkpoints. Use `load_from_checkpoint` method instead.*\n",
    "\n",
    "**Note that you shouldn't change code for custom functions and classes that were passed to model during initialization if you want to have correct model saving and loading.** "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "54579980"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.save(\"my_model.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "HPU available: False, using: 0 HPUs\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'rectools.models.nn.sasrec.SASRecModel'>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c3d274cc8064541b842dd0358bb6e79",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Predicting: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>score</th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>176549</td>\n",
       "      <td>2599</td>\n",
       "      <td>2.681841</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>176549</td>\n",
       "      <td>12225</td>\n",
       "      <td>2.516873</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>176549</td>\n",
       "      <td>2025</td>\n",
       "      <td>2.416028</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>176549</td>\n",
       "      <td>11749</td>\n",
       "      <td>2.410308</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>176549</td>\n",
       "      <td>14120</td>\n",
       "      <td>2.356824</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id     score  rank\n",
       "0   176549     2599  2.681841     1\n",
       "1   176549    12225  2.516873     2\n",
       "2   176549     2025  2.416028     3\n",
       "3   176549    11749  2.410308     4\n",
       "4   176549    14120  2.356824     5"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loaded = load_model(\"my_model.pkl\")\n",
    "print(type(loaded))\n",
    "loaded.recommend(users=VAL_USERS[:1], dataset=dataset, filter_viewed=True, k=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Configs for transformer models\n",
    "\n",
    "`from_config`, `from_params`, `get_config` and `get_params` methods are fully available for transformers just like for any other models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cls': 'SASRecModel',\n",
       " 'verbose': 0,\n",
       " 'data_preparator_type': 'rectools.models.nn.transformers.sasrec.SASRecDataPreparator',\n",
       " 'n_blocks': 1,\n",
       " 'n_heads': 1,\n",
       " 'n_factors': 64,\n",
       " 'use_pos_emb': True,\n",
       " 'use_causal_attn': True,\n",
       " 'use_key_padding_mask': False,\n",
       " 'dropout_rate': 0.2,\n",
       " 'session_max_len': 100,\n",
       " 'dataloader_num_workers': 0,\n",
       " 'batch_size': 128,\n",
       " 'loss': 'softmax',\n",
       " 'n_negatives': 1,\n",
       " 'gbce_t': 0.2,\n",
       " 'lr': 0.001,\n",
       " 'epochs': 2,\n",
       " 'deterministic': False,\n",
       " 'recommend_batch_size': 256,\n",
       " 'recommend_device': None,\n",
       " 'train_min_user_interactions': 2,\n",
       " 'item_net_block_types': ['rectools.models.nn.item_net.IdEmbeddingsItemNet',\n",
       "  'rectools.models.nn.item_net.CatFeaturesItemNet'],\n",
       " 'item_net_constructor_type': 'rectools.models.nn.item_net.SumOfEmbeddingsConstructor',\n",
       " 'pos_encoding_type': 'rectools.models.nn.transformers.net_blocks.LearnableInversePositionalEncoding',\n",
       " 'transformer_layers_type': 'rectools.models.nn.transformers.sasrec.SASRecTransformerLayers',\n",
       " 'lightning_module_type': 'rectools.models.nn.transformers.lightning.TransformerLightningModule',\n",
       " 'get_val_mask_func': None,\n",
       " 'get_trainer_func': None,\n",
       " 'data_preparator_kwargs': None,\n",
       " 'transformer_layers_kwargs': None,\n",
       " 'item_net_constructor_kwargs': None,\n",
       " 'pos_encoding_kwargs': None,\n",
       " 'lightning_module_kwargs': None}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = {\n",
    "    \"epochs\": 2,\n",
    "    \"n_blocks\": 1,\n",
    "    \"n_heads\": 1,\n",
    "    \"n_factors\": 64, \n",
    "}\n",
    "\n",
    "model = SASRecModel.from_config(config)\n",
    "model.get_params(simple_types=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Classes and functions in configs\n",
    "\n",
    "Transformer models in RecTools may accept functions and classes as arguments. These types of arguments are fully compatible with RecTools configs. You can eigther pass them as python objects or as strings that define their import paths.\n",
    "\n",
    "**Note that you shouldn't change code for those functions and classes if you want to have reproducible config and correct model saving and loading.** \n",
    "\n",
    "Below is an example of both approaches to pass them to configs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cls': 'SASRecModel',\n",
       " 'verbose': 0,\n",
       " 'data_preparator_type': 'rectools.models.nn.transformers.sasrec.SASRecDataPreparator',\n",
       " 'n_blocks': 2,\n",
       " 'n_heads': 4,\n",
       " 'n_factors': 256,\n",
       " 'use_pos_emb': True,\n",
       " 'use_causal_attn': True,\n",
       " 'use_key_padding_mask': False,\n",
       " 'dropout_rate': 0.2,\n",
       " 'session_max_len': 100,\n",
       " 'dataloader_num_workers': 0,\n",
       " 'batch_size': 128,\n",
       " 'loss': 'softmax',\n",
       " 'n_negatives': 1,\n",
       " 'gbce_t': 0.2,\n",
       " 'lr': 0.001,\n",
       " 'epochs': 3,\n",
       " 'deterministic': False,\n",
       " 'recommend_batch_size': 256,\n",
       " 'recommend_device': None,\n",
       " 'train_min_user_interactions': 2,\n",
       " 'item_net_block_types': ['rectools.models.nn.item_net.IdEmbeddingsItemNet',\n",
       "  'rectools.models.nn.item_net.CatFeaturesItemNet'],\n",
       " 'item_net_constructor_type': 'rectools.models.nn.item_net.SumOfEmbeddingsConstructor',\n",
       " 'pos_encoding_type': 'rectools.models.nn.transformers.net_blocks.LearnableInversePositionalEncoding',\n",
       " 'transformer_layers_type': 'rectools.models.nn.transformers.sasrec.SASRecTransformerLayers',\n",
       " 'lightning_module_type': 'rectools.models.nn.transformers.lightning.TransformerLightningModule',\n",
       " 'get_val_mask_func': '__main__.get_val_mask_func',\n",
       " 'get_trainer_func': '__main__.get_custom_trainer',\n",
       " 'data_preparator_kwargs': None,\n",
       " 'transformer_layers_kwargs': None,\n",
       " 'item_net_constructor_kwargs': None,\n",
       " 'pos_encoding_kwargs': None,\n",
       " 'lightning_module_kwargs': None}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = {\n",
    "    \"get_val_mask_func\": get_val_mask_func,  # function to get validation mask\n",
    "    \"get_trainer_func\": get_custom_trainer, # function to get custom trainer\n",
    "    # path to transformer layers class:\n",
    "    \"transformer_layers_type\": \"rectools.models.nn.transformers.sasrec.SASRecTransformerLayers\",\n",
    "}\n",
    "\n",
    "model = SASRecModel.from_config(config)\n",
    "model.get_params(simple_types=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that if you didn't pass custom `get_trainer_func`, you can still replace default `trainer` after model initialization. But this way custom trainer will not be saved with the model and will not appear in model config and params."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "model._trainer = trainer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multi-gpu training\n",
    "RecTools models use PyTorch Lightning to handle multi-gpu training.\n",
    "Please refer to Lightning documentation for details. We do not cover it in this guide."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rectools",
   "language": "python",
   "name": "rectools"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}