MLP.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MLP Models to Compute Heritage Values and Attributes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import os.path as osp\n",
    "from itertools import product\n",
    "from typing import Callable, List, Optional\n",
    "\n",
    "import numpy as np\n",
    "import scipy.sparse as sp\n",
    "\n",
    "from torch_geometric.data import (\n",
    "    Data,\n",
    "    InMemoryDataset,\n",
    "    download_url,\n",
    "    extract_zip,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from argparse import Namespace\n",
    "from collections import Counter\n",
    "import json\n",
    "import os\n",
    "import re\n",
    "import string\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import random\n",
    "\n",
    "import torch\n",
    "torch.manual_seed(1337)\n",
    "torch.backends.cudnn.benchmark = True\n",
    "\n",
    "from torch_geometric.transforms import RandomLinkSplit, ToUndirected\n",
    "import torch_geometric.transforms as T\n",
    "from torch_geometric.nn import GATConv, Linear, to_hetero\n",
    "\n",
    "import torch.nn.functional as F\n",
    "from torch_geometric import seed_everything\n",
    "\n",
    "from tqdm import tqdm\n",
    "import torch.optim as optim\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.nn import MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'D:\\\\surf\\\\TUD\\\\Paper\\\\Venice_Graph'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch version 1.10.2\n",
      "GPU-enabled installation? True\n"
     ]
    }
   ],
   "source": [
    "print(\"PyTorch version {}\".format(torch.__version__))\n",
    "print(\"GPU-enabled installation? {}\".format(torch.cuda.is_available()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "#device = torch.device(\"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed_everywhere(seed, cuda):\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    random.seed(seed)\n",
    "    if cuda:\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "\n",
    "def handle_dirs(dirpath):\n",
    "    if not os.path.exists(dirpath):\n",
    "        os.makedirs(dirpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = Namespace(\n",
    "    # Data and Path information\n",
    "    path = 'dataset/Venice',\n",
    "    save_dir='model_storage/MLP/',\n",
    "    model_state_file='model.pth',\n",
    "    \n",
    "    # Model hyper parameters\n",
    "    hidden_channels = 256,\n",
    "    num_layers = 3,\n",
    "    k=3,\n",
    "    \n",
    "    # Training hyper parameters\n",
    "    sample_nodes = 25,\n",
    "    batch_size=32,\n",
    "    early_stopping_criteria=30,\n",
    "    learning_rate=0.001,\n",
    "    l2=2e-4,\n",
    "    dropout_p=0.1,\n",
    "    num_epochs=300,\n",
    "    seed=42,\n",
    "    \n",
    "    # Runtime options\n",
    "    catch_keyboard_interrupt=True,\n",
    "    cuda=True,\n",
    "    expand_filepaths_to_save_dir=True,\n",
    "    reload_from_files=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expanded filepaths: \n",
      "\tmodel_storage/MLP/model.pth\n"
     ]
    }
   ],
   "source": [
    "if args.expand_filepaths_to_save_dir:\n",
    "    args.model_state_file = os.path.join(args.save_dir,\n",
    "                                         args.model_state_file)\n",
    "    \n",
    "    print(\"Expanded filepaths: \")\n",
    "    print(\"\\t{}\".format(args.model_state_file))\n",
    "\n",
    "# handle dirs\n",
    "handle_dirs(args.save_dir)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Constructing Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VEN(InMemoryDataset):\n",
    "    r\"\"\"A subset of Flickr post collected in Venice annotated with Heritage \n",
    "    Values and Attributes, as collected in the `\"Heri-Graphs: A Workflow of \n",
    "    Creating Datasets for Multi-modal Machine Learning on Graphs of Heritage \n",
    "    Values and Attributes with Social Media\" <https://arxiv.org/abs/2205.07545>`\n",
    "    paper.\n",
    "    VEN is a heterogeneous graph containing two types of nodes - nodes with only \n",
    "    visual features 'vis_only' (1,190 nodes), nodes with both visual and textual\n",
    "    features 'vis_tex' (1,761 nodes) and four types of links - social similarity\n",
    "    'SOC' (488,103 links), spatial similarity (445,779 links), temporal similarity\n",
    "    (501,191 links), and simple composed link (1,071,977 links).\n",
    "    Vis_only nodes are represented with 982-dimensional visual features and are\n",
    "    divided into 9 heritage attribute categories \n",
    "    ('architectural elements', 'form', 'gastronomy', 'interior',\n",
    "    'landscape scenery and natural features', 'monuments', 'people', 'product', \n",
    "    'urban scenery').\n",
    "    Vis_text nodes are represented with 1753-dimensional visual and textual \n",
    "    features and are divided into 9 heritage attribute categories plus 11 \n",
    "    heritage value categories ('criterion i-x', 'other').\n",
    "    Both types of nodes are also merged into a single type of node 'all' with \n",
    "    1753-dimensional features and 20-dimensional label categories.\n",
    "\n",
    "    Args:\n",
    "        root (string): Root directory where the dataset should be saved.\n",
    "        transform (callable, optional): A function/transform that takes in an\n",
    "            :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            every access. (default: :obj:`None`)\n",
    "        pre_transform (callable, optional): A function/transform that takes in\n",
    "            an :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            being saved to disk. (default: :obj:`None`)\n",
    "    \n",
    "    Stats:\n",
    "            * - #nodes\n",
    "              - #edges\n",
    "              - #features\n",
    "              - #classes\n",
    "            * - 2,951\n",
    "              - 1,071,977\n",
    "              - 1753\n",
    "              - 20\n",
    "    \"\"\"\n",
    "\n",
    "    url = 'https://drive.google.com/uc?export=download&id=1sxcKiZr1YGDv06wr03nsk5HVZledgzi9'\n",
    "\n",
    "    def __init__(self, root: str, transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "        \n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            'A_simp.npz', 'A_SOC.npz', 'A_SPA.npz', 'A_TEM.npz', 'labels.npz',\n",
    "            'node_types.npy', 'Textual_Features.npy', 'train_val_test_idx.npz',\n",
    "            'Visual_Features.npy'\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        os.remove(path)\n",
    "\n",
    "    def process(self):\n",
    "        data = HeteroData()\n",
    "\n",
    "        node_types = ['vis_only', 'vis_tex']\n",
    "        link_types = ['SOC', 'SPA', 'TEM', 'simp']\n",
    "\n",
    "        vis = np.load(osp.join(self.raw_dir, 'Visual_Features.npy'),allow_pickle=True)[:,2:].astype(float)\n",
    "        tex = np.load(osp.join(self.raw_dir, 'Textual_Features.npy'),allow_pickle=True)[:,5:].astype(float)\n",
    "\n",
    "        x = np.hstack([vis,np.nan_to_num(tex)])\n",
    "\n",
    "\n",
    "        node_type_idx = np.load(osp.join(self.raw_dir, 'node_types.npy'))\n",
    "        node_type_idx = torch.from_numpy(node_type_idx).to(torch.long)\n",
    "\n",
    "        data['vis_only'].num_nodes = int((node_type_idx == 0).sum())\n",
    "        data['vis_tex'].num_nodes = int((node_type_idx == 1).sum())\n",
    "        data['all'].num_nodes = len(node_type_idx)\n",
    "\n",
    "        data['vis_only'].x = torch.from_numpy(vis[node_type_idx==0]).to(torch.float)\n",
    "        data['vis_tex'].x = torch.from_numpy(x[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].x = torch.from_numpy(x).to(torch.float)\n",
    "\n",
    "\n",
    "        y_s = np.load(osp.join(self.raw_dir, 'labels.npz'), allow_pickle=True)\n",
    "        att_lab = y_s['ATT_LAB'][:,1:10].astype(float)\n",
    "        val_lab = np.nan_to_num(y_s['VAL_LAB'][:,2:13].astype(float))\n",
    "        ys = np.hstack([att_lab, val_lab])\n",
    "\n",
    "        data['vis_only'].y = torch.from_numpy(att_lab[node_type_idx==0]).to(torch.float)\n",
    "        data['vis_tex'].y = torch.from_numpy(ys[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].y = torch.from_numpy(ys).to(torch.float)\n",
    "\n",
    "        data.node_type = node_type_idx\n",
    "\n",
    "        split = np.load(osp.join(self.raw_dir, 'train_val_test_idx.npz'))\n",
    "        for name in ['train', 'val', 'test']:\n",
    "            idx = split[f'{name}_idx']\n",
    "            idx = torch.from_numpy(idx).to(torch.long)\n",
    "            mask = torch.zeros(data['all'].num_nodes, dtype=torch.bool)\n",
    "            mask[idx] = True\n",
    "            data['all'][f'{name}_mask'] = mask\n",
    "            data['vis_only'][f'{name}_mask'] = mask[node_type_idx==0]\n",
    "            data['vis_tex'][f'{name}_mask'] = mask[node_type_idx==1]\n",
    "\n",
    "        \n",
    "        s = {}\n",
    "        s['vis_only'] = np.arange(len(x))[node_type_idx==0]\n",
    "        s['vis_tex'] = np.arange(len(x))[node_type_idx==1]\n",
    "\n",
    "        for link in link_types:\n",
    "            A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz')).tocoo()\n",
    "            if A_sub.nnz>0:\n",
    "                row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                data['all', f'{link}_link', 'all'].edge_index = torch.stack([row, col], dim=0)\n",
    "                data['all', f'{link}_link', 'all'].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "        for src, dst in product(node_types, node_types):\n",
    "            for link in link_types:\n",
    "                A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz'))[s[src]][:,s[dst]].tocoo()\n",
    "                if A_sub.nnz>0:\n",
    "                    row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                    col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                    data[src, f'{link}_link', dst].edge_index = torch.stack([row, col], dim=0)\n",
    "                    data[src, f'{link}_link', dst].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "\n",
    "        if self.pre_transform is not None:\n",
    "            data = self.pre_transform(data)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.__class__.__name__}()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VEN_Homo(InMemoryDataset):\n",
    "    r\"\"\"A subset of Flickr post collected in Venice annotated with Heritage \n",
    "    Values and Attributes, as collected in the `\"Heri-Graphs: A Workflow of \n",
    "    Creating Datasets for Multi-modal Machine Learning on Graphs of Heritage \n",
    "    Values and Attributes with Social Media\" <https://arxiv.org/abs/2205.07545>`\n",
    "    paper.\n",
    "    VEN_Homo is a homogeneous graph containing 2951 nodes and 1,071,977 links.\n",
    "    Vis_only nodes are represented with 982-dimensional visual features and are\n",
    "    divided into 9 heritage attribute categories \n",
    "    ('architectural elements', 'form', 'gastronomy', 'interior',\n",
    "    'landscape scenery and natural features', 'monuments', 'people', 'product', \n",
    "    'urban scenery').\n",
    "    Vis_text nodes are represented with 1753-dimensional visual and textual \n",
    "    features and are divided into 9 heritage attribute categories plus 11 \n",
    "    heritage value categories ('criterion i-x', 'other').\n",
    "    Both types of nodes are also merged into a single type of node 'all' with \n",
    "    1753-dimensional features and 20-dimensional label categories.\n",
    "\n",
    "    Args:\n",
    "        root (string): Root directory where the dataset should be saved.\n",
    "        transform (callable, optional): A function/transform that takes in an\n",
    "            :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            every access. (default: :obj:`None`)\n",
    "        pre_transform (callable, optional): A function/transform that takes in\n",
    "            an :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            being saved to disk. (default: :obj:`None`)\n",
    "    \n",
    "    Stats:\n",
    "            * - #nodes\n",
    "              - #edges\n",
    "              - #features\n",
    "              - #classes\n",
    "            * - 2,951\n",
    "              - 1,071,977\n",
    "              - 1753\n",
    "              - 20\n",
    "    \"\"\"\n",
    "\n",
    "    url = 'https://drive.google.com/uc?export=download&id=1sxcKiZr1YGDv06wr03nsk5HVZledgzi9'\n",
    "\n",
    "    def __init__(self, root: str, transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "        \n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            'A_simp.npz', 'A_SOC.npz', 'A_SPA.npz', 'A_TEM.npz', 'labels.npz',\n",
    "            'node_types.npy', 'Textual_Features.npy', 'train_val_test_idx.npz',\n",
    "            'Visual_Features.npy'\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        os.remove(path)\n",
    "\n",
    "    def process(self):\n",
    "        data = Data()\n",
    "\n",
    "        link_types = ['simp']\n",
    "\n",
    "        vis = np.load(osp.join(self.raw_dir, 'Visual_Features.npy'),allow_pickle=True)[:,2:].astype(float)\n",
    "        tex = np.load(osp.join(self.raw_dir, 'Textual_Features.npy'),allow_pickle=True)[:,5:].astype(float)\n",
    "\n",
    "        x = np.hstack([vis,np.nan_to_num(tex)])\n",
    "\n",
    "        node_type_idx = np.load(osp.join(self.raw_dir, 'node_types.npy'))\n",
    "        node_type_idx = torch.from_numpy(node_type_idx).to(torch.long)\n",
    "\n",
    "        data.num_nodes = len(node_type_idx)\n",
    "\n",
    "        data.x = torch.from_numpy(x).to(torch.float)\n",
    "\n",
    "\n",
    "        y_s = np.load(osp.join(self.raw_dir, 'labels.npz'), allow_pickle=True)\n",
    "        att_lab = y_s['ATT_LAB'][:,1:10].astype(float)\n",
    "        val_lab = np.nan_to_num(y_s['VAL_LAB'][:,2:13].astype(float))\n",
    "        ys = np.hstack([att_lab, val_lab])\n",
    "\n",
    "        data.y = torch.from_numpy(ys).to(torch.float)\n",
    "\n",
    "        data.node_type = node_type_idx\n",
    "        \n",
    "        data.att_lab = torch.tensor(y_s['ATT_LAB'][:,-1].astype(bool))\n",
    "        data.val_lab = torch.tensor(y_s['VAL_LAB'][:,-1].astype(bool))\n",
    "\n",
    "        split = np.load(osp.join(self.raw_dir, 'train_val_test_idx.npz'))\n",
    "        for name in ['train', 'val', 'test']:\n",
    "            idx = split[f'{name}_idx']\n",
    "            idx = torch.from_numpy(idx).to(torch.long)\n",
    "            mask = torch.zeros(data.num_nodes, dtype=torch.bool)\n",
    "            mask[idx] = True\n",
    "            data[f'{name}_mask'] = mask\n",
    "                    \n",
    "        s = {}\n",
    "        \n",
    "        for link in link_types:\n",
    "            A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz')).tocoo()\n",
    "            if A_sub.nnz>0:\n",
    "                row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                data.edge_index = torch.stack([row, col], dim=0)\n",
    "                data.edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "        if self.pre_transform is not None:\n",
    "            data = self.pre_transform(data)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.__class__.__name__}()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VEN_XL(InMemoryDataset):\n",
    "    r\"\"\"A large subset of Flickr post collected in Venice annotated with Heritage \n",
    "    Values and Attributes, as collected in the `\"Heri-Graphs: A Workflow of \n",
    "    Creating Datasets for Multi-modal Machine Learning on Graphs of Heritage \n",
    "    Values and Attributes with Social Media\" <https://arxiv.org/abs/2205.07545>`\n",
    "    paper.\n",
    "    VEN_XL is a heterogeneous graph containing two types of nodes - nodes with only \n",
    "    visual features 'vis_only' (31,140 nodes), nodes with both visual and textual\n",
    "    features 'vis_tex' (49,823 nodes) and four types of links - social similarity\n",
    "    'SOC' (76,422,265 links), spatial similarity (202,173,159 links), temporal similarity\n",
    "    (71,135,671 links), and simple composed link (290,091,503 links).\n",
    "    Vis_only nodes are represented with 982-dimensional visual features and are\n",
    "    divided into 9 heritage attribute categories \n",
    "    ('architectural elements', 'form', 'gastronomy', 'interior',\n",
    "    'landscape scenery and natural features', 'monuments', 'people', 'product', \n",
    "    'urban scenery').\n",
    "    Vis_text nodes are represented with 1753-dimensional visual and textual \n",
    "    features and are divided into 9 heritage attribute categories plus 11 \n",
    "    heritage value categories ('criterion i-x', 'other').\n",
    "    Both types of nodes are also merged into a single type of node 'all' with \n",
    "    1753-dimensional features and 20-dimensional label categories.\n",
    "\n",
    "    Args:\n",
    "        root (string): Root directory where the dataset should be saved.\n",
    "        transform (callable, optional): A function/transform that takes in an\n",
    "            :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            every access. (default: :obj:`None`)\n",
    "        pre_transform (callable, optional): A function/transform that takes in\n",
    "            an :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            being saved to disk. (default: :obj:`None`)\n",
    "    \n",
    "    Stats:\n",
    "            * - #nodes\n",
    "              - #edges\n",
    "              - #features\n",
    "              - #classes\n",
    "            * - 80,963\n",
    "              - 290,091,503\n",
    "              - 1753\n",
    "              - 20\n",
    "    \"\"\"\n",
    "\n",
    "    url = 'https://drive.google.com/uc?export=download&id=1QZ5tyUWs6jYjh7mJrsnpou76iy-vb0CA'\n",
    "\n",
    "    def __init__(self, root: str, transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "        \n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            'A_simp.npz', 'A_SOC.npz', 'A_SPA.npz', 'A_TEM.npz', 'labels.npz',\n",
    "            'node_types.npy', 'Textual_Features.npy', 'train_val_test_idx.npz',\n",
    "            'Visual_Features.npy'\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        os.remove(path)\n",
    "\n",
    "    def process(self):\n",
    "        data = HeteroData()\n",
    "\n",
    "        node_types = ['vis_only', 'vis_tex']\n",
    "        link_types = ['SOC', 'SPA', 'TEM', 'simp']\n",
    "\n",
    "        vis = np.load(osp.join(self.raw_dir, 'Visual_Features.npy'),allow_pickle=True)[:,2:].astype(float)\n",
    "        tex = np.load(osp.join(self.raw_dir, 'Textual_Features.npy'),allow_pickle=True)[:,5:].astype(float)\n",
    "\n",
    "        x = np.hstack([vis,np.nan_to_num(tex)])\n",
    "\n",
    "\n",
    "        node_type_idx = np.load(osp.join(self.raw_dir, 'node_types.npy'))\n",
    "        node_type_idx = torch.from_numpy(node_type_idx).to(torch.long)\n",
    "\n",
    "        data['vis_only'].num_nodes = int((node_type_idx == 0).sum())\n",
    "        data['vis_tex'].num_nodes = int((node_type_idx == 1).sum())\n",
    "        data['all'].num_nodes = len(node_type_idx)\n",
    "\n",
    "        data['vis_only'].x = torch.from_numpy(vis[node_type_idx==0]).to(torch.float)\n",
    "        data['vis_tex'].x = torch.from_numpy(x[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].x = torch.from_numpy(x).to(torch.float)\n",
    "\n",
    "\n",
    "        y_s = np.load(osp.join(self.raw_dir, 'labels.npz'), allow_pickle=True)\n",
    "        att_lab = y_s['ATT_LAB'][:,1:10].astype(float)\n",
    "        val_lab = np.nan_to_num(y_s['VAL_LAB'][:,2:13].astype(float))\n",
    "        ys = np.hstack([att_lab, val_lab])\n",
    "\n",
    "        data['vis_only'].y = torch.from_numpy(att_lab[node_type_idx==0]).to(torch.float)\n",
    "        data['vis_tex'].y = torch.from_numpy(ys[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].y = torch.from_numpy(ys).to(torch.float)\n",
    "        \n",
    "        data['all'].att_lab = torch.tensor(y_s['ATT_LAB'][:,-1].astype(bool))\n",
    "        data['all'].val_lab = torch.tensor(y_s['VAL_LAB'][:,-1].astype(bool))\n",
    "        data['all'].node_type = node_type_idx\n",
    "\n",
    "        data.node_type = node_type_idx\n",
    "\n",
    "        split = np.load(osp.join(self.raw_dir, 'train_val_test_idx.npz'))\n",
    "        for name in ['train', 'val', 'test']:\n",
    "            idx = split[f'{name}_idx']\n",
    "            idx = torch.from_numpy(idx).to(torch.long)\n",
    "            mask = torch.zeros(data['all'].num_nodes, dtype=torch.bool)\n",
    "            mask[idx] = True\n",
    "            data['all'][f'{name}_mask'] = mask\n",
    "            data['vis_only'][f'{name}_mask'] = mask[node_type_idx==0]\n",
    "            data['vis_tex'][f'{name}_mask'] = mask[node_type_idx==1]\n",
    "\n",
    "        \n",
    "        s = {}\n",
    "        s['vis_only'] = np.arange(len(x))[node_type_idx==0]\n",
    "        s['vis_tex'] = np.arange(len(x))[node_type_idx==1]\n",
    "\n",
    "        for link in link_types:\n",
    "            A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz')).tocoo()\n",
    "            if A_sub.nnz>0:\n",
    "                row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                data['all', f'{link}_link', 'all'].edge_index = torch.stack([row, col], dim=0)\n",
    "                data['all', f'{link}_link', 'all'].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "        for src, dst in product(node_types, node_types):\n",
    "            for link in link_types:\n",
    "                A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz'))[s[src]][:,s[dst]].tocoo()\n",
    "                if A_sub.nnz>0:\n",
    "                    row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                    col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                    data[src, f'{link}_link', dst].edge_index = torch.stack([row, col], dim=0)\n",
    "                    data[src, f'{link}_link', dst].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "\n",
    "        if self.pre_transform is not None:\n",
    "            data = self.pre_transform(data)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.__class__.__name__}()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VEN_XL_Homo(InMemoryDataset):\n",
    "    r\"\"\"A large subset of Flickr post collected in Venice annotated with Heritage \n",
    "    Values and Attributes, as collected in the `\"Heri-Graphs: A Workflow of \n",
    "    Creating Datasets for Multi-modal Machine Learning on Graphs of Heritage \n",
    "    Values and Attributes with Social Media\" <https://arxiv.org/abs/2205.07545>`\n",
    "    paper.\n",
    "    VEN_XL is a heterogeneous graph containing two types of nodes - nodes with only \n",
    "    visual features 'vis_only' (31,140 nodes), nodes with both visual and textual\n",
    "    features 'vis_tex' (49,823 nodes) and four types of links - social similarity\n",
    "    'SOC' (76,422,265 links), spatial similarity (202,173,159 links), temporal similarity\n",
    "    (71,135,671 links), and simple composed link (290,091,503 links).\n",
    "    Vis_only nodes are represented with 982-dimensional visual features and are\n",
    "    divided into 9 heritage attribute categories \n",
    "    ('architectural elements', 'form', 'gastronomy', 'interior',\n",
    "    'landscape scenery and natural features', 'monuments', 'people', 'product', \n",
    "    'urban scenery').\n",
    "    Vis_text nodes are represented with 1753-dimensional visual and textual \n",
    "    features and are divided into 9 heritage attribute categories plus 11 \n",
    "    heritage value categories ('criterion i-x', 'other').\n",
    "    Both types of nodes are also merged into a single type of node 'all' with \n",
    "    1753-dimensional features and 20-dimensional label categories.\n",
    "\n",
    "    Args:\n",
    "        root (string): Root directory where the dataset should be saved.\n",
    "        transform (callable, optional): A function/transform that takes in an\n",
    "            :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            every access. (default: :obj:`None`)\n",
    "        pre_transform (callable, optional): A function/transform that takes in\n",
    "            an :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            being saved to disk. (default: :obj:`None`)\n",
    "    \n",
    "    Stats:\n",
    "            * - #nodes\n",
    "              - #edges\n",
    "              - #features\n",
    "              - #classes\n",
    "            * - 80,963\n",
    "              - 290,091,503\n",
    "              - 1753\n",
    "              - 20\n",
    "    \"\"\"\n",
    "\n",
    "    url = 'https://drive.google.com/uc?export=download&id=1sxcKiZr1YGDv06wr03nsk5HVZledgzi9'\n",
    "\n",
    "    def __init__(self, root: str, transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "        \n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            'A_simp.npz', 'A_SOC.npz', 'A_SPA.npz', 'A_TEM.npz', 'labels.npz',\n",
    "            'node_types.npy', 'Textual_Features.npy', 'train_val_test_idx.npz',\n",
    "            'Visual_Features.npy'\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        os.remove(path)\n",
    "\n",
    "    def process(self):\n",
    "        data = Data()\n",
    "\n",
    "        link_types = ['simp']\n",
    "\n",
    "        vis = np.load(osp.join(self.raw_dir, 'Visual_Features.npy'),allow_pickle=True)[:,2:].astype(float)\n",
    "        tex = np.load(osp.join(self.raw_dir, 'Textual_Features.npy'),allow_pickle=True)[:,5:].astype(float)\n",
    "\n",
    "        x = np.hstack([vis,np.nan_to_num(tex)])\n",
    "\n",
    "        node_type_idx = np.load(osp.join(self.raw_dir, 'node_types.npy'))\n",
    "        node_type_idx = torch.from_numpy(node_type_idx).to(torch.long)\n",
    "\n",
    "        data.num_nodes = len(node_type_idx)\n",
    "\n",
    "        data.x = torch.from_numpy(x).to(torch.float)\n",
    "\n",
    "\n",
    "        y_s = np.load(osp.join(self.raw_dir, 'labels.npz'), allow_pickle=True)\n",
    "        att_lab = y_s['ATT_LAB'][:,1:10].astype(float)\n",
    "        val_lab = np.nan_to_num(y_s['VAL_LAB'][:,2:13].astype(float))\n",
    "        ys = np.hstack([att_lab, val_lab])\n",
    "\n",
    "        data.y = torch.from_numpy(ys).to(torch.float)\n",
    "        \n",
    "        data.att_lab = torch.tensor(y_s['ATT_LAB'][:,-1].astype(bool))\n",
    "        data.val_lab = torch.tensor(y_s['VAL_LAB'][:,-1].astype(bool))\n",
    "\n",
    "        data.node_type = node_type_idx\n",
    "\n",
    "        split = np.load(osp.join(self.raw_dir, 'train_val_test_idx.npz'))\n",
    "        for name in ['train', 'val', 'test']:\n",
    "            idx = split[f'{name}_idx']\n",
    "            idx = torch.from_numpy(idx).to(torch.long)\n",
    "            mask = torch.zeros(data.num_nodes, dtype=torch.bool)\n",
    "            mask[idx] = True\n",
    "            data[f'{name}_mask'] = mask\n",
    "                    \n",
    "        s = {}\n",
    "        \n",
    "        for link in link_types:\n",
    "            A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz')).tocoo()\n",
    "            if A_sub.nnz>0:\n",
    "                row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                data.edge_index = torch.stack([row, col], dim=0)\n",
    "                data.edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "        if self.pre_transform is not None:\n",
    "            data = self.pre_transform(data)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.__class__.__name__}()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#path = osp.join(os.getcwd(), '../../data/VEN')\n",
    "transform = T.NormalizeFeatures()\n",
    "dataset = VEN_Homo('dataset/Venice_homo')\n",
    "data = dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Data(num_nodes=2951, x=[2951, 1753], y=[2951, 20], node_type=[2951], att_lab=[2951], val_lab=[2951], train_mask=[2951], val_mask=[2951], test_mask=[2951], edge_index=[2, 1071977], edge_attr=[1071977], n_id=[2951])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.n_id = torch.arange(data.num_nodes)\n",
    "data = data.to(device)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.2861, 0.3745, 1.1453,  ..., 1.0000, 0.0000, 0.0000],\n",
       "        [0.3977, 0.1582, 0.3059,  ..., 1.0000, 0.0000, 0.0000],\n",
       "        [0.5185, 0.5124, 1.3662,  ..., 1.0000, 0.0000, 0.0000],\n",
       "        ...,\n",
       "        [0.0124, 1.7083, 0.3258,  ..., 0.0000, 1.0000, 0.0000],\n",
       "        [0.4402, 0.8374, 0.4974,  ..., 0.0000, 1.0000, 0.0000],\n",
       "        [0.2102, 1.5535, 0.3352,  ..., 0.0000, 1.0000, 0.0000]],\n",
       "       device='cuda:0')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1356, device='cuda:0')"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.att_lab.sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Loader for Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.loader import NeighborLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.train_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.val_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.test_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Data(num_nodes=2915, x=[2915, 1753], y=[2915, 20], node_type=[2915], att_lab=[2915], val_lab=[2915], train_mask=[2915], val_mask=[2915], test_mask=[2915], edge_index=[2, 102001], edge_attr=[102001], n_id=[2915], batch_size=32)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_data = next(iter(train_loader))\n",
    "batch = sampled_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_train_state(args):\n",
    "    return {'stop_early': False,\n",
    "            'early_stopping_step': 0,\n",
    "            'early_stopping_best_ATT_acc_val': 0,\n",
    "            'early_stopping_best_VAL_acc_val': 0,\n",
    "            'early_stopping_best_ATT_acc_val_2': 0,\n",
    "            'early_stopping_lowest_loss': 1000,\n",
    "            'learning_rate': args.learning_rate,\n",
    "            'epoch_index': 0,\n",
    "            'train_loss': [],\n",
    "            'train_ATT_loss': [],\n",
    "            'train_VAL_loss':[],\n",
    "            'train_ATT_acc': [],\n",
    "            'train_VAL_acc': [],\n",
    "            'train_VAL_jac': [],\n",
    "            'train_VAL_acc_1':[], \n",
    "            'val_loss': [],\n",
    "            'val_ATT_loss': [],\n",
    "            'val_VAL_loss':[],\n",
    "            'val_ATT_acc': [],\n",
    "            'val_VAL_acc': [],\n",
    "            'val_VAL_jac': [],\n",
    "            'val_VAL_acc_1': [],\n",
    "            'test_loss': -1,\n",
    "            'test_ATT_loss': -1,\n",
    "            'test_VAL_loss':-1,\n",
    "            'test_ATT_acc': -1,\n",
    "            'test_VAL_acc': -1,\n",
    "            'test_VAL_jac': -1,\n",
    "            'test_VAL_acc_1': -1,\n",
    "            'model_filename': args.model_state_file}\n",
    "\n",
    "def update_train_state(args, model, train_state):\n",
    "    \"\"\"Handle the training state updates.\n",
    "\n",
    "    Components:\n",
    "     - Early Stopping: Prevent overfitting.\n",
    "     - Model Checkpoint: Model is saved if the model is better\n",
    "\n",
    "    :param args: main arguments\n",
    "    :param model: model to train\n",
    "    :param train_state: a dictionary representing the training state values\n",
    "    :returns:\n",
    "        a new train_state\n",
    "    \"\"\"\n",
    "\n",
    "    # Save one model at least\n",
    "    if train_state['epoch_index'] == 0:\n",
    "        torch.save(model.state_dict(), train_state['model_filename'])\n",
    "        train_state['stop_early'] = False\n",
    "\n",
    "    # Save model if performance improved\n",
    "    elif train_state['epoch_index'] >= 1:\n",
    "        ATT_acc_tm1, ATT_acc_t = train_state['val_ATT_acc'][-2:]\n",
    "        #ATT_acc_2_tm1, ATT_acc_2_t = train_state['val_ATT_acc_2'][-2:]\n",
    "        VAL_acc_tm1, VAL_acc_t = train_state['val_VAL_acc'][-2:]\n",
    "        loss_tm1, loss_t = train_state['val_loss'][-2:]\n",
    "\n",
    "        # If accuracy worsened\n",
    "        #if loss_t >= train_state['early_stopping_lowest_loss']:\n",
    "        #    train_state['early_stopping_step'] += 1\n",
    "        \n",
    "        if ATT_acc_t <= train_state['early_stopping_best_ATT_acc_val'] and VAL_acc_t <= train_state['early_stopping_best_VAL_acc_val']:# and ATT_acc_2_t <= train_state['early_stopping_best_ATT_acc_val_2']:\n",
    "            # Update step\n",
    "            train_state['early_stopping_step'] += 1\n",
    "        # Loss decreased\n",
    "        else:\n",
    "            # Save the best model from sklearn\n",
    "            if VAL_acc_t > train_state['early_stopping_best_VAL_acc_val']:\n",
    "                train_state['early_stopping_best_VAL_acc_val'] = VAL_acc_t\n",
    "                \n",
    "            if ATT_acc_t > train_state['early_stopping_best_ATT_acc_val']:\n",
    "                train_state['early_stopping_best_ATT_acc_val'] = ATT_acc_t\n",
    "            \n",
    "            #if ATT_acc_2_t > train_state['early_stopping_best_ATT_acc_val_2']:\n",
    "            #    train_state['early_stopping_best_ATT_acc_val_2'] = ATT_acc_2_t\n",
    "                \n",
    "            if loss_t < train_state['early_stopping_lowest_loss']:\n",
    "                train_state['early_stopping_lowest_loss'] = loss_t\n",
    "                torch.save(model.state_dict(), train_state['model_filename'])\n",
    "                \n",
    "            # Reset early stopping step\n",
    "            train_state['early_stopping_step'] = 0\n",
    "\n",
    "        # Stop early ?\n",
    "        train_state['stop_early'] = \\\n",
    "            train_state['early_stopping_step'] >= args.early_stopping_criteria\n",
    "\n",
    "    return train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_cross_entropy(y_pred, y_target):\n",
    "    y_target = y_target.cpu().float()\n",
    "    y_pred = y_pred.cpu().float()\n",
    "    criterion = nn.BCEWithLogitsLoss()\n",
    "    return criterion(y_target, y_pred)\n",
    "\n",
    "def compute_1_accuracy(y_pred, y_target):\n",
    "    y_target_indices = y_target.max(dim=1)[1]\n",
    "    y_pred_indices = y_pred.max(dim=1)[1]\n",
    "    n_correct = torch.eq(y_pred_indices, y_target_indices).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_k_accuracy(y_pred, y_target, k=3):\n",
    "    y_pred_indices = y_pred.topk(k, dim=1)[1]\n",
    "    y_target_indices = y_target.max(dim=1)[1]\n",
    "    n_correct = torch.tensor([y_pred_indices[i] in y_target_indices[i] for i in range(len(y_pred))]).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_k_jaccard_index(y_pred, y_target, k=3):\n",
    "    y_target_indices = y_target.topk(k, dim=1)[1]\n",
    "    y_pred_indices = y_pred.max(dim=1)[1]\n",
    "    jaccard = torch.tensor([len(np.intersect1d(y_target_indices[i], y_pred_indices[i]))/\n",
    "                            len(np.union1d(y_target_indices[i], y_pred_indices[i]))\n",
    "                            for i in range(len(y_pred))]).sum().item()\n",
    "    return jaccard / len(y_pred_indices)\n",
    "\n",
    "def compute_jaccard_index(y_pred, y_target, k=3, multilabel=False):\n",
    "    \n",
    "    threshold = 1.0/(k+1)\n",
    "    threshold_2 = 0.5\n",
    "    \n",
    "    if multilabel:\n",
    "        y_pred_indices = y_pred.gt(threshold_2)\n",
    "    else:\n",
    "        y_pred_indices = y_pred.gt(threshold)\n",
    "    \n",
    "    y_target_indices = y_target.gt(threshold)\n",
    "        \n",
    "    jaccard = ((y_target_indices*y_pred_indices).sum(axis=1)/((y_target_indices+y_pred_indices).sum(axis=1)+1e-8)).sum().item()\n",
    "    return jaccard / len(y_pred_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy(pred, soft_targets):\n",
    "    logsoftmax = nn.LogSoftmax(dim=1)\n",
    "    return torch.mean(torch.sum(- soft_targets * logsoftmax(pred), 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Searched Best Hyper-parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'42/hyperdict.p', 'rb') as fp:\n",
    "    hyperdict= pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "135"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(hyperdict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "hyper_df = pd.DataFrame(hyperdict).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>stop_early</th>\n",
       "      <th>early_stopping_step</th>\n",
       "      <th>early_stopping_best_ATT_acc_val</th>\n",
       "      <th>early_stopping_best_VAL_acc_val</th>\n",
       "      <th>early_stopping_best_ATT_acc_val_2</th>\n",
       "      <th>early_stopping_lowest_loss</th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>epoch_index</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_ATT_loss</th>\n",
       "      <th>...</th>\n",
       "      <th>val_VAL_acc</th>\n",
       "      <th>val_VAL_jac</th>\n",
       "      <th>test_loss</th>\n",
       "      <th>test_ATT_loss</th>\n",
       "      <th>test_VAL_loss</th>\n",
       "      <th>test_ATT_acc</th>\n",
       "      <th>test_ATT_acc_2</th>\n",
       "      <th>test_VAL_acc</th>\n",
       "      <th>test_VAL_jac</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <th>2</th>\n",
       "      <th>512</th>\n",
       "      <th>0.01</th>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>87.913669</td>\n",
       "      <td>96.978417</td>\n",
       "      <td>0</td>\n",
       "      <td>4.283473</td>\n",
       "      <td>0.01</td>\n",
       "      <td>81</td>\n",
       "      <td>[3.668607771396637, 2.8671470483144126, 2.6329...</td>\n",
       "      <td>[1.6286450495680282, 0.9909614489679521, 0.998...</td>\n",
       "      <td>...</td>\n",
       "      <td>[56.97841726618705, 55.97122302158273, 57.6978...</td>\n",
       "      <td>[0.1157656767385469, 0.10939705817819499, 0.12...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/MLP/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">0.2</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
       "      <th>256</th>\n",
       "      <th>0.01</th>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>87.194245</td>\n",
       "      <td>95.251799</td>\n",
       "      <td>0</td>\n",
       "      <td>4.247989</td>\n",
       "      <td>0.01</td>\n",
       "      <td>40</td>\n",
       "      <td>[3.4082926511764526, 2.7423266967137656, 2.622...</td>\n",
       "      <td>[1.3227104675076344, 0.9560339421116414, 0.841...</td>\n",
       "      <td>...</td>\n",
       "      <td>[58.992805755395686, 63.7410071942446, 87.9136...</td>\n",
       "      <td>[0.12546762785465598, 0.1313429269859259, 0.13...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/MLP/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>512</th>\n",
       "      <th>0.01</th>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>86.906475</td>\n",
       "      <td>98.129496</td>\n",
       "      <td>0</td>\n",
       "      <td>4.233886</td>\n",
       "      <td>0.01</td>\n",
       "      <td>142</td>\n",
       "      <td>[3.684901793797811, 2.8937514623006186, 2.7011...</td>\n",
       "      <td>[1.5665503717195295, 1.0540965808725753, 0.989...</td>\n",
       "      <td>...</td>\n",
       "      <td>[57.697841726618705, 55.10791366906475, 88.201...</td>\n",
       "      <td>[0.11342240872142983, 0.11150565833496533, 0.1...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/MLP/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <th>512</th>\n",
       "      <th>0.01</th>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>87.482014</td>\n",
       "      <td>98.705036</td>\n",
       "      <td>0</td>\n",
       "      <td>4.23534</td>\n",
       "      <td>0.01</td>\n",
       "      <td>112</td>\n",
       "      <td>[3.980452517668406, 3.006455639998118, 2.69875...</td>\n",
       "      <td>[1.9148970105971657, 0.9970721310195501, 1.037...</td>\n",
       "      <td>...</td>\n",
       "      <td>[87.4820143884892, 92.5179856115108, 94.244604...</td>\n",
       "      <td>[0.12217197778413622, 0.13116649723739074, 0.1...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/MLP/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <th>2</th>\n",
       "      <th>512</th>\n",
       "      <th>0.01</th>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>85.755396</td>\n",
       "      <td>99.42446</td>\n",
       "      <td>0</td>\n",
       "      <td>4.218476</td>\n",
       "      <td>0.01</td>\n",
       "      <td>61</td>\n",
       "      <td>[3.8542353312174478, 3.08571062485377, 2.85462...</td>\n",
       "      <td>[1.6187379809297684, 1.0744382905827996, 0.968...</td>\n",
       "      <td>...</td>\n",
       "      <td>[62.87769784172662, 56.83453237410072, 79.1366...</td>\n",
       "      <td>[0.11468996452770645, 0.12218568067756488, 0.1...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/MLP/model.pth</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               stop_early early_stopping_step early_stopping_best_ATT_acc_val  \\\n",
       "0.1 2 512 0.01       True                  30                       87.913669   \n",
       "0.2 2 256 0.01       True                  30                       87.194245   \n",
       "      512 0.01       True                  30                       86.906475   \n",
       "    3 512 0.01       True                  30                       87.482014   \n",
       "0.5 2 512 0.01       True                  30                       85.755396   \n",
       "\n",
       "               early_stopping_best_VAL_acc_val  \\\n",
       "0.1 2 512 0.01                       96.978417   \n",
       "0.2 2 256 0.01                       95.251799   \n",
       "      512 0.01                       98.129496   \n",
       "    3 512 0.01                       98.705036   \n",
       "0.5 2 512 0.01                        99.42446   \n",
       "\n",
       "               early_stopping_best_ATT_acc_val_2 early_stopping_lowest_loss  \\\n",
       "0.1 2 512 0.01                                 0                   4.283473   \n",
       "0.2 2 256 0.01                                 0                   4.247989   \n",
       "      512 0.01                                 0                   4.233886   \n",
       "    3 512 0.01                                 0                    4.23534   \n",
       "0.5 2 512 0.01                                 0                   4.218476   \n",
       "\n",
       "               learning_rate epoch_index  \\\n",
       "0.1 2 512 0.01          0.01          81   \n",
       "0.2 2 256 0.01          0.01          40   \n",
       "      512 0.01          0.01         142   \n",
       "    3 512 0.01          0.01         112   \n",
       "0.5 2 512 0.01          0.01          61   \n",
       "\n",
       "                                                       train_loss  \\\n",
       "0.1 2 512 0.01  [3.668607771396637, 2.8671470483144126, 2.6329...   \n",
       "0.2 2 256 0.01  [3.4082926511764526, 2.7423266967137656, 2.622...   \n",
       "      512 0.01  [3.684901793797811, 2.8937514623006186, 2.7011...   \n",
       "    3 512 0.01  [3.980452517668406, 3.006455639998118, 2.69875...   \n",
       "0.5 2 512 0.01  [3.8542353312174478, 3.08571062485377, 2.85462...   \n",
       "\n",
       "                                                   train_ATT_loss  ...  \\\n",
       "0.1 2 512 0.01  [1.6286450495680282, 0.9909614489679521, 0.998...  ...   \n",
       "0.2 2 256 0.01  [1.3227104675076344, 0.9560339421116414, 0.841...  ...   \n",
       "      512 0.01  [1.5665503717195295, 1.0540965808725753, 0.989...  ...   \n",
       "    3 512 0.01  [1.9148970105971657, 0.9970721310195501, 1.037...  ...   \n",
       "0.5 2 512 0.01  [1.6187379809297684, 1.0744382905827996, 0.968...  ...   \n",
       "\n",
       "                                                      val_VAL_acc  \\\n",
       "0.1 2 512 0.01  [56.97841726618705, 55.97122302158273, 57.6978...   \n",
       "0.2 2 256 0.01  [58.992805755395686, 63.7410071942446, 87.9136...   \n",
       "      512 0.01  [57.697841726618705, 55.10791366906475, 88.201...   \n",
       "    3 512 0.01  [87.4820143884892, 92.5179856115108, 94.244604...   \n",
       "0.5 2 512 0.01  [62.87769784172662, 56.83453237410072, 79.1366...   \n",
       "\n",
       "                                                      val_VAL_jac test_loss  \\\n",
       "0.1 2 512 0.01  [0.1157656767385469, 0.10939705817819499, 0.12...        -1   \n",
       "0.2 2 256 0.01  [0.12546762785465598, 0.1313429269859259, 0.13...        -1   \n",
       "      512 0.01  [0.11342240872142983, 0.11150565833496533, 0.1...        -1   \n",
       "    3 512 0.01  [0.12217197778413622, 0.13116649723739074, 0.1...        -1   \n",
       "0.5 2 512 0.01  [0.11468996452770645, 0.12218568067756488, 0.1...        -1   \n",
       "\n",
       "               test_ATT_loss test_VAL_loss test_ATT_acc test_ATT_acc_2  \\\n",
       "0.1 2 512 0.01            -1            -1           -1             -1   \n",
       "0.2 2 256 0.01            -1            -1           -1             -1   \n",
       "      512 0.01            -1            -1           -1             -1   \n",
       "    3 512 0.01            -1            -1           -1             -1   \n",
       "0.5 2 512 0.01            -1            -1           -1             -1   \n",
       "\n",
       "               test_VAL_acc test_VAL_jac               model_filename  \n",
       "0.1 2 512 0.01           -1           -1  model_storage/MLP/model.pth  \n",
       "0.2 2 256 0.01           -1           -1  model_storage/MLP/model.pth  \n",
       "      512 0.01           -1           -1  model_storage/MLP/model.pth  \n",
       "    3 512 0.01           -1           -1  model_storage/MLP/model.pth  \n",
       "0.5 2 512 0.01           -1           -1  model_storage/MLP/model.pth  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hyper_df[(hyper_df.early_stopping_best_VAL_acc_val + hyper_df.early_stopping_best_VAL_acc_val)>190]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>stop_early</th>\n",
       "      <th>early_stopping_step</th>\n",
       "      <th>early_stopping_best_ATT_acc_val</th>\n",
       "      <th>early_stopping_best_VAL_acc_val</th>\n",
       "      <th>early_stopping_best_ATT_acc_val_2</th>\n",
       "      <th>early_stopping_lowest_loss</th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>epoch_index</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_ATT_loss</th>\n",
       "      <th>...</th>\n",
       "      <th>val_VAL_acc</th>\n",
       "      <th>val_VAL_jac</th>\n",
       "      <th>test_loss</th>\n",
       "      <th>test_ATT_loss</th>\n",
       "      <th>test_VAL_loss</th>\n",
       "      <th>test_ATT_acc</th>\n",
       "      <th>test_ATT_acc_2</th>\n",
       "      <th>test_VAL_acc</th>\n",
       "      <th>test_VAL_jac</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">0.1</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">3</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">256</th>\n",
       "      <th>0.0010</th>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>91.510791</td>\n",
       "      <td>71.366906</td>\n",
       "      <td>0</td>\n",
       "      <td>4.11884</td>\n",
       "      <td>0.001</td>\n",
       "      <td>200</td>\n",
       "      <td>[3.3199336926142373, 2.6071802377700806, 2.500...</td>\n",
       "      <td>[1.1809954174337625, 0.8182199795490487, 0.780...</td>\n",
       "      <td>...</td>\n",
       "      <td>[60.431654676258994, 60.0, 59.13669064748201, ...</td>\n",
       "      <td>[0.13776978623095176, 0.13870503816673224, 0.1...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/MLP/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.0005</th>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>92.230216</td>\n",
       "      <td>73.381295</td>\n",
       "      <td>0</td>\n",
       "      <td>4.113167</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>201</td>\n",
       "      <td>[3.593160013357798, 2.682584981123606, 2.51767...</td>\n",
       "      <td>[1.5237200521035867, 0.9307283595658405, 0.791...</td>\n",
       "      <td>...</td>\n",
       "      <td>[64.74820143884892, 61.43884892086331, 61.7266...</td>\n",
       "      <td>[0.15906475053416738, 0.1399040783051964, 0.14...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/MLP/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <th>256</th>\n",
       "      <th>0.0005</th>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>89.928058</td>\n",
       "      <td>83.309353</td>\n",
       "      <td>0</td>\n",
       "      <td>4.127223</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>161</td>\n",
       "      <td>[3.8874157071113586, 2.8924161195755005, 2.640...</td>\n",
       "      <td>[1.942243436697117, 1.1255941513172478, 0.8200...</td>\n",
       "      <td>...</td>\n",
       "      <td>[69.92805755395683, 59.280575539568346, 59.424...</td>\n",
       "      <td>[0.15179856423851398, 0.16254196818783986, 0.1...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/MLP/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">0.2</th>\n",
       "      <th>2</th>\n",
       "      <th>512</th>\n",
       "      <th>0.0005</th>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>91.942446</td>\n",
       "      <td>74.100719</td>\n",
       "      <td>0</td>\n",
       "      <td>4.126685</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>144</td>\n",
       "      <td>[3.225165367126465, 2.5821815530459085, 2.4668...</td>\n",
       "      <td>[1.0910309771091324, 0.8361637635244227, 0.784...</td>\n",
       "      <td>...</td>\n",
       "      <td>[55.97122302158273, 58.70503597122302, 57.5539...</td>\n",
       "      <td>[0.13990407847672057, 0.14033573256979742, 0.1...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/MLP/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">3</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">256</th>\n",
       "      <th>0.0010</th>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>91.223022</td>\n",
       "      <td>75.395683</td>\n",
       "      <td>0</td>\n",
       "      <td>4.121039</td>\n",
       "      <td>0.001</td>\n",
       "      <td>101</td>\n",
       "      <td>[3.4373897711435952, 2.716141859690348, 2.5443...</td>\n",
       "      <td>[1.2536587077825023, 0.8496173643339374, 0.787...</td>\n",
       "      <td>...</td>\n",
       "      <td>[71.36690647482014, 68.63309352517986, 64.1726...</td>\n",
       "      <td>[0.13952038579707524, 0.1399280601268192, 0.14...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/MLP/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.0005</th>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>91.079137</td>\n",
       "      <td>73.81295</td>\n",
       "      <td>0</td>\n",
       "      <td>4.128302</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>103</td>\n",
       "      <td>[3.6888991792996726, 2.795999844868978, 2.5873...</td>\n",
       "      <td>[1.5622473000819663, 0.9837408716328586, 0.805...</td>\n",
       "      <td>...</td>\n",
       "      <td>[67.4820143884892, 60.57553956834533, 60.86330...</td>\n",
       "      <td>[0.16932853966308153, 0.14354916373602777, 0.1...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/MLP/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>512</th>\n",
       "      <th>0.0010</th>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>90.791367</td>\n",
       "      <td>67.338129</td>\n",
       "      <td>0</td>\n",
       "      <td>4.12224</td>\n",
       "      <td>0.001</td>\n",
       "      <td>81</td>\n",
       "      <td>[3.18015193939209, 2.6315991481145224, 2.53178...</td>\n",
       "      <td>[1.0501962424645463, 0.8789123669885862, 0.818...</td>\n",
       "      <td>...</td>\n",
       "      <td>[63.02158273381295, 59.280575539568346, 57.553...</td>\n",
       "      <td>[0.13764988295465921, 0.13853717124719414, 0.1...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/MLP/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <th>3</th>\n",
       "      <th>512</th>\n",
       "      <th>0.0010</th>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>90.647482</td>\n",
       "      <td>74.388489</td>\n",
       "      <td>0</td>\n",
       "      <td>4.127274</td>\n",
       "      <td>0.001</td>\n",
       "      <td>137</td>\n",
       "      <td>[3.5991070667902627, 2.949184517065684, 2.7242...</td>\n",
       "      <td>[1.3044477602121243, 0.9235615568478022, 0.822...</td>\n",
       "      <td>...</td>\n",
       "      <td>[64.31654676258992, 68.20143884892086, 65.3237...</td>\n",
       "      <td>[0.13964029147470597, 0.1406714653797287, 0.14...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/MLP/model.pth</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 stop_early early_stopping_step  \\\n",
       "0.1 3 256 0.0010       True                  30   \n",
       "          0.0005       True                  30   \n",
       "    5 256 0.0005       True                  30   \n",
       "0.2 2 512 0.0005       True                  30   \n",
       "    3 256 0.0010       True                  30   \n",
       "          0.0005       True                  30   \n",
       "      512 0.0010       True                  30   \n",
       "0.5 3 512 0.0010       True                  30   \n",
       "\n",
       "                 early_stopping_best_ATT_acc_val  \\\n",
       "0.1 3 256 0.0010                       91.510791   \n",
       "          0.0005                       92.230216   \n",
       "    5 256 0.0005                       89.928058   \n",
       "0.2 2 512 0.0005                       91.942446   \n",
       "    3 256 0.0010                       91.223022   \n",
       "          0.0005                       91.079137   \n",
       "      512 0.0010                       90.791367   \n",
       "0.5 3 512 0.0010                       90.647482   \n",
       "\n",
       "                 early_stopping_best_VAL_acc_val  \\\n",
       "0.1 3 256 0.0010                       71.366906   \n",
       "          0.0005                       73.381295   \n",
       "    5 256 0.0005                       83.309353   \n",
       "0.2 2 512 0.0005                       74.100719   \n",
       "    3 256 0.0010                       75.395683   \n",
       "          0.0005                        73.81295   \n",
       "      512 0.0010                       67.338129   \n",
       "0.5 3 512 0.0010                       74.388489   \n",
       "\n",
       "                 early_stopping_best_ATT_acc_val_2 early_stopping_lowest_loss  \\\n",
       "0.1 3 256 0.0010                                 0                    4.11884   \n",
       "          0.0005                                 0                   4.113167   \n",
       "    5 256 0.0005                                 0                   4.127223   \n",
       "0.2 2 512 0.0005                                 0                   4.126685   \n",
       "    3 256 0.0010                                 0                   4.121039   \n",
       "          0.0005                                 0                   4.128302   \n",
       "      512 0.0010                                 0                    4.12224   \n",
       "0.5 3 512 0.0010                                 0                   4.127274   \n",
       "\n",
       "                 learning_rate epoch_index  \\\n",
       "0.1 3 256 0.0010         0.001         200   \n",
       "          0.0005        0.0005         201   \n",
       "    5 256 0.0005        0.0005         161   \n",
       "0.2 2 512 0.0005        0.0005         144   \n",
       "    3 256 0.0010         0.001         101   \n",
       "          0.0005        0.0005         103   \n",
       "      512 0.0010         0.001          81   \n",
       "0.5 3 512 0.0010         0.001         137   \n",
       "\n",
       "                                                         train_loss  \\\n",
       "0.1 3 256 0.0010  [3.3199336926142373, 2.6071802377700806, 2.500...   \n",
       "          0.0005  [3.593160013357798, 2.682584981123606, 2.51767...   \n",
       "    5 256 0.0005  [3.8874157071113586, 2.8924161195755005, 2.640...   \n",
       "0.2 2 512 0.0005  [3.225165367126465, 2.5821815530459085, 2.4668...   \n",
       "    3 256 0.0010  [3.4373897711435952, 2.716141859690348, 2.5443...   \n",
       "          0.0005  [3.6888991792996726, 2.795999844868978, 2.5873...   \n",
       "      512 0.0010  [3.18015193939209, 2.6315991481145224, 2.53178...   \n",
       "0.5 3 512 0.0010  [3.5991070667902627, 2.949184517065684, 2.7242...   \n",
       "\n",
       "                                                     train_ATT_loss  ...  \\\n",
       "0.1 3 256 0.0010  [1.1809954174337625, 0.8182199795490487, 0.780...  ...   \n",
       "          0.0005  [1.5237200521035867, 0.9307283595658405, 0.791...  ...   \n",
       "    5 256 0.0005  [1.942243436697117, 1.1255941513172478, 0.8200...  ...   \n",
       "0.2 2 512 0.0005  [1.0910309771091324, 0.8361637635244227, 0.784...  ...   \n",
       "    3 256 0.0010  [1.2536587077825023, 0.8496173643339374, 0.787...  ...   \n",
       "          0.0005  [1.5622473000819663, 0.9837408716328586, 0.805...  ...   \n",
       "      512 0.0010  [1.0501962424645463, 0.8789123669885862, 0.818...  ...   \n",
       "0.5 3 512 0.0010  [1.3044477602121243, 0.9235615568478022, 0.822...  ...   \n",
       "\n",
       "                                                        val_VAL_acc  \\\n",
       "0.1 3 256 0.0010  [60.431654676258994, 60.0, 59.13669064748201, ...   \n",
       "          0.0005  [64.74820143884892, 61.43884892086331, 61.7266...   \n",
       "    5 256 0.0005  [69.92805755395683, 59.280575539568346, 59.424...   \n",
       "0.2 2 512 0.0005  [55.97122302158273, 58.70503597122302, 57.5539...   \n",
       "    3 256 0.0010  [71.36690647482014, 68.63309352517986, 64.1726...   \n",
       "          0.0005  [67.4820143884892, 60.57553956834533, 60.86330...   \n",
       "      512 0.0010  [63.02158273381295, 59.280575539568346, 57.553...   \n",
       "0.5 3 512 0.0010  [64.31654676258992, 68.20143884892086, 65.3237...   \n",
       "\n",
       "                                                        val_VAL_jac test_loss  \\\n",
       "0.1 3 256 0.0010  [0.13776978623095176, 0.13870503816673224, 0.1...        -1   \n",
       "          0.0005  [0.15906475053416738, 0.1399040783051964, 0.14...        -1   \n",
       "    5 256 0.0005  [0.15179856423851398, 0.16254196818783986, 0.1...        -1   \n",
       "0.2 2 512 0.0005  [0.13990407847672057, 0.14033573256979742, 0.1...        -1   \n",
       "    3 256 0.0010  [0.13952038579707524, 0.1399280601268192, 0.14...        -1   \n",
       "          0.0005  [0.16932853966308153, 0.14354916373602777, 0.1...        -1   \n",
       "      512 0.0010  [0.13764988295465921, 0.13853717124719414, 0.1...        -1   \n",
       "0.5 3 512 0.0010  [0.13964029147470597, 0.1406714653797287, 0.14...        -1   \n",
       "\n",
       "                 test_ATT_loss test_VAL_loss test_ATT_acc test_ATT_acc_2  \\\n",
       "0.1 3 256 0.0010            -1            -1           -1             -1   \n",
       "          0.0005            -1            -1           -1             -1   \n",
       "    5 256 0.0005            -1            -1           -1             -1   \n",
       "0.2 2 512 0.0005            -1            -1           -1             -1   \n",
       "    3 256 0.0010            -1            -1           -1             -1   \n",
       "          0.0005            -1            -1           -1             -1   \n",
       "      512 0.0010            -1            -1           -1             -1   \n",
       "0.5 3 512 0.0010            -1            -1           -1             -1   \n",
       "\n",
       "                 test_VAL_acc test_VAL_jac               model_filename  \n",
       "0.1 3 256 0.0010           -1           -1  model_storage/MLP/model.pth  \n",
       "          0.0005           -1           -1  model_storage/MLP/model.pth  \n",
       "    5 256 0.0005           -1           -1  model_storage/MLP/model.pth  \n",
       "0.2 2 512 0.0005           -1           -1  model_storage/MLP/model.pth  \n",
       "    3 256 0.0010           -1           -1  model_storage/MLP/model.pth  \n",
       "          0.0005           -1           -1  model_storage/MLP/model.pth  \n",
       "      512 0.0010           -1           -1  model_storage/MLP/model.pth  \n",
       "0.5 3 512 0.0010           -1           -1  model_storage/MLP/model.pth  \n",
       "\n",
       "[8 rows x 29 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hyper_df[(hyper_df.early_stopping_lowest_loss<4.13)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.1, 3, 256, 0.001)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(hyper_df['val_VAL_loss'].apply(lambda x: min(x)) + hyper_df['val_ATT_loss'].apply(lambda x: min(x))).index[25]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Re-run model and get Inference Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialization():\n",
    "    set_seed_everywhere(args.seed, args.cuda)\n",
    "    #transform = T.Compose([T.ToSparseTensor()])\n",
    "    dataset = VEN_Homo('dataset/Venice_homo')\n",
    "    data = dataset[0]\n",
    "    data.n_id = torch.arange(data.num_nodes)\n",
    "    data = data.to(device)\n",
    "    \n",
    "    train_loader = NeighborLoader(\n",
    "        data,\n",
    "        # Sample 25 neighbors for each node and edge type for 2 iterations\n",
    "        num_neighbors=[3*args.sample_nodes] * 2,\n",
    "        # Use a batch size of 32 for sampling training nodes\n",
    "        batch_size=args.batch_size,\n",
    "        input_nodes=data.train_mask,\n",
    "    )\n",
    "    val_loader = NeighborLoader(\n",
    "        data,\n",
    "        # Sample 25 neighbors for each node and edge type for 2 iterations\n",
    "        num_neighbors=[3*args.sample_nodes] * 2,\n",
    "        # Use a batch size of 32 for sampling validating nodes\n",
    "        batch_size=args.batch_size,\n",
    "        input_nodes=data.val_mask,\n",
    "    )\n",
    "    test_loader = NeighborLoader(\n",
    "        data,\n",
    "        # Sample 25 neighbors for each node and edge type for 2 iterations\n",
    "        num_neighbors=[3*args.sample_nodes] * 2,\n",
    "        # Use a batch size of 32 for sampling testing nodes\n",
    "        batch_size=args.batch_size,\n",
    "        input_nodes=data.test_mask,\n",
    "    )\n",
    " \n",
    "    model = MLP(in_channels=data.x.shape[-1], hidden_channels = 256, \n",
    "            out_channels = data.y.shape[-1], dropout = 0.1, num_layers=3).to(device)\n",
    "    return data, model, train_loader, val_loader, test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_Homo(model, optimizer, train_loader):\n",
    "    model.train()\n",
    "\n",
    "    total_examples = total_loss = 0\n",
    "    for batch in tqdm(train_loader):\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        batch = batch.to(device)\n",
    "        batch_size = args.batch_size\n",
    "        out = model(batch.x)[:batch_size]\n",
    "        out_att = out[:,:9]\n",
    "        out_val = out[:,9:]\n",
    "        y = batch.y\n",
    "        y_att = y[:,:9]\n",
    "        y_val = y[:,9:]\n",
    "        \n",
    "        loss = F.cross_entropy(out_att, y_att[:batch_size]) + F.cross_entropy(out_val, y_val[:batch_size])\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        total_examples += batch_size\n",
    "        total_loss += float(loss) * batch_size\n",
    "\n",
    "    return total_loss / total_examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def test_Homo(model, loader):\n",
    "    model.eval()\n",
    "\n",
    "    total_examples_att = total_examples_val = 0\n",
    "    running_loss_1 = running_loss_2 = 0.\n",
    "    running_1_acc = 0.\n",
    "    running_1_val = 0.\n",
    "    running_k_acc = 0.\n",
    "    running_k_jac = 0.\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        loss_1 = 0\n",
    "        acc_1_t = 0\n",
    "        loss_2 = 0\n",
    "        acc_1_val = 0\n",
    "        acc_k_t = 0\n",
    "        jac_k_t = 0\n",
    "\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch.batch_size\n",
    "        #edge_index = to_undirected(batch.edge_index)\n",
    "        out = model(batch.x)[:batch_size]\n",
    "        out_att = out[:,:9]\n",
    "        out_val = out[:,9:]\n",
    "        att_node = (batch.att_lab[:batch_size]).nonzero().squeeze()\n",
    "        val_node = (batch.val_lab[:batch_size]).nonzero().squeeze()\n",
    "\n",
    "        #print(type_node)\n",
    "\n",
    "        #pred_att = out_att.argmax(dim=-1)\n",
    "        #pred_val = out_val.argmax(dim=-1)\n",
    "\n",
    "        y = batch.y\n",
    "        y_att = y[:,:9]\n",
    "        y_val = y[:,9:]\n",
    "\n",
    "        if not att_node.shape[0]==0:\n",
    "            loss_1 = F.cross_entropy(out_att[att_node], y_att[:batch_size][att_node])\n",
    "            acc_1_t = compute_1_accuracy(y_att[:batch_size][att_node], out_att[att_node])\n",
    "\n",
    "        if not val_node.shape[0]==0:\n",
    "            loss_2 = F.cross_entropy(out_val[val_node], y_val[val_node])\n",
    "            acc_1_val = compute_1_accuracy(y_val[val_node], out_val[val_node])\n",
    "            acc_k_t = compute_k_accuracy(y_val[val_node], out_val[val_node], args.k)\n",
    "            jac_k_t = compute_jaccard_index(y_val[val_node], F.softmax(out_val[val_node],dim=-1), args.k)\n",
    "            #loss_3 = loss_1 + loss_2\n",
    "\n",
    "        total_examples_att += att_node.shape[0]\n",
    "        total_examples_val += val_node.shape[0]\n",
    "        #total_correct_att += int((pred_att == y_att[:batch_size]).sum())\n",
    "        #total_correct_val += int((pred_val == y_val[:batch_size]).sum())\n",
    "\n",
    "        running_loss_1 += float(loss_1) * att_node.shape[0]\n",
    "        running_loss_2 += float(loss_2) * val_node.shape[0]\n",
    "        running_1_acc += float(acc_1_t) * att_node.shape[0]\n",
    "        running_1_val += float(acc_1_val) * val_node.shape[0]\n",
    "        running_k_acc += float(acc_k_t) * val_node.shape[0]\n",
    "        running_k_jac += float(jac_k_t) * val_node.shape[0]\n",
    "    \n",
    "    return running_loss_1/total_examples_att, running_loss_2/total_examples_val, running_1_acc/ total_examples_att, running_k_acc/ total_examples_val, running_k_jac/ total_examples_val, running_1_val/total_examples_val\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_loop(verbose=False):\n",
    "    \n",
    "    _, model, train_loader, val_loader, test_loader = initialization()\n",
    "    if torch.cuda.device_count() > 1:\n",
    "        print(\"Use {} GPUs !\".format(torch.cuda.device_count()))\n",
    "        model = DataParallel(model)\n",
    "    \n",
    "    optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=args.l2)\n",
    "    #scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer,\n",
    "    #                                           mode='min', factor=0.5,\n",
    "    #                                           patience=1)\n",
    "\n",
    "    train_state = make_train_state(args)\n",
    "\n",
    "    try:\n",
    "        for epoch in range(args.num_epochs):\n",
    "            train_state['epoch_index'] = epoch\n",
    "            \n",
    "            loss = train_Homo(model, optimizer, train_loader)\n",
    "            train_loss_att, train_loss_val, train_att_acc, train_val_acc, train_val_jac, train_val_1 = test_Homo(model, train_loader)\n",
    "            val_loss_att, val_loss_val, val_att_acc, val_val_acc, val_val_jac, val_val_1 = test_Homo(model, val_loader)\n",
    "            if verbose:\n",
    "                print(f'Epoch: {epoch:02d}, Loss: {loss:.4f}, Train_ATT: {train_att_acc:.4f}, Train_VAL: {train_val_acc:.4f}, Val_vis_tex_ATT: {val_att_acc:.4f}, Val_vis_tex_VAL: {val_val_acc:.4f}')\n",
    "            \n",
    "            train_state['train_loss'].append(loss)\n",
    "            train_state['train_ATT_loss'].append(train_loss_att)\n",
    "            train_state['train_VAL_loss'].append(train_loss_val)\n",
    "            train_state['train_ATT_acc'].append(train_att_acc)\n",
    "            train_state['train_VAL_acc'].append(train_val_acc)\n",
    "            train_state['train_VAL_jac'].append(train_val_jac)\n",
    "            train_state['train_VAL_acc_1'].append(train_val_1)\n",
    "            \n",
    "            train_state['val_ATT_loss'].append(val_loss_att)\n",
    "            train_state['val_VAL_loss'].append(val_loss_val)\n",
    "            train_state['val_loss'].append(val_loss_att + 3*val_loss_val)\n",
    "            train_state['val_ATT_acc'].append(val_att_acc)\n",
    "            train_state['val_VAL_acc'].append(val_val_acc)\n",
    "            train_state['val_VAL_jac'].append(val_val_jac)\n",
    "            train_state['val_VAL_acc_1'].append(val_val_1)\n",
    "            \n",
    "            train_state = update_train_state(args=args, model=model,\n",
    "                                                train_state=train_state)\n",
    "            if train_state['stop_early']:\n",
    "                break\n",
    "\n",
    "    except KeyboardInterrupt:\n",
    "        print(\"Exiting loop\")\n",
    "        pass\n",
    "    \n",
    "    return train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_state = training_loop(verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MLP(in_channels=data.x.shape[-1], hidden_channels = 256, \n",
    "            out_channels = data.y.shape[-1], dropout = 0.1, num_layers=3).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load(args.save_dir+'MLP_best_model/model.pth',map_location=torch.device('cpu')))\n",
    "model.eval()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 42.21it/s]\n"
     ]
    }
   ],
   "source": [
    "test_loss_att, test_loss_val, test_att_acc, test_val_acc, test_val_jac, test_val_1 = test_Homo(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 51.19it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7234816146689439,\n",
       " 1.5954706130595748,\n",
       " 100.0,\n",
       " 100.0,\n",
       " 0.9048938222869282,\n",
       " 91.68975069252078)"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 39.86it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7870456847718091,\n",
       " 1.6269368919832954,\n",
       " 98.98373983739837,\n",
       " 99.50738916256158,\n",
       " 0.757799681771565,\n",
       " 80.78817733990148)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 35.16it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7920896851755753,\n",
       " 1.6384961319466431,\n",
       " 98.2107355864811,\n",
       " 99.47916666666667,\n",
       " 0.7413194552063942,\n",
       " 80.20833333333333)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 36.30it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 62.85it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 63.00it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 64.20it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 65.91it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 61.94it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 62.06it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 59.65it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 59.44it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 57.26it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 59.57it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 57.90it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 58.92it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 57.32it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 56.12it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 55.22it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 56.10it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 54.60it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 57.20it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 55.99it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test_Homo(model, val_loader))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test_Homo(model, test_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.00000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.0000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.787046</td>\n",
       "      <td>1.626937</td>\n",
       "      <td>98.98374</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>0.7578</td>\n",
       "      <td>80.788177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.787046</td>\n",
       "      <td>1.626937</td>\n",
       "      <td>98.98374</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>0.7578</td>\n",
       "      <td>80.788177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.787046</td>\n",
       "      <td>1.626937</td>\n",
       "      <td>98.98374</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>0.7578</td>\n",
       "      <td>80.788177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.787046</td>\n",
       "      <td>1.626937</td>\n",
       "      <td>98.98374</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>0.7578</td>\n",
       "      <td>80.788177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.787046</td>\n",
       "      <td>1.626937</td>\n",
       "      <td>98.98374</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>0.7578</td>\n",
       "      <td>80.788177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.787046</td>\n",
       "      <td>1.626937</td>\n",
       "      <td>98.98374</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>0.7578</td>\n",
       "      <td>80.788177</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss   ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.00000  10.000000    10.0000  10.000000\n",
       "mean    0.787046   1.626937  98.98374  99.507389     0.7578  80.788177\n",
       "std     0.000000   0.000000   0.00000   0.000000     0.0000   0.000000\n",
       "min     0.787046   1.626937  98.98374  99.507389     0.7578  80.788177\n",
       "25%     0.787046   1.626937  98.98374  99.507389     0.7578  80.788177\n",
       "50%     0.787046   1.626937  98.98374  99.507389     0.7578  80.788177\n",
       "75%     0.787046   1.626937  98.98374  99.507389     0.7578  80.788177\n",
       "max     0.787046   1.626937  98.98374  99.507389     0.7578  80.788177"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>7.920897e-01</td>\n",
       "      <td>1.638496e+00</td>\n",
       "      <td>9.821074e+01</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.741319</td>\n",
       "      <td>8.020833e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.170278e-16</td>\n",
       "      <td>2.340556e-16</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.497956e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>7.920897e-01</td>\n",
       "      <td>1.638496e+00</td>\n",
       "      <td>9.821074e+01</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.741319</td>\n",
       "      <td>8.020833e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>7.920897e-01</td>\n",
       "      <td>1.638496e+00</td>\n",
       "      <td>9.821074e+01</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.741319</td>\n",
       "      <td>8.020833e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>7.920897e-01</td>\n",
       "      <td>1.638496e+00</td>\n",
       "      <td>9.821074e+01</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.741319</td>\n",
       "      <td>8.020833e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.920897e-01</td>\n",
       "      <td>1.638496e+00</td>\n",
       "      <td>9.821074e+01</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.741319</td>\n",
       "      <td>8.020833e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>7.920897e-01</td>\n",
       "      <td>1.638496e+00</td>\n",
       "      <td>9.821074e+01</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.741319</td>\n",
       "      <td>8.020833e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ATT_loss      VAL_loss       ATT_acc     VAL_k_acc  VAL_k_jac  \\\n",
       "count  1.000000e+01  1.000000e+01  1.000000e+01  1.000000e+01  10.000000   \n",
       "mean   7.920897e-01  1.638496e+00  9.821074e+01  9.947917e+01   0.741319   \n",
       "std    1.170278e-16  2.340556e-16  1.497956e-14  1.497956e-14   0.000000   \n",
       "min    7.920897e-01  1.638496e+00  9.821074e+01  9.947917e+01   0.741319   \n",
       "25%    7.920897e-01  1.638496e+00  9.821074e+01  9.947917e+01   0.741319   \n",
       "50%    7.920897e-01  1.638496e+00  9.821074e+01  9.947917e+01   0.741319   \n",
       "75%    7.920897e-01  1.638496e+00  9.821074e+01  9.947917e+01   0.741319   \n",
       "max    7.920897e-01  1.638496e+00  9.821074e+01  9.947917e+01   0.741319   \n",
       "\n",
       "          VAL_1_acc  \n",
       "count  1.000000e+01  \n",
       "mean   8.020833e+01  \n",
       "std    1.497956e-14  \n",
       "min    8.020833e+01  \n",
       "25%    8.020833e+01  \n",
       "50%    8.020833e+01  \n",
       "75%    8.020833e+01  \n",
       "max    8.020833e+01  "
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'val_metrics.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'test_metrics.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_state['test_ATT_loss']=test_loss_att\n",
    "train_state['test_VAL_loss']=test_loss_val\n",
    "train_state['test_loss']=test_loss_att + 3*test_loss_val\n",
    "train_state['test_ATT_acc']=test_att_acc\n",
    "train_state['test_VAL_acc_1']=test_val_1\n",
    "train_state['test_VAL_acc']=test_val_acc\n",
    "train_state['test_VAL_jac']=test_val_jac"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': True,\n",
       " 'early_stopping_step': 30,\n",
       " 'early_stopping_best_ATT_acc_val': 98.98373983739837,\n",
       " 'early_stopping_best_VAL_acc_val': 100.0,\n",
       " 'early_stopping_best_ATT_acc_val_2': 0,\n",
       " 'early_stopping_lowest_loss': 5.667856360721696,\n",
       " 'learning_rate': 0.001,\n",
       " 'epoch_index': 125,\n",
       " 'train_loss': [3.315506915251414,\n",
       "  2.6138915419578552,\n",
       "  2.5112855633099875,\n",
       "  2.44793168703715,\n",
       "  2.4063382148742676,\n",
       "  2.396766722202301,\n",
       "  2.38583113749822,\n",
       "  2.4065025448799133,\n",
       "  2.404258131980896,\n",
       "  2.383147875467936,\n",
       "  2.3815468351046243,\n",
       "  2.3735955953598022,\n",
       "  2.3644070426623025,\n",
       "  2.3610894282658896,\n",
       "  2.378908177216848,\n",
       "  2.3610124985376992,\n",
       "  2.370857914288839,\n",
       "  2.359621544679006,\n",
       "  2.354199528694153,\n",
       "  2.3597083489100137,\n",
       "  2.3557668924331665,\n",
       "  2.3458130757013955,\n",
       "  2.3741236130396524,\n",
       "  2.3486496408780417,\n",
       "  2.355173031489054,\n",
       "  2.360405961672465,\n",
       "  2.3617889881134033,\n",
       "  2.3627221981684365,\n",
       "  2.365428904692332,\n",
       "  2.352561036745707,\n",
       "  2.355615556240082,\n",
       "  2.3668547868728638,\n",
       "  2.353724201520284,\n",
       "  2.3479246497154236,\n",
       "  2.36415026585261,\n",
       "  2.358995040257772,\n",
       "  2.34091184536616,\n",
       "  2.3573124210039773,\n",
       "  2.3512824376424155,\n",
       "  2.3540567557017007,\n",
       "  2.33818652232488,\n",
       "  2.360440254211426,\n",
       "  2.3555983106295266,\n",
       "  2.3537731766700745,\n",
       "  2.351346433162689,\n",
       "  2.337008833885193,\n",
       "  2.345582207043966,\n",
       "  2.349985738595327,\n",
       "  2.361271023750305,\n",
       "  2.3528001507123313,\n",
       "  2.351381699244181,\n",
       "  2.339920242627462,\n",
       "  2.338102340698242,\n",
       "  2.365539868672689,\n",
       "  2.349204142888387,\n",
       "  2.3440974950790405,\n",
       "  2.3507551352183023,\n",
       "  2.3245892922083535,\n",
       "  2.3482385675112405,\n",
       "  2.3481228748957315,\n",
       "  2.325235346953074,\n",
       "  2.3599313696225486,\n",
       "  2.3546549876530967,\n",
       "  2.357810060183207,\n",
       "  2.334756930669149,\n",
       "  2.349695344765981,\n",
       "  2.3566622336705527,\n",
       "  2.354045271873474,\n",
       "  2.3387094338734946,\n",
       "  2.352731704711914,\n",
       "  2.3481549421946206,\n",
       "  2.3294272422790527,\n",
       "  2.356828033924103,\n",
       "  2.3501622676849365,\n",
       "  2.3479803005854287,\n",
       "  2.3552624185880027,\n",
       "  2.3647202253341675,\n",
       "  2.33521976073583,\n",
       "  2.3560988306999207,\n",
       "  2.3305185238520303,\n",
       "  2.360434611638387,\n",
       "  2.359033465385437,\n",
       "  2.322071293989817,\n",
       "  2.3214895526568093,\n",
       "  2.327380140622457,\n",
       "  2.3262585202852883,\n",
       "  2.3314969539642334,\n",
       "  2.3274068236351013,\n",
       "  2.335904836654663,\n",
       "  2.3426684737205505,\n",
       "  2.342876374721527,\n",
       "  2.3349111080169678,\n",
       "  2.3432106177012124,\n",
       "  2.31166410446167,\n",
       "  2.311119635899862,\n",
       "  2.333995044231415,\n",
       "  2.3298935492833457,\n",
       "  2.33325187365214,\n",
       "  2.3317426443099976,\n",
       "  2.32907634973526,\n",
       "  2.3388867378234863,\n",
       "  2.3475815852483115,\n",
       "  2.3507424791653952,\n",
       "  2.338340441385905,\n",
       "  2.336793522040049,\n",
       "  2.3458454608917236,\n",
       "  2.3455958366394043,\n",
       "  2.3480361898740134,\n",
       "  2.3431071837743125,\n",
       "  2.33378537495931,\n",
       "  2.3274798591931662,\n",
       "  2.3301069140434265,\n",
       "  2.322732706864675,\n",
       "  2.3227789402008057,\n",
       "  2.3424896001815796,\n",
       "  2.3362234234809875,\n",
       "  2.345608949661255,\n",
       "  2.3446009357770285,\n",
       "  2.3434053460756936,\n",
       "  2.340348223845164,\n",
       "  2.325570027033488,\n",
       "  2.3255076805750527,\n",
       "  2.318283994992574,\n",
       "  2.3406328360239663,\n",
       "  2.3401795427004495,\n",
       "  2.321334183216095],\n",
       " 'train_ATT_loss': [1.1807323007372277,\n",
       "  0.8346732445370788,\n",
       "  0.7913130708348388,\n",
       "  0.7623249069475401,\n",
       "  0.7482898068560128,\n",
       "  0.742452864501615,\n",
       "  0.7379347345175176,\n",
       "  0.7374369029853483,\n",
       "  0.7390912944590285,\n",
       "  0.7372184049059479,\n",
       "  0.7390033031434564,\n",
       "  0.733921593576257,\n",
       "  0.7338595763468015,\n",
       "  0.7297192694737971,\n",
       "  0.7283553287923501,\n",
       "  0.729217949666475,\n",
       "  0.7287758095112534,\n",
       "  0.7294680968215921,\n",
       "  0.7337711770448658,\n",
       "  0.7265936550671374,\n",
       "  0.7293626403544418,\n",
       "  0.7261183915706222,\n",
       "  0.7301824016914473,\n",
       "  0.7289971601930021,\n",
       "  0.731632434265105,\n",
       "  0.7251490328120392,\n",
       "  0.734608584989141,\n",
       "  0.7283674869841156,\n",
       "  0.7373079433005272,\n",
       "  0.7289680954161774,\n",
       "  0.7280120438485925,\n",
       "  0.7297258428259239,\n",
       "  0.7370304886323924,\n",
       "  0.7395011733113234,\n",
       "  0.73822708687954,\n",
       "  0.7363933666591169,\n",
       "  0.7311161992622545,\n",
       "  0.7250727132747048,\n",
       "  0.7260351509598814,\n",
       "  0.7238614963692641,\n",
       "  0.7276673113870489,\n",
       "  0.7239226239870129,\n",
       "  0.7260297026660634,\n",
       "  0.7240592396490462,\n",
       "  0.7237978542940768,\n",
       "  0.7246481450971144,\n",
       "  0.7241129805836981,\n",
       "  0.7247084770506439,\n",
       "  0.7255178412572169,\n",
       "  0.7283508505187206,\n",
       "  0.7277214033451768,\n",
       "  0.7309553994696556,\n",
       "  0.7315554977123757,\n",
       "  0.7270622321112995,\n",
       "  0.7267661687409779,\n",
       "  0.7288050537624517,\n",
       "  0.7308093476163383,\n",
       "  0.7249674265404488,\n",
       "  0.7267682025637323,\n",
       "  0.7258350314195797,\n",
       "  0.727228611129803,\n",
       "  0.7229952803939333,\n",
       "  0.7271918144252492,\n",
       "  0.7261979233194916,\n",
       "  0.7259046392427587,\n",
       "  0.7257842893745761,\n",
       "  0.7288182277098256,\n",
       "  0.7239775679118086,\n",
       "  0.7249049091933507,\n",
       "  0.7245041936058086,\n",
       "  0.7222229415690139,\n",
       "  0.7227797488426568,\n",
       "  0.7255396539154475,\n",
       "  0.7252033742511041,\n",
       "  0.72587939791402,\n",
       "  0.7234525350322354,\n",
       "  0.7241668435345066,\n",
       "  0.7232431270739378,\n",
       "  0.7329989182982088,\n",
       "  0.7342511436945844,\n",
       "  0.725055743122365,\n",
       "  0.7232043693600599,\n",
       "  0.7234657991295707,\n",
       "  0.723130150184737,\n",
       "  0.7218681040563082,\n",
       "  0.7218564203901634,\n",
       "  0.7208370853659188,\n",
       "  0.7230133305296013,\n",
       "  0.7240380303020952,\n",
       "  0.7233188373560391,\n",
       "  0.7248994229242742,\n",
       "  0.7248899434742174,\n",
       "  0.7232259946186457,\n",
       "  0.7264378364726777,\n",
       "  0.7255939288152552,\n",
       "  0.7234816146689439,\n",
       "  0.7236339941909768,\n",
       "  0.7271425557268624,\n",
       "  0.7254917372296722,\n",
       "  0.7264345032356453,\n",
       "  0.7262202450112953,\n",
       "  0.7269052458601976,\n",
       "  0.7232261293482583,\n",
       "  0.722396399671021,\n",
       "  0.7229266439118214,\n",
       "  0.7218697114003992,\n",
       "  0.7271411451936759,\n",
       "  0.7228683122637529,\n",
       "  0.7218641931000178,\n",
       "  0.7222074669814176,\n",
       "  0.7225852073724911,\n",
       "  0.7239003841896797,\n",
       "  0.7222239069661275,\n",
       "  0.7222710694302482,\n",
       "  0.7216090794415355,\n",
       "  0.7220485555498224,\n",
       "  0.7219874924900129,\n",
       "  0.7218496791213503,\n",
       "  0.7224793310310702,\n",
       "  0.7253531017792192,\n",
       "  0.7286507335396024,\n",
       "  0.723357649886377,\n",
       "  0.7245045125979797,\n",
       "  0.723409932240885,\n",
       "  0.7242659728282707,\n",
       "  0.7221021492065155],\n",
       " 'train_VAL_loss': [1.6907099733062068,\n",
       "  1.6555064992561235,\n",
       "  1.638565930303114,\n",
       "  1.630911952570865,\n",
       "  1.6239503417318877,\n",
       "  1.6236811780533302,\n",
       "  1.6204671806906068,\n",
       "  1.61575301400182,\n",
       "  1.6157429139039523,\n",
       "  1.6124407581345197,\n",
       "  1.611905323501439,\n",
       "  1.610948794106037,\n",
       "  1.6116009811945569,\n",
       "  1.6088876377512544,\n",
       "  1.6092325320204208,\n",
       "  1.6070750783354952,\n",
       "  1.605357168784102,\n",
       "  1.6086414517458125,\n",
       "  1.603813540242055,\n",
       "  1.603957660640706,\n",
       "  1.605435041179287,\n",
       "  1.6032525373627935,\n",
       "  1.6037006810761554,\n",
       "  1.6038115084006186,\n",
       "  1.6053152325410922,\n",
       "  1.603896264554391,\n",
       "  1.6047955837276173,\n",
       "  1.6031061628848893,\n",
       "  1.6036990735669545,\n",
       "  1.605631413882459,\n",
       "  1.6067103959847024,\n",
       "  1.6036812833141423,\n",
       "  1.602365180395977,\n",
       "  1.6010099464506324,\n",
       "  1.6013355727341037,\n",
       "  1.5996637585420688,\n",
       "  1.5987914093973894,\n",
       "  1.598450897803267,\n",
       "  1.6008718878938881,\n",
       "  1.6002524724627465,\n",
       "  1.5999812762823131,\n",
       "  1.5993115631166919,\n",
       "  1.6002655716153724,\n",
       "  1.5998922916660678,\n",
       "  1.5986169322375776,\n",
       "  1.59853862329203,\n",
       "  1.599268313920399,\n",
       "  1.598477152245857,\n",
       "  1.5993426305105152,\n",
       "  1.5978638812115318,\n",
       "  1.6047673697616915,\n",
       "  1.5989506683851544,\n",
       "  1.5993412254259527,\n",
       "  1.5989407492476486,\n",
       "  1.5988503978523192,\n",
       "  1.6001930722239275,\n",
       "  1.5974638366302956,\n",
       "  1.6026746186523226,\n",
       "  1.5992718674139303,\n",
       "  1.6028340866691189,\n",
       "  1.5996959037727927,\n",
       "  1.5989634462340716,\n",
       "  1.5977278904901646,\n",
       "  1.5972424362504911,\n",
       "  1.5984178682443508,\n",
       "  1.5960366257009744,\n",
       "  1.5970546598249524,\n",
       "  1.5962258801896156,\n",
       "  1.5980048057445198,\n",
       "  1.600149948841317,\n",
       "  1.5963356108546587,\n",
       "  1.6004052066406715,\n",
       "  1.5982318895345249,\n",
       "  1.596180488528307,\n",
       "  1.5981065909948375,\n",
       "  1.5975109929853528,\n",
       "  1.5964571812146258,\n",
       "  1.5990626607245024,\n",
       "  1.596289495682122,\n",
       "  1.5971959360418557,\n",
       "  1.5971409815500317,\n",
       "  1.5988507987389604,\n",
       "  1.5963470661739234,\n",
       "  1.5956972999255743,\n",
       "  1.5987006491901472,\n",
       "  1.5955582913269297,\n",
       "  1.5960834075869617,\n",
       "  1.5974365664320969,\n",
       "  1.5974302328194276,\n",
       "  1.5962048666629105,\n",
       "  1.5963555283823831,\n",
       "  1.5969595195844233,\n",
       "  1.5972698517453308,\n",
       "  1.5960457724547452,\n",
       "  1.5983422093114035,\n",
       "  1.5954706130595748,\n",
       "  1.5962920248343344,\n",
       "  1.597828174562005,\n",
       "  1.5976084431122546,\n",
       "  1.5971800640349243,\n",
       "  1.5981049715977296,\n",
       "  1.5966067010345881,\n",
       "  1.5957710567934031,\n",
       "  1.5962867205162787,\n",
       "  1.5984755500201704,\n",
       "  1.5956118909275763,\n",
       "  1.5980440447535211,\n",
       "  1.5999906591431257,\n",
       "  1.596649578403568,\n",
       "  1.5986312660814321,\n",
       "  1.5973677658308245,\n",
       "  1.5956818479580233,\n",
       "  1.5979804104384954,\n",
       "  1.5964857584221541,\n",
       "  1.5986125726779081,\n",
       "  1.5967174058475653,\n",
       "  1.5985409222811544,\n",
       "  1.59631480173391,\n",
       "  1.5963259875939493,\n",
       "  1.5982450474662464,\n",
       "  1.5961493048971709,\n",
       "  1.5963151144519077,\n",
       "  1.5991577768589982,\n",
       "  1.5959841686602774,\n",
       "  1.6002521244112475,\n",
       "  1.596457786837443],\n",
       " 'train_ATT_acc': [81.4404432132964,\n",
       "  96.95290858725762,\n",
       "  98.61495844875347,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.44598337950139,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0],\n",
       " 'train_VAL_acc': [95.8448753462604,\n",
       "  98.89196675900277,\n",
       "  99.16897506925208,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.44598337950139,\n",
       "  98.33795013850416,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.16897506925208,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.16897506925208,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.16897506925208,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.44598337950139,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0],\n",
       " 'train_VAL_jac': [0.5572483929240473,\n",
       "  0.6671283595119487,\n",
       "  0.7262234621761248,\n",
       "  0.7414589245233509,\n",
       "  0.7548476507128771,\n",
       "  0.7506925313426517,\n",
       "  0.7280701824832821,\n",
       "  0.7816251242259863,\n",
       "  0.7853185701238151,\n",
       "  0.7756232739815752,\n",
       "  0.8074792296602455,\n",
       "  0.7973222653291232,\n",
       "  0.7903970522893763,\n",
       "  0.8176362045583963,\n",
       "  0.7973222653291232,\n",
       "  0.8259464538658755,\n",
       "  0.8314866200708616,\n",
       "  0.7968605812896982,\n",
       "  0.8337950191339297,\n",
       "  0.8268698166612113,\n",
       "  0.8310249413149509,\n",
       "  0.8554940051979636,\n",
       "  0.833333340378019,\n",
       "  0.8527239168119563,\n",
       "  0.8047091465577525,\n",
       "  0.842105268441409,\n",
       "  0.843028636520259,\n",
       "  0.8337950191339297,\n",
       "  0.8199446036214644,\n",
       "  0.8139427639440816,\n",
       "  0.8144044426999925,\n",
       "  0.8287165369683686,\n",
       "  0.8670360110803325,\n",
       "  0.8818097946716478,\n",
       "  0.8476454346463951,\n",
       "  0.8245614070311147,\n",
       "  0.8684210631986073,\n",
       "  0.8679593791591824,\n",
       "  0.8573407255051209,\n",
       "  0.851800564583649,\n",
       "  0.8522622380560455,\n",
       "  0.8439519887485663,\n",
       "  0.8767313072225724,\n",
       "  0.8790397115691547,\n",
       "  0.872114503812922,\n",
       "  0.8661126535685109,\n",
       "  0.8531856061348955,\n",
       "  0.8707294622616755,\n",
       "  0.8481071134023058,\n",
       "  0.8804247531204012,\n",
       "  0.8153278054953282,\n",
       "  0.8698060994663397,\n",
       "  0.8642659332613536,\n",
       "  0.8711911463011005,\n",
       "  0.8578024095445459,\n",
       "  0.8430286312367448,\n",
       "  0.8711911410175862,\n",
       "  0.8384118278270943,\n",
       "  0.8614958607258889,\n",
       "  0.8277931741730328,\n",
       "  0.8633425704660178,\n",
       "  0.8684210631986073,\n",
       "  0.8610341714029497,\n",
       "  0.8799630690809762,\n",
       "  0.866112658852025,\n",
       "  0.8993536507984278,\n",
       "  0.8679593844426966,\n",
       "  0.8822714734275585,\n",
       "  0.8813481106322227,\n",
       "  0.8591874458122781,\n",
       "  0.8878116396325447,\n",
       "  0.8444136727879913,\n",
       "  0.8518005593001348,\n",
       "  0.8970452517353596,\n",
       "  0.8771929859784832,\n",
       "  0.8748845816319009,\n",
       "  0.874884586915415,\n",
       "  0.8559556839538743,\n",
       "  0.8873499608766339,\n",
       "  0.8716528197734971,\n",
       "  0.8771929912619973,\n",
       "  0.8813481106322227,\n",
       "  0.8827331521834693,\n",
       "  0.8951985261446882,\n",
       "  0.8531856061348955,\n",
       "  0.8887350024278805,\n",
       "  0.892428443042195,\n",
       "  0.8855032352859624,\n",
       "  0.8771929859784832,\n",
       "  0.8891966864673054,\n",
       "  0.9021237339009208,\n",
       "  0.8882733236719697,\n",
       "  0.8647276173007785,\n",
       "  0.9104339832084001,\n",
       "  0.857340730788635,\n",
       "  0.9048938222869282,\n",
       "  0.8771929912619973,\n",
       "  0.8674976951197574,\n",
       "  0.8642659332613536,\n",
       "  0.881348115915737,\n",
       "  0.8541089530796886,\n",
       "  0.8887349971443662,\n",
       "  0.8938134845934416,\n",
       "  0.8905817227350378,\n",
       "  0.8545706371191135,\n",
       "  0.8891966811837912,\n",
       "  0.8628808917101071,\n",
       "  0.8342566978898405,\n",
       "  0.8711911463011005,\n",
       "  0.8494921602370666,\n",
       "  0.8790397115691547,\n",
       "  0.9104339832084001,\n",
       "  0.8481071186858201,\n",
       "  0.8938134898769559,\n",
       "  0.8457987090557236,\n",
       "  0.9030470966962566,\n",
       "  0.8702677782222504,\n",
       "  0.861957523631257,\n",
       "  0.8785780328132439,\n",
       "  0.8550323264420527,\n",
       "  0.8804247531204012,\n",
       "  0.8845798777741408,\n",
       "  0.8633425704660178,\n",
       "  0.889196675900277,\n",
       "  0.8513388805442239,\n",
       "  0.8762696231831474],\n",
       " 'train_VAL_acc_1': [74.23822714681441,\n",
       "  73.6842105263158,\n",
       "  78.39335180055402,\n",
       "  77.8393351800554,\n",
       "  83.37950138504155,\n",
       "  80.60941828254848,\n",
       "  79.22437673130194,\n",
       "  84.7645429362881,\n",
       "  81.7174515235457,\n",
       "  83.65650969529086,\n",
       "  88.6426592797784,\n",
       "  80.88642659279779,\n",
       "  85.31855955678671,\n",
       "  84.7645429362881,\n",
       "  85.0415512465374,\n",
       "  86.42659279778394,\n",
       "  86.98060941828255,\n",
       "  81.99445983379502,\n",
       "  86.14958448753463,\n",
       "  88.36565096952909,\n",
       "  84.48753462603878,\n",
       "  87.81163434903047,\n",
       "  85.59556786703601,\n",
       "  85.31855955678671,\n",
       "  83.37950138504155,\n",
       "  88.6426592797784,\n",
       "  84.7645429362881,\n",
       "  87.53462603878117,\n",
       "  85.31855955678671,\n",
       "  81.4404432132964,\n",
       "  83.93351800554017,\n",
       "  86.70360110803324,\n",
       "  84.7645429362881,\n",
       "  89.75069252077563,\n",
       "  86.70360110803324,\n",
       "  87.81163434903047,\n",
       "  87.25761772853186,\n",
       "  90.58171745152355,\n",
       "  88.9196675900277,\n",
       "  88.36565096952909,\n",
       "  86.70360110803324,\n",
       "  90.02770083102493,\n",
       "  88.9196675900277,\n",
       "  88.6426592797784,\n",
       "  89.19667590027701,\n",
       "  90.85872576177286,\n",
       "  89.19667590027701,\n",
       "  90.30470914127424,\n",
       "  87.53462603878117,\n",
       "  91.41274238227147,\n",
       "  85.87257617728532,\n",
       "  88.6426592797784,\n",
       "  90.02770083102493,\n",
       "  90.02770083102493,\n",
       "  87.81163434903047,\n",
       "  87.25761772853186,\n",
       "  91.13573407202216,\n",
       "  85.0415512465374,\n",
       "  85.87257617728532,\n",
       "  83.65650969529086,\n",
       "  85.0415512465374,\n",
       "  89.19667590027701,\n",
       "  90.30470914127424,\n",
       "  88.08864265927978,\n",
       "  89.19667590027701,\n",
       "  89.47368421052632,\n",
       "  90.58171745152355,\n",
       "  90.58171745152355,\n",
       "  89.47368421052632,\n",
       "  87.53462603878117,\n",
       "  91.41274238227147,\n",
       "  85.0415512465374,\n",
       "  89.47368421052632,\n",
       "  92.797783933518,\n",
       "  88.36565096952909,\n",
       "  90.02770083102493,\n",
       "  89.47368421052632,\n",
       "  86.42659279778394,\n",
       "  90.85872576177286,\n",
       "  92.5207756232687,\n",
       "  92.24376731301939,\n",
       "  90.58171745152355,\n",
       "  89.75069252077563,\n",
       "  92.797783933518,\n",
       "  86.98060941828255,\n",
       "  92.5207756232687,\n",
       "  93.90581717451524,\n",
       "  89.75069252077563,\n",
       "  90.02770083102493,\n",
       "  88.6426592797784,\n",
       "  90.30470914127424,\n",
       "  90.58171745152355,\n",
       "  91.13573407202216,\n",
       "  91.68975069252078,\n",
       "  88.6426592797784,\n",
       "  91.68975069252078,\n",
       "  90.85872576177286,\n",
       "  89.75069252077563,\n",
       "  89.75069252077563,\n",
       "  88.9196675900277,\n",
       "  86.70360110803324,\n",
       "  91.96675900277009,\n",
       "  91.13573407202216,\n",
       "  91.13573407202216,\n",
       "  88.9196675900277,\n",
       "  92.797783933518,\n",
       "  90.02770083102493,\n",
       "  86.42659279778394,\n",
       "  91.96675900277009,\n",
       "  88.6426592797784,\n",
       "  88.08864265927978,\n",
       "  92.5207756232687,\n",
       "  89.75069252077563,\n",
       "  91.13573407202216,\n",
       "  86.42659279778394,\n",
       "  89.19667590027701,\n",
       "  90.85872576177286,\n",
       "  90.02770083102493,\n",
       "  91.41274238227147,\n",
       "  86.42659279778394,\n",
       "  93.07479224376732,\n",
       "  91.41274238227147,\n",
       "  88.36565096952909,\n",
       "  91.68975069252078,\n",
       "  86.42659279778394,\n",
       "  89.47368421052632],\n",
       " 'val_loss': [6.786652300651103,\n",
       "  6.360190982442676,\n",
       "  6.2288531235948605,\n",
       "  6.115193779986594,\n",
       "  6.044656065460167,\n",
       "  5.985284018209315,\n",
       "  5.961756707269637,\n",
       "  5.924210196334464,\n",
       "  5.908337831325914,\n",
       "  5.903636410178327,\n",
       "  5.87087368710321,\n",
       "  5.856707908751739,\n",
       "  5.843579170557103,\n",
       "  5.801966933555108,\n",
       "  5.786363701161712,\n",
       "  5.77248363252172,\n",
       "  5.770100337454194,\n",
       "  5.785533872710923,\n",
       "  5.759090773127783,\n",
       "  5.760023836429445,\n",
       "  5.765080806424861,\n",
       "  5.7580672872931125,\n",
       "  5.7341249626228485,\n",
       "  5.736692881645875,\n",
       "  5.746843912217953,\n",
       "  5.73909239286313,\n",
       "  5.741158435123193,\n",
       "  5.727227590505192,\n",
       "  5.72991501514254,\n",
       "  5.728354542070708,\n",
       "  5.744533585207168,\n",
       "  5.729219903830241,\n",
       "  5.732681299282154,\n",
       "  5.728851157058736,\n",
       "  5.71557555749766,\n",
       "  5.713870016177123,\n",
       "  5.715237203562182,\n",
       "  5.700324931876589,\n",
       "  5.709363730755537,\n",
       "  5.6959390377595165,\n",
       "  5.6992769685366005,\n",
       "  5.688744846548062,\n",
       "  5.708723323084222,\n",
       "  5.695802763761501,\n",
       "  5.688843808900804,\n",
       "  5.691302401034373,\n",
       "  5.684530987017905,\n",
       "  5.691373654315328,\n",
       "  5.682943192441737,\n",
       "  5.689003685325405,\n",
       "  5.716702515593809,\n",
       "  5.6990228690924045,\n",
       "  5.6946407582189655,\n",
       "  5.695462360719121,\n",
       "  5.690095529397076,\n",
       "  5.7145096291801165,\n",
       "  5.6890394258330375,\n",
       "  5.697241916001676,\n",
       "  5.693458592736768,\n",
       "  5.706636462051932,\n",
       "  5.6947845874673595,\n",
       "  5.68982742624633,\n",
       "  5.680233423126165,\n",
       "  5.674671922052568,\n",
       "  5.684109267216336,\n",
       "  5.671584411102644,\n",
       "  5.6729896472484045,\n",
       "  5.670974756131384,\n",
       "  5.680017038471417,\n",
       "  5.695665499278186,\n",
       "  5.674621893544832,\n",
       "  5.700976373978021,\n",
       "  5.6785419995700055,\n",
       "  5.67320115099316,\n",
       "  5.68928554799797,\n",
       "  5.675877252121802,\n",
       "  5.669936943756737,\n",
       "  5.683813054928448,\n",
       "  5.6765442260409085,\n",
       "  5.684538671263677,\n",
       "  5.68371117868583,\n",
       "  5.687851887594508,\n",
       "  5.680666700604108,\n",
       "  5.672798564660926,\n",
       "  5.679808034400778,\n",
       "  5.667471908299955,\n",
       "  5.676391174906728,\n",
       "  5.675973177067179,\n",
       "  5.683262167316524,\n",
       "  5.667305806068875,\n",
       "  5.668211307108816,\n",
       "  5.6735543743586945,\n",
       "  5.671162366717272,\n",
       "  5.680413884546813,\n",
       "  5.686370772922315,\n",
       "  5.667856360721696,\n",
       "  5.675435796690373,\n",
       "  5.685923700841653,\n",
       "  5.664023541052215,\n",
       "  5.677959857582019,\n",
       "  5.679095527683926,\n",
       "  5.676072517540841,\n",
       "  5.670620768739529,\n",
       "  5.667820159987858,\n",
       "  5.670156581800049,\n",
       "  5.669598048034636,\n",
       "  5.674933631464703,\n",
       "  5.695524524858696,\n",
       "  5.667568649484501,\n",
       "  5.671138076887571,\n",
       "  5.669596618614087,\n",
       "  5.6668804301475415,\n",
       "  5.681345900811094,\n",
       "  5.668821531660833,\n",
       "  5.673083971944898,\n",
       "  5.660235869345116,\n",
       "  5.68127953238847,\n",
       "  5.667714484255884,\n",
       "  5.665798574125072,\n",
       "  5.671314066466701,\n",
       "  5.672873122032229,\n",
       "  5.663076616391899,\n",
       "  5.681835823347999,\n",
       "  5.667333615694408,\n",
       "  5.671488684555944,\n",
       "  5.6660093466548265],\n",
       " 'val_ATT_loss': [1.6766852186462744,\n",
       "  1.3852082844672164,\n",
       "  1.2839880741950942,\n",
       "  1.1837415091875123,\n",
       "  1.1186738221383676,\n",
       "  1.05767104465787,\n",
       "  1.0271153172584084,\n",
       "  1.0073281270944006,\n",
       "  0.9985874642928442,\n",
       "  0.9914708553048653,\n",
       "  0.9649423910834924,\n",
       "  0.9594839475746078,\n",
       "  0.9315312118307362,\n",
       "  0.8959515597519836,\n",
       "  0.8838345965718836,\n",
       "  0.8782044162110585,\n",
       "  0.8755546513854003,\n",
       "  0.8761198590926038,\n",
       "  0.8671912461276946,\n",
       "  0.8600453637479767,\n",
       "  0.8655406199092788,\n",
       "  0.8602316432852086,\n",
       "  0.8457089988439064,\n",
       "  0.838054743602993,\n",
       "  0.8466328845276097,\n",
       "  0.8358808444525169,\n",
       "  0.8337268091556502,\n",
       "  0.8266638835998086,\n",
       "  0.8299028426408768,\n",
       "  0.8310545664734956,\n",
       "  0.8363611427991371,\n",
       "  0.8276703965857746,\n",
       "  0.8325216413998022,\n",
       "  0.8374566094177526,\n",
       "  0.8254251707860125,\n",
       "  0.8277044742088008,\n",
       "  0.827335010093402,\n",
       "  0.8155477048178029,\n",
       "  0.8155911304359513,\n",
       "  0.8076290252005182,\n",
       "  0.8100979061630683,\n",
       "  0.8072429305411936,\n",
       "  0.8110978021854307,\n",
       "  0.807635560995195,\n",
       "  0.8024777381158457,\n",
       "  0.8041832505202875,\n",
       "  0.7998521622361207,\n",
       "  0.8046159440182089,\n",
       "  0.7989218603304731,\n",
       "  0.8059643782008954,\n",
       "  0.8085975292010036,\n",
       "  0.8103793641416038,\n",
       "  0.8087981048880554,\n",
       "  0.8070732419326053,\n",
       "  0.8020412031470275,\n",
       "  0.808596450623458,\n",
       "  0.802551025176436,\n",
       "  0.7995982794015388,\n",
       "  0.7990325657817406,\n",
       "  0.7962535305963299,\n",
       "  0.7998567663314866,\n",
       "  0.7962194774451294,\n",
       "  0.7963653231054787,\n",
       "  0.7905665826264435,\n",
       "  0.7942440737553729,\n",
       "  0.792532852025536,\n",
       "  0.7951430386401773,\n",
       "  0.7904857250490809,\n",
       "  0.7904034106712031,\n",
       "  0.7963409429885507,\n",
       "  0.7922456355356589,\n",
       "  0.7970931370083879,\n",
       "  0.791914638465013,\n",
       "  0.7900320108586211,\n",
       "  0.7937464104677604,\n",
       "  0.7895155838592266,\n",
       "  0.7944714637791238,\n",
       "  0.7962959967250747,\n",
       "  0.8028015610648365,\n",
       "  0.802805153335013,\n",
       "  0.7988972093273954,\n",
       "  0.7928283437480771,\n",
       "  0.7972296936725213,\n",
       "  0.7907223018204294,\n",
       "  0.7922865366790353,\n",
       "  0.789772924247796,\n",
       "  0.7890797632981122,\n",
       "  0.78469968205545,\n",
       "  0.7899029989068101,\n",
       "  0.78781774191837,\n",
       "  0.7933797692138005,\n",
       "  0.791774512791052,\n",
       "  0.7929446725099067,\n",
       "  0.7921237582113685,\n",
       "  0.7911802186229364,\n",
       "  0.7870456847718091,\n",
       "  0.7905193202621569,\n",
       "  0.7944193614449927,\n",
       "  0.7893145180087748,\n",
       "  0.7935179251480878,\n",
       "  0.7945324333944941,\n",
       "  0.7920692588497953,\n",
       "  0.7884698444750251,\n",
       "  0.7876273381516217,\n",
       "  0.7868955031158479,\n",
       "  0.7887746736528428,\n",
       "  0.7886732212169384,\n",
       "  0.793856091373335,\n",
       "  0.7935794027355628,\n",
       "  0.7854708567625139,\n",
       "  0.786472581872126,\n",
       "  0.7875829129441967,\n",
       "  0.7879339549841919,\n",
       "  0.7845570073137439,\n",
       "  0.7876636736034378,\n",
       "  0.7860485619403482,\n",
       "  0.7874290238066417,\n",
       "  0.7911131816908596,\n",
       "  0.7867677979837588,\n",
       "  0.7924009780815946,\n",
       "  0.796498745801003,\n",
       "  0.7829087280887899,\n",
       "  0.7871243494555233,\n",
       "  0.786917486084186,\n",
       "  0.7835216846892504,\n",
       "  0.7856695591434231],\n",
       " 'val_VAL_loss': [1.7033223606682764,\n",
       "  1.6583275659918197,\n",
       "  1.648288349799922,\n",
       "  1.643817423599694,\n",
       "  1.6419940811072664,\n",
       "  1.6425376578504816,\n",
       "  1.6448804633370762,\n",
       "  1.6389606897466875,\n",
       "  1.6365834556776901,\n",
       "  1.6373885182911538,\n",
       "  1.6353104320065728,\n",
       "  1.6324079870590436,\n",
       "  1.6373493195754554,\n",
       "  1.6353384579343748,\n",
       "  1.6341763681966095,\n",
       "  1.631426405436887,\n",
       "  1.6315152286895978,\n",
       "  1.6364713378727729,\n",
       "  1.6306331756666963,\n",
       "  1.6333261575604894,\n",
       "  1.6331800621718608,\n",
       "  1.6326118813359678,\n",
       "  1.629471987926314,\n",
       "  1.6328793793476273,\n",
       "  1.633403675896781,\n",
       "  1.6344038494702042,\n",
       "  1.6358105419891809,\n",
       "  1.6335212356351279,\n",
       "  1.6333373908338875,\n",
       "  1.6324333251990708,\n",
       "  1.6360574808026769,\n",
       "  1.6338498357481557,\n",
       "  1.6333865526274507,\n",
       "  1.6304648492136613,\n",
       "  1.6300501289038822,\n",
       "  1.628721847322774,\n",
       "  1.62930073115626,\n",
       "  1.6282590756862623,\n",
       "  1.6312575334398618,\n",
       "  1.6294366708529995,\n",
       "  1.6297263541245108,\n",
       "  1.6271673053356226,\n",
       "  1.632541840299597,\n",
       "  1.6293890675887686,\n",
       "  1.6287886902616528,\n",
       "  1.6290397168380286,\n",
       "  1.6282262749272614,\n",
       "  1.6289192367657064,\n",
       "  1.6280071107037548,\n",
       "  1.627679769041503,\n",
       "  1.6360349954642686,\n",
       "  1.6295478349836001,\n",
       "  1.6286142177769702,\n",
       "  1.6294630395955052,\n",
       "  1.6293514420833495,\n",
       "  1.6353043928522195,\n",
       "  1.6288294668855339,\n",
       "  1.6325478788667125,\n",
       "  1.6314753423183423,\n",
       "  1.636794310485201,\n",
       "  1.6316426070452912,\n",
       "  1.6312026496004002,\n",
       "  1.6279560333402285,\n",
       "  1.6280351131420416,\n",
       "  1.6299550644869876,\n",
       "  1.6263505196923693,\n",
       "  1.6259488695360758,\n",
       "  1.6268296770274346,\n",
       "  1.6298712092667378,\n",
       "  1.6331081854298783,\n",
       "  1.6274587526697244,\n",
       "  1.6346277456565443,\n",
       "  1.6288757870349977,\n",
       "  1.627723046711513,\n",
       "  1.6318463791767364,\n",
       "  1.628787222754192,\n",
       "  1.6251551599925376,\n",
       "  1.6291723527344577,\n",
       "  1.6245808883253576,\n",
       "  1.6272445059762213,\n",
       "  1.6282713231194783,\n",
       "  1.631674514615477,\n",
       "  1.6278123356438623,\n",
       "  1.6273587542801655,\n",
       "  1.6291738325739142,\n",
       "  1.62589966135072,\n",
       "  1.6291038038695387,\n",
       "  1.630424498337243,\n",
       "  1.631119722803238,\n",
       "  1.6264960213835016,\n",
       "  1.624943845965005,\n",
       "  1.627259953855881,\n",
       "  1.6260725647357885,\n",
       "  1.6294300421118149,\n",
       "  1.6317301847664594,\n",
       "  1.6269368919832954,\n",
       "  1.6283054921427385,\n",
       "  1.6305014464655534,\n",
       "  1.6249030076811466,\n",
       "  1.6281473108113105,\n",
       "  1.6281876980964773,\n",
       "  1.6280010862303484,\n",
       "  1.6273836414215013,\n",
       "  1.626730940612079,\n",
       "  1.6277536928947336,\n",
       "  1.6269411247939312,\n",
       "  1.6287534700825883,\n",
       "  1.6338894778284534,\n",
       "  1.6246630822496462,\n",
       "  1.628555740041686,\n",
       "  1.6277080122473204,\n",
       "  1.6264325057344484,\n",
       "  1.6311373152756339,\n",
       "  1.628088174782363,\n",
       "  1.6284734327804866,\n",
       "  1.6247291024682557,\n",
       "  1.6312835028606096,\n",
       "  1.625533767521675,\n",
       "  1.626343592047104,\n",
       "  1.6263043627950358,\n",
       "  1.6254581254104088,\n",
       "  1.6267226294343695,\n",
       "  1.6315704912974918,\n",
       "  1.6268053765367405,\n",
       "  1.6293223332888975,\n",
       "  1.6267799291704677],\n",
       " 'val_ATT_acc': [69.10569105691057,\n",
       "  80.28455284552845,\n",
       "  83.53658536585365,\n",
       "  89.63414634146342,\n",
       "  89.02439024390245,\n",
       "  86.78861788617886,\n",
       "  86.78861788617886,\n",
       "  87.60162601626017,\n",
       "  87.60162601626017,\n",
       "  88.6178861788618,\n",
       "  90.44715447154472,\n",
       "  91.46341463414635,\n",
       "  93.69918699186992,\n",
       "  94.71544715447155,\n",
       "  94.51219512195122,\n",
       "  94.3089430894309,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  94.71544715447155,\n",
       "  94.71544715447155,\n",
       "  94.10569105691057,\n",
       "  94.91869918699187,\n",
       "  94.91869918699187,\n",
       "  94.3089430894309,\n",
       "  95.32520325203252,\n",
       "  96.54471544715447,\n",
       "  96.34146341463415,\n",
       "  96.95121951219512,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  95.1219512195122,\n",
       "  94.51219512195122,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  95.1219512195122,\n",
       "  96.34146341463415,\n",
       "  96.54471544715447,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  96.54471544715447,\n",
       "  97.15447154471545,\n",
       "  96.95121951219512,\n",
       "  96.13821138211382,\n",
       "  96.54471544715447,\n",
       "  96.7479674796748,\n",
       "  97.15447154471545,\n",
       "  97.35772357723577,\n",
       "  96.7479674796748,\n",
       "  96.54471544715447,\n",
       "  96.7479674796748,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  96.34146341463415,\n",
       "  96.7479674796748,\n",
       "  97.5609756097561,\n",
       "  97.35772357723577,\n",
       "  97.15447154471545,\n",
       "  96.54471544715447,\n",
       "  96.7479674796748,\n",
       "  98.3739837398374,\n",
       "  96.7479674796748,\n",
       "  97.96747967479675,\n",
       "  97.96747967479675,\n",
       "  97.35772357723577,\n",
       "  97.96747967479675,\n",
       "  97.76422764227642,\n",
       "  98.17073170731707,\n",
       "  97.5609756097561,\n",
       "  98.3739837398374,\n",
       "  98.57723577235772,\n",
       "  97.5609756097561,\n",
       "  98.17073170731707,\n",
       "  97.15447154471545,\n",
       "  96.7479674796748,\n",
       "  96.95121951219512,\n",
       "  96.13821138211382,\n",
       "  97.5609756097561,\n",
       "  97.76422764227642,\n",
       "  96.34146341463415,\n",
       "  97.35772357723577,\n",
       "  96.95121951219512,\n",
       "  97.15447154471545,\n",
       "  97.5609756097561,\n",
       "  96.7479674796748,\n",
       "  97.15447154471545,\n",
       "  97.15447154471545,\n",
       "  97.5609756097561,\n",
       "  97.5609756097561,\n",
       "  97.96747967479675,\n",
       "  97.96747967479675,\n",
       "  97.76422764227642,\n",
       "  98.98373983739837,\n",
       "  98.17073170731707,\n",
       "  97.5609756097561,\n",
       "  98.57723577235772,\n",
       "  98.17073170731707,\n",
       "  97.35772357723577,\n",
       "  98.57723577235772,\n",
       "  97.76422764227642,\n",
       "  98.3739837398374,\n",
       "  97.96747967479675,\n",
       "  98.3739837398374,\n",
       "  98.98373983739837,\n",
       "  97.96747967479675,\n",
       "  98.57723577235772,\n",
       "  98.3739837398374,\n",
       "  98.17073170731707,\n",
       "  98.98373983739837,\n",
       "  96.95121951219512,\n",
       "  97.5609756097561,\n",
       "  97.96747967479675,\n",
       "  98.17073170731707,\n",
       "  98.17073170731707,\n",
       "  96.54471544715447,\n",
       "  96.95121951219512,\n",
       "  98.17073170731707,\n",
       "  96.7479674796748,\n",
       "  98.17073170731707,\n",
       "  97.96747967479675,\n",
       "  97.5609756097561,\n",
       "  98.17073170731707,\n",
       "  98.17073170731707],\n",
       " 'val_VAL_acc': [98.0295566502463,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  96.55172413793103,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  98.0295566502463,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  98.0295566502463,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  98.0295566502463,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  95.07389162561576,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  96.55172413793103,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  100.0,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  97.53694581280789,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315],\n",
       " 'val_VAL_jac': [0.5320197044334976,\n",
       "  0.687192122924504,\n",
       "  0.7019704480476567,\n",
       "  0.6978653593016375,\n",
       "  0.7454844084866529,\n",
       "  0.7397372828328551,\n",
       "  0.6527093666527659,\n",
       "  0.7454844108356043,\n",
       "  0.7331691352017408,\n",
       "  0.6765188912452735,\n",
       "  0.74876848230221,\n",
       "  0.7602627336098056,\n",
       "  0.6921182289499367,\n",
       "  0.7241379380813373,\n",
       "  0.7610837508892191,\n",
       "  0.7397372804839035,\n",
       "  0.708538590980868,\n",
       "  0.7364532090172979,\n",
       "  0.7504105168610371,\n",
       "  0.7101806325865496,\n",
       "  0.7200328493353181,\n",
       "  0.7323481155733756,\n",
       "  0.7660098616125548,\n",
       "  0.760262735958757,\n",
       "  0.718390814776491,\n",
       "  0.7356321917378844,\n",
       "  0.7471264430454799,\n",
       "  0.7266009922685295,\n",
       "  0.7290640464557215,\n",
       "  0.7454844084866529,\n",
       "  0.7528735686992777,\n",
       "  0.7224959058714617,\n",
       "  0.7142857189836174,\n",
       "  0.7569786597942484,\n",
       "  0.7709359652890361,\n",
       "  0.7553366228864697,\n",
       "  0.7610837532381706,\n",
       "  0.7627257877969976,\n",
       "  0.7610837485402676,\n",
       "  0.7348111721095193,\n",
       "  0.7668308765430168,\n",
       "  0.7668308741940654,\n",
       "  0.7561576472127379,\n",
       "  0.7446633888582878,\n",
       "  0.7487684846511615,\n",
       "  0.7635468050764112,\n",
       "  0.769293930730209,\n",
       "  0.7602627312608541,\n",
       "  0.7463054304639695,\n",
       "  0.7569786597942484,\n",
       "  0.7684729134507955,\n",
       "  0.7553366252354213,\n",
       "  0.7512315388383537,\n",
       "  0.7594417139814404,\n",
       "  0.7668308788919683,\n",
       "  0.7068965634688955,\n",
       "  0.7545156079560078,\n",
       "  0.7520525537688156,\n",
       "  0.7766830956407369,\n",
       "  0.6863711103429935,\n",
       "  0.7536945930255458,\n",
       "  0.7528735710482292,\n",
       "  0.769293935428112,\n",
       "  0.7561576425148349,\n",
       "  0.7479474626738449,\n",
       "  0.7750410610819097,\n",
       "  0.770935969986939,\n",
       "  0.7742200414535447,\n",
       "  0.7512315388383537,\n",
       "  0.725779974989116,\n",
       "  0.7586206990509785,\n",
       "  0.7717569872663526,\n",
       "  0.7512315341404506,\n",
       "  0.7668308788919683,\n",
       "  0.7413793173916822,\n",
       "  0.7495894995816236,\n",
       "  0.7709359629400845,\n",
       "  0.7446633888582878,\n",
       "  0.7783251325485154,\n",
       "  0.7577996794226134,\n",
       "  0.7742200414535447,\n",
       "  0.7454844061377013,\n",
       "  0.7553366275843728,\n",
       "  0.7676518938224304,\n",
       "  0.7520525537688156,\n",
       "  0.770935969986939,\n",
       "  0.7446633912072393,\n",
       "  0.7430213542994607,\n",
       "  0.749589497232672,\n",
       "  0.760262735958757,\n",
       "  0.7651888419841898,\n",
       "  0.7561576401658834,\n",
       "  0.7594417139814404,\n",
       "  0.7561576448637863,\n",
       "  0.7372742239477599,\n",
       "  0.757799681771565,\n",
       "  0.7413793197406336,\n",
       "  0.7438423715788742,\n",
       "  0.768472915799747,\n",
       "  0.7750410563840068,\n",
       "  0.7422003323221441,\n",
       "  0.7594417186793435,\n",
       "  0.7471264430454799,\n",
       "  0.7463054281150179,\n",
       "  0.7569786621431999,\n",
       "  0.7643678247047763,\n",
       "  0.749589501930575,\n",
       "  0.7167487778687125,\n",
       "  0.7364532090172979,\n",
       "  0.7660098569146518,\n",
       "  0.7717569872663526,\n",
       "  0.7487684799532585,\n",
       "  0.7512315341404506,\n",
       "  0.7619047681686326,\n",
       "  0.749589497232672,\n",
       "  0.725779974989116,\n",
       "  0.7233169208019238,\n",
       "  0.7561576448637863,\n",
       "  0.7643678247047763,\n",
       "  0.7635468097743142,\n",
       "  0.7733990241741312,\n",
       "  0.747126447743383,\n",
       "  0.7241379357323858,\n",
       "  0.7750410563840068,\n",
       "  0.7660098592636033,\n",
       "  0.7775041129201504],\n",
       " 'val_VAL_acc_1': [72.9064039408867,\n",
       "  75.86206896551724,\n",
       "  76.35467980295566,\n",
       "  77.83251231527093,\n",
       "  76.84729064039409,\n",
       "  80.29556650246306,\n",
       "  70.93596059113301,\n",
       "  79.80295566502463,\n",
       "  79.3103448275862,\n",
       "  76.35467980295566,\n",
       "  76.84729064039409,\n",
       "  80.29556650246306,\n",
       "  75.36945812807882,\n",
       "  77.33990147783251,\n",
       "  77.83251231527093,\n",
       "  78.32512315270937,\n",
       "  78.81773399014779,\n",
       "  78.81773399014779,\n",
       "  82.26600985221675,\n",
       "  77.33990147783251,\n",
       "  77.33990147783251,\n",
       "  78.32512315270937,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  78.32512315270937,\n",
       "  75.86206896551724,\n",
       "  78.81773399014779,\n",
       "  78.32512315270937,\n",
       "  78.81773399014779,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  76.84729064039409,\n",
       "  76.35467980295566,\n",
       "  78.81773399014779,\n",
       "  80.78817733990148,\n",
       "  77.33990147783251,\n",
       "  78.81773399014779,\n",
       "  81.77339901477832,\n",
       "  81.77339901477832,\n",
       "  77.83251231527093,\n",
       "  80.29556650246306,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  80.78817733990148,\n",
       "  78.81773399014779,\n",
       "  80.78817733990148,\n",
       "  83.2512315270936,\n",
       "  80.78817733990148,\n",
       "  82.75862068965517,\n",
       "  80.78817733990148,\n",
       "  78.81773399014779,\n",
       "  79.3103448275862,\n",
       "  80.29556650246306,\n",
       "  79.3103448275862,\n",
       "  79.80295566502463,\n",
       "  73.39901477832512,\n",
       "  78.32512315270937,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  74.8768472906404,\n",
       "  81.77339901477832,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  79.3103448275862,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  81.77339901477832,\n",
       "  79.3103448275862,\n",
       "  80.29556650246306,\n",
       "  76.35467980295566,\n",
       "  81.2807881773399,\n",
       "  78.81773399014779,\n",
       "  78.81773399014779,\n",
       "  81.2807881773399,\n",
       "  77.83251231527093,\n",
       "  79.3103448275862,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  79.3103448275862,\n",
       "  78.81773399014779,\n",
       "  78.81773399014779,\n",
       "  78.32512315270937,\n",
       "  81.77339901477832,\n",
       "  80.78817733990148,\n",
       "  82.26600985221675,\n",
       "  77.83251231527093,\n",
       "  79.80295566502463,\n",
       "  79.3103448275862,\n",
       "  82.75862068965517,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  78.81773399014779,\n",
       "  78.32512315270937,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  81.2807881773399,\n",
       "  79.80295566502463,\n",
       "  81.2807881773399,\n",
       "  77.33990147783251,\n",
       "  81.2807881773399,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  77.33990147783251,\n",
       "  78.81773399014779,\n",
       "  81.2807881773399,\n",
       "  81.77339901477832,\n",
       "  80.78817733990148,\n",
       "  78.32512315270937,\n",
       "  79.3103448275862,\n",
       "  80.78817733990148,\n",
       "  79.3103448275862,\n",
       "  78.32512315270937,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  82.26600985221675,\n",
       "  79.80295566502463,\n",
       "  77.83251231527093,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306],\n",
       " 'test_loss': 5.707578081015505,\n",
       " 'test_ATT_loss': 0.7920896851755753,\n",
       " 'test_VAL_loss': 1.6384961319466431,\n",
       " 'test_ATT_acc': 98.2107355864811,\n",
       " 'test_VAL_acc': 99.47916666666667,\n",
       " 'test_VAL_jac': 0.7413194552063942,\n",
       " 'test_VAL_acc_1': 80.20833333333333,\n",
       " 'model_filename': 'model_storage/MLP/model.pth'}"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'MLP_best_model/best_config.p', 'wb') as fp:\n",
    "    pickle.dump(train_state,fp, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'MLP_best_model/best_config.p', 'rb') as fp:\n",
    "    train_state = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "states = pd.DataFrame(train_state['val_VAL_acc'], columns=['val_VAL'])\n",
    "states['train_VAL'] = pd.DataFrame(train_state['train_VAL_acc'])\n",
    "states['val_ATT'] = pd.DataFrame(train_state['val_ATT_acc'])\n",
    "states['train_ATT'] = pd.DataFrame(train_state['train_ATT_acc'])\n",
    "states['val_VAL_1'] = pd.DataFrame(train_state['val_VAL_acc_1'])\n",
    "states['train_VAL_1'] = pd.DataFrame(train_state['train_VAL_acc_1'])\n",
    "states['train_VAL_jac'] = pd.DataFrame(train_state['train_VAL_jac'])\n",
    "states['val_VAL_jac'] = pd.DataFrame(train_state['val_VAL_jac'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>val_VAL</th>\n",
       "      <th>train_VAL</th>\n",
       "      <th>val_ATT</th>\n",
       "      <th>train_ATT</th>\n",
       "      <th>val_VAL_1</th>\n",
       "      <th>train_VAL_1</th>\n",
       "      <th>train_VAL_jac</th>\n",
       "      <th>val_VAL_jac</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>98.029557</td>\n",
       "      <td>95.844875</td>\n",
       "      <td>69.105691</td>\n",
       "      <td>81.440443</td>\n",
       "      <td>72.906404</td>\n",
       "      <td>74.238227</td>\n",
       "      <td>0.557248</td>\n",
       "      <td>0.532020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>99.014778</td>\n",
       "      <td>98.891967</td>\n",
       "      <td>80.284553</td>\n",
       "      <td>96.952909</td>\n",
       "      <td>75.862069</td>\n",
       "      <td>73.684211</td>\n",
       "      <td>0.667128</td>\n",
       "      <td>0.687192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>98.522167</td>\n",
       "      <td>99.168975</td>\n",
       "      <td>83.536585</td>\n",
       "      <td>98.614958</td>\n",
       "      <td>76.354680</td>\n",
       "      <td>78.393352</td>\n",
       "      <td>0.726223</td>\n",
       "      <td>0.701970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>99.507389</td>\n",
       "      <td>99.722992</td>\n",
       "      <td>89.634146</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>77.832512</td>\n",
       "      <td>77.839335</td>\n",
       "      <td>0.741459</td>\n",
       "      <td>0.697865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>89.024390</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>76.847291</td>\n",
       "      <td>83.379501</td>\n",
       "      <td>0.754848</td>\n",
       "      <td>0.745484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>99.014778</td>\n",
       "      <td>99.722992</td>\n",
       "      <td>98.170732</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>79.802956</td>\n",
       "      <td>91.412742</td>\n",
       "      <td>0.884580</td>\n",
       "      <td>0.747126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>98.522167</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>97.967480</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>77.832512</td>\n",
       "      <td>88.365651</td>\n",
       "      <td>0.863343</td>\n",
       "      <td>0.724138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>99.507389</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>97.560976</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>80.295567</td>\n",
       "      <td>91.689751</td>\n",
       "      <td>0.889197</td>\n",
       "      <td>0.775041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>99.014778</td>\n",
       "      <td>99.722992</td>\n",
       "      <td>98.170732</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>79.802956</td>\n",
       "      <td>86.426593</td>\n",
       "      <td>0.851339</td>\n",
       "      <td>0.766010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>99.014778</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>98.170732</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>80.295567</td>\n",
       "      <td>89.473684</td>\n",
       "      <td>0.876270</td>\n",
       "      <td>0.777504</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>126 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        val_VAL   train_VAL    val_ATT   train_ATT  val_VAL_1  train_VAL_1  \\\n",
       "0     98.029557   95.844875  69.105691   81.440443  72.906404    74.238227   \n",
       "1     99.014778   98.891967  80.284553   96.952909  75.862069    73.684211   \n",
       "2     98.522167   99.168975  83.536585   98.614958  76.354680    78.393352   \n",
       "3     99.507389   99.722992  89.634146  100.000000  77.832512    77.839335   \n",
       "4    100.000000  100.000000  89.024390  100.000000  76.847291    83.379501   \n",
       "..          ...         ...        ...         ...        ...          ...   \n",
       "121   99.014778   99.722992  98.170732  100.000000  79.802956    91.412742   \n",
       "122   98.522167  100.000000  97.967480  100.000000  77.832512    88.365651   \n",
       "123   99.507389  100.000000  97.560976  100.000000  80.295567    91.689751   \n",
       "124   99.014778   99.722992  98.170732  100.000000  79.802956    86.426593   \n",
       "125   99.014778  100.000000  98.170732  100.000000  80.295567    89.473684   \n",
       "\n",
       "     train_VAL_jac  val_VAL_jac  \n",
       "0         0.557248     0.532020  \n",
       "1         0.667128     0.687192  \n",
       "2         0.726223     0.701970  \n",
       "3         0.741459     0.697865  \n",
       "4         0.754848     0.745484  \n",
       "..             ...          ...  \n",
       "121       0.884580     0.747126  \n",
       "122       0.863343     0.724138  \n",
       "123       0.889197     0.775041  \n",
       "124       0.851339     0.766010  \n",
       "125       0.876270     0.777504  \n",
       "\n",
       "[126 rows x 8 columns]"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['KMP_DUPLICATE_LIB_OK']='True'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA2/UlEQVR4nO3deXhU5fXA8e/JJGQPIQkJIQHCDmELhM0NUQEVcUNUrFq0Iq3aurTWtf2prVrb2trauhT3Faos7kUEcUNAw74ECCGBLGSFhISsM/P+/pjJkBVC9hnO53l4JnPn3jvnhuTMm3PPfa8YY1BKKeVZvDo7AKWUUm1Pk7tSSnkgTe5KKeWBNLkrpZQH0uSulFIeyLuzAwCIiIgwcXFxnR2GUkq5lY0bNxYYY3o29lqXSO5xcXEkJSV1dhhKKeVWRORAU69pWUYppTyQJnellPJAmtyVUsoDaXJXSikPpMldKaU80EmTu4i8KiJ5IrKj1rIwEflCRFKcjz1qvfagiOwTkT0icmF7Ba6UUqppzRm5vw5cVG/ZA8BqY8xgYLXzOSISD8wFRji3eV5ELG0WrVJKqWY5aZ+7MeYbEYmrt/hyYKrz6zeAr4D7ncsXG2MqgTQR2QdMBNa1UbynJvljOLStU966wwVHwbh5YPFxPM/cCCmfg07p7JkCwiDxZvDxczwvzYdNr4O1qlPDUi0QORxGzm7z3bb0IqYoY8whAGPMIRGJdC6PAdbXWi/TuawBEVkALADo27dvC8M4gaoyWHIL2CoBafv9dzkGtiyC2Qth5zJY8ycwNk6PYz8dGSjOhAufALsdltwM6d+i/99uaOTsLpXcm9LYT1ajQ0djzEJgIcD48ePbfnh58HtHYr9hKQya1ua773J2LoeP7oJ/JQIGRs6BWc+AX0hnR6baw6e/gXXPwZCLIGe7I7Ff/hyMvaGzI1NdREuTe66IRDtH7dFAnnN5JtCn1nqxQHZrAmyx1DVg8YW+Z3bK23e4EVdCTCKsetTxYTbmOhAdxXms6X+A1C9h2QIoK4QhF0PC9Z0dlepCWtoK+REwz/n1PODDWsvnioiviPQHBgM/tC7EFtq3GvqdCd0COuXtO0VoX5jzKiT8RBO7p+sWCFe8CKU54BsElz2r/+eqjpOO3EVkEY6TpxEikgk8AjwFvCcitwAHgasBjDE7ReQ9YBdgBe4wxtjaKfamHc2G/GRHklPKU/WdBHPfheBeEBR58vXVaaU53TLXNfHSBU2s/wTwRGuCarXULx2PA8/v1DCUandDL+7sCFQX5ZlXqKZ+CUFREDWisyNR6rRkjOHa/6zjbyv3tPt7ZRwuY/KTq1mdnNus9attdi7793e8vjatnSPrXJ6X3O02x8nUgedrDVKpTrIvr5QNaYd56dv9FJZWttv7GGN4+IMd5Byt4PXv05u1zcdbs9mWWcyXe/LbLa6uwPOS+6GtUH5YSzLKbWQVlfN9akFnh9GmVu5yjKIrqu280UjSLSitZEtGUaPbWm12Vu3KxW4/eYf0J9sO8c3efAb2DOS7fQVkF5W79vH5zhyOVlTXWd9uN7z4dSoAu7KPnsIRuR/PS+7p3zke+0/p3DiUx7DZTbMSzak6cqyKxz/ZxXlPf8VPXtpAan5pm79HZ1m5K5cxfUKZER/FG+sOcKzSWuf1B5Zu48rn1/LR1oad0h9vy2b+m0ks25xVZ3mV1V7neXF5NY99vItRMd15ed4EjIHlzm0Wfrufn7+1kXP/soaXv91PRbWjr+PL3XnszS0loU8oBaWV5JVUtOVhdymel9wzNkCPOEcHgVJt4KevbuCBZW07jUWV1c4Vz6/l1bVpXDIqGouXsHRjZpu+R2fJKa5ga0YRM+Kj+MXUgRSXV7Poh4Ou1/NLKlmzJx8/bwu//u8Wvtxdt1b++Q7H8/98ner6UH1tbRoJf1jJ9/scf+FYbXbuW7KVw8cq+dPsUfSPCGRi/zCWbszkQOEx/rkqhbMHRTAypjuPf5rM+U9/xXtJGTz/1T5ie/jz2wuHAp49eves5G6MI7n3mdzZkSgPUVppZV1qIV/uzsO0cJ4eu93w5xW7eeGrVNeyD7dkcaCwjBduSOSZaxM4d0hPlm/OwuZMZss2ZfL7D3Y0tcsu7Qvnic0LR0Qxrm8PJg8I4+Vv01yj5w+3OI7z3VsnMTw6hNve3sSOrGIAKqptfL03nz5h/qTklbJ6dx4Zh8v4y4o9VFTbmP9mEpsOHuHBZdv5fGcuD18Sz8iY7gDMSYxlf8Exbn79R3wsXjx99RjeumUS786fRM9gX+5bso1NB4tYMGWAa5tdhxpP7na74aOt2Vzz4jpm/etbZv3rWxZ+k9rouhsPHObORZux2o7/ZbFiRw53vLuJ7ZnFDdbfn1/KdQvXu/b75xW7W/idPjHPSu5H0uBYPvSZ2NmRKA+x5WARdgMFpVWkF5a5ln+67RDPfLGXZ77Yy8f1Sgu7so/yweYsqm12jDH88dNdvPBVKn9esZuNB4646r7DegUzIz4KgKvGxXKouILvUwvIPFLGw8t38Nb6A+zJKenQ4z0VFdU2XvkuzfV9WLEjB2MMK3fmMCAikIE9gwC4e9oQco5W8K8vUzDG8H5SJgl9Qhnbtwev3zwBPx8Lz65OAeC7lALKq2384fKRxPbw5/mv9vHIRzsRgWW3n0VEkC/XvLiO9zdmctcFg7nl7P6ueGaOisbfx8L+/GPcO2MIvbo7JlU7c1AEH9xxFi9cP44bJ/fjmvF96O7vQ2wP/0ZH7lsyirj0399x56LNFB6rJCrYj6Kyal75Lq3RD/hXv0vno63ZbK2VyF/5bj+fbjvEpf/+jjve3eQ6qWy3Gx5Yup0d2cVEBfsRFexHd3+ftvtPqaWt55bpXAc3OB776shdtY0f0w/X+bp/RCBHjlVx5+LNrlE2QJCvN+cNi+RYpZX5b/xIdnEFz65OYVy/HizZmMkNk/vyZXIeDy3bzl3TBpOaf4x/zk1AnB1dFwyPpLu/D+8nZbrq095ewtJNmTw0c/gpxZxfUklogA8+lvYbu1Xb7Pzy3U2sSs6rs3x0bHeSDx3lZ2f3dx3b5AHhXDUulv98vZ+BPYPYk1vCH68YCUB4kC/zzujHs1/uY19eCSt35RDs581ZAyP4+ZQB/P7DnQD8flY8CX1CeWf+JOa99gPTh0dx97TBdd47yNebn0zqS/Kho9x4Rlyd10SEi0dFc/GoaNey+OiQBiP3bZlF3PDyBkL8vPn7NWO4PCEGi5fw5rp0/u/DnWQeKadP2PGr3iutNr7a4/gerN1XQGK/HpRWWtl8sIgbJ/ejR4AP//lmP2n5x1i0YDKf78jhh/TD/PmqUVw7oR0mTKzFs0buGevBNwR6ntovg1JN2XjgCMN6BRMa4EOSM9F/uTsPm93w4R1nsffxixkUGcTvPthBWZWVZ77YS3ZxBQ9cPAxvi7BkYyZXJ8byh8tG8uhlI9iTW8Kv39tCnzB/LqmVaPx8LFw2pjefbMtm9e487pk+mPOGRbJsU1adP/dPJPNIGb9+bwsTn1zFv7/c1y7fD3CMPu99fyurkvP44+UjSH/qElKfnMlf54ymoKSSapvhohF1z3k9fMlwgv28uff9rXSzeHHZ6N6u1+adGYefjxfPr0llVXIe5w+LpJu3F1eP70PPYF9GxoQw74x+APQJC+DL30zlwZnDXR8etf1+Vjzv3joZi9fJ26Dje4eQVnCMsirHh2lKbgnzXv2B0AAflt1+FrPHxbr2M75fGABJBw7X2cf3qYUcq7Lh6+3Fd87zARv2F2K1Gy4a2YtfzxjKf25MJCWvhJtf+4En/5fMxLgwrk7sQ3vzrJF7xg8QOwG8POszS7Wdapudn73+I727+3PXtMH0DvVvcl2rzc6mg0e4OjGWrKJyktKPAPDFrlx6hfgxKqY7Xl7CE1eM5NqF67nnv1tYlZzHTyb15RfnDuTWcwawNbOIMbGheHkJM0b0YkZ8FCt35bLgnAF41xtZz0mM5a31BxgeHcLNZ/WnX3ggX+zK5duUAs4bduLpBdbvL+Snr/wAAuGB3Vi9O5d7pg856fejotrG/Uu3MSAiiLvqjYRryzhcxu3vbCLnaAVWm50jZdX89sKhrhGyxUu4enwfLh3Tm9T8Ukb07l5n+7DAbjw0czi/XbKN6SOi6B5wvBQRHuTL3Al9XX3qM+IdHwx+PhY++dXZBPp6N/hetYX46BCMgd05JQyICOTGV37A2+LF27dMcpV0agztFUywrzc/ph/hyrGxruUrd+YS2M3CtRP68tb6dI5VWvluXwG+3l4k9nPcoG7q0Ej+ce1YfrVoExYv4YkrR+LVjA+f1vKc5F5eBHnJjtkRlccwxvD13nx255Sw4JwBrf6l+GhLNt+mFGDxEpZvyeKmM+O47dyB9Ajs1mDd5EMllFXZGB8XRnRROauS88gqKufrvfnMSYx1xTJpQDjXju/Df5MyiAjy5f4LhwGOhDeub486+3xy9igm9g/jmgkNR26jY7vz0MxhnD8sEh+LF+cNjSQssBtLNmaeNLkv3ZiJn48XK+6ewpKNmTyzai+Hj1URFtgNm93w2to0psdH0S880LWN1WbnzkWbWbkrFx+LcO2EPg2SGkDe0Qquf3kDxeXVzBzVCxBGxXTnuokNj8HPx9IgsdeYkxjLkbIqzh8W1eC1+ef05631B7CIcO7Qnq7lUSEN42kr8b0d02Hvyj7Kez9mkF9ayQe3n0VcRGCDdS1ewrh+PVx/vYHjL5gvduUydVgk5w+L5NW1afyQdpjvUgqY2D8MP5/jN6G7ZHQ0fj7jqbbZGRwV3G7HVJvnJPfMJMDoydQuLPnQUfqEBRDk27wfu22ZRTzxaTIb0hy/UKH+Psyd2PI6Ze0TmS/9dDz/WJXCS9/uZ9EPB10j7W7ex0eINfX28XE96F3kSDL/XLWX8mobM0bUTVAPzhzGwcNlzD+nf51RaX0RQb7MP2dAo6+JCAumDHQ97+btxeUJvXln/UGKyqoIDWj4AQSOD8C1+wo4a1AEvUP9OXtwBH//Yi/fpxYwa3Rvvt6bx+OfJvPqd2m8f9uZxIT6Y7XZuX/pdlbuyuXn5w7g5W/TeHVtmqu+vy+vhPSCMgzw9Od7KCit5J35kxhb78PqVNQ/vtpiewTw8ykDqKi2N/vno7ViQv0J8fNm8Y8H2ZF1lJ9PGcCo2MY/mAAmxPXg6ZX5rv+LzRlFFJRWMiM+ivFxPfD19mLppkxS8kqZkxjbYPsLhjf8UGtPnlO/yFgPYoGY8Z0diWrEhv2FXPzPbzn3L2t4bW0aldYTTxb6/b4C5ry4jtT8Uh67bAQT+4fxp//tpqAVl7Kv3p1HSl4pvzh3IH3CAvjbNWNYcdcUJvUP56+f7+Ev9VrSkg4cJibUn+ju/oyM6U43by/e35hJsJ83k/qH11k3NKAbixZMbvNf4KsT+1Bls/Pw8h11TuDWtr/gGNnFFZw9OAKA0THdCfbzZq2zBrxkYybd/X0oqbRy48sbeD8pgxnPfMPSTZncM20ID148nEtGRfPO+gMUl1WzZk8eF//zW+a/mcStbyaRVnCMl346vlWJvTnuu2gY/3dpfLu+R20iQnzvEHZkHSUm1P+EZSmA8XGOuvvGA47y3MpdOfhYhPOGReLnY2FCXBifbDsEwFmDIto3+GbwnOSemeSYKMw3qLMj6TSFpZWc+9c1Xe5imEqrjYeWbycm1J+hvYJ57ONdXLdwfZN941syipj/ZhJx4QF8cc+5zDszjievHEVZlZUnPk1uUQzGGNcFLLNGHz+RObRXMC/PG8/1k/ry6to0V7+1MYYf048wIc6R0Hy9LSTEhmIMrhN+HSG+dwgPzRzGp9sP8fDy7eQereB3H2xn4hOr2JvraJOsSeJnOxOKt8WLMwaE821KAUeOVbFqVx5XjYvltZsmkF1czm+XbMPiJSy8MZE7LxgEwC/OHcixKhsPLt/GbW9vZEhUMB/ccRYf//Jsvrv/vC6RrNpDfLRjpP6Hy0cQ0O3EfzGMiQ3FxyL8mH6EgtJKPtqSzeQB4YT4Of5Sq/lwDQvsRnx0598BzXOSe/mR0/6q1Cc+TeZAYRmvfd+1Zrtb+PV+UvOP8fiVI3ln/iQevTSeTQeL+KrexE3GGD7fmcNNr/1ARJAvb90yyVULHxQZxG3nDmT55ize+D7d1UGy8cAR7l+yjYzDZQ3et7ZvUwrY7LyApbGTc/ddNIywQF8eXr4dm92wOjmP/JJK12gNHOUZOH7Cr6MsmDKQX543iMU/ZnDmU1+y+IcMisqreX6NoyPm25QC+oT516mnnz04gswj5fx7zT6qbHbmJMYyPi6Md2+dzLPXjWXF3VOYMaKXq+MkvncIU4f25LPtOcSE+vPmzyaS0CeUUbHdiWzHundnm39Of569bmyz/uLy72ZhZEx3vtmbz09f+YEjZVV1TlrXfLieOTC8Q06Ynozn1Nxt1WBpvCbpqbZnFuPfzcKgyCDW7itg2eYsBvQMZEfWUXbnHGVYr7YfPezMLsZmN4yODW3W+mkFx/jXmn3MGh3NeUMdJwWvn9yPhd/s54WvUl0nCrdkFPGHj3ey6WARgyKDeO2mCQ1Opt1+3iA2pB3mkY928ua6dOLCA1m929FjXG238/drEhqNYVf2Ue54dxMDIgKbbEHr7u/D/10az52LNjPjma9JzT9GbA9/10VGAFeOjWF//jHOG9az0X20p9/MGIKXQFZRBXdeMIi31x/g1bXp3DVtCOtTC5k1JrrO+jUj7dfWphEfHeI6eTiub48GJ3lr3H/RMPx9LPx+VjzhQb7te0BdRO9Qfy47QcdUfRPiwlj4zX58LMLL8ybU+V7GR4cwe1wM14xv/zbH5vCg5F4Flva50qsrqqi2cd1L6ymrsjJ7XCxJ6YeJCw/g7VsmuUozD19yavXLwtJKbHaDiBAR1K1OH/H+/FL+tnIvn24/RDeLF6/eNMH1Z2hTjDH87oPt+Hp78X+zjsfiY/Hi1ikDeOzjXa7ugxte2UCofzeemj2KOYmxjY6u/XwsLF4wmc935vLXz3fzQ9phfjN9CIeOVvDejxn8evoQYnvUva1iWsExfvrqBoJ9vXlr/iT8u1ka7LfGpaOj+WhLFpsOFvH7WfHcMLkvvt7H1x8cFcyLNyae+JvYTkSEX88Y6np+y9kDeP37dO7+7xZKKq2cPajuB86AiEB6d/cju7ii0ZN7jRkeHcILN3TO8bmLacOjeHv9AZ6+egznDqn7PffykiYHGJ3Bw5L76TNy/z61gNJKK9OGR/HR1myqrHbevmUSvUP9OX9YJMs3Z3PfRcOafZXih1uyuGvxFtfz+y8axm1THZ0NOcUVXPLsd4jAr84fxBe7clnwVhJvz5/U5CgQ4IMtWazdV8gfrxjZ4E/7ayf04dnVKfzx02T255fSu7s/7/3iDCJOMmIUES4a2YsLR0RhjOMXKruonPd+zODlb9N49LLjN2jJLirnhpc3YAy8NX8SMScZoYkIL96QiIF2vbqzLfTq7sfssbH8NykDEUcpoDYR4ZzBPVm2OZPLE3o3sRd1qib2D2PbIzPape++rXX9CJvrNEvuK3fmEuzrzfPXj+Ore6fyzvxJrpH0nMQ+FJRW8s3e5t+MYO2+Arr7+/DElSMZ1iu4zlSsK3flUF5tY9ntZ/KbGUN585aJ9Az25aZXfyC5iYmXisqqePyTZBL6hHJ9I+2LAd28uenM/mzNKHKNqk+W2GsTEVdds3eoP5cnxPDfHzM4fKwKcMwXfsMrGzhaXs0bP5vomufkZLwtXl0+sddYcO4ARGBk7+6N9un/9qKhLF5wxmlTYuko7pDYQUfubslmN6xKdlw80c3bi96h/nWutJw6tCfhgd14dnUK27Pqzko3sX8YZw5sWE7ZllnM2L6hXD+pH+VVNh7/NJmMw2X0CQtg5c5cBvYMdNXwI4P9ePuWSVz94jpufOUHlvziDOIiAknNL+V/2w9htRs2HjhCUXk1b88e1eTJpZvPjuNIWRU/PaPfSUfVJ/OLcwewdFMm9y3ZxsiYEFbsyCG7qJy3bpnkmgHQ0wzsGcTDM4fXOZFaW0SQ7yl9YCrP4kHJ/fQ5obr54BEKSqvqnOyrzcfixU/PiOOZVXvrzFQHEBrgw/cPnF+n7ausysre3BJmOOcDmR4fxeOfJrNyVy5zxsWyfn8ht06pe+FNn7AA3p4/kWv+s57rX97AOYMjeH9jpqsXWwTunTGU4SdoCQvx86lTRmmNwVHBzB4bw7LNWaxKdvxV88INiUyo1e3iiZq6IEopD0ruVeB9eiT3msvFpw5tumvjrmmDXT3MNTYdPMJVL6xj8Q8Z/KzWVKk7so5iNzDGeXVev/BAhvUKZuXOHMIDu2G1m0Y/SAZFBvPGzRO57qX1LN2UyY2T+3HHeYOICHL8PzQ2sVN7+vu1CfztmjGu5x39/kp1JZ6R3I0Ba2WXHLn/mH6YZ1enENjNm7unD251e2JNL/iZAyMI9jtxd1D95JbYL4yJ/cN4+dv93DC5n+tCnG2ZRQB12htnxEfx7zX78BIhMtiXMU20Po6K7c5nd56Dt0VOOAlXR9GErpRDq84MiMhdIrJDRHaKyN3OZY+KSJaIbHH+m9kmkZ6I3QaYLpXcSyutzH8jiatfXMfunBLWphZw8T+/5b4lWxvcC/JUpOSVcqCwrMHcJs1129SBZBdX1DlhujWzmJhQf3oGH6/PTo/vhd3Auv2FTIuPOuFFGX3DA7pEYldKHdfikbuIjARuBSYCVcAKEfnU+fIzxpin2yC+5rE5OiS6Up/7+0kZrErO5TfThzD/nAFUWm3868t9vPJdGnERgdw+ddDJd9KIj7Zk4yUwvYVzmEwd0pNhvYJ58etUZo+NwctL2JpRxOh6EyaNjAkhursfh4ormqztK6W6rtaM3IcD640xZcYYK/A10Dnz7dqck0lZuk5nwJKNmYyK6c6vLhiMfzcLoQHd+P2seC4a0Yt/rkrhYOGJL5dvjM1uWLopkylDerb4knAR4bapA9mXV8rSTZkcOVbFwcNljOkT2mC9WaOjCQvsxhn1eqiVUl1fa5L7DmCKiISLSAAwE6i57vaXIrJNRF4VkUavchGRBSKSJCJJ+fnN78dulK3a8dhFRu67so+yM/too1cGPnrZCHwsXvzuwx1NTpz15GfJxD3wKXEPfMqoRz533fF9XWohh07hisOmXDq6N4n9evDkZ8l87eyFrz9yB7j3wqF8cc+UOldpKqXcQ4uTuzEmGfgz8AWwAtgKWIEXgIFAAnAI+FsT2y80xow3xozv2bOVc3W4yjIdW3P/Ie0wj328s8GkVUs3ZeJjES4b0/DKwF7d/bh3xhC+2ZvPDa9sYP4bP/LU/3Zjd7YQ5h2t4PW16ZwxIJw7LxhMj8BuPLBsOxXVNpZszCDEz5tprZxW1stLePLKUZRUWPndBzsQgVGN9IL7elv0Ahil3FSrTqgaY14xxowzxkwBDgMpxphcY4zNGGMHXsJRk29fNcndu2MT0Z9X7Oa1temc/7evePSjnRSUVlJts/PB5iwuGBbV6FWDADeeEcc142MpKqvm4OEyXvw6lUU/HgTglbVpWO12nrpqFL+ePoSnrhrFwcNl/OmzZFbszOHSMb3r3OGlpYb2CubWKQMorbQyICLwpJ03Sin30qpWSBGJNMbkiUhfYDZwhohEG2MOOVe5Ekf5pn1ZO/6E6v78UjYeOMKt5/SntNLKW+sP8H5SBlOHRVJ4rOqEpROLl/CXOY5+bGMMP3lpA3/+324mDwjnnfUHuWR0b9dVh2cOjGD2uBjeWHcAoNUlmdruPH8wn+/McU1VqpTyHK3tc18qIuFANXCHMeaIiLwlIgmAAdKBn7fyPU6uE8oySzdlYvESbj1nAJEhftxy9gD+tnIPn247RERQtzr3gTwREeHxK0dy8T++5eoX11FaaeXn9a4GfXjmcL7cnUd4YDcS6p34bA3/bhZW3DUF7y4w97RSqm21KrkbY85pZNmNrdlni7hOqHZMcrfZDcs2ZTFlcISra2VQZBAv3JDIjqxivEROafKpgT2DuP28gfxjVQpThvRsMBdKeJAvi26djI/Fq80v0umoOwoppTqWZ1yh6mqF7Jjk/n1qAYeKK/hdI/Olt3SSqtumDuRYpZVrJzQ+0f+J5mhRSqn6PCS5d2xZpuaGwxcMj2yzffp6W0755hpKKdUUz/ibvAPLMtU2O5/vzGHW6Og26VpRSqn24CHJvaYVsv2T+4HCY1RU2103S1ZKqa7IM5K7teNq7ntySgEYHBnc7u+llFIt5RnJvQPLMntzS/ASR3eMUkp1VR6S3DvuIqa9uSX0Cw/UertSqkvzkOTecbNC7s0tYUiUjtqVUl2bhyT3jpkVsqLaRnphGUOjtN6ulOraPCS5d0yf+/78Y9jshsGa3JVSXZxnJfd2nhUyJa8EcMyoqJRSXZlnJPeaWSG92veC2z05JXh7CXHOGRuVUqqr8ozkbqtylGTaeFKt+vbmljKgZ6BOtqWU6vI8I0vZqjusx13r7Uopd+Ahyb2q3ZN7WZWVjCPaKaOUcg8ektwr2z2578srxRi0x10p5RY8JLm3f1lm9yFHp4yWZZRS7sBDkntVu88I+dmOQ/Tu7kd/7ZRRSrkBz0ju1vYty+QereCbvfnMHheLl95vVCnlBjwjuduq23XqgeWbs7AbuCoxtt3eQyml2pKHJPf265YxxrBkYybj+/Wgf4SWZJRS7sFDknt1u80IuTWzmH15pczRUbtSyo14SHKvbLeyzJKNGfj5eDFzdHS77F8ppdpDq5K7iNwlIjtEZKeI3O1cFiYiX4hIivOx/W822k5lGWMMn+/MZdrwKEL82v9GIEop1VZanNxFZCRwKzARGAPMEpHBwAPAamPMYGC183n7slW3SyvkoeIK8ksqmRAX1ub7Vkqp9tSakftwYL0xpswYYwW+Bq4ELgfecK7zBnBFqyJsjnZqhdyaUQTAmD6hbb5vpZRqT61J7juAKSISLiIBwEygDxBljDkE4HyMbGxjEVkgIkkikpSfn9+KMGi3K1S3ZhbjYxGGR+tVqUop99Li5G6MSQb+DHwBrAC2AtZT2H6hMWa8MWZ8z549WxqGg62q0ROqxeXVVNvsLd7ttswihvUKwddbb4atlHIvrTqhaox5xRgzzhgzBTgMpAC5IhIN4HzMa32YJ2GrarQVcvbza/nlu5tatEu73bA9s5jRsd1bG51SSnW41nbLRDof+wKzgUXAR8A85yrzgA9b8x7N0kS3TOaRcj7fmcvnO3NOeZf7C45RUmnVertSyi21ts99qYjsAj4G7jDGHAGeAqaLSAow3fm8fTVSlqmotlFpdZRkHv1oJ6WVza4YAY6SDMCY2NC2iFAppTpUa8sy5xhj4o0xY4wxq53LCo0xFxhjBjsfD7dNqE0G0ejIvaTCkcxnj40h52gFf1+595R2uy2zmIBuFgZF6vztSin34/5XqNqqHY/1+tyPVjiWnzu0J3Mn9OGNdemUOJc1x9bMIkb27o5FZ4FUSrkhD0juVY7HeiP3o+WORB7i58N5QyOx2Q0peaXN2mW1zc7O7KN6MlUp5bY8N7k7yzIh/t4M7eXoU9+bU9KsXe7JKaHKateTqUopt+UByd1ZajnByL1PjwD8fLzYm9u8kfvKnTmIwMT+Ou2AUso9eUByr3Q8Nhi5O5O7vw9eXsLgyGD25p585G63G5ZuyuLsQRFEhfi1ebhKKdURPCC5NzVyd5ZlnLM5DolqXnJfn1ZIVlG5zt+ulHJrHpDca2rudfvcj1ZU42MR/HwchzgkKoi8kkqOHKs64e6WbMwk2NebC0f0apdwlVKqI3hOcveuO/3A0fJqQvx8EHG0Mg6pOal6gtF7aaWV/23PYdaY3vj56HwySin35d3ZAbSatelumRD/46P5IVHO5J5XyqQB4XXWzThcxrEqK1/tyae82sacxJj2jVkppdqZ+yf3psoy5dWE+B0/vN7d/Qjy9W7QDrkts4jLn1uLMY7nAyICGde3/W8epZRS7cmDknu9skxFdZ2Ru4gwOCqoQVnm+TWpBPt686fZo/ESGNG7u6uUo5RS7sqDknvDPvfe3f3rLBsaFcznO3MwxiAi7Msr5fNdOdwxdRCX6A2wlVIexHNOqDbolrES4l/3s2tIVDBHyqopKHVss/CbVLpZvLjprLiOiFQppTqMR4/ca3rca7hOquaWYLXbWb45i+sm9iUiqOGNPpRSyp15QHJvOCtkzVzutWvuAEN6OabvvfGVDa66+q3nDOiYOJVSqgO5f3K3Npx+oGYu99rdMgCRwX78afYoMg6XATA8OoQ+YQEdE6dSSnUg90/ujZRlas8rU991E/t2SFhKKdWZPOCEasO5ZWrPCKmUUqcjD0juDcsytedyV0qp05EHJHcduSulVH0ekNyrAAGv4xN9najmrpRSpwPPSO7evlBryoD6c7krpdTpxv2Tu7Wq0bsw1Z7LXSmlTjetyn4ico+I7BSRHSKySET8RORREckSkS3OfzPbKthG2aqamBHSRycAU0qdtlrcTiIiMcCdQLwxplxE3gPmOl9+xhjzdFsEeFK2qkZmhLQS7KedMkqp01dr6xbegL+IeAMBQHbrQzpFTY3c9WSqUuo01uLkbozJAp4GDgKHgGJjzErny78UkW0i8qqINHrnCxFZICJJIpKUn5/f0jCcyb1uzb2kouGkYUopdTppcXJ3Ju3Lgf5AbyBQRG4AXgAGAgk4kv7fGtveGLPQGDPeGDO+Z8+eLQ3D0efe6C32tCyjlDp9taYsMw1IM8bkG2OqgWXAmcaYXGOMzRhjB14CJrZFoE2yVdWZERIan+5XKaVOJ61J7geBySISII62lAuAZBGpfUujK4EdrQnwpKyVjbZCas1dKXU6a3HtwhizQUSWAJsAK7AZWAi8LCIJgAHSgZ+3PswTqFeWqbTaqKi2N5juVymlTietyoDGmEeAR+otvrE1+zxltirw6e566prLXUfuSqnTmPtfwlmvW0YnDVNKKY9J7scTuU73q5RSHpPcdeSulFK1eUByr3bMCulUVK7T/SqllPsnd2tlnbJMTnE5AL26+3VWREop1encP7nXK8tkF1UQ7OutZRml1GnNA5J73T73rKJyeof6d2JASinV+TwgudcfuZfTO1RLMkqp05t7J3e7HezVjSR3HbkrpU5vbp7cHZ0xNSdUy6qsHCmr1uSulDrtuXdyt1U5Hp2tkNlFFQDEaHJXSp3m3Du5W53J3VmWOeRsg9SRu1LqdOfeyb1m5O4sy2QX1SR3PaGqlDq9eUhyd5Rlsooq8BKICtHkrpQ6vbl5cq85oeooy2QXlRMV4oePxb0PSymlWsu9s6Ct0vFYqyyj9XallHL75F73hGp2UTnROqeMUkq5eXL3CYC4cyAoErvdkF1coW2QSilFK2+z1+l6DoWbPgGgsKSSKqtdyzJKKYW7j9xrOd4GqcldKaU8MLlrzV0ppTwmuWc5k7vW3JVSyoOSe3ZRBQHdLHTX2+sppVTrkruI3CMiO0Vkh4gsEhE/EQkTkS9EJMX52KOtgj2Rmh53EemIt1NKqS6txcldRGKAO4HxxpiRgAWYCzwArDbGDAZWO5+3u7ySCqJCfE++olJKnQZaW5bxBvxFxBsIALKBy4E3nK+/AVzRyvdolvJqO/4+7t3ZqZRSbaXFyd0YkwU8DRwEDgHFxpiVQJQx5pBznUNAZGPbi8gCEUkSkaT8/PyWhuFSWW3Dz8djTiEopVSrtKYs0wPHKL0/0BsIFJEbmru9MWahMWa8MWZ8z549WxqGS6XVjp+PpdX7UUopT9Caoe40IM0Yk2+MqQaWAWcCuSISDeB8zGt9mCdXUW3D11tH7kopBa1L7geBySISII4WlQuAZOAjYJ5znXnAh60LsXkqqm06cldKKacWn4E0xmwQkSXAJsAKbAYWAkHAeyJyC44PgKvbItCTqbDateaulFJOrWovMcY8AjxSb3EljlF8h7Ha7NjsBj9vHbkrpRR4yBWqFVY7AL46cldKKcBTknu1DUBr7kop5eQRyb3SOXLXsoxSSjl4RHKvGblrWUYppRw8Ihu6kruO3JVSCvCY5O4sy+jIXSmlAA9J7pVWPaGqlFK1eUZyd47cdfoBpZRy8IhsqK2QSilVl0ckd1crpCZ3pZQCPCS5Hx+5e8ThKKVUq3lENtRWSKWUqsszkrtVWyGVUqo2j8iGx7tldOSulFLgIcm9wmrDxyJYvKSzQ1FKqS7BM5J7tU0nDVNKqVo8IrlXWu34ahukUkq5eERyd9w/1SMORSml2oRHZMTKartOPaCUUrV4REZ0jNy1LKOUUjU8IrlXWu2a3JVSqhaPSO5ac1dKqbo8IiNWWG16AZNSStXi3dINRWQo8N9aiwYA/weEArcC+c7lDxljPmvp+zRHZbVdR+5KKVVLi5O7MWYPkAAgIhYgC1gO3Aw8Y4x5ui0CbI4Kq17EpJRStbXVcPcCINUYc6CN9ndKKqrt+OrIXSmlXNoqI84FFtV6/ksR2SYir4pIj8Y2EJEFIpIkIkn5+fmNrdJsFdVac1dKqdpandxFpBtwGfC+c9ELwEAcJZtDwN8a284Ys9AYM94YM75nz56tikFbIZVSqq62GLlfDGwyxuQCGGNyjTE2Y4wdeAmY2Abv0SS73VBl1ROqSilVW1tkxOuoVZIRkehar10J7GiD92hSzf1TtSyjlFLHtbhbBkBEAoDpwM9rLf6LiCQABkiv91qbq7Tq/VOVUqq+ViV3Y0wZEF5v2Y2tiugUVVTX3GJPR+5KKVXD7Ye7x2+O7faHopRSbcbtM2KFqyyjI3ellKrRqrJMV1DpKsu4/eeUUh6purqazMxMKioqOjsUt+Xn50dsbCw+Pj7N3sbtk3tNWUanH1Cqa8rMzCQ4OJi4uDhE9Cb2p8oYQ2FhIZmZmfTv37/Z27n9cLeiphVSR+5KdUkVFRWEh4drYm8hESE8PPyU//Jx+4xY6TqhqiN3pboqTeyt05Lvn9sn95qRu55QVUqp49w/uWsrpFJKNeD2GbGmLKMjd6VUWwgKCjrpOs888wx+fn4UFxdTWFhIQkICCQkJ9OrVi5iYGBISErBYLMTHx5OQkEBYWBj9+/cnISGBadOmdcBReEC3TKVVWyGVchePfbyTXdlH23Sf8b1DeOTSEW26z5NZtGgREyZMYPny5dx0001s2bIFgEcffZSgoCDuvffeOuvfdNNNzJo1izlz5nRYjG6fESt05K6UOoH777+f559/3vX80Ucf5bHHHuOCCy5g3LhxjBo1ig8//LDZ+0tNTaW0tJTHH3+cRYsWnXyDTuL2I/eKajteAt5eejZeqa6uo0fYAHPnzuXuu+/m9ttvB+C9995jxYoV3HPPPYSEhFBQUMDkyZO57LLLmtWVsmjRIq677jrOOecc9uzZQ15eHpGRke19GKfM7UfulVYbfj4WbbVSSjVq7Nix5OXlkZ2dzdatW+nRowfR0dE89NBDjB49mmnTppGVlUVubm6z9rd48WLmzp2Ll5cXs2fP5v333z/5Rp3AI0buWpJRSp3InDlzWLJkCTk5OcydO5d33nmH/Px8Nm7ciI+PD3Fxcc26SGjbtm2kpKQwffp0AKqqqhgwYAB33HFHex/CKXP7kXtFtQ0/bYNUSp3A3LlzWbx4MUuWLGHOnDkUFxcTGRmJj48Pa9as4cCBA83az6JFi3j00UdJT08nPT2d7OxssrKymr19R3L7rFhhteOrI3el1AmMGDGCkpISYmJiiI6O5vrrrycpKYnx48fzzjvvMGzYsGbtZ/HixVx55ZV1ll155ZUsXry4PcJuFTHGdHYMjB8/3iQlJbVo2wVvJnHwcBkr7p7SxlEppdpCcnIyw4cP7+ww3F5j30cR2WiMGd/Y+h4xcteau1JK1eUBJ1RtOvWAUqpNbd++nRtvrHvHUF9fXzZs2NBJEZ06t0/ulVY7of7Nn8BeKaVOZtSoUa6rTt2V2w95K6ttOvWAUkrV4/ZZsaLapjV3pZSqxwOSu11r7kopVU+Ls6KIDBWRLbX+HRWRu0UkTES+EJEU52OPtgy4vprpB5RSSh3X4uRujNljjEkwxiQAiUAZsBx4AFhtjBkMrHY+bzc6/YBS6kSKiorqzArZXDNnzqSoqKhF72m1WomIiODBBx8E4IknnnDN+W6xWFxfiwgJCQnEx8fj7+/vWr5kyZIWvW9tbdUtcwGQaow5ICKXA1Ody98AvgLub6P3qcMYQ4VVWyGVchv/ewBytrftPnuNgoufavLlmuReMytkDZvNhsXS9MDws88+a3FIK1euZOjQobz33ns8+eSTPPzwwzz88MOA42Yg9Ttx0tPTmTVrVpt26LRVVpwL1ExsHGWMOQTgfGx0LkwRWSAiSSKSlJ+f36I3rbYZjNG53JVSTXvggQdITU0lISGBCRMmcN555/GTn/yEUaNGAXDFFVeQmJjIiBEjWLhwoWu7uLg4CgoKSE9PZ/jw4dx6662MGDGCGTNmUF5efsL3XLRoEXfddRd9+/Zl/fr17Xp8TTLGtOof0A0owJHUAYrqvX7kZPtITEw0LVFcXmX63f+Jeemb1BZtr5Rqf7t27erU909LSzMjRowwxhizZs0aExAQYPbv3+96vbCw0BhjTFlZmRkxYoQpKCgwxhjTr18/k5+fb9LS0ozFYjGbN282xhhz9dVXm7feeqvJ9ysrKzPR0dHm2LFj5j//+Y/51a9+Vef1wMDAE8bYlMa+j0CSaSKvtsXI/WJgkzGmZjLkXBGJBnA+5rXBezRK78KklDpVEydOpH///q7nzz77LGPGjGHy5MlkZGSQkpLSYJua+58CJCYmkp6e3uT+P/nkE8477zwCAgK46qqrWL58OTabra0P46TaIrlfx/GSDMBHwDzn1/OA5t+/6hRVVjvun6o1d6VUcwUGBrq+/uqrr1i1ahXr1q1j69atjB07ttF53X19fV1fWywWrFZrk/tftGgRq1atIi4ujsTERAoLC1mzZk3bHkQztCorikgAMB1YVmvxU8B0EUlxvtb0mY5WqrTqyF0pdWLBwcGUlJQ0+lpxcTE9evQgICCA3bt3t7o+fvToUb777jsOHjzomvP9ueee65R7rbaqW8YYUwaE11tWiKN7pt1VOEfumtyVUk0JDw/nrLPOYuTIkfj7+xMVFeV67aKLLuLFF19k9OjRDB06lMmTJ7fqvZYtW8b5559fZ6R/+eWXc99991FZWVlneXtz6/nc0wqO8fTne7ht6kBGxnRvh8iUUq2l87m3jVOdz92tZ4XsHxHIc9eP6+wwlFKqy3Hr5K6UUp3ljjvuYO3atXWW3XXXXdx8882dFFFdmtyVUu3OGIOIdHYYbeq5557rsPdqSflcewiVUu3Kz8+PwsLCFiUo5UjshYWF+Pn5ndJ2OnJXSrWr2NhYMjMzaek0I8rxARkbG3tK22hyV0q1Kx8fnzpXhKqOoWUZpZTyQJrclVLKA2lyV0opD9QlrlAVkXzgQCt2EYFj2mF35e7xgx5DV+Du8YMew6nqZ4zp2dgLXSK5t5aIJDV1Ca47cPf4QY+hK3D3+EGPoS1pWUYppTyQJnellPJAnpLcF558lS7N3eMHPYauwN3jBz2GNuMRNXellFJ1ecrIXSmlVC2a3JVSygO5dXIXkYtEZI+I7BORBzo7nuYQkT4iskZEkkVkp4jc5VweJiJfiEiK87FHZ8d6IiJiEZHNIvKJ87m7xR8qIktEZLfz/+IMNzyGe5w/QztEZJGI+HX1YxCRV0UkT0R21FrWZMwi8qDz93uPiFzYOVEf10T8f3X+HG0TkeUiElrrtU6L322Tu4hYgOeAi4F44DoRie/cqJrFCvzGGDMcmAzc4Yz7AWC1MWYwsNr5vCu7C0iu9dzd4v8nsMIYMwwYg+NY3OYYRCQGuBMYb4wZCViAuXT9Y3gduKjeskZjdv5ezAVGOLd53vl735lep2H8XwAjjTGjgb3Ag9D58bttcgcmAvuMMfuNMVXAYuDyTo7ppIwxh4wxm5xfl+BIKjE4Yn/DudobwBWdEmAziEgscAnwcq3F7hR/CDAFeAXAGFNljCnCjY7ByRvwFxFvIADIposfgzHmG+BwvcVNxXw5sNgYU2mMSQP24fi97zSNxW+MWWmMsTqfrgdq5ubt1PjdObnHABm1nmc6l7kNEYkDxgIbgChjzCFwfAAAkZ0Y2sn8A7gPsNda5k7xDwDygdecpaWXRSQQNzoGY0wW8DRwEDgEFBtjVuJGx1BLUzG74+/4z4D/Ob/u1PjdObk3ds8ut+nrFJEgYClwtzHmaGfH01wiMgvIM8Zs7OxYWsEbGAe8YIwZCxyj65UvTshZl74c6A/0BgJF5IbOjarNudXvuIg8jKPs+k7NokZW67D43Tm5ZwJ9aj2PxfFnaZcnIj44Evs7xphlzsW5IhLtfD0ayOus+E7iLOAyEUnHUQo7X0Texn3iB8fPTqYxZoPz+RIcyd6djmEakGaMyTfGVAPLgDNxr2Oo0VTMbvM7LiLzgFnA9eb4xUOdGr87J/cfgcEi0l9EuuE4cfFRJ8d0UuK4S/ArQLIx5u+1XvoImOf8eh7wYUfH1hzGmAeNMbHGmDgc3/MvjTE34CbxAxhjcoAMERnqXHQBsAs3OgYc5ZjJIhLg/Jm6AMf5G3c6hhpNxfwRMFdEfEWkPzAY+KET4jshEbkIuB+4zBhTVuulzo3fGOO2/4CZOM5OpwIPd3Y8zYz5bBx/mm0Dtjj/zQTCcXQKpDgfwzo71mYcy1TgE+fXbhU/kAAkOf8fPgB6uOExPAbsBnYAbwG+Xf0YgEU4zhFU4xjZ3nKimIGHnb/fe4CLu2j8+3DU1mt+n1/sCvHr9ANKKeWB3Lkso5RSqgma3JVSygNpcldKKQ+kyV0ppTyQJnellPJAmtyVUsoDaXJXSikP9P9pA1Yuhv5IRgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_ATT','train_ATT']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABmmElEQVR4nO29e5wcVZ33/zlV1deZ7rllkkymExJIgJCQcEkgKOAFQZYfchMVVl13V+Fxl+dZcF0Vdb09K/usuo/P6vOs67Lq6iqCCCp4AUFEUeSSQAgJJDGBTEjPTGaSyUy6Z/redX5/nDpVp6qr+jLdPTPdOe/Xa149XV1dfarq1Pd8z+d8z/cQSikkEolE0l4o810AiUQikTQeadwlEomkDZHGXSKRSNoQadwlEomkDZHGXSKRSNoQbb4LAACLFi2iK1eunO9iSCQSSUvx3HPPHaWU9rt9tiCM+8qVK7Ft27b5LoZEIpG0FISQg16fSVlGIpFI2hBp3CUSiaQNkcZdIpFI2hBp3CUSiaQNkcZdIpFI2pCKxp0Q8i1CyDghZJewrZcQ8ighZJ/x2iN89nFCyH5CyF5CyFubVXCJRCKReFON5/5tAJc7tt0O4DFK6RoAjxnvQQg5A8ANANYZ3/kaIURtWGklEolEUhUV49wppU8QQlY6Nl8N4I3G/98B8BsAHzO230MpzQI4QAjZD+A8AE81qLyz4ujhQ3ht+6M450/+svRDSoEX7gLWXQf4w94HSR0D9v8K2PBO92M8+x/AzBH7dn8HsOWvAC3A3meOA3t+AWy8ASDE+7fGdwPTY8DJb7S2Df0eePW33t/hKCpw9nuBrkH2XtfZ+a2/jpUHAHIzwDNfB/KZysdzY8UWYPUl5ttdv/sJpvc+AQAgnYtw3jtvB1EMv2HyIPDC9wGqA4oGbPoLoHOxUbYisP17wIZ3Ab4g25ZJAM/+O1DIsffrrwMWr7V+e9ePgJNeB0SWsveUsmOI5+fG9BFg27cAveD+ueoDNr0f6Ohj74t5dt3Oejf7zMlLPwbGXi53lbwJdQPnf5DdKwBIjADP/xe7HiJLzwTOuMp6f+hZYN+j3seNLAU2/aVVtyaHWBlPv8LcZdfwcTzy0mEoeh7nj/8Am5f5oSlzoM52LwfO+TM8vOswzj2pB/2Zg8Cu+9j90/zA5pvYdQGAXMqon+nKx+0/DTjzeuv9yHb2jM0GRQPO/XMgsoS9N+vnOwFfiG3LJoFn/h0oZD2OoQJnvwfoihnH0Fl9Th1j71ddBKy62Nr/6a8D0QHgjKtnV+YyzHYS0xJK6SgAUEpHCSHG04pBAE8L+8WNbSUQQm4GcDMArFixYpbFqI6hB/8Jm0a+h/Trr0Eo2mv/8OgfgQduAQoZYPMHvA/yzL8Dv/0nYNk5wKLV9s9GdwAPfcR4w422kSd/YCNwypvY/y/9BPjp3zBjtews79965O+B+DbgowcA/uD99DZgYp9wfC8okJsGLvs8e3voGeDB/87O77yb2LaXHwQe+5+O8lYLBSLLgA/vNrcs+fWHsZ4eNd+P7n0zBtZuYW/+8FVg6zeM36HMyL/p4+yzVx9n10P1AWf9Kdu284fArz9v/dyR3cC7vsf+T4wC9/0FcMF/B956B9v22tOl5+fGtm8Bv/nHMudLAdUPXHgbe7v3IeCntwLBbmDdNfZdcyng/g8YDcUsrh/A6sXKC9n/z3wdePIrjmNRQAsCp17OjB8A/OIjwOgLHr9pHHflhczgAcBvvwjsuAe4/TUg0AkA+Opj+/DIy2O4UNmJ2/xfBV7FLM6hVljZMivegL+6axf+9i2n4n8c+0fWQHIiy4Cz383+f+XXwGOfMz4oVzYKEIVdI+P88MingKHfVfhemePpeeDNf8/eHvgtq5/cYAPAzvuAX/9DmbJR5qBc/o/s7fA24OHbrY93/hC49QVjVwo88UXg1D9ZUMbdizK1zrGR0jsB3AkAmzZtauqKIb2TOwAA2YnXSo17dpq9xreVN+7xrdar07jzz27bCXQbDdWRvcC/ngekJqz9Uket/b2Mu66zsmSmgGOvAIvWsFZ/Yh9wyaeBiz5c9lzxjUvZ90vKvc0yfvGtQCAKfOyg1XhUyzP/Djz0UeD4MOsdHB9GPz2K+xb/DyzZfB0u+vmboB/aCnDjHt8KrHoD8L4HgX97vVUeXia+Dzfu8W1ARz/wd/uAH90MHHiCPQSE2M+l3Pm5Ed8KLD4D+GuPTuRXz3aUTbjfTuM+uoMZ9hvvAU77E+/fdGNmAvjSyey43LjHtwGDm4CbHrP2e+nHwA//HBjbBQyewxqUsV3s/l/y6dLj8voW32oZ9/hWgBZZg2D81lQ6j/NX9eKfT+sDfgM89tZf4ZILNtd2DrUysh24840oHHwGlHZgKp1n57zuOuDarwOfXwwkhq39+f8feQXoWOR93H2PAnddz46/6iKgWACGnwfO+2/AFV+svZxfv8i7fnLjHt8GhPtY2dx639+63L0efXgv8OIPgEc/DcwcZec1eYDZh9im2staBbPtj40RQgYAwHgdN7bHASwX9osBGJl98RpAMY9Yei8AID8ZL/28YHT9xBviRNdZC+y1X3wb0LkE6BJOPWSMMacnrW28ayYaJyfHXmGGXfyt4efZa6yKhzC2mVX2Yt5+DGeFGzyndsMOWBXRLBs7l+GOdfD3rsA47YZv1Di/XAo4vMsqd2wT21/Xy5cttpk9OLHNwPRh62Hn+42+YMk2bsdwQqlx3DIPUWyzYQwNP8N8sF3uFf+twVk8lB19QO/J1nG5QXLeW/6e78cbFK860LcGCHRZZUtPsl6pWF4AiXQe0ZAPHVn2yB5Br/NIjWfJetYL4eeSGAWOH2LnogVYY+407qqfGdFyDJ7LXvn5HdkN5Geqe07ciG0G4s9Z8piXM8Hrp+sxNrF7xWWb+FagawWTzJz3lL/OtrwVmK1xfxDA+4z/3wfgAWH7DYSQACFkFYA1AJ6tr4h1MrYLfjBDoB8fLv2c63oT+y3j62RiP9PLieJh3F1uODfu4jG5oS9niPhn4m/Ft7L3y872/h4ntolJFGNGcFN8G/vusVdYWXIzwNhLs69QS84E1ICtbFn4MNF5GqJhP7brq9FxZDv7bPQF5jmaxn0zu44T+60eClFYeXIzVg+FG2FnQ8L3L3d+bkwYDWa5c45tZuMcx+OsYRzZzo4rNiSc+FagZyXQ6ZqvqTJiQzL+EnMwnA1PdBCIDJQ2Xl4NiqIAsXMtgzH8HHslis04JTMFRIM+hNKHcYR2YSrncqxGo/qAZWdDHWHlWHR8J9vO70d0GRt34CRG2LZy41IAEO4F+lbbPWxg9p5wbDOQS7JGkTsERAHGX2Zae3oKOLq3spNQzDKnBmBl4/sPnAUQ1X5PfR32MaUGUk0o5N1gA6KnEULihJD3A/gnAJcSQvYBuNR4D0rpSwDuBfAygIcB3EIpLbofeW4ovma1LTTh0okQB234A+GE34y1b7MMESd1jBkW5w1XfUz6ED13/n85QxTfyjywlRfZK8HiM4BAxP07IqJ3cHwYSI6wcvNtIy/YDW6taH4mKQnex266Cpo/iGjIx4z79EF2fs6HzSzbVquHcvqVTIcf2V7aQxE9Pm5wxXMpOT+PRjP+rP24bogNyZhhcNe+zWhIdjqOt60+b8tsSA4J18hxPEJYmcQ6UKlBiW22DFF8GwACnHaFrUeSyOQRDWlQp0dxmPYikc7P/jxqIbYJgaO74EcegzO7mGc+sIF9Fh10Me6uQ3UuxxUayvg2ILyIXadZlVGsn6+y53Xt24T6+Zx9v0rHEHsoAAvYWLrefk8Hz7EG1htMReNOKb2RUjpAKfVRSmOU0m9SSicopZdQStcYr8eE/e+glJ5CKT2NUvpQU0pdA5mhZ3CY9mCcdkNJuhj3ghAx4mkcDIO78U+ZYRzZLnxWpmsV6gbSghFPHQN8Yfv33H4rdi6w/HxmZLJJJmVU6410xYDOpew4/HzOu9nqCdQjKXBim5lHm0sBI9uxXV+NgE9BNKhhu77GOr/4VqBnlaWb9q0Bgl32cmz5K+u8zR7KOWyb5mfejmhwz7ja8mj5MTbfZPeInPAxhkWneZ+T2JCYZftr61w4vEGpy7gLDUl8G9Cx2Bqrse23memyM0era1Bimy1DxB2CU95kNiS6TjGdZZ47SYzgqLIIyYxH9FCjiW2GUsxiLTmIVZndwNINVhRZdFmpLBNdVuVxNwEz48DUa5Ulk0r0ncIG0MW6df4H2Su/VyBW/XQjuow1TPGtlpQr3rfYZubE5GaAwzuB5efNrqxV0PYzVJXhbdiur8YI7YU6PVq6Qz7FXsN9ZYzDNsPgGjfCqRF7SSahXocsc4wNbHnJO6Jkwh/UF+9lUkasykrAPb5Dz7Lf0ILsu0vWWZW292Qr5G82xDazRnHH3UAhg23F1QhoCjr8GnZiFXSozFs+tNVesRWFNSrxrax8gSiwfAvQewrbN/4ssHidFfkAsHMZeQE4+KTx/jzDozXOTw2whpCfnxvVjDEY0gE7rjGGsvx8FsXhNkBWzyDYkvWAFmK/c+hZb4PEr93LP6muQeEa9KFnrDEG/p1DzyKZLYBSIBLUgMQwJrV+JDJz5bmzcmxW9mJN4Y92oxZdxrzkXIp54FyWqeG42PcIk1OW19Ho8nEefl/8EVYHFp0q1M+1QDBaoUxG/Tz0rL2HwsubS7IopnJjKA2gvY37zARCyYPYrq/GYdoHf+pw6T481nvlhcZgim7/PDvNdNHYZqbx9Z5SOsCyZJ17jHW4t9Rz74p5G6KR7cygxzZbxuPpf2OvtVQC7vH98ZfM89X8hsfwnGVM6oF/3yjbdn01ApoKRSHQAp0YC53Cwj6nD7sPFI6/zMLMBs81tOLNhlF9rtRocg1z23+yHklXzDi/IXZ+y4TzEwfDOLWMMfDBsNf+IAzqbio17mqAjT3MFt6Q7HvEXdLjcI326a9b5StHuJf1jnb8wHAINrPG0mhIuATT48sDmSkk/f1zJ8tElyEdWoob1V8jiJz9XLgEkxhh0SPFXPWyDD+/Z/6dvW9E3R7fDbz6G0sy4dJPvMoedGwz60nsfYiFvPIeilg+/lzX04OuQHsbd6NbtF1fg1Hai4CbcefRMqsuBrLHjVhyAdHgAnaNT9eZwfSqUKEeS2enlP0f6rEMrbMhMSWTc62GZGIfkzL6HOGX5eDlsQ1ObgayCdaFrfcB6BpkHu3EPugdSzCCPgQ0VpWiIR9eDa61rqObsaY60zTFKJqZI+z6e0WN8HPh3pW5TbgvfDBMxHn/yhHbzAzL1Gv26zY5xCZBAewB5w1KPcQ2sYFl/htucI12Yl/1DUpss3DtNwOqxoxUfKspwfQbKupMYAkScyXLAJjo3oBTFNZ7pqJR4156YtiSZ6r13Pn5TeyrPuigHLFNAKjR6Ar1M3W08qC8eQyX+snpPZnZgIl99Q3KV0F7G/f4VhShYq9yCg7TXvgLSaZhi/AB1ZUXmd9xHgOA1eWNbbIGw47+kRlMT+MuyDLZBNPrQ72WoXUaovg2ZsTDRngaP+7gptrCFpedxTw+8Rg23a8B3oJxjNzAuQAIAj7DuAd92KOezvbRgkyCEBkU9ErXsjmuJW9IxM+4RyuUwzaQJVLLGINbOfjr8DYWNTP6QmO60vwYlQwS36/aBoVfj0CUyQl82+EXkZxhczp6i6yhyoSWIjlXsgyAsS7WOI3TbqRCgvEWPXc+sFqtcQesc6426KAc/DkHqqufbgxsZLNdxbJxROekiZIMcAIY99d8q7Csvxej1DCYCYfunk+zbh0f7Nv5Q+C571h/e37mbnD/8H+Bp79m3+Yk3Mu6x3rRMvLhXndDZMZiuxjhWiuBv4NJP+J3e43BIjeDOxuM42aWMGPtV1lVigQ1vEiMQVUuCYlw6QCwzm+J0bX26qE4rwP3aMVt4mCYSHwbO/dqxhj4YJhocPmDuv17wJP/wsYaGtI4GuV2jjF47VdtHTAdgnMth4D3SEbZZL7uPItxz3csRSI9d577SAe7Z9v11Uhkhd+NDLBXm+depSwD2D3segl1WwPv/Hj9a1nIYqVBeY4vxFJHiGUTmSPjviDWUG0aoy9ip34uTlncicNj3LgPA/2nWvsUMiyviaKwXC4vP8D0NhFx5uqSdWzSxbN3svfRGDMeboR6wKYjH7fkmVAPM2CBKHD4RWvf6XHWIxg4y9p28hvZgIyQx6Vq1lzKwgd5jhlFAVa/haUmcMuVUiunvAl4zIfEsgsBHEPAxzzpaMiHvROLWZdz9Vu8yxaIWA2m6mPH04LuPZQ1l7LcOuKs3tWXsoki3AgQwgzy6Iv27x5+sbaHaPVbmGTEx1D8YWDFBayR3/MzJo+suKD643kRHWCNbKV7u/JCdl2qrQOLz2A9HfHaG+cfGNsOYCM6jQlMNDKARGbc5SDN4VBgDY7SKH6rb8TKdAEDXcYH/jB7LhIjLMBB0dgzVi3LtwD+Tu/6VitrLmUGmkd5qRqrn4pafQ969aVsvEec2Mg55RLgiS+xmdtNpH2Ney4FpI9hX7EXK3rD2KMaN0oMuQJYZdKMpEBv/xbw1rHSY3HPAmCG6G9eYAYbYJXS64aHDOOVOmYNrIZ6mSHqijlie43Zs2JI3KI1wMfj9gGZannT3wNv/Lh923V3WjMw62XpmcAnhpEczwD4vaW5B31IZovAR7ZZ0omTyz7PdHCRd37XO4Tt7PcCG26w9wLe9Engjbfbv9O9nA2ecnSdXWO3MEMvrvw/pdfoPfezcESAednBrtLvzYabf+N9jThdMZYbpto6oGrArTssWQBgsyO7liN69AUAGxHKjAGhXoQ7IkjlRpEv6vCpze/EJ3UfLsj+P+Sh4lqnHMRj3fMp9rzVEvvd2Q989NXZPSduXPoPTEIVecd3ajvGGz8OvOGjHlFQ587+ua6B9jXuSSa/DBd7cE53CEnfIpblxjmRKZ+xMr6pmuXpliPQWb4rzeGeafoYkJq0byuJ7fXQGmdbARQFJapboydLaAFkCyyU1BpQ1Vh4XbnegaICcJRFLVMVCSmVd9zOLzrIBowLObb/zBEWblZLF9/tGmmB6upFrVTbg6q1Drhp87FNWLSf5fTzzYwC0UFEg+yaJzMF9HbUOUBcBelcEXnD5JRE6fDnIT9Tm97OaaShdKtb5epntccQabJhR/lfb3EMwzmKPsR6QtD8IUyr3aWee0Ew7o2Ge+7pSbssA7hPuQZqM0QLgGyeeeABjRnFSNCH6WwBut7UXHDucKNgNOw1R160M7HNiGZHcZI/wSbzRZchEmSNy1wNqqZzljdcMnmKPw+1xLhLytLGxp0Zy8O0F7GeMIJ+FZNav4vnnmKaZjPg+alFWSZobIsOMp2d5y2pNlnSAiNbYA+sFS2jgVIgmZ27gToTM6RuxP4qjYWpu2/xH2DXpWsQ0RAz7nM1qJrKF9kEKqB08lQ0xsINpw61nIOzUGlj4868tsO0B4PdIYT9KiaUReVlmUZjk2WOMa2Wd++iywBQNtEHYOWKDMwuU+M8ki1wz92Kcwdcut1zgRlSZ3jsLdobagpLN6AADVuU3WyiUHSZKcvM1SzVdK6IJVHmSLnKMgCbsCYb44bQWpakFhIjSKkRdHRGEfKrCPlUHFH6XGSZdPOMe6CLhdVxz51LMoC7l9mCRsgy7ka0jNnVXwie+zCg+FgyqRMdXxAHfKtxUeEP7H10cM4b4nS+gK6QD0GfUjp5SjTo0rg3hLY27keVRRjsZoY75NcwRvusHBacfLp5soyiMBmGa+5cgwdcvMzhlqzU2bwhywgDqsDceYM2gl0sH4itwVzWcr2hZvGyeioW6UbUT3SZt0TSJNK5IkI+FZGgz8VzH3T/XzJr2jdaJjGMUZ3p7QAQ8ikY5QsTJEfZpBeAGXeeqdGFf318P57czx6IpV1BfOn6jVAV76xzP39xFBMzWfzZBSvZBp5fJnUMaV8XPv/jnfjsVevgE71MSkETI3hc0fCN/3jadryQT8U/Xnem2Z0V+bffvIL1g1FctKb+Kcz7x5P4zh8O4rNXrSt7fr/YOYqJ6Szea5xfiSwTrOwN/urlMRw4OoObLj7Z3Pbdp4bw0K7S9BDvv3AVLlnL1rSklOKOn+/GNWcPYv0gC0fM5Iv48L07MJliYxdf8/WjW5RlDEOxf3wa//XUED7ztvLn9/CuURxJWucn8uqRaXzupy8jX9RBCPDBN5xiXvuiTvEPP3sZ7z5/BdYsqXOWpEAmX8Tf3vsCplKVDfDSaBBfeod3/dyur4G5mJvguZfrZW0bOoavPLYPRY8B8qBPxR3XrsdAF3Oi4pMpfOonu8x6Id6/VK6Ivs4AokHNZUBVCDd2cXJeOTKN7/xhCJ++8gxoZcI2H951GOPJjPX8AbjrmYOIBn1420bruP/rod3YGWfhzCf3d+Afrl4PMttsklXy4I4RTGcK+NPzm7usKKd9XZrECA4Vu9EfYSFHYb+G4aIhi4jSDJ/E5MH3n3kNew8nMTKVxo+eH8bEtMfCuAb/9dQQvv6bV6wNPAVB+hgO58O465nX8NqxFJvE5O80kyWRYg5PHPZjLJFBvqgjX9SRyOTx2J5xPH9w0vW3/u03+/Hj7cOun9XKL3YexnefPsjKVobv/GEI3/7DkPk+5yHLlMtZ8v1nX8P/e3y//bhPHcTew0nz3PNFHS8cmsL9z1urZx2ZzuIbvz+AR1625iLsH5/Gz3cyg7xr+DgO5LrssoxhKB7aOYr/euogxhLlFwW/Z+shfOvJIdfPfr1nHL/94xFkCzq2DU3arn18MoVv/2EIj+1p7KSgl0YS+MXOwzg2k7NdG+ffyFQaP9o+jCNJ7/r5bN5qTBEZQKdfAyHlG+KfvTiKp1+dcP3NmVwRv94zjif3W0tJ/m7fUTy+9whSuSKeOziJn+6wxrgyeea5R0O+0t5CIGLJmJ1LSsrx+J5x/NdTB3GwQv38wdbX8M3fH7Bt+88nh/BfTw2Z7wtFHd/43QEcnEhheCqN7z392pwMKn//mYP4j9+92vTf4bSn517IAjNH8Fqhx/ROgj4VfzSNuzCoytMPeB1K13HJ2sU4f1UfPvzDHUjlip77AkB8Mo3DhoH2qQrT2ZOjQHoSqQ7m0SXSeRa7zWN7edgm7cUd156JLSeziJnhqTRe/0+/du026zpFMltoWKWMT6bM11WLXDJcCudHhUk+pufus8sy5cLr4pMpHE/nkczkEQn6QClFfDKFd59/Ej515Rnmfn/2rWcRn0wL32P/i8aIX5v/efV6/MfvXsXR0UVA4iUrdayxkIf53Uwey+B9vxPpvKexi0+m0RnQcN8HL8A7//2pimVrBPy+/N8bzy7bI/jJ9mHc9oMXkMq51wdKKfZmezDT0YsOtQgEOqEAiAS0sg1xfDKFU/o78cMPvq7ks2yhiNP+/mGzjHx/VSG474MX4JqvPWk7dipXRNivIhr0YSrlsgRUdBlLzuUS/8+va3wyjVP6veeYJDKFknuQSOcxI0RvjR7PoKhT/M0lq9EV8uGD33sehyZT6Ao3aHKaV9nSBQxPpqHrFEqZ3mOjaE/P3YhzHqW9ZkRA2K/itXyX7XMAhixTxrgXKTRVQdjPPNN03tu4F4o6Dicy0Clw+LjhIYZ72ezGzHFMKywPtFnho8vY4g9C2CYfIwBgRTO4GPDpHMvN3Si9dHiKGadhwWA54ecnXgMeCslzy3QGvMsMMCPDf4P/5rGZHDJ53XbuADDYHbKVZ1gw0Bz+O9GQBoUQHCF9LAJpetyIvBi0/ValxjCRKSCRydsaMPP3p9IY7A6BEFJV2RoBL/dgT/lB/6CvfP1M5Yoo6sBo73m2/Ciu+rdAfDJdcl84AU3F4kig5DosjQahqQqijmOn80UEfSoiQY8Gpf80a3FvB3z/cvUTMBrnTMF2/xKZPMYSGbOXaV7T7jAGu8O2bc0kkckjV9RxtELvv1G0p3EXjCWXCUI+FcfyGqiiAZkE269YAPR8WeOeL+rwqwqChnEv57kfTmRMbfIQ92ZCvWyhBQAJw7ibXi2fcm147uOkDwNdlkTU4degEHcvmGuWjYpK4Z5nvMzDw89PvAbZArs+3BPRVAUdftXTyE2l8pgxvh8/ZjfyMYcBi/WEMDGTM71RXjbxnPm1iQZ90BSCcdJnrEZkLNlnyDLcu6w0YSeZySNfpGaPRCQ+mTbLGOsJ43Aig0JRdxy/sd37+GQavR1+hP3lO9mm8+FRP/n92L7xc8C77zW3M4nEu8zDwjm7EesJ2QyjeI2iQZ/teqS55x7yud+Hq/+f5zR/Xn6xl+BGMlNAUadmI5cr6MjkdZvDxetRrCdklrVcvW8U/FocmoPfAtrcuI/SXlMmCPlVFHUwbY+n/eW53MtEy+SLFJpCEDY8o0wZz93NkxPDHxNg3UnTe4wuY17m1GsoQoUWWWIbLFIUgk6PbjP3iBohA+g6xciU3dC6wc8pW9DNGajZvG4OpnKiIW9vUDw+/58/WE7vlD94VtnYg22XZQzPPeiDqhCM8UFznh0yOmicX8bYv/z14vfGrfzDkymzjIM9IRR1isOGhh83ewYN9tzLeM4ioQo9S35e4c4uW52MBjXPa3I8nUcyWyjbaxjsCdsM4/BU2tw/Ihw7X9RR0CnT3IM+JNKF0t5RIGJN/PMofyUPm/8e319sROJG/eH1eKA7iO6wD2G/WrFHUC+6Ts2yzEUvAWhb484nMNk9dwCgfsG481WYyskyug5NVcyHp5zn7qbBImw9SFPUMO6m577MXPNyUunFQG+pluhlKE3j3gAZYDyZRb5IjXJ7e0bi+XEjki0UTb3dLHPQZcDMPEaq5P9h05OyRy1x437I0asQGzt+HTqDGlSFYBxGTLtp3JfhyHQWOcPDLifL5Aq6eV7O8icyrLtvee4hW9ndytYI4pOpsp4zh9dvr/rJz4c7O5xyDTG/P877IhLrCWH0eBpFnSJXYLId3188Ni9XyK8iGtKQK+quvSMvLM/d2zDmi7r5O6aRF+6H1TtNYUk0gICmghCCWE+oYo+gXmZyBfCAo2b/FqdNjfsICloHphE2B1R5t7Xo6xSMu3GRPYw7pRT5IoVPJVVp7rxF7u3wW62zENt+zPTcBVkGAIafxyjtRczFQ/MylLzSNiKPC/eI+zr8ZT0Y0eOwjLtuRsqYZQ65hLoZ8AesT7hG8ckUIgENXSH7QJqph3KN3mNANRJghl1VCEZhNKbDz7OMi52LbQahnGctennHHY0A/21eJu5Nx8uUrV4opabOXwnufHj1LEX5SiTiFpZoYJ1zGc+9O4R8kWI8mcHh4xlQCrMeR4M+zOSKKBR1s1whv2rmtKnlWvF9y9XPpEujL/6GONYjntNgd6jp3rTYyDS7l8BpU+M+jFSQhVPxiRq88hd8nWwVJICFQQKesgzXz32qYg1YeUQjAMxILY4EsGpRh9U6C13giSKLQkmKA6oAkJvGa4VuVw8tGtJcvU3+sFLKBlfrgRuo80/uNSN93PezPA6u7TLjXovnziJOzlgWtQyj0JUXWRwJwKcSM0JHjHjhJDMFswFXFYLjeieLfspNm6ljxXKXy3lj6xE4yi/qtACwzDAOw1Npc6CZl6dRTBgDzdV47uEKPUtr4Nlu3J2DniLOc3ZD7MFYnr5h3I1ewnS2YJaLRcvUPnmKX9exZMYcxC/dJ1+yv/gb4riS2BuJOaSlZiBe47nQ94G2Ne4jSAaYcXfKMgVN9NyNi+wxialgGHdNJeaAlteAFWANJtkGmcKW5z6hO2UZaybeCO1zNXARL89d9F7r9BZ5ZTtvZa890seBq+eeL8LvMO6RoHuDxH+LXaOwTdJwMyCKQrDM8KomU3mkjcRTYm8lkc6bDbhKCAqUWo2mOZiaFspVnefuNNLDhuHi9yjoU9EfCSA+mTIHmiNl9OvZYHrOZWQRTshX3YAqv1acaMiH6Zx77294Ko2gTymbDlgckIw7InssD71glovHubMyVd8QJjLsPlMKjE6510+xzvHz5fcxEtQwPJVCUacYPW53JgZ7QmZobrMQyyFlmXpIjGBKYzMHnZ57TutwMe7unjvXaX2KYmmaFWSZwZ4wBrtDGJ0yIme4LENUTOTZQ2IamFCPGWPPs1c6cUYcmKdo64LW77n3dvhxqhFHfcij8sUn0+hweIi5oseAqseDwrvEYiTMsMOTEuF6KH8g1g5Ebb2VRCZvNuCqStigucO4D0+x81scCZQ1vjbjkC713IM+BX2CoeONOG881g5EkcoVPXs+tVKN58ypPKDqYdzLZPFken+47MxNMZQwPpkGITBnq4oeejpfMMqpVTWLWUTXKaazBawdiJq/5XqOmVKHh7+uHWA9xfFkBvkitV1Ts/fRRGlGLMfwVNo11LbRtJ9xL+aB5GFMKIvQ4VfN6BPebc2pHS7RMu4PT6Foee5BY9Aw4+EZ8YgT7pUWdMpmQ3JZJtSDdMHwNrlh5hOZwCJ7vGWZCp57nR7HsFHuQccgodv5rTYaAFOWybto7kaD5FaB+QAhP9fdowkkswVPAzbYHWJeoVGmM4wHnJ8/k2Usz72o61aPyHjlPQM2wFdOlvG+pmKMO4d3551la5Q0w8dCKsW4A2yegUK8PfdkpoCgT3G9V4BHdNBU+TBIgDUqizr9ZgO8NBo0e3JiYrJ0jjV4Yb+KLjP/UHXXKZllczr49fXyfN2iqPh9PGMgitHjGRycMK5pt2jcWQPFQ3ObgViOTF7HxIzLJK4G037GfXoMAMU46TO7hYA1ySOjhKuOluExzD5VASEEIZ/qqWnyiJPBbstIxifTbC1O1Q+Ee0293tb9M4z7YfSaHo8IW7auUJLbwx7rXa/nnsJgdwgDXSEQ4q4J8vNbs5hJS9wTc42WCWkl8fCAEVqXYaF1/OF6+lWW595r0C7WE8aRZBavHpkGAKwd4LN8XTx3hbDr5PTcjfNjjU45z710cpR1jUp7F4PdIYxMpXHImBJ/2lJWtkZ17+OTaUSDWskgqBuEMOmwXLSM23HKJXorN4FJhDfAzrBNMRUFn6vAE4cB1Xvu/HqeuiQChXgPSCZdxkwS6QIUwr5b1CmeM1J5iPdysHsuPXdWR+ZiULX9jDuPcdd7bWFfXDPPKB3MYy/mK0bL5M0BVWIcQ/Xs9oqDSVY3L8W881AvEOq1wuxEw2F4l4WOZSXaNWB1o6cd3eZEJm/KIfVo7nzGaKwnBL+mYEkk6FrJuRdpGnfDE3MbUDUfXqf3K4Q88ofrmQPHzG1u8Afv2aFJRIKauV9SeHhNzZ0bd74kXtegGXFieu5lGkJuHAKaUmKg3QZ9Yz0sUmT7oSksjgTQ3xkwy9QIhifTVentnKDPu36K10kkKujiItPZAqZS+bJhkBw+fuIcO4nYZBkhFNKjfnjBy9bb4cfSaNBzQJIfL6Aptjj3SNCH5b2sXLy+iY3Qok4/AprSVC2c1621Zu9DGvfamXoNADBMe2yeiqmZE6OyZpMVo2XyRhyuZqSMDfpUz26vOMvSDJPj3bzoMiA6UBKDCwDoXYUsCSDYMwA3vHJuJzJ580GqR5Y5Op1DtmBN/feK+eWVcc0SZty5J+YaCulhMKxp3yEzEua5IeNh8+j+83N8buiY6X0DMKeYJzN58xppCkGRUqBnFftyzyoz4oR9t/yAaiKTh0KAga6gIydKAcdmciUSxaBYtp5Qw1Poeg00exH2q57RXAnhOolYmSHdG+JqJKHBnhDiUyynkri/XZaxBlSDPgU+lVTd4zRj9I3GPe6luafzIIRFMolx7tGQZtbv54aOYVGn3xyjAFivZ7CnueGQiUweIZ+Kk/pYxBx3lppJ+xn34ecBLYQ9+QFbZeY3c4YYlS+brCJaxjDuVXnuVhx00KdiUWfAqizv+E/g8i+Ysb62QbcLbsHNgS9goNc9KZSXl5NIF8xwvHo8RatRMuK3PSo5P7/V/aycGXESU8mAqnvyMLF3wyNhZowp6T1hd+mBG4uZXBGxnrBlQNMsjYFOrWukcM/9lDcD738UWHaWMCgZNiOPvAazWOSND12OiT1e8d7LHWVr5OIXtcS4c0LlPPdMwVWWsRokZ0NsD2ssR6wnhFxBR1GnNk8/EjCyTmYKZrnCfjZxqFwIZknZ03wClo/VT0/PvWDOlzBDIdN5RAI+81mZyRVdr2mzwyETadbIdIV8RsSM9NxrJ74VWHY2JrPUHK0HLM99BoLnXiFahs/a5EmxQn5vzT0+mUZfh+URDPaErBvYsxL5cD/yRWpGW0wbla/o68STiSWeD7GpiToMeDKTR0/YXzaPSzXEHSF+sR4h0sdxfr0dfvR1svKnxAFVlxmqgHusuBhax8/ZOVApsjQaNPOTc2mFH9t66Nk10rhxJwRYfl7J+UVDmmfeGHZMJl04o33EBkJkmXDPBrvtZauX4+k8pssMNLtRrn4m0x6eu4f+bZ5zlZq72/88fUYykzfLxce+PJOHuZVdSDER6wnZcvqIJAwJRgx5ZT0WzQxdBdwlQGciuEYjjnk0+7c47WXcC1lgdAcQ22R6YRy/pkBTCJLUMOQ2WaZStIxh3Mt4Rs5p4k55g3/PXEPSMABjiQwKDo9HxNNzN7qbnkmYqsTZ/R7sFiJ9xP0M3dqZfdB9hqqHLGMMSnJD7pzK74amKlhqXLOYKH2kC0LstuG5EwKdwuaZi+dXKQQvkWYPoNOrjAuSm0jYr5mNdawnZDoTjYiWqSUMkhPyqZ4zVJlxKdXcvaSk4ck0/JqCRcY4QjnsE4Ls5eV5ZDL5IhRiX2u3as9dSJ0QM3L6jLrMxWDesc/WOCfSVo+Fl81NanImqWs0oiw2F5OmgDqNOyHkVkLILkLIS4SQ24xtGwkhTxFCdhJCfkoIiTakpNVweCdQzILGNpnGTyTkU5GkDs9d8VmLVjvgce5clgn5y2juk/YBt1h3CCNTGXNyCP/ekqh90M0raRbHbU1SSqnDEM2+QjojMryy5PHGS1UIApoizFB1n8QEuHjuUymHl2dJQeUQGwGfkX45mclbsy6DluYOwNbrEM+vkmedNBtM+5T8+GQKflUxB0xFxB4Pz+LZCFnGq7dQjrCH587qS8Hm7HB4Fk9ng8QjZarJOy7ev2UOT59P7EoZS+zxhr1S5JIIv8+dAc2sM27GkTdg4jORdBhV9upu3IHmRbEkM9aANnf8mh3rPmvjTghZD+AmAOcB2AjgSkLIGgDfAHA7pfRMAD8G8JFGFLQqjGRRmSXnoKjTEo0x5FdxXDce0Gyiilzu1iQmwFtztyIy7B5MrqjjiJG72TLuds+9krZpyTLWg5DOF1HQqeGl1Dcr0lluM9ZdGPDhETXWerTsOlDKkkWVRsuUlhkoTR8rps8th9ir4MdPZKwZhfwacUNUEIw7n1gGWJNqnHljOLzr7JwVPDyZxrLuoKuhExueclk8a0UcfK6WoEf9zBZ05Ip6ibPDcfOi41XEuHM6Axq6wz70RwJmz8557HS+iJCQtpjV2+oHVPmclXITjhKG9MQaZ2tAlddHUQZ0Yjo1TRpU5c4Y/62ZXBHHm7wweT0rMa0F8DSlNAUAhJDfArgWwGkAnjD2eRTALwF8qp5CVsNnH3wJ7zz4GM6IxnDcmJ3q1BhDfhVTRWZcf//SAcRf/iPeTBRc+MmHzH3esSmGO649E4BlJHgopDNa5p5nX8OnH3zJTDAmPgxirPuSaND0qBYbxp1XPh5R4/UQm4tfuEyNjwQ1RII+jCe9l46bSuVw/defwlduOAvrlpWuNDM8mcaKvtKYX3FChzOiJmzE+xd0Cp2ixLgHNBYRIVbemWwBk6m8zctb3uvtSYksd3hc3DOzoijsnrvukGX470TMXpC3LBMJ+hANasjkdWOwWGVerEcZedm4x+olN/z9T3bi3q1xc5+Hb7vIlDx++dJh3HbPC7YeR0HX0eFX0e0x0OxG2COay5qd6n6srpAP9z0fxwMvWCuU5Yo6bjxvedW/vbwnbPZwRaJBH0am0mxxbL9VT7pCPuwfn8apn3wIAU3Bdz9wPs5a3u16bNH7HugOQiHAR+/bgU/8aCfWLovigVteb+xnjJkEfcgWdKRyBUxnLVmGh0O6ORN82we+sw1qhbVUoyEND916sanhP/LSYdzquH8AcO5JPbj75i0AYFMSeD3efMevQEBwxZlL8S83nF32N2dDPcZ9F4A7CCF9ANIArgCwzdh+FYAHALwDgGsNIYTcDOBmAFixov4FY59+dQIfTOwATn+dZx6NkE/FZJF5FiPjRxBRC/D7w3j/2Sx07ucvjmLX8HFz/7wpy7h77jviU/ApBH/2ulXwqQqu3GAtwNvXwW48X06MT/rh+rGYn3pRZ6nHwzEXv3CZGh81DNErR7w9oJdHE9g/Po3do0lX457I5NEtNIIlkT4ojajhHmLWsX6qyJJoEIcTWc9jAKzyf/6a9XiLsYCyF+8+fwVO7u9Aj6Fvc02VXxMxzh2we+48Hw2AijMjxXEMgBmLQKeK4ak03nzaYtfvvO91K7Eh1m3Oo2BJ00qP/9zBKcR6QjhrRTd+9Pww9owmceEaVkeePXAMOqV4/0WrbN9Zv6yrpkWbQx6eO9enl7ossg4An7hiLZ56dcK2jQC47pxY1b/9mbed4dqziQY17MmwUMiwz3oe//L1q9AT9iOZKeC7Tx/EH8eSnsZd1M0DmoovXb8R+49MY2f8OH6//6gZy857XryHxtcB4Pfzqo3LoBKCU5eUptZeEg3iH65ehxGPvEqcsUSG3b/DCfRHmBO5degYio779/zBSTxzYAK5gg6fSmye+8Wn9uNvLz3VvFenL23cguoiszbulNLdhJAvgHnn0wB2ACgA+EsAXyWEfBrAgwBc59lSSu8EcCcAbNq0qW7xSUuNY6k+zvT2tN2j44T8Ko4XFAAE+ZkpLAnp6O6I4mOXnw4A2Dc2bRsE5dEy3HNnM1TtOV2WdgXN74s4dWc+6cfU3IX81JU8V2f0hjXA5Ks4MMW1Sa+B4HS+aKZm4MTESB8IIYyG5xM2xh6yxjGd0TIAjwgozd0unquqELxny0meZecsjgZx9VlWkrVoUMPR6VyJR8qNu5gEq6hTc3u5AdWikb+Ej2Pw/ToDGo4ks573aFl3yKYze8lkyUwe563qxYfecip+9PxwSV77Fb1h13pUC17RMpUknotP7cfFp/bX9dubVva6buf1M5UvmquZAcCaJRF89PLTcXQ6i+8+fbDsIjg8aRjn7eeyRudnL47g9/uPYngqjVMXs4Ry/JkArLrPjX0k6MMN53k7ku+9YGXF8xyeShv3T3w+0ljeE7Ldv3u3HcIzB47h8PEMFkX8KOjUrKdhv4a/uWRNxd+ql7oGVCml36SUnkMpvRjAMQD7KKV7KKWXUUrPBXA3gFcaUdBKnJzdAwCY6T/HCp1yyDJhv4pUngKBCPRMElGtYNPcA5piS/rENXc+iSnkZ911MyNhJu/Z1XVGjPBGoT8SsK0475Xu1nYsx+CTNZComZ6i1+CMadw9ogBSOftDB5TGujvjvENG99/y3EurkbOBMGen1qAhe8Ebu2S2gJBPNQd03Tz3gq6bck25AdVpoc5EhKgX7v1VM5kH8F6TlHtuA10stNPZM6r2+OUI+VQz3lzEGe46l0SDGpLZAlLZgrmamUilVMWA9wQsM23AZNrMPyMGBww7PPdGsCQSYPfPsfqU10Iz8amUYI/qEUpqp95omcXG6woA1wG4W9imAPh7AF+vt5CVKOoUpxf3IkdVHPSvts1oE+G5YXR/JwLFGXQqeVsYpE8lprcOWOkH/JrluQNApsBnmhY8K45lIAzP3ZzEoZmDbrpOSwYZ3XDmdBc990jQPY8LZ9g07qVxwXz1HLG7DBjZDifTZiMWn0wbky+M9Ml+DakKssxgdxjjyayZezteQ2hdJXgcMxtAs8peyXMPaAr8quKRZdOqM2IjUGvUilsWT12nSGYLiAY1M7TT6fnVEvLohdeCMsOTaUSCpYuhzAXRkA+UAkens7ZZoZygVj5VMcBlmVLDaCb8mkwLg+s+s06IqZ4bhaYqGOgKOnpeLqkphKgeLyWh2dQb534/IeRlAD8FcAuldBLAjYSQPwLYA2AEwH/W+RsVmc4UsIG8gj10BeLT1HMAiXneReS0DkRICmElb5vA5FMVc4V0oNRzdy5CnEy7xw4D1qAi11/5d8JGbo1EJo+jxvJvlbxZ5+IXCWFAVdSH3eCVMJUv/dzK92GvBrFue6QPTyzGCfkUZHJF03B7ee4AzLVL41PVh9ZVgvdWjqftybD4QJjdc6em504I8czpflyoM2L6hOEaPXc3WWY6Z3iVxr0SZ1nyHC48EqgevHK6uyU9myv4tTycyLgad0VhGVfLrXCW9PDceU6Y4am0rTfLn3tLlmmsUeWJ0gDv1BRLu4JmkjPRGZtL6pVlLqKUnkEp3UgpfczY9hVK6anG3+10DhIXJzJ5REgaE5TlbBaNn0jIpyCVKyJDOtCJNALI2VIP+B2yTN4Z5+5Yp9Kru8gRJ8OYhtRYrCCRLphrg1Z68JyLQNgHVMvHbnPj5Jaq2MzU53d67vZYYmfq17BfQypfQDZveO5umrsZLZQyj9UI7xRgD0nRmGgl3mPVJc69WKRQFcX2XdcFx4WwSjFTYnwyBU0hWBKprscRDfpKlj50em6xbmuCm5VMrQGyjMeCMrWmMWgk/P5k8rr5/DgJ+zVPz51SagtnFOE5YeKTKVvklCnLGNe40T2WWE/YfK68UlP4NQVLjB6ac+B/rmiLGaqJTB4+FFGAZt7ogKaURKCE/RrS+SKmEUKEpOHXM7akYT5VMScuAe7pBwArr4o4iu+GuD6lbYFgw1hXq4VGQ/aufiKTh984P7c4eE6hqJuREm6yTcaQapwPXUwwzHx5O7GMLCTUWuDYr5Y+tM5JIc50sPUgaqpi4+pm3As6tYXoeSUPE6e4iyGT8ck0BrqDZsRUxbIZMoS4+IVTc+VT6PNFvaF6eMhXKsvw+9eohrVWxPvjHLjnlEulncoVXeescLgXLeaf4dfZ1Nwb7bkb9y9X0MvKdjGXhmcuaQ/jni5AQwF5qKwblHbXwnmcekIPIkLSUIpZmyzj9NwLjlBI8eHJ5IvIFfWyrbEY5ZLJF0GM6dcRQ5etdqIK7wHwTpDYqHil1wWAsWTWNHRu3V4u1TgfOmsiUxpTKTa7UKy8PPsgl7DcPHeeEyY+mUYmX8TRae+Ik1rhD+9YImuXZbhxp+6aO/uu+8xIbhy6Qj50+FVjpmmh5kbJbQKXUyYc7AmZyxmK2UTrxRqctBqW2eSoaSRumVmdhPzl0yYA3pIGTzcsNs4hnwpNIRgzQnE7G+wxx3pCoMb980pNAVgLbydacUB1oZDI5KGhCM3nM2QZdy087FeRK+qYKATQpWRACmm7LOPU3IU1VPn3AeZNVKOjibJMKldE2Jh+zVdXik+m0RP2oSNQ/qZHQxp0yjLaAYYGGeRx1e6JxQAgfqx0QWsRMQ2rSNivobfDb1tlyK65q2YDB7hr7nzgcHgqXbNuXQlxLMVtQNU5GUgTjLtXwqqEYBzYPfKZA6q16NVuMpl4bMAue5kDzR31DzQ78/7w32C/OV+eu5C8r6zn7j5mlHRcOyc8JwxfoDwS1MyxFYBNAlQbMM5j+81uq2dbLjVFrCeM0eMZTBqrLknPfRYk0kyW6QgZLWXaPUSRG7HRjA8dSLP0Aw5ZRqeWccg50g8EhWgEcQDHC1HfZdOvVeM7zHCUWztUxLlyTSJTQMRoVLzycQNWt3RRp9/Vc08LUpETnrnOLT1CyK9Cp0AyyxdHcH9orfVPa8+TUg7xmov32ZlbRjdm0No89zKhioDl5UWDPkxM5zCWzNTkuVvpjl0mnYXs0+DjkylWBxo00Owc8Ge/0dhrXyuRKj33Wtd+5fB6uedw0rYffy7KPZ+zxWycp9JlU1MMGknO9o1Pw6+WysTNpi2MezJTgI8UEAmzVcxHHFoshxux0YwPIZpmWSGFOHefEfLIpZmCYxKT+PBU47kzzT1vfsc07iE26HboWKoqw+H0BhNClI5XPm7AerBP7u901TTFQV4npYbZ7rkDwFTKWvnGDZ722GvQabaI11z0hhRiN+5cntEcsoz7JKOCzcuLBDXsHUuC0tq8XreJUkmH5jrQHQQhfFHpVMN6NG6LZM8mR00jEY1yOc3da0C10nPGz2v3aMK2ZjK/1s2IUFnaFTSXoyzXsxPXCZ5rSQZoE+POZZlIB7uYB47OuLbY3CglqVDRBePOB075QCHX3PkDL4aaWd3FMp67kQOFUsqMu081v0MpMDQxU5XhcHqDYpROQFONZcVKDVZ8MoXFkQC6Qz5XTTMlhGc64XphfDKNzoA9RprvP8mNu4vmDjAPZyyRwdDEDIs48Zj+XitRD1mGy2emcTdeFcU+oJrJ6zb5DShNiRsN+sx1W2sxvuJKUdaxjayGxvEDmoolERZJUc0i1NXijOYCWB2oNUdNI+FZPAF4eq7lFsGp1EPmhvXVI9P2Rj9k9cAajV8zJMcK989WtjmWZIB2Me7pAnwoojPELrJO3VtsXsmSEFpaYRITn+nIPfe8TuFTiZnfg3tGqXyxqokJ0ZCGXJFFlaSErHj8OzqtznA4vcGkY1UdL2+Uz3z0SgXLHyi3hy7WE0Imr2NHfKpkMQ1+HXjeHE9ZppsNHG4dOoaB7mDDtM+IwwhzeMgjHysxx0wcnjtQKmPxjILWfmycA7CSg1WDW/RSIp1H2K/CJ0TcDPaEsH98Gkencw3zqp3RXICVirqWHDWNht+jsN/dQJeLlklW8Nz5co3iilzibzYr/HCwO4T9R6ZxJJn1vH8DXcyZ0SlMGXUuaQ/jnsnDR4roCFueodtN5Zr5tM1zt2vugGDcC7o5gQkQZqjWMKAKsIc7kysi5OMLFVhlq05zt+epEWUZ9jvug4S8y+ilaabLeO68XDvjx0s8k2plGf69nfHj5oy9RhAUUg7Y4tyJPStk0ZDVxDh3LxnLmb/EXLqPsG54tfAsns7QVacTEOsJmUnqGqWHu03ln88JTBx+XZ2T5Tjlo2XKx4jz5Rqd+zi190YT6wnhJX7/et2Ne9CnYrExP6IZ2n8l2sK4JzN5aCggFAxZK724eNQ8t8U0RONuj5YBYHbZC4bnbn7f8DxSOXFAtbzmDrAKmsoXbJkDOVVp7kKemowx7d/uZZYOEuo6xYgxeSXkc58kIq5I74T3KAo6LeldhExZhnvu3pq71zHqxU1TNXPLFLnnzmcYk5LvlSw47pizwAcCl0aDNo+7EpqqGOkl7CmanZrrYHfI7Fk06tq4TeWfzwlMHH6PQr7aPfdEms1Z8eodAtYz5DYW0yyjOtgj3L8yjgu/t1KWmSUszr0Iovqsi1lmQDWriLKM4Lk7ZZmibnuwVYXAr7Gp0olMHpoxddoLMUeJTXMXylbNgy3GTrtp/W5pZseTWTPHfMivmItriJjrWrrlhnFZVIPDPcTj6TwUAs8JPgNdIXA1oNGheG6aqplbhto1d2ecO1CariGZLZVlWLlr93qdE6XcEsyVW5putjin8icyeRxP5+ctDJITNT338pq722T2SrPAATHPv7gYSPMGVNlvVnf/+H5yQHWWTKczUEABxWe14h5x7gAQjvRYG20DqswI5AqG51ekJQsQsJH9gpnvopyWKXqJtmgZQQ+sZmo0z1OTzBZcNchIUEPSuerRlDXzMexnycVyjkWFM/kigj7FIw+3lRfb6ZkEBVmmnEfl1xQsiQSNYzTLcy+Ncy+Ui5bxmPTlTE5lLmY8C8PoXMnJLfEVP66mECyONGagGbBHnjjXx50veF31ipYJ+lVQCteFyxMZ96RhIrx+2j335g2ost+07l+5QAHLHknPfVakMkaCfVWzWkqXi8mNUrRbyD0thkJyWUbw3EXNHbC8DK9MdSLi4hDpvOW5W2spVu8V8lSybhqkW74UHsK4vMda1DrjyAyZyhU8B7nE8rktDA2wAVWvSBnrGN6r39RDxOXhdWaFLBTdPPfSAU9dpyXJqcT1LmvFuQarm/fJj7usO9TQSTZhv2b2yIbnOcadY2ruXtEyHgnPAGt1rHLwa2nX3Js7oMp/s1KggFvZ5oq2MO7pjLHij+Kzumgu3SBulHp6BeNeIVrGufgz1wer6S6KOUpSOWtRDOeajtUQDWo4NJnCi/Ep471dXyxZjNp4sJd1h6yBNkdmyHTOO5kTIKxd6jGgmsgUPPV25zEaL8v44FcV2+9rTs/dMcMYcE/XMJMrsIgGl279bHoczvvhjG4Sj9voHk3Qp5iDk2bemvnW3M1U0d6TmAAWhQawfDj7x5N4MT6F8US24nPmpms3W5ZZVuX9KycTN5u5b04aDKUU6UwGCABQfVi9mC2h5dbV7QgweSO2ZDHwkrHRLVpGiHPXHK0yH9lnHkWFtAFGZZtK5ZEt6KYHrakK+iMBs6zVsDgSxJP7J/DkfrYcWr+QpbAr5EPOWDOSN2DxyTR6O/wI+zXPVLDpfMHzgQOA1Ys7sW3oGPqM5e044nfKyTIAsLq/E2G/WlPESTUMRIPGZBLr/jgnMRX00mgZnjdG9Kyns7w3ZD2APIytlnvE6Qr5zBmTlFLXuhL0qYj1hGZ1/HIwz91awjGgKVjU6a/wreYy0BVEQFPMSCInzmyWzx2cxPVff8r8fP1g6fKQIqsWdUBViK2O8fvX6HrHCfpULO+tfP9OWdQJQtCwOR610PLGfSZXhEaNB1XRcOnaJfjlbRebiyKLBDQVv/ibi1hr+rsOID9TQZahJYOFludeqHjDgj4FPpVg3Mh7IWqOP/qr16G3o/qH7svv2oiXRxIAgO6wDyf1dZif8Yo8MpUxK1t8MmV6yyGXEDkAtkFeN25502rcuHlFybiC+J1KnvsHLjoZb9u4rKaIk2q49S1r8OevX2nb5jWJSWygCSEI+lSbvpvJl2bH3BDrxsO3XYTTl0ZrLtvSriDGEhkUdYpsgS0k7ua53XPzloZ7dDzvD2AtIjGfMe4A8M7Ny3Hhmn7PSUxO54OvY/r5a9ZjoCuIs1f0uH6PsyQaxC9vuwgrhWdi/WDXrO9ftdx905aKktGKvjB+edvFWN3f2Ea8GlreuCfSbHYqAED1QVEITiuz4OzJ/CIHIsy4C7IMN1Q8FJJFy5R67tPGwGalQRJCCKJBn5mdTvR43Rqfcgx0hTDQ5TUTzspVwo378FTaXHjXLRUswIx9Oc+9M6C5elt+TYGmEBR0WlFzD/lVrFzUUXaf2RARUvNyTM+d2kMhnZpoQFPM9V8BeC46MlvDEOsJo2Dkm+dlcqsrzdDCQ37VnFzmtvzbfBDQVKwqUwecK0jx8ZBLz1hStce7enHpM99Mww5Uf/9OXdKcBbAr0fKaeyKTh0a4516DFxQwLrjrJCbLODg9Th6NwNIKV24boyEfxpIZ87vNQEzRCzApQExV65ZQCmDRMrMtE/9eJVlmLrESh+nGa6nnDrDGSfTc+aIjzvGV2SLeDyu6aW78qBLPfZ719mrgHj2Xk+Yr/3m70fLGPZlhqQcAAOpsjLvV+nIvPS/KMg7DEParLLQxX6zYJQPYIN24i+feSBZHgvCpxBxEPTqdQ7agm56F9fCUeu5e4WmV4OdSSZaZS6yUv+x9wSXOHWANks24l1kLdjaIPSluqKqpK42Ap5rwWv5tIeJ0PhLpAnxq+Tkkksq0/NVzyjJVE4gAIIBq6d5+hyxTKLp47n7VXFu0mtlv0aAP48lSzb2RqArBQJe1LiePkuAPNv9d5xTvdD2eu3HMRnm7jUD19NztZQxoiinFAIIs0yBjwr1lvnAMMHfTz4PGgH8jl+9rNk7ZkEue8z1W0OosnCdzlvAl9gDULsv4QoBQgfyuA6rOwUTNlG2qGQyLhqz9m5nPmafoBVCyOEbZAdXZeu6+FvDcXeLcAWbEuRQDWLJMo84l6FOxqDNgLhwDzF0oXNgY8C+3QtBCw5kTJ5FxX0lNUhsL58mcJXyJPQA1eu5RW+oBwCVxmKvnLiy2XEVXu5plxhoBT9ELWDHupuZu5PRwDqg2wnNfSJq7p+euzq0sA1i57J2rMDUbniSOr8K1EAZUKxF09CyrCTOWVKblr2AyI8gySg2nc857gcFzbJtKZBlH4jDAnra0mgpoX6ygeZeb5U7PIlsoIj6ZQlfIiiYxF3EQljKjlCKdn73mHl6ImntJnHuZaBlBlskVvZcLnC2xnhBeHklUXEmo0YSMqfyvHJnxXP5toeHMQ++WRVNSOwvnyZwliUwBHZrhhdXiuZ/0OuC8m2ybnJ57wSX9gCitVCXLzJXnbnS/R6cyxvJ9Yvw+gaoQm+eeLeig1PKaasWUZRbQoJequs9QVUmpcc+5RMs08lxixjKFx9N5+LW5W2KN35d940nP5d8WGj6VzQexNPfqItEk5Vk4T+YsSaTziPqNClyL5u4C99JzhlabL9ISWUb0dKvT3AXj3qQBVUCM0EiXhMARQkwtlmOuwjRrWcZaVWih4MznPl/RMgC7H7miPuer8PD6uW9set4ThtWCmPCMrVcgPfd6aX3jnskjygNeavHcXSCEwKeS8pOYRM+9mmiZKlZ/bwS2RZddJq/wKAoO95JmKxXxhUcWlCxTRW4ZwBhQtRn3xssy3LDuHk3OqRfKewjjyWxDF0hpNiG/YNwzUnNvBAvnyZwlyUwBUb/xoNaiuXvgVxVLltFdomUMA60QoKMKwxgJzI0sM9DFstPtGjmOVK5Y4rU5l9rj+vtsZZnwQvTcnVkhvSYxqY4Zqg2exARYA5nDU+k59tytOtlKnnvYryGVLyJX0JHJ69JzbwAtb9wT6TwivB7U6bkDbMGOfJmUv9xAdwa0qvRMLsv4NaWhqV2daCpbtPeZV48BKA2Bc64wnzbS/85WlgkuRM2dOD13PqDqiHMv8dx1tuhIA++PKIvNpRcqOhCtEAbJCRr1s9KaqZLqWThP5ixJZAro4PWgTs0dYIM7onF3enNc06y28vEuebMmMIkM9oSwb3ya/e+Ydu5cR5VP9Z6tVMTPx9/ghGD1oCgEhFieO493dxrtUs29iICmNnTSTEdAQ0+4uWln3RDvZyukHuCwdRIKVuioHFCtm4XzZM6SRDqPTs1YnqsBnrtftby6gkv6AXPBjyq7jWYu6zmIlhA9teUOzb3Ecy+zfmo1LMRoGQBmQjNA9NzLh0JmC3pTzqPcwjHNwua515icbj6xcjbJvDKNYmE9mTVCKUUyU0CHr4HGXVOQL1JQSg3N3ctzr86zMBcHngPPPWZ4ap0BraR8pZq7YdzbaBITwDJDFml5zT2gqcgXqTngmivoTRkYthZunjsvlNdPVSFYEln4Me6ckN9aBAeYu1w87UxLG/dsQUeuqKNDNYx7Q2QZgnxBNw2Dz2WxDqD6yscXh5gbz91aFs8pMQR9XtEy7ZN+AGCGvFj0XiAbsHobPCoqW9Cb0kjFXFYIaja8fg50BT0XLl+IhI1orqSUZRpGXXefEHIrIWQXIeQlQshtxrazCCFPE0JeIIRsI4Sc15CSusC7cCF1FpOYPPBrCnJF3dTdfU7N3VfbwruEEESCvjnT3AF3rdXpuafq9NwX4gxVgOnupude9E4cBlghkExzb4Lnbhr3ORxQNe5LK+ntgLAIjpRlGsasazQhZD2AmwCcB2AjgCsJIWsAfBHA5yilZwH4tPG+KfAuXJhr7g0IheQDqvmie5c+aOSWqcWziIa0OZmhaC1GXfpgi3m+ASuPx2zlIh5CGZijmZfVoimkZCUm1SW3DCB47vkma+5zOaBq3I9WyCkjwgf85zrRWjtTjzVcC+BpSmkKAAghvwVwLQAKgC+B0gVgpK4SloGPrDfSc/epbGp6gXvujq6tX1VwSn8H1tawysu6ga45CUsb6AphsDuEc04qXZYs5NdsA6r1eu4nL+pAJKjhpAU2aKcKxt0zzt303C1ZphlRP2csiyIS1Bq+Tmo5fKqC1Ys7ca5LHVjIiIvgsDkkC8tpaEXqMe67ANxBCOkDkAZwBYBtAG4D8EtCyD+D9Qxe5/ZlQsjNAG4GgBUrVsyqAGfFuvHiZy9D+KmdbEMDNPeApmA6W7AMg8PrI4TgsQ+/saZjfv2959ZdrmrwawqevP3Nrp+FfCpyRdZoaaqCdL4Iv6rMWpc9qa8DOz/71nqK2xRUm+fuHS0DOGWZxhuTwe7QvFyjX/3tG+b8N+sl7FdR0CmOpXKIyFzuDWHW7gqldDeALwB4FMDDAHYAKAD4KwAfopQuB/AhAN/0+P6dlNJNlNJN/f39syqDorA1Sq0Fsut/QLksw7vsjV7Yeb5wrlOZzhXbcqUblZR67m6JwwBrYexmhUJKqofLlmPHM3IwtUHUVaMppd+klJ5DKb0YwDEA+wC8D8CPjF1+CKbJN5dinnntDWjtWbQMtaJl1PbwIIIuxr2ZKYjnC1W1a+4KQclMYj5OkBU19wU2MHyiweviWDIjB1MbRL3RMouN1xUArgNwN5jGzvuFbwYz+M1FLzREbwcMzb1oae7OSItWhacZ4Lp7Kj/7VZgWMqojzt3t/jllmVyxOaGQkurhi+CMJbLSuDeIel23+w3NPQ/gFkrpJCHkJgBfIYRoADIwdPWmUsw3zLj7jVzfPFqmXTz3kIvnPhex93ONapuhSuHWNgecA6r55oRCSqonZIQYH53O4pwV3fNbmDahLuNOKb3IZdvvAczNCCJHzzdkMBWwskLmPaJlWhXnOqrpfKE9PXeFWFkhi16euyHLSM19wcDrIqVydmqjaI8a3UDPnQ+o8iXaWmmWXzm4LJPJiZp7Oxp3xZZbxi0TJzfkVrSMlGXmG9siONK4N4T2sFx6oXGeu1OWaYFlyqrB6bmncsU5W/ptLlEVez53tzS+PKbdSj9QbGgud0nt2BbBkdEyDaE9anQxD6iNqRDMc6fW1PV28dwdmnumjsWxFzJ2z51W8Nx1FHWKfJFKzX2eCUnPveG0R41uqOZO7Lll2mRANeiMlmnXAVViX0PVzXM3NfeCNZ9ByjLzi1gX5RJ7jaE9jHuxcaGQvHvOPdx2GVDlccRmtEybhkJqimL2unSdluSVAeyhkM1YP1VSO7UuPC+pTHvU6GKuIUnDAMuYz2TZrFdn+oFWhXtGqVx7h0IqCqqPc88LnruMlplXxPEfKcs0hvao0Xpjo2UAy8Ntl0lMPNVAOl80ooFoW2rumqLYZqi6ae6EEPg1tuJWVsoyC4KApoDfKjmg2hjaw3IVGxstAwAzWWbcF9IaofVACDEy7xVM770do2UUW1ZI3XPRa77UnpRlFga8fgLSc28U7VGjG+i5c2POF5BuF1kGsBbs4IOq7ZhbxpnP3c1zB6xFsnnyMGnc55+Qv7aFcCTlaY8a3chJTBozBty7bSfjHjQW7LAWx26P2y+iOLJClvXc84Is04a9mFaD18dOGS3TENrj6W5oKKR94LFdZBmAee7pXNHslfB8Hu1E9Z47SxDHZZl2us+tStinIRLQPO+ZpDbao0YXCw2cxMQ9dy7LtMclAqylzOpdYm8hwxKHMW/cK7cMwMZWsvmi4Lm3z31uVYJ+VYZBNpD2qNEN9Nx9jgFVr259KxLyqXgxfhxfeGgvALRltIyqEBiOe3nP3cc096zU3BcMYZ8qJzA1kPa4ko1M+WuGQjLPvV0mMQHApWcswWQqh6l0Dues6Mbq/rlb23OusHnuuo6Ah/TEo2VyRRkKuVD4/zYMmPNLJPXTHsa9wYnDAKa5K6R0/c1W5gMXnYwPXHTyfBejqbCUv+z/Spr7dLaAbF6GQi4U3rPlpPkuQlvRHjW6wYnDACCVLbaV3n6ioBLRc6cl66dyAprqiJaR91rSXrRHjS7mGqe5GwOqM7lC26T7PZFga6iy/8tr7nwSk5RlJO1Jexj3Bq6hyrvn6Zz03FsRlRAUBc/da55CwEw/IGUZSXvSHjW6wSsxAYbnLo17y6GWxLm730PTuBvRMjLOXdJutEeNbmQopPGQZ/J62+RyP5FQq84toyJnJA7zqwoUKcFJ2ozWN+66DlC94fncgfZKPXCioCnETPlbLJaPluGJw6QkI2lHWr9W63n22uB87gDga5N0vycSYlbIIq2QW8bw3GWkjKQdaf1aXTSMe4MnMQHtNYHpRKHq3DI+FZSyRVlkpIykHWl962V67lKWkbCskDoFKKUVs0ICQCKdl7KMpC1p/VpdNKYrN8hzVxVirggjQyFbD27Mizo1NHfvaBkASGQKtgZdImkXWr9WN1hzByw5Rk5iaj141EtBpxXi3JkUk0jnZS53SVvS+sa9mGOvDfLcAUt3l5p768E9d53Sspq73/Tc8wjI+yxpQ1q/VnNZpkGaO2A9+FJzbz1Um+defg1VAEikCzJaRtKWtH6t1hsbLQMIsoz06FoO07gXKXTqndWTG/R0Xsa5S9qT1q/VDQ6FBKx1VNtpoY4TBW7Mc0ZCsHIzVN3+l0jahdY37g0OhQQEzV16dC2H07hXipZx/i+RtAt11WpCyK2EkF2EkJcIIbcZ235ACHnB+BsihLzQiIJ6YoZCymgZCcz87TzbY1Weu9TcJW3IrC0iIWQ9gJsAnAcgB+BhQsjPKaXvEvb53wCO113KcjTDczcHVOVD32pwz53nafdKCCYadCnLSNqReqzXWgBPU0pTlNICgN8CuJZ/SAghAN4J4O76iliBJmjuMhSydXEa90rRMs7/JZJ2oZ5avQvAxYSQPkJIGMAVAJYLn18EYIxSuq+eAlZEb3wopBUtI2WZVqNUc69mQFUad0n7MWtZhlK6mxDyBQCPApgGsAOAuHT5jSjjtRNCbgZwMwCsWLFitsUQPPcGau5clpFZIVsOy3Mvr7mLKQdk+gFJO1JXraaUfpNSeg6l9GIAxwDsAwBCiAbgOgA/KPPdOymlmyilm/r7+2dfCD5DtRnRMtJzbzm0qj13qblL2pu63F1CyGJK6TghZAWYMb/A+OgtAPZQSuP1FrAiemMThwGA34hzl5p766EY0TI5Y5Xscmuomv/LaBlJG1KvlnE/IaQPQB7ALZTSSWP7DWj2QCqnGZOYVJl+oFXh96xSnLumKuaSfFJzl7QjdRl3SulFHtv/vJ7j1kQTQiFl+oHWRSHVRcsAzHtP5YpSlpG0Ja1vvZoRCmkOqErPvdXgg+CVNHfAkmak5y5pR1q/VjchFFLGubcuXIWpFC0DWAOpUnOXtCOtX6ubEQqpEturpHWoyXP3cc9dyjKS9qP1jbtMPyARUE3PnWvu3veQ99BknLukHWn9Wt3gNVQBIVpGau4thzorz731HwOJxEnr1+omrqEqPbrWQ3VGy5SR1kzNXcoykjak9a1XMc8MO2mclx2Q6QdaFmfiMBktIzlRaf1aXcw1VG8H5CSmVqba3DKAYNxltIykDWn9Wq0XANXf0EPKrJCtS7VZIQEpy0jam9Y37sV8Q8MgAUtrl3HurUdNsowcUJW0Ma1fq/V8E2QZvkB261+eEw1nVsiqZBlp3CVtSOvX6mKhoWGQgEz528ooDs3dK3EYwHpoCpHzGSTtSevXaj3f0DBIAOgKscYiEmxsoyFpPrV47t0hv3mvJZJ2o7FWcT4o5hvuuW85uQ/3/rcLcNrSSEOPK2k+zqyQ5TT3my46GVduHJiTckkkc03rG3e90HDNXVEIzlvV29BjSuaGWjz3rrAPXWHpuUvak9aXZZoQLSNpXbjmzldiKue5SyTtTOsb9yZEy0hal1LPvfWruEQyG1q/5jdBc5e0LiVx7jLiSXKCIo27pK1wzlCVmT0lJyqtb9ylLCMRsLJC8jh3adwlJyatb9yl5y4RUBQCQoB8kQKwjL1EcqLR+sZdLzR8EpOkteEGXSFW9IxEcqLR+sZdeu4SB1yKkZEykhOZ1q/9UnOXOOCDqFJvl5zItL5xb0LiMElro5ieuzTukhOX1jfuTUgcJmltTM9dxrhLTmBa37hLzV3igMsxMlJGciLTHsZdau4SAVVq7hJJGxh3XXruEjvcY5eau+REpvWNu5RlJA641i41d8mJTGsbd0oBWpSyjMSG5bm3dvWWSOqhtWt/Mc9eZT53iYDU3CWSOo07IeRWQsguQshLhJDbhO3/gxCy19j+xbpL6YVuGHfpuUsEVBnnLpHMfpk9Qsh6ADcBOA9ADsDDhJCfA4gBuBrABkpplhCyuCEldcP03KVxl1iohhwjPXfJiUw9esZaAE9TSlMAQAj5LYBrAWwC8E+U0iwAUErH6y6lF3qBvUrPXSKgGv1R6blLTmTqkWV2AbiYENJHCAkDuALAcgCnAriIEPIMIeS3hJDNbl8mhNxMCNlGCNl25MiR2ZVAau4SF6TnLpHUYdwppbsBfAHAowAeBrADQAGsN9ADYAuAjwC4l5DSqYKU0jsppZsopZv6+/tnVwipuUtc4BGQMlpGciJTV+2nlH6TUnoOpfRiAMcA7AMQB/AjyngWgA5gUf1FdUFq7hIXNOm5SyR1ae4ghCymlI4TQlYAuA7ABWDG/M0AfkMIORWAH8DRukvqhjTuEhe4w67JSUySE5h6xer7CSF9APIAbqGUThJCvgXgW4SQXWBRNO+jlNJ6C+qKlGUkLkjPXSKp07hTSi9y2ZYD8J56jls10nOXuCDzuUskrT5D1d8JnPwmoKN5ofSS1kOuxCSR1C/LNI18Po94PI5MJlN+x83/C0gC2L17TsrVagSDQcRiMfh8J07vRpG5ZSSShWvc4/E4IpEIVq5cCZdISkkVUEoxMTGBeDyOVatWzXdx5gzuuSvSc5ecwCxY1yaTyaCvr08a9joghKCvr69y76fNkLllJJIFbNwBSMPeAE7EayizQkokC9y4SySzQXruEok07pI2RHruEok07g2ls7PT87NVq1Zh7969tm233XYbvvhFlu5++/btIITgl7/8ZdXHlLgj11CVSBZwtIzI5376El4eSTT0mGcsi+Izb1vX0GOW44YbbsA999yDz3zmMwAAXddx33334cknnwQA3H333bjwwgtx9913461vfeuclasdMddQlaGQkhMYWfvL8LGPfQxf+9rXzPef/exn8bnPfQ6XXHIJzjnnHJx55pl44IEHqjrWjTfeiHvuucd8/8QTT2DlypU46aSTQCnFfffdh29/+9t45JFHTrjolkZjeu4yt4zkBKYlPPe59LBFbrjhBtx2223467/+awDAvffei4cffhgf+tCHEI1GcfToUWzZsgVXXXVVxaiUDRs2QFEU7NixAxs3bsQ999yDG2+8EQDw5JNPYtWqVTjllFPwxje+Eb/4xS9w3XXXNf382hWpuUsk0nMvy9lnn43x8XGMjIxgx44d6OnpwcDAAD7xiU9gw4YNeMtb3oLh4WGMjY1VdTzuvRcKBTzwwAN4xzveAYBJMjfccAMA1qDcfffdTTunEwEZLSORtIjnPp9cf/31uO+++3D48GHccMMNuOuuu3DkyBE899xz8Pl8WLlyZdUyyo033ojLLrsMb3jDG7BhwwYsXrwYxWIR999/Px588EHccccd5qzSZDKJSCTS5LNrT2RuGYlEeu4V4QOh9913H66//nocP34cixcvhs/nw+OPP46DBw9WfaxTTjkFfX19uP32201J5le/+hU2btyIQ4cOYWhoCAcPHsTb3/52/OQnP2nSGbU/MiukRCKNe0XWrVuHZDKJwcFBDAwM4N3vfje2bduGTZs24a677sLpp59e0/FuvPFG7NmzB9deey0AJsnw/zlvf/vb8f3vfx8AkEqlEIvFzL8vf/nLjTmxNsby3GX1lpy4kGato1ELmzZtotu2bbNt2717N9auXTtPJWovTrRr+X8e/SO+8tg+fPKKtbjp4pPnuzgSSdMghDxHKd3k9pl0bSRth9TcJRI5oNpwdu7cife+9722bYFAAM8888w8lejEw9TcZZy75ARGGvcGc+aZZ+KFF16Y72Kc0Jj53E/AjJgSCUfKMpK2Q8a5SyTSuEvaEDlDVSKRxl3ShqhSc5dIpHGXtB+qjHOXSKRxL8fU1JQtK2S1XHHFFZiamqrpO9/+9rfNWauco0ePor+/H9lsFgBw9dVX44ILLrDt89nPfhb//M//XHMZ2xmZz10iaZVomYduBw7vbOwxl54J/Mk/ld2FG3eeFZJTLBahqqrn937xi1/UXJzrrrsOf/d3f4dUKoVwOAwAuO+++3DVVVchEAhgamoKzz//PDo7O3HgwAGsWrWq5t84UZCau0QiPfey3H777XjllVdw1llnYfPmzXjTm96EP/3TP8WZZ54JALjmmmtw7rnnYt26dbjzzjvN761cuRJHjx7F0NAQ1q5di5tuugnr1q3DZZddhnQ67fpb0WgUF198MX7605+a28S0wPfffz/e9ra3mbluJN7IaBmJBACldN7/zj33XOrk5ZdfLtk21xw4cICuW7eOUkrp448/TsPhMH311VfNzycmJiillKZSKbpu3Tp69OhRSimlJ510Ej1y5Ag9cOAAVVWVbt++nVJK6Tve8Q763e9+1/P37r33XnrNNddQSikdHh6mAwMDtFAoUEopveSSS+gTTzxB9+7dS88880zzO5/5zGfol770pbLnsRCu5Vzyk+1xetLHfkYf3zM230WRSJoKgG3Uw65Kz70GzjvvPJsc8tWvfhUbN27Eli1bcOjQIezbt6/kO6tWrcJZZ50FADj33HMxNDTkefwrr7wSv//975FIJHDvvffi+uuvh6qqGBsbw/79+3HhhRfi1FNPhaZp2LVrV6NPr22wPHdZvSUnLrL210BHR4f5/29+8xv86le/wlNPPYUdO3bg7LPPds3rHggEzP9VVUWhUPA8figUwuWXX44f//jHNknmBz/4ASYnJ7Fq1SqsXLkSQ0NDUpopg8wtI5FI416WSCSCZDLp+tnx48fR09ODcDiMPXv24Omnn27Ib95444348pe/jLGxMWzZsgUASwv88MMPY2hoCENDQ3juueekcS+DItdQlUikcS9HX18fXv/612P9+vX4yEc+Yvvs8ssvR6FQwIYNG/CpT33KNMT1ctlll2FkZATvete7QAjB0NAQXnvtNdvxV61ahWg0aiYj+/znP2/L+X6iw426zC0jOZGpK587IeRWADcBIAD+g1L6L4SQzxrbjhi7fYJSWjY2UOZzby4n2rWczhbwfx/bh7+97FQENO+QVYmk1SmXz33Wce6EkPVgRvw8ADkADxNCfm58/H8opXJmjWRe6Axo+PgVJ05jJpG4Uc8kprUAnqaUpgCAEPJbANeW/4oEAG655RY8+eSTtm233nor/uIv/mKeSiSRSNqNeoz7LgB3EEL6AKQBXAFgG4AJAP+dEPJnxvsPU0onnV8mhNwM4GYAWLFihesPUEpB2lA3/dd//dc5+616ZDeJRNK6zHpAlVK6G8AXADwK4GEAOwAUAPwbgFMAnAVgFMD/9vj+nZTSTZTSTf39/SWfB4NBTExMSONUB5RSTExMIBgMzndRJBLJHFNXbhlK6TcBfBMACCH/CCBOKR3jnxNC/gPAz2Zz7Fgshng8jiNHjlTeWeJJMBiUETQSyQlIXcadELKYUjpOCFkB4DoAFxBCBiilo8Yu14LJNzXj8/lkciyJRCKZJfVmhbzf0NzzAG6hlE4SQr5LCDkLAAUwBOC/1fkbEolEIqmRemWZi1y2vbeeY0okEomkfuQMVYlEImlD6pqh2rBCEHIEwME6DrEIwNEGFWc+aPXyA/IcFgKtXn5AnkOtnEQpLQ03xAIx7vVCCNnmNQW3FWj18gPyHBYCrV5+QJ5DI5GyjEQikbQh0rhLJBJJG9Iuxv3OyrssaFq9/IA8h4VAq5cfkOfQMNpCc5dIJBKJnXbx3CUSiUQiII27RCKRtCEtbdwJIZcTQvYSQvYTQm6f7/JUAyFkOSHkcULIbkLIS8ZqViCE9BJCHiWE7DNee+a7rOUghKiEkO2EkJ8Z71ut/N2EkPsIIXuMe3FBC57Dh4w6tIsQcjchJLjQz4EQ8i1CyDghZJewzbPMhJCPG8/3XkLIW+en1BYe5f+SUY9eJIT8mBDSLXw2b+VvWeNOCFEB/CuAPwFwBoAbCSFnzG+pqqIAluN+LYAtAG4xyn07gMcopWsAPGa8X8jcCmC38L7Vyv8VAA9TSk8HsBHsXFrmHAghgwD+BsAmSul6ACqAG7Dwz+HbAC53bHMts/Fc3ABgnfGdrxnP/XzybZSW/1EA6ymlGwD8EcDHgfkvf8sad7Dl/fZTSl+llOYA3APg6nkuU0UopaOU0ueN/5NgRmUQrOzfMXb7DoBr5qWAVUAIiQH4/wB8Q9jcSuWPArgYRrpqSmmOUjqFFjoHAw1AiBCiAQgDGMECPwdK6RMAjjk2e5X5agD3UEqzlNIDAPaDPffzhlv5KaWPUEoLxtunAfAc2/Na/lY27oMADgnv48a2loEQshLA2QCeAbCEp0o2XhfPY9Eq8S8APgpAF7a1UvlPBlvA/T8NaekbhJAOtNA5UEqHAfwzgNfAFsU5Til9BC10DgJeZW7FZ/wvATxk/D+v5W9l4+62/l7LxHUSQjoB3A/gNkppYr7LUy2EkCsBjFNKn5vvstSBBuAcAP9GKT0bwAwWnnxRFkOXvhrAKgDLAHQQQt4zv6VqOC31jBNCPgkmu97FN7nsNmflb2XjHgewXHgfA+uWLngIIT4ww34XpfRHxuYxQsiA8fkAgPH5Kl8FXg/gKkLIEJgU9mZCyPfQOuUHWN2JU0qfMd7fB2bsW+kc3gLgAKX0CKU0D+BHAF6H1joHjleZW+YZJ4S8D8CVAN5NrclD81r+VjbuWwGsIYSsIoT4wQYuHpznMlWEsBW/vwlgN6X0y8JHDwJ4n/H/+wA8MNdlqwZK6ccppTFK6Uqwa/5rSul70CLlBwBK6WEAhwghpxmbLgHwMlroHMDkmC2EkLBRpy4BG79ppXPgeJX5QQA3EEIChJBVANYAeHYeylcWQsjlAD4G4CpKaUr4aH7LTylt2T8AV4CNTr8C4JPzXZ4qy3whWNfsRQAvGH9XAOgDixTYZ7z2zndZqziXNwL4mfF/S5UfbAH3bcZ9+AmAnhY8h88B2AO2lOV3AQQW+jkAuBtsjCAP5tm+v1yZAXzSeL73AviTBVr+/WDaOn+ev74Qyi/TD0gkEkkb0sqyjEQikUg8kMZdIpFI2hBp3CUSiaQNkcZdIpFI2hBp3CUSiaQNkcZdIpFI2hBp3CUSiaQN+f8Bvr3OBClh+AYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_VAL','train_VAL']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD8CAYAAABuHP8oAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAB6aUlEQVR4nO29d5hkVZ0+/p7Koau6Os50T+qeYfIwzMBIjqIoiIDACmMOK66iIj9XFxddE7iYXV11v7qmRQRlEAkCEgSGDBMIkwPTk7qnc1dVV1eu8/vj3HPvubduVd2qruo0532efqq6wq1zq+5973ve8wmEUgoJCQkJiekH22QPQEJCQkKiMkgCl5CQkJimkAQuISEhMU0hCVxCQkJimkISuISEhMQ0hSRwCQkJiWkKSwROCLmBELKNELKdEPJ55bGvE0KOEkJeVf4uqelIJSQkJCR0cJR6ASFkFYBPADgVQArAI4SQvylP/4hS+v0ajk9CQkJCogBKEjiA5QBepJSOAQAh5GkA76npqCQkJCQkSoKUysQkhCwHcB+AMwDEATwBYBOAQQAfARBR/v8CpXS42Laam5tpR0fHuActISEhcTxh8+bNA5TSFuPjJQkcAAghHwdwPYBRADvAiPw2AAMAKIBvAWijlH7M5L3XAbgOAObPn3/KwYMHx7EbEhISEscfCCGbKaXrjI9bWsSklP6aUnoypfRcAEMA9lJKeymlWUppDsCvwDxys/f+klK6jlK6rqUl7wIiISEhIVEhrEahtCq38wFcCeBOQkib8JL3ANhW/eFJSEhISBSClUVMALiHENIEIA3gekrpMCHkdkLIGjALpQvAJ2szRAkJCQkJM1gicErpOSaPfbAaA0in0zhy5AgSiUQ1NidhAR6PB3PnzoXT6ZzsoUhISIwDVhV4zXDkyBEEAgF0dHSAEDLZw5nxoJRicHAQR44cQWdn52QPR0JCYhyY9FT6RCKBpqYmSd4TBEIImpqa5IxHQmIGYNIJHIAk7wmG/L4lJGYGpgSBS0hI1Bi5HLDldiAdn+yRSFQRksAlJI4HHHoeuP8zwJ5HJnskElWEJPAyUVdXV/C5zs5O7N69W/fY5z//eXz3u98FAGzduhWEEPz973+3vE0RGzduxMknnwyHw4ENGzaUOXKJ4xpHt7DbscHJHYdEVSEJvIq49tprcdddd6n/53I5bNiwAddccw0A4M4778TZZ5+NO++8s6Ltz58/H7/73e/wvve9ryrjlTiO0L2V3cZHJnUYEtXFpIcRivjGA9uxoztS1W2uaA/ia+9eWfD5f/u3f8OCBQvw6U9/GgDw9a9/HYQQbNy4EcPDw0in07jllltw+eWXl/ys9evX45prrsHXvvY1AEwxd3R0YMGCBaCUYsOGDXjsscdwzjnnIJFIwOPxlLUvvBCYzSavuxJloltR4ImRSR2GRHVx3DPBtddeiz/96U/q/3/+85/x0Y9+FPfeey+2bNmCJ598El/4whdgpejX6tWrYbPZ8NprrwEA7rrrLqxfvx4A8Nxzz6GzsxOLFi3C+eefj4ceeqg2OyQhYcTYEDDcxe7HixYMlZhmmFIKvJhSrhXWrl2Lvr4+dHd3o7+/Hw0NDWhra8ONN96IjRs3wmaz4ejRo+jt7cXs2bNLbm/9+vW46667sHLlStx333345je/CYDZJ9deey0AdtG4/fbbceWVV9Z03yQkAAA9r2r3pYUyozClCHyycPXVV2PDhg04duwYrr32Wtxxxx3o7+/H5s2b4XQ60dHRYTnxZf369bjoootw3nnnYfXq1WhtbUU2m8U999yD+++/H7feequaDRmNRhEIBGq8dxLHPfgCZutKSeDVQCYJHNsGzD1lskciLRRAW3zcsGEDrr76aoTDYbS2tsLpdOLJJ59EOTXMFy1ahKamJtx0002qffL444/jpJNOwuHDh9HV1YWDBw/iqquuwl//+tca7ZGEhIDurUDjIqChQ3rg1cD2e4H/vRAYnvzeBpLAAaxcuRLRaBRz5sxBW1sb3v/+92PTpk1Yt24d7rjjDixbtqys7a1fvx67du3Ce97DOs/deeed6n2Oq666Cn/84x8BAGNjY5g7d67698Mf/tB0u6+88grmzp2Lu+++G5/85CexcuXEW04S0xDdrwLtawFvSHrg1cDYIAAK9G6f7JFY68hTLaxbt45u2rRJ99jOnTuxfPnyCRuDBIP83icBf/000HQCcM7/V5vt73wQeO6/gI/9HeCRSqN9wPcXAxfdCkS6gc2/A27urs3n1wp/vBZYchGwLq/h1+Tg6e8BT94CvPWrwLn/OiEfOa6OPBISElXAvseBg8/XbvsHnweOvAykY9pjPP67fS3gbWDPZVK1G0O1QSmw/wngwDOTPRIN/Pvt3zW544Ak8IrwxhtvYM2aNbq/0047rWrbv/XWW/O2f+utt1Zt+xKTgFyOTb0r8aDTceCp7zA1XQyjvew2Oao91r0VAAHaTmIWCjC9fPBUDMimgGhP+e9NjgLP/hjIZqo8pjF227ezututADIKpQKceOKJePXVV2u2/Ztvvhk333xzzbYvMQlIjAC5DJAIl//eN58Gnvo28MbdwIcfAIJt5q/jBJ4SFPjQAaB+HuCuYwocYJEoda3lj2MyEB9it5UQ+L7Hgce/Bsw/A5hfPYGlKvCBPeziYJ88GpUKXEJiIhAbYLeVEHjfDnYb7QF+9y4gfNT8dVyhpwQFnowCbiVU1RNit9NpIZOPNXqM2SnlgM80olX2/LkCz6aAoTeru+0yIQlcQmIiEOtnt5XEYffvAoJzgA/ey0j64S8V+AwTAk9FmfoGNAU+nSyUMUWBZxLlj5t/19Fj1RwRkB4DbEo7Qn5xnSRIApeQmAhwAs8mgXSZ3ZD6dgCty4F5pwJLL2ZhgUZkkppaFS2U5Cjg4gQeYrfTSoEPaffLJWI+26nEfimGVAyYtQIAmfSFTEngEhITgbEB7X45NkouC/TvAVqUXITWZUDkCJAwFH3jFwiA2SYcqVETC2XE+ufXEq/eCTz+9eKvES82ZkS8/a/AY18zf69K4DVQ4L5mlhglFfjkYmRkBD//+c/Lft8ll1yCkZGRst7zu9/9Ts3O5BgYGEBLSwuSySQA4PLLL8cZZ5yhe83Xv/51fP/737f0GR/72MfQ2tqKVatWlTU2iRojJhL4iPX3DR1gqr11Bfuf3xqVH1/ABPIVOLdQPPXlf34tsfN+Frce7S38mjGBwCMmBL7rQWDTb83fq3rgVVbg6Tjg8rHfok8q8ElFIQLPZrNF3/fQQw8hFAqV9VlXXnklHnvsMYyNjamPbdiwAZdddhncbjdGRkawZcsWjIyM4MCBA2Vtm+MjH/kIHnlEdl2ZchAVcjkKvF8JVWvlClxJvjIqv1Fh+zoPfBRwKQrc7gDcwcotFEpZspB4gchlgR33sTDJcpGMAjTH3l8I8SHA7mb3zYg4OQokw8xCMqJWCjwVA5x+9lsM7sv/7H2P60M5a4ipFUb48E3AsTequ83ZJwIX31bw6Ztuugn79+/HmjVr4HQ6UVdXh7a2Nrz66qvYsWMHrrjiChw+fBiJRAI33HADrrvuOgCsNvemTZswOjqKiy++GGeffTaef/55zJkzB/fddx+8Xm/eZwWDQZx77rl44IEH1CYPd911F77yla8AAO655x68+93vxqxZs3DXXXfhy1/+ctm7e+6556Krq6vs90nUGJUSOI815hZK/XzA6ctXfmYKnFLFQhE6PnlDlVsoO+4D7v4wcMUvgDVKU5E3nwL+/CHg6t8Aq64qb3vc6tn+F+C068xfEx8G6mYByYg5EfOLVWwAqJ+jf45/z2bKfTxIjykKfDlAs4zEZyllLaLHgD9cBVz6ownJHD3uFfhtt92GRYsW4dVXX8X3vvc9vPzyy7j11luxYwdTOL/5zW+wefNmbNq0CT/5yU8wOJjfkmrv3r24/vrrsX37doRCIdxzzz0FP4+XmwWA7u5u7NmzBxdccAEAVjNl/fr1WL9+fcVdeySmKGKDgL+F3S+HQPt2Mq/V5Wf/22yMzPMUuBKBYnNoxJgeYwrXJRC4J1SZAs9lgaf+k92PCGF5XBVv+0v52+TjPPRC4dDIsSHA1wAE2goocGUb4gWSg3/Pqah+XWC8SI2xi6g6GxISevhFZmwo/301wNRS4EWU8kTh1FNPRWdnp/r/T37yE9x7770AgMOHD2Pv3r1oamrSvaezsxNr1qwBAJxyyilFFfCll16KT3/604hEIvjzn/+Mq6++Gna7Hb29vdi3bx/OPvtsEELgcDiwbdu2me9l9+8Gfn0R8MmNQMOCyR7N+DA2BPzsVOCaPwDzT9c/F+sHmhaz23I86L6dQIuhZk3rcjZNFzHay8jZZtcUOCctnQJvqMwD336v5ruLGaFc+e99jC2seoLWt5mMAh3nAF3PADv+Cpxxff5r4kNK+CMprcCNSISZ1ZGOMZ/dXUHp5t4dwG/fCVz3NNDYyWY1aYXAm04AiF2/HsHHkZoYC+W4V+BG+P1+9f5TTz2Fxx9/HC+88AJee+01rF271rQuuNvtVu/b7XZkMoVTd71eL975znfi3nvv1XXs+dOf/oTh4WF0dnaio6MDXV1duv6aMxa92xmhDO2f7JGMH8NdjKDNQsti/UDTQnbfKoFmUsDgXk3pcbQuZ8QZE2aDo73ManD5BQJXSMQlEFclFQmzGaa+W1cCjQv1dg0n82wS2F1ml6lkFGhfA8xeXVjBx4cBb6OiwE0InO/jWAECb1nK7le6kLn/H2w7g8rxmY4DoMxCcbjZdy5aNHwc1VT8RXDcE3ggEEA0av5lh8NhNDQ0wOfzYdeuXXjxxRer8pnr16/HD3/4Q/T29uL005lSu/POO/HII4+gq6sLXV1d2Lx58/FB4DzOdzyLPuk48PebJ+ykKQizOGyAEWB8CAjOBRxe6x740H6Wfm9G4IC2wAmwC0RdKyNrrv5SJgrcEyrfA3/jbubzXvBlRqRGBd64kO1bOTZKNgNk4mxRddWVwNFNwJ8+wPx0cVFzbAjwNbLyAaPH8hdLC1ko6Ti7qPC1A07+g/uBJ79tPauTFwNLKr9ZWglAcCpCr65Vf0Hj45AEPjFoamrCWWedhVWrVuGLX/yi7rl3vvOdyGQyWL16Nb761a+qZDteXHTRReju7sY111wDQgi6urpw6NAh3fY7OzsRDAbx0ksvAQBuueUWXc3wQli/fj3OOOMM7N69G3PnzsWvf/3rqoy5ZlBJbxwEfuhF4IX/Brqeq86YKoW6L2P6x8cUpexvZqF8Vgmc+9xGAm8x8V51Clz5LlUFbmKhWCWwbBp4+jamkpddyggrJhJ4PyP1lVcwtWpV3asXlwCw+lpgzinAwF5g3xPA8//Nnsvl2Fi9igeey+iVdi6n1SUxEjj/jnn0Dlfgm38HPP0dc8/cDCqBK+PlF2eXj93WzZpUAp9aHvgkgTdWMMLtduPhhx82fY773M3Nzdi2bZv6+L/+a+n6wA6HA/392gHU0dGBo0fzF3G2bGGtsE477TR8/etfL7ldAFNr8XNwP0sBd3oKv4bH+Y5HgXPSSMeKv67W4AtXxnFw0vG3MAK3qoD7dgHExrxzEcF2wF1vIPA+Rq7xIS3JhxO5MQolm1IiKTS7sCBe/SOzhtb/CSBEIawnhM/tBWavYir6hf9mYYYnf7D0djnBueqYuv7EP9j/910P7FX8/cQIW4T1NgIBpR9ttEcrxCVe9I0eOP+Og3PYZ3AC54SciJgX9OrbCTQvZYvF8RHN2uPfqarAOYG3AMdezx+HVOAS0xqpMeAXZwEv/qz467iFkhrHAa9uY5IJvJCFwlWZv5kRqFUFPriPRaAYL4CEMGXJvfZUjJFZXasFD1yoSFgKmSSw8XtMHS95B3usrpWF9PFZxmgf4G8F2k8GQgtYYo0VJAUFLiLUwaySdFz7Pn2KBw7offBiBM6/Y0+IkX+0hyl2XobA7DcY7gJ+fgawWUkMEptBi5E9gHbxq5vFvgNu7UwwgUsFXiNcf/31eO45/ZT+hhtuwEc/+tFxb3twcBAXXnhh3uNPPPFEXoTMpGFgN/M4zep2iIhXQYGPFSDOiYZ6ITFYKDGDAi9V11vd3jBL2TZD63LmFVOqba9uljUPnG/bGDdtxNbbgfBh4N0/ZhcNgJE1wGwUMot5w3Wt7PlFbwXe2GCtxCr/vfMIfD67HTnMLhSAYqEICty4DaCwheINaQugQ/u17yRpQuA9rwOgzPN/y8c1tW53aWNJGRX4LBYLHh9iF+jj0UKhlILwA2SG4Gc/K6E8x4GmpqZx1SMv2EZv72PAy79k02XbOCdnfHpfqug9tx3G44EXUr7VxIaPaR57/Rzgow+zKAQRhfZFVeAKgQ/ssfaZiZEiBL6C+bmRboHAW809cLeJAi8VCTNymLUOm3c6sEgQC3Wz2O1oHwCif6zzHKZee14F5uZ1/9JDVeCGsEMeSjpyiNknALNQ6mYhL5SQk7EnZKLAR5Tn6hn5H3kFOLpFeN5QSwbQjlUel350C5sB5bImCpwTuHJBG+1VCPw4s1A8Hg8GBwcLk4pEVUEpxeDgIDweE1/6zaeAvY8CI1Xots1PhqE3i1ffU6NQprCFEhsEtt3D4oDb1wJHNzNCyBsH9+JNFDixM6LxhKxbKImwVr/EiI6z2e2eh7VFNH8rU9ucuDmROwWv20pFwuGDwO8uYTbGJd/V1DcgEFafdmHiBN5xDrs9sLHkrqmKVpwdAMyGAYCRLr2FYneyC6CYRMT3s7GTjUXkENVCUQg8egzo3pL/vIi+HdoFZcdftWbQ7oD2en6MiQocYL8Bpdp3MkFx4JYUOCHkBgCfALvk/opS+mNCSCOAPwHoANAF4L2U0rJTvObOnYsjR47oFvUkaguPx2MeycIPvv5d7KQYDziB0yyLZZ59IjvAt94OrHyPpgpVC2UcBD42DgLv28Wq+53wtsKv6VL6Mb79m0DzEuC7nYykOIlyFLRQ+pk6s9m0KBRK9cRohkRYI1wjWlewxbZt97IIEECLQsmlWQw5LyUrzqZKVSQMHwF+ewkjoA/9lbViEyESFh9/nZJhWtfKImS6ninduLmQB143i9U+GT7IFmsBbdbAiZiDk2RDJ7M7UjHtgqBT4O2snvibT7FY9r7t2gVERP8udhEKHwY2/QYIHwJO/QSL8zYqcE7g3FIa7Wefn4lryUMT0K2n5NYJIavAyPtUACkAjxBC/qY89gSl9DZCyE0AbgLwb+UOwOl06jIfJSYRnMD7drC60+NB/y5G2sfeYGQ++0QW7nf/Z9nzJ3+ILfxUI4xwPBbKo19hyuxLRTqrdD3DTsr2tUwJzl7NmuxeYHhdQQtlQLNCPPXMGkhGi2ctUspItpACJ4RFfjx1G0sQIjZ2keALlqlRZjG4DAq3lIWy9Q9A5CjLjG1bnf+8vxkAYQqcGCwUAOg8l12kMynA4Sq8f6kCHrjNBoTmsVmgw8M+i38HxnR6rsAbOtjt2IBG4PERFnPvcGv+ef8u4NRPsuPbqMAzKbZovOxdwLy3aGVu29eyizWPJEoZFzEFC4W/pnEh0PsG+/75910jWLFQlgN4kVI6RinNAHgawHsAXA7g98prfg/gipqMUGLiwP27UiUys+ninc0TEaZilr2bdS7hapxPrXnmWjKs+ZyFFjETEVZSdeiAuW8JCMq3zItANs06uY8NFp8BHNgILDiTkTfASOrIK/lKu6CFoihwQGgszKfkY+aV/FIxNnspROAAsPJKABR4/W52gbDZNWJJjepLyXK4A8zO4WM1VtLr28EIyIy8AfYd+BoZYXHvndd4AZgPnh5jNlMxiGGERoQWMA88Psy+L5udPZ6nwJVt8NmirmRvWE/8HHNOYTaJ8Vga3KckTa1gM0SOtpPYhVYNIzRYKO4Au1CM9mqfz8czAT64FQLfBuBcQkgTIcQH4BIA8wDMopT2AIByO026pEoUhErgJRYe7/ln4O6PFH6eh7e1rWb1Ivj2uBXBVZTow5qRby4L/PQU4Cdr2N9/v8U8AaVSC6V7q3ZCjhwyf030GFt07DxHe6zzXGZTHBYyc3NZPSnrxjegkZxakzvMiPunJ7PGu0aIYXCF0LIEmLWKTdu5ElQJXAktNBIkIVo6/T9uAW5boKWJA+y3MiYOGVE3i12URvsAX5N2YQOABWcBINpvXQhJZXbAyVlEaD6zUNQ6KAoCbexzs2llG4KFAuRXfOQXS67AAWDOyew3MFooPGmqZRlT9HPfwu57goyk1USeMbZ/TqXaKCFKNqawJtC4UNvHGqMkgVNKdwL4DoDHADwC4DUAhYt9GEAIuY4QsokQskn63FMY4gIM77ZdCP27WTGlQouTnLBbl7O/vh1sQewwyyrVKrbxRapmcwU+sIeFq73lE8BJ72PxwcZoA56tB5RP4OJi23CBhduuZ9lth0Dg809nVf8OCCQVHwFAmZVhZqHkEfgI81ijPcBL/5NfjU/0cIuBq0VO4NySSI7qGxqL8ISYat/4PUb+B5XomkySkXlJAlfSx0d7NQ+Yw9fI7LJSC5nJiLn6BlgkSnyIXVS9jdrj/mYAVL9uQmxa6KGOwEcEBa4QuCsANC5SFLXBQunbyWYmzUrS1NW/Ba65g913BzXC54WsdAu7SjamSuBcgdd+IdNSFAql9NeU0pMppecCGAKwF0AvIaQNAJRb0+BWSukvKaXrKKXrWlpazF4iUWskIsC9/1K8xGUizFTlrFWshsRwkYYSsX72miMvmz/ft5Md5PXzGRmMHGQLSNkUe1xV4Mp4QvPME3l4HO6pnwBWXMbuG5WyaMOUa6Ec2Mgy9cy2q77maZb1KC7muQMscUVUmZxUAu16CyWdYCe/X4nP54o6EdasqmwKeMbQcUmMoiiGVVeyW+5DGy0UM5L0NrDve93H2b7x73lgL7NteP2QQuCExbM/jeg8l613/PZdwO8uZenxRiRHzS8ugBaJcmybXoH7lO+QlybgMwx+cTQqcP5dO73sfvsaYSHZoMD7dwFNi7TQ0NA8oPkEdt8dZIugmRQTCTyEUP0+Wtlnc3HBPfmpoMABgBDSqtzOB3AlgDsB3A/gw8pLPgygSFsNiUnF0c3Aa3dqSssM/ODrPJfdFrJRclntBDpQYJrcv5ORgM2mqbmXf8UUzuKLNAXOSS80n53QRnukeys7QZtO0IeXiRAvSkbvuRgySTYjWH4ZW6AsFDp54Bmg46z8qX7nOSxOmJ+k/GJUP5cRMp/mi2n0gEbI8RFt2n7iPwFbbtfPAsRElGJoXAic8wW2DUAjbL6IafTAAWDt+4ELbgbe9QOg/SSNwNWZ04rin8ktA15/xYg172PfGcAWsZ/5Yf5rCs0OAC0WPBNnip7DSOD8AuXysd9QrM5oDME8+/PA6Z9i993B/EQe3jjaDHyxORllM0mnCYFzD9xVp30nZpEuVYbVOPB7CCE7ADwA4HolXPA2AG8nhOwF8Hblf4mpCE4yYtEdIzjRcA+zEIGPDQFQiLbQNFn0UXnhpf1PMAXUvITZItmMRr6h+Uz5ZQyWTPdWpnxtdm2abLQ6eDhc3azyLJQjm9jndZ6rea5GhI+wmYhon3B0nsvGfPAF9v+YQOCANhYxiQfQe+D9u1gVv7d9g1kBG7+bv1+lFDgAXPgfwAlKso3ogRdS4Os+Bpz3JWYDtK9lSjeTZBdem4NdMIvB38q+u/ARcwU+ayXwofuAj/6NkebB5/K74hQjcH6xBvQWSp4CFy5QYhYkkB/Bc/aNLMIEyLdQUmNskdxYd51DtaUibM3EWEOmbhYbU7SbjUN9vXLejQ0BW+/Qx7BXCVYtlHMopSsopSdRSp9QHhuklF5IKV2s3E5MCwqJ8sGVQLEUbn7wh+YX77Y9JkwTj27OJ82xIXah4ATe2Kn1NOw8lxUuojn2eVy1BhXSEz3DbJqpt/a17H93HTuBjVaHqHzLIfADGxlpLjiTKT6jAk/HtZBHTo4i5p3GZhTc1+ezCU7gfDbAyYaTj0jgfTtYTZP6OcCa9cDrf9YiUqwsYppBJA9jOzUztK9l1lnvdnbhbTqhePgfoClMmjVX4CJ4pMyOv+ofL0bgviYt+aiYhSLaMP4W7RjO5dgxX2j2YrRQBvawMRZS4Dy5h9eA4QuYHPwi1reTjUOcBQFszei+T5cODqgAk56JKTEBUBW4BQL3N7MD2awpgfi6le9hJ/4hQ410cQETYOqZF9XvOEcoStTDyN5Tr5Ga6IP37WQqjxM4oISXGYhWVb7z2AljNaO36xkWz+0NaWFr/L2pGPDH9wL7nwQu/5k2fhFOL7vYDe5j/8cLKHCupDkR2eyMEMYGgf49+plKNqVdCDiBG1PNS4Grw0SYfX+uAiTJ0X4yu+3eai0CBdCrbjMFLoJHyhhrhaeKEDghmo2is1CU+0YPHNCnsadGmUgoNHvhi5L89y5lHYkXRb6IKYIv5A7sZQvyRgXOZ76lLnYVQBL48YCEFQWuHPy+ZqHbtkmsNyfw5e9WIjEMNgpPVxano60rWDz4/NP1RYl4txWuEsVFH+7LigTesMDEQuE++jx20hptGDNkkiyOm4cGhuazE5pv68EbWfTJe/4fsPYDhbfTdIJWbnRsiCl6foHiBG62GOmpZ/VCskmNNMSEEIBFUbgC5WfyOX1giTbKdkq1EQvNZ7/BwedZNb5CNoIIkYhKETjALvZHXmb1VTiKKXA+LkCvwB1u5eInNAFRFXizNjssFcHDk6lUhbyLHZ88/C/v9cpFNBFRFjFNLBSAzUj8Sjy+05/fbEISuERFUC2UIh54bIAd2A4XI5VcRlOXxtcBTLXOWaePxEgngBd/Acw9VUuDBoBzvwhcczs78EUFHle6rXAVJVoo3VvZeMSTKjSfJQiJiS/xIQBEiyaxYqOMDTK127iI/a8WUDrIvPldDwFrPwicdE3x7TQtAgbfVLImhxnZ8IsRt1DMyMQT0qo08ogPToS8WUKxOijFQIhS/1pZKC5loXAffNffUNRGEKEjcAukxCNltt/Lbim1QODKb2LMZPQ16j1wVYG3aPVQStlPIiEDzMuvn1v4YilaKGYKXLyI8bUOd0B/3hG7fjZRJUgCPx5g1QPnBx8nFTMfPNbPlKa3gSnY7q3a6v+W37M07LferI+TbT5BS833t7D3R48xJeVt0E5kMQywewsjFnE7oQWMeEeFbDxuw/CTzEooIVfa3CNVI1wOMWWcigKLjHnyJmhcpDTMPaYlnTiFRUSAkYndrfdNPfXMfgLR7BldlT8Ur4NSCm6BwAvFWouYczKL+ACsEbi3gRESYI3AGxcCbWuA7YqNkkkwgVBsbGYWCsB8cJ0HLhB4LsMumKVCMPmxwl8XPabP1iz0+mSUeeBmYYQcKoELRcV4pUKzpKVxQhL48QDRiyvkEYsE3ryETatfM+nuExtgJ5HNzkLwiB244yq2wv7MD1gUS+d5hcdis7OTXmehGDzDdIJ1AxftE0A7qUUbJT6sqHgDcRaD0ZcWI1wOPM3um0WeGNGkKPih/crFqFE7uUUCNxIJ/79hgXlNDT7GShQ4wLbJL3JWOrHz79nu1rIai8FmU2qA2/VRIsWw5B1K5muicC1wEQvPZwvFxogYkcB1HjiPBR8oHcHDH+fCJtqtz9Y0go8zEWYXbKfBQnF6WTy9OA4xe7NQvHwVIAmcw6zj9WQiOVq49ke54NvJJgvHpnJiBpiNctbnWLbloZcMrxOIvm01cM0fWATDz05n5HOBQX2bgde0UMnXsGrft50pVCOBhwSrg4Mr37IIXFHgfIrtDbGTeuQgi/tuXaHVLikGTuCD+/MvJNxCMSNirqzFRTN3kBVvUj3wCi0UQG+hWFHg/HtuWWLdc69rZceB1brx3AoLC40aii3QzloJfPzRfJL3NbEZXybJZmNcgfNjN9ZfOoZejQRSFjJLKXCnR2nqUECBA1pFRp6wlUfg1fe/AUngDH07gR8sLd09ZiLx4I3A3R8u/TorEBcHC9koIjEDwKnXsf+fvNXwugE9uS19J3DtnWxavPACLYGjGAJtWscV0TfmyuzYG+zWWMq0fh67FUMJVeVbBoFzX1r0V0MLWBTBoRe1ZKZSqJ/HTuzBfZoHbmahGImEE4iY8UgIi2YQLZRyQwg5XHWM3IDSHjjA1itC85nNYRUNnVrGoRWIs5xCpWStgCtwY7s4vubSu708C4X72sEiBM7HOjbIhIVRgQMaQasKPCgJfMLAV8fFUpWTjfBh1gyhGkhGtCme2UJmjreEEgjc5WfJDwee1uqBAIzojV1iFr8N+OwmtlBpBYE2bYHUa6LAw0eZT87jwzmcHvbeqlkoIe2xhgVsPzNxa/YJwOyghk72OxW6kBSzUIxha3UigY9UrsBF0raiwAHWYeiiW6x/xqU/At77+9Kv4xBnTyqBWxybCF8jszF4xAnfRssytqj+7I+1heBCCl+1UMLaTKWYAufb4r+NqQJXLBKjhUKp4oHXpoyIJHBAKFQTr+52h7uAO95bWU2EZFQr9jReJKNaXQczAo8Ps7Aq40G27mPswH76O9pjYmEmEaH51hVVoI2FXAHshLTZ2co+/56iPUyNmk3nQ/MNFoqifNWLgEULhdj0MdKhBcqYiLVZBEfTIqVYVwzwNSiLlUQg8BETAg+x21ZDzRG1QW6WHZPj8cA5rP4m9XPLWzT1NRb3jY0ItLFQvZGDhWuBW/pcxaLgF3H+uxPCFs8jR4DNv2eCpdCioRiFwkVbqX1xB7R1BWMiD6ApbD4+Vx1bDI8PM9UuFXgNUSsC3/8PYO/f2YJcJWNKhotXBbSKRERbDBo1qQgpJvGIcHpZjY1DL7JxZJJsTONVE+LJwknDVaed2NFjhU8oMZknm1ZsGFH5WohCSYwwEhX9W64Q21aXV4S/aZE2U/I2MCJx+oQwQhMrZNklrH6JqQLvFZr5Gt5nFSKBW1XgtQZv1KCzUMpMUgI0guTHgKjiF14AzD+TzSaLXfwciqedKEOBe+q115pZKGvez2YwvLQuV+Biv9IaQBI4oC3yZaqtwJWDrFj/wULgB3mp5rOlkMsxJRBawBJvzBS4sV6HiFkrmZ869Ka2+m9lga8YxJOFRzGIYVfFFpUaFjCLJZvRrJCyLZThfHLkES5W7RMOHksOaPvi8isNGaj5ImZoPqtfYlSIvKYGD8usWIErytbmyG+8PJngGa/qIuZ4FHgXuxVnUVyFA8W/O0K0bEyuwEspZHdAO0/MLJS21cCZn9W/PpfR1mukAq8hxPC1aoL/eDzN2ioo1cisWAlYK1A7d9frF8lE8OQcM2IWY8KLEX05ENU1j/PVKfCeIgpcKXwVOaJ9r94GLbnCqgduVNmzT2TT7uWXWd4NAFokCiDsi09pqGChq46IuhYAVMvuHK+F4qorHRE0keA1Z4p14ykFvv5ipsAB1qd06buA2auKb4cXtIoeY2Reyo93B7WyxcZEHtPXKxcWvtZTIwVe246b0wW1slD4QVYuCWeSSqIHKlPvIsQVfz5F58jl2NRWJXATYm5ZCoCwdGOx8tt4oFPgDdr4klGWvj82UFiBi0k3Nqe2De6jW03kMRJ4sB34coGa4MUgxinzbTr9zEKxWhKWg6u0gT3sttIoFP47VaJwa4nQfKVqn5KZaOYll0IhD1zEtXeUvnDxglbZtDUvX/wujan0xV7PL8bSQqkhJtJC2XEf8MMVxdW+uOhZrno3gu+bJ6gV4gdYkf3vLAAG9umzK41wellFwb4dxYm+HPgaGfnaHJoP6qpTPENlfIVOKh66duwN7Xv1GayLUkiMVK/ZbKBNU2RGC8VqVx0OTuD9u8t7nxGcYKYcgSsX395tSm/OCmYH3hAAIihwk320sl3VQikRA84hNqAuV4HbXZVfjEtAEjhQGwslJYQ6iSTc/SpLNy/UQADQJ9uM10LRKXCh5ObOB9jnPH2bEhrYVHjVvnUFi5UvtNhZLghhJw1f9AOYakyNagtFYi0VEaH5rF/hCz/XogJU5euz1tQhPlL5AqERhGhJKkYLxWpXHQ6u0gb2lvc+I7gvPFUWMDn4xbd3W2ULmAA7Rr0N2ndb6T566jULxQqBV6rAB99kF+YaWVmSwAF9v7tqwZhsot5XSL1QD0bAoMAF9b7ldpYdWQ7UBaN6IUwtx4pQETvwxgYWZVJMVbcsY9mGkW6mJio9+UQE2/Qq2KUsYkaVoveFFDghwAX/zjzwF37GHvOKPnoJBc57aFZTETUtYt8LV2ZGC8XqZ6llSRULZbxRKJXEWdcSXIHHh8c3O+A2it1dunZ5IageeJH1FhHuChV4+HDN7BNAEjgDJzkrpUitghO03a1X4NyGKKbARR9XfO+T39ZIyyrEFf+6WWxRrXcbm9qd/XlGev07tZPCDK3Lle4zz7NFpGqoiZOuZa23ONwBvQIvpop4uNjgPsWG4YrTX9oDT0XZYlS1LBQAWH0N6y/Jvxc+jnK66gBMubsC7DcntsrVJSfuqabA/c0a+Y3n4sKP1fFsw62EBebSFhW4QOBmUShGqNExNL/xcxUhCRzQfOJaKPDZq/QqmtsQRS0UQYFz9Z7LMn+4mHI3g84DVw6kbRvY7YrLtT6BxRQ4r1DX89r47ROOdR9jmZ4crjr2/YePMH+8WJEkMVxMtGGseODGSoTVwLJ3ARcLHQVdPlYzo5KuOvw38tRXfqF0TdFFTEK0lPpqKPDxXKA89VBbA1pR4JV64IBU4DVHLTzwkYOAw8sq+42ZELgVC8Xm1AgnNsBUsLEedimIHjhXAtv+wkhl1onAGdczsmwsUoWuaTFTuqA1SwlW1dTgfnZClSqS1HE2sOhCff9ESwQ+wm6rqcCN4Bejcj1wQFvIHI/F45qiChzQfq9xETjPHRjHNkRCLscDtzm1ZB0rrwdqFgMOyDBChmQNolCGu7RuJ+VaKHw89XO19/KEA14Pu9Ain+m2iL5bdvgwsOxSRpLeEPDZzcUXZhwuFi7Xv6t2BM7JZnCv9RTta/6gFW3i2yhloRgrEdYCTmURMz7MxlROVx1RgVeKqeqBA1NMgSsoVcgKEKKlLKhvgEVvEbvSN1Qq8NqBl6UEKosDf+4nwF8/nf/4yCGWuOBrYGosnWDb5wRjbM4rgifxhOZr6l0sd1uOjZKMsoOPEP2BJFbc8zWWztjjCT3VslCMUONmD1gncJdPb4VYUeBmlQirDZcPAGUX2nKJuBoEzr/LqWahAFrGa6lencXAj8FxeeCCAreikPl3aZZGbwZCtPdIAq8hxJrb5Voo0WOs3Oqev+c/N3KQTRe5lxsf1tR30wns/0L1vpNRpRrfHE0xipUSi5G/EYmINl10B5itA5SfMs7rdtSawK0uKpmhLAslVNlnWBqHQiyR7vKVfjUI3BsC3vVD4MT3Vr6NWqEqFko1FHhQ25aVcgOeMhU4IBB47SwUSeBizHW5FsqzP2KRK8Zqg/ER5n+G5mtKLz6k+d9zTmG3hWwU3i/QJ9gv0WMASPH3cWz9g0b8yYh2IHEVzhsXlwNeOa/WFgowPgLPJNiCbyFMlIUCKARergJXTvbxXmDe8nGgfs74tlELVNNCGY8C57+L1WONK3YrC5jqe6QCrz3Uym+N5Vko4aPApt+wMMFsklkxHFwhNyzQFlxEBT5nnf51eWNSbA+vYL9Eu9mBUDeruIUy3AXcdz2w9Q5t/8Tp4qILgLXvLz/CYd7prHBT+8nlvc8q3FUicKC4Ck+M5PeorDa4Sov2VE7g41HgUxnNi4FZq4D2NZVvQ1Xg47gI8HPCql3HO81bSeJRP0MZXw3DCOUiJlfPYpq5FTzzfVZ06vRPAs//lFkVvK0SV8ihBcwKAVg4IPe/5yoKvBAR827bKvkPaSVW7e7iCjyiJMLwIjrJqL4Bw7v/y/o+igjMAj63pbL3WoFOgZdRZ1q3DYHAxSgDEbwSYS2LPPF9yWXKV9J8hjNTCdzlBz713Pi2oUahVEOBl3GseYLlK3Cnv6aLyVKBcx+6rtV6Ik8izLIi136AqQlAb8VwYi5koTQvYSd5UQUe0Pvn0R6mTI0NDYzgXjkvoiN64FMZ4pS6YgVuoamDWSXCakM8ycsl4vq57KJfV+FF7HiAv4WFtY7HzuM9SMtpC+dvKZ7wZvb60Lyyh1YOpAIXFXh6jKnqUuqs+1W22Lb83Zp1IhL4yCHNAuELJPFhVonN4WVEU4yIk1Hm0XKiGVMU+JxT2AG0/V5WD9ssPI1Hqwy+qW1rKkYjGFFVBV4klLDaafSm4xgHgde1Ah9/nNVhlzCHOwD88+NMCFUKuwP4+GPF8x+M+KfflWehvO0b1qpjjgNSgScFBQ7ovexC6FashPa1GjmKC5k8AoV3Z7G7GQnzdmSEsOcLWSjiIibArJ1Yv6LAldZfkaPm7+UKPHKEZQMaPfCpCqeXKU+Hd/xV+IoqcJNSstWGeDGq5GIx9xTW/1OiMNrXlkemZmhbXZ64aVpUZhu5Wfp68TWAJPA8ArewkNm9lRGpr1HfX49jtJf9eAAja2+DYqEIHd0blO4klJqMaVSxUBSi4eVFA7O1ONpC6l2MFx/YzWyh6UDghLBFqcDs8aeQFyXwcG1DCIHxWSgSEmVAEngiwrwwTnJWIlG6twJzlGgMMwVu9Fl9jSwhJ9avEXhovtb01AijB96n9NTkHjhQWL1HerTV+e5X2e108MABtthjNcPUDKKFkk0DBzbmvyY+PAEWiqAMJYFL1BCSwHmcNFdNpQg8NsCUc/ta9r+7XtsOh3Ga7m3Uwgj5wgtPaOC9/Th4D0t3gHmpDg/Qu509F2gD6ucxq6HQAmi0B5h/OrvfvVUZ4zTwwAE23Zx9YuXv58SZHgNe/hXw+3ezhhUc2TT7bmtuoQgEXmu1L3FcQxI4j7nmnmOpSBSualUC5wpcIfBcLr8Tua9Bi0IRLRQgn4j5ogffrrdBI/lAGyukE5xjbqFQyiyU5sVsUVYl8GmiwD9wL/COb1f+fqfggfOKizwaByi/xVmlsDu1dm9SgUvUEJLAeZgdTzEvpcA5KbatYbcOF1PJ3ANPhgFQgwJvAEYOs4QfowLf95jeBxerBwKKjUJZ2BQPYQrNN7dQklEgHWNE37hIs16miwK3Owp3BbICrnx7twNHN7P74gVyIioRGsciCVyihpAEzv1mp1UC38LKq4q+Mm/IC5jX2vA2MmIFtKQaTxA47VMs7f2RL2skzhU4X5DjkSh1QonV0AJzBS42Q2hayBJJ+GcdD3C4WQW4bX9h/9sceotK7VEZqv1YVAKfgM+SOG5hicAJITcSQrYTQrYRQu4khHgIIV8nhBwlhLyq/F1S68HWBDzMjhM4t1BG+4GN32Nd0kV0b9XsEw7eIBUwr7XBSRjQJx+88z8Zib/0C+DRryjj4QpcIV1+IRDDlxoWMK/bGPLIQwgDs/Xd0qeLhTJeEKVsbirKyhU0LtRf6NRmDhOgwJ2+8XXVkZCwgJIETgiZA+BzANZRSlcBsAO4Vnn6R5TSNcrfQzUcZ+3APXCH4oHzrjy7HwL+cQuw9f+010Z6GEnyCBQOUYGblSsV74vV/AhhJL7qauDlX7IiTGILNECLRBEJnEdqiCGD4v/cQlHHd5wQOKAp31VX5ltNE1GJUByHO1i6MYWExDhg9ehyAPASQhwAfAC6azekCQb3wFULRVHgnIg3/kB7jPvfRgXuCWoeeCELhcOY/ksI0HkOq0kePpzvgXP1LqaXiyn2ItSGwLMMCnyaeODVACfwFVcoVpPogU9AJUJxHDICRaLGKEnglNKjAL4P4BCAHgBhSumjytOfIYS8Tgj5DSFkAualVQalQhght1AUD5xHLES7gc2/Zck1z/+UZVUaQ93cQcEDN5mm6ywUk3ranGwH92vNHHgBHL4dUYGLRa5ERI+xGHB3QEsRtruOr6y+ulms1nn9HGY1JUa03zLWx2yNiSDWwGwgOLf2nyNxXKNkLRSFmC8H0AlgBMDdhJAPAPgFgG+BdQb9FoAfAPiYyfuvA3AdAMyfP79a464OUqMAqMFCEQjc28hqUjzzQ2DHfcDhl4Erf5mfwit64GYLZZyE3UHz4vHc7hh6U6tlrXrgClmLCS5ijRQR0R6N6J1eRiDVbBM3HfBPv1X6d0Kf9NS2Guh5HWheaq2n4XhxyfeL1yWXkKgCrFgobwNwgFLaTylNA/gLgDMppb2U0iylNAfgVwBONXszpfSXlNJ1lNJ1LS01agZgBXsfA+7/rP4x0a4wJvLER1gI2AX/zpTbkVeAq38NnHh1/rbdAf0ipsOrV72chAt1swnMZjHMg/u0MeVFoQhdPQpaKMf0/f2aFh1f9gmgNKxQvp+QEGtPqT6DttbwNWrlhSUkagQr1QgPATidEOIDEAdwIYBNhJA2Sinv8/UeANtqNMbqYO9jwJb/Y8qIq2DuW3u4MiZaFEpCqZmx4EzgoltYB5sT3ma+bY9ioVCqpNGH9M9zxVyo/CUhLOxvcD+zPBweFl8OsN6VZ36OjcO4vTwC7wHmn6H9f/aNwGif+WceD+ClQkcOsjrpsb789QsJiWmMkgROKX2JELIBwBYAGQBbAfwSwP8SQtaAWShdAD5Zu2FWATy+OtbPai4DggKvZyTq8GhRKImwloRxpkG5G+EOADSndSI3LpI5XExRF6tf3LgIOPY6qx8sqmZ3ALjoW+bbEy0UnoUpeuWLLig+7pkObwP7noYP6itISkjMEFiqB04p/RqArxke/mD1h1NDcLIe7RUIXFncUjtOe/VRKKIdUQzcr05GFOVusp47d11x8mg6Adj5ACNlK7HDvL4KR3yYRbJU2gxhJoKX7R05CHTXMW+cN+CQkJgBOH4aOnAFLloKnNR5pqLTq49CsRpuJpaUjQ9ri2ciPnRf8W00LWJ1vnu3WfOteX0VDt5KrdJmCDMVDQtYNmY2BbSuOL4iciRmPI6fLANVgQsEzj1wTpgOjz4KxWodC1WBRytv2cUjUQb3WUu88TbqLRQxiUdCA0/mMcuglZCY5jiOCNxMgXMC5wrcxyyUdIItZpZN4OHKW3aVm3jjbdBbKGIavYSG0AJWhyY+LAlcYsbh+CFw1UIROs8nowCI5jk7PcxCKbfsKCfcsWH2OZUocF+jdsGw0sXa16i3UKQCNwcv2wtIApeYcTh+CJxbKDGDheIOaPUquIXCCbxcD5wXTqok048QzUaxpMAbmV3Dk0UiR9ljZolCxzP4eoTdzTxwCYkZhOODwCktvIgpkqXTZyBwqxaKsg1ed6PSWhvcRrFqoYBqY40cZenjEnrwZJ7Zq7TYegmJGYLjg8AzSa02tmihxIf0ZOv0MO+73LrRLgOBV1qutKkMBe4zZGOGj8raG2bwBBmJd5w92SORkKg6jo8wQq6+7S69Ah96U7946PCyRJ5yFbjNxkhcJfBQZeNULRSLUSgAi0RpWgREjgALzij+nuMVn3xa3yleQmKGYGYp8L99AXjxf/If5/53Qycj81SMecdDB1jRfw6eyKMq8DLaYXmCQPgIuz8RClxMp0+OsotOUFoopvA2yLUBiRmJmUXge/4O7Hkk/3GuwDlZj/Yxss0mNdIEFAKPazW9yyFwd4BtD6jcA29bA1x0K7DUQnMjsaRs5Ci7Xz+vss+tMR7b0YtvPrBjsochITHjMLMIPBnN71IDaDHgnKxH+7Ru5ToLRQgjdHjKy9oTbY9KG9nabMCZn7FmwYgKnCv/KbqI+cBr3fj9C13I5mjpF0tISFjGzCFwHmnCE1pE5CnwXlb5D9C3HnP62GLn2GD5JMxtD3eQdVevNTz1AAjzwLkCn6IWSk84jmyOoj+aLP1iCQkJy5g5BM4jTRIj+Z3lecYlV+CxPkbgTr8+c5Er7uix8m0QHgs+UV3IbXam1ONDigIn+qYPUwjdI6xA2NGR46y5hIREjTFzCJwvVAL5KpxbKA0dAIhmoTQtZAk0HA6RwCtU4BPZB5FXJAwfZQ0fJqLTTJnI5iiORRiB94Q1An/5wBCe3tM/WcOSKIDeSAJ3vHSw9AslpgRmDoGnRAI3+ODcQvHUs644o72saJRonwBaqNloJQTOW6CFynvfeOBtUCyUI1PW/+6LJlTvu1tQ4N/7+y584/7tkzUsiQK4Z8sR3HzvNvQqF12JqY2ZQ+BcZQOFFbgrwJRqpJtVqGsyEriiwMcGyydilcAnsLczr4cSPjpl/W9unxjvHxiI4dDQGDLZ3GQMS6IABkdTAPQXW4mpi5lD4CmRwI0KPMrsEbuD9Uw8sonV3hYjUACWyMNRrgKfaA8cUErKDitp9FMzC5PbJi67TSWFaCKNgdEUMjmqI3WJycdQjBO4/F2mA2YOgYsKnDc3EJ/jHnXdLK2KX56FIhJ4qLzPVz3wBvSE4zq/t2bwNjDyTo9NqAI/PDSGB1/vxoOvd+PpPf2gtHB4ICftVXOC6AkzUjg4OKY+f2AwVtvBSqjYemi4ZCjnoELgE3L8TlOkszm8dnhksocBYCYRuOqBE3MPnJeMFftS5lko41Dgggf+xbtfx033vFHe+yuBr5HNJIAJ9cC/cPdr+Mwft+Izf9yKD//mZWw7Gin42u6RBOrcDiydHVBJ4cCARtpdA5LAJwLbjobxnp8/j6f3FG9yPRRjoZ4yYqgw/u+Fg7ji589NiXWCmUPgXIGH5uUTeHJUq7FdN4vduusBX5P+dQ4hcafiKJQGdI/EJybmWfTbJ7CQ1b6+UVy6ug1//MRpAIDXjowUfG33SBxt9R6013sxMJpCIp1VSdvtsOnIXKJ2eFVRjH2R4sflkOKB90gLpSCe2NkLSoFDQ2OlX1xjzJxiVtwDb1rMeiAan3MJFgrA1LcYQgjoCx6Vu4jJydTXhMFYCkHvBHy1IoFPkAIPx9MYiqVw4px6nLGwCfVeJ7Z3F1bgPeEE2kNetIXY7OZYOIGuwTHMDnrQ6HehS1ooEwL+G4Xj6YKvoZRKC6UERpMZvNLFLNipsNA78xR40wlMgYu+bDIqKPBW5XUG+wTQp86Xq8DbTgIu/znSC9+GcDyNaCJj+a25HEWukjRzXg/F5tAuTOPEwGgSfZEEBkfNldpBhXA7mv0ghGBFWxA7usMFt9cTjqM95EF7iH233SNxdA3GsKDJh85mv84PnwhU9D2DxbMX8/qnGjLZnG68/DeKJAoT+Fgqi2SGRQUdnYIKnFJqKWopk83VrGzDc/sGkM7ysNjJ/45mDoGnoiyKpH4u64EoJvYko5oHzonOuIAJjC8KhRBg7fsxnGKqPprIWD7hf/z4Hlzyk2fK+zxAKykbaGOZmePEH148iHW3PI5Tv/0ETrnlcfx169G813DLo7PZDwBY2R7ErmNR0xMrkc5iYDSFtnov2uvZd9sdTqBrIIbOZj8WNPlweAJDCf+86TDOuO0JpCv4vCt//hy+9/fdNRhV9ZFIZ/GWWx/Hhs2sRk4mm8OuY+x8iMQLCwsegTKv0YuB0SSSmWztB1sG7njpEM647R9FyZlSig/++mV88vZNNRnDU7v7Ued2IOB2TIlZyswhcO5z856QYix4SvDAGzqAzvOAJRflb2M8USgK+EmQzVEk0taI4v7XurG3b7R8dcgtlCpFoOzpjcLvsuOWK1aBEOBNE3+6a4Ap5vmNzG5aOSeIZCaH/f35rz2mRJ20h7yYXe9RP2MwlkJHsx8dzX5kchRHhifmRDgwEENvJImjZX5eOJ7Ga0fCeONo4ZnGVEJ/NInhsTTuf41FY+3vj6nKupiFwu2TVe1MvPDfb6rg/le70R9NIlpkFnH/a9144c1BPLW7v+jrKgGlFE/t7sPZJzRjToNXWihVBY80CZoQeFLwwJ0e4MP3A3NOyd/GeKJQFPBFIACWDqADAzF0DY4hm6NFTy5TcAulSv53XySJtpAXHzh9AYIeJ0bGUnmv6RqMob3eA4+TKf6Vysm+3cRG4Qc4f31znQvP7x8AAHQ0+VUVP1GhhPFUtqLP26H4x1PhhLUCLiJeenMIY6mM+tvUuR1FLRQegbJqDvtNp4JFwBGOp7H50LB63wzxVBa3PbwLjX4XMjmK5/YNVHUMe3pH0RNO4IJlLWgPeafE9zNzCDxPgSuRKLkcs1SsdHq32QGbUk+kQgLnKgYAIhZ88Kd2a2Fd4nstwVXHbJTmpeW9rwD6R5NoDbDGBw0+J0bG8k+UrsEYOhTiBYCFzX64HTbThcxuQYEDQFu9V31dR7MPHU1sOxMVSsgJ/GCZn8cJsCecmBY+OCfwVDaHF/YPYnt3BG6HDavmBBEppsAV8cEJfCpYBBzP7h1QrZNCNtD/27gfPeEE/nv9WgTcDjy1u7q1dp5UztXzlrSird6D7inw/cwgAo+yWGzucXMFzqNTXBYIHGCRKO5gxZ7ykEDCo8nSBP6kcJANlUvghODm2b/Ag8F/Ku99BdAXTaBFIfB6nwsjJid714CewB12G5a1BU0VeI+iWLl90h7yqGvLCxr9aK5zoc7tmLCFzHiaEXhXmZ/HFfhYKlt0lhRPZfGh37yMvb1R0+dve3gX/vjSIcufe/uLB/Hlv7xR9kVDFAJP7u7D9u4wlrUF0eh3FR0/P/5WtrOchkpnHK8fGcGHfvMyEmnNQ39iZy8+f9dW3etuf/Egzvvekzjve0/i3T99tuD3xveDw2wWMRRL4X+e3o93rW7DmSc045wlzXhqd/Eks3Lx1O4+LJsdwOx6D9pDXoyMpTGWsh6sUAvMHAJPKQuV7jpGwFyBcwK3osABZrFU2pAB+pOnlIUST2Xx4puDOGMhi0fnU1irGE1mcMfOLL7y4D5Tu6McUMrqdesVuH6b4bE0hsfS6GjS95dc2R7Eju5I3snSHY6juc6l2i1t9VyJe+B12UEIwYIm34TFgo9xC6VsBR6B084Wp4tNmw8OxbBxTz+e2Ws+dd+w+TAe2W7ScKQA7t50GHe+fAiP7yyefGMEP47OWNiEJ3f1Y0d3BCvbg6j3OovOCodiKbgcNjT5XWj0u9QZVLl46I1j2LinX7dm8JetR/HXV7uRymjrQk/u6kMknsbaeSEcHIzhP+7bbkq4uRzF03v6sbiVncNmF6FdxyJIpHN436nzAQDnL2nFsUhCXbwdL6KJNDZ1DeP8pSyKTYuqmlwbZeYQuJisE5itKXCxkJUVOMZH4CIJlwolfOHNAaQyOVx5MvOwy7VQuMIdGUvjx4/vLXOkekSTGSTSOVWBh7z5Fgr3jrn1wbGyPYhIIpO3GNk9klBJG9AOevH9Hc3+CYsFT6gK3PrnJdJZ7OsfxenKRbaYrcB/bzPlyiNyrF6k00LkyK1/21FWRMhgLAWX3YZ3rW7D0ZE4IokMVrYHEfQ4i1sosRSa/C4QQtAe8lSswPlsbLtA4DtM4tBHxlJY2V6PH1+7Fl98x1K88OYg/r69N297O3oi6I8mccVadp6Y7QMn0jmKXXfeUpZxLSr38eC5fQPI5CguULbLo6om22aaOQQupssHZgMRbqEoV2DLCtw3roJUQ7EUvIriLKXAn9rdD6/TjotPZL69uABqBVwhrWwP4vYXDxadgpYCzxxtDTCSDflcGDYo8C5DCCFHoYVMnoXJwb3wjmZNwXc2+XFkOF5RaF+54NPdcj5v97EosjmKty1n1lwxUuO/d4+JcuURHVZ/4/39o+zivnYOugbH8Pvnuyy9j39Go9+FC5a1qo+tbK9H0OtEMpPTWRu698XY+wA2W6okG5NSqpI1X+8YTWbUWY84qxsZSyPkY2tO60+dj6WzAvj2QzvzLlZ8nejS1ew8MVPgRrtuVtCDle3BqvngT+7qR8DjwMkLWOQXP5YnO2N15hC4WLAq0K5ZKMkyPfC2k4D2NRUPY3A0hQWKxVBKgT+5uw9nndCkxpVWqsBvu3I1fC47bn1oZ8n3/H37MTxvsjrPU6xVBe5zIprI6GK0uwZjIASY16i3UJbNDsBuI3kLmTwLk4OrcaMCz+YoDlcpLfmRbT14Yme+igOAuBLWmS0jdJHv03lLWuC0k6K2Av+9zeqIcOIfjKUs+bLblfoynzp/Ed66rBU/fWJfweQqIzgRzwl5sWRWHew2gmWzAwh6GVkWikQZFAi8vd6aAh9LZfDDR3erF8feSFI9jvl3t7NHOy7EdZWRuEbgDrsNX710BQ4NjeG3z3XpPuPJ3f1YPbce8xt9sNuI6fi7w3E0+TW7DgDOX9qCzQeHC/r+v3n2APb1jZo+J4JSiqf29OHcxS1w2hllzgp6QMjk14yZGQTOI01cBguF98kErCvwK/8f8I5bKx7KUCylElwxAg/H0zg8FMepnSwUsLHOVfYiZvdIHDYCLGsLYP2p87FxT3/JWPJvP7QT//3kvrzH+0e5AtcsFEAfSdM1EEN7vVd3kgCAx2lHR5MPe4QZwHAshdFkBnMbNAJfNjuA0xc2qtNbAOhU1Hi1bJTv/X03brjrVdNaNPFURp1iW4182d4dRsDjwIImH2aXIDX+e5tNqznxJzM51Ysv/rkReJw2LGypw41vW4JoMoONe62pycFYCk11jIg/elYn/umUufA47Qh6WHmHQlEcQ7EkmjiBh7yIJjMlZ5Eb9wzgJ//Yh4feOKaMm83C1i1owN6+KFKZnM5K4bZcLkcxMpZCyOtSnzt7cTPOXNSEP79yWHh9ClsPDeP8JS0ghDAf32T83SN6sQAAp3Y2IZuj2NWTHyHVH03imw/usDSz2dkTRW8kqTtuXQ4bWurc0kKpCowkHWgDcmkg1q8pcLFrfA0xFEuhJeCG32UvSuCcCOaEGIE1+isg8HACrQEPnHYbWgNu5Gjxi0Y6m8OR4bjp5/QpldW4Am9QTmTRRjkwOKazP0R0NvvVJB/22ny/3O924K7rzsCy2dpvsUANJRy/AqeU1RcfTWbwfZOsyXg6i+Vt7LOtLmRu745gRVsQhJCStgL/7vuiyTyLpkcgfiu/8/buMJbNDsJuI1jeFmChmkWqPooQrZD1p87HbVetBgBVgRdSpMx6Yb8/r11jZgeJ4ATGvWauuq8+ZS7SWYo9vVFs747AbmOLwPx4iiYzyFGoCpzjohWz8OZATC3ZsHHvAHIUOF+xg4Ieh7mFEtbbdYDmh5vtg+rTFykDwfGUUsHx/CUtusfbpkAs+MwicK7AZ61gt8de1xoaW7VQxoFcjmJ4jC0EBTxOjCYLqxd+4LcpC3tNflfZFkr3SFx9f4OPnbAj8cLbODzEEobMPqd/NAmX3YZ65STnt+JCZtdALG8Bk6OjiS1G8hkAV7gdzeav52jyuxBwO6qiwMPxNOLpLJrrXPjz5sPYZsicHEtlMbfBi4DHoRJEMWRzFLuORbBCCatrr/cUnTJztUppfhajGDNc6nemlGJHT0QN53PYbVg2O1C0aJgIkcBF1BexUBLpLGKprKrc5yjHVSmLgAuRZ/b0I5PNYXt3GJ3NfnVmuaM7gu3dEayey9ZJwsrxxG9DPv04eZQH966f2t2HBp8TJ80NAWAXIVMLxUSBq5EiJipZs3eiJeumPLWrHyvbg2gNGi8Qkx8LbonACSE3EkK2E0K2EULuJIR4CCGNhJDHCCF7ldsJ7CVmgKqymQeeaD6R/d+9tWwL5c3+0Yp9rZF4GjnK1HSdx1FCgetXzZkCLy+MUPSYuZIZNkm+4eDx1sMmPmx/JImWgBtEqdDIT6ywckEYGUshHE/nLWByLGj2I5nJoTfK9qtrcAw2wupqFAMhBB3N/oKK+NXDIxi2eGHj3+mX3rEMDT4XvvnADt1+JtJZeF12dDT5cUD5Lo6FE3lEv69vFH/edBj/+8ybSKRz6iJte8iL3ojW4/PVwyM6NSjG/RtVX/dIAg5FhZr9zkOxFLYomYaHh+KIJjLq5wLAivZ6bO8Ol/TPk5ksRpMZ1QoREfQoBG6iYPmsQFzEBEov0nFrKJLIYOvhETZjaQ+io8kPv8uOrYdHsLcvilM7G+GwEVWB81tu1XF0NLMM3Sd397Hwwd39OHdJi6rgzSJpIok0RpMZlbA5fC4HQj6n6T7whdZ4Olt0NsYzQC9Y2pr3HJ+RGX+TF/YPqkljAKtF8+tnDyBmIS+kXJQkcELIHACfA7COUroKgB3AtQBuAvAEpXQxgCeU/ycHPNJEUdl/2RHFm7k2pA5tYeRObPpSsUVw459exTcfqKzZLj8xG/0uBEoSeBwOG0FznVt5jxtDFhe4AG4XxNFez6NGuGIuTHb8QM3kaJ6P2D+aVO0TgMWBA8BwjJ0snPznNxawUBRlzj+jayCG9pAXbkfphKiOAlUJI4k03vs/L+B7j1orIsXV4NLZAXz6/EV4uWtIrdmczuaQzlJ4nXYWujgQQzZH8fHfv4L3/+9LusXaz965FV/a8Dr+8+FdsBHgFCXyoC3kRSZHMTCaxMBoElf94nn84UWtg3s0kVErFBu98u6ROBbPYgJj0CQS5fuP7sbVv3geu45F1Gk9V+D8vlmophEaEbvznuMljs1iwY0E3hpwo87twD92FQ/D6x6JY9UcZvXc/2o3jgzHsbI9CJuNYHlbEA+90YN0lmJVez1CPqe6iMlvG/zOvG2ev7QFL+wfxKaDwxiMpXC+4D3Xe515Fgr/rsWQVY62evOaJXymwO8Xwl0vH0I2R3HhcjMC9yCezupmqX2RBNb/6kV880GNQ+585TC+9eAOPFvl1H7AuoXiAOAlhDgA+AB0A7gcwO+V538P4Iqqj84qknqVfSwcx2t0IWzHXtXCC421vwugJ5wwPcGsgL+vye9GwONEtMgVtyecwOx6j6osmvwupLO06HtEDMVSSGZyggLnirmwAhdtikGDCuyL6AmcLy7xE0317BvMFTX3xrmX3TUYK6jW897b5MOR4TFdkgcAPLd3AKlsDk/u6rN0YRNtKe6t8wsVz8L0uezoVD7vzpcPYXt3BOF4GluVhgc94Th29kTwubeegGf/7QJs/srb1f0QbYWnd/cjm6O6xdJoIq1aTMapdU84gVUKIRs9cEopntzVhxwFvvXgDtUzXjpby13gZF7KRuHHoJmFUkyBc1uHK3eH3YZPnb8Ij+/sLVpTpGckgWWzgzhlfgP+tOmwMtZ6dcz8eOSJRNw64UKj3ps/zvOXtiKZyeG2h3eCEODcxRqBB72OvAsQV9hGCwXgNodegUcTaXQNjuGyk9rhsttUNW5EfzSJn/5jH966rBVr5+cbDHz2LP7Wh4fZ8X/XK4exvTuM8FgaP3x0N05f2IiLVlSn5LOIkgROKT0K4PsADgHoARCmlD4KYBaltEd5TQ+A/EvURMHggQ/GUngjtxCO0W5gcL9l/5tS5mGXU8tbhKhiAm5H0RX8oyNxNRmAvwewHifMp+hcdfCpaDG7QZwqGklErIMCAAGPAzainWhqXRMTlcMfdzls6BqMgVKKA0X8ciM6mvzIUe3g5+ALYz3hBPb0lg73OjqSgNNO0Oxn6hEAoso6BJ/SehQFnqPArX/biVVzgnDYCJ5UlCb3Xi89qR1zG3zqYi6gtxWe2sNeJy7yRhIZtAbcqPfqp+18ir94Vh1cDlved8+LJK2ZF8Jz+wZx1yuHcEJLnS7aZ9nsIGwERWuvA9rvyr1sER6nHW6HrYCFos0eOT5+difmNXrxzQd2mJb8TWdz6Ism0F7vwfnLWtQLML/YcCL3K7aVmFvAVWuDL1+Bn9bZCI/Thi2HRrB6bghNddpxaWahcMvTaKEA5gp8Zw+bsZ80rx5LZtcVvCj+4NHdSKSzuPldy02f5wu94kImv++02fDNB3bgx0/sQTiexn9culK1J6sJKxZKA5ja7gTQDsBPCPmA1Q8ghFxHCNlECNnU31/d4jIqeO1vxQMfiqXweq6TPXboBcBdB0ppyboFkUSGqeAKy1AOCidPKQulJ6wtQAIsjFDcRikcVaNY2EGkLjoWUeAHB8ewUFGT4uekMjk1eobDZmMhW/xE6x6Jw+O05UUNiK9f0OhD10AMw2OsoUWpBUwO/joxtI+V7uzHOsW+EDPqYknzWussEsELm42oBB5LMuLmBO5z2VV1Hk9ncesVJ+KUBQ26RbP2eo+ati2CX7wOD49ho0Lg4vR5NJFBwONQKtVppKFWZQx5TRer+b799/vW4oTWOgyMpnT2CQB4XXYsailMNhxGK8QI4yIg9+3F2SOHx2nHv1+8HLt7o7hTCO3j6I0kkKNsv85fwvTbrKBbtQX54u/yNmapiAXS+G29N/948jjtOHNRMwComY/i+I3JSD3hOOw2oiahiWgLeRCOp3X+s2ZR1WNlm/nawrajYfxp02F8+MwOLGoxF4D8giGGEvL7N759CV46MITfPteFa94yX/0uqg0rFsrbAByglPZTStMA/gLgTAC9hJA2AFBuTc0ySukvKaXrKKXrWlpazF4yfiT1HvhgLIXttAOU2FQL5VfPvIlzvvNk0WQIfvBbtTEKvb/BxwncnExzOYpjhiQXPnW1GkrIw9L4RcBhtyHgcZhWEAQYSR8ZHlOnguLncDvFeAKEhIJWrLOOt6iKWKBEonClb6yZUgjcohCLTO3oiaAvmsR73zIPy9uCqkLe0xvFyd96DH83qSnSM5JQQ8n8bqZeeSQQj732Ou1Y1OKHjQBXnTwXJ80L4fylrdjRE8GR4TE8u3cA5y9rNd3PoNcBv8uOh9/oQTiehsNGdGsO0WQaAY+TJcEI03auxtvqvabhok/t7sPytiDmNvjw1UtZBBWP2hCxsj2IbSUUuNEKMUL0kF96cxAnfeNRvPTmIIZiKThsJK8V4DtXzcZpnY34r8f35kVrqLPAkBfL2wJor/dgtRItAgBLZgXgcdrUx+q9LvX7Gh5LIeB2wGE3p6C3KmGDb12mn9ibJSP1jCQwO6jZkSK0UEKNZLd3R9Bc50JrwI2Vc4IYHkvnLTr//Kl9CHmd+NyFi03HBwDNfjdcdpuuvnz3SAIBtwOfOKcTy2YHEHA78IWLlhTcxnhhhcAPATidEOIj7Ki+EMBOAPcD+LDymg8DuK82Q7QAQ6TJUCyFODyI15+gPt49ksBgLIUfPran4Gb4NHI0mamo9dZQLIWAxwGXw4aAx4lEOmeasj0wmkQ6S9UFSECwUCxGonSHE2rhIY6QSQEqjsPDY8hR4OQFIXWsHMYsTLPtHR1JFLRPODqbfTg4OIY3+9nvYVWBN/icCHgcOgXOFfH5S1rUjLpIIo1vPbgDyUzOdDHv6EhcPWHrlKSVUa7AFcXmddkR8rlw97+ciVuuWAUAuGAZExY/eHQPYqlsXrwvByEEbSEvXjsSht1GcM7iZt2MJ6oo8DZDHRFxit9oUOBakST2mectacGGfzkD1ypFmUSsbK9Hb4QtoBbCUCwJu42ofrcRQY9DXRd4/UgY2RzFNx7YgYHRJBqUOijGfX7fafMxMJrE64bm1VougweEENz+z6fhW5evUp93OWz4y6fOwg0KCYqLmOF4GiGTBUyOa98yD3/+5Bm6CwIfP6D38Y+OxE3tE0CzvUSbg0XK1IMQUnBtYeuhEZy7pMV0hsBhsxHMbfTqFuB5aK/DbsMd/3wa7v/s2eqMpBaw4oG/BGADgC0A3lDe80sAtwF4OyFkL4C3K/9PDgyRJpycIo1KOKEroHYkufPlQ7rUXhF8GkkpEKugTCQvBgRAncKPmtgoxjrZgDZ1tWqh8AgU8YRrKFACFtB6WS6bHYTPZdct1Gp1UAwELlgoPUVOEo4OJZTwxTeHWAhhgzUFTghhiUCDIoH3qbG3FyxtRSZH8c0HdqiV/ozZjNkcRW8koc5INAtFWcQUFDjAIku8LnZ/6awAZgc9uHfrUTjtBGed0FxwrPw3O3l+CAua/Or3QynFaCKDOjezUMRpuzjFbzKEi2pFkjSlua6jMS/bFbC2kDkUS6HB54LNRI0CeguFf987eiJ48PWegqr93MUtsBF96WNAI0VOkota6tRaJBwr2oOoV2y3Bp9T6buZzcvCNMJht6mx5CLq1WQkfcimWQQKkG9zJDNZ7O2Nqt/lstlBEKKPRBmKpdATTuTZWGbobNIft92KjQcATXVuywv5lcJSFAql9GuU0mWU0lWU0g9SSpOU0kFK6YWU0sXK7VBNR1oMvOs8IcgqyTQAMFy/kj3vrkMyk0Wj34Wg14lvPbjD1EMtt5Z3/vuTqpIOKErBzAc3C3vyuuzwOu1lLWIaD9p6r7NgHPgBJTqks9mfF3PeFzVX4A3KolMqk0P/aLLgScLBQwmf3tOHuQ0+uBzW88Q6mrRY8PBYGlsOjaikdvL8EAIeBzZsPoJFLX447SSPwPujSWRyVB2j12mHjWgXUC0KRW8RAOwCwlX4qZ2N8LvzX8PBZ03nL21VyrOm1fZ5mRxVLBT9tL1bmOI3+t2631gtkjQ/VPI7WqESeGEbZXA0VZCIAb2F0jUYw0lz67FuQQPGUtmCvnmD34U180J42lDZryccR9DjKPp96T6bR0opZYkLracUg9FCyeVo3nqSCK1mCbvY7O0dRSZHVXL2ux3obPbrLoqiR14KvJomn7H3mCQU1RIzIxNTKCU7MpZSmwYMBBUCd9Uhmc6hwefEjW9bguf3D5rWbNZPbStQ4EIqckCZwkZNsjG7C6yal5NO3z0SzztQQj4XwgUslK6BGAIeBxp8zryFNK7AjVO9eh8L++qNJECp+Sq/iAWK2hgQCnpZRUezH90jcSQzWTyxqxfZHFVtBYfdhnMWM1X81UtXwO92IG6YIfFQLm6hEELgdzvUCzFfwPa6zA/585RFOL4YVwj8Oz9/aQtCPicoZdN5vt7BFzEBTaF2C7OXpjoXYqksEuksKGV1rs9d3FLQCxYR8rECVVsOjhR8TaEsTA4xiqNrYAydzX587d0rQUjhhU8AuGBpK147EtbZN2bHYDHwiJOReJpZKL7Cn1ds/IBmoQzEmB05p8A4eJkJvmbEk7ZEcl7ZXo9tR7WFTE7mVhR4R7MfiTRLYEuksxiMpXTWaK0xMwicN3OAXkUP+BczW6WuFclMFh6nHVefMhcAmzYaMWTwJsvFkGChFFfgCfhc9jx/ranOWjp9JptDbySRR6gNgsdoBI/LJoTkXSj6ogk0+l15ijnkdSGazKjJMKVO1ragB25lG+VOHTuafMhRYH9fDD96fA+WzKrTxd5+7sLF+MZlK3H+0lb4nHbEDApcXSgUvpM6t0O1MRKqB26uFt+6rBVfePsSvHfdvKLjvPLkOfjqpSuwoi2oli8YHkupC98Bj0NdSOUKXJwtNQqL1b2RJI5FEnhLh/Uk5svWtOPxnb14TYlbN2IollIjmszA46gT6Sy6w3F0NPtx4tx6fP/qk/DxszsLvo+nuPPoG8A8fb0Y1NyCsbRioZSvwNVyAGp+gt7GMUN7yKte4J/dN4CWgBsLhIS0Uzsa0BNOqDPA7d0RzAl5LV1g+EJ918CYuhAqFXi5EBS4SIBx6gQ+8Q/g9E8hmcnB7bDB72aRBHzhToRIalb6WYrgMeT85ClG4LzwjnHByKoC740m1fAtESFlemxW26FrUIvL5lmfHP3RJFpMFlp4lhxfMyhlodhsRFXeVmPAOfiC59fv347DQ3H8x6UrdVEFy2YH8eEzOwAwuyluIHAxVI+jTqfA9R64ES6HDZ+9cLHq1xbC3AYfPn52J6uMJyhK/jsHPA7Mrtem7XyKz8clErg6VZ9jvYHI9RecgOY6N75ZwAYU12HMUO91Ipuj2NkTAaXahfaqU+aaJqtwrGwPornOrauvzfbLutrklslQLIlwPG0aA14K/Lzi52dPkRhwjnYl5T2TzWHjnn6cv6RFt0bAL07c49/eHbYc9qf2dR2M5UWGTQRmBoELzRxEYkpmskDrcsBTj2Q6p6Z1twTcavlUEYOxFHzKwpbZ4mMx8BhyTYErFopp4R3zqadVAlcPFMNULeRzgdL8z0xlcjg6HFfVAlf6nAD6R5NoDeYTOFc7fLZi5WTlB3S5Cpz75y93DeFty2fh7MWFFxL9bkdeTH93OI46t0MXfSFaKGImZrXAFWR4TLRQnLppO5/iqxaKX4v3394dASFQKyRaQZ3bgS+9Yyk2HxzG/a91655LZ3MIx9MlLRSARaAA1i+0NhvBeUtasHEvy0CNp7IYHkuXvKiL4AR+eCiOHNU88XLAk5G4j18qwQxg58nRkTi2Hh5BJJFRCZtjXqMPi1r8eGp3H2JK8wkr9gnABIPLbkPXQCwvN2MiYG31YaojOQr4mV86qCPwnHA/q2bVtQY8avlUEUOxJBY0+bGzJ2LZA39+/wDueOmQqgiNi5hmi6Hd4YSupCoH86b1F5ZwPI2fPLEXn3urpg6PmqhNQKyHovcXDw2xEEKuchv9LqQyOcRSWdS5HeiLJNHZmX8ic4tgR3cEIZ/TdAHQCE7cVkMI1c/yu1DvdWIslcFXCmS+cXhNLBRj9x9Ar8DjqSwIgWrxVAOihcKjRnj0S1u9F0/v6cexiH6KL4aLbu8Oo6PJr77HKq4+ZS7+78UufOvBHXh0ey8IAT5yZgfm8wt0MQJXLjqvKSGB5cyULljWgnu2HMHWQ8PquVQOWfFjkpcarkSBA1BqgmsJZl6nveiCaHvIi2Qmh79sOQq7jZiKgwuWtuL/XjiIrYdGQKm1BUwAsNsI5it9XflirjESp5aYIQpc8MCFFX6xtkYinVNP3paA27Tg/9BoSvXGrHjg4Xgan/njVjy/bwCHhsawak5QnYaqqdyGC0Eyk0V/NGk6zWr0u5FI53Tq8oX9A/j1swfww8dYQSdKKf7w4kE0+V15haW0ioR6Fc9T1PnrxbT9RDqLY5GEaY0Tvr19faOWldZFK2fjkhNnY16BminFcO2p83DTxctLkr/PxEIxdv8B9B54PJWF12mvajqzeMEcFSwUAHj3Se0Iep3oCSdw0rwQTprHCEENFx1NqZX7yoXNRvCf71mN9pAXu3ujeGbvAD7zx61qbLxZISsOPqt6/UgYDT5nSctIxLlLWhDwOPDTf+wTkpOsk5XfZYfDRtR4/0qiUAB9KOT+/lEsaPIV/V357Of+V4/ilPkNprHd5y9tRSqbw6+eeROAtQVMjo4mVoyte4Q18bZSwK1amDkKXE3iSSLocSCmxJuqL8lkdRbKxj16AqeU1cme2+Bl4WcWwgh/8sReDI+l8MBnzsYqg4/pcdrhstvyahf3htnnmlko6vR6NAVfI/tp+IXmDy8dwvtPX4A9vVG80jWMb7/nxLxY4ZBaE1z/mcZiP9o0PomhsRSyQliVbnvKolMmR9VCTqVwyoIGnLLgFEuvNeLLFxdX3hw+lwNjKX3dlO6ReN4++N0ONZV+LJ0t6H9XiqDHCaLUi8kpdhS3zj5+dqfpomDQ62AkNhjDkeE43ndafsKOFZw4tx73f+ZsAMDmg0O46hcv4LaHdgEoHk3CLZT9/aNYMy9U1mcGPU7ccOFi3PK3naoVVc6CHSEEIZ9LTXwxK2RlbRxaU4ft3RE1QqkQ+BhjqSzOX2aepPWWzgb4XHY8vacfDT5nWRemjiYfntnbj9age0IXMIEZo8BHdWn0TXVuuB02JNOihZKDx6kp8Ggyo1NxLMEgh6Y6VgiplIWyv38Uv3++C9esm5dH3hwBjyPPS+er4WaenbjAxdEXTcJGmHr52n3b8Z8P7cLytiCueUt+tAT3ZI3ZmLz1Gk/UET/HLKxK3Z6QKVeO11lr+Fx2XRx4MsM6vhu/U7GcQSKVVRN3qgW1XoywiFnKDiGEoMHvwrNKGKvVqXoxnLKgEZed1I6Xu1gqhlkhKw6eKk+ptu5QDj50RgcWNvvx8LZjIITFWZeDkM+pngOVWihBpa1aXzSB/mgSK0qsIYjHrlldbwBwO+xqAtdKJUvTKngC22uHR8oi/mpg+hL4328G7lwPZJJANqVLo2/0u+B22JDK6gmcK3BOZKKNolZxU7rpFGr8ynHr33bC47TjCxctLfgas4JW3UVWqnkEizFCpKnOjRvfvgQvvDmIoyNx/MelK0zrPqgK3JDM0x2OY3bQo8Yai1mf27sjCHocut6V6vjdDvVzJlpZFIORwI8JNTlE+N3MK2eFzKqvwAEtW3U0mYHfZTf9XYxo8rvUui/lTNWL4aaLl6kCpZgCF+2DctcpABat85VL2Uyppc5dVrIWwL4vHjxTSRw4oCUj7VDjtYtfBJv8LrjsNswOerBMKNFrBM87KPc34es+kURGKnDLOLoZ2P0QsPMB9r9Lq0TIY5p1Cjyd1XngAIt/5uCLn1aaMTy9px//2NWHz771hLzsRRF1JgWtdvdG4bAR08UfHsonXlj6lBC/D5y+ACfNC+Gqk+fijEVNpp9n1gYNUIo8CZ8nXih2dIcLKg5CiKrqywkXqzW8Lodu9tSrhITOMkTS+N0ONUsyns5WNQKFg5dIjSbSqn1SCpxgxcp940V7yIsbLlyCWUG3urhqBnGGUG6yFccFS1tx0YpZpgW3SkEk7WJ1RoohqAgsnnBTah3BZiM4aV49LlvTXlRZX7hsFurcjqKlFMwgXghL1QuqNqavB55Q0okf/wa7FeLA18wLwe2wGzzwHNyKQuFV9/QKXKmHrJSCLRRGmM7m8K0Hd2BBkw8fOauj6BADbmeel/707n6s62gwrXXRGnSDEH07rv4oC/Fz2m34y6fORDGBx4oYOfItlHBcVxTI77LD5bChL5LErmNRfPD0BQW3We9zsuyyKaTA/S47UllWKMxpt6kXSWMBp4BbiwSK18BCAZglMBRLoc7tUBcwS4ETeDXsExGfOn8Rrjt3YdFZgMNuU6NzKq3TQQjB/3zgFKs9UnTgC5dBj8PSbMUMQa8DkXga27vDmNfotXQh+PMnzyj5mtn1Hrz+tYsK1pEphLagBy6HDalMbkJjwIHprMATEcBdD4QPsf9drOb3sGCh8DDCTJbVqRAXMQGtBggg1kN2Kd10zC2UO148iH19o7j5kuUlV5uNSr4nHMeuY9GiPlxznVtXya4vmlCVud1GSnpzDX59QSuWSJLQpfcSQtDkd2HTwSEkMzmsnFNYwXAFPtHeXjFwIuY2StQQAcLhFwpaxWuwiAlo9WJGkxm1AmIpNKkEXv0a0VZIkRNeJRYKh83CsWgG7ntXap8AbPw5Crx8YBgr26xdBAmxNt5yyZu/h0evSQvFKpIRYPU/ASFFPbrrEIlnkMlRRuBOmxpGyImcWyiNfhfsNmLqgTf6XQUXMYdjKfzo8b0464QmvN1CeyRjY2OexXbBMnMCB6DUkmYEnstRDIymTJNsCkGsIAiwGUkqk8sj4Ea/C28UWcDkaPC5KlqsqiV4PDq3UbQ0dr0S8wsKfCyVsRTHXi5404tIIlOGhcJ+z1oQuBUEPA40+V0FS87WEpy4K13ABLSZ1sBoctK+QyP4xXCiLZTpSeC5LCNwXxPeXPVZAMDuWJ2aBNOoLFokDQTObQu7jSlQ0QMfiqXgUqaXhSyUHz++B9FEGl+9dIWlq3nQsBharNsLh9jNhYf4maW5F0K9z6WzUHg9DqMyaPSzrE23w6Z26THD7HoP5jf64LRQbGmiwJs18Hh5sZCUCNFCSaRzprbVeBHyORFNZBAeS1m2UOYooaonGmpdTxRm13uwZFbhxbxagqv/SrIwOYKCZVJs9jiRWDY7AL/LXnRNrBaYnh4478DjqcfDiQuwIfkDXNwTwlvrNRUteuD8VszCaw3qk3kGFeuFEMIsFAOB7+mN4g8vHcL7TptvmkVphoCHeY2UUqSzFM/uHcDla+cUJX+ewUep1jC3tQz12+BzqrW/Aa3Yj5HA+TR+WVuwaCW8L75jKf7lvEWWP38iwK0Q0UJx2klelqVooTAFXhsLBWDf8+kLzReXjbjspHasbA9OaMq1iO9etRrltyupDhqqoMBFz7va6wiV4l/OW4Qr1s6p2NevFFNHVpWDpFJJ0B3E9u4wDtA2PLW7X2gn5YbbKShwJRqFL2ICLOJD9MC5dw4w4k1ltb57lFJ868Ed8Lvs+P/eXjhs0IiAx6E0h8hi08Ghot1eONpDHoylsgjH0wXrdBdDyOvUNTY2K/IEWJ/Gh3wuzGusLFqhVuBWiEbgLALEeGEULZR4unaLmACQyuYsK3CXw1ZW/ZNqozXomTRLjH9flVQi5OAWCm+LNhXgdzsK9s6sJaYngfMIFE+9WhBoR08Eu5Ru0411Shy4QuAJVYFrJ3BrwJOnwJsKVBL8x64+PLN3ADe8bUnRGFsj6txaQaundveX7PYCaETbPZIo2CmnGOp9LkQSGbUiYU84DrfDlqd4+L5OFQ+xHPgMFgpvJmyEWLkukc7VZBFTVINWPfDjGdWxUNjvuqLMhJuZiGlK4EyBx2ysBsElJ7YBAO7degSAErjvsOcrcGGK3RJwY2A0qRLdkEGBA1o6/Xcf2Y2FLX586IzC4XZm4AfahT94Gr9+9gBO62wq2b1ErCXNPfpyFDgnarFeslkz4ua62oSyTQR8JlEoZhmQ/LseUC6EtVDgYsx1uUWpjkfwc6xxHBYKL/EwHcVHtTE9jzhFgR+IMEK+6uQ52NQ1hK7BMfhcdrXkZDLNPXD9IibAPPAcZcTdEnDrCVxQzmOpDHb3RvGvFy0peyHvnMUtuP6CRUgoF5DL17SXfM8cVYHH0R9Nos7tKCt6Qixo1eB3obtAzeZ3rmpDMpPD6jJqUU8V+JxGC8VcgfucdhACtYtMbRJ5RAU+PU+niUR7yItvv+dEvHPV7Iq3Ue9z4rtXrcZ5S4vbkccDpucRp3jgO0cYoa5qr8cFS1tx1yuHVRIWU+nNFjF5ZEdfNIGgly02qg2JBQulS+klWUnMbL3XiS++Y1lZ72muc8NpJ+gOJ1gWZpken7GgVc9IwrR8Zr3XiQ+d0VHWtqcKuJLmbdWiyYxpKQCbjcDvcqhWVG2iUDQFLi0Ua6i0gJeI95rUAjoeMa0tlDcGGOG1Bj1qHYMmlcDtqnWiWSh6BQ6wTEctBpz3sxQIXInoKLfDTKWw2Qhm13uYAo9UQOBCk4F0lvXqm0pZlNUADyOMiYuYBewLv9uuNu+ohQIPuB1qdmxQKnCJCcY0JXBmoWzpzag+2FknNMNhI6oCdzny48D1USjMVuiLJjEQ1cIPAW2VO5pIawQ+jqy1ctGmtIDqH61cgQ/GUloz4imURVkNeBz5Hngh+6LOrSnwWixi8oqEACxnYkpIVAvTk8CTYVCHF7v6tUysgMeJ6y84AVesnQNAs1ByOaqGAxoXMQGmwP/w4kE47QSrlKQAsRlD10AMzUqJ2YnCHKUJa18kUXaYVHvIg+Y6N/740kEcHeaVD2eWArfZCLxOO+IpFmM/miycBakj8BoocEBbyJQWisREY3pKhkQYGWcdMjmqi6K48e1L1PtcbaeyOSGVXjuBvS47Am4Hnt7dj1cODuGfz+7E3AYW72z0wDubJzYOuk2xUHK0vAgUgO3jl96xFF+653X8ciPrLmK1GcN0Ai8VG09nkc3Rggrc73aov38tFDgAtauNXMSUmGhMTwWeiGDMxoLmC4USuZSIkWQmpy5iepz63W0JuvFy1xAafS589sLF6uNOuw1epx2jyTQOCN3cJwptIS94Y3leObEcXH3KXKyaE8QTu/rY9qZQM4ZqgXemVxspFLFQOGpRCwXQ1h1kGKHERGN6Engyggj1os7tyOsLyeFW1FYykzVV4IAWifKFi5bmFfap8zhwLJJEfzQ5of43oFfMldRWsNkI/uPSlQBYtEmp2PPpCJ+TdaYXu8GbQSTVWinwBh+rvVOLKBcJiWKYlmd2dmwER+JOnLygoWD5R+53J9M5NQrF2D1k7fwGZHPUtD1ZwOPAdqVa34QrcEExV5oqfGpnI967bq5aJnemwedmXXkKlZLlEJV5rTzw1XPr1YbCEhITiWlJ4CPDgxjMtOELgudtBCfwVDaHRCYLp53kFZq56eJloJSapuMGPE68dngEANAxwR64GPY3nupm37lq9YxNNeZt1VQCLxhGWHsC/8hZnfjIWfkNjCUkao1pZ6EcHIwhGw+jqakZJxXpqm1U4IWaLxQiODGmd6IVeNDjUPsrNo6jZsRMJW8A8DodBgU+eRaKhMRkYdoR+Lcf2okgYjhxUfG6JJywmQeezSs1Wgr8xG8NuCfcQyaEoD3kRXOdq6IOIccD/G47xlIZjCbNa4Fz8N/R5bBNeKlPCYlaY1pZKIeHxvDk9iPweNJAfUPR16oWSoaFEZa7wMQJYaLVN8eiljoMjc1M/7oayLNQioQR8tdLSMw0TCsCPzoSRwCsNgncxYsw8TjwpELg5SpwPiWfaP+b47arTkQmN1ll96c+vE7WmT6SyIAQwF8gRLBOSbuX9onETMS0IvC+aBIBohC4pziBu+zcQskhmc7mRaCUAp96T3QIIcd4mr4eD/C57IgpYYR1LkdBq4nXZK/VAqaExGRiWnng/dEkAlDCtTzFawFrCjyLRCanxoVbBZ+Sd06ShSJRHD63HZQCA6PFe1H6pQKXmMEoqcAJIUsB/El4aCGA/wAQAvAJAP3K4/9OKX2o2gMU0RdNoMmuELi7BIGLHni6/EVMroAnS4FLFIdPIeTeSKJoEak66YFLzGCUJHBK6W4AawCAEGIHcBTAvQA+CuBHlNLv13KAIvqjSczxpoE0SlooWhQK88DLrVPBC84vmz053bslioOnxfdFEmiqKxwrz8ldZklKzESUa6FcCGA/pfRgLQZTCv3RJGa5lciMEhaKS40Dz1YUhVLnduDqU+bO6Fjq6QzeF7M3kixhoUgFLjFzUS6BXwvgTuH/zxBCXieE/IYQUjyurwrojyYx28X6RJZW4PpiVuVaKBJTG5yQ4+ls0SJSPDpFeuASMxGWWY0Q4gJwGYC7lYd+AWARmL3SA+AHBd53HSFkEyFkU39/v9lLLKMvmkSTMwmAAK7i1obeAy+ciSkxPeF1aqRdrA633Ubgc9nhrVElQgmJyUQ5svRiAFsopb0AQCntpZRmKaU5AL8CcKrZmyilv6SUrqOUrmtpqbwJaTqbY42HbWOAOwDYig/dYWeZd6oCd0oFPpPAo0uA0q3M/uW8Rbh0dVuthyQhMeEoR5ash2CfEELaKKU9yr/vAbCtmgMzgncWD9riJe0TDpfdxlLp0+Un8khMbYiedqkF6s8Jtd4lJGYSLBE4IcQH4O0APik8/F1CyBoAFECX4bmqg7fFCmCsZAghh9tpU1PppYUysyBaIrKRgsTxCktHPqV0DECT4bEP1mREBdAXUfoa5kZLRqBwuB02xNNZpLK5vG48EtMbfp0Cl70oJY5PTA9WGxuCrWsjAMCdiVm2UNwOOyLxjHpfYubAW4aFIiExUzE9CPyRm3DW5htQj1E401HLForLYUNEabklPfCZBZddKw9bLBNTQmImY3qw2lmfhzM7hs95HgZJhstQ4Da13KiMQplZIISo6fTGfqYSEscLpgerzVqBzf7z8D48DCTCZXngmgKXFspMA8/GlBaKxPGK6UHgAH7vXg83UgDNlWehxBmBy0XMmQdeD0VGoUgcr5g2rLZ1rBWb69/G/ilnETMhFzFnKnh6vIxCkTheMS0InFKK/tEkNi24Dgi0A60rLL3P7bAhq3S1kYuYMw9+tx1uh63sZh0SEjMF02LuGYlnkMrk4GxZBFy50/L7xCYOksBnHrwuh1TfEsc1pgWr9Y+yCoQtgcJ1n83gsmu7V25HHompj6DHgZBPErjE8YtpocB5Fma5BC6GDkoFPvNw49uXqIvUEhLHI6YFgfcrhaxaA56y3ieStuzIMvOwqKVusocgITGpmBaytGIF7pAeuISExMzFtGC1/tEk3A5bybrPRojRCZLAJSQkZhqmBavNa/DiopWzy+5PKZK2XMSUkJCYaZgWHvgHz+jAB8/oKPt9bqnAJSQkZjBmNKtx1W0jgMMmu8tLSEjMLMxsAlfiwD1Oe9n2i4SEhMRUx8wmcCUOXNonEhISMxEzmtk4cctCVhISEjMRM5zAGXHLZg4SEhIzETOa2VwOaaFISEjMXMxoZpMWioSExEzGDCdwRtyyG4+EhMRMxIxmNi0KRSpwCQmJmYcZTeC8Hrj0wCUkJGYiZjSzqQpcWigSEhIzEDOa2dQwQmmhSEhIzEDMcAKXFoqEhMTMxYxmNh4HLrvxSEhIzETMaAJ32AhsRN/YQUJCQmKmYFrUA68UhBDc/K4VOGNh02QPRUJCQqLqmNEEDgAfP7tzsocgISEhURNIb0FCQkJimkISuISEhMQ0RUkCJ4QsJYS8KvxFCCGfJ4Q0EkIeI4TsVW4bJmLAEhISEhIMJQmcUrqbUrqGUroGwCkAxgDcC+AmAE9QShcDeEL5X0JCQkJiglCuhXIhgP2U0oMALgfwe+Xx3wO4oorjkpCQkJAogXIJ/FoAdyr3Z1FKewBAuW2t5sAkJCQkJIrDMoETQlwALgNwdzkfQAi5jhCyiRCyqb+/v9zxSUhISEgUQDkK/GIAWyilvcr/vYSQNgBQbvvM3kQp/SWldB2ldF1LS8v4RishISEhoaKcRJ710OwTALgfwIcB3Kbc3ldqA5s3bx4ghBwsa4QamgEMVPjeqQK5D5OP6T5+QO7DVMBEj3+B2YOEUlrynYQQH4DDABZSSsPKY00A/gxgPoBDAP6JUjpUteHmj2ETpXRdrbY/EZD7MPmY7uMH5D5MBUyV8VtS4JTSMQBNhscGwaJSJCQkJCQmATITU0JCQmKaYjoR+C8newBVgNyHycd0Hz8g92EqYEqM35IHLiEhISEx9TCdFLiEhISEhIBpQeCEkHcSQnYTQvYRQqZ8zRVCyDxCyJOEkJ2EkO2EkBuUx6ddATBCiJ0QspUQ8qDy/7TaB0JIiBCygRCyS/k9zphO+0AIuVE5hrYRQu4khHim+vgJIb8hhPQRQrYJjxUcMyHky8q5vZsQ8o7JGbUeBfbhe8px9Doh5F5CSEh4blL2YcoTOCHEDuBnYIlEKwCsJ4SsmNxRlUQGwBcopcsBnA7gemXM07EA2A0Adgr/T7d9+C8Aj1BKlwE4CWxfpsU+EELmAPgcgHWU0lUA7GDlLKb6+H8H4J2Gx0zHrJwX1wJYqbzn58o5P9n4HfL34TEAqyilqwHsAfBlYHL3YcoTOIBTAeyjlL5JKU0BuAuskNaUBaW0h1K6RbkfBSONOZhmBcAIIXMBvAvA/woPT5t9IIQEAZwL4NcAQClNUUpHMI32ASzU10sIcQDwAejGFB8/pXQjAGNOSKExXw7gLkppklJ6AMA+sHN+UmG2D5TSRymlGeXfFwHMVe5P2j5MBwKfA5ZExHFEeWxagBDSAWAtgJcw/QqA/RjAlwDkhMem0z4sBNAP4LeKDfS/hBA/psk+UEqPAvg+WKJcD4AwpfRRTJPxG1BozNP1/P4YgIeV+5O2D9OBwInJY9MidIYQUgfgHgCfp5RGJns85YAQcimAPkrp5skeyzjgAHAygF9QStcCiGHq2Q0FofjElwPoBNAOwE8I+cDkjqrqmHbnNyHkZjCb9A7+kMnLJmQfpgOBHwEwT/h/Ltg0ckqDEOIEI+87KKV/UR62VABsiuAsAJcRQrrAbKu3EkL+gOm1D0cAHKGUvqT8vwGM0KfLPrwNwAFKaT+lNA3gLwDOxPQZv4hCY55W5zch5MMALgXwfqrFYE/aPkwHAn8FwGJCSKdS0vZasEJaUxaEEALmu+6klP5QeIoXAAMsFgCbLFBKv0wpnUsp7QD7zv9BKf0Aptc+HANwmBCyVHnoQgA7MH324RCA0wkhPuWYuhBsPWW6jF9EoTHfD+BaQoibENIJYDGAlydhfCVBCHkngH8DcJlSXoRj8vaBUjrl/wBcArbqux/AzZM9HgvjPRtsCvU6gFeVv0vA6sk8AWCvcts42WO1uD/nA3hQuT+t9gHAGgCblN/irwAaptM+APgGgF0AtgG4HYB7qo8frGppD4A0mDr9eLExA7hZObd3A7h4ssdfZB/2gXnd/Jz+n8neB5mJKSEhITFNMR0sFAkJCQkJE0gCl5CQkJimkAQuISEhMU0hCVxCQkJimkISuISEhMQ0hSRwCQkJiWkKSeASEhIS0xSSwCUkJCSmKf5/8ve/uGumpxgAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_VAL_1','train_VAL_1']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABpF0lEQVR4nO2dd3hcxbn/P7Or3rusYkuyce8F25hmMD2ETrDhQkJuQkgC6blACL8UkpvckEYSAiEkIRDAoVfHNGOqccO9y+qW1XvdNr8/5pxt2pVWsoq1zOd59Kz27Dm7c7Z8zzvfeecdIaVEo9FoNOGLZawboNFoNJqRRQu9RqPRhDla6DUajSbM0UKv0Wg0YY4Weo1GowlzIsa6AYHIyMiQhYWFY90MjUajGTds3769QUqZGeixk1LoCwsL2bZt21g3Q6PRaMYNQojyYI9p60aj0WjCHC30Go1GE+ZooddoNJowRwu9RqPRhDla6DUajSbM0UKv0Wg0YY4Weo1GowlztNBrNJrhpa0aDq4b61aETs1eKN801q0YUbTQazSa4WXzQ/DvG8BhG+uWhMab98Cr3xrrVowoWug1Gs3w0lYN0gVdjWPdktBoOAKdDWPdihFFC71Goxle2mvUbWedZ1vlFtj73Ni0pz/s3dBaCd3NEMar7YUk9EKIi4QQh4QQxUKIOwM8niqEeEEIsVsIsUUIMSfUYzUazTjE1gn2nsCPtVWr2856z7YP74dXv3PyiWlTibqVTuhp7X/fLX+F6h0j36YRYEChF0JYgQeAi4FZwBohxCy/3X4A7JRSzgNuAu4fxLEajWY8ISX842J44SuBHzMj+g4voW+vgZ4W6KgdlSaGTGOx5//upuD7SQnr71JiPw4JJaJfChRLKUuklDZgLXC53z6zgLcBpJQHgUIhRHaIx2o045v9L8G2v491K0aP47vU36F10N3i+1hvO9g71f/eEb0p8HUHRqWJIeMt9F39CH1PK7jsnh7AOCMUoc8DKr3uVxnbvNkFXAUghFgKFAD5IR6LcdwtQohtQoht9fX1gXbRaE5OPn4Q3vv18DyXywnb/hHcFjkZ2P20unXa4NB/fB8zo3nwePRSeoS+/uDIt28wNB71/N+f0JsDy977jyNCEXoRYJu/0fZLIFUIsRO4HdgBOEI8Vm2U8mEp5RIp5ZLMzIC18zWak5OmEmg7pnzrE6Vyi0r1G8mBy4PrVEQ+FJwO2PsszLgUkifBvud9H28/7vnfzGTpblYXBTg5I/rEXPV/f9aNeS6ddarXMs4IReirgIle9/OBau8dpJRtUsqbpZQLUB59JlAayrEazbimt8MTrQ5Ht77tmLqtGKEJPN0t8OzN8N59gR+Xsv/899KN6nznXQezr4CjG5SQm5gRfXSyx7rxjvJPJKL/+CH4943QUjnwvqHSWAwTT1X/e0f0794H2//pud/llX45Du2bUIR+KzBVCFEkhIgCVgMve+8ghEgxHgP4EvCelLItlGM1mnFNc6nnf2+/d6iYoljx8Yk/VyD2vQCOHmg91vexjnp47DK4f57vQKo3u5+GmGSYdiHMvhJcDjj4mudxM6KfMAc66ny3Zc1WEf1QMm+2/QPW3wEHXoEHV8CutSeewdPVpCyZvMUgLL4R/SePwZ5nvPb1mhMQjkIvpXQAtwGvAweAp6WU+4QQtwohbjV2mwnsE0IcRGXYfLO/Y4f/NDSaMcL7Rz8sQm+IYuMITeLZ+aS6bfPrWB/bDg+frayjrkZ4LUAqZG+HEtrZV0JENOQuhJQCdfFwt78GohIhtcjTfrPHM3kl9Lb1fe2B2P+Sas/UC+C2bZA9W2X8lGz07ONywuHXByf+5meXMQ1iUjwRvculPgef8YZhjuh7WqH0/RN/nhAJKY9eSrlOSjlNSjlFSvlzY9tDUsqHjP83SSmnSilnSCmvklI293esRhM2mD/6mJTQB+qqdwa3TtqP4x7aCjWqP/xG/wOJJg3FULUF4tKV+JoWjcsJj18FwgpffB3O+QEceLnvOMGBl8HepWwbACGU6Jds9Ng37cchcQIkZCrrxjvdcvLZ6rZ+ED69vRteuBXylsC1/4SMU+D6f6vHanZ7vQfr4cnPDa4nZF6Y00+BuDRPRN/dpDJsvFNBuxohMg4SJkDjMAj9xw+p3tNAufvDhJ4Zq9GcCE0lEJ8JOfNCj+i3/wM2/Czw4G3bcWUlWKND8+m7muDJa0PL+tn1pLIolt0KSOgwBLjtmMpxP+u7kLsATrtdCeu670G7l9h98jikTYFJp3m2TTlH2TfmRKL2GiX08ZlKLM3c+ahEdV4AdYPw6ZvL1cVl6S0QFae2xSRDbCo0l3n2M9/7mj2hP3djsbq4pRRAbJrHnjF7Vb1tYOtS/3c2qAtk2uThiehrdqsyES0VJ/5cIaCFXqM5EZpK1Y8//ZTQhb7+kLptO973sfbjkFqoRDGU6NR8zcPr+9/P5VS+9pRVkLfIeH3DQmkuV7cpBerWGgFXPqQuRG//VG1rOAIVH8GiG1Ukb5IzX92aWTztxyExB+Kz1P2OekP8syE+Q10ABhPRm2KeVuS7PaXA025QnwNA7d7Qn7uxGFILICJKRfRdZq/Ey7IxL4Zd3kI/yBTLTQ/AHxb6XjTNC5IWes2oU7NHebCa0Gkq8Qh9d7PHQnnnf+HJ69Rtybue/aX0ZJ60+3nVps2ROAEmLYfjOz0RZTBMoW86qqwZf5rL4N1fwV/PVZH7gjWQZExlaa3y7APqAmOSMVVF/jufUCK+43EV/c6/3vf5Y1OV6Fbv9G1/fIZ6vLNeRfQJE9T9zBn9R/SNR32zeMzBbu+2gRLoFi+hN8+hbn/w5+7zWsXqcwMl4qZ1450iag4odzaoc0qfrM6nt2Pg53e54PW74fUfqO9JmeHJ97R52j6cGUT9oIVe42H9XfDi10a2Hom9e2jH7XzKE12OJTV7lYUB6lzajnmEHpR4dLfA+7+Fys3Ki3/sMk93v7PBI2T+EX1PCzi6VUQ86TRliRzb3n97Gotxe/r+UX3ZB/CXs9TFxhoJF/wMZl3hEXp3RF+mLJ3kfN/jz/yuEvL1P1Dv/7SLVGTuT+4CdVHqbgZnr2p/ghHRd9Z5InqArJmqRxPoO+Z0wN/Oh7d+7NnWXKZsn7h0331TClQ07HJ59gOo3e/Z1h9SqouK+bnFpnou0t4Rvfl/VxPEZajPGnyzrYLx5j2w6U/KdoqMh6qtarv3XAId0WtGla4mKP9I+ZKhDOwNlqrt8MS18L95fWdThsKuJz0ZI2PJe7+Cl29XImlaB6lFvkJ/eL3yp294Fr78jtpetU3deueRt/mlOJqikpRj5HaLge2bxmJIn6JSF72Ffu/z8PiVKpL+xg740luw4nawWCEmSYmnKfQt5UrkrZG+zx2bAivvgvIPlGAvujFwG3IWKKE1BSwpR1k0oKwb/4je1u7pTXhT/Ynyyb0nczWVqmje2y4CFdE7bcpacTpUBcr4LFV+wTvSD0ZbtfL+TeGOS1MXWXu3iugtxnthDsh2GRF92hR133/gfdvfYd33fbftfV5NLLv4V8ouM4XetJeik6FVC71mNDm8XlXwg9CilcHw/m/gkXPVFz05D17+xuAvJo0l6pixrH4opboYImH/y54oPW0ypEwCS4QS3v0vqag5dxFkz1HZGmZk3mD488LqaxGAR3gTc1SEOWGOigjf+7Xq7gfCjEqnXagGb7tblP327BfV639xfV9/GyApF9q8rBt/a8Rkyc2QPlW16ZTzA+9j+vRHXvdqfxog1Htk71J2DqjUSFC9HX+OblC39Yd8I/W0AG1LMbY1l6vzcDlgxiVqWyj2TcNhdZsxTd3GpqnbriZ1wc2Ypj7P9hpln9m7DI/eeC/9B2Q/flDl+puZTF1NypqbuExdpPKXwPHdqrRF7T4l8hNP9Y3ot/9T/TZG4DuuhV6jOPCqyvQAz8DWsD33K0p0vrUHrntCeaH+0U9/2LvVj9nZO3TrZzhoOOyZ7bnveS+hL1LRcGqh8qqL34aZnwWLRQ1s5izwCH39IRVNZ0ztm09uRvSJOer2qkeUUGy4F+6bAr+YpP42/0U97nJ5Cf1FSuw+/D08f4sazL3xBRWpBiI5z3cw1hyI9ccaqZ7nppfVuQQid6G6PWT0KBInqH3j0j2DjqbQ55+qouIP7+8raEeN3o+9S0XlLlfwi5C5rbnM832dbgh9bQhTdRqOqNvM6erWfJ+6m9T7kpSreggdtZ5ZsXHpEJ2otnsLfXO5+m647J6LjHneE+Z4zttlV72V2n3qgpcyydejP/KGulj7916GAS30GpVdcfRtmG/kRw9nRC+lisbzFqkfSc48OPsOVS9l34uhPYf3hcd7oG60KftA3S64QUWkZe+r/HlTJNJPUe+jsxdmeRVpzVukojmHTVk3mdNUxN9H6I0I3xTFrBlww9PK/ln2FVhwvbJTdq019q9WdkP6FBUxxqbBB79Tt6uf9KQjBiIpV82OtXUqWyZYRA+QMlG1ORhxaarujdlbMW2a+EyoNQQvwfDoLVY449sqvbD4Lc9z9LSqHl+RkWtfd8CwZXqVNRaoTQh1QTD9+ezZ6jxCEvrDEJ3kaZd/RJ84QY0rtNd4JkuZA8zpU3yF/ujbnv+P71S3pj2TPVfd5i1Rt5WbfYW+u8lTO6dmt+oBjgBa6DWqy+zogTnXqAJPwxnRdzVBb6vH2wT1Q89dCK98I/Brlb4P6/7Hc987nc17mnrbcU9WBCjReujM4JORPvyD8q2HSvmHKto+87vq/uH1Ho8XPD59fJaKxE3yFinBqtuvIvrMGcrHDiT0MSkQGeu7PW+RGki9+Jcw91oVFfa2+074sVhVLyIyHq5fG3jQ1JukfBWtml5zf0IfCjnz1G1sKkTGqP8TMj0TgsyLF6gJV0n56nMyo/rS95V1uPxr6n7dfs93I1DbIqLVZ9FcrgITa5S6nz2nr9BLqb4XO57wbGs4pOwZM3o2L9ad9erCl5ijLlgdtR6bMc4Q+rTJvh598duQPFHl91fvVNtq9qqLSIIxVpGYrYR973NqjGLCHHUMqKi+u0XZOBPmBnmDTwwt9Bpl28SmQsHpyoYIFNH3tKnsjQ9+ryLKUGf0efvYJtZIuOYf6v+nb+xrx2x+CLb8xSOE3j8q74j+uS/Bwys9lsdbP1FR0da/B868OPKGmsXp6A2x7aVqoQmXS4lF2YfqPUqfAhPm9T2vdONiNvNSJbwm5kSho28r4cgwIvqOWnDaPfu116hIuz8KT1eCWLHZV+hBDfp9a3doYpGUC0iPV36iQp+7QN2athN4BmTBEzmDyls/41tGr8joJR3dAFEJMOVcJYD1B4Pn0JuYKZbNZcp6slgha5YKDLy/U5316ntx8FXPtoYjHn8ePFk99QfVRKbECSpzqL3GY92YEX3uQtXbKHlXfX4l78Ipq9RYhTmQXLunb3Sev9QT8WfP8dhlrZWeHoD5vRpmtNB/2nE5VWQ67WLlq6YWBY6yi9+Ed/8P3vqRqjPy9OdDe34zGk+f4rs9rQiu+qvyMl/7nme70+GpAXLsE9/nAF+hb61QmStPrYEjb6qLQ9ZsZWlUfNS3LfWH1I/Ye6JNMFqPwT8/q2aHbv+HumB11CihBTX1H3yFPmcBIGDu53yfK6VACcnOp9T9zBmGIErfafZm+YD+mLhMDRKWf6AugJFxHnGNjPGI0UAkGymW5R962ngi5Bg+vXf7zUlTETEq2vVm4X+px1++XY1fHN0AhWeqi0DWTGXdNJcaaZ8TCYg5acrMzAHInqU+Y3NSGngGPKu2qQt2T6t6rzOmevYxrZtaw2NPzFHn0tXgsdTMi8HCG5VV9cbdylO3tcMp5ymhr92nBm/rD3n8eZN8o0omQn0HUsyIvsLL09cRvWYkMKe/TzKshrRCJWj+E3XM6Prb+5THXvJOaF5oU4n6saZM6vvYtAuVDbLzX54p9Md3KasHVLodKI8/2TjeO1uns1FFQNU7VOpm2mT4/CvKvvCuPGgeZy6EMdAU9q4m+NdVqjuds0D1FPY8qx4rOEPdzrlaCZg5EAnKYvl+MRSc5vt8QqiovtFrANCdy+6VedN23FMbPRhR8eo1yz70pFYOZfDOfP3yj9T7FeoFIhhm5o1PRG88Z0J23zZGxsLn/qlSJB85X4n6lHPUY1kzlYfecCRw2qdJaoH6/jaVeKJ+M4r2/m6aPYPOOhU9mxPLzIFYUBeYqASoM45LnODphdQdVOmW5sUqMgbO+5ERpHxXXXiLzlLfFWevSj5w2jz+vIkp9GlFEJ2gLnTWaI/Qx2cNbLkNES30n3bMUX8zajIHvrzriIASocg4JRDLboWIWJVSNhBNJerHGhEd+PEVtyt/1Vy1qOQdT3tM8W86qgYbwRPR2zpVzvScq5R/HRkLVzwI8ekqzW7/S7511b3z1wcS+he+ovZZ8xRc83c1frHxF+qHaEaBqQVK1Kdf7HtsMME07ZuIWHXRSzIE0cyldzlVdD9QRA9QeIa6CNbs9dg2g8W0iDpq1bmcaKZHQqby3qdd5LXNiOiDnVPBCvjqR8ZFM1ZVpwRlvzhtUPpu4IFYk9RCQIKtwxPRp01WF2DvFEvvFMaqbX1TK01i0zzfezOiByX+cem+79Gcq9UAa8Nh1cuKSfZc9HcYE+r8I/oJc5WwmymmFmOSmin0IxTNgxZ6Tash9GbX3S30fvZNe7X68guhBq4WrFHiHKxuuUnjUd+BWH9iU1Vkv+dZZduUbFRf+CnnKKHv7VBd5+zZ6gdsCr07EyITVtwGd5SpsgGgBpW7mz0XDfAIvbD6WkFv/VhNUzeRUvnGi78ARWeqiPns/wGkEibvH3t0YugCaQp9hjFwakbupi3QWa+891CEvuAMlUrZXj10oY9JVmmecOL+vMlVD8Osyzz3TY8+oZ8oNTYFrv4r3FXpicozZ6jb7uYBsoG87Cbze2uxKgH3n30ak6y+P8e2q4FYS2Tf545LVbfCotpuZg/VH+o7M1cIuPB/1f/mBSq1SL2nZe8rQU+f6ntMRBRc/gCc6WVVpkxS38f6g30vDMOIFvpPO+6I3pj+7p4Q4if0bcd9BwqXfVV1U7f/o//nN2vB9Me861S3+tA6NUA3eaXKu/cW67TJ6qJgZt14Cz349himnKuyV0y7BVT3OzJeXUS8I/pd/1bld00661Uet7eArviGauPiL/R/Hv2RaxQSM0UsLk2JgRnRm4I/0GAsKJtNGIO9QxV679caLqH3J36AiN4bb3smczrusg7BBmJB9UTc/xd6HT/Dz6MvV9+fnPlGRH9E3e8zEzjN025rhMdGcdpUT9GfScvgK+8Z1UBREbppYWXNCDzvYN61noFrUD59zR71GiM0EAta6DUt5eqLbabExaaqWXvBInqTzGlqpuSWvyovOxBdTcr/9x+I9WfqBUqYX79bfeEnr/R0g02vPX2KIfTGa5kTlwJZJRFRKo/94Gue7Iv6g0pA0k/xCH17rTqv1iqPzeNfydF8vqse9njIQyE+XV0wFtyg7gthpFgaAu+eLBWCKEYnegTlRITeHJA90YHYYHh79IMhMtYTHPR3EUrM8ZQq8BH66WqCnZmf3lKhIue8JSrrpXaf70CsiRm1m5+BeaECT2qlPznzPb8d8Ii4vz8fDO+xK23dfAp591ee4lkjSWulZ/QflAClFfpG9C6XEdHn+B678k4VdT/zBWW7+BMotTIQEdEqi6W1Qvn1k05TVo01Wq0aZD5HbJqXdWMKfZCF5GdfoTx8c7almb+eNln98B02T6qbd13wQJUch4sL7vW9WHhPmnKXPwghogfl0yMGfm/7Y6Qj+uSJquc387KB9/Una6a67c+jt1jVdzch23dymNlrajisvrstlepilr9Yjbc0l/oOxJqYufRmQBMR5RXlhzhYbV6AQ7VhzCSDiNgTu2gPQEhCL4S4SAhxSAhRLIS4M8DjyUKIV4QQu4QQ+4QQN3s9ViaE2COE2CmE2DacjQ9bXC746I8qXdCbmj3DX3CspbJv+lqqXy59V6Oavm1mapjkL4FLf6fslfV3eFLXzBKuoQo9eFYtmrhMZZZYI1WE4+hRP+ToROXnmudvCn2wSKvgDNUzOfiqujh01Kgfd9pkj7Cbg73gOd+WMnUbKEtouEnM8ZQqbq/xeMOhcMa34YZngpc4CIUkw65LHaGI3mJRk7z6m1UbjAlz1fsx0EUoZ75HXE1MEa8/ZMxV6PVE9Cb+A7HgEXXvXpX5f7DvmT9FZ6nMnymrQtvf/J5lz/KdezHMDCj0Qggr8ABqLdhZwBohxCy/3b4O7JdSzgdWAr/xWiwc4Bwp5QIp5RI0A9NSpqpI1u73pDnau1Ua2oe/H77XcbmUbZHiJ/RpRUYJWKPImSlGiX4RPaiKhituh62PwM9z4JeT4I+LlOA3HgVEaBHjxGUql3/RTZ5t5gIZ5mBubKrvYGxkfPBp/hFRapD30H88qXZZMz02UtNRJfTmD9i8KDWX9Y0QR4qkXNVTktJYqSoreD0Zf+LSYGqQImOhMuUcJUwn0isYKZbdqurrxKb0v98VD6olBr1JLVKWTv1BT08tpUCJqnkhDST0/hE9eGynQB59IBInwFc/DP3iZv72RtC2gdAi+qVAsZSyREppA9YCl/vtI4FEIYQAEoAmIEBfXhMSx3erW+n0rIt5bLuqazKcCxV01qtoJ9kvek0tUlkdZinZtgEGCs/7CZz3Yzj1v1WOfUedqljZVKJ6C8FSK72xWNTU/Xlek41Mnz7dq5Rsd7MSxs76gbvTMy9Vg7fbH1X3zYgeVNuqd6oZjZHxHququXzkrAx/knLV+390gyqSZlZfHC0mLVfzDkL5fEab2BSV9TQQkbF9L8rWCOXB1x/2CL2ZQmpG9YE8ejOi97YoBxvRD5bEHFXKePZVI/P8BqGED3mAt7pUAcv89vkT8DJQDSQC10kpzTnoEnhDCCGBv0gpHw70IkKIW4BbACZNGoVu88lMzW5U1oFUs0MnLYfyTeox75mUJ4o7tdLv/fYuxZpa0H9ED55CVSYtlSrHPmFC/1kTA5EbIKJ39qqsmM76gW2OU85TKXV7n1NzAJInqR97dJKaEdpRo16jdp/Humku96RpjjTmhfOFW9W5rPrR6Lzup4HM6epCblpxpj256CaVTROd2PcYM2oPFNH7p1cOFxYrrH5i4P1O9GVC2CdQorB/weQLgZ1ALrAA+JMQIsl47HQp5SKU9fN1IcRZgV5ESvmwlHKJlHJJZmaIPmW4UrNHTRpJyveUtzWnqnuvfnOiuLu1ftZNhuFxmrnIbdXKLw01e2LVPSr9r7Vi4Iyb/sicDpf+Xk2XByX0YCzZ1zCw0EfFw+RzlCefMU31GoQxgHn4DbVP7kIVwTeVqLolbVWjF9GbA6+ddXDRLwe2KTShkzFd2XD1h9X3xIz6Z1wCn70/8DGFZ6rPwaygCV4ZOCMU0Y8SoQh9FeCtBPmoyN2bm4HnpaIYKAVmAEgpq43bOuAFlBWk6Y/ju1U1wLyFSuidDqjcoh4LFNE7bCp6GWzVSVPo/QdjE7NVVGNmpbQdVyIfqn+clAunf0P9fyL+rxBq4QtzhqXZte5u9qzhORAzL1W3ZhaH2SZnr7p4TZir7jeXqWheukZP6M30xqkXemrnaIaHzOmAVIkCoQ6sWyNh+VfV+I7JtAtVoDGCGTGjQSi/3K3AVCFEEXAMWA34rRBMBbAKeF8IkQ1MB0qEEPGARUrZbvx/AfDTYWt9ONJRpyyFCfM8dTNKNqpUQbM6Xm+HqpXhsMETVytbx2VXPuI3d6nHQqG1Us0YjEnq+1juQk9Win8OfSis+Ia6KM34zOCO6w8zou9qDM26ATXAGxnnmZkKnotP5gwV6aUVqfz9CsMeG6ksFH+ScuHqv6lexwgsNvGpxkyx7KxXkfpQSZusZrOOcwaM6KWUDuA24HXgAPC0lHKfEOJWIYQxJYx7gRVCiD3A28AdUsoGIBv4QAixC9gCvCalXN/3VTRuzIHYnHkecdr0J3VrRn1mVN9cCqXvqaj1gp8rO2NLwCGQwLRU9h2INcldqGYQ9rT1nRUbCtEJqos8nBkdptA3l6nB4lCEPj4dvrkbFt/s2WbaSeZgr9lGcxbuaEX0AHOvCT2jQxM66VM8s4dHI1X2JCekvriUch2wzm/bQ17/V6Oidf/jSoD5/ts1/VBj1LPOnqOsBYQSoLTJninSHbXqi2xaL8u+qqZjl2yEj/4AS78ceLDJn9bK4KKWswCQamC4vdqYoDPGmOlv5jJwoeacJ/jtZwq7KfTmpJySjSotb7C9F83JR0S06qk1FmuhR8+MPfmo2aNyfmNTlKViTv6YtMIzMGQOyJqr3Ztf5HPuUv61uaZof0gZeLKUiTmVu/wjlRPvPyt2LDAjerOOyVAHyPKWwMq7VAVCUHV+LJHKEkqZNKITVzSjiGnfjJYVdxKjhX6skBJ2P9O3Tow5EGti2jcFKzzV9EzrpsUoGWBmw+QtVmViP/rjwCtA9bSoBRP8M25MErJU1s/B19R9/1mxY0FkrEqXNMvMhhrR+2ONUOUbzB6CxeoRg9G0bTQjixkkjVQtn3GEFvrRoqnEd3mzhiPw/JfUCjvmupm97WrG5gQvt6tghfIaC89QwmSJ9IroK1REbvH6GM/+HyXie5/rvz3mxKv+urW5CzyZNyeLnRGb5rGshir0gTDtGx39hQ+zr4T5a/qvl/MpQQv9aHD0HfjjEs+gKniKZx142SPKZhEz74h+/hr4+hbPzL6EbN+I3l+ocxepOtj7Xuy/TcFSK32ea4Hn/8EOxo4Usam4p3GcSJ0Xf0KplqgZX0yYC1c+FHpacBijhX6kqT+s1leVzr41skFN7Hjtu7D+B/D6Xapk72SvCocWq1qswiQx2zei9xd6IVQkU/a+StUMhnuyVH8RvdcyeSdNRJ/quQ22xNxQMGfw6m6+JgzRQj+SdDXBk59TgpQ1y3dCU0u5KsN73b9UhcaPH1CLDq9+ynfChj8JE1REbzPKAAQS6tlXqok/3gtq+FOyUUXz/U3tNhd8jk4KPTd/pDFXARpO2wbURU1YR7y4lEYzFmihH0l2/Evluq9+Ui0M7L0OqxmNZ05Tk2Yu+TVc9seBu5mJhnXjvwSgN1kzVU8hmH3T06pSNmde1v9Enfh0Y33Tk8S2AU9EP9xCP2k53FF6YiUbNJqTFG1ejSR1B1Q9k0nLVK2argY14Bqd6Gu7mNP0QyEhW6UBNhor2QfKmhFCLZq98ZdqFSX/leUPv65mgs7yL0IagFO/pCYnnSy4hX4Eao/EJA//c2o0JwE6oh9J6g966lKbHrAZ1TeXD20ih5lKaRY7C/Ycs64AZGD7Zv9LynPPP3Xg1zv9m3DmdwffzpHCveLPp7zwnUYzCLTQjxRSepavA082R1Opiuq7m4aWymdOmqrcolItzdx6f7JmQNZs2PSAb059bwcUv6VqYFvG4cc/UtaNRhPGjMNf+klK9U741WTP4tKtVaoQmTlpI9Urog8lhz0Y7oj+E2Xb9CfWn/mN8vJf/JonV7/4TTX4G4ptczIyktaNRhOmaKEfLsreV9750Q3qfoORSmnWdo9NgZgUNTjrLl1wAhG9vXPgC0XBaXD+vWrd1Hf/Dyq3wiePqSqXBSsG/9onA2bu/Eit+KPRhCFa6IcLc5EOs268mTNvWjeg7JvmMt91LAdLfBbutWBC6REs/6pKt9z4C/jbeepCNPvK8VvPJW8JnHabWu9Uo9GEhM66GS7q9qvbyo/Vbf1BFXV6l6BNKzKWN6tQNdKHYj9YI9RxwXLo/RFCLaA863L1mgnZkD178K97shAZAxf+fKxbodGMK7TQDwcul4rgI+NUTZuOOjUj1juaBxXRH3hF7ZMyaeiLTSRkG0IfYo8gMlavYKTRfIrR1s1gOPImvPurvttbytWC1XOMldwrN/umVpqkFqmc9IpNJ1Yj2xyQ1XW2NRpNCGihHwzb/g7v/NyzCpSJ6c/PX6PKBh94RVWQDBTRg6oZfyIibQ7IaqHXaDQhEJLQCyEuEkIcEkIUCyHuDPB4shDiFSHELiHEPiHEzaEeO65oPKpuP/qD73bTn58wT9VM2f+Sum+mVpqYk6bgxIpnZUxVE4eC5dBrNBqNFwMKvRDCCjwAXAzMAtYIIWb57fZ1YL+Ucj6wEviNECIqxGPHBy6nSo2MiIW9z3syZ0BF9MmT1IpQE5epPHXwpFaaJOWpSU5wYtH48q+r0sXjccKTRqMZdUJRiqVAsZSyREppA9YC/rNtJJAohBBAAtAEOEI8dnzQWqnqw5z+DTWIuunPnsfqD6pCYqCKYwFEJ3ssFhOL1SPwJyL0EVF910HVaDSaIIQi9HlApdf9KmObN38CZgLVwB7gm1JKV4jHAiCEuEUIsU0Isa2+vj7E5o8ipm1TdDbMuQY++acqQ+y0q6XtTKGfuEzdZk4PnFVj+vR6gQuNRjNKhCL0gXIApd/9C4GdQC6wAPiTECIpxGPVRikfllIukVIuycw8CaNVU+jTp6io3t4Nb/9UpUo6barePKgc9/xTg888zZyhasCbU/k1Go1mhAklj74K8K6Fm4+K3L25GfillFICxUKIUmBGiMeOD5qOQlSCSm1MnAArblOLcEunetyM6AH++83gOfIr74BT/3voOfQajUYzSEKJ6LcCU4UQRUKIKGA14F/7tgJYBSCEyAamAyUhHjs+aCxW64qaAn3O3ZAxTdWOERb1v0l/Ih6TrBe30Gg0o8qAQi+ldAC3Aa8DB4CnpZT7hBC3CiFuNXa7F1ghhNgDvA3cIaVsCHbsSJzIiNN4FNK91m6NjIUrHlIinzZZTc3XaDSak5CQSiBIKdcB6/y2PeT1fzVwQajHjjscNjX7de41vtvzF8Nn7weLriSh0WhOXrRCBUNKsHWqRbFbytVi22kBLJdFN41+2zQajWYQ6Bk3gehshMcuh9/OUmuuujNuTun/OI1GozkJ0RG9P9U74d//pSpQuuyw6U+eImJ6EFWj0YxDdETvz9M3KZvmi+th9lWqkNmxbSrv3VzdSKPRaMYRWui9cTpUDZsFN0DeIjjzu2DrgH0vBPbnNZqTgHV7jtPQ0TvWzdCcII0dvXT2OkbkubXQe9PdBEhIyFL3s2fB9M+o/7U/H5ZsOFjLm/trx7oZQ6ap08bXnviEh98rGeumaE6Q3791hLPve4deh3PYn1sLvTedRo0d7yX+zvquus2YOvrt0Yw4v1p/iD9tOHJCz/HCjip+se6Azw+0uK6D2raeE23egJQ1dgKwvbzZZ/vWsiYcTteIv/5w4nJJ/t9Le3l9X81YN2XE2VHRzHee3onNoT6jzl4HL+w4xllTM4mOGP71nLXQe+MWeq9aO3mL4aaX4NQvjU2bNCNGj93JkboOOk6wu/yPD8v4y3slfOHvW2nqtPGbNw5xwe/e5QfP7xnwWFU1ZOhUNHYBsKeq1X2h2VHRzLUPbWLd3vElmH/7oJTHNpXz/CdVo/J6TteJvfcDcc+Le7nv9YMBH3vrQC3Pf3KMZ7armo+v7Kqmo9fBDctHZjEhLfTedJhCn+W7ffJKiE0Z7daMGT12J79afzDsfd8jtR04XZLO3qF3lV0uyeHadmbnJrGtvInlv3ibP24oJjUuih2VLf0KeVOnjUX3vsm6PcdDfr2ntlTwSYUnei83hN7mdLH3WBsAGw7WAXDweFvIz/vI+yUcrAl9/+FmX3UrvzJE0TynkWRHRTPzf/IGbx8YGdvuUE07j39cziu7An+2De02AP74djE9didPbK5gxoREFk0amWKHWui9CWTdfAp5Y38tf954lH9vrRx452HC5nDxgxf28M6hun73k1LycUkjrmGIxvZVtwIMOAAmpeSu53fzUXFDn8cqm7vosbu46bQCHr15KbNzk3jwhkV86/xpNHXaONbSHfR5X91dTXOXnbcP9H/OJj12J/e8uJeH3/X48eWNnSRGqyzpTwz7ZuMh9T0+UtcR0vNWt3Tzs9cO8Pim8pD2H266bU6+uXYnqXFRXL0on7LGzhPu6fRHbVsPX3l8Ox29Dj4I8JkOBw9uLAagoqkrYI+xvqOXuCgrNW09/OD5Pew51sr1yyYhRqjYoRZ6bzrrVTmDmJSxbokPI93F9Oe13arA6FCinfLGTv7w9hHuf+sID7xTTHOnbcBjpJT86OW9PLm5gh88v4cee/AI+839tax++GNe2HGs3+esbevh1d39F0rdb0S8HTZHv8JyqLadp7ZU8scNxX0fq2kHYFp2IqefksELXzudi+fmMC8vGVCWSjBe2qna5x2hO12Sp7dWBhyQ21fdisMlfSLv8qYuZuclMSktju3lzTR09LLnmHrNo0GE/htP7eDh9466728ubQTgoHEuo81/9h6nuK6D/7t6HgsmpdBjd1HbNjK9yR670y3yk9Li+v18hkpFYxcv76pmalYCAIdr+76vDR29LClMY8WUdJ7fcYzYSCtXLAy4VMewoIXem8465c+fREv0bThYy4KfvsHOypZReb3OXgcbD9UTF2VlR2XLoO2bh949ym/fPMzv3jrMfa8f4s8b+4qjP//aXMFTWyo5Z3omx1t7eGpLRb/7Ary4M7jQu1yS25/cwW1P7ug3Wt9XrQRTSuiyBb+4mBHyx6WN1LT6DrCaP+Kp2Yk+22fkJBJpFew+FlhIKpu62F7eTE5yDKUNnTQa7/Ob+2v5n+d28/q+vhfZHRUtgBL3Lps6r/LGTgrT41lckMr2imbeO6zaunJ6JmWNnX0uGIdq2nl5VzX//KjcfXHbXNKkzqWmPeRI+t9bK9g4QO8rVPYcayU20spZ0zIpTI8DPIPMg+F4a/Dek8kj75ews7KF335uPqtmZrGvum1YBq27bA5KGzpxuiR/ee8oERYLv7hqLuAJBrypb+8lMyGa716gqt5eNj+XpJjIE25HME4eRTsZ6Gw46WybHRUttPc4+Mrj26gbhSyOtw/W0etw8f0LpyMlvHPQ98fsdEm2ljUF9XNL6jtZXJBKyf9ewqXzcli7tbLfwc7dVS385OV9nDsji0c+fyqnTU7ngXeOuoXMm4rGLt47XE9GQhQfFjdQ1x74/fj3tkq2lCnxagrSo3C5JAeOtxEbqTIc+rsgbDxUR2ZiNFLCy7t8LzCHajvIT40lIdp3knl0hJXpExKDRowvGReqOy6aAcAnhohvOKgEfn913/fXFHop4XCtGkRu6LAxKT2ORQWp1Lf38q+Py8lIiOKKBXm4JJQ1+PrdzxqDf8daujlcqyL+zaVNWC2C9l5Hv1aTyYMbj3LHc3v41r930t5jH3D/gdhf3caMnESsFkFhejwAZQ2DE/qntlRw2i82DBgQ7axsZVp2AhfNyWFefjLddidH6wd/UfFGSsmt//qEc369kVn/bz1rt1Zy9eI8Fk1KJT7K2mesREpJY4eNjMQoFhek8fcvLOGOi2ecUBsGQgu9N531vhk3/dBf5LPpaCNH60PzRweirLGL5NhI2rod3Pqv7SOSY+vNa7uryUqM5qbTCslOinYP7LX32PnZq/s57Rdvc+1Dm7jo9+9zwe/e5YnNvr5uaUMnRRnxWCyCL55RRHuPg+e2B8+iWLu1kqgIC7+7bgFWi+C7F0yjoaOXxwL4xU9uqcBqEfz+uoW4JAEHuurae/jfdQdIjFHC29wVWOhVVOxkUUEKQNCLUUevg21lzVy1KI/5+cm8uMPXDjpc0850v2jeZG5eCnuOtfb5rkgpeXFnNUsL07hozgQirYLt5c24XJINB1VEfiDAQOqOimbm5StL6FBNmzvjpjA9nsXGIN4nFS2cNTWTqdnKNjhS54km7U4XL+w4xqJJ6pzfOlBLXVsPpQ2dXDBLlfk4eLx/++bxTWX83/qDnDY5nZYuO3//oKzf/QdCSsn+423MykkCIDclliirhbJBDMjWtanPHDy2YzCO1ndwimGpzM1LAVSw4c/3n9nF7948HNLrbzhYx3uH61mzdCI3Li/gwtnZ3H7uVCwWwfQJiX0ssbZuBzani8yEaADOnZFNWnxUSK81VLTQexOi0JfUdzDvJ2+wo6I54OPfXLuD/33twLA0qaKxk3n5yfz62vl8UtHC3z4oHZbnDUSHYdtcMjcHq0Vw7oxs3jtcT49dDZb946MyFkxM4Y9rFnLv5bOJtFq4+4W9VDWrH2Vnr4O69l6KMlRUtmhSKgsmpvDoR2UBB09dLsmb+2tZOT2T5FjVbV1SmMbK6Zn85d2jPhc1m8PFM9sqWTUjizOmZjA7N4mXA9g3P31lP712F//vUrW0Y7CI3hyIPbUwzWh74Avoh8UNOFySldOyuGxBHvuPt3HEsGtsDhdH6zuYNiGw0M/LT6a1205lk2+UvK+6jeK6Di5fmEtMpJXZuclsL29i97FWGjp6SYqJ6CP0tW09VLf2cNn8XGIjrRw43k65YW9MSotj+oRE4qNU7+Ts6ZlMyUxACJXPb7LxUD0NHTa+uvIU5uYls+FgHR+Xqp7PjacVAGo8Ihjby5u456V9nDczi8f+eykXzs7mkfdLaOmy0WVz8Ls3Dw968llVczftPQ5m56oLmNUimJgW6z43KSUPvFNMcV3wdv3klf30OlzMzEni9X21QYOwXoeTiqYupmQqoZ+cEU98lJW9fvbageNtPLO9ikfeLwnYs/TG7nTx89cOMDkznp9ePocfXjqLP9+wmNyUWACmT0jiUK2vJVbfoXqimYnR/T73cKKF3kRKlV4ZgtA//nE57T0OjtT2jdq7bU7q2nsHTK0LlbLGLialxfGZeTnkp8ZyeAQHzN4+UEuvw8Ulc3MAOG9mFp02J198dCsbDtbx48tm8/BNS/js/FxuPK2Qn14+G/BEgaavana/Ab54RhGlDZ1sPNzXz91R2UJ9ey8Xzp7gs/3zpxXS3GXnQ6+MiNf31dDYaeP6ZSrP+PIFueyqaqXUq4v/cUkjr+4+zldXTmGJIeDBIvr91W1EWAQLjUi4vTewBbHxUD0J0REsLkjls/NzsAjPIGpZYycOl+wnolfitftYi8/2N/fXYhFwyRz1Pi8uSGVXVSv/2Xsci1CiW9fe6/btwWPbLCpIZdqERA7VtFPepC6wBelxWI1zEQLOmppJTKSVialxPpk3z2yrJCMhipXTMzl3RhafVDSzfu9xEqIjWFqYRn5qbL8Dsn99r5Tk2Ej+sGYhkVYL3z5/Gh02B/e8tI9L//AB9799hC8/to07nt3dxwp7ZVc1v3+rb4RsjpPMyk1ybytMj3d/rsV1Hdz3+iF+GyS6fvtALa/tOc43zj2FG5cXUNHUFfRiVdHYhdMl3UJvsQhm5yX3GUf5x4elWAR02pys29P/XITHN5VT0tDJDz8zk0hrXzmdMSGRli67z+ByvZFaaUb0o8GnU+hbjylh98bWCY7uAYW+2+Z0WxGt3X3F4ViL+vE1ddqoaDqxfODWLjut3Xa3cOYkx1DdOnI+/QdHGkiLj2JJgRK/FVMyiI6w8NHRRtYsncR/LfOdzDHNEDjzh2X+OM2IHuDiOROYkBQTsIv/xr4aIq2Cc2b4zls4/ZQMkmIieHW3x5p5fFM5E9NiOWuq+nwum5+HELgn17hckp+9tp+c5BhuPXsKaXGqK9zUGVjA91W3MTU7kdQ41ZMIFNFLKXnvcD0rpqQTFWEhKzFGZdbsOIbN4XKL4rQgQj8tO5Eoq6WPT3+0voOJaXGkGt31JQWp2Bwu/rWpnCUFaZw2WY0TeYvujspmoqwWZucmMSM7kYM1bZQ3dpIeH0WiMYj3pTOL+O7509zPOzUrwZ1509jRy4aDdVy5MI9Iq4XzZmYjJazbU8OSwlQirBZmTEgMmntf2dTFG/truH7ZJOKilC02Y0ISn52Xyyu7qum2O/nnF5fytZVTeHp7JVc/+JFPL+5vH5Tyxw3FtPl5+vurW7EIfC6WBenxlDd2IaXk/SPqYv/W/ro+GVwul+QX/znIKVkJ3HLWFM6flY0Q8PrewL0Ks3djWjcA8/KS2V/dht0YkG3s6OXFndWsXjqJwvQ4ntkWPMW4x+7k/rePcObUDM6ZnhVwnxlGb897TKveuIBnnGwRvRDiIiHEISFEsRDizgCPf18IsdP42yuEcAoh0ozHyoQQe4zHtg33CQya8o/gd7Pgxa+B0+tLF2hWbABe3V1NW4+KVlq6+0aLlc2ebroZhQ25qU1G19zIRMhJju2T9TFYmjttQX3+4609FKTHYbGoXN7YKCuXL8jljFMy+Mlls/vk+CbGRJKfGuu2GcwBtMKMOPc+kVYLNyybxAfFDT7Rt5SS1/fVsHxyep9sg6gICxfMnsCb+2vpdTjZU9XKlrImPn9aobttE5JjWDUjiz9vPMpTWyp47pMq9h5r446LZhAbZSUxJgKrRfiIwwPvFLPkZ2/yyPsl7KtWvrA5iOodgT6zrZJf/OcAf/uglGMt3az0+hHffHohx1q6+dfH5RyuacdqEUzO9FzY/M9jZk4iu/2EvszIlDFZZFxYO21OVs3MYmaOEgdv+2ZHRQuzcpOIjrAyIyeR5i47W8ua3d8NgJXTs7jtXE+pjlOyEyip78ThdPHE5gocLsnnlkwEYHZuElmG0CwrSgeUcJc09M3UAXhsUxlCCG4yLB6TH35mJt+/cDrrv3kWZ0/L5H8umsFPLpvNwZp2Dht2S4/dyb7qVpwu2Wcuwv7jbUzJTCA2yjPtvygjjm67k/r2Xt4/Uk9iTAQ2p4uXd/n67+8XN1Bc18HXVk4hKsJCZmI0iyelBi2hYI6beQcic/OT6XW43L3zJzdXYHO4+OLphVyzOJ/NpU3usRB/Shs6ae2287klE4Pmv8+YoHoq3pk3De1K6E+qiF4IYQUeAC4GZgFrhBCzvPeRUt4npVwgpVwA3AW8K6Vs8trlHOPxJcPX9CGy80mVK7/rSXjyOug1uradxhcwIfCV2eSJzRVMyYwnNS4yYERfZUTxVosI6uGHSpnXYBuoiL6mtWfIk4Xq23s59zcbuW/9oYCPH2/tJifZd+3bX10zn399aRlREYG/KjMMGwGgtKGL7KRod8Rnct2pE7FahE/a5JG6Dsoau/rYNiafmZtDe4+DD4408I8PS4mPsvK5Uyf67HP/6oWcOTWDu57fw/97aR8LJqZw2fxcQHXLU+MiafKybnZWttDcZednrx2goaOXWbkeofcejP3NG4f5y7sl/Oy1A1iESlU0OWd6FmdOzeD+t4+wpbSJwvQ4YiKD1yaZm5/M3mOt7s9MSkl5Q5c7jRAgOymG/FTl6a6amUV6QjRZidHuPH+H08XuqhYWGoOo040osbiuw+eC4c8pmQnYnC4O1bbzjw9LWTUjy50GarEIVs1U3/Xlk9Pcz+t0SR9f33xv1m6t5JK5OeQkx/o8lpUUw9fPOYXkOM/F2oxuPz6q8vP3HGvF7lTnb6aqmuyrbvOxbUBF9KAyizaXNnHFgjxm5ya5ywWY/OPDUjITo/nMvBz3tgtmZ7P/eBuVAXrTR+s7yU2OId4rQ2peforRxhZ67E4e+7ics6dlckpWIlctykcIT6aSP+Y4gveFw5/kuEgmJMX49M7qO3qJsAj3uNRoEEpEvxQollKWSCltwFrg8n72XwM8NRyNG3YcvXDgZZhzNVz2RyjZCK99Rz3WaXjI/aRX7j3Wys7KFm5YVkBqXBQtXQGEvrmbqAgLiwtS2dFPqldLl40rHvgw4Ii/SYXXYBsoobc5XT7iNRjufXU/zV129lb3TfmTUnK8tYfspMEtcu4dBZY2dAT80mclxXDBrGye2Vbpngz1ulGHxcz28Me0bx79qIxXdldz7ZKJfSL/+OgIHrlpCatPnYjD5eKeS2e5I36AlLgon4i+rr2XFVPSefLLy7hqUR6fmZvj/tF7R/Qt3TZuPr2QN759Fi/fdoZ7YA1ACMEPPzOL9h47W8qa3KIbjLl5ybT3Otx+elOnjfZeB4V+79NZ0zKZMSHR7R/PzEnigDH2cbCmnR67yz2eYEaJ4PluBMIU9Z+9eoDmLjtfO8e31PYXVhRx4/IC91iC2ZPwz/t+bnsV7T0Ovnh6Yb/najIxLY68lFg2GwO9ZsG1pUVpbDxU7x67auq0cby1h9l+Qm9evJ7fUUWXzckZUzO4dnE+e4+1uXs5xXUdbDxUz43LC3yKgJmBwxsBBoWL6zqY4mXbABSkxZEYE8GTmys499cbqW/v5ctnTgZUBtAZp2Tw3CfHAgZXZiBWkB78MwA1p+KgX0SfkRDt810daUIR+jzA+5JWZWzrgxAiDrgIeM5rswTeEEJsF0LcEuxFhBC3CCG2CSG21dfXB9vtxCh+G3paYc41aq3XudcqsZcyJOvmxR3HiIqwcPWifJJig0T0zd3kp8SyuCCV/dVtQWd5/mdvDTsrW/i4pDHo65U1dpGVGO3u1k4woqnjLX3tmx67M2B7TN45VMfLu6qJi7JSEiBvuL3XQZfN2SeiHwjvKLCssStodHPDsgKau+ys31tDeWMnT22pYOGkFLKCXFhM++b9Iyrr5fMrCgPuF2G18Mur5/HJPeezuMC3TkhaXJTPYGx9Ww9ZiTGsmJLBbz+3gAnJMcRFWRHCE9H32J302F1kJEQzLTuROYYI+p/zmqVqvCKYP28yK0cd77a3AgxYA/z4s7N5/msr3BbAzJwkiuvasTlcPGWklS41BpjT4qPctou3TebPFMNS2lTSyNKiNBYX+C6cM31CIvdeMYcIYxCxMD2eqAiLj9DXtvXwu7cOc2phqvtCEwrLitLYUtqElJLt5c0UZcRz9aI8atp63Pn75lwB8z0yyU2JIdIqeHX3cawWwWlT0rl8QR6RVsGTmytwuiSPflRKlNXiHpw3KUiPZ3JGPJv9fldSSo7Wd7gvpCYWi2BefjK7qlrJTIrhyS8v44ypnmDvigV5HGvpDjhIXdbQSUaCZ4wkGNMnJFJc1+4eB6jv6CUjcWTTKf0JRegDXXaCeQefBT70s21Ol1IuQlk/XxdCnBXoQCnlw1LKJVLKJZmZoeWyD5q9zyJj09hqna9mw+Utgo5aaD/uFvrizhhagkTMx1t7mJgaS3JcJCnBrJvmLvJSY1k4MQWHS7rT+PwxC1lVBxBtk4rGLh9ByE2JMdrRd1LLd57eyXV/2RTwebpsDn74wl6mZMZzy1mTqWvv7TPRxfT+J/h1zQfCjAK3lDbR1GkLaiWsmJJOYXocf3qnmKv+/BHddic/+uzsfp/b7JKvmpHVb/cYCPhjS42PpNkYjHW5JPUdvWQl+fqiQgjioyLcQm9+pilx/f94v3P+NJYVpbFqRuAeicnU7ASsFuE1jhE4CoyKsPhYXjNzErE7Jf/Ze5yntlRw4/ICJnhdhM2exKS04O9LYkyk+8L9tZUDL5wTYbUwNSuBA4aouVyS7z2zi167i19ePW/A471ZNjmNxk4bxXUdfFLezKJJqZw1Tf2uzRm1+4+r34a/dRNhtTAxNQ6bw8WCiSkkxUSSGh/FhbMn8PjH5WpS0pZKLl+QS0YAn3tyZkKfwmg1bT102Zw+A7Em914+h8f/eykvfm0FK6b49ugXGHZZoF5wWWOn22bqj5kTkrA7pTvAaujoHVV/HkIT+irA2xzNB4LNSliNn20jpaw2buuAF1BW0Ohj64RD/6Gx4GKu/es2fvbaAchdpB479ony6KOTufGfu/jBC4HLy7b12EkyfLXk2MiA1k1lczf5qXHuL0igAdmmThsfGf5ldT8zEcsaO30G28wf+nG/Adnjrd2s31vDwZp26tv7lix4cUc1x1q6+dkVc93dfv+o3nzOwUb0ZhS43rBiggmyxSK4ftkkius6iI+O4LmvrmDBxJR+n/uMUzJYs3QS371g+qDaZJIWH+W2uZq7bNid0h0Je5MQHeG2bkyhH8g/TU+I5t9fOY25+X0jfm9iIq1Mzoj3ieitFkF+av/dfXMC0d0v7CUxJpJvrvJdD2Gm8fhAtsGCiSksmJjC2dNCC56mT0hkW1kTv3vzML958xDvH2ngnktn9YmEB8Ic4P331koaO20sKUwlJzmWGRMSefdwPS6XZFuZKgERaLKQeV5nnOIR3v+7eh73XTOPG5cXcN7MbG47N/BiQEUZcZQ1dvrYLea4Q6DzmJyZwJlTMwMOqBalq1z7fQFKWZQ3dg34/gPMyVOflVmDqKHdFvACNZKEsjj4VmCqEKIIOIYS8+v9dxJCJANnA//ltS0esEgp243/LwB+OhwNHzSH/gP2Lo5kXgjAox+VMSdrKtdYIqD6E7UYeHwGjXU23txfS1Onrc8XsLXb7t6WEsC66ex10NRpY2JaLFmJaoAtkNC/vq8Gp0uSmxxDdZD6HGY+vvegXUZ8NJFW0Ufo126pxPxOf1LR3GeA851DdeSlxLJ8cpr7C1/S0MF8L6GtMdoxYZAevRkFbjVKDvQXed+wrACnC65dkh/SFz3S6qkXMhRSDY9eSkmdcQHMSux7fvHRVnd6pXnxTokdvq71jJwkd2XJssYu8lJigw5umxRlqAtoR6+D/3fpLHfKpMmNywuYlBY34Pv4+9ULcLkIuSri11ZOoaa1hz9sOIKUcP6sbNYsnTjwgX4UpMcxISmGJ40BeNNWO3t6Jn//oJTLHviAvcfa3BaYP4UZ8XConjO9bJT46AiuXTJwWwrS4+l1uKht73EPHptpplOyBo7AvbFYBLNyk9jrV5Ki2+bkeGsPRSFE9EUZCcRHWdlT1cJVC/NURD+KqZUQQkQvpXQAtwGvAweAp6WU+4QQtwohbvXa9UrgDSmld6iYDXwghNgFbAFek1KuH77mD4J9L0BiDgej5wAq0rnrlWK6UqYZEX09rvhMbA4Xdqd01yLxpq3b7h4QTI6NpK3H7hM1mHVCzGht4aTUgJk3r+0+TmF6HGdPzwpq3Zg5+JO8vkgWiyA7KcbHunE4XazdWsFpk9OJslrcgmJic7j4qLiBldNVxDLJmFwTLKIf7GAsqCjQJUEINRAXjPjoCL66csqoRTNp8VE4XJJ2Y8Yu0Me6ARXRtxsRvWnbDWTdDIaZOYkca+mmtdtOeWNnSFFghJEzPzkj3j1r1ZuJaXH81/K+2/2JjrD6pC4OxClZiTz55eV8fNcq7rtmHr/53Pwhlc4VQrBschpdNieJMRGcYkTS583Mxu6UtHTZ+e3n5vOzK+YEPP68mdmcOyPLJxgJFU+9HI99U1zfQVJMxJAsk9m5Ktfeu4qs+fssGMBSBJWBZ07Mau2243DJUY/oQ8qjl1Kuk1JOk1JOkVL+3Nj2kJTyIa99HpVSrvY7rkRKOd/4m20eO+rYu+HoBpjxGeo67ERYBP/4wqmkxkWx1VYI1Tugsx5HbLr7kGe29a3P0tptd3fpk+OikBLaezzZGmZKl5kqNz8/merWHp8KkI0dvWwqaeQz83LIS4mhqdMWcMDWM2jnKwq5ybE+Ef3bB+uobevl5tMLmZOX1GdJuW3lTXTanO6ue3SElYmpsX2Evqa1h4yE6AEjzUCYk0Jyk2P7TTUcbVKNSVPNnTZ3QbhA1k38EKybwWDaLAePt7lrAYXCgzcsZu1XlgeccTnSZCfFBMx0GgymfbNoUqo7w+TUwjRevf0M3v7u2Vy1KB9rkMyT00/J4O9fOHVI524OUJd7VcA8WtfJlKyEIV205uap4melDZ6002C/z2CYE7PM3+5oTpaCT8vM2JJ3wd4F0y+hrk11m1Ljo5iWnch+MQV6WqDhCLZo9cWck5fE/uNtPgOpUkraehwkxSq3K8UQAu9JU1XNZkSvhN4c+PEW1df31eJ0SS6Zm+NO2wvk05uTNAr8BtsmJPtG9E9urmBCUgznzshicUEqu4+1+kx4efdQPZFWwQovr3NyZkKfomvHW3sG7c+bmL5/sIlDY4VpszV12vq1bgJ69MMY0Zt++4dHG2nvcYQ0gAfqsw7U3vGCmZ+/xC8bak5e8oisi2qSk6wKo5V6C32AjJtQMTOvzBW8wDM5MNTP0pyY9dFRNV/nZByMHf8cWgdRiVB4BnXtPe6oLiE6gt1S5cwinfREKaG/fmkBUVaLT1TfaXPidElPRG/cevv0Vc1dREdY3B/i5Az1xfKOBLaUNpKdFM2snCQvoe9r35Q1dpISF9lHcHJSYqht7cXlktS29fDekXquO3UiEVaVu29zuNz1Q0BNUDm1MM2njO6UTFVLxNt2qmnt8cnqGAxmRN/f5J2xwPS1m7ts1Lf3khgdEdDGSIj2ZN20dNmxWoR71abhICsxmrT4KNbvVZlWoUaB453JmQk8ctMSPh9i/v1w4S6MZlg3tW091LX3ur+ng2VKZjzRERb3YCqosZa0+KiQe37mxCyzGmzmSZheOb5xueDweph6HkREq4L/RpSUEBPBXlseRKj7XZEqAslNieH82dm8tNMzUcIUdLMra3q43pk3lU3d5KfGuruHeakqsijxmvp/uLaD6ROSEEKQawwUBRqQrWjqoiCA352T5Jk09f6RBqRUswHBM5Xe9OmPt3ZzqLa9T8bF5MwEeh0un9rjgWbFhkpmYjRfOXsyVy0auRVyhoJZx6a5005dew+ZAfx5UNaNW+i7bSTHRg7rkm5CCGbmJLrzx/0nS4Uz583KHtEFNYJRlBHvtlfMRAGzUulgibBamJmT5FPlMtSxFhNzYpbZlsyE0e2phb/QV3+icuWnXwKo2ZHmgFxCdAStNmCCyuzoiEhxb19SkEpzl50WQ+Db/LzbgBF9S5dP2pzVIihIj6PUsG6cLjVpY5ph6WQnRyNEX+umx+5kV2VLwMk4OSmeSVMfHKknPT6KmYZ1kpUYw6S0OLaVKaF/95C52pBvWYfJhtCY9k1nr4O2HseQI3ohBHddPHNQE2pGA++Ivq6tN6A/D/4evcNtyw0n5mdkER5rTzNyeBdG21bWTGyktU++/mCYk5fE/uo2d+BX1tA5qB6sxSKYm5eM3SmJslrcFvBoEf5Cf/A1EFaYej42h4umTpuPddPR60DmLACgzZICqB++x99V3q47oncPxpoevbd1093nR1yUEe+O6Cubuuh1uNwCHh1hJTMhus9M1zf219LW4wi4hqQZdVe3dvNBcSNnTM3wmUptLilX1dzF2q2V5CTHMC3b15uc4jd2UNM2tBz6k53E6AgiLMLt0QfzuxOirdidkl6Hk5Yum/szHk5mei2sMZL+tEZRmK4Ko9W197KltIlFBSknNKg9J1eVsqho6qLH7qS6tWfQVqU55yIjIWrEFgEPRvgL/aH/QMEKiE11Z7+YKYQJMRG4JNjyTwNhoTFS5Z/HR0W4058aO9Rga7CI3tze3mOnpcveJ72wKDOe8ka1lqS5vugpXsKbkxLbx7p5ZlsleSmxnDY5HX/MvOCNh+po6Oj1mVACuJeUO+fXG9l/vI3bz53a50uVHh9FUkwEJcbYgXtWbFJ4RZpCCFLjowyh7wka0XsqWKoyEsOZWmkyw5hBHGrGjebEMO2xPVWtHKhpY0nB0GwbE/eAbHWrO7uuv/ITgZhnrGg12hk3EO5C39kA9Qdg6gUAXpkXnogeoLXwErhtGw1WZXHER1t9Mjagr0cfHWElNtLqzrv2z7gxmZKRgN0pOdbc7V4EYqrXNOy8lBgfr7y6pZsPihu4elFewKJH6fFRRFqFexm9M6f6+u9nT80kPT6KKxfm8c73VvapBQJKACdnJrgj+qHOih0PpMVFGVGYK2AOPeBT2Kylyz4i1s3ULFWbXgv96OBdGE1KVVDtRJianUBUhIU/bSjm+R1qjk2oGTcm5jKQo51xA6HNjB2/dBgV7FLUbDpPLrUR0Zslam1OsjKn0NFbDKgffrrhxTUaQm/WoPceZU/2mh3ryaHvG9GDmol6pLad3OQYn7osucmxvHNQVfQTQvD8J+qLec3iwDMALRbBhOQYKpu6OSUroY+vPik9ju33nD/gWzMlM4EPipWH754VG4ZCnxof6S5IFdy6Ud+D9h6Hz1yJ4SQqwsI/v7hUC/0okZOsCqO9tb8Oq0UMWG5jIKIjrPz+ugXc++p+Htx4FCCkWbHe5KfGkpUYPSZjNOEt9F1GBbs4ZW/4z470r0Xe2esgwiKIjrC4B/L8I/qEGM9blhLnqXcTbAKF+cMuqe/kcG2Hu3SsSU5KLN12p4ok4yJ5dnsVy4rSfGrc+JOTFEtlU7fP9PDBMjMn0Viso5XjrT2kxkWeVJOdhos0w7qBwJOlwBPRt/fYaeuxkxw3Mqlvp03pa8VpRgazMFpJQyfz85N9atAPlUvm5nDujCz+9XE59R29g55rIYTgua+uGJExoIH4lAi9+oHVtfcihLI/wCPaHT0eoY+PjkAIQaRVkBQT4V63s63b7l61yMS7VHFpQxepcZGk+ImEWuotguL6Do7Wd7DC78eeZ1SkPNbSzYGaNsoau3xWCQpEjnHMiQj9506dyIMbj3Lvq/uJj44YdNXK8UKq1+cxkHVT09aDlIyIdaMZfQqNRIglQ0yrDERMpJUvGfXqh0J/JUJGkvD26P2Evr69h/T4aHf9bf+IvqPX6TOxKD0h2mPdeNW5MfEubFbW0BkwP9r0w987XO+TcWNiTpo63trDQ++WkJEQxaVeK+YEojA9nphIi3uK+VBIionk2+dPY3NpEx8UN4SlPw/4FKbLDGLdJBoXfHOcZSQGYzWjj5nnPtT8+XAizIXeKIsfpz5o/1zqQNZNfLTHvvDu9rf19PVuvT36ssbOoJ7d5Ix4t4hM9Ut1NLNo3thXw3uH67n59KIBLZRbzprMa98484S7o6tPnci07ARsDteQipmNB8weVnSEhaSYwO+X+T6an9FoLvGmGTnm5ScTF2U94YHYcCDMhb4RopPBqn643pOlwMu6MYXe5vBZ/MFb6Fu77X0mOZgevbtkaZCBNu/t/h59enwUUREWntleRWJ0RMBKhf7ER0cMuW6HNxFWC/dcqpb/NS2kcCMtXn32WUnRQXOXE4zP3Mx+0hF9eHD5/Dw23bUqYL37TxvhLfSdDe5oHuiTS+2dbQEqovexbuKjvKwbR8CIvtvu5Iix2n2wqe1msa/c5Bif5weVRZNr2CY3nlYw6tPFz5yayV9vWsL1ywa+wIxHTI++v+JgZi/OnKGcPIy16DVjh2WUF+A+mQlvoe9qdC/27XRJ6v1mR0ZHWIiwCPf0985eZx/rxly4ojWAR29mZ+wyFgEfKKL3j+ZN1GxJC188o2gIJ3ninD8rO2yjHvO8gmXcgOrZxERaOKY9ek2YEv5ZN0m5ADR29uKSvpkXQggSYiK8BmMdPr63uXBFW7cjqEcPsMMQ+mARfVFGPBZB0Op5t517Ci1d9lFfjODTgCei7/+9TYiOoMGYBa2jQE24EeZC3+QuWFbX5jsr1iQhOsKTXmnzs24SlEjUtquFhf3zX800vJ2VLWQmRvexZUzioiJ49OalQYsq+S9IrBk+zMVUJg0wuSXeEPr4KOuYLPSh0YwkIX2jhRAXCSEOCSGKhRB3Bnj8+0KIncbfXiGEUwiRFsqxI0pXo9ujNxfN9k+x815GrrNPRK8uCqVGUbJgEX1JffCMG5OzpmXqiH0MiI2y8trtZ3BDgFIQ3sQbA7L+8yA0mnBgQKEXQliBB4CLgVnAGiHELO99pJT3SSkXSCkXAHcB70opm0I5dsSwdYGjmy21goM1bdS1B15KzlxdqNfhxO6UfQZjwbOaTKCsG5PBFjjSjB5TsxMHTFk1P3dt22jCkVCsm6VAsZSyBEAIsRa4HNgfZP81wFNDPHb46FJLdj1zsIf/FG9iSaGqle4/OzIhJoKmThudvWr5vbgo38FYCB7Rp3hlZxRlnHi6o2bsMFNttdBrwpFQrJs8oNLrfpWxrQ9CiDjgIuC5IRx7ixBimxBiW319fQjNGgBjVmy7SCIvJZaNh+pJiYvsUwvc9OjNzBv/wVjwCL1/1k1iTARmanaRjujHNebnrjNuNOFIKEIfaJaJDLAN4LPAh1LKpsEeK6V8WEq5REq5JDMzM9Aug0J2KqGfmD+RZ756GmdNy2SuUVPaG3PxkU6bw33fJCbSSnyUNWhEb7EIt/h/mpaHC0cSjLRaLfSacCQU66YK8K6Zmw9UB9l3NR7bZrDHDivHqqvIBxZMm0xSTCSPfXEpUva9xriFPkBED6reTYVRgjhQ1TmzDEJBmhb68Yw5GKsnS2nCkVAi+q3AVCFEkRAiCiXmL/vvJIRIBs4GXhrssSPBkbJyAJbPnebdxj77JcRE0GVz0tZtRvS+1o73RKJA/m1KXCS5yTHERoVfid9PE9qj14QzA0b0UkqHEOI24HXACvxdSrlPCHGr8fhDxq5XAm9IKTsHOna4TyIQNdVVOLGQnp7V736mVWNm5fSJ6A2hj7JaiI7oe12cn59Cj905HE3WjCEJ2qPXhDEhTZiSUq4D1vlte8jv/qPAo6EcO9JUNHbh6mzEFptCrKX/Tov5A69pVXn28VG+b4kZ0SfFRgbsEdx7xZzhaLJmjHEPxuqIXhOGhOUUwDcP1JIq2rEmDjzj1Oyy17QFjujTEkyhD+9JxJ924nUevSaMCUuhr2ntJsPSTlTiwNk7ZkRf6xZ6X6/dtG60AIQ307MTSY+PYvIwlH/WaE42wjJMtTlcpIkOiDtlwH29hT7SKvrk2ZtlEEa7fLBmdJk+ITGkRdU1mvFIWEb0vQ4XqbS7lxDsD9O6qW3rDbhik47oNRrNeCcshd5md5BMW2hCb4h7Y2dvn4FY8B6MDcvOj0aj+RQQlkIvbG1E4IK4EAZjDaGXkoBlhtN0RK/RaMY5YSn00bZm9U8IEb23XeM/EAuQmRhNYkwEBQOUIdZoNJqTlbD0I6JtLeqfEIQ+0lhGrsfuCujRx0Ra+eB/znV7+RqNRjPeCMuIPsYt9Gn97meSEK1smUAePUByXCRWS6D6bBqNRnPyE5ZCH+tsUf+EENGDp75NoIheo9FoxjthKfRx9hb1T3xoa7Gatox/QTONRqMJB8JS6OOdbTiIgMjQFgMxs210RK/RaMKRsBT6SFcPNkssBChCFggt9BqNJpwJS6GPcvVgs8aGvL8p9IHy6DUajWa8E55CL7txWGJC3t/06OP04iEajSYMCUuhj3b1YB9URB9p3OqIXqPRhB9hJ/RSSmJkL86IwQi9Tq/UaDThS0hCL4S4SAhxSAhRLIS4M8g+K4UQO4UQ+4QQ73ptLxNC7DEe2zZcDQ+GzekiVvTiHIJHr4Veo9GEIwMqmxDCCjwAnA9UAVuFEC9LKfd77ZMC/Bm4SEpZIYTwX6j1HCllw/A1Ozg2h4tYenENJqKP0daNRqMJX0KJ6JcCxVLKEimlDVgLXO63z/XA81LKCgApZd3wNjN0eh0u4ujFFWIOPcB5M7P47vnTmJqlVxfSaDThRyhCnwdUet2vMrZ5Mw1IFUJsFEJsF0Lc5PWYBN4wtt8S7EWEELcIIbYJIbbV19eH2v4+9DqUdSMjQhf6lLgobl81FYuuZ6PRaMKQULyKQOonAzzPYmAVEAtsEkJ8LKU8DJwupaw27Jw3hRAHpZTv9XlCKR8GHgZYsmSJ//OHTK/dSSq9NA8iotdoNJpwJpSIvgqY6HU/H6gOsM96KWWn4cW/B8wHkFJWG7d1wAsoK2jEsDkcxIleiNJCr9FoNBCa0G8FpgohioQQUcBq4GW/fV4CzhRCRAgh4oBlwAEhRLwQIhFACBEPXADsHb7m98Xe3aX+idQLhWg0Gg2EYN1IKR1CiNuA1wEr8Hcp5T4hxK3G4w9JKQ8IIdYDuwEX8IiUcq8QYjLwglA1ZyKAJ6WU60fqZADsvZ0AiKjQs240Go0mnAkpn1BKuQ5Y57ftIb/79wH3+W0rwbBwRgtnjxJ6S5TOoNFoNBoIw5mxzt4OACzR2rrRaDQaCEuhVxG9VQu9RqPRAGEo9NIU+hgt9BqNRgNhKPQumxL6iBjt0Ws0Gg2EodBjCH2kFnqNRqMBwlDopb0bgAht3Wg0Gg0QhkIv7GrCVFSsjug1Go0GwlHoDetGC71Go9Eowk/ond24pEDoomYajUYDhKHQW+xd9IgoELrksEaj0UAYCr3V0U03MWPdDI1GozlpCD+hd3bRI7TQazQajUnYCX2EswebiB7rZmg0Gs1JQ1gKfa+O6DUajcZN2Al9pKsbm0ULvUaj0ZiEndBHubqxW/SiIxqNRmMShkLfg11H9BqNRuMmJKEXQlwkhDgkhCgWQtwZZJ+VQoidQoh9Qoh3B3PscBLt6sFu1RG9RqPRmAy4lKAQwgo8AJwPVAFbhRAvSyn3e+2TAvwZuEhKWSGEyAr12OEmWvbgtOqIXqPRaExCieiXAsVSyhIppQ1YC1zut8/1wPNSygoAKWXdII4dVmLoxRmhyx9oNBqNSShCnwdUet2vMrZ5Mw1IFUJsFEJsF0LcNIhjARBC3CKE2CaE2FZfXx9a6/1x2onEgVNbNxqNRuNmQOsGCFQ0RgZ4nsXAKiAW2CSE+DjEY9VGKR8GHgZYsmRJwH0GxChR7NIFzTQajcZNKEJfBUz0up8PVAfYp0FK2Ql0CiHeA+aHeOzwYTOEPkJH9BqNRmMSinWzFZgqhCgSQkQBq4GX/fZ5CThTCBEhhIgDlgEHQjx22JBGLXoZqVeX0mg0GpMBI3oppUMIcRvwOmAF/i6l3CeEuNV4/CEp5QEhxHpgN+ACHpFS7gUIdOwInQuO3k4iAbR1o9FoNG5CsW6QUq4D1vlte8jv/n3AfaEcO1I4ejoModfWjUaj0ZiEJPTjBXt3B7GAiNLWjUYzHNjtdqqqqujp6RnrpmgMYmJiyM/PJzIyMuRjwkroHb0dAFiitdBrNMNBVVUViYmJFBYWIvSqbWOOlJLGxkaqqqooKioK+biwqnXj6lVZNzqi12iGh56eHtLT07XInyQIIUhPTx90DyushN5pRPTWGC30Gs1woUX+5GIon0d4CX2Piuit0Qlj3BKNRqM5eQgroZc2HdFrNBqNP2Em9F3YpZWoKL1mrEbzaSQhIXhvvqioiEOHDvls+9a3vsWvfvUrAHbs2IEQgtdffz3k5/TmoYce4rHHHhtki0eHsMq6wdZJN9FER4TV9UujOSn4ySv72F/dNqzPOSs3iR99dvawPmcwVq9ezdq1a/nRj34EgMvl4tlnn+XDDz8E4KmnnuKMM87gqaee4sILLxz08996663D2t7hJLwU0d5FF9FER1jHuiUajWYYuOOOO/jzn//svv/jH/+Yn/zkJ6xatYpFixYxd+5cXnrppZCea82aNaxdu9Z9/7333qOwsJCCggKklDz77LM8+uijvPHGG0OaN/DjH/+YX//61wD89a9/5dRTT2X+/PlcffXVdHWp8cPa2lquvPJK5s+fz/z58/noo48G/TpDIbwiens3XTKaKB3RazTDzmhF3t6sXr2ab33rW3zta18D4Omnn2b9+vV8+9vfJikpiYaGBpYvX85ll102YDbKvHnzsFgs7Nq1i/nz57N27VrWrFkDwIcffkhRURFTpkxh5cqVrFu3jquuumrI7b7qqqv48pe/DMAPf/hD/va3v3H77bfzjW98g7PPPpsXXngBp9NJR0fHkF9jMISVIgpHl7ZuNJowYuHChdTV1VFdXc2uXbtITU0lJyeHH/zgB8ybN4/zzjuPY8eOUVtbG9LzmVG9w+HgpZde4tprrwWUbbN69WpAXVyeeuqpE2r33r17OfPMM5k7dy5PPPEE+/apEl8bNmzgq1/9KgBWq5Xk5OQTep1QCauI3mJYN5mRWug1mnDhmmuu4dlnn6WmpobVq1fzxBNPUF9fz/bt24mMjKSwsDBkq2XNmjVccMEFnH322cybN4+srCycTifPPfccL7/8Mj//+c/ds0/b29tJTEwcUpu/8IUv8OKLLzJ//nweffRRNm7cOKTnGS7CShGtjm66ZTTRVu3RazThgjmI+uyzz3LNNdfQ2tpKVlYWkZGRvPPOO5SXl4f8XFOmTCE9PZ0777zTbdu89dZbzJ8/n8rKSsrKyigvL+fqq6/mxRdfHHKb29vbycnJwW6388QTT7i3r1q1igcffBAAp9NJW9vwDm4HI7yE3mlYNzqi12jChtmzZ9Pe3k5eXh45OTnccMMNbNu2jSVLlvDEE08wY8aMQT3fmjVrOHjwIFdeeSWgbBvzf5Orr76aJ598EoCuri7y8/Pdf7/97W+DPrc5TnDvvfeybNkyzj//fJ/23X///bzzzjvMnTuXxYsXuy2dkUZIObRV+0aSJUuWyG3btg36uNZfzGRDVxGX/+Q1LBY9bVujOVEOHDjAzJkzx7oZ44Lbb7+dRYsWcfPNN4/4awX6XIQQ26WUSwLtH1ahb4Srm15itMhrNJpR5Z577mHz5s1cdtllY92UgITVYGyks4deS8xYN0Oj0Ywhe/bs4cYbb/TZFh0dzebNm4fl+X/+85/zzDPP+Gy79tpr2bJly7A8/0gQktALIS4C7kctB/iIlPKXfo+vRK0bW2psel5K+VPjsTKgHXACjmBdi+Hgnayb+Lgum8+P1AtoNJqTnrlz57Jz584Re/67776bu+++e8SefyQYUOiFEFbgAeB8oArYKoR4WUq532/X96WUlwZ5mnOklA0n1tSBeSv9BnY1j/jLaDQazbgiFI9+KVAspSyRUtqAtcDlI9usodHrcOlZsRqNRuNHKKqYB1R63a8ytvlzmhBilxDiP0II77nSEnhDCLFdCHHLCbR1QHodTl3nRqPRaPwIxaMPlMLin5P5CVAgpewQQlwCvAhMNR47XUpZLYTIAt4UQhyUUr7X50XUReAWgEmTJoXafh9sDpfOoddoNBo/QlHFKmCi1/18oNp7Byllm5Syw/h/HRAphMgw7lcbt3XACygrqA9SyoellEuklEsyMzMHfSJgWDdWLfQaTbjQ0tLiU70yVC655BJaWloGdcyjjz7qni1r0tDQQGZmJr29vQBcfvnlnHbaaT77eFetHIgVK1YMqk3DRSgR/VZgqhCiCDgGrAau995BCDEBqJVSSiHEUtQFpFEIEQ9YpJTtxv8XAD8d1jPwotfhIkZH9BrNyPCfO6Fmz/A+54S5cPEvgz5sCr1ZvdLE6XRi7afUybp16wbdlKuuuorvfe97dHV1ERcXB8Czzz7LZZddRnR0NC0tLXzyySckJCRQWlpKUVHRoF9jtMoS+zOgKkopHcBtwOvAAeBpKeU+IcStQgiz0v41wF4hxC7gD8BqqabcZgMfGNu3AK9JKdePxImAYd1oj16jCRvuvPNOjh49yoIFCzj11FM555xzuP7665k7dy4AV1xxBYsXL2b27Nk8/PDD7uMKCwtpaGigrKyMmTNn8uUvf5nZs2dzwQUX0N3dHfC1kpKSOOuss3jllVfc27xLGT/33HN89rOfddfeGQrmalUdHR1Ba+o/9thjzJs3j/nz5/eZDzBkpJQn3d/ixYvlUDj/txvlVx7bNqRjNRpNX/bv3z+mr19aWipnz54tpZTynXfekXFxcbKkpMT9eGNjo5RSyq6uLjl79mzZ0NAgpZSyoKBA1tfXy9LSUmm1WuWOHTuklFJee+218vHHHw/6ek8//bS84oorpJRSHjt2TObk5EiHwyGllHLVqlXyvffek4cOHZJz5851H/OjH/1I3nfffSGdT3x8vJRSSrvdLltbW6WUUtbX18spU6ZIl8sl9+7dK6dNmybr6+t9zs+fQJ8LsE0G0dSw8jl69WCsRhPWLF261Mcy+cMf/sD8+fNZvnw5lZWVHDlypM8xRUVFLFiwAIDFixdTVlYW9PkvvfRSPvjgA9ra2nj66ae55pprsFqt1NbWUlxczBlnnMG0adOIiIhg7969Qz4PKWXAmvobNmzgmmuuISMjA4C0tLQhv4Y3YaWKyroJq1PSaDRexMfHu//fuHEjb731Fps2bWLXrl0sXLgwYF366Oho9/9WqxWHwxH0+WNjY7nooot44YUXfGybf//73zQ3N1NUVERhYSFlZWVDtm8An5r6O3fuJDs7m56eHqSUA66UNRTCShV7tUev0YQViYmJtLe3B3ystbWV1NRU4uLiOHjwIB9//PGwvOaaNWv47W9/S21tLcuXLwdUKeP169dTVlZGWVkZ27dvPyGhD1ZTf9WqVTz99NM0NjYC0NTUdOInRLgJvd2pZ8ZqNGFEeno6p59+OnPmzOH73/++z2MXXXQRDoeDefPmcc8997hF+US54IILqK6u5rrrrkMIQVlZGRUVFT7PX1RURFJSkrtQ2s9+9jOfmvXBMKP1YDX1Z8+ezd13383ZZ5/N/Pnz+c53vjMs5xRW9ei/tXYHZ03L5KpFwd9ojUYTOroe/fDR2NjIokWLBrUiVjAGW48+rMoU/371wrFugkaj0fShurqalStX8r3vfW9MXj+shF6j0WhC4etf/zoffvihz7ZvfvObw7I6VGNjI6tWreqzfdOmTaSnp5/w8w8FLfQajaZfRioTZCx54IEHRuy509PTR7Qe/lDsdj1yqdFoghITE0NjY+OQxEUz/EgpaWxsJCZmcCvp6Yheo9EEJT8/n6qqKurr68e6KRqDmJiYfjN7AqGFXqPRBCUyMnJIxbs0JxfautFoNJowRwu9RqPRhDla6DUajSbMOSlnxgoh6oGhTh/LABqGsTmjzXhvP+hzOFkY7+cw3tsPo3sOBVLKgMvznZRCfyIIIbYFmwY8Hhjv7Qd9DicL4/0cxnv74eQ5B23daDQaTZijhV6j0WjCnHAU+ocH3uWkZry3H/Q5nCyM93MY7+2Hk+Qcws6j12g0Go0v4RjRazQajcYLLfQajUYT5oSN0AshLhJCHBJCFAsh7hzr9oSCEGKiEOIdIcQBIcQ+IcQ3je1pQog3hRBHjNvUsW5rfwghrEKIHUKIV4374639KUKIZ4UQB43P4rRxeA7fNr5De4UQTwkhYk72cxBC/F0IUSeE2Ou1LWibhRB3Gb/vQ0KIC8em1b4EOYf7jO/SbiHEC0KIFK/HxuQcwkLohRBW4AHgYmAWsEYIMWtsWxUSDuC7UsqZwHLg60a77wTellJOBd427p/MfBM44HV/vLX/fmC9lHIGMB91LuPmHIQQecA3gCVSyjmAFVjNyX8OjwIX+W0L2Gbjd7EamG0c82fjdz/WPErfc3gTmCOlnAccBu6CsT2HsBB6YClQLKUskVLagLXA5WPcpgGRUh6XUn5i/N+OEpg8VNv/aez2T+CKMWlgCAgh8oHPAI94bR5P7U8CzgL+BiCltEkpWxhH52AQAcQKISKAOKCak/wcpJTvAU1+m4O1+XJgrZSyV0pZChSjfvdjSqBzkFK+IaV0GHc/BsyawmN2DuEi9HlApdf9KmPbuEEIUQgsBDYD2VLK46AuBkDWGDZtIH4P/A/g8to2nto/GagH/mHYT48IIeIZR+cgpTwG/BqoAI4DrVLKNxhH5+BFsDaP19/4F4H/GP+P2TmEi9AHWuds3OSNCiESgOeAb0kp28a6PaEihLgUqJNSbh/rtpwAEcAi4EEp5UKgk5PP4ugXw8e+HCgCcoF4IcR/jW2rhp1x9xsXQtyNsmefMDcF2G1UziFchL4KmOh1Px/VdT3pEUJEokT+CSnl88bmWiFEjvF4DlA3Vu0bgNOBy4QQZSi77FwhxL8YP+0H9d2pklJuNu4/ixL+8XQO5wGlUsp6KaUdeB5Ywfg6B5NgbR5Xv3EhxOeBS4EbpGey0pidQ7gI/VZgqhCiSAgRhRrweHmM2zQgQq24/DfggJTyt14PvQx83vj/88BLo922UJBS3iWlzJdSFqLe8w1Syv9inLQfQEpZA1QKIaYbm1YB+xlH54CybJYLIeKM79Qq1HjPeDoHk2BtfhlYLYSIFkIUAVOBLWPQvgERQlwE3AFcJqXs8npo7M5BShkWf8AlqBHuo8DdY92eENt8BqrrthvYafxdAqSjMg6OGLdpY93WEM5lJfCq8f+4aj+wANhmfA4vAqnj8Bx+AhwE9gKPA9En+zkAT6HGFOyoaPe/+2szcLfx+z4EXDzW7e/nHIpRXrz5m35orM9Bl0DQaDSaMCdcrBuNRqPRBEELvUaj0YQ5Wug1Go0mzNFCr9FoNGGOFnqNRqMJc7TQazQaTZijhV6j0WjCnP8Pjttg+PctwRwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_VAL_jac','train_VAL_jac']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "unlabel_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=~(data.train_mask + data.val_mask + data.test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def predict_Homo(model, loader):\n",
    "    model.eval()\n",
    "    seed_everything(args.seed)\n",
    "    all_preds = []\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch.batch_size\n",
    "        #edge_index = to_undirected(batch.edge_index)\n",
    "        out = model(batch.x)[:batch_size]\n",
    "        out_att = out[:,:9].softmax(axis=1)\n",
    "        out_val = out[:,9:].softmax(axis=1)\n",
    "        IDs = batch.n_id[:batch_size].unsqueeze(dim=-1).int()\n",
    "        \n",
    "        now = torch.hstack([IDs, out_att, out_val])\n",
    "        all_preds.append(now)\n",
    "    \n",
    "    final = torch.vstack(all_preds)\n",
    "        \n",
    "    return final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 26.75it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 70.29it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 68.16it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 38/38 [00:00<00:00, 73.47it/s]\n"
     ]
    }
   ],
   "source": [
    "pred_train = predict_Homo(model, train_loader)\n",
    "pred_val = predict_Homo(model, val_loader)\n",
    "pred_test = predict_Homo(model, test_loader)\n",
    "pred_unlab = predict_Homo(model, unlabel_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = torch.vstack([pred_train, pred_val, pred_test, pred_unlab]).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df = pd.DataFrame(preds).sort_values(0).set_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.021555</td>\n",
       "      <td>0.099800</td>\n",
       "      <td>0.012440</td>\n",
       "      <td>0.009709</td>\n",
       "      <td>0.238287</td>\n",
       "      <td>0.136117</td>\n",
       "      <td>0.042519</td>\n",
       "      <td>0.006944</td>\n",
       "      <td>0.432629</td>\n",
       "      <td>0.056437</td>\n",
       "      <td>0.318523</td>\n",
       "      <td>0.264389</td>\n",
       "      <td>0.231813</td>\n",
       "      <td>0.015072</td>\n",
       "      <td>0.095696</td>\n",
       "      <td>0.002628</td>\n",
       "      <td>0.001783</td>\n",
       "      <td>0.002582</td>\n",
       "      <td>0.003020</td>\n",
       "      <td>0.008057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.011781</td>\n",
       "      <td>0.037312</td>\n",
       "      <td>0.006450</td>\n",
       "      <td>0.004553</td>\n",
       "      <td>0.040292</td>\n",
       "      <td>0.043186</td>\n",
       "      <td>0.010875</td>\n",
       "      <td>0.004713</td>\n",
       "      <td>0.840839</td>\n",
       "      <td>0.119712</td>\n",
       "      <td>0.412343</td>\n",
       "      <td>0.214445</td>\n",
       "      <td>0.125116</td>\n",
       "      <td>0.016628</td>\n",
       "      <td>0.082014</td>\n",
       "      <td>0.005115</td>\n",
       "      <td>0.003278</td>\n",
       "      <td>0.004943</td>\n",
       "      <td>0.005984</td>\n",
       "      <td>0.010422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.047308</td>\n",
       "      <td>0.089104</td>\n",
       "      <td>0.019904</td>\n",
       "      <td>0.012865</td>\n",
       "      <td>0.214152</td>\n",
       "      <td>0.330976</td>\n",
       "      <td>0.058205</td>\n",
       "      <td>0.014936</td>\n",
       "      <td>0.212550</td>\n",
       "      <td>0.109006</td>\n",
       "      <td>0.324120</td>\n",
       "      <td>0.258103</td>\n",
       "      <td>0.159002</td>\n",
       "      <td>0.012958</td>\n",
       "      <td>0.109527</td>\n",
       "      <td>0.004611</td>\n",
       "      <td>0.003401</td>\n",
       "      <td>0.003815</td>\n",
       "      <td>0.004318</td>\n",
       "      <td>0.011139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.030690</td>\n",
       "      <td>0.140461</td>\n",
       "      <td>0.035073</td>\n",
       "      <td>0.012410</td>\n",
       "      <td>0.009325</td>\n",
       "      <td>0.031827</td>\n",
       "      <td>0.684635</td>\n",
       "      <td>0.046914</td>\n",
       "      <td>0.008666</td>\n",
       "      <td>0.059222</td>\n",
       "      <td>0.040448</td>\n",
       "      <td>0.178818</td>\n",
       "      <td>0.074877</td>\n",
       "      <td>0.021621</td>\n",
       "      <td>0.581006</td>\n",
       "      <td>0.012006</td>\n",
       "      <td>0.008276</td>\n",
       "      <td>0.005939</td>\n",
       "      <td>0.007297</td>\n",
       "      <td>0.010490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.368223</td>\n",
       "      <td>0.291382</td>\n",
       "      <td>0.006805</td>\n",
       "      <td>0.076779</td>\n",
       "      <td>0.020741</td>\n",
       "      <td>0.173170</td>\n",
       "      <td>0.041746</td>\n",
       "      <td>0.010219</td>\n",
       "      <td>0.010934</td>\n",
       "      <td>0.189621</td>\n",
       "      <td>0.075349</td>\n",
       "      <td>0.100124</td>\n",
       "      <td>0.297412</td>\n",
       "      <td>0.015744</td>\n",
       "      <td>0.295089</td>\n",
       "      <td>0.005760</td>\n",
       "      <td>0.004349</td>\n",
       "      <td>0.002846</td>\n",
       "      <td>0.003156</td>\n",
       "      <td>0.010550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2946.0</th>\n",
       "      <td>0.027841</td>\n",
       "      <td>0.508675</td>\n",
       "      <td>0.119450</td>\n",
       "      <td>0.030187</td>\n",
       "      <td>0.012398</td>\n",
       "      <td>0.030492</td>\n",
       "      <td>0.203104</td>\n",
       "      <td>0.045802</td>\n",
       "      <td>0.022050</td>\n",
       "      <td>0.285283</td>\n",
       "      <td>0.189706</td>\n",
       "      <td>0.215706</td>\n",
       "      <td>0.182972</td>\n",
       "      <td>0.032010</td>\n",
       "      <td>0.042161</td>\n",
       "      <td>0.008416</td>\n",
       "      <td>0.007815</td>\n",
       "      <td>0.007482</td>\n",
       "      <td>0.010901</td>\n",
       "      <td>0.017548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2947.0</th>\n",
       "      <td>0.005524</td>\n",
       "      <td>0.029289</td>\n",
       "      <td>0.350625</td>\n",
       "      <td>0.023732</td>\n",
       "      <td>0.005754</td>\n",
       "      <td>0.005032</td>\n",
       "      <td>0.475743</td>\n",
       "      <td>0.094678</td>\n",
       "      <td>0.009623</td>\n",
       "      <td>0.253265</td>\n",
       "      <td>0.177734</td>\n",
       "      <td>0.251875</td>\n",
       "      <td>0.166296</td>\n",
       "      <td>0.024077</td>\n",
       "      <td>0.073485</td>\n",
       "      <td>0.008465</td>\n",
       "      <td>0.008517</td>\n",
       "      <td>0.006828</td>\n",
       "      <td>0.012216</td>\n",
       "      <td>0.017243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2948.0</th>\n",
       "      <td>0.082219</td>\n",
       "      <td>0.573929</td>\n",
       "      <td>0.071733</td>\n",
       "      <td>0.023898</td>\n",
       "      <td>0.024530</td>\n",
       "      <td>0.037873</td>\n",
       "      <td>0.124089</td>\n",
       "      <td>0.042005</td>\n",
       "      <td>0.019725</td>\n",
       "      <td>0.286234</td>\n",
       "      <td>0.194948</td>\n",
       "      <td>0.213093</td>\n",
       "      <td>0.188490</td>\n",
       "      <td>0.035445</td>\n",
       "      <td>0.033786</td>\n",
       "      <td>0.009093</td>\n",
       "      <td>0.007172</td>\n",
       "      <td>0.007361</td>\n",
       "      <td>0.010529</td>\n",
       "      <td>0.013848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2949.0</th>\n",
       "      <td>0.328473</td>\n",
       "      <td>0.250673</td>\n",
       "      <td>0.075527</td>\n",
       "      <td>0.046130</td>\n",
       "      <td>0.021026</td>\n",
       "      <td>0.164873</td>\n",
       "      <td>0.051524</td>\n",
       "      <td>0.040992</td>\n",
       "      <td>0.020783</td>\n",
       "      <td>0.322616</td>\n",
       "      <td>0.194243</td>\n",
       "      <td>0.150925</td>\n",
       "      <td>0.250832</td>\n",
       "      <td>0.022708</td>\n",
       "      <td>0.031271</td>\n",
       "      <td>0.003223</td>\n",
       "      <td>0.003686</td>\n",
       "      <td>0.003432</td>\n",
       "      <td>0.004769</td>\n",
       "      <td>0.012294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2950.0</th>\n",
       "      <td>0.043091</td>\n",
       "      <td>0.725594</td>\n",
       "      <td>0.035364</td>\n",
       "      <td>0.018479</td>\n",
       "      <td>0.014506</td>\n",
       "      <td>0.107470</td>\n",
       "      <td>0.029462</td>\n",
       "      <td>0.008090</td>\n",
       "      <td>0.017945</td>\n",
       "      <td>0.304681</td>\n",
       "      <td>0.171659</td>\n",
       "      <td>0.233563</td>\n",
       "      <td>0.197824</td>\n",
       "      <td>0.025985</td>\n",
       "      <td>0.030759</td>\n",
       "      <td>0.006103</td>\n",
       "      <td>0.005049</td>\n",
       "      <td>0.005106</td>\n",
       "      <td>0.006924</td>\n",
       "      <td>0.012346</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2951 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              1         2         3         4         5         6         7   \\\n",
       "0                                                                              \n",
       "0.0     0.021555  0.099800  0.012440  0.009709  0.238287  0.136117  0.042519   \n",
       "1.0     0.011781  0.037312  0.006450  0.004553  0.040292  0.043186  0.010875   \n",
       "2.0     0.047308  0.089104  0.019904  0.012865  0.214152  0.330976  0.058205   \n",
       "3.0     0.030690  0.140461  0.035073  0.012410  0.009325  0.031827  0.684635   \n",
       "4.0     0.368223  0.291382  0.006805  0.076779  0.020741  0.173170  0.041746   \n",
       "...          ...       ...       ...       ...       ...       ...       ...   \n",
       "2946.0  0.027841  0.508675  0.119450  0.030187  0.012398  0.030492  0.203104   \n",
       "2947.0  0.005524  0.029289  0.350625  0.023732  0.005754  0.005032  0.475743   \n",
       "2948.0  0.082219  0.573929  0.071733  0.023898  0.024530  0.037873  0.124089   \n",
       "2949.0  0.328473  0.250673  0.075527  0.046130  0.021026  0.164873  0.051524   \n",
       "2950.0  0.043091  0.725594  0.035364  0.018479  0.014506  0.107470  0.029462   \n",
       "\n",
       "              8         9         10        11        12        13        14  \\\n",
       "0                                                                              \n",
       "0.0     0.006944  0.432629  0.056437  0.318523  0.264389  0.231813  0.015072   \n",
       "1.0     0.004713  0.840839  0.119712  0.412343  0.214445  0.125116  0.016628   \n",
       "2.0     0.014936  0.212550  0.109006  0.324120  0.258103  0.159002  0.012958   \n",
       "3.0     0.046914  0.008666  0.059222  0.040448  0.178818  0.074877  0.021621   \n",
       "4.0     0.010219  0.010934  0.189621  0.075349  0.100124  0.297412  0.015744   \n",
       "...          ...       ...       ...       ...       ...       ...       ...   \n",
       "2946.0  0.045802  0.022050  0.285283  0.189706  0.215706  0.182972  0.032010   \n",
       "2947.0  0.094678  0.009623  0.253265  0.177734  0.251875  0.166296  0.024077   \n",
       "2948.0  0.042005  0.019725  0.286234  0.194948  0.213093  0.188490  0.035445   \n",
       "2949.0  0.040992  0.020783  0.322616  0.194243  0.150925  0.250832  0.022708   \n",
       "2950.0  0.008090  0.017945  0.304681  0.171659  0.233563  0.197824  0.025985   \n",
       "\n",
       "              15        16        17        18        19        20  \n",
       "0                                                                   \n",
       "0.0     0.095696  0.002628  0.001783  0.002582  0.003020  0.008057  \n",
       "1.0     0.082014  0.005115  0.003278  0.004943  0.005984  0.010422  \n",
       "2.0     0.109527  0.004611  0.003401  0.003815  0.004318  0.011139  \n",
       "3.0     0.581006  0.012006  0.008276  0.005939  0.007297  0.010490  \n",
       "4.0     0.295089  0.005760  0.004349  0.002846  0.003156  0.010550  \n",
       "...          ...       ...       ...       ...       ...       ...  \n",
       "2946.0  0.042161  0.008416  0.007815  0.007482  0.010901  0.017548  \n",
       "2947.0  0.073485  0.008465  0.008517  0.006828  0.012216  0.017243  \n",
       "2948.0  0.033786  0.009093  0.007172  0.007361  0.010529  0.013848  \n",
       "2949.0  0.031271  0.003223  0.003686  0.003432  0.004769  0.012294  \n",
       "2950.0  0.030759  0.006103  0.005049  0.005106  0.006924  0.012346  \n",
       "\n",
       "[2951 rows x 20 columns]"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df.to_csv(args.save_dir + 'preds.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "states.to_csv(args.save_dir + 'train_state.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Per-Class Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.021555</td>\n",
       "      <td>0.099800</td>\n",
       "      <td>0.012440</td>\n",
       "      <td>0.009709</td>\n",
       "      <td>0.238287</td>\n",
       "      <td>0.136117</td>\n",
       "      <td>0.042519</td>\n",
       "      <td>0.006944</td>\n",
       "      <td>0.432629</td>\n",
       "      <td>0.056437</td>\n",
       "      <td>0.318522</td>\n",
       "      <td>0.264389</td>\n",
       "      <td>0.231813</td>\n",
       "      <td>0.015072</td>\n",
       "      <td>0.095696</td>\n",
       "      <td>0.002628</td>\n",
       "      <td>0.001783</td>\n",
       "      <td>0.002582</td>\n",
       "      <td>0.003020</td>\n",
       "      <td>0.008057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.011781</td>\n",
       "      <td>0.037312</td>\n",
       "      <td>0.006450</td>\n",
       "      <td>0.004553</td>\n",
       "      <td>0.040292</td>\n",
       "      <td>0.043186</td>\n",
       "      <td>0.010875</td>\n",
       "      <td>0.004713</td>\n",
       "      <td>0.840839</td>\n",
       "      <td>0.119712</td>\n",
       "      <td>0.412343</td>\n",
       "      <td>0.214445</td>\n",
       "      <td>0.125116</td>\n",
       "      <td>0.016628</td>\n",
       "      <td>0.082014</td>\n",
       "      <td>0.005115</td>\n",
       "      <td>0.003278</td>\n",
       "      <td>0.004943</td>\n",
       "      <td>0.005984</td>\n",
       "      <td>0.010422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.047308</td>\n",
       "      <td>0.089104</td>\n",
       "      <td>0.019904</td>\n",
       "      <td>0.012865</td>\n",
       "      <td>0.214152</td>\n",
       "      <td>0.330976</td>\n",
       "      <td>0.058205</td>\n",
       "      <td>0.014936</td>\n",
       "      <td>0.212550</td>\n",
       "      <td>0.109006</td>\n",
       "      <td>0.324120</td>\n",
       "      <td>0.258103</td>\n",
       "      <td>0.159002</td>\n",
       "      <td>0.012958</td>\n",
       "      <td>0.109527</td>\n",
       "      <td>0.004611</td>\n",
       "      <td>0.003401</td>\n",
       "      <td>0.003815</td>\n",
       "      <td>0.004318</td>\n",
       "      <td>0.011139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.030690</td>\n",
       "      <td>0.140460</td>\n",
       "      <td>0.035073</td>\n",
       "      <td>0.012410</td>\n",
       "      <td>0.009325</td>\n",
       "      <td>0.031827</td>\n",
       "      <td>0.684635</td>\n",
       "      <td>0.046914</td>\n",
       "      <td>0.008666</td>\n",
       "      <td>0.059222</td>\n",
       "      <td>0.040448</td>\n",
       "      <td>0.178818</td>\n",
       "      <td>0.074877</td>\n",
       "      <td>0.021621</td>\n",
       "      <td>0.581005</td>\n",
       "      <td>0.012006</td>\n",
       "      <td>0.008276</td>\n",
       "      <td>0.005939</td>\n",
       "      <td>0.007297</td>\n",
       "      <td>0.010490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.368223</td>\n",
       "      <td>0.291382</td>\n",
       "      <td>0.006805</td>\n",
       "      <td>0.076779</td>\n",
       "      <td>0.020741</td>\n",
       "      <td>0.173170</td>\n",
       "      <td>0.041746</td>\n",
       "      <td>0.010219</td>\n",
       "      <td>0.010934</td>\n",
       "      <td>0.189621</td>\n",
       "      <td>0.075349</td>\n",
       "      <td>0.100124</td>\n",
       "      <td>0.297412</td>\n",
       "      <td>0.015744</td>\n",
       "      <td>0.295089</td>\n",
       "      <td>0.005760</td>\n",
       "      <td>0.004349</td>\n",
       "      <td>0.002846</td>\n",
       "      <td>0.003156</td>\n",
       "      <td>0.010550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2946.0</th>\n",
       "      <td>0.027841</td>\n",
       "      <td>0.508675</td>\n",
       "      <td>0.119450</td>\n",
       "      <td>0.030187</td>\n",
       "      <td>0.012398</td>\n",
       "      <td>0.030492</td>\n",
       "      <td>0.203104</td>\n",
       "      <td>0.045802</td>\n",
       "      <td>0.022050</td>\n",
       "      <td>0.285283</td>\n",
       "      <td>0.189706</td>\n",
       "      <td>0.215706</td>\n",
       "      <td>0.182972</td>\n",
       "      <td>0.032010</td>\n",
       "      <td>0.042161</td>\n",
       "      <td>0.008416</td>\n",
       "      <td>0.007815</td>\n",
       "      <td>0.007482</td>\n",
       "      <td>0.010901</td>\n",
       "      <td>0.017548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2947.0</th>\n",
       "      <td>0.005524</td>\n",
       "      <td>0.029289</td>\n",
       "      <td>0.350625</td>\n",
       "      <td>0.023732</td>\n",
       "      <td>0.005754</td>\n",
       "      <td>0.005032</td>\n",
       "      <td>0.475743</td>\n",
       "      <td>0.094678</td>\n",
       "      <td>0.009623</td>\n",
       "      <td>0.253265</td>\n",
       "      <td>0.177734</td>\n",
       "      <td>0.251875</td>\n",
       "      <td>0.166296</td>\n",
       "      <td>0.024077</td>\n",
       "      <td>0.073485</td>\n",
       "      <td>0.008465</td>\n",
       "      <td>0.008517</td>\n",
       "      <td>0.006828</td>\n",
       "      <td>0.012216</td>\n",
       "      <td>0.017243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2948.0</th>\n",
       "      <td>0.082219</td>\n",
       "      <td>0.573929</td>\n",
       "      <td>0.071733</td>\n",
       "      <td>0.023898</td>\n",
       "      <td>0.024530</td>\n",
       "      <td>0.037873</td>\n",
       "      <td>0.124089</td>\n",
       "      <td>0.042005</td>\n",
       "      <td>0.019725</td>\n",
       "      <td>0.286234</td>\n",
       "      <td>0.194948</td>\n",
       "      <td>0.213093</td>\n",
       "      <td>0.188490</td>\n",
       "      <td>0.035445</td>\n",
       "      <td>0.033786</td>\n",
       "      <td>0.009093</td>\n",
       "      <td>0.007172</td>\n",
       "      <td>0.007361</td>\n",
       "      <td>0.010529</td>\n",
       "      <td>0.013848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2949.0</th>\n",
       "      <td>0.328473</td>\n",
       "      <td>0.250673</td>\n",
       "      <td>0.075527</td>\n",
       "      <td>0.046130</td>\n",
       "      <td>0.021026</td>\n",
       "      <td>0.164873</td>\n",
       "      <td>0.051524</td>\n",
       "      <td>0.040992</td>\n",
       "      <td>0.020783</td>\n",
       "      <td>0.322615</td>\n",
       "      <td>0.194243</td>\n",
       "      <td>0.150925</td>\n",
       "      <td>0.250832</td>\n",
       "      <td>0.022708</td>\n",
       "      <td>0.031271</td>\n",
       "      <td>0.003223</td>\n",
       "      <td>0.003686</td>\n",
       "      <td>0.003432</td>\n",
       "      <td>0.004769</td>\n",
       "      <td>0.012294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2950.0</th>\n",
       "      <td>0.043091</td>\n",
       "      <td>0.725594</td>\n",
       "      <td>0.035364</td>\n",
       "      <td>0.018479</td>\n",
       "      <td>0.014506</td>\n",
       "      <td>0.107470</td>\n",
       "      <td>0.029462</td>\n",
       "      <td>0.008090</td>\n",
       "      <td>0.017945</td>\n",
       "      <td>0.304681</td>\n",
       "      <td>0.171659</td>\n",
       "      <td>0.233563</td>\n",
       "      <td>0.197824</td>\n",
       "      <td>0.025985</td>\n",
       "      <td>0.030759</td>\n",
       "      <td>0.006103</td>\n",
       "      <td>0.005049</td>\n",
       "      <td>0.005106</td>\n",
       "      <td>0.006924</td>\n",
       "      <td>0.012346</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2951 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               1         2         3         4         5         6         7  \\\n",
       "0                                                                              \n",
       "0.0     0.021555  0.099800  0.012440  0.009709  0.238287  0.136117  0.042519   \n",
       "1.0     0.011781  0.037312  0.006450  0.004553  0.040292  0.043186  0.010875   \n",
       "2.0     0.047308  0.089104  0.019904  0.012865  0.214152  0.330976  0.058205   \n",
       "3.0     0.030690  0.140460  0.035073  0.012410  0.009325  0.031827  0.684635   \n",
       "4.0     0.368223  0.291382  0.006805  0.076779  0.020741  0.173170  0.041746   \n",
       "...          ...       ...       ...       ...       ...       ...       ...   \n",
       "2946.0  0.027841  0.508675  0.119450  0.030187  0.012398  0.030492  0.203104   \n",
       "2947.0  0.005524  0.029289  0.350625  0.023732  0.005754  0.005032  0.475743   \n",
       "2948.0  0.082219  0.573929  0.071733  0.023898  0.024530  0.037873  0.124089   \n",
       "2949.0  0.328473  0.250673  0.075527  0.046130  0.021026  0.164873  0.051524   \n",
       "2950.0  0.043091  0.725594  0.035364  0.018479  0.014506  0.107470  0.029462   \n",
       "\n",
       "               8         9        10        11        12        13        14  \\\n",
       "0                                                                              \n",
       "0.0     0.006944  0.432629  0.056437  0.318522  0.264389  0.231813  0.015072   \n",
       "1.0     0.004713  0.840839  0.119712  0.412343  0.214445  0.125116  0.016628   \n",
       "2.0     0.014936  0.212550  0.109006  0.324120  0.258103  0.159002  0.012958   \n",
       "3.0     0.046914  0.008666  0.059222  0.040448  0.178818  0.074877  0.021621   \n",
       "4.0     0.010219  0.010934  0.189621  0.075349  0.100124  0.297412  0.015744   \n",
       "...          ...       ...       ...       ...       ...       ...       ...   \n",
       "2946.0  0.045802  0.022050  0.285283  0.189706  0.215706  0.182972  0.032010   \n",
       "2947.0  0.094678  0.009623  0.253265  0.177734  0.251875  0.166296  0.024077   \n",
       "2948.0  0.042005  0.019725  0.286234  0.194948  0.213093  0.188490  0.035445   \n",
       "2949.0  0.040992  0.020783  0.322615  0.194243  0.150925  0.250832  0.022708   \n",
       "2950.0  0.008090  0.017945  0.304681  0.171659  0.233563  0.197824  0.025985   \n",
       "\n",
       "              15        16        17        18        19        20  \n",
       "0                                                                   \n",
       "0.0     0.095696  0.002628  0.001783  0.002582  0.003020  0.008057  \n",
       "1.0     0.082014  0.005115  0.003278  0.004943  0.005984  0.010422  \n",
       "2.0     0.109527  0.004611  0.003401  0.003815  0.004318  0.011139  \n",
       "3.0     0.581005  0.012006  0.008276  0.005939  0.007297  0.010490  \n",
       "4.0     0.295089  0.005760  0.004349  0.002846  0.003156  0.010550  \n",
       "...          ...       ...       ...       ...       ...       ...  \n",
       "2946.0  0.042161  0.008416  0.007815  0.007482  0.010901  0.017548  \n",
       "2947.0  0.073485  0.008465  0.008517  0.006828  0.012216  0.017243  \n",
       "2948.0  0.033786  0.009093  0.007172  0.007361  0.010529  0.013848  \n",
       "2949.0  0.031271  0.003223  0.003686  0.003432  0.004769  0.012294  \n",
       "2950.0  0.030759  0.006103  0.005049  0.005106  0.006924  0.012346  \n",
       "\n",
       "[2951 rows x 20 columns]"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = pd.read_csv(args.save_dir + 'preds.csv', sep='\\t', index_col='0')\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = torch.tensor(np.array(preds)).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_k_confusion_matrix(y, y_pred, k=3):\n",
    "    dim = y.shape[-1]\n",
    "    y = y.topk(k=k, axis=1)[1]\n",
    "    y_pred = y_pred.topk(k=k, axis=1)[1]\n",
    "    conf = np.zeros((dim, dim))\n",
    "    for i in range(k):\n",
    "        for j in range(k):\n",
    "            conf = np.add(conf, confusion_matrix(y[:,i], y_pred[:,j], labels = range(dim)))\n",
    "    return conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ATT_conf = confusion_matrix(data.y[(data.att_lab) * data.test_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                                 pred[(data.att_lab) * data.test_mask][:,:9].argmax(axis=1).cpu(), labels = range(9))\n",
    "test_VAL_conf = confusion_matrix(data.y[(data.val_lab) * data.test_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                                 pred[(data.val_lab) * data.test_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "test_VAL_conf_k = (top_k_confusion_matrix(data.y[(data.val_lab) * data.test_mask][:,9:].cpu(),  \n",
    "                                 pred[(data.val_lab) * data.test_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 64,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0, 144,   0,   0,   0,   0,   1,   0,   0],\n",
       "       [  0,   0,  10,   0,   0,   0,   0,   1,   0],\n",
       "       [  1,   0,   0,  17,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,  72,   0,   0,   0,   0],\n",
       "       [  0,   1,   0,   0,   0, 101,   0,   0,   0],\n",
       "       [  0,   1,   0,   0,   2,   0,  74,   0,   0],\n",
       "       [  0,   0,   1,   0,   0,   0,   0,   1,   0],\n",
       "       [  1,   0,   0,   0,   0,   0,   0,   0,  11]], dtype=int64)"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[54,  3,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 1, 20,  0,  2,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 1,  6, 19,  2,  0,  7,  0,  0,  0,  0,  0],\n",
       "       [ 2,  1,  3, 26,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 1,  5,  2,  2,  0, 34,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  1,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0]], dtype=int64)"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 90,  61,  48,  47,   0,  36,   0,   0,   0,   0,   0],\n",
       "       [ 50,  87,  78,  64,   1,  29,   0,   0,   0,   0,   0],\n",
       "       [ 56,  84, 112,  93,   1,  61,   1,   0,   0,   0,   0],\n",
       "       [ 56,  79,  91, 118,   1,  54,   0,   0,   0,   0,   0],\n",
       "       [  0,   3,   3,   3,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [ 45,  40,  72,  74,   0,  77,   1,   0,   0,   0,   0],\n",
       "       [  0,   0,   1,   0,   0,   2,   2,   0,   0,   1,   0],\n",
       "       [  0,   0,   0,   0,   0,   1,   1,   0,   0,   1,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   1,   1,   0,   0,   1,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0]])"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_ATT_conf = confusion_matrix(data.y[(data.att_lab) * data.val_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                                pred[(data.att_lab) * data.val_mask][:,:9].argmax(axis=1).cpu())\n",
    "val_VAL_conf = confusion_matrix(data.y[(data.val_lab) * data.val_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                                 pred[(data.val_lab) * data.val_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "val_VAL_conf_k = (top_k_confusion_matrix(data.y[(data.val_lab) * data.val_mask][:,9:].cpu(),  \n",
    "                                 pred[(data.val_lab) * data.val_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 61,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0, 145,   0,   0,   0,   1,   0,   0,   0],\n",
       "       [  0,   0,   8,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,  22,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,  75,   0,   0,   0,   0],\n",
       "       [  0,   1,   0,   0,   0, 102,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,  62,   0,   0],\n",
       "       [  1,   0,   0,   0,   0,   0,   0,   4,   0],\n",
       "       [  0,   0,   0,   0,   2,   0,   0,   0,   8]], dtype=int64)"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[61,  3,  2,  2,  0,  1,  0,  0,  0,  0,  0],\n",
       "       [ 0, 14,  1,  3,  0,  1,  0,  0,  0,  0,  0],\n",
       "       [ 0,  1, 22,  4,  0,  4,  0,  0,  0,  0,  0],\n",
       "       [ 1,  1,  0, 32,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 4,  5,  4,  2,  0, 35,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0]], dtype=int64)"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 99,  65,  59,  60,   0,  41,   0,   0,   0,   0,   0],\n",
       "       [ 56,  88,  68,  70,   5,  34,   0,   0,   0,   0,   0],\n",
       "       [ 46,  64, 113,  96,   1,  73,   0,   0,   0,   0,   0],\n",
       "       [ 62,  89,  92, 123,   5,  55,   0,   0,   0,   0,   0],\n",
       "       [  0,   8,   4,   8,   4,   0,   0,   0,   0,   0,   0],\n",
       "       [ 43,  43,  93,  72,   0,  88,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0]])"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat([pd.DataFrame(val_ATT_conf),pd.DataFrame(test_ATT_conf)],axis=1).to_csv(args.save_dir+'confusion_matrix_ATT.csv')\n",
    "pd.concat([pd.DataFrame(val_VAL_conf),pd.DataFrame(test_VAL_conf)],axis=1).to_csv(args.save_dir+'confusion_matrix_VAL.csv')\n",
    "pd.concat([pd.DataFrame(val_VAL_conf_k),pd.DataFrame(test_VAL_conf_k)],axis=1).to_csv(args.save_dir+'confusion_matrix_VAL_k.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics(confusion_matrix, classes):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val/(row.sum()+0.000000001)\n",
    "        recall = val/(col.sum()+0.000000001)\n",
    "        F1 = 2*(precision*recall)/(precision+recall+0.000000001)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics_k(confusion_matrix, classes, k=3):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val*k/(row.sum()+0.000000001)\n",
    "        recall = val*k/(col.sum()+0.000000001)\n",
    "        F1 = 2*(precision*recall)/(precision+recall+0.000000001)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = ['Criterion i', 'Criterion ii', 'Criterion iii', 'Criterion iv', 'Criterion v', 'Criterion vi', \n",
    "              'Criterion vii', 'Criterion viii', 'Criterion ix', 'Criterion x', 'Others']\n",
    "categories = ['Building Elements',\n",
    " 'Urban Form Elements',\n",
    " 'Gastronomy',\n",
    " 'Interior Scenery',\n",
    " 'Natural Features and Land-scape Scenery',\n",
    " 'Monuments and Buildings',\n",
    " 'Peoples Activity and Association',\n",
    " 'Artifact Products',\n",
    " 'Urban Scenery']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_dict = {}\n",
    "metrics_dict['test_ATT'] = per_class_metrics(test_ATT_conf, categories)\n",
    "metrics_dict['val_ATT'] = per_class_metrics(val_ATT_conf, categories)\n",
    "metrics_dict['test_VAL'] = per_class_metrics(test_VAL_conf, classes)\n",
    "metrics_dict['val_VAL'] = per_class_metrics(val_VAL_conf, classes)\n",
    "metrics_dict['test_VAL_k'] = per_class_metrics_k(test_VAL_conf_k, classes)\n",
    "metrics_dict['val_VAL_k'] = per_class_metrics_k(val_VAL_conf_k, classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df = pd.DataFrame.from_dict({(i,j): metrics_dict[i][j] \n",
    "                           for i in metrics_dict.keys() \n",
    "                           for j in metrics_dict[i].keys()},\n",
    "                       orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df.to_csv(args.save_dir+'per_class_metrics.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Masking Visual and Textual Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MLP(in_channels=data.x.shape[-1], hidden_channels = 256, \n",
    "            out_channels = data.y.shape[-1], dropout = 0.1, num_layers=3).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MLP(1753, 256, 256, 20)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load(args.save_dir+'MLP_best_model/model.pth',map_location=torch.device('cpu')))\n",
    "model.eval()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Data(num_nodes=2921, x=[2921, 1753], y=[2921, 20], node_type=[2921], att_lab=[2921], val_lab=[2921], train_mask=[2921], val_mask=[2921], test_mask=[2921], edge_index=[2, 104214], edge_attr=[104214], n_id=[2921], batch_size=32)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_data = next(iter(train_loader))\n",
    "batch = sampled_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0000, 0.0000, 0.0000,  ..., 1.0000, 0.0000, 0.0000],\n",
       "        [0.0000, 0.0000, 0.0000,  ..., 0.0000, 0.0000, 1.0000],\n",
       "        [0.0000, 0.0000, 0.0000,  ..., 1.0000, 1.0000, 1.0000],\n",
       "        ...,\n",
       "        [0.4269, 2.1218, 0.6201,  ..., 0.0000, 1.0000, 0.0000],\n",
       "        [0.6513, 0.2455, 1.4309,  ..., 1.0000, 0.0000, 0.0000],\n",
       "        [0.0303, 0.2407, 0.1925,  ..., 0.0000, 0.0000, 0.0000]],\n",
       "       device='cuda:0')"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch.x[:32,:982] = torch.zeros(32,982)\n",
    "batch.x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def test_Mask(model, loader, mask = None):\n",
    "    model.eval()\n",
    "\n",
    "    total_examples_att = total_examples_val = 0\n",
    "    running_loss_1 = running_loss_2 = 0.\n",
    "    running_1_acc = 0.\n",
    "    running_1_val = 0.\n",
    "    running_k_acc = 0.\n",
    "    running_k_jac = 0.\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        loss_1 = 0\n",
    "        acc_1_t = 0\n",
    "        loss_2 = 0\n",
    "        acc_1_val = 0\n",
    "        acc_k_t = 0\n",
    "        jac_k_t = 0\n",
    "\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch.batch_size\n",
    "        \n",
    "        if mask == 'vis':\n",
    "            batch.x[:batch_size,:982] = torch.zeros(batch_size,982)\n",
    "        elif mask == 'tex':\n",
    "            batch.x[:batch_size,982:] = torch.zeros(batch_size,771)\n",
    "        \n",
    "        #edge_index = to_undirected(batch.edge_index)\n",
    "        out = model(batch.x)[:batch_size]\n",
    "        out_att = out[:,:9]\n",
    "        out_val = out[:,9:]\n",
    "        att_node = (batch.att_lab[:batch_size]).nonzero().squeeze()\n",
    "        val_node = (batch.val_lab[:batch_size]).nonzero().squeeze()\n",
    "\n",
    "        #print(type_node)\n",
    "\n",
    "        #pred_att = out_att.argmax(dim=-1)\n",
    "        #pred_val = out_val.argmax(dim=-1)\n",
    "\n",
    "        y = batch.y\n",
    "        y_att = y[:,:9]\n",
    "        y_val = y[:,9:]\n",
    "\n",
    "        if not att_node.shape[0]==0:\n",
    "            loss_1 = F.cross_entropy(out_att[att_node], y_att[:batch_size][att_node])\n",
    "            acc_1_t = compute_1_accuracy(y_att[:batch_size][att_node], out_att[att_node])\n",
    "\n",
    "        if not val_node.shape[0]==0:\n",
    "            loss_2 = F.cross_entropy(out_val[val_node], y_val[val_node])\n",
    "            acc_1_val = compute_1_accuracy(y_val[val_node], out_val[val_node])\n",
    "            acc_k_t = compute_k_accuracy(y_val[val_node], out_val[val_node], args.k)\n",
    "            jac_k_t = compute_jaccard_index(y_val[val_node], F.softmax(out_val[val_node],dim=-1), args.k)\n",
    "            #loss_3 = loss_1 + loss_2\n",
    "\n",
    "        total_examples_att += att_node.shape[0]\n",
    "        total_examples_val += val_node.shape[0]\n",
    "        #total_correct_att += int((pred_att == y_att[:batch_size]).sum())\n",
    "        #total_correct_val += int((pred_val == y_val[:batch_size]).sum())\n",
    "\n",
    "        running_loss_1 += float(loss_1) * att_node.shape[0]\n",
    "        running_loss_2 += float(loss_2) * val_node.shape[0]\n",
    "        running_1_acc += float(acc_1_t) * att_node.shape[0]\n",
    "        running_1_val += float(acc_1_val) * val_node.shape[0]\n",
    "        running_k_acc += float(acc_k_t) * val_node.shape[0]\n",
    "        running_k_jac += float(jac_k_t) * val_node.shape[0]\n",
    "    \n",
    "    return running_loss_1/total_examples_att, running_loss_2/total_examples_val, running_1_acc/ total_examples_att, running_k_acc/ total_examples_val, running_k_jac/ total_examples_val, running_1_val/total_examples_val\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 28.23it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2.1408802029828946,\n",
       " 1.6458789093672734,\n",
       " 13.296398891966758,\n",
       " 99.7229916897507,\n",
       " 0.7248384206248782,\n",
       " 76.17728531855956)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, train_loader, 'vis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 23.77it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7581068843685689,\n",
       " 1.9154196657302307,\n",
       " 99.7229916897507,\n",
       " 73.13019390581718,\n",
       " 0.10803324099722991,\n",
       " 31.024930747922436)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, train_loader, 'tex')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.16it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2.232890134419852,\n",
       " 1.6475393995275638,\n",
       " 14.227642276422765,\n",
       " 99.50738916256158,\n",
       " 0.7545156032581047,\n",
       " 80.78817733990148)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, val_loader, 'vis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.23it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7890427937594856,\n",
       " 2.052411281416569,\n",
       " 99.1869918699187,\n",
       " 60.09852216748769,\n",
       " 0.012315270935960592,\n",
       " 23.15270935960591)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, val_loader, 'tex')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 40.42it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2.249690012239794,\n",
       " 1.6597018192211788,\n",
       " 15.109343936381709,\n",
       " 100.0,\n",
       " 0.7708333358168602,\n",
       " 79.16666666666667)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, test_loader, 'vis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 40.84it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.796224286968618,\n",
       " 2.0458929960926375,\n",
       " 99.00596421471172,\n",
       " 59.895833333333336,\n",
       " 0.028645833333333332,\n",
       " 25.0)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, test_loader, 'tex')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 35.23it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 65.74it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 62.04it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 61.09it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 64.92it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 64.06it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 60.16it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 59.01it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 59.09it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 58.63it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 56.82it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 54.91it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 57.80it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 57.36it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 56.42it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 55.13it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 53.50it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 56.14it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 53.90it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 53.17it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 54.61it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 55.84it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 53.64it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 53.23it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 54.10it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 54.60it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 54.24it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 54.83it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 55.49it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 54.94it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 54.30it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 47.69it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 52.17it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 54.52it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 52.75it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 52.81it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 56.17it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 53.42it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 52.71it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 53.14it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers_vis = []\n",
    "val_numbers_tex = []\n",
    "test_numbers_vis = []\n",
    "test_numbers_tex = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers_vis.append(test_Mask(model, val_loader, 'vis'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers_tex.append(test_Mask(model, val_loader, 'tex'))\n",
    "    \n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers_vis.append(test_Mask(model, test_loader, 'vis'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers_tex.append(test_Mask(model, test_loader, 'tex'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df_vis = pd.DataFrame(val_numbers_vis, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "val_df_tex = pd.DataFrame(val_numbers_tex, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df_vis = pd.DataFrame(test_numbers_vis, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df_tex = pd.DataFrame(test_numbers_tex, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.232890e+00</td>\n",
       "      <td>1.647539e+00</td>\n",
       "      <td>14.227642</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>7.545156e-01</td>\n",
       "      <td>80.788177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>4.681111e-16</td>\n",
       "      <td>2.340556e-16</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.170278e-16</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.232890e+00</td>\n",
       "      <td>1.647539e+00</td>\n",
       "      <td>14.227642</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>7.545156e-01</td>\n",
       "      <td>80.788177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.232890e+00</td>\n",
       "      <td>1.647539e+00</td>\n",
       "      <td>14.227642</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>7.545156e-01</td>\n",
       "      <td>80.788177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.232890e+00</td>\n",
       "      <td>1.647539e+00</td>\n",
       "      <td>14.227642</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>7.545156e-01</td>\n",
       "      <td>80.788177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.232890e+00</td>\n",
       "      <td>1.647539e+00</td>\n",
       "      <td>14.227642</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>7.545156e-01</td>\n",
       "      <td>80.788177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.232890e+00</td>\n",
       "      <td>1.647539e+00</td>\n",
       "      <td>14.227642</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>7.545156e-01</td>\n",
       "      <td>80.788177</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ATT_loss      VAL_loss    ATT_acc  VAL_k_acc     VAL_k_jac  \\\n",
       "count  1.000000e+01  1.000000e+01  10.000000  10.000000  1.000000e+01   \n",
       "mean   2.232890e+00  1.647539e+00  14.227642  99.507389  7.545156e-01   \n",
       "std    4.681111e-16  2.340556e-16   0.000000   0.000000  1.170278e-16   \n",
       "min    2.232890e+00  1.647539e+00  14.227642  99.507389  7.545156e-01   \n",
       "25%    2.232890e+00  1.647539e+00  14.227642  99.507389  7.545156e-01   \n",
       "50%    2.232890e+00  1.647539e+00  14.227642  99.507389  7.545156e-01   \n",
       "75%    2.232890e+00  1.647539e+00  14.227642  99.507389  7.545156e-01   \n",
       "max    2.232890e+00  1.647539e+00  14.227642  99.507389  7.545156e-01   \n",
       "\n",
       "       VAL_1_acc  \n",
       "count  10.000000  \n",
       "mean   80.788177  \n",
       "std     0.000000  \n",
       "min    80.788177  \n",
       "25%    80.788177  \n",
       "50%    80.788177  \n",
       "75%    80.788177  \n",
       "max    80.788177  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df_vis.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>7.890428e-01</td>\n",
       "      <td>2.052411e+00</td>\n",
       "      <td>9.918699e+01</td>\n",
       "      <td>6.009852e+01</td>\n",
       "      <td>0.012315</td>\n",
       "      <td>2.315271e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.170278e-16</td>\n",
       "      <td>4.681111e-16</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.744889e-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>7.890428e-01</td>\n",
       "      <td>2.052411e+00</td>\n",
       "      <td>9.918699e+01</td>\n",
       "      <td>6.009852e+01</td>\n",
       "      <td>0.012315</td>\n",
       "      <td>2.315271e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>7.890428e-01</td>\n",
       "      <td>2.052411e+00</td>\n",
       "      <td>9.918699e+01</td>\n",
       "      <td>6.009852e+01</td>\n",
       "      <td>0.012315</td>\n",
       "      <td>2.315271e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>7.890428e-01</td>\n",
       "      <td>2.052411e+00</td>\n",
       "      <td>9.918699e+01</td>\n",
       "      <td>6.009852e+01</td>\n",
       "      <td>0.012315</td>\n",
       "      <td>2.315271e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.890428e-01</td>\n",
       "      <td>2.052411e+00</td>\n",
       "      <td>9.918699e+01</td>\n",
       "      <td>6.009852e+01</td>\n",
       "      <td>0.012315</td>\n",
       "      <td>2.315271e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>7.890428e-01</td>\n",
       "      <td>2.052411e+00</td>\n",
       "      <td>9.918699e+01</td>\n",
       "      <td>6.009852e+01</td>\n",
       "      <td>0.012315</td>\n",
       "      <td>2.315271e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ATT_loss      VAL_loss       ATT_acc     VAL_k_acc  VAL_k_jac  \\\n",
       "count  1.000000e+01  1.000000e+01  1.000000e+01  1.000000e+01  10.000000   \n",
       "mean   7.890428e-01  2.052411e+00  9.918699e+01  6.009852e+01   0.012315   \n",
       "std    1.170278e-16  4.681111e-16  1.497956e-14  1.497956e-14   0.000000   \n",
       "min    7.890428e-01  2.052411e+00  9.918699e+01  6.009852e+01   0.012315   \n",
       "25%    7.890428e-01  2.052411e+00  9.918699e+01  6.009852e+01   0.012315   \n",
       "50%    7.890428e-01  2.052411e+00  9.918699e+01  6.009852e+01   0.012315   \n",
       "75%    7.890428e-01  2.052411e+00  9.918699e+01  6.009852e+01   0.012315   \n",
       "max    7.890428e-01  2.052411e+00  9.918699e+01  6.009852e+01   0.012315   \n",
       "\n",
       "          VAL_1_acc  \n",
       "count  1.000000e+01  \n",
       "mean   2.315271e+01  \n",
       "std    3.744889e-15  \n",
       "min    2.315271e+01  \n",
       "25%    2.315271e+01  \n",
       "50%    2.315271e+01  \n",
       "75%    2.315271e+01  \n",
       "max    2.315271e+01  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df_tex.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.249690e+00</td>\n",
       "      <td>1.659702e+00</td>\n",
       "      <td>15.109344</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.770833</td>\n",
       "      <td>7.916667e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>4.681111e-16</td>\n",
       "      <td>2.340556e-16</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.497956e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.249690e+00</td>\n",
       "      <td>1.659702e+00</td>\n",
       "      <td>15.109344</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.770833</td>\n",
       "      <td>7.916667e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.249690e+00</td>\n",
       "      <td>1.659702e+00</td>\n",
       "      <td>15.109344</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.770833</td>\n",
       "      <td>7.916667e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.249690e+00</td>\n",
       "      <td>1.659702e+00</td>\n",
       "      <td>15.109344</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.770833</td>\n",
       "      <td>7.916667e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.249690e+00</td>\n",
       "      <td>1.659702e+00</td>\n",
       "      <td>15.109344</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.770833</td>\n",
       "      <td>7.916667e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.249690e+00</td>\n",
       "      <td>1.659702e+00</td>\n",
       "      <td>15.109344</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.770833</td>\n",
       "      <td>7.916667e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ATT_loss      VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  \\\n",
       "count  1.000000e+01  1.000000e+01  10.000000       10.0  10.000000   \n",
       "mean   2.249690e+00  1.659702e+00  15.109344      100.0   0.770833   \n",
       "std    4.681111e-16  2.340556e-16   0.000000        0.0   0.000000   \n",
       "min    2.249690e+00  1.659702e+00  15.109344      100.0   0.770833   \n",
       "25%    2.249690e+00  1.659702e+00  15.109344      100.0   0.770833   \n",
       "50%    2.249690e+00  1.659702e+00  15.109344      100.0   0.770833   \n",
       "75%    2.249690e+00  1.659702e+00  15.109344      100.0   0.770833   \n",
       "max    2.249690e+00  1.659702e+00  15.109344      100.0   0.770833   \n",
       "\n",
       "          VAL_1_acc  \n",
       "count  1.000000e+01  \n",
       "mean   7.916667e+01  \n",
       "std    1.497956e-14  \n",
       "min    7.916667e+01  \n",
       "25%    7.916667e+01  \n",
       "50%    7.916667e+01  \n",
       "75%    7.916667e+01  \n",
       "max    7.916667e+01  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df_vis.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>7.962243e-01</td>\n",
       "      <td>2.045893e+00</td>\n",
       "      <td>9.900596e+01</td>\n",
       "      <td>59.895833</td>\n",
       "      <td>0.028646</td>\n",
       "      <td>25.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.170278e-16</td>\n",
       "      <td>4.681111e-16</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>7.962243e-01</td>\n",
       "      <td>2.045893e+00</td>\n",
       "      <td>9.900596e+01</td>\n",
       "      <td>59.895833</td>\n",
       "      <td>0.028646</td>\n",
       "      <td>25.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>7.962243e-01</td>\n",
       "      <td>2.045893e+00</td>\n",
       "      <td>9.900596e+01</td>\n",
       "      <td>59.895833</td>\n",
       "      <td>0.028646</td>\n",
       "      <td>25.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>7.962243e-01</td>\n",
       "      <td>2.045893e+00</td>\n",
       "      <td>9.900596e+01</td>\n",
       "      <td>59.895833</td>\n",
       "      <td>0.028646</td>\n",
       "      <td>25.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.962243e-01</td>\n",
       "      <td>2.045893e+00</td>\n",
       "      <td>9.900596e+01</td>\n",
       "      <td>59.895833</td>\n",
       "      <td>0.028646</td>\n",
       "      <td>25.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>7.962243e-01</td>\n",
       "      <td>2.045893e+00</td>\n",
       "      <td>9.900596e+01</td>\n",
       "      <td>59.895833</td>\n",
       "      <td>0.028646</td>\n",
       "      <td>25.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ATT_loss      VAL_loss       ATT_acc  VAL_k_acc  VAL_k_jac  \\\n",
       "count  1.000000e+01  1.000000e+01  1.000000e+01  10.000000  10.000000   \n",
       "mean   7.962243e-01  2.045893e+00  9.900596e+01  59.895833   0.028646   \n",
       "std    1.170278e-16  4.681111e-16  1.497956e-14   0.000000   0.000000   \n",
       "min    7.962243e-01  2.045893e+00  9.900596e+01  59.895833   0.028646   \n",
       "25%    7.962243e-01  2.045893e+00  9.900596e+01  59.895833   0.028646   \n",
       "50%    7.962243e-01  2.045893e+00  9.900596e+01  59.895833   0.028646   \n",
       "75%    7.962243e-01  2.045893e+00  9.900596e+01  59.895833   0.028646   \n",
       "max    7.962243e-01  2.045893e+00  9.900596e+01  59.895833   0.028646   \n",
       "\n",
       "       VAL_1_acc  \n",
       "count       10.0  \n",
       "mean        25.0  \n",
       "std          0.0  \n",
       "min         25.0  \n",
       "25%         25.0  \n",
       "50%         25.0  \n",
       "75%         25.0  \n",
       "max         25.0  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df_tex.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df_vis.to_csv(args.save_dir + 'vis_masked_val_metrics.csv', sep='\\t')\n",
    "val_df_tex.to_csv(args.save_dir + 'tex_masked_val_metrics.csv', sep='\\t')\n",
    "test_df_vis.to_csv(args.save_dir + 'vis_masked_test_metrics.csv', sep='\\t')\n",
    "test_df_tex.to_csv(args.save_dir + 'tex_masked_test_metrics.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Direct Prediction on VEN-XL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cpu\")\n",
    "#device = torch.device(\"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#path = osp.join(os.getcwd(), '../../data/VEN')\n",
    "#transform = T.Compose([T.NormalizeFeatures(), T.ToSparseTensor()])\n",
    "dataset_XL = VEN_XL_Homo('dataset/Venice_XL_homo')\n",
    "data_XL = dataset_XL[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Data(num_nodes=80963, x=[80963, 1753], y=[80963, 20], att_lab=[80963], val_lab=[80963], node_type=[80963], train_mask=[80963], val_mask=[80963], test_mask=[80963], edge_index=[2, 290091503], edge_attr=[290091503], n_id=[80963])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_XL.n_id = torch.arange(data_XL.num_nodes)\n",
    "data_XL = data_XL.to(device)\n",
    "data_XL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.loader import NeighborLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data_XL,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data_XL.train_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "val_loader = NeighborLoader(\n",
    "    data_XL,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data_XL.val_mask,\n",
    ")\n",
    "seed_everything(args.seed)\n",
    "test_loader = NeighborLoader(\n",
    "    data_XL,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data_XL.test_mask,\n",
    ")\n",
    "unlabel_loader = NeighborLoader(\n",
    "    data_XL,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=~(data_XL.train_mask + data_XL.val_mask + data_XL.test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Data(num_nodes=53788, x=[53788, 1753], y=[53788, 20], att_lab=[53788], val_lab=[53788], node_type=[53788], train_mask=[53788], val_mask=[53788], test_mask=[53788], edge_index=[2, 170325], edge_attr=[170325], n_id=[53788], batch_size=32)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_data = next(iter(train_loader))\n",
    "batch = sampled_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MLP(in_channels=data_XL.x.shape[-1], hidden_channels = 256, \n",
    "            out_channels = data_XL.y.shape[-1], dropout = 0.1, num_layers=3).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load(args.save_dir+'MLP_best_model/model.pth',map_location=torch.device('cpu')))\n",
    "model.eval()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def predict_Homo(model, loader):\n",
    "    model.eval()\n",
    "    seed_everything(args.seed)\n",
    "    all_preds = []\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch.batch_size\n",
    "        #edge_index = to_undirected(batch.edge_index)\n",
    "        out = model(batch.x)[:batch_size]\n",
    "        out_att = out[:,:9].softmax(axis=1)\n",
    "        out_val = out[:,9:].softmax(axis=1)\n",
    "        IDs = batch.n_id[:batch_size].unsqueeze(dim=-1).int()\n",
    "        \n",
    "        now = torch.hstack([IDs, out_att, out_val])\n",
    "        all_preds.append(now)\n",
    "    \n",
    "    final = torch.vstack(all_preds)\n",
    "        \n",
    "    return final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 362/362 [01:51<00:00,  3.24it/s]\n"
     ]
    }
   ],
   "source": [
    "pred_train = predict_Homo(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 624/624 [03:18<00:00,  3.15it/s]\n"
     ]
    }
   ],
   "source": [
    "pred_val = predict_Homo(model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 624/624 [03:07<00:00,  3.33it/s]\n"
     ]
    }
   ],
   "source": [
    "pred_test = predict_Homo(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 921/921 [04:15<00:00,  3.60it/s]\n"
     ]
    }
   ],
   "source": [
    "pred_unlab = predict_Homo(model, unlabel_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([4.0000e+00, 6.4191e-03, 9.3232e-01, 4.7782e-03, 2.2493e-03, 1.4609e-03,\n",
       "        3.6227e-02, 8.7789e-03, 1.3225e-03, 6.4492e-03, 2.4002e-02, 1.2366e-01,\n",
       "        1.9101e-01, 4.7043e-01, 8.0229e-02, 9.2186e-02, 2.5333e-03, 2.0446e-03,\n",
       "        2.9066e-03, 3.2552e-03, 7.7464e-03])"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = torch.vstack([pred_train, pred_val, pred_test]).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = torch.vstack([pred_train, pred_val, pred_test, pred_unlab]).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df = pd.DataFrame(preds).sort_values(0).set_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.594535</td>\n",
       "      <td>0.048073</td>\n",
       "      <td>0.019625</td>\n",
       "      <td>0.041608</td>\n",
       "      <td>0.014619</td>\n",
       "      <td>0.190502</td>\n",
       "      <td>0.038185</td>\n",
       "      <td>0.023982</td>\n",
       "      <td>0.028871</td>\n",
       "      <td>0.189272</td>\n",
       "      <td>0.116634</td>\n",
       "      <td>0.180595</td>\n",
       "      <td>0.374547</td>\n",
       "      <td>0.029213</td>\n",
       "      <td>0.074395</td>\n",
       "      <td>0.005818</td>\n",
       "      <td>0.005557</td>\n",
       "      <td>0.004208</td>\n",
       "      <td>0.004887</td>\n",
       "      <td>0.014874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.362458</td>\n",
       "      <td>0.031278</td>\n",
       "      <td>0.002685</td>\n",
       "      <td>0.012323</td>\n",
       "      <td>0.004354</td>\n",
       "      <td>0.555326</td>\n",
       "      <td>0.007800</td>\n",
       "      <td>0.005447</td>\n",
       "      <td>0.018330</td>\n",
       "      <td>0.219277</td>\n",
       "      <td>0.125396</td>\n",
       "      <td>0.120398</td>\n",
       "      <td>0.418638</td>\n",
       "      <td>0.021438</td>\n",
       "      <td>0.072530</td>\n",
       "      <td>0.003081</td>\n",
       "      <td>0.003248</td>\n",
       "      <td>0.002418</td>\n",
       "      <td>0.003077</td>\n",
       "      <td>0.010500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.029422</td>\n",
       "      <td>0.369658</td>\n",
       "      <td>0.014396</td>\n",
       "      <td>0.015661</td>\n",
       "      <td>0.348402</td>\n",
       "      <td>0.030611</td>\n",
       "      <td>0.122547</td>\n",
       "      <td>0.009769</td>\n",
       "      <td>0.059534</td>\n",
       "      <td>0.065325</td>\n",
       "      <td>0.065170</td>\n",
       "      <td>0.111108</td>\n",
       "      <td>0.086168</td>\n",
       "      <td>0.018429</td>\n",
       "      <td>0.547296</td>\n",
       "      <td>0.039902</td>\n",
       "      <td>0.022893</td>\n",
       "      <td>0.012779</td>\n",
       "      <td>0.015562</td>\n",
       "      <td>0.015368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.020930</td>\n",
       "      <td>0.221499</td>\n",
       "      <td>0.030887</td>\n",
       "      <td>0.026482</td>\n",
       "      <td>0.068644</td>\n",
       "      <td>0.072682</td>\n",
       "      <td>0.492496</td>\n",
       "      <td>0.025842</td>\n",
       "      <td>0.040539</td>\n",
       "      <td>0.049214</td>\n",
       "      <td>0.130924</td>\n",
       "      <td>0.147752</td>\n",
       "      <td>0.228507</td>\n",
       "      <td>0.068703</td>\n",
       "      <td>0.262241</td>\n",
       "      <td>0.029974</td>\n",
       "      <td>0.021974</td>\n",
       "      <td>0.020983</td>\n",
       "      <td>0.020707</td>\n",
       "      <td>0.019022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.006419</td>\n",
       "      <td>0.932315</td>\n",
       "      <td>0.004778</td>\n",
       "      <td>0.002249</td>\n",
       "      <td>0.001461</td>\n",
       "      <td>0.036227</td>\n",
       "      <td>0.008779</td>\n",
       "      <td>0.001323</td>\n",
       "      <td>0.006449</td>\n",
       "      <td>0.024002</td>\n",
       "      <td>0.123658</td>\n",
       "      <td>0.191009</td>\n",
       "      <td>0.470429</td>\n",
       "      <td>0.080229</td>\n",
       "      <td>0.092186</td>\n",
       "      <td>0.002533</td>\n",
       "      <td>0.002045</td>\n",
       "      <td>0.002907</td>\n",
       "      <td>0.003255</td>\n",
       "      <td>0.007746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80958.0</th>\n",
       "      <td>0.053355</td>\n",
       "      <td>0.323336</td>\n",
       "      <td>0.013751</td>\n",
       "      <td>0.010410</td>\n",
       "      <td>0.013495</td>\n",
       "      <td>0.446532</td>\n",
       "      <td>0.037435</td>\n",
       "      <td>0.007922</td>\n",
       "      <td>0.093765</td>\n",
       "      <td>0.086527</td>\n",
       "      <td>0.306084</td>\n",
       "      <td>0.176422</td>\n",
       "      <td>0.301726</td>\n",
       "      <td>0.014634</td>\n",
       "      <td>0.093429</td>\n",
       "      <td>0.002878</td>\n",
       "      <td>0.002736</td>\n",
       "      <td>0.003200</td>\n",
       "      <td>0.002976</td>\n",
       "      <td>0.009389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80959.0</th>\n",
       "      <td>0.924019</td>\n",
       "      <td>0.016786</td>\n",
       "      <td>0.001945</td>\n",
       "      <td>0.030483</td>\n",
       "      <td>0.005963</td>\n",
       "      <td>0.008671</td>\n",
       "      <td>0.005393</td>\n",
       "      <td>0.005177</td>\n",
       "      <td>0.001564</td>\n",
       "      <td>0.188021</td>\n",
       "      <td>0.354948</td>\n",
       "      <td>0.172444</td>\n",
       "      <td>0.192030</td>\n",
       "      <td>0.010255</td>\n",
       "      <td>0.065905</td>\n",
       "      <td>0.001918</td>\n",
       "      <td>0.002253</td>\n",
       "      <td>0.001935</td>\n",
       "      <td>0.001957</td>\n",
       "      <td>0.008335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80960.0</th>\n",
       "      <td>0.830574</td>\n",
       "      <td>0.035322</td>\n",
       "      <td>0.003268</td>\n",
       "      <td>0.051942</td>\n",
       "      <td>0.009040</td>\n",
       "      <td>0.044200</td>\n",
       "      <td>0.012632</td>\n",
       "      <td>0.009196</td>\n",
       "      <td>0.003826</td>\n",
       "      <td>0.253837</td>\n",
       "      <td>0.203410</td>\n",
       "      <td>0.197328</td>\n",
       "      <td>0.168720</td>\n",
       "      <td>0.009609</td>\n",
       "      <td>0.139658</td>\n",
       "      <td>0.004125</td>\n",
       "      <td>0.004363</td>\n",
       "      <td>0.002813</td>\n",
       "      <td>0.003385</td>\n",
       "      <td>0.012753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80961.0</th>\n",
       "      <td>0.672005</td>\n",
       "      <td>0.164621</td>\n",
       "      <td>0.012071</td>\n",
       "      <td>0.043754</td>\n",
       "      <td>0.010095</td>\n",
       "      <td>0.022950</td>\n",
       "      <td>0.048975</td>\n",
       "      <td>0.019726</td>\n",
       "      <td>0.005803</td>\n",
       "      <td>0.140200</td>\n",
       "      <td>0.336401</td>\n",
       "      <td>0.193229</td>\n",
       "      <td>0.185162</td>\n",
       "      <td>0.015779</td>\n",
       "      <td>0.101258</td>\n",
       "      <td>0.004157</td>\n",
       "      <td>0.003922</td>\n",
       "      <td>0.003787</td>\n",
       "      <td>0.004161</td>\n",
       "      <td>0.011945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80962.0</th>\n",
       "      <td>0.722733</td>\n",
       "      <td>0.077768</td>\n",
       "      <td>0.005419</td>\n",
       "      <td>0.102405</td>\n",
       "      <td>0.008666</td>\n",
       "      <td>0.053822</td>\n",
       "      <td>0.012266</td>\n",
       "      <td>0.010382</td>\n",
       "      <td>0.006539</td>\n",
       "      <td>0.139460</td>\n",
       "      <td>0.441957</td>\n",
       "      <td>0.160908</td>\n",
       "      <td>0.157873</td>\n",
       "      <td>0.013420</td>\n",
       "      <td>0.062108</td>\n",
       "      <td>0.003416</td>\n",
       "      <td>0.003230</td>\n",
       "      <td>0.003581</td>\n",
       "      <td>0.003317</td>\n",
       "      <td>0.010731</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80963 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               1         2         3         4         5         6         7   \\\n",
       "0                                                                               \n",
       "0.0      0.594535  0.048073  0.019625  0.041608  0.014619  0.190502  0.038185   \n",
       "1.0      0.362458  0.031278  0.002685  0.012323  0.004354  0.555326  0.007800   \n",
       "2.0      0.029422  0.369658  0.014396  0.015661  0.348402  0.030611  0.122547   \n",
       "3.0      0.020930  0.221499  0.030887  0.026482  0.068644  0.072682  0.492496   \n",
       "4.0      0.006419  0.932315  0.004778  0.002249  0.001461  0.036227  0.008779   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80958.0  0.053355  0.323336  0.013751  0.010410  0.013495  0.446532  0.037435   \n",
       "80959.0  0.924019  0.016786  0.001945  0.030483  0.005963  0.008671  0.005393   \n",
       "80960.0  0.830574  0.035322  0.003268  0.051942  0.009040  0.044200  0.012632   \n",
       "80961.0  0.672005  0.164621  0.012071  0.043754  0.010095  0.022950  0.048975   \n",
       "80962.0  0.722733  0.077768  0.005419  0.102405  0.008666  0.053822  0.012266   \n",
       "\n",
       "               8         9         10        11        12        13        14  \\\n",
       "0                                                                               \n",
       "0.0      0.023982  0.028871  0.189272  0.116634  0.180595  0.374547  0.029213   \n",
       "1.0      0.005447  0.018330  0.219277  0.125396  0.120398  0.418638  0.021438   \n",
       "2.0      0.009769  0.059534  0.065325  0.065170  0.111108  0.086168  0.018429   \n",
       "3.0      0.025842  0.040539  0.049214  0.130924  0.147752  0.228507  0.068703   \n",
       "4.0      0.001323  0.006449  0.024002  0.123658  0.191009  0.470429  0.080229   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80958.0  0.007922  0.093765  0.086527  0.306084  0.176422  0.301726  0.014634   \n",
       "80959.0  0.005177  0.001564  0.188021  0.354948  0.172444  0.192030  0.010255   \n",
       "80960.0  0.009196  0.003826  0.253837  0.203410  0.197328  0.168720  0.009609   \n",
       "80961.0  0.019726  0.005803  0.140200  0.336401  0.193229  0.185162  0.015779   \n",
       "80962.0  0.010382  0.006539  0.139460  0.441957  0.160908  0.157873  0.013420   \n",
       "\n",
       "               15        16        17        18        19        20  \n",
       "0                                                                    \n",
       "0.0      0.074395  0.005818  0.005557  0.004208  0.004887  0.014874  \n",
       "1.0      0.072530  0.003081  0.003248  0.002418  0.003077  0.010500  \n",
       "2.0      0.547296  0.039902  0.022893  0.012779  0.015562  0.015368  \n",
       "3.0      0.262241  0.029974  0.021974  0.020983  0.020707  0.019022  \n",
       "4.0      0.092186  0.002533  0.002045  0.002907  0.003255  0.007746  \n",
       "...           ...       ...       ...       ...       ...       ...  \n",
       "80958.0  0.093429  0.002878  0.002736  0.003200  0.002976  0.009389  \n",
       "80959.0  0.065905  0.001918  0.002253  0.001935  0.001957  0.008335  \n",
       "80960.0  0.139658  0.004125  0.004363  0.002813  0.003385  0.012753  \n",
       "80961.0  0.101258  0.004157  0.003922  0.003787  0.004161  0.011945  \n",
       "80962.0  0.062108  0.003416  0.003230  0.003581  0.003317  0.010731  \n",
       "\n",
       "[80963 rows x 20 columns]"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df.to_csv(args.save_dir + 'preds_XL_trans.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.594534</td>\n",
       "      <td>0.048073</td>\n",
       "      <td>0.019625</td>\n",
       "      <td>0.041608</td>\n",
       "      <td>0.014619</td>\n",
       "      <td>0.190502</td>\n",
       "      <td>0.038185</td>\n",
       "      <td>0.023982</td>\n",
       "      <td>0.028871</td>\n",
       "      <td>0.189272</td>\n",
       "      <td>0.116634</td>\n",
       "      <td>0.180595</td>\n",
       "      <td>0.374547</td>\n",
       "      <td>0.029213</td>\n",
       "      <td>0.074395</td>\n",
       "      <td>0.005818</td>\n",
       "      <td>0.005557</td>\n",
       "      <td>0.004208</td>\n",
       "      <td>0.004887</td>\n",
       "      <td>0.014874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.362457</td>\n",
       "      <td>0.031278</td>\n",
       "      <td>0.002685</td>\n",
       "      <td>0.012323</td>\n",
       "      <td>0.004354</td>\n",
       "      <td>0.555326</td>\n",
       "      <td>0.007800</td>\n",
       "      <td>0.005447</td>\n",
       "      <td>0.018330</td>\n",
       "      <td>0.219277</td>\n",
       "      <td>0.125396</td>\n",
       "      <td>0.120398</td>\n",
       "      <td>0.418638</td>\n",
       "      <td>0.021438</td>\n",
       "      <td>0.072530</td>\n",
       "      <td>0.003081</td>\n",
       "      <td>0.003248</td>\n",
       "      <td>0.002418</td>\n",
       "      <td>0.003077</td>\n",
       "      <td>0.010500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.029422</td>\n",
       "      <td>0.369658</td>\n",
       "      <td>0.014396</td>\n",
       "      <td>0.015661</td>\n",
       "      <td>0.348402</td>\n",
       "      <td>0.030611</td>\n",
       "      <td>0.122547</td>\n",
       "      <td>0.009769</td>\n",
       "      <td>0.059534</td>\n",
       "      <td>0.065325</td>\n",
       "      <td>0.065170</td>\n",
       "      <td>0.111108</td>\n",
       "      <td>0.086168</td>\n",
       "      <td>0.018429</td>\n",
       "      <td>0.547296</td>\n",
       "      <td>0.039902</td>\n",
       "      <td>0.022893</td>\n",
       "      <td>0.012779</td>\n",
       "      <td>0.015562</td>\n",
       "      <td>0.015368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.020930</td>\n",
       "      <td>0.221499</td>\n",
       "      <td>0.030887</td>\n",
       "      <td>0.026482</td>\n",
       "      <td>0.068644</td>\n",
       "      <td>0.072682</td>\n",
       "      <td>0.492496</td>\n",
       "      <td>0.025842</td>\n",
       "      <td>0.040539</td>\n",
       "      <td>0.049214</td>\n",
       "      <td>0.130924</td>\n",
       "      <td>0.147752</td>\n",
       "      <td>0.228507</td>\n",
       "      <td>0.068703</td>\n",
       "      <td>0.262241</td>\n",
       "      <td>0.029974</td>\n",
       "      <td>0.021974</td>\n",
       "      <td>0.020983</td>\n",
       "      <td>0.020707</td>\n",
       "      <td>0.019022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.006419</td>\n",
       "      <td>0.932315</td>\n",
       "      <td>0.004778</td>\n",
       "      <td>0.002249</td>\n",
       "      <td>0.001461</td>\n",
       "      <td>0.036227</td>\n",
       "      <td>0.008779</td>\n",
       "      <td>0.001323</td>\n",
       "      <td>0.006449</td>\n",
       "      <td>0.024002</td>\n",
       "      <td>0.123658</td>\n",
       "      <td>0.191009</td>\n",
       "      <td>0.470429</td>\n",
       "      <td>0.080229</td>\n",
       "      <td>0.092186</td>\n",
       "      <td>0.002533</td>\n",
       "      <td>0.002045</td>\n",
       "      <td>0.002907</td>\n",
       "      <td>0.003255</td>\n",
       "      <td>0.007746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80958.0</th>\n",
       "      <td>0.053355</td>\n",
       "      <td>0.323336</td>\n",
       "      <td>0.013751</td>\n",
       "      <td>0.010410</td>\n",
       "      <td>0.013495</td>\n",
       "      <td>0.446532</td>\n",
       "      <td>0.037435</td>\n",
       "      <td>0.007922</td>\n",
       "      <td>0.093765</td>\n",
       "      <td>0.086527</td>\n",
       "      <td>0.306084</td>\n",
       "      <td>0.176422</td>\n",
       "      <td>0.301727</td>\n",
       "      <td>0.014634</td>\n",
       "      <td>0.093429</td>\n",
       "      <td>0.002878</td>\n",
       "      <td>0.002736</td>\n",
       "      <td>0.003200</td>\n",
       "      <td>0.002976</td>\n",
       "      <td>0.009389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80959.0</th>\n",
       "      <td>0.924019</td>\n",
       "      <td>0.016786</td>\n",
       "      <td>0.001945</td>\n",
       "      <td>0.030483</td>\n",
       "      <td>0.005963</td>\n",
       "      <td>0.008671</td>\n",
       "      <td>0.005393</td>\n",
       "      <td>0.005177</td>\n",
       "      <td>0.001564</td>\n",
       "      <td>0.188021</td>\n",
       "      <td>0.354948</td>\n",
       "      <td>0.172444</td>\n",
       "      <td>0.192030</td>\n",
       "      <td>0.010255</td>\n",
       "      <td>0.065905</td>\n",
       "      <td>0.001918</td>\n",
       "      <td>0.002253</td>\n",
       "      <td>0.001935</td>\n",
       "      <td>0.001957</td>\n",
       "      <td>0.008335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80960.0</th>\n",
       "      <td>0.830574</td>\n",
       "      <td>0.035322</td>\n",
       "      <td>0.003268</td>\n",
       "      <td>0.051942</td>\n",
       "      <td>0.009040</td>\n",
       "      <td>0.044200</td>\n",
       "      <td>0.012632</td>\n",
       "      <td>0.009196</td>\n",
       "      <td>0.003826</td>\n",
       "      <td>0.253837</td>\n",
       "      <td>0.203410</td>\n",
       "      <td>0.197328</td>\n",
       "      <td>0.168720</td>\n",
       "      <td>0.009609</td>\n",
       "      <td>0.139658</td>\n",
       "      <td>0.004125</td>\n",
       "      <td>0.004363</td>\n",
       "      <td>0.002813</td>\n",
       "      <td>0.003385</td>\n",
       "      <td>0.012753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80961.0</th>\n",
       "      <td>0.672005</td>\n",
       "      <td>0.164621</td>\n",
       "      <td>0.012071</td>\n",
       "      <td>0.043754</td>\n",
       "      <td>0.010095</td>\n",
       "      <td>0.022950</td>\n",
       "      <td>0.048975</td>\n",
       "      <td>0.019726</td>\n",
       "      <td>0.005803</td>\n",
       "      <td>0.140200</td>\n",
       "      <td>0.336401</td>\n",
       "      <td>0.193229</td>\n",
       "      <td>0.185162</td>\n",
       "      <td>0.015779</td>\n",
       "      <td>0.101258</td>\n",
       "      <td>0.004157</td>\n",
       "      <td>0.003922</td>\n",
       "      <td>0.003787</td>\n",
       "      <td>0.004161</td>\n",
       "      <td>0.011945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80962.0</th>\n",
       "      <td>0.722733</td>\n",
       "      <td>0.077768</td>\n",
       "      <td>0.005419</td>\n",
       "      <td>0.102405</td>\n",
       "      <td>0.008666</td>\n",
       "      <td>0.053822</td>\n",
       "      <td>0.012266</td>\n",
       "      <td>0.010382</td>\n",
       "      <td>0.006539</td>\n",
       "      <td>0.139460</td>\n",
       "      <td>0.441957</td>\n",
       "      <td>0.160908</td>\n",
       "      <td>0.157873</td>\n",
       "      <td>0.013420</td>\n",
       "      <td>0.062108</td>\n",
       "      <td>0.003416</td>\n",
       "      <td>0.003230</td>\n",
       "      <td>0.003581</td>\n",
       "      <td>0.003317</td>\n",
       "      <td>0.010731</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80963 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                1         2         3         4         5         6         7  \\\n",
       "0                                                                               \n",
       "0.0      0.594534  0.048073  0.019625  0.041608  0.014619  0.190502  0.038185   \n",
       "1.0      0.362457  0.031278  0.002685  0.012323  0.004354  0.555326  0.007800   \n",
       "2.0      0.029422  0.369658  0.014396  0.015661  0.348402  0.030611  0.122547   \n",
       "3.0      0.020930  0.221499  0.030887  0.026482  0.068644  0.072682  0.492496   \n",
       "4.0      0.006419  0.932315  0.004778  0.002249  0.001461  0.036227  0.008779   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80958.0  0.053355  0.323336  0.013751  0.010410  0.013495  0.446532  0.037435   \n",
       "80959.0  0.924019  0.016786  0.001945  0.030483  0.005963  0.008671  0.005393   \n",
       "80960.0  0.830574  0.035322  0.003268  0.051942  0.009040  0.044200  0.012632   \n",
       "80961.0  0.672005  0.164621  0.012071  0.043754  0.010095  0.022950  0.048975   \n",
       "80962.0  0.722733  0.077768  0.005419  0.102405  0.008666  0.053822  0.012266   \n",
       "\n",
       "                8         9        10        11        12        13        14  \\\n",
       "0                                                                               \n",
       "0.0      0.023982  0.028871  0.189272  0.116634  0.180595  0.374547  0.029213   \n",
       "1.0      0.005447  0.018330  0.219277  0.125396  0.120398  0.418638  0.021438   \n",
       "2.0      0.009769  0.059534  0.065325  0.065170  0.111108  0.086168  0.018429   \n",
       "3.0      0.025842  0.040539  0.049214  0.130924  0.147752  0.228507  0.068703   \n",
       "4.0      0.001323  0.006449  0.024002  0.123658  0.191009  0.470429  0.080229   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80958.0  0.007922  0.093765  0.086527  0.306084  0.176422  0.301727  0.014634   \n",
       "80959.0  0.005177  0.001564  0.188021  0.354948  0.172444  0.192030  0.010255   \n",
       "80960.0  0.009196  0.003826  0.253837  0.203410  0.197328  0.168720  0.009609   \n",
       "80961.0  0.019726  0.005803  0.140200  0.336401  0.193229  0.185162  0.015779   \n",
       "80962.0  0.010382  0.006539  0.139460  0.441957  0.160908  0.157873  0.013420   \n",
       "\n",
       "               15        16        17        18        19        20  \n",
       "0                                                                    \n",
       "0.0      0.074395  0.005818  0.005557  0.004208  0.004887  0.014874  \n",
       "1.0      0.072530  0.003081  0.003248  0.002418  0.003077  0.010500  \n",
       "2.0      0.547296  0.039902  0.022893  0.012779  0.015562  0.015368  \n",
       "3.0      0.262241  0.029974  0.021974  0.020983  0.020707  0.019022  \n",
       "4.0      0.092186  0.002533  0.002045  0.002907  0.003255  0.007746  \n",
       "...           ...       ...       ...       ...       ...       ...  \n",
       "80958.0  0.093429  0.002878  0.002736  0.003200  0.002976  0.009389  \n",
       "80959.0  0.065905  0.001918  0.002253  0.001935  0.001957  0.008335  \n",
       "80960.0  0.139658  0.004125  0.004363  0.002813  0.003385  0.012753  \n",
       "80961.0  0.101258  0.004157  0.003922  0.003787  0.004161  0.011945  \n",
       "80962.0  0.062108  0.003416  0.003230  0.003581  0.003317  0.010731  \n",
       "\n",
       "[80963 rows x 20 columns]"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = pd.read_csv(args.save_dir + 'preds_XL_trans.csv', sep='\\t', index_col='0')\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = torch.tensor(np.array(preds)).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df = pd.DataFrame(('val_ATT_loss', 'val_VAL_loss', 'val_ATT_acc', 'val_VAL_acc', 'val_VAL_acc_k', 'val_VAL_jac_k'), columns=['name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ATT_loss = F.cross_entropy(data_XL.y[data_XL.train_mask][:,:9], \n",
    "                pred[data_XL.train_mask][:,:9]).cpu().detach().item()\n",
    "train_VAL_loss = F.cross_entropy(data_XL.y[data_XL.train_mask][:,9:], \n",
    "                pred[data_XL.train_mask][:,9:]).cpu().detach().item()\n",
    "\n",
    "train_ATT_acc = compute_1_accuracy(data_XL.y[data_XL.train_mask][:,:9], \n",
    "                pred[data_XL.train_mask][:,:9])\n",
    "train_VAL_acc = compute_1_accuracy(data_XL.y[data_XL.train_mask][:,9:], \n",
    "                pred[data_XL.train_mask][:,9:])\n",
    "train_VAL_acc_k = compute_k_accuracy(data_XL.y[data_XL.train_mask][:,9:].cpu(),  \n",
    "                pred[data_XL.train_mask][:,9:].cpu(),3)\n",
    "train_VAL_jac_k = compute_jaccard_index(data_XL.y[data_XL.train_mask][:,9:].cpu(),  \n",
    "                pred[data_XL.train_mask][:,9:].cpu(),3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df['train'] = pd.DataFrame((train_ATT_loss, train_VAL_loss, train_ATT_acc, train_VAL_acc, train_VAL_acc_k, train_VAL_jac_k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_ATT_loss = F.cross_entropy(data_XL.y[(data_XL.att_lab) * data_XL.val_mask][:,:9], \n",
    "                pred[(data_XL.att_lab) * data_XL.val_mask][:,:9]).cpu().detach().item()\n",
    "val_VAL_loss = F.cross_entropy(data_XL.y[(data_XL.val_lab) * data_XL.val_mask][:,9:], \n",
    "                pred[(data_XL.val_lab) * data_XL.val_mask][:,9:]).cpu().detach().item()\n",
    "\n",
    "val_ATT_acc = compute_1_accuracy(data_XL.y[(data_XL.att_lab) * data_XL.val_mask][:,:9], \n",
    "                pred[(data_XL.att_lab) * data_XL.val_mask][:,:9])\n",
    "val_VAL_acc = compute_1_accuracy(data_XL.y[(data_XL.val_lab) * data_XL.val_mask][:,9:], \n",
    "                pred[(data_XL.val_lab) * data_XL.val_mask][:,9:])\n",
    "val_VAL_acc_k = compute_k_accuracy(data_XL.y[(data_XL.val_lab) * data_XL.val_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL.val_lab) * data_XL.val_mask][:,9:].cpu(),3)\n",
    "val_VAL_jac_k = compute_jaccard_index(data_XL.y[(data_XL.val_lab) * data_XL.val_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL.val_lab) * data_XL.val_mask][:,9:].cpu(),3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df['val'] = pd.DataFrame((val_ATT_loss, val_VAL_loss, val_ATT_acc, val_VAL_acc, val_VAL_acc_k, val_VAL_jac_k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ATT_loss = F.cross_entropy(data_XL.y[(data_XL.att_lab) * data_XL.test_mask][:,:9], \n",
    "                pred[(data_XL.att_lab) * data_XL.test_mask][:,:9]).cpu().detach().item()\n",
    "test_VAL_loss = F.cross_entropy(data_XL.y[(data_XL.val_lab) * data_XL.test_mask][:,9:], \n",
    "                pred[(data_XL.val_lab) * data_XL.test_mask][:,9:]).cpu().detach().item()\n",
    "\n",
    "test_ATT_acc = compute_1_accuracy(data_XL.y[(data_XL.att_lab) * data_XL.test_mask][:,:9], \n",
    "                pred[(data_XL.att_lab) * data_XL.test_mask][:,:9])\n",
    "test_VAL_acc = compute_1_accuracy(data_XL.y[(data_XL.val_lab) * data_XL.test_mask][:,9:], \n",
    "                pred[(data_XL.val_lab) * data_XL.test_mask][:,9:])\n",
    "test_VAL_acc_k = compute_k_accuracy(data_XL.y[(data_XL.val_lab) * data_XL.test_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL.val_lab) * data_XL.test_mask][:,9:].cpu(),3)\n",
    "test_VAL_jac_k = compute_jaccard_index(data_XL.y[(data_XL.val_lab) * data_XL.test_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL.val_lab) * data_XL.test_mask][:,9:].cpu(),3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df['test'] = pd.DataFrame((test_ATT_loss, test_VAL_loss, test_ATT_acc, test_VAL_acc, test_VAL_acc_k, test_VAL_jac_k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>train</th>\n",
       "      <th>val</th>\n",
       "      <th>test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>val_ATT_loss</td>\n",
       "      <td>1.749471</td>\n",
       "      <td>1.740666</td>\n",
       "      <td>1.740848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>val_VAL_loss</td>\n",
       "      <td>2.250396</td>\n",
       "      <td>2.248371</td>\n",
       "      <td>2.247688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>val_ATT_acc</td>\n",
       "      <td>91.580949</td>\n",
       "      <td>96.860673</td>\n",
       "      <td>96.794024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>val_VAL_acc</td>\n",
       "      <td>79.159824</td>\n",
       "      <td>80.527355</td>\n",
       "      <td>80.520394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>val_VAL_acc_k</td>\n",
       "      <td>98.668856</td>\n",
       "      <td>98.702764</td>\n",
       "      <td>98.860759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>val_VAL_jac_k</td>\n",
       "      <td>0.744237</td>\n",
       "      <td>0.752468</td>\n",
       "      <td>0.752192</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            name      train        val       test\n",
       "0   val_ATT_loss   1.749471   1.740666   1.740848\n",
       "1   val_VAL_loss   2.250396   2.248371   2.247688\n",
       "2    val_ATT_acc  91.580949  96.860673  96.794024\n",
       "3    val_VAL_acc  79.159824  80.527355  80.520394\n",
       "4  val_VAL_acc_k  98.668856  98.702764  98.860759\n",
       "5  val_VAL_jac_k   0.744237   0.752468   0.752192"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df.to_csv(args.save_dir+'eval_metrics_XL_trans.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Per-Class Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_k_confusion_matrix(y, y_pred, k=3):\n",
    "    dim = y.shape[-1]\n",
    "    y = y.topk(k=k, axis=1)[1]\n",
    "    y_pred = y_pred.topk(k=k, axis=1)[1]\n",
    "    conf = np.zeros((dim, dim))\n",
    "    for i in range(k):\n",
    "        for j in range(k):\n",
    "            conf = np.add(conf, confusion_matrix(y[:,i], y_pred[:,j], labels = range(dim)))\n",
    "    return conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ATT_conf = confusion_matrix(data_XL.y[(data_XL.att_lab) * data_XL.test_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                pred[(data_XL.att_lab) * data_XL.test_mask][:,:9].argmax(axis=1).cpu())\n",
    "test_VAL_conf = confusion_matrix(data_XL.y[(data_XL.val_lab) * data_XL.test_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                pred[(data_XL.val_lab) * data_XL.test_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "test_VAL_conf_k = (top_k_confusion_matrix(data_XL.y[(data_XL.val_lab) * data_XL.test_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL.val_lab) * data_XL.test_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1371,    2,    0,    5,    1,    8,    2,    0,    0],\n",
       "       [   4, 3194,    0,    2,    2,    5,    4,    0,    0],\n",
       "       [   0,    0,  233,    0,    0,    0,   31,    1,    0],\n",
       "       [  22,    1,    0,  603,    0,    1,    9,    1,    0],\n",
       "       [   1,    2,    0,    0, 2694,    0,    7,    0,    0],\n",
       "       [   4,   19,    0,    2,    5, 1551,    2,    0,    0],\n",
       "       [   3,   13,    0,    1,    5,    0, 2569,    2,    0],\n",
       "       [  14,    3,   20,   14,    0,    0,  150,  112,    0],\n",
       "       [   1,   11,    0,    0,   19,   13,    0,    0,  112]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 228,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 524,   52,   33,   15,    0,   56,    0,    0,    0,    0,    0],\n",
       "       [  23,  577,   70,   61,    0,  118,    0,    0,    0,    0,    0],\n",
       "       [  27,   96, 1066,   63,    0,  355,    0,    0,    0,    0,    0],\n",
       "       [  20,   17,   44,  393,    0,   62,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [  23,   28,  169,   43,    0, 3152,    0,    0,    0,    0,    0],\n",
       "       [   1,    0,    0,    0,    0,    4,    6,    0,    0,    1,    0],\n",
       "       [   0,    0,    0,    0,    0,    1,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    1,    0,    0,    0,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    2,    0,    0,    7,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1250,  852, 1229,  818,    0,  800,    4,    0,    0,    0,    0],\n",
       "       [ 784, 1722, 2330, 1774,    2, 1593,    3,    0,    0,    0,    0],\n",
       "       [ 806, 1544, 5855, 4774,    4, 5090,   11,    0,    0,    0,    0],\n",
       "       [ 735, 1443, 4845, 4611,    5, 4162,    3,    0,    0,    0,    0],\n",
       "       [   0,    7,   17,   18,    2,   10,    0,    0,    0,    0,    0],\n",
       "       [ 690,  987, 5311, 4439,    2, 5231,   10,    0,    0,    1,    0],\n",
       "       [   7,    0,   14,    6,    0,   25,   23,    1,    3,   17,    0],\n",
       "       [   1,    0,    1,    0,    0,    6,    6,    0,    1,    6,    0],\n",
       "       [   1,    0,    1,    0,    0,   10,   14,    1,    2,   13,    0],\n",
       "       [   1,    0,    2,    0,    0,   14,   19,    1,    3,   17,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]])"
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ True,  True, False,  ...,  True,  True,  True])"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_XL.val_mask[data_XL.val_mask + data_XL.train_mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_ATT_conf = confusion_matrix(data_XL.y[(data_XL.att_lab) * data_XL.val_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                pred[(data_XL.att_lab) * data_XL.val_mask][:,:9].argmax(axis=1).cpu())\n",
    "val_VAL_conf = confusion_matrix(data_XL.y[(data_XL.val_lab) * data_XL.val_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                pred[(data_XL.val_lab) * data_XL.val_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "val_VAL_conf_k = (top_k_confusion_matrix(data_XL.y[(data_XL.val_lab) * data_XL.val_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL.val_lab) * data_XL.val_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1342,    1,    0,    4,    0,    3,    1,    1,    0],\n",
       "       [   6, 3275,    0,    1,    0,    7,   10,    0,    0],\n",
       "       [   0,    0,  258,    0,    0,    0,   19,    3,    0],\n",
       "       [  33,    4,    0,  635,    0,    0,    7,    1,    0],\n",
       "       [   1,    3,    0,    0, 2674,    0,    8,    1,    0],\n",
       "       [   9,   19,    0,    3,    4, 1452,    1,    0,    0],\n",
       "       [   3,   12,    1,    0,    4,    0, 2609,    1,    0],\n",
       "       [  11,    1,   23,   10,    0,    0,  146,   95,    0],\n",
       "       [   0,    8,    0,    0,   26,    8,    0,    0,  125]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 233,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 509,   42,   45,   26,    0,   45,    0,    0,    0,    0,    0],\n",
       "       [  18,  569,   66,   78,    0,   98,    0,    0,    0,    0,    0],\n",
       "       [  21,   85, 1101,   75,    0,  361,    0,    0,    0,    1,    0],\n",
       "       [  22,   20,   42,  444,    0,   70,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    1,    3,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [  22,   33,  165,   33,    0, 3079,    1,    0,    0,    0,    0],\n",
       "       [   0,    0,    0,    0,    0,    1,    4,    0,    0,    0,    0],\n",
       "       [   0,    0,    0,    0,    0,    1,    1,    0,    0,    1,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    1,    0],\n",
       "       [   0,    0,    0,    0,    0,    1,    2,    0,    0,    5,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 234,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1263,  864, 1228,  857,    0,  802,    2,    0,    0,    0,    0],\n",
       "       [ 832, 1724, 2426, 1899,    8, 1644,    2,    0,    0,    0,    0],\n",
       "       [ 825, 1545, 5811, 4743,    9, 5017,   18,    0,    0,    2,    0],\n",
       "       [ 715, 1443, 4757, 4575,   15, 4054,    8,    0,    0,    0,    0],\n",
       "       [   0,    7,   18,   21,   11,    8,    1,    0,    0,    0,    0],\n",
       "       [ 661,  957, 5233, 4429,    2, 5142,   20,    0,    0,    2,    0],\n",
       "       [   9,    0,   23,   14,    2,   32,   23,    1,    2,   14,    0],\n",
       "       [   1,    0,    0,    0,    0,    3,    2,    0,    0,    3,    0],\n",
       "       [   2,    0,    3,    0,    1,    9,   15,    1,    2,   15,    0],\n",
       "       [   3,    0,    4,    1,    0,   11,   14,    1,    2,   15,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]])"
      ]
     },
     "execution_count": 235,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ATT_conf = confusion_matrix(data_XL.y[data_XL.train_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                                  pred[data_XL.train_mask][:,:9].argmax(axis=1).cpu())\n",
    "train_VAL_conf = confusion_matrix(data_XL.y[data_XL.train_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                                 pred[data_XL.train_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "train_VAL_conf_k = (top_k_confusion_matrix(data_XL.y[data_XL.train_mask][:,9:].cpu(),  \n",
    "                                pred[data_XL.train_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(11569)"
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_XL.train_mask.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1479,    2,    0,    1,    1,   12,    6,    0,    0],\n",
       "       [   0, 2617,    0,    3,    0,    6,   10,    0,    0],\n",
       "       [   2,    2,   92,    0,    0,    0,   42,    1,    0],\n",
       "       [  38,   14,    0,  403,    0,    0,   24,    1,    0],\n",
       "       [   5,    8,    0,    0, 2008,    0,   28,    2,    0],\n",
       "       [   6,   42,    0,    1,    8, 1449,    1,    0,    0],\n",
       "       [   2,   19,    0,    2,    1,    0, 2431,    2,    0],\n",
       "       [  22,    5,    8,    8,    0,    0,  578,   64,    0],\n",
       "       [   0,   15,    0,    0,   34,   12,    0,    0,   52]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 782,   66,   53,   22,    0,   89,    0,    0,    0,    0,    0],\n",
       "       [  26,  889,  149,  120,    0,  194,    0,    0,    0,    0,    0],\n",
       "       [  32,  144, 1872,   95,    0,  672,    0,    0,    0,    0,    0],\n",
       "       [  20,   47,   90,  797,    0,   89,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    0,    1,    0,    1,    0,    0,    0,    0,    0],\n",
       "       [  38,   50,  306,   91,    0, 4798,    0,    0,    0,    0,    0],\n",
       "       [   2,    0,    0,    1,    0,    5,   12,    0,    0,    1,    0],\n",
       "       [   1,    0,    0,    0,    0,    0,    0,    0,    0,    1,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    1,    0,    0,    0,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    4,    0,    0,    8,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 238,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1875, 1234, 1734, 1309,    0, 1233,    2,    0,    0,    2,    0],\n",
       "       [1169, 2880, 4035, 3253,    5, 2767,    3,    0,    0,    0,    0],\n",
       "       [1213, 2707, 9575, 7787,    4, 8289,   17,    0,    0,    0,    0],\n",
       "       [1137, 2517, 7831, 7531,    9, 6699,   10,    0,    0,    0,    0],\n",
       "       [   0,   25,   49,   53,    9,   26,    0,    0,    0,    0,    0],\n",
       "       [1052, 1684, 8529, 7104,    0, 8374,   19,    0,    0,    1,    0],\n",
       "       [  12,    2,   32,   17,    0,   43,   34,    2,    5,   27,    0],\n",
       "       [   6,    0,    7,    3,    0,   12,   14,    1,    0,   11,    0],\n",
       "       [   3,    0,    3,    0,    0,   10,   19,    1,    5,   16,    0],\n",
       "       [   4,    0,    8,    0,    0,   15,   26,    2,    5,   24,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]])"
      ]
     },
     "execution_count": 239,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat([pd.DataFrame(train_ATT_conf),pd.DataFrame(val_ATT_conf),pd.DataFrame(test_ATT_conf)],axis=1).to_csv(args.save_dir+'confusion_matrix_ATT_XL_trans.csv')\n",
    "pd.concat([pd.DataFrame(train_VAL_conf),pd.DataFrame(val_VAL_conf),pd.DataFrame(test_VAL_conf)],axis=1).to_csv(args.save_dir+'confusion_matrix_VAL_XL_trans.csv')\n",
    "pd.concat([pd.DataFrame(train_VAL_conf_k),pd.DataFrame(val_VAL_conf_k),pd.DataFrame(test_VAL_conf_k)],axis=1).to_csv(args.save_dir+'confusion_matrix_VAL_k_XL_trans.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics(confusion_matrix, classes):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val/(row.sum()+0.000000001)\n",
    "        recall = val/(col.sum()+0.000000001)\n",
    "        F1 = 2*(precision*recall)/(precision+recall+0.000000001)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics_k(confusion_matrix, classes, k=3):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val*k/(row.sum()+0.000000001)\n",
    "        recall = val*k/(col.sum()+0.000000001)\n",
    "        F1 = 2*(precision*recall)/(precision+recall+0.000000001)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = ['Criterion i', 'Criterion ii', 'Criterion iii', 'Criterion iv', 'Criterion v', 'Criterion vi', \n",
    "              'Criterion vii', 'Criterion viii', 'Criterion ix', 'Criterion x', 'Others']\n",
    "categories = ['Building Elements',\n",
    " 'Urban Form Elements',\n",
    " 'Gastronomy',\n",
    " 'Interior Scenery',\n",
    " 'Natural Features and Land-scape Scenery',\n",
    " 'Monuments and Buildings',\n",
    " 'Peoples Activity and Association',\n",
    " 'Artifact Products',\n",
    " 'Urban Scenery']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_dict = {}\n",
    "metrics_dict['test_ATT'] = per_class_metrics(test_ATT_conf, categories)\n",
    "metrics_dict['val_ATT'] = per_class_metrics(val_ATT_conf, categories)\n",
    "metrics_dict['test_VAL'] = per_class_metrics(test_VAL_conf, classes)\n",
    "metrics_dict['val_VAL'] = per_class_metrics(val_VAL_conf, classes)\n",
    "metrics_dict['test_VAL_k'] = per_class_metrics_k(test_VAL_conf_k, classes)\n",
    "metrics_dict['val_VAL_k'] = per_class_metrics_k(val_VAL_conf_k, classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df = pd.DataFrame.from_dict({(i,j): metrics_dict[i][j] \n",
    "                           for i in metrics_dict.keys() \n",
    "                           for j in metrics_dict[i].keys()},\n",
    "                       orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">test_ATT</th>\n",
       "      <th>Building Elements</th>\n",
       "      <td>0.987041</td>\n",
       "      <td>0.965493</td>\n",
       "      <td>0.976148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Urban Form Elements</th>\n",
       "      <td>0.994706</td>\n",
       "      <td>0.984284</td>\n",
       "      <td>0.989467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gastronomy</th>\n",
       "      <td>0.879245</td>\n",
       "      <td>0.920949</td>\n",
       "      <td>0.899614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Interior Scenery</th>\n",
       "      <td>0.946625</td>\n",
       "      <td>0.961722</td>\n",
       "      <td>0.954114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Natural Features and Land-scape Scenery</th>\n",
       "      <td>0.996302</td>\n",
       "      <td>0.988261</td>\n",
       "      <td>0.992265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">val_VAL_k</th>\n",
       "      <th>Criterion vii</th>\n",
       "      <td>0.575000</td>\n",
       "      <td>0.657143</td>\n",
       "      <td>0.613333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Criterion viii</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Criterion ix</th>\n",
       "      <td>0.125000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.222222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Criterion x</th>\n",
       "      <td>0.882353</td>\n",
       "      <td>0.882353</td>\n",
       "      <td>0.882353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Others</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>62 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   precision    recall  \\\n",
       "test_ATT  Building Elements                         0.987041  0.965493   \n",
       "          Urban Form Elements                       0.994706  0.984284   \n",
       "          Gastronomy                                0.879245  0.920949   \n",
       "          Interior Scenery                          0.946625  0.961722   \n",
       "          Natural Features and Land-scape Scenery   0.996302  0.988261   \n",
       "...                                                      ...       ...   \n",
       "val_VAL_k Criterion vii                             0.575000  0.657143   \n",
       "          Criterion viii                            0.000000  0.000000   \n",
       "          Criterion ix                              0.125000  1.000000   \n",
       "          Criterion x                               0.882353  0.882353   \n",
       "          Others                                    0.000000  0.000000   \n",
       "\n",
       "                                                         F1  \n",
       "test_ATT  Building Elements                        0.976148  \n",
       "          Urban Form Elements                      0.989467  \n",
       "          Gastronomy                               0.899614  \n",
       "          Interior Scenery                         0.954114  \n",
       "          Natural Features and Land-scape Scenery  0.992265  \n",
       "...                                                     ...  \n",
       "val_VAL_k Criterion vii                            0.613333  \n",
       "          Criterion viii                           0.000000  \n",
       "          Criterion ix                             0.222222  \n",
       "          Criterion x                              0.882353  \n",
       "          Others                                   0.000000  \n",
       "\n",
       "[62 rows x 3 columns]"
      ]
     },
     "execution_count": 256,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df.to_csv(args.save_dir+'per_class_metrics_XL_trans.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}