SAGE.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SAGE Models to Compute Heritage Values and Attributes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import os.path as osp\n",
    "from itertools import product\n",
    "from typing import Callable, List, Optional\n",
    "\n",
    "import numpy as np\n",
    "import scipy.sparse as sp\n",
    "\n",
    "from torch_geometric.data import (\n",
    "    HeteroData,\n",
    "    Data,\n",
    "    InMemoryDataset,\n",
    "    download_url,\n",
    "    extract_zip,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from argparse import Namespace\n",
    "from collections import Counter\n",
    "import json\n",
    "import os\n",
    "import re\n",
    "import string\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import random\n",
    "\n",
    "import torch\n",
    "torch.manual_seed(1337)\n",
    "torch.backends.cudnn.benchmark = True\n",
    "\n",
    "from torch_geometric.transforms import RandomLinkSplit, ToUndirected\n",
    "import torch_geometric.transforms as T\n",
    "from torch_geometric.nn import GATConv, Linear, to_hetero\n",
    "from torch_geometric.utils import to_undirected\n",
    "\n",
    "import torch.nn.functional as F\n",
    "from torch_geometric import seed_everything\n",
    "\n",
    "from tqdm import tqdm\n",
    "import torch.optim as optim\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.nn import SAGEConv\n",
    "from torch_cluster import knn_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'D:\\\\surf\\\\TUD\\\\Paper\\\\Venice_Graph'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch version 1.10.2\n",
      "GPU-enabled installation? True\n"
     ]
    }
   ],
   "source": [
    "print(\"PyTorch version {}\".format(torch.__version__))\n",
    "print(\"GPU-enabled installation? {}\".format(torch.cuda.is_available()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "#device = torch.device(\"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed_everywhere(seed, cuda):\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    random.seed(seed)\n",
    "    if cuda:\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "\n",
    "def handle_dirs(dirpath):\n",
    "    if not os.path.exists(dirpath):\n",
    "        os.makedirs(dirpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = Namespace(\n",
    "    # Data and Path information\n",
    "    path = 'dataset/Venice',\n",
    "    save_dir='model_storage/SAGE/',\n",
    "    model_state_file='model.pth',\n",
    "    \n",
    "    # Model hyper parameters\n",
    "    hidden_channels = 128,\n",
    "    num_layers = 3,\n",
    "    k=3,\n",
    "    use_gdc=True,\n",
    "    \n",
    "    # Training hyper parameters\n",
    "    sample_nodes = 25,\n",
    "    batch_size=32,\n",
    "    early_stopping_criteria=30,\n",
    "    learning_rate=0.001,\n",
    "    l2=2e-4,\n",
    "    dropout_p=0.5,\n",
    "    num_epochs=300,\n",
    "    seed=42,\n",
    "    \n",
    "    # Runtime options\n",
    "    catch_keyboard_interrupt=True,\n",
    "    cuda=True,\n",
    "    expand_filepaths_to_save_dir=True,\n",
    "    reload_from_files=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expanded filepaths: \n",
      "\tmodel_storage/SAGE/model.pth\n"
     ]
    }
   ],
   "source": [
    "if args.expand_filepaths_to_save_dir:\n",
    "    args.model_state_file = os.path.join(args.save_dir,\n",
    "                                         args.model_state_file)\n",
    "    \n",
    "    print(\"Expanded filepaths: \")\n",
    "    print(\"\\t{}\".format(args.model_state_file))\n",
    "\n",
    "# handle dirs\n",
    "handle_dirs(args.save_dir)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Constructing Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VEN(InMemoryDataset):\n",
    "    r\"\"\"A subset of Flickr post collected in Venice annotated with Heritage \n",
    "    Values and Attributes, as collected in the `\"Heri-Graphs: A Workflow of \n",
    "    Creating Datasets for Multi-modal Machine Learning on Graphs of Heritage \n",
    "    Values and Attributes with Social Media\" <https://arxiv.org/abs/2205.07545>`\n",
    "    paper.\n",
    "    VEN is a heterogeneous graph containing two types of nodes - nodes with only \n",
    "    visual features 'vis_only' (1,190 nodes), nodes with both visual and textual\n",
    "    features 'vis_tex' (1,761 nodes) and four types of links - social similarity\n",
    "    'SOC' (488,103 links), spatial similarity (445,779 links), temporal similarity\n",
    "    (501,191 links), and simple composed link (1,071,977 links).\n",
    "    Vis_only nodes are represented with 982-dimensional visual features and are\n",
    "    divided into 9 heritage attribute categories \n",
    "    ('architectural elements', 'form', 'gastronomy', 'interior',\n",
    "    'landscape scenery and natural features', 'monuments', 'people', 'product', \n",
    "    'urban scenery').\n",
    "    Vis_text nodes are represented with 1753-dimensional visual and textual \n",
    "    features and are divided into 9 heritage attribute categories plus 11 \n",
    "    heritage value categories ('criterion i-x', 'other').\n",
    "    Both types of nodes are also merged into a single type of node 'all' with \n",
    "    1753-dimensional features and 20-dimensional label categories.\n",
    "\n",
    "    Args:\n",
    "        root (string): Root directory where the dataset should be saved.\n",
    "        transform (callable, optional): A function/transform that takes in an\n",
    "            :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            every access. (default: :obj:`None`)\n",
    "        pre_transform (callable, optional): A function/transform that takes in\n",
    "            an :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            being saved to disk. (default: :obj:`None`)\n",
    "    \n",
    "    Stats:\n",
    "            * - #nodes\n",
    "              - #edges\n",
    "              - #features\n",
    "              - #classes\n",
    "            * - 2,951\n",
    "              - 1,071,977\n",
    "              - 1753\n",
    "              - 20\n",
    "    \"\"\"\n",
    "\n",
    "    url = 'https://drive.google.com/uc?export=download&id=1sxcKiZr1YGDv06wr03nsk5HVZledgzi9'\n",
    "\n",
    "    def __init__(self, root: str, transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "        \n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            'A_simp.npz', 'A_SOC.npz', 'A_SPA.npz', 'A_TEM.npz', 'labels.npz',\n",
    "            'node_types.npy', 'Textual_Features.npy', 'train_val_test_idx.npz',\n",
    "            'Visual_Features.npy'\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        os.remove(path)\n",
    "\n",
    "    def process(self):\n",
    "        data = HeteroData()\n",
    "\n",
    "        node_types = ['vis_only', 'vis_tex']\n",
    "        link_types = ['SOC', 'SPA', 'TEM', 'simp']\n",
    "\n",
    "        vis = np.load(osp.join(self.raw_dir, 'Visual_Features.npy'),allow_pickle=True)[:,2:].astype(float)\n",
    "        tex = np.load(osp.join(self.raw_dir, 'Textual_Features.npy'),allow_pickle=True)[:,5:].astype(float)\n",
    "\n",
    "        x = np.hstack([vis,np.nan_to_num(tex)])\n",
    "\n",
    "\n",
    "        node_type_idx = np.load(osp.join(self.raw_dir, 'node_types.npy'))\n",
    "        node_type_idx = torch.from_numpy(node_type_idx).to(torch.long)\n",
    "\n",
    "        data['vis_only'].num_nodes = int((node_type_idx == 0).sum())\n",
    "        data['vis_tex'].num_nodes = int((node_type_idx == 1).sum())\n",
    "        data['all'].num_nodes = len(node_type_idx)\n",
    "\n",
    "        data['vis_only'].x = torch.from_numpy(vis[node_type_idx==0]).to(torch.float)\n",
    "        data['vis_tex'].x = torch.from_numpy(x[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].x = torch.from_numpy(x).to(torch.float)\n",
    "\n",
    "\n",
    "        y_s = np.load(osp.join(self.raw_dir, 'labels.npz'), allow_pickle=True)\n",
    "        att_lab = y_s['ATT_LAB'][:,1:10].astype(float)\n",
    "        val_lab = np.nan_to_num(y_s['VAL_LAB'][:,2:13].astype(float))\n",
    "        ys = np.hstack([att_lab, val_lab])\n",
    "\n",
    "        data['vis_only'].y = torch.from_numpy(att_lab[node_type_idx==0]).to(torch.float)\n",
    "        data['vis_tex'].y = torch.from_numpy(ys[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].y = torch.from_numpy(ys).to(torch.float)\n",
    "\n",
    "        data.node_type = node_type_idx\n",
    "\n",
    "        split = np.load(osp.join(self.raw_dir, 'train_val_test_idx.npz'))\n",
    "        for name in ['train', 'val', 'test']:\n",
    "            idx = split[f'{name}_idx']\n",
    "            idx = torch.from_numpy(idx).to(torch.long)\n",
    "            mask = torch.zeros(data['all'].num_nodes, dtype=torch.bool)\n",
    "            mask[idx] = True\n",
    "            data['all'][f'{name}_mask'] = mask\n",
    "            data['vis_only'][f'{name}_mask'] = mask[node_type_idx==0]\n",
    "            data['vis_tex'][f'{name}_mask'] = mask[node_type_idx==1]\n",
    "\n",
    "        \n",
    "        s = {}\n",
    "        s['vis_only'] = np.arange(len(x))[node_type_idx==0]\n",
    "        s['vis_tex'] = np.arange(len(x))[node_type_idx==1]\n",
    "\n",
    "        for link in link_types:\n",
    "            A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz')).tocoo()\n",
    "            if A_sub.nnz>0:\n",
    "                row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                data['all', f'{link}_link', 'all'].edge_index = torch.stack([row, col], dim=0)\n",
    "                data['all', f'{link}_link', 'all'].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "        for src, dst in product(node_types, node_types):\n",
    "            for link in link_types:\n",
    "                A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz'))[s[src]][:,s[dst]].tocoo()\n",
    "                if A_sub.nnz>0:\n",
    "                    row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                    col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                    data[src, f'{link}_link', dst].edge_index = torch.stack([row, col], dim=0)\n",
    "                    data[src, f'{link}_link', dst].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "\n",
    "        if self.pre_transform is not None:\n",
    "            data = self.pre_transform(data)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.__class__.__name__}()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VEN_Homo(InMemoryDataset):\n",
    "    r\"\"\"A subset of Flickr post collected in Venice annotated with Heritage \n",
    "    Values and Attributes, as collected in the `\"Heri-Graphs: A Workflow of \n",
    "    Creating Datasets for Multi-modal Machine Learning on Graphs of Heritage \n",
    "    Values and Attributes with Social Media\" <https://arxiv.org/abs/2205.07545>`\n",
    "    paper.\n",
    "    VEN_Homo is a homogeneous graph containing 2951 nodes and 1,071,977 links.\n",
    "    Vis_only nodes are represented with 982-dimensional visual features and are\n",
    "    divided into 9 heritage attribute categories \n",
    "    ('architectural elements', 'form', 'gastronomy', 'interior',\n",
    "    'landscape scenery and natural features', 'monuments', 'people', 'product', \n",
    "    'urban scenery').\n",
    "    Vis_text nodes are represented with 1753-dimensional visual and textual \n",
    "    features and are divided into 9 heritage attribute categories plus 11 \n",
    "    heritage value categories ('criterion i-x', 'other').\n",
    "    Both types of nodes are also merged into a single type of node 'all' with \n",
    "    1753-dimensional features and 20-dimensional label categories.\n",
    "\n",
    "    Args:\n",
    "        root (string): Root directory where the dataset should be saved.\n",
    "        transform (callable, optional): A function/transform that takes in an\n",
    "            :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            every access. (default: :obj:`None`)\n",
    "        pre_transform (callable, optional): A function/transform that takes in\n",
    "            an :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            being saved to disk. (default: :obj:`None`)\n",
    "    \n",
    "    Stats:\n",
    "            * - #nodes\n",
    "              - #edges\n",
    "              - #features\n",
    "              - #classes\n",
    "            * - 2,951\n",
    "              - 1,071,977\n",
    "              - 1753\n",
    "              - 20\n",
    "    \"\"\"\n",
    "\n",
    "    url = 'https://drive.google.com/uc?export=download&id=1sxcKiZr1YGDv06wr03nsk5HVZledgzi9'\n",
    "\n",
    "    def __init__(self, root: str, transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "        \n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            'A_simp.npz', 'A_SOC.npz', 'A_SPA.npz', 'A_TEM.npz', 'labels.npz',\n",
    "            'node_types.npy', 'Textual_Features.npy', 'train_val_test_idx.npz',\n",
    "            'Visual_Features.npy'\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        os.remove(path)\n",
    "\n",
    "    def process(self):\n",
    "        data = Data()\n",
    "\n",
    "        link_types = ['simp']\n",
    "\n",
    "        vis = np.load(osp.join(self.raw_dir, 'Visual_Features.npy'),allow_pickle=True)[:,2:].astype(float)\n",
    "        tex = np.load(osp.join(self.raw_dir, 'Textual_Features.npy'),allow_pickle=True)[:,5:].astype(float)\n",
    "\n",
    "        x = np.hstack([vis,np.nan_to_num(tex)])\n",
    "\n",
    "        node_type_idx = np.load(osp.join(self.raw_dir, 'node_types.npy'))\n",
    "        node_type_idx = torch.from_numpy(node_type_idx).to(torch.long)\n",
    "\n",
    "        data.num_nodes = len(node_type_idx)\n",
    "\n",
    "        data.x = torch.from_numpy(x).to(torch.float)\n",
    "\n",
    "\n",
    "        y_s = np.load(osp.join(self.raw_dir, 'labels.npz'), allow_pickle=True)\n",
    "        att_lab = y_s['ATT_LAB'][:,1:10].astype(float)\n",
    "        val_lab = np.nan_to_num(y_s['VAL_LAB'][:,2:13].astype(float))\n",
    "        ys = np.hstack([att_lab, val_lab])\n",
    "\n",
    "        data.y = torch.from_numpy(ys).to(torch.float)\n",
    "\n",
    "        data.node_type = node_type_idx\n",
    "        \n",
    "        data.att_lab = torch.tensor(y_s['ATT_LAB'][:,-1].astype(bool))\n",
    "        data.val_lab = torch.tensor(y_s['VAL_LAB'][:,-1].astype(bool))\n",
    "\n",
    "        split = np.load(osp.join(self.raw_dir, 'train_val_test_idx.npz'))\n",
    "        for name in ['train', 'val', 'test']:\n",
    "            idx = split[f'{name}_idx']\n",
    "            idx = torch.from_numpy(idx).to(torch.long)\n",
    "            mask = torch.zeros(data.num_nodes, dtype=torch.bool)\n",
    "            mask[idx] = True\n",
    "            data[f'{name}_mask'] = mask\n",
    "                    \n",
    "        s = {}\n",
    "        \n",
    "        for link in link_types:\n",
    "            A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz')).tocoo()\n",
    "            if A_sub.nnz>0:\n",
    "                row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                data.edge_index = torch.stack([row, col], dim=0)\n",
    "                data.edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "        if self.pre_transform is not None:\n",
    "            data = self.pre_transform(data)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.__class__.__name__}()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VEN_XL(InMemoryDataset):\n",
    "    r\"\"\"A large subset of Flickr post collected in Venice annotated with Heritage \n",
    "    Values and Attributes, as collected in the `\"Heri-Graphs: A Workflow of \n",
    "    Creating Datasets for Multi-modal Machine Learning on Graphs of Heritage \n",
    "    Values and Attributes with Social Media\" <https://arxiv.org/abs/2205.07545>`\n",
    "    paper.\n",
    "    VEN_XL is a heterogeneous graph containing two types of nodes - nodes with only \n",
    "    visual features 'vis_only' (31,140 nodes), nodes with both visual and textual\n",
    "    features 'vis_tex' (49,823 nodes) and four types of links - social similarity\n",
    "    'SOC' (76,422,265 links), spatial similarity (202,173,159 links), temporal similarity\n",
    "    (71,135,671 links), and simple composed link (290,091,503 links).\n",
    "    Vis_only nodes are represented with 982-dimensional visual features and are\n",
    "    divided into 9 heritage attribute categories \n",
    "    ('architectural elements', 'form', 'gastronomy', 'interior',\n",
    "    'landscape scenery and natural features', 'monuments', 'people', 'product', \n",
    "    'urban scenery').\n",
    "    Vis_text nodes are represented with 1753-dimensional visual and textual \n",
    "    features and are divided into 9 heritage attribute categories plus 11 \n",
    "    heritage value categories ('criterion i-x', 'other').\n",
    "    Both types of nodes are also merged into a single type of node 'all' with \n",
    "    1753-dimensional features and 20-dimensional label categories.\n",
    "\n",
    "    Args:\n",
    "        root (string): Root directory where the dataset should be saved.\n",
    "        transform (callable, optional): A function/transform that takes in an\n",
    "            :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            every access. (default: :obj:`None`)\n",
    "        pre_transform (callable, optional): A function/transform that takes in\n",
    "            an :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            being saved to disk. (default: :obj:`None`)\n",
    "    \n",
    "    Stats:\n",
    "            * - #nodes\n",
    "              - #edges\n",
    "              - #features\n",
    "              - #classes\n",
    "            * - 80,963\n",
    "              - 290,091,503\n",
    "              - 1753\n",
    "              - 20\n",
    "    \"\"\"\n",
    "\n",
    "    url = 'https://drive.google.com/uc?export=download&id=1QZ5tyUWs6jYjh7mJrsnpou76iy-vb0CA'\n",
    "\n",
    "    def __init__(self, root: str, transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "        \n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            'A_simp.npz', 'A_SOC.npz', 'A_SPA.npz', 'A_TEM.npz', 'labels.npz',\n",
    "            'node_types.npy', 'Textual_Features.npy', 'train_val_test_idx.npz',\n",
    "            'Visual_Features.npy'\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        os.remove(path)\n",
    "\n",
    "    def process(self):\n",
    "        data = HeteroData()\n",
    "\n",
    "        node_types = ['vis_only', 'vis_tex']\n",
    "        link_types = ['SOC', 'SPA', 'TEM', 'simp']\n",
    "\n",
    "        vis = np.load(osp.join(self.raw_dir, 'Visual_Features.npy'),allow_pickle=True)[:,2:].astype(float)\n",
    "        tex = np.load(osp.join(self.raw_dir, 'Textual_Features.npy'),allow_pickle=True)[:,5:].astype(float)\n",
    "\n",
    "        x = np.hstack([vis,np.nan_to_num(tex)])\n",
    "\n",
    "\n",
    "        node_type_idx = np.load(osp.join(self.raw_dir, 'node_types.npy'))\n",
    "        node_type_idx = torch.from_numpy(node_type_idx).to(torch.long)\n",
    "\n",
    "        data['vis_only'].num_nodes = int((node_type_idx == 0).sum())\n",
    "        data['vis_tex'].num_nodes = int((node_type_idx == 1).sum())\n",
    "        data['all'].num_nodes = len(node_type_idx)\n",
    "\n",
    "        data['vis_only'].x = torch.from_numpy(vis[node_type_idx==0]).to(torch.float)\n",
    "        data['vis_tex'].x = torch.from_numpy(x[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].x = torch.from_numpy(x).to(torch.float)\n",
    "\n",
    "\n",
    "        y_s = np.load(osp.join(self.raw_dir, 'labels.npz'), allow_pickle=True)\n",
    "        att_lab = y_s['ATT_LAB'][:,1:10].astype(float)\n",
    "        val_lab = np.nan_to_num(y_s['VAL_LAB'][:,2:13].astype(float))\n",
    "        ys = np.hstack([att_lab, val_lab])\n",
    "\n",
    "        data['vis_only'].y = torch.from_numpy(att_lab[node_type_idx==0]).to(torch.float)\n",
    "        data['vis_tex'].y = torch.from_numpy(ys[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].y = torch.from_numpy(ys).to(torch.float)\n",
    "\n",
    "        data.node_type = node_type_idx\n",
    "\n",
    "        split = np.load(osp.join(self.raw_dir, 'train_val_test_idx.npz'))\n",
    "        for name in ['train', 'val', 'test']:\n",
    "            idx = split[f'{name}_idx']\n",
    "            idx = torch.from_numpy(idx).to(torch.long)\n",
    "            mask = torch.zeros(data['all'].num_nodes, dtype=torch.bool)\n",
    "            mask[idx] = True\n",
    "            data['all'][f'{name}_mask'] = mask\n",
    "            data['vis_only'][f'{name}_mask'] = mask[node_type_idx==0]\n",
    "            data['vis_tex'][f'{name}_mask'] = mask[node_type_idx==1]\n",
    "\n",
    "        \n",
    "        s = {}\n",
    "        s['vis_only'] = np.arange(len(x))[node_type_idx==0]\n",
    "        s['vis_tex'] = np.arange(len(x))[node_type_idx==1]\n",
    "\n",
    "        for link in link_types:\n",
    "            A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz')).tocoo()\n",
    "            if A_sub.nnz>0:\n",
    "                row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                data['all', f'{link}_link', 'all'].edge_index = torch.stack([row, col], dim=0)\n",
    "                data['all', f'{link}_link', 'all'].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "        for src, dst in product(node_types, node_types):\n",
    "            for link in link_types:\n",
    "                A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz'))[s[src]][:,s[dst]].tocoo()\n",
    "                if A_sub.nnz>0:\n",
    "                    row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                    col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                    data[src, f'{link}_link', dst].edge_index = torch.stack([row, col], dim=0)\n",
    "                    data[src, f'{link}_link', dst].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "\n",
    "        if self.pre_transform is not None:\n",
    "            data = self.pre_transform(data)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.__class__.__name__}()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VEN_XL_Homo(InMemoryDataset):\n",
    "    r\"\"\"A large subset of Flickr post collected in Venice annotated with Heritage \n",
    "    Values and Attributes, as collected in the `\"Heri-Graphs: A Workflow of \n",
    "    Creating Datasets for Multi-modal Machine Learning on Graphs of Heritage \n",
    "    Values and Attributes with Social Media\" <https://arxiv.org/abs/2205.07545>`\n",
    "    paper.\n",
    "    VEN_XL is a heterogeneous graph containing two types of nodes - nodes with only \n",
    "    visual features 'vis_only' (31,140 nodes), nodes with both visual and textual\n",
    "    features 'vis_tex' (49,823 nodes) and four types of links - social similarity\n",
    "    'SOC' (76,422,265 links), spatial similarity (202,173,159 links), temporal similarity\n",
    "    (71,135,671 links), and simple composed link (290,091,503 links).\n",
    "    Vis_only nodes are represented with 982-dimensional visual features and are\n",
    "    divided into 9 heritage attribute categories \n",
    "    ('architectural elements', 'form', 'gastronomy', 'interior',\n",
    "    'landscape scenery and natural features', 'monuments', 'people', 'product', \n",
    "    'urban scenery').\n",
    "    Vis_text nodes are represented with 1753-dimensional visual and textual \n",
    "    features and are divided into 9 heritage attribute categories plus 11 \n",
    "    heritage value categories ('criterion i-x', 'other').\n",
    "    Both types of nodes are also merged into a single type of node 'all' with \n",
    "    1753-dimensional features and 20-dimensional label categories.\n",
    "\n",
    "    Args:\n",
    "        root (string): Root directory where the dataset should be saved.\n",
    "        transform (callable, optional): A function/transform that takes in an\n",
    "            :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            every access. (default: :obj:`None`)\n",
    "        pre_transform (callable, optional): A function/transform that takes in\n",
    "            an :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            being saved to disk. (default: :obj:`None`)\n",
    "    \n",
    "    Stats:\n",
    "            * - #nodes\n",
    "              - #edges\n",
    "              - #features\n",
    "              - #classes\n",
    "            * - 80,963\n",
    "              - 290,091,503\n",
    "              - 1753\n",
    "              - 20\n",
    "    \"\"\"\n",
    "\n",
    "    url = 'https://drive.google.com/uc?export=download&id=1sxcKiZr1YGDv06wr03nsk5HVZledgzi9'\n",
    "\n",
    "    def __init__(self, root: str, transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "        \n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            'A_simp.npz', 'A_SOC.npz', 'A_SPA.npz', 'A_TEM.npz', 'labels.npz',\n",
    "            'node_types.npy', 'Textual_Features.npy', 'train_val_test_idx.npz',\n",
    "            'Visual_Features.npy'\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        os.remove(path)\n",
    "\n",
    "    def process(self):\n",
    "        data = Data()\n",
    "\n",
    "        link_types = ['simp']\n",
    "\n",
    "        vis = np.load(osp.join(self.raw_dir, 'Visual_Features.npy'),allow_pickle=True)[:,2:].astype(float)\n",
    "        tex = np.load(osp.join(self.raw_dir, 'Textual_Features.npy'),allow_pickle=True)[:,5:].astype(float)\n",
    "\n",
    "        x = np.hstack([vis,np.nan_to_num(tex)])\n",
    "\n",
    "        node_type_idx = np.load(osp.join(self.raw_dir, 'node_types.npy'))\n",
    "        node_type_idx = torch.from_numpy(node_type_idx).to(torch.long)\n",
    "\n",
    "        data.num_nodes = len(node_type_idx)\n",
    "\n",
    "        data.x = torch.from_numpy(x).to(torch.float)\n",
    "\n",
    "\n",
    "        y_s = np.load(osp.join(self.raw_dir, 'labels.npz'), allow_pickle=True)\n",
    "        att_lab = y_s['ATT_LAB'][:,1:10].astype(float)\n",
    "        val_lab = np.nan_to_num(y_s['VAL_LAB'][:,2:13].astype(float))\n",
    "        ys = np.hstack([att_lab, val_lab])\n",
    "\n",
    "        data.y = torch.from_numpy(ys).to(torch.float)\n",
    "        \n",
    "        data.att_lab = torch.tensor(y_s['ATT_LAB'][:,-1].astype(bool))\n",
    "        data.val_lab = torch.tensor(y_s['VAL_LAB'][:,-1].astype(bool))\n",
    "\n",
    "        data.node_type = node_type_idx\n",
    "\n",
    "        split = np.load(osp.join(self.raw_dir, 'train_val_test_idx.npz'))\n",
    "        for name in ['train', 'val', 'test']:\n",
    "            idx = split[f'{name}_idx']\n",
    "            idx = torch.from_numpy(idx).to(torch.long)\n",
    "            mask = torch.zeros(data.num_nodes, dtype=torch.bool)\n",
    "            mask[idx] = True\n",
    "            data[f'{name}_mask'] = mask\n",
    "                    \n",
    "        s = {}\n",
    "        \n",
    "        for link in link_types:\n",
    "            A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz')).tocoo()\n",
    "            if A_sub.nnz>0:\n",
    "                row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                data.edge_index = torch.stack([row, col], dim=0)\n",
    "                data.edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "        if self.pre_transform is not None:\n",
    "            data = self.pre_transform(data)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.__class__.__name__}()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#path = osp.join(os.getcwd(), '../../data/VEN')\n",
    "transform = T.NormalizeFeatures()\n",
    "dataset = VEN_Homo('dataset/Venice_Homo')\n",
    "data = dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Data(num_nodes=2951, x=[2951, 1753], y=[2951, 20], node_type=[2951], att_lab=[2951], val_lab=[2951], train_mask=[2951], val_mask=[2951], test_mask=[2951], edge_index=[2, 1071977], edge_attr=[1071977], n_id=[2951])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.n_id = torch.arange(data.num_nodes)\n",
    "data = data.to(device)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.2861, 0.3745, 1.1453,  ..., 1.0000, 0.0000, 0.0000],\n",
       "        [0.3977, 0.1582, 0.3059,  ..., 1.0000, 0.0000, 0.0000],\n",
       "        [0.5185, 0.5124, 1.3662,  ..., 1.0000, 0.0000, 0.0000],\n",
       "        ...,\n",
       "        [0.0124, 1.7083, 0.3258,  ..., 0.0000, 1.0000, 0.0000],\n",
       "        [0.4402, 0.8374, 0.4974,  ..., 0.0000, 1.0000, 0.0000],\n",
       "        [0.2102, 1.5535, 0.3352,  ..., 0.0000, 1.0000, 0.0000]],\n",
       "       device='cuda:0')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Loader for Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.loader import NeighborLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.train_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.val_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.test_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Data(num_nodes=2931, x=[2931, 1753], y=[2931, 20], node_type=[2931], att_lab=[2931], val_lab=[2931], train_mask=[2931], val_mask=[2931], test_mask=[2931], edge_index=[2, 103428], edge_attr=[103428], n_id=[2931], batch_size=32)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_data = next(iter(train_loader))\n",
    "batch = sampled_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(361, device='cuda:0')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch['train_mask'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2931, 1753])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch.x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2931, 20])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch.y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_train_state(args):\n",
    "    return {'stop_early': False,\n",
    "            'early_stopping_step': 0,\n",
    "            'early_stopping_best_ATT_acc_val': 0,\n",
    "            'early_stopping_best_VAL_acc_val': 0,\n",
    "            'early_stopping_best_ATT_acc_val_2': 0,\n",
    "            'early_stopping_lowest_loss': 1000,\n",
    "            'learning_rate': args.learning_rate,\n",
    "            'epoch_index': 0,\n",
    "            'train_loss': [],\n",
    "            'train_ATT_loss': [],\n",
    "            'train_VAL_loss':[],\n",
    "            'train_ATT_acc': [],\n",
    "            'train_VAL_acc': [],\n",
    "            'train_VAL_jac': [],\n",
    "            'train_VAL_acc_1':[], \n",
    "            'val_loss': [],\n",
    "            'val_ATT_loss': [],\n",
    "            'val_VAL_loss':[],\n",
    "            'val_ATT_acc': [],\n",
    "            'val_VAL_acc': [],\n",
    "            'val_VAL_jac': [],\n",
    "            'val_VAL_acc_1': [],\n",
    "            'test_loss': -1,\n",
    "            'test_ATT_loss': -1,\n",
    "            'test_VAL_loss':-1,\n",
    "            'test_ATT_acc': -1,\n",
    "            'test_VAL_acc': -1,\n",
    "            'test_VAL_jac': -1,\n",
    "            'test_VAL_acc_1': -1,\n",
    "            'model_filename': args.model_state_file}\n",
    "\n",
    "def update_train_state(args, model, train_state):\n",
    "    \"\"\"Handle the training state updates.\n",
    "\n",
    "    Components:\n",
    "     - Early Stopping: Prevent overfitting.\n",
    "     - Model Checkpoint: Model is saved if the model is better\n",
    "\n",
    "    :param args: main arguments\n",
    "    :param model: model to train\n",
    "    :param train_state: a dictionary representing the training state values\n",
    "    :returns:\n",
    "        a new train_state\n",
    "    \"\"\"\n",
    "\n",
    "    # Save one model at least\n",
    "    if train_state['epoch_index'] == 0:\n",
    "        torch.save(model.state_dict(), train_state['model_filename'])\n",
    "        train_state['stop_early'] = False\n",
    "\n",
    "    # Save model if performance improved\n",
    "    elif train_state['epoch_index'] >= 1:\n",
    "        ATT_acc_tm1, ATT_acc_t = train_state['val_ATT_acc'][-2:]\n",
    "        #ATT_acc_2_tm1, ATT_acc_2_t = train_state['val_ATT_acc_2'][-2:]\n",
    "        VAL_acc_tm1, VAL_acc_t = train_state['val_VAL_acc'][-2:]\n",
    "        loss_tm1, loss_t = train_state['val_loss'][-2:]\n",
    "\n",
    "        # If accuracy worsened\n",
    "        #if loss_t >= train_state['early_stopping_lowest_loss']:\n",
    "        #    train_state['early_stopping_step'] += 1\n",
    "        \n",
    "        if ATT_acc_t <= train_state['early_stopping_best_ATT_acc_val'] and VAL_acc_t <= train_state['early_stopping_best_VAL_acc_val']:# and ATT_acc_2_t <= train_state['early_stopping_best_ATT_acc_val_2']:\n",
    "            # Update step\n",
    "            train_state['early_stopping_step'] += 1\n",
    "        # Loss decreased\n",
    "        else:\n",
    "            # Save the best model from sklearn\n",
    "            if VAL_acc_t > train_state['early_stopping_best_VAL_acc_val']:\n",
    "                train_state['early_stopping_best_VAL_acc_val'] = VAL_acc_t\n",
    "                \n",
    "            if ATT_acc_t > train_state['early_stopping_best_ATT_acc_val']:\n",
    "                train_state['early_stopping_best_ATT_acc_val'] = ATT_acc_t\n",
    "            \n",
    "            #if ATT_acc_2_t > train_state['early_stopping_best_ATT_acc_val_2']:\n",
    "            #    train_state['early_stopping_best_ATT_acc_val_2'] = ATT_acc_2_t\n",
    "                \n",
    "            if loss_t < train_state['early_stopping_lowest_loss']:\n",
    "                train_state['early_stopping_lowest_loss'] = loss_t\n",
    "                torch.save(model.state_dict(), train_state['model_filename'])\n",
    "                \n",
    "            # Reset early stopping step\n",
    "            train_state['early_stopping_step'] = 0\n",
    "\n",
    "        # Stop early ?\n",
    "        train_state['stop_early'] = \\\n",
    "            train_state['early_stopping_step'] >= args.early_stopping_criteria\n",
    "\n",
    "    return train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_cross_entropy(y_pred, y_target):\n",
    "    y_target = y_target.cpu().float()\n",
    "    y_pred = y_pred.cpu().float()\n",
    "    criterion = nn.BCEWithLogitsLoss()\n",
    "    return criterion(y_target, y_pred)\n",
    "\n",
    "def compute_1_accuracy(y_pred, y_target):\n",
    "    y_target_indices = y_target.max(dim=1)[1]\n",
    "    y_pred_indices = y_pred.max(dim=1)[1]\n",
    "    n_correct = torch.eq(y_pred_indices, y_target_indices).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_k_accuracy(y_pred, y_target, k=3):\n",
    "    y_pred_indices = y_pred.topk(k, dim=1)[1]\n",
    "    y_target_indices = y_target.max(dim=1)[1]\n",
    "    n_correct = torch.tensor([y_pred_indices[i] in y_target_indices[i] for i in range(len(y_pred))]).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_k_jaccard_index(y_pred, y_target, k=3):\n",
    "    y_target_indices = y_target.topk(k, dim=1)[1]\n",
    "    y_pred_indices = y_pred.max(dim=1)[1]\n",
    "    jaccard = torch.tensor([len(np.intersect1d(y_target_indices[i], y_pred_indices[i]))/\n",
    "                            len(np.union1d(y_target_indices[i], y_pred_indices[i]))\n",
    "                            for i in range(len(y_pred))]).sum().item()\n",
    "    return jaccard / len(y_pred_indices)\n",
    "\n",
    "def compute_jaccard_index(y_pred, y_target, k=3, multilabel=False):\n",
    "    \n",
    "    threshold = 1.0/(k+1)\n",
    "    threshold_2 = 0.5\n",
    "    \n",
    "    if multilabel:\n",
    "        y_pred_indices = y_pred.gt(threshold_2)\n",
    "    else:\n",
    "        y_pred_indices = y_pred.gt(threshold)\n",
    "    \n",
    "    y_target_indices = y_target.gt(threshold)\n",
    "        \n",
    "    jaccard = ((y_target_indices*y_pred_indices).sum(axis=1)/((y_target_indices+y_pred_indices).sum(axis=1)+1e-8)).sum().item()\n",
    "    return jaccard / len(y_pred_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy(pred, soft_targets):\n",
    "    logsoftmax = nn.LogSoftmax(dim=1)\n",
    "    return torch.mean(torch.sum(- soft_targets * logsoftmax(pred), 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Searched Best Hyper-Parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'42/hyperdict.p', 'rb') as fp:\n",
    "    hyperdict= pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "135"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(hyperdict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "hyper_df = pd.DataFrame(hyperdict).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>stop_early</th>\n",
       "      <th>early_stopping_step</th>\n",
       "      <th>early_stopping_best_ATT_acc_val</th>\n",
       "      <th>early_stopping_best_VAL_acc_val</th>\n",
       "      <th>early_stopping_best_ATT_acc_val_2</th>\n",
       "      <th>early_stopping_lowest_loss</th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>epoch_index</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_ATT_loss</th>\n",
       "      <th>...</th>\n",
       "      <th>val_VAL_acc</th>\n",
       "      <th>val_VAL_jac</th>\n",
       "      <th>test_loss</th>\n",
       "      <th>test_ATT_loss</th>\n",
       "      <th>test_VAL_loss</th>\n",
       "      <th>test_ATT_acc</th>\n",
       "      <th>test_ATT_acc_2</th>\n",
       "      <th>test_VAL_acc</th>\n",
       "      <th>test_VAL_jac</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.1</th>\n",
       "      <th>3</th>\n",
       "      <th>512</th>\n",
       "      <th>0.0010</th>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>90.503597</td>\n",
       "      <td>98.129496</td>\n",
       "      <td>0</td>\n",
       "      <td>4.138431</td>\n",
       "      <td>0.001</td>\n",
       "      <td>140</td>\n",
       "      <td>[3.9107560515403748, 3.575341443220774, 3.3046...</td>\n",
       "      <td>[1.9028614198071805, 1.700502701743488, 1.3999...</td>\n",
       "      <td>...</td>\n",
       "      <td>[50.50359712230216, 60.86330935251799, 85.4676...</td>\n",
       "      <td>[0.13347722231912956, 0.13585132146053178, 0.1...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/SAGE/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <th>512</th>\n",
       "      <th>0.0001</th>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>89.784173</td>\n",
       "      <td>96.690647</td>\n",
       "      <td>0</td>\n",
       "      <td>4.139175</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>208</td>\n",
       "      <td>[4.131787121295929, 3.7653501431147256, 3.7225...</td>\n",
       "      <td>[1.956105706103951, 1.928340225008386, 1.90353...</td>\n",
       "      <td>...</td>\n",
       "      <td>[81.00719424460432, 81.00719424460432, 34.3884...</td>\n",
       "      <td>[0.13347722231912956, 0.13347722231912956, 0.1...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/SAGE/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <th>3</th>\n",
       "      <th>512</th>\n",
       "      <th>0.0001</th>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>89.928058</td>\n",
       "      <td>94.676259</td>\n",
       "      <td>0</td>\n",
       "      <td>4.1395</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>163</td>\n",
       "      <td>[4.014001568158467, 3.684295117855072, 3.60325...</td>\n",
       "      <td>[1.8947085921453968, 1.8424101859909015, 1.769...</td>\n",
       "      <td>...</td>\n",
       "      <td>[83.02158273381295, 89.92805755395683, 68.7769...</td>\n",
       "      <td>[0.13347722231912956, 0.13347722231912956, 0.1...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/SAGE/model.pth</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 stop_early early_stopping_step  \\\n",
       "0.1 3 512 0.0010       True                  30   \n",
       "    5 512 0.0001       True                  30   \n",
       "0.3 3 512 0.0001       True                  30   \n",
       "\n",
       "                 early_stopping_best_ATT_acc_val  \\\n",
       "0.1 3 512 0.0010                       90.503597   \n",
       "    5 512 0.0001                       89.784173   \n",
       "0.3 3 512 0.0001                       89.928058   \n",
       "\n",
       "                 early_stopping_best_VAL_acc_val  \\\n",
       "0.1 3 512 0.0010                       98.129496   \n",
       "    5 512 0.0001                       96.690647   \n",
       "0.3 3 512 0.0001                       94.676259   \n",
       "\n",
       "                 early_stopping_best_ATT_acc_val_2 early_stopping_lowest_loss  \\\n",
       "0.1 3 512 0.0010                                 0                   4.138431   \n",
       "    5 512 0.0001                                 0                   4.139175   \n",
       "0.3 3 512 0.0001                                 0                     4.1395   \n",
       "\n",
       "                 learning_rate epoch_index  \\\n",
       "0.1 3 512 0.0010         0.001         140   \n",
       "    5 512 0.0001        0.0001         208   \n",
       "0.3 3 512 0.0001        0.0001         163   \n",
       "\n",
       "                                                         train_loss  \\\n",
       "0.1 3 512 0.0010  [3.9107560515403748, 3.575341443220774, 3.3046...   \n",
       "    5 512 0.0001  [4.131787121295929, 3.7653501431147256, 3.7225...   \n",
       "0.3 3 512 0.0001  [4.014001568158467, 3.684295117855072, 3.60325...   \n",
       "\n",
       "                                                     train_ATT_loss  ...  \\\n",
       "0.1 3 512 0.0010  [1.9028614198071805, 1.700502701743488, 1.3999...  ...   \n",
       "    5 512 0.0001  [1.956105706103951, 1.928340225008386, 1.90353...  ...   \n",
       "0.3 3 512 0.0001  [1.8947085921453968, 1.8424101859909015, 1.769...  ...   \n",
       "\n",
       "                                                        val_VAL_acc  \\\n",
       "0.1 3 512 0.0010  [50.50359712230216, 60.86330935251799, 85.4676...   \n",
       "    5 512 0.0001  [81.00719424460432, 81.00719424460432, 34.3884...   \n",
       "0.3 3 512 0.0001  [83.02158273381295, 89.92805755395683, 68.7769...   \n",
       "\n",
       "                                                        val_VAL_jac test_loss  \\\n",
       "0.1 3 512 0.0010  [0.13347722231912956, 0.13585132146053178, 0.1...        -1   \n",
       "    5 512 0.0001  [0.13347722231912956, 0.13347722231912956, 0.1...        -1   \n",
       "0.3 3 512 0.0001  [0.13347722231912956, 0.13347722231912956, 0.1...        -1   \n",
       "\n",
       "                 test_ATT_loss test_VAL_loss test_ATT_acc test_ATT_acc_2  \\\n",
       "0.1 3 512 0.0010            -1            -1           -1             -1   \n",
       "    5 512 0.0001            -1            -1           -1             -1   \n",
       "0.3 3 512 0.0001            -1            -1           -1             -1   \n",
       "\n",
       "                 test_VAL_acc test_VAL_jac                model_filename  \n",
       "0.1 3 512 0.0010           -1           -1  model_storage/SAGE/model.pth  \n",
       "    5 512 0.0001           -1           -1  model_storage/SAGE/model.pth  \n",
       "0.3 3 512 0.0001           -1           -1  model_storage/SAGE/model.pth  \n",
       "\n",
       "[3 rows x 29 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hyper_df[(hyper_df.early_stopping_lowest_loss<4.14)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.1, 5, 512, 0.0001)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(hyper_df['val_VAL_loss'].apply(lambda x: min(x)) + hyper_df['val_ATT_loss'].apply(lambda x: min(x))).index[44]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Re-run model and get Inference Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SAGE(torch.nn.Module):\n",
    "    def __init__(self, in_channels, hidden_channels, out_channels, num_layers, dropout):\n",
    "        super().__init__()\n",
    "        self.num_layers = num_layers\n",
    "        self.p = dropout\n",
    "        self.convs = torch.nn.ModuleList()\n",
    "        for i in range(num_layers-1):\n",
    "            in_channels = in_channels if i == 0 else hidden_channels\n",
    "            self.convs.append(SAGEConv(in_channels, hidden_channels))\n",
    "        self.convs.append(SAGEConv(hidden_channels, out_channels))\n",
    "\n",
    "    def forward(self, x, edge_index):\n",
    "        for i, conv in enumerate(self.convs):\n",
    "            x = conv(x, edge_index)\n",
    "            if i != self.num_layers - 1:\n",
    "                x = x.relu()\n",
    "                x = F.dropout(x, p=self.p, training=self.training)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_Homo(model, optimizer, train_loader):\n",
    "    model.train()\n",
    "\n",
    "    total_examples = total_loss = 0\n",
    "    for batch in tqdm(train_loader):\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        batch = batch.to(device)\n",
    "        batch_size = args.batch_size\n",
    "        edge_index = to_undirected(batch.edge_index)\n",
    "        out = model(batch.x, edge_index)[:batch_size]\n",
    "        out_att = out[:,:9]\n",
    "        out_val = out[:,9:]\n",
    "        y = batch.y\n",
    "        y_att = y[:,:9]\n",
    "        y_val = y[:,9:]\n",
    "        \n",
    "        loss = F.cross_entropy(out_att, y_att[:batch_size]) + F.cross_entropy(out_val, y_val[:batch_size])\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        total_examples += batch_size\n",
    "        total_loss += float(loss) * batch_size\n",
    "\n",
    "    return total_loss / total_examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def test_Homo(model, loader):\n",
    "    model.eval()\n",
    "\n",
    "    total_examples_att = total_examples_val = 0\n",
    "    running_loss_1 = running_loss_2 = 0.\n",
    "    running_1_acc = 0.\n",
    "    running_1_val = 0.\n",
    "    running_k_acc = 0.\n",
    "    running_k_jac = 0.\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        loss_1 = 0\n",
    "        acc_1_t = 0\n",
    "        loss_2 = 0\n",
    "        acc_1_val = 0\n",
    "        acc_k_t = 0\n",
    "        jac_k_t = 0\n",
    "\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch.batch_size\n",
    "        edge_index = to_undirected(batch.edge_index)\n",
    "        out = model(batch.x, edge_index)[:batch_size]\n",
    "        out_att = out[:,:9]\n",
    "        out_val = out[:,9:]\n",
    "        att_node = (batch.att_lab[:batch_size]).nonzero().squeeze()\n",
    "        val_node = (batch.val_lab[:batch_size]).nonzero().squeeze()\n",
    "\n",
    "        #print(type_node)\n",
    "\n",
    "        #pred_att = out_att.argmax(dim=-1)\n",
    "        #pred_val = out_val.argmax(dim=-1)\n",
    "\n",
    "        y = batch.y\n",
    "        y_att = y[:,:9]\n",
    "        y_val = y[:,9:]\n",
    "\n",
    "        if not att_node.shape[0]==0:\n",
    "            loss_1 = F.cross_entropy(out_att[att_node], y_att[:batch_size][att_node])\n",
    "            acc_1_t = compute_1_accuracy(y_att[:batch_size][att_node], out_att[att_node])\n",
    "\n",
    "        if not val_node.shape[0]==0:\n",
    "            loss_2 = F.cross_entropy(out_val[val_node], y_val[val_node])\n",
    "            acc_1_val = compute_1_accuracy(y_val[val_node], out_val[val_node])\n",
    "            acc_k_t = compute_k_accuracy(y_val[val_node], out_val[val_node], args.k)\n",
    "            jac_k_t = compute_jaccard_index(y_val[val_node], F.softmax(out_val[val_node],dim=-1), args.k)\n",
    "            #loss_3 = loss_1 + loss_2\n",
    "\n",
    "        total_examples_att += att_node.shape[0]\n",
    "        total_examples_val += val_node.shape[0]\n",
    "        #total_correct_att += int((pred_att == y_att[:batch_size]).sum())\n",
    "        #total_correct_val += int((pred_val == y_val[:batch_size]).sum())\n",
    "\n",
    "        running_loss_1 += float(loss_1) * att_node.shape[0]\n",
    "        running_loss_2 += float(loss_2) * val_node.shape[0]\n",
    "        running_1_acc += float(acc_1_t) * att_node.shape[0]\n",
    "        running_1_val += float(acc_1_val) * val_node.shape[0]\n",
    "        running_k_acc += float(acc_k_t) * val_node.shape[0]\n",
    "        running_k_jac += float(jac_k_t) * val_node.shape[0]\n",
    "    \n",
    "    return running_loss_1/total_examples_att, running_loss_2/total_examples_val, running_1_acc/ total_examples_att, running_k_acc/ total_examples_val, running_k_jac/ total_examples_val, running_1_val/total_examples_val\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialization():\n",
    "    set_seed_everywhere(args.seed, args.cuda)\n",
    "    #transform = T.Compose([T.ToSparseTensor()])\n",
    "    dataset = VEN_Homo('dataset/Venice_homo')\n",
    "    data = dataset[0].to(device)\n",
    "    data.n_id = torch.arange(data.num_nodes)\n",
    "    data = data.to(device)\n",
    "    \n",
    "    train_loader = NeighborLoader(\n",
    "        data,\n",
    "        # Sample 25 neighbors for each node and edge type for 2 iterations\n",
    "        num_neighbors=[3*args.sample_nodes] * 2,\n",
    "        # Use a batch size of 32 for sampling training nodes\n",
    "        batch_size=args.batch_size,\n",
    "        input_nodes=data.train_mask,\n",
    "    )\n",
    "    val_loader = NeighborLoader(\n",
    "        data,\n",
    "        # Sample 25 neighbors for each node and edge type for 2 iterations\n",
    "        num_neighbors=[3*args.sample_nodes] * 2,\n",
    "        # Use a batch size of 32 for sampling validating nodes\n",
    "        batch_size=args.batch_size,\n",
    "        input_nodes=data.val_mask,\n",
    "    )\n",
    "    test_loader = NeighborLoader(\n",
    "        data,\n",
    "        # Sample 25 neighbors for each node and edge type for 2 iterations\n",
    "        num_neighbors=[3*args.sample_nodes] * 2,\n",
    "        # Use a batch size of 32 for sampling testing nodes\n",
    "        batch_size=args.batch_size,\n",
    "        input_nodes=data.test_mask,\n",
    "    )\n",
    " \n",
    "    model = SAGE(in_channels=data.x.shape[-1], hidden_channels = 512, \n",
    "            out_channels = data.y.shape[-1], dropout = 0.1, num_layers=5).to(device)\n",
    "    return data, model, train_loader, val_loader, test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_loop(verbose=False):\n",
    "    \n",
    "    _, model, train_loader, val_loader, test_loader = initialization()\n",
    "    if torch.cuda.device_count() > 1:\n",
    "        print(\"Use {} GPUs !\".format(torch.cuda.device_count()))\n",
    "        model = DataParallel(model)\n",
    "    \n",
    "    optimizer = optim.Adam(model.parameters(), lr=0.0001, weight_decay=args.l2)\n",
    "    #scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer,\n",
    "    #                                           mode='min', factor=0.5,\n",
    "    #                                           patience=1)\n",
    "\n",
    "    train_state = make_train_state(args)\n",
    "\n",
    "    try:\n",
    "        for epoch in range(args.num_epochs):\n",
    "            train_state['epoch_index'] = epoch\n",
    "            \n",
    "            loss = train_Homo(model, optimizer, train_loader)\n",
    "            train_loss_att, train_loss_val, train_att_acc, train_val_acc, train_val_jac, train_val_1 = test_Homo(model, train_loader)\n",
    "            val_loss_att, val_loss_val, val_att_acc, val_val_acc, val_val_jac, val_val_1 = test_Homo(model, val_loader)\n",
    "            if verbose:\n",
    "                print(f'Epoch: {epoch:02d}, Loss: {loss:.4f}, Train_ATT: {train_att_acc:.4f}, Train_VAL: {train_val_acc:.4f}, Val_vis_tex_ATT: {val_att_acc:.4f}, Val_vis_tex_VAL: {val_val_acc:.4f}')\n",
    "            \n",
    "            train_state['train_loss'].append(loss)\n",
    "            train_state['train_ATT_loss'].append(train_loss_att)\n",
    "            train_state['train_VAL_loss'].append(train_loss_val)\n",
    "            train_state['train_ATT_acc'].append(train_att_acc)\n",
    "            train_state['train_VAL_acc'].append(train_val_acc)\n",
    "            train_state['train_VAL_jac'].append(train_val_jac)\n",
    "            train_state['train_VAL_acc_1'].append(train_val_1)\n",
    "            \n",
    "            train_state['val_ATT_loss'].append(val_loss_att)\n",
    "            train_state['val_VAL_loss'].append(val_loss_val)\n",
    "            train_state['val_loss'].append(val_loss_att + 3*val_loss_val)\n",
    "            train_state['val_ATT_acc'].append(val_att_acc)\n",
    "            train_state['val_VAL_acc'].append(val_val_acc)\n",
    "            train_state['val_VAL_jac'].append(val_val_jac)\n",
    "            train_state['val_VAL_acc_1'].append(val_val_1)\n",
    "            \n",
    "            train_state = update_train_state(args=args, model=model,\n",
    "                                                train_state=train_state)\n",
    "            if train_state['stop_early']:\n",
    "                break\n",
    "\n",
    "    except KeyboardInterrupt:\n",
    "        print(\"Exiting loop\")\n",
    "        pass\n",
    "    \n",
    "    return train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_state = training_loop(verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': True,\n",
       " 'early_stopping_step': 30,\n",
       " 'early_stopping_best_ATT_acc_val': 96.7479674796748,\n",
       " 'early_stopping_best_VAL_acc_val': 100.0,\n",
       " 'early_stopping_best_ATT_acc_val_2': 0,\n",
       " 'early_stopping_lowest_loss': 5.674814448439726,\n",
       " 'learning_rate': 0.001,\n",
       " 'epoch_index': 169,\n",
       " 'train_loss': [4.098931749661763,\n",
       "  3.767206331094106,\n",
       "  3.7242619395256042,\n",
       "  3.700642983118693,\n",
       "  3.668950080871582,\n",
       "  3.6377055446306863,\n",
       "  3.6117473045984902,\n",
       "  3.5522724787394204,\n",
       "  3.4761929710706077,\n",
       "  3.3616212805112204,\n",
       "  3.23332671324412,\n",
       "  3.1092602411905923,\n",
       "  3.028814951578776,\n",
       "  2.898661951224009,\n",
       "  2.82891054948171,\n",
       "  2.812813639640808,\n",
       "  2.7504736383756003,\n",
       "  2.733060439427694,\n",
       "  2.6882276137669883,\n",
       "  2.664868414402008,\n",
       "  2.642411748568217,\n",
       "  2.6827587683995566,\n",
       "  2.6409761110941568,\n",
       "  2.5932233532269797,\n",
       "  2.5788277784983316,\n",
       "  2.548888703187307,\n",
       "  2.53569761912028,\n",
       "  2.5417450070381165,\n",
       "  2.529538651307424,\n",
       "  2.5021569530169168,\n",
       "  2.4645569125811257,\n",
       "  2.4980392257372537,\n",
       "  2.501281420389811,\n",
       "  2.4597434798876443,\n",
       "  2.485634922981262,\n",
       "  2.466936190923055,\n",
       "  2.467691481113434,\n",
       "  2.4658232728640237,\n",
       "  2.4621368249257407,\n",
       "  2.464854379494985,\n",
       "  2.4529733459154763,\n",
       "  2.4534698923428855,\n",
       "  2.436204969882965,\n",
       "  2.4329582850138345,\n",
       "  2.450833817323049,\n",
       "  2.421373744805654,\n",
       "  2.434440871079763,\n",
       "  2.450161556402842,\n",
       "  2.427618066469828,\n",
       "  2.4393926660219827,\n",
       "  2.4163631796836853,\n",
       "  2.4301605820655823,\n",
       "  2.435442308584849,\n",
       "  2.4121821324030557,\n",
       "  2.4129402240117392,\n",
       "  2.4047138492266336,\n",
       "  2.4033178885777793,\n",
       "  2.41557643810908,\n",
       "  2.3933849732081094,\n",
       "  2.3944159746170044,\n",
       "  2.392829736073812,\n",
       "  2.3958428303400674,\n",
       "  2.404998560746511,\n",
       "  2.3976862827936807,\n",
       "  2.417176087697347,\n",
       "  2.3936072985331216,\n",
       "  2.408930778503418,\n",
       "  2.3829866647720337,\n",
       "  2.393950899442037,\n",
       "  2.4159895380338035,\n",
       "  2.4185696244239807,\n",
       "  2.399583617846171,\n",
       "  2.3866588274637857,\n",
       "  2.3977599342664084,\n",
       "  2.385384738445282,\n",
       "  2.385758380095164,\n",
       "  2.3827030460039773,\n",
       "  2.3948446114857993,\n",
       "  2.3857273856798806,\n",
       "  2.4019908706347146,\n",
       "  2.392253816127777,\n",
       "  2.3864142696062722,\n",
       "  2.366167684396108,\n",
       "  2.3878723780314126,\n",
       "  2.374208470185598,\n",
       "  2.3906711538632712,\n",
       "  2.3854604959487915,\n",
       "  2.3820138772328696,\n",
       "  2.3843430280685425,\n",
       "  2.377971331278483,\n",
       "  2.3856988151868186,\n",
       "  2.3722445170084634,\n",
       "  2.38117923339208,\n",
       "  2.3817812403043113,\n",
       "  2.3885410825411477,\n",
       "  2.3754957914352417,\n",
       "  2.383930762608846,\n",
       "  2.3800458908081055,\n",
       "  2.391184707482656,\n",
       "  2.3534916838010154,\n",
       "  2.378878355026245,\n",
       "  2.3758183320363364,\n",
       "  2.3601855039596558,\n",
       "  2.3570021788279214,\n",
       "  2.371889809767405,\n",
       "  2.374414642651876,\n",
       "  2.373858153820038,\n",
       "  2.355741282304128,\n",
       "  2.366458535194397,\n",
       "  2.3738173047701516,\n",
       "  2.35603266954422,\n",
       "  2.353848874568939,\n",
       "  2.378168066342672,\n",
       "  2.3655735850334167,\n",
       "  2.3658991058667502,\n",
       "  2.355059822400411,\n",
       "  2.370965520540873,\n",
       "  2.368260939915975,\n",
       "  2.3701597849527993,\n",
       "  2.348038117090861,\n",
       "  2.372505327065786,\n",
       "  2.371385157108307,\n",
       "  2.3717514077822366,\n",
       "  2.363755484422048,\n",
       "  2.3688772519429526,\n",
       "  2.3616636395454407,\n",
       "  2.3614793618520102,\n",
       "  2.3517807920773826,\n",
       "  2.3396608233451843,\n",
       "  2.346040507157644,\n",
       "  2.35396538178126,\n",
       "  2.3598483403523765,\n",
       "  2.3532610734303794,\n",
       "  2.370964745680491,\n",
       "  2.3649390737215676,\n",
       "  2.3489114244778952,\n",
       "  2.3580377101898193,\n",
       "  2.3587553103764853,\n",
       "  2.3682141304016113,\n",
       "  2.366213579972585,\n",
       "  2.3549970189730325,\n",
       "  2.364099085330963,\n",
       "  2.3584629893302917,\n",
       "  2.360451857248942,\n",
       "  2.3638933499654136,\n",
       "  2.36885529756546,\n",
       "  2.374499042828878,\n",
       "  2.355612337589264,\n",
       "  2.359101116657257,\n",
       "  2.346854587395986,\n",
       "  2.3477829893430076,\n",
       "  2.366513510545095,\n",
       "  2.3481218417485556,\n",
       "  2.342003047466278,\n",
       "  2.350774347782135,\n",
       "  2.369879643122355,\n",
       "  2.365553597609202,\n",
       "  2.356052656968435,\n",
       "  2.3574883937835693,\n",
       "  2.355355203151703,\n",
       "  2.3598441084225974,\n",
       "  2.3543240427970886,\n",
       "  2.3405757745107016,\n",
       "  2.3648985425631204,\n",
       "  2.346867481867472,\n",
       "  2.365351438522339,\n",
       "  2.3487407565116882,\n",
       "  2.3562622467676797,\n",
       "  2.3468205531438193,\n",
       "  2.352146029472351],\n",
       " 'train_ATT_loss': [1.9546462502175752,\n",
       "  1.933227481604283,\n",
       "  1.9044952280303449,\n",
       "  1.8954763026118608,\n",
       "  1.8790527037306175,\n",
       "  1.85460715742983,\n",
       "  1.818479740058286,\n",
       "  1.7553564171381604,\n",
       "  1.6525168686362184,\n",
       "  1.5284845360098123,\n",
       "  1.3960980295804728,\n",
       "  1.2829446231229154,\n",
       "  1.1823519247060337,\n",
       "  1.100580136531608,\n",
       "  1.0436663112481877,\n",
       "  1.0261845717469742,\n",
       "  0.989991723005131,\n",
       "  0.9566390336055175,\n",
       "  0.94409316761672,\n",
       "  0.9302744825791124,\n",
       "  0.9226926218439667,\n",
       "  0.8990904911733401,\n",
       "  0.8791366349957326,\n",
       "  0.8851204942137911,\n",
       "  0.8496665183526988,\n",
       "  0.8268885950962923,\n",
       "  0.818720144082965,\n",
       "  0.8138482732455816,\n",
       "  0.80768588969582,\n",
       "  0.8035663227625501,\n",
       "  0.7936245870061859,\n",
       "  0.7901797676020382,\n",
       "  0.782620763349401,\n",
       "  0.7779420784305667,\n",
       "  0.7768213484756173,\n",
       "  0.7768766764458527,\n",
       "  0.7762275073997201,\n",
       "  0.7695193786039907,\n",
       "  0.7650198105960011,\n",
       "  0.763135122955671,\n",
       "  0.7609167265759941,\n",
       "  0.7586613754816663,\n",
       "  0.7562868400293704,\n",
       "  0.7554397617680875,\n",
       "  0.7554485241461989,\n",
       "  0.7529076639965301,\n",
       "  0.752861852460951,\n",
       "  0.7502899803943581,\n",
       "  0.750594558643172,\n",
       "  0.7475648231783733,\n",
       "  0.7478374434309983,\n",
       "  0.7448096587387149,\n",
       "  0.7439154650696097,\n",
       "  0.7456373215712339,\n",
       "  0.7428600963132863,\n",
       "  0.7422313085883608,\n",
       "  0.7449038992958386,\n",
       "  0.7427807301695657,\n",
       "  0.7407917812920674,\n",
       "  0.7399915585557509,\n",
       "  0.7388355208565984,\n",
       "  0.7381309044658312,\n",
       "  0.7380269539653429,\n",
       "  0.7392500615846417,\n",
       "  0.7391345398577956,\n",
       "  0.7372666285308774,\n",
       "  0.7363568491552676,\n",
       "  0.7358763165090884,\n",
       "  0.7341074466375103,\n",
       "  0.7390065029717549,\n",
       "  0.7348545637817594,\n",
       "  0.7341795948403694,\n",
       "  0.7336663625246931,\n",
       "  0.733937031839693,\n",
       "  0.7327201891473786,\n",
       "  0.7350096808245968,\n",
       "  0.7325471038633437,\n",
       "  0.7331557721312356,\n",
       "  0.7311574829912582,\n",
       "  0.7322029137875565,\n",
       "  0.7318318203875893,\n",
       "  0.7317600431864942,\n",
       "  0.7303660582967743,\n",
       "  0.7309218465126122,\n",
       "  0.7309670567182293,\n",
       "  0.7314814781548243,\n",
       "  0.7313714885975846,\n",
       "  0.7302612733312591,\n",
       "  0.7300168203184809,\n",
       "  0.7307250534398404,\n",
       "  0.7288559303719582,\n",
       "  0.7289511916379849,\n",
       "  0.7302039173171131,\n",
       "  0.7286603789250277,\n",
       "  0.7286888103405855,\n",
       "  0.7296452512371243,\n",
       "  0.7283628989124562,\n",
       "  0.7279192775570454,\n",
       "  0.7280225621696325,\n",
       "  0.728538278067211,\n",
       "  0.7275784352479548,\n",
       "  0.7278532022584508,\n",
       "  0.7293509553674186,\n",
       "  0.7294930807441226,\n",
       "  0.7271956263156478,\n",
       "  0.7281060940340945,\n",
       "  0.7268856430317887,\n",
       "  0.7270274472698941,\n",
       "  0.7263259800187108,\n",
       "  0.7269815693601677,\n",
       "  0.7270486617352494,\n",
       "  0.726385351858641,\n",
       "  0.7270347632859883,\n",
       "  0.7291333668780129,\n",
       "  0.7286997430872719,\n",
       "  0.7278061978044272,\n",
       "  0.7272645996217912,\n",
       "  0.7276377057104559,\n",
       "  0.7290878063093592,\n",
       "  0.7263797574426328,\n",
       "  0.7271127897286349,\n",
       "  0.7299188717250349,\n",
       "  0.7265173465921608,\n",
       "  0.7269514711609838,\n",
       "  0.7265304153645799,\n",
       "  0.7260907123954012,\n",
       "  0.7263158790952942,\n",
       "  0.7288298661358799,\n",
       "  0.7258909574175806,\n",
       "  0.7263430748289642,\n",
       "  0.7254435122838642,\n",
       "  0.7253453419479307,\n",
       "  0.7265390776861408,\n",
       "  0.7250594678347791,\n",
       "  0.724814727861135,\n",
       "  0.7252569731913114,\n",
       "  0.7248612411134461,\n",
       "  0.7250717183229336,\n",
       "  0.7271107253605639,\n",
       "  0.7270112359622839,\n",
       "  0.7266698002485027,\n",
       "  0.7254077214283295,\n",
       "  0.7294890723730388,\n",
       "  0.7264151256170299,\n",
       "  0.7269143237631737,\n",
       "  0.7249167837264465,\n",
       "  0.7256764726625585,\n",
       "  0.7281667552166038,\n",
       "  0.7262458584975668,\n",
       "  0.7250483608972332,\n",
       "  0.7260188921006433,\n",
       "  0.725769547379248,\n",
       "  0.7246643917382258,\n",
       "  0.7260264001394573,\n",
       "  0.7251703623589386,\n",
       "  0.7241784510850245,\n",
       "  0.7250886163222823,\n",
       "  0.7258201314141546,\n",
       "  0.7266337138463916,\n",
       "  0.725004944774913,\n",
       "  0.7258336479644035,\n",
       "  0.7251031674506592,\n",
       "  0.7245107887524317,\n",
       "  0.7244798600508565,\n",
       "  0.7243442258015894,\n",
       "  0.7258242599851867,\n",
       "  0.7253383705160295,\n",
       "  0.7245499924279316,\n",
       "  0.7239685027222884,\n",
       "  0.7250117207167882],\n",
       " 'train_VAL_loss': [1.8116718421682427,\n",
       "  1.8048931300805215,\n",
       "  1.794408496066804,\n",
       "  1.786028530789214,\n",
       "  1.7744940225437407,\n",
       "  1.758524922452805,\n",
       "  1.7416969118356045,\n",
       "  1.727210153503101,\n",
       "  1.724068412820388,\n",
       "  1.7271408925756524,\n",
       "  1.7252861050027228,\n",
       "  1.713301249488239,\n",
       "  1.705337117253248,\n",
       "  1.7061472602828387,\n",
       "  1.697877908347386,\n",
       "  1.6915473419543448,\n",
       "  1.69230836217093,\n",
       "  1.7007082212004305,\n",
       "  1.696528370690808,\n",
       "  1.6880398939190808,\n",
       "  1.6791018683494292,\n",
       "  1.685056095308214,\n",
       "  1.6835761819850044,\n",
       "  1.6691038747243274,\n",
       "  1.669677336790555,\n",
       "  1.6698398844357012,\n",
       "  1.6625222598416654,\n",
       "  1.6605119711804588,\n",
       "  1.6664237176942693,\n",
       "  1.6633091071966282,\n",
       "  1.6588391722734614,\n",
       "  1.6613311259039882,\n",
       "  1.6605271272712137,\n",
       "  1.6578142276431054,\n",
       "  1.652783023023209,\n",
       "  1.648089217677341,\n",
       "  1.649600606876067,\n",
       "  1.652168225713714,\n",
       "  1.6467887883041044,\n",
       "  1.6439999384893276,\n",
       "  1.6433866608506094,\n",
       "  1.6438393582927884,\n",
       "  1.6440409686096487,\n",
       "  1.64152337474506,\n",
       "  1.639261954048664,\n",
       "  1.639005054397266,\n",
       "  1.6384596933618476,\n",
       "  1.6385322138873495,\n",
       "  1.6366317688263023,\n",
       "  1.636057458425823,\n",
       "  1.6367604597123375,\n",
       "  1.6341615106260348,\n",
       "  1.6331871790899135,\n",
       "  1.634273741383962,\n",
       "  1.632514024375218,\n",
       "  1.6327536310845796,\n",
       "  1.6344124164607716,\n",
       "  1.6318661776936285,\n",
       "  1.632599066168978,\n",
       "  1.6313219575670617,\n",
       "  1.6293366602583275,\n",
       "  1.6280674125349093,\n",
       "  1.629100662519397,\n",
       "  1.6314236354959968,\n",
       "  1.6289402688969536,\n",
       "  1.6282597724090322,\n",
       "  1.6260940886600526,\n",
       "  1.624837441127386,\n",
       "  1.6260579629618046,\n",
       "  1.6277662218442583,\n",
       "  1.6255714123929306,\n",
       "  1.625792026849995,\n",
       "  1.6256230904455,\n",
       "  1.6286932431429708,\n",
       "  1.626376760963588,\n",
       "  1.6250780365143456,\n",
       "  1.6216324405987177,\n",
       "  1.62110808813671,\n",
       "  1.6212841420952964,\n",
       "  1.6205288815696484,\n",
       "  1.6216340841018593,\n",
       "  1.6186893306610657,\n",
       "  1.6181581053377188,\n",
       "  1.6190993138627663,\n",
       "  1.618933295609218,\n",
       "  1.6182240432649437,\n",
       "  1.6181097654754766,\n",
       "  1.6186893768918151,\n",
       "  1.618291619411796,\n",
       "  1.6188235973056995,\n",
       "  1.616722936445326,\n",
       "  1.6163123958328753,\n",
       "  1.6157647493473382,\n",
       "  1.6156388481568102,\n",
       "  1.614626459797994,\n",
       "  1.615075167196279,\n",
       "  1.6146018359799794,\n",
       "  1.6151762153963634,\n",
       "  1.6147568585800003,\n",
       "  1.6149412829459868,\n",
       "  1.6139556152998906,\n",
       "  1.6133016697917948,\n",
       "  1.614078189527559,\n",
       "  1.6134013697050946,\n",
       "  1.6126826955340905,\n",
       "  1.6121505568232233,\n",
       "  1.6124903558033654,\n",
       "  1.612349418722031,\n",
       "  1.612311630037683,\n",
       "  1.6116237234210704,\n",
       "  1.6118275699853237,\n",
       "  1.6128621880697742,\n",
       "  1.6115445509511679,\n",
       "  1.6126363792577938,\n",
       "  1.6120649602604704,\n",
       "  1.6133830227019714,\n",
       "  1.6129557343401077,\n",
       "  1.6105488178802658,\n",
       "  1.6111674008276984,\n",
       "  1.611330396911114,\n",
       "  1.6110662700727045,\n",
       "  1.6109023322028797,\n",
       "  1.6110514072169888,\n",
       "  1.6118645625101231,\n",
       "  1.609554374317053,\n",
       "  1.6085463471690042,\n",
       "  1.6081257919855725,\n",
       "  1.608558438491293,\n",
       "  1.6093426500331,\n",
       "  1.6087106420062585,\n",
       "  1.6081184128975274,\n",
       "  1.607255687343777,\n",
       "  1.6075764696353692,\n",
       "  1.6085331529131226,\n",
       "  1.60770972448703,\n",
       "  1.6077978924701088,\n",
       "  1.6075854278337263,\n",
       "  1.6074802145733398,\n",
       "  1.6089027954931074,\n",
       "  1.609567384640596,\n",
       "  1.6088831431317527,\n",
       "  1.6086879460104946,\n",
       "  1.6095650549410452,\n",
       "  1.6086326404951947,\n",
       "  1.6079769936955206,\n",
       "  1.6063219606050825,\n",
       "  1.6067882977694357,\n",
       "  1.608143498032377,\n",
       "  1.60669404053622,\n",
       "  1.6070644353565418,\n",
       "  1.6059624699674484,\n",
       "  1.6051409228026372,\n",
       "  1.6052963855854363,\n",
       "  1.6052211648539494,\n",
       "  1.605403248953357,\n",
       "  1.6048400610107465,\n",
       "  1.6042452608119087,\n",
       "  1.6046841309671587,\n",
       "  1.6048155808382747,\n",
       "  1.6058779419954463,\n",
       "  1.6048747611508145,\n",
       "  1.605228927987434,\n",
       "  1.6058536678800293,\n",
       "  1.6062448870112032,\n",
       "  1.6061633785675768,\n",
       "  1.60516725682816,\n",
       "  1.6044996108374767,\n",
       "  1.604426426900721,\n",
       "  1.605478557193048,\n",
       "  1.6040322097054478],\n",
       " 'train_ATT_acc': [27.977839335180054,\n",
       "  26.31578947368421,\n",
       "  27.977839335180054,\n",
       "  35.18005540166205,\n",
       "  34.903047091412745,\n",
       "  34.903047091412745,\n",
       "  36.28808864265928,\n",
       "  43.21329639889197,\n",
       "  47.36842105263158,\n",
       "  56.232686980609415,\n",
       "  65.92797783933518,\n",
       "  68.97506925207756,\n",
       "  78.11634349030471,\n",
       "  81.7174515235457,\n",
       "  82.54847645429363,\n",
       "  83.65650969529086,\n",
       "  84.21052631578948,\n",
       "  86.98060941828255,\n",
       "  86.14958448753463,\n",
       "  86.98060941828255,\n",
       "  90.02770083102493,\n",
       "  90.85872576177286,\n",
       "  91.13573407202216,\n",
       "  90.58171745152355,\n",
       "  93.07479224376732,\n",
       "  96.67590027700831,\n",
       "  97.50692520775624,\n",
       "  98.06094182825485,\n",
       "  98.33795013850416,\n",
       "  98.33795013850416,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  99.16897506925208,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  99.16897506925208,\n",
       "  99.44598337950139,\n",
       "  99.16897506925208,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0],\n",
       " 'train_VAL_acc': [68.42105263157895,\n",
       "  68.42105263157895,\n",
       "  72.29916897506925,\n",
       "  84.7645429362881,\n",
       "  83.10249307479225,\n",
       "  83.65650969529086,\n",
       "  86.14958448753463,\n",
       "  84.7645429362881,\n",
       "  82.27146814404432,\n",
       "  84.7645429362881,\n",
       "  84.21052631578948,\n",
       "  84.7645429362881,\n",
       "  86.70360110803324,\n",
       "  86.98060941828255,\n",
       "  90.30470914127424,\n",
       "  91.13573407202216,\n",
       "  86.98060941828255,\n",
       "  89.47368421052632,\n",
       "  91.96675900277009,\n",
       "  90.85872576177286,\n",
       "  91.68975069252078,\n",
       "  91.13573407202216,\n",
       "  91.96675900277009,\n",
       "  94.18282548476455,\n",
       "  94.18282548476455,\n",
       "  93.07479224376732,\n",
       "  96.67590027700831,\n",
       "  95.8448753462604,\n",
       "  95.56786703601108,\n",
       "  95.8448753462604,\n",
       "  96.1218836565097,\n",
       "  96.1218836565097,\n",
       "  96.39889196675901,\n",
       "  97.50692520775624,\n",
       "  96.67590027700831,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  99.44598337950139,\n",
       "  98.89196675900277,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  98.61495844875347,\n",
       "  99.16897506925208,\n",
       "  99.44598337950139,\n",
       "  98.89196675900277,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  98.89196675900277,\n",
       "  99.44598337950139,\n",
       "  99.16897506925208,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.16897506925208,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0],\n",
       " 'train_VAL_jac': [0.0,\n",
       "  0.0,\n",
       "  0.0,\n",
       "  0.0,\n",
       "  0.0,\n",
       "  0.10110803324099724,\n",
       "  0.2520775623268698,\n",
       "  0.3254847645429363,\n",
       "  0.37026777558049334,\n",
       "  0.3905817200932807,\n",
       "  0.44967683860799945,\n",
       "  0.44459834851716695,\n",
       "  0.4445983471962884,\n",
       "  0.44367498043831693,\n",
       "  0.46121884449036826,\n",
       "  0.48753463924756674,\n",
       "  0.5383195005295349,\n",
       "  0.5364727709762277,\n",
       "  0.5161588238216833,\n",
       "  0.5180055454497192,\n",
       "  0.5420129358603353,\n",
       "  0.5466297498370142,\n",
       "  0.5521699213255146,\n",
       "  0.5923361104611214,\n",
       "  0.5983379633472897,\n",
       "  0.6006463729773862,\n",
       "  0.641735927550086,\n",
       "  0.6398892072429287,\n",
       "  0.6246537501792168,\n",
       "  0.6394275232035037,\n",
       "  0.6551246643066406,\n",
       "  0.6463527309597364,\n",
       "  0.6417359328336002,\n",
       "  0.6523545864876618,\n",
       "  0.6606648252281125,\n",
       "  0.6777470131660102,\n",
       "  0.654201306794819,\n",
       "  0.6278855173211348,\n",
       "  0.6722068416775099,\n",
       "  0.6948291905368794,\n",
       "  0.7003693567418656,\n",
       "  0.680517090984989,\n",
       "  0.6680517223072845,\n",
       "  0.6754386035359137,\n",
       "  0.7063712017027625,\n",
       "  0.6948291852533652,\n",
       "  0.7036011186002694,\n",
       "  0.7049861601515159,\n",
       "  0.7211449800105636,\n",
       "  0.7197599384593171,\n",
       "  0.7119113679077487,\n",
       "  0.7216066534829602,\n",
       "  0.7220683428058994,\n",
       "  0.7340720327276933,\n",
       "  0.7183748916245564,\n",
       "  0.7192982597034063,\n",
       "  0.7174515341127348,\n",
       "  0.7336103486882682,\n",
       "  0.7036011133167552,\n",
       "  0.7469990801613087,\n",
       "  0.7354570689954256,\n",
       "  0.7525392463662948,\n",
       "  0.725300099380789,\n",
       "  0.7183748863410422,\n",
       "  0.734533711483604,\n",
       "  0.7562326975476379,\n",
       "  0.7465374119724263,\n",
       "  0.7590027806501309,\n",
       "  0.749307484507891,\n",
       "  0.740535556444501,\n",
       "  0.7585410913271917,\n",
       "  0.7603878274848916,\n",
       "  0.7659279884063637,\n",
       "  0.744229007625844,\n",
       "  0.7488458110354944,\n",
       "  0.7497691685473159,\n",
       "  0.7710064705719247,\n",
       "  0.7710064652884105,\n",
       "  0.7668513459181852,\n",
       "  0.766851340634671,\n",
       "  0.760849495673774,\n",
       "  0.7723915121231714,\n",
       "  0.7839335285725686,\n",
       "  0.7746999164697537,\n",
       "  0.7571560550594594,\n",
       "  0.7710064652884105,\n",
       "  0.7742382377138428,\n",
       "  0.7682363874694317,\n",
       "  0.7710064652884105,\n",
       "  0.7802400826747398,\n",
       "  0.8074792349437597,\n",
       "  0.7950138556990267,\n",
       "  0.8084025924555813,\n",
       "  0.8033240997229917,\n",
       "  0.8120960330698959,\n",
       "  0.8028624209670809,\n",
       "  0.8088642712114921,\n",
       "  0.7816251189424721,\n",
       "  0.7987073015968555,\n",
       "  0.7963988972502732,\n",
       "  0.8074792296602455,\n",
       "  0.8213296451727109,\n",
       "  0.8010157006599236,\n",
       "  0.808402587172067,\n",
       "  0.8139427533770532,\n",
       "  0.8190212408261286,\n",
       "  0.8070175509043348,\n",
       "  0.8042474678018416,\n",
       "  0.8097876340068279,\n",
       "  0.8116343490304709,\n",
       "  0.8107109915186493,\n",
       "  0.7899353682499513,\n",
       "  0.8070175509043348,\n",
       "  0.7987073015968555,\n",
       "  0.8051708305971774,\n",
       "  0.8042474678018416,\n",
       "  0.8060941933925132,\n",
       "  0.8139427586605674,\n",
       "  0.8125577223928351,\n",
       "  0.8130193958652316,\n",
       "  0.8287165422518827,\n",
       "  0.8162511630071497,\n",
       "  0.8171745258024855,\n",
       "  0.8093259605344313,\n",
       "  0.8264081379053005,\n",
       "  0.8379501385041551,\n",
       "  0.8347183819292655,\n",
       "  0.8254847698264505,\n",
       "  0.832871656338594,\n",
       "  0.824099728275204,\n",
       "  0.8250230910705397,\n",
       "  0.8370267809923336,\n",
       "  0.8296398944801902,\n",
       "  0.8254847698264505,\n",
       "  0.8305632572755259,\n",
       "  0.8319482935432582,\n",
       "  0.8070175509043348,\n",
       "  0.8264081326217863,\n",
       "  0.8319482988267725,\n",
       "  0.8287165316848544,\n",
       "  0.824561412314629,\n",
       "  0.8287165369683686,\n",
       "  0.8153278054953282,\n",
       "  0.8250230963540539,\n",
       "  0.832871656338594,\n",
       "  0.8421052631578947,\n",
       "  0.8314866147873474,\n",
       "  0.8176361992748821,\n",
       "  0.8162511577236355,\n",
       "  0.8217913239286216,\n",
       "  0.8402585481342516,\n",
       "  0.8397968693783409,\n",
       "  0.8374884597482444,\n",
       "  0.8393351853389159,\n",
       "  0.8494921549535525,\n",
       "  0.8471837558904843,\n",
       "  0.8490304761976416,\n",
       "  0.8494921549535525,\n",
       "  0.8536472848908062,\n",
       "  0.844875351543902,\n",
       "  0.8471837506069702,\n",
       "  0.8457987143392378,\n",
       "  0.8365651022364228,\n",
       "  0.823638044235779,\n",
       "  0.8319482935432582,\n",
       "  0.842105268441409,\n",
       "  0.8407202268901625,\n",
       "  0.841643584401984,\n",
       "  0.8264081326217863,\n",
       "  0.8628808864265928],\n",
       " 'train_VAL_acc_1': [19.94459833795014,\n",
       "  19.94459833795014,\n",
       "  21.606648199445985,\n",
       "  51.52354570637119,\n",
       "  49.58448753462604,\n",
       "  51.52354570637119,\n",
       "  54.84764542936288,\n",
       "  51.800554016620495,\n",
       "  47.64542936288089,\n",
       "  47.9224376731302,\n",
       "  47.64542936288089,\n",
       "  48.47645429362881,\n",
       "  50.41551246537396,\n",
       "  49.307479224376735,\n",
       "  53.46260387811634,\n",
       "  61.772853185595565,\n",
       "  58.448753462603875,\n",
       "  54.016620498614955,\n",
       "  57.61772853185595,\n",
       "  56.50969529085872,\n",
       "  62.04986149584487,\n",
       "  58.448753462603875,\n",
       "  61.21883656509695,\n",
       "  64.81994459833795,\n",
       "  66.7590027700831,\n",
       "  64.81994459833795,\n",
       "  66.4819944598338,\n",
       "  68.42105263157895,\n",
       "  67.59002770083103,\n",
       "  67.31301939058172,\n",
       "  67.86703601108033,\n",
       "  68.42105263157895,\n",
       "  68.42105263157895,\n",
       "  70.91412742382272,\n",
       "  71.74515235457064,\n",
       "  71.74515235457064,\n",
       "  70.3601108033241,\n",
       "  69.25207756232687,\n",
       "  73.6842105263158,\n",
       "  73.6842105263158,\n",
       "  74.79224376731302,\n",
       "  73.40720221606648,\n",
       "  70.6371191135734,\n",
       "  72.85318559556787,\n",
       "  75.62326869806094,\n",
       "  77.5623268698061,\n",
       "  75.62326869806094,\n",
       "  75.90027700831025,\n",
       "  76.45429362880887,\n",
       "  75.90027700831025,\n",
       "  74.51523545706371,\n",
       "  76.73130193905817,\n",
       "  78.39335180055402,\n",
       "  77.5623268698061,\n",
       "  76.45429362880887,\n",
       "  76.45429362880887,\n",
       "  73.9612188365651,\n",
       "  77.00831024930748,\n",
       "  75.62326869806094,\n",
       "  77.8393351800554,\n",
       "  78.11634349030471,\n",
       "  78.94736842105263,\n",
       "  78.94736842105263,\n",
       "  76.73130193905817,\n",
       "  77.8393351800554,\n",
       "  78.94736842105263,\n",
       "  78.94736842105263,\n",
       "  78.94736842105263,\n",
       "  78.39335180055402,\n",
       "  77.28531855955679,\n",
       "  79.50138504155125,\n",
       "  77.5623268698061,\n",
       "  76.73130193905817,\n",
       "  76.17728531855956,\n",
       "  77.8393351800554,\n",
       "  77.8393351800554,\n",
       "  79.50138504155125,\n",
       "  80.88642659279779,\n",
       "  80.05540166204986,\n",
       "  81.16343490304709,\n",
       "  81.7174515235457,\n",
       "  82.27146814404432,\n",
       "  81.99445983379502,\n",
       "  83.10249307479225,\n",
       "  81.4404432132964,\n",
       "  81.7174515235457,\n",
       "  81.99445983379502,\n",
       "  81.7174515235457,\n",
       "  80.88642659279779,\n",
       "  80.60941828254848,\n",
       "  81.4404432132964,\n",
       "  84.7645429362881,\n",
       "  83.65650969529086,\n",
       "  83.65650969529086,\n",
       "  82.82548476454294,\n",
       "  82.82548476454294,\n",
       "  82.54847645429363,\n",
       "  83.37950138504155,\n",
       "  82.82548476454294,\n",
       "  80.88642659279779,\n",
       "  82.82548476454294,\n",
       "  84.7645429362881,\n",
       "  83.10249307479225,\n",
       "  82.54847645429363,\n",
       "  83.37950138504155,\n",
       "  85.31855955678671,\n",
       "  84.7645429362881,\n",
       "  83.65650969529086,\n",
       "  83.93351800554017,\n",
       "  84.21052631578948,\n",
       "  83.65650969529086,\n",
       "  83.93351800554017,\n",
       "  83.37950138504155,\n",
       "  83.93351800554017,\n",
       "  84.7645429362881,\n",
       "  83.10249307479225,\n",
       "  81.7174515235457,\n",
       "  83.93351800554017,\n",
       "  83.93351800554017,\n",
       "  84.21052631578948,\n",
       "  83.37950138504155,\n",
       "  82.82548476454294,\n",
       "  83.65650969529086,\n",
       "  81.99445983379502,\n",
       "  82.82548476454294,\n",
       "  85.0415512465374,\n",
       "  86.14958448753463,\n",
       "  85.31855955678671,\n",
       "  85.31855955678671,\n",
       "  83.65650969529086,\n",
       "  83.65650969529086,\n",
       "  86.14958448753463,\n",
       "  86.14958448753463,\n",
       "  84.7645429362881,\n",
       "  85.31855955678671,\n",
       "  85.31855955678671,\n",
       "  85.0415512465374,\n",
       "  86.42659279778394,\n",
       "  85.31855955678671,\n",
       "  83.93351800554017,\n",
       "  81.4404432132964,\n",
       "  83.37950138504155,\n",
       "  82.54847645429363,\n",
       "  84.48753462603878,\n",
       "  84.48753462603878,\n",
       "  86.42659279778394,\n",
       "  85.59556786703601,\n",
       "  82.27146814404432,\n",
       "  83.37950138504155,\n",
       "  84.7645429362881,\n",
       "  85.0415512465374,\n",
       "  86.14958448753463,\n",
       "  86.42659279778394,\n",
       "  84.7645429362881,\n",
       "  85.59556786703601,\n",
       "  85.87257617728532,\n",
       "  88.36565096952909,\n",
       "  86.42659279778394,\n",
       "  85.59556786703601,\n",
       "  83.37950138504155,\n",
       "  86.14958448753463,\n",
       "  85.59556786703601,\n",
       "  83.65650969529086,\n",
       "  85.31855955678671,\n",
       "  83.93351800554017,\n",
       "  87.81163434903047,\n",
       "  87.81163434903047,\n",
       "  88.08864265927978,\n",
       "  86.98060941828255,\n",
       "  89.75069252077563],\n",
       " 'val_loss': [7.389335035278751,\n",
       "  7.351940003074334,\n",
       "  7.28385148578013,\n",
       "  7.247727541876688,\n",
       "  7.190594639592274,\n",
       "  7.124504636205227,\n",
       "  7.038099749521125,\n",
       "  6.932983508210402,\n",
       "  6.83111528790328,\n",
       "  6.744247649536818,\n",
       "  6.626269363061729,\n",
       "  6.491639192467233,\n",
       "  6.361750173758026,\n",
       "  6.331452633756015,\n",
       "  6.231342453424676,\n",
       "  6.238035516764386,\n",
       "  6.201665538890308,\n",
       "  6.1791061978302135,\n",
       "  6.160957223964952,\n",
       "  6.113802697247687,\n",
       "  6.069780767910462,\n",
       "  6.072291408439408,\n",
       "  6.015231831932107,\n",
       "  5.962008695979777,\n",
       "  5.970577143485686,\n",
       "  5.944798442663002,\n",
       "  5.913527129073635,\n",
       "  5.902649265380797,\n",
       "  5.918441751206985,\n",
       "  5.910239144055069,\n",
       "  5.873215003432722,\n",
       "  5.883730675410611,\n",
       "  5.867670532354313,\n",
       "  5.849226252703091,\n",
       "  5.83160365130034,\n",
       "  5.813149176423109,\n",
       "  5.816661082926389,\n",
       "  5.8284616717979425,\n",
       "  5.810774430887419,\n",
       "  5.798029430301003,\n",
       "  5.798642801080222,\n",
       "  5.79428024918248,\n",
       "  5.798609770790582,\n",
       "  5.785624395405589,\n",
       "  5.776498888804957,\n",
       "  5.770442308641925,\n",
       "  5.768002603649305,\n",
       "  5.7684758501403,\n",
       "  5.759240985845884,\n",
       "  5.759669658722387,\n",
       "  5.758764695988013,\n",
       "  5.747202527038331,\n",
       "  5.744368810346748,\n",
       "  5.753495160975909,\n",
       "  5.737434603560027,\n",
       "  5.736576697576714,\n",
       "  5.746213150490385,\n",
       "  5.737737711884982,\n",
       "  5.738964980041228,\n",
       "  5.73084300950231,\n",
       "  5.723768712214433,\n",
       "  5.719760086695486,\n",
       "  5.7269706511380285,\n",
       "  5.734066703607416,\n",
       "  5.73635173211852,\n",
       "  5.735371186004426,\n",
       "  5.721775338598569,\n",
       "  5.710841491544503,\n",
       "  5.7129367284932995,\n",
       "  5.7207307717159575,\n",
       "  5.719542234062414,\n",
       "  5.721878519560818,\n",
       "  5.720248056030864,\n",
       "  5.721766438726225,\n",
       "  5.716764444717083,\n",
       "  5.716359588240178,\n",
       "  5.7026988744468365,\n",
       "  5.704617501424748,\n",
       "  5.704324834018611,\n",
       "  5.705494319067981,\n",
       "  5.707634992606577,\n",
       "  5.698658765565457,\n",
       "  5.69399780613484,\n",
       "  5.7029845173319575,\n",
       "  5.691668502897212,\n",
       "  5.688194006733959,\n",
       "  5.690352249013141,\n",
       "  5.697788103906477,\n",
       "  5.693946429703117,\n",
       "  5.701836691814146,\n",
       "  5.689986224100545,\n",
       "  5.681576957339885,\n",
       "  5.682564651074138,\n",
       "  5.684316705722126,\n",
       "  5.688614256507838,\n",
       "  5.684231815878848,\n",
       "  5.681171642494572,\n",
       "  5.6862258232329985,\n",
       "  5.684235231689412,\n",
       "  5.681089871586852,\n",
       "  5.674814448439726,\n",
       "  5.67480370126382,\n",
       "  5.680939387853696,\n",
       "  5.672993344347107,\n",
       "  5.677565892187718,\n",
       "  5.675447280152769,\n",
       "  5.684449544847023,\n",
       "  5.677508145487205,\n",
       "  5.6668724232766206,\n",
       "  5.675441834700685,\n",
       "  5.673142204148506,\n",
       "  5.681848538318053,\n",
       "  5.677246046153296,\n",
       "  5.6795696078139954,\n",
       "  5.688473215015518,\n",
       "  5.694097299242608,\n",
       "  5.690042774118876,\n",
       "  5.6762843098682545,\n",
       "  5.683082944973769,\n",
       "  5.68078246830729,\n",
       "  5.6782622737462765,\n",
       "  5.675934083720695,\n",
       "  5.6726920290509675,\n",
       "  5.689148196593304,\n",
       "  5.674858198794453,\n",
       "  5.666599560244177,\n",
       "  5.665500469059093,\n",
       "  5.673181561549128,\n",
       "  5.666320657892667,\n",
       "  5.666699788904986,\n",
       "  5.662518403546616,\n",
       "  5.661674470632625,\n",
       "  5.667558635347975,\n",
       "  5.674706492353256,\n",
       "  5.6696041932904615,\n",
       "  5.66959130761501,\n",
       "  5.664970794256608,\n",
       "  5.668566283837753,\n",
       "  5.6851491607488995,\n",
       "  5.683687099741368,\n",
       "  5.679629395775459,\n",
       "  5.676242866661944,\n",
       "  5.677973161955691,\n",
       "  5.669689296938458,\n",
       "  5.660714911813707,\n",
       "  5.658746077733741,\n",
       "  5.675165803485865,\n",
       "  5.681277299532209,\n",
       "  5.670703209400406,\n",
       "  5.669041735028026,\n",
       "  5.670996687851281,\n",
       "  5.660797788021878,\n",
       "  5.666154042049253,\n",
       "  5.661422396913237,\n",
       "  5.6649407020947296,\n",
       "  5.6587948122437135,\n",
       "  5.664828681324308,\n",
       "  5.6619066436492265,\n",
       "  5.66408244088621,\n",
       "  5.6698946744410055,\n",
       "  5.65898779735779,\n",
       "  5.664001124619368,\n",
       "  5.6700209012608145,\n",
       "  5.657319169594923,\n",
       "  5.672919330271699,\n",
       "  5.673788656609629,\n",
       "  5.661035936953757,\n",
       "  5.661534674821042,\n",
       "  5.669836502068821,\n",
       "  5.6645986045864],\n",
       " 'val_ATT_loss': [1.9754252106678196,\n",
       "  1.95841538978786,\n",
       "  1.9339974968413995,\n",
       "  1.928011721954113,\n",
       "  1.9089900994203923,\n",
       "  1.894441720431413,\n",
       "  1.863403122599532,\n",
       "  1.8009332558973048,\n",
       "  1.7079340270379695,\n",
       "  1.611228049528308,\n",
       "  1.5005133612369135,\n",
       "  1.4064908873259536,\n",
       "  1.3029033776705827,\n",
       "  1.254273116104002,\n",
       "  1.175094438641052,\n",
       "  1.1865728817334988,\n",
       "  1.1424274873442766,\n",
       "  1.087599147626055,\n",
       "  1.0749053165195435,\n",
       "  1.0564707317730275,\n",
       "  1.0459723455634544,\n",
       "  1.0321546965013675,\n",
       "  0.9747485468784968,\n",
       "  0.9756261410751963,\n",
       "  0.9655944582650332,\n",
       "  0.9406097247833158,\n",
       "  0.9271048034836606,\n",
       "  0.9223230327290248,\n",
       "  0.9219872222198704,\n",
       "  0.9180276248513198,\n",
       "  0.8979151247720408,\n",
       "  0.8909011417772712,\n",
       "  0.8796277261846434,\n",
       "  0.8786799072977004,\n",
       "  0.8793070806962687,\n",
       "  0.8689187720296828,\n",
       "  0.8650327725865976,\n",
       "  0.8640053766529735,\n",
       "  0.862186959846233,\n",
       "  0.8562932406983724,\n",
       "  0.8580713931137953,\n",
       "  0.8542373077413901,\n",
       "  0.8538121439334823,\n",
       "  0.8491734982263751,\n",
       "  0.8457789324163422,\n",
       "  0.8426134529394832,\n",
       "  0.8418570248092093,\n",
       "  0.839810941519776,\n",
       "  0.8361607517411069,\n",
       "  0.837965819772666,\n",
       "  0.8349738077419561,\n",
       "  0.8301556547725104,\n",
       "  0.8259650345255689,\n",
       "  0.826118339852589,\n",
       "  0.823322027437086,\n",
       "  0.8194694425759277,\n",
       "  0.8259131753347754,\n",
       "  0.8235300736940974,\n",
       "  0.8231026258895068,\n",
       "  0.8205215777081203,\n",
       "  0.8150602331975612,\n",
       "  0.8142722985608791,\n",
       "  0.8176098377481709,\n",
       "  0.8144947742301274,\n",
       "  0.8193655526492654,\n",
       "  0.8178679946719146,\n",
       "  0.8166530796182834,\n",
       "  0.8100011850275645,\n",
       "  0.8107232366635547,\n",
       "  0.807168188497303,\n",
       "  0.8104208481263339,\n",
       "  0.810321339019915,\n",
       "  0.8071216801560022,\n",
       "  0.803669609795741,\n",
       "  0.8057034261585251,\n",
       "  0.8063663566500191,\n",
       "  0.8040095674313181,\n",
       "  0.8039398944474817,\n",
       "  0.8036975680085702,\n",
       "  0.8042957945083214,\n",
       "  0.8081155379370946,\n",
       "  0.8014098972809024,\n",
       "  0.8016714617246534,\n",
       "  0.8037425131817174,\n",
       "  0.7979980661132471,\n",
       "  0.8011693404457434,\n",
       "  0.7970080216967963,\n",
       "  0.7980884382637535,\n",
       "  0.7992046085315022,\n",
       "  0.7994071205214757,\n",
       "  0.8000339068532959,\n",
       "  0.7931234056387491,\n",
       "  0.7979505058468842,\n",
       "  0.7972169692923383,\n",
       "  0.7982184155200555,\n",
       "  0.7958054650362915,\n",
       "  0.7953570233127936,\n",
       "  0.7964241236932879,\n",
       "  0.7904736438417822,\n",
       "  0.7926887608155971,\n",
       "  0.7923866814471842,\n",
       "  0.791083546431084,\n",
       "  0.7990993584801511,\n",
       "  0.7929698214298342,\n",
       "  0.7942293293592406,\n",
       "  0.7918727926122464,\n",
       "  0.7940859079845553,\n",
       "  0.7934170847501212,\n",
       "  0.7886485401692429,\n",
       "  0.7929460532296964,\n",
       "  0.7898322537662538,\n",
       "  0.7904602130496405,\n",
       "  0.7899776802557271,\n",
       "  0.788233066356279,\n",
       "  0.7935539293822235,\n",
       "  0.7907354238072062,\n",
       "  0.7948543884889866,\n",
       "  0.7890621678131383,\n",
       "  0.7890478246580295,\n",
       "  0.7909488324227372,\n",
       "  0.7889211742373986,\n",
       "  0.789680158340834,\n",
       "  0.7871854208349213,\n",
       "  0.7877322906643395,\n",
       "  0.7863621809860555,\n",
       "  0.7883145421501098,\n",
       "  0.7851526780826289,\n",
       "  0.7895335541750358,\n",
       "  0.7831492169601161,\n",
       "  0.7849748963262977,\n",
       "  0.7851390524850628,\n",
       "  0.7850637517082013,\n",
       "  0.7864594558874766,\n",
       "  0.7874815333180312,\n",
       "  0.7872594135805844,\n",
       "  0.7850923414637403,\n",
       "  0.7842710569379775,\n",
       "  0.785236436661666,\n",
       "  0.7892928354866137,\n",
       "  0.7899342640870954,\n",
       "  0.790399633045119,\n",
       "  0.7816502203543981,\n",
       "  0.788775199918243,\n",
       "  0.7819638702927566,\n",
       "  0.7802266224128444,\n",
       "  0.7850008776517419,\n",
       "  0.7846237473129257,\n",
       "  0.7898857714684029,\n",
       "  0.7871777996784304,\n",
       "  0.7845709422012654,\n",
       "  0.7900229247847224,\n",
       "  0.7839975016630762,\n",
       "  0.7826116441226587,\n",
       "  0.783148912758362,\n",
       "  0.7789311774862491,\n",
       "  0.7814529627803865,\n",
       "  0.7818793008724848,\n",
       "  0.782503011386569,\n",
       "  0.7839939072122418,\n",
       "  0.7838839236798325,\n",
       "  0.7825043411031971,\n",
       "  0.7796242205592675,\n",
       "  0.784082246868591,\n",
       "  0.7774516816546277,\n",
       "  0.7828813461753411,\n",
       "  0.7854755804306124,\n",
       "  0.7830267116063978,\n",
       "  0.7860671107361956,\n",
       "  0.7846018443020378,\n",
       "  0.7814090743782075],\n",
       " 'val_VAL_loss': [1.8046366082036436,\n",
       "  1.797841537762158,\n",
       "  1.7832846629795769,\n",
       "  1.773238606640858,\n",
       "  1.7605348467239605,\n",
       "  1.743354305257938,\n",
       "  1.7248988756405308,\n",
       "  1.710683417437699,\n",
       "  1.7077270869551033,\n",
       "  1.7110065333361697,\n",
       "  1.7085853339416053,\n",
       "  1.6950494350470933,\n",
       "  1.6862822653624812,\n",
       "  1.6923931725506711,\n",
       "  1.6854160049278748,\n",
       "  1.683820878343629,\n",
       "  1.686412683848677,\n",
       "  1.6971690167347198,\n",
       "  1.6953506358151365,\n",
       "  1.6857773218248866,\n",
       "  1.6746028074490025,\n",
       "  1.6800455706460136,\n",
       "  1.68016109501787,\n",
       "  1.662127518301527,\n",
       "  1.6683275617402176,\n",
       "  1.668062905959895,\n",
       "  1.662140775196658,\n",
       "  1.6601087442172573,\n",
       "  1.6654848429957048,\n",
       "  1.66407050640125,\n",
       "  1.6584332928868937,\n",
       "  1.6642765112111133,\n",
       "  1.66268093538989,\n",
       "  1.656848781801797,\n",
       "  1.6507655235346903,\n",
       "  1.6480768014644753,\n",
       "  1.6505427701132638,\n",
       "  1.654818765048323,\n",
       "  1.6495291570137287,\n",
       "  1.6472453965342104,\n",
       "  1.6468571359888087,\n",
       "  1.6466809804803633,\n",
       "  1.6482658756190334,\n",
       "  1.6454836323930713,\n",
       "  1.6435733187962047,\n",
       "  1.6426096185674808,\n",
       "  1.642048526280032,\n",
       "  1.6428883028735082,\n",
       "  1.6410267447015923,\n",
       "  1.6405679463165734,\n",
       "  1.6412636294153524,\n",
       "  1.639015624088607,\n",
       "  1.639467925273726,\n",
       "  1.64245894037444,\n",
       "  1.6380375253743138,\n",
       "  1.6390357516669287,\n",
       "  1.6400999917185366,\n",
       "  1.638069212730295,\n",
       "  1.6386207847172403,\n",
       "  1.6367738105980634,\n",
       "  1.6362361596722907,\n",
       "  1.6351625960448692,\n",
       "  1.636453604463286,\n",
       "  1.6398573097924294,\n",
       "  1.638995393156418,\n",
       "  1.6391677304441705,\n",
       "  1.6350407529934285,\n",
       "  1.6336134355056462,\n",
       "  1.6340711639432484,\n",
       "  1.6378541944062182,\n",
       "  1.6363737953120265,\n",
       "  1.6371857268469674,\n",
       "  1.6377087919582873,\n",
       "  1.6393656096434945,\n",
       "  1.6370203395195195,\n",
       "  1.6366644105300527,\n",
       "  1.6328964356718392,\n",
       "  1.6335592023257552,\n",
       "  1.6335424220033468,\n",
       "  1.6337328415198866,\n",
       "  1.6331731515564942,\n",
       "  1.6324162894281848,\n",
       "  1.630775448136729,\n",
       "  1.63308066805008,\n",
       "  1.6312234789279882,\n",
       "  1.629008222096072,\n",
       "  1.6311147424387815,\n",
       "  1.633233221880908,\n",
       "  1.631580607057205,\n",
       "  1.63414319043089,\n",
       "  1.6299841057490834,\n",
       "  1.6294845172337122,\n",
       "  1.6282047150757513,\n",
       "  1.629033245476596,\n",
       "  1.6301319469959277,\n",
       "  1.629475450280852,\n",
       "  1.6286048730605929,\n",
       "  1.6299338998465702,\n",
       "  1.6312538626158766,\n",
       "  1.6294670369237514,\n",
       "  1.627475922330847,\n",
       "  1.6279067182775788,\n",
       "  1.6272800097911817,\n",
       "  1.626674507639091,\n",
       "  1.627778854276159,\n",
       "  1.6278581625135073,\n",
       "  1.6301212122874895,\n",
       "  1.628030353579028,\n",
       "  1.6260746277024594,\n",
       "  1.627498593823663,\n",
       "  1.6277699834607504,\n",
       "  1.6304627750894707,\n",
       "  1.6290894552991895,\n",
       "  1.6304455138192389,\n",
       "  1.6316397618777647,\n",
       "  1.6344539584784672,\n",
       "  1.6317294618766296,\n",
       "  1.6290740473517056,\n",
       "  1.6313450401052465,\n",
       "  1.6299445452948509,\n",
       "  1.629780366502959,\n",
       "  1.6287513084599536,\n",
       "  1.628502202738682,\n",
       "  1.6338053019763215,\n",
       "  1.6294986726027991,\n",
       "  1.626095006031356,\n",
       "  1.6267825969921543,\n",
       "  1.6278826691246973,\n",
       "  1.6277238136441836,\n",
       "  1.627241630859563,\n",
       "  1.6257931170205178,\n",
       "  1.6255369063081413,\n",
       "  1.6270330598201659,\n",
       "  1.629074986345075,\n",
       "  1.6274482599032924,\n",
       "  1.6281663220504234,\n",
       "  1.6268999124395436,\n",
       "  1.6277766157253621,\n",
       "  1.631952108420762,\n",
       "  1.631250945218091,\n",
       "  1.6297432542434467,\n",
       "  1.6315308821025154,\n",
       "  1.6297326540124828,\n",
       "  1.6292418088819005,\n",
       "  1.6268294298002872,\n",
       "  1.6245817333606665,\n",
       "  1.6301806853909797,\n",
       "  1.6304638426879356,\n",
       "  1.6278418032406585,\n",
       "  1.6281569309422534,\n",
       "  1.6269912543555198,\n",
       "  1.6256000954529335,\n",
       "  1.6278474659755313,\n",
       "  1.6260911613849585,\n",
       "  1.6286698415361602,\n",
       "  1.6257806164877755,\n",
       "  1.627649793483941,\n",
       "  1.6264678774208858,\n",
       "  1.6266961778913225,\n",
       "  1.6286702502537242,\n",
       "  1.6254944854181976,\n",
       "  1.6281256346867001,\n",
       "  1.6286462181307413,\n",
       "  1.6266224959800983,\n",
       "  1.6300126613654526,\n",
       "  1.629437692059672,\n",
       "  1.6260030751157863,\n",
       "  1.625155854694949,\n",
       "  1.628411552588928,\n",
       "  1.627729843402731],\n",
       " 'val_ATT_acc': [29.67479674796748,\n",
       "  12.601626016260163,\n",
       "  29.67479674796748,\n",
       "  33.739837398373986,\n",
       "  33.333333333333336,\n",
       "  34.959349593495936,\n",
       "  36.17886178861789,\n",
       "  39.63414634146341,\n",
       "  42.479674796747965,\n",
       "  45.32520325203252,\n",
       "  55.08130081300813,\n",
       "  59.34959349593496,\n",
       "  66.869918699187,\n",
       "  69.51219512195122,\n",
       "  71.7479674796748,\n",
       "  70.1219512195122,\n",
       "  70.9349593495935,\n",
       "  74.79674796747967,\n",
       "  75.8130081300813,\n",
       "  78.65853658536585,\n",
       "  80.08130081300813,\n",
       "  79.0650406504065,\n",
       "  85.5691056910569,\n",
       "  83.9430894308943,\n",
       "  84.95934959349593,\n",
       "  88.41463414634147,\n",
       "  88.6178861788618,\n",
       "  88.82113821138212,\n",
       "  88.82113821138212,\n",
       "  88.6178861788618,\n",
       "  89.22764227642277,\n",
       "  89.83739837398375,\n",
       "  90.65040650406505,\n",
       "  89.63414634146342,\n",
       "  91.0569105691057,\n",
       "  91.66666666666667,\n",
       "  91.46341463414635,\n",
       "  91.66666666666667,\n",
       "  91.0569105691057,\n",
       "  92.27642276422765,\n",
       "  93.4959349593496,\n",
       "  92.07317073170732,\n",
       "  92.88617886178862,\n",
       "  92.6829268292683,\n",
       "  92.88617886178862,\n",
       "  92.07317073170732,\n",
       "  92.6829268292683,\n",
       "  92.88617886178862,\n",
       "  93.4959349593496,\n",
       "  92.07317073170732,\n",
       "  93.08943089430895,\n",
       "  92.88617886178862,\n",
       "  93.29268292682927,\n",
       "  93.29268292682927,\n",
       "  93.08943089430895,\n",
       "  93.69918699186992,\n",
       "  92.6829268292683,\n",
       "  93.08943089430895,\n",
       "  92.6829268292683,\n",
       "  93.29268292682927,\n",
       "  94.51219512195122,\n",
       "  94.51219512195122,\n",
       "  94.3089430894309,\n",
       "  94.3089430894309,\n",
       "  93.69918699186992,\n",
       "  93.29268292682927,\n",
       "  93.4959349593496,\n",
       "  93.69918699186992,\n",
       "  94.10569105691057,\n",
       "  94.10569105691057,\n",
       "  95.1219512195122,\n",
       "  94.71544715447155,\n",
       "  94.51219512195122,\n",
       "  95.1219512195122,\n",
       "  94.91869918699187,\n",
       "  94.51219512195122,\n",
       "  94.91869918699187,\n",
       "  95.1219512195122,\n",
       "  95.1219512195122,\n",
       "  94.51219512195122,\n",
       "  94.3089430894309,\n",
       "  94.10569105691057,\n",
       "  94.71544715447155,\n",
       "  94.71544715447155,\n",
       "  95.32520325203252,\n",
       "  94.10569105691057,\n",
       "  94.71544715447155,\n",
       "  94.91869918699187,\n",
       "  95.52845528455285,\n",
       "  94.3089430894309,\n",
       "  94.71544715447155,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  94.91869918699187,\n",
       "  95.52845528455285,\n",
       "  94.71544715447155,\n",
       "  95.1219512195122,\n",
       "  94.71544715447155,\n",
       "  95.32520325203252,\n",
       "  95.32520325203252,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.1219512195122,\n",
       "  94.91869918699187,\n",
       "  95.1219512195122,\n",
       "  95.9349593495935,\n",
       "  95.32520325203252,\n",
       "  94.71544715447155,\n",
       "  95.9349593495935,\n",
       "  95.1219512195122,\n",
       "  95.1219512195122,\n",
       "  95.73170731707317,\n",
       "  95.1219512195122,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.1219512195122,\n",
       "  94.71544715447155,\n",
       "  94.91869918699187,\n",
       "  95.1219512195122,\n",
       "  95.1219512195122,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  94.71544715447155,\n",
       "  95.9349593495935,\n",
       "  95.32520325203252,\n",
       "  95.1219512195122,\n",
       "  95.73170731707317,\n",
       "  95.32520325203252,\n",
       "  95.1219512195122,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  96.34146341463415,\n",
       "  96.7479674796748,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  94.51219512195122,\n",
       "  95.52845528455285,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  96.13821138211382,\n",
       "  95.1219512195122,\n",
       "  95.52845528455285,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  96.7479674796748,\n",
       "  95.52845528455285,\n",
       "  96.7479674796748,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935],\n",
       " 'val_VAL_acc': [64.5320197044335,\n",
       "  64.5320197044335,\n",
       "  69.95073891625616,\n",
       "  90.14778325123153,\n",
       "  88.66995073891626,\n",
       "  91.13300492610837,\n",
       "  89.16256157635468,\n",
       "  92.61083743842364,\n",
       "  86.20689655172414,\n",
       "  84.23645320197045,\n",
       "  87.6847290640394,\n",
       "  92.11822660098522,\n",
       "  89.65517241379311,\n",
       "  87.6847290640394,\n",
       "  86.69950738916256,\n",
       "  89.65517241379311,\n",
       "  92.11822660098522,\n",
       "  93.5960591133005,\n",
       "  93.10344827586206,\n",
       "  94.08866995073892,\n",
       "  93.5960591133005,\n",
       "  94.08866995073892,\n",
       "  93.5960591133005,\n",
       "  96.05911330049261,\n",
       "  97.53694581280789,\n",
       "  96.55172413793103,\n",
       "  96.05911330049261,\n",
       "  96.55172413793103,\n",
       "  93.5960591133005,\n",
       "  96.05911330049261,\n",
       "  98.0295566502463,\n",
       "  96.55172413793103,\n",
       "  95.56650246305419,\n",
       "  97.04433497536945,\n",
       "  97.04433497536945,\n",
       "  98.0295566502463,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  97.53694581280789,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158],\n",
       " 'val_VAL_jac': [0.0,\n",
       "  0.0,\n",
       "  0.0,\n",
       "  0.0,\n",
       "  0.0,\n",
       "  0.1921182266009852,\n",
       "  0.34646962311467516,\n",
       "  0.45484400735112834,\n",
       "  0.4868637117846259,\n",
       "  0.4770114973848089,\n",
       "  0.4983579630922214,\n",
       "  0.551724140279986,\n",
       "  0.5500821057211589,\n",
       "  0.5032840726410814,\n",
       "  0.5065681476311144,\n",
       "  0.5254515591513347,\n",
       "  0.5476190538829183,\n",
       "  0.5812807922880051,\n",
       "  0.5878489358084542,\n",
       "  0.565681449298201,\n",
       "  0.5919540269034249,\n",
       "  0.5993431871160498,\n",
       "  0.5755336648724937,\n",
       "  0.6223316920801923,\n",
       "  0.5985221745345393,\n",
       "  0.6190476229625382,\n",
       "  0.6215106748007788,\n",
       "  0.6297208546417687,\n",
       "  0.645320197044335,\n",
       "  0.6379310368317102,\n",
       "  0.6724138001503028,\n",
       "  0.6486042755577952,\n",
       "  0.6444991821138729,\n",
       "  0.6871921205755525,\n",
       "  0.6995073915115131,\n",
       "  0.6896551747627446,\n",
       "  0.7044335022348488,\n",
       "  0.6945812831371289,\n",
       "  0.7044335022348488,\n",
       "  0.7126436844247902,\n",
       "  0.7183908100785881,\n",
       "  0.7068965587709924,\n",
       "  0.716748770821858,\n",
       "  0.7101806278886467,\n",
       "  0.7266009899195779,\n",
       "  0.7167487755197609,\n",
       "  0.7134647017042038,\n",
       "  0.7142857189836174,\n",
       "  0.7348111721095193,\n",
       "  0.7348111721095193,\n",
       "  0.73891626320449,\n",
       "  0.7216748815451937,\n",
       "  0.7241379380813373,\n",
       "  0.7241379380813373,\n",
       "  0.7298850613861836,\n",
       "  0.7126436820758387,\n",
       "  0.725779974989116,\n",
       "  0.7241379357323858,\n",
       "  0.7183908077296365,\n",
       "  0.7430213519505092,\n",
       "  0.7356321893889328,\n",
       "  0.7372742262967115,\n",
       "  0.717569790450223,\n",
       "  0.7389162608555385,\n",
       "  0.749589492534769,\n",
       "  0.7438423715788742,\n",
       "  0.7512315341404506,\n",
       "  0.7594417139814404,\n",
       "  0.7323481179223272,\n",
       "  0.748768477604307,\n",
       "  0.7676518938224304,\n",
       "  0.7479474603248935,\n",
       "  0.7643678223558248,\n",
       "  0.7577996794226134,\n",
       "  0.7454844084866529,\n",
       "  0.7520525514198642,\n",
       "  0.7422003346710957,\n",
       "  0.7487684799532585,\n",
       "  0.748768477604307,\n",
       "  0.7413793150427306,\n",
       "  0.747947457975942,\n",
       "  0.7602627312608541,\n",
       "  0.7389162585065869,\n",
       "  0.747947457975942,\n",
       "  0.7487684752553555,\n",
       "  0.7577996747247104,\n",
       "  0.7668308765430168,\n",
       "  0.749589497232672,\n",
       "  0.7422003346710957,\n",
       "  0.7602627312608541,\n",
       "  0.7619047658196811,\n",
       "  0.761083746191316,\n",
       "  0.7783251255016609,\n",
       "  0.7766830909428338,\n",
       "  0.7676518938224304,\n",
       "  0.7594417139814404,\n",
       "  0.7561576401658834,\n",
       "  0.7520525514198642,\n",
       "  0.7602627289119025,\n",
       "  0.7824302189455831,\n",
       "  0.7725780021968146,\n",
       "  0.7479474603248935,\n",
       "  0.7627257854480461,\n",
       "  0.7783251255016609,\n",
       "  0.779967164758391,\n",
       "  0.7914614207638896,\n",
       "  0.7783251255016609,\n",
       "  0.780788179688853,\n",
       "  0.779967164758391,\n",
       "  0.7750410563840068,\n",
       "  0.759441711632489,\n",
       "  0.7594417139814404,\n",
       "  0.7536945883276427,\n",
       "  0.759441711632489,\n",
       "  0.7676518938224304,\n",
       "  0.7692939283812574,\n",
       "  0.7717569825684496,\n",
       "  0.7725780021968146,\n",
       "  0.7824302189455831,\n",
       "  0.779967164758391,\n",
       "  0.7898193815071595,\n",
       "  0.7816092016661695,\n",
       "  0.7840722582023132,\n",
       "  0.7766830956407369,\n",
       "  0.7791461451300259,\n",
       "  0.7791461451300259,\n",
       "  0.7898193815071595,\n",
       "  0.7791461451300259,\n",
       "  0.7717569825684496,\n",
       "  0.769293926032306,\n",
       "  0.7881773422504293,\n",
       "  0.7865353076916023,\n",
       "  0.7947454898815437,\n",
       "  0.7766830909428338,\n",
       "  0.779967164758391,\n",
       "  0.7898193815071595,\n",
       "  0.7848932731327752,\n",
       "  0.7848932731327752,\n",
       "  0.7791461451300259,\n",
       "  0.778325130199564,\n",
       "  0.7742200391045932,\n",
       "  0.7807881820378045,\n",
       "  0.789819383856111,\n",
       "  0.7963875244403708,\n",
       "  0.7742200367556417,\n",
       "  0.779967164758391,\n",
       "  0.7668308765430168,\n",
       "  0.7561576401658834,\n",
       "  0.7766830956407369,\n",
       "  0.7717569872663526,\n",
       "  0.7750410563840068,\n",
       "  0.7865353076916023,\n",
       "  0.7758620736634203,\n",
       "  0.7914614160659865,\n",
       "  0.7922824356943515,\n",
       "  0.7742200414535447,\n",
       "  0.7922824356943515,\n",
       "  0.7906404011355245,\n",
       "  0.7709359676379876,\n",
       "  0.7610837485402676,\n",
       "  0.7840722511554586,\n",
       "  0.7775041082222474,\n",
       "  0.7775041082222474,\n",
       "  0.7610837485402676,\n",
       "  0.7528735663503262,\n",
       "  0.7848932707838236,\n",
       "  0.7742200367556417,\n",
       "  0.7840722535044101,\n",
       "  0.7865353076916023,\n",
       "  0.7750410563840068],\n",
       " 'val_VAL_acc_1': [15.270935960591133,\n",
       "  15.270935960591133,\n",
       "  16.748768472906406,\n",
       "  63.05418719211823,\n",
       "  60.59113300492611,\n",
       "  64.03940886699507,\n",
       "  64.03940886699507,\n",
       "  63.54679802955665,\n",
       "  58.62068965517241,\n",
       "  56.65024630541872,\n",
       "  59.60591133004926,\n",
       "  62.5615763546798,\n",
       "  61.083743842364534,\n",
       "  54.1871921182266,\n",
       "  51.724137931034484,\n",
       "  52.70935960591133,\n",
       "  55.66502463054187,\n",
       "  60.59113300492611,\n",
       "  65.02463054187191,\n",
       "  62.5615763546798,\n",
       "  67.48768472906404,\n",
       "  62.06896551724138,\n",
       "  62.5615763546798,\n",
       "  67.98029556650246,\n",
       "  70.44334975369458,\n",
       "  70.93596059113301,\n",
       "  72.41379310344827,\n",
       "  69.45812807881774,\n",
       "  67.98029556650246,\n",
       "  71.42857142857143,\n",
       "  73.39901477832512,\n",
       "  73.39901477832512,\n",
       "  72.9064039408867,\n",
       "  75.36945812807882,\n",
       "  72.9064039408867,\n",
       "  70.93596059113301,\n",
       "  71.92118226600985,\n",
       "  68.47290640394088,\n",
       "  73.39901477832512,\n",
       "  76.35467980295566,\n",
       "  73.39901477832512,\n",
       "  73.39901477832512,\n",
       "  72.41379310344827,\n",
       "  72.9064039408867,\n",
       "  78.81773399014779,\n",
       "  76.84729064039409,\n",
       "  78.32512315270937,\n",
       "  77.33990147783251,\n",
       "  80.78817733990148,\n",
       "  77.83251231527093,\n",
       "  77.83251231527093,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  78.81773399014779,\n",
       "  76.35467980295566,\n",
       "  78.81773399014779,\n",
       "  80.78817733990148,\n",
       "  77.33990147783251,\n",
       "  80.78817733990148,\n",
       "  82.75862068965517,\n",
       "  81.77339901477832,\n",
       "  78.32512315270937,\n",
       "  77.33990147783251,\n",
       "  80.29556650246306,\n",
       "  76.35467980295566,\n",
       "  80.29556650246306,\n",
       "  83.74384236453201,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  78.81773399014779,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  78.81773399014779,\n",
       "  79.3103448275862,\n",
       "  79.80295566502463,\n",
       "  78.81773399014779,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  79.3103448275862,\n",
       "  81.2807881773399,\n",
       "  79.3103448275862,\n",
       "  79.3103448275862,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  82.26600985221675,\n",
       "  80.78817733990148,\n",
       "  80.78817733990148,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  82.26600985221675,\n",
       "  78.81773399014779,\n",
       "  79.80295566502463,\n",
       "  77.83251231527093,\n",
       "  79.80295566502463,\n",
       "  77.83251231527093,\n",
       "  78.32512315270937,\n",
       "  81.2807881773399,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  82.26600985221675,\n",
       "  79.80295566502463,\n",
       "  79.3103448275862,\n",
       "  79.3103448275862,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  81.2807881773399,\n",
       "  79.80295566502463,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  80.78817733990148,\n",
       "  82.26600985221675,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  82.26600985221675,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  80.29556650246306,\n",
       "  81.77339901477832,\n",
       "  80.29556650246306,\n",
       "  81.77339901477832,\n",
       "  79.80295566502463,\n",
       "  82.75862068965517,\n",
       "  81.77339901477832,\n",
       "  81.77339901477832,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  81.77339901477832,\n",
       "  82.26600985221675,\n",
       "  80.78817733990148,\n",
       "  77.83251231527093,\n",
       "  79.3103448275862,\n",
       "  82.26600985221675,\n",
       "  81.2807881773399,\n",
       "  79.80295566502463,\n",
       "  81.2807881773399],\n",
       " 'test_loss': -1,\n",
       " 'test_ATT_loss': -1,\n",
       " 'test_VAL_loss': -1,\n",
       " 'test_ATT_acc': -1,\n",
       " 'test_VAL_acc': -1,\n",
       " 'test_VAL_jac': -1,\n",
       " 'test_VAL_acc_1': -1,\n",
       " 'model_filename': 'model_storage/SAGE/model.pth'}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SAGE(in_channels=data.x.shape[-1], hidden_channels = 512, \n",
    "            out_channels = data.y.shape[-1], dropout = 0.1, num_layers=5).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load(args.save_dir+'SAGE_best_model/model.pth',map_location=torch.device('cpu')))\n",
    "model.eval()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SAGE(\n",
       "  (convs): ModuleList(\n",
       "    (0): SAGEConv(1753, 512)\n",
       "    (1): SAGEConv(512, 512)\n",
       "    (2): SAGEConv(512, 512)\n",
       "    (3): SAGEConv(512, 512)\n",
       "    (4): SAGEConv(512, 20)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 12.95it/s]\n"
     ]
    }
   ],
   "source": [
    "test_loss_att, test_loss_val, test_att_acc, test_val_acc, test_val_jac, test_val_1 = test_Homo(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 13.60it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.727577265444885,\n",
       " 1.6139437128632352,\n",
       " 100.0,\n",
       " 99.44598337950139,\n",
       " 0.8111726755580744,\n",
       " 83.93351800554017)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 14.42it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7921790241952834,\n",
       " 1.627505142700496,\n",
       " 96.13821138211382,\n",
       " 100.0,\n",
       " 0.779967164758391,\n",
       " 79.80295566502463)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 13.37it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7921501777281107,\n",
       " 1.6524048553158839,\n",
       " 97.01789264413519,\n",
       " 99.47916666666667,\n",
       " 0.7413194502393404,\n",
       " 80.20833333333333)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 13.57it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.25it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.81it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.00it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.83it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.32it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.96it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.39it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.96it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.17it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.82it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.36it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.75it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.27it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.88it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.22it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 17.00it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 15.81it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.75it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.30it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test_Homo(model, val_loader))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test_Homo(model, test_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.792034</td>\n",
       "      <td>1.627620</td>\n",
       "      <td>95.711382</td>\n",
       "      <td>99.950739</td>\n",
       "      <td>0.771675</td>\n",
       "      <td>80.689655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000524</td>\n",
       "      <td>0.000174</td>\n",
       "      <td>0.223680</td>\n",
       "      <td>0.155777</td>\n",
       "      <td>0.003974</td>\n",
       "      <td>0.763149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.791345</td>\n",
       "      <td>1.627426</td>\n",
       "      <td>95.325203</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>0.766831</td>\n",
       "      <td>79.310345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.791626</td>\n",
       "      <td>1.627471</td>\n",
       "      <td>95.528455</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.768678</td>\n",
       "      <td>80.295567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.792003</td>\n",
       "      <td>1.627631</td>\n",
       "      <td>95.731707</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.770525</td>\n",
       "      <td>80.788177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.792291</td>\n",
       "      <td>1.627699</td>\n",
       "      <td>95.934959</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.774220</td>\n",
       "      <td>81.280788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.793165</td>\n",
       "      <td>1.627940</td>\n",
       "      <td>95.934959</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.777504</td>\n",
       "      <td>81.773399</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc   VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000   10.000000  10.000000  10.000000\n",
       "mean    0.792034   1.627620  95.711382   99.950739   0.771675  80.689655\n",
       "std     0.000524   0.000174   0.223680    0.155777   0.003974   0.763149\n",
       "min     0.791345   1.627426  95.325203   99.507389   0.766831  79.310345\n",
       "25%     0.791626   1.627471  95.528455  100.000000   0.768678  80.295567\n",
       "50%     0.792003   1.627631  95.731707  100.000000   0.770525  80.788177\n",
       "75%     0.792291   1.627699  95.934959  100.000000   0.774220  81.280788\n",
       "max     0.793165   1.627940  95.934959  100.000000   0.777504  81.773399"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.792685</td>\n",
       "      <td>1.652345</td>\n",
       "      <td>97.077535</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.754774</td>\n",
       "      <td>79.062500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000627</td>\n",
       "      <td>0.000295</td>\n",
       "      <td>0.230517</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.005164</td>\n",
       "      <td>0.685709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.791774</td>\n",
       "      <td>1.651828</td>\n",
       "      <td>96.620278</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.744792</td>\n",
       "      <td>78.125000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.792228</td>\n",
       "      <td>1.652276</td>\n",
       "      <td>97.017893</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.752170</td>\n",
       "      <td>78.645833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.792665</td>\n",
       "      <td>1.652408</td>\n",
       "      <td>97.117296</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.754340</td>\n",
       "      <td>79.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.793224</td>\n",
       "      <td>1.652524</td>\n",
       "      <td>97.216700</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.758247</td>\n",
       "      <td>79.557292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.793595</td>\n",
       "      <td>1.652687</td>\n",
       "      <td>97.415507</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.762153</td>\n",
       "      <td>80.208333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc     VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  1.000000e+01  10.000000  10.000000\n",
       "mean    0.792685   1.652345  97.077535  9.947917e+01   0.754774  79.062500\n",
       "std     0.000627   0.000295   0.230517  1.497956e-14   0.005164   0.685709\n",
       "min     0.791774   1.651828  96.620278  9.947917e+01   0.744792  78.125000\n",
       "25%     0.792228   1.652276  97.017893  9.947917e+01   0.752170  78.645833\n",
       "50%     0.792665   1.652408  97.117296  9.947917e+01   0.754340  79.166667\n",
       "75%     0.793224   1.652524  97.216700  9.947917e+01   0.758247  79.557292\n",
       "max     0.793595   1.652687  97.415507  9.947917e+01   0.762153  80.208333"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'val_metrics.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'test_metrics.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_state['test_ATT_loss']=test_loss_att\n",
    "train_state['test_VAL_loss']=test_loss_val\n",
    "train_state['test_loss']=test_loss_att + 3*test_loss_val\n",
    "train_state['test_ATT_acc']=test_att_acc\n",
    "train_state['test_VAL_acc_1']=test_val_1\n",
    "train_state['test_VAL_acc']=test_val_acc\n",
    "train_state['test_VAL_jac']=test_val_jac"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': True,\n",
       " 'early_stopping_step': 30,\n",
       " 'early_stopping_best_ATT_acc_val': 96.7479674796748,\n",
       " 'early_stopping_best_VAL_acc_val': 100.0,\n",
       " 'early_stopping_best_ATT_acc_val_2': 0,\n",
       " 'early_stopping_lowest_loss': 5.674814448439726,\n",
       " 'learning_rate': 0.001,\n",
       " 'epoch_index': 169,\n",
       " 'train_loss': [4.098931749661763,\n",
       "  3.767206331094106,\n",
       "  3.7242619395256042,\n",
       "  3.700642983118693,\n",
       "  3.668950080871582,\n",
       "  3.6377055446306863,\n",
       "  3.6117473045984902,\n",
       "  3.5522724787394204,\n",
       "  3.4761929710706077,\n",
       "  3.3616212805112204,\n",
       "  3.23332671324412,\n",
       "  3.1092602411905923,\n",
       "  3.028814951578776,\n",
       "  2.898661951224009,\n",
       "  2.82891054948171,\n",
       "  2.812813639640808,\n",
       "  2.7504736383756003,\n",
       "  2.733060439427694,\n",
       "  2.6882276137669883,\n",
       "  2.664868414402008,\n",
       "  2.642411748568217,\n",
       "  2.6827587683995566,\n",
       "  2.6409761110941568,\n",
       "  2.5932233532269797,\n",
       "  2.5788277784983316,\n",
       "  2.548888703187307,\n",
       "  2.53569761912028,\n",
       "  2.5417450070381165,\n",
       "  2.529538651307424,\n",
       "  2.5021569530169168,\n",
       "  2.4645569125811257,\n",
       "  2.4980392257372537,\n",
       "  2.501281420389811,\n",
       "  2.4597434798876443,\n",
       "  2.485634922981262,\n",
       "  2.466936190923055,\n",
       "  2.467691481113434,\n",
       "  2.4658232728640237,\n",
       "  2.4621368249257407,\n",
       "  2.464854379494985,\n",
       "  2.4529733459154763,\n",
       "  2.4534698923428855,\n",
       "  2.436204969882965,\n",
       "  2.4329582850138345,\n",
       "  2.450833817323049,\n",
       "  2.421373744805654,\n",
       "  2.434440871079763,\n",
       "  2.450161556402842,\n",
       "  2.427618066469828,\n",
       "  2.4393926660219827,\n",
       "  2.4163631796836853,\n",
       "  2.4301605820655823,\n",
       "  2.435442308584849,\n",
       "  2.4121821324030557,\n",
       "  2.4129402240117392,\n",
       "  2.4047138492266336,\n",
       "  2.4033178885777793,\n",
       "  2.41557643810908,\n",
       "  2.3933849732081094,\n",
       "  2.3944159746170044,\n",
       "  2.392829736073812,\n",
       "  2.3958428303400674,\n",
       "  2.404998560746511,\n",
       "  2.3976862827936807,\n",
       "  2.417176087697347,\n",
       "  2.3936072985331216,\n",
       "  2.408930778503418,\n",
       "  2.3829866647720337,\n",
       "  2.393950899442037,\n",
       "  2.4159895380338035,\n",
       "  2.4185696244239807,\n",
       "  2.399583617846171,\n",
       "  2.3866588274637857,\n",
       "  2.3977599342664084,\n",
       "  2.385384738445282,\n",
       "  2.385758380095164,\n",
       "  2.3827030460039773,\n",
       "  2.3948446114857993,\n",
       "  2.3857273856798806,\n",
       "  2.4019908706347146,\n",
       "  2.392253816127777,\n",
       "  2.3864142696062722,\n",
       "  2.366167684396108,\n",
       "  2.3878723780314126,\n",
       "  2.374208470185598,\n",
       "  2.3906711538632712,\n",
       "  2.3854604959487915,\n",
       "  2.3820138772328696,\n",
       "  2.3843430280685425,\n",
       "  2.377971331278483,\n",
       "  2.3856988151868186,\n",
       "  2.3722445170084634,\n",
       "  2.38117923339208,\n",
       "  2.3817812403043113,\n",
       "  2.3885410825411477,\n",
       "  2.3754957914352417,\n",
       "  2.383930762608846,\n",
       "  2.3800458908081055,\n",
       "  2.391184707482656,\n",
       "  2.3534916838010154,\n",
       "  2.378878355026245,\n",
       "  2.3758183320363364,\n",
       "  2.3601855039596558,\n",
       "  2.3570021788279214,\n",
       "  2.371889809767405,\n",
       "  2.374414642651876,\n",
       "  2.373858153820038,\n",
       "  2.355741282304128,\n",
       "  2.366458535194397,\n",
       "  2.3738173047701516,\n",
       "  2.35603266954422,\n",
       "  2.353848874568939,\n",
       "  2.378168066342672,\n",
       "  2.3655735850334167,\n",
       "  2.3658991058667502,\n",
       "  2.355059822400411,\n",
       "  2.370965520540873,\n",
       "  2.368260939915975,\n",
       "  2.3701597849527993,\n",
       "  2.348038117090861,\n",
       "  2.372505327065786,\n",
       "  2.371385157108307,\n",
       "  2.3717514077822366,\n",
       "  2.363755484422048,\n",
       "  2.3688772519429526,\n",
       "  2.3616636395454407,\n",
       "  2.3614793618520102,\n",
       "  2.3517807920773826,\n",
       "  2.3396608233451843,\n",
       "  2.346040507157644,\n",
       "  2.35396538178126,\n",
       "  2.3598483403523765,\n",
       "  2.3532610734303794,\n",
       "  2.370964745680491,\n",
       "  2.3649390737215676,\n",
       "  2.3489114244778952,\n",
       "  2.3580377101898193,\n",
       "  2.3587553103764853,\n",
       "  2.3682141304016113,\n",
       "  2.366213579972585,\n",
       "  2.3549970189730325,\n",
       "  2.364099085330963,\n",
       "  2.3584629893302917,\n",
       "  2.360451857248942,\n",
       "  2.3638933499654136,\n",
       "  2.36885529756546,\n",
       "  2.374499042828878,\n",
       "  2.355612337589264,\n",
       "  2.359101116657257,\n",
       "  2.346854587395986,\n",
       "  2.3477829893430076,\n",
       "  2.366513510545095,\n",
       "  2.3481218417485556,\n",
       "  2.342003047466278,\n",
       "  2.350774347782135,\n",
       "  2.369879643122355,\n",
       "  2.365553597609202,\n",
       "  2.356052656968435,\n",
       "  2.3574883937835693,\n",
       "  2.355355203151703,\n",
       "  2.3598441084225974,\n",
       "  2.3543240427970886,\n",
       "  2.3405757745107016,\n",
       "  2.3648985425631204,\n",
       "  2.346867481867472,\n",
       "  2.365351438522339,\n",
       "  2.3487407565116882,\n",
       "  2.3562622467676797,\n",
       "  2.3468205531438193,\n",
       "  2.352146029472351],\n",
       " 'train_ATT_loss': [1.9546462502175752,\n",
       "  1.933227481604283,\n",
       "  1.9044952280303449,\n",
       "  1.8954763026118608,\n",
       "  1.8790527037306175,\n",
       "  1.85460715742983,\n",
       "  1.818479740058286,\n",
       "  1.7553564171381604,\n",
       "  1.6525168686362184,\n",
       "  1.5284845360098123,\n",
       "  1.3960980295804728,\n",
       "  1.2829446231229154,\n",
       "  1.1823519247060337,\n",
       "  1.100580136531608,\n",
       "  1.0436663112481877,\n",
       "  1.0261845717469742,\n",
       "  0.989991723005131,\n",
       "  0.9566390336055175,\n",
       "  0.94409316761672,\n",
       "  0.9302744825791124,\n",
       "  0.9226926218439667,\n",
       "  0.8990904911733401,\n",
       "  0.8791366349957326,\n",
       "  0.8851204942137911,\n",
       "  0.8496665183526988,\n",
       "  0.8268885950962923,\n",
       "  0.818720144082965,\n",
       "  0.8138482732455816,\n",
       "  0.80768588969582,\n",
       "  0.8035663227625501,\n",
       "  0.7936245870061859,\n",
       "  0.7901797676020382,\n",
       "  0.782620763349401,\n",
       "  0.7779420784305667,\n",
       "  0.7768213484756173,\n",
       "  0.7768766764458527,\n",
       "  0.7762275073997201,\n",
       "  0.7695193786039907,\n",
       "  0.7650198105960011,\n",
       "  0.763135122955671,\n",
       "  0.7609167265759941,\n",
       "  0.7586613754816663,\n",
       "  0.7562868400293704,\n",
       "  0.7554397617680875,\n",
       "  0.7554485241461989,\n",
       "  0.7529076639965301,\n",
       "  0.752861852460951,\n",
       "  0.7502899803943581,\n",
       "  0.750594558643172,\n",
       "  0.7475648231783733,\n",
       "  0.7478374434309983,\n",
       "  0.7448096587387149,\n",
       "  0.7439154650696097,\n",
       "  0.7456373215712339,\n",
       "  0.7428600963132863,\n",
       "  0.7422313085883608,\n",
       "  0.7449038992958386,\n",
       "  0.7427807301695657,\n",
       "  0.7407917812920674,\n",
       "  0.7399915585557509,\n",
       "  0.7388355208565984,\n",
       "  0.7381309044658312,\n",
       "  0.7380269539653429,\n",
       "  0.7392500615846417,\n",
       "  0.7391345398577956,\n",
       "  0.7372666285308774,\n",
       "  0.7363568491552676,\n",
       "  0.7358763165090884,\n",
       "  0.7341074466375103,\n",
       "  0.7390065029717549,\n",
       "  0.7348545637817594,\n",
       "  0.7341795948403694,\n",
       "  0.7336663625246931,\n",
       "  0.733937031839693,\n",
       "  0.7327201891473786,\n",
       "  0.7350096808245968,\n",
       "  0.7325471038633437,\n",
       "  0.7331557721312356,\n",
       "  0.7311574829912582,\n",
       "  0.7322029137875565,\n",
       "  0.7318318203875893,\n",
       "  0.7317600431864942,\n",
       "  0.7303660582967743,\n",
       "  0.7309218465126122,\n",
       "  0.7309670567182293,\n",
       "  0.7314814781548243,\n",
       "  0.7313714885975846,\n",
       "  0.7302612733312591,\n",
       "  0.7300168203184809,\n",
       "  0.7307250534398404,\n",
       "  0.7288559303719582,\n",
       "  0.7289511916379849,\n",
       "  0.7302039173171131,\n",
       "  0.7286603789250277,\n",
       "  0.7286888103405855,\n",
       "  0.7296452512371243,\n",
       "  0.7283628989124562,\n",
       "  0.7279192775570454,\n",
       "  0.7280225621696325,\n",
       "  0.728538278067211,\n",
       "  0.7275784352479548,\n",
       "  0.7278532022584508,\n",
       "  0.7293509553674186,\n",
       "  0.7294930807441226,\n",
       "  0.7271956263156478,\n",
       "  0.7281060940340945,\n",
       "  0.7268856430317887,\n",
       "  0.7270274472698941,\n",
       "  0.7263259800187108,\n",
       "  0.7269815693601677,\n",
       "  0.7270486617352494,\n",
       "  0.726385351858641,\n",
       "  0.7270347632859883,\n",
       "  0.7291333668780129,\n",
       "  0.7286997430872719,\n",
       "  0.7278061978044272,\n",
       "  0.7272645996217912,\n",
       "  0.7276377057104559,\n",
       "  0.7290878063093592,\n",
       "  0.7263797574426328,\n",
       "  0.7271127897286349,\n",
       "  0.7299188717250349,\n",
       "  0.7265173465921608,\n",
       "  0.7269514711609838,\n",
       "  0.7265304153645799,\n",
       "  0.7260907123954012,\n",
       "  0.7263158790952942,\n",
       "  0.7288298661358799,\n",
       "  0.7258909574175806,\n",
       "  0.7263430748289642,\n",
       "  0.7254435122838642,\n",
       "  0.7253453419479307,\n",
       "  0.7265390776861408,\n",
       "  0.7250594678347791,\n",
       "  0.724814727861135,\n",
       "  0.7252569731913114,\n",
       "  0.7248612411134461,\n",
       "  0.7250717183229336,\n",
       "  0.7271107253605639,\n",
       "  0.7270112359622839,\n",
       "  0.7266698002485027,\n",
       "  0.7254077214283295,\n",
       "  0.7294890723730388,\n",
       "  0.7264151256170299,\n",
       "  0.7269143237631737,\n",
       "  0.7249167837264465,\n",
       "  0.7256764726625585,\n",
       "  0.7281667552166038,\n",
       "  0.7262458584975668,\n",
       "  0.7250483608972332,\n",
       "  0.7260188921006433,\n",
       "  0.725769547379248,\n",
       "  0.7246643917382258,\n",
       "  0.7260264001394573,\n",
       "  0.7251703623589386,\n",
       "  0.7241784510850245,\n",
       "  0.7250886163222823,\n",
       "  0.7258201314141546,\n",
       "  0.7266337138463916,\n",
       "  0.725004944774913,\n",
       "  0.7258336479644035,\n",
       "  0.7251031674506592,\n",
       "  0.7245107887524317,\n",
       "  0.7244798600508565,\n",
       "  0.7243442258015894,\n",
       "  0.7258242599851867,\n",
       "  0.7253383705160295,\n",
       "  0.7245499924279316,\n",
       "  0.7239685027222884,\n",
       "  0.7250117207167882],\n",
       " 'train_VAL_loss': [1.8116718421682427,\n",
       "  1.8048931300805215,\n",
       "  1.794408496066804,\n",
       "  1.786028530789214,\n",
       "  1.7744940225437407,\n",
       "  1.758524922452805,\n",
       "  1.7416969118356045,\n",
       "  1.727210153503101,\n",
       "  1.724068412820388,\n",
       "  1.7271408925756524,\n",
       "  1.7252861050027228,\n",
       "  1.713301249488239,\n",
       "  1.705337117253248,\n",
       "  1.7061472602828387,\n",
       "  1.697877908347386,\n",
       "  1.6915473419543448,\n",
       "  1.69230836217093,\n",
       "  1.7007082212004305,\n",
       "  1.696528370690808,\n",
       "  1.6880398939190808,\n",
       "  1.6791018683494292,\n",
       "  1.685056095308214,\n",
       "  1.6835761819850044,\n",
       "  1.6691038747243274,\n",
       "  1.669677336790555,\n",
       "  1.6698398844357012,\n",
       "  1.6625222598416654,\n",
       "  1.6605119711804588,\n",
       "  1.6664237176942693,\n",
       "  1.6633091071966282,\n",
       "  1.6588391722734614,\n",
       "  1.6613311259039882,\n",
       "  1.6605271272712137,\n",
       "  1.6578142276431054,\n",
       "  1.652783023023209,\n",
       "  1.648089217677341,\n",
       "  1.649600606876067,\n",
       "  1.652168225713714,\n",
       "  1.6467887883041044,\n",
       "  1.6439999384893276,\n",
       "  1.6433866608506094,\n",
       "  1.6438393582927884,\n",
       "  1.6440409686096487,\n",
       "  1.64152337474506,\n",
       "  1.639261954048664,\n",
       "  1.639005054397266,\n",
       "  1.6384596933618476,\n",
       "  1.6385322138873495,\n",
       "  1.6366317688263023,\n",
       "  1.636057458425823,\n",
       "  1.6367604597123375,\n",
       "  1.6341615106260348,\n",
       "  1.6331871790899135,\n",
       "  1.634273741383962,\n",
       "  1.632514024375218,\n",
       "  1.6327536310845796,\n",
       "  1.6344124164607716,\n",
       "  1.6318661776936285,\n",
       "  1.632599066168978,\n",
       "  1.6313219575670617,\n",
       "  1.6293366602583275,\n",
       "  1.6280674125349093,\n",
       "  1.629100662519397,\n",
       "  1.6314236354959968,\n",
       "  1.6289402688969536,\n",
       "  1.6282597724090322,\n",
       "  1.6260940886600526,\n",
       "  1.624837441127386,\n",
       "  1.6260579629618046,\n",
       "  1.6277662218442583,\n",
       "  1.6255714123929306,\n",
       "  1.625792026849995,\n",
       "  1.6256230904455,\n",
       "  1.6286932431429708,\n",
       "  1.626376760963588,\n",
       "  1.6250780365143456,\n",
       "  1.6216324405987177,\n",
       "  1.62110808813671,\n",
       "  1.6212841420952964,\n",
       "  1.6205288815696484,\n",
       "  1.6216340841018593,\n",
       "  1.6186893306610657,\n",
       "  1.6181581053377188,\n",
       "  1.6190993138627663,\n",
       "  1.618933295609218,\n",
       "  1.6182240432649437,\n",
       "  1.6181097654754766,\n",
       "  1.6186893768918151,\n",
       "  1.618291619411796,\n",
       "  1.6188235973056995,\n",
       "  1.616722936445326,\n",
       "  1.6163123958328753,\n",
       "  1.6157647493473382,\n",
       "  1.6156388481568102,\n",
       "  1.614626459797994,\n",
       "  1.615075167196279,\n",
       "  1.6146018359799794,\n",
       "  1.6151762153963634,\n",
       "  1.6147568585800003,\n",
       "  1.6149412829459868,\n",
       "  1.6139556152998906,\n",
       "  1.6133016697917948,\n",
       "  1.614078189527559,\n",
       "  1.6134013697050946,\n",
       "  1.6126826955340905,\n",
       "  1.6121505568232233,\n",
       "  1.6124903558033654,\n",
       "  1.612349418722031,\n",
       "  1.612311630037683,\n",
       "  1.6116237234210704,\n",
       "  1.6118275699853237,\n",
       "  1.6128621880697742,\n",
       "  1.6115445509511679,\n",
       "  1.6126363792577938,\n",
       "  1.6120649602604704,\n",
       "  1.6133830227019714,\n",
       "  1.6129557343401077,\n",
       "  1.6105488178802658,\n",
       "  1.6111674008276984,\n",
       "  1.611330396911114,\n",
       "  1.6110662700727045,\n",
       "  1.6109023322028797,\n",
       "  1.6110514072169888,\n",
       "  1.6118645625101231,\n",
       "  1.609554374317053,\n",
       "  1.6085463471690042,\n",
       "  1.6081257919855725,\n",
       "  1.608558438491293,\n",
       "  1.6093426500331,\n",
       "  1.6087106420062585,\n",
       "  1.6081184128975274,\n",
       "  1.607255687343777,\n",
       "  1.6075764696353692,\n",
       "  1.6085331529131226,\n",
       "  1.60770972448703,\n",
       "  1.6077978924701088,\n",
       "  1.6075854278337263,\n",
       "  1.6074802145733398,\n",
       "  1.6089027954931074,\n",
       "  1.609567384640596,\n",
       "  1.6088831431317527,\n",
       "  1.6086879460104946,\n",
       "  1.6095650549410452,\n",
       "  1.6086326404951947,\n",
       "  1.6079769936955206,\n",
       "  1.6063219606050825,\n",
       "  1.6067882977694357,\n",
       "  1.608143498032377,\n",
       "  1.60669404053622,\n",
       "  1.6070644353565418,\n",
       "  1.6059624699674484,\n",
       "  1.6051409228026372,\n",
       "  1.6052963855854363,\n",
       "  1.6052211648539494,\n",
       "  1.605403248953357,\n",
       "  1.6048400610107465,\n",
       "  1.6042452608119087,\n",
       "  1.6046841309671587,\n",
       "  1.6048155808382747,\n",
       "  1.6058779419954463,\n",
       "  1.6048747611508145,\n",
       "  1.605228927987434,\n",
       "  1.6058536678800293,\n",
       "  1.6062448870112032,\n",
       "  1.6061633785675768,\n",
       "  1.60516725682816,\n",
       "  1.6044996108374767,\n",
       "  1.604426426900721,\n",
       "  1.605478557193048,\n",
       "  1.6040322097054478],\n",
       " 'train_ATT_acc': [27.977839335180054,\n",
       "  26.31578947368421,\n",
       "  27.977839335180054,\n",
       "  35.18005540166205,\n",
       "  34.903047091412745,\n",
       "  34.903047091412745,\n",
       "  36.28808864265928,\n",
       "  43.21329639889197,\n",
       "  47.36842105263158,\n",
       "  56.232686980609415,\n",
       "  65.92797783933518,\n",
       "  68.97506925207756,\n",
       "  78.11634349030471,\n",
       "  81.7174515235457,\n",
       "  82.54847645429363,\n",
       "  83.65650969529086,\n",
       "  84.21052631578948,\n",
       "  86.98060941828255,\n",
       "  86.14958448753463,\n",
       "  86.98060941828255,\n",
       "  90.02770083102493,\n",
       "  90.85872576177286,\n",
       "  91.13573407202216,\n",
       "  90.58171745152355,\n",
       "  93.07479224376732,\n",
       "  96.67590027700831,\n",
       "  97.50692520775624,\n",
       "  98.06094182825485,\n",
       "  98.33795013850416,\n",
       "  98.33795013850416,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  99.16897506925208,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  99.16897506925208,\n",
       "  99.44598337950139,\n",
       "  99.16897506925208,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0],\n",
       " 'train_VAL_acc': [68.42105263157895,\n",
       "  68.42105263157895,\n",
       "  72.29916897506925,\n",
       "  84.7645429362881,\n",
       "  83.10249307479225,\n",
       "  83.65650969529086,\n",
       "  86.14958448753463,\n",
       "  84.7645429362881,\n",
       "  82.27146814404432,\n",
       "  84.7645429362881,\n",
       "  84.21052631578948,\n",
       "  84.7645429362881,\n",
       "  86.70360110803324,\n",
       "  86.98060941828255,\n",
       "  90.30470914127424,\n",
       "  91.13573407202216,\n",
       "  86.98060941828255,\n",
       "  89.47368421052632,\n",
       "  91.96675900277009,\n",
       "  90.85872576177286,\n",
       "  91.68975069252078,\n",
       "  91.13573407202216,\n",
       "  91.96675900277009,\n",
       "  94.18282548476455,\n",
       "  94.18282548476455,\n",
       "  93.07479224376732,\n",
       "  96.67590027700831,\n",
       "  95.8448753462604,\n",
       "  95.56786703601108,\n",
       "  95.8448753462604,\n",
       "  96.1218836565097,\n",
       "  96.1218836565097,\n",
       "  96.39889196675901,\n",
       "  97.50692520775624,\n",
       "  96.67590027700831,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  99.44598337950139,\n",
       "  98.89196675900277,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  98.61495844875347,\n",
       "  99.16897506925208,\n",
       "  99.44598337950139,\n",
       "  98.89196675900277,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  98.89196675900277,\n",
       "  99.44598337950139,\n",
       "  99.16897506925208,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.16897506925208,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0],\n",
       " 'train_VAL_jac': [0.0,\n",
       "  0.0,\n",
       "  0.0,\n",
       "  0.0,\n",
       "  0.0,\n",
       "  0.10110803324099724,\n",
       "  0.2520775623268698,\n",
       "  0.3254847645429363,\n",
       "  0.37026777558049334,\n",
       "  0.3905817200932807,\n",
       "  0.44967683860799945,\n",
       "  0.44459834851716695,\n",
       "  0.4445983471962884,\n",
       "  0.44367498043831693,\n",
       "  0.46121884449036826,\n",
       "  0.48753463924756674,\n",
       "  0.5383195005295349,\n",
       "  0.5364727709762277,\n",
       "  0.5161588238216833,\n",
       "  0.5180055454497192,\n",
       "  0.5420129358603353,\n",
       "  0.5466297498370142,\n",
       "  0.5521699213255146,\n",
       "  0.5923361104611214,\n",
       "  0.5983379633472897,\n",
       "  0.6006463729773862,\n",
       "  0.641735927550086,\n",
       "  0.6398892072429287,\n",
       "  0.6246537501792168,\n",
       "  0.6394275232035037,\n",
       "  0.6551246643066406,\n",
       "  0.6463527309597364,\n",
       "  0.6417359328336002,\n",
       "  0.6523545864876618,\n",
       "  0.6606648252281125,\n",
       "  0.6777470131660102,\n",
       "  0.654201306794819,\n",
       "  0.6278855173211348,\n",
       "  0.6722068416775099,\n",
       "  0.6948291905368794,\n",
       "  0.7003693567418656,\n",
       "  0.680517090984989,\n",
       "  0.6680517223072845,\n",
       "  0.6754386035359137,\n",
       "  0.7063712017027625,\n",
       "  0.6948291852533652,\n",
       "  0.7036011186002694,\n",
       "  0.7049861601515159,\n",
       "  0.7211449800105636,\n",
       "  0.7197599384593171,\n",
       "  0.7119113679077487,\n",
       "  0.7216066534829602,\n",
       "  0.7220683428058994,\n",
       "  0.7340720327276933,\n",
       "  0.7183748916245564,\n",
       "  0.7192982597034063,\n",
       "  0.7174515341127348,\n",
       "  0.7336103486882682,\n",
       "  0.7036011133167552,\n",
       "  0.7469990801613087,\n",
       "  0.7354570689954256,\n",
       "  0.7525392463662948,\n",
       "  0.725300099380789,\n",
       "  0.7183748863410422,\n",
       "  0.734533711483604,\n",
       "  0.7562326975476379,\n",
       "  0.7465374119724263,\n",
       "  0.7590027806501309,\n",
       "  0.749307484507891,\n",
       "  0.740535556444501,\n",
       "  0.7585410913271917,\n",
       "  0.7603878274848916,\n",
       "  0.7659279884063637,\n",
       "  0.744229007625844,\n",
       "  0.7488458110354944,\n",
       "  0.7497691685473159,\n",
       "  0.7710064705719247,\n",
       "  0.7710064652884105,\n",
       "  0.7668513459181852,\n",
       "  0.766851340634671,\n",
       "  0.760849495673774,\n",
       "  0.7723915121231714,\n",
       "  0.7839335285725686,\n",
       "  0.7746999164697537,\n",
       "  0.7571560550594594,\n",
       "  0.7710064652884105,\n",
       "  0.7742382377138428,\n",
       "  0.7682363874694317,\n",
       "  0.7710064652884105,\n",
       "  0.7802400826747398,\n",
       "  0.8074792349437597,\n",
       "  0.7950138556990267,\n",
       "  0.8084025924555813,\n",
       "  0.8033240997229917,\n",
       "  0.8120960330698959,\n",
       "  0.8028624209670809,\n",
       "  0.8088642712114921,\n",
       "  0.7816251189424721,\n",
       "  0.7987073015968555,\n",
       "  0.7963988972502732,\n",
       "  0.8074792296602455,\n",
       "  0.8213296451727109,\n",
       "  0.8010157006599236,\n",
       "  0.808402587172067,\n",
       "  0.8139427533770532,\n",
       "  0.8190212408261286,\n",
       "  0.8070175509043348,\n",
       "  0.8042474678018416,\n",
       "  0.8097876340068279,\n",
       "  0.8116343490304709,\n",
       "  0.8107109915186493,\n",
       "  0.7899353682499513,\n",
       "  0.8070175509043348,\n",
       "  0.7987073015968555,\n",
       "  0.8051708305971774,\n",
       "  0.8042474678018416,\n",
       "  0.8060941933925132,\n",
       "  0.8139427586605674,\n",
       "  0.8125577223928351,\n",
       "  0.8130193958652316,\n",
       "  0.8287165422518827,\n",
       "  0.8162511630071497,\n",
       "  0.8171745258024855,\n",
       "  0.8093259605344313,\n",
       "  0.8264081379053005,\n",
       "  0.8379501385041551,\n",
       "  0.8347183819292655,\n",
       "  0.8254847698264505,\n",
       "  0.832871656338594,\n",
       "  0.824099728275204,\n",
       "  0.8250230910705397,\n",
       "  0.8370267809923336,\n",
       "  0.8296398944801902,\n",
       "  0.8254847698264505,\n",
       "  0.8305632572755259,\n",
       "  0.8319482935432582,\n",
       "  0.8070175509043348,\n",
       "  0.8264081326217863,\n",
       "  0.8319482988267725,\n",
       "  0.8287165316848544,\n",
       "  0.824561412314629,\n",
       "  0.8287165369683686,\n",
       "  0.8153278054953282,\n",
       "  0.8250230963540539,\n",
       "  0.832871656338594,\n",
       "  0.8421052631578947,\n",
       "  0.8314866147873474,\n",
       "  0.8176361992748821,\n",
       "  0.8162511577236355,\n",
       "  0.8217913239286216,\n",
       "  0.8402585481342516,\n",
       "  0.8397968693783409,\n",
       "  0.8374884597482444,\n",
       "  0.8393351853389159,\n",
       "  0.8494921549535525,\n",
       "  0.8471837558904843,\n",
       "  0.8490304761976416,\n",
       "  0.8494921549535525,\n",
       "  0.8536472848908062,\n",
       "  0.844875351543902,\n",
       "  0.8471837506069702,\n",
       "  0.8457987143392378,\n",
       "  0.8365651022364228,\n",
       "  0.823638044235779,\n",
       "  0.8319482935432582,\n",
       "  0.842105268441409,\n",
       "  0.8407202268901625,\n",
       "  0.841643584401984,\n",
       "  0.8264081326217863,\n",
       "  0.8628808864265928],\n",
       " 'train_VAL_acc_1': [19.94459833795014,\n",
       "  19.94459833795014,\n",
       "  21.606648199445985,\n",
       "  51.52354570637119,\n",
       "  49.58448753462604,\n",
       "  51.52354570637119,\n",
       "  54.84764542936288,\n",
       "  51.800554016620495,\n",
       "  47.64542936288089,\n",
       "  47.9224376731302,\n",
       "  47.64542936288089,\n",
       "  48.47645429362881,\n",
       "  50.41551246537396,\n",
       "  49.307479224376735,\n",
       "  53.46260387811634,\n",
       "  61.772853185595565,\n",
       "  58.448753462603875,\n",
       "  54.016620498614955,\n",
       "  57.61772853185595,\n",
       "  56.50969529085872,\n",
       "  62.04986149584487,\n",
       "  58.448753462603875,\n",
       "  61.21883656509695,\n",
       "  64.81994459833795,\n",
       "  66.7590027700831,\n",
       "  64.81994459833795,\n",
       "  66.4819944598338,\n",
       "  68.42105263157895,\n",
       "  67.59002770083103,\n",
       "  67.31301939058172,\n",
       "  67.86703601108033,\n",
       "  68.42105263157895,\n",
       "  68.42105263157895,\n",
       "  70.91412742382272,\n",
       "  71.74515235457064,\n",
       "  71.74515235457064,\n",
       "  70.3601108033241,\n",
       "  69.25207756232687,\n",
       "  73.6842105263158,\n",
       "  73.6842105263158,\n",
       "  74.79224376731302,\n",
       "  73.40720221606648,\n",
       "  70.6371191135734,\n",
       "  72.85318559556787,\n",
       "  75.62326869806094,\n",
       "  77.5623268698061,\n",
       "  75.62326869806094,\n",
       "  75.90027700831025,\n",
       "  76.45429362880887,\n",
       "  75.90027700831025,\n",
       "  74.51523545706371,\n",
       "  76.73130193905817,\n",
       "  78.39335180055402,\n",
       "  77.5623268698061,\n",
       "  76.45429362880887,\n",
       "  76.45429362880887,\n",
       "  73.9612188365651,\n",
       "  77.00831024930748,\n",
       "  75.62326869806094,\n",
       "  77.8393351800554,\n",
       "  78.11634349030471,\n",
       "  78.94736842105263,\n",
       "  78.94736842105263,\n",
       "  76.73130193905817,\n",
       "  77.8393351800554,\n",
       "  78.94736842105263,\n",
       "  78.94736842105263,\n",
       "  78.94736842105263,\n",
       "  78.39335180055402,\n",
       "  77.28531855955679,\n",
       "  79.50138504155125,\n",
       "  77.5623268698061,\n",
       "  76.73130193905817,\n",
       "  76.17728531855956,\n",
       "  77.8393351800554,\n",
       "  77.8393351800554,\n",
       "  79.50138504155125,\n",
       "  80.88642659279779,\n",
       "  80.05540166204986,\n",
       "  81.16343490304709,\n",
       "  81.7174515235457,\n",
       "  82.27146814404432,\n",
       "  81.99445983379502,\n",
       "  83.10249307479225,\n",
       "  81.4404432132964,\n",
       "  81.7174515235457,\n",
       "  81.99445983379502,\n",
       "  81.7174515235457,\n",
       "  80.88642659279779,\n",
       "  80.60941828254848,\n",
       "  81.4404432132964,\n",
       "  84.7645429362881,\n",
       "  83.65650969529086,\n",
       "  83.65650969529086,\n",
       "  82.82548476454294,\n",
       "  82.82548476454294,\n",
       "  82.54847645429363,\n",
       "  83.37950138504155,\n",
       "  82.82548476454294,\n",
       "  80.88642659279779,\n",
       "  82.82548476454294,\n",
       "  84.7645429362881,\n",
       "  83.10249307479225,\n",
       "  82.54847645429363,\n",
       "  83.37950138504155,\n",
       "  85.31855955678671,\n",
       "  84.7645429362881,\n",
       "  83.65650969529086,\n",
       "  83.93351800554017,\n",
       "  84.21052631578948,\n",
       "  83.65650969529086,\n",
       "  83.93351800554017,\n",
       "  83.37950138504155,\n",
       "  83.93351800554017,\n",
       "  84.7645429362881,\n",
       "  83.10249307479225,\n",
       "  81.7174515235457,\n",
       "  83.93351800554017,\n",
       "  83.93351800554017,\n",
       "  84.21052631578948,\n",
       "  83.37950138504155,\n",
       "  82.82548476454294,\n",
       "  83.65650969529086,\n",
       "  81.99445983379502,\n",
       "  82.82548476454294,\n",
       "  85.0415512465374,\n",
       "  86.14958448753463,\n",
       "  85.31855955678671,\n",
       "  85.31855955678671,\n",
       "  83.65650969529086,\n",
       "  83.65650969529086,\n",
       "  86.14958448753463,\n",
       "  86.14958448753463,\n",
       "  84.7645429362881,\n",
       "  85.31855955678671,\n",
       "  85.31855955678671,\n",
       "  85.0415512465374,\n",
       "  86.42659279778394,\n",
       "  85.31855955678671,\n",
       "  83.93351800554017,\n",
       "  81.4404432132964,\n",
       "  83.37950138504155,\n",
       "  82.54847645429363,\n",
       "  84.48753462603878,\n",
       "  84.48753462603878,\n",
       "  86.42659279778394,\n",
       "  85.59556786703601,\n",
       "  82.27146814404432,\n",
       "  83.37950138504155,\n",
       "  84.7645429362881,\n",
       "  85.0415512465374,\n",
       "  86.14958448753463,\n",
       "  86.42659279778394,\n",
       "  84.7645429362881,\n",
       "  85.59556786703601,\n",
       "  85.87257617728532,\n",
       "  88.36565096952909,\n",
       "  86.42659279778394,\n",
       "  85.59556786703601,\n",
       "  83.37950138504155,\n",
       "  86.14958448753463,\n",
       "  85.59556786703601,\n",
       "  83.65650969529086,\n",
       "  85.31855955678671,\n",
       "  83.93351800554017,\n",
       "  87.81163434903047,\n",
       "  87.81163434903047,\n",
       "  88.08864265927978,\n",
       "  86.98060941828255,\n",
       "  89.75069252077563],\n",
       " 'val_loss': [7.389335035278751,\n",
       "  7.351940003074334,\n",
       "  7.28385148578013,\n",
       "  7.247727541876688,\n",
       "  7.190594639592274,\n",
       "  7.124504636205227,\n",
       "  7.038099749521125,\n",
       "  6.932983508210402,\n",
       "  6.83111528790328,\n",
       "  6.744247649536818,\n",
       "  6.626269363061729,\n",
       "  6.491639192467233,\n",
       "  6.361750173758026,\n",
       "  6.331452633756015,\n",
       "  6.231342453424676,\n",
       "  6.238035516764386,\n",
       "  6.201665538890308,\n",
       "  6.1791061978302135,\n",
       "  6.160957223964952,\n",
       "  6.113802697247687,\n",
       "  6.069780767910462,\n",
       "  6.072291408439408,\n",
       "  6.015231831932107,\n",
       "  5.962008695979777,\n",
       "  5.970577143485686,\n",
       "  5.944798442663002,\n",
       "  5.913527129073635,\n",
       "  5.902649265380797,\n",
       "  5.918441751206985,\n",
       "  5.910239144055069,\n",
       "  5.873215003432722,\n",
       "  5.883730675410611,\n",
       "  5.867670532354313,\n",
       "  5.849226252703091,\n",
       "  5.83160365130034,\n",
       "  5.813149176423109,\n",
       "  5.816661082926389,\n",
       "  5.8284616717979425,\n",
       "  5.810774430887419,\n",
       "  5.798029430301003,\n",
       "  5.798642801080222,\n",
       "  5.79428024918248,\n",
       "  5.798609770790582,\n",
       "  5.785624395405589,\n",
       "  5.776498888804957,\n",
       "  5.770442308641925,\n",
       "  5.768002603649305,\n",
       "  5.7684758501403,\n",
       "  5.759240985845884,\n",
       "  5.759669658722387,\n",
       "  5.758764695988013,\n",
       "  5.747202527038331,\n",
       "  5.744368810346748,\n",
       "  5.753495160975909,\n",
       "  5.737434603560027,\n",
       "  5.736576697576714,\n",
       "  5.746213150490385,\n",
       "  5.737737711884982,\n",
       "  5.738964980041228,\n",
       "  5.73084300950231,\n",
       "  5.723768712214433,\n",
       "  5.719760086695486,\n",
       "  5.7269706511380285,\n",
       "  5.734066703607416,\n",
       "  5.73635173211852,\n",
       "  5.735371186004426,\n",
       "  5.721775338598569,\n",
       "  5.710841491544503,\n",
       "  5.7129367284932995,\n",
       "  5.7207307717159575,\n",
       "  5.719542234062414,\n",
       "  5.721878519560818,\n",
       "  5.720248056030864,\n",
       "  5.721766438726225,\n",
       "  5.716764444717083,\n",
       "  5.716359588240178,\n",
       "  5.7026988744468365,\n",
       "  5.704617501424748,\n",
       "  5.704324834018611,\n",
       "  5.705494319067981,\n",
       "  5.707634992606577,\n",
       "  5.698658765565457,\n",
       "  5.69399780613484,\n",
       "  5.7029845173319575,\n",
       "  5.691668502897212,\n",
       "  5.688194006733959,\n",
       "  5.690352249013141,\n",
       "  5.697788103906477,\n",
       "  5.693946429703117,\n",
       "  5.701836691814146,\n",
       "  5.689986224100545,\n",
       "  5.681576957339885,\n",
       "  5.682564651074138,\n",
       "  5.684316705722126,\n",
       "  5.688614256507838,\n",
       "  5.684231815878848,\n",
       "  5.681171642494572,\n",
       "  5.6862258232329985,\n",
       "  5.684235231689412,\n",
       "  5.681089871586852,\n",
       "  5.674814448439726,\n",
       "  5.67480370126382,\n",
       "  5.680939387853696,\n",
       "  5.672993344347107,\n",
       "  5.677565892187718,\n",
       "  5.675447280152769,\n",
       "  5.684449544847023,\n",
       "  5.677508145487205,\n",
       "  5.6668724232766206,\n",
       "  5.675441834700685,\n",
       "  5.673142204148506,\n",
       "  5.681848538318053,\n",
       "  5.677246046153296,\n",
       "  5.6795696078139954,\n",
       "  5.688473215015518,\n",
       "  5.694097299242608,\n",
       "  5.690042774118876,\n",
       "  5.6762843098682545,\n",
       "  5.683082944973769,\n",
       "  5.68078246830729,\n",
       "  5.6782622737462765,\n",
       "  5.675934083720695,\n",
       "  5.6726920290509675,\n",
       "  5.689148196593304,\n",
       "  5.674858198794453,\n",
       "  5.666599560244177,\n",
       "  5.665500469059093,\n",
       "  5.673181561549128,\n",
       "  5.666320657892667,\n",
       "  5.666699788904986,\n",
       "  5.662518403546616,\n",
       "  5.661674470632625,\n",
       "  5.667558635347975,\n",
       "  5.674706492353256,\n",
       "  5.6696041932904615,\n",
       "  5.66959130761501,\n",
       "  5.664970794256608,\n",
       "  5.668566283837753,\n",
       "  5.6851491607488995,\n",
       "  5.683687099741368,\n",
       "  5.679629395775459,\n",
       "  5.676242866661944,\n",
       "  5.677973161955691,\n",
       "  5.669689296938458,\n",
       "  5.660714911813707,\n",
       "  5.658746077733741,\n",
       "  5.675165803485865,\n",
       "  5.681277299532209,\n",
       "  5.670703209400406,\n",
       "  5.669041735028026,\n",
       "  5.670996687851281,\n",
       "  5.660797788021878,\n",
       "  5.666154042049253,\n",
       "  5.661422396913237,\n",
       "  5.6649407020947296,\n",
       "  5.6587948122437135,\n",
       "  5.664828681324308,\n",
       "  5.6619066436492265,\n",
       "  5.66408244088621,\n",
       "  5.6698946744410055,\n",
       "  5.65898779735779,\n",
       "  5.664001124619368,\n",
       "  5.6700209012608145,\n",
       "  5.657319169594923,\n",
       "  5.672919330271699,\n",
       "  5.673788656609629,\n",
       "  5.661035936953757,\n",
       "  5.661534674821042,\n",
       "  5.669836502068821,\n",
       "  5.6645986045864],\n",
       " 'val_ATT_loss': [1.9754252106678196,\n",
       "  1.95841538978786,\n",
       "  1.9339974968413995,\n",
       "  1.928011721954113,\n",
       "  1.9089900994203923,\n",
       "  1.894441720431413,\n",
       "  1.863403122599532,\n",
       "  1.8009332558973048,\n",
       "  1.7079340270379695,\n",
       "  1.611228049528308,\n",
       "  1.5005133612369135,\n",
       "  1.4064908873259536,\n",
       "  1.3029033776705827,\n",
       "  1.254273116104002,\n",
       "  1.175094438641052,\n",
       "  1.1865728817334988,\n",
       "  1.1424274873442766,\n",
       "  1.087599147626055,\n",
       "  1.0749053165195435,\n",
       "  1.0564707317730275,\n",
       "  1.0459723455634544,\n",
       "  1.0321546965013675,\n",
       "  0.9747485468784968,\n",
       "  0.9756261410751963,\n",
       "  0.9655944582650332,\n",
       "  0.9406097247833158,\n",
       "  0.9271048034836606,\n",
       "  0.9223230327290248,\n",
       "  0.9219872222198704,\n",
       "  0.9180276248513198,\n",
       "  0.8979151247720408,\n",
       "  0.8909011417772712,\n",
       "  0.8796277261846434,\n",
       "  0.8786799072977004,\n",
       "  0.8793070806962687,\n",
       "  0.8689187720296828,\n",
       "  0.8650327725865976,\n",
       "  0.8640053766529735,\n",
       "  0.862186959846233,\n",
       "  0.8562932406983724,\n",
       "  0.8580713931137953,\n",
       "  0.8542373077413901,\n",
       "  0.8538121439334823,\n",
       "  0.8491734982263751,\n",
       "  0.8457789324163422,\n",
       "  0.8426134529394832,\n",
       "  0.8418570248092093,\n",
       "  0.839810941519776,\n",
       "  0.8361607517411069,\n",
       "  0.837965819772666,\n",
       "  0.8349738077419561,\n",
       "  0.8301556547725104,\n",
       "  0.8259650345255689,\n",
       "  0.826118339852589,\n",
       "  0.823322027437086,\n",
       "  0.8194694425759277,\n",
       "  0.8259131753347754,\n",
       "  0.8235300736940974,\n",
       "  0.8231026258895068,\n",
       "  0.8205215777081203,\n",
       "  0.8150602331975612,\n",
       "  0.8142722985608791,\n",
       "  0.8176098377481709,\n",
       "  0.8144947742301274,\n",
       "  0.8193655526492654,\n",
       "  0.8178679946719146,\n",
       "  0.8166530796182834,\n",
       "  0.8100011850275645,\n",
       "  0.8107232366635547,\n",
       "  0.807168188497303,\n",
       "  0.8104208481263339,\n",
       "  0.810321339019915,\n",
       "  0.8071216801560022,\n",
       "  0.803669609795741,\n",
       "  0.8057034261585251,\n",
       "  0.8063663566500191,\n",
       "  0.8040095674313181,\n",
       "  0.8039398944474817,\n",
       "  0.8036975680085702,\n",
       "  0.8042957945083214,\n",
       "  0.8081155379370946,\n",
       "  0.8014098972809024,\n",
       "  0.8016714617246534,\n",
       "  0.8037425131817174,\n",
       "  0.7979980661132471,\n",
       "  0.8011693404457434,\n",
       "  0.7970080216967963,\n",
       "  0.7980884382637535,\n",
       "  0.7992046085315022,\n",
       "  0.7994071205214757,\n",
       "  0.8000339068532959,\n",
       "  0.7931234056387491,\n",
       "  0.7979505058468842,\n",
       "  0.7972169692923383,\n",
       "  0.7982184155200555,\n",
       "  0.7958054650362915,\n",
       "  0.7953570233127936,\n",
       "  0.7964241236932879,\n",
       "  0.7904736438417822,\n",
       "  0.7926887608155971,\n",
       "  0.7923866814471842,\n",
       "  0.791083546431084,\n",
       "  0.7990993584801511,\n",
       "  0.7929698214298342,\n",
       "  0.7942293293592406,\n",
       "  0.7918727926122464,\n",
       "  0.7940859079845553,\n",
       "  0.7934170847501212,\n",
       "  0.7886485401692429,\n",
       "  0.7929460532296964,\n",
       "  0.7898322537662538,\n",
       "  0.7904602130496405,\n",
       "  0.7899776802557271,\n",
       "  0.788233066356279,\n",
       "  0.7935539293822235,\n",
       "  0.7907354238072062,\n",
       "  0.7948543884889866,\n",
       "  0.7890621678131383,\n",
       "  0.7890478246580295,\n",
       "  0.7909488324227372,\n",
       "  0.7889211742373986,\n",
       "  0.789680158340834,\n",
       "  0.7871854208349213,\n",
       "  0.7877322906643395,\n",
       "  0.7863621809860555,\n",
       "  0.7883145421501098,\n",
       "  0.7851526780826289,\n",
       "  0.7895335541750358,\n",
       "  0.7831492169601161,\n",
       "  0.7849748963262977,\n",
       "  0.7851390524850628,\n",
       "  0.7850637517082013,\n",
       "  0.7864594558874766,\n",
       "  0.7874815333180312,\n",
       "  0.7872594135805844,\n",
       "  0.7850923414637403,\n",
       "  0.7842710569379775,\n",
       "  0.785236436661666,\n",
       "  0.7892928354866137,\n",
       "  0.7899342640870954,\n",
       "  0.790399633045119,\n",
       "  0.7816502203543981,\n",
       "  0.788775199918243,\n",
       "  0.7819638702927566,\n",
       "  0.7802266224128444,\n",
       "  0.7850008776517419,\n",
       "  0.7846237473129257,\n",
       "  0.7898857714684029,\n",
       "  0.7871777996784304,\n",
       "  0.7845709422012654,\n",
       "  0.7900229247847224,\n",
       "  0.7839975016630762,\n",
       "  0.7826116441226587,\n",
       "  0.783148912758362,\n",
       "  0.7789311774862491,\n",
       "  0.7814529627803865,\n",
       "  0.7818793008724848,\n",
       "  0.782503011386569,\n",
       "  0.7839939072122418,\n",
       "  0.7838839236798325,\n",
       "  0.7825043411031971,\n",
       "  0.7796242205592675,\n",
       "  0.784082246868591,\n",
       "  0.7774516816546277,\n",
       "  0.7828813461753411,\n",
       "  0.7854755804306124,\n",
       "  0.7830267116063978,\n",
       "  0.7860671107361956,\n",
       "  0.7846018443020378,\n",
       "  0.7814090743782075],\n",
       " 'val_VAL_loss': [1.8046366082036436,\n",
       "  1.797841537762158,\n",
       "  1.7832846629795769,\n",
       "  1.773238606640858,\n",
       "  1.7605348467239605,\n",
       "  1.743354305257938,\n",
       "  1.7248988756405308,\n",
       "  1.710683417437699,\n",
       "  1.7077270869551033,\n",
       "  1.7110065333361697,\n",
       "  1.7085853339416053,\n",
       "  1.6950494350470933,\n",
       "  1.6862822653624812,\n",
       "  1.6923931725506711,\n",
       "  1.6854160049278748,\n",
       "  1.683820878343629,\n",
       "  1.686412683848677,\n",
       "  1.6971690167347198,\n",
       "  1.6953506358151365,\n",
       "  1.6857773218248866,\n",
       "  1.6746028074490025,\n",
       "  1.6800455706460136,\n",
       "  1.68016109501787,\n",
       "  1.662127518301527,\n",
       "  1.6683275617402176,\n",
       "  1.668062905959895,\n",
       "  1.662140775196658,\n",
       "  1.6601087442172573,\n",
       "  1.6654848429957048,\n",
       "  1.66407050640125,\n",
       "  1.6584332928868937,\n",
       "  1.6642765112111133,\n",
       "  1.66268093538989,\n",
       "  1.656848781801797,\n",
       "  1.6507655235346903,\n",
       "  1.6480768014644753,\n",
       "  1.6505427701132638,\n",
       "  1.654818765048323,\n",
       "  1.6495291570137287,\n",
       "  1.6472453965342104,\n",
       "  1.6468571359888087,\n",
       "  1.6466809804803633,\n",
       "  1.6482658756190334,\n",
       "  1.6454836323930713,\n",
       "  1.6435733187962047,\n",
       "  1.6426096185674808,\n",
       "  1.642048526280032,\n",
       "  1.6428883028735082,\n",
       "  1.6410267447015923,\n",
       "  1.6405679463165734,\n",
       "  1.6412636294153524,\n",
       "  1.639015624088607,\n",
       "  1.639467925273726,\n",
       "  1.64245894037444,\n",
       "  1.6380375253743138,\n",
       "  1.6390357516669287,\n",
       "  1.6400999917185366,\n",
       "  1.638069212730295,\n",
       "  1.6386207847172403,\n",
       "  1.6367738105980634,\n",
       "  1.6362361596722907,\n",
       "  1.6351625960448692,\n",
       "  1.636453604463286,\n",
       "  1.6398573097924294,\n",
       "  1.638995393156418,\n",
       "  1.6391677304441705,\n",
       "  1.6350407529934285,\n",
       "  1.6336134355056462,\n",
       "  1.6340711639432484,\n",
       "  1.6378541944062182,\n",
       "  1.6363737953120265,\n",
       "  1.6371857268469674,\n",
       "  1.6377087919582873,\n",
       "  1.6393656096434945,\n",
       "  1.6370203395195195,\n",
       "  1.6366644105300527,\n",
       "  1.6328964356718392,\n",
       "  1.6335592023257552,\n",
       "  1.6335424220033468,\n",
       "  1.6337328415198866,\n",
       "  1.6331731515564942,\n",
       "  1.6324162894281848,\n",
       "  1.630775448136729,\n",
       "  1.63308066805008,\n",
       "  1.6312234789279882,\n",
       "  1.629008222096072,\n",
       "  1.6311147424387815,\n",
       "  1.633233221880908,\n",
       "  1.631580607057205,\n",
       "  1.63414319043089,\n",
       "  1.6299841057490834,\n",
       "  1.6294845172337122,\n",
       "  1.6282047150757513,\n",
       "  1.629033245476596,\n",
       "  1.6301319469959277,\n",
       "  1.629475450280852,\n",
       "  1.6286048730605929,\n",
       "  1.6299338998465702,\n",
       "  1.6312538626158766,\n",
       "  1.6294670369237514,\n",
       "  1.627475922330847,\n",
       "  1.6279067182775788,\n",
       "  1.6272800097911817,\n",
       "  1.626674507639091,\n",
       "  1.627778854276159,\n",
       "  1.6278581625135073,\n",
       "  1.6301212122874895,\n",
       "  1.628030353579028,\n",
       "  1.6260746277024594,\n",
       "  1.627498593823663,\n",
       "  1.6277699834607504,\n",
       "  1.6304627750894707,\n",
       "  1.6290894552991895,\n",
       "  1.6304455138192389,\n",
       "  1.6316397618777647,\n",
       "  1.6344539584784672,\n",
       "  1.6317294618766296,\n",
       "  1.6290740473517056,\n",
       "  1.6313450401052465,\n",
       "  1.6299445452948509,\n",
       "  1.629780366502959,\n",
       "  1.6287513084599536,\n",
       "  1.628502202738682,\n",
       "  1.6338053019763215,\n",
       "  1.6294986726027991,\n",
       "  1.626095006031356,\n",
       "  1.6267825969921543,\n",
       "  1.6278826691246973,\n",
       "  1.6277238136441836,\n",
       "  1.627241630859563,\n",
       "  1.6257931170205178,\n",
       "  1.6255369063081413,\n",
       "  1.6270330598201659,\n",
       "  1.629074986345075,\n",
       "  1.6274482599032924,\n",
       "  1.6281663220504234,\n",
       "  1.6268999124395436,\n",
       "  1.6277766157253621,\n",
       "  1.631952108420762,\n",
       "  1.631250945218091,\n",
       "  1.6297432542434467,\n",
       "  1.6315308821025154,\n",
       "  1.6297326540124828,\n",
       "  1.6292418088819005,\n",
       "  1.6268294298002872,\n",
       "  1.6245817333606665,\n",
       "  1.6301806853909797,\n",
       "  1.6304638426879356,\n",
       "  1.6278418032406585,\n",
       "  1.6281569309422534,\n",
       "  1.6269912543555198,\n",
       "  1.6256000954529335,\n",
       "  1.6278474659755313,\n",
       "  1.6260911613849585,\n",
       "  1.6286698415361602,\n",
       "  1.6257806164877755,\n",
       "  1.627649793483941,\n",
       "  1.6264678774208858,\n",
       "  1.6266961778913225,\n",
       "  1.6286702502537242,\n",
       "  1.6254944854181976,\n",
       "  1.6281256346867001,\n",
       "  1.6286462181307413,\n",
       "  1.6266224959800983,\n",
       "  1.6300126613654526,\n",
       "  1.629437692059672,\n",
       "  1.6260030751157863,\n",
       "  1.625155854694949,\n",
       "  1.628411552588928,\n",
       "  1.627729843402731],\n",
       " 'val_ATT_acc': [29.67479674796748,\n",
       "  12.601626016260163,\n",
       "  29.67479674796748,\n",
       "  33.739837398373986,\n",
       "  33.333333333333336,\n",
       "  34.959349593495936,\n",
       "  36.17886178861789,\n",
       "  39.63414634146341,\n",
       "  42.479674796747965,\n",
       "  45.32520325203252,\n",
       "  55.08130081300813,\n",
       "  59.34959349593496,\n",
       "  66.869918699187,\n",
       "  69.51219512195122,\n",
       "  71.7479674796748,\n",
       "  70.1219512195122,\n",
       "  70.9349593495935,\n",
       "  74.79674796747967,\n",
       "  75.8130081300813,\n",
       "  78.65853658536585,\n",
       "  80.08130081300813,\n",
       "  79.0650406504065,\n",
       "  85.5691056910569,\n",
       "  83.9430894308943,\n",
       "  84.95934959349593,\n",
       "  88.41463414634147,\n",
       "  88.6178861788618,\n",
       "  88.82113821138212,\n",
       "  88.82113821138212,\n",
       "  88.6178861788618,\n",
       "  89.22764227642277,\n",
       "  89.83739837398375,\n",
       "  90.65040650406505,\n",
       "  89.63414634146342,\n",
       "  91.0569105691057,\n",
       "  91.66666666666667,\n",
       "  91.46341463414635,\n",
       "  91.66666666666667,\n",
       "  91.0569105691057,\n",
       "  92.27642276422765,\n",
       "  93.4959349593496,\n",
       "  92.07317073170732,\n",
       "  92.88617886178862,\n",
       "  92.6829268292683,\n",
       "  92.88617886178862,\n",
       "  92.07317073170732,\n",
       "  92.6829268292683,\n",
       "  92.88617886178862,\n",
       "  93.4959349593496,\n",
       "  92.07317073170732,\n",
       "  93.08943089430895,\n",
       "  92.88617886178862,\n",
       "  93.29268292682927,\n",
       "  93.29268292682927,\n",
       "  93.08943089430895,\n",
       "  93.69918699186992,\n",
       "  92.6829268292683,\n",
       "  93.08943089430895,\n",
       "  92.6829268292683,\n",
       "  93.29268292682927,\n",
       "  94.51219512195122,\n",
       "  94.51219512195122,\n",
       "  94.3089430894309,\n",
       "  94.3089430894309,\n",
       "  93.69918699186992,\n",
       "  93.29268292682927,\n",
       "  93.4959349593496,\n",
       "  93.69918699186992,\n",
       "  94.10569105691057,\n",
       "  94.10569105691057,\n",
       "  95.1219512195122,\n",
       "  94.71544715447155,\n",
       "  94.51219512195122,\n",
       "  95.1219512195122,\n",
       "  94.91869918699187,\n",
       "  94.51219512195122,\n",
       "  94.91869918699187,\n",
       "  95.1219512195122,\n",
       "  95.1219512195122,\n",
       "  94.51219512195122,\n",
       "  94.3089430894309,\n",
       "  94.10569105691057,\n",
       "  94.71544715447155,\n",
       "  94.71544715447155,\n",
       "  95.32520325203252,\n",
       "  94.10569105691057,\n",
       "  94.71544715447155,\n",
       "  94.91869918699187,\n",
       "  95.52845528455285,\n",
       "  94.3089430894309,\n",
       "  94.71544715447155,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  94.91869918699187,\n",
       "  95.52845528455285,\n",
       "  94.71544715447155,\n",
       "  95.1219512195122,\n",
       "  94.71544715447155,\n",
       "  95.32520325203252,\n",
       "  95.32520325203252,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.1219512195122,\n",
       "  94.91869918699187,\n",
       "  95.1219512195122,\n",
       "  95.9349593495935,\n",
       "  95.32520325203252,\n",
       "  94.71544715447155,\n",
       "  95.9349593495935,\n",
       "  95.1219512195122,\n",
       "  95.1219512195122,\n",
       "  95.73170731707317,\n",
       "  95.1219512195122,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.1219512195122,\n",
       "  94.71544715447155,\n",
       "  94.91869918699187,\n",
       "  95.1219512195122,\n",
       "  95.1219512195122,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  94.71544715447155,\n",
       "  95.9349593495935,\n",
       "  95.32520325203252,\n",
       "  95.1219512195122,\n",
       "  95.73170731707317,\n",
       "  95.32520325203252,\n",
       "  95.1219512195122,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  96.34146341463415,\n",
       "  96.7479674796748,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  94.51219512195122,\n",
       "  95.52845528455285,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  96.13821138211382,\n",
       "  95.1219512195122,\n",
       "  95.52845528455285,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  96.7479674796748,\n",
       "  95.52845528455285,\n",
       "  96.7479674796748,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935],\n",
       " 'val_VAL_acc': [64.5320197044335,\n",
       "  64.5320197044335,\n",
       "  69.95073891625616,\n",
       "  90.14778325123153,\n",
       "  88.66995073891626,\n",
       "  91.13300492610837,\n",
       "  89.16256157635468,\n",
       "  92.61083743842364,\n",
       "  86.20689655172414,\n",
       "  84.23645320197045,\n",
       "  87.6847290640394,\n",
       "  92.11822660098522,\n",
       "  89.65517241379311,\n",
       "  87.6847290640394,\n",
       "  86.69950738916256,\n",
       "  89.65517241379311,\n",
       "  92.11822660098522,\n",
       "  93.5960591133005,\n",
       "  93.10344827586206,\n",
       "  94.08866995073892,\n",
       "  93.5960591133005,\n",
       "  94.08866995073892,\n",
       "  93.5960591133005,\n",
       "  96.05911330049261,\n",
       "  97.53694581280789,\n",
       "  96.55172413793103,\n",
       "  96.05911330049261,\n",
       "  96.55172413793103,\n",
       "  93.5960591133005,\n",
       "  96.05911330049261,\n",
       "  98.0295566502463,\n",
       "  96.55172413793103,\n",
       "  95.56650246305419,\n",
       "  97.04433497536945,\n",
       "  97.04433497536945,\n",
       "  98.0295566502463,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  97.53694581280789,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158],\n",
       " 'val_VAL_jac': [0.0,\n",
       "  0.0,\n",
       "  0.0,\n",
       "  0.0,\n",
       "  0.0,\n",
       "  0.1921182266009852,\n",
       "  0.34646962311467516,\n",
       "  0.45484400735112834,\n",
       "  0.4868637117846259,\n",
       "  0.4770114973848089,\n",
       "  0.4983579630922214,\n",
       "  0.551724140279986,\n",
       "  0.5500821057211589,\n",
       "  0.5032840726410814,\n",
       "  0.5065681476311144,\n",
       "  0.5254515591513347,\n",
       "  0.5476190538829183,\n",
       "  0.5812807922880051,\n",
       "  0.5878489358084542,\n",
       "  0.565681449298201,\n",
       "  0.5919540269034249,\n",
       "  0.5993431871160498,\n",
       "  0.5755336648724937,\n",
       "  0.6223316920801923,\n",
       "  0.5985221745345393,\n",
       "  0.6190476229625382,\n",
       "  0.6215106748007788,\n",
       "  0.6297208546417687,\n",
       "  0.645320197044335,\n",
       "  0.6379310368317102,\n",
       "  0.6724138001503028,\n",
       "  0.6486042755577952,\n",
       "  0.6444991821138729,\n",
       "  0.6871921205755525,\n",
       "  0.6995073915115131,\n",
       "  0.6896551747627446,\n",
       "  0.7044335022348488,\n",
       "  0.6945812831371289,\n",
       "  0.7044335022348488,\n",
       "  0.7126436844247902,\n",
       "  0.7183908100785881,\n",
       "  0.7068965587709924,\n",
       "  0.716748770821858,\n",
       "  0.7101806278886467,\n",
       "  0.7266009899195779,\n",
       "  0.7167487755197609,\n",
       "  0.7134647017042038,\n",
       "  0.7142857189836174,\n",
       "  0.7348111721095193,\n",
       "  0.7348111721095193,\n",
       "  0.73891626320449,\n",
       "  0.7216748815451937,\n",
       "  0.7241379380813373,\n",
       "  0.7241379380813373,\n",
       "  0.7298850613861836,\n",
       "  0.7126436820758387,\n",
       "  0.725779974989116,\n",
       "  0.7241379357323858,\n",
       "  0.7183908077296365,\n",
       "  0.7430213519505092,\n",
       "  0.7356321893889328,\n",
       "  0.7372742262967115,\n",
       "  0.717569790450223,\n",
       "  0.7389162608555385,\n",
       "  0.749589492534769,\n",
       "  0.7438423715788742,\n",
       "  0.7512315341404506,\n",
       "  0.7594417139814404,\n",
       "  0.7323481179223272,\n",
       "  0.748768477604307,\n",
       "  0.7676518938224304,\n",
       "  0.7479474603248935,\n",
       "  0.7643678223558248,\n",
       "  0.7577996794226134,\n",
       "  0.7454844084866529,\n",
       "  0.7520525514198642,\n",
       "  0.7422003346710957,\n",
       "  0.7487684799532585,\n",
       "  0.748768477604307,\n",
       "  0.7413793150427306,\n",
       "  0.747947457975942,\n",
       "  0.7602627312608541,\n",
       "  0.7389162585065869,\n",
       "  0.747947457975942,\n",
       "  0.7487684752553555,\n",
       "  0.7577996747247104,\n",
       "  0.7668308765430168,\n",
       "  0.749589497232672,\n",
       "  0.7422003346710957,\n",
       "  0.7602627312608541,\n",
       "  0.7619047658196811,\n",
       "  0.761083746191316,\n",
       "  0.7783251255016609,\n",
       "  0.7766830909428338,\n",
       "  0.7676518938224304,\n",
       "  0.7594417139814404,\n",
       "  0.7561576401658834,\n",
       "  0.7520525514198642,\n",
       "  0.7602627289119025,\n",
       "  0.7824302189455831,\n",
       "  0.7725780021968146,\n",
       "  0.7479474603248935,\n",
       "  0.7627257854480461,\n",
       "  0.7783251255016609,\n",
       "  0.779967164758391,\n",
       "  0.7914614207638896,\n",
       "  0.7783251255016609,\n",
       "  0.780788179688853,\n",
       "  0.779967164758391,\n",
       "  0.7750410563840068,\n",
       "  0.759441711632489,\n",
       "  0.7594417139814404,\n",
       "  0.7536945883276427,\n",
       "  0.759441711632489,\n",
       "  0.7676518938224304,\n",
       "  0.7692939283812574,\n",
       "  0.7717569825684496,\n",
       "  0.7725780021968146,\n",
       "  0.7824302189455831,\n",
       "  0.779967164758391,\n",
       "  0.7898193815071595,\n",
       "  0.7816092016661695,\n",
       "  0.7840722582023132,\n",
       "  0.7766830956407369,\n",
       "  0.7791461451300259,\n",
       "  0.7791461451300259,\n",
       "  0.7898193815071595,\n",
       "  0.7791461451300259,\n",
       "  0.7717569825684496,\n",
       "  0.769293926032306,\n",
       "  0.7881773422504293,\n",
       "  0.7865353076916023,\n",
       "  0.7947454898815437,\n",
       "  0.7766830909428338,\n",
       "  0.779967164758391,\n",
       "  0.7898193815071595,\n",
       "  0.7848932731327752,\n",
       "  0.7848932731327752,\n",
       "  0.7791461451300259,\n",
       "  0.778325130199564,\n",
       "  0.7742200391045932,\n",
       "  0.7807881820378045,\n",
       "  0.789819383856111,\n",
       "  0.7963875244403708,\n",
       "  0.7742200367556417,\n",
       "  0.779967164758391,\n",
       "  0.7668308765430168,\n",
       "  0.7561576401658834,\n",
       "  0.7766830956407369,\n",
       "  0.7717569872663526,\n",
       "  0.7750410563840068,\n",
       "  0.7865353076916023,\n",
       "  0.7758620736634203,\n",
       "  0.7914614160659865,\n",
       "  0.7922824356943515,\n",
       "  0.7742200414535447,\n",
       "  0.7922824356943515,\n",
       "  0.7906404011355245,\n",
       "  0.7709359676379876,\n",
       "  0.7610837485402676,\n",
       "  0.7840722511554586,\n",
       "  0.7775041082222474,\n",
       "  0.7775041082222474,\n",
       "  0.7610837485402676,\n",
       "  0.7528735663503262,\n",
       "  0.7848932707838236,\n",
       "  0.7742200367556417,\n",
       "  0.7840722535044101,\n",
       "  0.7865353076916023,\n",
       "  0.7750410563840068],\n",
       " 'val_VAL_acc_1': [15.270935960591133,\n",
       "  15.270935960591133,\n",
       "  16.748768472906406,\n",
       "  63.05418719211823,\n",
       "  60.59113300492611,\n",
       "  64.03940886699507,\n",
       "  64.03940886699507,\n",
       "  63.54679802955665,\n",
       "  58.62068965517241,\n",
       "  56.65024630541872,\n",
       "  59.60591133004926,\n",
       "  62.5615763546798,\n",
       "  61.083743842364534,\n",
       "  54.1871921182266,\n",
       "  51.724137931034484,\n",
       "  52.70935960591133,\n",
       "  55.66502463054187,\n",
       "  60.59113300492611,\n",
       "  65.02463054187191,\n",
       "  62.5615763546798,\n",
       "  67.48768472906404,\n",
       "  62.06896551724138,\n",
       "  62.5615763546798,\n",
       "  67.98029556650246,\n",
       "  70.44334975369458,\n",
       "  70.93596059113301,\n",
       "  72.41379310344827,\n",
       "  69.45812807881774,\n",
       "  67.98029556650246,\n",
       "  71.42857142857143,\n",
       "  73.39901477832512,\n",
       "  73.39901477832512,\n",
       "  72.9064039408867,\n",
       "  75.36945812807882,\n",
       "  72.9064039408867,\n",
       "  70.93596059113301,\n",
       "  71.92118226600985,\n",
       "  68.47290640394088,\n",
       "  73.39901477832512,\n",
       "  76.35467980295566,\n",
       "  73.39901477832512,\n",
       "  73.39901477832512,\n",
       "  72.41379310344827,\n",
       "  72.9064039408867,\n",
       "  78.81773399014779,\n",
       "  76.84729064039409,\n",
       "  78.32512315270937,\n",
       "  77.33990147783251,\n",
       "  80.78817733990148,\n",
       "  77.83251231527093,\n",
       "  77.83251231527093,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  78.81773399014779,\n",
       "  76.35467980295566,\n",
       "  78.81773399014779,\n",
       "  80.78817733990148,\n",
       "  77.33990147783251,\n",
       "  80.78817733990148,\n",
       "  82.75862068965517,\n",
       "  81.77339901477832,\n",
       "  78.32512315270937,\n",
       "  77.33990147783251,\n",
       "  80.29556650246306,\n",
       "  76.35467980295566,\n",
       "  80.29556650246306,\n",
       "  83.74384236453201,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  78.81773399014779,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  78.81773399014779,\n",
       "  79.3103448275862,\n",
       "  79.80295566502463,\n",
       "  78.81773399014779,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  79.3103448275862,\n",
       "  81.2807881773399,\n",
       "  79.3103448275862,\n",
       "  79.3103448275862,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  82.26600985221675,\n",
       "  80.78817733990148,\n",
       "  80.78817733990148,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  82.26600985221675,\n",
       "  78.81773399014779,\n",
       "  79.80295566502463,\n",
       "  77.83251231527093,\n",
       "  79.80295566502463,\n",
       "  77.83251231527093,\n",
       "  78.32512315270937,\n",
       "  81.2807881773399,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  82.26600985221675,\n",
       "  79.80295566502463,\n",
       "  79.3103448275862,\n",
       "  79.3103448275862,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  81.2807881773399,\n",
       "  79.80295566502463,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  80.78817733990148,\n",
       "  82.26600985221675,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  82.26600985221675,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  80.29556650246306,\n",
       "  81.77339901477832,\n",
       "  80.29556650246306,\n",
       "  81.77339901477832,\n",
       "  79.80295566502463,\n",
       "  82.75862068965517,\n",
       "  81.77339901477832,\n",
       "  81.77339901477832,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  81.77339901477832,\n",
       "  82.26600985221675,\n",
       "  80.78817733990148,\n",
       "  77.83251231527093,\n",
       "  79.3103448275862,\n",
       "  82.26600985221675,\n",
       "  81.2807881773399,\n",
       "  79.80295566502463,\n",
       "  81.2807881773399],\n",
       " 'test_loss': 5.751885582350387,\n",
       " 'test_ATT_loss': 0.7928359644787449,\n",
       " 'test_VAL_loss': 1.6530165392905474,\n",
       " 'test_ATT_acc': 97.01789264413519,\n",
       " 'test_VAL_acc': 99.47916666666667,\n",
       " 'test_VAL_jac': 0.7517361169060072,\n",
       " 'test_VAL_acc_1': 78.64583333333333,\n",
       " 'model_filename': 'model_storage/SAGE/model.pth'}"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'SAGE_best_model/best_config.p', 'wb') as fp:\n",
    "    pickle.dump(train_state,fp, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'SAGE_best_model/best_config.p', 'rb') as fp:\n",
    "    train_state = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "states = pd.DataFrame(train_state['val_VAL_acc'], columns=['val_VAL'])\n",
    "states['train_VAL'] = pd.DataFrame(train_state['train_VAL_acc'])\n",
    "states['val_ATT'] = pd.DataFrame(train_state['val_ATT_acc'])\n",
    "states['train_ATT'] = pd.DataFrame(train_state['train_ATT_acc'])\n",
    "states['val_VAL_1'] = pd.DataFrame(train_state['val_VAL_acc_1'])\n",
    "states['train_VAL_1'] = pd.DataFrame(train_state['train_VAL_acc_1'])\n",
    "states['train_VAL_jac'] = pd.DataFrame(train_state['train_VAL_jac'])\n",
    "states['val_VAL_jac'] = pd.DataFrame(train_state['val_VAL_jac'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['KMP_DUPLICATE_LIB_OK']='True'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAu40lEQVR4nO3deXzU1b3/8dcnk3WyL4RsQMIqe4CAIOIGetVSwbW4Va0tbbWttreLrb9eba+2tr1tb721tW5XbL1QxY1a1yKKIKIB2bdASEhC9n2fZOb8/phJTEIIIZNkJjOf5+PBY2a+852ZT76O7zlz5pzzFWMMSimlfF+ApwtQSik1PDTwlVLKT2jgK6WUn9DAV0opP6GBr5RSfiLQ0wUAJCQkmPT0dE+XoZRSI8qOHTsqjDGj+ru/VwR+eno62dnZni5DKaVGFBHJP5v9tUtHKaX8hAa+Ukr5CQ18pZTyExr4SinlJzTwlVLKT5wx8EXkGREpE5F9XbbFici7IpLjuoztct+PReSoiBwWkX8bqsKVUkqdnf608J8FLu+x7T5gozFmErDRdRsRmQasAqa7HvMnEbEMWrVKKaUG7Izj8I0xm0UkvcfmFcBFrutrgPeBH7m2rzPGtALHReQosADYNkj1Kk8o3Q9H3oa2Zk9XopTvSZwKM64Zlpca6MSr0caYYgBjTLGIJLq2pwIfd9mv0LXtFCKyGlgNMHbs2AGWoU5RcwIkAKLTwN4OJz+D9maoL4X9L0NDKZyzHIwdDmyA5uq+n89ucz4GABny8pXyOzOu8frAP53eEqHXM6wYY54AngDIysrSs7D0h8MBhZ/AoX9C9BiYvhIiEqGuGPa/AvvWQ9EO574pc6G2ABrLP398ZDJEpcDGnzlvj1kISTPP8KICKZkwbSVE9HsGt1LKCw008EtFJNnVuk8GylzbC4ExXfZLA066U6DfMQY+edIZyraG3vcJCARHO7z5g+7bk2bBpT8Hhx0OvQ7jznMFdSIEWSE5EwICoLYQEIju9cuXUspHDTTwNwC3AY+4Ll/rsv3/ROR3QAowCfjE3SJ9mr0dct+HfS85W+RNVVC2H8ZfDGPOPXX/+Akw5QpnaB95C9paINgKU66EhEmf77fke6d/zei0Qf8zlFLe74yBLyJrcf5AmyAihcADOIP+BRG5EzgBXA9gjNkvIi8AB4B24G5jjH2Iah/5yg7BK6uheDeERMPo6WCNg+W/h3l3gPTRZ5441flPKaX6qT+jdG48zV1LT7P/w8DD7hTlFw5sgJe/BsHhcM2TMG0FBIZ4uiqllA/ziuWR/c6Rd2D9V5w/hn7peYgc7emKlPJqDa3thAQGEGTxzsUBGlrbiQhxxqndYWizOwgN8r4pSN559HxZbRG8cKuz++aWlzTslepDZUMrD//zAFkPvcs3/7YDY4ZnQF+73cGx8gba7I4z7vuXD44x68G3eXNvMcYY7lzzKUt/+wHl9a3DUOnZ0Rb+cDu+GdpbYMVjEBrt6WrUCLSnsIbJoyO7tSDL61ux2R2kxoSd9nFPfZhLTmkD37hoAhkJ4UNaY3l9K802O2PjrX3u53AYfv76AUICA/jaBeNJiHB2a9Y02Xhicy7PfpRHS5ud2WNi+NfBMl7dVcSVM5PZf7KOOWNiENfvXHUtbfzinweJCgtidZfneXlnIVtyKnjo6hlYg51xV1DVxK/eOkRUWBDnTYhn4fj4zv2PlNbzx/eOsulQGfWt7cweE8NjN80hLdaKw2E4WFJHS5udmakx2OwO1nyUx2/ePkxIYAA/fW0f+VVNvH+4HBH49tqd/OWWLHaeqGZbbiU5pfVclZnC8lkp5JQ2UFDdBEBSVCizx8QMxX+GU8hwfWL2JSsry/jNGa9e/x7sfRF+lO8cIqlGpJzSegwweXRk57aWNjtv7iumpc0ZvBdMds5b2J5bydh4K8nRpw9ju8Pw2q4int9+gqToUM6fmMCKzJTOkOrwp/eP8uu3DnNOUiR/vmUeGQnhNNvsXPGHzVQ02PjrnQuYM9a5tNWbe4t5astx/v3SydS1tPGNv+0EwBIgnD8xgbljY8mrbGRvUS12hyE8xML89Dgum5bEwvFxnWHaG2MMW45WUFjdjK3dwe6CGnIrGvmv62eRFmvlykc/pLCqmZ9+cRq3nDsWEeHt/SU8/sExVmamsmrBGEICLfzhXzn8/l9HALAGW7hw8iiSo8N4IbuARls7y2elcM/SSWQkhHP94x9xrLyR8GALJ2tb+M11s7g+awyHS+r5+l+zKahuxhhDSKCFi6aMwhIgvL6nGICrZqfwh1WZvLWvhB+9tAeHK/YaWtsBSIsNIzBAyK9qIjw4kC/OTiY9Ppw/vncUhzEkRoVS1WijtrkNgNCgANrsBrvDcMWMJO6+eCIrH9tKu8NwbkYc12eN4fsv7u48XsGWAEZFhlBU00ywJQBbl28Oy2cl88eb5p7+zdYHEdlhjMnq9/4a+MPs8fPBGg9ffu3M+6pe2R2Gl3cWcm5GfLcWZHZeFfmVTZw7Po602M+3Vzfa2H68imPlznkNc8bEcN7EhD5fI6e0nh++tIcrZiSx+oIJnduNMTy3LZ+H/nkAa3Ag737vAhIjQ2lps/PVNdlsOVrRue/189IIDwnk2Y/ySIoK5e9fX0igJYCtRytYNnU0Atz38h62Hauk3WFostmZmBhBQ0s7JXUtJESE8JXz07l4SiJBFuGlnUX8+f1jXDh5FLsLa2i3G35z3Sw+K6jhic25jI4Koclm52tLxlNQ1cSLOwoJDgyg3e4gODCAKUlR/OnmuTz3UR4bD5VxtKyBhIhg5oyNJSzIQkVDKzvyq2ltd7AgI45/v3Qy546PB6C2uY1PjleRU1aPMfDO/hJ2F9Z2/q3x4cHY7A4SI0NYMmkUz36UR+aYGHYV1DAtOYr0BCtv7C0h1hpEdVMb8eHBzBsXy7sHS7k6M5W7Lp7IUx/msvlIOSdrW7hiRhL3LpvMlKTPP1CPljWw4o9bmJwUSbPNTnWTjRe/fh7X/+UjjIHHbp5LfHgwT36Yy+YjFZysbeabF04gNMjC7949Qnq8lbzKJmakRvGnm+aREhPK3qJatuVWcrjE+Xelx1u5Y3EGseHBAORVNPLn94/R3Gbv/EC0Bgey/Xgl1mAL501IYNH4eAIChMc2HeXJD3N59a7FpCeE879bj1PR0Mp5ExKYNy6WYEsAb+0v4ePcSmanxTAlKRIRiAoNYkxc39+ETkcD35vZGuGXabDk+3DJ/Z6uxiMaW9tptLUTEmghOixoQM/xm7cP8dimYwQGCFfPSeWy6UnsKazhf9472rnPgvQ47liczlv7S/jH7pOdLboOX79wPD+4bAqBXX4E3FVQwz/3nKS13cGL2YW0tNsJEGHDtxYTYw3msU1H+ehoBXmVTZw/MYFP8qpYek4iD3xxOj9Yv5stRyv45dUzuXDKKNZ+UsD/vJeDMbBq/hje3l+CiFDf0kab3RAebCEiNJDqxjaunZdKSKCFhePjuWzaaEQgO7+a3797hI+OVXare0VmCr+9fjal9a3c9fxOdhfUIAKr5o/lW5dM5Nant5Nb3kiAwJcXpXPvskn87B8H+OR4FS9+YxEpXbp86lvaiAgJ7NaSb2mzs+6TEzz2/jHK61tZkBFHs83O/pO13Y5hWmwY37lkEksmJ2ARYVRkCNtyK7nlqe04jPPD7lfXzuKvH+fzxt5i9p+s4+o5qfy/5VP59Hg1L2QXsC23kpToUNatXkRYsLN7yhhDc5v9lG82XesLCQxgR3411z2+jciQQNocDl65azFTk6M69zPG0Nru/OHU4TB8e91n7C6o4e6LJ3LdvLQh+/HX1u78cB0uGvje7PiHsGY53PQiTL7M09UMu10FNdz05Mc02ZxTMyaMCmdGajQWEeaOi+Vm11f/3tS3tFFU08zewlp+sH4PV89JJTosiHWfnqClzfn1+IasNL68KJ2tRyt4estxyupbCQuycPO5Y/m3GUlMT4nCYeCXbxzk+e0niAoNZEFGPIsmxGNrd/Dbdw4TIEJwYACZY2L46fJp3PzUdqJCA6lqstHa5mDxxHiWTR3NDVljeHzzMX791mGCLIIx8PDVM/jS/M/XhdqeW0lru4MLJo9iX1Et31n7GQsnxHPV7BT++nE+R0sb+M31s5iVFnPaY1ZU0+z8BmB3sHB8POPirZ3HyNbu4JdvHuSzEzU8d+cCokKDcDgMbQ4HASLdQs0Y02cXTU8tbXb+9nE+f/04n9FRoSwa7zxOM1OjCbQIwZaAXp/vrx/n89KOQtbcsYBo6+cf6L29/tnW1NM3/7aDN/eV8N9fymTlnL5njbv7Wt5KA9+bffhb2Phz+OFx5wQrH3O0rIHxCeEEBJz6P1Zru50v/s8W6prbufuSidS3OLsIjpU30NZuKKlr4bJpo/nNdbO7BUVpXQv/t/0Ez2w5Tr2rv3VWWjQvfH0RoUEWWtrs7CqowWEM5034vJumpc3OhzkVZI6JYVTkqfMbNh0u4539JXx0rJL8SuePZ8umjua3N8zu9s3jzb3FfPP5nd36zDu02x18e+1nRIYG8u1LJg34a7kamPqWNvYV1bFoQrynS/EYDXxvtvZGqDgC394xLC9XVt/Cz/9xgHuWTmJSlx8Xz6TZZuc3bx9mdFQIX79wwin3l9S2sCO/GoMhKSqUyUmRPPT6AV7ILuTO8zP46fJppzzmd+8c5tH3jvLM7Vlcck73oajGGJ7ecpxH3jyENdjCrYvGUdPUxrZjleRWNAJw+fQkls9OJjAggMUT44kMHVh3UG9O1jRTVNPMvLGxvX5Y9TYqRilvcLaBr8Myh4vDAQWfwKTh68r5j1f389b+EsrrW1m3eiGlda3sPFHNnLExOAzsLqgha1wsiVGhgDPoP82r4hdvHORQST0BAhdNSSQkMIBvr/2MtNgwYqxBvLSzCFv756MMRJxrvs0eE8PTW46TGhPG4ZJ6TlQ1ccfidLLzq3licy5Xz0k9Jeydjxe+umQ8501I4HfvHuGxTceICAlkQUYcNy4Yy0VTRp3VB9bZSokJ69a33VNfXS5KjSQa+MPl5GfQVAHjLxyWl3tjbzFv7S9h9pgYth+v4uktx3nyw1xK67pPBkmNCWPd6oVs2H2SP2zMwdbuINYaxB9WZfLTV/fxn68foKKhlaLqZqoabZTUtXDd3DRuWTiOkKAAcssb+exENedPSuDcjHhuevJjfv76AYIDAxgVEcLqvzq/zdyycGyvLf+upqVE8dRtWZTXtxJrDer2g6pSyn3apTNc3n0Atv0RfnAUwmLPvL8bqhttXPr7D0iKDmX9N87jykc/JLe8kbjwYH597Szyq5oQnCMtvv/iblraHdjaHVwxI4kvzR/D/PQ4wkMCeXJzLg+/cRARePaOBVwwKYE2u+lzFEJFQysv7ShkRWYqCRHBbNh9EmtwIJfPSBrSv1kpf6RdOt7IGDi4ATIuGLSwP1nTzBObc0mNCWNCYjifnajBEiB848IJ/OfrB6hpauO5r5xLaJCFh1bM4JdvHuJX185iWkpUt+f5W3QoP3hxDzcvHMutC8d1G8nw5fPGsTmnnIunJHKhaxJRcGDfIx0SIrr3+18zV5diVspbaAt/qNjb4fgHzhOVRKfBn8+DL/wO5t/p9lNvPlLOPes+o76lnXbX4GhLgGB3GMbFW8mvbOI7l0zke5dNcfu1lFLeS1v43uDI2/Da3Z+fXjAyGRA45wsDerqKhlZKaluYkRrNicomvromm4yEcF765lzCQwLJLW9kRmoU2XnV3Pv3XUwZHcndl0wcvL9HKeUTNPAH27H34O+3wKgpzhOZ1JyAf/0Mxi2GyLPvxy6qaeaGx7dRWtfC2tULeXZrHpYAYc1XFpAU7RxdM9o1yubicxLZ/IOLCQiAkEAdQqiU6k4DfzDVFsG6myFhMnx5w+eTq6atHNDJTQqqmrj5qe3UtbSRHBPK157LpqapjXuWTuoM+566TlpSSqmuNPAHU8470NYE1z7dfSbtAE4W/t6hUr779904jOG5ryzAGhzIyse2khgZwtcvHD+IRSul/IUG/mA6/gFEpji7c/qpsqGVPNfU/uToUEZHhfL7d4/wx01HmZYcxZ9vmcu4eOd0/vXfXESwJeC0C0sppVRfNDkGi8PhPLnJxEv7PPm4MYb3D5fzwZFyth2r5HBpfbf7Y6xB1DS1ceOCMTzwxendpvNPT9ETpiilBk4Df7CU7YemyjPOpN1TWMsdz35KaFAA89PjuCozhWnJUYjAMdes1aVTE7l6jo5fV0oNLg38wZL7gfMyo+/A/+R4FQAf/ODiztE1HS6aApAxBMUppZSexHzgjHH+63D8A4ifeMYfaLPzqxgbZz0l7JVSaqhp4A/Uh/8F/zMXqvOgdL/z5CZnaN0bY9iRX03WuKFdS0cppXqjXToDtev/oCoX1nwR2logLAYW39PnQ/Irm6hosDEvXQNfKTX8tIU/EJXHnGE/51ZoqgbjcE60ih3X58Oy86sByBrne2e7Ukp5P23hD0TOO87LJf8OF3wfLMEQlXLKbq/tKmLjwTL+sCoTEWFHfhVRoYFMSowY5oKVUkpb+AOT8y7ET4K4DIhN7zXsAd7eX8KG3SfZeLAMgOy8auaO6/00ekopNdS0hX+2bE2QtwXmf/WMuxZVNwPw6Hs5VDXZyClr4KZzxw51hUop1Stt4Z8NY2DH/4K9FSYtO+PuhdXNxIUHs6ewlvte2sPiifHcurDvfn6llBoqGvj91dYML94Ob/8E0pfAuPP73L3ZZqey0caXF40jNSaMpKhQHl01R8/TqpTyGO3S6a93fgoHXoWlDziHXwb0vd58UY2zOyc9PpyX7zqPIEsAceHBw1CoUkr1TgO/Pw6/CZ8+CYu+BUu+16+HFFY7V8BMiw3TWbVKKa+g/QtnUnYIXr0LkmbC0v/o98M6WvipsWFDVZlSSp0VDfy+VB6D51aAJQiuX3NWZ60qrG4mMEBIjNTWvVLKO2iXTl9evQvsNrjjDYifcFYPLapuJiUmDIuOuVdKeQm3Wvgi8l0R2S8i+0RkrYiEikiciLwrIjmuy5G5cIytCYqyYd7tkDj1rB9eVNNMaox25yilvMeAA19EUoHvAFnGmBmABVgF3AdsNMZMAja6bo88xbvA0Q5p8wf08MLqJu2/V0p5FXf78AOBMBEJBKzASWAFsMZ1/xpgpZuv4RmFnzovzyLw7Q7DW/uKqW1qo6y+VVv4SimvMuA+fGNMkYj8F3ACaAbeMca8IyKjjTHFrn2KRSSxt8eLyGpgNcDYsV643EDBJxCbARGj+v2Q57fn8x+v7Wfu2BiMcQ7JVEopb+FOl04sztZ8BpAChIvILf19vDHmCWNMljEma9So/ofqsDDG2cI/i9Z9XUsb//2vHOLCg9l5ogbQIZlKKe/iTpfOMuC4MabcGNMGvAycB5SKSDKA67LM/TKHWW0BNJTCmAX9fshjm45S3WRjzR0L+MKsZMA5y1YppbyFO4F/AlgoIlYREWApcBDYANzm2uc24DX3SvSAgk+cl/1s4Te2tvPs1jxWZqYyMy2a314/m5e+uYgU7cNXSnkRd/rwt4vIemAn0A58BjwBRAAviMidOD8Urh+MQodV0U4IDIPR0/u1+8e5lbS2O7h2bhoAoUEW5ulZrZRSXsatiVfGmAeAB3psbsXZ2h+56gohZoxzhm0/fHCknLAgC1l6rlqllBfTpRV601gB4f3/IXnzkXIWTYgnNKjvFTSVUsqTNPB701AG4Qn92jWvopG8yiYumNS//ZVSylM08HvTWA7hvU4fOMXmnHIALpzSv/2VUspTNPB7ardBS02/unSMMby9v4SxcVbS461DX5tSSrlBA7+npkrnZT+6dNZ8lMfWo5XcfO5YnCNTlVLKe2ng99TomicW0XcXzY78ah7650GWTR3N15aMH4bClFLKPRr4PTU6++TP1KXzzNbjxFiD+O0NswnQNe+VUiOABn5PjRXOyzME/p7CGhZkxBEd1r+x+kop5Wka+D11tvBP34df1WijoKqZWWkxw1OTUkoNAg38nhrKwBICIVGn3WVPYQ0As9Kih6kopZRynwZ+Tx2zbPsYdbOnsBaAmaka+EqpkUMDv6fG8jMOydxTWMv4UeFEhmr/vVJq5NDA76mx7JQhmU9vOc7hkvrO23sKa5it/fdKqRFGA7+nHgunHS6p5z9fP8D9r+zFGENJbQtl9a3af6+UGnHcWh7Z5xhzSpfOq7uKAMjOr2ZbbiXFNS0AOkJHKTXiaOB31VoHdlvnwmkOh2HDrpMsGh9PbkUDD7y2n4LqJs5JitQfbJVSI4526XTV0H2WbXZ+NUU1zXxp/hhWXzCBnLIG0mKt/O2r5xIcqIdOKTWyaAu/qx6Trl7dVURYkIVLp43GEiAYY7gqM4WEiBAPFqmUUgOjgd9Vl3V07A7D2/tKWDZtNOEhzsP0VV0kTSk1gmm/RFfVec7LqBR2FdRQ2Whj2VQ9sYlSyjdo4HeV9yHET4LwBDYeLMUSIFw0WQNfKeUbNPA72Nsg/yMYfyEAGw+WMT89lmirzqZVSvkGDfwORTvA1gAZF1BQ1cTh0nqWTR3t6aqUUmrQaOB3yP0AEEhfwsaDpQAs1cBXSvkQDfwOxz+A5FlgjWN3YS3J0aFkJIR7uiqllBo0GvgAtkYo+AQynP33JbUtpMSEebgopZQaXBr4AGUHwdEGYxcCUFrXQlJUqIeLUkqpwaWBD59PuIpIcq6IWdfCaA18pZSP0cCHLicuj6e+tZ0mm52kaF0+QSnlWzTwAZo6An8UJbXO5Y+1ha+U8jUa+OBs4QeGQXB4Z+BrH75Sytdo4AM0VXaukFlS5wz85GgdpaOU8i0a+OD80dYaD0Cpq4WfGKV9+Eop36KBD67z2H7ewo+1BhEaZPFwUUopNbg08MHZpWN1Bn6pDslUSvkoDXzo1sIvrm0hKVoDXynlezTwbY3Q3vx5H77OslVK+Si3Al9EYkRkvYgcEpGDIrJIROJE5F0RyXFdxg5WsUOic9JVArZ2BxUNNu3SUUr5JHdb+H8A3jLGnAPMBg4C9wEbjTGTgI2u296rY9KVNYGyetcYfO3SUUr5oAEHvohEARcATwMYY2zGmBpgBbDGtdsaYKV7JQ6xxkrnZXgCpXU66Uop5bvcaeGPB8qB/xWRz0TkKREJB0YbY4oBXJe9nhRWRFaLSLaIZJeXl7tRhps6W/jxlNS2AtrCV0r5JncCPxCYC/zZGDMHaOQsum+MMU8YY7KMMVmjRo1yoww3denDz6tsBCA1VmfZKqV8jzuBXwgUGmO2u26vx/kBUCoiyQCuyzL3ShxiTRUQEAQhURworiMtNoyoUD1xuVLK9ww48I0xJUCBiExxbVoKHAA2ALe5tt0GvOZWhUOt0bWOjggHi+uYlhzl6YqUUmpIBLr5+G8Dz4tIMJAL3IHzQ+QFEbkTOAFc7+ZrDK2mCrAm0GRr53hFI1fNTvF0RUopNSTcCnxjzC4gq5e7lrrzvMOqsQLC4zlUUo8xMFVb+EopH6UzbV0t/IPFdQDapaOU8lka+K4+/AMn64gMDSRNR+gopXyUfwd+WwvY6p2B7/rBVkQ8XZVSSg0J/w78RueIUYc1kcMl9UxL0e4cpZTv8u/Ab3DO8C0zUTTZ7PqDrVLKp/l34Lta+GV2Z9CPibV6shqllBpS/h34Da7AN87Aj48I9mQ1Sik1pPw78F0t/OJ2Z+DHhWvgK6V8l38HfkM5hERR3uIcmRMTpmvoKKV8l38HfmMZhI+iqrGVGGsQgRb/PhxKKd/m3wnXUA4RiVQ3tml3jlLK5/l34Lta+JWNrcRr4CulfJyfB76zhV/VaNMWvlLK5/lv4NvboLkawjsCP8TTFSml1JDy38BvdM6ydYSPorqpTbt0lFI+z38D3zXpqikoDrvDaJeOUsrn+W/gu1r4NRIN6CxbpZTv89/Ad7XwqyQGgFirBr5Syrf5b+C7llUodeiyCkop/+C/gd9QDkFWyludyylol45Sytf5b+B3WVYBtIWvlPJ9/hv4DWUQkUhlo42IkEBCAi2erkgppYaU/wZ+zQmIStFZtkopv+Gfgd9SC9XHIWmmBr5Sym/4Z+CX7HNeJs2mssGms2yVUn7BTwN/j/MyeRbVTTZiNfCVUn7APwO/eA+EJ+IIH01lo7bwlVL+wT8Dv2QPJM/iQHEdtnYHU5IiPV2RUkoNOf8L/PZWKD8ESbP44IhzPZ0LJo/ycFFKKTX0/C/wyw6Aox2SnYE/IzWKhAhdC18p5fv8L/CLdwNQHzednfnVXKite6WUn/C/wC/ZB8GRbK2IoN1huGCSBr5Syj/4X+DXnIC4dDYfrSQiJJC542I9XZFSSg0L/wv82kJMVCofHC7nvAnxBFn87xAopfyT/6VdXSG1wUkU1TRz4RTtzlFK+Q//CvzWemipJafFeVpD7b9XSvkTtwNfRCwi8pmIvO66HSci74pIjuvSezrJa4sAyK62Mn5UOGPirB4uSCmlhs9gtPDvAQ52uX0fsNEYMwnY6LrtHeoKAfiwNESHYyql/I5bgS8iacAXgKe6bF4BrHFdXwOsdOc1BlWtM/Dz2uM08JVSfsfdFv5/Az8EHF22jTbGFAO4LhN7e6CIrBaRbBHJLi8vd7OMfqotwkEA1YHxnJsRPzyvqZRSXmLAgS8iy4EyY8yOgTzeGPOEMSbLGJM1atQwtbZrC6kKiGXWmHjCgvWUhkop/xLoxmMXA1eJyJVAKBAlIn8DSkUk2RhTLCLJQNlgFDoY7DUFnGiPIyvde35HVkqp4TLgFr4x5sfGmDRjTDqwCnjPGHMLsAG4zbXbbcBrblc5SGxVBZw08WSNi/N0KUopNeyGYhz+I8ClIpIDXOq67XnGENRYTJGJZ+5YbeErpfyPO106nYwx7wPvu65XAksH43kHVVMlgY5W7BEpRFuDPF2NUkoNO7+ZaeuoLgAgcnS6ZwtRSikP8ZvALy44CkDK2EkerkQppTzDbwK/6rMNtBkLk8+Z6elSlFLKI/wi8PP2fsT00n/wYdy1jElN8XQ5SinlEYPyo63Xeft+56kMp6/EHpNB82v3UyuRZN78sKcrU0opj/HNwN//KjSUQt6HWICpwGeZP2dOQq+rPCillF/wvcB32KG+GM6/F/vML3Hvs+9hs0Tw+IpbPF2ZUkp5lO8FfkMpGDtby0Mpyrfyj6qx/OnmuYiIpytTSimP8r3ArzsJwNN7Wnlv1x4mJUZw+fQkDxellFKe53OBX1uaRzQw85yphFqSuPnccQQEaOteKaV8LvDLCo8RDVyQlcl3p030dDlKKeU1fG4cfn3ZCVpMEFPGj/N0KUop5VV8LvDttYVUBCQQEaoLpCmlVFc+F/ghjSU0hemPtEop1ZNPBX51o404RwVE6vIJSinVk08F/r7CKkZTTVjCWE+XopRSXmdEB35pXQtPbs6loKoJgNy8PILETkJKhocrU0op7zPiA//hNw5yuKQegLrS4wDawldKqV6M6MC3BjunETTa2gEIaS513hGlffhKKdXTiA788BALAE02OwDWlo7AT/NUSUop5bVGdOBHNJ7gj0GP4qgrBiCqtRQbQWCN83BlSinlfUZ04IcGB3FFwHamHn8OHHbmtWyjIHgC6MqYSil1ihEd+EEJ43ndLGZG8UuQ/QypjpNsSrjR02UppZRXGtGBD/Cc5RqCHc3w5g/JJY1jcRd5uiSllPJKIz7wS0Iy2BO5BIyDJ81KwkKCPV2SUkp5pREf+OEhFl6MXY1Z/F3W2xZ0jtxRSinV3YhfD98aHEieGU3LhT+lbeNbnWPzlVLeoa2tjcLCQlpaWjxdyogVGhpKWloaQUHurQI84tMxPMRCk83eOflKW/hKeZfCwkIiIyNJT0/Xc0sPgDGGyspKCgsLychwb9mYEd+lYw0OpLG1naZWe+dtpZT3aGlpIT4+XsN+gESE+Pj4QfmGNOIDPzzYQnObnaa29s7bSinvomHvnsE6fiM+8K0hgTS22mnsaOGHaAtfKaV6M+IDPzzYQpOtnSabtvCVUqovIz7ww4IDabLZaWhxBr724Sul3BEREXHGfX7/+98TGhpKbW0tlZWVZGZmkpmZSVJSEqmpqWRmZmKxWJg2bRqZmZnExcWRkZFBZmYmy5YtG4a/oncjPh07WvQVjTbnbR2lo5TX+tk/9nPgZN2gPue0lCge+OL0QX3OM1m7di3z58/nlVde4fbbb2fXrl0APPjgg0RERPD973+/2/633347y5cv57rrrhvWOnsa8S38jj778vpW521t4SuluvjRj37En/70p87bDz74ID/72c9YunQpc+fOZebMmbz22mv9fr5jx47R0NDAQw89xNq1a4ei5CEz4tOxo4X/eeBrC18pbzXcLXGAVatWce+993LXXXcB8MILL/DWW2/x3e9+l6ioKCoqKli4cCFXXXVVv0bDrF27lhtvvJElS5Zw+PBhysrKSExMHOo/Y1AMuIUvImNEZJOIHBSR/SJyj2t7nIi8KyI5rsvYwSv3VB0t+o7ADwvSwFdKfW7OnDmUlZVx8uRJdu/eTWxsLMnJyfzkJz9h1qxZLFu2jKKiIkpLS/v1fOvWrWPVqlUEBARwzTXX8OKLLw7xXzB43GnhtwP/bozZKSKRwA4ReRe4HdhojHlERO4D7gN+5H6pvevosy9vaMUabCEgQMf7KqW6u+6661i/fj0lJSWsWrWK559/nvLycnbs2EFQUBDp6en9mti0Z88ecnJyuPTSSwGw2WyMHz+eu+++e6j/hEEx4Ba+MabYGLPTdb0eOAikAiuANa7d1gAr3ayxTx0t/Ir6Vu2/V0r1atWqVaxbt47169dz3XXXUVtbS2JiIkFBQWzatIn8/Px+Pc/atWt58MEHycvLIy8vj5MnT1JUVNTvx3vaoPxoKyLpwBxgOzDaGFMMzg8FoNfOLRFZLSLZIpJdXl4+4NfubOHXt+oIHaVUr6ZPn059fT2pqakkJydz8803k52dTVZWFs8//zznnHNOv55n3bp1XH311d22XX311axbt24oyh50Yoxx7wlEIoAPgIeNMS+LSI0xJqbL/dXGmD778bOyskx2dvaAXr+gqoklv94EwNTkKN68Z8mAnkcpNTQOHjzI1KlTPV3GiNfbcRSRHcaYrP4+h1stfBEJAl4CnjfGvOzaXCoiya77k4Eyd17jTLqOytFZtkopdXoD7vQW5/ilp4GDxpjfdblrA3Ab8Ijrsv8DXAcgvMvaObqOjlJqMOzdu5dbb72127aQkBC2b9/uoYoGhzsJuRi4FdgrIrtc236CM+hfEJE7gRPA9W5VeAYhgQEECDiMtvCVUoNj5syZnbNnfcmAA98YswU43RjIpQN93rMlIoQHB1Lf2q6jdJRSqg8jfmkFAKtrdI6O0lFKqdPzicAPd7XstYWvlFKn5xOB39nC1z58pZQ6Ld8I/CBXC19H6Sileqipqem2WmZ/XXnlldTU1AzoNdvb20lISODHP/4xAA8//HDnmvkWi6XzuoiQmZnJtGnTCAsL69y+fv36Ab3umfhEQna08HWlTKW83Jv3QcnewX3OpJlwxSOnvbsj8DtWy+xgt9uxWE6fGW+88caAS3rnnXeYMmUKL7zwAr/4xS+4//77uf/++wHnCVZ6jgDKy8tj+fLlQz4yyCda+J/34WvgK6W6u++++zh27BiZmZnMnz+fiy++mJtuuomZM2cCsHLlSubNm8f06dN54oknOh+Xnp5ORUUFeXl5TJ06la997WtMnz6dyy67jObm5j5fc+3atdxzzz2MHTuWjz/+eEj/vrPhGy384I4+fJ/4c5TyXX20xIfKI488wr59+9i1axfvv/8+X/jCF9i3bx8ZGRkAPPPMM8TFxdHc3Mz8+fO59tpriY+P7/YcOTk5rF27lieffJIbbriBl156iVtuuaXX12tubmbjxo385S9/oaamhrVr17Jo0aIh/zv7wzda+CEdffjawldK9W3BggWdYQ/w6KOPMnv2bBYuXEhBQQE5OTmnPKbjfLQA8+bNIy8v77TP//rrr3PxxRdjtVq59tpreeWVV7Db7YP9ZwyITzSJtYWvlOqv8PDwzuvvv/8+//rXv9i2bRtWq5WLLrqo13XxQ0JCOq9bLJY+u3TWrl3L1q1bSU9PB6CyspJNmzZ59OTlHXyqha8Tr5RSPUVGRlJfX9/rfbW1tcTGxmK1Wjl06JDb/e11dXVs2bKFEydOdK6Z/9hjj3nNuW99oknc0cIP0xa+UqqH+Ph4Fi9ezIwZMwgLC2P06NGd911++eU8/vjjzJo1iylTprBw4UK3Xuvll1/mkksu6faNYMWKFfzwhz+ktbW123ZPcHs9/MHgznr4AEU1zfz90wK+u2xSv05CrJQaProe/uAYjPXwfaJJnBoTxvcunezpMpRSyqv5ROArpdRwu/vuu9m6dWu3bffccw933HGHhyo6Mw18pdSQM8b4XHfrY489NmyvNVhd7z4xSkcp5b1CQ0OprKwctNDyN8YYKisrCQ0Ndfu5tIWvlBpSaWlpFBYWUl5e7ulSRqzQ0FDS0tLcfh4NfKXUkAoKCuo2s1V5jnbpKKWUn9DAV0opP6GBr5RSfsIrZtqKSDmQ78ZTJAAVg1TOcNGah4fWPDy05uHRs+ZxxphR/X2wVwS+u0Qk+2ymF3sDrXl4aM3DQ2seHu7WrF06SinlJzTwlVLKT/hK4D9x5l28jtY8PLTm4aE1Dw+3avaJPnyllFJn5istfKWUUmegga+UUn5iRAe+iFwuIodF5KiI3OfpenojImNEZJOIHBSR/SJyj2v7gyJSJCK7XP+u9HStXYlInojsddWW7doWJyLvikiO6zLW03V2EJEpXY7lLhGpE5F7ve04i8gzIlImIvu6bDvtcRWRH7ve34dF5N+8qObfiMghEdkjIq+ISIxre7qINHc53o97Uc2nfS948XH+e5d680Rkl2v7wI6zMWZE/gMswDFgPBAM7AamebquXupMBua6rkcCR4BpwIPA9z1dXx915wEJPbb9GrjPdf0+4FeerrOP90YJMM7bjjNwATAX2Hem4+p6n+wGQoAM1/vd4iU1XwYEuq7/qkvN6V3387Lj3Ot7wZuPc4/7fwv8hzvHeSS38BcAR40xucYYG7AOWOHhmk5hjCk2xux0Xa8HDgKpnq1qwFYAa1zX1wArPVdKn5YCx4wx7szeHhLGmM1AVY/NpzuuK4B1xphWY8xx4CjO9/2w6q1mY8w7xph2182PAffX7h1EpznOp+O1x7mDOM8ecwOw1p3XGMmBnwoUdLldiJcHqYikA3OA7a5N33J9JX7Gm7pHXAzwjojsEJHVrm2jjTHF4PwgAxI9Vl3fVtH9fwxvPs5w+uM6Ut7jXwHe7HI7Q0Q+E5EPRGSJp4o6jd7eCyPhOC8BSo0xOV22nfVxHsmB39v50rx2jKmIRAAvAfcaY+qAPwMTgEygGOfXNW+y2BgzF7gCuFtELvB0Qf0hIsHAVcCLrk3efpz74vXvcRG5H2gHnndtKgbGGmPmAN8D/k9EojxVXw+ney94/XEGbqR7I2ZAx3kkB34hMKbL7TTgpIdq6ZOIBOEM++eNMS8DGGNKjTF2Y4wDeBIPfIXsizHmpOuyDHgFZ32lIpIM4Los81yFp3UFsNMYUwref5xdTndcvfo9LiK3AcuBm42rY9nVLVLpur4DZ3/4ZM9V+bk+3gvefpwDgWuAv3dsG+hxHsmB/ykwSUQyXK26VcAGD9d0Clff29PAQWPM77psT+6y29XAvp6P9RQRCReRyI7rOH+g24fz+N7m2u024DXPVNinbi0hbz7OXZzuuG4AVolIiIhkAJOATzxQ3ylE5HLgR8BVxpimLttHiYjFdX08zppzPVNld328F7z2OLssAw4ZYwo7Ngz4OA/3L9GD/Kv2lThHvRwD7vd0Paep8XycXw/3ALtc/64E/grsdW3fACR7utYuNY/HOWphN7C/49gC8cBGIMd1GefpWnvUbQUqgegu27zqOOP8MCoG2nC2LO/s67gC97ve34eBK7yo5qM4+7073tOPu/a91vWe2Q3sBL7oRTWf9r3grcfZtf1Z4Bs99h3QcdalFZRSyk+M5C4dpZRSZ0EDXyml/IQGvlJK+QkNfKWU8hMa+Eop5Sc08JVSyk9o4CullJ/4/0khLJE2uTO7AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_ATT','train_ATT']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA8RElEQVR4nO3dd5xU5fX48c+Zme2dLbDUXapIhwVRFDVYEEVULKAxGo2aqIklGkkVfzHfJJpoNDEmmNgVVCyIigrGioouHQQEZAuwvfed8vz+uLONnaXsbGP2vF+vfc3MrWcuw5lnzn3uc8UYg1JKqcBn6+4AlFJKdQ1N+Eop1UtowldKqV5CE75SSvUSmvCVUqqXcHR3AAAJCQkmJSWlu8NQSqnjyvr16wuNMYlHu3yPSPgpKSmkp6d3dxhKKXVcEZHMY1leSzpKKdVLaMJXSqleQhO+Ukr1EprwlVKql9CEr5RSvcQRE76IPCki+SKyrdm0PiKyWkR2ex/jms37pYjsEZFdInJuZwWulFLq2BxNC/9pYPYh0xYBHxhjRgAfeF8jIicCC4Ax3nX+KSL2DotWKaVUux2xH74x5hMRSTlk8jzgDO/zZ4CPgHu805cZY+qAfSKyB5gGfNFB8aoO9O62HMb0j2FQn/BW89IzirHZhMmD43ysaVmfWYxNhEmHWcZf2cXVLF+/H1/DeIfXFzE2fwUOj5P4yBBGJEVaM0RwnzCXpZnR5JfXEh0WxA9npGK3CeszS6h3eTh5WHzrneXvhO2vg/E0bofRc6HfuKZlCnbBgQ0wYQGIsOabPIbFB5G672WoLgKgvNbJNznlYOBg1Hgy+pzCRRP7MzQxksriHL798HnGnvdjgsOjWuze4zEs+zqb0sIcxuW9QZCnjrKQZLYnXcApI5KYPjQej7OOLSv+xqioOsKC7FTUOVlXFMGWxAs4ZXgi06s/smJsEBIFU6+H4AjrtTHWe+wzFPpPpKokn+0rH8FTX0NcRDCj+lox7SusIq+8FptNGNs/mvBgBxV1Tr45WNHi32JPn5mUxo7hynFR9Nn5IjhrmvY9YDKMOq/pdXUxrH+6cZm8ijoKok5k7PcWgNtpzavM9/k5yCiqIres1uc8gP6xYQzuEw4jZ8PAKW0u13gMNi+F4n2t56XMgKFnNL2uKYWtr8D4yyE0hu82fohtz2pS4iNgxDkwaGrTsiUZkP0VjLvM+uw07GvLS1C0t+14kkbD2EsOH3MHkaMZD9+b8N8yxoz1vi41xsQ2m19ijIkTkX8AXxpjnvdO/y+wyhiz3Mc2bwRuBBg8ePCUzMxjun5A+Sm/vJaT/vgBc8Yl89iVk1vMM8Yw40//w2G38fHdZyANH95Dljn1zx8SFerg3dtndkqMdS435z/6GXvyKzk0hBGynyeDHmCgFOIx1kwRsJ4ZnLZQbq69mdWeNACeunYqZ4xK5My/fEReeR3v3zGz5Rfd7tXwyrVQX0nDVsCAIwzmP2El/t1rvMtUwNhLWTfh99z03094OvxRJnq2A4Kx1gIDNrH+b/3VeSkroq/ivav7UfXUfBKcB8mNHE2/m96AqH6NITy1dh/PvbWGp4IeYIitKfG95Z7Or8zNvHnTFEJeu5bkkq/xIEjjHuEt90mUSxRX2tZ412r2HpInwMKXICIRVt0N6U+CPQTOWkzBh4+RWL+/xTEEK0818h7XhvfVwCaGOhPEn10LuCHsQ5Jd+1vuF+C0u+DMX0PJPnjxcija02qZ4ok/oU/5Tvjuw2bzmphD42mDTQzYgmDeYzDhirYX3P669e/Y8Oaa78keDLess74QSzLghcuhcBckjoaJV+JavRgHbu8Og+DCR2HilZD5BSy7EmqK4Yrnrc8LwI6V8NL3feyrmbGXwKVPHvkN+iAi640xaUe9fAcn/MeALw5J+O8YY1493PbT0tKMXmnbtZ79IoPfrdhOeLCdDb89m9AgO7jqYdUvqN63jvNzrmefSeatn57K2AExLVf+8l+419xHvdPFG54ZzP3NciJDmv1YdDvh3V9C7hb4wZsQFNo6ALcT3l0EWV9aH/bEUdb0PR/A23fCST/moZKZxK+9l/kJ2URe9SyIzfqPWrQX3HUQkQRXLqMuaTznP/oZ1XUu3rtjJsV5+yl9cj4TZC/GEUaty41dBIdNqHNZrXexCSGTFiJzHoQNz8CqX0DfsXDlSxDd34qlsgCWLYT9X1uJ31VjtfZHzoZPHqQeBzYMbmPj7WG/5ZIf3MYja3bz8JpveeIHaZw9Mhbe/BlsWUatCSJY3BSZSF6wX8wN7mWE292ILQgAD4Y6p4cQcSER8cjCZTBgCnz+d8zq3+HEjg2Dxwj3O27m2arpnJTah3X7inh32mZGbXkQwfBaxBVcdOfj2OzeSuq378PyH1qtapsd3PVw8q3Wr5SszykyUawa+xBXXHIpFz22ljzvL6I6p4f37pjJaxv287sV2737KuaxKydz/vhka9vVxVYyy1xLsYlk4yn/ZNa585r+fd++EzY8a325eFwQGgMLXoQhJ5NZVMWsB9fw/xxPc6XjfxibA5n7CExqSI6WepeHuX//jLIaJ+/fOZPo0KBWH6XiqnrOfuhjRsW4eD76MWwZn1r/XodKngAX/ROeuRDC4uCmj61j0qAiF/4+BVJnwql3wtIF4HHCzF/Axw9AXRmfuMdxi/M2TkyOZGnsv7Dt+8j72aiF+GHWZ9RVC7d8ZW3zsWkQHAU3fQL21gUVY4zPBtXR6qqEvws4wxiTIyLJwEfGmFEi8ksAY8wfvcu9Byw2xhy2pKMJvwMV7AKbw/rwHcaCJV8Qnv0JW5yD+MPV3+PcYeHw0lWw7xPqbOHUuIWbXHeRNnMOP5qRwqsvPcWZ58xjWGQ9/GMa2WGj2F4Wwmz713wz6ylOPOVC2P46efm5FHz9KmPrNgLgOfM3mNPu4r6V29mRUw5AuKeKRZV/ZHT1eus/g80Gp93FR1v2cFrec9RJKOGmmiyTxGDJt5YRm9X8tDlg4kJwhMKUayFmIAAbskqY//jnpCZEUFPvxlVXzf9O3UmUp5z/7cwns7ia0cnRbM4uJS0ljozMDObbPyXX3o9+7lzSQ6bxSMwiam0tE0WwqeO8qhVEmQqqJIJV4fOotYUxvPxLhpSvZ+6E/qyon8qD2yKYMjiOzftLOW9sMo8unGRtwBjY9AKffrGWTQerWRd3AQ/dMJefPvI8c8ynxIZZSaCy1kWdy8NlJw0l6uTrIG5IUxB7/8fOtW/y0beFfGQ7iQdvv547XtpEemYJ549L5rGrJsPe//H5jkyu/CyJMf2jiQsP5p7ZJzBuYAyb0teS8dHTiPHwbdAJfBU6A4epZ1L+G2wKP5knb7+U0CA72w+WMe8fa3F5DM9cN43TRybi8RgWLPmSrzKKmT2mH49/f3LLBOWqw/P1k9y2oS8fFUSw+o7T6RcT2vTet7yEM2cbH+4u4U05k7roFO6/aCyvbzzAn1bt5BfnjuTb1U9y5slTmTf3ErKKqlm8cjsVtU4Aymtc7Mqr4L/XpDFrdN82P89vbTnIrS9u5NfnDuOGkDVQmddyAY/L+vJx1YHHyb+H/p2pZ8xl8uA4vs4o5rUN+7l37hhC1z0KaxZbrfeYAeTOfY4H0j3ck2Zn77q3+MHmE/n57LH8+d2dLDpnKD8O+xAqciA4kk39L+PrLz7mhn23w4SF1n43L4VrVlpfIj7c+uIGRiRFcdtZI9p8b4fTVQn/QaDIGPMnEVkE9DHG/EJExgAvYtXt+2Od0B1hjHEfbvua8DtIdbHVQrEHw0/TrfqtDwVl1ax88Dquc6winRN5ftRj/C36BUh/EjPvMa54Fx5x/YFQTzXfD3uMq6I2cWXun8mwD2HwkKFI9jrm2x/BFpnEg/k3EhsRStyAkbBnNQD1OFgSdSvDStdyVtBWXj1lBYvWFDFlSBwDyOfuot/Rz7mf5f3vYuHlV1ktqfxvAFjnSGNJ4q+4uOJFzq1agfPcBwk/YRa8uACM22qBx6X4fF8vrMvk7S052ES4/rRUzhyVBMDqb/K44dl0Qhw2pgyJ4/nrT+JP7+4kdver/KjkEf4XMYfnYm7EHGP/grNG9+W6U1OpqHVy74rt5JbXEh8Zwn0XjqFPRHCLZSvrXPzh7R384OQhjE6O5vO9hTz+0V7cHuv/nwhcNyO1zaRmjOEv7+9i4qA4zj6xL5lFVTz+0V7uOncUCZEhjcs8+N4uNmWXsiu3gpiwIF644STm/v0zHDYbQxMjWmwzxGHjrnNHMaZ/0y+45ev3U1xVx40zmxoM2cXVPPbhHn5+zigSo0J8xpdRWMXsRz5hxrAE/nNNWosvhftWbueptRlMH9qHTdmlzBiWQEFlHQArbpnBDc+u59PdBbxz22n88rWtbD9QxoRBsY3rnzoigZvPGH7YfwtjDDc9t56Pvi1g1W2nMSwxsvVCuVth6UK2BI3jwv3fZ0BsGK/+5BQuemwtueW13HLmMO6elQpLzoTQGMzlz3LV0r18vreI00YkUF3vpqrOxbu3z+TmF9azZkc+7/zsVIYnRVFR6+Tchz/hYFktKwe+wLjCt619TrgSLn7cZ8zvbM3h5hc28IvZo474/trS4QlfRJZinaBNAPKAe4E3gJeBwUAWcJkxpti7/K+B6wAXcLsxZtWRgtCE30HeuRu+/g8YD2bG7cgJ58Nbd0JtGUQnw+XPQVRf9j7xA4YdWEF13zTC89L5l7mEm2xvIFN/xLYJv+GCv3/GE7OEs9cu5HnXLM61p1MXFEu0s4BoqSZnyt2cvHYSf54/jk0fvMwfa38PYud/Q+/mru0pPHjldL43LoW7/7OS+/dfRy0huBwRxEcGI9XFYHfwyrD/4+71cTx5bRrfG5nAf95fz2Mf7WXVoovoF+ttZbudYPf+hPc01E2PvdNXrdPNlN+vpqreze8vGsvV05u1npvvI4Cs3VPIVf9ZR3xEMGU1Tlb+9FRGJ0d36j7/8+l33P/2Dh6+YgJzx1tlsQ1ZpVyx5At+MH0I980by38/28fv37K+3O+ZfQI/OWMYeeW1nPXQxwTbbRRV1fPApeO5PG3QMe8/37udkX2jePGG6Thsgs3W9MXjcnvYnFXM5Uu+4KShiXzxXRF9woMpqa7npNR4vsoo5vWbT+HEvuEgdl5ef4Bfvb6VU4cn8NmeQgDuPHskP5s1goKKOs5++GNSEyJYduN07lv5Dcu+yuKk1HjW7StkxbUnWMc7PJ5WJ6CA0hons//2CckxYbx+8yk47O27JKpTWvidTRN+O+39EGpLred1FbDydkj7IfU1lci25YjYcMQkkx01kX7Zb+MeexmhU66EZ+byQtB8rly0hKq/zyCydCd1wbGE3L6R375/kBe/yiL912cR+s7PCNu+FA+C+7oP+OMHWUTtWcm/3HNx2UL4+tdncf/b39B/59Ncfv5sZr3mYc64ZB5ZYJUzDpbW8H8PP8TZrOPsMX0JD3KAIxim30J93PAWtdkF//6SsGA7r/7klE45VD9dupG3thxk3a9mkRTl45xCAPrla1tY+lU2t80awR1nj+z0/bk9hkv/9Tkbs0pbTB8QG8b7d8wkIsSB22O47F+fsyGrlI/uOoOUBOtXx8vp2fxi+RZOG5HAs9dNa3dde/n6/dz1ymYAokIdPHXtVNJS+vDL17ay9KssAPrHhPLeHTN5aPW3PLU2g5tmDuXmM4Zz9sMfk19R12J7pwyL57nrT2LhE1/y1b5iVt8xkxHenkxvbDzA7S9talz2htNS+emsEZz90Mfklbfcji8Om/j9RawJv7fI2w6PH5IcI5LglnXc98ZGbtt1NbvNQOrmP8stb2TxE+cz/NjxFiZmIAdK6/jPhGUsviQN177PMM/M40+2H3HKZXfyo2fT+f5JQ/j9RWOhMp/6R6dRP+pCIuc/Snmtk5e/zqam3s2IvpHMHpvMc19m8ts3tpESH05FrYvVd57eopyxObsUoMVP9ObzLv7nWk4bkcjH3xbwm/NH86PThnbK4TpQWsOOg+WcdWLbdeBAU13vYvU3eZw3NplgR9dcVJ9fXsvyDftxu5tKVeeP709qQlM5KbeslvTMYi7w/goAqyTzztZcTh4W36ocdiyMMbyx6QD7i2tY9nU2wQ4bt581gtuWbWLuhP6MTIrkvHHJDE+KpNbp5r3tuZw7ph+hQXZ251Xw7rbcxm0FOWxcNmUg8ZEh5FfU8uV3xVw4oWXMb24+SFZRNTHhQVyeNojQIDt78itYtTXXV3gtTBkSxynDE9r9XkETfu+x7VVYfh0sWAp9Uq1p0f35KLOOa5/6mhunxbPim3Lyq5wE2W1EmCo+Db+bSGcxN9Xfzjnzb2D+FOuE57dZBzn/35tweQz9Y8J4746ZTb1u6iogONLnz1KA7QfLOP/RzwBa9uA4Sn9ctYN/f/wdAGsXfY8BsT56VyjVDg1lLREY3S+aFbfOIKidpZOe6lgTfmC9+96k4UKOoWdYF24kjYbQGP60aifDEiP4+YVTuf+SCRhj1R2H9E/mr1H3sHXEzbznmcqkwbGNmxo5uD+3nmn1EvjjJeNadrEMiWoz2QOM6htFbHgQ543tx5xx/dpcri13nDWSoYkR1gldTfaqA80YnsCVJw3GYRMeuHR8wCX79ugRd7xS7VC0F6IHQHDTxUPfFVSyM7eCe+eeSIjDztkn9vXWrEPIK69l6VdDKO93MjFheS1+YgP8bNZwFk4bRFL0sdW3HXYbq247jT4Rwe2qu4YG2Xn9JzMwdP8vTRV47p83ltvPGtFrztsciX7l9UAej+H5LzOpqW+7N6sp2kuOvT9l1c7Gaau89cfZY5ta2n2jQxERJg6Kpdbp4Z2tOUwaHNsqOYvIMSf7BskxYYQ42j9kUkx4ELHh7a/bKtUWm0002TejCb8H2pBVwm/e2MbrGw+0uYyzYDf/K4jmNysaBzFl1TYrmSfHtC6NNIyJU+N0M2lQ5419o5TquTTh90Df5lUCsDGrxPcC1cUE15eyz/Rj5eaDvL89l6yiarYdKGfOWN8nTQfGhZEQabWim9fvlVK9h9bwu4gxht+u2MZ5Y5OZcYSuWLvzKwDY6O3S2GjHSrAHUx8SSzDQL3UMo6ui+fkrmxuvtmxezmnOKuvEsWZHns8ukkqpwKcJv4scLKvl+S+zyCyqPmLC35Nf2fhYVuMkJiwIPB546w5A2DvmDkYD48ZP5vQhY3nwvV3UON2cO6afz6GOG1x3agrjBsRY21NK9Tqa8LtIQ3nm871FlFTVE3eYi0t251WSHBNKTlktm7NLmTkyEXI2QlUBANFbnsSNMHHCREJCwljyg6PrhnvKsAROGebfhR5KqeOX1vC7yMasUkSsy89Xf5PX5nLltU5yy2u5eNIARGi6TP3b9/EgFJkoBtTupjSoHyEh2m9dKXX0NOF3kY1ZJaQNiWNQnzDe2ZbT5nIN5ZzJg+MYkRTJxmzrl4HZ/T5bzHA+jZoDQET/Ezo/aKVUQNGE3wXqXG62HSxn8uA45oxNZu2eQip3fQJ/n0JVcQ6ZRVWNy+7x9tAZ0TeSyYPj2JhViqnIQw5uYI1rIpHTrwUgtG/nD4allAosmvC7wI6cCupdHiYNjuX88ck43YYNX6yGoj28/tSDXP7vpvvD7M6vIMRhY2BcOJMHx1FW4yQn/U0APvRMYuToiXDZ03Dyzd3zZpRSxy1N+F2g4YTtpMFxjB8Yy4UT+vPtXmvAsJPL3iGvvLbxqtrd+ZUMS4zEbhPOPCGJGbZt9PlsMQXBA8gNG8GgPmEw5uI2bwKilFJt0YTfBTZmldI/JpS+3qELFl84hn4O63Z/w2w5pMkucstrAauHzuyovfDYSSQumcizwX8ix8Tz06D7mDQkzq/7XyqlejdN+F1gR045Y5rdCLxPRDDTkzxk2AbjdESw0PEhOWU11DrdTC1fzS3ZP7duNj18Ft+kXMPc6t/yZVE4kwbrkAhKqfbTfvidzBhDdkk1p49MbDE9gTISho+hPCSJi7c8y7b1/6J6h4u/Bf+TgvhpJF7/MoTFkVReS+X/fQDAJL1CVinlB23hd7KCyjpqnR4Gxx9yBWxVPkQmEjT7/1jlmcr4bx6kT/rDvOKaSfac5yHMas33jQ4lbUgcIjBeE75Syg9+tfBF5DbgBkCAJ4wxfxORxd5pBd7FfmWMecevKI9j2cXVAC2HPPB4oKoQIpIIi4jkN46fE9TvQ2Kio7l78wS+SoxpsY07zx7JxuzSljcmUUqpY9TuDCIiY7ES+zSgHnhXRN72zn7YGPOXDojvuJddXAPAoLhmCb+mGIwbIpMA6BsTzsuhl5ESFU5oUCaJ3oHQGpwyPMHve18qpZQ/TcbRwJfGmGoAEfkYuLhDogogWd4W/sC4ZsMgVOZbjxFWXT85JpTc8hpsYn0xaE8cpVRn8KeGvw2YKSLxIhIOzAEGeefdKiJbRORJEfHZtUREbhSRdBFJLygo8LVIQMgurqZvdAihQc3uCFXlTfjeFn6/mDByy2rJLqlh8GFGu1RKKX+0O+EbY3YAfwZWA+8CmwEX8DgwDJgI5AB/bWP9JcaYNGNMWmJioq9FAkJWcXXrJF7p/YKLsBJ+ckwohZX1ZBRWHXZ4Y6WU8odfvXSMMf81xkw2xswEioHdxpg8Y4zbGOMBnsCq8fda2cXVrZN4Ywvf+qLrF2NdkFXjdGvCV0p1Gr8SvogkeR8HA5cAS0Wk+T32LsYq/fRK9S4POeW1DI4Ntlr1VUVgjFXDtwdDaCxgtfAbDIrTIY+VUp3D335+r4pIPOAEbjHGlIjIcyIyETBABnCTn/s4Liz9KoupKXEMT4oiv6KWd7flcsqweIyBK/Yugs8/thY8+/9ZNzKJSATvydnmCb9Vf32llOogfiV8Y8xpPqZd7c82j0dZRdX88rWtDE+K5K2fnsrtyzbx+d4iThthdaVMKN0KQ06F6kLY/jqEJzT20AHrpG2DFt03lVKqA+mVPB1glfeGJnvyK7n831+wZX8ZA+PC+HR3IdFUElRXDCPPBVcdfHg/RA+Evic2rh8Z4iAqxEGww0aEXlyllOokOrRCO7215SC/WL4Zj8fwzrZcJgyMYf7kgWzZX8bJQ+N57SenEBMWxEiH9wRt/DAYeY71vHx/Yw+dBv1iQhmoJ2yVUp1Im5PttPSrLNbuKSImLIjN2aUsOu8EFk4dTFx4ED88NZWk6FD+vnASzo27YAfQZxgkjoLIflCZ29hDp8Gt3xtOiMPue2dKKdUBNOG3g9tj2Jxdhk3giU/3AXDe2H7EhAfxmwuaSjUzRybCgUrYIdAn1TpJO+Js2Phcqxb+vIkDuvQ9KKV6Hy3ptMOe/Eoq61z8/JxRRIY4GNM/miHxEb4XLtoDsYPA4R0fZ+S51mNkku/llVKqk2gLvx0ablk4Z1wyp41IICzoMKWY4r1WOafBiHNh1r0w4pxOjlIppVrqdQm/tLqe0monKQlttMiPwsasUmLDg0iJP8JAZ8ZA0Xcw/vKmaY5gOO3Odu9bKaXaq9eVdB5a/S0/ePIrv7axKbuUSYNijzyqZVUh1JVZPXSUUqqb9bqEn11cTWFlXbvXr6h18m1+xdHdX7Z4r/XYRxO+Uqr79bqEX1BZR3W9G4/HtJpXWFnHU2v3YUzreQ227C/DGJg0OPbIOyvyJnxt4SuleoDel/ArrNZ9Vb2r1by3t+Rw38pv+K6wqs31d+SUAzCmv/c2hF//FzYv871w0W6wOSB2iH9BK6VUB+hVCd/jMRRW1gNQVeduNb+4yprXcJcqX7KLq4kKdRAXHmRN+Pzv8MHvrRO0AOXWMAsYA7tWQf/JYO9158aVUj1Qr8pEJdX1uL2lnMq61i38shonAPsPl/BLappuQ+hxQ1k2eFyQ/w24auGJWXDho5B4AhTshLmPds6bUUqpY9SrEn5Bs5O1VT4Sfkn1kVv4WcXVDE+MtF6UH7CSPcDu9701ewNrFsOQGRAUAWMv6ajwlVLKL72ipNNwgrahfg9tJXyrhZ9dXONzO8ZV772DlXc445JM69EWZA17vO01GHwy1JTAjjdh7MUQEtWB70Qppdov4BN+dnE1o3/3LpuzS1skfJ8lncO18PO2wx8HMtr9bdM9aku9CX/MRZCzGZxV1g1OJl9jTW94VEqpHiDgSzrbD5ZR5/LwdUZxY/0efPfSaWrhV2OMaXlhVfqTiLuO4bYDTfedLckEscGUH8LWV6y6/cCp0HcsnHABDOrVt/NVSvUwAZ/wG1rru/MqiQpteruVPnrplFTX47AJFXUuymqcxIYHWzOcNbDlFQDiqGhK+KWZ1s1MBk+3SjlTrrVGxAwOhxFnder7UkqpY9ULSjpWPX53fgUFlXUkRFpJ/NAavsvtoaLWxci+Vs29RVnnmzetIRKAPlLBgNhmNfy4IWCzw3XvwoQFnfxulFKq/fxK+CJym4hsE5HtInK7d1ofEVktIru9j0cxBkHnaUjce/IryS+vY3CfcERaJ/yGLpnjB8a0WA+wxq+PS6Xc0YcBwdWENoyOWZKhF1UppY4b7U74IjIWuAGYBkwALhCREcAi4ANjzAjgA+/rLvXBjjy+/K4IsOrxAOW1LnbklpMUFUpEsKPVSduG+v04b8Jv7KnjqoeMz2DsJRQTQ78g73RnjXXnqriUzn9DSinVAfxp4Y8GvjTGVBtjXMDHwMXAPOAZ7zLPABf5FWE7/PX9b/njOzvweAz7S2oaW+2l1U4So0KICLFTfUgNv6zG6qEzMC6cPhHBTS38yjzAQOwQCt0RJNgqreml2dZjnLbwlVLHB38S/jZgpojEi0g4MAcYBPQ1xuQAeB993tpJRG4UkXQRSS8oKPAjjNZcHg/bD5aTVVxNvdvDmaOaQrASvoPKQ3rplFRZLfzYsCAG9Qlnf0nzhA/O8CTyXBHEYo2l09glU0s6SqnjRLsTvjFmB/BnYDXwLrAZaN3Xse31lxhj0owxaYmJiUde4Ri4PAaXx7BqWy4Ak4fEERNmjX2TGBVCZIijVQ2/4SrbuPBgBsWFkVnkTfgV1jbyTCzFJooId5l3hQzrUVv4SqnjhF8nbY0x/zXGTDbGzASKgd1AnogkA3gf8/0P89g09LdfsekAAIP7hDMiyRoOISEyhIjg1gm/4aRtbEQQQxMi2F9STZ3LbdXpgQOuGIqJIthZZo2hU5IBjlCI7NtF70oppfzjby+dJO/jYOASYCnwJtBwiek1wAp/9tEeLreV8HfmViACA2LDGNHXSviNJZ1Davgl1fXYbUJUiIPhfaPwGNhXWAUVeSA2surCKTFRiPFAbZlV0okdYvW7V0qp44C/F169KiLxgBO4xRhTIiJ/Al4WkeuBLOAyf4M8Vi6Pp/F5cnQowQ4bJ/SLtl7HhBIZYvdR0nESGxaEiDT+GtidV8kJlbkQnkBuuZNi4x0Xp7qoqQ++UkodJ/xK+MaY03xMKwJm+bNdf7k9BrtNcHtM41WxV0wdxIi+kfSNDiXCRw2/rNpJjHeM+9SECGwCu/MrrRZ+VF9yymtxh8aBByvhl2bCoJO6+q0ppVS7BeSVti6PYUx/q0XfkPBDg+ycMiwBgMgQX/3w64nzDqUQGmRnSHwEe/IrrBp+ZD9yy2qxR1rrU7TXKutoC18pdRwJyITvdhvGDYghPiK4sQ9+cxEhDupcHlzuptJPSbWz6S5WwPCkSHbnVUJlvtXCL6slNNrbm+jgButRu2QqpY4jAZnwXR5DeLCdz+75Ht8/qXVSDg+2hkZofpvDsup6YsKCG1+PSIoks7ACU5nvbeHXEB7r7c9/wJvwtYWvlDqOBGTCt2r4NsKC7dhsrXvRRIZYpy6aX3x1aAt/RN9Ioj3liHHjDE+ipNpJQlys1RUzb5u1kA6roJQ6jgRkwnd5PDh8JPoGEd6E33DittbppsbpJi6ieQs/iiQpAaDE1geAfrHhEB4P7noIjYXQ1uUipZTqqQIu4Xs8Bo8Bh73thN/Ywvcm/IaLrhquxgUYlhhJkq0UgDyPldiTY0Ih3Er+Ws5RSh1vAi7hu4110dWxtPCbD6vQICzYzqiIKgAOuq0eP/1iQq0WPugJW6XUcSfwEr53WAW7re23FhHScNLWSvhFlVbCj48MbrHchBjrHrh7aqwLsfpFN0v42sJXSh1nAi7huzxHbuFHNrbwrV46DTc3T4wKabHc2Jgaykw4r24uJDrUYf0yCPOWdLSFr5Q6zgRewvf2rbcfTUnH20unrYQ/wF5GIXF8V1hFcoz3toaNLfzUjgxbKaU6XeAl/IYW/jGctC2orCPEYSMqpNlIE1WF2PO24oqwRsPsFxNqTY/wXm2rJR2l1HHG38HTepymGn7bCT/EYcNuk8YafkFFHYlRIUjDyJfF38FzF0NlHhUn/Qo+8PbQARg7H8QG8cM79X0opVRHC9wW/mESvogQEWxvUcNvUc75+EGoKoRr3mLM6ZeREBnC6GSrpw7hfWDq9TosslLquBN4LXz3kXvpQMsB1Aoq6hgSH940M387DJoGg6YSBqxddCbB9oD7blRK9TIBl8UaxsJv1cJ31UPutsaXzYdILqxs1sL3eKDgW0g8oXHZEIe9qdyjlFLHqYBL+O62TtpuWQb/Ps26cQlWn/vc8lqcbg/F1fVNCb80E1w1LRK+UkoFgoBL+G3W8At2gfFA1heANfzxnvxKiirrMaZZl8yCndZj0uiuClkppbpEwCX8Nq+0LcmwHr0Jf0RSFBW1LrYfLAMgMdKb8PN3WI8JIzs7VKWU6lIBl/DbbOGXWqUcsr4EaLxv7ed7i4DmLfxdENUfwmI7PVallOpKfiV8EblDRLaLyDYRWSoioSKyWEQOiMgm79+cjgr2aLR5pW1JFtgcVsmmupjhfdtK+DsgSev3SqnA0+6ELyIDgJ8BacaYsYAdWOCd/bAxZqL3750OiPOo+Wzh15RAXRmMONd6nf0ViZEhxIQFsSOnHICEyJBmPXS0fq+UCjz+lnQcQJiIOIBw4KD/IfnH55W23p45jLkYbEGQ9QUi0ljWiQ51EBpkb9ZDZ1RXh62UUp2u3QnfGHMA+AuQBeQAZcaY972zbxWRLSLypIjE+VpfRG4UkXQRSS8oKGhvGK24PIYzbRuJLVjXNLHhhG3iKOg/samO3zeSq+3vkxaeZ83XHjpKqQDmT0knDpgHpAL9gQgR+T7wODAMmIj1RfBXX+sbY5YYY9KMMWmJiYntDaMVt8fD3Y6XGZj+QNPEhhO2cUOg/2TI3QrGMLqP8Pugp/ld/cPgccNXT0BIDCSd2GHxKKVUT+FPSecsYJ8xpsAY4wReA04xxuQZY9zGGA/wBDCtIwI9Wi63IYR6wop3WEkcrJJOwz1o41LAWQXVRYwJs+5Zm+L6Dl65FvZ+AGf+EkIiuzJkpZTqEv4k/CxguoiEizXuwCxgh4gkN1vmYmCbz7U7idtjCMaFzV0LhbutiaWZTcMZx6VYjyWZpDoKASgLSoIdb1pX1079UVeGq5RSXcafGv46YDmwAdjq3dYS4AER2SoiW4AzgTs6ItCj5fIYgsQaI4eczdZjSWbTHaoaEn9pBnF11jnmzNMfsoY7nvMXsAehlFKByK/RMo0x9wL3HjL5an+26S+rhe+0XuRshnGXQWkWjJptTWtI/CWZSEUuhEQzfsYFcOrc7glYKaW6SMANj+zyGILw1u5zt0BlLrjrmhJ9SKR1m8LSTCjPsabrSJhKqV4g4BK+2+Np2cL/5k3red8xTQvFDrHKPBU5eucqpVSvEXBj6ThdHkLEhTuiL9SVw5rFkHo6DD65aaG4FKtvfmlWU8tfKaUCXMAlfOO2Wveu/mnWBHc9nPfnlmWbuCFQsg+c1XozcqVUrxF4Cd9VZz32m2D1u5/+k9ZXzjZv1WsLXynVSwRcDd+4agGQkEj42SbrgqtDNW/VawtfKdVLBF7C95Z0bEGhEN7H90ItWviDuyAqpZTqfgGX8MVb0rEFhbS9UMwgEBuEJ0BwRBdFppRS3SvgEr5x1QNgcwS3vZAjGKIHQFS/LopKKaW6X8AlfNxWwhfHYVr4ANNugDCfIzcrpVRACtiEj/0wLXyAGbd1fixKKdWDBFy3THFbNfwjJnyllOplAi7hN7bwj1TSUUqpXibgEr4cbUlHKaV6mYBL+Hj74WvCV0qplgIu4ds83hq+lnSUUqqFgEv4WtJRSinfAi7h2zxa0lFKKV8CLuGLR3vpKKWUL34lfBG5Q0S2i8g2EVkqIqEi0kdEVovIbu9jl17Oam8s6ejNyJVSqrl2J3wRGQD8DEgzxowF7MACYBHwgTFmBPCB93WXkcaSjrbwlVKqOX9LOg4gTEQcQDhwEJgHPOOd/wxwkZ/7OCZ2j560VUopX9qd8I0xB4C/AFlADlBmjHkf6GuMyfEukwMk+VpfRG4UkXQRSS8oKGhvGK3YjBM3NrAH3jBBSinlD39KOnFYrflUoD8QISLfP9r1jTFLjDFpxpi0xMTE9obRis3txIXW75VS6lD+lHTOAvYZYwqMMU7gNeAUIE9EkgG8j/n+h3n0HKYel2jrXimlDuVPws8CpotIuIgIMAvYAbwJXONd5hpghX8hHhubx4lLtIWvlFKHandT2BizTkSWAxsAF7ARWAJEAi+LyPVYXwqXdUSgR8thNOErpZQvftU+jDH3AvceMrkOq7XfLeya8JVSyqeAu9LWbpy4NeErpVQrAZfwHcaJy6Z98JVS6lABmfA92ktHKaVaCciE79YWvlJKtRJwCT9Ia/hKKeVTwCV8u3HhsWnCV0qpQwVcwg9CSzpKKeVLQCZ8beErpVRrgZfwjQuPtvCVUqqVwEv4ODXhK6WUDwGX8INxYfT2hkop1UpAJXxjDEFoSUcppXwJqITvMRCME6O3N1RKqVYCKuG7XE4c4sFoC18ppVoJqITvdtYBaAtfKaV8CKiE76zXhK+UUm0JqITv8bbw0V46SinVSkAlfLezFgBjD+nmSJRSqudp98DxIjIKeKnZpKHA74BY4AagwDv9V8aYd9q7n2PhcXlb+A5N+EopdSh/bmK+C5gIICJ24ADwOvBD4GFjzF86IsBj4a7Xko5SSrWlo0o6s4C9xpjMDtpeuxhvC1+0ha+UUq10VMJfACxt9vpWEdkiIk+KSFwH7eOI3I0nbbWXjlJKHcrvhC8iwcCFwCveSY8Dw7DKPTnAX9tY70YRSReR9IKCAl+LHDPjqre2rS18pZRqpSNa+OcBG4wxeQDGmDxjjNsY4wGeAKb5WskYs8QYk2aMSUtMTOyAMMDjsnrp4NAWvlJKHaojEv5CmpVzRCS52byLgW0dsI+j0tDCt2lJRymlWml3Lx0AEQkHzgZuajb5ARGZCBgg45B5naoh4Wu3TKWUas2vhG+MqQbiD5l2tV8R+aHhSlubI7S7QlBKqR4roK60NW6rhi9aw1dKqVYCKuHjcgJgC9KSjlJKHSqgEr5xey+80oSvlFKtBFTCx3vS1qEnbZVSqpWATPha0lFKqdYCK+F7Szr2ID1pq5RShwqwhF9PvbFjt9u7OxKllOpxAirhi7ueeoJw2KS7Q1FKqR4n4BK+Ewd2TfhKKdVKQCV83PXU48BhC6y3pZRSHSGgMmNwfQklJkpb+Eop5UNAJfyQumKKTZTW8JVSyoeASvih9SUUEY3drglfKaUOFVgJ31lCkYkmSGv4SinVSuBkRlc9Ia4Kik201vCVUsqHwEn41UUAFBGtNXyllPIhgBJ+IQDFRGPThK+UUq0ETsKvKgCgTKK7ORCllOqZAijhWyWdEonp5kCUUqpnanfCF5FRIrKp2V+5iNwuIn1EZLWI7PY+xnVkwG3ylnTKbbFdsjullDretDvhG2N2GWMmGmMmAlOAauB1YBHwgTFmBPCB93XnqyrAjZ0qieyS3Sml1PGmo0o6s4C9xphMYB7wjHf6M8BFHbSPw6sqpNoRo0MjK6VUGzoq4S8Alnqf9zXG5AB4H5N8rSAiN4pIuoikFxQU+B9BdRGV9ljtg6+UUm1w+LsBEQkGLgR+eSzrGWOWAEsA0tLSjL9xUFVAuS2GyCC/35JSqoM4nU72799PbW1td4dyXAsNDWXgwIEEBQX5tZ2OyI7nARuMMXne13kikmyMyRGRZCC/A/ZxZFWFlMggokI14SvVU+zfv5+oqChSUlIQ0V/f7WGMoaioiP3795OamurXtjqipLOQpnIOwJvANd7n1wArOmAfR1ZdSJGJIirUv29ApVTHqa2tJT4+XpO9H0SE+Pj4DvmV5FfCF5Fw4GzgtWaT/wScLSK7vfP+5M8+joqrHmrLyPdEaQtfqR5Gk73/OuoY+pUdjTHVQPwh04qweu10He84OnmuSKK1ha+UUj4FxpW23ouuDtZHagtfKaXaEBgJ3zuOTo4rUmv4Sql2i4xs+8LN1NRUdu3a1WLa7bffzgMPPADAxo0bERHee++9o95mVzu+m8PlByH9KSjYCUAxWsNXqqe6b+V2vjlY3qHbPLF/NPfOHdOh22zLggULWLZsGffeey8AHo+H5cuXs3btWgCWLl3KqaeeytKlSzn33HO7JKZjdXy38Cty4JMHYcdKXBHJHDTxRIdpC18pZbnnnnv45z//2fh68eLF3HfffcyaNYvJkyczbtw4Vqw4uo6ECxcuZNmyZY2vP/nkE1JSUhgyZAjGGJYvX87TTz/N+++/33OvOzDGdPvflClTjL+27i81Q+55y7y7LcfvbSmlOsY333zTrfvfsGGDmTlzZuPr0aNHm8zMTFNWVmaMMaagoMAMGzbMeDweY4wxERERh93eiSeeaDZt2mSMMeamm24y//jHP4wxxnz66afme9/7njHGmIULF5pXX321cZ0jbfNo+TqWQLo5hlx7fLfwmymvdQJoSUcp1WjSpEnk5+dz8OBBNm/eTFxcHMnJyfzqV79i/PjxnHXWWRw4cIC8vLwjb4ymVr7L5WLFihVcdtllgFXOWbBgAWCVfpYuXXq4zXSbgMmOFbUuAO2WqZRq4dJLL2X58uXk5uayYMECXnjhBQoKCli/fj1BQUGkpKQcdQlm4cKFnHPOOZx++umMHz+epKQk3G43r776Km+++SZ/+MMfGq+MraioICoqqpPf3bEJmBa+JnyllC8NJ1uXL1/OpZdeSllZGUlJSQQFBfHhhx+SmZl51NsaNmwY8fHxLFq0iIULFwKwZs0aJkyYQHZ2NhkZGWRmZjJ//nzeeOONTnpH7RcwCb+8Rks6SqnWxowZQ0VFBQMGDCA5OZmrrrqK9PR00tLSeOGFFzjhhBOOaXsLFy5k586dXHzxxYBVzml43mD+/Pm8+OKLAFRXVzNw4MDGv4ceeqhj3lg7iFX3715paWkmPT3dr208smY3D6/5lt1/OI8ge8B8jyl1XNuxYwejR4/u7jACgq9jKSLrjTFpR7uNgMmMFbVOwoLsmuyVUqoNAVP/qKh1ER0WMG9HKdVNtm7dytVXX91iWkhICOvWreumiDpOwGTI8lqnDquglPLbuHHj2LRpU3eH0SkCpv5RUevSE7ZKKXUYAZTwndolUymlDiNgEn65tvCVUuqwAibhV2gNXymlDitgEn55rYtobeErpZopLS1tMVrm0ZozZw6lpaXHtM7TTz/dePVtg8LCQhITE6mrqwNg3rx5nHzyyS2WWbx4MX/5y1+OOcb2CIgMWet0U+/y6NDISvVkqxZB7taO3Wa/cXBe27fNbkj4N998c4vpbrcbu93e5nrvvPPOMYdyySWXcNddd1FdXU14eDgAy5cv58ILLyQkJITS0lI2bNhAZGQk+/btIzU19Zj34S9/b2IeKyLLRWSniOwQkZNFZLGIHBCRTd6/OR0VbFsaxtHRGr5SqrlFixaxd+9eJk6cyNSpUznzzDO58sorGTduHAAXXXQRU6ZMYcyYMSxZsqRxvZSUFAoLC8nIyGD06NHccMMNjBkzhnPOOYeamhqf+4qOjmbmzJmsXLmycdqyZcsaW/2vvvoqc+fObRzbp1scy1jKh/4BzwA/8j4PBmKBxcBdx7Idf8fD35tfYYbc85Z5bUO2X9tRSnWs7h4Pf9++fWbMmDHGGGM+/PBDEx4ebr777rvG+UVFRcYYY6qrq82YMWNMYWGhMcaYIUOGmIKCArNv3z5jt9vNxo0bjTHGXHbZZea5555rc38vv/yyueiii4wxxhw4cMAkJycbl8tljDFm1qxZ5pNPPjG7du0y48aNa1zn3nvvNQ8++OAR30u3jocvItHATOC/3i+OemNMqT9fPu3V2MIP0ZKOUqpt06ZNa1FKefTRR5kwYQLTp08nOzub3bt3t1onNTWViRMnAjBlyhQyMjLa3P4FF1zAZ599Rnl5OS+//DKXXnopdrudvLw89uzZw6mnnsrIkSNxOBxs27ato9/eEflT0hkKFABPichGEfmPiER4590qIltE5EkRifO1sojcKCLpIpJeUFDgRxjNhkbWGr5S6jAiIiIan3/00UesWbOGL774gs2bNzNp0iSf4+KHhIQ0Prfb7bhcrja3HxYWxuzZs3n99ddblHNeeuklSkpKSE1NJSUlhYyMjG4p6/iT8B3AZOBxY8wkoApYBDwODAMmAjnAX32tbIxZYoxJM8akJSYm+hGG3u1KKeVbVFQUFRUVPueVlZURFxdHeHg4O3fu5Msvv+yQfS5cuJCHHnqIvLw8pk+fDlhDKL/77rtkZGSQkZHB+vXrj7uEvx/Yb4xpGFFoOTDZGJNnjHEbYzzAE8A0f4Nsy87ccs5+6GPuW7kd0ISvlGopPj6eGTNmMHbsWO6+++4W82bPno3L5WL8+PH89re/bUzO/jrnnHM4ePAgV1xxBSJCRkYGWVlZLbafmppKdHR044Bs999/f4sx8zuLX+Phi8inWCdtd4nIYiACeMgYk+OdfwdwkjFmweG2097x8DMKq3jgvZ0AJEWF8rsLTsRmk2PejlKqc+h4+B2nI8bD97dJ/FPgBREJBr4Dfgg8KiITAQNkADf5uY82pSRE8M+rpnTW5pVSKqD4lfCNMZuAQ79drvaxqFJKBYxbbrmFtWvXtph222238cMf/rCbIjo6WvRWSnUqYwwigVVqfeyxx7p0f/6U3psLmLF0lFI9T2hoKEVFRR2WsHojYwxFRUWEhob6vS1t4SulOs3AgQPZv38//l5r09uFhoZ2SO8dTfhKqU4TFBTULYOEKd+0pKOUUr2EJnyllOolNOErpVQv4deVth0WhEgBkOnHJhKAwg4Kp6tozF1DY+4aGnPXODTmIcaYox6MrEckfH+JSPqxXF7cE2jMXUNj7hoac9fwN2Yt6SilVC+hCV8ppXqJQEn4S468SI+jMXcNjblraMxdw6+YA6KGr5RS6sgCpYWvlFLqCDThK6VUL3FcJ3wRmS0iu0Rkj4gs6u54fBGRQSLyoYjsEJHtInKbd/piETkgIpu8f3O6O9bmRCRDRLZ6Y0v3TusjIqtFZLf30ecN6ruDiIxqdiw3iUi5iNze046ziDwpIvkisq3ZtDaPq4j80vv53iUi5/agmB8UkZ0iskVEXheRWO/0FBGpaXa8/9WDYm7zs9CDj/NLzeLNEJFN3untO87GmOPyD7ADe4GhQDCwGTixu+PyEWcy1r1+AaKAb4ETgcXAXd0d32HizgASDpn2ALDI+3wR8OfujvMwn41cYEhPO87ATGAysO1Ix9X7OdkMhACp3s+7vYfEfA7g8D7/c7OYU5ov18OOs8/PQk8+zofM/yvwO3+O8/Hcwp8G7DHGfGeMqQeWAfO6OaZWjDE5xpgN3ucVwA5gQPdG1W7zgGe8z58BLuq+UA5rFrDXGOPP1dudwhjzCVB8yOS2jus8YJkxps4Ysw/Yg/W571K+YjbGvG+McXlffgl03p2326GN49yWHnucG4h1B5nLgaX+7ON4TvgDgOxmr/fTwxOpiKQAk4B13km3en8SP9mTyiNeBnhfRNaLyI3eaX2N9wb13sekbovu8BbQ8j9GTz7O0PZxPV4+49cBq5q9ThWRjSLysYic1l1BtcHXZ+F4OM6nAXnGmN3Nph3zcT6eE76ve6b12D6mIhIJvArcbowpBx4HhgETgRysn2s9yQxjzGTgPOAWEZnZ3QEdDREJBi4EXvFO6unH+XB6/GdcRH4NuIAXvJNygMHGmEnAncCLIhLdXfEdoq3PQo8/zsBCWjZi2nWcj+eEvx8Y1Oz1QOBgN8VyWCIShJXsXzDGvAZgjMkzxriNMR7gCbrhJ+ThGGMOeh/zgdex4ssTkWQA72N+90XYpvOADcaYPOj5x9mrrePaoz/jInINcAFwlfEWlr1lkSLv8/VY9fCR3Rdlk8N8Fnr6cXYAlwAvNUxr73E+nhP+18AIEUn1tuoWAG92c0yteGtv/wV2GGMeajY9udliFwPbDl23u4hIhIhENTzHOkG3Dev4XuNd7BpgRfdEeFgtWkI9+Tg309ZxfRNYICIhIpIKjAC+6ob4WhGR2cA9wIXGmOpm0xNFxO59PhQr5u+6J8qWDvNZ6LHH2essYKcxZn/DhHYf564+E93BZ7XnYPV62Qv8urvjaSPGU7F+Hm4BNnn/5gDPAVu9098Ekrs71mYxD8XqtbAZ2N5wbIF44ANgt/exT3fHekjc4UARENNsWo86zlhfRjmAE6tlef3hjivwa+/nexdwXg+KeQ9W3bvhM/0v77LzvZ+ZzcAGYG4PirnNz0JPPc7e6U8DPz5k2XYdZx1aQSmleonjuaSjlFLqGGjCV0qpXkITvlJK9RKa8JVSqpfQhK+UUr2EJnyllOolNOErpVQv8f8BTryl/D8M+igAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_VAL','train_VAL']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD4CAYAAAD4k815AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABNLElEQVR4nO3dd3hUVfrA8e+ZSSMVUgkJKUDoHUSQoggi9oIo2LC7a1ldu+vuT7e467q76jZdO+6KIIKKXalSlJLQWwglgZBCekL6zJzfH2fSE5hAEjL4fp4nz8zcuXPvmcvwzjvvPfccpbVGCCHE2cNyphsghBCibUlgF0KIs4wEdiGEOMtIYBdCiLOMBHYhhDjLeHTkzkJDQ3VcXFxH7lIIIdxeUlJSrtY6zNX1OzSwx8XFkZiY2JG7FEIIt6eUSmvN+lKKEUKIs4wEdiGEOMtIYBdCiLOMBHYhhDjLuBTYlVIPKaV2KqV2KaUedi4LVkotVUqlOG+7tWtLhRBCuOSkgV0pNRi4GxgDDAMuV0olAE8By7XWCcBy52MhhBBnmCsZ+wBgvda6TGttA74HrgGuAt5zrvMecHW7tFAIIUSruBLYdwKTlFIhSilf4FKgJxChtc4EcN6Gt18zhRDCTZXlw3e/gdz9HbbLkwZ2rfUe4M/AUuAbYBtgc3UHSql7lFKJSqnEnJycU26oEEK4pcM/wg//gNJjHbZLl06eaq3f1lqP1FpPAvKBFCBbKRUJ4LxtttVa6ze01qO11qPDwly+IlYIIc4OqevA6g09RnbYLl3tFRPuvI0BrgXmA58Bc5yrzAGWtEcDhRDCraWtg+hzwNOnw3bpaj/2xUqp3cDnwP1a6wLgBeAipVQKcJHzsRBCuJ8t8+DrNuzYZ68Ghx0qiiBrO8Se13bbdoFLg4BprSc2sywPmNLmLRJCiI7247/h2C4Ydx90jWn966tKIflrcNjgaBJsWwCx42H0HaAdEDe+7dt8Ah06uqMQQnQ6JVkmqAPsXAwTftn6bSTNhW9/Ze5bvSByOOz7GorSweIB0WPaqrUukSEFhBA/bQdXmVu/cNix6NS2kbUD/LvDg5vhsRS4/WvoPgSyd5iTpl6+bdZcV0hgF0L8tB1YAb6hMPERyN4Jx/a0fhvZuyBiEIT0hi5dweoBl70MKIif1NYtPikJ7EKIM2fTW5CytPWvy90PS5+F6grX1s/eDd8+A3u+aLhcaziwEnpdAINngLLA1g9a1xa7DXKSIWJgw+U9z4G7V8CEh1u3vTYggV0I4bplz8G8608cUFf9Gf4YZf6W/77l9XKS4cvH4KPboTizde34/s+w7hX4/CETnE/W5tfGwY//gq8eN4G4RvYuc+FQ78ngHw6DrjUnUmvKM2Duv3oepP3Y/PbzD4K9EsIHNX0uaiR4B7TuvbUBCexCCNccXg9rX4aUb+GLh5sPqFrDlvdNz5LuQ8wVl8UZzW9v9V/BswvYq+pOPLqiqgz2fmlq2tsXwPpXW173aBKsfQWGXA9X/gtKMkz7qytg2W/hv1eak5u9Jpv1r3gFQvvCR7dBQapZtu1Dc3L1vStg6/ym+zi229yGD3D9PbQzCexCdEZHk+CzB+F4JxmGw26DLx6BwCgY/zBsmw9vTYX/XgXpSXXr5R2AosNwzp1wzX9MX+51/4DMbfDBDTD3clhwEyS+AzsXmfUmPgq7PjYlEVfs+xqqS+HaN6D3FFjzt+a/ZBx202b/CLjsbzBsNgT0gE1vm2O79iWIGQdzPoegKPMa7wCY/QFUl5vMHSBtrSnV9BwDXz5iujY67PD5w+bL7thuU8IJ63caB7htSXdHIVqrLN9kmQHdT387tkoIjGz63Pcvwr5vTLCb9QFEDm3dtrWGgkMQ3Ov02lhj6/sma71hHvS7FNBwZJMJ6t//GW5aaNY76AzOvS+EbnEwbJYJ4klzwdsfQhLMl9beL8DDB877BfgEmSx/xR9MAFXqxG3ZscgE6LgJkJcCB5ZD4WHoFlv33vd+Aetfg8ytcN074BNonhs1B1b9ydy/8Ncw6fGm2w/uZdqf/LVpX+FhGHsfdB8Kcy81y70DIeldyNgMQT0huLf59dFJSMYuRGstuR/euAAqik9vO4vvhHcuNtlffWX5sH8Z9LvMPPf+DCgvbN22Vz4P/xhxaicmm7PrE1Oi6H8ZWCxw0e/gjq9h7M8g5TsT/MD0MOkWV/eFMvFRQEPkMPj5D+Y1D++E2Qtg9nxT1/bwNicYjybWfTE0pjVseAO++7V5T4OvBYu1bvyVjC116+5eAh/eDIVH4OI/mrp5jZG3gkcXGHg1THys5ffb71IoOgIbXzePY8eb7D4wCnZ8ZL6sUOaXyL5vm544PcMksAvRGlqbn98lmbDyj6e+nZIsk40XppkgXt/uJeYKxgueNGWBslyTzYIpzax5CebPhvTEujbVt+sTWP0Xc3/jmw2fs9ugvMBcOLP2Zfjv1ScfTra8EFLXmmDXOJseOccsS5prLqM/tKauXg2m+99D2+G2L00QB9MVsN8lJiuuMeJmk4V//5fm25C1Hb5+HDa8bsolI242yyMGgcWzLrA7HObXTmg/+MUWGHd/wzYH9oCHt8N17574l0HfiwHl3F+Qcz8W84Wyf5mp05/3oDmX4KiG8M4V2KUUIzqFL7dn0ifcn37dO74HQasUHobyfHPibuPrpq7qH25qva0Z5Gnnx4A2P+kT33EGEqcdi0x23H2oCT7n3GUCdEmmyQ4d1SbY7F8GI24xy7x8YeZcyN0Hn/wMep4LMWNNfbsgra5M8daFJsusYfWG+bPg7uWmJAKml4dPV/ANNo/3LzNfNP0ubfo+uvaEhIth8//Me6kqaRiwoflSU2M1WfvXT5jyTvSohs/vWGQC+KPJde2qeV3EIFMSAUj+0pSMrn3TfIE0x9+FqSP8w01N/cgG834sVrN8yEz44Z+AgjF3Q3A8fPFLiBh88m12IMnYxRlXZXPwyw+38uqqjpuI4JTVZIbXvGaC+xcPw4IbYdEdJltsLGlu04wczM/5yGFw7r0mMNeUMgoPm9EAB19Xl1Fe+GsIiITUNSbI378JHtpqAnfi2xDax5SF3rzQ9OaIHGbq8mPuqcumAXL2maA+dBZc8iLctwFu+cTU4hfdYU4YZmw1XfsW3VHX1uSvzAU80aObPybn3mO6DC57Fjx9Ib7J0FKuGTLTnIRM+c48Tk+E7R+Z47pzMfSZ0jCo14gaCRnbTNnq+xdNGah++eVU9bvE3MbWG+el+1DT26f/ZSZbHzkHrv8v9J1++vtrQ5KxizNuX3YJVXYHyVklHbfTsnxYeKsJmjFjXX9dxhaTOcaOhwc2mpJG8tew/Lew4ncw6nZzoq5LNxNQP38IUHDhM6amq5TpOZKxGS76PQy6xvTqWPEHuPwVWHy3Oak47Ia6ffoEwf3rzRgk9U/Q3foZVBab54sz4dOfm6A2/U8mkwUTcLb8D85/0gRogCm/gaDouu1c9hJ8/gt49xJT6rFVmFp33gFzYjBlGQy4oi5rbaz3hfCLreZ1XYLNez8VvsHQY4Sp009+2vQ6yd5hJqooPmrq+s3pMcL86lnxe1Oyueb1lrP11hh8nSmLDbi8bplScPs3poskmGMy8KrT31cbk8Au2tzCxCPMW5/G+3edS4CP50nX33m0CICDOaVU2x14Wjvgh+TS/zMZ8Jq/wU0fuf66jM3QfbAJnB7epu9yWH/IP2Bq1mtfBmU12eWBlaZE4xtiArdfuOmVsXMxoMyVjkFR5gTj6r+Y9UuPmV4c3eIa7remTFKfUnXLAyPh1k+brnPuz0xA3/I/c9t9aMOgDqZNfqHw8T0m673xQ/MrJPEdUyOvLIL+zZRh6guOd/EAnkTvC82/yeENJqh7B5lfJZ6+dRl0Yz1GmNu1L5sTnEOub/Vu//P9Adbtz+XtOefg5eH8/HXtCfesarqyt3+rt9/RpBQj2ty761LZll7E7z7f7dL6OzNMYK+yO0jNLW2fRmltLhVf+UdTr93yP1PeSFladyHKyTgc5id/TSCpoZTJtq//L1z1Kpz3gOnSFxwP171t+luHDTD719qUYWLH1/WdvvDX5nUVRTDpCRg8g71Zxfx9WQr6BFdV/rA/lzdWHzjhOsRPMvX21X+FIxtNCaGR/ceO86+MvuifrYW7lpl6f//LYPN/4esnTRfEhIubbruRlOwSHlm4lYcXbGHBxsMnXb+xovJq/nuslxnm9stHTFnmzm9NF8lhs8HLr3bdgznHeXnpPhwObY6th4/Joi97yZzkbKVPtxxlTUou/1yRcsL1qu0O3lpzkFXJzU9zt/5gHnPXHWr1/tuaZOyiTe0/VsKezGJ6hfrxUVI6Fw2MYNqgE/f33nG0mGA/L/JLq0jOLiEhoo1PoNoqzYUqW9+vWxYYberLr54LSe/BBU/B7s9M+cRWASNvgaE3NCx9FBwy2WvjwA5g9Wz4k/zC35gAVVMSGXKdKRUkf2VOcI79ecPXj7jJZPDOE7C//2I36/bnMSKmK5P6NpxS0mZ38Jfvknn9+4MAXDa0B1FdW+hDrZT5spg3wzxuJut9a81BFmw6wrRBk+jb3Zl5j77DlCG6xZseJC6UNt79IZXPtmYQ7OfFp1sziOzahfMbtf1E3l57iFe3+jHL1xev7J2md034ALh/Q5N1//jVHpbtOcZFAyMYHBVkzj0ERDbodlhps5OaW1Z7Qv5Qbil+XlbCA33QWpOYVsDo2G4cr7SRnF1CgI8H/165n/BAH0L9vDivTyhBXep+cWYUlvPAB5vZfLiQsABv1j45GW+PhuWpf65IYd3+PHp07XLSz317koxdnNQJM8JGPt+WiVLw3zvH0DfCn3+4kAHtySzmsiGRWBTsa1Rnd3XfLa2nS7LM1Y5b3zcB7pe7YOpzcP17ENbX1KA3vQUvDYCP7zK13KpSUxt//zrTha9GzYlTV+autHrWBXUwgR3gs1+YzHLg1WitG7RbO9ffkV7Euv15ALy++kCTTf9r5X5e//4gExNCgbpSVov6TIGoUc7L/Bte6KS1Zk1KLgCr99W7yjX+fJP93vJx7QnLE/1baK1ZvS+HC/qFs/qJySSE+/PEom0UllU1WKclZVU2/vtjKjY8+MHhvDS/5phZrGCx1r4+JbuEZXtMxrw6xdnmi583v5TqeXThNqb/fTW7M4qpqLZz3Ws/cO/7SWit+WxbBjP/8yNf7chiy+FCtIYXZwwlupsvv/l0Jz+ft5lLXlnNptT8Btvbl32cuybEk1NSyadbjjbYn83uYMvhQgCe/ngHuccra587VuziYGVtRAK7OKHkrBJG/WFZw//0LdBa8/n2DM6NDya6my9TBkSQnFVCpc3e4msO5BynyuZgVGw34kL9SM42gb2orJr7P9jMlL99T3lVy6+v2e/P39/M9FdWsyez7qKhdRs3kfO3cdizdpiugBc+Y+rLE35Z18Nj3P3mKtKYcSaDf3Az3PejGVckbS1845wurbLElDN8Q01N/SR+8+lO7py7qW5Btzgz72VZLvSewpEKH2b+50cuenk1O48W8emWo4z4/VKe/ng7/1yRQoC3Bw9M7sO6/XkNAve2I4X8c8V+rhkRxRu3jMaiYNfJArtS5oKgOZ9z41sbeP7LuhLZgZxSjhaWA7DaGeBrX3POnbUXGr299hDnPL+Mr3Y0P1hXal4Z6QXlnN83FB9PKy/fMJy841W88PXe2naP+sMy/r1yP3ZH0wD/UWI6hWXV/OrS/nxcNZYyrzDoX3fSsrCsiokvruTRhdv4x4r9+HhaiAvxZc2+3CbbAliy9ShfbM9Ea3hzzUE+Skonr7SKLYcL2ZRaUPtrZ8nWoySlFWBRMLFvGN8+PImlv5zE+3eei6eHhRte/5FVycfYdqSQHw/m8dCUBJ65bACDegTyxuqDphTktDerhLIqOw9M7kNJpY0/fGGO886jRYx7YQXf7co68b9TG3KpFKOU+iVwF6CBHcDtgC/wIRAHpALXO+dCFW6i0mbnd5/v5mfn96ZncNOJAGq6IeaXVvHF9owmJQGAV5btY0xcMOf1CWV3ZjEHc0q5c4L5OT+4RxDVds2+rOMMiW7m5B8mOwUYHBVEv4gA9maVcCi3lJveXE92SSV2h+ajpCPcOi6uxfcxb8NhvtmVRYAXFL12MTt7jSH82hewfPUoXXQ593r+kX8kXEGzUx3ETYBfZzddPvIWUzL54R/mCkZ7pXl8yycnLUuUVdlYlJROebWdI/lldcd2yExI38S+iOnM+PsaUODrZeXKf63FoaF3mB8LNh1Ba7h3Ui/uOb8Xc39I5YEPNhMTYurLezOLCQ/w5rkrB9HFy0pCeAA7ThbYAfzDKSit4ocDu0hMLeDuSb0ID/Cp/cKeOiCCNSk5VFTb8fFsWF7YlVHEC1/vwdNq4b55m5k9pif/d/kgCsur+PuyFGaPiWHrkUKA2s/I4Kggbjo3hg82HubhqX3518r9FJVX85dvk/nxQB5v3zYabw8r/165nw2H8tmRXsio2G7cM6k3M3ZdzkVFU1lc5UN3Z4Xp/fVppBeUk1GYjkPDnHGx+HhZeWftIUorbfh51/2bZBVV8JtPdzIipivDorvy/vo01h/MY3BUIEcLynn0o60cyS8nulsXViXnkF1cQf/ugfg7t5EQEUBCRABfPDiBGa/9wBOLtjMgMpAAbw9mjemJUop7JvXioQVbWbH3GFMHRgCQlGbC3+xzY6hy1uIfndaP11cfpIunlbG9Q07+79RGTpqxK6WigF8Ao7XWgwErMAt4CliutU4AljsfCzey5XAh8zYc5pNGPyn3ZZfwv/VpPLV4O7szi+kZ3IU1KblNfkqn5pbyyrIU3vsxFYANB83P1qkDzAd9SJQJ5vUDT7XdwYq92bXb2pVRjK+XlfhQP/pGBJCaV8qD8zdTVm3n45+fx8iYrry15hA2u4MfD+SR3egn7f5jx3n+yz1MTAjlh8sLGWvZzeDUuWx76WrGsY29Ax5kWUEEjy7cxv/Wp7H5cCtyj6nPwQVP48jYAgdXsbH/4/zvWBz/W59W+3fIebK3tNLGd7uysDs0S3dnU15tfmV8sb1ehjviFsqm/plb1/cgPNCbr34xka8fmsS1I6N5bFpfvn3YZIqXDunOXRN7EejjyVOX9KerrxfF5dUUl1cTF+rHv24cUVv7HRQVyM4M14Y2qHnvVXYHc9elArAmJYf4UD9uGhtDpc3RoPQA5sv/kQ+30dXXi1WPX8DPL+jNgk1HuOyfa7jk72tYsOkI983bzNc7M4kJ9iU2pO4E510Te2F3aJ79bCdLd2dz/wW9ef6awazdn8vLS1P4fFsGf/k2mczCcnqF+fPkdPNL6ImL+1FQVsX0v69m6e5sKqrtzP0hlfP7hrHgnnFcMrg7P7ugN5MSwqi2azYcyqvdp9aaxxdto9queen64dw9qRcayCyq4P4L+jDnvDiO5JcTFuDN32YOo8ruYFt6EaNim3bRDPDx5KXrh1NQVsX3+3K4aWxsbS+vy4ZEEujjwfK9dUlBYloBkUE+RHXtwu3j47BaFL/7Yjdfbs/gxnNjCHShh1hbcfXkqQfQRSlVjcnUM4CngQucz78HrAKebOP2iXZU8xM/Ma1hsPvlh1vZ5QwWN4+NYVCPIJ7+eAf7jx1vcGLzi+0Zzu0U124vPMCbiEBzArBncBcCfTxqe70ALNmawWMfbWPJ/eMZ1rMruzKKGBAZiNWi6Nc9AK3N9v5940iG9ezKPZN687P3k7j+9R/ZfLiQ83qH8MHdpt/5l9szeerj7Xh7WnhxxmAC3n8QHT6QdHswF+WtJS+gP+fMfJL7uu7n1VUH+Hqn+Sl87/m9uO28OLysFkL869XBAYdDY3No0+XNYmVHn5/zy40jsVQeZN+WnrBlZ4P1fTwt3H9BHz7ecpRDuaU8Nq0vW48UERHoTfegLny+LYOfX9AbAO3ZhcfTxpBXnsXbd4yozeT/OnNY7fbG9wllfJ/Q2sc3j43l5rGxLf4bDokK4uPNR8kurqCrryeeFgsWi7mwqXH2nZhWgIdFMTEhlPfXpzFjVDTrD+Yzc3Q058YH42W1sHpfDhMT6n6ZvbR0H8nZJbxz22jCA3x4cnp/xvcO5ZGFW+nZzZdnr4jj0YXbOFpYzs1jG04C3TPYl0uHRPLF9ky8PSzMOS+OEH9vdh4t4vXVB/D38mBYz64s/tk4POp1cT23VwhfPDiBB+dv4e7/JjIipiu5x6u49/xejIkPZky8qfkH+3nh42nhu13ZDIwMqv1MrknJ5fdXDyY+1HzJXD86mi2HC5k2qDvn9gph7g+p/Pz83oyJDya6WxfSC8oZHdd83/vBUUE8Ob0///n+ILePj6td7mG1MDK2G4mpdf93klLzGen8gogM6sJVw6NYlJSOp1U1eG1HOGlg11ofVUr9FTgMlAPfaa2/U0pFaK0znetkKqWavU5XKXUPcA9ATMwpzP4t2k1N8N6SVoDdobFaFDkllezKKObBC/twm/M/YnpBGWBqsPUD++fbTDZ6tLCcgtIqdmYUmR4KTkopBkcFNagRbzpkMsK9WcUMjQ4iOauEK4b1AGBApBmB76rhPbhsaCRUHuei8CJ6hfqx5Ugho2O78cOBPHakF7Evu4RHP9rGsJ5d+eesEUQe/RZyk1HXvUPPPlOxL32WkDH3gNWDJ6b3566JvaiyOfjHihRe//5gbY31N5cPrC0d7coo4hfzt+DlYeWT+87jQM5xZvznB0L8vHjxjhn07x7Y4Pgdr7Txf0t28rel++ge6MN5vUN4ZVkKSsGt4+KI6tqF332xm/3HjhPdrQu//2I3X+7I5PGL+zGoR/OlqdaqOd4/HsjjpaX7OL9vGL+/ejCJqfnMfnM9n9w3vnadpNQCBkUF8YspCVzz6g9M+dv3AJzfNwxfLw/GxAezKCmd8/uGMyEhlI2H8nlj9UFmj4nhwv4RtfuckBDK2icvxMOisFgU+48d598rD3B+36Yh4N5JvflieybXjYqu/RJ95rKBrN2fS05JJS9dP6xBUK/RK8yfj+87j798k8xbaw8xJCqIcb0aljK8PayM6xXCgk1HWLDpSO3ySX3DuPnculjz/NVDcGjz+Q7282Ljr6biaVUopbhyWA9eXXWg2Yy9xl0TTSLQuJ2jY7uxKjmHwrIqyqrsZBRVcHe97dwzqReLktK5clgUkUEdO/LjSQO7UqobcBUQDxQCHymlbnZ1B1rrN4A3AEaPHu169wrR7nYcLcLLaqGk0sa+7BIGRAaydr+puU4b2L32P2J0N196hfmxel9ObRBMziohObuEaQMj+G53NptS89l/7DjTG3XxGhwVxNx1qbUXHiWm5Ttff5zs4kqKK2y13dHiQ/14744xnFOTPa36E9aNb/Le7UkUWwLpGezL+D+t4I9f7WHH0SLGxAcz765zzQVNi/9u+jsPvBosVqxXvNKgHcF+XgD88ZohXDG0B6l5pXy1I5M/f72X83qHsP5gHn/6ai8BPh7klZbyp6/2sP5gPkFdPPn8wQmENsrsAcICvHnv9jGsTD7GyJhuKAXTXl7NsZJKrhzWg+5BPvz+y93c9u5GbHZNVnEFd0+M52fn926Tfz+AgZGBKGVO1pZU2vh061F+ffkAFiYeodquWZSUzuCoIKpsDralF3Lz2FhGxHTj7TmjOVZSia+Xlcn9TEB+7sqB3Pu/JG55ZwMDIwNJLyinZzdffn1Z0wkkai/iAR6e2peRMd1qt1PfkOgg5t11LkPrnWPx9/Zg4b3jKCitpndYyxf7eHtY+fXlA7liWA9CA7xRzQza9burTGmnhodFMX1w9wbrWiwKC3WP67f9gQv7MCEhlOhuJ55surkvn1Gx5pfD5sMFlFSYWZlGx9YNedA3IoD5d49lYGRgk9e2N1dKMVOBQ1rrHACl1MfAeUC2UirSma1HAs332BdnVFmVja93ZHHNiKjan+g1yw/kHOeaEVF8vPkoiWkFDIgMZPW+XIL9vBjUo+GHcVJCGAs2Ha79ef/F9gwsCp6Y3p/vdmfzUZI5qVWbsWsNK5/ngT0fM8NaSfa6Y/iNms2BHFOTDkj9Gv8DH/Kd13Gi1/vCZg8YfC3nT3zMXGCiNez5DOyV9MxaBqNvB+DGc2N4ffVB/Lys/G3mMBPUj242XREv+UvLl73XM653CON6hzBtYAQXv7Kaa15dR0W1g6kDwnnxumG8vHQf7/2YBsC7t53TbFCvYbEopgyoy2ZfvWkk3+7KYmh0EEopHpnaly1HCrFaFH8eO7RV/bpd4eftQa9QPw7klDKuVwg/Hsxj2e5jfOMsO325I5PfXD6QXRlFVNocjHZmlPXbXKNPeABfPDiRV5bvIyXb/Mp48MKEBicmm+NptTS7vRr1S0s1IoO6uJzFDuvZtcXnegb7MnvMqVcCfL08OK930/a5YnjPrnhYFImpBSSlFdAjyIcBkQ2vwRjXgSdM63MlsB8GxiqlfDGlmClAIlAKzAFecN4uaa9GilP39Y4sHv1oG8F+XkzuH05FtR2lYHdGMVrDJYMjWZOSy+a0Am4aE8OalFwm9Ak1XwLZu81kEr7BTOobytwfUklMLWBCQijf7crm3PgQ+oT70zO4Cyv2mu/12sC+8U1Y/Rc8e4wjqHgvXkmvsznMDJQ0PLiKO/Jewt4lmBQdRVxEBFQVmjHEM7bCjDch/1DdwFg7FtUG9jsmxPPd7mwempJQ19sk8R1zyXn98VVcEOLvzV+uG8bji7bx9CUJ3DouFqUUT1/an23phYztFcLk/i6MBFjP6LhgRsfVZW0PTklo1etPxdQBEYT4F/LObecw/s8r+P0XuymusHHdqGgWJaWz4VAeu51ltxOVHAC6eFl5+pLOM8VbZ9bFy8qgHoEs3pxOdnElv75sQLOZ/ZngSo19g1JqEbAZsAFbMKUVf2ChUupOTPCf2Z4NFacmvcD0Uf58ewYX9Avj1nc2UlhWxbUjzXghQ6KCGB3bjcS0fPZkFZN7vNJc+FKWD29ONlce3rWUc+ND8LQq1qTk0Cfcn+TsEp6+xPRiGNwjiCP55YT4eRHZxWaC+jdPQd9L8L5hHh/+/n4eKprP3pRkPCyKPwUsxre0nN92e5ZvbV257MapJkPf+IZ53dL/MyMn4uxLveltKDoKQVFEBPqw4tHz635qlxeasVeGzGx+PJWTmNw/nMRfX9Rgma+XB0vuH9/sT//O6OlL6wLxJYO7M2/DYbr6evJ/Vwzkqx2Z/GvFfg7llhIf6kd4YCuGFhYnNTK2G++uSyXQx4NZp/HLoa259PWitX5Wa91faz1Ya32L1rpSa52ntZ6itU5w3uaffEuio2UWmcC+fFcG63YfYuOhfPZlm3E2Qv29iAj0ZlRsN47kl3PL2xsBZ1/krfPMpfW5yfDxvfh5WhgdG8z3+3JY47zar36fZYCLw/JRLw+Crx4zVzpe+wYWq5Woc83l7LlJnzEz7AgDsj/nLfulfJjmS7+ak7FKmSFsx9xjAnniO+YiorH3AdrMiUnNqgoyt8MrQ80sQdVl5hL4NuQuQb2xmhPRlwyOJNDHk4sGRvDDgTw8rRb+MauZoRDEaampqd88Nra2H3xn0HlaItpFRlEFXlYLd9vnM+CjVXT3fY3Jg3syf+MRxkWZOvCVw3twIKeUKpuDhAh/Ivy9IPFd6DnWzBjz9ROw4TUm9Z3On7/Zy+LN6YT6e9PfedKzJrBfq1aY2d9v/8YMhesMjtdOm0JWYiQXVv1Iv8rl2AN78o9j11CNpm/jcWEmPwO7PjWzyY+5y4wu2GOEmRZt+E1143H/+C8oyzPtC+4FPYZ30BHt3MbEBfPIRX25ergZYOzhqX3pFerPHRPiXBppU7TOhf3DuX9yb+6Z2HYnxNtC5ygIiXaTUVjO+QnBXO+xhhCK+HXfdP7v8kGM7RXMJYNND5bwAB/+dO0Q/nb9MNNj49D3Zhjac+40GXTCNFj5RyZHmnFT1h/MZ1JCqMlq17zEaL2DwT0CGFKyDnqdD7HjGkw7ZrFa8Bt6BROtOwmvOIjl0j/j1cUE9H7dG/WK8AmES/9SNy8lmJOix7Ng0e1marfSPBP8h82GK/9phggQgDmZ+4spCcSEmPMP8aF+PDQ1QYJ6O+niZeXxi/sT5Nu5jq8E9rOY1prMwnImeCUTjqmUTXOspouXlQX3jOOGc2LMtGmNZ/5JmmsmTBhwpQnQl7wIDhv9Nv+WC/xS6UGuKcNUFMHy3+H3zSN8MbMr3sePND99GhAw9Epzp9+lqP6X1XZx7Ne9ma5gA6+Ep9NNtg7Q8xy4/GU4uAo+/Zkp09gr27z8IsTZQkoxbmx7eiGPLtzGh/eOq+2nXV9xuY3SKjtjjq9Ee/lT3vcqfPcsMgHZy98MI7v2ZTPW9ow3zcnH6gozNdmwWXVzeAbHw6THUCv+wFy+otjbl8q4vWbMcbQZznbJ/WbdlqYIix0P0/9sSidAv4gANh7KJyG8hX7MjcdjGXEzHM+G5c5ZdGLGdbqZ4YXoLCRjd2Nf78wi5dhxfjjQ/Ah3GUXleGKjd85yVP/L8D33dpPprvwT/O9qE9T7TIUDy+GtqaaHyaHV5mRkv0YTMkx8DG7/hvxxTxOoyggr2AJHNgHKXBiUtd2cMG1p4mKLBcb+rHYi4bsn9uLvs4aftI90wzY8CjfMMz1mpPwiRIsksHdCSWkF2OzNTIzceD3nOBX1x6uoL7OonPGWHXhVF5n5G6NHQ9dY2PAaZO8y5Y2bF5up4XL3wYb/mFnevfybTkisFMSOI/iCB8ycnwdWQvpGMxHChc+YdVqauqwZMSG+XOU8wdcqAy6Hx5LNLD9CiGZJKaaT2Xm0iBmv/cAL1w45Yb/YmkvEoW640MaOFlYw3rILbfVGxU8ywfm6d6HosKmF10wE0ftC83j9q2D1NhMzeLRwtaW3v5lqbf9ys52BV8OAq+Dq15qddk0I0fEkY+9kauZSXJV84oktai4RTwj3p0fWCqqXPNTkJGhmYTljLXtMpl5TL48eBYOuaRq4Jz1uau+lx1o8AVqr9wVmouGKIug5xpRZht94ShcICSHangT2TqZmFpt1B3JPWI6pydIfGBPEix6v4bllrplPs578/DwGWtJQcRNOvuOokaZbo7Ka2xPpfWHd/egxJ9+2EKJDSWDvREoqqtmcVkCvMD9KKmy1pZbmJKYW0DO4C5dkvkoXKjnuFQ6rXzSX5jsF5iRhxWF6pLjiyn+aGYJqLgJqSeRw8Olq/kL6uLZtIUSHkcB+BmmtySqqILOonGrnDEE2h+aJi/thUfB9C/M56ooiHjxwD59W34fXrg9Z5HMtH/jdDJnbsCd/U7teTMlmbHiYuTZdEdDdXGB0Mharufx/1G2mDCOE6FTk5OkZ9Lfv9vGvlfsBM9tQXIgfvl5WLuwfwbCeXVmTksMjF/Vt8rqc5I0MYj9Hg86DvjPZV3Il72/K4FLvUPIWv8CAJy/Gw6IYVLWTrMCBRHudeKzpUzL5V22/TSFEm5B06wxxOMwkCCNjuvL7qwbhcMCalFzG9QrBy8PCxIQwth0ppLCsqslri9LN7OfHJv0Zpv2e+6cN5fkZI8iOmkb/ql28tmwnWw4cZbA6SEGYi9m6EOKsIYG9nSWlFdTO4F5fYloBWcUVzDkvjlvGxfHVQxO5d1Iv7ptsLqOfOiAch4bvdmU3ea3t2D7KtReRMaa+HRbgzQ3nxDBq0uV4q2p+WP0db7z3Lp7KTvfhJzkRKoQ460hgb0cV1Xbu+W8iTy3e3uS5z7dl4ONpYapz5pmgLp48femA2um2hkQFERviy+fOCaNLK22UVprptzwLD5BKJOGBjWagiRmHRnFpwEFuDd6N9gogbNCFCCF+WqTG3o4WJaWTV1pFflkVxRXVBDpH2LPZHXy1I5MpAyJavKRe5ezlLa+XyUgroGTJWGYcuIzIrr68d8cYAktT2eXZhwGWRmOG+wajIgYxx/cIHNsDCReBR9MxZIQQZzfJ2NuJ3aF5a81BAnw80Bq2HC6sfW7DoXzySqu40jkpQhOluTDvenqVbaU7+QRseYOw3PUkpRXgqCon1JZFiX9c86+NPc+M91KaI1eCCvETJYG9nXy65SipeWX83+UDsaiGl/1vSs1HKZjQzCS/OOywcA6UHsN66yc80vXvZOpgHvNZwvFKG0cP7sKCxt6thYH9a/qsWzzM0ABCiJ+ckwZ2pVQ/pdTWen/FSqmHlVLBSqmlSqkU5+2JZ8n9ibDZHbz4zV4eW7SNwVGBXDsymgGRgSSl1c0cuPNoMb1C/Zovw6Stg7S1MP0FiBrFrRP78k3Q9Yxw7OZctYfkXZsB8Izo13wDagJ77HnQRf5JhPgpOmlg11ona62Ha62HA6OAMuAT4ClgudY6AVjufPyT9/GWo7y66gDXj+rJwnvHYbUoRsV2Y8vhwtohAnZlFDEkqoVxVQ6sNNn2YDNP6A3nxHD7g8+h/cJ53HMh2Qd3ANC1ZwszyfuHwflPmmF2hRA/Sa0txUwBDmit04CrgPecy98Drm7DdrmtVcnHiAzy4YUZQ/D1Mhn5qNhulFXZ2ZtVQu7xSjKLKmrnCW3iwAoz/opPvZmFPLugLnyG0ZZkLjr+GVm6G9ER4S03YvKvXLuCVAhxVmptYJ8FzHfej9BaZwI4b5uNNEqpe5RSiUqpxJycE49Y6O5sdgdrU3KZWDMfqNPoONOFcVNqPjuPFgEwqEczgb00DzK3Qe/JTZ8bcStpvoMJV4Uc1D2I6tal6TpCCEErArtSygu4EvioNTvQWr+htR6ttR4dFhbW2va5le1HiyiusJn5QOuJ6tqFPuH+LN6cXhfYo5qZ6/PQKkA3HD2xhsXCzhHPYteKTK9YPK1y3lsI0bzWRIdLgM1a65pLIbOVUpEAzttjbd04d7N6Xw5KwfjeTXu73DUhnp1Hi/lgw2HiQnxr+7Q3cGCFGdO8x4hmt99zwBhmVj3LivA5bd10IcRZpDWBfTZ1ZRiAz4CaCDMHWNJWjXJXa1JyGRrdlW6NJ5Z2OJh58Ff83HcFGUUVDKpfX68ogo9ug7/2g20fQvz5ZvTEZvSNCGCnpT/B4dHt9yaEEG7PpStPlVK+wEXAvfUWvwAsVErdCRwGZrZ989xHcUU1W48Uct8FzfQvT9+Ide/nPIFinyWIIVE3meW5+2HBbMg/aOYk9fSBUbe3uA8fTyvv3n4OvcL82uldCCHOBi4Fdq11GRDSaFkeppfMT86xkgrSC8oZGVPXTzwluwS7QzdYVmvHIvDwwRGSwL9yXqNMRcLevfDJz8HqAbcuAVdmOQLGN3dRkxBC1CNn4E7BP5fv55a3NqDrzVaUXlAOmHHVG7DbYNcn0O8SrDd+SJewXoSseAwW3Ahde8LdK10O6kII4QoZBOwUHMw9TmmVndzjVYQFmEmhawJ7VBc7bJ0Pw2aBUqanS1muKbUERcHP1sLhHyFrp5kA2tv/DL4TIcTZSAL7KUjLKwMgvaCsXmAvI9Tfiy6b34CVz0NAhOm2uGMReAeZkRbBBPvY88yfEEK0AynFtFKVzUFGocnOa7L0mvtRXbvADmc3/x2LobwAdn0Kg68BD+8z0FohxE+RZOytlF5QhkPX3G8Y2KcFH4PD+8zgW3s+g9AEsJXD6DvOUGuFED9FkrG3Ulp+We399AJz3+HQHC0oZ3L1KjOA16V/hcpiWPlHiBoNkcPOUGuFED9FEthdUFRezcz//EByVgmHnfX17oE+tRl7zvFKqu02hhQsh95TYODV4BcG9krJ1oUQHU4Cuwu2HSlkU2oBH29JJzWvFF8vK8N7dq3N2NMLyhioDuNXkQWDrjF904fNAr9wGHztGW69EOKn5qwJ7Mcrbazbn8valFyOlVS06bYP5hwHYPW+XA7nlRET7EvP4C6kF5SjteZIfjkDLalm5Z5jzO2U5+DBJPCUURiFEB3rrDl5+pdv9vLej2kABHh78Mdrh3BFS3OKttLB3FIA9mQWk1NSwajYbkR386XS5iD3eJUzY09De/qhusWbF1k9wNrMCI5CCNHOzpqMfcuRQob37Mq8u86lT4Q/D87fwqKk9DbZ9oGc4wT4mO/A3ONVxIb4Ee0cDz29oIz0gnKGeRxGdR8MlrPmkAoh3JTbRqFqu4MXvt5LfmkVVTYHezNLODc+mPF9Qll47zjCA7z5YX9um+zrYE4pU/qHE+IctTEm2Jfobr6A6eZ4NL+U/ioVug9pk/0JIcTpcNtSzL7sEv7z/QG6+XoyISGUKrujdjhcT6uF+FC/Bl0T6zuSX4bFoswFRSdRVmUjs6iC3mH+aGDJ1gziGmTs5djzD+GryyWwCyE6BbfN2O3Oq4RWp+TUzkpUf4Lo2BDf2kv/G/v5vCRufXsDDodu9vn6DuaY+nrvcH8uGdwdL6uFvhH++Hl7EOznxdtrDxFYlGxWlsAuhOgE3Daw25xBedOhAjYeKsDf24PYYN/a52ND/Mg9Xklppa3B64orqtmVUcyBnFKW721m0qeqMijOrH1Yc+K0V5gf0wdHkvibqYTv+wDmXk5MsC+5xyu5Ja4IrawQPrAd3qkQQrSO2wb2mmy7yu7g8+0ZDOwRiMVSN4F0jDPIH25UjtlyuBCtwctq4fXvDzTd8Oq/wOuTwOEATFdHpSAuxExuEejjCQe/h9Q1vDg9go/vO4/x/pmo0L7StVEI0Sm4bWC31SujVNkcDcowUBeI0/JKGyxPSivAouChqQkkphWQlJbfcMMFqVB6DPJN0D+QU0pU1y74eNabrq7oCAB9daqZWCNzu5RhhBCdhkuBXSnVVSm1SCm1Vym1Ryk1TikVrJRaqpRKcd42M3VQ+6mpsQd4m/O/g6Ma9hmPCTEZe+M6e1JaPgMiA7l9fBxeHha+2ZnVcMNlzp40GVsAk7H3Cms0ZnqhCexk7TD3SzIgauTpviUhhGgTrmbsfwe+0Vr3B4YBe4CngOVa6wRgufNxh6nJ2Cf1CwNgaHTXBs8HdfGkq69ng54xNruDLYcLGRXbDV8vDyKDfMgurmy44dI8c5uxBa01h3JL6RVab47R6nKT0YPJ1NN+MPdjx7fZexNCiNNx0u6OSqlAYBJwG4DWugqoUkpdBVzgXO09YBXwZHs0sjk1NfbbzovjpjEx9G6cVWNOoB6ul7HvzSqhrMrOqFjz4yI8wLvp8AM1GfvRzeQcr6Ssyk58/cBe5LzoSVlNxu7tDz5BEDGo7d6cEEKcBlcy9l5ADvCuUmqLUuotpZQfEKG1zgRw3oY392Kl1D1KqUSlVGJOTk6bNbwmY+/iaeW8FiZ4jg32JbVejT0prQCA0XHBAIQH+HCsfsauNZQ5M/as7RzOKQbqyjoAFJphC4gbD3n7Yf8KiBkHlno1eCGEOINcCewewEjgNa31CKCUVpRdtNZvaK1Ha61Hh4WFnWIzm7I7e61Y6/WEaSw2xJeMwnKqbGbdxLQCIoN8ai9MCgvw5lhJvcBeUQgOmxk/vbqM/LRdZjvB9QO7s77e/3JAQ3G6lGGEEJ2KK4E9HUjXWm9wPl6ECfTZSqlIAOdtM53C209Nxu5xgsAeE+yLQ8NR51R2San5jIytO8cbHujN8UobZVXOvu5lzh4yfaYC4Di6GYuidvgAwPSIUda6OUzBZO9CCNFJnDSwa62zgCNKqX7ORVOA3cBnwBznsjnAknZpYQtqesVYThDY45y18UO5x8koLCejqILR9QJ7RIAPQF05ptRZX+85FrwC8MvdTo+uXfDyqHeYCg9DUBR0ize1dS9/6C4zJAkhOg9Xx4p5EJinlPICDgK3Y74UFiql7gQOAzPbp4nNs7uQsQ/qEUgXTysr9h6jtNIOwOjY4NrnwwPNBNPHSirNl0DNiVP/MIgcSujRZGK7+zbcaOERCIoBpaD3hWYqPKvbDrkjhDgLuRSRtNZbgdHNPDWlTVvTCjWlmBPV2H29PJgyIJyvdpi+6l08rfSPDKh9PrwmY6/pGVOTsfuGQmgC3dMWExPs7BFTkAZdY0wpJn6SWTZzrjnhKoQQnYjbXnlal7Gf+C1cMawH+aVVLEpKZ3jPrnha69YPDzAZe21f9pqM3S+UiqBedKOEfoFVkLsf/j4U1r0CxRkQ1LNuB6rlLxYhhDgT3D6wn2xei/P7hhHg7UFFtYPRcQ0vju3q64mX1VIvY88DTz/w7EKWpwne/T2y4GiieX7FHwBtMnchhOik3D6wnyxj9/G0Mm1Qd4AGPWIAlFKEBXiTUz9j9wsB4JCOAiDGcdRcYWr1BounWa9rT4QQorNy28DuSo29xu3j45jUN4wxccFNngsPrNeXvTTX1NeBPRVdqdQehFYdhqzt0H0wnP8EKAuEJLTdGxFCiDbmtoG95gKlE/WKqTE4Koj/3jEGP++m54obDCtQlgt+JrCn5VeSrrrjVXDQDB3QfQhM+CU8vMN0dxRCiE7KjQO7uXUlYz+R8ACfuoy9LL82Yz+UV0qOdwykrTVXpHYfYk6UBkWf1v6EEKK9uXFgP/mQAq4ID/CmsKyaimq7sxQTjM3uYEd6EdVde0OFmXaP7kNPt8lCCNEh3Daw19bYG3c3XPEHWPM3l7cTEWj6sufmF4CtHPxC2ZNZQnm1Hf+oAc61lEx7J4RwG24b2O0OjUU1GlJAa9j0Nmx6x+XthDmvPi3IyTALfENJdM6qFJ3gHCogpI8ZnlcIIdyAWwf2JmWYwsNQnm9GXCzOcGk73Z0Ze0GucwJrv1AS0wroEeRDWJxzjHWZ9k4I4UbOrsDunM4OgCMbXdpOn3B/Arw9OJjqHGfdN5TNaQWMiguGLt1gxM0wbFYbtVoIIdqf2wZ2m0M3vTgpY7O5iMjqDembXNqOp9XCuN4hZKSnApDlCCCz/iiQV/0b+l7chi0XQoj25bbDEraYsUcMAg8flzN2gEl9wzi+7zDay4uNeWYSjlGNrlIVQgh34bYZe5PA7nBAxjaIGgk9z4HMrWCrbPH19U1KCKOPSqfAN5ZPtmXj7+1B/+4BJ3+hEEJ0Qm4b2G2NA3vBIagsgh4jIHoM2KvMGC8uiAnxZYBHBptKw1iZnMMvL+qLh9VtD40Q4ifObaOX3eHAo34lpubEaY8REH2OuZ/uYjmmupxIfYzdVZGM7RXM7efFtWVThRCiQ7ltYPepyuPbqjmQus4sOLLR1NbDBkBgJPh3h+xdrm0sNwULmvKuCfx15rATTrcnhBCdndsG9sCqYwRyHJLeBYcd9nwGvafUTVMX1g9y9rq2sZxkAH415+qGE1cLIYQbcimwK6VSlVI7lFJblVKJzmXBSqmlSqkU522HdiNR9mpzZ+9XkLIUSjJhyHV1K4T1g5x9J566rrrCPJ+zF5QVgnu3b6OFEKIDtCZjn6y1Hq61rpn79ClgudY6AVjufNxhlMMZ2KtL4ctHwMsf+k6vWyGsH1SVtHwFanUF/H0YfPdrE9hDeoOHV/s3XAgh2tnplGKuAt5z3n8PuPq0W9Ma9qq6+8VHof9l4FWvjBLW39y2VI5J+Q6OZ8H6VyF1LYT2bb+2CiFEB3I1sGvgO6VUklLqHueyCK11JoDzNry5Fyql7lFKJSqlEnNyck6/xU7Wmow95jxzO2RmwxVqA3ty8xvYuciMve4basZbr1lfCCHcnKtXno7XWmcopcKBpUopF89Kgtb6DeANgNGjR5+g4N1KDpu5nfgIZE6BXpMbPu8XCr4hzWfsFcWQ/A2MmmP6vH98l5n6TgghzgIuBXatdYbz9phS6hNgDJCtlIrUWmcqpSKBY+3YziYsNRl7UE9IuKj5lUL7NZ+x7/0S7JUmy48+B7rFQo+R7ddYIYToQCctxSil/JRSATX3gWnATuAzYI5ztTnAkvZqZLPtcjhr7FbPlleq6fJYv2eMvRo2vQVdY0xQVwp6jqnrJimEEG7OlWgWAXyizExFHsAHWutvlFKbgIVKqTuBw8DME2yjzVlrSjEnDOz9Tf18/zIoSIV+l8K6V+BoIlz7pgnqQghxljlpYNdaHwSGNbM8D5jSHo1yhdI1GfsJuiiG9TO385z9279+ArQDxj0AQ69v3wYKIcQZ4rb1B0ttxn6CwN7zXBg6y4z4GDcBdnwEVWVw0e86ppFCCHEGuHFgd548PVEpxssXrn297nHEoPZtlBBCdAJuO1aMVbuQsQshxE+Q2wZ2i3Zm7JYTZOxCCPET5LaB3eqoxo4FGs97KoQQP3FuGxUt2oZdSbYuhBCNuW1g95DALoQQzXLbwG7V1diV23bqEUKIduPGgd0mgV0IIZrhtoHdQ1fjkB4xQgjRhNsGditSYxdCiOa4bWD30DYcFinFCCFEY24b2K3SK0YIIZrltoHdExtaauxCCNGE2wZ2D6QUI4QQzXHLwO5waBPYlQwAJoQQjbllYLc5tCnFnGjIXiGE+IlyObArpaxKqS1KqS+cj4OVUkuVUinO227t18yG7A6NF3bpxy6EEM1oTcb+ELCn3uOngOVa6wRgufNxh7BrLSdPhRCiBS4FdqVUNHAZ8Fa9xVcB7znvvwdc3aYtOwG7vSawy8lTIYRozNWM/RXgCcBRb1mE1joTwHkb3twLlVL3KKUSlVKJOTk5p9PWWjaHAw9ll4xdCCGacdLArpS6HDimtU46lR1ord/QWo/WWo8OCws7lU00YWrsNrRMiyeEEE24UssYD1yplLoU8AEClVLvA9lKqUitdaZSKhI41p4Nrc+uNT5SYxdCiGadNGPXWj+ttY7WWscBs4AVWuubgc+AOc7V5gBL2q2VjdhqauySsQshRBOn04/9BeAipVQKcJHzcYewO/uxKzl5KoQQTbQqMmqtVwGrnPfzgClt36STs9kdeGKXjF0IIZrhlleeOhw2LEqDBHYhhGjCLQO7vbrK3JEhBYQQogm3DOyO6kpzRzJ2IYRowj0Du80EdiUZuxBCNOGWgV3bq80dCexCCNGEWwZ2h7PGrqQUI4QQTbhnYLc5A7uHBHYhhGjMPQO7XQK7EEK0xC0Du7ZLd0chhGiJewZ2mzl5arF6n+GWCCFE5+OmgV1KMUII0RL3DOzOUozFUwK7EEI05paBHbt0dxRCiJa4dWC3esjJUyGEaMwtA3tdjV1OngohRGNuGdiVw/SKsUqNXQghmnDLwF4zVoxFesUIIUQTbhnYa2rsFinFCCFEEycN7EopH6XURqXUNqXULqXUb53Lg5VSS5VSKc7bbu3fXCfnBUoekrELIUQTrmTslcCFWuthwHBgulJqLPAUsFxrnQAsdz7uEBaHM2P3ksAuhBCNnTSwa+O486Gn808DVwHvOZe/B1zdHg1stk0OGwBWKcUIIUQTLtXYlVJWpdRW4BiwVGu9AYjQWmcCOG/DW3jtPUqpRKVUYk5OTps0WtX2Y5eMXQghGnMpsGut7Vrr4UA0MEYpNdjVHWit39Baj9Zajw4LCzvFZjakHNXYtAUPD4822Z4QQpxNWtUrRmtdCKwCpgPZSqlIAOftsbZuXEuUo5pqPLBYVEftUggh3IYrvWLClFJdnfe7AFOBvcBnwBznanOAJe3UxqZtsldhw9pRuxNCCLfiSi0jEnhPKWXFfBEs1Fp/oZT6EViolLoTOAzMbMd2NqAcNqpdaroQQvz0nDQ6aq23AyOaWZ4HTGmPRp2MxVmKEUII0ZRbXnmqHNXYlAR2IYRojlsGdsnYhRCiZe4Z2HU1dgnsQgjRLLeMjhaHTUoxQrSj6upq0tPTqaioONNN+Unx8fEhOjoaT8/Tm0TILaOjRVdjQ2ZPEqK9pKenExAQQFxcHErJ9SIdQWtNXl4e6enpxMfHn9a23LIUY5WTp0K0q4qKCkJCQiSodyClFCEhIW3yK8ktA7tFS2AXor1JUO94bXXM3TKwWx1y8lQIIVriloHdou3YldTYhRCiOW4Z2K26GruUYoQQ9fj7+7f4XHx8PMnJyQ2WPfzww7z44osAbNmyBaUU3377rcvbrG/16tWMHDkSDw8PFi1a1MqWtz23jI5WXY3dKhm7EB3ht5/vYndGcZtuc2CPQJ69YlCbbvNEZs2axYIFC3j22WcBcDgcLFq0iHXr1gEwf/58JkyYwPz587n44otbvf2YmBjmzp3LX//61zZt96lyy4zdQ9skYxfiLPfkk0/y6quv1j5+7rnn+O1vf8uUKVMYOXIkQ4YMYckS1waVnT17NgsWLKh9vHr1auLi4oiNjUVrzaJFi5g7dy7ffffdKfVKiYuLY+jQoVgsnSOkumV0tGobDqmxC9EhOjKzrm/WrFk8/PDD3HfffQAsXLiQb775hl/+8pcEBgaSm5vL2LFjufLKK0/am6Qm6G7bto1hw4axYMECZs+eDcC6deuIj4+nd+/eXHDBBXz11Vdce+217f7+2lPn+HppJatk7EKc9UaMGMGxY8fIyMhg27ZtdOvWjcjISH71q18xdOhQpk6dytGjR8nOznZpezVZu81mY8mSJcycaUYanz9/PrNmzQLMl8n8+fPb7T11FLeMjh5U47BIxi7E2e66665j0aJFZGVlMWvWLObNm0dOTg5JSUl4enoSFxfnculk9uzZTJs2jfPPP5+hQ4cSHh6O3W5n8eLFfPbZZzz//PO1V3+WlJQQEBDQzu+u/bhlxu6hbdglsAtx1qs56blo0SKuu+46ioqKCA8Px9PTk5UrV5KWlubytnr37k1ISAhPPfVUbRlm2bJlDBs2jCNHjpCamkpaWhozZszg008/bad31DHcL7AfTcKLavI9I890S4QQ7WzQoEGUlJQQFRVFZGQkN910E4mJiYwePZp58+bRv3//Vm1v9uzZ7N27l2uuuQYwZZia+zVmzJjBBx98AEBZWRnR0dG1fy+99FKz2920aRPR0dF89NFH3HvvvQwadGbOS9RQWusTr6BUT+C/QHfAAbyhtf67UioY+BCIA1KB67XWBSfa1ujRo3ViYuLptXjJ/ZRvWcQTcQv5523nn962hBDN2rNnDwMGDDjTzfhJau7YK6WStNajXd2GKxm7DXhUaz0AGAvcr5QaCDwFLNdaJwDLnY/bV3kB7FjMSs9J2Dxcu3BACCF+alyZ8zQTyHTeL1FK7QGigKuAC5yrvQesAp5sl1bmHYDj2bDvW7CV86nvJXhZZIAiIURDO3bs4JZbbmmwzNvbmw0bNrTJ9p9//nk++uijBstmzpzJM8880ybbbyut6hWjlIrDTGy9AYhwBn201plKqfAWXnMPcA+Yq7NOyfpXYdNb5n70OewriGeYBHYhRCNDhgxh69at7bb9Z555ptMF8ea4HNiVUv7AYuBhrXWxq8NLaq3fAN4AU2M/lUZy7s9gwBXmfvggbP/egVUCuxBCNMulwK6U8sQE9Xla64+di7OVUpHObD0SONZejSQ0wfw52R0aDwnsQgjRrJOePFUmNX8b2KO1rt/X5zNgjvP+HMC1QRvagM2hsXaSMRmEEKKzcSVjHw/cAuxQSm11LvsV8AKwUCl1J3AYmNkuLWyGw6GxSlwXQohmnTQ8aq3Xaq2V1nqo1nq48+8rrXWe1nqK1jrBeZvfEQ0Gk7F7SMYuxFmtsLCwweiOrrr00kspLCxs1Wvmzp1bezVqjdzcXMLCwqisrATgqquuYty4cQ3Wee6551weqveOO+4gPDycwYMHt6ptp8Itx4qxO7ScPBWio3z9FGTtaNttdh8Cl7xwwlVqAnvN6I417HY7Vqu1xdd99dVXrW7Otddey2OPPUZZWRm+vr4ALFq0iCuvvBJvb28KCwvZvHkz/v7+HDp0iPj4+Fbv47bbbuOBBx7g1ltvbfVrW8st016bwyEnT4U4yz311FMcOHCA4cOHc8455zB58mRuvPFGhgwZAsDVV1/NqFGjGDRoEG+88Ubt6+Li4sjNzSU1NZUBAwZw9913M2jQIKZNm0Z5eXmz+woMDGTSpEl8/vnntcvqD+27ePFirrjiitqxa07FpEmTCA4OPqXXtpbbZexaa+wOjUUCuxAd4ySZdXt54YUX2LlzJ1u3bmXVqlVcdtll7Ny5szZbfueddwgODqa8vJxzzjmHGTNmEBIS0mAbKSkpzJ8/nzfffJPrr7+exYsXc/PNNze7v9mzZ/PBBx9www03kJGRwb59+5g8eTJgxpR59tlniYiI4LrrruPpp59u3zd/mtwuY1+VnEO1XdMvwn2H1BRCtN6YMWMalED+8Y9/MGzYMMaOHcuRI0dISUlp8pr4+HiGDx8OwKhRo0hNTW1x+5dffjlr166luLiYhQsXct1112G1WsnOzmb//v1MmDCBvn374uHhwc6dO9v67bUptwvs//n+AJFBPlw2VEZ3FOKnxM/Pr/b+qlWrWLZsGT/++CPbtm1jxIgRzY7L7u3tXXvfarVis9la3H6XLl2YPn06n3zySYMyzIcffkhBQQHx8fHExcWRmpp6yuWYjuJWgX3rkUI2HMrnzgnxeEp/RyHOagEBAZSUlDT7XFFREd26dcPX15e9e/eyfv36Ntnn7Nmzeemll8jOzmbs2LGAKcN88803pKamkpqaSlJSkgT2tvDP5Slc9NL33Dl3EwE+Hswac4pjzggh3EZISAjjx49n8ODBPP744w2emz59OjabjaFDh/Kb3/ymNgifrmnTppGRkcENN9yAUorU1FQOHz7cYPvx8fEEBgbWDiz2hz/8ocGY7S2ZPXs248aNIzk5mejoaN5+++02aXNzTjoee1s61fHYF2w8zOqUHAAuHtSdq4ZHtXXThBD1yHjsZ05bjMfuFr1iZo2JkSxdCCFc5BaBXQgh2sr999/PunXrGix76KGHuP32209723l5eUyZMqXJ8uXLlzfpitmeJLALIZqltcbV4bndyb///e9223ZISMhpjQffVqVxtzh5KoToWD4+PuTl5bVZoBEnp7UmLy8PHx+f096WZOxCiCaio6NJT08nJyfnTDflJ8XHx+eEPWtcJYFdCNGEp6fnKQ10JToHKcUIIcRZRgK7EEKcZSSwCyHEWaZDrzxVSuUAaaf48lAgtw2b0xGkzR1D2twxpM0do7k2x2qtw1zdQIcG9tOhlEpszSW1nYG0uWNImzuGtLljtEWbpRQjhBBnGQnsQghxlnGnwP7GyVfpdKTNHUPa3DGkzR3jtNvsNjV2IYQQrnGnjF0IIYQLJLALIcRZxi0Cu1JqulIqWSm1Xyn11JluT3OUUj2VUiuVUnuUUruUUg85lz+nlDqqlNrq/Lv0TLe1PqVUqlJqh7Ntic5lwUqppUqpFOdttzPdzhpKqX71juVWpVSxUurhznaclVLvKKWOKaV21lvW4nFVSj3t/HwnK6Uu7kRt/otSaq9SartS6hOlVFfn8jilVHm94/2fTtTmFj8Lnfg4f1ivvalKqa3O5ad2nLXWnfoPsAIHgF6AF7ANGHim29VMOyOBkc77AcA+YCDwHPDYmW7fCdqdCoQ2WvYi8JTz/lPAn890O0/w2cgCYjvbcQYmASOBnSc7rs7PyTbAG4h3ft6tnaTN0wAP5/0/12tzXP31Otlxbvaz0JmPc6Pn/wb83+kcZ3fI2McA+7XWB7XWVcAC4Koz3KYmtNaZWuvNzvslwB7AXSdnvQp4z3n/PeDqM9eUE5oCHNBan+rVzO1Ga70ayG+0uKXjehWwQGtdqbU+BOzHfO47VHNt1lp/p7W2OR+uB05/TNk21MJxbkmnPc41lJnZ5Hpg/unswx0CexRwpN7jdDp5wFRKxQEjgA3ORQ84f8q+05nKGk4a+E4plaSUuse5LEJrnQnmCwsIP2OtO7FZNPwP0JmPM7R8XN3lM34H8HW9x/FKqS1Kqe+VUhPPVKNa0NxnwR2O80QgW2udUm9Zq4+zOwT25ubm6rR9NJVS/sBi4GGtdTHwGtAbGA5kYn5mdSbjtdYjgUuA+5VSk850g1yhlPICrgQ+ci7q7Mf5RDr9Z1wp9QxgA+Y5F2UCMVrrEcAjwAdKqcAz1b5GWvosdPrjDMymYbJySsfZHQJ7OtCz3uNoIOMMteWElFKemKA+T2v9MYDWOltrbddaO4A3OQM//U5Ea53hvD0GfIJpX7ZSKhLAeXvszLWwRZcAm7XW2dD5j7NTS8e1U3/GlVJzgMuBm7Sz8OssZ+Q57ydh6tV9z1wr65zgs9DZj7MHcC3wYc2yUz3O7hDYNwEJSql4Z5Y2C/jsDLepCWdt7G1gj9b6pXrLI+utdg2ws/FrzxSllJ9SKqDmPuZE2U7M8Z3jXG0OsOTMtPCEGmQ2nfk419PScf0MmKWU8lZKxQMJwMYz0L4mlFLTgSeBK7XWZfWWhymlrM77vTBtPnhmWtnQCT4LnfY4O00F9mqt02sWnPJx7ugzwqd4FvlSTC+TA8AzZ7o9LbRxAuZn3XZgq/PvUuB/wA7n8s+AyDPd1npt7oXpJbAN2FVzbIEQYDmQ4rwNPtNtbdRuXyAPCKq3rFMdZ8yXTiZQjckU7zzRcQWecX6+k4FLOlGb92Pq0jWf6f84153h/MxsAzYDV3SiNrf4Weisx9m5fC7ws0brntJxliEFhBDiLOMOpRghhBCtIIFdCCHOMhLYhRDiLCOBXQghzjIS2IUQ4iwjgV0IIc4yEtiFEOIs8/80loRH7alSRAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_VAL_1','train_VAL_1']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD4CAYAAAAJmJb0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABBsklEQVR4nO3dd3hU1dbA4d/OpFdSIRAgoQSpoRelXpSmCCIoiNgLKnb97Fe9Xq/XXq4FsWEBEbCAiogoXXrvLQQIgZCE9DLJzOzvjx1CAgESSJ2s93nyzMw5Z86sOQxr9qyzz95Ka40QQgjn5FLdAQghhKg8kuSFEMKJSZIXQggnJkleCCGcmCR5IYRwYq7V9cIhISE6MjKyul5eCCFqpfXr1ydrrUPLun21JfnIyEjWrVtXXS8vhBC1klLqYHm2l3KNEEI4MUnyQgjhxCTJCyGEE5MkL4QQTkySvBBCODFJ8kII4cQkyQshhBOTJC+EEFVp8X8hdnGVvZwkeSGEqCp5GSbJH1pdZS8pSV4IIc4lfh0k7a6YfSVsBDREdKmY/ZVBtQ1rIIQQNcbu+eDmCU16QdohOLYFctNg9zzYtxA8AuC2+VC/zcW9Tvxac9tIkrwQQlSN7T/CrFvMfeUC2nFqnVcgDHgG1n0O31wLzfrBoZXgFw6Nu0P/p82XQ1kdWQ/BLcx+q4gkeSFE7ZZ+BP56Cfo8CiEtS99GaziwBBw28K0PYW3BxcWUYeZMgojucNmDcHi1ScKNuoB3sPlzdYdWw2DqMNgzH5peBtnJsOJd89rXfgpKnXqtrOPw6yPmuQGNzWP/hmb/8eug+T+q5rgUkiQvhCiftMPwy8MQeRl0mgA+Iefedtts6DER3LwqPhZrFnx7PRzbCkc3w51/lf46+/6EadeeeuwdAgGN4Ng206oeM9U8bn1V6a/ToB08vANcPcFSmDaXvQV/vmi+FAY8ZZbZbTD7NvNl4e4Dualg8QC7FQIiIPs4RHSt0ENwPnLiVQhRdg4H/HQPxC6ChS/AuzHmZKLWsPYz2PlzyW2/v8NsN20MWDNPrctLhwPLYPMMsOWXfI349RC3wuzzXOwF8P3tkLgdej8Cx3fAb0+Uvu3qyaYFf+tvcM3H0Kw/eAZA74fNsoBG53/vHr6nEjyY58bcAEtehSMbzLK//gVxy2D4e/B/B+DpBHjiAPiEws8PmW2qOMlLS14IcUpuGix/C3reC34Nzly/9pNTSSyiG0y/3vxFD4ENX5qa9vXT4JJhZtvDqyBmHGyZCZ8Ngiv+Bcl74M+XwJZr9mmzQpeboSAX/nge1nxslge3MCWOmHGw6xfT7fDyF0wN3F5gWsx75sOVb0K3OwANy9+GdoW185NS9sO+P6D/U9D0UvMXM/bij5VSMPS/sP9P+PVR6HCdKeF0uRU6jjPbuPuY2z6PwvwnzS+B+u0u/rXLE6Y+37dlJenatauWSUOEqGEWvmiSfKthMHZ6yVpzdjK8097UpMfPMuuO7zTJ25oB3e8yNefjO0wiPfg3RPaG8bNND5VfHzE9V8B8KXS/05R9QlvD+Jkw9wHzRdHjHgiPMa3vo5vA3Q/yC38FDPynabV/fzts+x4GvwK97jXrCvLgg27g4Q93LwUXi1n+25Ow9lN4eDv41a/4Y7ZlFvxwh7l/yVUw5suSLf6Tsf2vM9RrCrf9dlEvp5Rar7Uu888BackLURNs/g6W/BfuWmzKCMXlZZiufG1Glq8nR3llp8CaKaa0sHueKb20Hm7WKWXWFeTA4P+cSv5hrWHCT5C0CzreADkpJgHnppnnXvGS2bblFTBpHWydDV71zJeIUiYprv0U0uNhy3fQ5RbTOgbT2t71q1neaphpzS99AzITTYIf+PypBA/m2Fz+Isy+FdZPha63weZvzf22IysnwQO0Hw27fgaHHUZ/fmaCPxnbLb+e+uKpQtKSF6K65aXDe50hJxmufAu63X5qXewSmHMfpB+GnvfBkP9U7Ose3wkh0ab1+9e/YMV7MHE5/DQRkveaunhgJFz/NXw+xPQjHze94mKIW2F6rTS9DA6uMC3w8JjSt02Ngw96gC0POlxvauvFf2mAiffzIaZM5BEA1nRo2htGf1Z6+akWqpSWvFJqCPAuYAE+1Vr/97T1AcA3QJPCfb6htf6izFELUdtpfSrhZCbCD3dC55tMK09rU3c+Wyt82ZsmwfuFw8avTZLPzzb16bWfQFBzaHUlrPoQ2l4DjbuZ5x3dDLvmwZF1ptYbFAV9/w88/c8fb9ph+PIqkziLa3et6UlyzRRY8Y7phbLxG/i4r0mulz14oUeodE16mq6GB1dAo65nT/BgvmwGvwz7F8Hwd89M8GCWjZ0OW2eZslFYG1MWqoYWdE1x3pa8UsoC7AGuAOKBtcA4rfWOYts8DQRorZ9QSoUCu4EGWuv80vYJ0pIXtVzGUZg7CYa+BsHNTW05drE5sbjkNXPFpIub+fm+8RvTsnxgE3gHldxP8j74qBe0Gw3hHczJufGzYcGzpg93z3vgH8+Z/t0f9gKLGwx8zrTAl71p9hHWxqxP2mVKKb3ugz0LTK+Xvo+b/uCJO0yS9go0Xw4Ln4ecVFMayU4yX0JegaaFfPqXxJEN8NVIk/xvnVfxx3LOfeYYjfgQOo2v+P07mcpoyXcH9mmtYwtfYAYwAthRbBsN+CmlFOALnABsZY5aiNpm+w/mZOLPD8I/njVXRLr7wnc3gourSe6LX4WZEwAFaNj/l2nZn2QvMCfs3LzNCUVXD/jjn6a7obsvTPgRmg84tf2oj02XxNm3mccdb4TB/z519eSUAbBxGnS708SVmQBZx8AnzNT7i/MKMvsvyxgqjTrDAxvMF0xl6H636e/e9prK2X8dV5Yk3wg4XOxxPNDjtG3eB+YCCYAfcL3Wxa8NFsLJ7F0AFnfTnXD69eDXEO5ZYXqHhLQyXQgbdoZFL5skNv062PuHSfILnjO9TFwsprV93VfgH272236M6RZ44/fQsFPJ14zsbXqIHPzbdFWMvKzk+k7jTVe+Bc+aBN98oPnyAdOfu/VVptUe1taURVzdy/5+z3XB08UK7wDXfVmupyRlWnGzKOp5l+M91FFlSfKlFL44vcYzGNgE/ANoDvyhlFqmtc4osSOl7gLuAmjSpEm5gxWiRrBmmROGPe42Y5EcWmn6ansHmQtkTgqKMpe8A7S43PTVPrYV/n7PtN4LcswVo21GnHrOVe+Atp9x1WZuvh0vd4v5YojqU3pc7a6F+U+bfuZhbU3ZZ8U7pjdLl1tLr2FXAa01by/cS1xyNu+O7Yi6yDj+3JnIA99upENEPb69q+dF7SuvwM7uY5kkZ1npGhlEgJcbVpudVbEnaBHmS8MAz4uOt7qV5YrXeKBxsccRmBZ7cbcCP2hjH3AAuOT0HWmtp2itu2qtu4aGhl5ozEJUrwNLwFEA0YNNWWbEBybBnkvLQaZ74Y/3mAT/8HbTpfCqd0pu5+p+RoKfv+0YMS8uYP62o+d+Da/AU5flX/agqcX3ecR0JSxjoopPzWHS9A3sSMg477brD6byyHebyMgrOGOdw6GZv+0Ymw6n8caC3bz3517mbk5gzYETRdscSctl4tfr+X37sTLFBjBt9UHu+Mqcy1sZm0J8ak6Zn3vSyfOQDofm+imrGPHBCm7/ch3//sVUoD9eEsvNn6/hsv/+xbhPVuFwVE8PxIpSliS/FmiplIpSSrkDYzGlmeIOAQMBlFL1gVZAbEUGKkSlSI0zl9GXx94/TM28cU8z8FSnG8+fRFsMBBQkboXON4N3EPagFqX3qS4mKdPK0z9uJd/u4MWfd5BtPc+prr6Pm4uS2o0q89ux2R3YHZrcfDt3fbWeX7YcZcJnq9mekM6iXcdZFZtyxnNSsqzc8816fth4hKd/2MrpHTjmbz/GxG/WM/KDFXywaD9jukRQz9uNz1ccAEwLeuLX65m//Rh3f72ecVNW8dCMjby7cC9Wm73UOBdsP8azP22jf3QoP95nSlVzN5/e3jy3HzfG0/vVRexJzGT+9mNsPpzGo1dEM7JjQ+ZsSuBoei5frzpI96ggbu8dxarYEyXev9aa+duOkp575hfbSXkFdl75bSdH0swVvXHJ2Tzz41Z6/Gchd3y59oxjVdnOW67RWtuUUpOA3zFdKD/XWm9XSk0sXD8ZeAmYqpTaiinvPKG1Tq7EuIW4eHkZMPUqyDxqTkJG9T3/cxx2k+Sb9S9R0y6wO8jJtxPgdZaTk95BZhiAI+uh171k5BUw5O2ljO4SwSODWp2xebbVxrqDqXyyNJYsq41XRrXnqR+28sLc7QT6uHMkNZcwfw8GtAqjb3SxX8VhrWHY62U+BJl5BYz+aCWJmXmEB3ix61gGL17dlv/9tY8r31tetN3Twy7h5ksjWReXSkZuAdPXHCItt4DrukYwc108vVuEMLb7qRLsVyvjiAj04plhrcmz2bk6phGhC3bz0ZL9bDyUyqfLD7D1SDofje9MXEoOs9Yf5khaLj9tSuD37cd4bHA0EYHeNA/1xeKiWBWbwgMzNtKhUQAfjO+Mt7srXZoGMmdjAjf2bMqcTQmM7NgQP8+znxw+fCKHZ3/cRna+nUnTzVgzzUN9uHdAC+JSsvlpUwJ3frWOpEwrr4/uQM9mwcxeH8/0NYe4tIU5J7F4dxITv9lAs1AfPru5G1EhPme8zop9yXy8JJbVsSeYclMXxn+6mhPZ+bRr5M/Cncf5ZtVBJvSKLPO/0cUqUz95rfU8YN5pyyYXu58ADKrY0ISoZL8/BRlHzHCw302AOxaWHKrWZjX1d5/gU8sWPAsZ8aa/djGPzdrMnE0JNAny5s4+UaX/J77iRUg9CPWaMH/dYRLS83jvr31EN/Djqg4NizYrsDu45sMV7EnMQin419VtGde9CeviUpm1Ph43iyIi0JtFu/P4YkUcV3YI598j2hHoU76TkA6H5tGZm9mXlMXgtvVZHXuCJ4aYZH5Zi2C+W3uYS1uEMHt9PP+Zt4u3/9hLbsGpVvZLI9sxvnsTjqbn8exP2/DxcGV4TEP2JGayKvYETw69hKHtw4u2n9CrKR8vjeWaD/82x2xQdNH6e/o3B0y9/bFZm7ltqinJtAzz5Yo29flkWSyNg7z59OZueLubtDWyY0Oem7OdK95aQmKGlV1HM3j5mvZFr/fpslhSsvO5b0ALtNY8NmszSqmiL0yAd8d2xOKiaB7qS7/oUJbsSaJZqA99W4bi4qIY1bkR36w6SEqWlWBfD37enICfhyup2flc8dYSPFxdaB3uz3d398LiYn7NrTlwAqVg0+E0Br+9lOx8O7Pu7kWHiABu/mIt/5m3iz4tQ4ks5QuiMsiwBsL5ZB03l8I77KYPe6srTX26uN3zTd/s3o+YwbE++Yfp637LLya5L33dXA6fkwIDnjYlkNVTzAVJPe81l8kXOpiSzc+bE+gXHUpOvo3n5mzH38uNER1PG9nw5OBYwNzCL4RQPw8en7WFDo3q0STYG4Dv18ezJzGLl0a2Y0THhvgXtk7/NaItV3ZoQNfIIPw93ci3Ofh4yX7e+2svHhYX3rq+43kPzbH0PP7YmciaAyc4mJLNlvh0/nlVG27rHVViuxZhfjxzpZkFqW/LUKLD/EjMzOOK1vUJr+eJj7srjYNMvB+M78wdU9fxwIyNbE/I4HBqDu6uLlzXtXGJfYYHePHkkEtISM9lfI+mtAjzPSO+ga3rs/jxAexJzCQ2KYspS2P5cPF++rQM4f0bOpf4pTSsfTgv/boTVxcXBrWpz/Q1hxjbrQntIwJIy8nntfm7ybc7+H59PNlWG9n5dt4YE8PoLhGk5xawLu5EiS/X23tHsWRPErddFoVLYcK+oXsTvlgRx+z18dx8aSQLdiRyZftwJv2jBdPXHOJYeh4/bjzCvK1HGR5j9rXqwAm6NAkkuoEf01cf4rXRHYhpXA+A167twKC3l/D5igP8a0TVDFQmwxqIWiEly8qbf+zh/wa3Mt3mspJOjdF9ul8egXWfnXrcoL0ZEMvF1XRR9AqED3uaqznvWmT6p694D/54Du5aYsoxi/5NXrPBnLBCwyO/n5ox6JKrTJfHYldQ/nPONmasOczyJwYQ4O3GTZ+tYeOhNKbe1o1Lm5/Z9fB4Rh49X/mTSQNacH33JvR+9S8euTya+we2xGqzM+D1xYT6e/LTvZeWqWfHy7/u4LPlB/jjkX40Dz0zceYV2Hnx5x0s25tEfKqpEzeq50VEoBd9WoZw34AWF92DJDffzmOzNjNv21G0hlGdG/HWdR0vap9gzhdsjk8jJqIerpYzTyHGJmUR6ueBQ8PAN5cQEejF9/dcyjerDvL83O28Mqo9v245Sn1/T27q1bQo2Z7Nlvg02jUMKEryAOOmrGLbkXTuHdCCV+fv4uvbu9OnpSmRORyaK95egpvFhd8e7EN2vp2YFxcwsV8zHr48mn1JWVzSoOTFZbuPZdIizLeo5V9e5b0YSpK8qBXe+H037y/ax/8NacW9QRvJm/MQWXhjv38T9esV+9lrs8Ib0eZE59X/M5f9L/4PnDgAaDOiYf02pjZ+5yLTRxvMOC5vtYHGPeDQKvIj+zPk6F3EJmfx58AEmjsOQuuroXF3bA7NnsQs9h7PxNvdlfu/3cDVMQ15bbS5JD89p4DRk//mSFouX9/egy5NS0719umyWP79607+fNQk5VEfriDf7uCX+/swdcUBXvh5R4lEcj7JWVb6vLqIwW3r887Ykn3rrTZzMnXp3iSGtG1Apyb1GNAqjBZhvpXSNTAp08rf+5O5rEUIIb4eFb7/c/lxYzwPf7eZxwe3Yt7WoygFv9x/lu6m5RCfmsPV76/gRHY+wT7urH56YIkvnO/Xx/PorM18dnNX3Cwu3PT5Gr66rXvJcyUVSEahFLWTww4uFrTWLJz9MZ7Z8fS52YxgmG9z8POaXfzH9Qv6LdkBOpETOpiG6jiT3p/CvbfdTpuGha2l3b9BXhp0HG/G8u4wxvyBGa9l1i1weDWbW07igW9SmPeADR8PVzPyY+ebYNWHaIs7j6SOIj4tFz9PN15J6MSnN98FQEZeAVe+t4zDJ3KLQlcK7uzTrOhxgLcb0+7owXUfr+SWL9Zwb/8WXN+tMUE+7qTl5PPNqoO0bxRQ1Ooe0q4B/5m3i72Jmby/aD89ooLo3aLsFx+F+HpwU6+mTFkWy57ELIJ93XnuqjaE+Xnw8HebWLInif+Oal/ixGhlCfXzOLNMVUVGdmzEX7uSeHPBbhzalLcqQkSgN+/f0IkJn61heEzDM35RXN2xIe/8uYcXf95Bz2ZBWFzUGV/s1Ula8qJq2G3mEvvSyivx68wkySM+YOo+H8auG4OnKsA66FU8Lp3InE1H2D/rnzziNpv59m7EBvTgoxNd2OB9Pwsc3XjN60F+f6gvnm4W7N9chyVxi+mHXtqgVLZ8jm/7i3/84CArXxfVaAFIPYh+vyt/+F/LXUeH879xndh3PIt3/9zLwkf60iLMr6gV/tLIdnSLDCQrz4abxaXUMsCRtFwem7mZlbEpeLq5cHOvSFbFprDzWCZTbz1VyjmYkk2/1xfTPNSH/UnZzJrYi26RQWfs71xSs/N5fu52cvLtbDyUSpbVRj1vN1Ky8nlxRFvG92harv3VVpl5BVz1v+UcTc9jzdMDK/SK2APJ2TSs54mH65mfqw2HUrnx09Xk5Nvp2LgeP913WSl7qBjSkhc106J/w6rJ8OjOorFWHA7N16sOMvLQFALy0siffScdC8JxWFxZZmvFZX88DeFtmLHSwfvuf2JvdjlPHbib1OQCRnVqhJvnNQzZ+gP/l5LCr99Nob41lp6H/mBv9K20dLFw+EQOa+PMxTc9mwXTsJ4X2uLG4xuCcHCC8AA3ftgQX3QibmeqL0tbzWDyhhyeGHIJw2MakpJlZfKS/Xy4eD+vj45h6t9xdIsMZELP8yfNRvW8+Paunuw+lsnkJfuZsiwWi1J8PKFLiVp902AfLmngx65jmfSLDi13ggcI9HHnvXGmVHM8M4//m72FQydymDKh63nr0M7Ez9ONGXf15Gh6XoUPeVBad8mTOjcJ5LObu3Hr1DUMvCSsQl/3YkmSF5UjPwf96UBU74fNlaFrPjXTve1fVHShzkdL9vO/37dwrcdctvj0oEH2bjq67Kdg0Gs8Pj+MOe7PE/ztePrk9CfYNRV63cvooHA+XX7AdLnLHYtl49f87fsEAfvMZRnHVDD37WrPuBUHeGvBHjILLx4K8nHn81u68cOGeJbsSeKfV7UhM8/GO3/uYXVsCvdN30hylhWAG3tGMrGfKb8E+3pw86WRTFkaS1KmlfjUXJ4Z1rpch6JVAz/evr4j9w1oTk6+nQ4R9c7YZlj7cHYdy+SxUvrMl1eYnydTb+2O1rrWX5J/IcIDvAgPqIRJw8+jV/Ng1jxzOT7uNSutSrlGnFVegR1Ptwsbhztz39/4fTMUq4s37l3Go9Z+Aq5eZqTBaz5ibdwJxk5ZxZMR27jz+MuMy3+G3u1acE/Dfbj0fZR7v93EkbjdfOl4lnr2FOzB0VgmrSG3wMH+pCzaNQowE0V/0B2HNYsPXG/Cv9M1DOvcjOH/W86xjDxah/vz+ugOFNgdTJq+segKxDv7RPHU0NYcSculz2uLcLMovN1deWNMDM1CfWgW4lMiOdodmmd/2sa3aw7RqJ4XSx7vX2pPj4uRV2Bn3/HC9yXEOUjvGnHB0nMKSMzMIzU7n4+XxrJ0TxKzJvaiU5Pyn0Ra/8NbdNnyIlbthocqYJ9vV7JcA4kp2IR+ZDcD316G3aH5K/x9LEk72TNuJdEN/IuS66x1h3l89hZaq4N85/8u/sNfKf1SfWumGQ3S9VRPjh0JGczbepR7+jc3J1WBhLRcnvlxK8PahzOmWP/t6z9eycZDaXx9e3d6NAs+Y/cnaW1KS81DfbmsHCdFhahoUpMXF8RmdzDsvWVFrV1/T1c83SxMXrKfjycUfp4K8uDrayCqL7r/k2eWApa8bk6uXvkm6XGbyMaL5S0e5R/7XuFf6UMJtKfwrvsf7NiwjAPJ2Xw0PAzXhYvg0vtpFV6yBduvlel+dtCtGfb7t8DZuuN5+J2xqE1D/1O9bQo1rOfFF7d2P2Pb98Z1IjUn/4y+zKdTSnFTFV6KLkRFkSRfx8QlZxMR6HVGuWHp3iSOpOXy4MCWtA73o1fzED5dFsv7i/YRm5RFs1BfWPoaHPobe/xahi1tyr0j+jGiYyO01uSdOIzX0tdAa7L7/hP/9F2k+LZk8ITHST9xK1P8Ahn2yk84HIqEtXPwdh/M5SdmmP6Hxec0LRTm58noLhG0qu9HYCX2t67v70l9/0qcHFuIalaxhUVRo207ks6ANxfzyMzNZ4yEN3NtPCE+bkwa0Jwh7cIJ8HLjpl6RuFkUn835g+lTP8S+7G3WuXfDbndwo+0H3l24F4dD8+6fe/n27cfBng+OAjYtn0c0h3BvZC40CggKwdPNwuVd27DJ0YJWib9yZ/MM3DZ9DTFjoV7p/bffGBPDnX2blbpOCFE2kuSdTLbVxsIdidgLx8DWWhcl9A8X7wPM8KyfLjtQ9JyULCuLdibwi/vTuL0eBV8Mg4MrCfV155uw6bwcfws3xD1Fuks93vF/jL0NR3CD22JGpX3Ozs/uJH3Jh9zo+hdrffpj1a6k/f0F/iqX0BadS8Q2tltjXi0YSxDpPBR3txmTvfcjVXRkhKibpFzjRLYnpHP/txuJTcrm/4a04u6+zbnnm/UcTs3l8cHR/LbtGPf2b05sUjav/LbTjJLXrgHL9yYxmJU0yN0LrYebiZ6/GQUxY+l+4meORt9IcKfhBDXuwje+oZDWBv3B79zj+jM58R48b8lFKwvdbn2TtJn3MOT4WgAs4e1LxNcs1Be35n2YlBTIFx5vQssrzABiQohKI71rnMTyvcnc9uVaAr3diArxYf3BVIa2C2fu5gS83CzkFtjxcrOw4sl/4OHqwjsL9/Dd2sNk5NlQOFjm+zQRQb4wcQXkJJtx1pN3Q6thcP20M0dxzM9m6uojvPDrHl4b4MN1HYLNODBLXjPzmgI8dQQ8Sg6YlZqdT57NTrifO6DO3K8Q4pykC2Ud4HBo0nILCCocP3xt3Alu+mwNTYO9mXZHD5RSDHp7KclZVsZ1b8JdfZvx0IyNDGodwn16hpnUOTSa3Hw7B+MP0+jwL/gtegau/cxMNA1muN6N30C3O8Cz9J4nBXYHf+48zhVt6p8aUe/Qavh8EARGwYObquBoCFG3SJJ3Uja7o6hHzIs/b2fa6kPMvLsX9bzcuPr95YT4efDdXb0I9XGF5L2syQ5j3tajPDXsklNjbRxeC59dbiZ5vmsRLHkVlr1p1oW1hYnLSh/vpTzsBfBqFDTvD9d/c3H7EkKcQfrJO6G9iZmM+Xgl9/RrztB24Xy98iA2h2bi1+vx93LFxUXx5a3dCbUdhS/uhsOr6H7THLpf3b/kjuKWmtvj2+GLoWa43fbXmR4ujbtffIIHsLjBuG/BL/z82wohKp0k+RrO4dA8+cNW0nIK+O/8Xfy8JQFXi+KTm7tyzzfrOZ6Zx1e39aCxZx68189MbOERAOs+N/OQFndgGYS1gUadTSmm9XC4ZnLFJPfioi5+DG8hRMWQJF/DTVtziPUHU3lpZDumrz7EtiMZ3NO/OQNahfHVbT3IzCugd8sQk9Tz0uCOv2D7D7B6sqmr+xaOiGfLh8OrodMEuPx5aHoZtB1V8QleCFGjSJKvwbTWvPfnXno2C+LGHk0Y0CqUr1ceLJr0uHtUsSFpt34PIdGmle7hByvfh03TofdDZv2R9VCQA1F9zWQaHW+o+jckhKhy0n+tBjueaSUp08qQtg1QShER6M1Tg5oXTexcJCMBDq6AdqPNMAGh0dDkUjMRtd0MtUvcMkBBZOVNZiCEqHkkyddgO45mANA6vLAL496F8EoEHN9ZcsNtPwD6VPdHgF73QeoBU7bRGvb/ZSa09qo505IJISqfJPkaIjYpizGT/+Z4Zl7Rsp2FSf6Sk0l+6etgt8KWmSWfvP0HCO9Y8urRS66E6CHmwqSf7oVDK81Y7kKIOkWSfA0xf/sx1salMndTQtGynUczaVTPiwAvNzi0Cg6vAldP2P6jaZ0D2KyQsAlaXF5yh0rBsDcABZunQ69J0PvhKns/QoiaQZJ8DbHhYCoA87YeLVq282jGqVLNivdMqeXyF0wZ5tgWszxpN2g71C9lZvp6jWHMVBj+Hgz6t0n8Qog6RZJ8DaC1Zv3BVNwtLmw4lMbR9FzyCuzEJmWZyS+2/QC7f4Vud5qLl5TFtOYBEreb29KSPED0IOhysyR4IeooSfI1QGxyNo1yd7O43r9or2KZv+0Yu49l4tDQlw3ww53QpJcpt/gEQ7N+sP0nU7I5vh0sHhAkozkKIc4kSb4GWH8wladdp9Mwewdfe77OmvXr2HoknebqCJ3XPGxa6Td8B+7e5gmtrzYlm6RdpiUf2goscsmDEOJMkuSr0WfLD/DHjkTSty/kUssOdPe78XRVPJvyBMt/nsqH7v9DuXnBuBngWWwO1OjB5nb3b5C4A+q3q543IISo8aT5V5X2LDDdIBt2Ynf4cF76JQOlND+6f8gJ11CCrvgXHjFjqTfjNiZnvmWec81s8G9Ycj/+DSE8BjZ/aybOrt+m6t+LEKJWkCRflbbNNrMuJW6j0brpRHi9zzUhh+mYtIdFzZ5hgJsnqlFnfB5cY8aicfM0syeVJnooLPmvuR8mSV4IUTop11Slo1ug+QB2DZ2BryOTt5qu5GHLTLK8I+gyctKp7VzdoedE6HLL2fd1smQDUq4RQpyVtOSrSkEuJO/B2nIYjy534VG6M+DgJyhtx3fkZPDxLt/+wjuCbwMzGfbJkSaFEOI00pKvJFvi0ziecWqIAhJ3gLbz3g4vdh/LxGfIcyjtMCNHdriu/C/g4gJ9HoXud0sfeCHEWUlLvhJorZnw2Rp6NQtm8oQupGRZmffTXCYAPyeG8O64TvToEA5ek02Sv9Ax3XvcVaFxCyGcjyT5SpCQnkd6bgGLdh8n22rjixVxNEzcSq67H+/fO5IOjQtHgowZW72BCiGcXpnKNUqpIUqp3UqpfUqpJ8+yTX+l1Cal1Hal1JKKDbN22ZOYCYDV5mDhzkRmr4+nh9cRvBp3PJXghRCiCpy3Ja+UsgAfAFcA8cBapdRcrfWOYtvUAz4EhmitDyml6vSZwL2FST7Ay43//raLpIxsIr0PQIOB1RyZEKKuKUtLvjuwT2sdq7XOB2YAI07b5gbgB631IQCt9fGKDbPm01qTV2AHYE9iFqF+Hlwd05Cj6bkM9dqJxWGF8A7VHKUQoq4pS5JvBBwu9ji+cFlx0UCgUmqxUmq9Uuqm0naklLpLKbVOKbUuKSnpwiKugfIK7Nz51Tr6vraI3Hw7exMzia7vy9iwQyxzf4j39X/AxQ0ad6/uUIUQdUxZTryW1j9Pl7KfLsBAwAtYqZRapbXeU+JJWk8BpgB07dr19H3USrn5dm75Yg2rD5wAYPHu4+w9nsUbTVbT5o93yPBtTGbf9/BrN1T6swshqlxZWvLxQONijyOAhFK2ma+1ztZaJwNLgZiKCbFm0Vrz29ajHEzJBuD7DfGsPnCC10d3IMjHnU+WxWLNz2dQwgeoyD4EPLgCv543S4IXQlSLsiT5tUBLpVSUUsodGAvMPW2bOUAfpZSrUsob6AGcNtt07Zdvc/B/s7dwz7QNPDfHTNbx585EmgR5M7pLBIPb1mfDoTRaqCO4OqzQ6Ubw8KvmqIUQddl5k7zW2gZMAn7HJO6ZWuvtSqmJSqmJhdvsBOYDW4A1wKda622VF3b1+OecbcxaH0+r+n4s35vE4RM5rNifwsDWYSilGNouHIB2Ks48Idwpf8wIIWqRMl0MpbWeB8w7bdnk0x6/DrxecaHVLFpr/tiRyIiODblvQAsGvb2Up37YSr7NwcBL6gPQq3kw9bzd6M4hsPhAcItqjloIUdfJ2DVlFJeSQ0p2Pj2bBRNd34/W4f4s35eMr4cr3aOCAHCzuPDgwJb08T0CDdpf+HAFQghRQSTJl9HaONN7plukuWJ1ZEczkUff6BDcXU8dxlt7NaFh7j4p1QghagRJ8mW0Lu4E9bzdaBbiC8CIjo3w9XBlRMfTLhlI2Q8F2ZLkhRA1ggxQVkbr4lLp2jQQFxdz2UCDAE+2PD+o6HGRo5vNrSR5IUQNUOdb8labnX3HsyiwO866TXKWldjkbLpGBpVYfkaCBzi6CSweENqqgiMVQojyq/Mt+Y+XxPLWH3vwdHPh1suieGLIJWdss/5gKgBdm55jBEmtYe2nsPYziOgGFrfKClkIIcqszrfkdyRkUN/fg+j6fvy44Uip2yzfm4yHqwvtGgWUvpPE7fDlcJj3GDS9FEZ/VokRCyFE2dX5JB+Xkk37RgEMbRfOsQwz2UdxBXYHv22JZ0S0J55upXSJPLwWJveBxG1w1dtw4/fg16CKohdCiHOr00ne4dAcSM4mKsSH6Pqm18y+45kltlm+L5lR1p94LXYkvN0OVk8puZNN34CbF0xaD11vk/lWhRA1Sp1O8kcz8rDaHESF+BJd34wxsycxq8Q2czYeoavbAbR3CPiEwB/PQW6aWWm3wc5fIHow+ARXcfRCCHF+dTrJH0gyI0lGhnjTqJ4XXm6Woqn7AHLybSzYkUgHz+OoiK6mHGPLg62zzAaH/oacZGhz+hwqQghRM9TtJJ9sWu3NQnxxcVG0rO/L3mIt+d+2HiMvv4Cw/HgIaQnhHaF+e9j4tdlgxxxw84YWV1RD9EIIcX51PMnn4OVmob6/BwAtw/xKtOS/XnWQXsHZuDjyISTa1Ns7TzAXPG34GnbMhZZXgLt3db0FIYQ4pzqe5LOICvFBFZ4sja7vy/FMK+k5BWyNT2fT4TRuaZlvNg5uaW7bjzEXO82dBLknzMlWIYSooer0xVAHkrNp2/BU3/eWhT1s9hzPZNa6w3i5WegTlGZWhkSbW+8gmPAj5GebOVu96lVt0EIIUQ51NskX2B0cTs3lqg4Ni5a1DDM9bN7/ax+rYlMY1TkCz/TfwCuwZO+ZyMuqOlwhhLggdbZcc/hEDnaHJirEp2hZo3pe+Hu6smRPEl2aBvLAwBaQvPdUK14IIWqZOtuSP3mCNbJYkndxUcy4qxfuri60CDOlG5L3QvSg6ghRCCEuWp1N8r9uPUY9bzfaNfIvsbxNw2KPc9Mg+/ipk65CCFHL1KlyzZb4NJIyrWTkFbBg+zGGd2iIh+s5puhL2WdupVwjhKil6kxLPt/mYNyUVTSs58XY7k2w2hyM6tzo3E+KX2tu67et/ACFEKIS1JmW/NYj6WTn29l7PIt//7qDZqE+dGxc79xP2v0bhLSCwKZVEqMQQlS0OpPkVx9IAWBiv+ZoDdd2jii6CKpUeRlwcIUZfEwIIWqpOlOuWR17gpZhvjwxpBX9W4XS5VyzPAHs/xMcNmg1tGoCFEKISlAnWvI2u4N1cSfoHhWEUoqezYJxs5znre/5HTzrQUT3KolRCCEqQ51I8tsTMsjOt9OjWRnHfHfYYe8CaDkILHXmx44QwgnViSR/sh7fMyqobE/Y8zvkpMAlV1ZiVEIIUfnqRJJfF5dKZLA3Yf6eZXvCyg/AP0KSvBCi1qsTST4x00qTYJ+SC08cgC+vNt0ki0vYCAeXQ8+JYHGruiCFEKIS1ImCc1pOPk2DTpvYY/+fcGCJ+Wva2wwZ7O5jxqpx94PON1VLrEIIUZHqSJIvIND7tFZ5apyZ/KPv47BzDuSlmb7xmQnQ51HwDChtV0IIUas4fZK3OzQZeQUEeLuXXJEaB4GR0O9x83eS1maaPyGEcAJOX5NPzy1Aa0pvyQdGnvkESfBCCCfi9Ek+LcfM0VqveJLXGlIPlp7khRDCiTh9kk/NKQCgXvFyTW4qWDMkyQshnJ7TJ/n0XNOSDyye5E8cMLeS5IUQTs7pk3xqdmFL3qtYuSZVkrwQom4oU5JXSg1RSu1WSu1TSj15ju26KaXsSqnRFRfixUnLNUm+REs+Nc7cyjjxQggnd94kr5SyAB8AQ4E2wDilVJuzbPcq8HtFB3kx0nLycVHg51mst2hqHPiEmYufhBDCiZWlJd8d2Ke1jtVa5wMzgBGlbHc/8D1wvALju2hpOQUEeLnh4lKsa+TZuk8KIYSTKUuSbwQcLvY4vnBZEaVUI+AaYPK5dqSUuksptU4ptS4pKam8sV6Q1Jz8kqUakO6TQog6oyxJvrSrg/Rpj98BntBa28+1I631FK11V61119DQ0DKGeHHScgoIKN5H3pYPGfGS5IUQdUJZhjWIBxoXexwBJJy2TVdgRuGcqSHAMKWUTWv9U0UEeTHScvMJ8ys2xPCxraAdENyi+oISQogqUpYkvxZoqZSKAo4AY4Ebim+gtY46eV8pNRX4pSYkeDBdKKPD/E4t2PQNuHrKBN1CiDrhvElea21TSk3C9JqxAJ9rrbcrpSYWrj9nHb66pecWnLraNT8Hts6GNiPN0MJCCOHkyjQKpdZ6HjDvtGWlJnet9S0XH1bFyLc5yLLaTo1bs2OOGc6g84TqDUwIIaqIU1/xml50IVRhkt/4DQQ1g6aXVWNUQghRdZw6yZ8agdIdbFY4tBLajJDhhIUQdYZTJ/lTI1C6QdJu0HZo0L6aoxJCiKrj1En+ZEs+0Nsdju8wC8PaVmNEQghRtZw8yZuWfICXGyRuB4s7BDev5qiEEKLqOHeSPzmWvE9hSz6kFVjczvMsIYRwHs6d5HMKcHVR+LhbIHEH1D9j8EwhhHBqTp3ks6w2fD1dUXlpkJkAYa2rOyQhhKhSTp/kfdxdTSse5KSrEKLOceokn2214evheqpnjZRrhBB1jJMneTs+HhaT5D0CwL/R+Z8khBBOxKmTfJbVho+HKyTvhdBWcqWrEKLOceokX1SuyTgCARHVHY4QQlQ5p0/yPu4WyEgA/4bVHY4QQlQ5p07yWVYboa65YMuTJC+EqJOcNslrrcmy2ggjxSzwC6/egIQQoho4bZLPK3Dg0BCqC5O89KwRQtRBTpvks6w2AIIcJ5O8tOSFEHWP0yb57MIkX68gySzwbVCN0QghRPVw2iR/siXvV5AEPmHg6l7NEQkhRNVz2iR/siXvYz0upRohRJ3lvEk+3yR5r9xEOekqhKiznDbJZ1ntALjnHJPuk0KIOstpk3y21YYH+VisaVKuEULUWU6d5BuoE+aBlGuEEHWU0yb5LKuNBqSaB1KuEULUUU6b5LOtNpq4FSZ5ackLIeoop03yWVY7jV3TzQOpyQsh6ijX6g6gsmRbbdR3yQAXL/Dwq+5whBCiWjh1kve3WMFVErwQou5y4nKNDX+VCx6+1R2KEEJUG6dO8r4qT0o1Qog6zWmTfLbVhjd54C5JXghRdzltks+y2vHROVKuEULUaU6b5LOtNjx1jpRrhBB1mlMmebtDk1tgx8ORC+7SkhdC1F1OmeRPDjPsYc+Wco0Qok4rU5JXSg1RSu1WSu1TSj1ZyvrxSqkthX9/K6ViKj7Ussu22rBgx9VhBQ//6gxFCCGq1XmTvFLKAnwADAXaAOOUUm1O2+wA0E9r3QF4CZhS0YGWR7bVhg+55oGUa4QQdVhZWvLdgX1a61itdT4wAxhRfAOt9d9a68LRwFgFRFRsmOWTZbXjdzLJS7lGCFGHlSXJNwIOF3scX7jsbG4HfitthVLqLqXUOqXUuqSkpLJHWU7ZVhs+Ks88kN41Qog6rCxJXpWyTJe6oVIDMEn+idLWa62naK27aq27hoaGlj3Kcsqy2vAtKtdIkhdC1F1lGaAsHmhc7HEEkHD6RkqpDsCnwFCtdUrFhHdhcvPt+Cop1wghRFla8muBlkqpKKWUOzAWmFt8A6VUE+AHYILWek/Fh1k+Ofl2fJByjRBCnLclr7W2KaUmAb8DFuBzrfV2pdTEwvWTgX8CwcCHSikAm9a6a+WFfW65BcVa8tK7RghRh5VpPHmt9Txg3mnLJhe7fwdwR8WGduFy84vV5KUlL4Sow5zyitfcAjt+J3vXSEteCFGHOWWSz8m3U89iBYsHuLpXdzhCCFFtnDLJ5xXY8bfkSc8aIUSd55RJPiffjr+ySj1eCFHnOWWSz80vrMnLhVBCiDrOOZP8yS6UUq4RQtRxzpnk8+1mFEop1wgh6jjnTPIFdry1zAolhBDOmeTz7XhpKdcIIYRzJvkCe+Ek3jIrlBCibivTsAa1Ta61AA+kXCNEWRUUFBAfH09eXl51hyIKeXp6EhERgZub20XtxymTvLJlm3cm5RohyiQ+Ph4/Pz8iIyMpHGRQVCOtNSkpKcTHxxMVFXVR+3K6co3dofGwZZsH0rtGiDLJy8sjODhYEnwNoZQiODi4Qn5ZOV2Szy2wn5r6T8o1QpSZJPiapaL+PZwuyefIMMNCCFHE6ZJ8Xr6j2NR/kuSFEHWb0yX5nAIbvki5Rghn5ut79v/bUVFR7N69u8Syhx56iNdeew2AjRs3opTi999/L/M+i5s8eTJfffVVOSOuPk7XuyY3306AyjIPvAKrNxghaqEXf97OjoSMCt1nm4b+PD+8bYXu82zGjh3LjBkzeP755wFwOBzMnj2bFStWAPDtt9/Su3dvvv32WwYPHlzu/U+cOLFC461sTteSzy2wE0SmeeAdXL3BCCHK5IknnuDDDz8sevzCCy/w4osvMnDgQDp37kz79u2ZM2dOmfY1btw4ZsyYUfR46dKlREZG0rRpU7TWzJ49m6lTp7JgwYIL6r3ywgsv8MYbbwDwySef0K1bN2JiYrj22mvJyckBIDExkWuuuYaYmBhiYmL4+++/y/06FcUpW/KBKhOHxRMXd+/qDkeIWqeqWtzFjR07loceeoh7770XgJkzZzJ//nwefvhh/P39SU5OpmfPnlx99dXn7XXSoUMHXFxc2Lx5MzExMcyYMYNx48YBsGLFCqKiomjevDn9+/dn3rx5jBo16oLjHjVqFHfeeScAzz77LJ999hn3338/DzzwAP369ePHH3/EbreTlZV1wa9xsZy2JW/3CqruUIQQZdSpUyeOHz9OQkICmzdvJjAwkPDwcJ5++mk6dOjA5ZdfzpEjR0hMTCzT/k625m02G3PmzGHMmDGAKdWMHTsWMF8s33777UXFvW3bNvr06UP79u2ZNm0a27dvB+Cvv/7innvuAcBisRAQEHBRr3MxnK4ln5NvJ0hloiXJC1GrjB49mtmzZ3Ps2DHGjh3LtGnTSEpKYv369bi5uREZGVnm8sq4ceMYNGgQ/fr1o0OHDoSFhWG32/n++++ZO3cuL7/8ctFVpZmZmfj5XVhPvFtuuYWffvqJmJgYpk6dyuLFiy9oP5XJ6VryeQUmyUs9Xoja5eQJ09mzZzN69GjS09MJCwvDzc2NRYsWcfDgwTLvq3nz5gQHB/Pkk08WlWoWLlxITEwMhw8fJi4ujoMHD3Lttdfy008/XXDMmZmZhIeHU1BQwLRp04qWDxw4kI8++ggAu91ORkbFnsguD6dL8jn5dgLJxMUnpLpDEUKUQ9u2bcnMzKRRo0aEh4czfvx41q1bR9euXZk2bRqXXHJJufY3btw4du3axTXXXAOYUs3J+ydde+21TJ8+HYCcnBwiIiKK/t56662z7vvkeYGXXnqJHj16cMUVV5SI791332XRokW0b9+eLl26FJVxqoPSWlfLC3ft2lWvW7euwvf79h97uG15P/x73Iga9nqF718IZ7Rz505at25d3WHUCvfffz+dO3fm1ltvrfTXKu3fRSm1Xmvdtaz7cLqWfF6+lQCVg/KWlrwQomI999xzrF69mquvvrq6QykzpzvxqnLSzB1vOfEqhDPbunUrEyZMKLHMw8OD1atXV8j+X375ZWbNmlVi2ZgxY1izZk2F7L+qOF2Sd8k7Ye5IkhfCqbVv355NmzZV2v6feeYZnnnmmUrbf1VxunKNm/VkkpfeNUIIIUleCCGcmNMleY/8NHNHkrwQQjhfkvcsSDN35IpXIYRwviTvbUsnT3mBm2d1hyKEKKO0tLQSo1CW1bBhw0hLSyvXc6ZOnVp0FexJycnJhIaGYrVaARgxYgS9evUqsU3x0SfP59JLLy1XTJXJ6XrX+NjTyXYNQFK8EBfotyfh2NaK3WeD9jD0v2ddfTLJnxyF8iS73Y7FYjnr8+bNm1fuUEaNGsVjjz1GTk4O3t5mpNrZs2dz9dVX4+HhQVpaGhs2bMDX15cDBw4QFRVV7teozqGFT+d0LXk/Rzq5rvWqOwwhRDk8+eST7N+/n44dO9KtWzcGDBjADTfcQPv27QEYOXIkXbp0oW3btkyZMqXoeZGRkSQnJxMXF0fr1q258847adu2LYMGDSI3N7fU1/L396dv3778/PPPRcuKD0f8/fffM3z48KKxdC7EyVmmsrKyzjom/ldffUWHDh2IiYk5o79/hdJaV8tfly5ddGXY8s9Oet+bV1TKvoVwVjt27KjW1z9w4IBu27at1lrrRYsWaW9vbx0bG1u0PiUlRWutdU5Ojm7btq1OTk7WWmvdtGlTnZSUpA8cOKAtFoveuHGj1lrrMWPG6K+//vqsrzdz5kw9cuRIrbXWR44c0eHh4dpms2mttR44cKBeunSp3r17t27fvn3Rc55//nn9+uuvl+n9+Pj4aK21Ligo0Onp6VprrZOSknTz5s21w+HQ27Zt09HR0TopKanE+ztdaf8uwDpdjlzrVC15rTX+ZGJ1l2n/hKjNunfvXqJM8t577xETE0PPnj05fPgwe/fuPeM5UVFRdOzYEYAuXboQFxd31v1fddVVLF++nIyMDGbOnMno0aOxWCwkJiayb98+evfuTXR0NK6urmzbtu2C34fWutQx8f/66y9Gjx5NSIgZfiUoqPI6ipQpySulhiildiul9imlnixlvVJKvVe4fotSqnPFh3p+VpuDQDIp8JAkL0Rt5uPjU3R/8eLFLFy4kJUrV7J582Y6depU6rjyHh4eRfctFgs2m+2s+/fy8mLIkCH8+OOPJUo13333HampqURFRREZGUlcXNwFl2yAEmPib9q0ifr165OXl4fW+rwzXFWU8yZ5pZQF+AAYCrQBximl2py22VCgZeHfXcBHFRxnmeTm5uKvcrF7SvdJIWoTPz8/MjMzS12Xnp5OYGAg3t7e7Nq1i1WrVlXIa44bN4633nqLxMREevbsCZjhiOfPn09cXBxxcXGsX7/+opL82cbEHzhwIDNnziQlJQWAEydOXPwbOouy9K7pDuzTWscCKKVmACOAHcW2GQF8VVgvWqWUqqeUCtdaH63ogLcs/h7/pc+Xus5FOwgEHJ7SkheiNgkODuayyy6jXbt2eHl5Ub9+/aJ1Q4YMYfLkyXTo0IFWrVoVJeSLNWjQIG6++WZuv/12lFLExcVx6NChEvuPiorC39+/aNCzf//737zzzjtF6+Pj40vd98lW+vjx4xk+fDhdu3alY8eORWPOt23blmeeeYZ+/fphsVjo1KkTU6dOrZD3dUYs+jzjySulRgNDtNZ3FD6eAPTQWk8qts0vwH+11ssLH/8JPKG1Xnfavu7CtPRp0qRJl/LM9HLSrrULyVny7lnXO1zciRj9Cg2aRJd730LUVTKefMVJSUmhc+fO5ZrJ6mwqYjz5srTkSyscnf7NUJZt0FpPAaaAmTSkDK99hku6XQ7dLr+QpwohRKVKSEigf//+PPbYY9UdSpGyJPl4oHGxxxFAwgVsI4QQVeq+++5jxYoVJZY9+OCDFTKrU0pKCgMHDjxj+cqVKwkOrjljZ5Ulya8FWiqlooAjwFjghtO2mQtMKqzX9wDSK6MeL4SoPFXZ46OqfPDBB5W27+Dg4Eodz/58pfSyOm+S11rblFKTgN8BC/C51nq7Umpi4frJwDxgGLAPyAEqf/JDIUSF8fT0JCUlheDgYKdL9LWR1pqUlBQ8PS9+gBanm8hbCFF+BQUFxMfHl9r/XFQPT09PIiIicHNzK7G8Mk68CiGcnJub2wUNxCVqPqca1kAIIURJkuSFEMKJSZIXQggnVm0nXpVSScCFXhIWAiRXYDhVQWKuGrUxZqidcUvMVeP0mJtqrUPL+uRqS/IXQym1rjxnl2sCiblq1MaYoXbGLTFXjYuNWco1QgjhxCTJCyGEE6utSX7K+TepcSTmqlEbY4baGbfEXDUuKuZaWZMXQghRNrW1JS+EEKIMJMkLIYQTq3VJ/nyTitcESqnGSqlFSqmdSqntSqkHC5e/oJQ6opTaVPg3rLpjLU4pFaeU2loY27rCZUFKqT+UUnsLb2vM3IpKqVbFjuUmpVSGUuqhmnaclVKfK6WOK6W2FVt21uOqlHqq8PO9Wyk1uAbF/LpSapdSaotS6kelVL3C5ZFKqdxix3tyDYr5rJ+FGnycvysWb5xSalPh8gs7zlrrWvOHGep4P9AMcAc2A22qO65S4gwHOhfe9wP2YCZBfwF4rLrjO0fccUDIacteA54svP8k8Gp1x3mOz8YxoGlNO85AX6AzsO18x7Xwc7IZ8ACiCj/vlhoS8yDAtfD+q8Vijiy+XQ07zqV+FmrycT5t/ZvAPy/mONe2lnzRpOJa63zg5KTiNYrW+qjWekPh/UxgJ9CoeqO6YCOALwvvfwmMrL5QzmkgsF9rffETa1YwrfVS4MRpi892XEcAM7TWVq31AcwcDd2rIs7iSotZa71Aa20rfLgKMwNcjXGW43w2NfY4n6TMwP7XAd9ezGvUtiTfCDhc7HE8NTx5KqUigU7A6sJFkwp/7n5ek0ofhTSwQCm1vnDSdYD6unCWr8LbsGqL7tzGUvI/Q00+znD241pbPuO3Ab8VexyllNqolFqilOpTXUGdRWmfhdpwnPsAiVrrvcWWlfs417YkX6YJw2sKpZQv8D3wkNY6A/gIaA50BI5iforVJJdprTsDQ4H7lFJ9qzugslBKuQNXA7MKF9X043wuNf4zrpR6BrAB0woXHQWaaK07AY8A05VS/tUV32nO9lmo8ccZGEfJhssFHefaluRrzYThSik3TIKfprX+AUBrnai1tmutHcAnVMPPw3PRWicU3h4HfsTEl6iUCgcovD1efRGe1VBgg9Y6EWr+cS50tuNaoz/jSqmbgauA8bqwUFxY8kgpvL8eU9+Orr4oTznHZ6GmH2dXYBTw3cllF3qca1uSL5pUvLD1NhYziXiNUlhL+wzYqbV+q9jy8GKbXQNsO/251UUp5aOU8jt5H3OSbRvm+N5cuNnNwJzqifCcSrR4avJxLuZsx3UuMFYp5aGUigJaAmuqIb4zKKWGAE8AV2utc4otD1VKWQrvN8PEHFs9UZZ0js9CjT3OhS4Hdmmt408uuODjXNVnkyvgbPQwTG+V/cAz1R3PWWLsjfnptwXYVPg3DPga2Fq4fC4QXt2xFou5Gaa3wWZg+8ljCwQDfwJ7C2+DqjvW0+L2BlKAgGLLatRxxnwBHQUKMC3I2891XIFnCj/fu4GhNSjmfZg69snP9OTCba8t/MxsBjYAw2tQzGf9LNTU41y4fCow8bRtL+g4y7AGQgjhxGpbuUYIIUQ5SJIXQggnJkleCCGcmCR5IYRwYpLkhRDCiUmSF0IIJyZJXgghnNj/A8Gg/oEuKui0AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_VAL_jac','train_VAL_jac']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "unlabel_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=~(data.train_mask + data.val_mask + data.test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def predict_Homo(model, loader):\n",
    "    model.eval()\n",
    "    seed_everything(args.seed)\n",
    "    all_preds = []\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch.batch_size\n",
    "        edge_index = to_undirected(batch.edge_index)\n",
    "        out = model(batch.x, edge_index)[:batch_size]\n",
    "        out_att = out[:,:9].softmax(axis=1)\n",
    "        out_val = out[:,9:].softmax(axis=1)\n",
    "        IDs = batch.n_id[:batch_size].unsqueeze(dim=-1).int()\n",
    "        \n",
    "        now = torch.hstack([IDs, out_att, out_val])\n",
    "        all_preds.append(now)\n",
    "    \n",
    "    final = torch.vstack(all_preds)\n",
    "        \n",
    "    return final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 15.55it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.41it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 20.76it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 38/38 [00:01<00:00, 22.56it/s]\n"
     ]
    }
   ],
   "source": [
    "pred_train = predict_Homo(model, train_loader)\n",
    "pred_val = predict_Homo(model, val_loader)\n",
    "pred_test = predict_Homo(model, test_loader)\n",
    "pred_unlab = predict_Homo(model, unlabel_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = torch.vstack([pred_train, pred_val, pred_test, pred_unlab]).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([2.0000, 2.0000, 2.0000,  ..., 2.0000, 2.0000, 2.0000])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds[:,1:].sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df = pd.DataFrame(preds).sort_values(0).set_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.021085</td>\n",
       "      <td>0.080077</td>\n",
       "      <td>0.015716</td>\n",
       "      <td>0.004451</td>\n",
       "      <td>0.212755</td>\n",
       "      <td>0.075620</td>\n",
       "      <td>0.037683</td>\n",
       "      <td>0.004144</td>\n",
       "      <td>0.548468</td>\n",
       "      <td>0.085739</td>\n",
       "      <td>0.365993</td>\n",
       "      <td>0.173247</td>\n",
       "      <td>0.236994</td>\n",
       "      <td>0.015982</td>\n",
       "      <td>0.098432</td>\n",
       "      <td>0.004647</td>\n",
       "      <td>0.004069</td>\n",
       "      <td>0.003527</td>\n",
       "      <td>0.003366</td>\n",
       "      <td>0.008006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.007347</td>\n",
       "      <td>0.030522</td>\n",
       "      <td>0.007316</td>\n",
       "      <td>0.001726</td>\n",
       "      <td>0.056869</td>\n",
       "      <td>0.034187</td>\n",
       "      <td>0.009252</td>\n",
       "      <td>0.001284</td>\n",
       "      <td>0.851497</td>\n",
       "      <td>0.153359</td>\n",
       "      <td>0.434726</td>\n",
       "      <td>0.139513</td>\n",
       "      <td>0.161096</td>\n",
       "      <td>0.009869</td>\n",
       "      <td>0.081890</td>\n",
       "      <td>0.004033</td>\n",
       "      <td>0.003355</td>\n",
       "      <td>0.002760</td>\n",
       "      <td>0.002715</td>\n",
       "      <td>0.006684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.038959</td>\n",
       "      <td>0.089459</td>\n",
       "      <td>0.013668</td>\n",
       "      <td>0.009166</td>\n",
       "      <td>0.255841</td>\n",
       "      <td>0.180218</td>\n",
       "      <td>0.051883</td>\n",
       "      <td>0.005855</td>\n",
       "      <td>0.354952</td>\n",
       "      <td>0.139353</td>\n",
       "      <td>0.401640</td>\n",
       "      <td>0.158797</td>\n",
       "      <td>0.161027</td>\n",
       "      <td>0.009512</td>\n",
       "      <td>0.107023</td>\n",
       "      <td>0.004194</td>\n",
       "      <td>0.004084</td>\n",
       "      <td>0.003055</td>\n",
       "      <td>0.003088</td>\n",
       "      <td>0.008226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.049263</td>\n",
       "      <td>0.243927</td>\n",
       "      <td>0.023052</td>\n",
       "      <td>0.018801</td>\n",
       "      <td>0.015961</td>\n",
       "      <td>0.050762</td>\n",
       "      <td>0.547270</td>\n",
       "      <td>0.043546</td>\n",
       "      <td>0.007417</td>\n",
       "      <td>0.049004</td>\n",
       "      <td>0.041706</td>\n",
       "      <td>0.139258</td>\n",
       "      <td>0.069855</td>\n",
       "      <td>0.015055</td>\n",
       "      <td>0.655422</td>\n",
       "      <td>0.006946</td>\n",
       "      <td>0.005289</td>\n",
       "      <td>0.003414</td>\n",
       "      <td>0.005688</td>\n",
       "      <td>0.008363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.226045</td>\n",
       "      <td>0.307158</td>\n",
       "      <td>0.010621</td>\n",
       "      <td>0.060181</td>\n",
       "      <td>0.009664</td>\n",
       "      <td>0.304781</td>\n",
       "      <td>0.042088</td>\n",
       "      <td>0.018450</td>\n",
       "      <td>0.021013</td>\n",
       "      <td>0.194195</td>\n",
       "      <td>0.066356</td>\n",
       "      <td>0.092250</td>\n",
       "      <td>0.198467</td>\n",
       "      <td>0.018021</td>\n",
       "      <td>0.398030</td>\n",
       "      <td>0.006777</td>\n",
       "      <td>0.005326</td>\n",
       "      <td>0.003530</td>\n",
       "      <td>0.005200</td>\n",
       "      <td>0.011848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2946.0</th>\n",
       "      <td>0.027497</td>\n",
       "      <td>0.447962</td>\n",
       "      <td>0.065789</td>\n",
       "      <td>0.030700</td>\n",
       "      <td>0.007843</td>\n",
       "      <td>0.031089</td>\n",
       "      <td>0.337795</td>\n",
       "      <td>0.037865</td>\n",
       "      <td>0.013461</td>\n",
       "      <td>0.337518</td>\n",
       "      <td>0.171166</td>\n",
       "      <td>0.189892</td>\n",
       "      <td>0.195490</td>\n",
       "      <td>0.011729</td>\n",
       "      <td>0.059820</td>\n",
       "      <td>0.006806</td>\n",
       "      <td>0.006765</td>\n",
       "      <td>0.004519</td>\n",
       "      <td>0.005840</td>\n",
       "      <td>0.010456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2947.0</th>\n",
       "      <td>0.004697</td>\n",
       "      <td>0.018286</td>\n",
       "      <td>0.223953</td>\n",
       "      <td>0.013178</td>\n",
       "      <td>0.003370</td>\n",
       "      <td>0.003730</td>\n",
       "      <td>0.640505</td>\n",
       "      <td>0.088873</td>\n",
       "      <td>0.003408</td>\n",
       "      <td>0.291351</td>\n",
       "      <td>0.151943</td>\n",
       "      <td>0.220524</td>\n",
       "      <td>0.195674</td>\n",
       "      <td>0.013143</td>\n",
       "      <td>0.084207</td>\n",
       "      <td>0.008385</td>\n",
       "      <td>0.007127</td>\n",
       "      <td>0.005984</td>\n",
       "      <td>0.009803</td>\n",
       "      <td>0.011858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2948.0</th>\n",
       "      <td>0.052124</td>\n",
       "      <td>0.565919</td>\n",
       "      <td>0.038446</td>\n",
       "      <td>0.033455</td>\n",
       "      <td>0.009417</td>\n",
       "      <td>0.039590</td>\n",
       "      <td>0.222424</td>\n",
       "      <td>0.028003</td>\n",
       "      <td>0.010621</td>\n",
       "      <td>0.322396</td>\n",
       "      <td>0.202790</td>\n",
       "      <td>0.193366</td>\n",
       "      <td>0.181756</td>\n",
       "      <td>0.010406</td>\n",
       "      <td>0.056317</td>\n",
       "      <td>0.006385</td>\n",
       "      <td>0.006474</td>\n",
       "      <td>0.004193</td>\n",
       "      <td>0.005828</td>\n",
       "      <td>0.010089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2949.0</th>\n",
       "      <td>0.352359</td>\n",
       "      <td>0.308921</td>\n",
       "      <td>0.021057</td>\n",
       "      <td>0.051812</td>\n",
       "      <td>0.012259</td>\n",
       "      <td>0.164115</td>\n",
       "      <td>0.051383</td>\n",
       "      <td>0.021903</td>\n",
       "      <td>0.016191</td>\n",
       "      <td>0.379453</td>\n",
       "      <td>0.191668</td>\n",
       "      <td>0.144466</td>\n",
       "      <td>0.202435</td>\n",
       "      <td>0.009357</td>\n",
       "      <td>0.041763</td>\n",
       "      <td>0.005173</td>\n",
       "      <td>0.005717</td>\n",
       "      <td>0.003795</td>\n",
       "      <td>0.006305</td>\n",
       "      <td>0.009869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2950.0</th>\n",
       "      <td>0.039927</td>\n",
       "      <td>0.762812</td>\n",
       "      <td>0.015547</td>\n",
       "      <td>0.016468</td>\n",
       "      <td>0.007742</td>\n",
       "      <td>0.103283</td>\n",
       "      <td>0.031658</td>\n",
       "      <td>0.007951</td>\n",
       "      <td>0.014612</td>\n",
       "      <td>0.330286</td>\n",
       "      <td>0.174305</td>\n",
       "      <td>0.185595</td>\n",
       "      <td>0.200411</td>\n",
       "      <td>0.012938</td>\n",
       "      <td>0.057548</td>\n",
       "      <td>0.007806</td>\n",
       "      <td>0.007699</td>\n",
       "      <td>0.004849</td>\n",
       "      <td>0.007607</td>\n",
       "      <td>0.010956</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2951 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              1         2         3         4         5         6         7   \\\n",
       "0                                                                              \n",
       "0.0     0.021085  0.080077  0.015716  0.004451  0.212755  0.075620  0.037683   \n",
       "1.0     0.007347  0.030522  0.007316  0.001726  0.056869  0.034187  0.009252   \n",
       "2.0     0.038959  0.089459  0.013668  0.009166  0.255841  0.180218  0.051883   \n",
       "3.0     0.049263  0.243927  0.023052  0.018801  0.015961  0.050762  0.547270   \n",
       "4.0     0.226045  0.307158  0.010621  0.060181  0.009664  0.304781  0.042088   \n",
       "...          ...       ...       ...       ...       ...       ...       ...   \n",
       "2946.0  0.027497  0.447962  0.065789  0.030700  0.007843  0.031089  0.337795   \n",
       "2947.0  0.004697  0.018286  0.223953  0.013178  0.003370  0.003730  0.640505   \n",
       "2948.0  0.052124  0.565919  0.038446  0.033455  0.009417  0.039590  0.222424   \n",
       "2949.0  0.352359  0.308921  0.021057  0.051812  0.012259  0.164115  0.051383   \n",
       "2950.0  0.039927  0.762812  0.015547  0.016468  0.007742  0.103283  0.031658   \n",
       "\n",
       "              8         9         10        11        12        13        14  \\\n",
       "0                                                                              \n",
       "0.0     0.004144  0.548468  0.085739  0.365993  0.173247  0.236994  0.015982   \n",
       "1.0     0.001284  0.851497  0.153359  0.434726  0.139513  0.161096  0.009869   \n",
       "2.0     0.005855  0.354952  0.139353  0.401640  0.158797  0.161027  0.009512   \n",
       "3.0     0.043546  0.007417  0.049004  0.041706  0.139258  0.069855  0.015055   \n",
       "4.0     0.018450  0.021013  0.194195  0.066356  0.092250  0.198467  0.018021   \n",
       "...          ...       ...       ...       ...       ...       ...       ...   \n",
       "2946.0  0.037865  0.013461  0.337518  0.171166  0.189892  0.195490  0.011729   \n",
       "2947.0  0.088873  0.003408  0.291351  0.151943  0.220524  0.195674  0.013143   \n",
       "2948.0  0.028003  0.010621  0.322396  0.202790  0.193366  0.181756  0.010406   \n",
       "2949.0  0.021903  0.016191  0.379453  0.191668  0.144466  0.202435  0.009357   \n",
       "2950.0  0.007951  0.014612  0.330286  0.174305  0.185595  0.200411  0.012938   \n",
       "\n",
       "              15        16        17        18        19        20  \n",
       "0                                                                   \n",
       "0.0     0.098432  0.004647  0.004069  0.003527  0.003366  0.008006  \n",
       "1.0     0.081890  0.004033  0.003355  0.002760  0.002715  0.006684  \n",
       "2.0     0.107023  0.004194  0.004084  0.003055  0.003088  0.008226  \n",
       "3.0     0.655422  0.006946  0.005289  0.003414  0.005688  0.008363  \n",
       "4.0     0.398030  0.006777  0.005326  0.003530  0.005200  0.011848  \n",
       "...          ...       ...       ...       ...       ...       ...  \n",
       "2946.0  0.059820  0.006806  0.006765  0.004519  0.005840  0.010456  \n",
       "2947.0  0.084207  0.008385  0.007127  0.005984  0.009803  0.011858  \n",
       "2948.0  0.056317  0.006385  0.006474  0.004193  0.005828  0.010089  \n",
       "2949.0  0.041763  0.005173  0.005717  0.003795  0.006305  0.009869  \n",
       "2950.0  0.057548  0.007806  0.007699  0.004849  0.007607  0.010956  \n",
       "\n",
       "[2951 rows x 20 columns]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df.to_csv(args.save_dir + 'preds.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "states.to_csv(args.save_dir + 'train_state.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Per-Class Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = pd.read_csv(args.save_dir + 'preds.csv', sep='\\t', index_col='0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.021085</td>\n",
       "      <td>0.080077</td>\n",
       "      <td>0.015716</td>\n",
       "      <td>0.004451</td>\n",
       "      <td>0.212755</td>\n",
       "      <td>0.075620</td>\n",
       "      <td>0.037683</td>\n",
       "      <td>0.004144</td>\n",
       "      <td>0.548468</td>\n",
       "      <td>0.085739</td>\n",
       "      <td>0.365993</td>\n",
       "      <td>0.173247</td>\n",
       "      <td>0.236994</td>\n",
       "      <td>0.015982</td>\n",
       "      <td>0.098432</td>\n",
       "      <td>0.004647</td>\n",
       "      <td>0.004069</td>\n",
       "      <td>0.003527</td>\n",
       "      <td>0.003366</td>\n",
       "      <td>0.008006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.007347</td>\n",
       "      <td>0.030522</td>\n",
       "      <td>0.007316</td>\n",
       "      <td>0.001726</td>\n",
       "      <td>0.056869</td>\n",
       "      <td>0.034187</td>\n",
       "      <td>0.009252</td>\n",
       "      <td>0.001284</td>\n",
       "      <td>0.851497</td>\n",
       "      <td>0.153359</td>\n",
       "      <td>0.434726</td>\n",
       "      <td>0.139513</td>\n",
       "      <td>0.161096</td>\n",
       "      <td>0.009869</td>\n",
       "      <td>0.081890</td>\n",
       "      <td>0.004033</td>\n",
       "      <td>0.003355</td>\n",
       "      <td>0.002760</td>\n",
       "      <td>0.002715</td>\n",
       "      <td>0.006684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.038959</td>\n",
       "      <td>0.089459</td>\n",
       "      <td>0.013668</td>\n",
       "      <td>0.009166</td>\n",
       "      <td>0.255841</td>\n",
       "      <td>0.180218</td>\n",
       "      <td>0.051883</td>\n",
       "      <td>0.005855</td>\n",
       "      <td>0.354952</td>\n",
       "      <td>0.139353</td>\n",
       "      <td>0.401640</td>\n",
       "      <td>0.158797</td>\n",
       "      <td>0.161027</td>\n",
       "      <td>0.009512</td>\n",
       "      <td>0.107023</td>\n",
       "      <td>0.004194</td>\n",
       "      <td>0.004084</td>\n",
       "      <td>0.003055</td>\n",
       "      <td>0.003088</td>\n",
       "      <td>0.008226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.049263</td>\n",
       "      <td>0.243927</td>\n",
       "      <td>0.023052</td>\n",
       "      <td>0.018801</td>\n",
       "      <td>0.015961</td>\n",
       "      <td>0.050762</td>\n",
       "      <td>0.547270</td>\n",
       "      <td>0.043546</td>\n",
       "      <td>0.007417</td>\n",
       "      <td>0.049004</td>\n",
       "      <td>0.041706</td>\n",
       "      <td>0.139258</td>\n",
       "      <td>0.069855</td>\n",
       "      <td>0.015055</td>\n",
       "      <td>0.655422</td>\n",
       "      <td>0.006946</td>\n",
       "      <td>0.005289</td>\n",
       "      <td>0.003414</td>\n",
       "      <td>0.005688</td>\n",
       "      <td>0.008363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.226045</td>\n",
       "      <td>0.307158</td>\n",
       "      <td>0.010621</td>\n",
       "      <td>0.060181</td>\n",
       "      <td>0.009664</td>\n",
       "      <td>0.304781</td>\n",
       "      <td>0.042088</td>\n",
       "      <td>0.018450</td>\n",
       "      <td>0.021013</td>\n",
       "      <td>0.194195</td>\n",
       "      <td>0.066356</td>\n",
       "      <td>0.092250</td>\n",
       "      <td>0.198467</td>\n",
       "      <td>0.018021</td>\n",
       "      <td>0.398030</td>\n",
       "      <td>0.006777</td>\n",
       "      <td>0.005326</td>\n",
       "      <td>0.003530</td>\n",
       "      <td>0.005200</td>\n",
       "      <td>0.011848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2946.0</th>\n",
       "      <td>0.027497</td>\n",
       "      <td>0.447962</td>\n",
       "      <td>0.065789</td>\n",
       "      <td>0.030700</td>\n",
       "      <td>0.007843</td>\n",
       "      <td>0.031089</td>\n",
       "      <td>0.337795</td>\n",
       "      <td>0.037865</td>\n",
       "      <td>0.013461</td>\n",
       "      <td>0.337518</td>\n",
       "      <td>0.171166</td>\n",
       "      <td>0.189892</td>\n",
       "      <td>0.195490</td>\n",
       "      <td>0.011729</td>\n",
       "      <td>0.059820</td>\n",
       "      <td>0.006806</td>\n",
       "      <td>0.006765</td>\n",
       "      <td>0.004519</td>\n",
       "      <td>0.005840</td>\n",
       "      <td>0.010456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2947.0</th>\n",
       "      <td>0.004697</td>\n",
       "      <td>0.018286</td>\n",
       "      <td>0.223953</td>\n",
       "      <td>0.013178</td>\n",
       "      <td>0.003370</td>\n",
       "      <td>0.003730</td>\n",
       "      <td>0.640505</td>\n",
       "      <td>0.088873</td>\n",
       "      <td>0.003408</td>\n",
       "      <td>0.291351</td>\n",
       "      <td>0.151943</td>\n",
       "      <td>0.220524</td>\n",
       "      <td>0.195674</td>\n",
       "      <td>0.013143</td>\n",
       "      <td>0.084207</td>\n",
       "      <td>0.008385</td>\n",
       "      <td>0.007127</td>\n",
       "      <td>0.005984</td>\n",
       "      <td>0.009803</td>\n",
       "      <td>0.011858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2948.0</th>\n",
       "      <td>0.052124</td>\n",
       "      <td>0.565919</td>\n",
       "      <td>0.038446</td>\n",
       "      <td>0.033455</td>\n",
       "      <td>0.009417</td>\n",
       "      <td>0.039590</td>\n",
       "      <td>0.222424</td>\n",
       "      <td>0.028003</td>\n",
       "      <td>0.010621</td>\n",
       "      <td>0.322396</td>\n",
       "      <td>0.202790</td>\n",
       "      <td>0.193366</td>\n",
       "      <td>0.181756</td>\n",
       "      <td>0.010406</td>\n",
       "      <td>0.056317</td>\n",
       "      <td>0.006385</td>\n",
       "      <td>0.006474</td>\n",
       "      <td>0.004193</td>\n",
       "      <td>0.005828</td>\n",
       "      <td>0.010089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2949.0</th>\n",
       "      <td>0.352359</td>\n",
       "      <td>0.308921</td>\n",
       "      <td>0.021057</td>\n",
       "      <td>0.051812</td>\n",
       "      <td>0.012259</td>\n",
       "      <td>0.164115</td>\n",
       "      <td>0.051383</td>\n",
       "      <td>0.021903</td>\n",
       "      <td>0.016191</td>\n",
       "      <td>0.379453</td>\n",
       "      <td>0.191668</td>\n",
       "      <td>0.144466</td>\n",
       "      <td>0.202435</td>\n",
       "      <td>0.009357</td>\n",
       "      <td>0.041763</td>\n",
       "      <td>0.005173</td>\n",
       "      <td>0.005717</td>\n",
       "      <td>0.003795</td>\n",
       "      <td>0.006305</td>\n",
       "      <td>0.009869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2950.0</th>\n",
       "      <td>0.039927</td>\n",
       "      <td>0.762812</td>\n",
       "      <td>0.015547</td>\n",
       "      <td>0.016468</td>\n",
       "      <td>0.007742</td>\n",
       "      <td>0.103283</td>\n",
       "      <td>0.031658</td>\n",
       "      <td>0.007951</td>\n",
       "      <td>0.014612</td>\n",
       "      <td>0.330286</td>\n",
       "      <td>0.174305</td>\n",
       "      <td>0.185595</td>\n",
       "      <td>0.200411</td>\n",
       "      <td>0.012938</td>\n",
       "      <td>0.057548</td>\n",
       "      <td>0.007806</td>\n",
       "      <td>0.007699</td>\n",
       "      <td>0.004849</td>\n",
       "      <td>0.007607</td>\n",
       "      <td>0.010956</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2951 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               1         2         3         4         5         6         7  \\\n",
       "0                                                                              \n",
       "0.0     0.021085  0.080077  0.015716  0.004451  0.212755  0.075620  0.037683   \n",
       "1.0     0.007347  0.030522  0.007316  0.001726  0.056869  0.034187  0.009252   \n",
       "2.0     0.038959  0.089459  0.013668  0.009166  0.255841  0.180218  0.051883   \n",
       "3.0     0.049263  0.243927  0.023052  0.018801  0.015961  0.050762  0.547270   \n",
       "4.0     0.226045  0.307158  0.010621  0.060181  0.009664  0.304781  0.042088   \n",
       "...          ...       ...       ...       ...       ...       ...       ...   \n",
       "2946.0  0.027497  0.447962  0.065789  0.030700  0.007843  0.031089  0.337795   \n",
       "2947.0  0.004697  0.018286  0.223953  0.013178  0.003370  0.003730  0.640505   \n",
       "2948.0  0.052124  0.565919  0.038446  0.033455  0.009417  0.039590  0.222424   \n",
       "2949.0  0.352359  0.308921  0.021057  0.051812  0.012259  0.164115  0.051383   \n",
       "2950.0  0.039927  0.762812  0.015547  0.016468  0.007742  0.103283  0.031658   \n",
       "\n",
       "               8         9        10        11        12        13        14  \\\n",
       "0                                                                              \n",
       "0.0     0.004144  0.548468  0.085739  0.365993  0.173247  0.236994  0.015982   \n",
       "1.0     0.001284  0.851497  0.153359  0.434726  0.139513  0.161096  0.009869   \n",
       "2.0     0.005855  0.354952  0.139353  0.401640  0.158797  0.161027  0.009512   \n",
       "3.0     0.043546  0.007417  0.049004  0.041706  0.139258  0.069855  0.015055   \n",
       "4.0     0.018450  0.021013  0.194195  0.066356  0.092250  0.198467  0.018021   \n",
       "...          ...       ...       ...       ...       ...       ...       ...   \n",
       "2946.0  0.037865  0.013461  0.337518  0.171166  0.189892  0.195490  0.011729   \n",
       "2947.0  0.088873  0.003408  0.291351  0.151943  0.220524  0.195674  0.013143   \n",
       "2948.0  0.028003  0.010621  0.322396  0.202790  0.193366  0.181756  0.010406   \n",
       "2949.0  0.021903  0.016191  0.379453  0.191668  0.144466  0.202435  0.009357   \n",
       "2950.0  0.007951  0.014612  0.330286  0.174305  0.185595  0.200411  0.012938   \n",
       "\n",
       "              15        16        17        18        19        20  \n",
       "0                                                                   \n",
       "0.0     0.098432  0.004647  0.004069  0.003527  0.003366  0.008006  \n",
       "1.0     0.081890  0.004033  0.003355  0.002760  0.002715  0.006684  \n",
       "2.0     0.107023  0.004194  0.004084  0.003055  0.003088  0.008226  \n",
       "3.0     0.655422  0.006946  0.005289  0.003414  0.005688  0.008363  \n",
       "4.0     0.398030  0.006777  0.005326  0.003530  0.005200  0.011848  \n",
       "...          ...       ...       ...       ...       ...       ...  \n",
       "2946.0  0.059820  0.006806  0.006765  0.004519  0.005840  0.010456  \n",
       "2947.0  0.084207  0.008385  0.007127  0.005984  0.009803  0.011858  \n",
       "2948.0  0.056317  0.006385  0.006474  0.004193  0.005828  0.010089  \n",
       "2949.0  0.041763  0.005173  0.005717  0.003795  0.006305  0.009869  \n",
       "2950.0  0.057548  0.007806  0.007699  0.004849  0.007607  0.010956  \n",
       "\n",
       "[2951 rows x 20 columns]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = torch.tensor(np.array(preds)).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2951, 20])"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2951, 20])"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([695, 20])"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.y[data.test_mask].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([695, 20])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred[data.test_mask].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_k_confusion_matrix(y, y_pred, k=3):\n",
    "    dim = y.shape[-1]\n",
    "    y = y.topk(k=k, axis=1)[1]\n",
    "    y_pred = y_pred.topk(k=k, axis=1)[1]\n",
    "    conf = np.zeros((dim, dim))\n",
    "    for i in range(k):\n",
    "        for j in range(k):\n",
    "            conf = np.add(conf, confusion_matrix(y[:,i], y_pred[:,j], labels = range(dim)))\n",
    "    return conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ATT_conf = confusion_matrix(data.y[(data.att_lab) * data.test_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                                 pred[(data.att_lab) * data.test_mask][:,:9].argmax(axis=1).cpu(), labels = range(9))\n",
    "test_VAL_conf = confusion_matrix(data.y[(data.val_lab) * data.test_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                                 pred[(data.val_lab) * data.test_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "test_VAL_conf_k = (top_k_confusion_matrix(data.y[(data.val_lab) * data.test_mask][:,9:].cpu(),  \n",
    "                                 pred[(data.val_lab) * data.test_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 64,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0, 144,   0,   0,   0,   1,   0,   0,   0],\n",
       "       [  0,   0,  10,   0,   0,   0,   0,   1,   0],\n",
       "       [  2,   0,   0,  16,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,  71,   0,   1,   0,   0],\n",
       "       [  0,   2,   0,   0,   0, 100,   0,   0,   0],\n",
       "       [  0,   2,   0,   0,   0,   0,  74,   1,   0],\n",
       "       [  0,   0,   2,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   1,   0,   0,   1,   0,   0,   0,  10]], dtype=int64)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[53,  4,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 1, 21,  0,  1,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 2,  5, 17,  7,  0,  4,  0,  0,  0,  0,  0],\n",
       "       [ 1,  2,  2, 27,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 2,  2,  4,  3,  0, 33,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  1,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0]], dtype=int64)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 90,  59,  54,  42,   0,  37,   0,   0,   0,   0,   0],\n",
       "       [ 52,  85,  77,  63,   0,  32,   0,   0,   0,   0,   0],\n",
       "       [ 57,  79, 120,  88,   0,  64,   0,   0,   0,   0,   0],\n",
       "       [ 54,  75,  92, 116,   0,  62,   0,   0,   0,   0,   0],\n",
       "       [  0,   3,   3,   3,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [ 46,  35,  73,  72,   0,  83,   0,   0,   0,   0,   0],\n",
       "       [  2,   0,   2,   0,   0,   2,   0,   0,   0,   0,   0],\n",
       "       [  1,   0,   1,   0,   0,   1,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  1,   0,   1,   0,   0,   1,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0]])"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_ATT_conf = confusion_matrix(data.y[(data.att_lab) * data.val_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                                pred[(data.att_lab) * data.val_mask][:,:9].argmax(axis=1).cpu())\n",
    "val_VAL_conf = confusion_matrix(data.y[(data.val_lab) * data.val_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                                 pred[(data.val_lab) * data.val_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "val_VAL_conf_k = (top_k_confusion_matrix(data.y[(data.val_lab) * data.val_mask][:,9:].cpu(),  \n",
    "                                 pred[(data.val_lab) * data.val_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 61,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  1, 143,   0,   0,   1,   1,   0,   0,   0],\n",
       "       [  0,   0,   8,   0,   0,   0,   0,   0,   0],\n",
       "       [  1,   0,   0,  21,   0,   0,   0,   0,   0],\n",
       "       [  0,   1,   0,   0,  74,   0,   0,   0,   0],\n",
       "       [  1,   1,   0,   0,   1, 100,   0,   0,   0],\n",
       "       [  0,   2,   0,   0,   1,   0,  59,   0,   0],\n",
       "       [  1,   0,   3,   0,   0,   0,   1,   0,   0],\n",
       "       [  0,   1,   0,   0,   2,   2,   0,   0,   5]], dtype=int64)"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[65,  3,  1,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 1, 13,  1,  3,  0,  1,  0,  0,  0,  0,  0],\n",
       "       [ 1,  3, 18,  5,  0,  4,  0,  0,  0,  0,  0],\n",
       "       [ 1,  2,  0, 30,  0,  1,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 6,  5,  0,  2,  0, 37,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0]], dtype=int64)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[102,  70,  52,  53,   0,  47,   0,   0,   0,   0,   0],\n",
       "       [ 60,  88,  66,  66,   2,  39,   0,   0,   0,   0,   0],\n",
       "       [ 55,  66, 111,  88,   0,  73,   0,   0,   0,   0,   0],\n",
       "       [ 68,  88,  95, 118,   2,  55,   0,   0,   0,   0,   0],\n",
       "       [  0,   8,   6,   8,   2,   0,   0,   0,   0,   0,   0],\n",
       "       [ 51,  43,  87,  66,   0,  92,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0]])"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat([pd.DataFrame(val_ATT_conf),pd.DataFrame(test_ATT_conf)],axis=1).to_csv(args.save_dir+'confusion_matrix_ATT.csv')\n",
    "pd.concat([pd.DataFrame(val_VAL_conf),pd.DataFrame(test_VAL_conf)],axis=1).to_csv(args.save_dir+'confusion_matrix_VAL.csv')\n",
    "pd.concat([pd.DataFrame(val_VAL_conf_k),pd.DataFrame(test_VAL_conf_k)],axis=1).to_csv(args.save_dir+'confusion_matrix_VAL_k.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics(confusion_matrix, classes):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val/(row.sum()+0.000000001)\n",
    "        recall = val/(col.sum()+0.000000001)\n",
    "        F1 = 2*(precision*recall)/(precision+recall+0.000000001)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics_k(confusion_matrix, classes, k=3):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val*k/(row.sum()+0.000000001)\n",
    "        recall = val*k/(col.sum()+0.000000001)\n",
    "        F1 = 2*(precision*recall)/(precision+recall+0.000000001)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = ['Criterion i', 'Criterion ii', 'Criterion iii', 'Criterion iv', 'Criterion v', 'Criterion vi', \n",
    "              'Criterion vii', 'Criterion viii', 'Criterion ix', 'Criterion x', 'Others']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = ['Building Elements',\n",
    " 'Urban Form Elements',\n",
    " 'Gastronomy',\n",
    " 'Interior Scenery',\n",
    " 'Natural Features and Land-scape Scenery',\n",
    " 'Monuments and Buildings',\n",
    " 'Peoples Activity and Association',\n",
    " 'Artifact Products',\n",
    " 'Urban Scenery']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_dict = {}\n",
    "metrics_dict['test_ATT'] = per_class_metrics(test_ATT_conf, categories)\n",
    "metrics_dict['val_ATT'] = per_class_metrics(val_ATT_conf, categories)\n",
    "metrics_dict['test_VAL'] = per_class_metrics(test_VAL_conf, classes)\n",
    "metrics_dict['val_VAL'] = per_class_metrics(val_VAL_conf, classes)\n",
    "metrics_dict['test_VAL_k'] = per_class_metrics_k(test_VAL_conf_k, classes)\n",
    "metrics_dict['val_VAL_k'] = per_class_metrics_k(val_VAL_conf_k, classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df = pd.DataFrame.from_dict({(i,j): metrics_dict[i][j] \n",
    "                           for i in metrics_dict.keys() \n",
    "                           for j in metrics_dict[i].keys()},\n",
    "                       orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">test_ATT</th>\n",
       "      <th>Building Elements</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.969697</td>\n",
       "      <td>0.984615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Urban Form Elements</th>\n",
       "      <td>0.993103</td>\n",
       "      <td>0.966443</td>\n",
       "      <td>0.979592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gastronomy</th>\n",
       "      <td>0.909091</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.869565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Interior Scenery</th>\n",
       "      <td>0.888889</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.941176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Natural Features and Land-scape Scenery</th>\n",
       "      <td>0.986111</td>\n",
       "      <td>0.986111</td>\n",
       "      <td>0.986111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">val_VAL_k</th>\n",
       "      <th>Criterion vii</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Criterion viii</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Criterion ix</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Criterion x</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Others</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>62 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   precision    recall  \\\n",
       "test_ATT  Building Elements                         1.000000  0.969697   \n",
       "          Urban Form Elements                       0.993103  0.966443   \n",
       "          Gastronomy                                0.909091  0.833333   \n",
       "          Interior Scenery                          0.888889  1.000000   \n",
       "          Natural Features and Land-scape Scenery   0.986111  0.986111   \n",
       "...                                                      ...       ...   \n",
       "val_VAL_k Criterion vii                             0.000000  0.000000   \n",
       "          Criterion viii                            0.000000  0.000000   \n",
       "          Criterion ix                              0.000000  0.000000   \n",
       "          Criterion x                               0.000000  0.000000   \n",
       "          Others                                    0.000000  0.000000   \n",
       "\n",
       "                                                         F1  \n",
       "test_ATT  Building Elements                        0.984615  \n",
       "          Urban Form Elements                      0.979592  \n",
       "          Gastronomy                               0.869565  \n",
       "          Interior Scenery                         0.941176  \n",
       "          Natural Features and Land-scape Scenery  0.986111  \n",
       "...                                                     ...  \n",
       "val_VAL_k Criterion vii                            0.000000  \n",
       "          Criterion viii                           0.000000  \n",
       "          Criterion ix                             0.000000  \n",
       "          Criterion x                              0.000000  \n",
       "          Others                                   0.000000  \n",
       "\n",
       "[62 rows x 3 columns]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df.to_csv(args.save_dir+'per_class_metrics.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explanation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.nn import GNNExplainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "explainer = GNNExplainer(model, epochs=200, return_type='log_prob')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def explain_features(model, loader):\n",
    "    model.eval()\n",
    "    seed_everything(args.seed)\n",
    "    explainer = GNNExplainer(model, epochs=200, return_type='log_prob')\n",
    "    all_preds = {}\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch.batch_size\n",
    "        edge_index = to_undirected(batch.edge_index)\n",
    "        for node_idx in tqdm(range(batch_size)):\n",
    "            node_feat_mask, _ = explainer.explain_node(node_idx, batch.x, edge_index)\n",
    "            all_preds[batch.n_id[node_idx]] = node_feat_mask\n",
    "    \n",
    "    out = torch.vstack([torch.hstack([i, all_preds[i]]) for i in all_preds.keys()])\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_explain = explain_features(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_explain = explain_features(model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_explain = explain_features(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_explain_df = pd.DataFrame(train_explain.cpu()).sort_values(0).set_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_explain_df = pd.DataFrame(val_explain.cpu()).sort_values(0).set_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_explain_df = pd.DataFrame(test_explain.cpu()).sort_values(0).set_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_explain_df = pd.concat([train_explain_df, val_explain_df, test_explain_df]).reset_index().rename(columns={0:'ID'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_explain_df['ID'] = all_explain_df['ID'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>1744</th>\n",
       "      <th>1745</th>\n",
       "      <th>1746</th>\n",
       "      <th>1747</th>\n",
       "      <th>1748</th>\n",
       "      <th>1749</th>\n",
       "      <th>1750</th>\n",
       "      <th>1751</th>\n",
       "      <th>1752</th>\n",
       "      <th>1753</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.191380</td>\n",
       "      <td>0.070536</td>\n",
       "      <td>0.890955</td>\n",
       "      <td>0.154555</td>\n",
       "      <td>0.667271</td>\n",
       "      <td>0.859189</td>\n",
       "      <td>0.577962</td>\n",
       "      <td>0.842605</td>\n",
       "      <td>0.131733</td>\n",
       "      <td>...</td>\n",
       "      <td>0.165758</td>\n",
       "      <td>0.111742</td>\n",
       "      <td>0.264100</td>\n",
       "      <td>0.882734</td>\n",
       "      <td>0.833117</td>\n",
       "      <td>0.817715</td>\n",
       "      <td>0.831687</td>\n",
       "      <td>0.847712</td>\n",
       "      <td>0.870059</td>\n",
       "      <td>0.123290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28</td>\n",
       "      <td>0.916585</td>\n",
       "      <td>0.142908</td>\n",
       "      <td>0.815150</td>\n",
       "      <td>0.144299</td>\n",
       "      <td>0.852302</td>\n",
       "      <td>0.131787</td>\n",
       "      <td>0.852444</td>\n",
       "      <td>0.827218</td>\n",
       "      <td>0.877387</td>\n",
       "      <td>...</td>\n",
       "      <td>0.171603</td>\n",
       "      <td>0.820932</td>\n",
       "      <td>0.131418</td>\n",
       "      <td>0.884365</td>\n",
       "      <td>0.741325</td>\n",
       "      <td>0.684704</td>\n",
       "      <td>0.112249</td>\n",
       "      <td>0.683977</td>\n",
       "      <td>0.173271</td>\n",
       "      <td>0.110680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>30</td>\n",
       "      <td>0.656934</td>\n",
       "      <td>0.874703</td>\n",
       "      <td>0.897564</td>\n",
       "      <td>0.831838</td>\n",
       "      <td>0.611965</td>\n",
       "      <td>0.835433</td>\n",
       "      <td>0.857550</td>\n",
       "      <td>0.892003</td>\n",
       "      <td>0.152759</td>\n",
       "      <td>...</td>\n",
       "      <td>0.824198</td>\n",
       "      <td>0.417188</td>\n",
       "      <td>0.135931</td>\n",
       "      <td>0.890457</td>\n",
       "      <td>0.857191</td>\n",
       "      <td>0.848484</td>\n",
       "      <td>0.187467</td>\n",
       "      <td>0.818267</td>\n",
       "      <td>0.900504</td>\n",
       "      <td>0.134084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>34</td>\n",
       "      <td>0.878631</td>\n",
       "      <td>0.839939</td>\n",
       "      <td>0.815494</td>\n",
       "      <td>0.835174</td>\n",
       "      <td>0.871223</td>\n",
       "      <td>0.849590</td>\n",
       "      <td>0.179989</td>\n",
       "      <td>0.816892</td>\n",
       "      <td>0.848545</td>\n",
       "      <td>...</td>\n",
       "      <td>0.835072</td>\n",
       "      <td>0.838939</td>\n",
       "      <td>0.131701</td>\n",
       "      <td>0.855418</td>\n",
       "      <td>0.876120</td>\n",
       "      <td>0.886572</td>\n",
       "      <td>0.858278</td>\n",
       "      <td>0.880519</td>\n",
       "      <td>0.817320</td>\n",
       "      <td>0.146757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>35</td>\n",
       "      <td>0.899845</td>\n",
       "      <td>0.808518</td>\n",
       "      <td>0.853702</td>\n",
       "      <td>0.832495</td>\n",
       "      <td>0.859872</td>\n",
       "      <td>0.842544</td>\n",
       "      <td>0.886377</td>\n",
       "      <td>0.794908</td>\n",
       "      <td>0.848853</td>\n",
       "      <td>...</td>\n",
       "      <td>0.832578</td>\n",
       "      <td>0.889195</td>\n",
       "      <td>0.121868</td>\n",
       "      <td>0.861419</td>\n",
       "      <td>0.842121</td>\n",
       "      <td>0.874372</td>\n",
       "      <td>0.836251</td>\n",
       "      <td>0.897768</td>\n",
       "      <td>0.191089</td>\n",
       "      <td>0.809527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1746</th>\n",
       "      <td>2931</td>\n",
       "      <td>0.172773</td>\n",
       "      <td>0.132829</td>\n",
       "      <td>0.858535</td>\n",
       "      <td>0.839368</td>\n",
       "      <td>0.879682</td>\n",
       "      <td>0.792292</td>\n",
       "      <td>0.734532</td>\n",
       "      <td>0.803669</td>\n",
       "      <td>0.834399</td>\n",
       "      <td>...</td>\n",
       "      <td>0.176462</td>\n",
       "      <td>0.878227</td>\n",
       "      <td>0.138987</td>\n",
       "      <td>0.666388</td>\n",
       "      <td>0.904704</td>\n",
       "      <td>0.198043</td>\n",
       "      <td>0.090357</td>\n",
       "      <td>0.136764</td>\n",
       "      <td>0.265421</td>\n",
       "      <td>0.111996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1747</th>\n",
       "      <td>2940</td>\n",
       "      <td>0.853294</td>\n",
       "      <td>0.822491</td>\n",
       "      <td>0.888527</td>\n",
       "      <td>0.233365</td>\n",
       "      <td>0.865722</td>\n",
       "      <td>0.839907</td>\n",
       "      <td>0.852623</td>\n",
       "      <td>0.888329</td>\n",
       "      <td>0.229653</td>\n",
       "      <td>...</td>\n",
       "      <td>0.310373</td>\n",
       "      <td>0.877840</td>\n",
       "      <td>0.141607</td>\n",
       "      <td>0.855834</td>\n",
       "      <td>0.737038</td>\n",
       "      <td>0.171399</td>\n",
       "      <td>0.899665</td>\n",
       "      <td>0.872952</td>\n",
       "      <td>0.896432</td>\n",
       "      <td>0.108591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1748</th>\n",
       "      <td>2944</td>\n",
       "      <td>0.849596</td>\n",
       "      <td>0.115164</td>\n",
       "      <td>0.850300</td>\n",
       "      <td>0.150556</td>\n",
       "      <td>0.840395</td>\n",
       "      <td>0.158579</td>\n",
       "      <td>0.844938</td>\n",
       "      <td>0.839588</td>\n",
       "      <td>0.517085</td>\n",
       "      <td>...</td>\n",
       "      <td>0.826763</td>\n",
       "      <td>0.802192</td>\n",
       "      <td>0.819940</td>\n",
       "      <td>0.864570</td>\n",
       "      <td>0.331629</td>\n",
       "      <td>0.183510</td>\n",
       "      <td>0.858444</td>\n",
       "      <td>0.132697</td>\n",
       "      <td>0.839593</td>\n",
       "      <td>0.161466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1749</th>\n",
       "      <td>2947</td>\n",
       "      <td>0.846088</td>\n",
       "      <td>0.153487</td>\n",
       "      <td>0.860638</td>\n",
       "      <td>0.094728</td>\n",
       "      <td>0.852733</td>\n",
       "      <td>0.126536</td>\n",
       "      <td>0.802176</td>\n",
       "      <td>0.864283</td>\n",
       "      <td>0.824211</td>\n",
       "      <td>...</td>\n",
       "      <td>0.830477</td>\n",
       "      <td>0.857355</td>\n",
       "      <td>0.865812</td>\n",
       "      <td>0.872492</td>\n",
       "      <td>0.262918</td>\n",
       "      <td>0.783094</td>\n",
       "      <td>0.818889</td>\n",
       "      <td>0.129730</td>\n",
       "      <td>0.829582</td>\n",
       "      <td>0.154047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1750</th>\n",
       "      <td>2948</td>\n",
       "      <td>0.841361</td>\n",
       "      <td>0.157474</td>\n",
       "      <td>0.894473</td>\n",
       "      <td>0.827944</td>\n",
       "      <td>0.885683</td>\n",
       "      <td>0.438405</td>\n",
       "      <td>0.751103</td>\n",
       "      <td>0.840052</td>\n",
       "      <td>0.195704</td>\n",
       "      <td>...</td>\n",
       "      <td>0.446827</td>\n",
       "      <td>0.176274</td>\n",
       "      <td>0.444130</td>\n",
       "      <td>0.861291</td>\n",
       "      <td>0.844293</td>\n",
       "      <td>0.866857</td>\n",
       "      <td>0.833613</td>\n",
       "      <td>0.131990</td>\n",
       "      <td>0.831900</td>\n",
       "      <td>0.103366</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1751 rows × 1754 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        ID         1         2         3         4         5         6  \\\n",
       "0        1  0.191380  0.070536  0.890955  0.154555  0.667271  0.859189   \n",
       "1       28  0.916585  0.142908  0.815150  0.144299  0.852302  0.131787   \n",
       "2       30  0.656934  0.874703  0.897564  0.831838  0.611965  0.835433   \n",
       "3       34  0.878631  0.839939  0.815494  0.835174  0.871223  0.849590   \n",
       "4       35  0.899845  0.808518  0.853702  0.832495  0.859872  0.842544   \n",
       "...    ...       ...       ...       ...       ...       ...       ...   \n",
       "1746  2931  0.172773  0.132829  0.858535  0.839368  0.879682  0.792292   \n",
       "1747  2940  0.853294  0.822491  0.888527  0.233365  0.865722  0.839907   \n",
       "1748  2944  0.849596  0.115164  0.850300  0.150556  0.840395  0.158579   \n",
       "1749  2947  0.846088  0.153487  0.860638  0.094728  0.852733  0.126536   \n",
       "1750  2948  0.841361  0.157474  0.894473  0.827944  0.885683  0.438405   \n",
       "\n",
       "             7         8         9  ...      1744      1745      1746  \\\n",
       "0     0.577962  0.842605  0.131733  ...  0.165758  0.111742  0.264100   \n",
       "1     0.852444  0.827218  0.877387  ...  0.171603  0.820932  0.131418   \n",
       "2     0.857550  0.892003  0.152759  ...  0.824198  0.417188  0.135931   \n",
       "3     0.179989  0.816892  0.848545  ...  0.835072  0.838939  0.131701   \n",
       "4     0.886377  0.794908  0.848853  ...  0.832578  0.889195  0.121868   \n",
       "...        ...       ...       ...  ...       ...       ...       ...   \n",
       "1746  0.734532  0.803669  0.834399  ...  0.176462  0.878227  0.138987   \n",
       "1747  0.852623  0.888329  0.229653  ...  0.310373  0.877840  0.141607   \n",
       "1748  0.844938  0.839588  0.517085  ...  0.826763  0.802192  0.819940   \n",
       "1749  0.802176  0.864283  0.824211  ...  0.830477  0.857355  0.865812   \n",
       "1750  0.751103  0.840052  0.195704  ...  0.446827  0.176274  0.444130   \n",
       "\n",
       "          1747      1748      1749      1750      1751      1752      1753  \n",
       "0     0.882734  0.833117  0.817715  0.831687  0.847712  0.870059  0.123290  \n",
       "1     0.884365  0.741325  0.684704  0.112249  0.683977  0.173271  0.110680  \n",
       "2     0.890457  0.857191  0.848484  0.187467  0.818267  0.900504  0.134084  \n",
       "3     0.855418  0.876120  0.886572  0.858278  0.880519  0.817320  0.146757  \n",
       "4     0.861419  0.842121  0.874372  0.836251  0.897768  0.191089  0.809527  \n",
       "...        ...       ...       ...       ...       ...       ...       ...  \n",
       "1746  0.666388  0.904704  0.198043  0.090357  0.136764  0.265421  0.111996  \n",
       "1747  0.855834  0.737038  0.171399  0.899665  0.872952  0.896432  0.108591  \n",
       "1748  0.864570  0.331629  0.183510  0.858444  0.132697  0.839593  0.161466  \n",
       "1749  0.872492  0.262918  0.783094  0.818889  0.129730  0.829582  0.154047  \n",
       "1750  0.861291  0.844293  0.866857  0.833613  0.131990  0.831900  0.103366  \n",
       "\n",
       "[1751 rows x 1754 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_explain_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_explain_df.to_csv(args.save_dir + 'all_explain.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_counts = pd.concat([(all_explain_df>0.2).sum().loc[513:982],(all_explain_df>0.2).sum().loc[1751:]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "thres = []\n",
    "for i in range(20):\n",
    "    thres.append((0.05*i, (((all_explain_df>(0.05*i)).sum().loc[513:982])>0).sum()/473))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.0, 0.9936575052854123),\n",
       " (0.05, 0.9936575052854123),\n",
       " (0.1, 0.9936575052854123),\n",
       " (0.15000000000000002, 0.9936575052854123),\n",
       " (0.2, 0.30655391120507397),\n",
       " (0.25, 0.29809725158562367),\n",
       " (0.30000000000000004, 0.2959830866807611),\n",
       " (0.35000000000000003, 0.2917547568710359),\n",
       " (0.4, 0.2832980972515856),\n",
       " (0.45, 0.2832980972515856),\n",
       " (0.5, 0.27906976744186046),\n",
       " (0.55, 0.2748414376321353),\n",
       " (0.6000000000000001, 0.2727272727272727),\n",
       " (0.65, 0.27061310782241016),\n",
       " (0.7000000000000001, 0.266384778012685),\n",
       " (0.75, 0.2536997885835095),\n",
       " (0.8, 0.23678646934460887),\n",
       " (0.8500000000000001, 0.18816067653276955),\n",
       " (0.9, 0.042283298097251586),\n",
       " (0.9500000000000001, 0.0)]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "thres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['KMP_DUPLICATE_LIB_OK']='True'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAcCUlEQVR4nO3de3Rc5X3u8e9vZqSZ0cUaX+SLLsb4EmNDbI5RCCEQDEnBhgSSJqWQrJCm4bhucFdYKwm46Wqa0540B0hyWhIacFOatocU0kIIDQZyEgKEJqQWHNtgjEE2F98tYyzrYt3f88ds2dJ4JI00W957Rs9nrVkzs/c7mp83wzN73v3u/ZpzDhERKXyRoAsQERF/KNBFRIqEAl1EpEgo0EVEioQCXUSkSMSCeuMZM2a4efPmBfX2IiIF6fnnnz/snKvOti6wQJ83bx6NjY1Bvb2ISEEyszeHW6cuFxGRIqFAFxEpEgp0EZEioUAXESkSowa6md1rZofM7KVh1puZ3WlmTWa21cxW+F+miIiMJpc99B8Aq0ZYvxpY5N3WAN/LvywRERmrUQPdOfcMcGSEJtcA/+zSngNSZjbHrwJFRCQ3foxDrwV2D3q+x1u2P7Ohma0hvRfP3Llzx/VmOw608ujWfeN6bViUx2P84UVnUhLVIQwR8Y8fgW5ZlmW9yLpzbgOwAaChoWFcF2JvOtTGd37ZNJ6XhsLA5effXVfFhQtmBFuMiBQVPwJ9D1A/6HkdMGG70Fctm8NVy66aqD8/4d56u4MP3PFL9r5zPOhSRKTI+PGb/xHgBm+0ywVAi3PulO4WSZtVFccM9h3tDLoUESkyo+6hm9m/AiuBGWa2B/gLoATAOXc3sBG4EmgCOoDPTlSxxSAei1JdEWffUe2hi4i/Rg1059z1o6x3wE2+VTQJ1KSS7GtRoIuIvzTMIgC1qSR7tYcuIj5ToAegJpVg39HjODeugT4iIlkp0ANQk0rS2dPPOx09QZciIkVEgR6AmlQSQAdGRcRXCvQA1HqBrn50EfGTAj0A2kMXkYmgQA/A1LISEiURBbqI+EqBHgAzS49F19miIuIjBXpANBZdRPymQA9ITVVSXS4i4isFekBqUkkOtXbR1dsXdCkiUiQU6AGpSSUAONjSFXAlIlIsFOgB0Vh0EfGbAj0gGosuIn5ToAdkdlW6y0WBLiJ+UaAHJFESZUZFXNdFFxHfKNADVJtKsFcnF4mITxToAUqfLao9dBHxhwI9QAOBrokuRMQPCvQA1aSSdHT30XJcE12ISP4U6AGq9U4u0lh0EfGDAj1AJ8ei68CoiORPgR4gnVwkIn5SoAdoenkppTFNdCEi/lCgB8jMdF10EfGNAj1gNamE9tBFxBcK9IClJ7rQQVERyZ8CPWA1qSQHWzvp6esPuhQRKXAK9IDVppI4BwdatJcuIvlRoAdMQxdFxC8K9IANTEWny+iKSL5yCnQzW2VmO8ysyczWZ1lfZWb/YWZbzGybmX3W/1KLk84WFRG/jBroZhYF7gJWA0uB681saUazm4CXnXPLgZXAt8ys1Odai1KiJMr08lKNRReRvOWyh34+0OSc2+Wc6wbuB67JaOOASjMzoAI4AvT6WmkR03XRRcQPuQR6LbB70PM93rLBvgssAfYBLwJfcM6dMg7PzNaYWaOZNTY3N4+z5OKjk4tExA+5BLplWZY5I8MVwGagBjgX+K6ZTTnlRc5tcM41OOcaqqurx1hq8apJJdn7jia6EJH85BLoe4D6Qc/rSO+JD/ZZ4CGX1gS8DpzlT4nFrzaVpL27j2Od6qUSkfHLJdA3AYvM7EzvQOd1wCMZbd4CPghgZrOAxcAuPwstZhqLLiJ+GDXQnXO9wDrgCWA78CPn3DYzW2tma71mfwVcaGYvAr8AbnXOHZ6ooouNAl1E/BDLpZFzbiOwMWPZ3YMe7wMu97e0yePEyUUKdBHJg84UDYEZ5XFKoxH26uQiEcmDAj0EIhFjjoYuikieFOghkb4uugJdRMZPgR4SOltURPKlQA+J2lSCA8c66dVEFyIyTgr0kKhJJel3cLC1K+hSRKRAKdBDQmPRRSRfCvSQUKCLSL4U6CExcHKRrosuIuOlQA+JstIYU8tKtIcuIuOmQA+R9NBFnS0qIuOjQA8RjUUXkXwo0EOkNpVUH7qIjJsCPURqUglaO3s51tkTdCkiUoAU6CEyMHRxv/rRRWQcFOghorHoIpIPBXqI1HqBrn50ERkPBXqIVFfEKYma9tBFZFwU6CESiRizqzTRhYiMjwI9ZNITXeigqIiMnQI9ZDQWXUTGS4EeMjWpJAeOddLX74IuRUQKjAI9ZGpSSfr6HYda1e0iImOjQA+Zgcvo6sCoiIyVAj1kTo5F1x66iIyNAj1k5uhsUREZJwV6yFTEY1QlNdGFiIydAj2EdF10ERkPBXoI1aYS6kMXkTFToIeQ9tBFZDxyCnQzW2VmO8ysyczWD9NmpZltNrNtZva0v2VOLjWpJC3He2jr6g26FBEpIKMGuplFgbuA1cBS4HozW5rRJgX8HXC1c+5s4Pf8L3XyODnRhfbSRSR3ueyhnw80Oed2Oee6gfuBazLafBJ4yDn3FoBz7pC/ZU4utd7JRbqmi4iMRS6BXgvsHvR8j7dssHcBU83sKTN73sxu8KvAyejkzEU6MCoiuYvl0MayLMu8clQMOA/4IJAEfmNmzznnXh3yh8zWAGsA5s6dO/ZqJ4mZlQmiEU10ISJjk8se+h6gftDzOmBfljaPO+fanXOHgWeA5Zl/yDm3wTnX4JxrqK6uHm/NRS8aMWZP0UQXIjI2uQT6JmCRmZ1pZqXAdcAjGW1+AlxsZjEzKwPeC2z3t9TJRddFF5GxGrXLxTnXa2brgCeAKHCvc26bma311t/tnNtuZo8DW4F+4PvOuZcmsvBiV5NK8Pxb7wRdhogUkFz60HHObQQ2Ziy7O+P5HcAd/pU2udWkkjz64n76+h3RSLbDGCIiQ+lM0ZCqSSXp6XMcbusKuhQRKRAK9JA6eV109aOLSG4U6CFVo+uii8gYKdBDSlPRichYKdBDqjJRQmUiprNFRSRnCvQQ01h0ERkLBXqI6broIjIWCvQQq0np9H8RyZ0CPcRqUkne6eiho1sTXYjI6BToIVary+iKyBgo0ENMY9FFZCwU6CGmQBeRsVCgh9isyjgRU6CLSG4U6CEWi0aYPSXBXvWhi0gOFOghp7HoIpIrBXrI1aSS7GtRoIvI6BToIVeTSrL/aCf9/ZnzcouIDKVAD7naVILuvn4Ot2uiCxEZmQI95Gp0cpGI5EiBHnIaiy4iuVKgh5wCXURypUAPuSmJGBXxmK6LLiKjUqCHnJnpMroikhMFegFIn1ykg6IiMjIFegHQ2aIikgsFegGoTSV5u72bzp6+oEsRkRBToBeAmlQC0EgXERmZAr0A1FTp5CIRGZ0CvQBoLLqI5EKBXgBmVyUwQ2PRRWRECvQCUBKNMKtSY9FFZGQ5BbqZrTKzHWbWZGbrR2j3HjPrM7NP+FeiQPrAqK6LLiIjGTXQzSwK3AWsBpYC15vZ0mHa3QY84XeRopOLRGR0ueyhnw80Oed2Oee6gfuBa7K0+xPgQeCQj/WJpzaVZO/R4ziniS5EJLtcAr0W2D3o+R5v2QlmVgt8DLh7pD9kZmvMrNHMGpubm8da66RWk0rS3dvP2+3dQZciIiGVS6BblmWZu4l/A9zqnBvxVEbn3AbnXINzrqG6ujrHEgU0dFFERhfLoc0eoH7Q8zpgX0abBuB+MwOYAVxpZr3OuYf9KFKGni26rC4VbDEiEkq5BPomYJGZnQnsBa4DPjm4gXPuzIHHZvYD4KcKc3/Venvoe3VgVESGMWqgO+d6zWwd6dErUeBe59w2M1vrrR+x31z8UZUsoaw0qi4XERlWLnvoOOc2AhszlmUNcufcH+RflmRKT3Shy+iKyPB0pmgBUaCLyEgU6AWkNpVQH7qIDEuBXkBqqpIcbuvSRBcikpUCvYAMjEU/0KK9dBE5lQK9gOjkIhEZiQK9gJwci65AF5FTKdALyKyqOGaaik5EslOgF5B4LEp1RVxdLiKSlQK9wNSkkproQkSyUqAXmIHroouIZFKgF5iaVHpuUU10ISKZFOgFpiaVpLOnn3c6eoIuRURCJqeLc0l4DIxFX/t/nmfe9DJmT0kwqyqRvvdu08tLiUSyzUsiIsVMgV5g3nvmND68bA5vHengqR3NHG7roj+j96UkasysTDBrSpzZVemQnz0lweyqBDMrE8ycEidZEqU0FknfoumbvgRECpsCvcCkykr57idXnHje29dPc1sXB491caClk4PHOjlwrJODLen7Vw608vSOZtq7R7/+SyxilMYilEQHBf2g+5KoeV8CUSriUcpLY1QkYlTE07fyeIzKRGzI8vL4yfWJkgjerFYiMgEU6AUuFo0wpyrJnKrk0IkCM7R19Z4I/ObW9AW+uvv66e7tP3nf20/PkGXOu+/z1jm6e/tp6ehm39E+2jp7ae/qpa27l1yO0UYjRnlplMpECdWV8RO/GmZNSTC7Ks6sypPdR+VxfTRFxkr/10wSFfEYC2dWsHBmhe9/u7/fcbynj/auXlq7vJDv7KWtK30bvLy9q49jx3s41NpFU3Mb/9l0mNau3lP+ZmU8NuTYwOyq+IljBDMr45RE8zueX1YaPfErIlkS1S8HKQoKdMlbJGKUe90rM8fx+vau3nQ3kXc70NLl3ae7jXbuPMyh1i76Mg8W+CRipLuLBrqIBrqLsnQppZ9HqYiXUB6PUundV3jr9OUgQVKgS+DK4zEWVFewoHr4Xw99/Y6327s42NLFodbOvMK930FnT98pvybau07+qmjr6uXgsc4hvzRyecuBL4chXwKZxxjiUeKx6MljFbEI8WiEkphRGo0OOV4RzzimURJNLxt8rCMWMX2JCKBAlwIRjaRH7sysTABVp/39nXN09vQP7UbqPPVLYLjlh1o7ae/qo7Wzh/buPl9/bZhxYqTSiZFLA4E/eNmgL4ayeDSnXyTpL6AYZSVRjYIqAAp0kRyYGcnSKMnSKNWV8bz/Xm9f+sBzT6+jq2/oQechB6r7+unJeJ79APbQdd19J9d3ecs6uns5eryfrp5+Orr7TnzZ5PLlYkY68L0vgLqpSRZWV7BgZvqX1cKZFUwrL817u0h+FOgiAYhFI8SiESgFKAmsDuccXb3eL4+MA9lDHnf20tbVR1tXD8eO9/LmkQ5+s/Ntunr7T/ytqWUlJ7rOFs6sYMHMchZUV1A3tYyo9u5PCwW6yCRmZiRKoiRKosyoGNsvj/5+x96jx9nZ3EbToTZ2Nrezs7mNn28/yAONu0+0K41FmD+j3Av7chbMrODsmirmzyhXN47PFOgiMi6RiFE/rYz6aWWsXDx0fNM77d3sOtzGzkPtNDW3sfNQG9v2tfDYS/tPHFyujMdYVl/F8roUy+pSnFufYnZVIoB/SfFQoIuI76aWl3Je+TTOO2PakOWdPX28fridF/e2sHXPUbbsbmHDM7vo9VJ+1pQ4y+tSLK9PsbwuxbvrqqhKBtclVWgU6CJy2iRKoiyZM4Ulc6ZwbUP61ObOnj5e3n+MLbuPsmX3UbbuaeFnLx888Zr51eWc64X8sroqlsyZQqIkGtQ/IdQU6CISqERJlBVzp7Ji7tQTy1o6eti6Nx3wW/a08Kumwzz0//YC6YvPfeK8em65YjFTNbJmCAW6iIROVVkJFy+q5uJF1UB6NM6BY51s2X2UZ147zAObdvP4S/u5ddVZXNtQr4OrHgtq5puGhgbX2NgYyHuLSGF75cAxvvrwNv7rjSOcW5/if370HM6pPf0nnAXBzJ53zjVkW6cZi0Sk4Jw1ewoP/NEFfPva5ex5p4OPfPdZ/vzhl2iZ5DN5KdBFpCCZGb+7oo5ffHElN1xwBvf99k0u+9ZT/Fvjbvon6EJuYZdToJvZKjPbYWZNZrY+y/pPmdlW7/ZrM1vuf6kiIqeqSpbwP645h0fWXcTc6WV8+d+3cu09v2H7/mNBl3bajRroZhYF7gJWA0uB681saUaz14FLnHPLgL8CNvhdqIjISM6preLBtRdy+8eXsetwOx/+zrP85X+8TGvn5OmGyWUP/XygyTm3yznXDdwPXDO4gXPu1865d7ynzwF1/pYpIjK6SMS49j31PPnFS7juPfX8469f57JvPc1PNu8lqAEgp1MugV4L7B70fI+3bDifAx7LtsLM1phZo5k1Njc3516liMgYpMpK+frH3s3Dn38/c6oSfOH+zVz/98/x2sHWoEubULkEerYBnlm/6szsUtKBfmu29c65Dc65BudcQ3V1de5VioiMw/L6FD/+/Pv5+sfOYfv+Vlb/7a/4xsbtHM9h0vRClMuJRXsYOv1wHbAvs5GZLQO+D6x2zr3tT3kiIvmJRoxPvfcMVp09m9sf38E9z+yi5XgP/+vjy4IuzXe57KFvAhaZ2ZlmVgpcBzwyuIGZzQUeAj7tnHvV/zJFRPIzvSLObZ9YxucuOpMfNe5mx4Hi634ZNdCdc73AOuAJYDvwI+fcNjNba2ZrvWZfBaYDf2dmm81Mp4CKSCj9yWULqYjH+MZj24MuxXc5XcvFObcR2Jix7O5Bj28EbvS3NBER/6XKSll32UL+euMrPPvaYS5aNCPoknyjM0VFZNK54X3zqE0l+euN24vqrFIFuohMOomSKLesWszL+4/xY++yvMVAgS4ik9JHltWwrK6Kb/5sB509xTGMUYEuIpNSJGJ85col7G/p5B+efT3ocnyhQBeRSeuC+dP50JJZfO+pnbzd1hV0OXlToIvIpLZ+9Vkc7+njzl+8FnQpeVOgi8iktnBmBde9p577fvsWu5rbgi4nLwp0EZn0bv7Qu4jHItz2+CtBl5IXBbqITHrVlXHWXrKAJ7YdZNMbR4IuZ9wU6CIiwI0Xz2fWlDhff3R7wV47XYEuIgIkS6N88fLFbN59lEdf3B90OeOiQBcR8Xx8RR1nza7k9sd30NVbeCcbKdBFRDzRiPGnVy7hrSMd/Mtv3gy6nDFToIuIDHLJu6q5eNEMvvNkEy0dhTXBtAJdRCTDV65cwrHOHu56qinoUsZEgS4ikmHJnCl8fEUdP/jPN9h9pCPocnKmQBcRyeKLl7+LSATueGJH0KXkTIEuIpLFnKokN140n0e27GPL7qNBl5MTBbqIyDD+6JL5TC8v5esbC+NkIwW6iMgwKhMl3PyhRfzX60f4+fZDQZczKgW6iMgIrjt/LvOry/nGY9vp6esPupwRKdBFREZQEo2wftVZ7Gpu5/5Nu4MuZ0QKdBGRUfzO0lmcP28af/vzV2ntDO/JRgp0EZFRmBlfuWoJh9u6uefpXUGXMywFuohIDs6tT/GR5TV8/9ld7G85HnQ5WSnQRURydMsVi+l38OE7n+XeZ1+nsydcV2RUoIuI5Kh+Whn/vvZ9LJ5dyV/+9GUu++ZT/GjTbnpDMvpFgS4iMgbL6lL88L9fwH03vpfqKQlueXArl//NMzy6dT/9/cGefKRAFxEZh/cvnMHDn7+Qez59HlEzbvrhC1x917M8teNQYGeVKtBFRMbJzLji7Nk8fvMH+Pa1yzna0cMf/OMmfv+e52gMYLJpC+qbpKGhwTU2Ngby3iIiE6G7t58HNr3FnU820dzaxaWLq/nSFYs5u6bKt/cws+edcw3Z1uW0h25mq8xsh5k1mdn6LOvNzO701m81sxX5Fi0iUmhKYxE+/b55PP3lldy66ixeeOsoV935LOt++AK7mtsm/P1HDXQziwJ3AauBpcD1ZrY0o9lqYJF3WwN8z+c6RUQKRllpjD9euYBnbrmUdZcu5MlXDvE7//sZ1j+4lX1HJ24Mey576OcDTc65Xc65buB+4JqMNtcA/+zSngNSZjbH51pFRApKVbKEL12xmKe/fCmfvuAMHnphLyu/+RTf/9XEnG2aS6DXAoOvSLPHWzbWNpjZGjNrNLPG5ubmsdYqIlKQqivjfO3qs3nyS5fw0XNrqJtaNiHvE8uhjWVZlnkkNZc2OOc2ABsgfVA0h/cWESkadVPLuP0Tyyfs7+eyh74HqB/0vA7YN442IiIygXIJ9E3AIjM708xKgeuARzLaPALc4I12uQBocc7t97lWEREZwahdLs65XjNbBzwBRIF7nXPbzGytt/5uYCNwJdAEdACfnbiSRUQkm1z60HHObSQd2oOX3T3osQNu8rc0EREZC536LyJSJBToIiJFQoEuIlIkFOgiIkUisKstmlkz8OY4Xz4DOOxjOX4Le30Q/hpVX35UX37CXN8ZzrnqbCsCC/R8mFnjcJePDIOw1wfhr1H15Uf15Sfs9Q1HXS4iIkVCgS4iUiQKNdA3BF3AKMJeH4S/RtWXH9WXn7DXl1VB9qGLiMipCnUPXUREMijQRUSKRKgDPcyTU5tZvZn90sy2m9k2M/tCljYrzazFzDZ7t6+ervq893/DzF703rsxy/ogt9/iQdtls5kdM7ObM9qc9u1nZvea2SEze2nQsmlm9n/N7DXvfuowrx3x8zqB9d1hZq94/w1/bGapYV474udhAuv7mpntHfTf8cphXhvU9ntgUG1vmNnmYV474dsvb865UN5IX6p3JzAfKAW2AEsz2lwJPEZ6xqQLgN+exvrmACu8x5XAq1nqWwn8NMBt+AYwY4T1gW2/LP+tD5A+YSLQ7Qd8AFgBvDRo2e3Aeu/xeuC2Yf4NI35eJ7C+y4GY9/i2bPXl8nmYwPq+Bnwph89AINsvY/23gK8Gtf3yvYV5Dz3Uk1M75/Y7517wHrcC28kyj2rIhWVy7w8CO51z4z1z2DfOuWeAIxmLrwH+yXv8T8BHs7w0l8/rhNTnnPuZc67Xe/oc6RnDAjHM9stFYNtvgJkZcC3wr36/7+kS5kD3bXLqiWZm84D/Bvw2y+r3mdkWM3vMzM4+vZXhgJ+Z2fNmtibL+lBsP9KzYA33P1GQ22/ALOfNwOXdz8zSJizb8g9J/+rKZrTPw0Ra53UJ3TtMl1UYtt/FwEHn3GvDrA9y++UkzIHu2+TUE8nMKoAHgZudc8cyVr9AuhthOfAd4OHTWRvwfufcCmA1cJOZfSBjfRi2XylwNfBvWVYHvf3GIgzb8s+AXuC+YZqM9nmYKN8DFgDnAvtJd2tkCnz7Adcz8t55UNsvZ2EO9NBPTm1mJaTD/D7n3EOZ651zx5xzbd7jjUCJmc04XfU55/Z594eAH5P+WTtYGCb3Xg284Jw7mLki6O03yMGBrijv/lCWNkF/Fj8DfBj4lPM6fDPl8HmYEM65g865PudcP/D3w7xv0NsvBvwu8MBwbYLafmMR5kAP9eTUXn/bPwDbnXPfHqbNbK8dZnY+6e399mmqr9zMKgcekz5w9lJGszBM7j3sXlGQ2y/DI8BnvMefAX6SpU0un9cJYWargFuBq51zHcO0yeXzMFH1DT4u87Fh3jew7ef5EPCKc25PtpVBbr8xCfqo7Eg30qMwXiV99PvPvGVrgbXeYwPu8ta/CDScxtouIv2TcCuw2btdmVHfOmAb6SP2zwEXnsb65nvvu8WrIVTbz3v/MtIBXTVoWaDbj/SXy36gh/Re4+eA6cAvgNe8+2le2xpg40if19NUXxPp/ueBz+HdmfUN93k4TfX9i/f52ko6pOeEaft5y38w8Lkb1Pa0b798bzr1X0SkSIS5y0VERMZAgS4iUiQU6CIiRUKBLiJSJBToIiJFQoEuIlIkFOgiIkXi/wOZ+Zx8bp92PQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.DataFrame(thres)[1].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_counts = feature_counts.sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1752    1430\n",
       "595     1333\n",
       "1751    1209\n",
       "514     1203\n",
       "513     1173\n",
       "        ... \n",
       "587        1\n",
       "554        1\n",
       "766        1\n",
       "813        1\n",
       "758        1\n",
       "Length: 148, dtype: int64"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_counts[feature_counts>0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAOkklEQVR4nO3df6zd9V3H8efbVia0rrB1u86W2GIJSUMTWW/GUGNunRllrLAsS6RhSpXR+AeJPzBagjHzDyOoqIGhy3UgqJUbgtug0IUZpdk/OKE41iJjK9CNC4wy0euKRFZ9+8f5Es4u997ee37cc86b5yO5uef7Oef7Pa9ze8+r537O93y/kZlIkmr5gUEHkCT1nuUuSQVZ7pJUkOUuSQVZ7pJU0MpBBwBYu3ZtbtiwoaN1X3nlFVatWtXbQH0yKllHJSeMTtZRyQlm7Yd+5Tx48OB3MvNdc16ZmQP/2rp1a3bqwQcf7Hjd5TYqWUclZ+boZB2VnJlm7Yd+5QQeyXl61WkZSSrIcpekgix3SSrIcpekgix3SSrIcpekgix3SSrIcpekgobiE6rdOPTcDLv23N/TbR69/uKebk+Slpuv3CWpIMtdkgqy3CWpIMtdkgqy3CWpIMtdkgrqS7lHxEci4i8j4p6I+GA/7kOSNL9Fl3tE3BYRxyLi8Kzx7RHxZEQciYg9AJn5+cy8CtgF/HxPE0uSTmopr9xvB7a3D0TECuAW4CJgM7AzIja33eR3muslScto0eWemV8CXp41/D7gSGY+nZmvAVPApdFyA/CFzHy0d3ElSYsRrdPwLfLGERuA+zLz3Gb5Y8D2zPxEs/wLwPnA14ErgIeBr2Tmp+fY1m5gN8DY2NjWqampjh7AsZdnePHVjlad15Z1a3q7wcbx48dZvXp1X7bdS6OSE0Yn66jkBLP2Q79ybtu27WBmjs91XbfHlok5xjIzbwJuWmjFzJwEJgHGx8dzYmKiowA3772HGw/19hA5Ry/vLMvJHDhwgE4f53IalZwwOllHJSeYtR8GkbPbvWWmgTPbltcDz3e5TUlSl7ot94eBsyNiY0ScAlwG3Nt9LElSN5ayK+SdwEPAORExHRFXZuYJ4GrgAeAJ4K7MfHwJ29wREZMzMzNLzS1JWsCiJ6szc+c84/uB/Z3ceWbuA/aNj49f1cn6kqS5efgBSSrIcpekgix3SSpooOXuG6qS1B8DLffM3JeZu9es6c8nQiXprcppGUkqyHKXpIIsd0kqyDdUJakg31CVpIKclpGkgix3SSrIcpekgix3SSrIvWUkqSD3lpGkgpyWkaSCLHdJKshyl6SCLHdJKshyl6SCLHdJKsj93CWpIPdzl6SCnJaRpIIsd0kqyHKXpIIsd0kqyHKXpIIsd0kqyHKXpIL8EJMkFeSHmCSpIKdlJKkgy12SCrLcJakgy12SCrLcJakgy12SCrLcJakgy12SCrLcJakgy12SCvLYMpJUkMeWkaSCnJaRpIIsd0kqyHKXpIIsd0kqyHKXpIIsd0kqyHKXpIIsd0kqyHKXpIIsd0kqyHKXpIIsd0kqyHKXpIIsd0kqyOO5S1JBHs9dkgpyWkaSCrLcJakgy12SCrLcJakgy12SCrLcJakgy12SCrLcJakgy12SCrLcJakgy12SCrLcJakgy12SCrLcJakgy12SCrLcJakgy12SCrLcJakgy12SCrLcJakgy12SCrLcJakgy12SCup5uUfEWRFxa0Tc3ettS5IWZ1HlHhG3RcSxiDg8a3x7RDwZEUciYg9AZj6dmVf2I6wkaXEW+8r9dmB7+0BErABuAS4CNgM7I2JzT9NJkjoSmbm4G0ZsAO7LzHOb5QuAT2bmhc3ytQCZ+QfN8t2Z+bEFtrcb2A0wNja2dWpqqqMHcOzlGV58taNV57Vl3ZrebrBx/PhxVq9e3Zdt99Ko5ITRyToqOcGs/dCvnNu2bTuYmeNzXbeyi+2uA55tW54Gzo+IdwK/D5wXEde+XvazZeYkMAkwPj6eExMTHYW4ee893Hiom4fxZkcv7yzLyRw4cIBOH+dyGpWcMDpZRyUnmLUfBpGzm1aMOcYyM/8d+JUutitJ6lI3e8tMA2e2La8Hnu8ujiSpF7op94eBsyNiY0ScAlwG3NubWJKkbix2V8g7gYeAcyJiOiKuzMwTwNXAA8ATwF2Z+fhS7jwidkTE5MzMzFJzS5IWsKg598zcOc/4fmB/p3eemfuAfePj41d1ug1J0pt5+AFJKshyl6SCBlruzrlLUn8MtNwzc19m7l6zpj+fCJWktyqnZSSpIMtdkgqy3CWpIMtdkgrq7eEUlygidgA7Nm3aNMgYb7Jhz/093d7R6y/u6fYk6WTcW0aSCnJaRpIKstwlqSDLXZIKstwlqSD3llkGr+99c82WE+zqwZ447n0j6WTcW0aSCnJaRpIKstwlqSDLXZIKstwlqSDLXZIK8jR7klSQu0JKUkFOy0hSQZa7JBVkuUtSQZa7JBVkuUtSQZa7JBVkuUtSQZa7JBXkJ1QlqSA/oSpJBTktI0kFWe6SVJDlLkkFWe6SVJDlLkkFWe6SVJDlLkkFWe6SVJDlLkkFrRzknUfEDmDHpk2bBhlD0pDYsOf+Jd3+mi0n2LXAOkevv7jbSCPLww9IUkFOy0hSQZa7JBVkuUtSQZa7JBVkuUtSQZa7JBVkuUtSQZa7JBVkuUtSQZa7JBVkuUtSQZa7JBVkuUtSQZa7JBVkuUtSQQMt94jYERGTMzMzg4whSeV4sg5JKshpGUkqyHKXpIIsd0kqyHKXpIIsd0kqyHKXpIIsd0kqyHKXpIIsd0kqyHKXpIIsd0kqyHKXpIIsd0kqyHKXpIIsd0kqyHKXpIIsd0kqyHKXpIIsd0kqyHKXpIIsd0kqyHKXpIIsd0kqaGWvNxgRq4A/B14DDmTm3l7fhyRpYYt65R4Rt0XEsYg4PGt8e0Q8GRFHImJPM/xR4O7MvAq4pMd5JUmLsNhpmduB7e0DEbECuAW4CNgM7IyIzcB64NnmZv/bm5iSpKWIzFzcDSM2APdl5rnN8gXAJzPzwmb52uam08B/ZOZ9ETGVmZfNs73dwG6AsbGxrVNTUx09gGMvz/Diqx2tuuzGTqUnWbesW9P9Rtocem7m+5Z7kbPXGedz/PhxVq9evSz31Y1RyQmDzTr7d/FkevWc6reFcnbzXNm2bdvBzByf67pu5tzX8cYrdGiV+vnATcCnIuJiYN98K2fmJDAJMD4+nhMTEx2FuHnvPdx4qOdvHfTFNVtO9CTr0csnug/TZtee+79vuRc5e51xPgcOHKDT353lNCo5YbBZZ/8unkyvnlP9tlDOfj1XuvmpxBxjmZmvAL/UxXYlSV3qZlfIaeDMtuX1wPPdxZEk9UI35f4wcHZEbIyIU4DLgHuXsoGI2BERkzMzS5tnkyQtbLG7Qt4JPAScExHTEXFlZp4ArgYeAJ4A7srMx5dy55m5LzN3r1mzPG++SdJbxaLm3DNz5zzj+4H9PU0kSeqahx+QpIIsd0kqaKDl7huqktQfi/6Eal9DRLwEfLPD1dcC3+lhnH4alayjkhNGJ+uo5ASz9kO/cv5YZr5rriuGoty7ERGPzPfx22EzKllHJSeMTtZRyQlm7YdB5HTOXZIKstwlqaAK5T456ABLMCpZRyUnjE7WUckJZu2HZc858nPukqQ3q/DKXZI0i+UuSQWNbLnPc/7WQeY5MyIejIgnIuLxiPjVZvwdEfEPEfGN5vsZbetc2+R/MiIuXOa8KyLiXyPiviHPeXpE3B0RX2t+thcMY9aI+PXm3/1wRNwZET80LDnnOgdyJ9kiYmtEHGquuyki5jqnQz+y/lHz7//ViPhcRJw+rFnbrvvNiMiIWDuwrJk5cl/ACuAp4CzgFOAxYPOAM70HeG9z+YeBr9M6t+wfAnua8T3ADc3lzU3utwEbm8ezYhnz/gbwd7ROncgQ57wD+ERz+RTg9GHLSuusZM8ApzbLdwG7hiUn8DPAe4HDbWNLzgb8C3ABrRP1fAG4aJmyfhBY2Vy+YZizNuNn0jpa7jeBtYPKOqqv3N8HHMnMpzPzNWAKuHSQgTLzhcx8tLn8XVqHQV7X5LqjudkdwEeay5cCU5n5P5n5DHCE1uPqu4hYD1wMfKZteBhzvp3WE+hWgMx8LTP/cxiz0jrC6qkRsRI4jdaJa4YiZ2Z+CXh51vCSskXEe4C3Z+ZD2Wqkv25bp69ZM/OL2TrEOMA/0zox0FBmbfwp8FtA+94qy551VMt9rvO3rhtQljeJ1snEzwO+DIxl5gvQ+g8AeHdzs0E+hj+j9cv3f21jw5jzLOAl4K+aKaTPRMSqYcuamc8Bfwx8C3gBmMnMLw5bzlmWmm1dc3n2+HL7ZVqvbmEIs0bEJcBzmfnYrKuWPeuolvuc529d9hRziIjVwN8Dv5aZ/7XQTecY6/tjiIgPA8cy8+BiV5ljbLl+1itp/dn7F5l5HvAKrSmE+QzqZ3oGrVdmG4EfBVZFxMcXWmWOsaH4/WX+bAPPHBHXASeAva8PzXGzgWWNiNOA64DfnevqOcb6mnVUy30oz98aET9Iq9j3ZuZnm+EXmz+9aL4fa8YH9Rh+CrgkIo7Sms762Yj42yHM+fp9T2fml5vlu2mV/bBl/Tngmcx8KTO/B3wW+MkhzNluqdmmeWM6pH18WUTEFcCHgcub6QsYvqw/Tus/+Mea59d64NGI+JFBZB3Vcu/6/K291rzDfSvwRGb+SdtV9wJXNJevAO5pG78sIt4WERuBs2m9sdJXmXltZq7PzA20fm7/lJkfH7acTdZvA89GxDnN0AeAfxvCrN8C3h8RpzW/Bx+g9Z7LsOVst6RszdTNdyPi/c1j/MW2dfoqIrYDvw1ckpn/PesxDE3WzDyUme/OzA3N82ua1k4W3x5I1l6/g7xcX8CHaO2R8hRw3RDk+Wlaf059FfhK8/Uh4J3APwLfaL6/o22d65r8T9KHd/MXkXmCN/aWGcqcwE8AjzQ/188DZwxjVuD3gK8Bh4G/obVXxFDkBO6k9V7A92gVzpWdZAPGm8f3FPApmk+4L0PWI7Tmq19/Xn16WLPOuv4ozd4yg8jq4QckqaBRnZaRJC3Acpekgix3SSrIcpekgix3SSrIcpekgix3SSro/wFeCwPROh6NxwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "feature_counts[feature_counts>0].hist(log=True, bins=15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TEM_links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.loader import NeighborLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "#path = osp.join(os.getcwd(), '../../data/VEN')\n",
    "#transform = T.Compose([T.NormalizeFeatures(), T.ToSparseTensor()])\n",
    "dataset = VEN_tem('dataset/Venice_tem')\n",
    "data = dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.train_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.val_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.test_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 24.38it/s]\n"
     ]
    }
   ],
   "source": [
    "test_loss_att, test_loss_val, test_att_acc, test_val_acc, test_val_jac, test_val_1 = test_Homo(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 33.17it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7332217283856506,\n",
       " 1.619803889636518,\n",
       " 99.7229916897507,\n",
       " 99.7229916897507,\n",
       " 0.7807017667141648,\n",
       " 80.88642659279779)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 23.92it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8056002129626468,\n",
       " 1.6308585041262247,\n",
       " 95.1219512195122,\n",
       " 99.01477832512315,\n",
       " 0.7454844061377013,\n",
       " 78.81773399014779)"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 24.48it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8034103550683434,\n",
       " 1.6558503558238347,\n",
       " 96.02385685884691,\n",
       " 99.47916666666667,\n",
       " 0.7256944527228674,\n",
       " 77.60416666666667)"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.44it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.83it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 40.79it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.73it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 39.48it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.48it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 40.62it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.61it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 40.80it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.84it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 40.09it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.88it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 39.85it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.73it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 40.24it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.90it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 40.45it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.09it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 40.99it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.06it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test_Homo(model, val_loader))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test_Homo(model, test_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.805953</td>\n",
       "      <td>1.631010</td>\n",
       "      <td>95.020325</td>\n",
       "      <td>9.901478e+01</td>\n",
       "      <td>0.749836</td>\n",
       "      <td>79.064039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000287</td>\n",
       "      <td>0.000073</td>\n",
       "      <td>0.172731</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.001898</td>\n",
       "      <td>0.478732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.805414</td>\n",
       "      <td>1.630935</td>\n",
       "      <td>94.715447</td>\n",
       "      <td>9.901478e+01</td>\n",
       "      <td>0.747126</td>\n",
       "      <td>78.325123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.805793</td>\n",
       "      <td>1.630958</td>\n",
       "      <td>94.918699</td>\n",
       "      <td>9.901478e+01</td>\n",
       "      <td>0.748768</td>\n",
       "      <td>78.817734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.805968</td>\n",
       "      <td>1.630979</td>\n",
       "      <td>95.020325</td>\n",
       "      <td>9.901478e+01</td>\n",
       "      <td>0.749589</td>\n",
       "      <td>78.817734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.806075</td>\n",
       "      <td>1.631077</td>\n",
       "      <td>95.121951</td>\n",
       "      <td>9.901478e+01</td>\n",
       "      <td>0.751026</td>\n",
       "      <td>79.310345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.806440</td>\n",
       "      <td>1.631118</td>\n",
       "      <td>95.325203</td>\n",
       "      <td>9.901478e+01</td>\n",
       "      <td>0.753695</td>\n",
       "      <td>79.802956</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc     VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  1.000000e+01  10.000000  10.000000\n",
       "mean    0.805953   1.631010  95.020325  9.901478e+01   0.749836  79.064039\n",
       "std     0.000287   0.000073   0.172731  1.497956e-14   0.001898   0.478732\n",
       "min     0.805414   1.630935  94.715447  9.901478e+01   0.747126  78.325123\n",
       "25%     0.805793   1.630958  94.918699  9.901478e+01   0.748768  78.817734\n",
       "50%     0.805968   1.630979  95.020325  9.901478e+01   0.749589  78.817734\n",
       "75%     0.806075   1.631077  95.121951  9.901478e+01   0.751026  79.310345\n",
       "max     0.806440   1.631118  95.325203  9.901478e+01   0.753695  79.802956"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.803585</td>\n",
       "      <td>1.655908</td>\n",
       "      <td>96.143141</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.730556</td>\n",
       "      <td>77.604167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000256</td>\n",
       "      <td>0.000158</td>\n",
       "      <td>0.167649</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.001793</td>\n",
       "      <td>0.245523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.803203</td>\n",
       "      <td>1.655547</td>\n",
       "      <td>95.825050</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.727431</td>\n",
       "      <td>77.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.803394</td>\n",
       "      <td>1.655856</td>\n",
       "      <td>96.023857</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.730035</td>\n",
       "      <td>77.604167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.803593</td>\n",
       "      <td>1.655970</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.730903</td>\n",
       "      <td>77.604167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.803733</td>\n",
       "      <td>1.655993</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.730903</td>\n",
       "      <td>77.604167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.804032</td>\n",
       "      <td>1.656063</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.733507</td>\n",
       "      <td>78.125000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc     VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  1.000000e+01  10.000000  10.000000\n",
       "mean    0.803585   1.655908  96.143141  9.947917e+01   0.730556  77.604167\n",
       "std     0.000256   0.000158   0.167649  1.497956e-14   0.001793   0.245523\n",
       "min     0.803203   1.655547  95.825050  9.947917e+01   0.727431  77.083333\n",
       "25%     0.803394   1.655856  96.023857  9.947917e+01   0.730035  77.604167\n",
       "50%     0.803593   1.655970  96.222664  9.947917e+01   0.730903  77.604167\n",
       "75%     0.803733   1.655993  96.222664  9.947917e+01   0.730903  77.604167\n",
       "max     0.804032   1.656063  96.421471  9.947917e+01   0.733507  78.125000"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'TEM_val_metrics_transfer.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'TEM_test_metrics_transfer.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SPA_links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.loader import NeighborLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "#path = osp.join(os.getcwd(), '../../data/VEN')\n",
    "#transform = T.Compose([T.NormalizeFeatures(), T.ToSparseTensor()])\n",
    "dataset = VEN_spa('dataset/Venice_spa')\n",
    "data = dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.train_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.val_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.test_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.05it/s]\n"
     ]
    }
   ],
   "source": [
    "test_loss_att, test_loss_val, test_att_acc, test_val_acc, test_val_jac, test_val_1 = test_Homo(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 23.03it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7324363260718264,\n",
       " 1.6200073138498534,\n",
       " 100.0,\n",
       " 99.44598337950139,\n",
       " 0.7765466367769109,\n",
       " 80.33240997229917)"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 18.02it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7985899805780349,\n",
       " 1.6324272866319554,\n",
       " 95.52845528455285,\n",
       " 99.50738916256158,\n",
       " 0.7257799726401644,\n",
       " 78.32512315270937)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.82it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8011263183523599,\n",
       " 1.6576240280022223,\n",
       " 96.62027833001989,\n",
       " 99.47916666666667,\n",
       " 0.7213541716337204,\n",
       " 76.5625)"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 17.10it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.46it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.24it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 20.98it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.27it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.42it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.44it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.32it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.14it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.32it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.12it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.60it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.37it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.36it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.01it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.74it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.48it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.61it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.63it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.72it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test_Homo(model, val_loader))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test_Homo(model, test_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.798616</td>\n",
       "      <td>1.632383</td>\n",
       "      <td>95.487805</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>0.731691</td>\n",
       "      <td>78.275862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000227</td>\n",
       "      <td>0.000099</td>\n",
       "      <td>0.085699</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005712</td>\n",
       "      <td>0.363480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.798338</td>\n",
       "      <td>1.632231</td>\n",
       "      <td>95.325203</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>0.723317</td>\n",
       "      <td>77.832512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.798447</td>\n",
       "      <td>1.632329</td>\n",
       "      <td>95.528455</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>0.728243</td>\n",
       "      <td>77.955665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.798598</td>\n",
       "      <td>1.632383</td>\n",
       "      <td>95.528455</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>0.730706</td>\n",
       "      <td>78.325123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.798689</td>\n",
       "      <td>1.632429</td>\n",
       "      <td>95.528455</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>0.735632</td>\n",
       "      <td>78.325123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.799110</td>\n",
       "      <td>1.632544</td>\n",
       "      <td>95.528455</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>0.740558</td>\n",
       "      <td>78.817734</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    0.798616   1.632383  95.487805  99.507389   0.731691  78.275862\n",
       "std     0.000227   0.000099   0.085699   0.000000   0.005712   0.363480\n",
       "min     0.798338   1.632231  95.325203  99.507389   0.723317  77.832512\n",
       "25%     0.798447   1.632329  95.528455  99.507389   0.728243  77.955665\n",
       "50%     0.798598   1.632383  95.528455  99.507389   0.730706  78.325123\n",
       "75%     0.798689   1.632429  95.528455  99.507389   0.735632  78.325123\n",
       "max     0.799110   1.632544  95.528455  99.507389   0.740558  78.817734"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.801399</td>\n",
       "      <td>1.657411</td>\n",
       "      <td>96.580517</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.721094</td>\n",
       "      <td>77.135417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000255</td>\n",
       "      <td>0.000218</td>\n",
       "      <td>0.156821</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.002590</td>\n",
       "      <td>0.456039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.801039</td>\n",
       "      <td>1.657132</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.716146</td>\n",
       "      <td>76.562500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.801211</td>\n",
       "      <td>1.657247</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.719401</td>\n",
       "      <td>77.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.801398</td>\n",
       "      <td>1.657386</td>\n",
       "      <td>96.620278</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.721354</td>\n",
       "      <td>77.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.801501</td>\n",
       "      <td>1.657514</td>\n",
       "      <td>96.620278</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.723307</td>\n",
       "      <td>77.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.801903</td>\n",
       "      <td>1.657799</td>\n",
       "      <td>96.819085</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.723958</td>\n",
       "      <td>78.125000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc     VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  1.000000e+01  10.000000  10.000000\n",
       "mean    0.801399   1.657411  96.580517  9.947917e+01   0.721094  77.135417\n",
       "std     0.000255   0.000218   0.156821  1.497956e-14   0.002590   0.456039\n",
       "min     0.801039   1.657132  96.421471  9.947917e+01   0.716146  76.562500\n",
       "25%     0.801211   1.657247  96.421471  9.947917e+01   0.719401  77.083333\n",
       "50%     0.801398   1.657386  96.620278  9.947917e+01   0.721354  77.083333\n",
       "75%     0.801501   1.657514  96.620278  9.947917e+01   0.723307  77.083333\n",
       "max     0.801903   1.657799  96.819085  9.947917e+01   0.723958  78.125000"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'SPA_val_metrics_transfer.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'SPA_test_metrics_transfer.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SOC_links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.loader import NeighborLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "#path = osp.join(os.getcwd(), '../../data/VEN')\n",
    "#transform = T.Compose([T.NormalizeFeatures(), T.ToSparseTensor()])\n",
    "dataset = VEN_soc('dataset/Venice_soc')\n",
    "data = dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.train_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.val_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.test_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.53it/s]\n"
     ]
    }
   ],
   "source": [
    "test_loss_att, test_loss_val, test_att_acc, test_val_acc, test_val_jac, test_val_1 = test_Homo(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 26.79it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.738962907375061,\n",
       " 1.6204001081287036,\n",
       " 99.7229916897507,\n",
       " 99.44598337950139,\n",
       " 0.7894736894940405,\n",
       " 80.33240997229917)"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.83it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8110466261462468,\n",
       " 1.6310881370394101,\n",
       " 94.71544715447155,\n",
       " 98.52216748768473,\n",
       " 0.7619047681686326,\n",
       " 79.3103448275862)"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 23.08it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8238236066361307,\n",
       " 1.6575363489488761,\n",
       " 95.42743538767395,\n",
       " 99.47916666666667,\n",
       " 0.732638897995154,\n",
       " 78.125)"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 20.51it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 28.09it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.76it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.23it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.93it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.34it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.86it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.18it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.45it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 28.22it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.11it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.47it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.84it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.53it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.67it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.60it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.23it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.59it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.45it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.25it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test_Homo(model, val_loader))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test_Homo(model, test_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.811307</td>\n",
       "      <td>1.631118</td>\n",
       "      <td>94.898374</td>\n",
       "      <td>9.852217e+01</td>\n",
       "      <td>0.760591</td>\n",
       "      <td>79.261084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000181</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.177966</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.003816</td>\n",
       "      <td>0.489866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.811035</td>\n",
       "      <td>1.631001</td>\n",
       "      <td>94.715447</td>\n",
       "      <td>9.852217e+01</td>\n",
       "      <td>0.753695</td>\n",
       "      <td>78.817734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.811142</td>\n",
       "      <td>1.631049</td>\n",
       "      <td>94.715447</td>\n",
       "      <td>9.852217e+01</td>\n",
       "      <td>0.758621</td>\n",
       "      <td>78.817734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.811354</td>\n",
       "      <td>1.631105</td>\n",
       "      <td>94.918699</td>\n",
       "      <td>9.852217e+01</td>\n",
       "      <td>0.761084</td>\n",
       "      <td>79.310345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.811472</td>\n",
       "      <td>1.631200</td>\n",
       "      <td>95.071138</td>\n",
       "      <td>9.852217e+01</td>\n",
       "      <td>0.761084</td>\n",
       "      <td>79.310345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.811508</td>\n",
       "      <td>1.631229</td>\n",
       "      <td>95.121951</td>\n",
       "      <td>9.852217e+01</td>\n",
       "      <td>0.768473</td>\n",
       "      <td>80.295567</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc     VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  1.000000e+01  10.000000  10.000000\n",
       "mean    0.811307   1.631118  94.898374  9.852217e+01   0.760591  79.261084\n",
       "std     0.000181   0.000084   0.177966  1.497956e-14   0.003816   0.489866\n",
       "min     0.811035   1.631001  94.715447  9.852217e+01   0.753695  78.817734\n",
       "25%     0.811142   1.631049  94.715447  9.852217e+01   0.758621  78.817734\n",
       "50%     0.811354   1.631105  94.918699  9.852217e+01   0.761084  79.310345\n",
       "75%     0.811472   1.631200  95.071138  9.852217e+01   0.761084  79.310345\n",
       "max     0.811508   1.631229  95.121951  9.852217e+01   0.768473  80.295567"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.823987</td>\n",
       "      <td>1.657562</td>\n",
       "      <td>95.447316</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.732118</td>\n",
       "      <td>78.177083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000367</td>\n",
       "      <td>0.000188</td>\n",
       "      <td>0.255802</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.002496</td>\n",
       "      <td>0.384305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.823404</td>\n",
       "      <td>1.657290</td>\n",
       "      <td>95.029821</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.727431</td>\n",
       "      <td>77.604167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.823846</td>\n",
       "      <td>1.657448</td>\n",
       "      <td>95.228628</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.730903</td>\n",
       "      <td>78.125000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.824042</td>\n",
       "      <td>1.657567</td>\n",
       "      <td>95.526839</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.732639</td>\n",
       "      <td>78.125000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.824139</td>\n",
       "      <td>1.657680</td>\n",
       "      <td>95.626243</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.732639</td>\n",
       "      <td>78.515625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.824624</td>\n",
       "      <td>1.657861</td>\n",
       "      <td>95.825050</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.736111</td>\n",
       "      <td>78.645833</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc     VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  1.000000e+01  10.000000  10.000000\n",
       "mean    0.823987   1.657562  95.447316  9.947917e+01   0.732118  78.177083\n",
       "std     0.000367   0.000188   0.255802  1.497956e-14   0.002496   0.384305\n",
       "min     0.823404   1.657290  95.029821  9.947917e+01   0.727431  77.604167\n",
       "25%     0.823846   1.657448  95.228628  9.947917e+01   0.730903  78.125000\n",
       "50%     0.824042   1.657567  95.526839  9.947917e+01   0.732639  78.125000\n",
       "75%     0.824139   1.657680  95.626243  9.947917e+01   0.732639  78.515625\n",
       "max     0.824624   1.657861  95.825050  9.947917e+01   0.736111  78.645833"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'SOC_val_metrics_transfer.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'SOC_test_metrics_transfer.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Masking Visual and Textual Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SAGE(in_channels=data.x.shape[-1], hidden_channels = 512, \n",
    "            out_channels = data.y.shape[-1], dropout = 0.1, num_layers=5).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load(args.save_dir+'SAGE_best_model/model.pth',map_location=torch.device('cpu')))\n",
    "model.eval()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SAGE(\n",
       "  (convs): ModuleList(\n",
       "    (0): SAGEConv(1753, 512)\n",
       "    (1): SAGEConv(512, 512)\n",
       "    (2): SAGEConv(512, 512)\n",
       "    (3): SAGEConv(512, 512)\n",
       "    (4): SAGEConv(512, 20)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def test_Mask(model, loader, mask = None):\n",
    "    model.eval()\n",
    "\n",
    "    total_examples_att = total_examples_val = 0\n",
    "    running_loss_1 = running_loss_2 = 0.\n",
    "    running_1_acc = 0.\n",
    "    running_1_val = 0.\n",
    "    running_k_acc = 0.\n",
    "    running_k_jac = 0.\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        loss_1 = 0\n",
    "        acc_1_t = 0\n",
    "        loss_2 = 0\n",
    "        acc_1_val = 0\n",
    "        acc_k_t = 0\n",
    "        jac_k_t = 0\n",
    "\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch.batch_size\n",
    "        edge_index = to_undirected(batch.edge_index)\n",
    "        \n",
    "        if mask == 'vis':\n",
    "            batch.x[:batch_size,:982] = torch.zeros(batch_size,982)\n",
    "        elif mask == 'tex':\n",
    "            batch.x[:batch_size,982:] = torch.zeros(batch_size,771)\n",
    "        \n",
    "        out = model(batch.x, edge_index)[:batch_size]\n",
    "        out_att = out[:,:9]\n",
    "        out_val = out[:,9:]\n",
    "        att_node = (batch.att_lab[:batch_size]).nonzero().squeeze()\n",
    "        val_node = (batch.val_lab[:batch_size]).nonzero().squeeze()\n",
    "\n",
    "        #print(type_node)\n",
    "\n",
    "        #pred_att = out_att.argmax(dim=-1)\n",
    "        #pred_val = out_val.argmax(dim=-1)\n",
    "\n",
    "        y = batch.y\n",
    "        y_att = y[:,:9]\n",
    "        y_val = y[:,9:]\n",
    "\n",
    "        if not att_node.shape[0]==0:\n",
    "            loss_1 = F.cross_entropy(out_att[att_node], y_att[:batch_size][att_node])\n",
    "            acc_1_t = compute_1_accuracy(y_att[:batch_size][att_node], out_att[att_node])\n",
    "\n",
    "        if not val_node.shape[0]==0:\n",
    "            loss_2 = F.cross_entropy(out_val[val_node], y_val[val_node])\n",
    "            acc_1_val = compute_1_accuracy(y_val[val_node], out_val[val_node])\n",
    "            acc_k_t = compute_k_accuracy(y_val[val_node], out_val[val_node], args.k)\n",
    "            jac_k_t = compute_jaccard_index(y_val[val_node], F.softmax(out_val[val_node],dim=-1), args.k)\n",
    "            #loss_3 = loss_1 + loss_2\n",
    "\n",
    "        total_examples_att += att_node.shape[0]\n",
    "        total_examples_val += val_node.shape[0]\n",
    "        #total_correct_att += int((pred_att == y_att[:batch_size]).sum())\n",
    "        #total_correct_val += int((pred_val == y_val[:batch_size]).sum())\n",
    "\n",
    "        running_loss_1 += float(loss_1) * att_node.shape[0]\n",
    "        running_loss_2 += float(loss_2) * val_node.shape[0]\n",
    "        running_1_acc += float(acc_1_t) * att_node.shape[0]\n",
    "        running_1_val += float(acc_1_val) * val_node.shape[0]\n",
    "        running_k_acc += float(acc_k_t) * val_node.shape[0]\n",
    "        running_k_jac += float(jac_k_t) * val_node.shape[0]\n",
    "    \n",
    "    return running_loss_1/total_examples_att, running_loss_2/total_examples_val, running_1_acc/ total_examples_att, running_k_acc/ total_examples_val, running_k_jac/ total_examples_val, running_1_val/total_examples_val\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:01<00:00,  6.92it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2.171622239981992,\n",
       " 1.632508054334371,\n",
       " 21.606648199445985,\n",
       " 100.0,\n",
       " 0.7543859719569663,\n",
       " 78.94736842105263)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, train_loader, 'vis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 15.36it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7405485812977081,\n",
       " 1.7726872670683504,\n",
       " 100.0,\n",
       " 81.16343490304709,\n",
       " 0.10803324099722991,\n",
       " 32.686980609418285)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, train_loader, 'tex')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.57it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2.271485351450075,\n",
       " 1.630924281228352,\n",
       " 22.357723577235774,\n",
       " 100.0,\n",
       " 0.7758620783613233,\n",
       " 78.81773399014779)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, val_loader, 'vis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 17.13it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7868161172401614,\n",
       " 1.8005158537127115,\n",
       " 96.34146341463415,\n",
       " 71.92118226600985,\n",
       " 0.07389162561576355,\n",
       " 18.226600985221676)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, val_loader, 'tex')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.23it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2.267704428782757,\n",
       " 1.6548299727340539,\n",
       " 23.45924453280318,\n",
       " 99.47916666666667,\n",
       " 0.7743055646618208,\n",
       " 77.08333333333333)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, test_loader, 'vis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 13.96it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7934355283115303,\n",
       " 1.8172860238701105,\n",
       " 96.62027833001989,\n",
       " 77.60416666666667,\n",
       " 0.0798611119389534,\n",
       " 21.875)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, test_loader, 'tex')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 15.59it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 17.22it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.61it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 13.12it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.20it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.46it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.23it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.04it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.68it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.58it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.01it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:02<00:00,  9.11it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.60it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.53it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 15.98it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 15.94it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.49it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.27it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.26it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.27it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.60it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.97it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.05it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 15.98it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.45it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.94it/s]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.43it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.32it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.75it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 17.12it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.38it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.11it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.73it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.83it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.44it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.32it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.75it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.99it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.06it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 16.21it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers_vis = []\n",
    "val_numbers_tex = []\n",
    "test_numbers_vis = []\n",
    "test_numbers_tex = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers_vis.append(test_Mask(model, val_loader, 'vis'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers_tex.append(test_Mask(model, val_loader, 'tex'))\n",
    "    \n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers_vis.append(test_Mask(model, test_loader, 'vis'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers_tex.append(test_Mask(model, test_loader, 'tex'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df_vis = pd.DataFrame(val_numbers_vis, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "val_df_tex = pd.DataFrame(val_numbers_tex, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df_vis = pd.DataFrame(test_numbers_vis, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df_tex = pd.DataFrame(test_numbers_tex, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.273715</td>\n",
       "      <td>1.631051</td>\n",
       "      <td>22.337398</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.777915</td>\n",
       "      <td>79.261084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.001810</td>\n",
       "      <td>0.000224</td>\n",
       "      <td>0.223680</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005035</td>\n",
       "      <td>0.431328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.269818</td>\n",
       "      <td>1.630612</td>\n",
       "      <td>21.951220</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.769294</td>\n",
       "      <td>78.817734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.273107</td>\n",
       "      <td>1.630950</td>\n",
       "      <td>22.154472</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.774425</td>\n",
       "      <td>78.817734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.273843</td>\n",
       "      <td>1.631037</td>\n",
       "      <td>22.357724</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.777915</td>\n",
       "      <td>79.310345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.274964</td>\n",
       "      <td>1.631199</td>\n",
       "      <td>22.560976</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.781404</td>\n",
       "      <td>79.679803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.275932</td>\n",
       "      <td>1.631404</td>\n",
       "      <td>22.560976</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.785714</td>\n",
       "      <td>79.802956</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000       10.0  10.000000  10.000000\n",
       "mean    2.273715   1.631051  22.337398      100.0   0.777915  79.261084\n",
       "std     0.001810   0.000224   0.223680        0.0   0.005035   0.431328\n",
       "min     2.269818   1.630612  21.951220      100.0   0.769294  78.817734\n",
       "25%     2.273107   1.630950  22.154472      100.0   0.774425  78.817734\n",
       "50%     2.273843   1.631037  22.357724      100.0   0.777915  79.310345\n",
       "75%     2.274964   1.631199  22.560976      100.0   0.781404  79.679803\n",
       "max     2.275932   1.631404  22.560976      100.0   0.785714  79.802956"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df_vis.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.787288</td>\n",
       "      <td>1.800153</td>\n",
       "      <td>96.077236</td>\n",
       "      <td>70.295567</td>\n",
       "      <td>0.077094</td>\n",
       "      <td>19.704433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000459</td>\n",
       "      <td>0.000463</td>\n",
       "      <td>0.254404</td>\n",
       "      <td>0.616584</td>\n",
       "      <td>0.004086</td>\n",
       "      <td>0.985222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.786485</td>\n",
       "      <td>1.799571</td>\n",
       "      <td>95.731707</td>\n",
       "      <td>69.458128</td>\n",
       "      <td>0.068144</td>\n",
       "      <td>17.241379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.787002</td>\n",
       "      <td>1.799766</td>\n",
       "      <td>95.934959</td>\n",
       "      <td>69.950739</td>\n",
       "      <td>0.076355</td>\n",
       "      <td>19.704433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.787295</td>\n",
       "      <td>1.800067</td>\n",
       "      <td>96.138211</td>\n",
       "      <td>70.443350</td>\n",
       "      <td>0.077176</td>\n",
       "      <td>19.704433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.787667</td>\n",
       "      <td>1.800520</td>\n",
       "      <td>96.138211</td>\n",
       "      <td>70.443350</td>\n",
       "      <td>0.078818</td>\n",
       "      <td>20.197044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.787912</td>\n",
       "      <td>1.800971</td>\n",
       "      <td>96.544715</td>\n",
       "      <td>71.428571</td>\n",
       "      <td>0.082923</td>\n",
       "      <td>20.689655</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    0.787288   1.800153  96.077236  70.295567   0.077094  19.704433\n",
       "std     0.000459   0.000463   0.254404   0.616584   0.004086   0.985222\n",
       "min     0.786485   1.799571  95.731707  69.458128   0.068144  17.241379\n",
       "25%     0.787002   1.799766  95.934959  69.950739   0.076355  19.704433\n",
       "50%     0.787295   1.800067  96.138211  70.443350   0.077176  19.704433\n",
       "75%     0.787667   1.800520  96.138211  70.443350   0.078818  20.197044\n",
       "max     0.787912   1.800971  96.544715  71.428571   0.082923  20.689655"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df_tex.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.266135</td>\n",
       "      <td>1.654548</td>\n",
       "      <td>23.618290</td>\n",
       "      <td>99.166667</td>\n",
       "      <td>0.774566</td>\n",
       "      <td>77.656250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.002176</td>\n",
       "      <td>0.000358</td>\n",
       "      <td>0.225704</td>\n",
       "      <td>0.268957</td>\n",
       "      <td>0.002590</td>\n",
       "      <td>0.456039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.262557</td>\n",
       "      <td>1.653949</td>\n",
       "      <td>23.260437</td>\n",
       "      <td>98.958333</td>\n",
       "      <td>0.771701</td>\n",
       "      <td>77.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.264983</td>\n",
       "      <td>1.654285</td>\n",
       "      <td>23.459245</td>\n",
       "      <td>98.958333</td>\n",
       "      <td>0.772352</td>\n",
       "      <td>77.213542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.266050</td>\n",
       "      <td>1.654528</td>\n",
       "      <td>23.658052</td>\n",
       "      <td>98.958333</td>\n",
       "      <td>0.774306</td>\n",
       "      <td>77.604167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.267182</td>\n",
       "      <td>1.654826</td>\n",
       "      <td>23.658052</td>\n",
       "      <td>99.479167</td>\n",
       "      <td>0.776259</td>\n",
       "      <td>78.125000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.270030</td>\n",
       "      <td>1.655055</td>\n",
       "      <td>24.055666</td>\n",
       "      <td>99.479167</td>\n",
       "      <td>0.779514</td>\n",
       "      <td>78.125000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    2.266135   1.654548  23.618290  99.166667   0.774566  77.656250\n",
       "std     0.002176   0.000358   0.225704   0.268957   0.002590   0.456039\n",
       "min     2.262557   1.653949  23.260437  98.958333   0.771701  77.083333\n",
       "25%     2.264983   1.654285  23.459245  98.958333   0.772352  77.213542\n",
       "50%     2.266050   1.654528  23.658052  98.958333   0.774306  77.604167\n",
       "75%     2.267182   1.654826  23.658052  99.479167   0.776259  78.125000\n",
       "max     2.270030   1.655055  24.055666  99.479167   0.779514  78.125000"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df_vis.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.793832</td>\n",
       "      <td>1.817320</td>\n",
       "      <td>96.799205</td>\n",
       "      <td>77.187500</td>\n",
       "      <td>0.080903</td>\n",
       "      <td>22.447917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000429</td>\n",
       "      <td>0.000630</td>\n",
       "      <td>0.112852</td>\n",
       "      <td>0.878410</td>\n",
       "      <td>0.006709</td>\n",
       "      <td>1.376901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.793258</td>\n",
       "      <td>1.816637</td>\n",
       "      <td>96.620278</td>\n",
       "      <td>75.520833</td>\n",
       "      <td>0.067708</td>\n",
       "      <td>19.270833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.793534</td>\n",
       "      <td>1.817036</td>\n",
       "      <td>96.819085</td>\n",
       "      <td>77.083333</td>\n",
       "      <td>0.081163</td>\n",
       "      <td>22.005208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.793744</td>\n",
       "      <td>1.817087</td>\n",
       "      <td>96.819085</td>\n",
       "      <td>77.343750</td>\n",
       "      <td>0.082465</td>\n",
       "      <td>22.656250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.794065</td>\n",
       "      <td>1.817295</td>\n",
       "      <td>96.819085</td>\n",
       "      <td>77.604167</td>\n",
       "      <td>0.084418</td>\n",
       "      <td>23.437500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.794706</td>\n",
       "      <td>1.818721</td>\n",
       "      <td>97.017893</td>\n",
       "      <td>78.645833</td>\n",
       "      <td>0.087674</td>\n",
       "      <td>23.958333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    0.793832   1.817320  96.799205  77.187500   0.080903  22.447917\n",
       "std     0.000429   0.000630   0.112852   0.878410   0.006709   1.376901\n",
       "min     0.793258   1.816637  96.620278  75.520833   0.067708  19.270833\n",
       "25%     0.793534   1.817036  96.819085  77.083333   0.081163  22.005208\n",
       "50%     0.793744   1.817087  96.819085  77.343750   0.082465  22.656250\n",
       "75%     0.794065   1.817295  96.819085  77.604167   0.084418  23.437500\n",
       "max     0.794706   1.818721  97.017893  78.645833   0.087674  23.958333"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df_tex.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df_vis.to_csv(args.save_dir + 'vis_masked_val_metrics.csv', sep='\\t')\n",
    "val_df_tex.to_csv(args.save_dir + 'tex_masked_val_metrics.csv', sep='\\t')\n",
    "test_df_vis.to_csv(args.save_dir + 'vis_masked_test_metrics.csv', sep='\\t')\n",
    "test_df_tex.to_csv(args.save_dir + 'tex_masked_test_metrics.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using kNN Feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialization_knn():\n",
    "    set_seed_everywhere(args.seed, args.cuda)\n",
    "    #transform = T.Compose([T.ToSparseTensor()])\n",
    "    dataset = VEN_Homo('dataset/Venice_homo')\n",
    "    data = dataset[0]\n",
    "    data.n_id = torch.arange(data.num_nodes)\n",
    "    edge_index = knn_graph(data.x[:,:982].to(device), k=3, loop=False, cosine=True)\n",
    "    data.edge_index = edge_index\n",
    "    data = data.to(device)\n",
    "    \n",
    "    train_loader = NeighborLoader(\n",
    "        data,\n",
    "        # Sample 25 neighbors for each node and edge type for 2 iterations\n",
    "        num_neighbors=[3*args.sample_nodes] * 2,\n",
    "        # Use a batch size of 32 for sampling training nodes\n",
    "        batch_size=args.batch_size,\n",
    "        input_nodes=data.train_mask,\n",
    "    )\n",
    "    val_loader = NeighborLoader(\n",
    "        data,\n",
    "        # Sample 25 neighbors for each node and edge type for 2 iterations\n",
    "        num_neighbors=[3*args.sample_nodes] * 2,\n",
    "        # Use a batch size of 32 for sampling validating nodes\n",
    "        batch_size=args.batch_size,\n",
    "        input_nodes=data.val_mask,\n",
    "    )\n",
    "    test_loader = NeighborLoader(\n",
    "        data,\n",
    "        # Sample 25 neighbors for each node and edge type for 2 iterations\n",
    "        num_neighbors=[3*args.sample_nodes] * 2,\n",
    "        # Use a batch size of 32 for sampling testing nodes\n",
    "        batch_size=args.batch_size,\n",
    "        input_nodes=data.test_mask,\n",
    "    )\n",
    " \n",
    "    model = SAGE(in_channels=data.x.shape[-1], hidden_channels = 512, \n",
    "            out_channels = data.y.shape[-1], dropout = 0.1, num_layers=5).to(device)\n",
    "    return data, model, train_loader, val_loader, test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_loop_knn(verbose=False):\n",
    "    \n",
    "    _, model, train_loader, val_loader, test_loader = initialization_knn()\n",
    "    if torch.cuda.device_count() > 1:\n",
    "        print(\"Use {} GPUs !\".format(torch.cuda.device_count()))\n",
    "        model = DataParallel(model)\n",
    "    \n",
    "    optimizer = optim.Adam(model.parameters(), lr=0.0001, weight_decay=args.l2)\n",
    "    #scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer,\n",
    "    #                                           mode='min', factor=0.5,\n",
    "    #                                           patience=1)\n",
    "\n",
    "    train_state = make_train_state(args)\n",
    "\n",
    "    try:\n",
    "        for epoch in range(args.num_epochs):\n",
    "            train_state['epoch_index'] = epoch\n",
    "            \n",
    "            loss = train_Homo(model, optimizer, train_loader)\n",
    "            train_loss_att, train_loss_val, train_att_acc, train_val_acc, train_val_jac, train_val_1 = test_Homo(model, train_loader)\n",
    "            val_loss_att, val_loss_val, val_att_acc, val_val_acc, val_val_jac, val_val_1 = test_Homo(model, val_loader)\n",
    "            if verbose:\n",
    "                print(f'Epoch: {epoch:02d}, Loss: {loss:.4f}, Train_ATT: {train_att_acc:.4f}, Train_VAL: {train_val_acc:.4f}, Val_vis_tex_ATT: {val_att_acc:.4f}, Val_vis_tex_VAL: {val_val_acc:.4f}')\n",
    "            \n",
    "            train_state['train_loss'].append(loss)\n",
    "            train_state['train_ATT_loss'].append(train_loss_att)\n",
    "            train_state['train_VAL_loss'].append(train_loss_val)\n",
    "            train_state['train_ATT_acc'].append(train_att_acc)\n",
    "            train_state['train_VAL_acc'].append(train_val_acc)\n",
    "            train_state['train_VAL_jac'].append(train_val_jac)\n",
    "            train_state['train_VAL_acc_1'].append(train_val_1)\n",
    "            \n",
    "            train_state['val_ATT_loss'].append(val_loss_att)\n",
    "            train_state['val_VAL_loss'].append(val_loss_val)\n",
    "            train_state['val_loss'].append(val_loss_att + 3*val_loss_val)\n",
    "            train_state['val_ATT_acc'].append(val_att_acc)\n",
    "            train_state['val_VAL_acc'].append(val_val_acc)\n",
    "            train_state['val_VAL_jac'].append(val_val_jac)\n",
    "            train_state['val_VAL_acc_1'].append(val_val_1)\n",
    "            \n",
    "            train_state = update_train_state(args=args, model=model,\n",
    "                                                train_state=train_state)\n",
    "            if train_state['stop_early']:\n",
    "                break\n",
    "\n",
    "    except KeyboardInterrupt:\n",
    "        print(\"Exiting loop\")\n",
    "        pass\n",
    "    \n",
    "    return train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_state = training_loop_knn(verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "#path = osp.join(os.getcwd(), '../../data/VEN')\n",
    "transform = T.Compose([T.NormalizeFeatures(), T.ToSparseTensor()])\n",
    "dataset = VEN_Homo('dataset/Venice_homo')\n",
    "data_0 = dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "edge_index = knn_graph(data_0.x[:,:982].to(device), k=3, loop=False, cosine=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Data(num_nodes=2951, x=[2951, 1753], y=[2951, 20], node_type=[2951], att_lab=[2951], val_lab=[2951], train_mask=[2951], val_mask=[2951], test_mask=[2951], edge_index=[2, 8853], edge_attr=[1071977])"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_0.edge_index = edge_index\n",
    "data_0 = data_0.to(device)\n",
    "data_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data_0,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.train_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "val_loader = NeighborLoader(\n",
    "    data_0,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.val_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "test_loader = NeighborLoader(\n",
    "    data_0,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data.test_mask,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SAGE(in_channels=data_0.x.shape[-1], hidden_channels = 512, \n",
    "            out_channels = data.y.shape[-1], dropout = 0.1, num_layers=5).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load(args.save_dir+'SAGE_knn_feature/model.pth',map_location=torch.device('cpu')))\n",
    "model.eval()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 75.91it/s]\n"
     ]
    }
   ],
   "source": [
    "test_loss_att, test_loss_val, test_att_acc, test_val_acc, test_val_jac, test_val_1 = test_Homo(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 233.67it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7405274784465906,\n",
       " 1.6204533534036778,\n",
       " 100.0,\n",
       " 99.7229916897507,\n",
       " 0.7783933623675825,\n",
       " 82.54847645429363)"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 66.97it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8412722705340967,\n",
       " 1.6451545190341368,\n",
       " 91.46341463414635,\n",
       " 100.0,\n",
       " 0.7422003346710957,\n",
       " 77.33990147783251)"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 85.93it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8280698758944366,\n",
       " 1.6704634825388591,\n",
       " 95.02982107355865,\n",
       " 98.4375,\n",
       " 0.6935763930281004,\n",
       " 76.04166666666667)"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Homo(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 81.46it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 283.49it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 261.48it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 350.67it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 308.98it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 349.15it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 286.58it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 295.24it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 290.94it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 321.68it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 271.86it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 324.44it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 375.63it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 209.46it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 319.35it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 281.95it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 271.78it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 308.87it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 294.08it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 271.61it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test_Homo(model, val_loader))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test_Homo(model, test_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>8.412721e-01</td>\n",
       "      <td>1.645154e+00</td>\n",
       "      <td>9.146341e+01</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.7422</td>\n",
       "      <td>7.733990e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.233101e-07</td>\n",
       "      <td>5.549804e-08</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>1.497956e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>8.412720e-01</td>\n",
       "      <td>1.645154e+00</td>\n",
       "      <td>9.146341e+01</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.7422</td>\n",
       "      <td>7.733990e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>8.412720e-01</td>\n",
       "      <td>1.645154e+00</td>\n",
       "      <td>9.146341e+01</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.7422</td>\n",
       "      <td>7.733990e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>8.412721e-01</td>\n",
       "      <td>1.645154e+00</td>\n",
       "      <td>9.146341e+01</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.7422</td>\n",
       "      <td>7.733990e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>8.412722e-01</td>\n",
       "      <td>1.645154e+00</td>\n",
       "      <td>9.146341e+01</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.7422</td>\n",
       "      <td>7.733990e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>8.412724e-01</td>\n",
       "      <td>1.645154e+00</td>\n",
       "      <td>9.146341e+01</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.7422</td>\n",
       "      <td>7.733990e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ATT_loss      VAL_loss       ATT_acc  VAL_k_acc  VAL_k_jac  \\\n",
       "count  1.000000e+01  1.000000e+01  1.000000e+01       10.0    10.0000   \n",
       "mean   8.412721e-01  1.645154e+00  9.146341e+01      100.0     0.7422   \n",
       "std    1.233101e-07  5.549804e-08  1.497956e-14        0.0     0.0000   \n",
       "min    8.412720e-01  1.645154e+00  9.146341e+01      100.0     0.7422   \n",
       "25%    8.412720e-01  1.645154e+00  9.146341e+01      100.0     0.7422   \n",
       "50%    8.412721e-01  1.645154e+00  9.146341e+01      100.0     0.7422   \n",
       "75%    8.412722e-01  1.645154e+00  9.146341e+01      100.0     0.7422   \n",
       "max    8.412724e-01  1.645154e+00  9.146341e+01      100.0     0.7422   \n",
       "\n",
       "          VAL_1_acc  \n",
       "count  1.000000e+01  \n",
       "mean   7.733990e+01  \n",
       "std    1.497956e-14  \n",
       "min    7.733990e+01  \n",
       "25%    7.733990e+01  \n",
       "50%    7.733990e+01  \n",
       "75%    7.733990e+01  \n",
       "max    7.733990e+01  "
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.0000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>8.280698e-01</td>\n",
       "      <td>1.670463e+00</td>\n",
       "      <td>95.029821</td>\n",
       "      <td>98.4375</td>\n",
       "      <td>6.935764e-01</td>\n",
       "      <td>7.604167e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>9.614008e-08</td>\n",
       "      <td>1.976403e-07</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>1.170278e-16</td>\n",
       "      <td>1.497956e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>8.280697e-01</td>\n",
       "      <td>1.670463e+00</td>\n",
       "      <td>95.029821</td>\n",
       "      <td>98.4375</td>\n",
       "      <td>6.935764e-01</td>\n",
       "      <td>7.604167e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>8.280698e-01</td>\n",
       "      <td>1.670463e+00</td>\n",
       "      <td>95.029821</td>\n",
       "      <td>98.4375</td>\n",
       "      <td>6.935764e-01</td>\n",
       "      <td>7.604167e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>8.280698e-01</td>\n",
       "      <td>1.670463e+00</td>\n",
       "      <td>95.029821</td>\n",
       "      <td>98.4375</td>\n",
       "      <td>6.935764e-01</td>\n",
       "      <td>7.604167e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>8.280699e-01</td>\n",
       "      <td>1.670463e+00</td>\n",
       "      <td>95.029821</td>\n",
       "      <td>98.4375</td>\n",
       "      <td>6.935764e-01</td>\n",
       "      <td>7.604167e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>8.280700e-01</td>\n",
       "      <td>1.670464e+00</td>\n",
       "      <td>95.029821</td>\n",
       "      <td>98.4375</td>\n",
       "      <td>6.935764e-01</td>\n",
       "      <td>7.604167e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ATT_loss      VAL_loss    ATT_acc  VAL_k_acc     VAL_k_jac  \\\n",
       "count  1.000000e+01  1.000000e+01  10.000000    10.0000  1.000000e+01   \n",
       "mean   8.280698e-01  1.670463e+00  95.029821    98.4375  6.935764e-01   \n",
       "std    9.614008e-08  1.976403e-07   0.000000     0.0000  1.170278e-16   \n",
       "min    8.280697e-01  1.670463e+00  95.029821    98.4375  6.935764e-01   \n",
       "25%    8.280698e-01  1.670463e+00  95.029821    98.4375  6.935764e-01   \n",
       "50%    8.280698e-01  1.670463e+00  95.029821    98.4375  6.935764e-01   \n",
       "75%    8.280699e-01  1.670463e+00  95.029821    98.4375  6.935764e-01   \n",
       "max    8.280700e-01  1.670464e+00  95.029821    98.4375  6.935764e-01   \n",
       "\n",
       "          VAL_1_acc  \n",
       "count  1.000000e+01  \n",
       "mean   7.604167e+01  \n",
       "std    1.497956e-14  \n",
       "min    7.604167e+01  \n",
       "25%    7.604167e+01  \n",
       "50%    7.604167e+01  \n",
       "75%    7.604167e+01  \n",
       "max    7.604167e+01  "
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'knn_val_metrics.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'knn_test_metrics.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_state['test_ATT_loss']=test_loss_att\n",
    "train_state['test_VAL_loss']=test_loss_val\n",
    "train_state['test_loss']=test_loss_att + 3*test_loss_val\n",
    "train_state['test_ATT_acc']=test_att_acc\n",
    "train_state['test_VAL_acc_1']=test_val_1\n",
    "train_state['test_VAL_acc']=test_val_acc\n",
    "train_state['test_VAL_jac']=test_val_jac"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': True,\n",
       " 'early_stopping_step': 30,\n",
       " 'early_stopping_best_ATT_acc_val': 92.6829268292683,\n",
       " 'early_stopping_best_VAL_acc_val': 100.0,\n",
       " 'early_stopping_best_ATT_acc_val_2': 0,\n",
       " 'early_stopping_lowest_loss': 5.776735078451071,\n",
       " 'learning_rate': 0.001,\n",
       " 'epoch_index': 76,\n",
       " 'train_loss': [4.079040586948395,\n",
       "  3.597453316052755,\n",
       "  3.31153013308843,\n",
       "  2.9931045174598694,\n",
       "  2.793036421140035,\n",
       "  2.7115094860394797,\n",
       "  2.646265963713328,\n",
       "  2.610660672187805,\n",
       "  2.5842005014419556,\n",
       "  2.549887776374817,\n",
       "  2.5355554024378457,\n",
       "  2.517131825288137,\n",
       "  2.502668301264445,\n",
       "  2.4962843855222068,\n",
       "  2.4849180579185486,\n",
       "  2.480564912160238,\n",
       "  2.4669495820999146,\n",
       "  2.457028806209564,\n",
       "  2.452991763750712,\n",
       "  2.441280722618103,\n",
       "  2.441769540309906,\n",
       "  2.438609302043915,\n",
       "  2.4342857201894126,\n",
       "  2.4341718355814614,\n",
       "  2.4290005962053933,\n",
       "  2.4356675942738852,\n",
       "  2.4316309293111167,\n",
       "  2.428731143474579,\n",
       "  2.4142324725786843,\n",
       "  2.4095099568367004,\n",
       "  2.405249377091726,\n",
       "  2.395916302998861,\n",
       "  2.3913696805636087,\n",
       "  2.3957393566767373,\n",
       "  2.383979777495066,\n",
       "  2.380181829134623,\n",
       "  2.3802981972694397,\n",
       "  2.3737219174702964,\n",
       "  2.372760613759359,\n",
       "  2.36648561557134,\n",
       "  2.3700998226801553,\n",
       "  2.367711285750071,\n",
       "  2.366552710533142,\n",
       "  2.365387439727783,\n",
       "  2.359295149644216,\n",
       "  2.36155500014623,\n",
       "  2.3575908541679382,\n",
       "  2.3566491405169168,\n",
       "  2.3562656243642173,\n",
       "  2.353931168715159,\n",
       "  2.3548521796862283,\n",
       "  2.357053498427073,\n",
       "  2.3580503265062966,\n",
       "  2.3567169507344565,\n",
       "  2.3554275830586753,\n",
       "  2.3652203480402627,\n",
       "  2.361345370610555,\n",
       "  2.3626489837964377,\n",
       "  2.365090827147166,\n",
       "  2.35643337170283,\n",
       "  2.358904242515564,\n",
       "  2.3474533756573996,\n",
       "  2.3465211192766824,\n",
       "  2.3470300237337747,\n",
       "  2.349738816420237,\n",
       "  2.349606176217397,\n",
       "  2.34664777914683,\n",
       "  2.349816699822744,\n",
       "  2.349886476993561,\n",
       "  2.34809410572052,\n",
       "  2.351617972056071,\n",
       "  2.3558602333068848,\n",
       "  2.3598453402519226,\n",
       "  2.3527770042419434,\n",
       "  2.3674462040265403,\n",
       "  2.369896193345388,\n",
       "  2.3728488286336265],\n",
       " 'train_ATT_loss': [1.8751792184533835,\n",
       "  1.6545100681009055,\n",
       "  1.3574906977259882,\n",
       "  1.0857674649547673,\n",
       "  0.9737451310963512,\n",
       "  0.9113743572684206,\n",
       "  0.8690371992185175,\n",
       "  0.8534104411291614,\n",
       "  0.8428553943818956,\n",
       "  0.8196919202144126,\n",
       "  0.8090765482831199,\n",
       "  0.8067683704672097,\n",
       "  0.7995722498589936,\n",
       "  0.7966604574565412,\n",
       "  0.7994167225182551,\n",
       "  0.7980283282470175,\n",
       "  0.7899649035897611,\n",
       "  0.7848090156293642,\n",
       "  0.7831649246968722,\n",
       "  0.7850135262653107,\n",
       "  0.7860869837929998,\n",
       "  0.7908982603173507,\n",
       "  0.7931267802404895,\n",
       "  0.7893298408661523,\n",
       "  0.7748672026016045,\n",
       "  0.7809562232355662,\n",
       "  0.7618798953674507,\n",
       "  0.7624308589092582,\n",
       "  0.766778495007935,\n",
       "  0.764218131921298,\n",
       "  0.7532358278528145,\n",
       "  0.7562867634184143,\n",
       "  0.752852009604182,\n",
       "  0.7535381378229306,\n",
       "  0.7478275512063932,\n",
       "  0.7466728634451235,\n",
       "  0.7470096262207983,\n",
       "  0.7491350149183722,\n",
       "  0.743314209572166,\n",
       "  0.744519478891695,\n",
       "  0.7420872250091997,\n",
       "  0.7416498807988999,\n",
       "  0.7414254163110685,\n",
       "  0.7400538166804327,\n",
       "  0.7479124630587253,\n",
       "  0.739975144163063,\n",
       "  0.7405274285834251,\n",
       "  0.7418571530616844,\n",
       "  0.7419316251852506,\n",
       "  0.7412637721468537,\n",
       "  0.7419446599450468,\n",
       "  0.7425993505937571,\n",
       "  0.7408827414473008,\n",
       "  0.7476807756767379,\n",
       "  0.7410231937662056,\n",
       "  0.7576263994391275,\n",
       "  0.7444544962898846,\n",
       "  0.7429988027609616,\n",
       "  0.7480762541459208,\n",
       "  0.7569716609085696,\n",
       "  0.7489872977674172,\n",
       "  0.7400121127469388,\n",
       "  0.7414438957322668,\n",
       "  0.7453742114790919,\n",
       "  0.75446717214056,\n",
       "  0.7417474339543287,\n",
       "  0.7498283496854048,\n",
       "  0.767608121656645,\n",
       "  0.746598457530595,\n",
       "  0.7527270284055673,\n",
       "  0.7688304554062206,\n",
       "  0.7861440800894001,\n",
       "  0.7607322411193742,\n",
       "  0.7594077435226652,\n",
       "  0.7771633782545285,\n",
       "  0.7527807125424414,\n",
       "  0.757453288067741],\n",
       " 'train_VAL_loss': [1.8040795798446994,\n",
       "  1.7909160578349952,\n",
       "  1.7675341542737966,\n",
       "  1.752111307471743,\n",
       "  1.7467298217097147,\n",
       "  1.7393652606869008,\n",
       "  1.7262788287820579,\n",
       "  1.7150241820105556,\n",
       "  1.7106104309869274,\n",
       "  1.7003582138103792,\n",
       "  1.6954768007812078,\n",
       "  1.6883915233479974,\n",
       "  1.6851117075975581,\n",
       "  1.681208502552846,\n",
       "  1.67844591576637,\n",
       "  1.6751393324120223,\n",
       "  1.6693084494228838,\n",
       "  1.6658248006471967,\n",
       "  1.6622384628100408,\n",
       "  1.6619687499762241,\n",
       "  1.6612477005353594,\n",
       "  1.6580805837942953,\n",
       "  1.657446759559441,\n",
       "  1.6528140630088024,\n",
       "  1.6521299093383832,\n",
       "  1.6535247161447837,\n",
       "  1.6533862276420699,\n",
       "  1.6500046064979152,\n",
       "  1.6457106819443426,\n",
       "  1.641864057723175,\n",
       "  1.638795854312231,\n",
       "  1.636646347693129,\n",
       "  1.6346776089179549,\n",
       "  1.6362792078477855,\n",
       "  1.6340090244430585,\n",
       "  1.6308583107021046,\n",
       "  1.6302400900056158,\n",
       "  1.629005649743648,\n",
       "  1.6303406260350404,\n",
       "  1.6258913546057618,\n",
       "  1.626773554202262,\n",
       "  1.6290812409815696,\n",
       "  1.628967619668744,\n",
       "  1.6260409437718484,\n",
       "  1.622909429330905,\n",
       "  1.6224067346541176,\n",
       "  1.6204534121827736,\n",
       "  1.6199863983983809,\n",
       "  1.6196983253526556,\n",
       "  1.6186466154299284,\n",
       "  1.6178653203879698,\n",
       "  1.618566032922169,\n",
       "  1.620904985227083,\n",
       "  1.6196326047099527,\n",
       "  1.6161460787305542,\n",
       "  1.6183966612881902,\n",
       "  1.6193833351135254,\n",
       "  1.6160628914502848,\n",
       "  1.6160029202617106,\n",
       "  1.6148645184376893,\n",
       "  1.616496271043603,\n",
       "  1.616814982527841,\n",
       "  1.6149950955383006,\n",
       "  1.6144998717506176,\n",
       "  1.6153087936279846,\n",
       "  1.6168730404238292,\n",
       "  1.6172085534832814,\n",
       "  1.614788784544884,\n",
       "  1.6133358045958417,\n",
       "  1.614261681683506,\n",
       "  1.6129908928250343,\n",
       "  1.612423645162186,\n",
       "  1.6127310350661133,\n",
       "  1.6129618064188231,\n",
       "  1.6138186425053183,\n",
       "  1.6157215777526601,\n",
       "  1.611759000868018],\n",
       " 'train_ATT_acc': [28.808864265927976,\n",
       "  44.59833795013851,\n",
       "  62.04986149584487,\n",
       "  73.6842105263158,\n",
       "  82.82548476454294,\n",
       "  90.02770083102493,\n",
       "  93.62880886426593,\n",
       "  93.90581717451524,\n",
       "  94.18282548476455,\n",
       "  96.39889196675901,\n",
       "  97.22991689750693,\n",
       "  97.22991689750693,\n",
       "  97.50692520775624,\n",
       "  97.50692520775624,\n",
       "  96.95290858725762,\n",
       "  96.95290858725762,\n",
       "  98.06094182825485,\n",
       "  98.06094182825485,\n",
       "  98.61495844875347,\n",
       "  98.61495844875347,\n",
       "  98.33795013850416,\n",
       "  97.22991689750693,\n",
       "  97.22991689750693,\n",
       "  98.06094182825485,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  99.44598337950139,\n",
       "  99.16897506925208,\n",
       "  98.61495844875347,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.16897506925208,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  98.89196675900277,\n",
       "  99.44598337950139,\n",
       "  99.16897506925208,\n",
       "  98.33795013850416,\n",
       "  97.50692520775624,\n",
       "  98.89196675900277,\n",
       "  99.16897506925208,\n",
       "  98.61495844875347,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139],\n",
       " 'train_VAL_acc': [69.25207756232687,\n",
       "  70.3601108033241,\n",
       "  79.50138504155125,\n",
       "  80.33240997229917,\n",
       "  79.22437673130194,\n",
       "  79.50138504155125,\n",
       "  83.10249307479225,\n",
       "  88.9196675900277,\n",
       "  88.9196675900277,\n",
       "  90.85872576177286,\n",
       "  93.35180055401662,\n",
       "  94.45983379501385,\n",
       "  93.90581717451524,\n",
       "  95.29085872576178,\n",
       "  96.1218836565097,\n",
       "  95.29085872576178,\n",
       "  96.1218836565097,\n",
       "  95.29085872576178,\n",
       "  95.56786703601108,\n",
       "  96.67590027700831,\n",
       "  95.29085872576178,\n",
       "  96.39889196675901,\n",
       "  96.95290858725762,\n",
       "  97.22991689750693,\n",
       "  97.78393351800554,\n",
       "  96.67590027700831,\n",
       "  98.06094182825485,\n",
       "  98.06094182825485,\n",
       "  98.89196675900277,\n",
       "  99.7229916897507,\n",
       "  98.61495844875347,\n",
       "  98.61495844875347,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.16897506925208,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0],\n",
       " 'train_VAL_jac': [0.0110803324099723,\n",
       "  0.06509695290858726,\n",
       "  0.1814404432132964,\n",
       "  0.26131117310880625,\n",
       "  0.3028624262505951,\n",
       "  0.3758079417854795,\n",
       "  0.41366575299207525,\n",
       "  0.4007386949914314,\n",
       "  0.3986611339854402,\n",
       "  0.4695290911560904,\n",
       "  0.4838411854244665,\n",
       "  0.521237307308123,\n",
       "  0.5198522710403908,\n",
       "  0.546168060514075,\n",
       "  0.5577100690382009,\n",
       "  0.5803324205393277,\n",
       "  0.5858725788190424,\n",
       "  0.6084949329619262,\n",
       "  0.6098799745131728,\n",
       "  0.6186519025765628,\n",
       "  0.6140350938833982,\n",
       "  0.6403508833570823,\n",
       "  0.6565097137831585,\n",
       "  0.6726685257169348,\n",
       "  0.684210536882818,\n",
       "  0.6218836670767237,\n",
       "  0.6288088748329564,\n",
       "  0.6712834947327168,\n",
       "  0.6925207914738114,\n",
       "  0.7017543982931121,\n",
       "  0.6966759108440368,\n",
       "  0.7197599384593171,\n",
       "  0.7156048085220633,\n",
       "  0.6975992683558583,\n",
       "  0.7119113679077487,\n",
       "  0.7211449800105636,\n",
       "  0.7119113626242345,\n",
       "  0.7248384206248782,\n",
       "  0.7206832959711387,\n",
       "  0.7428439607910833,\n",
       "  0.7400738776885902,\n",
       "  0.7483841269960694,\n",
       "  0.7493074950749193,\n",
       "  0.7576177338153701,\n",
       "  0.7650046256110279,\n",
       "  0.7631579053038705,\n",
       "  0.7783933623675825,\n",
       "  0.7825484817378079,\n",
       "  0.789012016021644,\n",
       "  0.788550331982219,\n",
       "  0.790858731045287,\n",
       "  0.7714681493278356,\n",
       "  0.752077567610384,\n",
       "  0.7774699942887324,\n",
       "  0.779316719879404,\n",
       "  0.7908587363288013,\n",
       "  0.790858731045287,\n",
       "  0.7862419276356367,\n",
       "  0.7917820938406228,\n",
       "  0.8088642712114921,\n",
       "  0.7742382324303286,\n",
       "  0.7770083155328217,\n",
       "  0.7996306643921913,\n",
       "  0.7922437725965336,\n",
       "  0.7987072963133413,\n",
       "  0.7825484764542936,\n",
       "  0.7797783986353147,\n",
       "  0.8010157006599236,\n",
       "  0.7917820885571086,\n",
       "  0.7945521716596017,\n",
       "  0.8176361992748821,\n",
       "  0.815789484251239,\n",
       "  0.8167128417630605,\n",
       "  0.790858731045287,\n",
       "  0.825484780393479,\n",
       "  0.809787639290342,\n",
       "  0.8347183819292655],\n",
       " 'train_VAL_acc_1': [20.775623268698062,\n",
       "  23.822714681440445,\n",
       "  40.7202216066482,\n",
       "  44.875346260387815,\n",
       "  44.04432132963989,\n",
       "  45.70637119113574,\n",
       "  50.41551246537396,\n",
       "  56.50969529085872,\n",
       "  53.73961218836565,\n",
       "  59.2797783933518,\n",
       "  63.71191135734072,\n",
       "  63.71191135734072,\n",
       "  60.11080332409972,\n",
       "  62.880886426592795,\n",
       "  66.20498614958449,\n",
       "  64.54293628808864,\n",
       "  64.54293628808864,\n",
       "  66.7590027700831,\n",
       "  69.25207756232687,\n",
       "  68.97506925207756,\n",
       "  68.69806094182826,\n",
       "  69.80609418282549,\n",
       "  70.3601108033241,\n",
       "  73.40720221606648,\n",
       "  74.23822714681441,\n",
       "  66.7590027700831,\n",
       "  68.69806094182826,\n",
       "  70.91412742382272,\n",
       "  74.23822714681441,\n",
       "  76.17728531855956,\n",
       "  75.90027700831025,\n",
       "  75.90027700831025,\n",
       "  77.00831024930748,\n",
       "  75.34626038781164,\n",
       "  76.17728531855956,\n",
       "  77.28531855955679,\n",
       "  78.39335180055402,\n",
       "  80.33240997229917,\n",
       "  80.33240997229917,\n",
       "  83.10249307479225,\n",
       "  80.60941828254848,\n",
       "  81.7174515235457,\n",
       "  80.88642659279779,\n",
       "  80.60941828254848,\n",
       "  81.7174515235457,\n",
       "  82.54847645429363,\n",
       "  82.54847645429363,\n",
       "  82.82548476454294,\n",
       "  80.88642659279779,\n",
       "  81.7174515235457,\n",
       "  80.88642659279779,\n",
       "  82.54847645429363,\n",
       "  82.27146814404432,\n",
       "  82.54847645429363,\n",
       "  82.82548476454294,\n",
       "  82.82548476454294,\n",
       "  80.05540166204986,\n",
       "  81.4404432132964,\n",
       "  81.99445983379502,\n",
       "  82.27146814404432,\n",
       "  79.77839335180056,\n",
       "  79.22437673130194,\n",
       "  82.27146814404432,\n",
       "  83.10249307479225,\n",
       "  81.7174515235457,\n",
       "  83.10249307479225,\n",
       "  81.99445983379502,\n",
       "  80.88642659279779,\n",
       "  82.27146814404432,\n",
       "  85.87257617728532,\n",
       "  86.70360110803324,\n",
       "  83.93351800554017,\n",
       "  81.7174515235457,\n",
       "  85.59556786703601,\n",
       "  85.31855955678671,\n",
       "  81.99445983379502,\n",
       "  83.93351800554017],\n",
       " 'val_loss': [7.283423665945424,\n",
       "  7.032016599824992,\n",
       "  6.649818128838195,\n",
       "  6.334902016730736,\n",
       "  6.242974912752889,\n",
       "  6.178060978907068,\n",
       "  6.096450957607686,\n",
       "  6.04537403308408,\n",
       "  6.01878436211722,\n",
       "  5.9778561140609225,\n",
       "  5.962879366951013,\n",
       "  5.948586948946726,\n",
       "  5.936176787680063,\n",
       "  5.9299637997550025,\n",
       "  5.923684646760517,\n",
       "  5.9115414453521264,\n",
       "  5.888859509464507,\n",
       "  5.878514199803267,\n",
       "  5.874385382200855,\n",
       "  5.878727769579192,\n",
       "  5.873476698972556,\n",
       "  5.870474363367359,\n",
       "  5.874277972920579,\n",
       "  5.857426038860912,\n",
       "  5.857624456888588,\n",
       "  5.869214574114008,\n",
       "  5.8520210438824485,\n",
       "  5.846357591313849,\n",
       "  5.844857775458771,\n",
       "  5.822866945373859,\n",
       "  5.808364538591125,\n",
       "  5.797680629523603,\n",
       "  5.797840254317368,\n",
       "  5.809072073912027,\n",
       "  5.7997883530104986,\n",
       "  5.78938616082777,\n",
       "  5.788906185965672,\n",
       "  5.796900606322735,\n",
       "  5.790675301254428,\n",
       "  5.7841939130983215,\n",
       "  5.78561019228702,\n",
       "  5.798208586219635,\n",
       "  5.797086109919401,\n",
       "  5.783889703328754,\n",
       "  5.795797323103642,\n",
       "  5.783515444345648,\n",
       "  5.776735078451071,\n",
       "  5.781217983957907,\n",
       "  5.772141902909567,\n",
       "  5.772884587114438,\n",
       "  5.762641144733276,\n",
       "  5.77226971740934,\n",
       "  5.779605943883126,\n",
       "  5.794451155768383,\n",
       "  5.776907586052133,\n",
       "  5.80998762700771,\n",
       "  5.792775426537242,\n",
       "  5.779059259793947,\n",
       "  5.774846563319445,\n",
       "  5.7878766293779975,\n",
       "  5.798806684566058,\n",
       "  5.790579144272507,\n",
       "  5.789221655337065,\n",
       "  5.791984789685472,\n",
       "  5.814299270892617,\n",
       "  5.797098533905327,\n",
       "  5.807927744259507,\n",
       "  5.837984281963458,\n",
       "  5.805557146571754,\n",
       "  5.810840704042035,\n",
       "  5.818489433817318,\n",
       "  5.8413098094702685,\n",
       "  5.819335513321489,\n",
       "  5.801585035156232,\n",
       "  5.827289187266985,\n",
       "  5.801322369165223,\n",
       "  5.793010800660929],\n",
       " 'val_ATT_loss': [1.8916375099158869,\n",
       "  1.6814340084064296,\n",
       "  1.3885113555725996,\n",
       "  1.116481484194112,\n",
       "  1.0026963040596102,\n",
       "  0.9529044144764179,\n",
       "  0.9161934533981773,\n",
       "  0.9078676761165867,\n",
       "  0.8973125184212274,\n",
       "  0.8781239470330681,\n",
       "  0.8705134778245678,\n",
       "  0.8717500881208637,\n",
       "  0.8631645437420868,\n",
       "  0.8662610126704704,\n",
       "  0.8692642548462239,\n",
       "  0.8664933470933418,\n",
       "  0.8605464575736503,\n",
       "  0.8568617115175821,\n",
       "  0.8620002669532124,\n",
       "  0.8660328798904652,\n",
       "  0.86252817487329,\n",
       "  0.8670043770859881,\n",
       "  0.8742728034655253,\n",
       "  0.8710771027376981,\n",
       "  0.8620050559198953,\n",
       "  0.8701007753852906,\n",
       "  0.8517858791884368,\n",
       "  0.842140210353262,\n",
       "  0.8546489265149202,\n",
       "  0.8491516171432123,\n",
       "  0.8401392385727022,\n",
       "  0.8403092542799507,\n",
       "  0.8444284361794712,\n",
       "  0.843937008119211,\n",
       "  0.8357021886158765,\n",
       "  0.8386896032143415,\n",
       "  0.8372522515010058,\n",
       "  0.8426581560596218,\n",
       "  0.8333705927782912,\n",
       "  0.8405453234426374,\n",
       "  0.836771784516854,\n",
       "  0.8332459234852132,\n",
       "  0.8351281924945552,\n",
       "  0.8329193630838782,\n",
       "  0.8537204797432675,\n",
       "  0.8401425593509907,\n",
       "  0.8412721132844444,\n",
       "  0.8427224983044757,\n",
       "  0.8396269276859315,\n",
       "  0.8410020336145307,\n",
       "  0.8323019087556901,\n",
       "  0.8414715973100042,\n",
       "  0.8394660394850785,\n",
       "  0.8556199139211236,\n",
       "  0.8429225048398584,\n",
       "  0.8705318171561249,\n",
       "  0.8502733266450525,\n",
       "  0.8445998671335903,\n",
       "  0.8492163420934987,\n",
       "  0.8684407324810338,\n",
       "  0.8609539530868453,\n",
       "  0.849225049217542,\n",
       "  0.849676482561158,\n",
       "  0.8604948781127852,\n",
       "  0.8719587782776452,\n",
       "  0.8436513423192792,\n",
       "  0.8644170914965916,\n",
       "  0.902438215002781,\n",
       "  0.8658201196571675,\n",
       "  0.8711568844027635,\n",
       "  0.8972064242130373,\n",
       "  0.9244964206364097,\n",
       "  0.8854932260464846,\n",
       "  0.8806169847162758,\n",
       "  0.904015441250995,\n",
       "  0.8653959693705163,\n",
       "  0.8625128219525019],\n",
       " 'val_VAL_loss': [1.7972620520098457,\n",
       "  1.7835275304728542,\n",
       "  1.7537689244218648,\n",
       "  1.7394735108455415,\n",
       "  1.7467595362310926,\n",
       "  1.7417188548102167,\n",
       "  1.7267525014031697,\n",
       "  1.7125021189891647,\n",
       "  1.7071572812319977,\n",
       "  1.699910722342618,\n",
       "  1.6974552963754814,\n",
       "  1.692278953608621,\n",
       "  1.6910040813126588,\n",
       "  1.6879009290281775,\n",
       "  1.6848067973047642,\n",
       "  1.6816826994195948,\n",
       "  1.6761043506302857,\n",
       "  1.673884162761895,\n",
       "  1.6707950384158807,\n",
       "  1.670898296562909,\n",
       "  1.6703161746997552,\n",
       "  1.6678233287604571,\n",
       "  1.6666683898183512,\n",
       "  1.6621163120410714,\n",
       "  1.665206466989564,\n",
       "  1.6663712662429058,\n",
       "  1.6667450548980036,\n",
       "  1.6680724603201955,\n",
       "  1.6634029496479503,\n",
       "  1.6579051094102155,\n",
       "  1.656075100006141,\n",
       "  1.6524571250812174,\n",
       "  1.6511372727126323,\n",
       "  1.655045021930939,\n",
       "  1.654695388131541,\n",
       "  1.650232185871143,\n",
       "  1.650551311488222,\n",
       "  1.6514141500877042,\n",
       "  1.652434902825379,\n",
       "  1.6478828632185611,\n",
       "  1.6496128025900554,\n",
       "  1.6549875542448071,\n",
       "  1.6539859724749486,\n",
       "  1.650323446748292,\n",
       "  1.6473589477867916,\n",
       "  1.6477909616648858,\n",
       "  1.645154321722209,\n",
       "  1.646165161884477,\n",
       "  1.6441716584078785,\n",
       "  1.643960851166636,\n",
       "  1.6434464119925287,\n",
       "  1.6435993733664451,\n",
       "  1.6467133014660162,\n",
       "  1.646277080615753,\n",
       "  1.6446616937374245,\n",
       "  1.6464852699505284,\n",
       "  1.647500699964063,\n",
       "  1.6448197975534524,\n",
       "  1.6418767404086485,\n",
       "  1.6398119656323211,\n",
       "  1.645950910493071,\n",
       "  1.6471180316849883,\n",
       "  1.6465150575919691,\n",
       "  1.6438299705242287,\n",
       "  1.6474468308716572,\n",
       "  1.651149063862016,\n",
       "  1.6478368842543052,\n",
       "  1.6451820223202258,\n",
       "  1.646579008971529,\n",
       "  1.6465612732130905,\n",
       "  1.6404276698680933,\n",
       "  1.638937796277953,\n",
       "  1.6446140957583348,\n",
       "  1.6403226834799856,\n",
       "  1.6410912486719968,\n",
       "  1.6453087999315685,\n",
       "  1.6434993262361424],\n",
       " 'val_ATT_acc': [30.48780487804878,\n",
       "  39.63414634146341,\n",
       "  54.8780487804878,\n",
       "  75.40650406504065,\n",
       "  80.08130081300813,\n",
       "  84.34959349593495,\n",
       "  87.8048780487805,\n",
       "  85.77235772357723,\n",
       "  86.78861788617886,\n",
       "  90.44715447154472,\n",
       "  90.2439024390244,\n",
       "  88.82113821138212,\n",
       "  90.65040650406505,\n",
       "  89.83739837398375,\n",
       "  89.63414634146342,\n",
       "  90.04065040650407,\n",
       "  90.44715447154472,\n",
       "  90.2439024390244,\n",
       "  89.02439024390245,\n",
       "  89.83739837398375,\n",
       "  91.0569105691057,\n",
       "  90.04065040650407,\n",
       "  90.04065040650407,\n",
       "  89.63414634146342,\n",
       "  90.04065040650407,\n",
       "  89.02439024390245,\n",
       "  90.65040650406505,\n",
       "  91.869918699187,\n",
       "  90.2439024390244,\n",
       "  90.04065040650407,\n",
       "  91.66666666666667,\n",
       "  91.0569105691057,\n",
       "  91.0569105691057,\n",
       "  91.0569105691057,\n",
       "  91.66666666666667,\n",
       "  91.46341463414635,\n",
       "  91.66666666666667,\n",
       "  91.869918699187,\n",
       "  92.47967479674797,\n",
       "  91.26016260162602,\n",
       "  92.27642276422765,\n",
       "  92.27642276422765,\n",
       "  92.6829268292683,\n",
       "  91.66666666666667,\n",
       "  89.22764227642277,\n",
       "  91.0569105691057,\n",
       "  91.46341463414635,\n",
       "  91.26016260162602,\n",
       "  91.26016260162602,\n",
       "  89.83739837398375,\n",
       "  92.47967479674797,\n",
       "  90.44715447154472,\n",
       "  91.66666666666667,\n",
       "  89.4308943089431,\n",
       "  90.85365853658537,\n",
       "  88.82113821138212,\n",
       "  90.65040650406505,\n",
       "  90.85365853658537,\n",
       "  91.26016260162602,\n",
       "  89.02439024390245,\n",
       "  89.22764227642277,\n",
       "  89.83739837398375,\n",
       "  90.2439024390244,\n",
       "  88.6178861788618,\n",
       "  89.02439024390245,\n",
       "  91.0569105691057,\n",
       "  88.6178861788618,\n",
       "  85.97560975609755,\n",
       "  88.82113821138212,\n",
       "  88.82113821138212,\n",
       "  86.17886178861788,\n",
       "  84.95934959349593,\n",
       "  86.78861788617886,\n",
       "  87.39837398373983,\n",
       "  86.3821138211382,\n",
       "  88.41463414634147,\n",
       "  89.63414634146342],\n",
       " 'val_VAL_acc': [64.5320197044335,\n",
       "  68.47290640394088,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  82.26600985221675,\n",
       "  83.2512315270936,\n",
       "  86.69950738916256,\n",
       "  90.14778325123153,\n",
       "  91.13300492610837,\n",
       "  92.61083743842364,\n",
       "  95.56650246305419,\n",
       "  94.58128078817734,\n",
       "  92.61083743842364,\n",
       "  92.11822660098522,\n",
       "  95.07389162561576,\n",
       "  95.07389162561576,\n",
       "  96.55172413793103,\n",
       "  96.55172413793103,\n",
       "  96.05911330049261,\n",
       "  95.07389162561576,\n",
       "  95.07389162561576,\n",
       "  95.07389162561576,\n",
       "  96.55172413793103,\n",
       "  96.05911330049261,\n",
       "  95.56650246305419,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  97.04433497536945,\n",
       "  97.53694581280789,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  97.53694581280789,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  99.50738916256158],\n",
       " 'val_VAL_jac': [0.0024630541871921183,\n",
       "  0.04433497536945813,\n",
       "  0.22824302212945347,\n",
       "  0.3243021348427082,\n",
       "  0.3661740566122121,\n",
       "  0.4080459817583338,\n",
       "  0.403119868832856,\n",
       "  0.40640394323565104,\n",
       "  0.45894909844609905,\n",
       "  0.46715928063604045,\n",
       "  0.4802955712003661,\n",
       "  0.5139573125416422,\n",
       "  0.5714285784754259,\n",
       "  0.5640394159138497,\n",
       "  0.5771757017802722,\n",
       "  0.5894909703672813,\n",
       "  0.6288998397113067,\n",
       "  0.6272578028035282,\n",
       "  0.6321839111779124,\n",
       "  0.6338259480856909,\n",
       "  0.6215106748007788,\n",
       "  0.6510673250470843,\n",
       "  0.6658456572170915,\n",
       "  0.6888341574833311,\n",
       "  0.6921182312988883,\n",
       "  0.65599343577042,\n",
       "  0.6658456525191885,\n",
       "  0.6847290687373119,\n",
       "  0.6970443396732725,\n",
       "  0.6896551747627446,\n",
       "  0.6855500860167254,\n",
       "  0.6880131402039176,\n",
       "  0.6921182312988883,\n",
       "  0.6970443396732725,\n",
       "  0.7052545195142624,\n",
       "  0.7134647017042038,\n",
       "  0.7093596106092331,\n",
       "  0.7241379380813373,\n",
       "  0.717569790450223,\n",
       "  0.7356321870399813,\n",
       "  0.7307060857124517,\n",
       "  0.7118226694943283,\n",
       "  0.7126436844247902,\n",
       "  0.7216748838941452,\n",
       "  0.7315271029918652,\n",
       "  0.7463054210681633,\n",
       "  0.7422003346710957,\n",
       "  0.7422003370200472,\n",
       "  0.7619047658196811,\n",
       "  0.7471264453944314,\n",
       "  0.7315271006429137,\n",
       "  0.7643678200068732,\n",
       "  0.7249589530117994,\n",
       "  0.7200328469863666,\n",
       "  0.740558297763317,\n",
       "  0.7454844108356043,\n",
       "  0.7692939283812574,\n",
       "  0.7586206990509785,\n",
       "  0.7610837532381706,\n",
       "  0.7635468027274597,\n",
       "  0.7528735686992777,\n",
       "  0.7430213519505092,\n",
       "  0.7183908124275395,\n",
       "  0.7298850684330381,\n",
       "  0.7044335022348488,\n",
       "  0.6855500860167254,\n",
       "  0.6822660145501198,\n",
       "  0.7019704503966082,\n",
       "  0.7257799726401644,\n",
       "  0.7077175737014545,\n",
       "  0.7356321870399813,\n",
       "  0.749589497232672,\n",
       "  0.7134647040531553,\n",
       "  0.7602627336098056,\n",
       "  0.7454844061377013,\n",
       "  0.7192118320559046,\n",
       "  0.7495894948837205],\n",
       " 'val_VAL_acc_1': [15.270935960591133,\n",
       "  21.182266009852217,\n",
       "  46.79802955665025,\n",
       "  50.24630541871921,\n",
       "  51.724137931034484,\n",
       "  48.275862068965516,\n",
       "  46.30541871921182,\n",
       "  55.172413793103445,\n",
       "  59.60591133004926,\n",
       "  60.59113300492611,\n",
       "  61.576354679802954,\n",
       "  66.99507389162562,\n",
       "  65.51724137931035,\n",
       "  67.98029556650246,\n",
       "  69.45812807881774,\n",
       "  68.96551724137932,\n",
       "  70.44334975369458,\n",
       "  71.92118226600985,\n",
       "  70.93596059113301,\n",
       "  66.50246305418719,\n",
       "  69.45812807881774,\n",
       "  70.44334975369458,\n",
       "  71.42857142857143,\n",
       "  77.83251231527093,\n",
       "  74.38423645320196,\n",
       "  71.42857142857143,\n",
       "  68.47290640394088,\n",
       "  72.9064039408867,\n",
       "  73.89162561576354,\n",
       "  74.38423645320196,\n",
       "  76.35467980295566,\n",
       "  75.86206896551724,\n",
       "  76.84729064039409,\n",
       "  73.39901477832512,\n",
       "  74.8768472906404,\n",
       "  77.33990147783251,\n",
       "  73.89162561576354,\n",
       "  74.8768472906404,\n",
       "  73.39901477832512,\n",
       "  77.33990147783251,\n",
       "  75.36945812807882,\n",
       "  73.89162561576354,\n",
       "  74.38423645320196,\n",
       "  76.35467980295566,\n",
       "  78.81773399014779,\n",
       "  77.33990147783251,\n",
       "  77.33990147783251,\n",
       "  79.3103448275862,\n",
       "  80.78817733990148,\n",
       "  78.81773399014779,\n",
       "  76.84729064039409,\n",
       "  76.35467980295566,\n",
       "  76.35467980295566,\n",
       "  74.38423645320196,\n",
       "  79.80295566502463,\n",
       "  79.3103448275862,\n",
       "  79.3103448275862,\n",
       "  79.3103448275862,\n",
       "  75.86206896551724,\n",
       "  76.84729064039409,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  75.86206896551724,\n",
       "  76.35467980295566,\n",
       "  79.3103448275862,\n",
       "  76.35467980295566,\n",
       "  73.39901477832512,\n",
       "  74.8768472906404,\n",
       "  77.83251231527093,\n",
       "  77.33990147783251,\n",
       "  77.83251231527093,\n",
       "  78.81773399014779,\n",
       "  80.29556650246306,\n",
       "  76.84729064039409,\n",
       "  80.78817733990148,\n",
       "  79.3103448275862,\n",
       "  79.80295566502463],\n",
       " 'test_loss': 5.839460189574607,\n",
       " 'test_ATT_loss': 0.8280697997000298,\n",
       " 'test_VAL_loss': 1.6704634632915258,\n",
       " 'test_ATT_acc': 95.02982107355865,\n",
       " 'test_VAL_acc': 98.4375,\n",
       " 'test_VAL_jac': 0.6935763930281004,\n",
       " 'test_VAL_acc_1': 76.04166666666667,\n",
       " 'model_filename': 'model_storage/SAGE/model.pth'}"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'SAGE_knn_feature/best_config.p', 'wb') as fp:\n",
    "    pickle.dump(train_state,fp, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'SAGE_knn_feature/best_config.p', 'rb') as fp:\n",
    "    train_state = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "states = pd.DataFrame(train_state['val_VAL_acc'], columns=['val_VAL'])\n",
    "states['train_VAL'] = pd.DataFrame(train_state['train_VAL_acc'])\n",
    "states['val_ATT'] = pd.DataFrame(train_state['val_ATT_acc'])\n",
    "states['train_ATT'] = pd.DataFrame(train_state['train_ATT_acc'])\n",
    "states['val_VAL_1'] = pd.DataFrame(train_state['val_VAL_acc_1'])\n",
    "states['train_VAL_1'] = pd.DataFrame(train_state['train_VAL_acc_1'])\n",
    "states['train_VAL_jac'] = pd.DataFrame(train_state['train_VAL_jac'])\n",
    "states['val_VAL_jac'] = pd.DataFrame(train_state['val_VAL_jac'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>val_VAL</th>\n",
       "      <th>train_VAL</th>\n",
       "      <th>val_ATT</th>\n",
       "      <th>train_ATT</th>\n",
       "      <th>val_VAL_1</th>\n",
       "      <th>train_VAL_1</th>\n",
       "      <th>train_VAL_jac</th>\n",
       "      <th>val_VAL_jac</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>64.532020</td>\n",
       "      <td>69.252078</td>\n",
       "      <td>30.487805</td>\n",
       "      <td>28.808864</td>\n",
       "      <td>15.270936</td>\n",
       "      <td>20.775623</td>\n",
       "      <td>0.011080</td>\n",
       "      <td>0.002463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>68.472906</td>\n",
       "      <td>70.360111</td>\n",
       "      <td>39.634146</td>\n",
       "      <td>44.598338</td>\n",
       "      <td>21.182266</td>\n",
       "      <td>23.822715</td>\n",
       "      <td>0.065097</td>\n",
       "      <td>0.044335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>81.280788</td>\n",
       "      <td>79.501385</td>\n",
       "      <td>54.878049</td>\n",
       "      <td>62.049861</td>\n",
       "      <td>46.798030</td>\n",
       "      <td>40.720222</td>\n",
       "      <td>0.181440</td>\n",
       "      <td>0.228243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>80.788177</td>\n",
       "      <td>80.332410</td>\n",
       "      <td>75.406504</td>\n",
       "      <td>73.684211</td>\n",
       "      <td>50.246305</td>\n",
       "      <td>44.875346</td>\n",
       "      <td>0.261311</td>\n",
       "      <td>0.324302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>82.266010</td>\n",
       "      <td>79.224377</td>\n",
       "      <td>80.081301</td>\n",
       "      <td>82.825485</td>\n",
       "      <td>51.724138</td>\n",
       "      <td>44.044321</td>\n",
       "      <td>0.302862</td>\n",
       "      <td>0.366174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>99.507389</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>86.788618</td>\n",
       "      <td>98.891967</td>\n",
       "      <td>80.295567</td>\n",
       "      <td>81.717452</td>\n",
       "      <td>0.816713</td>\n",
       "      <td>0.713465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>99.014778</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>87.398374</td>\n",
       "      <td>99.168975</td>\n",
       "      <td>76.847291</td>\n",
       "      <td>85.595568</td>\n",
       "      <td>0.790859</td>\n",
       "      <td>0.760263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>99.507389</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>86.382114</td>\n",
       "      <td>98.614958</td>\n",
       "      <td>80.788177</td>\n",
       "      <td>85.318560</td>\n",
       "      <td>0.825485</td>\n",
       "      <td>0.745484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>99.722992</td>\n",
       "      <td>88.414634</td>\n",
       "      <td>99.722992</td>\n",
       "      <td>79.310345</td>\n",
       "      <td>81.994460</td>\n",
       "      <td>0.809788</td>\n",
       "      <td>0.719212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>99.507389</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>89.634146</td>\n",
       "      <td>99.445983</td>\n",
       "      <td>79.802956</td>\n",
       "      <td>83.933518</td>\n",
       "      <td>0.834718</td>\n",
       "      <td>0.749589</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>77 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       val_VAL   train_VAL    val_ATT  train_ATT  val_VAL_1  train_VAL_1  \\\n",
       "0    64.532020   69.252078  30.487805  28.808864  15.270936    20.775623   \n",
       "1    68.472906   70.360111  39.634146  44.598338  21.182266    23.822715   \n",
       "2    81.280788   79.501385  54.878049  62.049861  46.798030    40.720222   \n",
       "3    80.788177   80.332410  75.406504  73.684211  50.246305    44.875346   \n",
       "4    82.266010   79.224377  80.081301  82.825485  51.724138    44.044321   \n",
       "..         ...         ...        ...        ...        ...          ...   \n",
       "72   99.507389  100.000000  86.788618  98.891967  80.295567    81.717452   \n",
       "73   99.014778  100.000000  87.398374  99.168975  76.847291    85.595568   \n",
       "74   99.507389  100.000000  86.382114  98.614958  80.788177    85.318560   \n",
       "75  100.000000   99.722992  88.414634  99.722992  79.310345    81.994460   \n",
       "76   99.507389  100.000000  89.634146  99.445983  79.802956    83.933518   \n",
       "\n",
       "    train_VAL_jac  val_VAL_jac  \n",
       "0        0.011080     0.002463  \n",
       "1        0.065097     0.044335  \n",
       "2        0.181440     0.228243  \n",
       "3        0.261311     0.324302  \n",
       "4        0.302862     0.366174  \n",
       "..            ...          ...  \n",
       "72       0.816713     0.713465  \n",
       "73       0.790859     0.760263  \n",
       "74       0.825485     0.745484  \n",
       "75       0.809788     0.719212  \n",
       "76       0.834718     0.749589  \n",
       "\n",
       "[77 rows x 8 columns]"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA1jElEQVR4nO3deXhU5dn48e+dfd8TCGvYdwgQEFAUFBUVlYr6Yqvi3qptsbVa0PZXfV9bfdu31mqt1lorVQSRSsW1slcR0SD7EtZsZF/Jvs3z++NMQiJJSDIJM8zcn+vKNXPOmXPOncnknuc82xFjDEoppdyLl7MDUEop1f00uSullBvS5K6UUm5Ik7tSSrkhTe5KKeWGfJwdAEBMTIxJSEhwdhhKKXVe2bFjR4ExJra1bS6R3BMSEkhOTnZ2GEopdV4RkbS2tmm1jFJKuSFN7kop5YY0uSullBvS5K6UUm5Ik7tSSrmhsyZ3EXlNRPJEZF+zdVEisk5EjtgfI5ttWyoiR0UkRUSu7KnAlVJKta0jJffXgbnfWrcE2GCMGQZssC8jIqOBhcAY+z5/FhHvbotWKaVUh5y1n7sx5j8ikvCt1dcDs+zPlwGbgZ/b1680xtQAJ0TkKDAV2NZN8SpPYAyU5UD2bsg/CCG9IX48xIwAb5/T23P2QN5BCOnVcjuc3j93P9RVtX0uEYhMgN7jIXYEePs2238P5O5rf/+uiBxoP99I8PGz1pXnnT5fbUX3nu9sIvpb8cSNAh9/a11Fwen3LzCile2FkLMbcvZCTXn7xw/vZ/194ka3sv8+qCnrXLyDZ0HChWeub6iHPW9DcWrL9cPnQr/JnTuHG+jqIKZexphsAGNMtojE2df3Bb5s9rpM+7oziMh9wH0AAwYM6GIYyq2UnoRPlkD6NqjIP3O7TwDEDLMSb2vbvf0hdjiU5UJFXrMN0s5Jm93PoHH/8jwoz+3g/p3V/Hx+1hdSRT6U5/TQ+ToRj5ev9QVXVQynTp75Ui9fiBkO1SXf2t7B99fLx/p9q0vhVGYH92/leP/5LYy6Dq74H+uLGeDYRvhkKeQf+tYxDXz2e7j8v2H6g9aXubPUVsKmX1t/797jrS+83uMgMPLs+3ZBd49Qbe2da/VuIMaYV4BXAJKSkvSOIeer4lQrGTRqqIeCFKsUmrPH+mcLjbd/mCdYP/2mnC6xNjq2Ef55D9TXwOj59g++vbTYWErP3m2V1HuPP/3PETfaSsTZu1vf3mssBIS1Hb+tAQqP2Y+/C3IPQK9xp8/fe1z7+3eWzQZFx6xYc/ZYJeNeY6zzxU+wny+8+87XkXiKT5yOJ2ef9Z42//2rik+//7n7oddoe6z27UFR7R+/JPX05yFnr/U3jZ9w+hzt7f9tdVXwxZ/g82fh8L9h2v2QnwKHP7YS/cK3YMTVp5N4dSn86wH49HHI2A7Xv2j9PUvSYddbsHslYFp+PgdMB/+Qrr2XpekQGHXmZ6bgKKy6HfIOQGhv6wqj0fiFcMNfOn++s5CO3InJXi3zgTFmrH05BZhlL7XHA5uNMSNEZCmAMeZp++v+DTxhjGm3WiYpKcno9APnGVsDbH4a/vO71rf7Bln/+LEjoSzb+uduLJ0GxcCEhTDxVqsk95/fWceKGwU3/8MqnSvVntKTsOFJK0n6hcDFj1iJvrHapzljYNufYN2vrC+AiP5wfIu1bfAsKxFn77G+5MCq5rvs/8GE74LXWZolj2+GlI/tX157obbMusIcfb31+R54ERxcC+/90KryW/BXGDoHyvOtaqnsPRAxAMbd2KW3QUR2GGOSWt3WxeT+O6DQGPOMiCwBoowxj4rIGOAtrHr2PliNrcOMMQ3tHV+Tew+wNcCxTXBgzbfqcAWih5wu2UYM7PylakUB/PNu64Od+D0YdW3L40cNts7h9a229PI8yPzaKi2lfAy2OqtUX5ZtlV7mPQt+wV38hZVHKjxmXekEx5z9tWlfwLv3WZ/3xFsh8RYrsTaqLoXMZKugkfk1xCfCVf8LA6a1frzkv8MHPwHfQOsKsfFKMWcv7F0NNaWnP999k+DmZVb7QzdyKLmLyAqsxtMYIBf4FfAvYBUwAEgHbjLGFNlf/zhwF1APPGSM+fhsAWpyd0BlEdQ2a9CqKYN971qXnGVZEBABIXGntzfUQUkaGJu1HBgJs5bC1Ps6luTTt8M7d0BVEVz9fzDptq7FXVEAe1bBkX/DmO/ApEXOrQ9VnsOY9j9rxsDed6ySflkWjF0Ac560SvyNvnzJah8adoV1tekb2PIYtZVw6AMryceNhNm/OLMqshs4XHLvaZrcu8AY2PpH2PDf8O0LI/GCIZdZiXf4VWd+qOqqrLrlnN1wYC0c3wRjboDrngf/0NbPV1kEm34Dya9ZH/Kb37BKKkq5q9oK+Pw5+OJ5QODCxdbP9pes/7tR18KC13okaXeUJndXU19jNaxk2xscG2pPbxNvGHKpVS/n3UZ7d1UJ/Ot+SPnI6jUw/MqW+w+6GMJb7aR0JpsNtj4HG/8HoobAf71h1X03aqizEvqm30DNKUi6Cy79pdU9TilPUJIO6/4f7F9jXelWFcO4m2D+y23/j54jmtydwRgr+W59vmW1SUMtFB0HW7217BvU8pKurgrqKq2+3Ym3WI06YX1Oby9IgXfutLqiXfEUXPCD7qnOOPEZrL7LqtaJHnJ6fWWRdWk66BKY+7TVs0MpT5T2Bax/wupRM/eZM9uUnECT+7mWewD+vdRqcIweavUYaSRirWvsShY5qGWLfH2tVQ+980048unpuvHmQvtYjTP9p3Zv3GU5sPGpll0bvbxh3M0w8hqtE1fKxWhyP1ca6uDTX8BXf7Xqrmc/ZlVjNI567KxT2ZDyodU408jbz+o21ZHeAUopt9ZecneJ2+y5ja3PwfaXIeluuPQXnRuc0ZqweJhyT7eEppTyLJrcu0veIdjyW6tb37xnnR2NUsrD6Xzu3cHWAGt/ZA3Aueq3zo5GKaW05N4tvnoFMr+C77zScsCQUko5iZbcHVV0whrQMPRyGH+zs6NRSilAS+5dZ2uAgiPw0c+sEaHXPqddBZVSLkOTe0fUVVsjSnP2WKNKG6c+rbffxGHec90+IZBSSjlCk3t7Kotg5fes+vTGEaX+YdZUtkl3WoOQ+k6ybnCglFIuRJN7e75ZBulfWJMF9ZlknyI34exzPCullJNpcm+LzQY7XoeBF1q36FKqk4wx5J6qISbEDx9vLRCoc0uTe1tObLFuITf7F86ORDnJoZxTvPvNSa4c04vJA1sfbVxbb6Oq7vSUy/UNNnakFbPlcD5bDueTWVzF2L5hPPdfExka14VbtznAGMPrX6QyJSGKsX3P4a37lEvQ5N6WHX+37oXY4i5Dyt3VNdhYdyCXZV+ksv1EEQCvb03lmQXjuGHS6UZzYwxrdp7kV2v3U1Zdf8Zxgv28mTE0hpuT+vP6F6nMe+EzHr96FLdOG4ico15V/zlSwJPvH6BXmD+fLL6YyGDnzTuuzj1N7q0pz4NDH1rT6foGODsat7EzvZil7+7lrosGcXNS/7PvAHxxtIBBscHEhwee/cXfUltv4/Oj+fh5ezMwOog+EYF4e1mJtbSyjtTCClILK0grrGx6PJZfTkllHf0iA1l61UiuHNObpe/u5aerdnM8v4KfXj6csup6HvvXXj7ck82UhEjmjo1vOqcAo+LDmDwwEj8fqypm4dT+PPLOHn753n42HMrj2ZsTierhRNtgMzz90UF6hflTVFHLknf38PKtk8/4YvkmvZjBMcFEBGnidzea3Fuz802rd8ykRc6OxG18sCeLh1ftpt5mWPLPPUQH+3HZqF7t7vP61hM88f4BvL2EK8f04vbpCVwwKOqsJd/cU9Us357OW9vTKSivaVrv6y30jQikpKqOksq6FvvEhwcwMDqIuWN6c9moXlw6Mq7pi2DZXVP55b/28adNRzmQfYqD2afIL6vhkStH8INLhjS9ri1xoQG8fucU3vgyjac+PMijq/fw6qJWJ/IDwGYzHMg+xZbD+Ww9WoCfjxcJ0cEkRAcxMDqYYP/T/7ZeAuP6hePv03Ju8TU7T3Iop4wXbplIdmkVv/noEKuSM/ivKQOazvGH9Yd5YeNRpg6KYuW90/A6y+/RWcYYjuVX0DcikEA/58997mk0uX+bzWb1khl4EcQOd3Y0PaqwvIbnNxzhqnHxTBsc3SPnMMbwp41H+f26w0xJiOTZmxN5YPk3/PCtnay4bxqJ/SNa3e/jvdk8+cEB5oyKY0hsCCu/zuCjvTkM7xXC4Ji2664rauvZdqyQBmOYPSKOW6cNIMjPh9SCClILK8koriQi0JeE6GAG2pPlwOggAnzbTj5+Pl48s2AcQ+NC+M3HBxkUHcy7D8xgfL/WY2+NiHD79AQqahr4308OsTklj1kjWk5VUd9g46kPD/LBniwKyq27c42OD0MEvj5RREVt6/eZnzgggmV3TSUswJpaurqugWc/TWF8v3CuGWddVWxOyeeJtQeYOiia+PAAHn5nNx/uySaxfwRfnShixdfpfO+CgWccu6a+AT9vrw5VJRljKKyo5cvjhWxJsdoc8spquHRkHH9blNTj1VHGGP762XHG9glnxlCdEtuh+dxFZDFwL9bV6F+NMc+JSBTwNpAApAI3G2OK2zwILjaf+7GN8MZ3YMHfrHnTe1BNfQNvbEvj4uGxDO/Vxr1Le8iR3DLuWvY1GUVViMD3Lx7CTy8f3lSV0B3qG2w8unoP7+48yQ0T+/L0gnH4+3iTX1bDgpe+oLymnnfvn0FCTHCL/b5OLeJ7r25nbJ8w3rp3GgG+3lTVNvD+7ize2ZHBqaoz67gbicBFQ2O4bfpABkYHt/m6rjqeX06fiMB2vwzaU1PfwNznPgPgk4dmtihxP/3xQf6y5TjXjItnzug4LhoaS2yoP2AlroLyWtKLKqiuO30DlxMFFTyxdj9j+obzj7umEh7oy8tbjvHMx4d4694LmDHESnLZpVXMfe4zBkQF4eUl7MksYelVI7l35mBu/dt2dmeUsu6nF7eo/tqbWcrtr21n7tjePH1D6/fLffvrdD4/WkhaYQUnCiqa2h/CAnyYOTyWED8f3k7O4I8LE7k+sYO3fuyi1Tsy+dk7u4kO9mPjz2YRHtjF+yicR3rkZh0iMhZYCUwFaoFPgPuxkn2RMeYZEVkCRBpjft7esVwqua+63brl3MOHwMe/U7ueLKli/otbCfLztkqEUUEMiQ3mpqT+LS6lGz2+Zi/Lt6fjJXDL1AH85PLhxISc/ZzphZVsOZzHlsP5fHWiiPCg0yXRwTEhLJjcr90P9n8O5/Pg8m/w9/Xm+VsSeX93Fiu+ymBMnzD+uDCRoXGOf9EYY3hszV5WfJXBTy8fzo8uHdqi5HY8v5wFL31BWKAvj109ikExwQyICiKzuJIFL20jOtiPf94/wy0bATen5HHH37/m53NHcv8s65aGH+3N5oHl33DrtAE8NX9cp4736f4cHnzrG0bFh/HCLROZ98LnTEmI4rU7prR43cd7s7l/+TcE+nrz3MJErhzTG7A+T1c8t4WLhsbw19utEvaujBJu+9t2autt1NTbePG7k7hmfHyL4636OoNH/7mHvhGBDI4NbvoMJvaPILF/BD7eXjTYDAte+oL0okrW//SSTrU1FJbXsO14IRP6RdA/Kqjd16YVVnD1Hz+jf1QQh3PLuH16Ak9c5/63hOyp5H4TcKUx5h778i+BGuBuYJYxJltE4oHNxph2h3C6THKvKIDfj7AaUq/8dad3/79/p/DnzUe5alw8GUWVTSWZSfbL5tCA0wm38R/jjhkJALzxZRpBvt48MHsol46MY0BUUFM9ZVVtg3Wpa+9ed6KgAoD+UYFcOCSGytoGUpuVnAbHBPPaHVPOKBEbY3jjyzSefP8Aw+JC+NsdU+gbYZXUPt2fw5J391JeU9+0DqyS8Li+4VwyPJaZw6ySZHVdA9tPFLElJZ9dGcVcN6EPi2YktEjez284wrPrDvPg7CE8cuVIWvNNejG3vbq9RXWDn48XYQG+rHlgxln/oc9n9yxL5otjBWx8eBZl1XXMf3Erw3uH8vZ907t09bThYC73v/kNXl5WQ/LHiy9mRO8zv6TX7s5iRK/QM7a9+tlxnvrwIC/cMpG+kYEs+ttXRAT78ubdF/Djlbs4kV/OJw9dTB/7Z2NPZgk3vryNKQmRLLtzarv9+FNyypj3wmdcMy6e5xZObPN1xhj2nixl/YFcthzOZ8/JUoyx2kPefWBGm43q9Q02bvrLNo7mWTG+tPkoK77K4MMfX8TI3mEdefucpjH/drXKqqeS+yjgPWA6UAVsAJKB24wxEc1eV2yMiWxl//uA+wAGDBgwOS0trUtxdKuD78Pbt8Ld6zp9f9L6BhsX/u9GRseH8fc7rX2NMfx7fw4/fGsnY/uGs8x+2dz4jzE1IYrX75yCj7cXR/PKeebjg6w/mNd0zN5hAfQK8+dgThm19TYCfL2YNjiaWcNjuWREHAnRQS0+FMYYtp8o4v43d2CAl2+d3FSXvjujhKc+PMDXqcVcNjKOP94ykZBvXU3klVXz4sajFDdrbKypbyA5tZjCCqsOeFhcCOlFldTU2/Dz8WJgVBBH8sq5eHgs/3fjeOLCAliVnMGjq/dww6S+/P6mCe1+cMtr6jmeX05qYSWpBRXklVVz67SBLv9P6aj0wkrm/GELs4bHcjSvnFPVdXzwo5n0Du9676xNKXl8/40dLJjUj6dv6Fzpv8FmuOHPW8korqK23kZ0iB8r7p1Gn4hAUgsquPr5zxjfL5zl90yjtKqOa1/4HID3f3RRh0rjf1h3mD9uOMLf75zC7G+1NVTXNfDBnmz+sS2VPZmleAlMHBDJJcNjGRoXwqOr99AvMpBVP5je1K7Q2rGfv2Ui103oQ3FFLbN/v5kRvUJZed+0c9b1tLNq6208vmYvI3qHcs/MwV06Ro/dQ1VE7gYeBMqBA1hJ/s6OJPfmXKbkvuF/4PM/wGMnwbdzXe/WH8jlnn8k8/Ktk5k7tneLbesO5PLA8h2Mig/jjwsncuur24HW/zEO5ZzicG45afYGwOzSKkbHh3HJiFimJER1qK43taDCXp9eydKrRrH3ZClrdp4kJsSPh68Ywc1J/c/aw6O55r03vjpRxODYYC4ZHssFg6IJ8PVi+fZ0nvrwAIG+3tw2PYEXNx1lxpBoXrtjCr46MrNNz36awvMbj+LtJbx1zwVc0A2N2oXlNUQE+XXq79voUM4p5j3/Of2jglhx77QWXzSNV5qPXDmCL44V8HVqMf/8wQzG9evY4Kia+gbmPf85FTX1vH7XVLJLq0krrOBoXjkf7MmmqKKWoXEhLJo+kGsn9GnRNfPzIwXc8fevmJIQxet3TWnRTrEjrYibXt7G/MS+PPtfiU3rl29P4/E1+3jhlolcO6FPp9+LnlZcUcsP3tzB9hNFPDRnGA/N6VrnjXNyg2wR+Q2QCSzmfK2WeXMBlOXA/Vs7ves9y75mV0Yp25Ze2mpCa7xsthmDl5d06h+jK0or63jgrR1sPVqIn48X984cxA8uGdKiaqg7Hc0r56G3d7Lv5CnG9Anj7e9PP+PKQLVUVdvAfW8kc824eBZOHeDscAArwceHBRIe1PJzYozhgeXf8PG+HAB+d+N4burgWIVGO9KKufHlL2iecgJ9vZk5LIY7ZiQwfUh0m6XsNTsz+cnbu7luQh8euXIEnx0pYMvhPD47UkB0iB8f/Xhmi892g81w/YufU1BWy4aHL2m1zQus9o+D2WVNyyJwzbj4Hq0SPJ5fzt3LkjlZXMVvbxzP/Ildb2juyZJ7nDEmT0QGAJ9iVdE8BhQ2a1CNMsY82t5xXCK5GwO/GwrD58L8Fzu1a05pNTOe2cD3LxnCz+e2Xr8M1mXzw6t28/jVo1gwueenCK5rsPGvnSeZPiSafpE9X39dW2/j/d1ZzBoRS3QHGobV+aWkspYFL33B7BFx/GLe6C4dY92BXIora5v67ceG+ne42uTPm4/y209Smpb7RgRy8fBY7r5oUKtTO+xIK2LBS9vabPfJKqli1u82U9tga7F+WFwIHy+e2SPzAW07VsgP3tyBj5fwyu2T25zWoqPaS+6OFq3+KSLRQB3woDGmWESeAVbZq2zSgZscPMe5UZoJlQXQJ7HTu76TnIHNwMIp7ZdkZo+IY8cv5pyzOkBfb69Ol64c4efjdU6+tJRzRAT5sf6nlzj0+b18dPsD19pz/yVDCPbzoa7BxqwRsQyJDWk3lskDo7g+sQ+vfnaCW6cNPKNB9sVNRzEYNv9sVlMV1PqDufzwrZ2s3pHZ7VdTaYUV3LPsa/pEBPLaHVN6vMOAQ19NxpiZxpjRxpgJxpgN9nWFxpjLjDHD7I9F3RNqD8veZT32Od2aX13XwHPrD5NdWtXmbjab4e3kDGYMie5Qv2pXbdxRqiOc+fkVERbNSOCemYMZGhfaoVh+dsUIjLEaXZvLKKpkVXIGC6cMICEmmABfbwJ8vblmXDyTBkTwh/WHqaxtezxFZ9U12Fi8chfeXsKyu6aek55g2trVKGsXiDf0Ot039p0dmTy3/gg/XrGTBlvr1VefHy0gs7jKZepMlVKn9Y8K4vbpA1m9I5NDOaea1r+46SiC8MDsIS1eLyI8dvUock/V8NrnJzp1rvKaet5JziA59czy7Asbj7Iro4Tf3DCuqTtpT9Pk3ih7F8SNauolY4zhjW2phAf68nVqMS9tPtrqbiu/TicyyJcrx3T9clMp1XN+eOlQQvx9+N+PDwFWN9R3dmTy3QsGtNp3PikhiitG9+LlLcdbzE3UlqN55fzqvX1M+80GHlm9hxtf3sZTHxyg2j4V9I60Iv608QgLJvVj3vhz13NHkztYjalZOyE+sWnV9hNFHM4t5/GrR3HthD48t/4IuzJKWuz25fFC1h3I5YZJ/c6YuEkp5Roigvx4cPZQNqXk88WxAl7YeAQfL2kaHdyaR+eOpKqugRc2HGnzNQ02w+KVO5nz7BZWfJXBFaN78c4PpnP79IG8+vkJ5r+4leTUIhav3EXfyECeuK5rjdBdpX3VwN6YWtiiMfWNbWmEB/py7YQ+XDm2N9+kFfPQyp18+OOZ+Hp78Yf1h3l5yzEGRgVx90WDnBe7UuqsFs1I4B/b0vjFv/aRVljJHTMS6BXW9oCxoXEhLJzSn+Xb07njwkEMijmzPe33n6bw3q4svn/JYO6dObhp6pApCVHMHhHXVIr39hJWfX96j3VDbouW3OGMxtTcU9X8e38ON03uR6CfN+GBvjx78wTSiir52Tu7ueGlrby0+RgLp/Tnwx/PPGd1aEqprgnw9ebhK4ZzPL8CX2/hB5e0XWpvtHjOMPx8vPjhW9+Qe6q6xbZP9mXz583HuGXqAJZeNeqMOaFmj4zjk4dmsmBSP564djSTB7Y7jrNHaHKHMxpTV3yVTr3NcOu001OgXjA4mvsvGcLH+3I4WVzFX26bzNM3jG9zcIRSyrXMT+zLrBGx/PiyYU2zbbYnLjSAF787idSCCq7/01b2nSwFrDr2h1ftZkL/iHarWmJC/Pn9zRO4bXpCd/0KnaKZCaz6dntjal2Djbe2p3PJ8NgzJt76yeXDGRAVxKUj44hr55JOKeV6vLyE1+/s3JxRs0fGsfr+Gdz9+tfc9PI2nr5hHC9sPEKArzcvfW+SS7e1acndGKtaxl7f/un+XPLKarh9+pk3LvD19mLh1AGa2JXyIKPiw/jXDy9keO9QHnp7F6mFlbzw3YkuXx2rJffGxlR7T5l/bEulX2TgGXfJUUp5rrjQAN6+bxrPfHyIsX3Dm26C4so0uTdrTD2aV872E0X8fO7ILs2qp5RyXwG+3ufVDUC0WiZrZ1Nj6vu7sxCBBZN69nZgSinV0zS5Z+2CuNEYnwA+3JvN1IQorVNXSp33PDu5NzWmTiAlt4yjeeXMc8GJ/ZVSqrM8O7k3a0z9cE82XgJzx/Q++35KKeXiPDu55x0EwPQawwd7spk+JLpDgxuUUsrVeXZyL7DmeE5p6MOJggquGadVMkop9+DhyT0FgmJ473A13l5yxo2tlVLqfOXhyf0IJmYYH+7JZsaQaKKC/c6+j1JKnQc8PLkfpihoEOlFlcwbH+/saJRSqts4lNxF5Ccisl9E9onIChEJEJEoEVknIkfsj+d+rsuOqCiEykJ2Vsbi4yVcqb1klFJupMvJXUT6Aj8GkowxYwFvYCGwBNhgjBkGbLAvux57Y+rH2aFcNCyGiCCtklFKuQ9Hq2V8gEAR8QGCgCzgemCZffsyYL6D5+gZBSkAbC+P5ZpxWiWjlHIvXU7uxpiTwP8B6UA2UGqM+RToZYzJtr8mG2h1ekURuU9EkkUkOT8/v6thdF3BEerEnxyJ4fLRenNrpZR7caRaJhKrlD4I6AMEi8itHd3fGPOKMSbJGJMUGxvb1TC6ruAw6dKHpIRorZJRSrkdR6pl5gAnjDH5xpg64F1gBpArIvEA9sc8x8PsfnW5h9hf15vLR2tDqlLK/TiS3NOBaSISJCICXAYcBNYCi+yvWQS851iIPaCuCp9TGRyz9eEKrZJRSrmhLt+swxizXURWA98A9cBO4BUgBFglIndjfQHc1B2BdqvCowiGyvAh9I8KcnY0SinV7Ry6E5Mx5lfAr761ugarFO+yyk8eIAToN2yCs0NRSqke4ZEjVNNSdmMzwqSJSc4ORSmleoRHJvfKrP1kSRxjB2p9u1LKPXlccq+uayCk7AQVYYOx2oGVUsr9eFxy33okj0FkEdRntLNDUUqpHuNxyT15924CpI7eQ8Y5OxSllOoxHpXcbTbDySN7APCNG+nkaJRSqud4VHI/nFdGbE2qtRA7wqmxKKVUT/Ko5F5QVssQyaLOPwqCopwdjlJK9RiPSu4lVbUM8cqmLnKos0NRSqke5VnJvbKOoXISYoc7OxSllOpRHpXca07lES1l+PbS+nallHvzqOTuXXwcAN84Te5KKffmUcndtyzTehIx0LmBKKVUD/Oo5B5QmWU9iejv3ECUUqqHeVRyD6vK4pRXOPgFOzsUpZTqUR6V3CPqsiny1dvqKaXcn0cl99j6XE75xzs7DKWU6nEek9yNzUZvk09VUB9nh6KUUj2uy8ldREaIyK5mP6dE5CERiRKRdSJyxP4Y2Z0Bd1V1SS4BUkdNSD9nh6KUUj2uy8ndGJNijEk0xiQCk4FKYA2wBNhgjBkGbLAvO11FntXH3RY+wMmRKKVUz+uuapnLgGPGmDTgemCZff0yYH43ncMh1fknAPCO1G6QSin3113JfSGwwv68lzEmG8D+GNdN53BIQ1EaAH5ROoBJKeX+HE7uIuIHXAe808n97hORZBFJzs/PdzSMs5+vNINiE0JIhE71q5Ryf91Rcr8K+MYYk2tfzhWReAD7Y15rOxljXjHGJBljkmJjY7shjPb5lGWQaWKICPLr8XMppZSzdUdyv4XTVTIAa4FF9ueLgPe64RwOC6g4yUkTS0Sgr7NDUUqpHudQcheRIOBy4N1mq58BLheRI/Ztzzhyjm5hDKHVWWQRS5Cft7OjUUqpHufjyM7GmEog+lvrCrF6z7iOykJ8bTUU+vZGRJwdjVJK9TjPGKFaYvWU0akHlFKewkOSezoAlYE69YBSyjN4VHKvDenr5ECUUurc8JDknsEpgvEL0T7uSinP4CHJPZ0sE0NEkHaDVEp5Bo9I7raSNNJtMdrHXSnlMdw/uRuDlKSTaWK15K6U8hjun9yripG6SjJNLOE69YBSykO4f3K393E/abRaRinlOTwguVvdILVaRinlSTwmuWeYGCICtVpGKeUZPCK513qHcIpgwrXkrpTyEB6Q3DMo9e+Nlwih/g7Nk6aUUucND0ju6RT69CI80BcvL50RUinlGdw7uRsDJenkePXSOzAppTyKeyf3qmKoLSPLxBCu3SCVUh7EvZN7aSYA6bZo7QaplPIo7p3cK6x7c2fUhuoAJqWUR3Hz5F4IQEZNkNa5K6U8iqM3yI4QkdUickhEDorIdBGJEpF1InLE/hjZXcF2WkU+AGnVQVrnrpTyKI6W3P8IfGKMGQlMAA4CS4ANxphhwAb7snNUFmC8fDhFkNa5K6U8SpeTu4iEARcDfwMwxtQaY0qA64Fl9pctA+Y7FqIDKgpoCIjC4KXJXSnlURwpuQ8G8oG/i8hOEXlVRIKBXsaYbAD7Y1xrO4vIfSKSLCLJ+fn5DoTRjspC6vytW+vpvDJKKU/iSHL3ASYBLxljJgIVdKIKxhjzijEmyRiTFBsb60AY7agooNrPqvLXeWWUUp7EkeSeCWQaY7bbl1djJftcEYkHsD/mORaiAyoLqPCJANCukEopj9Ll5G6MyQEyRGSEfdVlwAFgLbDIvm4R8J5DETqiopAyr3AA7QqplPIojk6T+CNguYj4AceBO7G+MFaJyN1AOnCTg+fomvoaqCmlWKzkHhagM0IqpTyHQxnPGLMLSGpl02WOHLdbVFoDmIpMGKEBPvh4u/d4LaWUas59M15FAQD5tlDtBqmU8jjum9wrreSeUx+s3SCVUh7HfZO7fV6ZrLpgLbkrpTyO+yZ3e8k9oyZY55VRSnkc903uFQUgXmRU+WvJXSnlcdw4uedjgqIprqrXOnellMdx3+ReWYgtMBqbQUvuSimP477JvaKgadIwrXNXSnka903ulacnDdOpB5RSnsZ9k3tFAeXe1tQDMSGa3JVSnsU9k3tDHVSXUIKV3OPCApwckFJKnVvumdwriwAoMGGAltyVUp7HPZO7/cbYuQ0hhAf64u/j7eSAlFLq3HLP5G4fnXqyNpjYUH8nB6OUUueeeyZ3+4yQ6TVBxIZocldKeR73TO72udyPVwYQF6bJXSnledwzuVcUYBCOlftryV0p5ZHcM7lXFkBgFBV1RuvclVIeyT2Te0U+dQHW1AOa3JVSnsihe6iKSCpQBjQA9caYJBGJAt4GEoBU4GZjTLFjYXZSRWHT1ANxoTqASSnlebqj5D7bGJNojGm8UfYSYIMxZhiwwb58blUWUOEdAWjJXSnlmXqiWuZ6YJn9+TJgfg+co30VBZR6WVMPaHJXSnkiR5O7AT4VkR0icp99XS9jTDaA/TGutR1F5D4RSRaR5Pz8fAfDaMbWAFXFFJlQfLyECJ3uVynlgRyqcwcuNMZkiUgcsE5EDnV0R2PMK8ArAElJScbBOE6rLAIMuQ2hxIT44+Ul3XZopZQ6XzhUcjfGZNkf84A1wFQgV0TiAeyPeY4G2Sn2qQey6kJ0AJNSymN1ObmLSLCIhDY+B64A9gFrgUX2ly0C3nM0yE6xTxqWqVMPKKU8mCPVMr2ANSLSeJy3jDGfiMjXwCoRuRtIB25yPMxOsM8rc6I6kARtTFVKeaguJ3djzHFgQivrC4HLHAnKIY3zylQEMEWTu1LKQ7nfCFV7yb3IhBCnyV0p5aHcL7lXFlDvH049PtrHXSnlsdwvuVcUUOsXDegAJqWU53K/5F5ZSIVvBACxITqvjFLKM7lfcq/Ip0ynHlBKeTg3TO4FFBFGqL8PgX56Y2yllGdyr+Rus0FVEfm2UC21K6U8mnsl96piMDZy6kOI0eSulPJg7pXc7fPKnKwJ1pK7UsqjuVdyt88rk1YdqAOYlFIezb2Se3kuAGm1WueulPJsbpbcrZJ7gQnXGSGVUh7NvZJ7RR5GvCkmREvuSimP5l7JvTyXGv8oDF6a3JVSHs3Nkns+Fb7WvDJxoTr1gFLKc7lZcs/llHckXgJRwX7OjkYppZzGvZJ7RT5FhBMd4o+33hhbKeXB3Ce5GwPleeTatKeMUkq5T3KvKgZbHVkNocSFaXJXSnk2h5O7iHiLyE4R+cC+HCUi60TkiP0x0vEwO6BpdGqoltyVUh6vO0rui4GDzZaXABuMMcOADfblnmcfnXqiOki7QSqlPJ5DyV1E+gHXAK82W309sMz+fBkw35FzdFh5HoBV567JXSnl4RwtuT8HPArYmq3rZYzJBrA/xrW2o4jcJyLJIpKcn5/vYBg0VcvkG03uSinl09UdRWQekGeM2SEiszq7vzHmFeAVgKSkJNPVOJqU52ITH0oJpn9kkMOHU0p1j7q6OjIzM6murnZ2KOetgIAA+vXrh6+vb4f36XJyBy4ErhORq4EAIExE3gRyRSTeGJMtIvFAngPn6LjyfCp9IzFVXgyJCzknp1RKnV1mZiahoaEkJCQgouNPOssYQ2FhIZmZmQwaNKjD+3W5WsYYs9QY088YkwAsBDYaY24F1gKL7C9bBLzX1XN0SnkuxV6R9A4LIMTfke8spVR3qq6uJjo6WhN7F4kI0dHRnb7y6Yl+7s8Al4vIEeBy+3LPq8gjtyGMIXHB5+R0SqmO08TumK68f91SxDXGbAY2258XApd1x3E7FUN5Phm1wxkaq1UySinlHiNUbTaoyCO7IUzr25VSCndJ7tUliK2eAhOuJXellENCQs6eQ/7whz8QEBBAaWkphYWFJCYmkpiYSO/evenbty+JiYl4e3szevRoEhMTiYqKYtCgQSQmJjJnzpxz8Ft0U7WM09lHp+abcC25K+XCnnx/PweyTnXrMUf3CeNX147p1mOezYoVK5gyZQpr1qzhjjvuYNeuXQA88cQThISE8LOf/azF6++44w7mzZvHjTfeeM5idI+Su310arlvFHE6gEkp1czPf/5z/vznPzctP/HEEzz55JNcdtllTJo0iXHjxvHeex3v1Hfs2DHKy8t56qmnWLFiRU+E3C3co+RuH50aFNVHW+WVcmHnuoQNsHDhQh566CEeeOABAFatWsUnn3zCT37yE8LCwigoKGDatGlcd911HcofK1as4JZbbmHmzJmkpKSQl5dHXFyrA/Gdyk1K7la1TGRcXycHopRyNRMnTiQvL4+srCx2795NZGQk8fHxPPbYY4wfP545c+Zw8uRJcnNzO3S8lStXsnDhQry8vLjhhht45513evg36Bq3KLnXlOYgxpv4XvHODkUp5YJuvPFGVq9eTU5ODgsXLmT58uXk5+ezY8cOfH19SUhI6NAgoT179nDkyBEuv/xyAGpraxk8eDAPPvhgT/8KneYWJfeKwiwKCGdor1Bnh6KUckELFy5k5cqVrF69mhtvvJHS0lLi4uLw9fVl06ZNpKWldeg4K1as4IknniA1NZXU1FSysrI4efJkh/c/l9wiudeW5FBgwhmi3SCVUq0YM2YMZWVl9O3bl/j4eL73ve+RnJxMUlISy5cvZ+TIkR06zsqVK/nOd77TYt13vvMdVq5c2RNhO0SMcXxCRkclJSWZ5OTkLu+f+9up7C8PYuavNuLr7RbfV0q5jYMHDzJq1Chnh3Hea+19FJEdxpik1l7vFpnQv6aAGv9oTexKKWV3/jeo2myENhRjQmOdHYlSyk3s3buX2267rcU6f39/tm/f7qSIOu+8T+515QX4YsMvXHvKKKW6x7hx45pGnZ6vzvt6jOyT6QCERvdxciRKKeU6zvvknpudAUBU7/5OjkQppVzHeZ/cS/Kt5N6n7wAnR6KUUq7jvE/ulUXZAARHaZ27Uko1Ou+Te/2pPOrwhYAIZ4eilHJBJSUlLWaF7Kirr76akpKSLp2zvr6emJgYli5dCsCvf/3rpjnfvb29m56LCImJiYwePZrAwMCm9atXr+7SeZvr8iAmEQkA/gP4Y/W6WW2M+ZWIRAFvAwlAKnCzMaa4vWN1dRCTMYa1T1zHJX4pRDx+uNP7K6V6XovBNx8vgZy93XuC3uPgqrZv1Zyamsq8efPYt29fi/UNDQ14e3t3byx2H330Eb/+9a/Jycnh6NGjLWabDAkJoby8vEMxNncuBzHVAJcaYyYAicBcEZkGLAE2GGOGARvsyz0ir6yGCFsJ9YExPXUKpdR5bsmSJRw7dozExESmTJnC7Nmz+e53v8u4ceMAmD9/PpMnT2bMmDG88sorTfslJCRQUFBAamoqo0aN4t5772XMmDFcccUVVFVVtXvOFStWsHjxYgYMGMCXX37Zo79fm4wxDv8AQcA3wAVAChBvXx8PpJxt/8mTJ5uuSE4tNPt/Oc4UvjK/S/srpXregQMHnHr+EydOmDFjxhhjjNm0aZMJCgoyx48fb9peWFhojDGmsrLSjBkzxhQUFBhjjBk4cKDJz883J06cMN7e3mbnzp3GGGNuuukm88Ybb7R5vsrKShMfH28qKirMX/7yF/OjH/2oxfbg4OB2Y2xLa+8jkGzayKsO1bmLiLeI7ALygHXGmO1AL2NMtv2LIxvosVnsJw+MYlRYNRGxOo+7Uqpjpk6dyqBBg5qWn3/+eSZMmMC0adPIyMjgyJEjZ+zTeP9TgMmTJ5Oamtrm8T/44ANmz55NUFAQCxYsYM2aNTQ0NHT3r3FWDiV3Y0yDMSYR6AdMFZGxHd1XRO4TkWQRSc7Pz+9aALYGpKIAr9BeXdtfKeVxgoODm55v3ryZ9evXs23bNnbv3s3EiRNbndfd3//07Tu9vb2pr69v8/grVqxg/fr1JCQkMHnyZAoLC9m0aVP3/hId0C29ZYwxJcBmYC6QKyLxAPbHvDb2ecUYk2SMSYqN7eK8MJVFYBogxPVucaWUcg2hoaGUlZW1uq20tJTIyEiCgoI4dOiQw/Xjp06d4vPPPyc9Pb1pzvcXX3zRKfda7XJyF5FYEYmwPw8E5gCHgLXAIvvLFgEdv/NsZ1XYvzeCddIwpVTroqOjufDCCxk7diyPPPJIi21z586lvr6e8ePH88tf/pJp06Y5dK53332XSy+9tEVJ//rrr2ft2rXU1NQ4dOzOcqQr5HhgGeCN9SWxyhjz3yISDawCBgDpwE3GmKL2jtXl+dwLjsDGp2DmwxA/vvP7K6V6nM7n3j062xWyy7NCGmP2ABNbWV8IXNbV43ZKzDC4edk5OZVSSp1Pzvspf5VSyhkefPBBtm7d2mLd4sWLufPOO50UUUua3JVSPc4Y02KUpjt48cUXz9m5ulJ9ft7PLaOUcm0BAQEUFhZ2KUEpK7EXFhYSEBDQqf205K6U6lH9+vUjMzOTLo9nUQQEBNCvX79O7aPJXSnVo3x9fVuMCFXnhlbLKKWUG9LkrpRSbkiTu1JKuaEuj1Dt1iBE8oE0Bw4RAxR0Uzg9QeNzjMbnGI3PMa4c30BjTKvzr7hEcneUiCS3NQTXFWh8jtH4HKPxOcbV42uLVssopZQb0uSulFJuyF2S+ytnf4lTaXyO0fgco/E5xtXja5Vb1LkrpZRqyV1K7koppZrR5K6UUm7ovE7uIjJXRFJE5KiILHGBeF4TkTwR2ddsXZSIrBORI/bHSCfG119ENonIQRHZLyKLXSlGEQkQka9EZLc9viddKb5mcXqLyE4R+cDV4hORVBHZKyK7RCTZBeOLEJHVInLI/jmc7irxicgI+/vW+HNKRB5ylfg667xN7iLiDbwIXAWMBm4RkdHOjYrXsW4S3twSYIMxZhiwwb7sLPXAw8aYUcA04EH7e+YqMdYAlxpjJgCJwFwRmeZC8TVaDBxstuxq8c02xiQ265vtSvH9EfjEGDMSmID1PrpEfMaYFPv7lghMBiqBNa4SX6cZY87LH2A68O9my0uBpS4QVwKwr9lyChBvfx4PpDg7xmaxvQdc7ooxAkHAN8AFrhQf0A/rH/xS4ANX+xsDqUDMt9a5RHxAGHACe0cOV4vvWzFdAWx11fg68nPeltyBvkBGs+VM+zpX08sYkw1gf4xzcjwAiEgC1j1wt+NCMdqrPHYBecA6Y4xLxQc8BzwK2Jqtc6X4DPCpiOwQkfvs61wlvsFAPvB3e7XWqyIS7ELxNbcQWGF/7orxndX5nNxbu2eX9uvsABEJAf4JPGSMOeXseJozxjQY67K4HzBVRMY6OaQmIjIPyDPG7HB2LO240BgzCau68kERudjZATXjA0wCXjLGTAQqcMEqDhHxA64D3nF2LI44n5N7JtC/2XI/IMtJsbQnV0TiAeyPec4MRkR8sRL7cmPMu/bVLhUjgDGmBNiM1YbhKvFdCFwnIqnASuBSEXnTheLDGJNlf8zDqi+e6kLxZQKZ9qsxgNVYyd5V4mt0FfCNMSbXvuxq8XXI+ZzcvwaGicgg+zftQmCtk2NqzVpgkf35Iqx6bqcQ6w7FfwMOGmOebbbJJWIUkVgRibA/DwTmAIdcJT5jzFJjTD9jTALW522jMeZWV4lPRIJFJLTxOVa98T5Xic8YkwNkiMgI+6rLgAO4SHzN3MLpKhlwvfg6xtmV/g42elwNHAaOAY+7QDwrgGygDquUcjcQjdUAd8T+GOXE+C7CqrraA+yy/1ztKjEC44Gd9vj2Af/Pvt4l4vtWrLM43aDqEvFh1Wnvtv/sb/yfcJX47LEkAsn2v/G/gEgXiy8IKATCm61zmfg686PTDyillBs6n6tllFJKtUGTu1JKuSFN7kop5YY0uSullBvS5K6UUm5Ik7tSSrkhTe5KKeWG/j9YEMCV/3rlgAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_ATT','train_ATT']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA1rElEQVR4nO3deXiU5dX48e/JQkJCFgIJBAIk7BAgYRFREBcEgSq4AILLa9VK61LF2lasb6u2ta9W3Kv+StVqLWJZyuKGLC4ICsqSsO8JJCRkX8lCZub+/fEMIYGEJZMwk8n5XNdcM/OsJ0M4c+d+7vs8YoxBKaWUd/FxdwBKKaUanyZ3pZTyQprclVLKC2lyV0opL6TJXSmlvJCfuwMAaN++vYmNjXV3GEop1axs3rw51xgTWdc6j0jusbGxbNq0yd1hKKVUsyIih+tbp90ySinlhTS5K6WUF9LkrpRSXkiTu1JKeSFN7kop5YXOmdxF5F0RyRaRHTWWRYjIKhHZ73xuW2PdEyJyQET2ish1TRW4Ukqp+p1Py/09YPxpy2YDa4wxvYA1zveISH9gOhDv3OdNEfFttGiVUkqdl3OOczfGrBWR2NMWTwaucr5+H/gaeNy5/CNjTCWQIiIHgOHA940Ur1INV3QUdi6BiqLz30cEul4GcVeCTxP3YpYcg90fQ0QcdL8amxEWb0lnUEw4/aJDz7m7regY21e+R1xQOeGtWzVOTCLQ7wboOBC7w7B061ESu4bTI7LNmdsez7U+39LsCztHp8HQayz4+tdeXpYPO/8LJVkXdrzoBOh93ZnHO5fyAiv+4swL2+9cOsRDnwkY31Z8si2TqJAALu3eDsoLYcdiCG4P/Sc37jlp+CSmDsaYTABjTKaIRDmXdwY21Ngu3bnsDCIyE5gJ0LVr1waGodQ52E7A3s9g67/h4BowDkAu4ADO+x2EdYXBt0Pi7RDepfHis1fB/pWw5QPr2dgBqGrTif/aR/N64QiyfDow69re/Hx0d/x8T/uCsdvgwCqOb/gngSmrGIwDB4Lhwn7K+hlY9wp5o//Mz3fGs+lIIV0iWvPpw1cQGugPDjsc/BK2/Av2fg6Oqgs8s/PzbdMBEqZbn29RmvV57P0M7CcadrzgSOt4g++EyD71b+5wQMo31u/H7o/BXulc0Tif3sl4HIERrPK/ijdyhxMupTzR8UcGFX+D2CpgwJQmSe5yPjfrcLbcPzHGDHC+LzTGhNdYX2CMaSsibwDfG2P+7Vz+DvCZMWbx2Y4/bNgwozNUFXkH4T93wKhfwaCpF7Zv1i7Y+oHVEqooPrXcYbMSTkinU8k5Iu78j1tVAXs+sY596GtAoMfVMPgO6Hs9+AVcWJw1pf0Ii+6BoiPO5DYDx8Bb+ea7b/FLmsdIn234YDghAdgdBh8R/P0EnxqJxzhsiKOKXBPGx3IlJN7BHzfauGVIDHOmJjQ8tpPHP55L5rt30invO5ZxJYdH/JlX16ZzZx/DUzFbkKQPoSQDgtrBoOkw5E6I6nf+J7BXwf5V1ue774vqLzdatz11vA7xF3A8GxxY7TzeCuvff8AtcMOrEBBSe9ucvbDwbsjeCYHhMGia9WUQPej8z3cuDjsb1yymaP27XGV+oJVYP1+xac3X/lfSe+ID9B082voLqQFEZLMxZlid6xqY3PcCVzlb7dHA18aYPiLyBIAx5v+c230BPG2MOWu3jCZ3xfE8eOdayD8EAaHw4EYI7XT2fYyB5I/gx3/A0c3g4w99xkPbGslbfCB2FPS4BnxcvPxTcBiS5sHWeVCc7kxAt1oJoeOA8z+OMfDDXPjiSetnHP9/0Os60otP8NtF2/juYB5j+kbx/Ni2tE/5BMryOJBTyrf7c7E7DJEhp75QKqocrCnthr3nOP4yZQgdQgN5aeVeXvvyAK9OT2Ry4qk/nDOLyvnrir0kxITxP5fF4uNTO6EcK6rguc93c7SwvHpZcbmN/VlFzIlcwU0l85Co/qRXtiamaBMOfPDpOcb6ouszkewyB8+v2Eu/6BDuHhmH72nHzy6p4LnP95CWX1bnxxJuz+eyim8p9GnLj4EjsMn5dS2N6deBn42KO/OvmtJs+PFtWPsCRPTATPsXf9kESWmFXF7+DfcXv0KlBPB+yEw2BI6i6jzPd7rRvSL5xVU98D/t/IVlJ/j9sp18nJxBQpdwXrmhC3HHvoDAMDa0uozHlu4ns6icR8b05pFrezXo3E2R3F8A8owxz4nIbCDCGPNbEYkHPsTqZ++EdbG1lzEnv47rpsm9hauqgH9NhoytVgvrk1lWMp7+4dlbNN/9DVY+CZH9rBbeoFut/sum5rDDoa+sP+X3fGp1HUQnWjEMmAKtw+vft7IUPn7Y+guj9wS46S1MYDgLN6fzx493YYzhDzf0Z9qwLshpP3tWcQXPr9jDsaKK6mU+IvxkUDTTLzm1vc3uYPrcDew5VsKnD4+ia0QQy5Iy+MOyHZRW2nAYuKx7O16YOoiYtkEYY1ienMHvl+6gym5I7BJe/bGLwPgB0dw+vCs+B9fA0l9gWgWzwHYlbxVeytu/nEzPqDZ8si2D/126g+LyKhwGhsdGMGdqAl3bBQHw2fZMnlyynbITdoZ0bdvQhuoZjlfaSE4vYnDXcF6alkhc++AzN0pdBwvvpqq8mMcrfsrVYRncUL6cva3681rEkxT4Nvx3puyEnaS0QgZ2DuOlaQn06mD9dfDV3mweX7SN/OMneHhMLx64qscZXz7FFVX88eNd9Ihsw/1X9WjQ+V1K7iIyH+viaXsgC3gKWAosALoCR4Cpxph85/ZPAvcANmCWMebzcwWoyb0Fczhg8b3WhbMp/4QBN8P612DV70+9r8EYYyWxXcthwf9YF/umvt/0FzvrYY7nItsXWn3E2TvBLxCCo06t57T/X5UlUFmM4+r/xVz+CPllNn63ZAerd2cxPC6CF6cm0CUiyOW40gvKmPjqt8S2DyambWs+236Mod3aMmdqAj+k5PHHj3chIsye0JfvD+bx6fZMhnQN58X6EmT1D2T9PFkllUx49Vs6hAbSM6pNdev0xakJJKcV8vTyndiNYfaEvmw+XMCypAwSYsJ4cVoiPaPquBjrgpNfTJU2O7+b2I87Lu12xl8le/bvo+SDO7nEZ4+1YMSDMPaZC7/oWocVOzL53RLri/Oxsb1JzStj/g9H6N2hDS9NS2RA57Cz7l/9O90ALrfcm5om9xZs9TOw7iW49mkY9ai1zG6zumgK0+ChHyEoAmMMS7Ye5dlPd3NX11x+mTYL6TgQ7voY/Fu7JfRPt2Xy9Mc7uax7O/44qT/hRbtg20Jr1AWQXljG1iOFVNoc1fs4jPBfxxVscPSvXtbKz4ffXteHe0bGnZGUXLFiRya/+PcWWvn68OjY3swc3b26qyQtv4zHFibzQ0o+/r7Co2N78/PRPc7oSjmbL/dkcc97m/DzER4Z04v7a7ROjxaW89tFyaw/kIefj9Tbem0sWcUV/HbRNr7Zl8Oonu3565RBdAq3fi9KK23c8Po6qk5UsnLEDoKi+0C/6xv1/DkllfxuyXZW7cpCBGZe0Z1Hx/Ym0L9pR4JrcleeafP7VhfF0J/C9a/U7oI5tgPmXgkDppA37jWeXLKDFTuPcUX7Ul4u+TWVPq3JuGU5lwzoe9HDLiw7wR+W7WR5cgY9IoM5nFdGRHArnp8yiKv7RFFUVsVTy3ewNCmDgZ3DGNe/Q73H8vERrovvQM+okHq3ccWKHcfoHhlM7w5nHt/hsLpj+kaH0LfjuYda1mXVrixi2rauc6imw2H4ZHsmPSKDie909tZrYzDG8OEPR3j20934+gjPTIrnpsGdeWxhMku3HmX+fSOsIYhNeP4vdh4jKjSQIV3bnnuHRqDJXXmeA2tg3lTofhXc9p+6/zz+8llY+1d20R2bQ+gYFkikIxt71Qnu9fsL3+S35aeXx/L4+L60btU0LaRDOaW1LjDmllby3Od7yCs91Ze651gJv1qQxL6sUiYndmLjoXxySyt56JqePHh1zzMutKmmdTjvOI8tSGbT4QISu4STlFbII2N68ejY3u4OrdFpcleeJWsnvHMdtO0Gd38OgWe2+oorqnh2WRLx25+nX2A+/aJDaRPgZ30JjHqU8g5DeX7FHt77LpXukcG8NC2RxC7hjRrmDyn5TJ/7PY7T/ov0imrDy7fW7kutqLLz8qp9zP32ED0i2/DytEQGxjR9a1XVze4wvP3tIV5cuY/ELuF8eN+lTdYl5E6a3JXnKM6Et8dYk4l+tgbCzpzj9t3BXH6zcBuZReU8cFVPHh7Ti1Z+df/HXH8gl98sTCarpJIHr+rBQ9fUv+2FKCw7wcRXv8Xfz4cXpiTgUz16RBjQOZQAv7r/UkjLLyMqNKDe9eriyi6uILS1f5P3fbvL2ZK7R9xmT3kp2wn49FdwPOfUspw91rTrez4/I7FXVNl5fsUe/rk+lbj2wSy6//Jz9l2O7NmeFY+O5pnlu3jtywOs2ZPNlb3rvKVknXx9hOsHdaJPx1N90sYYHl+8jZzSShbffzmDYsLP+3iNMdJFNZ6o0EB3h+A2mtyVayqKrElHdQ3l2vK+NVOww4BTE4iCI+H6l636HzUkpRXyqwVJHMo5zl2XdWP2hH7n3Y8eGujPi9MSGBffgaeX7+Qf3x467/DtDsPfvznEr8b15r4rrNEk/954hC92ZvHkxH4XlNiV8iSa3FXDGAPrX4E1f4Srn4TRv669/kQZrJ1jFd26+/N6JyNV2R28vmY/b3x9kKiQAP5976WM6tWwSSXXxXfkuviOF7RPXmklTy7ZwXOf72H1rixmju7Onz7ZxZW9I7l31AWUKVDKw2ifu7pw5YWw9AHY+6k1Yae8AH6xjoq2vXh4/lZySiuZXLaYn5a+wx8i/sqgkT/hliGdz5iosS/LGmWy42gxNw/pzFM3xBPW2vVJJRfKGMPSpKP8YdlOSipstG8TwOePXFFrmr9Snkj73FXjydxmzQwtSoPxz1tFmd4YDssfYs0l77FyVxaju7bilrKFbAscykZ7P/61MJmVO4/xl5sH0r6NVQTr3XUpvLByLyEBfvy/O4YyfsCFtbgbk4hw0+AYLo1rx2tr9jNlaIwmdtXsactdnZvdZpXL3fIvq9JecKQ15b/rpdb6bQvhvz9jfsQDvFI6hu8v34zP13+B+77EHj2Ed9YdYs4X+wgJ9OO34/uweMtRfkjJZ2z/DvyfM+ErpS6cttxVw1SVw7cvWRdFSzKtpH7pL2DkLGhTY0TKwCmcSPoPkw++Q8WA4fh8/zerHG7nofgCM0f34MreUfxqQRKPL95OSIAfc6Ym1NlVo5RqHNpyV3XLT7G6X45tg17XWRUPe4+vt9DSgjXfM2HtTQT7g4+tAu7/Djr0r7XNCZuD5ckZXNajHZ3D3VMPRilvoi13L7blSAFFZVXV7/18heFxEXVPoslPgfnTYeyfoPe4+g+65zNY8gsQIWPi+0QMvuGck0Dm7baTHXw3D5W9CQOnnpHYwSqQNWVozHn/bEqphtPk3oz9mJrP1P935n1Q+nYM4cVpCbWLNRkDHz9iTSL64gmrXrrvaf/8xsCXf4JvX4ToBHZf8Tcm/CuNnuvWnXU6/YHsUpLTi7hh4r0QOgh6neWLQyl1UXhfsYUW5LU1+2nfphWL77+cpQ+OZOmDI3l9xmDyjp/gxjfW88ZXB7DZneVmt/7buldk3+sh7wBs+8+ZB0yebyX2wXfCPSt58cdKQgP9KKmo4qY31/Pq6v1U2R1n7LZkazo+ApMSYyBxBgQ3XeU9pdT50eTuyXL3w9tjrXtLniYprZC0A9v5JOiPDK3YQGKXcBK7hHNDQidWzhrNuPiOvPDFXqb+/XtyMlKtW7p1GwXTPrDuNv/Nc1Z5gJNKsmDFE9akoxteY2dOJat3Z/OzK7qzctaVXD8ompdX7+OWt77jQHZJ9W4Oh2Hp1gyu6BXZoqd6K+VpNLl7quO5MG8KpP8AC39q3YKuhvdWbeJfAS/QsXibdZPf9M3V69oGt+KN24bw2ozB7D1WTMr792PslTDpNeuORdf8LxQescoDnPT5b6zRMZNeBx8f3vzqICEBftx1eSxhQf68Mn0wb94+hLT8Mn7y2jreWZeCw2HYmJLP0cJybh5yZgEwpZT7aHL3RFXlMH8GlByz7iMa1B4+vNVKyMCuI1nckfoEnSQfbp0HbaJg/q1QkFrrMJMSOvHu8AyGV37Ht51nQjvnfRp7jLFa6GvnWGUCdi2HXcvgqtnQvhcHskv4bEemldhrzBidODCaLx4dzaie7fnTJ7u47e0NvLMuheBWvozr775JSEqpM2lyb0KllTZ+9v4mdhwtOv+dHA5rpEr6j3DzXOj7E7h9oXUT6XnToLyQsgU/Z5jPPionvWXdLuz2hdZNmudNq77FG2X5sPHvjNj1LGmBfbh33yVsPJRnrROBa34Ppcfg2znw2a+h4yC4/JcAvPHVQVr7+3JPHbVVokICefuuYfz1lkHsOFrM6t1ZTBgY3WQ3y1BKNYxLyV1EHhGRHSKyU0RmOZc9LSJHRSTJ+ZjYKJE2Q+v257J6dxYPzNtCSUXVuXcAWPMM7FoKY/8I/Sdby6L6wq0fQN5+bK8PZ1jpV3zT9SGCB0+x1kf2sVrw+YesFv/Cu+HFPvD5byGsCxF3/pPOESHM+k8SBced/eyxI60RM9++aHUBTf4b+PpzOO84y5KOcseIbkQEt6ozRBFh2iVd+PyRK7j90q480MA7tyulmk6Dk7uIDADuA4YDCcD1ItLLufplY0yi8/FZI8TZLJ28+fDRwnKeXLKDc04YS11nVVocdk91K7rSZqe00kZp55FUTHgFv7IsFpgxDJz2h9r7xl0Bk9+AI9/Doa+sY/xiHfz8G4I7x/P6jCHkllby28XbKKmoorTSRtmoJzDiy4kRv6Q0Ip7SSht/+/IA/r4+/OyKc1dE7BIRxLM3DaR7ZOPezV4p5TpXxrn3AzYYY8oAROQb4KZGicpL/JCax7BuEYzs2Y45K/cxqld7pg3rUvfGxsCaP0FINFz3FxAhu7iCq+Z8TdkJu3OjCDrzKj+54hKm1VWPJeFW6DwUwmLAv/bIlYExYTw+vi9//nQ3A59eWb08itfI/iocvjo1Iuenl8cSFaIjX5RqzlxJ7juAZ0WkHVAOTAQ2AXnAQyLyP873jxljCk7fWURmAjMBunbt6kIYnqm4oopdGcX88ppe3H9VT9YdyOWpZTsZ0rUtPaPqaOkeWA1pG+AnL4G/NTV//cFcyk7YeeCqHrQNsrpI/Hz7n32WZ/ue9a66Z2QckSEBZBdX1ruNr49wyxCdRapUc9fg5G6M2S0izwOrgFIgGbABbwF/Aozz+UXgnjr2nwvMBau2TEPj8FSbUwtwGLi0ewS+PsIrtw5mwqtr+eX8rSx54PLa0/lPzgwN72ZNIHL6ISWf0EA/HhvXB18f1wts+fgIkxN1yKJSLYFLF1SNMe8YY4YYY0YD+cB+Y0yWMcZujHEA/8Dqk29xNjr72wd3se4B2jEskDk39mZCztus+urL2hvvXg6ZydZQRL9WtY5xSWxEoyR2pVTL4upomSjnc1fgZmC+iETX2OQmrO6bFueHlDwGxYTXGiJ4TeY/eNhvKWO/ux2SndP/HXb48llo3xsG3Vq9bU5JJYdyjjM8LuJih66U8gKuFg5b7OxzrwIeNMYUiMgHIpKI1S2TCvzcxXM0O2UnbGxLL+K+0d1PLUzfjGx4k/0dJpCfmcKlS2ZC2kbrRtG5e2Hqe6duIo3VJQNocldKNYhLyd0Yc0Udy+6sa9uWZOuRQmwOcyox207A8oegTUdCb3mNCS9vYH7cSi7Z9I61vuNA6De51jF+SMkjqJUvAzrXXYlRKaXORmeoNoGNKfn4CAzrZvW3s+5lyN4F179Eh6goLuvZgUcLbsYx7d/Qrqc19NHH54xjDO3WFn9f/SdSSl04zRxN4IeUPOI7hRES6A/Zu2HtCzBgCvSZAMBNgzuTXlDOptYj4ZebIW50rf0Ly06wN6uE4bHaJaOUahhN7o2s0mZn65FCq0vGYYdlD0FACEx4vnqb6+I70trflyVb0+s8xo+pBRij/e1KqYbT5N7ItqUXUWlzWIn58HdwdJNVJya4ffU2wQF+TBjQkU+2ZVJRZT/jGD+k5NHKz4eELuEXMXKllDfR5N7ITo5yuSQ2AvavBB9/iL/xjO1uGtKZkgoba3Zn13mMxC7h57xvqVJK1UeTeyPbcCiPPh1CrIqKB1ZDt8utbpnTXN6jPR1CA87omimttLEjo5hLtUtGKeUCTe6NyGZ3sPlwgdUlU5hmjZCp52bRvj7CjYmd+XpvDrmlp2q9bD5cgL3mMEqllGoATe6NaO3+HMpO2LmsRzs4sMpa2GtsvdvfMjQGhzFMn7uBbemFgNXf7usjDOna9iJErJTyVprcG4kxhte/PEDn8NZc268D7F8F4V2tsgL16N0hhPfvGU5phY2b3vyOl1ft47uDeQzoHEZwgKuTh5VSLZkm90by/cE8th4p5BdX9aAVVXDoG6tLRs5e9OuKXpF8MWs0kxI68eqa/Ww9UsgI7ZJRSrlIk3sjee3L/USFBDB1aIw1BLLqeL397acLC/Ln5VsTeev2IfSPDuWGhE5NHK1Sytvp3/6N4MfUfDYcyuf31/e3hi/uXwW+ARB7Rumds5owMJoJA6PPvaFSSp2DttwbwetfHqBdcCtuG+68o9T+lRA7CloFuTcwpVSLpcndRclphazdl8PPruhu1W7PT4G8/efdJaOUUk1Bk7uL/vbVAcJa+3PHCGer/cBq6/ksQyCVUqqpaXJ3wYZDeazalcXdI2OtCpBgdclEdId2PdwbnFKqRdPk3kAFx08w66Mk4toHc98VzjsuVZVDylrtklFKuZ2OlmkAYwy/XbyNvOOVLLlr5KkJR/tWgK0Ceo93b4BKqRZPW+4N8MGGw6zalcXj4/vWvg1e8kcQ0umMm28opdTF5lJyF5FHRGSHiOwUkVnOZREiskpE9jufvapIyq6MYv786W6u7hPJvaPiTq0ozbbGtw+aVutG10op5Q4NTu4iMgC4DxgOJADXi0gvYDawxhjTC1jjfO8VKm12fjl/C+Gt/ZkzNQGpWVpg+0Iwdki8zX0BKqWUkyst937ABmNMmTHGBnwD3ARMBt53bvM+cKNLEXqQTakFHMw5ztOT4mnXJqD2yqT50GkIRPZxT3BKKVWDK8l9BzBaRNqJSBAwEegCdDDGZAI4n6Pq2llEZorIJhHZlJOT40IYF09SWiEAI3u0r73i2HbI2q6tdqWUx2hwcjfG7AaeB1YBK4BkwHYB+881xgwzxgyLjIxsaBgX1bb0QuLaBxMW5F97RdJ863Z6A25xT2BKKXUaly6oGmPeMcYMMcaMBvKB/UCWiEQDOJ/PvEloM5WcVsSgmLDaC+1VsH0B9L4OgrRUr1LKM7g6WibK+dwVuBmYDywH7nJuchewzJVzeIrs4gqOFVeQEBNee8WBNXA8R7tklFIexdVJTItFpB1QBTxojCkQkeeABSJyL3AEmOpqkJ4gOb0IgIQup7Xckz+EoHbQU2vJKKU8h0vJ3RhzRsFyY0weMMaV43qi5LRCQnwqSVz/IHxZdGpF2g9wyb3g18p9wSml1Gl0hup5Sk4vZEJEFr77PoPKEhAf6xE3GobPdHd4SilVi9aWOQ/GGLalF3FDTAmUAlPf06qPSimPpi3383A4r4yi8ir6BhZYC8Ji3BuQUkqdgyb385CcXghAF588aNMR/ALOvoNSSrmZJvfzkJxWRKC/D2GVmRDe1d3hKKXUOWlyPw/b0guJ7xSGT9ERTe5KqWZBk/s52OwOdmQUkdg5BIqOQngXd4eklFLnpMn9HPZllVJR5WB45AlwVGnLXSnVLGhyP4dtzoupA9sUWwvCNLkrpTyfJvdzSE4vJDTQj2iHs/6ZttyVUs2AJvdzSE4rIqFLOFJ0xFqgfe5KqWZAk/tZVFTZ2ZtVYlWCLEyD4Ejwb+3usJRS6pw0uZ/Fzowi7A5j1XAv1GGQSqnmQ5P7WSSnnSzzGw5FaRCmXTJKqeZBk/tZbEsvpGNoIB3atLK6ZbTlrpRqJjS5n0VyuvO2esezwV6pyV0p1Wxocq9HUXkVKbnHrS6ZwjRroSZ3pVQzocm9HttP3lYvJhwKD1sLNbkrpZoJTe71OFnmd+DJkTKgF1SVUs2GS8ldRB4VkZ0iskNE5otIoIg8LSJHRSTJ+ZjYWMFeTMlphcS1Dyastb81UqZ1BAS0cXdYSil1Xhp8mz0R6Qw8DPQ3xpSLyAJgunP1y8aYOY0RoLtsSy9iRPcI642OcVdKNTOudsv4Aa1FxA8IAjJcD8n9soorOFZcwaCYcGtBYZqWHVBKNSsNTu7GmKPAHOAIkAkUGWNWOlc/JCLbRORdEWlb1/4iMlNENonIppycnIaG0SSS0woBSOgSBsY4W+7d3BuUUkpdgAYnd2fSngzEAZ2AYBG5A3gL6AEkYiX9F+va3xgz1xgzzBgzLDIysqFhNIlt6UX4+gjxncLgeC7YyrVbRinVrLjSLXMtkGKMyTHGVAH/BS43xmQZY+zGGAfwD2B4YwR6MSWnF9KnQwiB/r5QpCNllFLNjyvJ/QgwQkSCRESAMcBuEYmusc1NwA5XArzYjDFsSy+yumTg1DBIbbkrpZqRBo+WMcZsFJFFwBbABmwF5gJvi0giYIBU4Oeuh3nxHM4ro6i8ypq8BDVmp2rLXSnVfDQ4uQMYY54Cnjpt8Z2uHNPdTk5eOjVS5ggEhlkPpZRqJnSG6mmS04oI9PehdwfnhCUd466UaoY0uZ9mW3ohAzqF4efr/GiK0vSm2EqpZkeTew02u4MdGUWnumSqx7hrcldKNS+a3GtYvTubiioHiV3DrQXlBXCiVC+mKqWaHU3uTseKKnjiv9uI7xTKdfEdrIX5h6xnnZ2qlGpmNLkDdofhkY+2Umlz8PqMwQT4+VorMpOs5+hBbotNKaUawqWhkN7ija8OsDElnzlTE+geWaOsb0aSVepXZ6cqpZqZFt9y/zE1n1dW7+PGxE7cMqRz7ZUZSdApEUTcEZpSSjVYi07uJRVVPDJ/K10igvjTjQOQmkm8qgJydkN0otviU0qphmrR3TLvf5dKRlEFi++/nJBA/9ors3aCwwadBrsnOKWUckGLbbkfr7TxzroUrukbxdBudZScz9xqPXdKvKhxKaVUY2ixyX3exsMUlFXx0DU9695AL6YqpZqxFpncK6rszF2bwqie7RnStc4bRVnDIPViqlKqmfL65F5UXoUxptayj344Qm5pZf2t9qoKyNaLqUqp5surk3tafhlD/rSKmR9sJre0EoBKm52/rz3E8NgIRnRvV/eO2ScvpiZevGCVUqoRefVombSCMuwOw6pdWWw+XMBfbhpI/vETZBZV8PwtZ5l1mpFkPWvLXSnVTHl1ci+tsAHw8q0JvLMuhV/8ezOB/j4kdAnnil7t698xMwlat9VqkEqpZsuru2VKK63kPqRrW5Y8MJJHxvRCEB4b27v2hKXTZSRZrXa9mKqUaqa8uuVe4my5twnww9/Xh0fH9uaRMb3w8TlL0rZVWhdTL3/oIkWplFKNz6WWu4g8KiI7RWSHiMwXkUARiRCRVSKy3/lcz1jDpney5d4m0Pkddmw7Pu+Og/RN9e+UtRMcVTozVSnVrDU4uYtIZ+BhYJgxZgDgC0wHZgNrjDG9gDXO925RXFFFKz8fq4Rv0VGYNw3Sf4Cl91vDHetSXeY38WKFqZRSjc7VPnc/oLWI+AFBQAYwGXjfuf594EYXz9FgpRU2QgL8oLIEPrwVKoth3LOQuw/WvlD3ThlJejFVKdXsNTi5G2OOAnOAI0AmUGSMWQl0MMZkOrfJBKLq2l9EZorIJhHZlJOT09Awzqq00kZ4gMDCuyF7F0x93+pLT7gN1r8CmdvO3Cljq15MVUo1e650y7TFaqXHAZ2AYBG543z3N8bMNcYMM8YMi4yMbGgYZ1VSXsVj9rfhwCr4yYvQ61prxXXPWq3z5Q+B3XZqh5MzU3XyklKqmXOlW+ZaIMUYk2OMqQL+C1wOZIlINIDzOdv1MBumbelBJlZ+Dpc9BMPuPrUiKAImzoHMZPj+b5B3EFY/A68lWhdTu410V8hKKdUoXBkKeQQYISJBQDkwBtgEHAfuAp5zPi9zNciGal2ZZb3oP/nMlf0nQ9/rYc0fYfVTID7Qc6yzhT/24gaqlFKNrMHJ3RizUUQWAVsAG7AVmAu0ARaIyL1YXwBTGyPQhmhVmW+9CKqjhoyIlcgdduhyidUPHxp9cQNUSqkm4tIkJmPMU8BTpy2uxGrFu11rW5H1Iiii7g1COsJtH128gJRS6iLx2vIDxhiCbIU48IWAMHeHo5RSF5XXJveKKgdtTTEV/mHg47U/plJK1clrs15JZRVtpYQTAW6rfqCUUm7jvcm9wkaElGDT5K6UaoG8NrmXVthoSwmO1vVcTFVKKS/mvcm90kZbKcHUNQxSKaW8nNcm95LyStpSik/wWe64pJRSXsprk3tFSQF+4sCvjSZ3pVTL47XJ3V6aC0Cr0KYpSqaUUp7Me5P78TwAAjS5K6VaIK9N7lJmtdy1W0Yp1RJ5b3IvP0vRMKWU8nJem9z9KjS5K6VaLq9N7q1OFFJJK2gV7O5QlFLqovPa5B5wooBS31C9F6pSqkXy2uQeZCviuG+4u8NQSim38Nrk3sZeRIV/uLvDUEopt/Da5B5iiqlsFe7uMJRSyi0afJs9EekD/KfGou7AH4Bw4D4gx7n8d8aYzxp6noYwxtDWFFGg5X6VUi2UKzfI3gskAoiIL3AUWALcDbxsjJnTGAE2RFl5BWFShl3L/SqlWqjG6pYZAxw0xhxupOO55Hih9UeDaa1j3JVSLVNjJffpwPwa7x8SkW0i8q6IXPS+kYribABEy/0qpVool5O7iLQCJgELnYveAnpgddlkAi/Ws99MEdkkIptycnLq2qTBKoqs4/lqXRmlVAvVGC33CcAWY0wWgDEmyxhjN8Y4gH8Aw+vayRgz1xgzzBgzLDKycSs32kqc5X5DNLkrpVqmxkjuM6jRJSMi0TXW3QTsaIRzXBD7cSu5B4RpuV+lVMvU4NEyACISBIwFfl5j8V9FJBEwQOpp6y4K46zl3jos6mKfWimlPIJLyd0YUwa0O23ZnS5F1AikPJ8S05o2wVo0TCnVMnnlDFW/ijwKTBvaBLj03aWUUs2WVyZ3/8oCiiQUXx+tCKmUapm8MrkHniikxCfU3WEopZTbeGVyb20rpFTL/SqlWjCvTO7B9iLKtdyvUqoF877kXlVOoKnQcr9KqRbN+5J7mXVj7Cot96uUasG8MLlbE5hsgVruVynVcnltcjeBWu5XKdVyeV1ydzjrykiwJnelVMvldcm9svhkuV9N7kqplsvrkrutJAeHEfw1uSulWjDvS+6leRQSTJvWge4ORSml3Mbrkrs5nkuBCaFNoBYNU0q1XF6X3KU8n3xCtCKkUqpF87rk7lOeT4EJIVRb7kqpFszrkrt/ZQH52i2jlGrhvCu5G0PAiQIKCCEk0N/d0SillNt4V3KvLMHXVFFACEH+vu6ORiml3KbByV1E+ohIUo1HsYjMEpEIEVklIvudzxevgpez9ECZbzg+ehcmpVQL1uDkbozZa4xJNMYkAkOBMmAJMBtYY4zpBaxxvr84Dn0NQFGrDhftlEop5Ykaq1tmDHDQGHMYmAy871z+PnBjI53j7IozYdVT7A1MZF/rhItySqWU8lSNldynA/OdrzsYYzIBnM9Rde0gIjNFZJOIbMrJyXHt7MbAp78CeyV/D3+Y4MBWrh1PKaWaOZfHC4pIK2AS8MSF7GeMmQvMBRg2bJhxKYidS2DvZzD2Txzc2oHwIB0po5QnqKqqIj09nYqKCneH0qwFBgYSExODv//557bGGAw+AdhijMlyvs8SkWhjTKaIRAPZjXCO+pXlw2e/gU6DYcQDlGxYR0xEUJOeUil1ftLT0wkJCSE2NhYRHeTQEMYY8vLySE9PJy4u7rz3a4xumRmc6pIBWA7c5Xx9F7CsEc5RvxVPQEUhTPob+PpRUmEjREsPKOURKioqaNeunSZ2F4gI7dq1u+C/flxK7iISBIwF/ltj8XPAWBHZ71z3nCvnOKuUtbDtIxj1K+g4AIDSChshOjtVKY+hid11DfkMXcqCxpgyoN1py/KwRs80va6XwcQ5MOR/ALDZHZRX2WkToH3uSqmWrXk3cX39Yfh91W9LK20AWldGKdXieVX5gZIKK7lrt4xSqiHatGlT77q4uDj27t1ba9msWbP461//CsDWrVsREb744ovzPmZT8qosWJ3c9YKqUh7nmY93siujuFGP2b9TKE/dEN+ox6zP9OnT+eijj3jqqacAcDgcLFq0iPXr1wMwf/58Ro0axfz587nuuusuSkxn41Ut9yP5xwGIDm/t5kiUUp7g8ccf580336x+//TTT/PMM88wZswYhgwZwsCBA1m27PwG9M2YMYOPPvqo+v3atWuJjY2lW7duGGNYtGgR7733HitXrvSIcf1e1cTdnVmCj0CfDiHuDkUpdZqL1cKuafr06cyaNYsHHngAgAULFrBixQoeffRRQkNDyc3NZcSIEUyaNOmcI1IGDRqEj48PycnJJCQk8NFHHzFjxgwA1q9fT1xcHD169OCqq67is88+4+abb27yn+9svKrlvjuzmNj2wbRupeV+lVIwePBgsrOzycjIIDk5mbZt2xIdHc3vfvc7Bg0axLXXXsvRo0fJyso698E41Xq32WwsW7aMqVOnAlaXzPTp0wHrC2X+/PlnO8xF4VUt9z3HShgYE+buMJRSHmTKlCksWrSIY8eOMX36dObNm0dOTg6bN2/G39+f2NjY8+5GmTFjBuPGjePKK69k0KBBREVFYbfbWbx4McuXL+fZZ5+tnlFaUlJCSIj7ehG8puVeUlHFkfwy+nXULhml1CknL4QuWrSIKVOmUFRURFRUFP7+/nz11VccPnz4vI/Vo0cP2rVrx+zZs6u7ZFavXk1CQgJpaWmkpqZy+PBhbrnlFpYuXdpEP9H58ZrkvvdYCQD9okPdHIlSypPEx8dTUlJC586diY6O5vbbb2fTpk0MGzaMefPm0bdv3ws63owZM9izZw833XQTYHXJnHx90i233MKHH34IQFlZGTExMdWPl156qXF+sHMQY1wryNgYhg0bZjZt2uTSMT7YcJjfL93Bd7OvoZOOllHKI+zevZt+/fq5OwyvUNdnKSKbjTHD6trea1ruuzOLCQ30Izos0N2hKKWU23nNBdXdmcX0iw7VIkVKKZds376dO++8s9aygIAANm7c6KaIGsYrkrvDYdh7rIRpw7q4OxSlVDM3cOBAkpKS3B2Gy7yiW+ZIfhllJ+z0i9aRMkopBV6S3HdnWvUqdKSMUkpZvCO5H7PKDvTWsgNKKQV4S3LPLCaufTCB/lp2QCmlwIuSu3bJKKVOV1hYWKsq5PmaOHEihYWFF7TPe++9Vz1r9aTc3FwiIyOprKwEYPLkyVx22WW1tnn66aeZM2fOBcd4Ls1+tExxRRXpBeXMGN7V3aEopc7m89lwbHvjHrPjQJhQ/22aTyb3k1UhT7Lb7fj61v+X/meffXbBodx88838+te/pqysjKCgIAAWLVrEpEmTCAgIoLCwkC1bttCmTRtSUlKIi4u74HNcCFdvkB0uIotEZI+I7BaRy0TkaRE5KiJJzsfExgq2LifLDvTXlrtS6jSzZ8/m4MGDJCYmcskll3D11Vdz2223MXDgQABuvPFGhg4dSnx8PHPnzq3eLzY2ltzcXFJTU+nXrx/33Xcf8fHxjBs3jvLy8jrPFRoayujRo/n444+rl9UsC7x48WJuuOGG6lo3Tc4Y0+AH8D7wM+frVkA48DTw6ws5ztChQ01Dvf9diun2+Ccmo7CswcdQSjWNXbt2ufX8KSkpJj4+3hhjzFdffWWCgoLMoUOHqtfn5eUZY4wpKysz8fHxJjc31xhjTLdu3UxOTo5JSUkxvr6+ZuvWrcYYY6ZOnWo++OCDes+3YMECc+ONNxpjjDl69KiJjo42NpvNGGPMmDFjzNq1a83evXvNwIEDq/d56qmnzAsvvHDOn6WuzxLYZOrJqw1uuYtIKDAaeMf5JXHCGFPoyhdNQ+zOLCE8yJ+OoVp2QCl1dsOHD6/VHfLaa6+RkJDAiBEjSEtLY//+/WfsExcXR2JiIgBDhw4lNTW13uNff/31rFu3juLiYhYsWMCUKVPw9fUlKyuLAwcOMGrUKHr37o2fnx87duxo7B+vFle6ZboDOcA/RWSriLwtIsHOdQ+JyDYReVdE2ta1s4jMFJFNIrIpJyenwUHsziymX0ctO6CUOrfg4ODq119//TWrV6/m+++/Jzk5mcGDB9dZ1z0gIKD6ta+vLzabrd7jt27dmvHjx7NkyZJaXTL/+c9/KCgoIC4ujtjYWFJTU5u8a8aV5O4HDAHeMsYMBo4Ds4G3gB5AIpAJvFjXzsaYucaYYcaYYZGRkQ0KwO4sO9BXZ6YqpeoQEhJCSUlJneuKiopo27YtQUFB7Nmzhw0bNjTKOWfMmMFLL71EVlYWI0aMAKyywCtWrCA1NZXU1FQ2b97s0ck9HUg3xpysprMIGGKMyTLG2I0xDuAfwHBXg6zPkfwyyqvsOgxSKVWndu3aMXLkSAYMGMBvfvObWuvGjx+PzWZj0KBB/P73v69OxK4aN24cGRkZ3HrrrYgIqampHDlypNbx4+LiCA0NrS5G9uc//7lWzffG4FI9dxH5FuuC6l4ReRoIBl4yxmQ61z8KXGqMmX624zS0nvuB7FJeXLmXR8f21tmpSnkgrefeeC60nrur49x/CcwTkVbAIeBu4DURSQQMkAr83MVz1KtnVBveumNoUx1eKaWaLZeSuzEmCTj9W+POOjZVSimv8eCDD7J+/fpayx555BHuvvtuN0V0pmY/Q1Up5dmMMV43mu2NN964qOdrSPe5V9SWUUp5psDAQPLy8hqUnJTFGENeXh6BgRc2l0db7kqpJhMTE0N6ejquzGVR1pfkhY6i0eSulGoy/v7+TV4gS9VNu2WUUsoLaXJXSikvpMldKaW8kEszVBstCJEc4LALh2gP5DZSOE1B43ONxucajc81nhxfN2NMncW5PCK5u0pENtU3BdcTaHyu0fhco/G5xtPjq492yyillBfS5K6UUl7IW5L73HNv4lYan2s0PtdofK7x9Pjq5BV97koppWrzlpa7UkqpGjS5K6WUF2rWyV1ExovIXhE5ICKzPSCed0UkW0R21FgWISKrRGS/87nOG4ZfpPi6iMhXIrJbRHaKyCOeFKOIBIrIDyKS7IzvGU+Kr0acvs6bwn/iafGJSKqIbBeRJBHZ5IHxhYvIIhHZ4/w9vMxT4hORPs7P7eSjWERmeUp8F6rZJncR8QXeACYA/YEZItLfvVHxHjD+tGWzgTXGmF7AGud7d7EBjxlj+gEjgAedn5mnxFgJXGOMScC6wfp4ERnhQfGd9Aiwu8Z7T4vvamNMYo2x2Z4U36vACmNMXyAB63P0iPiMMXudn1siMBQoA5Z4SnwXzBjTLB/AZcAXNd4/ATzhAXHFAjtqvN8LRDtfRwN73R1jjdiWAWM9MUYgCNgCXOpJ8QExWP/BrwE+8bR/Y6xbW7Y/bZlHxAeEAik4B3J4WnynxTQOWO+p8Z3Po9m23IHOQFqN9+nOZZ6mg3HeMNz5HOXmeAAQkVhgMLARD4rR2eWRBGQDq4wxHhUf8ArwW8BRY5knxWeAlSKyWURmOpd5SnzdgRzgn85urbdFJNiD4qtpOjDf+doT4zun5pzc67pvl47rPA8i0gZYDMwyxhS7O56ajDF2Y/1ZHAMMF5EBbg6pmohcD2QbYza7O5azGGmMGYLVXfmgiIx2d0A1+AFDgLeMMYOB43hgF4eItAImAQvdHYsrmnNyTwe61HgfA2S4KZazyRKRaADnc7Y7gxERf6zEPs8Y81/nYo+KEcAYUwh8jXUNw1PiGwlMEpFU4CPgGhH5twfFhzEmw/mcjdVfPNyD4ksH0p1/jQEswkr2nhLfSROALcaYLOd7T4vvvDTn5P4j0EtE4pzftNOB5W6OqS7Lgbucr+/C6ud2C7HuUvwOsNsY81KNVR4Ro4hEiki483Vr4Fpgj6fEZ4x5whgTY4yJxfp9+9IYc4enxCciwSIScvI1Vr/xDk+JzxhzDEgTkT7ORWOAXXhIfDXM4FSXDHhefOfH3Z3+Ll70mAjsAw4CT3pAPPOBTKAKq5VyL9AO6wLcfudzhBvjG4XVdbUNSHI+JnpKjMAgYKszvh3AH5zLPSK+02K9ilMXVD0iPqw+7WTnY+fJ/xOeEp8zlkRgk/PfeCnQ1sPiCwLygLAayzwmvgt5aPkBpZTyQs25W0YppVQ9NLkrpZQX0uSulFJeSJO7Ukp5IU3uSinlhTS5K6WUF9LkrpRSXuj/AxDJiW4Zh+JjAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_VAL','train_VAL']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD8CAYAAABuHP8oAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABEDElEQVR4nO3dd3iUVdrA4d9JJ50EEkICCSXSIfSOKIIUFaQoYEFRsa1t/XbFtvZd17X3BV1EpUhTEAEp0pQiIQQINYSShPRAGiF1zvfHO0ACKZNGZshzX9dcM/POW56ZwDNnTlVaa4QQQtgeu/oOQAghRPVIAhdCCBslCVwIIWyUJHAhhLBRksCFEMJGSQIXQggbZVECV0o9pZSKUkodUEo9bd7mo5Rap5SKNt83rtNIhRBClFJpAldKdQYeAvoA3YBblFKhwExgg9Y6FNhgfi6EEOIqsaQE3gHYobXO1VoXAZuB24GxwFzzPnOBcXUSoRBCiDI5WLBPFPCWUsoXOA+MBsIBf611IoDWOlEp5VfZiZo0aaJDQkJqEK4QQjQ8u3fvTtNaN718e6UJXGt9SCn1b2AdkAPsBYosvbBSagYwA6Bly5aEh4dbHLQQQghQSp0qa7tFjZha66+11j201kOAM0A0kKyUCjCfPABIKefYWVrrXlrrXk2bXvEFIoQQopos7YXiZ75vCYwHFgArgGnmXaYBy+siQCGEEGWzpA4cYKm5DrwQeFxrfVYp9TawSCn1ABALTKqrIIUQQlzJogSutR5cxrZ0YFhNAygsLCQ+Pp68vLyankpYyMXFhaCgIBwdHes7FCFEDVhaAq8z8fHxeHh4EBISglKqvsO55mmtSU9PJz4+nlatWtV3OEKIGqj3ofR5eXn4+vpK8r5KlFL4+vrKLx4hrgH1nsABSd5XmXzeQlwbrCKBCyFEvclOgl1fQeH5+o6kyiSBV5G7u3u5r7Vq1YojR46U2vb000/zzjvvALBnzx6UUvz6668Wn7OkLVu20KNHDxwcHFiyZEkVIxdClFKYB1vehY97wC/PwrZP6zuiKpMEXosmT57MwoULLz43mUwsWbKEO++8E4AFCxYwaNAgFixYUK3zt2zZkm+++YapU6fWSrxCNFgHV8BnveG3N6DNDdDqetj2CeSeqe/IqqTBJ/DnnnuOzz///OLzV199lddee41hw4bRo0cPunTpwvLllo1RmjJlSqkEvmXLFkJCQggODkZrzZIlS/jmm29Yu3ZttRoRQ0JC6Nq1K3Z2Df7PJkT1Ra+HRfeAsyfcuwImz4OR/4L8LNj2ceXHFxdB5HyI+Q3Opdd9vBWo926EJb328wEOJmTV6jk7NvfklVs7lfv65MmTefrpp3nssccAWLRoEWvWrOGZZ57B09OTtLQ0+vXrx2233VZp49+F5Lp37166devGwoULmTJlCgB//PEHrVq1ok2bNgwdOpRVq1Yxfvz42nujQgjL7JoN7v7w0EZwcDK2+XeCLhNhx5fQ91Hw8C//+APL4KdHLz33DIQWfWD0e+DmW7exX6bBF+W6d+9OSkoKCQkJ7N27l8aNGxMQEMALL7xA165duemmmzh9+jTJyckWne9CKbyoqIjly5czaZIxQHXBggVMnjwZML40qluNIoSogcx4iF4L3e++lLwvGPo8FBfA1vcqPkfkPPBuCfcuh+FvQPAAOLgctl/9OnSrKoFXVFKuSxMnTmTJkiUkJSUxefJk5s2bR2pqKrt378bR0ZGQkBCLqzymTJnCiBEjuP766+natSt+fn4UFxezdOlSVqxYwVtvvXVxME12djYeHh51/O6EEBdFfAtaQ49pV77m2wZ63APh/4MBfzGS9OUy4+H4Zrj+OWg91LiB0YNl9xwY8jdwcq3Ld1BKgy+Bw6XGxyVLljBx4kQyMzPx8/PD0dGRjRs3cupUmTM5lqlNmzb4+voyc+bMi9Un69evp1u3bsTFxXHy5ElOnTrFhAkT+Omnn+roHQkhrlBcZCTwtsOgcXDZ+wz5Oyg72PTvsl/fuxDQ0G1y6e39HoPzZ2H/orKPKyqodtgVkQQOdOrUiezsbAIDAwkICOCuu+4iPDycXr16MW/ePNq3b1+l802ZMoXDhw9z++23A0b1yYXHF0yYMIH58+cDkJubS1BQ0MXb+++/X+Z5d+3aRVBQEIsXL+bhhx+mU6f6+cUihE2KXgvZidDz/vL38QqE3g/C3vmQcqj0a1rD3gUQPBB8LpuGIngANOti1KFrXfq1tGj4uDuc/L123kcJSl9+sTrUq1cvffmCDocOHaJDhw5XLQZhkM9dNDjzJkHSfng6CuwrqD0+lw6f9gKf1vDAWrCzN7bH/QlfD4exnxl16JeLnG80bt7zk9E1EYyqla9ugqwEeOR34wuiGpRSu7XWvS7fLiVwIS53Lh0ivgNTcX1H0jCdOQF/fAR55fRIyz1T9T7bGbEQvQ6631Nx8gajJ8mof8PpcNj55aXtkfPA0RU6ji37uM4TwK0p7Pji0rY1MyE5CsbPqnbyrohVNWLaiv3793PPPfeU2ubs7MzOnTtr5fxvvfUWixcvLrVt0qRJvPjii7VyflGJNc/B/sWgTdCzjMau2mAylX5ekGP8R0/cB0n7jOQ09lNwa1I318/PhqQo41qJ+yD9mPF+L3D2gPGzr263uPxsowfI9s+M3iAHfoK7l4Krz6V9spPhu3GQctCorrhjLgRdUTC9UsS3xn2Peyre74Iuk2D/EtjwBrQbBR4BEPUjdLjN+GzK4uAMvR6AzW9Degwk7IHd38DApyB0uGXXrSKpQmmg5HMvR1IUfDkI7B2NgR5P7IZG3rV3/uwk2PC60RimyynhuzWFvExoNQSmLoaqDtyKXAC/Pm/0T27WFQK6gncwpB259AWRHgOY/++7+kLTDsZ7BmP7iS1Gw9zNb1X9PeaeMd6nf0fL9tfaqH7Y8BrkJEO3KRAyGFY+DU2uM6ok3JsaPUC+HWtURwx/3Rh0k5UIN/8T+jwE5Y3TKC6CDzoZn8Ndi8vepyyZp+HzfhDQzei1suxBo+vghZ4nZclONq4VOtz4DP07wX2/lPhsq6e8KhQpgQtR0sa3jMR957fw7TjY8p/qJbHLFebBjs9g6/tQlG+U7N2bXXrdwQn8OhoJ16OZ0ZXtl7/Cto9g0DOWXydmI6z4i3EeV184tt5okLvAu6XxWtc7LyV3j4Ark99Pj8Gfs40kXtlP/+SDcGgFJEQaXw5Zp43tk+dD+zGVx7x/MSx/DIJ6w+QFENTT2O7ZHBZOhTmj4LaPYdnDkJcB9/wILfsZVRY/PQqr/wax2419Li8dm0zGL6qcJOj1YeWxlOQVCCPegJ+fMn4deQZByJCKj/HwN+LatxBcvGHC1zVO3hWRBC7EBfHhcGQV3PiSUcrqcY9RB9pjGjS97tJ+J7YYpdx+jxoJsDIntsLyxyHjFLQbYyQF3zYVH9NrOpzcavyEb9nfSFhglFaPrDJ6VPSYBoE9Lh2TfAAW3QtN2hklRRdPY3t2klEH7Nu2dHVERa5/DvYtgi3vwK0fXfl6XiZELYU938Pp3UbXO99Qc2+MrkZ98a8vQJth4OhS/nVMJqPaxK8jTF9b+tdGmxvg7mVG4+OcUdCoMUxbAc27G6+7+hgJ/48PjTlNkvbDHd9eKvmbimHFE0YsA56E60Za9t5L6jHNeJ8nthjVI5b8Ghr4JJz6A8a8B94tqn7NqtBaX7Vbz5499eUOHjx4xTZR9+RzL8M3t2r979Za52Ubz7NTtP5nkNbfTTCeFxdrvfk/Wr/qrfUrnlq/4qX1iieN/cpSXKz1lveM/T/qrvWx36oWz/lMrT/spvV7HbQ+l651UpQR4yuel2L48VGtsxK1zjxt7PduO60z4qv7CZS28lmtX/PROu1Y6e37Fmv9ZjPj+p/103rbZ1rnpJbeJ2aj8fqWdyu+xsGfjf32LS5/n/jdWs+frHXSgfL3Ob5F63faav2Gv9aRC7QuKtB60X3GuTf+S2uTqeI4KnLmhNbz7qza51qT65UBCNdl5FSL6sCVUs8AD2JUmu0H7gdcgR+AEOAkcIfW+mxF55E6cOshn/tljm8y6ldv/hf0f+zS9m2fwtoX4fb/GiWx6LXQeSIMf81obPtzltEzYeBTRonRr5NR4jx/Fn58BI6ugU63w22flN/4VZGESPh6OCbPIOwyThrVOze8aMzb8fsHRo8HeyejjvhcGty/2rJfBZbIToKPwqDDrTBhtrEt4jujVNuyv1G11Lx7+XXPC+8yqnSeCDeqQy6nNXw1DHLT4S+7K+8dYkm8S6YbpV+f1nDmuDHUfeCTNTuvBfIKi3FxtK+z81e7G6FSKhB4Euilte4M2AOTgZnABq11KLDB/NzmZGRklJqN0FKjR48mIyOjSsd88803F0dnXpCWlkbTpk3Jz88HYOzYsfTv37/UPq+++irvvvuuRdeYPn06fn5+dO7cuUqxNWhaG1UVnoFG1UVJfWYYVQM/Pmwk+THvwYSvwCvImMHu0e1G9cZvb8DsG+GfzeHz/vDFQDi2AUb9BybOqV7yBnRAN9a0eJriM6dIvO5ueHIP9J1hVB+MeAMe32l8cWQlGD0yait5g1EX3/dho446+QDsnGXUr7e5wegdEtij/OQNMOJNMBXC+lfLfv3EZqP6ZeDTNU/eF+K9d4Vxvow4GP3uVUnekXEZdH11LV9ujqnza13O0uZtB6CRUsoBo+SdAIwF5ppfnwuMq/XoroLyEnhxccV9gFetWoW3t3eVrjV+/HjWrVtHbm7uxW1Llizhtttuw9nZmYyMDCIiIsjIyODEiRNVOvcF9913H2vWrKnWsQ3G0bXwSU/4oIv51sno83v9c1fW1zo4GY1jIYNh+hpjlF7JpNX0OqNnw1P74I7vYPBfwauF0Vg4fY2RbKu5hJ3Wmv/8eoRHDofR0/Qttx0fS5rJrfROvm2M6VCfj4e2N1XrOhUa+JTx5bNgstFY2G4MTFlo2XwfPq2g/19g3w/GIBggJjWHj9ZHM+PbcAo3vWs05IbV4vz29g7Gr6MXEoyeKbUgOjmbJxfsITmr7PmQFv4ZS0GxibdXH+arrcdr5ZqWqvRrT2t9Win1LhALnAfWaq3XKqX8tdaJ5n0SlVJ+dRxrnZg5cyYxMTGEhYXh6OiIu7s7AQEBREZGcvDgQcaNG0dcXBx5eXk89dRTzJgxAzDm5g4PDycnJ4dRo0YxaNAgtm3bRmBgIMuXL6dRo0ZXXMvT05MhQ4bw888/X1zkYeHChbz00ksALF26lFtvvRV/f38WLlzI888/X+X3M2TIEE6ePFn9D+Rad/ak0R3MzQ9CBl3a7tak/EQSPADuW1nxeRsHG7eOt9VaqB+sj+bzTTFM6dOSu/u15PbPt/F/i/fyv2m9sbO77EvBwbna1ykoMrHuYDK9Qxrj53nZF5irDwx4wuid03mCUZVUlV4Vg59FR84nddFTPGL/KhFJRSgF3TiGo/NWo5Reg9jLdflMg9UUk5rDlNk7ScvJp5mXCy+MLl3tmFdYzC/7EhkX1pyCYhNv/nIIBzvFfQMvDbVPyDjPqv2JTOrZAi/X2u2RUmkCV0o1xihttwIygMVKqTLGkZZ7/AxgBhgrylRo9UyjJbk2NesCo94u9+W3336bqKgoIiMj2bRpE2PGjCEqKopWrYw/wP/+9z98fHw4f/48vXv3ZsKECfj6lh7cEB0dzYIFC5g9ezZ33HEHS5cu5e67y/6IpkyZwvz587nzzjtJSEjg6NGj3HCDMex2wYIFvPLKK/j7+zNx4sRqJXBRgaICWHy/UWVy1+Ir57OwIp9siObjDdHc0SuIt8Z1xs5O8fKYDry8/ABf/36Ch4a0rrVrLYuIZ+ay/SgFfUJ8uKVbc0Z09Mfd2Zweej+BvV93ikOGQJGCoiIc7BXODhbU+Tq7s6HFE9x06EXmqXuJa3UzvoMf4MiyleQUu+Ne0bwk9exk2jmmzt4BaPqE+PDDrjieuek6Gjldet/rDiaTnV/EHb1a0LuVD0XFEbz680EKizWO9oqV+xIJP2U0DQY1bsTIzgG1GqMlFU83ASe01qkASqllwAAgWSkVYC59BwApZR2stZ4FzAKjEbN2wq47ffr0uZi8AT7++GN+/PFHAOLi4oiOjr4igbdq1YqwsDAAevbsWWEJ+JZbbuGxxx4jKyuLRYsWMXHiROzt7UlOTubYsWMMGjQIpRQODg5ERUVJXXZtWv8qJEQYXc2sOHn/sCuW99YdZXz3QP41vuvF0vbd/YL5/Vga/15zmN6tfAhr4V0r19t6LA0/D2em9m3Jz3sTePmnKF7+KaqMPddffOTkYMeqJwfR1q/iuv3cgiKeO9qO2/w+5JXA3Vx34EdYtJwBwIdFE7i70JEmlRTAdx5PZ/o3u1gwox9dg7yr/P6qI+5MLlNn76CwWLPgoX6czS1g8qwd/Lw3gTt6X+oauCwinuZeLvRr7YudneLTqT149PvdvLXKmAirnb8Hzw6/jlu6NadVE7fyLldtliTwWKCfUsoVowplGBAOnAOmAW+b7y1bd6wiFZSUrxY3t0sf8qZNm1i/fj3bt2/H1dWVoUOHljkvuLPzpX+B9vb2nD9f/urWjRo1YuTIkfz4448sXLiQDz74AIAffviBs2fPXvzyyMrKYuHChbz55pu19dYatsOrjIE0fWaUP5dFHTmZdo6520/y1LBQvF0r/ml/NDmbV1YcYFDbJvxnUjfsS1SVKKX494SujP5oK08u2MPKJwfh6VKzn+Qmk2bbsTRubO/P0zddx1PDQjmclM22mHSKLx/uf+EYDe+vO8o3207y5rguFZ5//s5Y0s8VMOaesRByP4x8Gw4u5+yR3/lf5A34HUhmat+Kf5m/t+4o5wqK+Wh9NF/f17va79VSBxOyeOjbcHILi5n/YD/aNfNAa007fw/mbj/JpF5BKKVIyc5jS3QaDw9pffFL1snBjs/v7sHyPQl0b+lNqH/dzvdvSR34TqXUEiACKAL2YJSo3YFFSqkHMJL8pLoMtK54eHiQnZ1d5muZmZk0btwYV1dXDh8+zI4dO2rlmlOmTOH5558nKyuLfv2MARoLFixgzZo1F3ugnDhxguHDh0sCrw3pMcaIvYBuRp3rVaS15rml+9h54gy7T53luwf64tWo7KSbV1jMX+ZH4O7swPt3lk7eF3i7OvHxlO7cOWsHLyzbzydTule61F9FDiZmcTa3kEGhxq9KpRQdAjzpEOBZ4XHHUnJYFnGav49sX+6XSF5hMf/dcpz+rX3pFWIeQOTsDt3vwjtsKj5xm1gdlVhhAt9xPJ0/T5zhOn93NhxOIep0Jp0Dvar1XitTVGziy80xfLQhGq9GTnw3vS8dmxufg1KKe/oH89JPUUTEZtAzuDErIhMoNmnG9yg9UtXZwb5UKb0uWdQLRWv9ita6vda6s9b6Hq11vtY6XWs9TGsdar63reWczXx9fRk4cCCdO3fmb3/7W6nXRo4cSVFREV27duXll1++mGxrasSIESQkJHDnnXeilOLkyZPExsaWOn+rVq3w9PS8OEHWm2++WWrO8PJMmTKF/v37c+TIEYKCgvj6669rJWabdXgVzDJP7TlxTt00mFXgl/2J7DxxhvHdAzmUmMW0//1Jdl5hmfu+vvIgR5NzeP+OMPw8yh+92CvEh2duCmXlvkQWhcfVKL7fj6UBMLBt1SbNurd/MLkFxSzbHV/uPj/siiM1O58nhrW94jWlFKO6BLA9Jp2M3PIXO/jkt2iauDvz/YN98XBx4JPfoqsUp6WOpeQw4cvtvLv2KCM6NWPdM0PoElT6i+L27oF4ODvw7faTACyLOE23IK9Kq5HqVFmje+rqJiMxrUdFn/vq/Ql6zMdbdG5+0VWMqJYVFWq99mVjJN6Xg43RdFfZufxC3e+f6/WoD7foomKTXhOVqNs8/4ue8PkfOievsNS+K/cm6ODnVup/rTpk0bmLik16yqztut1Lq/TRpKxqx3jX7B16xPubq3Xs2E9/1ze8u1Gbyhh1mFdYpPv9c72e+MUfZb6utdZ7487q4OdW6kW7Yst8PfzkGR383Eo9a3OM1lrr99Ye0cHPrdSHEjOrFOeB05l6wL826Mfm7dar9yfo8wXGv+vzBUV61b4E/dj3u3Xoi6t0t9d+1SsiT1d4rleWR+m2L/yitx5N1cHPrdTf/HGiSrFUF+WMxJT5wMUVftmfRNTpLFbsPV3foVTL6bgTnP54uDGndK/pxhwbjUMsPj41O5+vth6noKjsOmBLfbEphsTMPF4b2wl7O8XNnZrx8ZTu7InLYPKsHbzw4/6Lt5nL9hHWwptnR1xX+YkBezvFh3eG4ebkwF/m7yGvsOpzl+cVFvPnyTMMCq3elLX39g/meOo5/jiWfsVrS3efJjEzjyduDC23iqdLoBeB3o1YHZVU5uuf/BaNj5sTd/UzqlimDwzBzcmeT347VqU43193lIzcAnbEpPPI9xH0fGMd93y9k55vrOPReRHsOJ7Onb1asPbpIdzarYwRoyXc0z+YwmLNUwv34GCnKt2/rkkCryOPP/44YWFhpW5z5syplXOnp6dfce6wsDDS06/8j1QdEeZuT3O3nUJfxemGy1MUs5mcFX+HgtxK99XHN+M25wYaZxzgYN934JYPKp5MqQyfbTzGm78c4r21R6obMrHpufx3y3HGhTWnd8ilCaRGdwngo8lhnDlXwNoDyRdvQY1d+WRKdxztLf8v6efpwnt3dONIcjav/XyQYlPV/lbhJ89SUGRiUBWrTy4Y3SUAXzcn5pqrFC4oLDbx+aZjdGvhzeAKvhyUUozq3Iyt0alkXVattDcug01HUnlwcCtcnYymOm9XJ6YNCGHV/kSOpZTdbnW5AwmZrD+UzIwhbdj5wjC+e6APt3Rtzumz57mla3O+f6AvO18YxhvjOl/ZB74MbZq6Mzi0CennChjazg8ft9rpb15dMhthHfnss8/q7Ny+vr5ERkbWybkTM89zOuM87Zt5cDAxi4jYs/QMtnAGu9pmMpGy+p/47noPd0zkZsTieve8S0tcXbYvf3wAG94k3eTPY0XP0yyx+8WhwpYqKDKxYm8CLo52RgNcG1+Gtqv6GLU3fjmIg51i5qgr55u5pWtzbulaOyW3oe38ePj61vx383E2HEpmTNcAbunanB4tvStt3Pz9WBqO9oo+rar393VxtOfO3i34cnMMpzPOE+jdiPizufx9yT7iz57n9bGdKo1hVJcAvvr9BL8dSmFc90uNgZ/8dgyvRo7c2z+k1P4PDGrFnD9O8tnGGD64M6zSGD/97Rgezg7cNzAEB3s7Boc2ZXBo0+q83YvuGxDC1ug0JvUqvy3qarGKErg1lPIakoo+74hTGQD849aOeLg48O32U7UfQF6msdpKymEwFfPrgSRu/eR33l97hKPJRsmqKDuNU5/egt+u/7BODeCdosm4Hl8Na1+68nzn0mDhFNjwOhvsBvB/jT9kxNChbD6aysm0c1UKbfPRVM6cK+D9O8KMPryL9pJSzhDq8mw4lMy6g8k8cWMozbyqVvqvjudubs8nU7oT1sKbeTtjmfDFNnq9uZ7B7/x28Tb2sz+IO1P6F8wfx9Lo3rIxbs7VL8fd1c9Y3f37HadYtCuOkR9uZW9cBv+e0IUb2/tXenz3Ft74ezqzan8iJpNm5/F0XvppP+sPJTN9YKtLg4nMfN2dubtfS5ZHnuZEJX/bI0nZrI5K4r6BIeX2/KmOYR38WfvMEEZ0rPz91bV6L4G7uLiQnp6Or69vjbpDCctorUlPT8fFpezEsvvUWVwc7egd4sPEnkF8v+MUL43pSFOPWuq9YTLB4vsg5jfjqYMLzQpbMMMukNytELEFYho50rM4kmbFZ1ng9xQ3T3uRn5cf4PvoLO7e8bmxuky/R4yRlX/Ogs3vQGEuG1r/jQcPhvHD3b0JaeLGF5tijPhvsXBlGIyBGU3cnRje0Z9QP3du/fR3nv4hku8e6Ftmt77LJWfl8bcl+2jfzIPpg0Kq+SFVjZ25LvbWbs3Jzitk3cFktsekl6pSWXcomScW7GHxI/1xtLfj7LkCohIyeeYmy+rcyxPo3YibOvjz5eYYtIZ+rX34z8RutPCxYK4Uc+yjOgcwf2cs/f61gZTsfFwc7RgX1pwHBpc92OqhIa35dvspPt94jP9M6lbuuT/deAw3J3umD6z9QVvX1XH/bkvVewIPCgoiPj6e1NTU+g6lwXBxcSm3K+Lu2LN0DfLG0d6Oe/oFM+ePkyz8M5YnhoXWzsX/+MBI3je+TJF7c35Z+ysBRdGMcjuKQpNXWExeoYlU7cnhGz5j8vUjLvbBnbp/KkOD8wlaM9OYrjVqibGWY9ubSOz3Mo9+k8it3ZrRt7XRp/nmzs1YFB7HsyPalRr+XJ6M3AI2HErh7n7BONrbEervwWu3deK5pfv5cnMMj99wZXe4kopNmqcXRnK+oJhPp/awbKh5LfNwcWR8jyDG9yj99121P5HH5kXw7tojPD+qA9ti0tGaajdglvTw9W3YF5/Jw9e3Zlr/kCvnaanExJ5B/Lw3gbAW3tzarTnDOvhdrPcui5+HC1P6tOT7Had4clhomV8WMak5rNyXwIwhrWlcz/XUdaneE7ijo2Opoeui/uQVFnMwIZMHBxvzbLQ2N9jM2xnLo0Pb4FCFBrYyndoGv70FncbD4Gd5Z/VhZmV48eXdPXAwzxHhZr75Au1LHNq3lQ+h/l48kf8oywIzUZvfNqZ5nboYrhvBq9+FY68UL4y+dNS0/iH8si+R5ZGnmdynknl4gJ/3JVJQbCo1MOOOXi34/Vg67687SusmbozqUv5cFp9vPMb24+m8M7Erbf3cq/jh1K3RXQKY2rcl/918nAFtmvD7sTQ8XBzoWguDYnoGN2bHC8OqfXznQC92v1y1RX8fub4N83fG8vmmGP41/srRoJ9tPIazgx0PDa69OWOskVXUgQvrsP90JoXFmp4tG1/cNq1/CElZeaw7mFyzk59LhyUPGDP23foRG4+mMmvLce7pF2zRBD8XSuF7kgrYe/1smDQXHtsO143g1wNJ/Hogmb/c2JYAr0uzQPYOaUz7Zh58u/1SbxqtNcsi4nn0+92kZJeu214WEU87fw86Nfcsdd23x3chrIU3TyzYU+7n8OeJM3yw/ihjw5ozqWf9N26V5R+3dKSdvwd//SGSjYdT6N/at+ZfyvWkmZcLd/QOYsnuOBIySk9dEXU6k+WRCdzVN5gm7ld34NbVZpt/PVEndpu7D3Zv6X1x2w3t/Qj0bnRFV7EqMZmMBRFy09AT57A/TfN/i/bSvpkHL46xfFWgCyPh5kRkQqdxYO/IlqOpPLFgD10CvXhgUOlfckop7u0fwsHELHafOktaTj4Pf7ebvy7ay+qoJO4yTxMKcDw1hz2xGYzvEXhFW4ybswNz7u9Np0AvHpu3m42HS8/bdiQpm6cW7qGFjytvjutstW05Lo72fDK1O+cKikjKyquV6pP69Mj1bdAa/ltiIYWjydnc+78/8fNw5pHrK1l39BogCVxctPvUWVo1ccO3RKnFPuMk33l8whvxD7D/VJkTTlZuz7dwbB0bWj7FsPkZ3Prp7xQUmfh0avcqLUPl5uzAhJ5BrNqfSGp2PtuOpfHQt+G0aerOdw/0KfNc47o3x8PFgTdWHmTEB1vYdDSVF0a3Z96DfYk7m8vdX+3k7LkCftxzGjtFqa5sJXm6OPLt9D60a+bBw9/vZllEPJ9siObmD7Zw84dbOJtbwCdTuuNRw8ml6tp1/h68PrYzLo523FCN7pHWJKixKxN6BLFgVxwpWXkcS8lh6uydONgp5j/Ur/Ya3q2YRWti1pay1sQU1kFrTa831zO0nR/v3dEN8rKM1cJ3fI4uLkSheT3wS/7x0JTKT1bCmZw8Cj/qRVK+I7cXvkG/1k24pWtzRnZuVq1BEDGpOQx7bzOjOjdj05FUWvg0YsFD/Up96Vzu9Z8P8r8/TtAl0Iv37uh2sQfB79FpTJ+7i7ZN3ck8X0jrpm5890DfCq+fkVvAlNk7OZSYBRjVNLd0bc6oLs0qnL/E2hQWm6o0aMhanUo/x43vbWZkp2bsOnkGk4aFM/pZXRtETZW3Jma9N2IK63AqPZf0cwX0DG4Mcbtg4VQ4lwLdpqK63w3fjCbn5G4OJoy5OENbZdYdTGblkm/4yBTHnvZvsOPWm2qc5C6MhFsdlUSbpm7Me7Di5A3wzPBQegR7c3OnZqWS1qDQJsy6pyczvt1NQbGJv93crtLre7s6Mf/Bvqw7lMzg0Cal6txtybWQvAGCfd0YG9acZRGn8XVzYsE1mLwrIglcAJfqv3u29IZfpoOdAzz0GwT2BJMJ7eROD32KTzdG8/ldPSs817n8Iv6x/ABLI+JZ6r6aQhc/Rt7xSK0tc/XsiHa4OTnw+thOFv1M9nBxLHfk49B2fvz33p4sizjNzZ2aWXT9xm5O3NHr6kwXKir3zE3XkZ1XxF+HX2c1/bOvFkngAjD6f3s4OxB6PhLidxkregeaE7WdHSqgG4PTE3k+Kono5OwKJ6r/cnMMy/bE82o/O3pG7oHBL9Va8gYIa+HNl/dU/CVSFTe087P5+uCGrIWPK7PvvaJ2oUG4Nn5HiRqLOHWWsJbe2P3xgbHgb/fL1vRs1pXm+TG4OSo+3Vj+bHBaa37Zl8jANk24z/5XsHeGXta77qEQtkwSuA3aH5/Jp79Fcy6/qFbOl51XyJHkbEb5JhmjJPs/Do6X1e0GdEMV5vJkGPy8N4HjqTllnutocg7H085x23WNYO9C6DrJWPFdCFHrJIHbmL1xGUydvYN31x5l1Edb+fNEzRdCiozLQGsYkT4PXLyMObQvF9AVgClBZ3FysOOzjTFX7gOsjkpEKRhduBaKzkPfR2scnxCibJLAbUjU6Uzu+Xon3m6OfDa1BwB3ztrOW78crNaE/mB8Ibyy4gDtHRLxjVtrLPrrUkYvkybtwMEFj7MHmdonmJ8iTxObfuX83Kv3J9E32Av3vXMgZDA061ytuIQQlas0gSul2imlIkvcspRSTyulfJRS65RS0eb7xpWdS1TfwYQs7v56Jx4ujsx/sB9jugaw+qnBTO3TktlbTzDxy21VmtC/oMjEe2uPMP6LbZwvKGbuddtQDi7Q95GyD7B3AL+OkLiXh69vjb2d4vNNpevCY1JzOJKczaNN9kJWPPR7rCZvWQhRiUoTuNb6iNY6TGsdBvQEcoEfgZnABq11KLDB/FzUgaPJ2dz99U5cHOxZ8FC/i7OvuTk78NbtXXjr9s5Enc5i10nLqlOOJGVz++d/8MlvxxgXFsiv97XE/+Ry6HlfxfXVAd0gaR/+Hs5M7t2CpRHxnC4xD8WaqCQ8yWHQ8Q+hWRe47uYavGshRGWqWoUyDIjRWp8CxsLFBU/mAuNqMS5hZgwP3oGDnWLBjH609L1y6sxxYYE4O9ixen9ihecqNmm+2BTDrZ/8TmrmOZYMP8976kM8vxpg9Pse8JeKgwnoaizGkHHq4jwTX266VBe+an8iH3ovwv58Ooz9rOyVc4QQtaaqCXwysMD82F9rnQhgvi+zI61SaoZSKlwpFS5zflfNibRzTJ29AzDmdmjVxK3M/dycHRjarilrDiRhKqca5UTaOSZ9uY1/rznMzKB9bHf9K722PgDHN0LP+2HGJvCqZBa9APPk+Yl7ae7diIk9g/hhVxxJmXnEpufSJGkrN+ath0FPX9pXCFFnLE7gSikn4DZgcVUuoLWepbXupbXu1bRpzdaiu1YtjzzN9G92MX9nLGfOFQDGHA9TZu2gyKSZ/1DfiocHFxUwKfg86Vnn2BN39oqXtxxNZdRHW4hLOcvG9j8xPfmf2HsGwKRv4NkjMPod8LNgVkC/TqDsIXEfAI8NbUux1vx3SwzrI4/xluPXFDZuC0P+Xp2PQQhRRVUZiTkKiNBaX5gQOVkpFaC1TlRKBQDVnKquYVseeZqnf4jEzcmB3w6n8PLyKAa2bUJMSg55RcXMf7Bf5cOD18zkpvCvOejsQPoPbaBdH6ME3KwrJr9OvPnLQXp4ZDHX4zMcT+6FgU/DjS8bDZNV4egCTdtD4l7AGAF3e/dA5u+M5Z1Gc2mu0rEbv6DKq8ALIaqnKv+Dp3Cp+gRgBTANeNt8v7wW42oQVu5L4JkfIunbyoc59/XheFoOK/cl8vPeBM4VFPH9A30rnzgqcR+E/w863MaG0y40yTlCs8O/oPZ8B4BC8YWpGS2cz+GYoWDyAmg/uvpBB3S9uJ4lwOM3tCV+zzrGFq5mb9BUurXoU/1zCyGqxKIErpRyBYYDD5fY/DawSCn1ABALTKr98K5da6KSeGphJD2DG/P1tN40crKnU3MvOjX34u83t6OwWOPkUEkNl9aw+jlo1Bhu+5icAzk8umQfK+4fQFfPc+jEvXz/00paFcbQupUPjPwX+NRw+bqAbrB3AWQngUczWjXK40vXLzhR4I/XmNdrdm4hRJVYlMC11rkYyxSW3JaO0StFVNHW6FSeWBBBtyAv5tzfBzfn0n8GpRRODhas6nLgR4jdBrd8AI0aM7yjGw52ilVRyXQd1Z71px15OQPem9QNVVvLfDUzRmSSuM+YM+XHh/HSWRwZuYi+AdLGIcTVJCMxr7K8wmKeX7afYF83vpneB3fnak4IWZALa182+lv3mAYYc1X3b+PLmqhEtNZ88ls0LX1cGRtW9lSq1dLMvIBs4l7Y9jEcW4e6+Z/0HXBD7V1DCGERSeBX2X83Hyf+7HleH9sJz5osv7XtY2O046h3SvW3Ht0lgJPpuXy5+Tj74jN5rDZWky/JxRN8WsP+RbDhdeg4Fno/WHvnF0JYTBJ4HfhxTzy3ffo78WdLzxUSfzaXzzcdY0yXAAa0qcEMfRlx8PuH0Gk8BA8o9dKIjv7YKXjn18MEejdifI86WCE9oBukHQXvFnDbJ2Cli/gKca2TBF4HFv4Zx774TKbO3kli5qWh5v9cdQil4IUqrMR+Ba1htbmf9fArGw193Z3p28oXreGRoW0qbwitjpb9jXm+J84xZi8UQtQLSeC1LLegiIjYs9zQrilnzxUwdfZOkrPy2HYsjVX7k3hsaFsCvS+ba/v4Zlj+uFGvXZmopXBkFdz4olECLsN9A0MY0MaXSbXVcHm53g/CXw9BYI+6Ob8QwiKypFot23niDIXFmumDWvGXG0O59+udTJ29AzulaOHTiBlDWpc+QGv49QVIjoLzGXDHt+XPIXIuzSh9B/ascKa/mzs1s3h9x2qxswc338r3E0LUKSmB17I/otNwcrCjd4gPPYMbM+f+PiRk5BGdksNLYzri4nhZcj651UjerYfC4ZXw64vln3z13yEvSyaKEkIAUgKvdb8fS6N3SOOLibpPKx/mPdSXPbEZjOjof+UBO74EV1+YstDo1bHjc2gcDP0uW8nm8Cqj+uSGFy2bt0QIcc2TBF6LUrLzOJyUzd9Htiu1vUfLxvRoWcZ6F2eOG/XZg5811qAc8SZkxMKa58HZ49KMfsUFsPIZ8O8Mg565Cu9ECGELJIHXou0x6QAMamthF8E/ZxtVIRf6UdvZw/jZMPdWo1GzJGUPUxeCfQ36jgshrimSwGvR1ug0vF0d6dTcgq51eVkQ8R10uh08Ay5td3KFe5fDiS2gS6xz6dMG/DvWftBCCJslCbw27F+CXv8K/5eZxwsOdth/6ARNQuHupeU3NkbOh4Lssldtd3av2YyBQogGQXqh1IYdX1BUVMyWok6c9R9grMR+fGOpaVdLMZlg55cQ1AeCel7dWIUQ1wxJ4DWVGQ+nw9kXMIm/Fz2M4/gv4I7vwK0phM8p+5joX+HsiSt7mgghRBVIAq+pQz8D8GNeD1r4NDIWHXZwgrC74OgayEoovb/WsPkd8GoJHW6th4CFENcKSeA1dXA52q8jy+NcS/c+6TnNaITc833p/Q//AgkRMPQ56VEihKgRSeA1kZ0EsTtIDLyZ7PwiBrUtsaCBT2tofQPsngsmc28SUzH89ib4hkLXyfUTsxDimiEJvCYO/QxoPkvqhFLQv81l84P0vM+Ys/vYeuP5/iWQesiYiKqqCwoLIcRlLErgSilvpdQSpdRhpdQhpVR/pZSPUmqdUirafF/GUMNr29nwpZwgkB9OufHCqA74uDmV3qH9GGPZsfA5UFwIm/5prGjTYWz9BCyEuKZYWgL/CFijtW4PdAMOATOBDVrrUGCD+XmDkJNfxBs/bMYzeQfbnAfx0+MDeejyWQbBqOPufrfR62TT23D2JNz4D7CTHz5CiJqrNJMopTyBIcDXAFrrAq11BjAWmGvebS4wrm5CtC7bY9K5+YMt5O5bgb3STLr3cToHVjDysuc0o+fJ1nehRT8IHX71ghVCXNMsKQq2BlKBOUqpPUqpr5RSboC/1joRwHzvV4dx1ru8wmJe+/kAU2bvwNFeMTPkKDRuhVPzrhUf2DgE2txoPB72siw/JoSoNZYkcAegB/CF1ro7cI4qVJcopWYopcKVUuGpqanVDLN+HUvJZvTHW5nzx0mm9Q9m1YwueCVuMxb0tSQh3/wWjHkPQgbVfbBCiAbDkq4Q8UC81nqn+fkSjASerJQK0FonKqUCgJSyDtZazwJmAfTq1UvXQsxX3bu/HiUtO595D/ZlYNsmxjwmpiIjgVvCr4PM4S2EqHWVlsC11klAnFLqwiTXw4CDwApgmnnbNGB5nURYz3ILith0NIVx3QON5A1w9FfwaA7Nu9dvcEKIBs3SzshPAPOUUk7AceB+jOS/SCn1ABALTKqbEOvXpiOp5BWaGNnZvMakyWQsgxY6QuqzhRD1yqIErrWOBHqV8dKwWo3GCq2OSsLXzYk+IT7GhpQDkJsOra6v38CEEA2edEiuQF5hMb8dSmZEJ38c7M0f1fHNxn1rSeBCiPolCbwCW46mcq6gmFGdS6yYc2KzMZeJZ/P6C0wIIZAEXqE1UUl4NXK8NMdJcSGc2ialbyGEVZAEXo6CIhPrDiUzvKM/jheqT07vhoIcaDWkfoMTQggkgZct4lty/zucvLw8Rl3ofQLm+m8FIYPrLTQhhLhAEnhZ9i/GOzWce503Myi0xCINJzZDQFdw9am/2IQQwkwS+OWKC9Hx4QA86fAjzqZ8Y3tBLsT9Kd0HhRBWQxL45RL3ogpzmVN0M17FZ+DPWcb22O1gKpQGTCGE1ZAEfrlT2wD4itspbnMT/PEh5GUa1Sd2jtCyf/3GJ4QQZpLALxe7gySH5vj4t8B+2Mtw/ixs/8xowAzqDU5u9R2hEEIAksBLM5kgdjvhpnaE+rlD8zBjxsFtn0LiXqk+EUJYFUngJaUdhfNn2JwfShs/d2PbDS9C0XlASwOmEMKqSAIvKdao/951oQQO0LQdhN0FjXwgsGc9BieEEKVJAi/p1HbynH05qZsR6u9xafuY9+GxHeDgVP6xQghxlUkCLyl2Bydcu+HkYE+Lxo0ubXdwAg//+otLCCHKIAn8gsx4yIxlj2pP6yZul6aPFUIIKyVZ6oJT2wHYkNuathfqv4UQwoo1iAS+7mAyH2+Irnin2G1oJ3c2ZTYj1M+j4n2FEMIKWLomps36ZV8iTyyIwKThxvZ+dA70KnvHU9vJ8etJcZYdof5SAhdCWD+LSuBKqZNKqf1KqUilVLh5m49Sap1SKtp837huQ626NVFJPLlwD91bNqaRoz3fbj9Z9o65ZyD1EHHu3QAudSEUQggrVpUqlBu01mFa6wuLG88ENmitQ4EN5udWY8OhZJ5YEEHXIC/mTu/DuO6BLI9MICO34Mqd43YCsNeuAw52imBfGS4vhLB+NakDHwvMNT+eC4yrcTS1ZFtMGo9+H0GHAE/mTu+Du7MD9/YPJr/IxKLwuCsPOPk72Dnye24wwb6uODk0iKYBIYSNszRTaWCtUmq3UmqGeZu/1joRwHzvVxcBVse/Vx+mmZcL303vi6eLIwAdAjzpE+LD9ztiMZl06QOOroFWgzmUXigNmEIIm2FpAh+ote4BjAIeV0pZvCikUmqGUipcKRWempparSCrIjIug73xmTwwqBVero6lXrunfzCxZ3LZfLREHKlHIf0YhaEjOZWeKw2YQgibYVEC11onmO9TgB+BPkCyUioAwHyfUs6xs7TWvbTWvZo2bVo7UVfg2+0ncXOyZ3yPwCteu7lTM/w8nJlbsjHzyC8AxDYZSrFJSx9wIYTNqDSBK6XclFIeFx4DI4AoYAUwzbzbNGB5XQVpqfScfFbuS2R8jyA8XByveN3JwY4pfVqy+Wgqp9LPGRuPrIaAbhzKNapOJIELIWyFJSVwf+B3pdRe4E/gF631GuBtYLhSKhoYbn5er34Ij6OgyMS9/YPL3Wdq35bYK8X3O05BToqxzmW70RxLyUEpaNNUErgQwjZUOpBHa30c6FbG9nRgWF0EVR3FJs28HbH0b+1beibBy/h7ujCyczMW/BnHo57b8UFDu9FEb8yhpY8rLo72VzFqIYSovmumv9yGQ8mczjjPtAHll74veG5kexRwbOsitFcQNOvCseQcGcAjhLAp10wC/27HKQK8XLipQ+XTvrbwceWdsaF0yY8gwqU/RSbN8bQc2koXQiGEDbkmEnhMag5bo9O4q29Li6eBHeV6mEaqgPfj2jJvZyyFxdIDRQhhW66JyayWRcTjYKe4s3dLyw868gva2ZMznr159ecDgMyBIoSwLddECTw5Kx9/TxeaejhbdoCpGI6sQYUO56O7+uBsHjrfRhK4EMKGXBMl8Oy8Qtydq/BW4sMhNw3ajeY6fw/emxTGjuPpVTuHEELUs2siY+XkF+HhYuFb0Rp2fA52DhA6HIAxXQMY0zWgDiMUQojad01UoWTnFeFuaQLf+E84+BMMnQku5SzuIIQQNuCaSOA5eUWlh85Hr4OfHof0mNI7RnwHW96B7vfA4P+7ukEKIUQtuyYSeFZeUen6641vQeT38FlfWPsS5GVCzG+w8mlocyPc8gEoVW/xCiFEbbhG6sAL8bxQhZIeAwl7YMCTcP4MbPsUIhdAUT40aQeT5oL9lRNdCSGErbH5BF5YbCKv0HSpBB611Ljv+zB4BUHvB+HXFyEzDu5aDC6e9ResEELUIptP4Dl5RQBGI6bWsH8xBA80kjdA8+5w/yrjNak2EUJcQ2y+Djwn30jgHi6OkLQf0o5C5wlX7ijJWwhxjbH5BJ6VVwhgVKFELTH6d3ccV79BCSHEVWDzCfxCFYqnsx1ELYPWN4Cbbz1HJYQQdc/mE3i2OYH7Ze41Giq7TKzniIQQ4uqw+QR+oQ7c79RKcHCB9mPqOSIhhLg6bD6BZ+cVYk8x7jEr4bqR4CyLMgghGgaLE7hSyl4ptUcptdL83EcptU4pFW2+b1x3YZYvO7+IAXYHsMtNk+oTIUSDUpUS+FPAoRLPZwIbtNahwAbz86suO6+IYfaRaEdXaDu8PkIQQoh6YVECV0oFAWOAr0psHgvMNT+eC4yr1cgslJNXRHOHTJRnIDi61EcIQghRLywtgX8I/B0wldjmr7VOBDDf+5V1oFJqhlIqXCkVnpqaWpNYy5SdV0hTlQVuTWv93EIIYc0qTeBKqVuAFK317upcQGs9S2vdS2vdq2nT2k+yOflF+JIFbk1q/dxCCGHNLJkLZSBwm1JqNOACeCqlvgeSlVIBWutEpVQAkFKXgZYnK68Ib50pJXAhRINTaQlca/281jpIax0CTAZ+01rfDawAppl3mwYsr7MoK5B7Ph9PLVUoQoiGpyb9wN8GhiulooHh5udXnUP+GeOBVKEIIRqYKk0nq7XeBGwyP04HhtV+SFXjlJduPJASuBCigbHpkZhaaxoVXiiBSwIXQjQsNp3A84tMeJmyjCdShSKEaGBsOoFn5xXhqzKNJ1ICF0I0MDaewAvxVVmYlD24eNd3OEIIcVXZdAK/MIin0NkH7Gz6rQghRJXZdNbLziuiicqiqJGswCOEaHhsPoH7qky0qzRgCiEaHhtP4IX4koVylwZMIUTDY9MJPCe/CF+Vhb17mRMhCiHENa1KIzGtzflzObirPIo9JYELIRoemy6Bm84Z84vbe0gCF0I0PDadwNW5NOOBDOIRQjRANp3A7XLNK/xIAhdCNEA2ncAdZSpZIUQDZtMJ3OlCApd+4EKIBsimE7hrwRkKlDM4udV3KEIIcdXZdAJ3KzpLjkNjUKq+QxFCiKvOphO4hymD846N6zsMIYSoF5UmcKWUi1LqT6XUXqXUAaXUa+btPkqpdUqpaPP9Vc2kJpPG25RJvrPP1bysEEJYDUtK4PnAjVrrbkAYMFIp1Q+YCWzQWocCG8zPr5rcwmJ8VRaFLjIToRCiYao0gWtDjvmpo/mmgbHAXPP2ucC4ugiwPNnnC/Alk+JG0gNFCNEwWVQHrpSyV0pFAinAOq31TsBfa50IYL6/quPZc7PO4KSK0a4yiEcI0TBZlMC11sVa6zAgCOijlOps6QWUUjOUUuFKqfDU1NRqhnmlvMxkAOxkKlkhRANVpV4oWusMYBMwEkhWSgUAmO9Tyjlmlta6l9a6V9OmtZdsCzKNy9l7SgIXQjRMlvRCaaqU8jY/bgTcBBwGVgDTzLtNA5bXUYxlMmUbCdzZq9nVvKwQQlgNS+YDDwDmKqXsMRL+Iq31SqXUdmCRUuoBIBaYVIdxXsGUY1THOHv5X83LCiGE1ag0gWut9wHdy9ieDgyri6Asoc4bU8m6NZa5wIUQDZPNjsS0P59OpnbFrZFrfYcihBD1wmYTuFNeOmfwws5O5kERQjRMNpvAnfPPkGnnXd9hCCFEvbHZBO5adIZse+/6DkMIIeqNzSZw96IMzslMhEKIBsw2E3hxEe6mLM47ykyEQoiGyzYT+Pkz2KEpcJEELoRouGwzgZ8zBvEUuchMhEKIhsumE7ipkcwFLoRouGwygReb50HRbjIKUwjRcNlkAr84E6FMJSuEaMBsM4Fnp1Ck7XD0kEZMIUTDZXsJXGucTm3hlPbHs5FTfUcjhBD1xvYS+MmtNErZw5zikbg7O9Z3NEIIUW9sL4FvfY98lyYsLr4eDxdLpjMXQohrk20l8NO74fgmjrWZRj5OuEsCF0I0YLaVwLe+Dy5eRAVMAMDDWRK4EKLhsp0EnnIYDq+EPjM4U+QCgIeL1IELIRou20ngf3wIjq7Q91Fy8guxt1O4ONpO+EIIUdssWZW+hVJqo1LqkFLqgFLqKfN2H6XUOqVUtPm+7uZ2PXsK9i2CnveBmy/pOQV4NXJEKVmNRwjRcFlShC0CntVadwD6AY8rpToCM4ENWutQYIP5ed3Y9gkoO+j/FwB2nzpLl0CvOrucEELYAktWpU8EEs2Ps5VSh4BAYCww1LzbXGAT8FydRNn9LvDrAF6BpOfkE52Sw+09AuvkUkIIYSuq1I1DKRUCdAd2Av7m5I7WOlEpVXczSzXvbtyAXSfPANC3lQyjF0I0bBa3Aiql3IGlwNNa66wqHDdDKRWulApPTU2tToyl7Dh+BhdHO7oEetf4XEIIYcssSuBKKUeM5D1Pa73MvDlZKRVgfj0ASCnrWK31LK11L611r6ZNaz574M4TZ+gZ3BgnB+mBIoRo2CzphaKAr4FDWuv3S7y0AphmfjwNWF774ZWWmVvI4aQs+oTIQg5CCGFJHfhA4B5gv1Iq0rztBeBtYJFS6gEgFphUJxGWsOvkGbSGvq2l/lsIISzphfI7UF6H62G1G07Fdp5Ix8nejrAW3lfzskIIYZVsqiJ554kzhLXwxsXRvr5DEUKIemczCTwnv4io05lSfSKEEGY2k8DDT57BpKFvK2nAFEIIsKEEvvPEGRzsFD2Cves7FCGEsAo2k8D/PHGGLkFeuDrJHOBCCAE2ksDPFxSzLz5Dqk+EEKIEm0jgEbFnKSzWMv+JEEKUYBMJfOeJM9gp6BVSd1OOCyGErbGJBB7o7cLEnkGyhJoQQpRgEy2Cd/ZuyZ29W9Z3GEIIYVVsogQuhBDiSpLAhRDCRkkCF0IIGyUJXAghbJQkcCGEsFGSwIUQwkZJAhdCCBslCVwIIWyU0lpfvYsplQqcqubhTYC0Wgyntkl8NSPx1YzEV3PWHGOw1rrp5RuvagKvCaVUuNa6V33HUR6Jr2YkvpqR+GrOFmK8nFShCCGEjZIELoQQNsqWEvis+g6gEhJfzUh8NSPx1ZwtxFiKzdSBCyGEKM2WSuBCCCFKsIkErpQaqZQ6opQ6ppSaaQXx/E8plaKUiiqxzUcptU4pFW2+r7flg5RSLZRSG5VSh5RSB5RST1lTjEopF6XUn0qpveb4XrOm+Myx2Cul9iilVlpbbOZ4Tiql9iulIpVS4dYWo1LKWym1RCl12PzvsL+1xKeUamf+3C7cspRST1tLfFVh9QlcKWUPfAaMAjoCU5RSHes3Kr4BRl62bSawQWsdCmwwP68vRcCzWusOQD/gcfNnZi0x5gM3aq27AWHASKVUPyuKD+Ap4FCJ59YU2wU3aK3DSnR9s6YYPwLWaK3bA90wPkuriE9rfcT8uYUBPYFc4Edria9KtNZWfQP6A7+WeP488LwVxBUCRJV4fgQIMD8OAI7Ud4wlYlsODLfGGAFXIALoay3xAUEY/4FvBFZa498XOAk0uWybVcQIeAInMLexWVt8l8U0AvjDWuOr7Gb1JXAgEIgr8TzevM3a+GutEwHM9371HA8ASqkQoDuwEyuK0VxFEQmkAOu01tYU34fA3wFTiW3WEtsFGlirlNqtlJph3mYtMbYGUoE55mqor5RSblYUX0mTgQXmx9YYX4VsIYGrMrZJ1xkLKKXcgaXA01rrrPqOpyStdbE2fsIGAX2UUp3rOSQAlFK3ACla6931HUslBmqte2BULT6ulBpS3wGV4AD0AL7QWncHzmGF1RFKKSfgNmBxfcdSXbaQwOOBFiWeBwEJ9RRLRZKVUgEA5vuU+gxGKeWIkbznaa2XmTdbVYwAWusMYBNGm4I1xDcQuE0pdRJYCNyolPreSmK7SGudYL5Pwai/7YP1xBgPxJt/VQEswUjo1hLfBaOACK11svm5tcVXKVtI4LuAUKVUK/M35mRgRT3HVJYVwDTz42kY9c71QimlgK+BQ1rr90u8ZBUxKqWaKqW8zY8bATcBh60hPq3181rrIK11CMa/td+01ndbQ2wXKKXclFIeFx5j1ONGYSUxaq2TgDilVDvzpmHAQawkvhKmcKn6BKwvvsrVdyW8hQ0No4GjQAzwohXEswBIBAoxShsPAL4YDV/R5nufeoxvEEY10z4g0nwbbS0xAl2BPeb4ooB/mLdbRXwl4hzKpUZMq4kNo455r/l24ML/CSuLMQwIN/+NfwIaW1l8rkA64FVim9XEZ+lNRmIKIYSNsoUqFCGEEGWQBC6EEDZKErgQQtgoSeBCCGGjJIELIYSNkgQuhBA2ShK4EELYKEngQghho/4fTs5fbN7pPq4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_VAL_1','train_VAL_1']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA7v0lEQVR4nO3dd3iUVdrH8e9JJyQQSAiEUBJ6LxJ6F0GwgAIi2AB1lUWxrO6qq772Xde1uwqiUkSKiIKICBaKSiAQOoGElpBOCul1MnPeP54AAQJMQpKZTO7Pdc2VzDMzz9wh+puT85yitNYIIYSo/ZxsXYAQQoiqIYEuhBAOQgJdCCEchAS6EEI4CAl0IYRwEC62emM/Pz8dFBRkq7cXQohaaffu3Wla6yblPWazQA8KCiI8PNxWby+EELWSUurU5R6TLhchhHAQEuhCCOEgJNCFEMJB2KwPvTwmk4n4+HgKCwttXYoow8PDgxYtWuDq6mrrUoQQV2BXgR4fH4+3tzdBQUEopWxdjgC01qSnpxMfH09wcLCtyxFCXIFddbkUFhbi6+srYW5HlFL4+vrKX01C1AJ2FeiAhLkdkt+JELWD3QW6EEI4tC3/gcS91XJqu+pDF0IIhxb1E2z5F5iLoXnvKj+9tNCvgZeX12UfCw4OJioq6oJjTzzxBG+99RYAe/fuRSnFxo0brT5nWfPmzePLL7+sYMVCiGqXdgwKsy89nn8G1j4GTbvB8Geq5a0l0KvJ1KlTWbFixbn7FouFVatWceeddwKwfPlyhgwZwvLlyyt1/lmzZnHfffdVSa1CiCqSGQtzB8HnN0BWwoWPrX8aCjLg9nng4lYtb2+3XS6v/BDB4cRyPuWuQZfmDXjp1q6XffyZZ56hdevWzJ49G4CXX34ZpRS///47GRkZmEwmXn/9dSZMmHDV95o2bRp33nknL730EgC///47QUFBtG7dGq01q1at4pdffmHo0KEUFhbi4eFRoZ/l5ZdfxsvLi6effprPPvuM+fPnU1xcTLt27ViyZAmenp6cPn2aWbNmcfLkSQDmzp3LoEGDKvQ+QtQ5x36F3NPQoDk0CDS+agsU5Ri3kgJo1hOcymkPb/sAtIbsRFgwFu5bA75tIWINHPoWRr4AzbpXW+nSQi9j6tSpfP311+fur1y5kpkzZ7J69Wr27NnD5s2beeqpp7BmH9YePXrg5OTE/v37AVixYgXTpk0DYNu2bQQHB9O2bVtGjBjB+vXrr6nuiRMnsmvXLvbv30/nzp354osvAHjssccYPnw4+/fvZ8+ePXTtevkPMyEEcPh7WDoJvp8NS26Dj/vCvwPhzZbwXhf4pD/MHwFrZl362uwk2LMEet0FM34AU54R6ic2wY9/M/rMhzxZreVb1UJXSo0FPgCcgc+11m9e9HhD4CugVek539ZaL7yWwq7Ukq4uvXv3JiUlhcTERFJTU2nUqBEBAQE8+eST/P777zg5OZGQkMDp06dp1qzZVc83bdo0VqxYQdeuXfn+++959dVXAaO7ZerUqYDxIbJkyRImTpxY6boPHTrECy+8QGZmJrm5udx4440AbNq06Vw/u7OzMw0bNqz0ewhRabFhENADXOvZupIry4yFtXMgsA/cPh9yk42WdnYiODmDu7dxOxUKuz6HrhOh49jzrw/9ECwlRmg3DoaZG4wPhSW3g7M73DYPnKu3U+SqZ1dKOQMfA6OBeGCXUmqt1vpwmac9AhzWWt+qlGoCRCmllmqti6ul6mo0efJkVq1aRXJyMlOnTmXp0qWkpqaye/duXF1dCQoKsnqSzbRp0xgzZgzDhw+nR48e+Pv7Yzab+fbbb1m7di1vvPHGuZmYOTk5eHt7V6rmGTNmsGbNGnr27MmiRYvYsmVLpc4jRJX741347RUIuR9uec/W1VyeuQS+fdDoLpn0hRHIfu3Kf26nW+HUdlj3JLQeCB4NITcVwhdCjynGawGadID7N8J3f4Eed4J/p2r/MazpcukHHNdanywN6BXAxZ3IGvBWxgwUL+AMUFKlldaQsxczV61axeTJk8nKysLf3x9XV1c2b97MqVOXXYr4Em3btsXX15dnn332XHfLr7/+Ss+ePYmLiyMmJoZTp04xadIk1qxZU+mac3JyCAgIwGQysXTp0nPHR40axdy5cwEwm81kZ1ftNQkhrmjbB0aYe/rC3q+Mlq418tKMgK0KWsP+FZC478rP2/JviAszPnQaX2WJCxc3mPCR0YL/+UXj2Pb/QUkhDH3qwuf6tIT7N0DIzEr/CBVhTaAHAnFl7seXHivrf0BnIBE4CDyutbZcfCKl1ENKqXClVHhqamolS65eXbt2JScnh8DAQAICArj77rsJDw8nJCSEpUuX0qlTxT5lp02bRmRkJLfffjtgdLec/f6sSZMmsWzZMgDy8/Np0aLFudu777572XOfncH52muv0b9/f0aPHn1BfR988AGbN2+me/fu9OnTh4iIiArVLkSlhf4Pfvk/6DYJHvgFLGYj4K8mcR+82wU+6Al/vm8M9TsrNcoI0Pe7w9a3jLC+mp2fweqHYf5wowWeUU6D7ORW+OMd6HUPdJ9s3c8X2AcGPgp7FhsXPHd9Dt0mgl97615fTdTVLvAppe4AbtRaP1h6/16gn9Z6TpnnTAYGA38D2gK/AD211pdtEoaEhOiLdyw6cuQInTt3ruSPUrfMmTOH6667jpkza+aTX343tYTWRkuxKvuri/OM2zkK6vtBeUtClBTDjk/g15egy21G94WzC6yZbYzyeOIgePmX/z6F2fDpMCgpMro7on8HV0+jrzrtKMTvBCcX8OsIKREwaA6Mfq38OsDo6158K7QdBc26wfaPjdEqff9ijFxJjTQ+JJIPQsMW8PBWcKtv/b+LqQDmDoYzJwENf90OTbtY//pKUkrt1lqHlPeYNT308UDLMvdbYLTEy5oJvKmNT4fjSqlooBOwsxL1iqt48cUXCQsL4+WXX7Z1KaImWSwQvRUOrzFCz9nNuDm5QE4inImGjBhjaN2IZ43btb5f2Fz47VXjQ6Is7wBoe71xazUQkg8YI0Qi10NRFnS+FSZ9fv4i4JC/wf7lEPoRjHnt0vfSGn54zLgwOeNHo286+SDsmAcHV0KjYBjzutEX7ekHP/3dOJepAMb999IhhNmJsHI6+LSGSZ8Z/dwhDxizNHd8AmjjPP6doffdMGB2xcIcjA/N8R/Bopug0y01EuZXY02g7wLaK6WCgQRgKnDXRc+JBUYBfyilmgIdgZNVWai9OnjwIPfee+8Fx9zd3QkLC6uS87/xxht88803Fxy744472LlTPivrjIwY2Lcc9i2FrDhwbwD1fIzWsLkIzCbwamr0/bbsD9kJRp9wg0C47t6rnf3y77lmNpzaBh3GQfsbzj9WUmy0lqPWGzWd5dEQOt8CXSZAuxuMkSFn+bUzWtq7voDBT0B93wvfb/dCiFgNo/7PCHMwxmvf9jHc+oFxrrIt8ZveNlrvoR8aoX7L++cn65QUwdf3gikfpv9g1AXQMBAmfGyMBXd2Nf7KuFZBg40Ln02q/4KnNa7a5QKglLoJeB9j2OICrfUbSqlZAFrreUqp5sAiIABQGK31r650TulyqV3kd1ODshONft1Tf0LMNsiIBhS0HQm974GON4PrFSaimU2wbIrRZXH3N0Yr+mrMJsgpHaaXsBs2v2G857j/GOOqy+vWsJghaZ8x4sO/EwQNu/IMyJQj8MkAGPo0jHrx/PHkg/DZKAgaAnevKn/CTnm0hq3/MT68nFygcVto0hGKc42x31O+ND5cHMyVulysCvTqIIFeu8jvpobsWw5rHzXGM3v4QOvBRiuw863g08r68xRmw8JxxkXABzZC08vM68hNha9uh+RDGIPVSgUNhds+qdh7WuPre+HkFrj9UzhzwujHPv6bEc6z/gSvJhU/57FfIHa70R+eGgmZccZokxHVs16KrV1rH7oQorppbYwC+fUlCB4ON/4L/LtY31q9mEcDuGulsabI0jvgwV+NC4EXW/+UEYTDnjYuDDYING5NOlX+va9k2N/hyFpYYQzjpb6/0bof9VLlwhyg/WjjdpbFUj211wIS6ELYmsUCPz9vXKzrNglumwsu7td+3oaBcPdKY/r50ilw/0/GTMezIlYbFzJHvQRD/2bVKZOyCrjn8zAGtPHlH2M70bBeBfeZDegBM9aDcjK6RzwbV+z11qijYQ4S6ELUnKwE2PUZ7P/aCOyzCz/lpxl9vv3/arTMrxJIyVmF+Hu74+R09Z2k8ht35nPfF5mT/E/UNzNg2tfGyJO8NPjxKWN9kUGPWVW+xaJ5auV+4jIKiN4Zyy+HT/PK+K6M7dbsgl2tTGYL8RkFxKTlcTItj6TMAsZ0bUa/4NLwDhps1fuJipNALyMzM5Nly5adW23RWjfddBPLli3Dx8fH6tcsWrSIjRs3XrB8blpaGp07dyY+Ph53d3cmTJhASkoK27dvP/ecsqssXs2gQYMIDQ2t0M8iqkH8bqP1fXiNMQ66/Y3g5mks5hQXBgWZcMMrMPjxy4+pLrXhUDJ/Xbqb6zv689FdvfF0u/L/wotDT/FudEu8mj/G/cffM7pYbnnfWMq1KAcmfGL1+iKf/3mS0BPp/GdSd7oENOTZ7w7w16V7GNXJn1a+nsSk5RGTnk/cmXxKLOf7412cFJ//Gc20fi15dmxnGnpWsFVvxywWzT++PUCvlj7cM6C1rcuRQC8rMzOTTz755JJAN5vNODs7X+ZVVGq1xIkTJ/L000+Tn5+Pp6cnAKtWrWL8+PG4u7uTmZnJnj178PLyIjo6muDgq0xHLoeEuY2dCjVGYZzcAm7e0O9h6P8QNAqq1On2x2XyxNd7ad3Yk81RKdz56Q6+mBGCv3f5I15yCk18+vsJ3F2ceDWxL7cPnkOj3R9Bboox5PD6F60eOx2RmMV/N0ZxY9emTAlpiVKK7x8ZzIJt0bz3yzFCT6QT5FefLgENuKl7M4J869OmSX2CfOtTz82ZD349xud/RvPL4RReHt+Fm7sHOMReteGnMli1O55Vu+OJScvjnzd1tuovp+piv4H+07PGcKaq1Kw7jHvzsg8/++yznDhxgl69euHq6oqXlxcBAQHs27ePw4cPc9tttxEXF0dhYSGPP/44Dz30EABBQUGEh4eTm5vLuHHjGDJkCKGhoQQGBvL9999Tr96ls/YaNGjAsGHD+OGHH85terFixQpeeOEFAL799ltuvfVWmjZtyooVK3juuecq/ON6eXmRm5tLbm4uEyZMKHdN9y+//JK3334bpRQ9evRgyZIlFX6fOs9iMZZKLcw2Wr2Zscb46Jg/oH4TGP0q9JlpXKispPiMfB5YHI6flzur/jqI/XGZPLpsLxM/CWXRzH608790p6sFf8aQmW9i4Yy+PLQknA8td/JStyQ4tAoCehnjwa1QUGzm8RX7aOTpxpsTe5wLYhdnJx4a1paZg4NxcVJXDOjnburMrT2b89x3B3l02V6+bh/HS7d2oZ1/5Rakq0knUnNp0age7i6XNupW702gnqszE68L5PM/o0nMKuDdKb3wcL18A/DXw6cJCWqEj2fVb3JRd68elOPNN9+kbdu27Nu3j//+97/s3LmTN954g8OHjYUlFyxYwO7duwkPD+fDDz8kPT39knMcO3aMRx55hIiICHx8fPj2228v+35nl9cFSExM5OjRo4wcORIw1nyZNm0a06ZNq/SuRmd5eHiUu6Z7REQEb7zxBps2bWL//v188IEVa22I83JTjLVF3mwJ/25xfr3sZXcYU9Vv/Dc8fsDoSrlMmBeXWFi+M5YNh5JIzy0q9znZhSYeWBROUYmZhTP64uflzqjOTfn64QEUmsxMmhtKeMyZC16TmV/M53+cZEyXpozs5M8tPZrzzZ5Ecsd9CCOfhzsWWtXVUlBs5tV1ERxPyeWdKT1pVP/SEHJ1drKqtd0tsCGrZw/ipVu7sC8uk7Hv/8Fr6w6TXWi66mttQWvNx5uPc8O7W3l57eFLHi80mfnxQCJjuzXj9du68cLNnVl/MJl7Pg8jI6/8hWZj0/OZvXQPb/8cVe7j18p+W+hXaEnXlH79+l3Q1fHhhx+yevVqAOLi4jh27Bi+vhfOeAsODqZXr14A9OnTh5iYmMue/5ZbbmH27NlkZ2ezcuVKJk+ejLOzM6dPn+b48eMMGTIEpRQuLi4cOnSIbt26Vern0Frzz3/+85I13Tdt2sTkyZPx8zNmzDVuXA0jDmq7wizY+Lwx27BJR2M4n6evMeMxfIExU7PrRGje6/x62R4NjfHjV1lPJavAxKwlu9l+8nzDoL2/FyFBjS8YPbIr5gwnUnNZNLMf7Zueb9H2aOHD6tmDmb5gJ/d8Ecbcu/swspOxTspnf5wkt7iEv43pAMD0QUGs3pvAt/tTmT78H1esKzGzgF+PnGZTZArbT6RTVGLhwSHBDG1fyWGFZbg4OzFzcDDjezbn7Z+PsmBbNGv2JjD3nj7nL5ragUKTmWe+PcD3+xLx83Ln293xPHlDe/wbnO/e2hKVQnZhCbf1DkQpxYND2xDQsB5PrtzHU9/s54vpIZd80L21MRInJ3h0ZPUs4mW/gW4H6tc/v7bDli1b+PXXX9m+fTuenp6MGDGi3HXR3d3PDzdzdnamoKDgsuevV68eY8eOZfXq1axYsYL33jPWi/7666/JyMg492GSnZ3NihUreP311yv1c1xuTXettUP0Y1YbreH7RyFynbFmStn1TJSzsa7IsKeNLcYqKCGzgJkLdxKdlsfbd/Qk2K8+O6PPEBadzo8HEiksOb9YqbuzE/+a2J0h7S+dqt6ysScrZw1kxsKd/OXLcN6Z0pMh7fxYuC2Gm7sH0KmZ8ZdBr5Y+9Gzpw+LQGO4d0Pqy/by7Ys5w9+dhFJdYCPL15K7+rbi+kz+D21bBNPkyfL3c+ffE7tzdvxWPLtvD4yv2suGJYRUfBlkNTmcX8tCX4eyPz+LvN3bk5u4BXP/OFr7YFs1z485Prlu9NwE/L3cGtz3fqLu5RwBJWQW8/uMRfjiQxPie58f+74nNYN2BJB67vh3NGlZsy0lrSaCX4e3tTU5OTrmPZWVl0ahRIzw9PYmMjGTHjh1V8p7Tpk3jueeeIzs7mwEDBgBGd8uGDRsYONBY0yI6OprRo0dXOtAvt6b7qFGjuP3223nyySfx9fXlzJkz0kovK2weHFnLR873EdrkTsa3LmF44zMEWFJQHcZA4zaVOm1EYhYzF+6ioNjM4pn9GNTOCMs+rRvx1xEV/3Dw83Jn+V8G8ODicJ74eh89WvhQaDLzxA0dLnjezEFBPPH1Pn4/lsqIjpeueJiQWcCsJbtp4VOPz6aH0LbJpf3yVa1bYEM+mNqbiXNDeeWHCN6d0qva3/NKsgtNTPwklIz8Yj69tw83djV2JrupewDLdsTyyMh2NPBwJTO/mM2RqdwzoDUuzhf2XM8cHMwP+xN5ZW0EQ9v50ai+G1pr/vXjEfy83HloeMV/x9aSPvQyfH19GTx4MN26dePvf//7BY+NHTuWkpISevTowYsvvngufK/VmDFjSExM5M4770QpRUxMDLGxsRecPzg4mAYNGpxb8Ov111+/YM30yznb+r7cmu5du3bl+eefZ/jw4fTs2ZO//c26ySV1Qtwu9M8vsJkQVrjcxpl8C89tyWPQd+4M/70jO7N8KnzKohIz87ae4I5523F2Uqz666BzYX6tvD1cWXx/P0Z1asr+uExu6x14yYXSm7oH4OflzuLQmEten19cwoOLwyk2W2oszM/q2dKHR0a247s9CWw4lHzBY9mFJj749RjHU8pvaFW19345SmJWAUse6HcuzAFmDW9LTlEJX+0wGkM/Hkyi2Gxh4nUXbw0Bzk6KNyf1IKvAxOs/HgFgY0Qy4acy+NvoDni5V2M7Wmttk1ufPn30xQ4fPnzJMVE5aWlpulWrVlV2Pof93ZSYdML2lbooJkzrEpNxLC9dm/7bSce/1E6Pen21jk3P01prHZ+Rr5dsj9HD39qku720QUcmZVv9Nr8dSdbD39qkWz+zTt+/cKdOyiyojp9Gm0rM+utdsfpMblG5j7/7c5Ru/cw6vSUqRZeYLVprrc1mi561JFwHP7tOb4o8XS11XU1xiVnf/OHv+rpXf9apOYVaa63/PJaqB/37N936mXV69LtbdKGp5Jrfx2y26P9tOqb3xWZc8lhEQpYOfnadfn71gXJfe8/nO3Sf137RBcUlevLcbXrUO1u0xWK57Hv9d0Okbv3MunO/+xve2aJNJeZr/hmAcH2ZXJXFuRxQYmIiI0aMYM6cOcyZM+fqL7CCTX83GTHg1ezKKwxeTs5pY0JP2+sv2U3GkpVE/OfTaJWzF4AiJ09Mgf1wK8mBpP1MV6/zyqx76ND0wqF18Rn5TJobikLx3exBNPe5/MXPqOQc/v3TEbZEpdKmSX3+75Yu5XZ31JSU7ELGvP87mfkmfDxdGd6hCe4uTqwMj+f5mzrzl2GV60aqCkdP53DLR38yrH0TAn08WLz9FG386jOlb0ve/CmSR0e24+kbO17Te3z2+0neWH8Eb3cXlj80gG6BxtK6FotmyqfbOZmWx6anhpc7pDD0RBp3fRbGX0e0Ze6WE/z9xo48MvIy+45iXFi96cM/iD9TQLHZwsIZfc9dtL4WsjiXjT3yyCNs27btgmOPP/54lew2lJ6ezqhRoy45vn379ktG4NRKEavhmxnGruktQozRI8HDjKVWr3RBtyjX2Odx24fGGHHlDH1mGJs+ePlTfHwrhcun41eSz9fN/06urodHQih9Tx2hnUrkVf0gz/5l2iVhDtCikSeLZvZjyrztzFi4k28eHnTJ7Mf4jHze++UY3+2Nx8vdhedv6sz0QUG4udi2l9O/gQdb/z6SP46lsikyha1RqaTnFTPxukAeHFrxyWtVqUNTb54e04F/rY8EYObgIP5xYyfquTlzPCWXuVtPcGPXZnRv0bBS5z+cmM1/N0YxtL0fJ1Jymb5gJytnDaRtEy++25tA+KkM3prc47Ljwwe28aVni4bM3XICgAm9ylnsrAwPV2fenNiDKZ9uZ3A7X0Z0vPZRQldjdy30Tp06ycgLO6O1JjIysuZb6BmnYN5QY+OGoCEQ86exM462QP9ZMPbNS0PdYjH2edz8L8hLIa3VOJaoWxhRtIWep1ejXNwxtRuLy5HVRFuaEt7/A6bcNAalFOm5Razem8DWw3HMvqEbA9te+QMx9Hga0xfupHfLRkzt15K8ohLyis3EnslnVXg8KJgxKIi/Dm9b7vhte2CxaI6n5tLGr/4lF/dswWzRzN1ynD6tG1/w75+Vb2LM+1tp5OnG2keHVPiDsdBk5taP/iSrwMSGJ4aRkV/MlHnbcXdx4osZfbnn8zBa+3qyatagK8703HAoiVlf7aFfcGNWPjzQqvfecTKdDk29aVxF/w3UmvXQo6Oj8fb2xtfXV0LdTmitSU9PJycnp1LLD1Sa2WSs550aBbP+OD9dvjDLCOuweTD8WRhZZgatqcDYZSfiOzL9ruNfJfewMrkZDTxcyC0qoRVJPOO6knFOYay3DEDf+hE39+1Q7ttba+3+RB5fsfeC/YpdnBQTrwvkiRs6XLE7RlTMb0dO88DicB4b1Z6/ja7Y7+3ltREsCo3hy/v7MayD0VI+lJDFtPk7KDCZsWjND3OG0LX5lVv/ZovmH6sOMKFX83PnqWm1psulRYsWxMfHk5qaautSRBkeHh5XHE1TLTb/C+J3weQFF6594tHQaJkX5cLWN40ZmAMfgdwU9PK7UAm7WOg5g1fiRxPo48nrt7XljpAWFJdY2H0qg53RQ/g1KY47hvVmQBWMrR7fszn9gxtTUGzG090ZL3cXPFycbbqeh6Ma1bkpE3sH8snm44zu3NTqrpfNUSksCo3h/sHBF4Rwt8CGfDGjL9MX7OTu/q2uGuZgjGB5Z0rPSv8M1c2uWuiijjEVws75RhdJk07QcZyxEmFKBHx5m7Hd2oT/lf9acwmsmglH1mIZ/iyFu77EKT+Nx4tnE9VoBLNHtuP23oG42kE3gqg6mfnFjH3/DzSa72YPJvAqfwElZBYw4X9/4uflzppHBpe7xkpuUQn13ZxrTa9ArelyEY5vZ/QZ5m85xjTPMEYmfIpTdjy0HGAsaJWTCChw8QCflvDQlsvuxF5itrAv5jS+a6cTnBVGivbhFa8XGTN6HDd3D7CL/mBRPSKTs7lj3naaNvBg1ayBl72ImVNo4o5520nIKOC72YMuWDahNqs1XS7CfmXlm4hIzCIiMZv4jHwev6FDhS7ymC2a/206zrLfwljo/jZdiCZCBxPW7gOG3TiZdk3qQ9J+OLrB6GoZ/doFYa615kRqHmHR6Ww/kc4fx9LIKjDh7TSLZ/y60XzwPXzUt7d0ddQBnZo1YP69IUxfsJMHF4fz1YP9L2l5l5gtPLpsL8dSclk0s6/DhPnVSAtdXNHxlBxmfbWH4ym5Fxx/eFgbnrvJulEvSVkFPLFiH3uiU9jY6C2CS6JJGPJvPjzdgzX7kyk2W2jjV5/+bRrTL7gxPVv4kJFfTHRaPjFpeRxLySE8JoP00hXsmni7M6x9E67v5M/QDn408LD9+h+i5q07kMic5XsZ06Upn9zdB+fSD3OtNS9+f4ivdsTy74ndmdavije6tjHpchGVUmgyc9vH20jJKeLBocF0a96QXkRx9Id3eTp7Ct8/O+mqiylFJGZxz+dhFJVY+CH4O9qe+homL4RuEwFIzSni+30JbD+Rzs6YM+QUllzwemcnRctG9biudSP6BzemX7AvQb6etaa/U1SvBX9G8+q6w3i7u9CleQO6BTbEbNEsCo2pUKOjNpEuF1Epb/x4hMjkHGOGW7uGxsiT0A8J0RZuKGnGVztCrjhTLu5MPjMW7qKeqzMbh8fiv/lrY//K0jAHo7X94NA2PDi0DWaLJjI5m4iEbPy83QjyrU/Lxp5yYVNc1v1Dgglo6MG2E2kcSsjmqx2nKCqxMK5bM54Z28nW5dU4CXRRrg2Hkliy4xR/GRrMSJ/TMP82Y/RJ73vhVCjj86O4f1s0DwwJLnfkwJm8YqYv2EmRycy3kzzxX/McBA83dpi/DGcnRdfmDa0aPibEWeO6BzCuewBg9J0nZRXSolG9OvlXnDR9xCXiM/L5x6oD9GjRkGdaHIb5I42d6e9aaQwjbD+arqZD5OTmsmp3/CWvLyg288DiXcRnFrBoWkda/fKwsRbLZOt2yRGislycnWjZuO52yUmgiwsUl1h4YsU+LBoW9DqKy5q/QIu+MHsHdLjReFKbkTibC5nSNJH5v5+kxHx+M4ZCk5k5y/eyLy6TD6f24rqMnyA7wZggVN8B1pYRwo5JoItzcgpN3L9oF+GnMlja8yB+vz5pdJPcswo8y2x8ETQYnFyY2Sya2DP5/HQoGa01Px1MYtQ7W/n1yGleHd+VsV2bwe5FEBgCLfva7OcSoq6Qv38FYAwtnLlwF8dTcvm+9256HngHOoyDOxZdumytuze06Edw9k7aNLmFD387xvKdsYSeSKdTM2+W/2WAsbBS7A5IjYTxl5ntKYSoUtJCFxxJyub2j0OJz8jnt5Ad9DzyDnS5De5ccvk1yNuORCUd4LEBjTmWksvhpGxeu60b6+YMOb9K3u7F4OZ9wagWIUT1kRZ6HXc8JZc75m3Hy82ZLb224Ld/LvS8C8Z/dOULmG2vh81vML7BMZynDWRI6d6J5xRkGmuZ95p22en7QoiqJS30Ou71Hw/jhIVfu/xohHnIAzDh46uPRmneGzwa4nRyM7f2bH7pet8Hv4GSAmNTCSFEjZAWeh22OSqFrVGn+Sl4FV7718DAR2HM61feCegsJ2dj56ATm0HrC1+jNYQvhIBeEGC/S40K4WikhV5HmcwW3vjxCH9rsJlOSWtg2D+sD/Oz2oyE7HhIP37h8YTdxiQkaZ0LUaOkhe5gotPy2HEynaTMAlJyikjJKaLQZOaRke0Y3O78hg7LwmIpTI3mr/WXQ/sxMPKfFQtzgLYjja8nNl+4AfPuReBaH7pPvvYfSAhhNQn0WiarwMR7vxzFw9UZf293/Bu44+HiTOiJdDZHpRCdlgcY2exb340m3h5kF5i454swZo9oyxM3dCCvqIT3folicYMvcbYouPndioc5QOM2xm5CJzdD/4eMrpbYHXDoWyPM3evGkqVC2AsJ9Frmy9AYFoXG4OqsMJnPr5Tp5uLEwDa+TB/YmuEd/WnZqN65TR7yi0t4eW0EH28+wfYT6bRq7MnI4i301Lth3FvGZhKV1WakcQF052cQvgBSDoOHDwyYfY0/qRCioiTQaxGT2cJXYacY2t6PxTP7kVlgIiWnkJzCEro2b4CnWzm/Tq3xzE/irdu7MridH8+vPkRMbCx/1F8KAX2h74PXVlTbkbB7Iax/2rgAOv4j6DZJhioKYQMS6LXIhkPJnM4u4l+3d8fJSdG4vtuVdw2K2QabXoPY7eDRkAnBwxly/RBS9m/EMzPfCF+nS1dKrJAO44yLqa0GQeB1leu6EUJUCQn0WmRxaAytfT0Z2dH/8k8qKYKkA7DlX3Bik7HK4fUvQMYpOLEJ3yNr8QUY/gz4V8Hi/y5uMGjOtZ9HCHHNrAp0pdRY4APAGfhca/1mOc8ZAbwPuAJpWuvhVVal4FBCFuGnMnjh5s4X7pt55iT88ARknoK8dCjOMY7Xa2zsy9n3QXDzNI5pDWnH4PRB6Dy+xn8GIUT1umqgK6WcgY+B0UA8sEsptVZrfbjMc3yAT4CxWutYpdQVmpCiMhaFxuDp5swdIWUuYJYUwcrpRpi3HwOefsYStd4BRmB7NLjwJEpBkw7GTQjhcKxpofcDjmutTwIopVYAE4DDZZ5zF/Cd1joWQGudUtWF1mXpuUWs3Z/IlJAWF+7h+fMLkHwApq2AjuNsV6AQwi5YM1M0EIgrcz++9FhZHYBGSqktSqndSqn7yjuRUuohpVS4Uio8NTW1chXXQSt2xVFcYmH6wKDzBw9/Dzvnw4BHJMyFEIB1gV7esAV90X0XoA9wM3Aj8KJS6pK/67XW87XWIVrrkCZNmlS42LrIZLawZPsphrTzo33T0ok6Z6Lh+zkQ2AdueNmm9Qkh7Ic1gR4PlJ150gJILOc5G7TWeVrrNOB3QFZlukZpuUU8uDic5OxCZg4OMg5mxsKq+43vJy8wRpkIIQTW9aHvAtorpYKBBGAqRp95Wd8D/1NKuQBuQH/gvaostK7581gaT67cZ0z1H+PD9ekrYP4aSNwDygmmfGlMuxdCiFJXDXStdYlS6lFgI8awxQVa6wil1KzSx+dprY8opTYABwALxtDGQ9VZuKMymS28+8tR5m09QbsmXqwZmUbgL3eBthhrkN/wMnSZYKyjIoQQZSitL+4OrxkhISE6PDzcJu9tr1JyCnl06V52xpxhWr9WvDS4Hh4LRoJfB6N7pVFrW5cohLAxpdRurXVIeY/JTNEadvR0DjMW7GREJ3/uHxxEO3/jQueumDPMXrqH3MIS3r+zF7d194MvxhhjxyXMhRBWkECvYd/vSyA5u5BVu+NZFhbL8A5N6BbYgE+3nqRFo3oseaAfnZo1gJ+egaR9cOdSCXMhhFUk0GvYpshUQoIaM/fu61gWFsuXO06x9Wgqo7s05Z0pPWng4QpH1kHYPOg/CzrfYuuShRC1hAR6DUrKKuBIUjbPjuuEr5c7c0a15+HhbYlOy6O9v5exRkvcLvh+trEf5+hXbV2yEKIWkT1Fa9CmSGNFhOs7nV/qxs3FiY7NvHGymOC312DBGHBvAHcsAhd3G1UqhKiNpIVegzZHphDoU4/2/l4XPnA6AlY/DMkHodfdMPbf4NHQNkUKIWotCfQaUmgys+14OneEtECV3QTi1Hb4crwR4FOXQ6ebbFekEKJWk0CvITtOplNgMjOyTHcLpkJY+yh4N4O/bIb6frYrUAhR60mg15DNkSl4uBobOZ/z+38h/Tjc852EuRDimslF0RqgtWZTVAqD2/rh4Vq6h2fyIdj2PvS8C9qNsml9QgjHIIFeA06k5hJ3puB8d4vFDGvngIcP3PiGTWsTQjgO6XKpAZsjEgB9PtDDPjVWTZz0BXg2tmltQgjHIYFe3WK2cf/WW5ju4YzbAn/w9DU2am5/I3SbZOvqhBAORAK9mpm2vk2W9uJE8/H0b6ohL80I9VveNRbeEkKIKiKBXp1OH8Y1ehMLS6Zw/djXoXUjW1ckhHBgclG0GplDP6IQNw41n8R1rXxsXY4QwsFJoFeXnGQ4sJKvS4bz4Jg+F84OFUKIaiCBXk1KdnyK0mZ2Np3KkHYyaUgIUf2kD706FOdh3vk5v5hDmDp2uLTOhRA1Qlro1cC0ewnupmz+8JPWuRCi5kgLvapZzBT+/hEHLO0Zd9MEaZ0LIWqMtNCrkrkE87qn8C6IZ1OjKdI6F0LUKGmhV5XCLIqW34f7qS3MK7mVQbfMlNa5EKJGSaBXhYwY8hdNxi3rJC9YHqbv5McZ3L6JrasSQtQxEujXyJIRS9EnIzAVF/N8vZeZNWMmHZt527osIUQdJIFeSWaL5qdDScSv/4BZpgzebjmfV+6ZSAMPV1uXJoSooyTQK0hrzZp9Cfxv03FOpObxpVcEeZ6BvPDAFOkzF0LYlIxyqaCNEad58uv9uDo78fG03gx1O0b9dkMlzIUQNict9ApadyAR3/pu/PjYUJzPHIf8NGg90NZlCSGEtNArotBkZnNkCmO6NsXZScGpUOOBVoNsW5gQQiCBXiHbjqeRV2xmbLcA40DsDvD0A7/2ti1MCCGQQK+Qnw4l4+3hwsA2vsaB2FBoNUB2HhJC2AUJdCuZzBZ+PXKaGzo3xc3FCbKTICMGWkt3ixDCPkigW2ln9Bky803c2LWZcSD2bP/5ANsVJYQQZUigW2nDoWTquTozvEPplP5T28G1PjTradvChBCilAS6FSwWzcaIZEZ0bEI9N2fjYOx2aNkXnGXkpxDCPkigW2FvXCYpOUWM7Vba3VKQCacjZLiiEMKuSKBfrDgfdi+GotxzhzYcSsLVWTGyk79xIC4M0DKhSAhhV6wKdKXUWKVUlFLquFLq2Ss8r69SyqyUmlx1JVa/lOxCtNbGne3/gx8eg89GQkokWms2RCQzuJ3f+YW3YreDkysEhtiuaCGEuMhVO4CVUs7Ax8BoIB7YpZRaq7U+XM7z/gNsrI5Cq8vGiGQeXrKbgIYe3NC+IS8cm4ezf3d0ThLq0+GsaPY0cWe68OjIdudfdGo7NO8Fbp42q1sIIS5mTQu9H3Bca31Sa10MrAAmlPO8OcC3QEoV1lftFofG0LSBOz1b+KAPrsK9KJ3p8bcwKOMVdpuCuCfhdZb6f8V4972QfBByUyBxjwxXFELYHWuGaAQCcWXuxwP9yz5BKRUI3A5cD/S93ImUUg8BDwG0atWqorVWuei0PEJPpPP3GzvyyIi2WD55hDxTJ4b2msx9TbxoETABHf5fBod+CN+tv/DFckFUCGFnrAn08ua164vuvw88o7U2X2kZWa31fGA+QEhIyMXnqHHLd8bi4qS4I6QFnNyMU+oR6t82l1m9ynSvjHkVhjwBGdGQGQeZsWAqgHY32KxuIYQojzWBHg+0LHO/BZB40XNCgBWlYe4H3KSUKtFar6mKIqtDUYmZVbvjGd2lKf7eHvD9x+DVFLpNuvTJno2NW2Cfmi9UCCGsZE2g7wLaK6WCgQRgKnBX2SdorYPPfq+UWgSss+cwB2Pm55m8Yu7q3wpSjsDxX+H6F8DF3dalCSFEpVz1oqjWugR4FGP0yhFgpdY6Qik1Syk1q7oLrC7LwmJp1diTwW39YMcn4FIP+txv67KEEKLSrJq3rrVeD6y/6Ni8yzx3xrWXVb2Op+QQFn2GZ8Z2wik/DfZ/Db3vhvq+ti5NCCEqrU7OFF0WFoerc+nF0Kj1YC6Cvg/auiwhhLgmdS7QC01mvt0Tz5iuzfDzcjem8Xv6gn8XW5cmhBDXpM4F+vqDSWQVmLi7X+k4+Ngd0FJ2HRJC1H51LtCXhcUS7FefgW19jVmfZ05Aq/5Xf6EQQti5OhXoR0/nEH4qg2n9WqKUKl01EaOFLoQQtVydCvRlYbG4OTsxuU/pPKnYHeDsbiy0JYQQtVydCfSCYuNi6LjuzWhc3804GBcGzXvLZCIhhEOoM4G+7kAiOYUl3HX2YqipABL3Sf+5EMJh1JlAX7Yzlnb+XvQLbmwcSNgDFhO0kl2HhBCOoU4E+pGkbPbGZjKtXyvOrQYZt8P42lJa6EIIx1AnAn1ZWCxuLk5Mui7w/MHYMPDrYKyiKIQQDsDhAz2/uIQ1exO4pXsAPp6lF0MtFuOCqLTOhRAOxOEDfWtUKjlFJdzZt8yS7mlHoTBTtpETQjgUhw/0w0nZODsperXyOX8wdrvxVS6ICiEciMMHemRyDsF+9XF3cT5/MC4MPP2gcRvbFSaEEFXM4QM9KjmHjs28LzwYu8PobpEFuYQQDsShAz2vqITYM/l0bFom0HNTjA2f5YKoEMLBOHSgHz2dA3BhC/3EZuOr9J8LIRyMQwd6VLIR6J3KBvqexdAoCAL72KYoIYSoJg4d6JHJOXi6OdOykadxICUSTm2DPjPByaF/dCFEHeTQqRaVnEP7pt44OZVe/Ny9EJxcofc9ti1MCCGqgcMGutaaqNM5dDp7QbQ4H/Ythy4ToL6fbYsTQohq4LCBnpZbzJm84vMXRCNWQ1EWhNxv28KEEKKaOGygX3JBNHwB+HWE1oNsWJUQQlQfhw30yORsoHTIYtJ+SAg3WucymUgI4aAcNtCjknPw83LH18sdwheCiwf0vNPWZQkhRLVx3EA/nWN0txTlwMFvoNskqNfI1mUJIUS1cchAN1s0R0+XruFydCMU58J199m6LCGEqFYOGeixZ/IpNFmMQE/YDS71IDDE1mUJIUS1cshAjzp7QbSpN8SHQ/Ne4Oxi26KEEKKaOWSgRybnoBR08PMwRrjIui1CiDrAIQM9KjmH1o09qZcRCeYiCLzO1iUJIUS1c9hAP9d/DtJCF0LUCQ4X6IUmMzHpeXRs1gAS9oCnL/i0tnVZQghR7Rwu0I+dzsWiS6f8J+w2WucyO1QIUQc4XKBHp+cB0N4HSI2U7hYhRJ3hcIGemlMEQEBeJKAl0IUQdYZDBrqrs6J+2n7jQHMZ4SKEqBscLtDTcovw83JHJew29g6t72vrkoQQokZYFehKqbFKqSil1HGl1LPlPH63UupA6S1UKdWz6ku1TmpOEU283Y0RLtLdIoSoQ64a6EopZ+BjYBzQBZimlOpy0dOigeFa6x7Aa8D8qi7UWmm5RbT1yIXseAl0IUSdYk0LvR9wXGt9UmtdDKwAJpR9gtY6VGudUXp3B9Ciasu0XmpOET2dThp3JNCFEHWINYEeCMSVuR9feuxyHgB+Ku8BpdRDSqlwpVR4amqq9VVayWLRpOcV09F8FJQzNOtR5e8hhBD2yppAL29Wji73iUqNxAj0Z8p7XGs9X2sdorUOadKkifVVWikjvxizRdO64DA07QJunlX+HkIIYa+sCfR4oGWZ+y2AxIufpJTqAXwOTNBap1dNeRWTlluMwkKTnMPS3SKEqHOsCfRdQHulVLBSyg2YCqwt+wSlVCvgO+BerfXRqi/TOqk5RQSrZFxNORLoQog656q7PmitS5RSjwIbAWdggdY6Qik1q/TxecD/Ab7AJ8pYN6VEa13jWwSl5RbRVcUYd5r3rum3F0IIm7JqGx+t9Xpg/UXH5pX5/kHgwaotreLOttAB8G1n22KEEKKGOdRM0bTcIto6J6MbtADXerYuRwghapRDBXpqbhHtnE+jfNvYuhQhhKhxjhXoOUW0Igkat7V1KUIIUeMcKtCLstPw1rngK4EuhKh7HCrQPXNPGd9IC10IUQc5TKCbLZrGhbHGHWmhCyHqIIcJ9DN5xbRWyVhwMtZBF0KIOsZhAj0t1xiDXujZHFzcbV2OEELUOIcJ9NScIoJUMiafYFuXIoQQNuEwgZ6WU0iQSkZJ/7kQoo5ymEDPPZNEA1WAe9MOti5FCCFswmECXaefAMDNX9ZwEULUTQ4T6K6Z0QAoWZRLCFFHOUyge+WdwowT+LSydSlCCGETDhPojQrjSHMNAGdXW5cihBA24TCB7m9KIMNDWudCiLrLIQK9pMRMC51EvldrW5cihBA24xCBnpkSR31VhMlH1kEXQtRdDhHoOYlRADjJpCIhRB3mEIFenHIMAI9m7W1ciRBC2I5DBDrpxynSLjRsKl0uQoi6yyEC3TUrhjjtj19D2RhaCFF3OUSge+WeIlY1x9PNxdalCCGEzdT+QLdY8CmKJ8U10NaVCCGETdX+QM9OwE0Xk1VPJhUJIeq22h/oZ4xVFmVSkRCirqv9gR6zDTOKAt8utq5ECCFsqtYHuo5cR7ilI/V9/G1dihBC2FTtDvQzJ1Eph/nZHEITb9kYWghRt9XuQI9cD8DPlj74ebnZuBghhLCtWh7oP5LTsCNxuqm00IUQdV7tDfTcVIjbwU73gdRzdaadv5etKxJCCJuqvYF+dANoC58kd+LWngF4e8hORUKIuq32BnrUenI9Athd3JK7+ssYdCGEqJ2BXpyHPrGJ33QIXQIa0rNFQ1tXJIQQNlc7A/3EJlRJIStyenBX/1YopWxdkRBC2FztDPTIH8l39uaQS1cm9Gpu62qEEMIu1L5AN5dgifqJn029uaVXS7kYKoQQpawKdKXUWKVUlFLquFLq2XIeV0qpD0sfP6CUuq7qSy0VG4pTYSY/lfThrn5yMVQIIc66aqArpZyBj4FxQBdgmlLq4pWwxgHtS28PAXOruM5ztKsnv7sM5kzAELrLxVAhhDjHmhZ6P+C41vqk1roYWAFMuOg5E4AvtWEH4KOUCqjiWgHYY27DfbmPMKl/h+o4vRBC1FrWBHogEFfmfnzpsYo+p8oM69CEW3vKxVAhhCjLmk04yxsTqCvxHJRSD2F0ydCqVeV2GOrTujFf3t+vUq8VQghHZk0LPR5oWeZ+CyCxEs9Baz1fax2itQ5p0qRJRWsVQghxBdYE+i6gvVIqWCnlBkwF1l70nLXAfaWjXQYAWVrrpCquVQghxBVctctFa12ilHoU2Ag4Awu01hFKqVmlj88D1gM3AceBfGBm9ZUshBCiPNb0oaO1Xo8R2mWPzSvzvQYeqdrShBBCVETtmykqhBCiXBLoQgjhICTQhRDCQUigCyGEg1DG9UwbvLFSqcCpSr7cD0irwnKqmr3XB/Zfo9R3baS+a2PP9bXWWpc7kcdmgX4tlFLhWusQW9dxOfZeH9h/jVLftZH6ro2913c50uUihBAOQgJdCCEcRG0N9Pm2LuAq7L0+sP8apb5rI/VdG3uvr1y1sg9dCCHEpWprC10IIcRFJNCFEMJB1LpAv9qG1TaoZ4FSKkUpdajMscZKqV+UUsdKvzayYX0tlVKblVJHlFIRSqnH7alGpZSHUmqnUmp/aX2v2FN9Zep0VkrtVUqts7f6lFIxSqmDSql9SqlwO6zPRym1SikVWfrf4UB7qU8p1bH03+3sLVsp9YS91FdRtSrQrdywuqYtAsZedOxZ4DetdXvgt9L7tlICPKW17gwMAB4p/TezlxqLgOu11j2BXsDY0jX17aW+sx4HjpS5b2/1jdRa9yozdtqe6vsA2KC17gT0xPh3tIv6tNZRpf9uvYA+GMt/r7aX+ipMa11rbsBAYGOZ+88Bz9lBXUHAoTL3o4CA0u8DgChb11imtu+B0fZYI+AJ7AH621N9GDtw/QZcD6yzt98xEAP4XXTMLuoDGgDRlA7AsLf6LqppDLDNXuuz5larWujU8GbU16CpLt2xqfSrv43rAUApFQT0BsKwoxpLuzP2ASnAL1pru6oPeB/4B2Apc8ye6tPAz0qp3aX79oL91NcGSAUWlnZZfa6Uqm9H9ZU1FVhe+r091ndVtS3QrdqMWlxKKeUFfAs8obXOtnU9ZWmtzdr4k7cF0E8p1c3GJZ2jlLoFSNFa77Z1LVcwWGt9HUZX5CNKqWG2LqgMF+A6YK7WujeQhx12X5Rurzke+MbWtVyL2hboVm1GbQdOK6UCAEq/ptiyGKWUK0aYL9Vaf1d62K5qBNBaZwJbMK5J2Et9g4HxSqkYYAVwvVLqKzuqD611YunXFIz+3352VF88EF/6VxfAKoyAt5f6zhoH7NFany69b2/1WaW2Bbo1G1bbg7XA9NLvp2P0W9uEUkoBXwBHtNbvlnnILmpUSjVRSvmUfl8PuAGItJf6tNbPaa1baK2DMP5726S1vsde6lNK1VdKeZ/9HqMf+JC91Ke1TgbilFIdSw+NAg5jJ/WVMY3z3S1gf/VZx9ad+JW4cHETcBQ4ATxvB/UsB5IAE0Zr5AHAF+Mi2rHSr41tWN8QjG6pA8C+0ttN9lIj0APYW1rfIeD/So/bRX0X1TqC8xdF7aI+jD7q/aW3iLP/T9hLfaW19ALCS3/Ha4BGdlafJ5AONCxzzG7qq8hNpv4LIYSDqG1dLkIIIS5DAl0IIRyEBLoQQjgICXQhhHAQEuhCCOEgJNCFEMJBSKALIYSD+H9mFF3e7RUwBgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_VAL_jac','train_VAL_jac']].plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Direct Prediction on VEN-XL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cpu\")\n",
    "#device = torch.device(\"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#path = osp.join(os.getcwd(), '../../data/VEN')\n",
    "#transform = T.Compose([T.NormalizeFeatures(), T.ToSparseTensor()])\n",
    "dataset_XL = VEN_XL_Homo('dataset/Venice_XL_homo')\n",
    "data_XL = dataset_XL[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Data(num_nodes=80963, x=[80963, 1753], y=[80963, 20], att_lab=[80963], val_lab=[80963], node_type=[80963], train_mask=[80963], val_mask=[80963], test_mask=[80963], edge_index=[2, 290091503], edge_attr=[290091503], n_id=[80963])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_XL.n_id = torch.arange(data_XL.num_nodes)\n",
    "data_XL = data_XL.to(device)\n",
    "data_XL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.loader import NeighborLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data_XL,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data_XL.train_mask,\n",
    ")\n",
    "seed_everything(args.seed)\n",
    "val_loader = NeighborLoader(\n",
    "    data_XL,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data_XL.val_mask,\n",
    ")\n",
    "seed_everything(args.seed)\n",
    "test_loader = NeighborLoader(\n",
    "    data_XL,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=data_XL.test_mask,\n",
    ")\n",
    "unlabel_loader = NeighborLoader(\n",
    "    data_XL,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors=[3*args.sample_nodes] * 2,\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=~(data_XL.train_mask + data_XL.val_mask + data_XL.test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SAGE(in_channels=data_XL.x.shape[-1], hidden_channels = 512, \n",
    "            out_channels = data_XL.y.shape[-1], dropout = 0.1, num_layers=5).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load(args.save_dir+'SAGE_best_model/model.pth',map_location=torch.device('cpu')))\n",
    "model.eval()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def predict_Homo(model, loader):\n",
    "    model.eval()\n",
    "    seed_everything(args.seed)\n",
    "    all_preds = []\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch.batch_size\n",
    "        edge_index = to_undirected(batch.edge_index)\n",
    "        out = model(batch.x, edge_index)[:batch_size]\n",
    "        out_att = out[:,:9].softmax(axis=1)\n",
    "        out_val = out[:,9:].softmax(axis=1)\n",
    "        IDs = batch.n_id[:batch_size].unsqueeze(dim=-1).int()\n",
    "        \n",
    "        now = torch.hstack([IDs, out_att, out_val])\n",
    "        all_preds.append(now)\n",
    "    \n",
    "    final = torch.vstack(all_preds)\n",
    "        \n",
    "    return final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 362/362 [1:22:29<00:00, 13.67s/it]\n"
     ]
    }
   ],
   "source": [
    "pred_train = predict_Homo(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 624/624 [2:19:07<00:00, 13.38s/it]\n"
     ]
    }
   ],
   "source": [
    "pred_val = predict_Homo(model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 624/624 [2:20:47<00:00, 13.54s/it]\n"
     ]
    }
   ],
   "source": [
    "pred_test = predict_Homo(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([4.0000e+00, 1.3500e-02, 9.0181e-01, 5.1935e-03, 2.7402e-03, 2.6608e-03,\n",
       "        5.0749e-02, 1.2765e-02, 2.1375e-03, 8.4420e-03, 4.8005e-02, 1.2251e-01,\n",
       "        1.3997e-01, 4.8272e-01, 5.3826e-02, 1.2716e-01, 4.9087e-03, 3.4084e-03,\n",
       "        3.9171e-03, 4.9782e-03, 8.5965e-03])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = torch.vstack([pred_train, pred_val, pred_test]).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([2.0000, 2.0000, 2.0000,  ..., 2.0000, 2.0000, 2.0000])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds[:,1:].sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df = pd.DataFrame(preds).sort_values(0).set_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.633332</td>\n",
       "      <td>0.057277</td>\n",
       "      <td>0.015404</td>\n",
       "      <td>0.028697</td>\n",
       "      <td>0.013699</td>\n",
       "      <td>0.175408</td>\n",
       "      <td>0.017617</td>\n",
       "      <td>0.012913</td>\n",
       "      <td>0.045652</td>\n",
       "      <td>0.236449</td>\n",
       "      <td>0.075887</td>\n",
       "      <td>0.074907</td>\n",
       "      <td>0.507199</td>\n",
       "      <td>0.027443</td>\n",
       "      <td>0.057173</td>\n",
       "      <td>0.003742</td>\n",
       "      <td>0.003125</td>\n",
       "      <td>0.002563</td>\n",
       "      <td>0.003162</td>\n",
       "      <td>0.008349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.372060</td>\n",
       "      <td>0.024885</td>\n",
       "      <td>0.001671</td>\n",
       "      <td>0.005992</td>\n",
       "      <td>0.004620</td>\n",
       "      <td>0.570282</td>\n",
       "      <td>0.004931</td>\n",
       "      <td>0.002767</td>\n",
       "      <td>0.012791</td>\n",
       "      <td>0.232787</td>\n",
       "      <td>0.090658</td>\n",
       "      <td>0.085741</td>\n",
       "      <td>0.475766</td>\n",
       "      <td>0.018762</td>\n",
       "      <td>0.078831</td>\n",
       "      <td>0.002459</td>\n",
       "      <td>0.002418</td>\n",
       "      <td>0.001957</td>\n",
       "      <td>0.001963</td>\n",
       "      <td>0.008659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.013500</td>\n",
       "      <td>0.901812</td>\n",
       "      <td>0.005193</td>\n",
       "      <td>0.002740</td>\n",
       "      <td>0.002661</td>\n",
       "      <td>0.050749</td>\n",
       "      <td>0.012765</td>\n",
       "      <td>0.002137</td>\n",
       "      <td>0.008442</td>\n",
       "      <td>0.048005</td>\n",
       "      <td>0.122507</td>\n",
       "      <td>0.139968</td>\n",
       "      <td>0.482724</td>\n",
       "      <td>0.053826</td>\n",
       "      <td>0.127160</td>\n",
       "      <td>0.004909</td>\n",
       "      <td>0.003408</td>\n",
       "      <td>0.003917</td>\n",
       "      <td>0.004978</td>\n",
       "      <td>0.008597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5.0</th>\n",
       "      <td>0.012320</td>\n",
       "      <td>0.908419</td>\n",
       "      <td>0.003180</td>\n",
       "      <td>0.002311</td>\n",
       "      <td>0.001448</td>\n",
       "      <td>0.058789</td>\n",
       "      <td>0.006633</td>\n",
       "      <td>0.001524</td>\n",
       "      <td>0.005376</td>\n",
       "      <td>0.047455</td>\n",
       "      <td>0.109753</td>\n",
       "      <td>0.180022</td>\n",
       "      <td>0.424235</td>\n",
       "      <td>0.048640</td>\n",
       "      <td>0.162746</td>\n",
       "      <td>0.005463</td>\n",
       "      <td>0.003685</td>\n",
       "      <td>0.003980</td>\n",
       "      <td>0.005649</td>\n",
       "      <td>0.008372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7.0</th>\n",
       "      <td>0.019720</td>\n",
       "      <td>0.111638</td>\n",
       "      <td>0.009613</td>\n",
       "      <td>0.015328</td>\n",
       "      <td>0.641959</td>\n",
       "      <td>0.029642</td>\n",
       "      <td>0.132666</td>\n",
       "      <td>0.009407</td>\n",
       "      <td>0.030028</td>\n",
       "      <td>0.054278</td>\n",
       "      <td>0.080581</td>\n",
       "      <td>0.196898</td>\n",
       "      <td>0.108560</td>\n",
       "      <td>0.013309</td>\n",
       "      <td>0.513272</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.006606</td>\n",
       "      <td>0.004321</td>\n",
       "      <td>0.005031</td>\n",
       "      <td>0.008895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80956.0</th>\n",
       "      <td>0.118134</td>\n",
       "      <td>0.037216</td>\n",
       "      <td>0.006811</td>\n",
       "      <td>0.007781</td>\n",
       "      <td>0.208601</td>\n",
       "      <td>0.431651</td>\n",
       "      <td>0.030975</td>\n",
       "      <td>0.005117</td>\n",
       "      <td>0.153713</td>\n",
       "      <td>0.115312</td>\n",
       "      <td>0.172132</td>\n",
       "      <td>0.138203</td>\n",
       "      <td>0.352614</td>\n",
       "      <td>0.018661</td>\n",
       "      <td>0.180005</td>\n",
       "      <td>0.003568</td>\n",
       "      <td>0.004234</td>\n",
       "      <td>0.002963</td>\n",
       "      <td>0.002565</td>\n",
       "      <td>0.009743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80958.0</th>\n",
       "      <td>0.054562</td>\n",
       "      <td>0.223858</td>\n",
       "      <td>0.010701</td>\n",
       "      <td>0.006221</td>\n",
       "      <td>0.016337</td>\n",
       "      <td>0.557420</td>\n",
       "      <td>0.021155</td>\n",
       "      <td>0.003952</td>\n",
       "      <td>0.105794</td>\n",
       "      <td>0.120240</td>\n",
       "      <td>0.311770</td>\n",
       "      <td>0.104964</td>\n",
       "      <td>0.358159</td>\n",
       "      <td>0.015200</td>\n",
       "      <td>0.072098</td>\n",
       "      <td>0.002357</td>\n",
       "      <td>0.002840</td>\n",
       "      <td>0.002320</td>\n",
       "      <td>0.001984</td>\n",
       "      <td>0.008068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80959.0</th>\n",
       "      <td>0.943707</td>\n",
       "      <td>0.010062</td>\n",
       "      <td>0.001336</td>\n",
       "      <td>0.018816</td>\n",
       "      <td>0.004341</td>\n",
       "      <td>0.010590</td>\n",
       "      <td>0.005909</td>\n",
       "      <td>0.004065</td>\n",
       "      <td>0.001174</td>\n",
       "      <td>0.255703</td>\n",
       "      <td>0.303520</td>\n",
       "      <td>0.162176</td>\n",
       "      <td>0.174101</td>\n",
       "      <td>0.004954</td>\n",
       "      <td>0.086379</td>\n",
       "      <td>0.001446</td>\n",
       "      <td>0.002177</td>\n",
       "      <td>0.001417</td>\n",
       "      <td>0.001391</td>\n",
       "      <td>0.006736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80960.0</th>\n",
       "      <td>0.888731</td>\n",
       "      <td>0.018761</td>\n",
       "      <td>0.002371</td>\n",
       "      <td>0.031335</td>\n",
       "      <td>0.005960</td>\n",
       "      <td>0.035472</td>\n",
       "      <td>0.008148</td>\n",
       "      <td>0.006448</td>\n",
       "      <td>0.002774</td>\n",
       "      <td>0.339152</td>\n",
       "      <td>0.181970</td>\n",
       "      <td>0.155669</td>\n",
       "      <td>0.128935</td>\n",
       "      <td>0.004691</td>\n",
       "      <td>0.171414</td>\n",
       "      <td>0.002452</td>\n",
       "      <td>0.002963</td>\n",
       "      <td>0.001756</td>\n",
       "      <td>0.001853</td>\n",
       "      <td>0.009145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80962.0</th>\n",
       "      <td>0.753498</td>\n",
       "      <td>0.040989</td>\n",
       "      <td>0.003485</td>\n",
       "      <td>0.080890</td>\n",
       "      <td>0.008331</td>\n",
       "      <td>0.087939</td>\n",
       "      <td>0.011595</td>\n",
       "      <td>0.007848</td>\n",
       "      <td>0.005426</td>\n",
       "      <td>0.194627</td>\n",
       "      <td>0.394414</td>\n",
       "      <td>0.137414</td>\n",
       "      <td>0.185982</td>\n",
       "      <td>0.004836</td>\n",
       "      <td>0.069143</td>\n",
       "      <td>0.001371</td>\n",
       "      <td>0.002292</td>\n",
       "      <td>0.001451</td>\n",
       "      <td>0.001514</td>\n",
       "      <td>0.006956</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>51491 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               1         2         3         4         5         6         7   \\\n",
       "0                                                                               \n",
       "0.0      0.633332  0.057277  0.015404  0.028697  0.013699  0.175408  0.017617   \n",
       "1.0      0.372060  0.024885  0.001671  0.005992  0.004620  0.570282  0.004931   \n",
       "4.0      0.013500  0.901812  0.005193  0.002740  0.002661  0.050749  0.012765   \n",
       "5.0      0.012320  0.908419  0.003180  0.002311  0.001448  0.058789  0.006633   \n",
       "7.0      0.019720  0.111638  0.009613  0.015328  0.641959  0.029642  0.132666   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80956.0  0.118134  0.037216  0.006811  0.007781  0.208601  0.431651  0.030975   \n",
       "80958.0  0.054562  0.223858  0.010701  0.006221  0.016337  0.557420  0.021155   \n",
       "80959.0  0.943707  0.010062  0.001336  0.018816  0.004341  0.010590  0.005909   \n",
       "80960.0  0.888731  0.018761  0.002371  0.031335  0.005960  0.035472  0.008148   \n",
       "80962.0  0.753498  0.040989  0.003485  0.080890  0.008331  0.087939  0.011595   \n",
       "\n",
       "               8         9         10        11        12        13        14  \\\n",
       "0                                                                               \n",
       "0.0      0.012913  0.045652  0.236449  0.075887  0.074907  0.507199  0.027443   \n",
       "1.0      0.002767  0.012791  0.232787  0.090658  0.085741  0.475766  0.018762   \n",
       "4.0      0.002137  0.008442  0.048005  0.122507  0.139968  0.482724  0.053826   \n",
       "5.0      0.001524  0.005376  0.047455  0.109753  0.180022  0.424235  0.048640   \n",
       "7.0      0.009407  0.030028  0.054278  0.080581  0.196898  0.108560  0.013309   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80956.0  0.005117  0.153713  0.115312  0.172132  0.138203  0.352614  0.018661   \n",
       "80958.0  0.003952  0.105794  0.120240  0.311770  0.104964  0.358159  0.015200   \n",
       "80959.0  0.004065  0.001174  0.255703  0.303520  0.162176  0.174101  0.004954   \n",
       "80960.0  0.006448  0.002774  0.339152  0.181970  0.155669  0.128935  0.004691   \n",
       "80962.0  0.007848  0.005426  0.194627  0.394414  0.137414  0.185982  0.004836   \n",
       "\n",
       "               15        16        17        18        19        20  \n",
       "0                                                                    \n",
       "0.0      0.057173  0.003742  0.003125  0.002563  0.003162  0.008349  \n",
       "1.0      0.078831  0.002459  0.002418  0.001957  0.001963  0.008659  \n",
       "4.0      0.127160  0.004909  0.003408  0.003917  0.004978  0.008597  \n",
       "5.0      0.162746  0.005463  0.003685  0.003980  0.005649  0.008372  \n",
       "7.0      0.513272  0.008250  0.006606  0.004321  0.005031  0.008895  \n",
       "...           ...       ...       ...       ...       ...       ...  \n",
       "80956.0  0.180005  0.003568  0.004234  0.002963  0.002565  0.009743  \n",
       "80958.0  0.072098  0.002357  0.002840  0.002320  0.001984  0.008068  \n",
       "80959.0  0.086379  0.001446  0.002177  0.001417  0.001391  0.006736  \n",
       "80960.0  0.171414  0.002452  0.002963  0.001756  0.001853  0.009145  \n",
       "80962.0  0.069143  0.001371  0.002292  0.001451  0.001514  0.006956  \n",
       "\n",
       "[51491 rows x 20 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df.to_csv(args.save_dir + 'preds_XL_trans.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.633332</td>\n",
       "      <td>0.057277</td>\n",
       "      <td>0.015404</td>\n",
       "      <td>0.028697</td>\n",
       "      <td>0.013699</td>\n",
       "      <td>0.175408</td>\n",
       "      <td>0.017617</td>\n",
       "      <td>0.012913</td>\n",
       "      <td>0.045652</td>\n",
       "      <td>0.236449</td>\n",
       "      <td>0.075887</td>\n",
       "      <td>0.074907</td>\n",
       "      <td>0.507199</td>\n",
       "      <td>0.027443</td>\n",
       "      <td>0.057173</td>\n",
       "      <td>0.003742</td>\n",
       "      <td>0.003125</td>\n",
       "      <td>0.002563</td>\n",
       "      <td>0.003162</td>\n",
       "      <td>0.008349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.372060</td>\n",
       "      <td>0.024885</td>\n",
       "      <td>0.001671</td>\n",
       "      <td>0.005992</td>\n",
       "      <td>0.004620</td>\n",
       "      <td>0.570282</td>\n",
       "      <td>0.004931</td>\n",
       "      <td>0.002767</td>\n",
       "      <td>0.012791</td>\n",
       "      <td>0.232787</td>\n",
       "      <td>0.090658</td>\n",
       "      <td>0.085741</td>\n",
       "      <td>0.475766</td>\n",
       "      <td>0.018762</td>\n",
       "      <td>0.078831</td>\n",
       "      <td>0.002459</td>\n",
       "      <td>0.002418</td>\n",
       "      <td>0.001957</td>\n",
       "      <td>0.001963</td>\n",
       "      <td>0.008659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.017046</td>\n",
       "      <td>0.247266</td>\n",
       "      <td>0.016916</td>\n",
       "      <td>0.012148</td>\n",
       "      <td>0.426693</td>\n",
       "      <td>0.032296</td>\n",
       "      <td>0.175406</td>\n",
       "      <td>0.012946</td>\n",
       "      <td>0.059284</td>\n",
       "      <td>0.078895</td>\n",
       "      <td>0.068871</td>\n",
       "      <td>0.141617</td>\n",
       "      <td>0.120624</td>\n",
       "      <td>0.017452</td>\n",
       "      <td>0.527566</td>\n",
       "      <td>0.014055</td>\n",
       "      <td>0.007664</td>\n",
       "      <td>0.005529</td>\n",
       "      <td>0.008120</td>\n",
       "      <td>0.009607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.041138</td>\n",
       "      <td>0.434027</td>\n",
       "      <td>0.025498</td>\n",
       "      <td>0.013622</td>\n",
       "      <td>0.087983</td>\n",
       "      <td>0.120305</td>\n",
       "      <td>0.214133</td>\n",
       "      <td>0.017339</td>\n",
       "      <td>0.045954</td>\n",
       "      <td>0.047354</td>\n",
       "      <td>0.158037</td>\n",
       "      <td>0.138289</td>\n",
       "      <td>0.267976</td>\n",
       "      <td>0.026557</td>\n",
       "      <td>0.328015</td>\n",
       "      <td>0.007392</td>\n",
       "      <td>0.005137</td>\n",
       "      <td>0.004794</td>\n",
       "      <td>0.006497</td>\n",
       "      <td>0.009951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.013500</td>\n",
       "      <td>0.901812</td>\n",
       "      <td>0.005193</td>\n",
       "      <td>0.002740</td>\n",
       "      <td>0.002661</td>\n",
       "      <td>0.050749</td>\n",
       "      <td>0.012765</td>\n",
       "      <td>0.002137</td>\n",
       "      <td>0.008442</td>\n",
       "      <td>0.048005</td>\n",
       "      <td>0.122507</td>\n",
       "      <td>0.139968</td>\n",
       "      <td>0.482724</td>\n",
       "      <td>0.053826</td>\n",
       "      <td>0.127160</td>\n",
       "      <td>0.004909</td>\n",
       "      <td>0.003408</td>\n",
       "      <td>0.003917</td>\n",
       "      <td>0.004978</td>\n",
       "      <td>0.008597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80958.0</th>\n",
       "      <td>0.054562</td>\n",
       "      <td>0.223858</td>\n",
       "      <td>0.010701</td>\n",
       "      <td>0.006221</td>\n",
       "      <td>0.016337</td>\n",
       "      <td>0.557420</td>\n",
       "      <td>0.021155</td>\n",
       "      <td>0.003952</td>\n",
       "      <td>0.105794</td>\n",
       "      <td>0.120240</td>\n",
       "      <td>0.311770</td>\n",
       "      <td>0.104964</td>\n",
       "      <td>0.358159</td>\n",
       "      <td>0.015200</td>\n",
       "      <td>0.072098</td>\n",
       "      <td>0.002357</td>\n",
       "      <td>0.002840</td>\n",
       "      <td>0.002320</td>\n",
       "      <td>0.001984</td>\n",
       "      <td>0.008068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80959.0</th>\n",
       "      <td>0.943707</td>\n",
       "      <td>0.010062</td>\n",
       "      <td>0.001336</td>\n",
       "      <td>0.018816</td>\n",
       "      <td>0.004341</td>\n",
       "      <td>0.010590</td>\n",
       "      <td>0.005909</td>\n",
       "      <td>0.004065</td>\n",
       "      <td>0.001174</td>\n",
       "      <td>0.255703</td>\n",
       "      <td>0.303520</td>\n",
       "      <td>0.162176</td>\n",
       "      <td>0.174101</td>\n",
       "      <td>0.004954</td>\n",
       "      <td>0.086379</td>\n",
       "      <td>0.001446</td>\n",
       "      <td>0.002177</td>\n",
       "      <td>0.001417</td>\n",
       "      <td>0.001391</td>\n",
       "      <td>0.006736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80960.0</th>\n",
       "      <td>0.888731</td>\n",
       "      <td>0.018761</td>\n",
       "      <td>0.002371</td>\n",
       "      <td>0.031335</td>\n",
       "      <td>0.005960</td>\n",
       "      <td>0.035472</td>\n",
       "      <td>0.008148</td>\n",
       "      <td>0.006448</td>\n",
       "      <td>0.002774</td>\n",
       "      <td>0.339152</td>\n",
       "      <td>0.181970</td>\n",
       "      <td>0.155669</td>\n",
       "      <td>0.128935</td>\n",
       "      <td>0.004691</td>\n",
       "      <td>0.171414</td>\n",
       "      <td>0.002452</td>\n",
       "      <td>0.002963</td>\n",
       "      <td>0.001756</td>\n",
       "      <td>0.001853</td>\n",
       "      <td>0.009145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80961.0</th>\n",
       "      <td>0.603690</td>\n",
       "      <td>0.165463</td>\n",
       "      <td>0.013494</td>\n",
       "      <td>0.053913</td>\n",
       "      <td>0.010511</td>\n",
       "      <td>0.035217</td>\n",
       "      <td>0.089893</td>\n",
       "      <td>0.022010</td>\n",
       "      <td>0.005809</td>\n",
       "      <td>0.177895</td>\n",
       "      <td>0.374887</td>\n",
       "      <td>0.149664</td>\n",
       "      <td>0.191111</td>\n",
       "      <td>0.008633</td>\n",
       "      <td>0.080931</td>\n",
       "      <td>0.001965</td>\n",
       "      <td>0.002751</td>\n",
       "      <td>0.002150</td>\n",
       "      <td>0.001940</td>\n",
       "      <td>0.008073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80962.0</th>\n",
       "      <td>0.753498</td>\n",
       "      <td>0.040989</td>\n",
       "      <td>0.003485</td>\n",
       "      <td>0.080890</td>\n",
       "      <td>0.008331</td>\n",
       "      <td>0.087939</td>\n",
       "      <td>0.011595</td>\n",
       "      <td>0.007848</td>\n",
       "      <td>0.005426</td>\n",
       "      <td>0.194627</td>\n",
       "      <td>0.394414</td>\n",
       "      <td>0.137414</td>\n",
       "      <td>0.185982</td>\n",
       "      <td>0.004836</td>\n",
       "      <td>0.069143</td>\n",
       "      <td>0.001371</td>\n",
       "      <td>0.002292</td>\n",
       "      <td>0.001451</td>\n",
       "      <td>0.001514</td>\n",
       "      <td>0.006956</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80963 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                1         2         3         4         5         6         7  \\\n",
       "0                                                                               \n",
       "0.0      0.633332  0.057277  0.015404  0.028697  0.013699  0.175408  0.017617   \n",
       "1.0      0.372060  0.024885  0.001671  0.005992  0.004620  0.570282  0.004931   \n",
       "2.0      0.017046  0.247266  0.016916  0.012148  0.426693  0.032296  0.175406   \n",
       "3.0      0.041138  0.434027  0.025498  0.013622  0.087983  0.120305  0.214133   \n",
       "4.0      0.013500  0.901812  0.005193  0.002740  0.002661  0.050749  0.012765   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80958.0  0.054562  0.223858  0.010701  0.006221  0.016337  0.557420  0.021155   \n",
       "80959.0  0.943707  0.010062  0.001336  0.018816  0.004341  0.010590  0.005909   \n",
       "80960.0  0.888731  0.018761  0.002371  0.031335  0.005960  0.035472  0.008148   \n",
       "80961.0  0.603690  0.165463  0.013494  0.053913  0.010511  0.035217  0.089893   \n",
       "80962.0  0.753498  0.040989  0.003485  0.080890  0.008331  0.087939  0.011595   \n",
       "\n",
       "                8         9        10        11        12        13        14  \\\n",
       "0                                                                               \n",
       "0.0      0.012913  0.045652  0.236449  0.075887  0.074907  0.507199  0.027443   \n",
       "1.0      0.002767  0.012791  0.232787  0.090658  0.085741  0.475766  0.018762   \n",
       "2.0      0.012946  0.059284  0.078895  0.068871  0.141617  0.120624  0.017452   \n",
       "3.0      0.017339  0.045954  0.047354  0.158037  0.138289  0.267976  0.026557   \n",
       "4.0      0.002137  0.008442  0.048005  0.122507  0.139968  0.482724  0.053826   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80958.0  0.003952  0.105794  0.120240  0.311770  0.104964  0.358159  0.015200   \n",
       "80959.0  0.004065  0.001174  0.255703  0.303520  0.162176  0.174101  0.004954   \n",
       "80960.0  0.006448  0.002774  0.339152  0.181970  0.155669  0.128935  0.004691   \n",
       "80961.0  0.022010  0.005809  0.177895  0.374887  0.149664  0.191111  0.008633   \n",
       "80962.0  0.007848  0.005426  0.194627  0.394414  0.137414  0.185982  0.004836   \n",
       "\n",
       "               15        16        17        18        19        20  \n",
       "0                                                                    \n",
       "0.0      0.057173  0.003742  0.003125  0.002563  0.003162  0.008349  \n",
       "1.0      0.078831  0.002459  0.002418  0.001957  0.001963  0.008659  \n",
       "2.0      0.527566  0.014055  0.007664  0.005529  0.008120  0.009607  \n",
       "3.0      0.328015  0.007392  0.005137  0.004794  0.006497  0.009951  \n",
       "4.0      0.127160  0.004909  0.003408  0.003917  0.004978  0.008597  \n",
       "...           ...       ...       ...       ...       ...       ...  \n",
       "80958.0  0.072098  0.002357  0.002840  0.002320  0.001984  0.008068  \n",
       "80959.0  0.086379  0.001446  0.002177  0.001417  0.001391  0.006736  \n",
       "80960.0  0.171414  0.002452  0.002963  0.001756  0.001853  0.009145  \n",
       "80961.0  0.080931  0.001965  0.002751  0.002150  0.001940  0.008073  \n",
       "80962.0  0.069143  0.001371  0.002292  0.001451  0.001514  0.006956  \n",
       "\n",
       "[80963 rows x 20 columns]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = pd.read_csv(args.save_dir + 'preds_XL_trans.csv', sep='\\t', index_col='0')\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = torch.tensor(np.array(preds)).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df = pd.DataFrame(('val_ATT_loss', 'val_VAL_loss', 'val_ATT_acc', 'val_VAL_acc', 'val_VAL_acc_k', 'val_VAL_jac_k'), columns=['name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([51491])"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(data_XL.train_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ATT_loss = F.cross_entropy(data_XL.y[data_XL.train_mask][:,:9], \n",
    "                pred[data_XL.train_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,:9]).cpu().detach().item()\n",
    "train_VAL_loss = F.cross_entropy(data_XL.y[data_XL.train_mask][:,9:], \n",
    "                pred[data_XL.train_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:]).cpu().detach().item()\n",
    "\n",
    "train_ATT_acc = compute_1_accuracy(data_XL.y[data_XL.train_mask][:,:9], \n",
    "                pred[data_XL.train_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,:9])\n",
    "train_VAL_acc = compute_1_accuracy(data_XL.y[data_XL.train_mask][:,9:], \n",
    "                pred[data_XL.train_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:])\n",
    "train_VAL_acc_k = compute_k_accuracy(data_XL.y[data_XL.train_mask][:,9:].cpu(),  \n",
    "                pred[data_XL.train_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:].cpu(),3)\n",
    "train_VAL_jac_k = compute_jaccard_index(data_XL.y[data_XL.train_mask][:,9:].cpu(),  \n",
    "                pred[data_XL.train_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:].cpu(),3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ATT_loss = F.cross_entropy(data_XL.y[data_XL.train_mask][:,:9], \n",
    "                pred[data_XL.train_mask][:,:9]).cpu().detach().item()\n",
    "train_VAL_loss = F.cross_entropy(data_XL.y[data_XL.train_mask][:,9:], \n",
    "                pred[data_XL.train_mask][:,9:]).cpu().detach().item()\n",
    "\n",
    "train_ATT_acc = compute_1_accuracy(data_XL.y[data_XL.train_mask][:,:9], \n",
    "                pred[data_XL.train_mask][:,:9])\n",
    "train_VAL_acc = compute_1_accuracy(data_XL.y[data_XL.train_mask][:,9:], \n",
    "                pred[data_XL.train_mask][:,9:])\n",
    "train_VAL_acc_k = compute_k_accuracy(data_XL.y[data_XL.train_mask][:,9:].cpu(),  \n",
    "                pred[data_XL.train_mask][:,9:].cpu(),3)\n",
    "train_VAL_jac_k = compute_jaccard_index(data_XL.y[data_XL.train_mask][:,9:].cpu(),  \n",
    "                pred[data_XL.train_mask][:,9:].cpu(),3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df['train'] = pd.DataFrame((train_ATT_loss, train_VAL_loss, train_ATT_acc, train_VAL_acc, train_VAL_acc_k, train_VAL_jac_k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_ATT_loss = F.cross_entropy(data_XL.y[(data_XL.att_lab) * data_XL.val_mask][:,:9], \n",
    "                pred[(data_XL.att_lab[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]) * data_XL.val_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,:9]).cpu().detach().item()\n",
    "val_VAL_loss = F.cross_entropy(data_XL.y[(data_XL.val_lab) * data_XL.val_mask][:,9:], \n",
    "                pred[(data_XL.val_lab[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]) * data_XL.val_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:]).cpu().detach().item()\n",
    "\n",
    "val_ATT_acc = compute_1_accuracy(data_XL.y[(data_XL.att_lab) * data_XL.val_mask][:,:9], \n",
    "                pred[(data_XL.att_lab[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]) * data_XL.val_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,:9])\n",
    "val_VAL_acc = compute_1_accuracy(data_XL.y[(data_XL.val_lab) * data_XL.val_mask][:,9:], \n",
    "                pred[(data_XL.val_lab[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]) * data_XL.val_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:])\n",
    "val_VAL_acc_k = compute_k_accuracy(data_XL.y[(data_XL.val_lab) * data_XL.val_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL.val_lab[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]) * data_XL.val_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:].cpu(),3)\n",
    "val_VAL_jac_k = compute_jaccard_index(data_XL.y[(data_XL.val_lab) * data_XL.val_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL.val_lab[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]) * data_XL.val_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:].cpu(),3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_ATT_loss = F.cross_entropy(data_XL.y[(data_XL.att_lab) * data_XL.val_mask][:,:9], \n",
    "                pred[(data_XL.att_lab) * data_XL.val_mask][:,:9]).cpu().detach().item()\n",
    "val_VAL_loss = F.cross_entropy(data_XL.y[(data_XL.val_lab) * data_XL.val_mask][:,9:], \n",
    "                pred[(data_XL.val_lab) * data_XL.val_mask][:,9:]).cpu().detach().item()\n",
    "\n",
    "val_ATT_acc = compute_1_accuracy(data_XL.y[(data_XL.att_lab) * data_XL.val_mask][:,:9], \n",
    "                pred[(data_XL.att_lab) * data_XL.val_mask][:,:9])\n",
    "val_VAL_acc = compute_1_accuracy(data_XL.y[(data_XL.val_lab) * data_XL.val_mask][:,9:], \n",
    "                pred[(data_XL.val_lab) * data_XL.val_mask][:,9:])\n",
    "val_VAL_acc_k = compute_k_accuracy(data_XL.y[(data_XL.val_lab) * data_XL.val_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL.val_lab) * data_XL.val_mask][:,9:].cpu(),3)\n",
    "val_VAL_jac_k = compute_jaccard_index(data_XL.y[(data_XL.val_lab) * data_XL.val_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL.val_lab) * data_XL.val_mask][:,9:].cpu(),3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df['val'] = pd.DataFrame((val_ATT_loss, val_VAL_loss, val_ATT_acc, val_VAL_acc, val_VAL_acc_k, val_VAL_jac_k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ATT_loss = F.cross_entropy(data_XL.y[(data_XL.att_lab) * data_XL.test_mask][:,:9], \n",
    "                pred[(data_XL.att_lab[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]) * data_XL.test_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,:9]).cpu().detach().item()\n",
    "test_VAL_loss = F.cross_entropy(data_XL.y[(data_XL.val_lab) * data_XL.test_mask][:,9:], \n",
    "                pred[(data_XL.val_lab[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]) * data_XL.test_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:]).cpu().detach().item()\n",
    "\n",
    "test_ATT_acc = compute_1_accuracy(data_XL.y[(data_XL.att_lab) * data_XL.test_mask][:,:9], \n",
    "                pred[(data_XL.att_lab[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]) * data_XL.test_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,:9])\n",
    "test_VAL_acc = compute_1_accuracy(data_XL.y[(data_XL.val_lab) * data_XL.test_mask][:,9:], \n",
    "                pred[(data_XL.val_lab[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]) * data_XL.test_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:])\n",
    "test_VAL_acc_k = compute_k_accuracy(data_XL.y[(data_XL.val_lab) * data_XL.test_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL.val_lab[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]) * data_XL.test_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:].cpu(),3)\n",
    "test_VAL_jac_k = compute_jaccard_index(data_XL.y[(data_XL.val_lab) * data_XL.test_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL.val_lab[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]) * data_XL.test_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:].cpu(),3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ATT_loss = F.cross_entropy(data_XL.y[(data_XL.att_lab) * data_XL.test_mask][:,:9], \n",
    "                pred[(data_XL.att_lab) * data_XL.test_mask][:,:9]).cpu().detach().item()\n",
    "test_VAL_loss = F.cross_entropy(data_XL.y[(data_XL.val_lab) * data_XL.test_mask][:,9:], \n",
    "                pred[(data_XL.val_lab) * data_XL.test_mask][:,9:]).cpu().detach().item()\n",
    "\n",
    "test_ATT_acc = compute_1_accuracy(data_XL.y[(data_XL.att_lab) * data_XL.test_mask][:,:9], \n",
    "                pred[(data_XL.att_lab) * data_XL.test_mask][:,:9])\n",
    "test_VAL_acc = compute_1_accuracy(data_XL.y[(data_XL.val_lab) * data_XL.test_mask][:,9:], \n",
    "                pred[(data_XL.val_lab) * data_XL.test_mask][:,9:])\n",
    "test_VAL_acc_k = compute_k_accuracy(data_XL.y[(data_XL.val_lab) * data_XL.test_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL.val_lab) * data_XL.test_mask][:,9:].cpu(),3)\n",
    "test_VAL_jac_k = compute_jaccard_index(data_XL.y[(data_XL.val_lab) * data_XL.test_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL.val_lab) * data_XL.test_mask][:,9:].cpu(),3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df['test'] = pd.DataFrame((test_ATT_loss, test_VAL_loss, test_ATT_acc, test_VAL_acc, test_VAL_acc_k, test_VAL_jac_k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>train</th>\n",
       "      <th>val</th>\n",
       "      <th>test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>val_ATT_loss</td>\n",
       "      <td>1.756584</td>\n",
       "      <td>1.739774</td>\n",
       "      <td>1.740664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>val_VAL_loss</td>\n",
       "      <td>2.245626</td>\n",
       "      <td>2.239503</td>\n",
       "      <td>2.238977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>val_ATT_acc</td>\n",
       "      <td>90.085574</td>\n",
       "      <td>94.692672</td>\n",
       "      <td>94.101626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>val_VAL_acc</td>\n",
       "      <td>75.918403</td>\n",
       "      <td>78.186689</td>\n",
       "      <td>78.213783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>val_VAL_acc_k</td>\n",
       "      <td>98.435474</td>\n",
       "      <td>98.688663</td>\n",
       "      <td>98.368495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>val_VAL_jac_k</td>\n",
       "      <td>0.727310</td>\n",
       "      <td>0.755476</td>\n",
       "      <td>0.752813</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            name      train        val       test\n",
       "0   val_ATT_loss   1.756584   1.739774   1.740664\n",
       "1   val_VAL_loss   2.245626   2.239503   2.238977\n",
       "2    val_ATT_acc  90.085574  94.692672  94.101626\n",
       "3    val_VAL_acc  75.918403  78.186689  78.213783\n",
       "4  val_VAL_acc_k  98.435474  98.688663  98.368495\n",
       "5  val_VAL_jac_k   0.727310   0.755476   0.752813"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df.to_csv(args.save_dir+'eval_metrics_XL_trans.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Per-Class Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_k_confusion_matrix(y, y_pred, k=3):\n",
    "    dim = y.shape[-1]\n",
    "    y = y.topk(k=k, axis=1)[1]\n",
    "    y_pred = y_pred.topk(k=k, axis=1)[1]\n",
    "    conf = np.zeros((dim, dim))\n",
    "    for i in range(k):\n",
    "        for j in range(k):\n",
    "            conf = np.add(conf, confusion_matrix(y[:,i], y_pred[:,j], labels = range(dim)))\n",
    "    return conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ATT_conf = confusion_matrix(data_XL.y[data_XL.test_mask][:,:9].argmax(axis=1).cpu(), pred[data_XL.test_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,:9].argmax(axis=1).cpu())\n",
    "test_VAL_conf = confusion_matrix(data_XL.y[data_XL.test_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                                 pred[data_XL.test_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "test_VAL_conf_k = (top_k_confusion_matrix(data_XL.y[data_XL.test_mask][:,9:].cpu(),  pred[data_XL.test_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ATT_conf = confusion_matrix(data_XL.y[(data_XL.att_lab) * data_XL.test_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                pred[(data_XL.att_lab) * data_XL.test_mask][:,:9].argmax(axis=1).cpu())\n",
    "test_VAL_conf = confusion_matrix(data_XL.y[(data_XL.val_lab) * data_XL.test_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                pred[(data_XL.val_lab) * data_XL.test_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "test_VAL_conf_k = (top_k_confusion_matrix(data_XL.y[(data_XL.val_lab) * data_XL.test_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL.val_lab) * data_XL.test_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1313,   21,    0,    9,    0,   45,    0,    1,    0],\n",
       "       [   1, 3176,    0,    0,    0,   28,    6,    0,    0],\n",
       "       [   0,    0,  247,    0,    0,    0,   16,    2,    0],\n",
       "       [  47,   23,    0,  544,    2,    2,   14,    5,    0],\n",
       "       [   7,   15,    0,    0, 2666,    3,   11,    1,    1],\n",
       "       [   3,   23,    0,    1,    3, 1553,    0,    0,    0],\n",
       "       [   1,   97,    4,    1,   10,    0, 2472,    8,    0],\n",
       "       [  31,    6,   94,   19,    0,    0,  109,   54,    0],\n",
       "       [   0,   16,    0,    0,   24,   48,    0,    0,   68]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 603,    5,   13,   15,    0,   44,    0,    0,    0,    0,    0],\n",
       "       [  63,  486,   55,   83,    0,  162,    0,    0,    0,    0,    0],\n",
       "       [  52,   27,  839,   81,    0,  608,    0,    0,    0,    0,    0],\n",
       "       [  46,   11,   19,  373,    0,   87,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [  39,   10,   75,   34,    0, 3257,    0,    0,    0,    0,    0],\n",
       "       [   3,    0,    4,    0,    0,    4,    0,    0,    0,    1,    0],\n",
       "       [   0,    0,    1,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    0,    0,    0,    1,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    6,    0,    0,    0,    0,    0,    0,    3,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1427,  624, 1139,  862,    0,  901,    0,    0,    0,    0,    0],\n",
       "       [ 967, 1244, 2271, 2058,    1, 1667,    0,    0,    0,    0,    0],\n",
       "       [1058,  847, 5797, 5169,    1, 5212,    0,    0,    0,    0,    0],\n",
       "       [ 944,  935, 4751, 4903,    1, 4270,    0,    0,    0,    0,    0],\n",
       "       [   1,    6,   18,   18,    0,   11,    0,    0,    0,    0,    0],\n",
       "       [ 892,  388, 5301, 4754,    0, 5336,    0,    0,    0,    0,    0],\n",
       "       [  16,    0,   31,    9,    0,   25,    4,    0,    0,   11,    0],\n",
       "       [   3,    0,    6,    2,    0,    5,    2,    0,    0,    3,    0],\n",
       "       [   8,    0,   14,    1,    0,    9,    2,    0,    0,    8,    0],\n",
       "       [   9,    0,   19,    2,    0,   12,    4,    0,    0,   11,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]])"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ True,  True, False,  ...,  True,  True,  True])"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_XL.val_mask[data_XL.val_mask + data_XL.train_mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_ATT_conf = confusion_matrix(data_XL.y[data_XL.val_mask][:,:9].argmax(axis=1).cpu(), pred[data_XL.val_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,:9].argmax(axis=1).cpu())\n",
    "val_VAL_conf = confusion_matrix(data_XL.y[data_XL.val_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                                 pred[data_XL.val_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "val_VAL_conf_k = (top_k_confusion_matrix(data_XL.y[data_XL.val_mask][:,9:].cpu(),  pred[data_XL.val_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_ATT_conf = confusion_matrix(data_XL.y[(data_XL.att_lab) * data_XL.val_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                pred[(data_XL.att_lab) * data_XL.val_mask][:,:9].argmax(axis=1).cpu())\n",
    "val_VAL_conf = confusion_matrix(data_XL.y[(data_XL.val_lab) * data_XL.val_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                pred[(data_XL.val_lab) * data_XL.val_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "val_VAL_conf_k = (top_k_confusion_matrix(data_XL.y[(data_XL.val_lab) * data_XL.val_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL.val_lab) * data_XL.val_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1305,    4,    0,    7,    1,   32,    1,    2,    0],\n",
       "       [   1, 3262,    0,    0,    0,   33,    3,    0,    0],\n",
       "       [   1,    1,  265,    0,    0,    0,   11,    2,    0],\n",
       "       [  56,   25,    2,  565,    0,    3,   21,    8,    0],\n",
       "       [   1,   15,    0,    1, 2654,    3,   10,    3,    0],\n",
       "       [   2,   26,    0,    1,    4, 1455,    0,    0,    0],\n",
       "       [   2,   72,    4,    0,    7,    0, 2533,   12,    0],\n",
       "       [  25,    6,   80,   12,    0,    0,  109,   54,    0],\n",
       "       [   1,   19,    0,    0,   24,   30,    0,    0,   93]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 585,   10,   15,    8,    0,   49,    0,    0,    0,    0,    0],\n",
       "       [  77,  458,   68,   88,    0,  138,    0,    0,    0,    0,    0],\n",
       "       [  53,   23,  893,   73,    0,  602,    0,    0,    0,    0,    0],\n",
       "       [  48,   13,   31,  423,    0,   83,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    1,    3,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [  34,    8,   70,   36,    0, 3185,    0,    0,    0,    0,    0],\n",
       "       [   1,    0,    2,    0,    0,    2,    0,    0,    0,    0,    0],\n",
       "       [   1,    0,    2,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    1,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    6,    0,    0,    1,    0,    0,    0,    1,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1434,  636, 1167,  878,    0,  901,    0,    0,    0,    0,    0],\n",
       "       [1035, 1244, 2388, 2125,    0, 1743,    0,    0,    0,    0,    0],\n",
       "       [1048,  879, 5786, 5115,    0, 5141,    0,    0,    0,    1,    0],\n",
       "       [ 913,  953, 4683, 4840,    0, 4178,    0,    0,    0,    0,    0],\n",
       "       [   0,    8,   22,   22,    0,   14,    0,    0,    0,    0,    0],\n",
       "       [ 858,  393, 5245, 4713,    0, 5236,    0,    0,    0,    1,    0],\n",
       "       [  13,    0,   38,   25,    0,   37,    1,    0,    0,    6,    0],\n",
       "       [   2,    0,    3,    2,    0,    1,    0,    0,    0,    1,    0],\n",
       "       [   8,    0,   16,    5,    0,   11,    1,    0,    0,    7,    0],\n",
       "       [   8,    0,   17,    5,    0,   12,    1,    0,    0,    8,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]])"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ATT_conf = confusion_matrix(data_XL.y[data_XL.train_mask][:,:9].argmax(axis=1).cpu(), pred[data_XL.train_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,:9].argmax(axis=1).cpu())\n",
    "train_VAL_conf = confusion_matrix(data_XL.y[data_XL.train_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                                 pred[data_XL.train_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "train_VAL_conf_k = (top_k_confusion_matrix(data_XL.y[data_XL.train_mask][:,9:].cpu(),  pred[data_XL.train_mask[data_XL.val_mask + data_XL.train_mask + data_XL.test_mask]][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ATT_conf = confusion_matrix(data_XL.y[data_XL.train_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                                  pred[data_XL.train_mask][:,:9].argmax(axis=1).cpu())\n",
    "train_VAL_conf = confusion_matrix(data_XL.y[data_XL.train_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                                 pred[data_XL.train_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "train_VAL_conf_k = (top_k_confusion_matrix(data_XL.y[data_XL.train_mask][:,9:].cpu(),  \n",
    "                                pred[data_XL.train_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1424,   15,    0,    2,    0,   51,    9,    0,    0],\n",
       "       [   0, 2611,    0,    0,    0,   17,    8,    0,    0],\n",
       "       [   1,    2,  109,    0,    0,    0,   26,    1,    0],\n",
       "       [  56,   24,    0,  370,    0,    3,   24,    3,    0],\n",
       "       [   2,   11,    0,    0, 2007,    1,   26,    3,    1],\n",
       "       [   2,   24,    0,    0,    7, 1474,    0,    0,    0],\n",
       "       [   3,   72,    2,    2,   10,    0, 2362,    6,    0],\n",
       "       [  66,   13,   33,   32,    0,    0,  510,   31,    0],\n",
       "       [   0,   17,    0,    0,   33,   29,    0,    0,   34]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 901,   14,   19,   11,    0,   67,    0,    0,    0,    0,    0],\n",
       "       [ 117,  684,  109,  210,    0,  258,    0,    0,    0,    0,    0],\n",
       "       [  79,   37, 1411,  141,    0, 1147,    0,    0,    0,    0,    0],\n",
       "       [  53,   18,   45,  806,    0,  121,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    0,    1,    0,    1,    0,    0,    0,    0,    0],\n",
       "       [  64,   14,  143,   86,    0, 4976,    0,    0,    0,    0,    0],\n",
       "       [   3,    0,    7,    0,    0,   10,    0,    0,    0,    1,    0],\n",
       "       [   1,    0,    1,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    1,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    7,    0,    0,    0,    0,    0,    0,    5,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2143,  857, 1608, 1356,    0, 1425,    0,    0,    0,    0,    0],\n",
       "       [1467, 1984, 3939, 3719,    0, 3003,    0,    0,    0,    0,    0],\n",
       "       [1673, 1438, 9506, 8425,    0, 8550,    0,    0,    0,    0,    0],\n",
       "       [1539, 1548, 7711, 7977,    0, 6959,    0,    0,    0,    0,    0],\n",
       "       [   0,   23,   54,   54,    0,   31,    0,    0,    0,    0,    0],\n",
       "       [1452,  603, 8485, 7652,    0, 8571,    0,    0,    0,    0,    0],\n",
       "       [  33,    0,   57,   23,    0,   44,    4,    0,    0,   13,    0],\n",
       "       [  15,    0,   18,    5,    0,   14,    0,    0,    0,    2,    0],\n",
       "       [  10,    0,   19,    3,    0,   10,    4,    0,    0,   11,    0],\n",
       "       [  17,    0,   28,    6,    0,   16,    4,    0,    0,   13,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]])"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat([pd.DataFrame(train_ATT_conf),pd.DataFrame(val_ATT_conf),pd.DataFrame(test_ATT_conf)],axis=1).to_csv(args.save_dir+'confusion_matrix_ATT_XL_trans.csv')\n",
    "pd.concat([pd.DataFrame(train_VAL_conf),pd.DataFrame(val_VAL_conf),pd.DataFrame(test_VAL_conf)],axis=1).to_csv(args.save_dir+'confusion_matrix_VAL_XL_trans.csv')\n",
    "pd.concat([pd.DataFrame(train_VAL_conf_k),pd.DataFrame(val_VAL_conf_k),pd.DataFrame(test_VAL_conf_k)],axis=1).to_csv(args.save_dir+'confusion_matrix_VAL_k_XL_trans.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics(confusion_matrix, classes):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val/(row.sum()+0.000000001)\n",
    "        recall = val/(col.sum()+0.000000001)\n",
    "        F1 = 2*(precision*recall)/(precision+recall+0.000000001)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics_k(confusion_matrix, classes, k=3):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val*k/(row.sum()+0.000000001)\n",
    "        recall = val*k/(col.sum()+0.000000001)\n",
    "        F1 = 2*(precision*recall)/(precision+recall+0.000000001)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = ['Criterion i', 'Criterion ii', 'Criterion iii', 'Criterion iv', 'Criterion v', 'Criterion vi', \n",
    "              'Criterion vii', 'Criterion viii', 'Criterion ix', 'Criterion x', 'Others']\n",
    "categories = ['Building Elements',\n",
    " 'Urban Form Elements',\n",
    " 'Gastronomy',\n",
    " 'Interior Scenery',\n",
    " 'Natural Features and Land-scape Scenery',\n",
    " 'Monuments and Buildings',\n",
    " 'Peoples Activity and Association',\n",
    " 'Artifact Products',\n",
    " 'Urban Scenery']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_dict = {}\n",
    "metrics_dict['test_ATT'] = per_class_metrics(test_ATT_conf, categories)\n",
    "metrics_dict['val_ATT'] = per_class_metrics(val_ATT_conf, categories)\n",
    "metrics_dict['test_VAL'] = per_class_metrics(test_VAL_conf, classes)\n",
    "metrics_dict['val_VAL'] = per_class_metrics(val_VAL_conf, classes)\n",
    "metrics_dict['test_VAL_k'] = per_class_metrics_k(test_VAL_conf_k, classes)\n",
    "metrics_dict['val_VAL_k'] = per_class_metrics_k(val_VAL_conf_k, classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df = pd.DataFrame.from_dict({(i,j): metrics_dict[i][j] \n",
    "                           for i in metrics_dict.keys() \n",
    "                           for j in metrics_dict[i].keys()},\n",
    "                       orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">test_ATT</th>\n",
       "      <th>Building Elements</th>\n",
       "      <td>0.945284</td>\n",
       "      <td>0.935852</td>\n",
       "      <td>0.940544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Urban Form Elements</th>\n",
       "      <td>0.989100</td>\n",
       "      <td>0.940480</td>\n",
       "      <td>0.964177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gastronomy</th>\n",
       "      <td>0.932075</td>\n",
       "      <td>0.715942</td>\n",
       "      <td>0.809836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Interior Scenery</th>\n",
       "      <td>0.854003</td>\n",
       "      <td>0.947735</td>\n",
       "      <td>0.898431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Natural Features and Land-scape Scenery</th>\n",
       "      <td>0.985947</td>\n",
       "      <td>0.985582</td>\n",
       "      <td>0.985764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">val_VAL_k</th>\n",
       "      <th>Criterion vii</th>\n",
       "      <td>0.025000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.048780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Criterion viii</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Criterion ix</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Criterion x</th>\n",
       "      <td>0.470588</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.640000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Others</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>62 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   precision    recall  \\\n",
       "test_ATT  Building Elements                         0.945284  0.935852   \n",
       "          Urban Form Elements                       0.989100  0.940480   \n",
       "          Gastronomy                                0.932075  0.715942   \n",
       "          Interior Scenery                          0.854003  0.947735   \n",
       "          Natural Features and Land-scape Scenery   0.985947  0.985582   \n",
       "...                                                      ...       ...   \n",
       "val_VAL_k Criterion vii                             0.025000  1.000000   \n",
       "          Criterion viii                            0.000000  0.000000   \n",
       "          Criterion ix                              0.000000  0.000000   \n",
       "          Criterion x                               0.470588  1.000000   \n",
       "          Others                                    0.000000  0.000000   \n",
       "\n",
       "                                                         F1  \n",
       "test_ATT  Building Elements                        0.940544  \n",
       "          Urban Form Elements                      0.964177  \n",
       "          Gastronomy                               0.809836  \n",
       "          Interior Scenery                         0.898431  \n",
       "          Natural Features and Land-scape Scenery  0.985764  \n",
       "...                                                     ...  \n",
       "val_VAL_k Criterion vii                            0.048780  \n",
       "          Criterion viii                           0.000000  \n",
       "          Criterion ix                             0.000000  \n",
       "          Criterion x                              0.640000  \n",
       "          Others                                   0.000000  \n",
       "\n",
       "[62 rows x 3 columns]"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df.to_csv(args.save_dir+'per_class_metrics_XL_trans.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 921/921 [3:14:16<00:00, 12.66s/it]\n"
     ]
    }
   ],
   "source": [
    "pred_unlab = predict_Homo(model, unlabel_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = torch.vstack([pred_train, pred_val, pred_test, pred_unlab]).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df = pd.DataFrame(preds).sort_values(0).set_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.633332</td>\n",
       "      <td>0.057277</td>\n",
       "      <td>0.015404</td>\n",
       "      <td>0.028697</td>\n",
       "      <td>0.013699</td>\n",
       "      <td>0.175408</td>\n",
       "      <td>0.017617</td>\n",
       "      <td>0.012913</td>\n",
       "      <td>0.045652</td>\n",
       "      <td>0.236449</td>\n",
       "      <td>0.075887</td>\n",
       "      <td>0.074907</td>\n",
       "      <td>0.507199</td>\n",
       "      <td>0.027443</td>\n",
       "      <td>0.057173</td>\n",
       "      <td>0.003742</td>\n",
       "      <td>0.003125</td>\n",
       "      <td>0.002563</td>\n",
       "      <td>0.003162</td>\n",
       "      <td>0.008349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.372060</td>\n",
       "      <td>0.024885</td>\n",
       "      <td>0.001671</td>\n",
       "      <td>0.005992</td>\n",
       "      <td>0.004620</td>\n",
       "      <td>0.570282</td>\n",
       "      <td>0.004931</td>\n",
       "      <td>0.002767</td>\n",
       "      <td>0.012791</td>\n",
       "      <td>0.232787</td>\n",
       "      <td>0.090658</td>\n",
       "      <td>0.085741</td>\n",
       "      <td>0.475766</td>\n",
       "      <td>0.018762</td>\n",
       "      <td>0.078831</td>\n",
       "      <td>0.002459</td>\n",
       "      <td>0.002418</td>\n",
       "      <td>0.001957</td>\n",
       "      <td>0.001963</td>\n",
       "      <td>0.008659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.017046</td>\n",
       "      <td>0.247266</td>\n",
       "      <td>0.016916</td>\n",
       "      <td>0.012148</td>\n",
       "      <td>0.426693</td>\n",
       "      <td>0.032296</td>\n",
       "      <td>0.175406</td>\n",
       "      <td>0.012946</td>\n",
       "      <td>0.059284</td>\n",
       "      <td>0.078895</td>\n",
       "      <td>0.068871</td>\n",
       "      <td>0.141617</td>\n",
       "      <td>0.120624</td>\n",
       "      <td>0.017452</td>\n",
       "      <td>0.527566</td>\n",
       "      <td>0.014055</td>\n",
       "      <td>0.007664</td>\n",
       "      <td>0.005529</td>\n",
       "      <td>0.008120</td>\n",
       "      <td>0.009607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.041138</td>\n",
       "      <td>0.434027</td>\n",
       "      <td>0.025498</td>\n",
       "      <td>0.013622</td>\n",
       "      <td>0.087983</td>\n",
       "      <td>0.120305</td>\n",
       "      <td>0.214133</td>\n",
       "      <td>0.017339</td>\n",
       "      <td>0.045954</td>\n",
       "      <td>0.047354</td>\n",
       "      <td>0.158037</td>\n",
       "      <td>0.138289</td>\n",
       "      <td>0.267976</td>\n",
       "      <td>0.026557</td>\n",
       "      <td>0.328015</td>\n",
       "      <td>0.007392</td>\n",
       "      <td>0.005137</td>\n",
       "      <td>0.004794</td>\n",
       "      <td>0.006497</td>\n",
       "      <td>0.009951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.013500</td>\n",
       "      <td>0.901812</td>\n",
       "      <td>0.005193</td>\n",
       "      <td>0.002740</td>\n",
       "      <td>0.002661</td>\n",
       "      <td>0.050749</td>\n",
       "      <td>0.012765</td>\n",
       "      <td>0.002137</td>\n",
       "      <td>0.008442</td>\n",
       "      <td>0.048005</td>\n",
       "      <td>0.122507</td>\n",
       "      <td>0.139968</td>\n",
       "      <td>0.482724</td>\n",
       "      <td>0.053826</td>\n",
       "      <td>0.127160</td>\n",
       "      <td>0.004909</td>\n",
       "      <td>0.003408</td>\n",
       "      <td>0.003917</td>\n",
       "      <td>0.004978</td>\n",
       "      <td>0.008597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80958.0</th>\n",
       "      <td>0.054562</td>\n",
       "      <td>0.223858</td>\n",
       "      <td>0.010701</td>\n",
       "      <td>0.006221</td>\n",
       "      <td>0.016337</td>\n",
       "      <td>0.557420</td>\n",
       "      <td>0.021155</td>\n",
       "      <td>0.003952</td>\n",
       "      <td>0.105794</td>\n",
       "      <td>0.120240</td>\n",
       "      <td>0.311770</td>\n",
       "      <td>0.104964</td>\n",
       "      <td>0.358159</td>\n",
       "      <td>0.015200</td>\n",
       "      <td>0.072098</td>\n",
       "      <td>0.002357</td>\n",
       "      <td>0.002840</td>\n",
       "      <td>0.002320</td>\n",
       "      <td>0.001984</td>\n",
       "      <td>0.008068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80959.0</th>\n",
       "      <td>0.943707</td>\n",
       "      <td>0.010062</td>\n",
       "      <td>0.001336</td>\n",
       "      <td>0.018816</td>\n",
       "      <td>0.004341</td>\n",
       "      <td>0.010590</td>\n",
       "      <td>0.005909</td>\n",
       "      <td>0.004065</td>\n",
       "      <td>0.001174</td>\n",
       "      <td>0.255703</td>\n",
       "      <td>0.303520</td>\n",
       "      <td>0.162176</td>\n",
       "      <td>0.174101</td>\n",
       "      <td>0.004954</td>\n",
       "      <td>0.086379</td>\n",
       "      <td>0.001446</td>\n",
       "      <td>0.002177</td>\n",
       "      <td>0.001417</td>\n",
       "      <td>0.001391</td>\n",
       "      <td>0.006736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80960.0</th>\n",
       "      <td>0.888731</td>\n",
       "      <td>0.018761</td>\n",
       "      <td>0.002371</td>\n",
       "      <td>0.031335</td>\n",
       "      <td>0.005960</td>\n",
       "      <td>0.035472</td>\n",
       "      <td>0.008148</td>\n",
       "      <td>0.006448</td>\n",
       "      <td>0.002774</td>\n",
       "      <td>0.339152</td>\n",
       "      <td>0.181970</td>\n",
       "      <td>0.155669</td>\n",
       "      <td>0.128935</td>\n",
       "      <td>0.004691</td>\n",
       "      <td>0.171414</td>\n",
       "      <td>0.002452</td>\n",
       "      <td>0.002963</td>\n",
       "      <td>0.001756</td>\n",
       "      <td>0.001853</td>\n",
       "      <td>0.009145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80961.0</th>\n",
       "      <td>0.603690</td>\n",
       "      <td>0.165463</td>\n",
       "      <td>0.013494</td>\n",
       "      <td>0.053913</td>\n",
       "      <td>0.010511</td>\n",
       "      <td>0.035217</td>\n",
       "      <td>0.089893</td>\n",
       "      <td>0.022010</td>\n",
       "      <td>0.005809</td>\n",
       "      <td>0.177895</td>\n",
       "      <td>0.374887</td>\n",
       "      <td>0.149664</td>\n",
       "      <td>0.191111</td>\n",
       "      <td>0.008633</td>\n",
       "      <td>0.080931</td>\n",
       "      <td>0.001965</td>\n",
       "      <td>0.002751</td>\n",
       "      <td>0.002150</td>\n",
       "      <td>0.001940</td>\n",
       "      <td>0.008073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80962.0</th>\n",
       "      <td>0.753498</td>\n",
       "      <td>0.040989</td>\n",
       "      <td>0.003485</td>\n",
       "      <td>0.080890</td>\n",
       "      <td>0.008331</td>\n",
       "      <td>0.087939</td>\n",
       "      <td>0.011595</td>\n",
       "      <td>0.007848</td>\n",
       "      <td>0.005426</td>\n",
       "      <td>0.194627</td>\n",
       "      <td>0.394414</td>\n",
       "      <td>0.137414</td>\n",
       "      <td>0.185982</td>\n",
       "      <td>0.004836</td>\n",
       "      <td>0.069143</td>\n",
       "      <td>0.001371</td>\n",
       "      <td>0.002292</td>\n",
       "      <td>0.001451</td>\n",
       "      <td>0.001514</td>\n",
       "      <td>0.006956</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80963 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               1         2         3         4         5         6         7   \\\n",
       "0                                                                               \n",
       "0.0      0.633332  0.057277  0.015404  0.028697  0.013699  0.175408  0.017617   \n",
       "1.0      0.372060  0.024885  0.001671  0.005992  0.004620  0.570282  0.004931   \n",
       "2.0      0.017046  0.247266  0.016916  0.012148  0.426693  0.032296  0.175406   \n",
       "3.0      0.041138  0.434027  0.025498  0.013622  0.087983  0.120305  0.214133   \n",
       "4.0      0.013500  0.901812  0.005193  0.002740  0.002661  0.050749  0.012765   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80958.0  0.054562  0.223858  0.010701  0.006221  0.016337  0.557420  0.021155   \n",
       "80959.0  0.943707  0.010062  0.001336  0.018816  0.004341  0.010590  0.005909   \n",
       "80960.0  0.888731  0.018761  0.002371  0.031335  0.005960  0.035472  0.008148   \n",
       "80961.0  0.603690  0.165463  0.013494  0.053913  0.010511  0.035217  0.089893   \n",
       "80962.0  0.753498  0.040989  0.003485  0.080890  0.008331  0.087939  0.011595   \n",
       "\n",
       "               8         9         10        11        12        13        14  \\\n",
       "0                                                                               \n",
       "0.0      0.012913  0.045652  0.236449  0.075887  0.074907  0.507199  0.027443   \n",
       "1.0      0.002767  0.012791  0.232787  0.090658  0.085741  0.475766  0.018762   \n",
       "2.0      0.012946  0.059284  0.078895  0.068871  0.141617  0.120624  0.017452   \n",
       "3.0      0.017339  0.045954  0.047354  0.158037  0.138289  0.267976  0.026557   \n",
       "4.0      0.002137  0.008442  0.048005  0.122507  0.139968  0.482724  0.053826   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80958.0  0.003952  0.105794  0.120240  0.311770  0.104964  0.358159  0.015200   \n",
       "80959.0  0.004065  0.001174  0.255703  0.303520  0.162176  0.174101  0.004954   \n",
       "80960.0  0.006448  0.002774  0.339152  0.181970  0.155669  0.128935  0.004691   \n",
       "80961.0  0.022010  0.005809  0.177895  0.374887  0.149664  0.191111  0.008633   \n",
       "80962.0  0.007848  0.005426  0.194627  0.394414  0.137414  0.185982  0.004836   \n",
       "\n",
       "               15        16        17        18        19        20  \n",
       "0                                                                    \n",
       "0.0      0.057173  0.003742  0.003125  0.002563  0.003162  0.008349  \n",
       "1.0      0.078831  0.002459  0.002418  0.001957  0.001963  0.008659  \n",
       "2.0      0.527566  0.014055  0.007664  0.005529  0.008120  0.009607  \n",
       "3.0      0.328015  0.007392  0.005137  0.004794  0.006497  0.009951  \n",
       "4.0      0.127160  0.004909  0.003408  0.003917  0.004978  0.008597  \n",
       "...           ...       ...       ...       ...       ...       ...  \n",
       "80958.0  0.072098  0.002357  0.002840  0.002320  0.001984  0.008068  \n",
       "80959.0  0.086379  0.001446  0.002177  0.001417  0.001391  0.006736  \n",
       "80960.0  0.171414  0.002452  0.002963  0.001756  0.001853  0.009145  \n",
       "80961.0  0.080931  0.001965  0.002751  0.002150  0.001940  0.008073  \n",
       "80962.0  0.069143  0.001371  0.002292  0.001451  0.001514  0.006956  \n",
       "\n",
       "[80963 rows x 20 columns]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df.to_csv(args.save_dir + 'preds_XL_trans.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}