HGT.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HGT Models to Compute Heritage Values and Attributes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import os.path as osp\n",
    "from itertools import product\n",
    "from typing import Callable, List, Optional\n",
    "\n",
    "import numpy as np\n",
    "import scipy.sparse as sp\n",
    "\n",
    "from torch_geometric.data import (\n",
    "    HeteroData,\n",
    "    InMemoryDataset,\n",
    "    download_url,\n",
    "    extract_zip,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from argparse import Namespace\n",
    "from collections import Counter\n",
    "import json\n",
    "import os\n",
    "import re\n",
    "import string\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import random\n",
    "\n",
    "import torch\n",
    "torch.manual_seed(1337)\n",
    "torch.backends.cudnn.benchmark = True\n",
    "\n",
    "from torch_geometric.transforms import RandomLinkSplit, ToUndirected\n",
    "import torch_geometric.transforms as T\n",
    "from torch_geometric.nn import GATConv, Linear, to_hetero\n",
    "from torch_geometric.utils import to_undirected\n",
    "\n",
    "import torch.nn.functional as F\n",
    "from torch_geometric import seed_everything\n",
    "\n",
    "from tqdm import tqdm\n",
    "import torch.optim as optim\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.nn import  HGTConv, Linear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'D:\\\\surf\\\\TUD\\\\Paper\\\\Venice_Graph'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch version 1.10.2\n",
      "GPU-enabled installation? True\n"
     ]
    }
   ],
   "source": [
    "print(\"PyTorch version {}\".format(torch.__version__))\n",
    "print(\"GPU-enabled installation? {}\".format(torch.cuda.is_available()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "#device = torch.device(\"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed_everywhere(seed, cuda):\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    random.seed(seed)\n",
    "    if cuda:\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "\n",
    "def handle_dirs(dirpath):\n",
    "    if not os.path.exists(dirpath):\n",
    "        os.makedirs(dirpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = Namespace(\n",
    "    # Data and Path information\n",
    "    path = 'dataset/Venice',\n",
    "    save_dir='model_storage/HGT/',\n",
    "    model_state_file='model.pth',\n",
    "    \n",
    "    # Model hyper parameters\n",
    "    hidden_channels = 128,\n",
    "    num_layers = 3,\n",
    "    num_heads = 2,\n",
    "    group = 'sum',\n",
    "    k=3,\n",
    "    \n",
    "    # Training hyper parameters\n",
    "    sample_nodes = 25,\n",
    "    batch_size=32,\n",
    "    early_stopping_criteria=100,\n",
    "    learning_rate=0.001,\n",
    "    l2=2e-4,\n",
    "    dropout_p=0.5,\n",
    "    num_epochs=300,\n",
    "    seed=42,\n",
    "    \n",
    "    # Runtime options\n",
    "    catch_keyboard_interrupt=True,\n",
    "    cuda=True,\n",
    "    expand_filepaths_to_save_dir=True,\n",
    "    reload_from_files=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expanded filepaths: \n",
      "\tmodel_storage/HGT/model.pth\n"
     ]
    }
   ],
   "source": [
    "if args.expand_filepaths_to_save_dir:\n",
    "    args.model_state_file = os.path.join(args.save_dir,\n",
    "                                         args.model_state_file)\n",
    "    \n",
    "    print(\"Expanded filepaths: \")\n",
    "    print(\"\\t{}\".format(args.model_state_file))\n",
    "\n",
    "# handle dirs\n",
    "handle_dirs(args.save_dir)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Constructing Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VEN(InMemoryDataset):\n",
    "    r\"\"\"A subset of Flickr post collected in Venice annotated with Heritage \n",
    "    Values and Attributes, as collected in the `\"Heri-Graphs: A Workflow of \n",
    "    Creating Datasets for Multi-modal Machine Learning on Graphs of Heritage \n",
    "    Values and Attributes with Social Media\" <https://arxiv.org/abs/2205.07545>`\n",
    "    paper.\n",
    "    VEN is a heterogeneous graph containing two types of nodes - nodes with only \n",
    "    visual features 'vis_only' (1,190 nodes), nodes with both visual and textual\n",
    "    features 'vis_tex' (1,761 nodes) and four types of links - social similarity\n",
    "    'SOC' (488,103 links), spatial similarity (445,779 links), temporal similarity\n",
    "    (501,191 links), and simple composed link (1,071,977 links).\n",
    "    Vis_only nodes are represented with 982-dimensional visual features and are\n",
    "    divided into 9 heritage attribute categories \n",
    "    ('architectural elements', 'form', 'gastronomy', 'interior',\n",
    "    'landscape scenery and natural features', 'monuments', 'people', 'product', \n",
    "    'urban scenery').\n",
    "    Vis_text nodes are represented with 1753-dimensional visual and textual \n",
    "    features and are divided into 9 heritage attribute categories plus 11 \n",
    "    heritage value categories ('criterion i-x', 'other').\n",
    "    Both types of nodes are also merged into a single type of node 'all' with \n",
    "    1753-dimensional features and 20-dimensional label categories.\n",
    "\n",
    "    Args:\n",
    "        root (string): Root directory where the dataset should be saved.\n",
    "        transform (callable, optional): A function/transform that takes in an\n",
    "            :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            every access. (default: :obj:`None`)\n",
    "        pre_transform (callable, optional): A function/transform that takes in\n",
    "            an :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            being saved to disk. (default: :obj:`None`)\n",
    "    \n",
    "    Stats:\n",
    "            * - #nodes\n",
    "              - #edges\n",
    "              - #features\n",
    "              - #classes\n",
    "            * - 2,951\n",
    "              - 1,071,977\n",
    "              - 1753\n",
    "              - 20\n",
    "    \"\"\"\n",
    "\n",
    "    url = 'https://drive.google.com/uc?export=download&id=1sxcKiZr1YGDv06wr03nsk5HVZledgzi9'\n",
    "\n",
    "    def __init__(self, root: str, transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "        \n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            'A_simp.npz', 'A_SOC.npz', 'A_SPA.npz', 'A_TEM.npz', 'labels.npz',\n",
    "            'node_types.npy', 'Textual_Features.npy', 'train_val_test_idx.npz',\n",
    "            'Visual_Features.npy'\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        os.remove(path)\n",
    "\n",
    "    def process(self):\n",
    "        data = HeteroData()\n",
    "\n",
    "        node_types = ['vis_only', 'vis_tex']\n",
    "        link_types = ['SOC', 'SPA', 'TEM', 'simp']\n",
    "\n",
    "        vis = np.load(osp.join(self.raw_dir, 'Visual_Features.npy'),allow_pickle=True)[:,2:].astype(float)\n",
    "        tex = np.load(osp.join(self.raw_dir, 'Textual_Features.npy'),allow_pickle=True)[:,5:].astype(float)\n",
    "\n",
    "        x = np.hstack([vis,np.nan_to_num(tex)])\n",
    "\n",
    "\n",
    "        node_type_idx = np.load(osp.join(self.raw_dir, 'node_types.npy'))\n",
    "        node_type_idx = torch.from_numpy(node_type_idx).to(torch.long)\n",
    "\n",
    "        data['vis_only'].num_nodes = int((node_type_idx == 0).sum())\n",
    "        data['vis_tex'].num_nodes = int((node_type_idx == 1).sum())\n",
    "        data['all'].num_nodes = len(node_type_idx)\n",
    "\n",
    "        data['vis_only'].x = torch.from_numpy(vis[node_type_idx==0]).to(torch.float)\n",
    "        data['vis_tex'].x = torch.from_numpy(x[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].x = torch.from_numpy(x).to(torch.float)\n",
    "\n",
    "\n",
    "        y_s = np.load(osp.join(self.raw_dir, 'labels.npz'), allow_pickle=True)\n",
    "        att_lab = y_s['ATT_LAB'][:,1:10].astype(float)\n",
    "        val_lab = np.nan_to_num(y_s['VAL_LAB'][:,2:13].astype(float))\n",
    "        ys = np.hstack([att_lab, val_lab])\n",
    "\n",
    "        data['vis_only'].y = torch.from_numpy(att_lab[node_type_idx==0]).to(torch.float)\n",
    "        data['vis_tex'].y = torch.from_numpy(ys[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].y = torch.from_numpy(ys).to(torch.float)\n",
    "\n",
    "        data.node_type = node_type_idx\n",
    "\n",
    "        split = np.load(osp.join(self.raw_dir, 'train_val_test_idx.npz'))\n",
    "        for name in ['train', 'val', 'test']:\n",
    "            idx = split[f'{name}_idx']\n",
    "            idx = torch.from_numpy(idx).to(torch.long)\n",
    "            mask = torch.zeros(data['all'].num_nodes, dtype=torch.bool)\n",
    "            mask[idx] = True\n",
    "            data['all'][f'{name}_mask'] = mask\n",
    "            data['vis_only'][f'{name}_mask'] = mask[node_type_idx==0]\n",
    "            data['vis_tex'][f'{name}_mask'] = mask[node_type_idx==1]\n",
    "\n",
    "        \n",
    "        s = {}\n",
    "        s['vis_only'] = np.arange(len(x))[node_type_idx==0]\n",
    "        s['vis_tex'] = np.arange(len(x))[node_type_idx==1]\n",
    "\n",
    "        for link in link_types:\n",
    "            A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz')).tocoo()\n",
    "            if A_sub.nnz>0:\n",
    "                row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                data['all', f'{link}_link', 'all'].edge_index = torch.stack([row, col], dim=0)\n",
    "                data['all', f'{link}_link', 'all'].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "        for src, dst in product(node_types, node_types):\n",
    "            for link in link_types:\n",
    "                A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz'))[s[src]][:,s[dst]].tocoo()\n",
    "                if A_sub.nnz>0:\n",
    "                    row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                    col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                    data[src, f'{link}_link', dst].edge_index = torch.stack([row, col], dim=0)\n",
    "                    data[src, f'{link}_link', dst].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "\n",
    "        if self.pre_transform is not None:\n",
    "            data = self.pre_transform(data)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.__class__.__name__}()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VEN_links(InMemoryDataset):\n",
    "    r\"\"\"A subset of Flickr post collected in Venice annotated with Heritage \n",
    "    Values and Attributes, as collected in the `\"Heri-Graphs: A Workflow of \n",
    "    Creating Datasets for Multi-modal Machine Learning on Graphs of Heritage \n",
    "    Values and Attributes with Social Media\" <https://arxiv.org/abs/2205.07545>`\n",
    "    paper.\n",
    "    VEN is a heterogeneous graph containing two types of nodes - nodes with only \n",
    "    visual features 'vis_only' (1,190 nodes), nodes with both visual and textual\n",
    "    features 'vis_tex' (1,761 nodes) and four types of links - social similarity\n",
    "    'SOC' (488,103 links), spatial similarity (445,779 links), temporal similarity\n",
    "    (501,191 links), and simple composed link (1,071,977 links).\n",
    "    Vis_only nodes are represented with 982-dimensional visual features and are\n",
    "    divided into 9 heritage attribute categories \n",
    "    ('architectural elements', 'form', 'gastronomy', 'interior',\n",
    "    'landscape scenery and natural features', 'monuments', 'people', 'product', \n",
    "    'urban scenery').\n",
    "    Vis_text nodes are represented with 1753-dimensional visual and textual \n",
    "    features and are divided into 9 heritage attribute categories plus 11 \n",
    "    heritage value categories ('criterion i-x', 'other').\n",
    "    Both types of nodes are also merged into a single type of node 'all' with \n",
    "    1753-dimensional features and 20-dimensional label categories.\n",
    "\n",
    "    Args:\n",
    "        root (string): Root directory where the dataset should be saved.\n",
    "        transform (callable, optional): A function/transform that takes in an\n",
    "            :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            every access. (default: :obj:`None`)\n",
    "        pre_transform (callable, optional): A function/transform that takes in\n",
    "            an :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            being saved to disk. (default: :obj:`None`)\n",
    "    \n",
    "    Stats:\n",
    "            * - #nodes\n",
    "              - #edges\n",
    "              - #features\n",
    "              - #classes\n",
    "            * - 2,951\n",
    "              - 1,071,977\n",
    "              - 1753\n",
    "              - 20\n",
    "    \"\"\"\n",
    "\n",
    "    url = 'https://drive.google.com/uc?export=download&id=1sxcKiZr1YGDv06wr03nsk5HVZledgzi9'\n",
    "\n",
    "    def __init__(self, root: str, transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "        \n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            'A_simp.npz', 'A_SOC.npz', 'A_SPA.npz', 'A_TEM.npz', 'labels.npz',\n",
    "            'node_types.npy', 'Textual_Features.npy', 'train_val_test_idx.npz',\n",
    "            'Visual_Features.npy'\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        os.remove(path)\n",
    "\n",
    "    def process(self):\n",
    "        data = HeteroData()\n",
    "\n",
    "        node_types = ['all']\n",
    "        link_types = ['SOC', 'SPA', 'TEM', 'simp']\n",
    "\n",
    "        vis = np.load(osp.join(self.raw_dir, 'Visual_Features.npy'),allow_pickle=True)[:,2:].astype(float)\n",
    "        tex = np.load(osp.join(self.raw_dir, 'Textual_Features.npy'),allow_pickle=True)[:,5:].astype(float)\n",
    "\n",
    "        x = np.hstack([vis,np.nan_to_num(tex)])\n",
    "\n",
    "\n",
    "        node_type_idx = np.load(osp.join(self.raw_dir, 'node_types.npy'))\n",
    "        node_type_idx = torch.from_numpy(node_type_idx).to(torch.long)\n",
    "\n",
    "        #data['vis_only'].num_nodes = int((node_type_idx == 0).sum())\n",
    "        #data['vis_tex'].num_nodes = int((node_type_idx == 1).sum())\n",
    "        data['all'].num_nodes = len(node_type_idx)\n",
    "\n",
    "        #data['vis_only'].x = torch.from_numpy(vis[node_type_idx==0]).to(torch.float)\n",
    "        #data['vis_tex'].x = torch.from_numpy(x[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].x = torch.from_numpy(x).to(torch.float)\n",
    "\n",
    "\n",
    "        y_s = np.load(osp.join(self.raw_dir, 'labels.npz'), allow_pickle=True)\n",
    "        att_lab = y_s['ATT_LAB'][:,1:10].astype(float)\n",
    "        val_lab = np.nan_to_num(y_s['VAL_LAB'][:,2:13].astype(float))\n",
    "        ys = np.hstack([att_lab, val_lab])\n",
    "\n",
    "        #data['vis_only'].y = torch.from_numpy(att_lab[node_type_idx==0]).to(torch.float)\n",
    "        #data['vis_tex'].y = torch.from_numpy(ys[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].y = torch.from_numpy(ys).to(torch.float)\n",
    "\n",
    "        data['all'].node_type = node_type_idx\n",
    "        \n",
    "        data['all'].att_lab = torch.tensor(y_s['ATT_LAB'][:,-1].astype(bool))\n",
    "        data['all'].val_lab = torch.tensor(y_s['VAL_LAB'][:,-1].astype(bool))\n",
    "\n",
    "        split = np.load(osp.join(self.raw_dir, 'train_val_test_idx.npz'))\n",
    "        for name in ['train', 'val', 'test']:\n",
    "            idx = split[f'{name}_idx']\n",
    "            idx = torch.from_numpy(idx).to(torch.long)\n",
    "            mask = torch.zeros(data['all'].num_nodes, dtype=torch.bool)\n",
    "            mask[idx] = True\n",
    "            data['all'][f'{name}_mask'] = mask\n",
    "            #data['vis_only'][f'{name}_mask'] = mask[node_type_idx==0]\n",
    "            #data['vis_tex'][f'{name}_mask'] = mask[node_type_idx==1]\n",
    "\n",
    "        \n",
    "        s = {}\n",
    "        #s['vis_only'] = np.arange(len(x))[node_type_idx==0]\n",
    "        #s['vis_tex'] = np.arange(len(x))[node_type_idx==1]\n",
    "\n",
    "        for link in link_types:\n",
    "            A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz')).tocoo()\n",
    "            if A_sub.nnz>0:\n",
    "                row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                data['all', f'{link}_link', 'all'].edge_index = torch.stack([row, col], dim=0)\n",
    "                data['all', f'{link}_link', 'all'].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "        if self.pre_transform is not None:\n",
    "            data = self.pre_transform(data)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.__class__.__name__}()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VEN_XL(InMemoryDataset):\n",
    "    r\"\"\"A large subset of Flickr post collected in Venice annotated with Heritage \n",
    "    Values and Attributes, as collected in the `\"Heri-Graphs: A Workflow of \n",
    "    Creating Datasets for Multi-modal Machine Learning on Graphs of Heritage \n",
    "    Values and Attributes with Social Media\" <https://arxiv.org/abs/2205.07545>`\n",
    "    paper.\n",
    "    VEN_XL is a heterogeneous graph containing two types of nodes - nodes with only \n",
    "    visual features 'vis_only' (31,140 nodes), nodes with both visual and textual\n",
    "    features 'vis_tex' (49,823 nodes) and four types of links - social similarity\n",
    "    'SOC' (76,422,265 links), spatial similarity (202,173,159 links), temporal similarity\n",
    "    (71,135,671 links), and simple composed link (290,091,503 links).\n",
    "    Vis_only nodes are represented with 982-dimensional visual features and are\n",
    "    divided into 9 heritage attribute categories \n",
    "    ('architectural elements', 'form', 'gastronomy', 'interior',\n",
    "    'landscape scenery and natural features', 'monuments', 'people', 'product', \n",
    "    'urban scenery').\n",
    "    Vis_text nodes are represented with 1753-dimensional visual and textual \n",
    "    features and are divided into 9 heritage attribute categories plus 11 \n",
    "    heritage value categories ('criterion i-x', 'other').\n",
    "    Both types of nodes are also merged into a single type of node 'all' with \n",
    "    1753-dimensional features and 20-dimensional label categories.\n",
    "\n",
    "    Args:\n",
    "        root (string): Root directory where the dataset should be saved.\n",
    "        transform (callable, optional): A function/transform that takes in an\n",
    "            :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            every access. (default: :obj:`None`)\n",
    "        pre_transform (callable, optional): A function/transform that takes in\n",
    "            an :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            being saved to disk. (default: :obj:`None`)\n",
    "    \n",
    "    Stats:\n",
    "            * - #nodes\n",
    "              - #edges\n",
    "              - #features\n",
    "              - #classes\n",
    "            * - 80,963\n",
    "              - 290,091,503\n",
    "              - 1753\n",
    "              - 20\n",
    "    \"\"\"\n",
    "\n",
    "    url = 'https://drive.google.com/uc?export=download&id=1QZ5tyUWs6jYjh7mJrsnpou76iy-vb0CA'\n",
    "\n",
    "    def __init__(self, root: str, transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "        \n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            'A_simp.npz', 'A_SOC.npz', 'A_SPA.npz', 'A_TEM.npz', 'labels.npz',\n",
    "            'node_types.npy', 'Textual_Features.npy', 'train_val_test_idx.npz',\n",
    "            'Visual_Features.npy'\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        os.remove(path)\n",
    "\n",
    "    def process(self):\n",
    "        data = HeteroData()\n",
    "\n",
    "        node_types = ['vis_only', 'vis_tex']\n",
    "        link_types = ['SOC', 'SPA', 'TEM', 'simp']\n",
    "\n",
    "        vis = np.load(osp.join(self.raw_dir, 'Visual_Features.npy'),allow_pickle=True)[:,2:].astype(float)\n",
    "        tex = np.load(osp.join(self.raw_dir, 'Textual_Features.npy'),allow_pickle=True)[:,5:].astype(float)\n",
    "\n",
    "        x = np.hstack([vis,np.nan_to_num(tex)])\n",
    "\n",
    "\n",
    "        node_type_idx = np.load(osp.join(self.raw_dir, 'node_types.npy'))\n",
    "        node_type_idx = torch.from_numpy(node_type_idx).to(torch.long)\n",
    "\n",
    "        data['vis_only'].num_nodes = int((node_type_idx == 0).sum())\n",
    "        data['vis_tex'].num_nodes = int((node_type_idx == 1).sum())\n",
    "        data['all'].num_nodes = len(node_type_idx)\n",
    "\n",
    "        data['vis_only'].x = torch.from_numpy(vis[node_type_idx==0]).to(torch.float)\n",
    "        data['vis_tex'].x = torch.from_numpy(x[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].x = torch.from_numpy(x).to(torch.float)\n",
    "\n",
    "\n",
    "        y_s = np.load(osp.join(self.raw_dir, 'labels.npz'), allow_pickle=True)\n",
    "        att_lab = y_s['ATT_LAB'][:,1:10].astype(float)\n",
    "        val_lab = np.nan_to_num(y_s['VAL_LAB'][:,2:13].astype(float))\n",
    "        ys = np.hstack([att_lab, val_lab])\n",
    "\n",
    "        data['vis_only'].y = torch.from_numpy(att_lab[node_type_idx==0]).to(torch.float)\n",
    "        data['vis_tex'].y = torch.from_numpy(ys[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].y = torch.from_numpy(ys).to(torch.float)\n",
    "\n",
    "        data.node_type = node_type_idx\n",
    "\n",
    "        split = np.load(osp.join(self.raw_dir, 'train_val_test_idx.npz'))\n",
    "        for name in ['train', 'val', 'test']:\n",
    "            idx = split[f'{name}_idx']\n",
    "            idx = torch.from_numpy(idx).to(torch.long)\n",
    "            mask = torch.zeros(data['all'].num_nodes, dtype=torch.bool)\n",
    "            mask[idx] = True\n",
    "            data['all'][f'{name}_mask'] = mask\n",
    "            data['vis_only'][f'{name}_mask'] = mask[node_type_idx==0]\n",
    "            data['vis_tex'][f'{name}_mask'] = mask[node_type_idx==1]\n",
    "\n",
    "        \n",
    "        s = {}\n",
    "        s['vis_only'] = np.arange(len(x))[node_type_idx==0]\n",
    "        s['vis_tex'] = np.arange(len(x))[node_type_idx==1]\n",
    "\n",
    "        for link in link_types:\n",
    "            A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz')).tocoo()\n",
    "            if A_sub.nnz>0:\n",
    "                row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                data['all', f'{link}_link', 'all'].edge_index = torch.stack([row, col], dim=0)\n",
    "                data['all', f'{link}_link', 'all'].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "        for src, dst in product(node_types, node_types):\n",
    "            for link in link_types:\n",
    "                A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz'))[s[src]][:,s[dst]].tocoo()\n",
    "                if A_sub.nnz>0:\n",
    "                    row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                    col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                    data[src, f'{link}_link', dst].edge_index = torch.stack([row, col], dim=0)\n",
    "                    data[src, f'{link}_link', dst].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "\n",
    "        if self.pre_transform is not None:\n",
    "            data = self.pre_transform(data)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.__class__.__name__}()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VEN_XL_Links(InMemoryDataset):\n",
    "    r\"\"\"A large subset of Flickr post collected in Venice annotated with Heritage \n",
    "    Values and Attributes, as collected in the `\"Heri-Graphs: A Workflow of \n",
    "    Creating Datasets for Multi-modal Machine Learning on Graphs of Heritage \n",
    "    Values and Attributes with Social Media\" <https://arxiv.org/abs/2205.07545>`\n",
    "    paper.\n",
    "    VEN_XL is a heterogeneous graph containing two types of nodes - nodes with only \n",
    "    visual features 'vis_only' (31,140 nodes), nodes with both visual and textual\n",
    "    features 'vis_tex' (49,823 nodes) and four types of links - social similarity\n",
    "    'SOC' (76,422,265 links), spatial similarity (202,173,159 links), temporal similarity\n",
    "    (71,135,671 links), and simple composed link (290,091,503 links).\n",
    "    Vis_only nodes are represented with 982-dimensional visual features and are\n",
    "    divided into 9 heritage attribute categories \n",
    "    ('architectural elements', 'form', 'gastronomy', 'interior',\n",
    "    'landscape scenery and natural features', 'monuments', 'people', 'product', \n",
    "    'urban scenery').\n",
    "    Vis_text nodes are represented with 1753-dimensional visual and textual \n",
    "    features and are divided into 9 heritage attribute categories plus 11 \n",
    "    heritage value categories ('criterion i-x', 'other').\n",
    "    Both types of nodes are also merged into a single type of node 'all' with \n",
    "    1753-dimensional features and 20-dimensional label categories.\n",
    "\n",
    "    Args:\n",
    "        root (string): Root directory where the dataset should be saved.\n",
    "        transform (callable, optional): A function/transform that takes in an\n",
    "            :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            every access. (default: :obj:`None`)\n",
    "        pre_transform (callable, optional): A function/transform that takes in\n",
    "            an :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            being saved to disk. (default: :obj:`None`)\n",
    "    \n",
    "    Stats:\n",
    "            * - #nodes\n",
    "              - #edges\n",
    "              - #features\n",
    "              - #classes\n",
    "            * - 80,963\n",
    "              - 290,091,503\n",
    "              - 1753\n",
    "              - 20\n",
    "    \"\"\"\n",
    "\n",
    "    url = 'https://drive.google.com/uc?export=download&id=1QZ5tyUWs6jYjh7mJrsnpou76iy-vb0CA'\n",
    "\n",
    "    def __init__(self, root: str, transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "        \n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            'A_simp.npz', 'A_SOC.npz', 'A_SPA.npz', 'A_TEM.npz', 'labels.npz',\n",
    "            'node_types.npy', 'Textual_Features.npy', 'train_val_test_idx.npz',\n",
    "            'Visual_Features.npy'\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        os.remove(path)\n",
    "\n",
    "    def process(self):\n",
    "        data = HeteroData()\n",
    "\n",
    "        node_types = ['all']\n",
    "        link_types = ['SOC', 'SPA', 'TEM', 'simp']\n",
    "\n",
    "        vis = np.load(osp.join(self.raw_dir, 'Visual_Features.npy'),allow_pickle=True)[:,2:].astype(float)\n",
    "        tex = np.load(osp.join(self.raw_dir, 'Textual_Features.npy'),allow_pickle=True)[:,5:].astype(float)\n",
    "\n",
    "        x = np.hstack([vis,np.nan_to_num(tex)])\n",
    "\n",
    "\n",
    "        node_type_idx = np.load(osp.join(self.raw_dir, 'node_types.npy'))\n",
    "        node_type_idx = torch.from_numpy(node_type_idx).to(torch.long)\n",
    "\n",
    "        #data['vis_only'].num_nodes = int((node_type_idx == 0).sum())\n",
    "        #data['vis_tex'].num_nodes = int((node_type_idx == 1).sum())\n",
    "        data['all'].num_nodes = len(node_type_idx)\n",
    "\n",
    "        #data['vis_only'].x = torch.from_numpy(vis[node_type_idx==0]).to(torch.float)\n",
    "        #data['vis_tex'].x = torch.from_numpy(x[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].x = torch.from_numpy(x).to(torch.float)\n",
    "\n",
    "\n",
    "        y_s = np.load(osp.join(self.raw_dir, 'labels.npz'), allow_pickle=True)\n",
    "        att_lab = y_s['ATT_LAB'][:,1:10].astype(float)\n",
    "        val_lab = np.nan_to_num(y_s['VAL_LAB'][:,2:13].astype(float))\n",
    "        ys = np.hstack([att_lab, val_lab])\n",
    "\n",
    "        #data['vis_only'].y = torch.from_numpy(att_lab[node_type_idx==0]).to(torch.float)\n",
    "        #data['vis_tex'].y = torch.from_numpy(ys[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].y = torch.from_numpy(ys).to(torch.float)\n",
    "\n",
    "        data['all'].node_type = node_type_idx\n",
    "        \n",
    "        data['all'].att_lab = torch.tensor(y_s['ATT_LAB'][:,-1].astype(bool))\n",
    "        data['all'].val_lab = torch.tensor(y_s['VAL_LAB'][:,-1].astype(bool))\n",
    "\n",
    "        split = np.load(osp.join(self.raw_dir, 'train_val_test_idx.npz'))\n",
    "        for name in ['train', 'val', 'test']:\n",
    "            idx = split[f'{name}_idx']\n",
    "            idx = torch.from_numpy(idx).to(torch.long)\n",
    "            mask = torch.zeros(data['all'].num_nodes, dtype=torch.bool)\n",
    "            mask[idx] = True\n",
    "            data['all'][f'{name}_mask'] = mask\n",
    "            #data['vis_only'][f'{name}_mask'] = mask[node_type_idx==0]\n",
    "            #data['vis_tex'][f'{name}_mask'] = mask[node_type_idx==1]\n",
    "\n",
    "        \n",
    "        s = {}\n",
    "        #s['vis_only'] = np.arange(len(x))[node_type_idx==0]\n",
    "        #s['vis_tex'] = np.arange(len(x))[node_type_idx==1]\n",
    "\n",
    "        for link in link_types:\n",
    "            A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz')).tocoo()\n",
    "            if A_sub.nnz>0:\n",
    "                row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                data['all', f'{link}_link', 'all'].edge_index = torch.stack([row, col], dim=0)\n",
    "                data['all', f'{link}_link', 'all'].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "        if self.pre_transform is not None:\n",
    "            data = self.pre_transform(data)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.__class__.__name__}()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#path = osp.join(os.getcwd(), '../../data/VEN')\n",
    "dataset = VEN_links('dataset/Venice_links')\n",
    "data = dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=2951,\n",
       "    x=[2951, 1753],\n",
       "    y=[2951, 20],\n",
       "    node_type=[2951],\n",
       "    att_lab=[2951],\n",
       "    val_lab=[2951],\n",
       "    train_mask=[2951],\n",
       "    val_mask=[2951],\n",
       "    test_mask=[2951],\n",
       "    n_id=[2951]\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 488103],\n",
       "    edge_attr=[488103]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 445779],\n",
       "    edge_attr=[445779]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 501191],\n",
       "    edge_attr=[501191]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 1071977],\n",
       "    edge_attr=[1071977]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['all'].n_id = torch.arange(data.num_nodes)\n",
    "data = data.to(device)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(361, device='cuda:0')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['all']['train_mask'].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Loader for Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.loader import NeighborLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('all', 'SOC_link', 'all'),\n",
       " ('all', 'SPA_link', 'all'),\n",
       " ('all', 'TEM_link', 'all'),\n",
       " ('all', 'simp_link', 'all')]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.edge_types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].train_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].val_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=2950,\n",
       "    x=[2950, 1753],\n",
       "    y=[2950, 20],\n",
       "    node_type=[2950],\n",
       "    att_lab=[2950],\n",
       "    val_lab=[2950],\n",
       "    train_mask=[2950],\n",
       "    val_mask=[2950],\n",
       "    test_mask=[2950],\n",
       "    n_id=[2950],\n",
       "    batch_size=32\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 26610],\n",
       "    edge_attr=[26610]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 28393],\n",
       "    edge_attr=[28393]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 26680],\n",
       "    edge_attr=[26680]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_hetero_data = next(iter(train_loader))\n",
    "batch = sampled_hetero_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(361, device='cuda:0')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch['all']['train_mask'].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_train_state(args):\n",
    "    return {'stop_early': False,\n",
    "            'early_stopping_step': 0,\n",
    "            'early_stopping_best_ATT_acc_val': 0,\n",
    "            'early_stopping_best_VAL_acc_val': 0,\n",
    "            'early_stopping_best_ATT_acc_val_2': 0,\n",
    "            'early_stopping_lowest_loss': 1000,\n",
    "            'learning_rate': args.learning_rate,\n",
    "            'epoch_index': 0,\n",
    "            'train_loss': [],\n",
    "            'train_ATT_loss': [],\n",
    "            'train_VAL_loss':[],\n",
    "            'train_ATT_acc': [],\n",
    "            'train_VAL_acc': [],\n",
    "            'train_VAL_jac': [],\n",
    "            'train_VAL_acc_1':[], \n",
    "            'val_loss': [],\n",
    "            'val_ATT_loss': [],\n",
    "            'val_VAL_loss':[],\n",
    "            'val_ATT_acc': [],\n",
    "            'val_VAL_acc': [],\n",
    "            'val_VAL_jac': [],\n",
    "            'val_VAL_acc_1': [],\n",
    "            'test_loss': -1,\n",
    "            'test_ATT_loss': -1,\n",
    "            'test_VAL_loss':-1,\n",
    "            'test_ATT_acc': -1,\n",
    "            'test_VAL_acc': -1,\n",
    "            'test_VAL_jac': -1,\n",
    "            'test_VAL_acc_1': -1,\n",
    "            'model_filename': args.model_state_file}\n",
    "\n",
    "def update_train_state(args, model, train_state):\n",
    "    \"\"\"Handle the training state updates.\n",
    "\n",
    "    Components:\n",
    "     - Early Stopping: Prevent overfitting.\n",
    "     - Model Checkpoint: Model is saved if the model is better\n",
    "\n",
    "    :param args: main arguments\n",
    "    :param model: model to train\n",
    "    :param train_state: a dictionary representing the training state values\n",
    "    :returns:\n",
    "        a new train_state\n",
    "    \"\"\"\n",
    "\n",
    "    # Save one model at least\n",
    "    if train_state['epoch_index'] == 0:\n",
    "        torch.save(model.state_dict(), train_state['model_filename'])\n",
    "        train_state['stop_early'] = False\n",
    "\n",
    "    # Save model if performance improved\n",
    "    elif train_state['epoch_index'] >= 1:\n",
    "        ATT_acc_tm1, ATT_acc_t = train_state['val_ATT_acc'][-2:]\n",
    "        #ATT_acc_2_tm1, ATT_acc_2_t = train_state['val_ATT_acc_2'][-2:]\n",
    "        VAL_acc_tm1, VAL_acc_t = train_state['val_VAL_acc'][-2:]\n",
    "        loss_tm1, loss_t = train_state['val_loss'][-2:]\n",
    "\n",
    "        # If accuracy worsened\n",
    "        #if loss_t >= train_state['early_stopping_lowest_loss']:\n",
    "        #    train_state['early_stopping_step'] += 1\n",
    "        \n",
    "        if ATT_acc_t <= train_state['early_stopping_best_ATT_acc_val'] and VAL_acc_t <= train_state['early_stopping_best_VAL_acc_val']:# and ATT_acc_2_t <= train_state['early_stopping_best_ATT_acc_val_2']:\n",
    "            # Update step\n",
    "            train_state['early_stopping_step'] += 1\n",
    "        # Loss decreased\n",
    "        else:\n",
    "            # Save the best model from sklearn\n",
    "            if VAL_acc_t > train_state['early_stopping_best_VAL_acc_val']:\n",
    "                train_state['early_stopping_best_VAL_acc_val'] = VAL_acc_t\n",
    "                \n",
    "            if ATT_acc_t > train_state['early_stopping_best_ATT_acc_val']:\n",
    "                train_state['early_stopping_best_ATT_acc_val'] = ATT_acc_t\n",
    "            \n",
    "            #if ATT_acc_2_t > train_state['early_stopping_best_ATT_acc_val_2']:\n",
    "            #    train_state['early_stopping_best_ATT_acc_val_2'] = ATT_acc_2_t\n",
    "                \n",
    "            if loss_t < train_state['early_stopping_lowest_loss']:\n",
    "                train_state['early_stopping_lowest_loss'] = loss_t\n",
    "                torch.save(model.state_dict(), train_state['model_filename'])\n",
    "                \n",
    "            # Reset early stopping step\n",
    "            train_state['early_stopping_step'] = 0\n",
    "\n",
    "        # Stop early ?\n",
    "        train_state['stop_early'] = \\\n",
    "            train_state['early_stopping_step'] >= args.early_stopping_criteria\n",
    "\n",
    "    return train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_cross_entropy(y_pred, y_target):\n",
    "    y_target = y_target.cpu().float()\n",
    "    y_pred = y_pred.cpu().float()\n",
    "    criterion = nn.BCEWithLogitsLoss()\n",
    "    return criterion(y_target, y_pred)\n",
    "\n",
    "def compute_1_accuracy(y_pred, y_target):\n",
    "    y_target_indices = y_target.max(dim=1)[1]\n",
    "    y_pred_indices = y_pred.max(dim=1)[1]\n",
    "    n_correct = torch.eq(y_pred_indices, y_target_indices).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_k_accuracy(y_pred, y_target, k=3):\n",
    "    y_pred_indices = y_pred.topk(k, dim=1)[1]\n",
    "    y_target_indices = y_target.max(dim=1)[1]\n",
    "    n_correct = torch.tensor([y_pred_indices[i] in y_target_indices[i] for i in range(len(y_pred))]).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_k_jaccard_index(y_pred, y_target, k=3):\n",
    "    y_target_indices = y_target.topk(k, dim=1)[1]\n",
    "    y_pred_indices = y_pred.max(dim=1)[1]\n",
    "    jaccard = torch.tensor([len(np.intersect1d(y_target_indices[i], y_pred_indices[i]))/\n",
    "                            len(np.union1d(y_target_indices[i], y_pred_indices[i]))\n",
    "                            for i in range(len(y_pred))]).sum().item()\n",
    "    return jaccard / len(y_pred_indices)\n",
    "\n",
    "def compute_jaccard_index(y_pred, y_target, k=3, multilabel=False):\n",
    "    \n",
    "    threshold = 1.0/(k+1)\n",
    "    threshold_2 = 0.5\n",
    "    \n",
    "    if multilabel:\n",
    "        y_pred_indices = y_pred.gt(threshold_2)\n",
    "    else:\n",
    "        y_pred_indices = y_pred.gt(threshold)\n",
    "    \n",
    "    y_target_indices = y_target.gt(threshold)\n",
    "        \n",
    "    jaccard = ((y_target_indices*y_pred_indices).sum(axis=1)/((y_target_indices+y_pred_indices).sum(axis=1)+1e-8)).sum().item()\n",
    "    return jaccard / len(y_pred_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy(pred, soft_targets):\n",
    "    logsoftmax = nn.LogSoftmax(dim=1)\n",
    "    return torch.mean(torch.sum(- soft_targets * logsoftmax(pred), 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Searched Best Hyper-Parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'42/hyperdict.p', 'rb') as fp:\n",
    "    hyperdict= pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "192"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(hyperdict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "hyper_df = pd.DataFrame(hyperdict).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>stop_early</th>\n",
       "      <th>early_stopping_step</th>\n",
       "      <th>early_stopping_best_ATT_acc_val</th>\n",
       "      <th>early_stopping_best_VAL_acc_val</th>\n",
       "      <th>early_stopping_best_ATT_acc_val_2</th>\n",
       "      <th>early_stopping_lowest_loss</th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>epoch_index</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_ATT_loss</th>\n",
       "      <th>...</th>\n",
       "      <th>val_VAL_jac</th>\n",
       "      <th>val_VAL_acc_1</th>\n",
       "      <th>test_loss</th>\n",
       "      <th>test_ATT_loss</th>\n",
       "      <th>test_VAL_loss</th>\n",
       "      <th>test_ATT_acc</th>\n",
       "      <th>test_VAL_acc</th>\n",
       "      <th>test_VAL_jac</th>\n",
       "      <th>test_VAL_acc_1</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">2</th>\n",
       "      <th>2</th>\n",
       "      <th>sum</th>\n",
       "      <th>64</th>\n",
       "      <th>0.0100</th>\n",
       "      <td>True</td>\n",
       "      <td>100</td>\n",
       "      <td>89.064748</td>\n",
       "      <td>98.116416</td>\n",
       "      <td>0</td>\n",
       "      <td>4.260714</td>\n",
       "      <td>0.01</td>\n",
       "      <td>238</td>\n",
       "      <td>[4.073777973651886, 3.071266452471415, 2.70923...</td>\n",
       "      <td>[1.5183023914405844, 1.0559576274285356, 0.904...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.26566383523153075, 0.3079354757937258, 0.31...</td>\n",
       "      <td>[43.413124046217575, 36.670590800087204, 39.42...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HGT/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">3</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">sum</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">32</th>\n",
       "      <th>0.0010</th>\n",
       "      <td>True</td>\n",
       "      <td>100</td>\n",
       "      <td>89.496403</td>\n",
       "      <td>98.942228</td>\n",
       "      <td>0</td>\n",
       "      <td>4.207492</td>\n",
       "      <td>0.001</td>\n",
       "      <td>252</td>\n",
       "      <td>[4.071910460789998, 3.4655648271242776, 3.2366...</td>\n",
       "      <td>[1.788339904470787, 1.5794255941528363, 1.3938...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.2376455202803311, 0.259545095542274, 0.2871...</td>\n",
       "      <td>[29.764115979943316, 34.3684325267059, 36.0313...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HGT/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.0005</th>\n",
       "      <td>False</td>\n",
       "      <td>98</td>\n",
       "      <td>88.920863</td>\n",
       "      <td>98.953564</td>\n",
       "      <td>0</td>\n",
       "      <td>4.223594</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>299</td>\n",
       "      <td>[4.231941819190979, 3.713909467061361, 3.45286...</td>\n",
       "      <td>[1.9538542296417531, 1.7716821270306025, 1.613...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.06249400512881598, 0.21880968061022593, 0.2...</td>\n",
       "      <td>[14.242860257248745, 29.35687813385655, 34.577...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HGT/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <th>32</th>\n",
       "      <th>0.0005</th>\n",
       "      <td>True</td>\n",
       "      <td>100</td>\n",
       "      <td>89.208633</td>\n",
       "      <td>98.10508</td>\n",
       "      <td>0</td>\n",
       "      <td>4.238065</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>238</td>\n",
       "      <td>[4.248942116896312, 3.837139884630839, 3.53874...</td>\n",
       "      <td>[1.972144102125617, 1.784775425852831, 1.62302...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.051972967413533654, 0.11621103192352897, 0....</td>\n",
       "      <td>[14.452147373010682, 18.649226073686503, 34.15...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HGT/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">4</th>\n",
       "      <th>3</th>\n",
       "      <th>sum</th>\n",
       "      <th>64</th>\n",
       "      <th>0.0100</th>\n",
       "      <td>False</td>\n",
       "      <td>95</td>\n",
       "      <td>89.640288</td>\n",
       "      <td>98.116416</td>\n",
       "      <td>0</td>\n",
       "      <td>4.250843</td>\n",
       "      <td>0.01</td>\n",
       "      <td>299</td>\n",
       "      <td>[4.173254390557607, 2.9642478823661804, 2.6236...</td>\n",
       "      <td>[1.4151781909683734, 0.95903942459508, 0.84767...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.2324700263093857, 0.3140731074988309, 0.317...</td>\n",
       "      <td>[41.727490734684984, 40.04185742315239, 39.634...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HGT/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <th>sum</th>\n",
       "      <th>32</th>\n",
       "      <th>0.0001</th>\n",
       "      <td>True</td>\n",
       "      <td>100</td>\n",
       "      <td>88.201439</td>\n",
       "      <td>98.314367</td>\n",
       "      <td>0</td>\n",
       "      <td>4.251635</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>285</td>\n",
       "      <td>[4.508403778076172, 4.371051947275798, 4.22484...</td>\n",
       "      <td>[2.1526270024994405, 2.085895083947855, 2.0104...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.031393067364290386...</td>\n",
       "      <td>[3.9877915849138876, 14.661434488772617, 20.31...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HGT/model.pth</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   stop_early early_stopping_step  \\\n",
       "2 2 sum  64 0.0100       True                 100   \n",
       "  3 sum  32 0.0010       True                 100   \n",
       "            0.0005      False                  98   \n",
       "    mean 32 0.0005       True                 100   \n",
       "4 3 sum  64 0.0100      False                  95   \n",
       "  5 sum  32 0.0001       True                 100   \n",
       "\n",
       "                   early_stopping_best_ATT_acc_val  \\\n",
       "2 2 sum  64 0.0100                       89.064748   \n",
       "  3 sum  32 0.0010                       89.496403   \n",
       "            0.0005                       88.920863   \n",
       "    mean 32 0.0005                       89.208633   \n",
       "4 3 sum  64 0.0100                       89.640288   \n",
       "  5 sum  32 0.0001                       88.201439   \n",
       "\n",
       "                   early_stopping_best_VAL_acc_val  \\\n",
       "2 2 sum  64 0.0100                       98.116416   \n",
       "  3 sum  32 0.0010                       98.942228   \n",
       "            0.0005                       98.953564   \n",
       "    mean 32 0.0005                        98.10508   \n",
       "4 3 sum  64 0.0100                       98.116416   \n",
       "  5 sum  32 0.0001                       98.314367   \n",
       "\n",
       "                   early_stopping_best_ATT_acc_val_2  \\\n",
       "2 2 sum  64 0.0100                                 0   \n",
       "  3 sum  32 0.0010                                 0   \n",
       "            0.0005                                 0   \n",
       "    mean 32 0.0005                                 0   \n",
       "4 3 sum  64 0.0100                                 0   \n",
       "  5 sum  32 0.0001                                 0   \n",
       "\n",
       "                   early_stopping_lowest_loss learning_rate epoch_index  \\\n",
       "2 2 sum  64 0.0100                   4.260714          0.01         238   \n",
       "  3 sum  32 0.0010                   4.207492         0.001         252   \n",
       "            0.0005                   4.223594        0.0005         299   \n",
       "    mean 32 0.0005                   4.238065        0.0005         238   \n",
       "4 3 sum  64 0.0100                   4.250843          0.01         299   \n",
       "  5 sum  32 0.0001                   4.251635        0.0001         285   \n",
       "\n",
       "                                                           train_loss  \\\n",
       "2 2 sum  64 0.0100  [4.073777973651886, 3.071266452471415, 2.70923...   \n",
       "  3 sum  32 0.0010  [4.071910460789998, 3.4655648271242776, 3.2366...   \n",
       "            0.0005  [4.231941819190979, 3.713909467061361, 3.45286...   \n",
       "    mean 32 0.0005  [4.248942116896312, 3.837139884630839, 3.53874...   \n",
       "4 3 sum  64 0.0100  [4.173254390557607, 2.9642478823661804, 2.6236...   \n",
       "  5 sum  32 0.0001  [4.508403778076172, 4.371051947275798, 4.22484...   \n",
       "\n",
       "                                                       train_ATT_loss  ...  \\\n",
       "2 2 sum  64 0.0100  [1.5183023914405844, 1.0559576274285356, 0.904...  ...   \n",
       "  3 sum  32 0.0010  [1.788339904470787, 1.5794255941528363, 1.3938...  ...   \n",
       "            0.0005  [1.9538542296417531, 1.7716821270306025, 1.613...  ...   \n",
       "    mean 32 0.0005  [1.972144102125617, 1.784775425852831, 1.62302...  ...   \n",
       "4 3 sum  64 0.0100  [1.4151781909683734, 0.95903942459508, 0.84767...  ...   \n",
       "  5 sum  32 0.0001  [2.1526270024994405, 2.085895083947855, 2.0104...  ...   \n",
       "\n",
       "                                                          val_VAL_jac  \\\n",
       "2 2 sum  64 0.0100  [0.26566383523153075, 0.3079354757937258, 0.31...   \n",
       "  3 sum  32 0.0010  [0.2376455202803311, 0.259545095542274, 0.2871...   \n",
       "            0.0005  [0.06249400512881598, 0.21880968061022593, 0.2...   \n",
       "    mean 32 0.0005  [0.051972967413533654, 0.11621103192352897, 0....   \n",
       "4 3 sum  64 0.0100  [0.2324700263093857, 0.3140731074988309, 0.317...   \n",
       "  5 sum  32 0.0001  [0.0, 0.0, 0.0, 0.0, 0.0, 0.031393067364290386...   \n",
       "\n",
       "                                                        val_VAL_acc_1  \\\n",
       "2 2 sum  64 0.0100  [43.413124046217575, 36.670590800087204, 39.42...   \n",
       "  3 sum  32 0.0010  [29.764115979943316, 34.3684325267059, 36.0313...   \n",
       "            0.0005  [14.242860257248745, 29.35687813385655, 34.577...   \n",
       "    mean 32 0.0005  [14.452147373010682, 18.649226073686503, 34.15...   \n",
       "4 3 sum  64 0.0100  [41.727490734684984, 40.04185742315239, 39.634...   \n",
       "  5 sum  32 0.0001  [3.9877915849138876, 14.661434488772617, 20.31...   \n",
       "\n",
       "                   test_loss test_ATT_loss test_VAL_loss test_ATT_acc  \\\n",
       "2 2 sum  64 0.0100        -1            -1            -1           -1   \n",
       "  3 sum  32 0.0010        -1            -1            -1           -1   \n",
       "            0.0005        -1            -1            -1           -1   \n",
       "    mean 32 0.0005        -1            -1            -1           -1   \n",
       "4 3 sum  64 0.0100        -1            -1            -1           -1   \n",
       "  5 sum  32 0.0001        -1            -1            -1           -1   \n",
       "\n",
       "                   test_VAL_acc test_VAL_jac test_VAL_acc_1  \\\n",
       "2 2 sum  64 0.0100           -1           -1             -1   \n",
       "  3 sum  32 0.0010           -1           -1             -1   \n",
       "            0.0005           -1           -1             -1   \n",
       "    mean 32 0.0005           -1           -1             -1   \n",
       "4 3 sum  64 0.0100           -1           -1             -1   \n",
       "  5 sum  32 0.0001           -1           -1             -1   \n",
       "\n",
       "                                 model_filename  \n",
       "2 2 sum  64 0.0100  model_storage/HGT/model.pth  \n",
       "  3 sum  32 0.0010  model_storage/HGT/model.pth  \n",
       "            0.0005  model_storage/HGT/model.pth  \n",
       "    mean 32 0.0005  model_storage/HGT/model.pth  \n",
       "4 3 sum  64 0.0100  model_storage/HGT/model.pth  \n",
       "  5 sum  32 0.0001  model_storage/HGT/model.pth  \n",
       "\n",
       "[6 rows x 30 columns]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hyper_df[(hyper_df.early_stopping_best_VAL_acc_val + hyper_df.early_stopping_best_VAL_acc_val)>196]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>stop_early</th>\n",
       "      <th>early_stopping_step</th>\n",
       "      <th>early_stopping_best_ATT_acc_val</th>\n",
       "      <th>early_stopping_best_VAL_acc_val</th>\n",
       "      <th>early_stopping_best_ATT_acc_val_2</th>\n",
       "      <th>early_stopping_lowest_loss</th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>epoch_index</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_ATT_loss</th>\n",
       "      <th>...</th>\n",
       "      <th>val_VAL_jac</th>\n",
       "      <th>val_VAL_acc_1</th>\n",
       "      <th>test_loss</th>\n",
       "      <th>test_ATT_loss</th>\n",
       "      <th>test_VAL_loss</th>\n",
       "      <th>test_ATT_acc</th>\n",
       "      <th>test_VAL_acc</th>\n",
       "      <th>test_VAL_jac</th>\n",
       "      <th>test_VAL_acc_1</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <th>2</th>\n",
       "      <th>sum</th>\n",
       "      <th>128</th>\n",
       "      <th>0.0005</th>\n",
       "      <td>True</td>\n",
       "      <td>100</td>\n",
       "      <td>90.503597</td>\n",
       "      <td>92.861565</td>\n",
       "      <td>0</td>\n",
       "      <td>4.188147</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>276</td>\n",
       "      <td>[3.8720219135284424, 3.336054523785909, 3.0626...</td>\n",
       "      <td>[1.72003666383738, 1.4708360638314668, 1.24814...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.2638441987180076, 0.30983068525232427, 0.33...</td>\n",
       "      <td>[52.62175713974275, 43.20383693045563, 42.1233...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HGT/model.pth</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   stop_early early_stopping_step  \\\n",
       "4 2 sum 128 0.0005       True                 100   \n",
       "\n",
       "                   early_stopping_best_ATT_acc_val  \\\n",
       "4 2 sum 128 0.0005                       90.503597   \n",
       "\n",
       "                   early_stopping_best_VAL_acc_val  \\\n",
       "4 2 sum 128 0.0005                       92.861565   \n",
       "\n",
       "                   early_stopping_best_ATT_acc_val_2  \\\n",
       "4 2 sum 128 0.0005                                 0   \n",
       "\n",
       "                   early_stopping_lowest_loss learning_rate epoch_index  \\\n",
       "4 2 sum 128 0.0005                   4.188147        0.0005         276   \n",
       "\n",
       "                                                           train_loss  \\\n",
       "4 2 sum 128 0.0005  [3.8720219135284424, 3.336054523785909, 3.0626...   \n",
       "\n",
       "                                                       train_ATT_loss  ...  \\\n",
       "4 2 sum 128 0.0005  [1.72003666383738, 1.4708360638314668, 1.24814...  ...   \n",
       "\n",
       "                                                          val_VAL_jac  \\\n",
       "4 2 sum 128 0.0005  [0.2638441987180076, 0.30983068525232427, 0.33...   \n",
       "\n",
       "                                                        val_VAL_acc_1  \\\n",
       "4 2 sum 128 0.0005  [52.62175713974275, 43.20383693045563, 42.1233...   \n",
       "\n",
       "                   test_loss test_ATT_loss test_VAL_loss test_ATT_acc  \\\n",
       "4 2 sum 128 0.0005        -1            -1            -1           -1   \n",
       "\n",
       "                   test_VAL_acc test_VAL_jac test_VAL_acc_1  \\\n",
       "4 2 sum 128 0.0005           -1           -1             -1   \n",
       "\n",
       "                                 model_filename  \n",
       "4 2 sum 128 0.0005  model_storage/HGT/model.pth  \n",
       "\n",
       "[1 rows x 30 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hyper_df[(hyper_df.early_stopping_lowest_loss<4.19)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Re-run Model and get Inference Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HGT_L(torch.nn.Module):\n",
    "    def __init__(self, metadata, hidden_channels, out_channels, num_heads, num_layers, group='sum'):\n",
    "        super().__init__()\n",
    "\n",
    "        self.lin_dict = torch.nn.ModuleDict()\n",
    "        \n",
    "        self.lin_dict['all'] = Linear(-1, hidden_channels)\n",
    "\n",
    "        self.convs = torch.nn.ModuleList()\n",
    "        \n",
    "        metadata = (metadata[0], metadata[-1][:-1])\n",
    "        \n",
    "        for _ in range(num_layers):\n",
    "            conv = HGTConv(hidden_channels, hidden_channels, metadata,\n",
    "                           num_heads, group=group)\n",
    "            self.convs.append(conv)\n",
    "        \n",
    "        self.lin1 = Linear(-1, hidden_channels)\n",
    "        self.lin2 = Linear(2*hidden_channels, out_channels)\n",
    "\n",
    "\n",
    "    def forward(self, x_dict, edge_index_dict):\n",
    "        \n",
    "        x_0 = self.lin1(x_dict['all']).relu()\n",
    "        \n",
    "        x_dict = {\n",
    "            node_type: self.lin_dict[node_type](x).relu_()\n",
    "            for node_type, x in x_dict.items() if node_type=='all'\n",
    "        }\n",
    "\n",
    "        edge_index_dict = {key: edge_index_dict[key] for key in edge_index_dict if 'all' in key and not 'simp_link' in key}\n",
    "                           \n",
    "        for conv in self.convs:\n",
    "            x_dict = conv(x_dict, edge_index_dict)\n",
    "            \n",
    "        x = self.lin2(torch.hstack([x_0,x_dict['all']]))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def init_params(model, train_loader):\n",
    "    # Initialize lazy parameters via forwarding a single batch to the model:\n",
    "    batch = next(iter(train_loader))\n",
    "    batch = batch.to(device)\n",
    "    batch = batch.to(device, 'edge_index')\n",
    "    out = model(batch.x_dict, batch.edge_index_dict)\n",
    "\n",
    "def train(model, optimizer, train_loader):\n",
    "    model.train()\n",
    "\n",
    "    total_examples = total_loss = 0\n",
    "    for batch in tqdm(train_loader):\n",
    "        optimizer.zero_grad()\n",
    "        batch = batch.to(device)\n",
    "        batch_size = args.batch_size\n",
    "        new_dict = {}\n",
    "        for edge_type in [edge_type for edge_type in batch.edge_index_dict if 'all' in edge_type and not 'simp_link' in edge_type]:\n",
    "            edge_index = batch.edge_index_dict[edge_type]\n",
    "            edge_index = to_undirected(edge_index)\n",
    "            new_dict[edge_type] = edge_index\n",
    "        batch.edge_index_dict = new_dict\n",
    "        out = model(batch.x_dict, batch.edge_index_dict)[:batch_size]\n",
    "        out_att = out[:,:9]\n",
    "        out_val = out[:,9:]\n",
    "        y = batch.y_dict['all']\n",
    "        y_att = y[:,:9]\n",
    "        y_val = y[:,9:]\n",
    "        \n",
    "        loss = F.cross_entropy(out_att, y_att[:batch_size]) + F.cross_entropy(out_val, y_val[:batch_size])\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        total_examples += batch_size\n",
    "        total_loss += float(loss) * batch_size\n",
    "\n",
    "    return total_loss / total_examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def test(model, loader, mode='train'):\n",
    "    model.eval()\n",
    "\n",
    "    total_examples_att = total_examples_val = 0\n",
    "    running_loss_1 = running_loss_2 = 0.\n",
    "    running_1_acc = 0.\n",
    "    running_k_acc = 0.\n",
    "    running_k_jac = 0.\n",
    "    running_1_val = 0.\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        loss_1 = 0\n",
    "        acc_1_t = 0\n",
    "        loss_2 = 0\n",
    "        acc_1_val = 0\n",
    "        acc_k_t = 0\n",
    "        jac_k_t = 0\n",
    "\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch['all'].batch_size\n",
    "        new_dict = {}\n",
    "        for edge_type in [edge_type for edge_type in batch.edge_index_dict if 'all' in edge_type and not 'simp_link' in edge_type]:\n",
    "            edge_index = batch.edge_index_dict[edge_type]\n",
    "            edge_index = to_undirected(edge_index)\n",
    "            new_dict[edge_type] = edge_index\n",
    "        batch.edge_index_dict = new_dict\n",
    "\n",
    "        out = model(batch.x_dict, batch.edge_index_dict)[:batch_size]\n",
    "        out_att = out[:,:9]\n",
    "        out_val = out[:,9:]\n",
    "        att_node = (batch['all'].att_lab[:batch_size]).nonzero().squeeze()\n",
    "        val_node = (batch['all'].val_lab[:batch_size]).nonzero().squeeze()\n",
    "\n",
    "        #print(type_node)\n",
    "\n",
    "        #pred_att = out_att.argmax(dim=-1)\n",
    "        #pred_val = out_val.argmax(dim=-1)\n",
    "\n",
    "        y = batch.y_dict['all']\n",
    "        y_att = y[:,:9]\n",
    "        y_val = y[:,9:]\n",
    "\n",
    "        if not att_node.shape[0]==0:\n",
    "            loss_1 = F.cross_entropy(out_att[att_node], y_att[:batch_size][att_node])\n",
    "            acc_1_t = compute_1_accuracy(y_att[:batch_size][att_node], out_att[att_node])\n",
    "\n",
    "        if not val_node.shape[0]==0:\n",
    "            loss_2 = F.cross_entropy(out_val[val_node], y_val[val_node])\n",
    "            acc_1_val = compute_1_accuracy(y_val[val_node], out_val[val_node])\n",
    "            acc_k_t = compute_k_accuracy(y_val[val_node], out_val[val_node], args.k)\n",
    "            jac_k_t = compute_jaccard_index(y_val[val_node], F.softmax(out_val[val_node],dim=-1), args.k)\n",
    "            #loss_3 = loss_1 + loss_2\n",
    "\n",
    "        total_examples_att += att_node.shape[0]\n",
    "        total_examples_val += val_node.shape[0]\n",
    "        #total_correct_att += int((pred_att == y_att[:batch_size]).sum())\n",
    "        #total_correct_val += int((pred_val == y_val[:batch_size]).sum())\n",
    "\n",
    "        running_loss_1 += float(loss_1) * att_node.shape[0]\n",
    "        running_loss_2 += float(loss_2) * val_node.shape[0]\n",
    "        running_1_acc += float(acc_1_t) * att_node.shape[0]\n",
    "        running_1_val += float(acc_1_val) * val_node.shape[0]\n",
    "        running_k_acc += float(acc_k_t) * val_node.shape[0]\n",
    "        running_k_jac += float(jac_k_t) * val_node.shape[0]\n",
    "\n",
    "    return running_loss_1/total_examples_att, running_loss_2/total_examples_val, running_1_acc/ total_examples_att, running_k_acc/ total_examples_val, running_k_jac/ total_examples_val, running_1_val/total_examples_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialization():\n",
    "    set_seed_everywhere(args.seed, args.cuda)\n",
    "    #transform = T.Compose([T.ToSparseTensor()])\n",
    "    dataset = VEN_links('dataset/Venice_links')\n",
    "    data = dataset[0].to(device)\n",
    "    \n",
    "    train_loader = NeighborLoader(\n",
    "        data,\n",
    "        # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "        num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "        # Use a batch size of 128 for sampling training nodes of type paper\n",
    "        batch_size=args.batch_size,\n",
    "        input_nodes=('all', data['all'].train_mask),\n",
    "    )\n",
    "    val_loader = NeighborLoader(\n",
    "        data,\n",
    "        # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "        num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "        # Use a batch size of 128 for sampling training nodes of type paper\n",
    "        batch_size=args.batch_size,\n",
    "        input_nodes=('all', data['all'].val_mask),\n",
    "    )\n",
    "    test_loader = NeighborLoader(\n",
    "        data,\n",
    "        # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "        num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "        # Use a batch size of 128 for sampling training nodes of type paper\n",
    "        batch_size=args.batch_size,\n",
    "        input_nodes=('all', data['all'].test_mask),\n",
    "    )\n",
    " \n",
    "    model = HGT_L(data.metadata(), hidden_channels=32, out_channels=data.y_dict['all'].shape[-1],\n",
    "                  num_layers=3, num_heads = 2, group='mean').to(device)\n",
    "    return data, model, train_loader, val_loader, test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_loop(verbose=False):\n",
    "    \n",
    "    _, model, train_loader, val_loader, test_loader = initialization()\n",
    "    if torch.cuda.device_count() > 1:\n",
    "        print(\"Use {} GPUs !\".format(torch.cuda.device_count()))\n",
    "        model = DataParallel(model)\n",
    "    \n",
    "    init_params(model, train_loader)\n",
    "    optimizer = optim.Adam(model.parameters(), lr=0.0005, weight_decay=args.l2)\n",
    "    #scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer,\n",
    "    #                                           mode='min', factor=0.5,\n",
    "    #                                           patience=1)\n",
    "\n",
    "    train_state = make_train_state(args)\n",
    "\n",
    "    try:\n",
    "        for epoch in range(args.num_epochs):\n",
    "            train_state['epoch_index'] = epoch\n",
    "            \n",
    "            loss = train(model, optimizer, train_loader)\n",
    "            train_loss_att, train_loss_val, train_att_acc, train_val_acc, train_val_jac, train_val_1 = test(model, train_loader)\n",
    "            val_loss_att, val_loss_val, val_att_acc, val_val_acc, val_val_jac, val_val_1 = test(model, val_loader, 'val')\n",
    "            if verbose:\n",
    "                print(f'Epoch: {epoch:02d}, Loss: {loss:.4f}, Train_ATT: {train_att_acc:.4f}, Train_VAL: {train_val_acc:.4f}, Val_vis_tex_ATT: {val_att_acc:.4f}, Val_vis_tex_VAL: {val_val_acc:.4f}')\n",
    "            \n",
    "            train_state['train_loss'].append(loss)\n",
    "            train_state['train_ATT_loss'].append(train_loss_att)\n",
    "            train_state['train_VAL_loss'].append(train_loss_val)\n",
    "            train_state['train_ATT_acc'].append(train_att_acc)\n",
    "            train_state['train_VAL_acc'].append(train_val_acc)\n",
    "            train_state['train_VAL_jac'].append(train_val_jac)\n",
    "            train_state['train_VAL_acc_1'].append(train_val_1)\n",
    "            \n",
    "            train_state['val_ATT_loss'].append(val_loss_att)\n",
    "            train_state['val_VAL_loss'].append(val_loss_val)\n",
    "            train_state['val_loss'].append(val_loss_att + 3*val_loss_val)\n",
    "            train_state['val_ATT_acc'].append(val_att_acc)\n",
    "            train_state['val_VAL_acc'].append(val_val_acc)\n",
    "            train_state['val_VAL_jac'].append(val_val_jac)\n",
    "            train_state['val_VAL_acc_1'].append(val_val_1)\n",
    "            \n",
    "            train_state = update_train_state(args=args, model=model,\n",
    "                                                train_state=train_state)\n",
    "            if train_state['stop_early']:\n",
    "                break\n",
    "\n",
    "    except KeyboardInterrupt:\n",
    "        print(\"Exiting loop\")\n",
    "        pass\n",
    "    \n",
    "    return train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_state = training_loop(verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = HGT_L(data.metadata(), hidden_channels=32, out_channels=data.y_dict['all'].shape[-1],\n",
    "                  num_layers=3, num_heads = 2, group='mean').to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load(args.save_dir+'HGT_best_model/model.pth',map_location=torch.device('cpu')))\n",
    "model.eval()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HGT_L(\n",
       "  (lin_dict): ModuleDict(\n",
       "    (all): Linear(1753, 32, bias=True)\n",
       "  )\n",
       "  (convs): ModuleList(\n",
       "    (0): HGTConv(-1, 32, heads=2)\n",
       "    (1): HGTConv(-1, 32, heads=2)\n",
       "    (2): HGTConv(-1, 32, heads=2)\n",
       "  )\n",
       "  (lin1): Linear(1753, 32, bias=True)\n",
       "  (lin2): Linear(64, 20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.38it/s]\n"
     ]
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test_loss_att, test_loss_val, test_att_acc, test_val_acc, test_val_jac, test_val_1 = test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 30.49it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.725351293496478,\n",
       " 1.5993330835305422,\n",
       " 100.0,\n",
       " 100.0,\n",
       " 0.872576187852347,\n",
       " 90.30470914127424)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 32.42it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8408805747342304,\n",
       " 1.6251420863156247,\n",
       " 96.95121951219512,\n",
       " 98.0295566502463,\n",
       " 0.738095243576125,\n",
       " 79.3103448275862)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, val_loader, 'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.41it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8526318166886362,\n",
       " 1.6444950339694817,\n",
       " 96.42147117296223,\n",
       " 99.47916666666667,\n",
       " 0.740451397995154,\n",
       " 78.64583333333333)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, test_loader, 'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 33.50it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 45.86it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 49.36it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 48.92it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 43.77it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 48.67it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 43.82it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 48.17it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 48.00it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 50.45it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 47.17it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 42.41it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 44.14it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 50.58it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 48.30it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 39.44it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 49.82it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 46.69it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 49.32it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 48.08it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test(model, val_loader, 'val'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test(model, test_loader, 'val'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.840882</td>\n",
       "      <td>1.625143</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.738095</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.497956e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.840878</td>\n",
       "      <td>1.625140</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.738095</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.840879</td>\n",
       "      <td>1.625142</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.738095</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.840882</td>\n",
       "      <td>1.625143</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.738095</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.840883</td>\n",
       "      <td>1.625144</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.738095</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.840884</td>\n",
       "      <td>1.625148</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.738095</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss       ATT_acc     VAL_k_acc  VAL_k_jac  \\\n",
       "count  10.000000  10.000000  1.000000e+01  1.000000e+01  10.000000   \n",
       "mean    0.840882   1.625143  9.695122e+01  9.802956e+01   0.738095   \n",
       "std     0.000002   0.000002  1.497956e-14  1.497956e-14   0.000000   \n",
       "min     0.840878   1.625140  9.695122e+01  9.802956e+01   0.738095   \n",
       "25%     0.840879   1.625142  9.695122e+01  9.802956e+01   0.738095   \n",
       "50%     0.840882   1.625143  9.695122e+01  9.802956e+01   0.738095   \n",
       "75%     0.840883   1.625144  9.695122e+01  9.802956e+01   0.738095   \n",
       "max     0.840884   1.625148  9.695122e+01  9.802956e+01   0.738095   \n",
       "\n",
       "          VAL_1_acc  \n",
       "count  1.000000e+01  \n",
       "mean   7.931034e+01  \n",
       "std    1.497956e-14  \n",
       "min    7.931034e+01  \n",
       "25%    7.931034e+01  \n",
       "50%    7.931034e+01  \n",
       "75%    7.931034e+01  \n",
       "max    7.931034e+01  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.852633</td>\n",
       "      <td>1.644495</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>7.404514e-01</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>1.170278e-16</td>\n",
       "      <td>1.497956e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.852631</td>\n",
       "      <td>1.644491</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>7.404514e-01</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.852633</td>\n",
       "      <td>1.644493</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>7.404514e-01</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.852633</td>\n",
       "      <td>1.644494</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>7.404514e-01</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.852634</td>\n",
       "      <td>1.644497</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>7.404514e-01</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.852636</td>\n",
       "      <td>1.644499</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>7.404514e-01</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc     VAL_k_acc     VAL_k_jac  \\\n",
       "count  10.000000  10.000000  10.000000  1.000000e+01  1.000000e+01   \n",
       "mean    0.852633   1.644495  96.421471  9.947917e+01  7.404514e-01   \n",
       "std     0.000001   0.000003   0.000000  1.497956e-14  1.170278e-16   \n",
       "min     0.852631   1.644491  96.421471  9.947917e+01  7.404514e-01   \n",
       "25%     0.852633   1.644493  96.421471  9.947917e+01  7.404514e-01   \n",
       "50%     0.852633   1.644494  96.421471  9.947917e+01  7.404514e-01   \n",
       "75%     0.852634   1.644497  96.421471  9.947917e+01  7.404514e-01   \n",
       "max     0.852636   1.644499  96.421471  9.947917e+01  7.404514e-01   \n",
       "\n",
       "          VAL_1_acc  \n",
       "count  1.000000e+01  \n",
       "mean   7.864583e+01  \n",
       "std    1.497956e-14  \n",
       "min    7.864583e+01  \n",
       "25%    7.864583e+01  \n",
       "50%    7.864583e+01  \n",
       "75%    7.864583e+01  \n",
       "max    7.864583e+01  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'val_metrics.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'test_metrics.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_state['test_ATT_loss']=test_loss_att\n",
    "train_state['test_VAL_loss']=test_loss_val\n",
    "train_state['test_loss']=test_loss_att + 3*test_loss_val\n",
    "train_state['test_ATT_acc']=test_att_acc\n",
    "train_state['test_VAL_acc_1']=test_val_1\n",
    "train_state['test_VAL_acc']=test_val_acc\n",
    "train_state['test_VAL_jac']=test_val_jac"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': False,\n",
       " 'early_stopping_step': 55,\n",
       " 'early_stopping_best_ATT_acc_val': 96.95121951219512,\n",
       " 'early_stopping_best_VAL_acc_val': 100.0,\n",
       " 'early_stopping_best_ATT_acc_val_2': 0,\n",
       " 'early_stopping_lowest_loss': 5.716309051993083,\n",
       " 'learning_rate': 0.001,\n",
       " 'epoch_index': 299,\n",
       " 'train_loss': [4.212754587332408,\n",
       "  3.744509994983673,\n",
       "  3.4710254867871604,\n",
       "  3.263322631518046,\n",
       "  3.1269689798355103,\n",
       "  2.989909013112386,\n",
       "  2.8954469362894693,\n",
       "  2.8212348222732544,\n",
       "  2.7362569173177085,\n",
       "  2.674421548843384,\n",
       "  2.6485327084859214,\n",
       "  2.5981449484825134,\n",
       "  2.576626718044281,\n",
       "  2.550933837890625,\n",
       "  2.535131653149923,\n",
       "  2.484558125336965,\n",
       "  2.4908601442972818,\n",
       "  2.476132074991862,\n",
       "  2.4557361006736755,\n",
       "  2.4715222318967185,\n",
       "  2.4546250303586326,\n",
       "  2.4369125167528787,\n",
       "  2.4401920239130654,\n",
       "  2.4267801443735757,\n",
       "  2.422055800755819,\n",
       "  2.4224125146865845,\n",
       "  2.4179813067118325,\n",
       "  2.4229325453440347,\n",
       "  2.4207166830698648,\n",
       "  2.404494285583496,\n",
       "  2.405641794204712,\n",
       "  2.4137187401453652,\n",
       "  2.4108275373776755,\n",
       "  2.3856456875801086,\n",
       "  2.4094346165657043,\n",
       "  2.406758944193522,\n",
       "  2.4062544107437134,\n",
       "  2.4073506196339927,\n",
       "  2.3848799069722495,\n",
       "  2.3935265938440957,\n",
       "  2.3904134035110474,\n",
       "  2.3775036931037903,\n",
       "  2.3968611558278403,\n",
       "  2.3905741373697915,\n",
       "  2.4076212644577026,\n",
       "  2.386153817176819,\n",
       "  2.3838980992635093,\n",
       "  2.389664272467295,\n",
       "  2.3803069988886514,\n",
       "  2.3829084634780884,\n",
       "  2.3663163582483926,\n",
       "  2.38978111743927,\n",
       "  2.3866490920384726,\n",
       "  2.3895376125971475,\n",
       "  2.3759404023488364,\n",
       "  2.3816418250401816,\n",
       "  2.375000258286794,\n",
       "  2.365959664185842,\n",
       "  2.3773799737294516,\n",
       "  2.3851634860038757,\n",
       "  2.3808287382125854,\n",
       "  2.3740285833676658,\n",
       "  2.3576398690541587,\n",
       "  2.369821528593699,\n",
       "  2.370358169078827,\n",
       "  2.3829912344614663,\n",
       "  2.369228482246399,\n",
       "  2.3756791949272156,\n",
       "  2.3709821303685508,\n",
       "  2.3732546965281167,\n",
       "  2.381136973698934,\n",
       "  2.3519471685091653,\n",
       "  2.3688732584317527,\n",
       "  2.3717807133992515,\n",
       "  2.362366239229838,\n",
       "  2.3591893712679544,\n",
       "  2.357397178808848,\n",
       "  2.3710108598073325,\n",
       "  2.3500359058380127,\n",
       "  2.376711130142212,\n",
       "  2.3509661157925925,\n",
       "  2.362232426802317,\n",
       "  2.349884112675985,\n",
       "  2.3621522585550943,\n",
       "  2.3759855031967163,\n",
       "  2.3620211482048035,\n",
       "  2.358452637990316,\n",
       "  2.3501272598902383,\n",
       "  2.365586002667745,\n",
       "  2.360858937104543,\n",
       "  2.3614619970321655,\n",
       "  2.3498252232869468,\n",
       "  2.3578995068868003,\n",
       "  2.3472864230473838,\n",
       "  2.367964188257853,\n",
       "  2.363624354203542,\n",
       "  2.357446094353994,\n",
       "  2.362629532814026,\n",
       "  2.363881528377533,\n",
       "  2.367944320042928,\n",
       "  2.343201518058777,\n",
       "  2.3689687649408975,\n",
       "  2.3411059776941934,\n",
       "  2.348759651184082,\n",
       "  2.3659634590148926,\n",
       "  2.3593635161717734,\n",
       "  2.362978160381317,\n",
       "  2.362177928288778,\n",
       "  2.3488836089769998,\n",
       "  2.3406652013460794,\n",
       "  2.3452983299891152,\n",
       "  2.3503600557645163,\n",
       "  2.3508934378623962,\n",
       "  2.348996182282766,\n",
       "  2.369202673435211,\n",
       "  2.3639399210611978,\n",
       "  2.346254030863444,\n",
       "  2.338806966940562,\n",
       "  2.374669154485067,\n",
       "  2.3546056151390076,\n",
       "  2.3466368913650513,\n",
       "  2.3508933782577515,\n",
       "  2.357291559378306,\n",
       "  2.3650964498519897,\n",
       "  2.365654468536377,\n",
       "  2.3643471797307334,\n",
       "  2.3507268031438193,\n",
       "  2.3511754473050437,\n",
       "  2.3601104021072388,\n",
       "  2.344610035419464,\n",
       "  2.3571850260098777,\n",
       "  2.3505373001098633,\n",
       "  2.353857080141703,\n",
       "  2.34756871064504,\n",
       "  2.3511706590652466,\n",
       "  2.3492911060651145,\n",
       "  2.3498776157697043,\n",
       "  2.354318598906199,\n",
       "  2.3642939925193787,\n",
       "  2.341994325319926,\n",
       "  2.333648761113485,\n",
       "  2.3443042238553367,\n",
       "  2.3460735281308494,\n",
       "  2.3484234611193338,\n",
       "  2.3518704771995544,\n",
       "  2.3579606413841248,\n",
       "  2.3519768714904785,\n",
       "  2.3508107662200928,\n",
       "  2.352656821409861,\n",
       "  2.3524768352508545,\n",
       "  2.3491548697153726,\n",
       "  2.3669060269991555,\n",
       "  2.359908481438955,\n",
       "  2.3599456747372947,\n",
       "  2.34403787056605,\n",
       "  2.3450807332992554,\n",
       "  2.3444391886393228,\n",
       "  2.3474087516466775,\n",
       "  2.3497478564580283,\n",
       "  2.3577657341957092,\n",
       "  2.3491291403770447,\n",
       "  2.347566326459249,\n",
       "  2.3513152996699014,\n",
       "  2.3244027892748513,\n",
       "  2.3337082465489707,\n",
       "  2.343714455763499,\n",
       "  2.3473713994026184,\n",
       "  2.3553279439608255,\n",
       "  2.346401254336039,\n",
       "  2.3440206249554953,\n",
       "  2.359449883302053,\n",
       "  2.351641595363617,\n",
       "  2.358983834584554,\n",
       "  2.344243307908376,\n",
       "  2.3482866287231445,\n",
       "  2.3505460222562156,\n",
       "  2.361703554789225,\n",
       "  2.3473101456960044,\n",
       "  2.3717046976089478,\n",
       "  2.3561343351999917,\n",
       "  2.3647903203964233,\n",
       "  2.3589191834131875,\n",
       "  2.361755092938741,\n",
       "  2.367590347925822,\n",
       "  2.3586204449335733,\n",
       "  2.3649255832036338,\n",
       "  2.357851763566335,\n",
       "  2.337674101193746,\n",
       "  2.334989289442698,\n",
       "  2.332011659940084,\n",
       "  2.3330437938372293,\n",
       "  2.3505683541297913,\n",
       "  2.3588008681933084,\n",
       "  2.3391478061676025,\n",
       "  2.335266133149465,\n",
       "  2.3422364592552185,\n",
       "  2.34785129626592,\n",
       "  2.352286159992218,\n",
       "  2.348785122235616,\n",
       "  2.343895355860392,\n",
       "  2.3352490663528442,\n",
       "  2.34445987145106,\n",
       "  2.3410446445147195,\n",
       "  2.3357056180636087,\n",
       "  2.3401461044947305,\n",
       "  2.351550837357839,\n",
       "  2.3394398291905723,\n",
       "  2.3300169110298157,\n",
       "  2.3361928860346475,\n",
       "  2.346657911936442,\n",
       "  2.346759259700775,\n",
       "  2.3463246623675027,\n",
       "  2.3523252805074057,\n",
       "  2.345177471637726,\n",
       "  2.3353580435117087,\n",
       "  2.3378560543060303,\n",
       "  2.337415258089701,\n",
       "  2.3452074925104776,\n",
       "  2.3243465224901834,\n",
       "  2.351166566212972,\n",
       "  2.3297932147979736,\n",
       "  2.3511283795038858,\n",
       "  2.34217498699824,\n",
       "  2.337269345919291,\n",
       "  2.349517742792765,\n",
       "  2.3423258463541665,\n",
       "  2.344554603099823,\n",
       "  2.3304055531819663,\n",
       "  2.3330251177152,\n",
       "  2.3499587972958884,\n",
       "  2.3367531100908914,\n",
       "  2.341305692990621,\n",
       "  2.3527392943700156,\n",
       "  2.3433781464894614,\n",
       "  2.338826755682627,\n",
       "  2.3447272181510925,\n",
       "  2.3433652917544046,\n",
       "  2.3477142254511514,\n",
       "  2.3489569624265036,\n",
       "  2.3471329609553018,\n",
       "  2.333854019641876,\n",
       "  2.3313928047815957,\n",
       "  2.347913404305776,\n",
       "  2.33931036790212,\n",
       "  2.3428726394971213,\n",
       "  2.3403084675470986,\n",
       "  2.3460219899813333,\n",
       "  2.345427393913269,\n",
       "  2.341270923614502,\n",
       "  2.346359054247538,\n",
       "  2.344762146472931,\n",
       "  2.331865668296814,\n",
       "  2.344398339589437,\n",
       "  2.319320499897003,\n",
       "  2.347334643205007,\n",
       "  2.33759476741155,\n",
       "  2.3323342402776084,\n",
       "  2.3492140571276345,\n",
       "  2.3465505242347717,\n",
       "  2.3188069661458335,\n",
       "  2.3441200852394104,\n",
       "  2.3285131255785623,\n",
       "  2.3323670427004495,\n",
       "  2.31796803077062,\n",
       "  2.327207068602244,\n",
       "  2.321818232536316,\n",
       "  2.3293274442354837,\n",
       "  2.3387466073036194,\n",
       "  2.3333200017611184,\n",
       "  2.3300774693489075,\n",
       "  2.3380286494890847,\n",
       "  2.3365246852238974,\n",
       "  2.3469033439954123,\n",
       "  2.333066165447235,\n",
       "  2.350557724634806,\n",
       "  2.3417248129844666,\n",
       "  2.344898442427317,\n",
       "  2.3511628905932107,\n",
       "  2.3355544209480286,\n",
       "  2.3335918188095093,\n",
       "  2.3326813777287803,\n",
       "  2.3499688704808555,\n",
       "  2.3349210619926453,\n",
       "  2.3357057174046836,\n",
       "  2.3434485594431558,\n",
       "  2.3388755520184836,\n",
       "  2.350087285041809,\n",
       "  2.333504319190979,\n",
       "  2.3510329524676004,\n",
       "  2.334171732266744,\n",
       "  2.342860201994578,\n",
       "  2.345601042111715,\n",
       "  2.3313573598861694,\n",
       "  2.3268946607907615,\n",
       "  2.3433027267456055,\n",
       "  2.3462418913841248,\n",
       "  2.3399335741996765,\n",
       "  2.3453603386878967,\n",
       "  2.3375792304674783,\n",
       "  2.344885766506195],\n",
       " 'train_ATT_loss': [1.9179202255449797,\n",
       "  1.730461289347704,\n",
       "  1.5928260332329451,\n",
       "  1.458375739588962,\n",
       "  1.3386530506313672,\n",
       "  1.2377030865967769,\n",
       "  1.1513480047109714,\n",
       "  1.079639149834905,\n",
       "  1.0202431622634633,\n",
       "  0.9738469384713847,\n",
       "  0.9359683372967791,\n",
       "  0.9061746471806577,\n",
       "  0.8813384467544978,\n",
       "  0.8609489341191637,\n",
       "  0.8441518796117682,\n",
       "  0.8309625274917095,\n",
       "  0.8202618528931425,\n",
       "  0.8115657110623705,\n",
       "  0.804042853972258,\n",
       "  0.7989996673327734,\n",
       "  0.7930962251824355,\n",
       "  0.7895156330018823,\n",
       "  0.7849255712738988,\n",
       "  0.7812670084578178,\n",
       "  0.778695235952446,\n",
       "  0.7754131677738517,\n",
       "  0.7730409805134063,\n",
       "  0.7710750500911491,\n",
       "  0.7684160901899153,\n",
       "  0.7667722418037478,\n",
       "  0.7647889530559656,\n",
       "  0.7639658068025541,\n",
       "  0.7616029541908539,\n",
       "  0.7606888172368924,\n",
       "  0.7588537282560671,\n",
       "  0.7579682967999636,\n",
       "  0.7567136168810139,\n",
       "  0.7558138902167534,\n",
       "  0.7545182222804865,\n",
       "  0.7535423555532651,\n",
       "  0.7532196119221294,\n",
       "  0.7516896462176315,\n",
       "  0.7508227143261241,\n",
       "  0.7501313927431186,\n",
       "  0.7494237041539432,\n",
       "  0.748763811885485,\n",
       "  0.7481039836796367,\n",
       "  0.7471523218868181,\n",
       "  0.7469054467129905,\n",
       "  0.7459406700821134,\n",
       "  0.7460284893532539,\n",
       "  0.7447854906238017,\n",
       "  0.7448807961392601,\n",
       "  0.7437700467426691,\n",
       "  0.7433806228505607,\n",
       "  0.7433063693323955,\n",
       "  0.7423549684130915,\n",
       "  0.7417077177779496,\n",
       "  0.7416324759124059,\n",
       "  0.741514409843244,\n",
       "  0.7409268635792085,\n",
       "  0.7404435770663528,\n",
       "  0.7401373947756442,\n",
       "  0.739652427279718,\n",
       "  0.7396005043692866,\n",
       "  0.7396083247958788,\n",
       "  0.7384689915873668,\n",
       "  0.7383819139895347,\n",
       "  0.7382073990195742,\n",
       "  0.7378254707830434,\n",
       "  0.7377064138898559,\n",
       "  0.7373056353954728,\n",
       "  0.7374693279121061,\n",
       "  0.7366111935671017,\n",
       "  0.7372921865402496,\n",
       "  0.7367331192433999,\n",
       "  0.7358797701111791,\n",
       "  0.7356353770332653,\n",
       "  0.7352226482203793,\n",
       "  0.7351649472918207,\n",
       "  0.735038668495136,\n",
       "  0.7345882073003499,\n",
       "  0.7348598867572246,\n",
       "  0.7348306576961295,\n",
       "  0.7340189396839722,\n",
       "  0.7341829510276667,\n",
       "  0.7337448747204282,\n",
       "  0.7338080957655762,\n",
       "  0.733219530113516,\n",
       "  0.7330381290734309,\n",
       "  0.7329260221478682,\n",
       "  0.7333715371477967,\n",
       "  0.7326746637471164,\n",
       "  0.732329926497388,\n",
       "  0.7322432899078835,\n",
       "  0.7323843037652837,\n",
       "  0.7324475479258065,\n",
       "  0.7319101126570451,\n",
       "  0.7318413109000039,\n",
       "  0.7318213354187328,\n",
       "  0.7319731535673802,\n",
       "  0.7316640300103502,\n",
       "  0.7313063329606836,\n",
       "  0.7311777614490478,\n",
       "  0.7311765611006612,\n",
       "  0.7307195658498854,\n",
       "  0.7306853207524794,\n",
       "  0.7313897723636469,\n",
       "  0.7312008736866663,\n",
       "  0.7312175790028559,\n",
       "  0.7313037831697438,\n",
       "  0.730939183043641,\n",
       "  0.7309892060353815,\n",
       "  0.7299540967492185,\n",
       "  0.7301046177951253,\n",
       "  0.730229613523404,\n",
       "  0.7294393221427199,\n",
       "  0.7294285867022675,\n",
       "  0.7297390324587307,\n",
       "  0.7298888174450628,\n",
       "  0.7299711225435674,\n",
       "  0.729981311305408,\n",
       "  0.7295424441221348,\n",
       "  0.7295650902547335,\n",
       "  0.7292750157477783,\n",
       "  0.7287966258968342,\n",
       "  0.7294444856220995,\n",
       "  0.7298087439708763,\n",
       "  0.7300669045659643,\n",
       "  0.7293339408005374,\n",
       "  0.7295752705959733,\n",
       "  0.7314005081343189,\n",
       "  0.7334380661681749,\n",
       "  0.7303090329975963,\n",
       "  0.7312713210932766,\n",
       "  0.7335147221993211,\n",
       "  0.7328971355245384,\n",
       "  0.7331521866063992,\n",
       "  0.7334098611181793,\n",
       "  0.7310208103663374,\n",
       "  0.7305506563582909,\n",
       "  0.7298616137861215,\n",
       "  0.729304235067394,\n",
       "  0.7285008529546848,\n",
       "  0.7280038402021096,\n",
       "  0.7276374222168962,\n",
       "  0.7277264058424825,\n",
       "  0.7293061391138304,\n",
       "  0.7306996696213276,\n",
       "  0.7325081605660287,\n",
       "  0.7299723455119992,\n",
       "  0.7287706788887277,\n",
       "  0.728049054700582,\n",
       "  0.7266982040907207,\n",
       "  0.7265526180122037,\n",
       "  0.7269668742560284,\n",
       "  0.7272258235807234,\n",
       "  0.7288152758764759,\n",
       "  0.7308749866947903,\n",
       "  0.7299640612919245,\n",
       "  0.7308688200081485,\n",
       "  0.7313490934319113,\n",
       "  0.7287349603512941,\n",
       "  0.7287181604271781,\n",
       "  0.7292153987197665,\n",
       "  0.7298712481091888,\n",
       "  0.728649608976623,\n",
       "  0.7271891584686956,\n",
       "  0.7283465307505177,\n",
       "  0.7278014005386269,\n",
       "  0.7267639237427646,\n",
       "  0.726692106750203,\n",
       "  0.7262359916998739,\n",
       "  0.7268212360028085,\n",
       "  0.7277786347014091,\n",
       "  0.7272223798852218,\n",
       "  0.7277878484897666,\n",
       "  0.728595595775879,\n",
       "  0.7265337930161537,\n",
       "  0.7266570613655027,\n",
       "  0.7284535154741557,\n",
       "  0.7317194231989641,\n",
       "  0.7353221869864952,\n",
       "  0.7366005101362424,\n",
       "  0.7307340195634688,\n",
       "  0.7282915840188552,\n",
       "  0.7267332012633537,\n",
       "  0.7272839580876675,\n",
       "  0.7276370214953647,\n",
       "  0.7274112858270344,\n",
       "  0.7263759425802574,\n",
       "  0.7266331466941622,\n",
       "  0.7260539150304081,\n",
       "  0.725207494896865,\n",
       "  0.724815733049715,\n",
       "  0.7257243074538635,\n",
       "  0.7253615596948237,\n",
       "  0.7247233570777809,\n",
       "  0.7249168808110202,\n",
       "  0.7251450079299736,\n",
       "  0.7256074430539667,\n",
       "  0.7253668976952825,\n",
       "  0.7253736269110788,\n",
       "  0.7250275152872143,\n",
       "  0.7249370829880732,\n",
       "  0.7250308387827675,\n",
       "  0.7244548866953546,\n",
       "  0.7243211300749528,\n",
       "  0.7243175988712469,\n",
       "  0.7241013788120239,\n",
       "  0.7240509947251085,\n",
       "  0.724091169246346,\n",
       "  0.7244901044547063,\n",
       "  0.7244291868566476,\n",
       "  0.7247323631579856,\n",
       "  0.725165950954786,\n",
       "  0.7248204226639132,\n",
       "  0.7252216532289817,\n",
       "  0.7255966978720351,\n",
       "  0.726182557704376,\n",
       "  0.7261994104636343,\n",
       "  0.7263639399219418,\n",
       "  0.7260203832074216,\n",
       "  0.7256594110393788,\n",
       "  0.7257163853196226,\n",
       "  0.7254063690137995,\n",
       "  0.7244037300595947,\n",
       "  0.7240909037497565,\n",
       "  0.7242289013809775,\n",
       "  0.7240866600971803,\n",
       "  0.7250823931680822,\n",
       "  0.7258910786081879,\n",
       "  0.7271773993473634,\n",
       "  0.7289754188622134,\n",
       "  0.7308340840392495,\n",
       "  0.730435959685212,\n",
       "  0.7286478016845407,\n",
       "  0.7273535538578297,\n",
       "  0.7247646000246593,\n",
       "  0.7244523918859846,\n",
       "  0.7248735683776665,\n",
       "  0.7258059763842343,\n",
       "  0.7259113562404284,\n",
       "  0.7254609356626579,\n",
       "  0.7253514630642625,\n",
       "  0.7253553015373421,\n",
       "  0.7250284162914984,\n",
       "  0.7242939984038926,\n",
       "  0.7245623936613511,\n",
       "  0.725027663720942,\n",
       "  0.7242815099924885,\n",
       "  0.7243345921389615,\n",
       "  0.7233269792514495,\n",
       "  0.7230261011136866,\n",
       "  0.7229476732890692,\n",
       "  0.7228461135787647,\n",
       "  0.7232182754043727,\n",
       "  0.7231810419843467,\n",
       "  0.7229808852943357,\n",
       "  0.7227529865222625,\n",
       "  0.7229162601552842,\n",
       "  0.7229301848570066,\n",
       "  0.7232122454286612,\n",
       "  0.7229092707264126,\n",
       "  0.7227525258658666,\n",
       "  0.7227545682743316,\n",
       "  0.7227533230160742,\n",
       "  0.723191446709831,\n",
       "  0.7228970379050088,\n",
       "  0.7228648160633288,\n",
       "  0.7228283083009588,\n",
       "  0.7227490575690019,\n",
       "  0.7223939656220645,\n",
       "  0.722583297547211,\n",
       "  0.7229618028920773,\n",
       "  0.7233841089000332,\n",
       "  0.7231277691360326,\n",
       "  0.7229189313019412,\n",
       "  0.7229670651071289,\n",
       "  0.7235180549013978,\n",
       "  0.7243029011253505,\n",
       "  0.7247728122568526,\n",
       "  0.723895517742865,\n",
       "  0.7235657463773796,\n",
       "  0.7234663063469355,\n",
       "  0.7234675978359423,\n",
       "  0.723138042103881,\n",
       "  0.722853322603696,\n",
       "  0.723550601018763,\n",
       "  0.7232185787111108,\n",
       "  0.7230532329498566,\n",
       "  0.7228010889565846,\n",
       "  0.7223732393864449,\n",
       "  0.7227346439771044,\n",
       "  0.7232246545873521,\n",
       "  0.7231852379201852,\n",
       "  0.7249012435902519,\n",
       "  0.7263695072599395,\n",
       "  0.7283258010475919,\n",
       "  0.727797235973654],\n",
       " 'train_VAL_loss': [1.9992596548349904,\n",
       "  1.8534058507459645,\n",
       "  1.7388558962338518,\n",
       "  1.7010312126614051,\n",
       "  1.6883885758075028,\n",
       "  1.679924990661917,\n",
       "  1.6714080076798838,\n",
       "  1.6638898829674127,\n",
       "  1.6586003095489459,\n",
       "  1.6547908191865832,\n",
       "  1.651838499753429,\n",
       "  1.648395194901654,\n",
       "  1.6459720408817407,\n",
       "  1.643724456388204,\n",
       "  1.6409738040696882,\n",
       "  1.6392127057521957,\n",
       "  1.6373842931520246,\n",
       "  1.6356257031829073,\n",
       "  1.6341173642229836,\n",
       "  1.6327492606276621,\n",
       "  1.631828456043867,\n",
       "  1.630595743160829,\n",
       "  1.629612270154451,\n",
       "  1.6285647510491579,\n",
       "  1.6279436112110635,\n",
       "  1.6271286697599037,\n",
       "  1.626262467323578,\n",
       "  1.6257326008540443,\n",
       "  1.6249901284141224,\n",
       "  1.6244672328811602,\n",
       "  1.6239575880716381,\n",
       "  1.623326761240444,\n",
       "  1.6228608289253679,\n",
       "  1.6224600605026838,\n",
       "  1.6220917123836824,\n",
       "  1.6216266937863464,\n",
       "  1.6210879794118147,\n",
       "  1.6207028452379222,\n",
       "  1.6206919252046919,\n",
       "  1.620117875677727,\n",
       "  1.619651531908981,\n",
       "  1.6195250201423412,\n",
       "  1.6188407578296609,\n",
       "  1.618546873248515,\n",
       "  1.6181421956857487,\n",
       "  1.6179768058401727,\n",
       "  1.6180978977118834,\n",
       "  1.6174682322631582,\n",
       "  1.617227462850449,\n",
       "  1.6168072818058679,\n",
       "  1.6166669156742888,\n",
       "  1.6162871412953512,\n",
       "  1.6158557010819707,\n",
       "  1.6154091784168148,\n",
       "  1.615216005211722,\n",
       "  1.61505846138476,\n",
       "  1.6147889377667963,\n",
       "  1.6144255419517157,\n",
       "  1.614319777884972,\n",
       "  1.6139421971550938,\n",
       "  1.6140103399588461,\n",
       "  1.6140014520972719,\n",
       "  1.6136697593488192,\n",
       "  1.613207912841332,\n",
       "  1.6130122507708224,\n",
       "  1.6126371463580145,\n",
       "  1.6123471996460594,\n",
       "  1.6120708170359814,\n",
       "  1.6120115138817361,\n",
       "  1.6117944364072214,\n",
       "  1.6115074121390685,\n",
       "  1.611314772899131,\n",
       "  1.6112440312668228,\n",
       "  1.6111369420286692,\n",
       "  1.611190248064057,\n",
       "  1.610839849363734,\n",
       "  1.610294990592386,\n",
       "  1.6102219463385374,\n",
       "  1.61009021445985,\n",
       "  1.6099268567859302,\n",
       "  1.6097973796469354,\n",
       "  1.6096548542091391,\n",
       "  1.6097782602600774,\n",
       "  1.6094180144101298,\n",
       "  1.6090318809916109,\n",
       "  1.6089959309702104,\n",
       "  1.6090086837224353,\n",
       "  1.6090390487390873,\n",
       "  1.6086879384154429,\n",
       "  1.6083208877294017,\n",
       "  1.6083139663257757,\n",
       "  1.6083846425745956,\n",
       "  1.608005747240336,\n",
       "  1.6076641561582148,\n",
       "  1.6074991533300553,\n",
       "  1.6075157587548041,\n",
       "  1.6074194419417025,\n",
       "  1.6072567869751737,\n",
       "  1.6072981730722655,\n",
       "  1.6072697167251249,\n",
       "  1.6071526357011452,\n",
       "  1.606747054987667,\n",
       "  1.6066802320718105,\n",
       "  1.6066008267970626,\n",
       "  1.6064962640693643,\n",
       "  1.6064004544736277,\n",
       "  1.6063189849959185,\n",
       "  1.60650718839545,\n",
       "  1.6063628216529486,\n",
       "  1.606541355891241,\n",
       "  1.6065393805173624,\n",
       "  1.6062583018538035,\n",
       "  1.6057409041476052,\n",
       "  1.6054127239454485,\n",
       "  1.6056565779398022,\n",
       "  1.605260570623868,\n",
       "  1.6053867974109597,\n",
       "  1.605335146106181,\n",
       "  1.6051832021438515,\n",
       "  1.6052122829363287,\n",
       "  1.6048694680602267,\n",
       "  1.6050508841253053,\n",
       "  1.6047854479660288,\n",
       "  1.6050042987200033,\n",
       "  1.6047170030113072,\n",
       "  1.604717291953491,\n",
       "  1.6044048394522838,\n",
       "  1.6042299257421098,\n",
       "  1.604115958028883,\n",
       "  1.6040911591944602,\n",
       "  1.6044611452028692,\n",
       "  1.6045181533306259,\n",
       "  1.6045875404019765,\n",
       "  1.6046258273877596,\n",
       "  1.604597869011834,\n",
       "  1.6041454638140353,\n",
       "  1.6039174451722333,\n",
       "  1.6037487088808393,\n",
       "  1.6036420700622727,\n",
       "  1.603789371136483,\n",
       "  1.6038857097440808,\n",
       "  1.603786821510653,\n",
       "  1.6040118787427358,\n",
       "  1.6044892511869733,\n",
       "  1.6060641921458152,\n",
       "  1.608543578607554,\n",
       "  1.611085522538077,\n",
       "  1.6137675093151527,\n",
       "  1.61417878400586,\n",
       "  1.6161708689792664,\n",
       "  1.6156755586740383,\n",
       "  1.6126132090666287,\n",
       "  1.6096852089889822,\n",
       "  1.6087332214344903,\n",
       "  1.6071577511335675,\n",
       "  1.6078454434376344,\n",
       "  1.6073288204267084,\n",
       "  1.6065754626266184,\n",
       "  1.6063312265681429,\n",
       "  1.6053981856956376,\n",
       "  1.6056417372088023,\n",
       "  1.606915128858466,\n",
       "  1.6069442294641214,\n",
       "  1.6089312221204806,\n",
       "  1.609852678888062,\n",
       "  1.6098707161451642,\n",
       "  1.61150603083032,\n",
       "  1.6130699972696911,\n",
       "  1.6151066202866404,\n",
       "  1.6169089584139245,\n",
       "  1.6172729397084245,\n",
       "  1.6173427603581605,\n",
       "  1.6160343049966066,\n",
       "  1.612284115146732,\n",
       "  1.6094975514425136,\n",
       "  1.6061756845326305,\n",
       "  1.6041404018771945,\n",
       "  1.6027706755825688,\n",
       "  1.6037482700189396,\n",
       "  1.6087133933962878,\n",
       "  1.619743457461328,\n",
       "  1.6328416537710173,\n",
       "  1.6341438498193208,\n",
       "  1.6206792695370407,\n",
       "  1.6084681356382502,\n",
       "  1.6065308542462928,\n",
       "  1.6075476817477112,\n",
       "  1.6066479048900657,\n",
       "  1.6053379480198149,\n",
       "  1.6051729359455056,\n",
       "  1.605408986849798,\n",
       "  1.604943660157539,\n",
       "  1.6034283278061081,\n",
       "  1.6024556625582835,\n",
       "  1.6011616503432846,\n",
       "  1.6004272438482565,\n",
       "  1.6002743848473082,\n",
       "  1.6002790551436574,\n",
       "  1.600327986099053,\n",
       "  1.6004347883763406,\n",
       "  1.6010049195170732,\n",
       "  1.6012626608983302,\n",
       "  1.6014884449438376,\n",
       "  1.6019439046732937,\n",
       "  1.601637015052119,\n",
       "  1.6021256344470292,\n",
       "  1.6027783888528881,\n",
       "  1.6022291199982661,\n",
       "  1.6011646738342962,\n",
       "  1.6000430045695846,\n",
       "  1.5996969218399386,\n",
       "  1.5995519055554082,\n",
       "  1.5994314206273932,\n",
       "  1.5992989807578004,\n",
       "  1.599387977922392,\n",
       "  1.5994149687547763,\n",
       "  1.5994470522344277,\n",
       "  1.5995644162566378,\n",
       "  1.5998264666739592,\n",
       "  1.6001793813177092,\n",
       "  1.6005915816140637,\n",
       "  1.600491642291526,\n",
       "  1.6004423147423446,\n",
       "  1.6010350226695518,\n",
       "  1.6009953487612865,\n",
       "  1.6004490148988126,\n",
       "  1.6003082359926852,\n",
       "  1.5997761270015853,\n",
       "  1.59956679945177,\n",
       "  1.59912045411456,\n",
       "  1.5987512243091235,\n",
       "  1.5987248787259132,\n",
       "  1.5988383312965033,\n",
       "  1.5993773124885031,\n",
       "  1.5999603538962281,\n",
       "  1.6004200097266326,\n",
       "  1.6012238244270685,\n",
       "  1.602163992760254,\n",
       "  1.6025725078054411,\n",
       "  1.6024831143772833,\n",
       "  1.6027407811289018,\n",
       "  1.601814528911728,\n",
       "  1.6007651229314197,\n",
       "  1.6000460359858675,\n",
       "  1.5993338757274553,\n",
       "  1.5987264010384472,\n",
       "  1.5983191272888817,\n",
       "  1.5985795464211885,\n",
       "  1.5993219085677508,\n",
       "  1.5998107990729842,\n",
       "  1.6002040900021708,\n",
       "  1.600156125269438,\n",
       "  1.5995411585572683,\n",
       "  1.5995563995144704,\n",
       "  1.5993241722233738,\n",
       "  1.5988884738277531,\n",
       "  1.5985087176108954,\n",
       "  1.5982397005498574,\n",
       "  1.5980153341372587,\n",
       "  1.5978941293304316,\n",
       "  1.5977841123649619,\n",
       "  1.5978419229264404,\n",
       "  1.597835879246614,\n",
       "  1.5980548861944774,\n",
       "  1.5981458484961386,\n",
       "  1.598198971259627,\n",
       "  1.59834548409956,\n",
       "  1.5987089575162554,\n",
       "  1.5989391691466779,\n",
       "  1.5990638115399431,\n",
       "  1.5986931016900863,\n",
       "  1.5985479856792248,\n",
       "  1.598650830604363,\n",
       "  1.5988552722244052,\n",
       "  1.5984564606832996,\n",
       "  1.5983275670424062,\n",
       "  1.5981219571052827,\n",
       "  1.5980900321310576,\n",
       "  1.5983006105528643,\n",
       "  1.5985442499044529,\n",
       "  1.5983220352030196,\n",
       "  1.5981387626431325,\n",
       "  1.5978812808145115,\n",
       "  1.5975184358057883,\n",
       "  1.597812654899428,\n",
       "  1.5982190978163828,\n",
       "  1.5981831448230057,\n",
       "  1.5983520911340898,\n",
       "  1.5991938394852954,\n",
       "  1.6005977161042908,\n",
       "  1.6005785854899652,\n",
       "  1.6000014969516658,\n",
       "  1.5995781586771196,\n",
       "  1.599636060048999,\n",
       "  1.5995530553801898,\n",
       "  1.600078907039357,\n",
       "  1.5995839665801241,\n",
       "  1.5981537840042748,\n",
       "  1.5984265853823718,\n",
       "  1.5989003036160878],\n",
       " 'train_ATT_acc': [28.25484764542936,\n",
       "  47.9224376731302,\n",
       "  59.83379501385041,\n",
       "  72.85318559556787,\n",
       "  76.73130193905817,\n",
       "  80.88642659279779,\n",
       "  85.31855955678671,\n",
       "  86.98060941828255,\n",
       "  90.30470914127424,\n",
       "  91.41274238227147,\n",
       "  93.07479224376732,\n",
       "  95.01385041551247,\n",
       "  95.29085872576178,\n",
       "  95.56786703601108,\n",
       "  96.95290858725762,\n",
       "  98.06094182825485,\n",
       "  98.33795013850416,\n",
       "  98.33795013850416,\n",
       "  98.61495844875347,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  99.16897506925208,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0],\n",
       " 'train_VAL_acc': [74.23822714681441,\n",
       "  89.47368421052632,\n",
       "  91.41274238227147,\n",
       "  94.18282548476455,\n",
       "  94.18282548476455,\n",
       "  95.56786703601108,\n",
       "  96.95290858725762,\n",
       "  97.78393351800554,\n",
       "  97.50692520775624,\n",
       "  98.06094182825485,\n",
       "  97.50692520775624,\n",
       "  97.50692520775624,\n",
       "  97.78393351800554,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.16897506925208,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  98.06094182825485,\n",
       "  98.06094182825485,\n",
       "  98.89196675900277,\n",
       "  99.44598337950139,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0],\n",
       " 'train_VAL_jac': [0.09695290858725762,\n",
       "  0.2423822754307797,\n",
       "  0.41551246801571834,\n",
       "  0.5221606727452159,\n",
       "  0.5383194978877778,\n",
       "  0.5517082320025753,\n",
       "  0.6029547693986972,\n",
       "  0.6343490410379425,\n",
       "  0.6514312184088118,\n",
       "  0.6694367585750167,\n",
       "  0.6777470025989818,\n",
       "  0.689289016406622,\n",
       "  0.7086796060493448,\n",
       "  0.6966759029187655,\n",
       "  0.7119113652659915,\n",
       "  0.7197599384593171,\n",
       "  0.7326869911764468,\n",
       "  0.7409972404839259,\n",
       "  0.7409972404839259,\n",
       "  0.7423822820351725,\n",
       "  0.7442290023423298,\n",
       "  0.7409972404839259,\n",
       "  0.7433056448305082,\n",
       "  0.753000935689234,\n",
       "  0.7465374066889121,\n",
       "  0.7493074897914052,\n",
       "  0.753000935689234,\n",
       "  0.7590027806501309,\n",
       "  0.7571560603429737,\n",
       "  0.7603878222013775,\n",
       "  0.7631579053038705,\n",
       "  0.764542946855117,\n",
       "  0.7613111849967132,\n",
       "  0.7631579053038705,\n",
       "  0.7631579053038705,\n",
       "  0.7714681546113498,\n",
       "  0.7659279884063637,\n",
       "  0.7700831130601032,\n",
       "  0.7714681546113498,\n",
       "  0.7742382377138428,\n",
       "  0.764542946855117,\n",
       "  0.7714681546113498,\n",
       "  0.7802400826747398,\n",
       "  0.7816251242259863,\n",
       "  0.7857802488797259,\n",
       "  0.7839335285725686,\n",
       "  0.779778403918829,\n",
       "  0.7825484870213221,\n",
       "  0.7963989025337874,\n",
       "  0.7880886532263082,\n",
       "  0.7834718498166579,\n",
       "  0.789012016021644,\n",
       "  0.7950138609825409,\n",
       "  0.7982456228409448,\n",
       "  0.7922437778800479,\n",
       "  0.7959372237778767,\n",
       "  0.7908587363288013,\n",
       "  0.7996306643921913,\n",
       "  0.7963989025337874,\n",
       "  0.7991689856362805,\n",
       "  0.8000923431481021,\n",
       "  0.8042474678018416,\n",
       "  0.8000923431481021,\n",
       "  0.797783944085034,\n",
       "  0.797322260045609,\n",
       "  0.7987073015968555,\n",
       "  0.8033241102900202,\n",
       "  0.8116343595974993,\n",
       "  0.8060941933925132,\n",
       "  0.8047091518412667,\n",
       "  0.8102493180462528,\n",
       "  0.8116343595974993,\n",
       "  0.8060941933925132,\n",
       "  0.8120960383534102,\n",
       "  0.8024007527781986,\n",
       "  0.8116343595974993,\n",
       "  0.8130194011487459,\n",
       "  0.8116343595974993,\n",
       "  0.8171745258024855,\n",
       "  0.8199446089049786,\n",
       "  0.8148661267394174,\n",
       "  0.8259464591493897,\n",
       "  0.8194829301490678,\n",
       "  0.8264081379053005,\n",
       "  0.8328716616221082,\n",
       "  0.8301015785196151,\n",
       "  0.8259464538658755,\n",
       "  0.8287165369683686,\n",
       "  0.8264081379053005,\n",
       "  0.8277931794565471,\n",
       "  0.8259464538658755,\n",
       "  0.8250230963540539,\n",
       "  0.8264081379053005,\n",
       "  0.8361034287640262,\n",
       "  0.8416435949690124,\n",
       "  0.8397968693783409,\n",
       "  0.8384118278270943,\n",
       "  0.8439519940320804,\n",
       "  0.8287165369683686,\n",
       "  0.8356417447246013,\n",
       "  0.8356417447246013,\n",
       "  0.8425669524808339,\n",
       "  0.8361034287640262,\n",
       "  0.8328716616221082,\n",
       "  0.8370267862758478,\n",
       "  0.8370267862758478,\n",
       "  0.8384118278270943,\n",
       "  0.8314866200708616,\n",
       "  0.827331495417122,\n",
       "  0.8291782157242793,\n",
       "  0.8254847751099648,\n",
       "  0.8282548582124578,\n",
       "  0.8370267862758478,\n",
       "  0.8370267862758478,\n",
       "  0.8310249413149509,\n",
       "  0.8416435896854981,\n",
       "  0.8347183819292655,\n",
       "  0.8402585481342516,\n",
       "  0.8457987143392378,\n",
       "  0.8416435896854981,\n",
       "  0.8434903152761697,\n",
       "  0.8462603983786627,\n",
       "  0.8448753568274162,\n",
       "  0.8374884650317586,\n",
       "  0.8305632572755259,\n",
       "  0.833333340378019,\n",
       "  0.8416435896854981,\n",
       "  0.8444136727879913,\n",
       "  0.8397968640948267,\n",
       "  0.8374884650317586,\n",
       "  0.832871656338594,\n",
       "  0.832871656338594,\n",
       "  0.8365651022364228,\n",
       "  0.836103423480512,\n",
       "  0.8347183819292655,\n",
       "  0.8379501490711836,\n",
       "  0.8457987143392378,\n",
       "  0.8457987143392378,\n",
       "  0.8462603983786627,\n",
       "  0.8494921602370666,\n",
       "  0.8467220771345736,\n",
       "  0.8425669524808339,\n",
       "  0.8370267862758478,\n",
       "  0.8342567031733548,\n",
       "  0.8328716616221082,\n",
       "  0.815789484251239,\n",
       "  0.7982456228409448,\n",
       "  0.7940904981872051,\n",
       "  0.7871652904309725,\n",
       "  0.7746999164697537,\n",
       "  0.7774699995722467,\n",
       "  0.7940904981872051,\n",
       "  0.8088642764950063,\n",
       "  0.8130194011487459,\n",
       "  0.8204062876608893,\n",
       "  0.818097883314307,\n",
       "  0.8222530079680467,\n",
       "  0.8208679664168002,\n",
       "  0.8157894789677247,\n",
       "  0.8254847698264505,\n",
       "  0.8282548529289436,\n",
       "  0.8287165316848544,\n",
       "  0.8190212408261286,\n",
       "  0.8051708305971774,\n",
       "  0.7954755397384516,\n",
       "  0.8019390687387736,\n",
       "  0.7945521822266302,\n",
       "  0.7825484870213221,\n",
       "  0.7700831130601032,\n",
       "  0.7719298333672605,\n",
       "  0.7737765536744179,\n",
       "  0.7668513459181852,\n",
       "  0.7700831130601032,\n",
       "  0.7894736947775548,\n",
       "  0.807017556187849,\n",
       "  0.8291782157242793,\n",
       "  0.836103423480512,\n",
       "  0.8614958501588604,\n",
       "  0.8388735065830051,\n",
       "  0.8240997335587182,\n",
       "  0.7626962212644456,\n",
       "  0.7068328857421875,\n",
       "  0.7059095229468517,\n",
       "  0.7728531908790821,\n",
       "  0.8397968640948267,\n",
       "  0.8416435896854981,\n",
       "  0.835641739441087,\n",
       "  0.8416435896854981,\n",
       "  0.8457987143392378,\n",
       "  0.8370267862758478,\n",
       "  0.8319482988267725,\n",
       "  0.833333340378019,\n",
       "  0.8481071186858201,\n",
       "  0.8642659332613536,\n",
       "  0.8730378613247436,\n",
       "  0.8831948362228943,\n",
       "  0.8873499608766339,\n",
       "  0.890120043979127,\n",
       "  0.8878116396325447,\n",
       "  0.8771929912619973,\n",
       "  0.866112658852025,\n",
       "  0.8647276173007785,\n",
       "  0.8591874510957924,\n",
       "  0.8508772017883132,\n",
       "  0.8550323264420527,\n",
       "  0.8467220771345736,\n",
       "  0.8411819109295874,\n",
       "  0.8499538389929774,\n",
       "  0.8614958501588604,\n",
       "  0.8762696284666616,\n",
       "  0.8767313072225724,\n",
       "  0.883656514978805,\n",
       "  0.8845798724906266,\n",
       "  0.880886431876312,\n",
       "  0.8818097893881336,\n",
       "  0.8762696231831474,\n",
       "  0.8850415565300516,\n",
       "  0.880424747836887,\n",
       "  0.8790397062856404,\n",
       "  0.872114503812922,\n",
       "  0.8711911410175862,\n",
       "  0.8670360163638466,\n",
       "  0.8674976951197574,\n",
       "  0.8725761825688327,\n",
       "  0.8684210579150932,\n",
       "  0.8725761825688327,\n",
       "  0.8767313072225724,\n",
       "  0.8859649140418732,\n",
       "  0.8827331521834693,\n",
       "  0.889658359939702,\n",
       "  0.8850415565300516,\n",
       "  0.8905817227350378,\n",
       "  0.8896583652232162,\n",
       "  0.8767313125060866,\n",
       "  0.8698061047498539,\n",
       "  0.8674977004032716,\n",
       "  0.8656509800961143,\n",
       "  0.845798719622752,\n",
       "  0.8388735118665194,\n",
       "  0.8379501490711836,\n",
       "  0.8384118278270943,\n",
       "  0.8434903152761697,\n",
       "  0.8610341766864639,\n",
       "  0.8684210631986073,\n",
       "  0.872576187852347,\n",
       "  0.8781163487738189,\n",
       "  0.8873499555931197,\n",
       "  0.8845798724906266,\n",
       "  0.8716528197734971,\n",
       "  0.8776546700179082,\n",
       "  0.8771929859784832,\n",
       "  0.8753462656713259,\n",
       "  0.8707294622616755,\n",
       "  0.8711911410175862,\n",
       "  0.8767313072225724,\n",
       "  0.880886431876312,\n",
       "  0.8868882768372089,\n",
       "  0.8910434014909485,\n",
       "  0.889658359939702,\n",
       "  0.8933518058375308,\n",
       "  0.8979686145306954,\n",
       "  0.8979686145306954,\n",
       "  0.8896583652232162,\n",
       "  0.8841181990182301,\n",
       "  0.8845798777741408,\n",
       "  0.8767313072225724,\n",
       "  0.8781163487738189,\n",
       "  0.8670360163638466,\n",
       "  0.8679593791591824,\n",
       "  0.8684210579150932,\n",
       "  0.872114503812922,\n",
       "  0.8767313072225724,\n",
       "  0.8642659332613536,\n",
       "  0.8670360163638466,\n",
       "  0.8767313072225724,\n",
       "  0.8725761825688327,\n",
       "  0.8790397062856404,\n",
       "  0.880424747836887,\n",
       "  0.8767313072225724,\n",
       "  0.8698060994663397,\n",
       "  0.8795013903250655,\n",
       "  0.8767313072225724,\n",
       "  0.8734995400806543,\n",
       "  0.8938134845934416,\n",
       "  0.8947368473887773,\n",
       "  0.8970452464518455,\n",
       "  0.8975069304912705,\n",
       "  0.8878116343490304,\n",
       "  0.8753462603878116,\n",
       "  0.8725761772853186,\n",
       "  0.8656509695290858,\n",
       "  0.8822714681440443,\n",
       "  0.8836565096952909,\n",
       "  0.8850415512465374,\n",
       "  0.8905817174515236,\n",
       "  0.8767313019390581,\n",
       "  0.8836565096952909,\n",
       "  0.8928901217981059,\n",
       "  0.8831948309393801,\n",
       "  0.8739612241200793],\n",
       " 'train_VAL_acc_1': [26.31578947368421,\n",
       "  49.03047091412743,\n",
       "  55.6786703601108,\n",
       "  63.988919667590025,\n",
       "  64.81994459833795,\n",
       "  67.03601108033241,\n",
       "  70.08310249307479,\n",
       "  71.46814404432133,\n",
       "  73.9612188365651,\n",
       "  72.02216066481995,\n",
       "  73.6842105263158,\n",
       "  72.85318559556787,\n",
       "  74.23822714681441,\n",
       "  73.40720221606648,\n",
       "  76.45429362880887,\n",
       "  76.17728531855956,\n",
       "  77.5623268698061,\n",
       "  78.39335180055402,\n",
       "  78.94736842105263,\n",
       "  79.22437673130194,\n",
       "  77.28531855955679,\n",
       "  78.67036011080333,\n",
       "  79.77839335180056,\n",
       "  80.05540166204986,\n",
       "  79.50138504155125,\n",
       "  79.77839335180056,\n",
       "  80.60941828254848,\n",
       "  79.22437673130194,\n",
       "  80.88642659279779,\n",
       "  80.33240997229917,\n",
       "  80.88642659279779,\n",
       "  81.16343490304709,\n",
       "  81.16343490304709,\n",
       "  81.16343490304709,\n",
       "  81.16343490304709,\n",
       "  81.99445983379502,\n",
       "  82.54847645429363,\n",
       "  81.4404432132964,\n",
       "  81.4404432132964,\n",
       "  82.82548476454294,\n",
       "  81.99445983379502,\n",
       "  83.37950138504155,\n",
       "  83.65650969529086,\n",
       "  83.37950138504155,\n",
       "  83.37950138504155,\n",
       "  81.7174515235457,\n",
       "  81.7174515235457,\n",
       "  83.10249307479225,\n",
       "  83.10249307479225,\n",
       "  83.65650969529086,\n",
       "  83.10249307479225,\n",
       "  82.27146814404432,\n",
       "  83.93351800554017,\n",
       "  85.0415512465374,\n",
       "  84.48753462603878,\n",
       "  84.21052631578948,\n",
       "  83.93351800554017,\n",
       "  84.7645429362881,\n",
       "  84.48753462603878,\n",
       "  84.48753462603878,\n",
       "  83.37950138504155,\n",
       "  83.93351800554017,\n",
       "  84.21052631578948,\n",
       "  84.7645429362881,\n",
       "  84.21052631578948,\n",
       "  85.87257617728532,\n",
       "  85.59556786703601,\n",
       "  85.59556786703601,\n",
       "  85.0415512465374,\n",
       "  84.7645429362881,\n",
       "  85.59556786703601,\n",
       "  85.59556786703601,\n",
       "  85.0415512465374,\n",
       "  85.0415512465374,\n",
       "  84.7645429362881,\n",
       "  85.59556786703601,\n",
       "  85.87257617728532,\n",
       "  86.42659279778394,\n",
       "  85.87257617728532,\n",
       "  86.42659279778394,\n",
       "  86.42659279778394,\n",
       "  85.87257617728532,\n",
       "  84.7645429362881,\n",
       "  85.59556786703601,\n",
       "  87.53462603878117,\n",
       "  86.42659279778394,\n",
       "  85.87257617728532,\n",
       "  84.21052631578948,\n",
       "  85.87257617728532,\n",
       "  87.53462603878117,\n",
       "  87.25761772853186,\n",
       "  85.59556786703601,\n",
       "  86.98060941828255,\n",
       "  87.25761772853186,\n",
       "  86.70360110803324,\n",
       "  86.42659279778394,\n",
       "  86.70360110803324,\n",
       "  87.25761772853186,\n",
       "  86.98060941828255,\n",
       "  86.42659279778394,\n",
       "  87.81163434903047,\n",
       "  86.70360110803324,\n",
       "  86.14958448753463,\n",
       "  86.42659279778394,\n",
       "  86.42659279778394,\n",
       "  86.42659279778394,\n",
       "  86.42659279778394,\n",
       "  86.70360110803324,\n",
       "  86.14958448753463,\n",
       "  85.59556786703601,\n",
       "  85.87257617728532,\n",
       "  85.87257617728532,\n",
       "  86.70360110803324,\n",
       "  86.70360110803324,\n",
       "  86.98060941828255,\n",
       "  87.53462603878117,\n",
       "  86.70360110803324,\n",
       "  86.42659279778394,\n",
       "  86.70360110803324,\n",
       "  86.42659279778394,\n",
       "  86.98060941828255,\n",
       "  86.98060941828255,\n",
       "  86.98060941828255,\n",
       "  87.25761772853186,\n",
       "  88.08864265927978,\n",
       "  88.08864265927978,\n",
       "  88.6426592797784,\n",
       "  89.19667590027701,\n",
       "  88.6426592797784,\n",
       "  88.36565096952909,\n",
       "  89.19667590027701,\n",
       "  87.81163434903047,\n",
       "  87.81163434903047,\n",
       "  87.81163434903047,\n",
       "  88.36565096952909,\n",
       "  88.9196675900277,\n",
       "  89.19667590027701,\n",
       "  89.75069252077563,\n",
       "  89.19667590027701,\n",
       "  89.19667590027701,\n",
       "  88.9196675900277,\n",
       "  88.08864265927978,\n",
       "  86.42659279778394,\n",
       "  86.14958448753463,\n",
       "  85.59556786703601,\n",
       "  84.21052631578948,\n",
       "  83.93351800554017,\n",
       "  81.4404432132964,\n",
       "  81.4404432132964,\n",
       "  80.60941828254848,\n",
       "  79.77839335180056,\n",
       "  82.27146814404432,\n",
       "  83.93351800554017,\n",
       "  84.21052631578948,\n",
       "  85.0415512465374,\n",
       "  85.87257617728532,\n",
       "  86.42659279778394,\n",
       "  86.70360110803324,\n",
       "  86.70360110803324,\n",
       "  87.25761772853186,\n",
       "  86.98060941828255,\n",
       "  86.42659279778394,\n",
       "  86.70360110803324,\n",
       "  83.65650969529086,\n",
       "  83.10249307479225,\n",
       "  83.10249307479225,\n",
       "  82.54847645429363,\n",
       "  81.16343490304709,\n",
       "  80.33240997229917,\n",
       "  78.67036011080333,\n",
       "  78.39335180055402,\n",
       "  79.22437673130194,\n",
       "  80.05540166204986,\n",
       "  82.82548476454294,\n",
       "  84.48753462603878,\n",
       "  85.59556786703601,\n",
       "  87.53462603878117,\n",
       "  89.19667590027701,\n",
       "  86.98060941828255,\n",
       "  83.65650969529086,\n",
       "  77.5623268698061,\n",
       "  70.6371191135734,\n",
       "  69.25207756232687,\n",
       "  78.39335180055402,\n",
       "  85.87257617728532,\n",
       "  86.98060941828255,\n",
       "  86.14958448753463,\n",
       "  86.14958448753463,\n",
       "  88.36565096952909,\n",
       "  88.9196675900277,\n",
       "  89.47368421052632,\n",
       "  89.75069252077563,\n",
       "  90.58171745152355,\n",
       "  90.85872576177286,\n",
       "  91.96675900277009,\n",
       "  91.68975069252078,\n",
       "  91.13573407202216,\n",
       "  90.85872576177286,\n",
       "  90.30470914127424,\n",
       "  90.30470914127424,\n",
       "  88.08864265927978,\n",
       "  88.36565096952909,\n",
       "  86.70360110803324,\n",
       "  86.42659279778394,\n",
       "  86.14958448753463,\n",
       "  84.7645429362881,\n",
       "  83.93351800554017,\n",
       "  84.48753462603878,\n",
       "  86.98060941828255,\n",
       "  90.02770083102493,\n",
       "  91.96675900277009,\n",
       "  92.24376731301939,\n",
       "  92.797783933518,\n",
       "  92.24376731301939,\n",
       "  92.24376731301939,\n",
       "  91.68975069252078,\n",
       "  91.96675900277009,\n",
       "  90.58171745152355,\n",
       "  90.30470914127424,\n",
       "  90.02770083102493,\n",
       "  89.19667590027701,\n",
       "  89.47368421052632,\n",
       "  89.47368421052632,\n",
       "  88.6426592797784,\n",
       "  89.75069252077563,\n",
       "  90.58171745152355,\n",
       "  91.13573407202216,\n",
       "  91.13573407202216,\n",
       "  90.85872576177286,\n",
       "  91.96675900277009,\n",
       "  90.85872576177286,\n",
       "  90.85872576177286,\n",
       "  90.85872576177286,\n",
       "  90.85872576177286,\n",
       "  90.02770083102493,\n",
       "  87.25761772853186,\n",
       "  85.87257617728532,\n",
       "  85.59556786703601,\n",
       "  85.0415512465374,\n",
       "  85.87257617728532,\n",
       "  85.59556786703601,\n",
       "  86.42659279778394,\n",
       "  87.53462603878117,\n",
       "  89.47368421052632,\n",
       "  90.30470914127424,\n",
       "  90.85872576177286,\n",
       "  93.35180055401662,\n",
       "  92.5207756232687,\n",
       "  92.24376731301939,\n",
       "  90.85872576177286,\n",
       "  90.02770083102493,\n",
       "  90.02770083102493,\n",
       "  91.13573407202216,\n",
       "  90.30470914127424,\n",
       "  90.58171745152355,\n",
       "  90.85872576177286,\n",
       "  91.41274238227147,\n",
       "  90.85872576177286,\n",
       "  91.13573407202216,\n",
       "  91.13573407202216,\n",
       "  91.41274238227147,\n",
       "  91.41274238227147,\n",
       "  90.85872576177286,\n",
       "  91.41274238227147,\n",
       "  90.85872576177286,\n",
       "  90.85872576177286,\n",
       "  91.41274238227147,\n",
       "  90.58171745152355,\n",
       "  88.9196675900277,\n",
       "  88.36565096952909,\n",
       "  89.19667590027701,\n",
       "  89.47368421052632,\n",
       "  88.6426592797784,\n",
       "  88.9196675900277,\n",
       "  89.19667590027701,\n",
       "  89.75069252077563,\n",
       "  90.30470914127424,\n",
       "  90.30470914127424,\n",
       "  91.41274238227147,\n",
       "  89.19667590027701,\n",
       "  90.30470914127424,\n",
       "  92.24376731301939,\n",
       "  91.68975069252078,\n",
       "  93.07479224376732,\n",
       "  91.13573407202216,\n",
       "  89.75069252077563,\n",
       "  90.30470914127424,\n",
       "  89.19667590027701,\n",
       "  89.47368421052632,\n",
       "  87.25761772853186,\n",
       "  87.53462603878117,\n",
       "  88.08864265927978,\n",
       "  87.25761772853186,\n",
       "  87.53462603878117,\n",
       "  86.70360110803324,\n",
       "  86.70360110803324,\n",
       "  85.87257617728532,\n",
       "  88.9196675900277,\n",
       "  88.36565096952909,\n",
       "  86.98060941828255],\n",
       " 'val_loss': [7.913739697025317,\n",
       "  7.327657641867354,\n",
       "  6.848139567229138,\n",
       "  6.626440548538479,\n",
       "  6.501437664743352,\n",
       "  6.403325627790568,\n",
       "  6.313234879424944,\n",
       "  6.23496278300222,\n",
       "  6.1733208903281565,\n",
       "  6.120505636495647,\n",
       "  6.078910224054648,\n",
       "  6.04856843931881,\n",
       "  6.014430889224442,\n",
       "  5.98819338841198,\n",
       "  5.960374797577304,\n",
       "  5.936094549762467,\n",
       "  5.919699245299569,\n",
       "  5.902136724055078,\n",
       "  5.889328531820352,\n",
       "  5.880859915112637,\n",
       "  5.865386326697158,\n",
       "  5.86153322611701,\n",
       "  5.851132504842829,\n",
       "  5.839694566345863,\n",
       "  5.837009426161144,\n",
       "  5.826247739235187,\n",
       "  5.822169410966487,\n",
       "  5.816247836862387,\n",
       "  5.808393587773175,\n",
       "  5.8045342657699415,\n",
       "  5.796730387943819,\n",
       "  5.791170649078001,\n",
       "  5.790243116890554,\n",
       "  5.786654503690719,\n",
       "  5.784056414130443,\n",
       "  5.780621945995263,\n",
       "  5.780571405719262,\n",
       "  5.7751786217342325,\n",
       "  5.772220060469806,\n",
       "  5.771379181997258,\n",
       "  5.764759709918101,\n",
       "  5.767183701292195,\n",
       "  5.76258000792221,\n",
       "  5.76178635746471,\n",
       "  5.764192471918882,\n",
       "  5.762760932397831,\n",
       "  5.761477374687724,\n",
       "  5.756793454611701,\n",
       "  5.755513787095331,\n",
       "  5.7571536799439,\n",
       "  5.752267231382727,\n",
       "  5.752428688220221,\n",
       "  5.751879591016623,\n",
       "  5.746617053686258,\n",
       "  5.750714112437073,\n",
       "  5.751646642496901,\n",
       "  5.748402572145087,\n",
       "  5.7463116067473194,\n",
       "  5.743334547017032,\n",
       "  5.74553997773477,\n",
       "  5.746186622333085,\n",
       "  5.748676780979941,\n",
       "  5.749952837117032,\n",
       "  5.743620945535867,\n",
       "  5.748089880264441,\n",
       "  5.748763552770234,\n",
       "  5.745951509989779,\n",
       "  5.74693638023,\n",
       "  5.744443934599544,\n",
       "  5.745887472861554,\n",
       "  5.744681030699247,\n",
       "  5.743414443997952,\n",
       "  5.741111015876874,\n",
       "  5.739737094977009,\n",
       "  5.741863458776957,\n",
       "  5.74195325697226,\n",
       "  5.738172209766177,\n",
       "  5.741230573204532,\n",
       "  5.737906653380775,\n",
       "  5.7367670517767655,\n",
       "  5.735387418139677,\n",
       "  5.735425863029926,\n",
       "  5.74069598450707,\n",
       "  5.737938695079912,\n",
       "  5.73322796167576,\n",
       "  5.737502155187605,\n",
       "  5.736229215783333,\n",
       "  5.73926237736501,\n",
       "  5.733544452464492,\n",
       "  5.732399592690734,\n",
       "  5.732524518641975,\n",
       "  5.734564326790977,\n",
       "  5.7285089136766985,\n",
       "  5.72944079527671,\n",
       "  5.729211175727364,\n",
       "  5.731893239165151,\n",
       "  5.731654257601572,\n",
       "  5.730408346529091,\n",
       "  5.73154425032661,\n",
       "  5.734140587425227,\n",
       "  5.733068950135586,\n",
       "  5.733282994074469,\n",
       "  5.731613009566516,\n",
       "  5.733281165639507,\n",
       "  5.731012894340852,\n",
       "  5.731348746007384,\n",
       "  5.727871492148792,\n",
       "  5.733358858282801,\n",
       "  5.733057983865645,\n",
       "  5.73429954252432,\n",
       "  5.732772968198917,\n",
       "  5.73232017297069,\n",
       "  5.728855203187406,\n",
       "  5.728506614841517,\n",
       "  5.7307455933701785,\n",
       "  5.728222289526032,\n",
       "  5.730672551066637,\n",
       "  5.729653426904912,\n",
       "  5.731692363012825,\n",
       "  5.731765003257149,\n",
       "  5.732571825223364,\n",
       "  5.730007002946182,\n",
       "  5.729631101026653,\n",
       "  5.7307087786982365,\n",
       "  5.732272947744425,\n",
       "  5.73540684866962,\n",
       "  5.734307622517099,\n",
       "  5.7351608360191815,\n",
       "  5.732895754498219,\n",
       "  5.734589197885322,\n",
       "  5.736482320131458,\n",
       "  5.742344598017785,\n",
       "  5.747297271116962,\n",
       "  5.741907232204643,\n",
       "  5.7409980226578305,\n",
       "  5.741430995260248,\n",
       "  5.742003155940334,\n",
       "  5.738795263247608,\n",
       "  5.743153556693784,\n",
       "  5.7351672661552024,\n",
       "  5.738688457212817,\n",
       "  5.7379363807485735,\n",
       "  5.737988907644949,\n",
       "  5.738416200106023,\n",
       "  5.742067249430864,\n",
       "  5.750190660842662,\n",
       "  5.758560621671952,\n",
       "  5.767203691221123,\n",
       "  5.770494314892242,\n",
       "  5.777255724309429,\n",
       "  5.774826839312,\n",
       "  5.761999699052169,\n",
       "  5.751127103527639,\n",
       "  5.747787523282902,\n",
       "  5.744618480844646,\n",
       "  5.750311697330634,\n",
       "  5.747700847973286,\n",
       "  5.745629726050661,\n",
       "  5.7480076907068405,\n",
       "  5.743076756565532,\n",
       "  5.745332565284079,\n",
       "  5.74997694526345,\n",
       "  5.748780076929812,\n",
       "  5.756707751595544,\n",
       "  5.761065001591024,\n",
       "  5.760276819351219,\n",
       "  5.765953722786662,\n",
       "  5.768659563218236,\n",
       "  5.774113041920289,\n",
       "  5.777707712866168,\n",
       "  5.7783713292097305,\n",
       "  5.778352969163715,\n",
       "  5.772490121494321,\n",
       "  5.760784871834167,\n",
       "  5.7500998299907415,\n",
       "  5.736630997278005,\n",
       "  5.7315504315609,\n",
       "  5.727545223438776,\n",
       "  5.72532913061227,\n",
       "  5.738805109552266,\n",
       "  5.769195781758624,\n",
       "  5.801236202414352,\n",
       "  5.806506168605272,\n",
       "  5.773195097772788,\n",
       "  5.7437105618103645,\n",
       "  5.7449497017537805,\n",
       "  5.753512558959178,\n",
       "  5.752867247862907,\n",
       "  5.745762896100813,\n",
       "  5.742060110378769,\n",
       "  5.7395540209553255,\n",
       "  5.738830613410346,\n",
       "  5.732204902414503,\n",
       "  5.728592161110517,\n",
       "  5.7244365546004765,\n",
       "  5.724852131704645,\n",
       "  5.7257566522930565,\n",
       "  5.723257977294499,\n",
       "  5.7226512640782685,\n",
       "  5.723215963880803,\n",
       "  5.725058831641235,\n",
       "  5.725278080387011,\n",
       "  5.725774201538418,\n",
       "  5.724375432961512,\n",
       "  5.724342203328485,\n",
       "  5.7259948116943615,\n",
       "  5.724364120217046,\n",
       "  5.726036004829242,\n",
       "  5.721688758693029,\n",
       "  5.719662642521554,\n",
       "  5.718486824933426,\n",
       "  5.718240630856919,\n",
       "  5.718761715650525,\n",
       "  5.719793566638064,\n",
       "  5.72253005734562,\n",
       "  5.72398376615413,\n",
       "  5.726699321808004,\n",
       "  5.7243242027078605,\n",
       "  5.726745384438738,\n",
       "  5.727749561527889,\n",
       "  5.73256237703436,\n",
       "  5.730340054645795,\n",
       "  5.731920595549865,\n",
       "  5.732759968889953,\n",
       "  5.7327434759393245,\n",
       "  5.732039191998799,\n",
       "  5.73230552672385,\n",
       "  5.7294985724694305,\n",
       "  5.73100892619069,\n",
       "  5.728726732206409,\n",
       "  5.729636434662437,\n",
       "  5.728872248919253,\n",
       "  5.727530315882419,\n",
       "  5.728948702018565,\n",
       "  5.733242062714003,\n",
       "  5.733270748668536,\n",
       "  5.73001101811894,\n",
       "  5.730115880241187,\n",
       "  5.725686090261507,\n",
       "  5.721739744097729,\n",
       "  5.720643342537326,\n",
       "  5.7195345501162285,\n",
       "  5.716290025594996,\n",
       "  5.717957780053974,\n",
       "  5.716309051993083,\n",
       "  5.7190826920743,\n",
       "  5.718617773496697,\n",
       "  5.724502677385194,\n",
       "  5.7260871608838375,\n",
       "  5.73372530800113,\n",
       "  5.735151650968844,\n",
       "  5.732234721692192,\n",
       "  5.7328722967834,\n",
       "  5.73163780892474,\n",
       "  5.731631453221717,\n",
       "  5.72988547164107,\n",
       "  5.726986676988071,\n",
       "  5.728595836867539,\n",
       "  5.727685375265113,\n",
       "  5.724820085489028,\n",
       "  5.72352867343707,\n",
       "  5.72423604716849,\n",
       "  5.725080632642816,\n",
       "  5.724223089848824,\n",
       "  5.7207618754458105,\n",
       "  5.723107381679398,\n",
       "  5.724348265904918,\n",
       "  5.723710390906429,\n",
       "  5.727998555877468,\n",
       "  5.724573850511759,\n",
       "  5.72347298514736,\n",
       "  5.72241976202809,\n",
       "  5.721030940526817,\n",
       "  5.720661061323755,\n",
       "  5.718197605062426,\n",
       "  5.7189065533319745,\n",
       "  5.719113979264668,\n",
       "  5.720621995494499,\n",
       "  5.720830699943122,\n",
       "  5.724665788257551,\n",
       "  5.727847261778717,\n",
       "  5.725803202417765,\n",
       "  5.726187128929192,\n",
       "  5.725320754965509,\n",
       "  5.726116920436329,\n",
       "  5.728774912259982,\n",
       "  5.730544504982684,\n",
       "  5.732371765852686,\n",
       "  5.739928302585283,\n",
       "  5.746494724251462,\n",
       "  5.746792562182691,\n",
       "  5.744332496878754,\n",
       "  5.744043010940655,\n",
       "  5.742860775951333,\n",
       "  5.743176692627906,\n",
       "  5.744854712740259,\n",
       "  5.74655975605025,\n",
       "  5.7403884262223706,\n",
       "  5.745851414348496,\n",
       "  5.745241742088332],\n",
       " 'val_ATT_loss': [1.9395230984300134,\n",
       "  1.7996639035096982,\n",
       "  1.6692374796886755,\n",
       "  1.5701349433360061,\n",
       "  1.4874738726189467,\n",
       "  1.4102656637750022,\n",
       "  1.3420366189344142,\n",
       "  1.281380820322812,\n",
       "  1.2299540938158346,\n",
       "  1.185800143010248,\n",
       "  1.1483365867922946,\n",
       "  1.123885364673002,\n",
       "  1.0938028900361643,\n",
       "  1.0732721399243286,\n",
       "  1.051208352291487,\n",
       "  1.0313794087103711,\n",
       "  1.0174231122179729,\n",
       "  1.0028826521664131,\n",
       "  0.9923880338911119,\n",
       "  0.9849253768116478,\n",
       "  0.9719210485132729,\n",
       "  0.9691074646827651,\n",
       "  0.959434579542982,\n",
       "  0.9504539972640634,\n",
       "  0.9480712217528645,\n",
       "  0.9391369868100174,\n",
       "  0.9358309532811002,\n",
       "  0.9298440460267106,\n",
       "  0.9239244280549569,\n",
       "  0.9203719507630278,\n",
       "  0.9135912762182515,\n",
       "  0.9091737338197909,\n",
       "  0.9076503795094606,\n",
       "  0.904670549359748,\n",
       "  0.9038890701968495,\n",
       "  0.8993869930263457,\n",
       "  0.9005544980367025,\n",
       "  0.8958948370887012,\n",
       "  0.8921390105553759,\n",
       "  0.8923159445204386,\n",
       "  0.8866615180319887,\n",
       "  0.887762062554437,\n",
       "  0.8848776449032916,\n",
       "  0.8832159523314577,\n",
       "  0.886793350906876,\n",
       "  0.8853680760395236,\n",
       "  0.8836867004875245,\n",
       "  0.8808962226640887,\n",
       "  0.8784862238217176,\n",
       "  0.8800246102538535,\n",
       "  0.8767114494874225,\n",
       "  0.876297493654538,\n",
       "  0.8761638807572001,\n",
       "  0.8727755636219087,\n",
       "  0.8749298028102735,\n",
       "  0.8754502381008815,\n",
       "  0.8729439857529431,\n",
       "  0.8714103913161813,\n",
       "  0.8688993365541706,\n",
       "  0.8707536876928516,\n",
       "  0.870839156028701,\n",
       "  0.8717745460145842,\n",
       "  0.8739229463707141,\n",
       "  0.8693768704325203,\n",
       "  0.8729244148343559,\n",
       "  0.8749092656180142,\n",
       "  0.8718632320320703,\n",
       "  0.8725512993287264,\n",
       "  0.870642921667758,\n",
       "  0.8720172615797539,\n",
       "  0.8706952192919041,\n",
       "  0.8698941566110626,\n",
       "  0.8667997261130713,\n",
       "  0.8662860605532561,\n",
       "  0.8668993149346452,\n",
       "  0.866955355173204,\n",
       "  0.8643883189292458,\n",
       "  0.8671151859973504,\n",
       "  0.8640555514068138,\n",
       "  0.8638806360039285,\n",
       "  0.8624728748468848,\n",
       "  0.8629901570275547,\n",
       "  0.8670790776973818,\n",
       "  0.8652604152032031,\n",
       "  0.8618169881221724,\n",
       "  0.865151594567105,\n",
       "  0.8634224118498283,\n",
       "  0.8646878629195981,\n",
       "  0.8607903343390643,\n",
       "  0.861383940630812,\n",
       "  0.8611933101968068,\n",
       "  0.8612604412606092,\n",
       "  0.8566357122688759,\n",
       "  0.8574617606837575,\n",
       "  0.8572310629656644,\n",
       "  0.859741324089407,\n",
       "  0.8597230870064682,\n",
       "  0.8585754073247677,\n",
       "  0.8599469350363181,\n",
       "  0.8619862482315157,\n",
       "  0.861226122432608,\n",
       "  0.8612710207216139,\n",
       "  0.8586583450073149,\n",
       "  0.8593275513106245,\n",
       "  0.857329290814516,\n",
       "  0.8576463678988014,\n",
       "  0.8555038411442827,\n",
       "  0.8595359296333499,\n",
       "  0.8582689913550043,\n",
       "  0.8584816035458712,\n",
       "  0.8565785422073147,\n",
       "  0.8561953593318056,\n",
       "  0.8533883984011363,\n",
       "  0.854374130203472,\n",
       "  0.8554751121900915,\n",
       "  0.8546133139511434,\n",
       "  0.855892132816276,\n",
       "  0.8551875767184467,\n",
       "  0.8567769798806043,\n",
       "  0.8566903946845512,\n",
       "  0.8585331430280112,\n",
       "  0.8555210357516762,\n",
       "  0.8547962845825567,\n",
       "  0.8548181989813238,\n",
       "  0.8561702207094286,\n",
       "  0.8586859803616516,\n",
       "  0.8581331532175948,\n",
       "  0.8588609978920076,\n",
       "  0.8566899460747959,\n",
       "  0.8580665681662598,\n",
       "  0.8576585506278325,\n",
       "  0.8634016098288017,\n",
       "  0.8680261214816474,\n",
       "  0.8619853161214813,\n",
       "  0.8597455600170585,\n",
       "  0.8606337576619978,\n",
       "  0.8625644602911259,\n",
       "  0.8587674382256298,\n",
       "  0.8628415637384586,\n",
       "  0.8542166731464185,\n",
       "  0.8565347741047541,\n",
       "  0.8556019546297507,\n",
       "  0.8546756329090615,\n",
       "  0.8526840678802351,\n",
       "  0.8495452984319469,\n",
       "  0.848483326958447,\n",
       "  0.8467106355157324,\n",
       "  0.845880512784167,\n",
       "  0.8479510986708044,\n",
       "  0.8506470526137003,\n",
       "  0.850082006032874,\n",
       "  0.8483523016538077,\n",
       "  0.8476054019075099,\n",
       "  0.8484580905699148,\n",
       "  0.850964163740476,\n",
       "  0.8525833980581625,\n",
       "  0.8522312857998096,\n",
       "  0.8532502853531179,\n",
       "  0.8565563647969952,\n",
       "  0.8547998747932233,\n",
       "  0.8556870398967247,\n",
       "  0.85586639717827,\n",
       "  0.8534532082759267,\n",
       "  0.853821212925562,\n",
       "  0.8536528936488842,\n",
       "  0.8530937747015217,\n",
       "  0.851934684849367,\n",
       "  0.848702664176623,\n",
       "  0.8487730841568815,\n",
       "  0.847942725066247,\n",
       "  0.8484439871660093,\n",
       "  0.8493560479908455,\n",
       "  0.8490727112545231,\n",
       "  0.8514273246613945,\n",
       "  0.8510568994816726,\n",
       "  0.8491202283923219,\n",
       "  0.850511495659991,\n",
       "  0.8519358132185975,\n",
       "  0.848549081058037,\n",
       "  0.8498498940370917,\n",
       "  0.8508573338025953,\n",
       "  0.8487497595994453,\n",
       "  0.8508973332439981,\n",
       "  0.8519263016741451,\n",
       "  0.8481976755750857,\n",
       "  0.8467540092826858,\n",
       "  0.8480644587094222,\n",
       "  0.8502305501360234,\n",
       "  0.8487623325450634,\n",
       "  0.8479150767491116,\n",
       "  0.8464639120227923,\n",
       "  0.8471920613835497,\n",
       "  0.8450597084876967,\n",
       "  0.845068019822361,\n",
       "  0.8455289662853489,\n",
       "  0.8481780932686194,\n",
       "  0.8495158086705014,\n",
       "  0.8480772941820021,\n",
       "  0.848085013831534,\n",
       "  0.8494739172662177,\n",
       "  0.850719495396304,\n",
       "  0.8498529348431564,\n",
       "  0.8488943896642546,\n",
       "  0.8475848906408481,\n",
       "  0.8477260512791998,\n",
       "  0.8479055728369612,\n",
       "  0.8454486922762259,\n",
       "  0.8471742598264198,\n",
       "  0.8454867643796331,\n",
       "  0.8448660464548483,\n",
       "  0.8442296268251853,\n",
       "  0.8431422074393529,\n",
       "  0.8432945673300968,\n",
       "  0.8432588674188629,\n",
       "  0.8443786581841911,\n",
       "  0.8450143522605663,\n",
       "  0.8458133308383508,\n",
       "  0.8436563224569569,\n",
       "  0.8433193970259613,\n",
       "  0.842833229560193,\n",
       "  0.8442979541009035,\n",
       "  0.8431911910694789,\n",
       "  0.8445524655706514,\n",
       "  0.8428662021954855,\n",
       "  0.8440084341095715,\n",
       "  0.8451556530667514,\n",
       "  0.8459465116020141,\n",
       "  0.8462700167807137,\n",
       "  0.848055909561917,\n",
       "  0.8463193640960911,\n",
       "  0.8489372822327342,\n",
       "  0.8499075018051194,\n",
       "  0.8495987979619484,\n",
       "  0.8507248664774546,\n",
       "  0.8540514850519537,\n",
       "  0.8529291324983768,\n",
       "  0.8494463531467004,\n",
       "  0.8468714515125848,\n",
       "  0.8428345395297538,\n",
       "  0.8402722924947739,\n",
       "  0.839186614485291,\n",
       "  0.8399124477452379,\n",
       "  0.8398903247302141,\n",
       "  0.8413208380220382,\n",
       "  0.8408804658224912,\n",
       "  0.8425830439096544,\n",
       "  0.8414047370112039,\n",
       "  0.8432506586962599,\n",
       "  0.8423481063871849,\n",
       "  0.8447683677683032,\n",
       "  0.8437952373812838,\n",
       "  0.8427367169197982,\n",
       "  0.8438781421116697,\n",
       "  0.8435111572829689,\n",
       "  0.8431338974373127,\n",
       "  0.8434710929064246,\n",
       "  0.8432063067831644,\n",
       "  0.8440881502337572,\n",
       "  0.8449881647418185,\n",
       "  0.8431872829673736,\n",
       "  0.8437590733533953,\n",
       "  0.8447123752619193,\n",
       "  0.8458931467882017,\n",
       "  0.8443065857499595,\n",
       "  0.8417122401599961,\n",
       "  0.8431572041860441,\n",
       "  0.8433386819149421,\n",
       "  0.8439851824830218,\n",
       "  0.844358280664537,\n",
       "  0.8430098640482601,\n",
       "  0.8435635684224648,\n",
       "  0.8418169330532957,\n",
       "  0.8408109706834079,\n",
       "  0.8402254753481082,\n",
       "  0.8381860142316275,\n",
       "  0.8400036846476842,\n",
       "  0.8396394697388982,\n",
       "  0.8402151712557164,\n",
       "  0.8415095299724641,\n",
       "  0.8424831672412593,\n",
       "  0.8433625866484836,\n",
       "  0.8427640256600651,\n",
       "  0.8413564990448757,\n",
       "  0.8416555004148949,\n",
       "  0.8400575569490107,\n",
       "  0.841231946295839,\n",
       "  0.8415506656576948,\n",
       "  0.8430977629694513,\n",
       "  0.8455301741274391,\n",
       "  0.8445857111516037,\n",
       "  0.8449244860226546,\n",
       "  0.8452870770683133,\n",
       "  0.8456323347925171,\n",
       "  0.8451367030298806,\n",
       "  0.8446769230976338,\n",
       "  0.8451377928741579,\n",
       "  0.8485001295320387,\n",
       "  0.849092800563913,\n",
       "  0.8530909220377604,\n",
       "  0.8502759595469731],\n",
       " 'val_VAL_loss': [1.9914055328651015,\n",
       "  1.8426645794525522,\n",
       "  1.7263006958468208,\n",
       "  1.6854352017341576,\n",
       "  1.6713212640414685,\n",
       "  1.664353321338522,\n",
       "  1.6570660868301768,\n",
       "  1.6511939875598025,\n",
       "  1.647788932170774,\n",
       "  1.6449018311617998,\n",
       "  1.6435245457541179,\n",
       "  1.6415610248819361,\n",
       "  1.6402093330627592,\n",
       "  1.638307082829217,\n",
       "  1.636388815095272,\n",
       "  1.6349050470173652,\n",
       "  1.634092044360532,\n",
       "  1.633084690629555,\n",
       "  1.6323134993097466,\n",
       "  1.631978179433663,\n",
       "  1.6311550927279619,\n",
       "  1.6308085871447484,\n",
       "  1.6305659750999488,\n",
       "  1.6297468563606,\n",
       "  1.6296460681360931,\n",
       "  1.6290369174750567,\n",
       "  1.6287794858951288,\n",
       "  1.6288012636118923,\n",
       "  1.628156386572739,\n",
       "  1.6280541050023045,\n",
       "  1.6277130372418558,\n",
       "  1.6273323050860702,\n",
       "  1.6275309124603647,\n",
       "  1.6273279847769901,\n",
       "  1.6267224479778646,\n",
       "  1.6270783176563057,\n",
       "  1.6266723025608532,\n",
       "  1.6264279282151772,\n",
       "  1.6266936833048102,\n",
       "  1.6263544124922729,\n",
       "  1.6260327306287041,\n",
       "  1.6264738795792528,\n",
       "  1.625900787672973,\n",
       "  1.6261901350444175,\n",
       "  1.6257997070040022,\n",
       "  1.6257976187861025,\n",
       "  1.6259302247333995,\n",
       "  1.6252990773158709,\n",
       "  1.6256758544245378,\n",
       "  1.6257096898966823,\n",
       "  1.625185260631768,\n",
       "  1.625377064855228,\n",
       "  1.6252385700864744,\n",
       "  1.6246138300214494,\n",
       "  1.6252614365422666,\n",
       "  1.6253988014653398,\n",
       "  1.6251528621307147,\n",
       "  1.6249670718103795,\n",
       "  1.6248117368209538,\n",
       "  1.6249287633473062,\n",
       "  1.6251158221014614,\n",
       "  1.6256340783217857,\n",
       "  1.6253432969154396,\n",
       "  1.6247480250344488,\n",
       "  1.6250551551433619,\n",
       "  1.6246180957174066,\n",
       "  1.6246960926525698,\n",
       "  1.624795026967091,\n",
       "  1.6246003376439286,\n",
       "  1.6246234037606,\n",
       "  1.6246619371357809,\n",
       "  1.6245067624622964,\n",
       "  1.6247704299212677,\n",
       "  1.624483678141251,\n",
       "  1.6249880479474372,\n",
       "  1.6249993005996854,\n",
       "  1.6245946302789773,\n",
       "  1.6247051290690606,\n",
       "  1.6246170339913204,\n",
       "  1.624295471924279,\n",
       "  1.624304847764264,\n",
       "  1.624145235334124,\n",
       "  1.624538968936563,\n",
       "  1.6242260932922363,\n",
       "  1.6238036578511956,\n",
       "  1.6241168535401669,\n",
       "  1.6242689346445018,\n",
       "  1.6248581714818042,\n",
       "  1.6242513727084757,\n",
       "  1.6236718840199738,\n",
       "  1.6237770694817228,\n",
       "  1.6244346285101228,\n",
       "  1.6239577338026074,\n",
       "  1.6239930115309842,\n",
       "  1.6239933709205665,\n",
       "  1.6240506383585813,\n",
       "  1.6239770568650345,\n",
       "  1.623944313068108,\n",
       "  1.6238657717634304,\n",
       "  1.6240514463979034,\n",
       "  1.623947609234326,\n",
       "  1.6240039911176183,\n",
       "  1.6243182215197334,\n",
       "  1.6246512047762942,\n",
       "  1.6245612011754453,\n",
       "  1.6245674593695278,\n",
       "  1.6241225503348364,\n",
       "  1.6246076428831504,\n",
       "  1.6249296641702136,\n",
       "  1.6252726463261495,\n",
       "  1.625398141997201,\n",
       "  1.625374937879628,\n",
       "  1.625155601595423,\n",
       "  1.6247108282126816,\n",
       "  1.6250901603933625,\n",
       "  1.6245363251916294,\n",
       "  1.6249268060834536,\n",
       "  1.6248219500621552,\n",
       "  1.624971794377407,\n",
       "  1.6250248695241993,\n",
       "  1.6246795607317845,\n",
       "  1.624828655731502,\n",
       "  1.6249449388146988,\n",
       "  1.6252968599056374,\n",
       "  1.6253675756783321,\n",
       "  1.625573622769323,\n",
       "  1.6253914897665014,\n",
       "  1.6254332793757247,\n",
       "  1.625401936141141,\n",
       "  1.6255075432396875,\n",
       "  1.626274589834542,\n",
       "  1.6263143293963278,\n",
       "  1.626423716545105,\n",
       "  1.6266406386943872,\n",
       "  1.6270841542135905,\n",
       "  1.62693241253275,\n",
       "  1.6264795652164028,\n",
       "  1.6266759416739929,\n",
       "  1.6267706643184419,\n",
       "  1.626983531002928,\n",
       "  1.6273845610360207,\n",
       "  1.6274448087062743,\n",
       "  1.6277710915786292,\n",
       "  1.628577377408596,\n",
       "  1.6308406503329724,\n",
       "  1.6339024446280719,\n",
       "  1.6372833287187398,\n",
       "  1.6404410594789853,\n",
       "  1.6408477387404794,\n",
       "  1.6422028905652426,\n",
       "  1.641581611093042,\n",
       "  1.6378824657994537,\n",
       "  1.6345072338733766,\n",
       "  1.633109810904329,\n",
       "  1.6312181057013901,\n",
       "  1.6325760997574905,\n",
       "  1.631823187391159,\n",
       "  1.630793146899181,\n",
       "  1.6304837753032815,\n",
       "  1.6294256272574363,\n",
       "  1.629881841795785,\n",
       "  1.63137018269506,\n",
       "  1.6317756228846283,\n",
       "  1.6342955128899936,\n",
       "  1.6358040359807131,\n",
       "  1.6357276815498991,\n",
       "  1.6380063459790986,\n",
       "  1.639985633013871,\n",
       "  1.6417799859211362,\n",
       "  1.6432549959333071,\n",
       "  1.643309114014574,\n",
       "  1.64299897372429,\n",
       "  1.641139136746599,\n",
       "  1.6364525157242573,\n",
       "  1.6330143101696897,\n",
       "  1.6291702562952277,\n",
       "  1.6270129786336363,\n",
       "  1.6252031367400597,\n",
       "  1.625593349851411,\n",
       "  1.6296517385050582,\n",
       "  1.6394461493186763,\n",
       "  1.6508288142716356,\n",
       "  1.6518696117870912,\n",
       "  1.6404229320328811,\n",
       "  1.631837628745093,\n",
       "  1.632731897490365,\n",
       "  1.6351493667499186,\n",
       "  1.6342122325756279,\n",
       "  1.63233352118525,\n",
       "  1.6313816778765524,\n",
       "  1.6310300363108443,\n",
       "  1.630546184008932,\n",
       "  1.629048397975602,\n",
       "  1.6278413804293854,\n",
       "  1.626302529438376,\n",
       "  1.6255580128120084,\n",
       "  1.6254136145408518,\n",
       "  1.6250602277041657,\n",
       "  1.6248554167489113,\n",
       "  1.6245806822048618,\n",
       "  1.6247797787483103,\n",
       "  1.625141715181285,\n",
       "  1.6256266039580547,\n",
       "  1.6255968474402216,\n",
       "  1.6255387173497617,\n",
       "  1.6260297462858002,\n",
       "  1.62630514264694,\n",
       "  1.6262872483342738,\n",
       "  1.6254006647711317,\n",
       "  1.6249321986889016,\n",
       "  1.6247523993694137,\n",
       "  1.6250328078058554,\n",
       "  1.6251557161068093,\n",
       "  1.6255115664064004,\n",
       "  1.6260504663871427,\n",
       "  1.626323137964521,\n",
       "  1.6269619969898843,\n",
       "  1.6268892934169676,\n",
       "  1.6278086624709256,\n",
       "  1.6283054439892322,\n",
       "  1.6294214743111521,\n",
       "  1.6290496211921053,\n",
       "  1.6291227099930712,\n",
       "  1.629964588898156,\n",
       "  1.6295783472765843,\n",
       "  1.6289611796440162,\n",
       "  1.6287863383739454,\n",
       "  1.6277428518962391,\n",
       "  1.6276510055429243,\n",
       "  1.6274691227034395,\n",
       "  1.6268997174765676,\n",
       "  1.6263215823713781,\n",
       "  1.6259771726401568,\n",
       "  1.6260746118470366,\n",
       "  1.6263968592206832,\n",
       "  1.6267805387233865,\n",
       "  1.6268548883240799,\n",
       "  1.627748142909534,\n",
       "  1.627617183577251,\n",
       "  1.6271558172009848,\n",
       "  1.627152242684012,\n",
       "  1.62654070079033,\n",
       "  1.6254665669549275,\n",
       "  1.6255456473439784,\n",
       "  1.6251428620568638,\n",
       "  1.6254998827215486,\n",
       "  1.6257376788284978,\n",
       "  1.627084006229645,\n",
       "  1.6279130181655508,\n",
       "  1.6296523134109422,\n",
       "  1.6304521378625203,\n",
       "  1.6298326682574644,\n",
       "  1.6296647182239101,\n",
       "  1.629375550547257,\n",
       "  1.629499185261468,\n",
       "  1.6288047929115483,\n",
       "  1.6279267900683023,\n",
       "  1.6281692288779273,\n",
       "  1.6275657368410985,\n",
       "  1.6272109341738847,\n",
       "  1.6265898666945584,\n",
       "  1.6265078906355233,\n",
       "  1.6263958286182048,\n",
       "  1.6266388346996214,\n",
       "  1.6263498784286048,\n",
       "  1.6266500591644513,\n",
       "  1.627003194663325,\n",
       "  1.6265750694744692,\n",
       "  1.627880091737644,\n",
       "  1.6271879954878332,\n",
       "  1.6266364722416318,\n",
       "  1.6268676096582648,\n",
       "  1.626739989947803,\n",
       "  1.6268118619918823,\n",
       "  1.6266705302769326,\n",
       "  1.6263009562280966,\n",
       "  1.6264915031752563,\n",
       "  1.6268022747462607,\n",
       "  1.6264403899902193,\n",
       "  1.6273942070054304,\n",
       "  1.6281615583767444,\n",
       "  1.6276797255859,\n",
       "  1.6282768766281053,\n",
       "  1.627888418183538,\n",
       "  1.6286864544957729,\n",
       "  1.6291809886547144,\n",
       "  1.6296646131083297,\n",
       "  1.6297580009610781,\n",
       "  1.6314660428192815,\n",
       "  1.633969671033286,\n",
       "  1.6339560253866787,\n",
       "  1.6330151399368136,\n",
       "  1.6328035587160459,\n",
       "  1.6325746909738175,\n",
       "  1.632833256510091,\n",
       "  1.6332389732887005,\n",
       "  1.6326865421727372,\n",
       "  1.630431875219486,\n",
       "  1.6309201641035784,\n",
       "  1.6316552608471198],\n",
       " 'val_ATT_acc': [30.89430894308943,\n",
       "  50.609756097560975,\n",
       "  57.31707317073171,\n",
       "  63.41463414634146,\n",
       "  67.07317073170732,\n",
       "  70.32520325203252,\n",
       "  73.3739837398374,\n",
       "  76.21951219512195,\n",
       "  77.84552845528455,\n",
       "  78.86178861788618,\n",
       "  80.08130081300813,\n",
       "  82.72357723577235,\n",
       "  84.14634146341463,\n",
       "  83.53658536585365,\n",
       "  85.77235772357723,\n",
       "  85.97560975609755,\n",
       "  86.17886178861788,\n",
       "  86.78861788617886,\n",
       "  87.1951219512195,\n",
       "  86.78861788617886,\n",
       "  87.8048780487805,\n",
       "  87.60162601626017,\n",
       "  88.00813008130082,\n",
       "  88.82113821138212,\n",
       "  88.82113821138212,\n",
       "  90.2439024390244,\n",
       "  90.44715447154472,\n",
       "  90.65040650406505,\n",
       "  91.26016260162602,\n",
       "  91.66666666666667,\n",
       "  92.07317073170732,\n",
       "  92.47967479674797,\n",
       "  92.07317073170732,\n",
       "  92.88617886178862,\n",
       "  92.88617886178862,\n",
       "  93.29268292682927,\n",
       "  92.88617886178862,\n",
       "  93.08943089430895,\n",
       "  93.29268292682927,\n",
       "  93.69918699186992,\n",
       "  94.10569105691057,\n",
       "  93.90243902439025,\n",
       "  93.90243902439025,\n",
       "  94.10569105691057,\n",
       "  94.3089430894309,\n",
       "  94.10569105691057,\n",
       "  93.90243902439025,\n",
       "  94.10569105691057,\n",
       "  93.90243902439025,\n",
       "  94.10569105691057,\n",
       "  94.71544715447155,\n",
       "  94.3089430894309,\n",
       "  94.51219512195122,\n",
       "  94.71544715447155,\n",
       "  94.71544715447155,\n",
       "  95.1219512195122,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.1219512195122,\n",
       "  95.52845528455285,\n",
       "  95.1219512195122,\n",
       "  95.32520325203252,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.32520325203252,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.32520325203252,\n",
       "  96.13821138211382,\n",
       "  95.1219512195122,\n",
       "  95.32520325203252,\n",
       "  95.9349593495935,\n",
       "  95.1219512195122,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.1219512195122,\n",
       "  95.73170731707317,\n",
       "  95.1219512195122,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.32520325203252,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.1219512195122,\n",
       "  95.32520325203252,\n",
       "  95.32520325203252,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.1219512195122,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.32520325203252,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  95.9349593495935,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  96.34146341463415,\n",
       "  96.54471544715447,\n",
       "  96.54471544715447,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.32520325203252,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  96.54471544715447,\n",
       "  96.7479674796748,\n",
       "  96.54471544715447,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.95121951219512,\n",
       "  96.54471544715447,\n",
       "  96.7479674796748,\n",
       "  96.13821138211382,\n",
       "  96.54471544715447,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  96.54471544715447,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  96.54471544715447,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.54471544715447,\n",
       "  96.54471544715447,\n",
       "  96.54471544715447,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  96.54471544715447,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.7479674796748,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317],\n",
       " 'val_VAL_acc': [71.42857142857143,\n",
       "  88.66995073891626,\n",
       "  93.5960591133005,\n",
       "  95.56650246305419,\n",
       "  97.04433497536945,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  96.05911330049261,\n",
       "  95.56650246305419,\n",
       "  97.53694581280789,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  97.53694581280789,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  97.53694581280789,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463],\n",
       " 'val_VAL_jac': [0.09770114962103332,\n",
       "  0.2454844014397983,\n",
       "  0.4991789827205865,\n",
       "  0.5968801352778091,\n",
       "  0.6288998397113067,\n",
       "  0.6469622386500166,\n",
       "  0.6642036179603614,\n",
       "  0.6896551794606477,\n",
       "  0.7085385956787711,\n",
       "  0.7192118320559046,\n",
       "  0.7142857189836174,\n",
       "  0.7192118273580016,\n",
       "  0.7348111697605678,\n",
       "  0.7512315341404506,\n",
       "  0.75862070139993,\n",
       "  0.75862070139993,\n",
       "  0.7684729181486984,\n",
       "  0.7717569896153041,\n",
       "  0.769293935428112,\n",
       "  0.7619047728665357,\n",
       "  0.7660098592636033,\n",
       "  0.7758620760123718,\n",
       "  0.7725780068947177,\n",
       "  0.7610837508892191,\n",
       "  0.7594417139814404,\n",
       "  0.7643678223558248,\n",
       "  0.7594417139814404,\n",
       "  0.749589497232672,\n",
       "  0.7684729134507955,\n",
       "  0.7660098592636033,\n",
       "  0.7569786597942484,\n",
       "  0.7660098569146518,\n",
       "  0.7660098592636033,\n",
       "  0.7619047681686326,\n",
       "  0.7643678223558248,\n",
       "  0.7627257830990947,\n",
       "  0.7651888372862867,\n",
       "  0.7643678223558248,\n",
       "  0.7717569849174011,\n",
       "  0.770114945660671,\n",
       "  0.770114945660671,\n",
       "  0.7668308741940654,\n",
       "  0.7643678200068732,\n",
       "  0.7619047658196811,\n",
       "  0.7692939283812574,\n",
       "  0.7758620760123718,\n",
       "  0.7709359676379876,\n",
       "  0.7684729134507955,\n",
       "  0.7635468050764112,\n",
       "  0.7610837508892191,\n",
       "  0.7717569825684496,\n",
       "  0.7643678200068732,\n",
       "  0.7619047658196811,\n",
       "  0.7733990218251797,\n",
       "  0.7643678200068732,\n",
       "  0.7569786574452969,\n",
       "  0.7668308741940654,\n",
       "  0.7668308741940654,\n",
       "  0.7742200367556417,\n",
       "  0.7766830909428338,\n",
       "  0.7717569825684496,\n",
       "  0.7717569825684496,\n",
       "  0.7586206931785997,\n",
       "  0.7586206931785997,\n",
       "  0.7586206931785997,\n",
       "  0.7536945848042155,\n",
       "  0.7536945848042155,\n",
       "  0.7545156032581047,\n",
       "  0.7536945848042155,\n",
       "  0.7536945848042155,\n",
       "  0.7512315306170233,\n",
       "  0.7512315306170233,\n",
       "  0.7504105156865614,\n",
       "  0.7487684764298312,\n",
       "  0.7561576389914075,\n",
       "  0.7512315306170233,\n",
       "  0.7463054222426391,\n",
       "  0.7536945848042155,\n",
       "  0.7561576389914075,\n",
       "  0.7561576389914075,\n",
       "  0.7536945848042155,\n",
       "  0.7561576389914075,\n",
       "  0.7528735698737534,\n",
       "  0.7536945848042155,\n",
       "  0.754515602083629,\n",
       "  0.7512315306170233,\n",
       "  0.7512315306170233,\n",
       "  0.7553366240609456,\n",
       "  0.7536945848042155,\n",
       "  0.7487684764298312,\n",
       "  0.7463054222426391,\n",
       "  0.7536945848042155,\n",
       "  0.7463054222426391,\n",
       "  0.7512315306170233,\n",
       "  0.7487684764298312,\n",
       "  0.7487684764298312,\n",
       "  0.7471264395220526,\n",
       "  0.7487684764298312,\n",
       "  0.7487684764298312,\n",
       "  0.7463054222426391,\n",
       "  0.7487684764298312,\n",
       "  0.7438423704043985,\n",
       "  0.7438423704043985,\n",
       "  0.7487684787787827,\n",
       "  0.7463054245915907,\n",
       "  0.7487684787787827,\n",
       "  0.7463054245915907,\n",
       "  0.7463054245915907,\n",
       "  0.7454844073121771,\n",
       "  0.7536945883276427,\n",
       "  0.7536945883276427,\n",
       "  0.7487684787787827,\n",
       "  0.7487684787787827,\n",
       "  0.7487684787787827,\n",
       "  0.7487684764298312,\n",
       "  0.7471264395220526,\n",
       "  0.7471264395220526,\n",
       "  0.7495894937092448,\n",
       "  0.7520525478964368,\n",
       "  0.754515602083629,\n",
       "  0.754515602083629,\n",
       "  0.754515602083629,\n",
       "  0.754515602083629,\n",
       "  0.7577996747247104,\n",
       "  0.7561576389914075,\n",
       "  0.754515602083629,\n",
       "  0.754515602083629,\n",
       "  0.7487684787787827,\n",
       "  0.7438423704043985,\n",
       "  0.7487684787787827,\n",
       "  0.754515602083629,\n",
       "  0.7643678188323975,\n",
       "  0.7610837497147434,\n",
       "  0.7438423704043985,\n",
       "  0.7438423704043985,\n",
       "  0.7339901513066786,\n",
       "  0.7438423704043985,\n",
       "  0.73399015365563,\n",
       "  0.7364532078428222,\n",
       "  0.7282430303507837,\n",
       "  0.723316914929545,\n",
       "  0.7266009863961507,\n",
       "  0.7397372804839035,\n",
       "  0.7323481120499484,\n",
       "  0.7348111697605678,\n",
       "  0.7282430268273565,\n",
       "  0.7233169161040207,\n",
       "  0.7142857213325688,\n",
       "  0.7224958988246072,\n",
       "  0.7216748815451937,\n",
       "  0.7249589553607508,\n",
       "  0.7249589577097023,\n",
       "  0.7307060845379759,\n",
       "  0.7364532125407252,\n",
       "  0.7454844131845558,\n",
       "  0.7536945906765943,\n",
       "  0.7561576413403591,\n",
       "  0.7586206955275512,\n",
       "  0.7619047681686326,\n",
       "  0.7520525514198642,\n",
       "  0.7512315364894021,\n",
       "  0.7520525537688156,\n",
       "  0.7536945906765943,\n",
       "  0.7463054281150179,\n",
       "  0.7405583001122686,\n",
       "  0.7323481179223272,\n",
       "  0.7323481179223272,\n",
       "  0.7315271006429137,\n",
       "  0.7233169208019238,\n",
       "  0.7266009969664324,\n",
       "  0.7249589577097023,\n",
       "  0.7233169208019238,\n",
       "  0.7183908124275395,\n",
       "  0.7151067362630309,\n",
       "  0.729064048804673,\n",
       "  0.7438423739278258,\n",
       "  0.7487684799532585,\n",
       "  0.744663387683812,\n",
       "  0.7454844061377013,\n",
       "  0.7348111721095193,\n",
       "  0.7077175737014545,\n",
       "  0.666666669798602,\n",
       "  0.6715927805219378,\n",
       "  0.7134647017042038,\n",
       "  0.7134646993552523,\n",
       "  0.7405582930654141,\n",
       "  0.7364532043193949,\n",
       "  0.7282430233039292,\n",
       "  0.7307060798400729,\n",
       "  0.7274220083734672,\n",
       "  0.7274220083734672,\n",
       "  0.7372742274711872,\n",
       "  0.7257799714656886,\n",
       "  0.7233169172784965,\n",
       "  0.7364532090172979,\n",
       "  0.7430213496015576,\n",
       "  0.7380952412271734,\n",
       "  0.7397372804839035,\n",
       "  0.7446633888582878,\n",
       "  0.7380952412271734,\n",
       "  0.7430213496015576,\n",
       "  0.7454844037887498,\n",
       "  0.7413793126937791,\n",
       "  0.7454844037887498,\n",
       "  0.7495894948837205,\n",
       "  0.7520525490709127,\n",
       "  0.759441711632489,\n",
       "  0.7561576401658834,\n",
       "  0.7528735663503262,\n",
       "  0.7536945859786912,\n",
       "  0.7528735686992777,\n",
       "  0.7577996770736619,\n",
       "  0.7504105145120855,\n",
       "  0.7479474603248935,\n",
       "  0.7553366205375183,\n",
       "  0.7553366228864697,\n",
       "  0.7504105145120855,\n",
       "  0.7553366228864697,\n",
       "  0.7446633888582878,\n",
       "  0.7397372804839035,\n",
       "  0.7471264430454799,\n",
       "  0.7463054257660664,\n",
       "  0.7463054257660664,\n",
       "  0.7348111721095193,\n",
       "  0.7372742239477599,\n",
       "  0.739737278134952,\n",
       "  0.7356321870399813,\n",
       "  0.7307060786655971,\n",
       "  0.7356321870399813,\n",
       "  0.7315270982939621,\n",
       "  0.7463054257660664,\n",
       "  0.749589492534769,\n",
       "  0.7422003299731926,\n",
       "  0.7389162585065869,\n",
       "  0.7380952412271734,\n",
       "  0.7372742215988084,\n",
       "  0.7331691305038377,\n",
       "  0.738095243576125,\n",
       "  0.7380952388782219,\n",
       "  0.7389162608555385,\n",
       "  0.7364532066683463,\n",
       "  0.7364532066683463,\n",
       "  0.7339901571790574,\n",
       "  0.7339901571790574,\n",
       "  0.738095243576125,\n",
       "  0.7561576401658834,\n",
       "  0.7389162608555385,\n",
       "  0.7454844037887498,\n",
       "  0.7331691328527892,\n",
       "  0.7298850613861836,\n",
       "  0.727422009547943,\n",
       "  0.7298850637351351,\n",
       "  0.7372742239477599,\n",
       "  0.73891626320449,\n",
       "  0.7413793173916822,\n",
       "  0.739737278134952,\n",
       "  0.7471264430454799,\n",
       "  0.7512315341404506,\n",
       "  0.7520525514198642,\n",
       "  0.7446633888582878,\n",
       "  0.7487684799532585,\n",
       "  0.7471264406965283,\n",
       "  0.7520525490709127,\n",
       "  0.7520525490709127,\n",
       "  0.7471264406965283,\n",
       "  0.7372742239477599,\n",
       "  0.739737278134952,\n",
       "  0.7372742239477599,\n",
       "  0.7479474556269904,\n",
       "  0.7446633865093363,\n",
       "  0.7430213472526062,\n",
       "  0.7479474556269904,\n",
       "  0.7471264383475769,\n",
       "  0.7405582930654141,\n",
       "  0.7446633865093363,\n",
       "  0.7438423692299228,\n",
       "  0.7430213519505092,\n",
       "  0.7454844061377013,\n",
       "  0.7430213519505092,\n",
       "  0.7389162585065869,\n",
       "  0.7479474603248935,\n",
       "  0.7356321893889328,\n",
       "  0.740558297763317,\n",
       "  0.7389162655534416,\n",
       "  0.7339901571790574,\n",
       "  0.7339901548301058,\n",
       "  0.7249589553607508,\n",
       "  0.7266009922685295,\n",
       "  0.7101806278886467,\n",
       "  0.7134647064021068,\n",
       "  0.7019704503966082,\n",
       "  0.7249589600586539,\n",
       "  0.727422014245846,\n",
       "  0.7266009922685295,\n",
       "  0.7290640441067701,\n",
       "  0.7290640441067701,\n",
       "  0.7266009899195779,\n",
       "  0.7323481155733756,\n",
       "  0.7216748838941452,\n",
       "  0.7372742262967115],\n",
       " 'val_VAL_acc_1': [23.645320197044335,\n",
       "  55.172413793103445,\n",
       "  66.50246305418719,\n",
       "  68.96551724137932,\n",
       "  71.42857142857143,\n",
       "  71.92118226600985,\n",
       "  73.39901477832512,\n",
       "  76.84729064039409,\n",
       "  78.32512315270937,\n",
       "  77.33990147783251,\n",
       "  76.84729064039409,\n",
       "  77.83251231527093,\n",
       "  78.81773399014779,\n",
       "  78.81773399014779,\n",
       "  79.3103448275862,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  82.26600985221675,\n",
       "  81.77339901477832,\n",
       "  82.75862068965517,\n",
       "  82.26600985221675,\n",
       "  82.75862068965517,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  82.75862068965517,\n",
       "  81.2807881773399,\n",
       "  82.26600985221675,\n",
       "  81.77339901477832,\n",
       "  80.78817733990148,\n",
       "  82.26600985221675,\n",
       "  83.2512315270936,\n",
       "  82.26600985221675,\n",
       "  81.77339901477832,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  82.75862068965517,\n",
       "  82.26600985221675,\n",
       "  82.75862068965517,\n",
       "  81.2807881773399,\n",
       "  81.77339901477832,\n",
       "  82.75862068965517,\n",
       "  82.26600985221675,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  81.77339901477832,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  81.77339901477832,\n",
       "  81.2807881773399,\n",
       "  82.26600985221675,\n",
       "  83.2512315270936,\n",
       "  83.74384236453201,\n",
       "  82.75862068965517,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  83.2512315270936,\n",
       "  83.2512315270936,\n",
       "  81.2807881773399,\n",
       "  83.2512315270936,\n",
       "  82.26600985221675,\n",
       "  82.75862068965517,\n",
       "  82.26600985221675,\n",
       "  82.75862068965517,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  81.77339901477832,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  79.3103448275862,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  79.3103448275862,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  78.81773399014779,\n",
       "  79.3103448275862,\n",
       "  78.81773399014779,\n",
       "  78.32512315270937,\n",
       "  78.32512315270937,\n",
       "  78.32512315270937,\n",
       "  78.32512315270937,\n",
       "  76.84729064039409,\n",
       "  76.35467980295566,\n",
       "  77.83251231527093,\n",
       "  76.84729064039409,\n",
       "  77.83251231527093,\n",
       "  76.35467980295566,\n",
       "  76.84729064039409,\n",
       "  78.32512315270937,\n",
       "  78.81773399014779,\n",
       "  78.32512315270937,\n",
       "  77.33990147783251,\n",
       "  76.35467980295566,\n",
       "  76.84729064039409,\n",
       "  76.84729064039409,\n",
       "  77.33990147783251,\n",
       "  77.83251231527093,\n",
       "  77.33990147783251,\n",
       "  77.33990147783251,\n",
       "  76.84729064039409,\n",
       "  76.84729064039409,\n",
       "  76.35467980295566,\n",
       "  76.84729064039409,\n",
       "  77.33990147783251,\n",
       "  77.83251231527093,\n",
       "  76.84729064039409,\n",
       "  78.32512315270937,\n",
       "  77.83251231527093,\n",
       "  78.81773399014779,\n",
       "  78.32512315270937,\n",
       "  76.84729064039409,\n",
       "  76.35467980295566,\n",
       "  76.84729064039409,\n",
       "  75.86206896551724,\n",
       "  76.35467980295566,\n",
       "  75.86206896551724,\n",
       "  76.84729064039409,\n",
       "  77.33990147783251,\n",
       "  77.33990147783251,\n",
       "  77.33990147783251,\n",
       "  78.32512315270937,\n",
       "  79.3103448275862,\n",
       "  79.3103448275862,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  79.80295566502463,\n",
       "  77.33990147783251,\n",
       "  72.9064039408867,\n",
       "  71.42857142857143,\n",
       "  76.84729064039409,\n",
       "  79.3103448275862,\n",
       "  75.86206896551724,\n",
       "  76.84729064039409,\n",
       "  77.83251231527093,\n",
       "  78.81773399014779,\n",
       "  76.84729064039409,\n",
       "  76.84729064039409,\n",
       "  77.83251231527093,\n",
       "  77.83251231527093,\n",
       "  77.83251231527093,\n",
       "  78.32512315270937,\n",
       "  79.80295566502463,\n",
       "  78.81773399014779,\n",
       "  79.3103448275862,\n",
       "  78.81773399014779,\n",
       "  78.81773399014779,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  78.32512315270937,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  78.81773399014779,\n",
       "  78.81773399014779,\n",
       "  77.83251231527093,\n",
       "  77.33990147783251,\n",
       "  77.33990147783251,\n",
       "  76.35467980295566,\n",
       "  75.86206896551724,\n",
       "  75.86206896551724,\n",
       "  77.33990147783251,\n",
       "  76.84729064039409,\n",
       "  76.84729064039409,\n",
       "  77.33990147783251,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  79.3103448275862,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  77.33990147783251,\n",
       "  77.83251231527093,\n",
       "  78.81773399014779,\n",
       "  77.83251231527093,\n",
       "  78.81773399014779,\n",
       "  79.3103448275862,\n",
       "  80.29556650246306,\n",
       "  77.83251231527093,\n",
       "  79.3103448275862,\n",
       "  79.3103448275862,\n",
       "  79.3103448275862,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  81.77339901477832,\n",
       "  81.77339901477832,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  81.77339901477832,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  79.3103448275862,\n",
       "  78.81773399014779,\n",
       "  78.81773399014779,\n",
       "  79.3103448275862,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  79.3103448275862,\n",
       "  80.78817733990148,\n",
       "  78.81773399014779,\n",
       "  79.80295566502463,\n",
       "  79.3103448275862,\n",
       "  78.32512315270937,\n",
       "  77.33990147783251,\n",
       "  77.83251231527093,\n",
       "  77.83251231527093,\n",
       "  77.83251231527093,\n",
       "  78.81773399014779,\n",
       "  77.83251231527093,\n",
       "  78.32512315270937,\n",
       "  78.32512315270937,\n",
       "  77.83251231527093,\n",
       "  77.33990147783251,\n",
       "  76.84729064039409],\n",
       " 'test_loss': 5.786125828680503,\n",
       " 'test_ATT_loss': 0.8526348501266116,\n",
       " 'test_VAL_loss': 1.6444969928512971,\n",
       " 'test_ATT_acc': 96.42147117296223,\n",
       " 'test_VAL_acc': 99.47916666666667,\n",
       " 'test_VAL_jac': 0.740451397995154,\n",
       " 'test_VAL_acc_1': 78.64583333333333,\n",
       " 'model_filename': 'model_storage/HGT/model.pth'}"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'HGT_best_model/best_config.p', 'wb') as fp:\n",
    "    pickle.dump(train_state,fp, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'HGT_best_model/best_config.p', 'rb') as fp:\n",
    "    train_state = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': False,\n",
       " 'early_stopping_step': 55,\n",
       " 'early_stopping_best_ATT_acc_val': 96.95121951219512,\n",
       " 'early_stopping_best_VAL_acc_val': 100.0,\n",
       " 'early_stopping_best_ATT_acc_val_2': 0,\n",
       " 'early_stopping_lowest_loss': 5.716309051993083,\n",
       " 'learning_rate': 0.001,\n",
       " 'epoch_index': 299,\n",
       " 'train_loss': [4.212754587332408,\n",
       "  3.744509994983673,\n",
       "  3.4710254867871604,\n",
       "  3.263322631518046,\n",
       "  3.1269689798355103,\n",
       "  2.989909013112386,\n",
       "  2.8954469362894693,\n",
       "  2.8212348222732544,\n",
       "  2.7362569173177085,\n",
       "  2.674421548843384,\n",
       "  2.6485327084859214,\n",
       "  2.5981449484825134,\n",
       "  2.576626718044281,\n",
       "  2.550933837890625,\n",
       "  2.535131653149923,\n",
       "  2.484558125336965,\n",
       "  2.4908601442972818,\n",
       "  2.476132074991862,\n",
       "  2.4557361006736755,\n",
       "  2.4715222318967185,\n",
       "  2.4546250303586326,\n",
       "  2.4369125167528787,\n",
       "  2.4401920239130654,\n",
       "  2.4267801443735757,\n",
       "  2.422055800755819,\n",
       "  2.4224125146865845,\n",
       "  2.4179813067118325,\n",
       "  2.4229325453440347,\n",
       "  2.4207166830698648,\n",
       "  2.404494285583496,\n",
       "  2.405641794204712,\n",
       "  2.4137187401453652,\n",
       "  2.4108275373776755,\n",
       "  2.3856456875801086,\n",
       "  2.4094346165657043,\n",
       "  2.406758944193522,\n",
       "  2.4062544107437134,\n",
       "  2.4073506196339927,\n",
       "  2.3848799069722495,\n",
       "  2.3935265938440957,\n",
       "  2.3904134035110474,\n",
       "  2.3775036931037903,\n",
       "  2.3968611558278403,\n",
       "  2.3905741373697915,\n",
       "  2.4076212644577026,\n",
       "  2.386153817176819,\n",
       "  2.3838980992635093,\n",
       "  2.389664272467295,\n",
       "  2.3803069988886514,\n",
       "  2.3829084634780884,\n",
       "  2.3663163582483926,\n",
       "  2.38978111743927,\n",
       "  2.3866490920384726,\n",
       "  2.3895376125971475,\n",
       "  2.3759404023488364,\n",
       "  2.3816418250401816,\n",
       "  2.375000258286794,\n",
       "  2.365959664185842,\n",
       "  2.3773799737294516,\n",
       "  2.3851634860038757,\n",
       "  2.3808287382125854,\n",
       "  2.3740285833676658,\n",
       "  2.3576398690541587,\n",
       "  2.369821528593699,\n",
       "  2.370358169078827,\n",
       "  2.3829912344614663,\n",
       "  2.369228482246399,\n",
       "  2.3756791949272156,\n",
       "  2.3709821303685508,\n",
       "  2.3732546965281167,\n",
       "  2.381136973698934,\n",
       "  2.3519471685091653,\n",
       "  2.3688732584317527,\n",
       "  2.3717807133992515,\n",
       "  2.362366239229838,\n",
       "  2.3591893712679544,\n",
       "  2.357397178808848,\n",
       "  2.3710108598073325,\n",
       "  2.3500359058380127,\n",
       "  2.376711130142212,\n",
       "  2.3509661157925925,\n",
       "  2.362232426802317,\n",
       "  2.349884112675985,\n",
       "  2.3621522585550943,\n",
       "  2.3759855031967163,\n",
       "  2.3620211482048035,\n",
       "  2.358452637990316,\n",
       "  2.3501272598902383,\n",
       "  2.365586002667745,\n",
       "  2.360858937104543,\n",
       "  2.3614619970321655,\n",
       "  2.3498252232869468,\n",
       "  2.3578995068868003,\n",
       "  2.3472864230473838,\n",
       "  2.367964188257853,\n",
       "  2.363624354203542,\n",
       "  2.357446094353994,\n",
       "  2.362629532814026,\n",
       "  2.363881528377533,\n",
       "  2.367944320042928,\n",
       "  2.343201518058777,\n",
       "  2.3689687649408975,\n",
       "  2.3411059776941934,\n",
       "  2.348759651184082,\n",
       "  2.3659634590148926,\n",
       "  2.3593635161717734,\n",
       "  2.362978160381317,\n",
       "  2.362177928288778,\n",
       "  2.3488836089769998,\n",
       "  2.3406652013460794,\n",
       "  2.3452983299891152,\n",
       "  2.3503600557645163,\n",
       "  2.3508934378623962,\n",
       "  2.348996182282766,\n",
       "  2.369202673435211,\n",
       "  2.3639399210611978,\n",
       "  2.346254030863444,\n",
       "  2.338806966940562,\n",
       "  2.374669154485067,\n",
       "  2.3546056151390076,\n",
       "  2.3466368913650513,\n",
       "  2.3508933782577515,\n",
       "  2.357291559378306,\n",
       "  2.3650964498519897,\n",
       "  2.365654468536377,\n",
       "  2.3643471797307334,\n",
       "  2.3507268031438193,\n",
       "  2.3511754473050437,\n",
       "  2.3601104021072388,\n",
       "  2.344610035419464,\n",
       "  2.3571850260098777,\n",
       "  2.3505373001098633,\n",
       "  2.353857080141703,\n",
       "  2.34756871064504,\n",
       "  2.3511706590652466,\n",
       "  2.3492911060651145,\n",
       "  2.3498776157697043,\n",
       "  2.354318598906199,\n",
       "  2.3642939925193787,\n",
       "  2.341994325319926,\n",
       "  2.333648761113485,\n",
       "  2.3443042238553367,\n",
       "  2.3460735281308494,\n",
       "  2.3484234611193338,\n",
       "  2.3518704771995544,\n",
       "  2.3579606413841248,\n",
       "  2.3519768714904785,\n",
       "  2.3508107662200928,\n",
       "  2.352656821409861,\n",
       "  2.3524768352508545,\n",
       "  2.3491548697153726,\n",
       "  2.3669060269991555,\n",
       "  2.359908481438955,\n",
       "  2.3599456747372947,\n",
       "  2.34403787056605,\n",
       "  2.3450807332992554,\n",
       "  2.3444391886393228,\n",
       "  2.3474087516466775,\n",
       "  2.3497478564580283,\n",
       "  2.3577657341957092,\n",
       "  2.3491291403770447,\n",
       "  2.347566326459249,\n",
       "  2.3513152996699014,\n",
       "  2.3244027892748513,\n",
       "  2.3337082465489707,\n",
       "  2.343714455763499,\n",
       "  2.3473713994026184,\n",
       "  2.3553279439608255,\n",
       "  2.346401254336039,\n",
       "  2.3440206249554953,\n",
       "  2.359449883302053,\n",
       "  2.351641595363617,\n",
       "  2.358983834584554,\n",
       "  2.344243307908376,\n",
       "  2.3482866287231445,\n",
       "  2.3505460222562156,\n",
       "  2.361703554789225,\n",
       "  2.3473101456960044,\n",
       "  2.3717046976089478,\n",
       "  2.3561343351999917,\n",
       "  2.3647903203964233,\n",
       "  2.3589191834131875,\n",
       "  2.361755092938741,\n",
       "  2.367590347925822,\n",
       "  2.3586204449335733,\n",
       "  2.3649255832036338,\n",
       "  2.357851763566335,\n",
       "  2.337674101193746,\n",
       "  2.334989289442698,\n",
       "  2.332011659940084,\n",
       "  2.3330437938372293,\n",
       "  2.3505683541297913,\n",
       "  2.3588008681933084,\n",
       "  2.3391478061676025,\n",
       "  2.335266133149465,\n",
       "  2.3422364592552185,\n",
       "  2.34785129626592,\n",
       "  2.352286159992218,\n",
       "  2.348785122235616,\n",
       "  2.343895355860392,\n",
       "  2.3352490663528442,\n",
       "  2.34445987145106,\n",
       "  2.3410446445147195,\n",
       "  2.3357056180636087,\n",
       "  2.3401461044947305,\n",
       "  2.351550837357839,\n",
       "  2.3394398291905723,\n",
       "  2.3300169110298157,\n",
       "  2.3361928860346475,\n",
       "  2.346657911936442,\n",
       "  2.346759259700775,\n",
       "  2.3463246623675027,\n",
       "  2.3523252805074057,\n",
       "  2.345177471637726,\n",
       "  2.3353580435117087,\n",
       "  2.3378560543060303,\n",
       "  2.337415258089701,\n",
       "  2.3452074925104776,\n",
       "  2.3243465224901834,\n",
       "  2.351166566212972,\n",
       "  2.3297932147979736,\n",
       "  2.3511283795038858,\n",
       "  2.34217498699824,\n",
       "  2.337269345919291,\n",
       "  2.349517742792765,\n",
       "  2.3423258463541665,\n",
       "  2.344554603099823,\n",
       "  2.3304055531819663,\n",
       "  2.3330251177152,\n",
       "  2.3499587972958884,\n",
       "  2.3367531100908914,\n",
       "  2.341305692990621,\n",
       "  2.3527392943700156,\n",
       "  2.3433781464894614,\n",
       "  2.338826755682627,\n",
       "  2.3447272181510925,\n",
       "  2.3433652917544046,\n",
       "  2.3477142254511514,\n",
       "  2.3489569624265036,\n",
       "  2.3471329609553018,\n",
       "  2.333854019641876,\n",
       "  2.3313928047815957,\n",
       "  2.347913404305776,\n",
       "  2.33931036790212,\n",
       "  2.3428726394971213,\n",
       "  2.3403084675470986,\n",
       "  2.3460219899813333,\n",
       "  2.345427393913269,\n",
       "  2.341270923614502,\n",
       "  2.346359054247538,\n",
       "  2.344762146472931,\n",
       "  2.331865668296814,\n",
       "  2.344398339589437,\n",
       "  2.319320499897003,\n",
       "  2.347334643205007,\n",
       "  2.33759476741155,\n",
       "  2.3323342402776084,\n",
       "  2.3492140571276345,\n",
       "  2.3465505242347717,\n",
       "  2.3188069661458335,\n",
       "  2.3441200852394104,\n",
       "  2.3285131255785623,\n",
       "  2.3323670427004495,\n",
       "  2.31796803077062,\n",
       "  2.327207068602244,\n",
       "  2.321818232536316,\n",
       "  2.3293274442354837,\n",
       "  2.3387466073036194,\n",
       "  2.3333200017611184,\n",
       "  2.3300774693489075,\n",
       "  2.3380286494890847,\n",
       "  2.3365246852238974,\n",
       "  2.3469033439954123,\n",
       "  2.333066165447235,\n",
       "  2.350557724634806,\n",
       "  2.3417248129844666,\n",
       "  2.344898442427317,\n",
       "  2.3511628905932107,\n",
       "  2.3355544209480286,\n",
       "  2.3335918188095093,\n",
       "  2.3326813777287803,\n",
       "  2.3499688704808555,\n",
       "  2.3349210619926453,\n",
       "  2.3357057174046836,\n",
       "  2.3434485594431558,\n",
       "  2.3388755520184836,\n",
       "  2.350087285041809,\n",
       "  2.333504319190979,\n",
       "  2.3510329524676004,\n",
       "  2.334171732266744,\n",
       "  2.342860201994578,\n",
       "  2.345601042111715,\n",
       "  2.3313573598861694,\n",
       "  2.3268946607907615,\n",
       "  2.3433027267456055,\n",
       "  2.3462418913841248,\n",
       "  2.3399335741996765,\n",
       "  2.3453603386878967,\n",
       "  2.3375792304674783,\n",
       "  2.344885766506195],\n",
       " 'train_ATT_loss': [1.9179202255449797,\n",
       "  1.730461289347704,\n",
       "  1.5928260332329451,\n",
       "  1.458375739588962,\n",
       "  1.3386530506313672,\n",
       "  1.2377030865967769,\n",
       "  1.1513480047109714,\n",
       "  1.079639149834905,\n",
       "  1.0202431622634633,\n",
       "  0.9738469384713847,\n",
       "  0.9359683372967791,\n",
       "  0.9061746471806577,\n",
       "  0.8813384467544978,\n",
       "  0.8609489341191637,\n",
       "  0.8441518796117682,\n",
       "  0.8309625274917095,\n",
       "  0.8202618528931425,\n",
       "  0.8115657110623705,\n",
       "  0.804042853972258,\n",
       "  0.7989996673327734,\n",
       "  0.7930962251824355,\n",
       "  0.7895156330018823,\n",
       "  0.7849255712738988,\n",
       "  0.7812670084578178,\n",
       "  0.778695235952446,\n",
       "  0.7754131677738517,\n",
       "  0.7730409805134063,\n",
       "  0.7710750500911491,\n",
       "  0.7684160901899153,\n",
       "  0.7667722418037478,\n",
       "  0.7647889530559656,\n",
       "  0.7639658068025541,\n",
       "  0.7616029541908539,\n",
       "  0.7606888172368924,\n",
       "  0.7588537282560671,\n",
       "  0.7579682967999636,\n",
       "  0.7567136168810139,\n",
       "  0.7558138902167534,\n",
       "  0.7545182222804865,\n",
       "  0.7535423555532651,\n",
       "  0.7532196119221294,\n",
       "  0.7516896462176315,\n",
       "  0.7508227143261241,\n",
       "  0.7501313927431186,\n",
       "  0.7494237041539432,\n",
       "  0.748763811885485,\n",
       "  0.7481039836796367,\n",
       "  0.7471523218868181,\n",
       "  0.7469054467129905,\n",
       "  0.7459406700821134,\n",
       "  0.7460284893532539,\n",
       "  0.7447854906238017,\n",
       "  0.7448807961392601,\n",
       "  0.7437700467426691,\n",
       "  0.7433806228505607,\n",
       "  0.7433063693323955,\n",
       "  0.7423549684130915,\n",
       "  0.7417077177779496,\n",
       "  0.7416324759124059,\n",
       "  0.741514409843244,\n",
       "  0.7409268635792085,\n",
       "  0.7404435770663528,\n",
       "  0.7401373947756442,\n",
       "  0.739652427279718,\n",
       "  0.7396005043692866,\n",
       "  0.7396083247958788,\n",
       "  0.7384689915873668,\n",
       "  0.7383819139895347,\n",
       "  0.7382073990195742,\n",
       "  0.7378254707830434,\n",
       "  0.7377064138898559,\n",
       "  0.7373056353954728,\n",
       "  0.7374693279121061,\n",
       "  0.7366111935671017,\n",
       "  0.7372921865402496,\n",
       "  0.7367331192433999,\n",
       "  0.7358797701111791,\n",
       "  0.7356353770332653,\n",
       "  0.7352226482203793,\n",
       "  0.7351649472918207,\n",
       "  0.735038668495136,\n",
       "  0.7345882073003499,\n",
       "  0.7348598867572246,\n",
       "  0.7348306576961295,\n",
       "  0.7340189396839722,\n",
       "  0.7341829510276667,\n",
       "  0.7337448747204282,\n",
       "  0.7338080957655762,\n",
       "  0.733219530113516,\n",
       "  0.7330381290734309,\n",
       "  0.7329260221478682,\n",
       "  0.7333715371477967,\n",
       "  0.7326746637471164,\n",
       "  0.732329926497388,\n",
       "  0.7322432899078835,\n",
       "  0.7323843037652837,\n",
       "  0.7324475479258065,\n",
       "  0.7319101126570451,\n",
       "  0.7318413109000039,\n",
       "  0.7318213354187328,\n",
       "  0.7319731535673802,\n",
       "  0.7316640300103502,\n",
       "  0.7313063329606836,\n",
       "  0.7311777614490478,\n",
       "  0.7311765611006612,\n",
       "  0.7307195658498854,\n",
       "  0.7306853207524794,\n",
       "  0.7313897723636469,\n",
       "  0.7312008736866663,\n",
       "  0.7312175790028559,\n",
       "  0.7313037831697438,\n",
       "  0.730939183043641,\n",
       "  0.7309892060353815,\n",
       "  0.7299540967492185,\n",
       "  0.7301046177951253,\n",
       "  0.730229613523404,\n",
       "  0.7294393221427199,\n",
       "  0.7294285867022675,\n",
       "  0.7297390324587307,\n",
       "  0.7298888174450628,\n",
       "  0.7299711225435674,\n",
       "  0.729981311305408,\n",
       "  0.7295424441221348,\n",
       "  0.7295650902547335,\n",
       "  0.7292750157477783,\n",
       "  0.7287966258968342,\n",
       "  0.7294444856220995,\n",
       "  0.7298087439708763,\n",
       "  0.7300669045659643,\n",
       "  0.7293339408005374,\n",
       "  0.7295752705959733,\n",
       "  0.7314005081343189,\n",
       "  0.7334380661681749,\n",
       "  0.7303090329975963,\n",
       "  0.7312713210932766,\n",
       "  0.7335147221993211,\n",
       "  0.7328971355245384,\n",
       "  0.7331521866063992,\n",
       "  0.7334098611181793,\n",
       "  0.7310208103663374,\n",
       "  0.7305506563582909,\n",
       "  0.7298616137861215,\n",
       "  0.729304235067394,\n",
       "  0.7285008529546848,\n",
       "  0.7280038402021096,\n",
       "  0.7276374222168962,\n",
       "  0.7277264058424825,\n",
       "  0.7293061391138304,\n",
       "  0.7306996696213276,\n",
       "  0.7325081605660287,\n",
       "  0.7299723455119992,\n",
       "  0.7287706788887277,\n",
       "  0.728049054700582,\n",
       "  0.7266982040907207,\n",
       "  0.7265526180122037,\n",
       "  0.7269668742560284,\n",
       "  0.7272258235807234,\n",
       "  0.7288152758764759,\n",
       "  0.7308749866947903,\n",
       "  0.7299640612919245,\n",
       "  0.7308688200081485,\n",
       "  0.7313490934319113,\n",
       "  0.7287349603512941,\n",
       "  0.7287181604271781,\n",
       "  0.7292153987197665,\n",
       "  0.7298712481091888,\n",
       "  0.728649608976623,\n",
       "  0.7271891584686956,\n",
       "  0.7283465307505177,\n",
       "  0.7278014005386269,\n",
       "  0.7267639237427646,\n",
       "  0.726692106750203,\n",
       "  0.7262359916998739,\n",
       "  0.7268212360028085,\n",
       "  0.7277786347014091,\n",
       "  0.7272223798852218,\n",
       "  0.7277878484897666,\n",
       "  0.728595595775879,\n",
       "  0.7265337930161537,\n",
       "  0.7266570613655027,\n",
       "  0.7284535154741557,\n",
       "  0.7317194231989641,\n",
       "  0.7353221869864952,\n",
       "  0.7366005101362424,\n",
       "  0.7307340195634688,\n",
       "  0.7282915840188552,\n",
       "  0.7267332012633537,\n",
       "  0.7272839580876675,\n",
       "  0.7276370214953647,\n",
       "  0.7274112858270344,\n",
       "  0.7263759425802574,\n",
       "  0.7266331466941622,\n",
       "  0.7260539150304081,\n",
       "  0.725207494896865,\n",
       "  0.724815733049715,\n",
       "  0.7257243074538635,\n",
       "  0.7253615596948237,\n",
       "  0.7247233570777809,\n",
       "  0.7249168808110202,\n",
       "  0.7251450079299736,\n",
       "  0.7256074430539667,\n",
       "  0.7253668976952825,\n",
       "  0.7253736269110788,\n",
       "  0.7250275152872143,\n",
       "  0.7249370829880732,\n",
       "  0.7250308387827675,\n",
       "  0.7244548866953546,\n",
       "  0.7243211300749528,\n",
       "  0.7243175988712469,\n",
       "  0.7241013788120239,\n",
       "  0.7240509947251085,\n",
       "  0.724091169246346,\n",
       "  0.7244901044547063,\n",
       "  0.7244291868566476,\n",
       "  0.7247323631579856,\n",
       "  0.725165950954786,\n",
       "  0.7248204226639132,\n",
       "  0.7252216532289817,\n",
       "  0.7255966978720351,\n",
       "  0.726182557704376,\n",
       "  0.7261994104636343,\n",
       "  0.7263639399219418,\n",
       "  0.7260203832074216,\n",
       "  0.7256594110393788,\n",
       "  0.7257163853196226,\n",
       "  0.7254063690137995,\n",
       "  0.7244037300595947,\n",
       "  0.7240909037497565,\n",
       "  0.7242289013809775,\n",
       "  0.7240866600971803,\n",
       "  0.7250823931680822,\n",
       "  0.7258910786081879,\n",
       "  0.7271773993473634,\n",
       "  0.7289754188622134,\n",
       "  0.7308340840392495,\n",
       "  0.730435959685212,\n",
       "  0.7286478016845407,\n",
       "  0.7273535538578297,\n",
       "  0.7247646000246593,\n",
       "  0.7244523918859846,\n",
       "  0.7248735683776665,\n",
       "  0.7258059763842343,\n",
       "  0.7259113562404284,\n",
       "  0.7254609356626579,\n",
       "  0.7253514630642625,\n",
       "  0.7253553015373421,\n",
       "  0.7250284162914984,\n",
       "  0.7242939984038926,\n",
       "  0.7245623936613511,\n",
       "  0.725027663720942,\n",
       "  0.7242815099924885,\n",
       "  0.7243345921389615,\n",
       "  0.7233269792514495,\n",
       "  0.7230261011136866,\n",
       "  0.7229476732890692,\n",
       "  0.7228461135787647,\n",
       "  0.7232182754043727,\n",
       "  0.7231810419843467,\n",
       "  0.7229808852943357,\n",
       "  0.7227529865222625,\n",
       "  0.7229162601552842,\n",
       "  0.7229301848570066,\n",
       "  0.7232122454286612,\n",
       "  0.7229092707264126,\n",
       "  0.7227525258658666,\n",
       "  0.7227545682743316,\n",
       "  0.7227533230160742,\n",
       "  0.723191446709831,\n",
       "  0.7228970379050088,\n",
       "  0.7228648160633288,\n",
       "  0.7228283083009588,\n",
       "  0.7227490575690019,\n",
       "  0.7223939656220645,\n",
       "  0.722583297547211,\n",
       "  0.7229618028920773,\n",
       "  0.7233841089000332,\n",
       "  0.7231277691360326,\n",
       "  0.7229189313019412,\n",
       "  0.7229670651071289,\n",
       "  0.7235180549013978,\n",
       "  0.7243029011253505,\n",
       "  0.7247728122568526,\n",
       "  0.723895517742865,\n",
       "  0.7235657463773796,\n",
       "  0.7234663063469355,\n",
       "  0.7234675978359423,\n",
       "  0.723138042103881,\n",
       "  0.722853322603696,\n",
       "  0.723550601018763,\n",
       "  0.7232185787111108,\n",
       "  0.7230532329498566,\n",
       "  0.7228010889565846,\n",
       "  0.7223732393864449,\n",
       "  0.7227346439771044,\n",
       "  0.7232246545873521,\n",
       "  0.7231852379201852,\n",
       "  0.7249012435902519,\n",
       "  0.7263695072599395,\n",
       "  0.7283258010475919,\n",
       "  0.727797235973654],\n",
       " 'train_VAL_loss': [1.9992596548349904,\n",
       "  1.8534058507459645,\n",
       "  1.7388558962338518,\n",
       "  1.7010312126614051,\n",
       "  1.6883885758075028,\n",
       "  1.679924990661917,\n",
       "  1.6714080076798838,\n",
       "  1.6638898829674127,\n",
       "  1.6586003095489459,\n",
       "  1.6547908191865832,\n",
       "  1.651838499753429,\n",
       "  1.648395194901654,\n",
       "  1.6459720408817407,\n",
       "  1.643724456388204,\n",
       "  1.6409738040696882,\n",
       "  1.6392127057521957,\n",
       "  1.6373842931520246,\n",
       "  1.6356257031829073,\n",
       "  1.6341173642229836,\n",
       "  1.6327492606276621,\n",
       "  1.631828456043867,\n",
       "  1.630595743160829,\n",
       "  1.629612270154451,\n",
       "  1.6285647510491579,\n",
       "  1.6279436112110635,\n",
       "  1.6271286697599037,\n",
       "  1.626262467323578,\n",
       "  1.6257326008540443,\n",
       "  1.6249901284141224,\n",
       "  1.6244672328811602,\n",
       "  1.6239575880716381,\n",
       "  1.623326761240444,\n",
       "  1.6228608289253679,\n",
       "  1.6224600605026838,\n",
       "  1.6220917123836824,\n",
       "  1.6216266937863464,\n",
       "  1.6210879794118147,\n",
       "  1.6207028452379222,\n",
       "  1.6206919252046919,\n",
       "  1.620117875677727,\n",
       "  1.619651531908981,\n",
       "  1.6195250201423412,\n",
       "  1.6188407578296609,\n",
       "  1.618546873248515,\n",
       "  1.6181421956857487,\n",
       "  1.6179768058401727,\n",
       "  1.6180978977118834,\n",
       "  1.6174682322631582,\n",
       "  1.617227462850449,\n",
       "  1.6168072818058679,\n",
       "  1.6166669156742888,\n",
       "  1.6162871412953512,\n",
       "  1.6158557010819707,\n",
       "  1.6154091784168148,\n",
       "  1.615216005211722,\n",
       "  1.61505846138476,\n",
       "  1.6147889377667963,\n",
       "  1.6144255419517157,\n",
       "  1.614319777884972,\n",
       "  1.6139421971550938,\n",
       "  1.6140103399588461,\n",
       "  1.6140014520972719,\n",
       "  1.6136697593488192,\n",
       "  1.613207912841332,\n",
       "  1.6130122507708224,\n",
       "  1.6126371463580145,\n",
       "  1.6123471996460594,\n",
       "  1.6120708170359814,\n",
       "  1.6120115138817361,\n",
       "  1.6117944364072214,\n",
       "  1.6115074121390685,\n",
       "  1.611314772899131,\n",
       "  1.6112440312668228,\n",
       "  1.6111369420286692,\n",
       "  1.611190248064057,\n",
       "  1.610839849363734,\n",
       "  1.610294990592386,\n",
       "  1.6102219463385374,\n",
       "  1.61009021445985,\n",
       "  1.6099268567859302,\n",
       "  1.6097973796469354,\n",
       "  1.6096548542091391,\n",
       "  1.6097782602600774,\n",
       "  1.6094180144101298,\n",
       "  1.6090318809916109,\n",
       "  1.6089959309702104,\n",
       "  1.6090086837224353,\n",
       "  1.6090390487390873,\n",
       "  1.6086879384154429,\n",
       "  1.6083208877294017,\n",
       "  1.6083139663257757,\n",
       "  1.6083846425745956,\n",
       "  1.608005747240336,\n",
       "  1.6076641561582148,\n",
       "  1.6074991533300553,\n",
       "  1.6075157587548041,\n",
       "  1.6074194419417025,\n",
       "  1.6072567869751737,\n",
       "  1.6072981730722655,\n",
       "  1.6072697167251249,\n",
       "  1.6071526357011452,\n",
       "  1.606747054987667,\n",
       "  1.6066802320718105,\n",
       "  1.6066008267970626,\n",
       "  1.6064962640693643,\n",
       "  1.6064004544736277,\n",
       "  1.6063189849959185,\n",
       "  1.60650718839545,\n",
       "  1.6063628216529486,\n",
       "  1.606541355891241,\n",
       "  1.6065393805173624,\n",
       "  1.6062583018538035,\n",
       "  1.6057409041476052,\n",
       "  1.6054127239454485,\n",
       "  1.6056565779398022,\n",
       "  1.605260570623868,\n",
       "  1.6053867974109597,\n",
       "  1.605335146106181,\n",
       "  1.6051832021438515,\n",
       "  1.6052122829363287,\n",
       "  1.6048694680602267,\n",
       "  1.6050508841253053,\n",
       "  1.6047854479660288,\n",
       "  1.6050042987200033,\n",
       "  1.6047170030113072,\n",
       "  1.604717291953491,\n",
       "  1.6044048394522838,\n",
       "  1.6042299257421098,\n",
       "  1.604115958028883,\n",
       "  1.6040911591944602,\n",
       "  1.6044611452028692,\n",
       "  1.6045181533306259,\n",
       "  1.6045875404019765,\n",
       "  1.6046258273877596,\n",
       "  1.604597869011834,\n",
       "  1.6041454638140353,\n",
       "  1.6039174451722333,\n",
       "  1.6037487088808393,\n",
       "  1.6036420700622727,\n",
       "  1.603789371136483,\n",
       "  1.6038857097440808,\n",
       "  1.603786821510653,\n",
       "  1.6040118787427358,\n",
       "  1.6044892511869733,\n",
       "  1.6060641921458152,\n",
       "  1.608543578607554,\n",
       "  1.611085522538077,\n",
       "  1.6137675093151527,\n",
       "  1.61417878400586,\n",
       "  1.6161708689792664,\n",
       "  1.6156755586740383,\n",
       "  1.6126132090666287,\n",
       "  1.6096852089889822,\n",
       "  1.6087332214344903,\n",
       "  1.6071577511335675,\n",
       "  1.6078454434376344,\n",
       "  1.6073288204267084,\n",
       "  1.6065754626266184,\n",
       "  1.6063312265681429,\n",
       "  1.6053981856956376,\n",
       "  1.6056417372088023,\n",
       "  1.606915128858466,\n",
       "  1.6069442294641214,\n",
       "  1.6089312221204806,\n",
       "  1.609852678888062,\n",
       "  1.6098707161451642,\n",
       "  1.61150603083032,\n",
       "  1.6130699972696911,\n",
       "  1.6151066202866404,\n",
       "  1.6169089584139245,\n",
       "  1.6172729397084245,\n",
       "  1.6173427603581605,\n",
       "  1.6160343049966066,\n",
       "  1.612284115146732,\n",
       "  1.6094975514425136,\n",
       "  1.6061756845326305,\n",
       "  1.6041404018771945,\n",
       "  1.6027706755825688,\n",
       "  1.6037482700189396,\n",
       "  1.6087133933962878,\n",
       "  1.619743457461328,\n",
       "  1.6328416537710173,\n",
       "  1.6341438498193208,\n",
       "  1.6206792695370407,\n",
       "  1.6084681356382502,\n",
       "  1.6065308542462928,\n",
       "  1.6075476817477112,\n",
       "  1.6066479048900657,\n",
       "  1.6053379480198149,\n",
       "  1.6051729359455056,\n",
       "  1.605408986849798,\n",
       "  1.604943660157539,\n",
       "  1.6034283278061081,\n",
       "  1.6024556625582835,\n",
       "  1.6011616503432846,\n",
       "  1.6004272438482565,\n",
       "  1.6002743848473082,\n",
       "  1.6002790551436574,\n",
       "  1.600327986099053,\n",
       "  1.6004347883763406,\n",
       "  1.6010049195170732,\n",
       "  1.6012626608983302,\n",
       "  1.6014884449438376,\n",
       "  1.6019439046732937,\n",
       "  1.601637015052119,\n",
       "  1.6021256344470292,\n",
       "  1.6027783888528881,\n",
       "  1.6022291199982661,\n",
       "  1.6011646738342962,\n",
       "  1.6000430045695846,\n",
       "  1.5996969218399386,\n",
       "  1.5995519055554082,\n",
       "  1.5994314206273932,\n",
       "  1.5992989807578004,\n",
       "  1.599387977922392,\n",
       "  1.5994149687547763,\n",
       "  1.5994470522344277,\n",
       "  1.5995644162566378,\n",
       "  1.5998264666739592,\n",
       "  1.6001793813177092,\n",
       "  1.6005915816140637,\n",
       "  1.600491642291526,\n",
       "  1.6004423147423446,\n",
       "  1.6010350226695518,\n",
       "  1.6009953487612865,\n",
       "  1.6004490148988126,\n",
       "  1.6003082359926852,\n",
       "  1.5997761270015853,\n",
       "  1.59956679945177,\n",
       "  1.59912045411456,\n",
       "  1.5987512243091235,\n",
       "  1.5987248787259132,\n",
       "  1.5988383312965033,\n",
       "  1.5993773124885031,\n",
       "  1.5999603538962281,\n",
       "  1.6004200097266326,\n",
       "  1.6012238244270685,\n",
       "  1.602163992760254,\n",
       "  1.6025725078054411,\n",
       "  1.6024831143772833,\n",
       "  1.6027407811289018,\n",
       "  1.601814528911728,\n",
       "  1.6007651229314197,\n",
       "  1.6000460359858675,\n",
       "  1.5993338757274553,\n",
       "  1.5987264010384472,\n",
       "  1.5983191272888817,\n",
       "  1.5985795464211885,\n",
       "  1.5993219085677508,\n",
       "  1.5998107990729842,\n",
       "  1.6002040900021708,\n",
       "  1.600156125269438,\n",
       "  1.5995411585572683,\n",
       "  1.5995563995144704,\n",
       "  1.5993241722233738,\n",
       "  1.5988884738277531,\n",
       "  1.5985087176108954,\n",
       "  1.5982397005498574,\n",
       "  1.5980153341372587,\n",
       "  1.5978941293304316,\n",
       "  1.5977841123649619,\n",
       "  1.5978419229264404,\n",
       "  1.597835879246614,\n",
       "  1.5980548861944774,\n",
       "  1.5981458484961386,\n",
       "  1.598198971259627,\n",
       "  1.59834548409956,\n",
       "  1.5987089575162554,\n",
       "  1.5989391691466779,\n",
       "  1.5990638115399431,\n",
       "  1.5986931016900863,\n",
       "  1.5985479856792248,\n",
       "  1.598650830604363,\n",
       "  1.5988552722244052,\n",
       "  1.5984564606832996,\n",
       "  1.5983275670424062,\n",
       "  1.5981219571052827,\n",
       "  1.5980900321310576,\n",
       "  1.5983006105528643,\n",
       "  1.5985442499044529,\n",
       "  1.5983220352030196,\n",
       "  1.5981387626431325,\n",
       "  1.5978812808145115,\n",
       "  1.5975184358057883,\n",
       "  1.597812654899428,\n",
       "  1.5982190978163828,\n",
       "  1.5981831448230057,\n",
       "  1.5983520911340898,\n",
       "  1.5991938394852954,\n",
       "  1.6005977161042908,\n",
       "  1.6005785854899652,\n",
       "  1.6000014969516658,\n",
       "  1.5995781586771196,\n",
       "  1.599636060048999,\n",
       "  1.5995530553801898,\n",
       "  1.600078907039357,\n",
       "  1.5995839665801241,\n",
       "  1.5981537840042748,\n",
       "  1.5984265853823718,\n",
       "  1.5989003036160878],\n",
       " 'train_ATT_acc': [28.25484764542936,\n",
       "  47.9224376731302,\n",
       "  59.83379501385041,\n",
       "  72.85318559556787,\n",
       "  76.73130193905817,\n",
       "  80.88642659279779,\n",
       "  85.31855955678671,\n",
       "  86.98060941828255,\n",
       "  90.30470914127424,\n",
       "  91.41274238227147,\n",
       "  93.07479224376732,\n",
       "  95.01385041551247,\n",
       "  95.29085872576178,\n",
       "  95.56786703601108,\n",
       "  96.95290858725762,\n",
       "  98.06094182825485,\n",
       "  98.33795013850416,\n",
       "  98.33795013850416,\n",
       "  98.61495844875347,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  99.16897506925208,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0],\n",
       " 'train_VAL_acc': [74.23822714681441,\n",
       "  89.47368421052632,\n",
       "  91.41274238227147,\n",
       "  94.18282548476455,\n",
       "  94.18282548476455,\n",
       "  95.56786703601108,\n",
       "  96.95290858725762,\n",
       "  97.78393351800554,\n",
       "  97.50692520775624,\n",
       "  98.06094182825485,\n",
       "  97.50692520775624,\n",
       "  97.50692520775624,\n",
       "  97.78393351800554,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.16897506925208,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  98.06094182825485,\n",
       "  98.06094182825485,\n",
       "  98.89196675900277,\n",
       "  99.44598337950139,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0],\n",
       " 'train_VAL_jac': [0.09695290858725762,\n",
       "  0.2423822754307797,\n",
       "  0.41551246801571834,\n",
       "  0.5221606727452159,\n",
       "  0.5383194978877778,\n",
       "  0.5517082320025753,\n",
       "  0.6029547693986972,\n",
       "  0.6343490410379425,\n",
       "  0.6514312184088118,\n",
       "  0.6694367585750167,\n",
       "  0.6777470025989818,\n",
       "  0.689289016406622,\n",
       "  0.7086796060493448,\n",
       "  0.6966759029187655,\n",
       "  0.7119113652659915,\n",
       "  0.7197599384593171,\n",
       "  0.7326869911764468,\n",
       "  0.7409972404839259,\n",
       "  0.7409972404839259,\n",
       "  0.7423822820351725,\n",
       "  0.7442290023423298,\n",
       "  0.7409972404839259,\n",
       "  0.7433056448305082,\n",
       "  0.753000935689234,\n",
       "  0.7465374066889121,\n",
       "  0.7493074897914052,\n",
       "  0.753000935689234,\n",
       "  0.7590027806501309,\n",
       "  0.7571560603429737,\n",
       "  0.7603878222013775,\n",
       "  0.7631579053038705,\n",
       "  0.764542946855117,\n",
       "  0.7613111849967132,\n",
       "  0.7631579053038705,\n",
       "  0.7631579053038705,\n",
       "  0.7714681546113498,\n",
       "  0.7659279884063637,\n",
       "  0.7700831130601032,\n",
       "  0.7714681546113498,\n",
       "  0.7742382377138428,\n",
       "  0.764542946855117,\n",
       "  0.7714681546113498,\n",
       "  0.7802400826747398,\n",
       "  0.7816251242259863,\n",
       "  0.7857802488797259,\n",
       "  0.7839335285725686,\n",
       "  0.779778403918829,\n",
       "  0.7825484870213221,\n",
       "  0.7963989025337874,\n",
       "  0.7880886532263082,\n",
       "  0.7834718498166579,\n",
       "  0.789012016021644,\n",
       "  0.7950138609825409,\n",
       "  0.7982456228409448,\n",
       "  0.7922437778800479,\n",
       "  0.7959372237778767,\n",
       "  0.7908587363288013,\n",
       "  0.7996306643921913,\n",
       "  0.7963989025337874,\n",
       "  0.7991689856362805,\n",
       "  0.8000923431481021,\n",
       "  0.8042474678018416,\n",
       "  0.8000923431481021,\n",
       "  0.797783944085034,\n",
       "  0.797322260045609,\n",
       "  0.7987073015968555,\n",
       "  0.8033241102900202,\n",
       "  0.8116343595974993,\n",
       "  0.8060941933925132,\n",
       "  0.8047091518412667,\n",
       "  0.8102493180462528,\n",
       "  0.8116343595974993,\n",
       "  0.8060941933925132,\n",
       "  0.8120960383534102,\n",
       "  0.8024007527781986,\n",
       "  0.8116343595974993,\n",
       "  0.8130194011487459,\n",
       "  0.8116343595974993,\n",
       "  0.8171745258024855,\n",
       "  0.8199446089049786,\n",
       "  0.8148661267394174,\n",
       "  0.8259464591493897,\n",
       "  0.8194829301490678,\n",
       "  0.8264081379053005,\n",
       "  0.8328716616221082,\n",
       "  0.8301015785196151,\n",
       "  0.8259464538658755,\n",
       "  0.8287165369683686,\n",
       "  0.8264081379053005,\n",
       "  0.8277931794565471,\n",
       "  0.8259464538658755,\n",
       "  0.8250230963540539,\n",
       "  0.8264081379053005,\n",
       "  0.8361034287640262,\n",
       "  0.8416435949690124,\n",
       "  0.8397968693783409,\n",
       "  0.8384118278270943,\n",
       "  0.8439519940320804,\n",
       "  0.8287165369683686,\n",
       "  0.8356417447246013,\n",
       "  0.8356417447246013,\n",
       "  0.8425669524808339,\n",
       "  0.8361034287640262,\n",
       "  0.8328716616221082,\n",
       "  0.8370267862758478,\n",
       "  0.8370267862758478,\n",
       "  0.8384118278270943,\n",
       "  0.8314866200708616,\n",
       "  0.827331495417122,\n",
       "  0.8291782157242793,\n",
       "  0.8254847751099648,\n",
       "  0.8282548582124578,\n",
       "  0.8370267862758478,\n",
       "  0.8370267862758478,\n",
       "  0.8310249413149509,\n",
       "  0.8416435896854981,\n",
       "  0.8347183819292655,\n",
       "  0.8402585481342516,\n",
       "  0.8457987143392378,\n",
       "  0.8416435896854981,\n",
       "  0.8434903152761697,\n",
       "  0.8462603983786627,\n",
       "  0.8448753568274162,\n",
       "  0.8374884650317586,\n",
       "  0.8305632572755259,\n",
       "  0.833333340378019,\n",
       "  0.8416435896854981,\n",
       "  0.8444136727879913,\n",
       "  0.8397968640948267,\n",
       "  0.8374884650317586,\n",
       "  0.832871656338594,\n",
       "  0.832871656338594,\n",
       "  0.8365651022364228,\n",
       "  0.836103423480512,\n",
       "  0.8347183819292655,\n",
       "  0.8379501490711836,\n",
       "  0.8457987143392378,\n",
       "  0.8457987143392378,\n",
       "  0.8462603983786627,\n",
       "  0.8494921602370666,\n",
       "  0.8467220771345736,\n",
       "  0.8425669524808339,\n",
       "  0.8370267862758478,\n",
       "  0.8342567031733548,\n",
       "  0.8328716616221082,\n",
       "  0.815789484251239,\n",
       "  0.7982456228409448,\n",
       "  0.7940904981872051,\n",
       "  0.7871652904309725,\n",
       "  0.7746999164697537,\n",
       "  0.7774699995722467,\n",
       "  0.7940904981872051,\n",
       "  0.8088642764950063,\n",
       "  0.8130194011487459,\n",
       "  0.8204062876608893,\n",
       "  0.818097883314307,\n",
       "  0.8222530079680467,\n",
       "  0.8208679664168002,\n",
       "  0.8157894789677247,\n",
       "  0.8254847698264505,\n",
       "  0.8282548529289436,\n",
       "  0.8287165316848544,\n",
       "  0.8190212408261286,\n",
       "  0.8051708305971774,\n",
       "  0.7954755397384516,\n",
       "  0.8019390687387736,\n",
       "  0.7945521822266302,\n",
       "  0.7825484870213221,\n",
       "  0.7700831130601032,\n",
       "  0.7719298333672605,\n",
       "  0.7737765536744179,\n",
       "  0.7668513459181852,\n",
       "  0.7700831130601032,\n",
       "  0.7894736947775548,\n",
       "  0.807017556187849,\n",
       "  0.8291782157242793,\n",
       "  0.836103423480512,\n",
       "  0.8614958501588604,\n",
       "  0.8388735065830051,\n",
       "  0.8240997335587182,\n",
       "  0.7626962212644456,\n",
       "  0.7068328857421875,\n",
       "  0.7059095229468517,\n",
       "  0.7728531908790821,\n",
       "  0.8397968640948267,\n",
       "  0.8416435896854981,\n",
       "  0.835641739441087,\n",
       "  0.8416435896854981,\n",
       "  0.8457987143392378,\n",
       "  0.8370267862758478,\n",
       "  0.8319482988267725,\n",
       "  0.833333340378019,\n",
       "  0.8481071186858201,\n",
       "  0.8642659332613536,\n",
       "  0.8730378613247436,\n",
       "  0.8831948362228943,\n",
       "  0.8873499608766339,\n",
       "  0.890120043979127,\n",
       "  0.8878116396325447,\n",
       "  0.8771929912619973,\n",
       "  0.866112658852025,\n",
       "  0.8647276173007785,\n",
       "  0.8591874510957924,\n",
       "  0.8508772017883132,\n",
       "  0.8550323264420527,\n",
       "  0.8467220771345736,\n",
       "  0.8411819109295874,\n",
       "  0.8499538389929774,\n",
       "  0.8614958501588604,\n",
       "  0.8762696284666616,\n",
       "  0.8767313072225724,\n",
       "  0.883656514978805,\n",
       "  0.8845798724906266,\n",
       "  0.880886431876312,\n",
       "  0.8818097893881336,\n",
       "  0.8762696231831474,\n",
       "  0.8850415565300516,\n",
       "  0.880424747836887,\n",
       "  0.8790397062856404,\n",
       "  0.872114503812922,\n",
       "  0.8711911410175862,\n",
       "  0.8670360163638466,\n",
       "  0.8674976951197574,\n",
       "  0.8725761825688327,\n",
       "  0.8684210579150932,\n",
       "  0.8725761825688327,\n",
       "  0.8767313072225724,\n",
       "  0.8859649140418732,\n",
       "  0.8827331521834693,\n",
       "  0.889658359939702,\n",
       "  0.8850415565300516,\n",
       "  0.8905817227350378,\n",
       "  0.8896583652232162,\n",
       "  0.8767313125060866,\n",
       "  0.8698061047498539,\n",
       "  0.8674977004032716,\n",
       "  0.8656509800961143,\n",
       "  0.845798719622752,\n",
       "  0.8388735118665194,\n",
       "  0.8379501490711836,\n",
       "  0.8384118278270943,\n",
       "  0.8434903152761697,\n",
       "  0.8610341766864639,\n",
       "  0.8684210631986073,\n",
       "  0.872576187852347,\n",
       "  0.8781163487738189,\n",
       "  0.8873499555931197,\n",
       "  0.8845798724906266,\n",
       "  0.8716528197734971,\n",
       "  0.8776546700179082,\n",
       "  0.8771929859784832,\n",
       "  0.8753462656713259,\n",
       "  0.8707294622616755,\n",
       "  0.8711911410175862,\n",
       "  0.8767313072225724,\n",
       "  0.880886431876312,\n",
       "  0.8868882768372089,\n",
       "  0.8910434014909485,\n",
       "  0.889658359939702,\n",
       "  0.8933518058375308,\n",
       "  0.8979686145306954,\n",
       "  0.8979686145306954,\n",
       "  0.8896583652232162,\n",
       "  0.8841181990182301,\n",
       "  0.8845798777741408,\n",
       "  0.8767313072225724,\n",
       "  0.8781163487738189,\n",
       "  0.8670360163638466,\n",
       "  0.8679593791591824,\n",
       "  0.8684210579150932,\n",
       "  0.872114503812922,\n",
       "  0.8767313072225724,\n",
       "  0.8642659332613536,\n",
       "  0.8670360163638466,\n",
       "  0.8767313072225724,\n",
       "  0.8725761825688327,\n",
       "  0.8790397062856404,\n",
       "  0.880424747836887,\n",
       "  0.8767313072225724,\n",
       "  0.8698060994663397,\n",
       "  0.8795013903250655,\n",
       "  0.8767313072225724,\n",
       "  0.8734995400806543,\n",
       "  0.8938134845934416,\n",
       "  0.8947368473887773,\n",
       "  0.8970452464518455,\n",
       "  0.8975069304912705,\n",
       "  0.8878116343490304,\n",
       "  0.8753462603878116,\n",
       "  0.8725761772853186,\n",
       "  0.8656509695290858,\n",
       "  0.8822714681440443,\n",
       "  0.8836565096952909,\n",
       "  0.8850415512465374,\n",
       "  0.8905817174515236,\n",
       "  0.8767313019390581,\n",
       "  0.8836565096952909,\n",
       "  0.8928901217981059,\n",
       "  0.8831948309393801,\n",
       "  0.8739612241200793],\n",
       " 'train_VAL_acc_1': [26.31578947368421,\n",
       "  49.03047091412743,\n",
       "  55.6786703601108,\n",
       "  63.988919667590025,\n",
       "  64.81994459833795,\n",
       "  67.03601108033241,\n",
       "  70.08310249307479,\n",
       "  71.46814404432133,\n",
       "  73.9612188365651,\n",
       "  72.02216066481995,\n",
       "  73.6842105263158,\n",
       "  72.85318559556787,\n",
       "  74.23822714681441,\n",
       "  73.40720221606648,\n",
       "  76.45429362880887,\n",
       "  76.17728531855956,\n",
       "  77.5623268698061,\n",
       "  78.39335180055402,\n",
       "  78.94736842105263,\n",
       "  79.22437673130194,\n",
       "  77.28531855955679,\n",
       "  78.67036011080333,\n",
       "  79.77839335180056,\n",
       "  80.05540166204986,\n",
       "  79.50138504155125,\n",
       "  79.77839335180056,\n",
       "  80.60941828254848,\n",
       "  79.22437673130194,\n",
       "  80.88642659279779,\n",
       "  80.33240997229917,\n",
       "  80.88642659279779,\n",
       "  81.16343490304709,\n",
       "  81.16343490304709,\n",
       "  81.16343490304709,\n",
       "  81.16343490304709,\n",
       "  81.99445983379502,\n",
       "  82.54847645429363,\n",
       "  81.4404432132964,\n",
       "  81.4404432132964,\n",
       "  82.82548476454294,\n",
       "  81.99445983379502,\n",
       "  83.37950138504155,\n",
       "  83.65650969529086,\n",
       "  83.37950138504155,\n",
       "  83.37950138504155,\n",
       "  81.7174515235457,\n",
       "  81.7174515235457,\n",
       "  83.10249307479225,\n",
       "  83.10249307479225,\n",
       "  83.65650969529086,\n",
       "  83.10249307479225,\n",
       "  82.27146814404432,\n",
       "  83.93351800554017,\n",
       "  85.0415512465374,\n",
       "  84.48753462603878,\n",
       "  84.21052631578948,\n",
       "  83.93351800554017,\n",
       "  84.7645429362881,\n",
       "  84.48753462603878,\n",
       "  84.48753462603878,\n",
       "  83.37950138504155,\n",
       "  83.93351800554017,\n",
       "  84.21052631578948,\n",
       "  84.7645429362881,\n",
       "  84.21052631578948,\n",
       "  85.87257617728532,\n",
       "  85.59556786703601,\n",
       "  85.59556786703601,\n",
       "  85.0415512465374,\n",
       "  84.7645429362881,\n",
       "  85.59556786703601,\n",
       "  85.59556786703601,\n",
       "  85.0415512465374,\n",
       "  85.0415512465374,\n",
       "  84.7645429362881,\n",
       "  85.59556786703601,\n",
       "  85.87257617728532,\n",
       "  86.42659279778394,\n",
       "  85.87257617728532,\n",
       "  86.42659279778394,\n",
       "  86.42659279778394,\n",
       "  85.87257617728532,\n",
       "  84.7645429362881,\n",
       "  85.59556786703601,\n",
       "  87.53462603878117,\n",
       "  86.42659279778394,\n",
       "  85.87257617728532,\n",
       "  84.21052631578948,\n",
       "  85.87257617728532,\n",
       "  87.53462603878117,\n",
       "  87.25761772853186,\n",
       "  85.59556786703601,\n",
       "  86.98060941828255,\n",
       "  87.25761772853186,\n",
       "  86.70360110803324,\n",
       "  86.42659279778394,\n",
       "  86.70360110803324,\n",
       "  87.25761772853186,\n",
       "  86.98060941828255,\n",
       "  86.42659279778394,\n",
       "  87.81163434903047,\n",
       "  86.70360110803324,\n",
       "  86.14958448753463,\n",
       "  86.42659279778394,\n",
       "  86.42659279778394,\n",
       "  86.42659279778394,\n",
       "  86.42659279778394,\n",
       "  86.70360110803324,\n",
       "  86.14958448753463,\n",
       "  85.59556786703601,\n",
       "  85.87257617728532,\n",
       "  85.87257617728532,\n",
       "  86.70360110803324,\n",
       "  86.70360110803324,\n",
       "  86.98060941828255,\n",
       "  87.53462603878117,\n",
       "  86.70360110803324,\n",
       "  86.42659279778394,\n",
       "  86.70360110803324,\n",
       "  86.42659279778394,\n",
       "  86.98060941828255,\n",
       "  86.98060941828255,\n",
       "  86.98060941828255,\n",
       "  87.25761772853186,\n",
       "  88.08864265927978,\n",
       "  88.08864265927978,\n",
       "  88.6426592797784,\n",
       "  89.19667590027701,\n",
       "  88.6426592797784,\n",
       "  88.36565096952909,\n",
       "  89.19667590027701,\n",
       "  87.81163434903047,\n",
       "  87.81163434903047,\n",
       "  87.81163434903047,\n",
       "  88.36565096952909,\n",
       "  88.9196675900277,\n",
       "  89.19667590027701,\n",
       "  89.75069252077563,\n",
       "  89.19667590027701,\n",
       "  89.19667590027701,\n",
       "  88.9196675900277,\n",
       "  88.08864265927978,\n",
       "  86.42659279778394,\n",
       "  86.14958448753463,\n",
       "  85.59556786703601,\n",
       "  84.21052631578948,\n",
       "  83.93351800554017,\n",
       "  81.4404432132964,\n",
       "  81.4404432132964,\n",
       "  80.60941828254848,\n",
       "  79.77839335180056,\n",
       "  82.27146814404432,\n",
       "  83.93351800554017,\n",
       "  84.21052631578948,\n",
       "  85.0415512465374,\n",
       "  85.87257617728532,\n",
       "  86.42659279778394,\n",
       "  86.70360110803324,\n",
       "  86.70360110803324,\n",
       "  87.25761772853186,\n",
       "  86.98060941828255,\n",
       "  86.42659279778394,\n",
       "  86.70360110803324,\n",
       "  83.65650969529086,\n",
       "  83.10249307479225,\n",
       "  83.10249307479225,\n",
       "  82.54847645429363,\n",
       "  81.16343490304709,\n",
       "  80.33240997229917,\n",
       "  78.67036011080333,\n",
       "  78.39335180055402,\n",
       "  79.22437673130194,\n",
       "  80.05540166204986,\n",
       "  82.82548476454294,\n",
       "  84.48753462603878,\n",
       "  85.59556786703601,\n",
       "  87.53462603878117,\n",
       "  89.19667590027701,\n",
       "  86.98060941828255,\n",
       "  83.65650969529086,\n",
       "  77.5623268698061,\n",
       "  70.6371191135734,\n",
       "  69.25207756232687,\n",
       "  78.39335180055402,\n",
       "  85.87257617728532,\n",
       "  86.98060941828255,\n",
       "  86.14958448753463,\n",
       "  86.14958448753463,\n",
       "  88.36565096952909,\n",
       "  88.9196675900277,\n",
       "  89.47368421052632,\n",
       "  89.75069252077563,\n",
       "  90.58171745152355,\n",
       "  90.85872576177286,\n",
       "  91.96675900277009,\n",
       "  91.68975069252078,\n",
       "  91.13573407202216,\n",
       "  90.85872576177286,\n",
       "  90.30470914127424,\n",
       "  90.30470914127424,\n",
       "  88.08864265927978,\n",
       "  88.36565096952909,\n",
       "  86.70360110803324,\n",
       "  86.42659279778394,\n",
       "  86.14958448753463,\n",
       "  84.7645429362881,\n",
       "  83.93351800554017,\n",
       "  84.48753462603878,\n",
       "  86.98060941828255,\n",
       "  90.02770083102493,\n",
       "  91.96675900277009,\n",
       "  92.24376731301939,\n",
       "  92.797783933518,\n",
       "  92.24376731301939,\n",
       "  92.24376731301939,\n",
       "  91.68975069252078,\n",
       "  91.96675900277009,\n",
       "  90.58171745152355,\n",
       "  90.30470914127424,\n",
       "  90.02770083102493,\n",
       "  89.19667590027701,\n",
       "  89.47368421052632,\n",
       "  89.47368421052632,\n",
       "  88.6426592797784,\n",
       "  89.75069252077563,\n",
       "  90.58171745152355,\n",
       "  91.13573407202216,\n",
       "  91.13573407202216,\n",
       "  90.85872576177286,\n",
       "  91.96675900277009,\n",
       "  90.85872576177286,\n",
       "  90.85872576177286,\n",
       "  90.85872576177286,\n",
       "  90.85872576177286,\n",
       "  90.02770083102493,\n",
       "  87.25761772853186,\n",
       "  85.87257617728532,\n",
       "  85.59556786703601,\n",
       "  85.0415512465374,\n",
       "  85.87257617728532,\n",
       "  85.59556786703601,\n",
       "  86.42659279778394,\n",
       "  87.53462603878117,\n",
       "  89.47368421052632,\n",
       "  90.30470914127424,\n",
       "  90.85872576177286,\n",
       "  93.35180055401662,\n",
       "  92.5207756232687,\n",
       "  92.24376731301939,\n",
       "  90.85872576177286,\n",
       "  90.02770083102493,\n",
       "  90.02770083102493,\n",
       "  91.13573407202216,\n",
       "  90.30470914127424,\n",
       "  90.58171745152355,\n",
       "  90.85872576177286,\n",
       "  91.41274238227147,\n",
       "  90.85872576177286,\n",
       "  91.13573407202216,\n",
       "  91.13573407202216,\n",
       "  91.41274238227147,\n",
       "  91.41274238227147,\n",
       "  90.85872576177286,\n",
       "  91.41274238227147,\n",
       "  90.85872576177286,\n",
       "  90.85872576177286,\n",
       "  91.41274238227147,\n",
       "  90.58171745152355,\n",
       "  88.9196675900277,\n",
       "  88.36565096952909,\n",
       "  89.19667590027701,\n",
       "  89.47368421052632,\n",
       "  88.6426592797784,\n",
       "  88.9196675900277,\n",
       "  89.19667590027701,\n",
       "  89.75069252077563,\n",
       "  90.30470914127424,\n",
       "  90.30470914127424,\n",
       "  91.41274238227147,\n",
       "  89.19667590027701,\n",
       "  90.30470914127424,\n",
       "  92.24376731301939,\n",
       "  91.68975069252078,\n",
       "  93.07479224376732,\n",
       "  91.13573407202216,\n",
       "  89.75069252077563,\n",
       "  90.30470914127424,\n",
       "  89.19667590027701,\n",
       "  89.47368421052632,\n",
       "  87.25761772853186,\n",
       "  87.53462603878117,\n",
       "  88.08864265927978,\n",
       "  87.25761772853186,\n",
       "  87.53462603878117,\n",
       "  86.70360110803324,\n",
       "  86.70360110803324,\n",
       "  85.87257617728532,\n",
       "  88.9196675900277,\n",
       "  88.36565096952909,\n",
       "  86.98060941828255],\n",
       " 'val_loss': [7.913739697025317,\n",
       "  7.327657641867354,\n",
       "  6.848139567229138,\n",
       "  6.626440548538479,\n",
       "  6.501437664743352,\n",
       "  6.403325627790568,\n",
       "  6.313234879424944,\n",
       "  6.23496278300222,\n",
       "  6.1733208903281565,\n",
       "  6.120505636495647,\n",
       "  6.078910224054648,\n",
       "  6.04856843931881,\n",
       "  6.014430889224442,\n",
       "  5.98819338841198,\n",
       "  5.960374797577304,\n",
       "  5.936094549762467,\n",
       "  5.919699245299569,\n",
       "  5.902136724055078,\n",
       "  5.889328531820352,\n",
       "  5.880859915112637,\n",
       "  5.865386326697158,\n",
       "  5.86153322611701,\n",
       "  5.851132504842829,\n",
       "  5.839694566345863,\n",
       "  5.837009426161144,\n",
       "  5.826247739235187,\n",
       "  5.822169410966487,\n",
       "  5.816247836862387,\n",
       "  5.808393587773175,\n",
       "  5.8045342657699415,\n",
       "  5.796730387943819,\n",
       "  5.791170649078001,\n",
       "  5.790243116890554,\n",
       "  5.786654503690719,\n",
       "  5.784056414130443,\n",
       "  5.780621945995263,\n",
       "  5.780571405719262,\n",
       "  5.7751786217342325,\n",
       "  5.772220060469806,\n",
       "  5.771379181997258,\n",
       "  5.764759709918101,\n",
       "  5.767183701292195,\n",
       "  5.76258000792221,\n",
       "  5.76178635746471,\n",
       "  5.764192471918882,\n",
       "  5.762760932397831,\n",
       "  5.761477374687724,\n",
       "  5.756793454611701,\n",
       "  5.755513787095331,\n",
       "  5.7571536799439,\n",
       "  5.752267231382727,\n",
       "  5.752428688220221,\n",
       "  5.751879591016623,\n",
       "  5.746617053686258,\n",
       "  5.750714112437073,\n",
       "  5.751646642496901,\n",
       "  5.748402572145087,\n",
       "  5.7463116067473194,\n",
       "  5.743334547017032,\n",
       "  5.74553997773477,\n",
       "  5.746186622333085,\n",
       "  5.748676780979941,\n",
       "  5.749952837117032,\n",
       "  5.743620945535867,\n",
       "  5.748089880264441,\n",
       "  5.748763552770234,\n",
       "  5.745951509989779,\n",
       "  5.74693638023,\n",
       "  5.744443934599544,\n",
       "  5.745887472861554,\n",
       "  5.744681030699247,\n",
       "  5.743414443997952,\n",
       "  5.741111015876874,\n",
       "  5.739737094977009,\n",
       "  5.741863458776957,\n",
       "  5.74195325697226,\n",
       "  5.738172209766177,\n",
       "  5.741230573204532,\n",
       "  5.737906653380775,\n",
       "  5.7367670517767655,\n",
       "  5.735387418139677,\n",
       "  5.735425863029926,\n",
       "  5.74069598450707,\n",
       "  5.737938695079912,\n",
       "  5.73322796167576,\n",
       "  5.737502155187605,\n",
       "  5.736229215783333,\n",
       "  5.73926237736501,\n",
       "  5.733544452464492,\n",
       "  5.732399592690734,\n",
       "  5.732524518641975,\n",
       "  5.734564326790977,\n",
       "  5.7285089136766985,\n",
       "  5.72944079527671,\n",
       "  5.729211175727364,\n",
       "  5.731893239165151,\n",
       "  5.731654257601572,\n",
       "  5.730408346529091,\n",
       "  5.73154425032661,\n",
       "  5.734140587425227,\n",
       "  5.733068950135586,\n",
       "  5.733282994074469,\n",
       "  5.731613009566516,\n",
       "  5.733281165639507,\n",
       "  5.731012894340852,\n",
       "  5.731348746007384,\n",
       "  5.727871492148792,\n",
       "  5.733358858282801,\n",
       "  5.733057983865645,\n",
       "  5.73429954252432,\n",
       "  5.732772968198917,\n",
       "  5.73232017297069,\n",
       "  5.728855203187406,\n",
       "  5.728506614841517,\n",
       "  5.7307455933701785,\n",
       "  5.728222289526032,\n",
       "  5.730672551066637,\n",
       "  5.729653426904912,\n",
       "  5.731692363012825,\n",
       "  5.731765003257149,\n",
       "  5.732571825223364,\n",
       "  5.730007002946182,\n",
       "  5.729631101026653,\n",
       "  5.7307087786982365,\n",
       "  5.732272947744425,\n",
       "  5.73540684866962,\n",
       "  5.734307622517099,\n",
       "  5.7351608360191815,\n",
       "  5.732895754498219,\n",
       "  5.734589197885322,\n",
       "  5.736482320131458,\n",
       "  5.742344598017785,\n",
       "  5.747297271116962,\n",
       "  5.741907232204643,\n",
       "  5.7409980226578305,\n",
       "  5.741430995260248,\n",
       "  5.742003155940334,\n",
       "  5.738795263247608,\n",
       "  5.743153556693784,\n",
       "  5.7351672661552024,\n",
       "  5.738688457212817,\n",
       "  5.7379363807485735,\n",
       "  5.737988907644949,\n",
       "  5.738416200106023,\n",
       "  5.742067249430864,\n",
       "  5.750190660842662,\n",
       "  5.758560621671952,\n",
       "  5.767203691221123,\n",
       "  5.770494314892242,\n",
       "  5.777255724309429,\n",
       "  5.774826839312,\n",
       "  5.761999699052169,\n",
       "  5.751127103527639,\n",
       "  5.747787523282902,\n",
       "  5.744618480844646,\n",
       "  5.750311697330634,\n",
       "  5.747700847973286,\n",
       "  5.745629726050661,\n",
       "  5.7480076907068405,\n",
       "  5.743076756565532,\n",
       "  5.745332565284079,\n",
       "  5.74997694526345,\n",
       "  5.748780076929812,\n",
       "  5.756707751595544,\n",
       "  5.761065001591024,\n",
       "  5.760276819351219,\n",
       "  5.765953722786662,\n",
       "  5.768659563218236,\n",
       "  5.774113041920289,\n",
       "  5.777707712866168,\n",
       "  5.7783713292097305,\n",
       "  5.778352969163715,\n",
       "  5.772490121494321,\n",
       "  5.760784871834167,\n",
       "  5.7500998299907415,\n",
       "  5.736630997278005,\n",
       "  5.7315504315609,\n",
       "  5.727545223438776,\n",
       "  5.72532913061227,\n",
       "  5.738805109552266,\n",
       "  5.769195781758624,\n",
       "  5.801236202414352,\n",
       "  5.806506168605272,\n",
       "  5.773195097772788,\n",
       "  5.7437105618103645,\n",
       "  5.7449497017537805,\n",
       "  5.753512558959178,\n",
       "  5.752867247862907,\n",
       "  5.745762896100813,\n",
       "  5.742060110378769,\n",
       "  5.7395540209553255,\n",
       "  5.738830613410346,\n",
       "  5.732204902414503,\n",
       "  5.728592161110517,\n",
       "  5.7244365546004765,\n",
       "  5.724852131704645,\n",
       "  5.7257566522930565,\n",
       "  5.723257977294499,\n",
       "  5.7226512640782685,\n",
       "  5.723215963880803,\n",
       "  5.725058831641235,\n",
       "  5.725278080387011,\n",
       "  5.725774201538418,\n",
       "  5.724375432961512,\n",
       "  5.724342203328485,\n",
       "  5.7259948116943615,\n",
       "  5.724364120217046,\n",
       "  5.726036004829242,\n",
       "  5.721688758693029,\n",
       "  5.719662642521554,\n",
       "  5.718486824933426,\n",
       "  5.718240630856919,\n",
       "  5.718761715650525,\n",
       "  5.719793566638064,\n",
       "  5.72253005734562,\n",
       "  5.72398376615413,\n",
       "  5.726699321808004,\n",
       "  5.7243242027078605,\n",
       "  5.726745384438738,\n",
       "  5.727749561527889,\n",
       "  5.73256237703436,\n",
       "  5.730340054645795,\n",
       "  5.731920595549865,\n",
       "  5.732759968889953,\n",
       "  5.7327434759393245,\n",
       "  5.732039191998799,\n",
       "  5.73230552672385,\n",
       "  5.7294985724694305,\n",
       "  5.73100892619069,\n",
       "  5.728726732206409,\n",
       "  5.729636434662437,\n",
       "  5.728872248919253,\n",
       "  5.727530315882419,\n",
       "  5.728948702018565,\n",
       "  5.733242062714003,\n",
       "  5.733270748668536,\n",
       "  5.73001101811894,\n",
       "  5.730115880241187,\n",
       "  5.725686090261507,\n",
       "  5.721739744097729,\n",
       "  5.720643342537326,\n",
       "  5.7195345501162285,\n",
       "  5.716290025594996,\n",
       "  5.717957780053974,\n",
       "  5.716309051993083,\n",
       "  5.7190826920743,\n",
       "  5.718617773496697,\n",
       "  5.724502677385194,\n",
       "  5.7260871608838375,\n",
       "  5.73372530800113,\n",
       "  5.735151650968844,\n",
       "  5.732234721692192,\n",
       "  5.7328722967834,\n",
       "  5.73163780892474,\n",
       "  5.731631453221717,\n",
       "  5.72988547164107,\n",
       "  5.726986676988071,\n",
       "  5.728595836867539,\n",
       "  5.727685375265113,\n",
       "  5.724820085489028,\n",
       "  5.72352867343707,\n",
       "  5.72423604716849,\n",
       "  5.725080632642816,\n",
       "  5.724223089848824,\n",
       "  5.7207618754458105,\n",
       "  5.723107381679398,\n",
       "  5.724348265904918,\n",
       "  5.723710390906429,\n",
       "  5.727998555877468,\n",
       "  5.724573850511759,\n",
       "  5.72347298514736,\n",
       "  5.72241976202809,\n",
       "  5.721030940526817,\n",
       "  5.720661061323755,\n",
       "  5.718197605062426,\n",
       "  5.7189065533319745,\n",
       "  5.719113979264668,\n",
       "  5.720621995494499,\n",
       "  5.720830699943122,\n",
       "  5.724665788257551,\n",
       "  5.727847261778717,\n",
       "  5.725803202417765,\n",
       "  5.726187128929192,\n",
       "  5.725320754965509,\n",
       "  5.726116920436329,\n",
       "  5.728774912259982,\n",
       "  5.730544504982684,\n",
       "  5.732371765852686,\n",
       "  5.739928302585283,\n",
       "  5.746494724251462,\n",
       "  5.746792562182691,\n",
       "  5.744332496878754,\n",
       "  5.744043010940655,\n",
       "  5.742860775951333,\n",
       "  5.743176692627906,\n",
       "  5.744854712740259,\n",
       "  5.74655975605025,\n",
       "  5.7403884262223706,\n",
       "  5.745851414348496,\n",
       "  5.745241742088332],\n",
       " 'val_ATT_loss': [1.9395230984300134,\n",
       "  1.7996639035096982,\n",
       "  1.6692374796886755,\n",
       "  1.5701349433360061,\n",
       "  1.4874738726189467,\n",
       "  1.4102656637750022,\n",
       "  1.3420366189344142,\n",
       "  1.281380820322812,\n",
       "  1.2299540938158346,\n",
       "  1.185800143010248,\n",
       "  1.1483365867922946,\n",
       "  1.123885364673002,\n",
       "  1.0938028900361643,\n",
       "  1.0732721399243286,\n",
       "  1.051208352291487,\n",
       "  1.0313794087103711,\n",
       "  1.0174231122179729,\n",
       "  1.0028826521664131,\n",
       "  0.9923880338911119,\n",
       "  0.9849253768116478,\n",
       "  0.9719210485132729,\n",
       "  0.9691074646827651,\n",
       "  0.959434579542982,\n",
       "  0.9504539972640634,\n",
       "  0.9480712217528645,\n",
       "  0.9391369868100174,\n",
       "  0.9358309532811002,\n",
       "  0.9298440460267106,\n",
       "  0.9239244280549569,\n",
       "  0.9203719507630278,\n",
       "  0.9135912762182515,\n",
       "  0.9091737338197909,\n",
       "  0.9076503795094606,\n",
       "  0.904670549359748,\n",
       "  0.9038890701968495,\n",
       "  0.8993869930263457,\n",
       "  0.9005544980367025,\n",
       "  0.8958948370887012,\n",
       "  0.8921390105553759,\n",
       "  0.8923159445204386,\n",
       "  0.8866615180319887,\n",
       "  0.887762062554437,\n",
       "  0.8848776449032916,\n",
       "  0.8832159523314577,\n",
       "  0.886793350906876,\n",
       "  0.8853680760395236,\n",
       "  0.8836867004875245,\n",
       "  0.8808962226640887,\n",
       "  0.8784862238217176,\n",
       "  0.8800246102538535,\n",
       "  0.8767114494874225,\n",
       "  0.876297493654538,\n",
       "  0.8761638807572001,\n",
       "  0.8727755636219087,\n",
       "  0.8749298028102735,\n",
       "  0.8754502381008815,\n",
       "  0.8729439857529431,\n",
       "  0.8714103913161813,\n",
       "  0.8688993365541706,\n",
       "  0.8707536876928516,\n",
       "  0.870839156028701,\n",
       "  0.8717745460145842,\n",
       "  0.8739229463707141,\n",
       "  0.8693768704325203,\n",
       "  0.8729244148343559,\n",
       "  0.8749092656180142,\n",
       "  0.8718632320320703,\n",
       "  0.8725512993287264,\n",
       "  0.870642921667758,\n",
       "  0.8720172615797539,\n",
       "  0.8706952192919041,\n",
       "  0.8698941566110626,\n",
       "  0.8667997261130713,\n",
       "  0.8662860605532561,\n",
       "  0.8668993149346452,\n",
       "  0.866955355173204,\n",
       "  0.8643883189292458,\n",
       "  0.8671151859973504,\n",
       "  0.8640555514068138,\n",
       "  0.8638806360039285,\n",
       "  0.8624728748468848,\n",
       "  0.8629901570275547,\n",
       "  0.8670790776973818,\n",
       "  0.8652604152032031,\n",
       "  0.8618169881221724,\n",
       "  0.865151594567105,\n",
       "  0.8634224118498283,\n",
       "  0.8646878629195981,\n",
       "  0.8607903343390643,\n",
       "  0.861383940630812,\n",
       "  0.8611933101968068,\n",
       "  0.8612604412606092,\n",
       "  0.8566357122688759,\n",
       "  0.8574617606837575,\n",
       "  0.8572310629656644,\n",
       "  0.859741324089407,\n",
       "  0.8597230870064682,\n",
       "  0.8585754073247677,\n",
       "  0.8599469350363181,\n",
       "  0.8619862482315157,\n",
       "  0.861226122432608,\n",
       "  0.8612710207216139,\n",
       "  0.8586583450073149,\n",
       "  0.8593275513106245,\n",
       "  0.857329290814516,\n",
       "  0.8576463678988014,\n",
       "  0.8555038411442827,\n",
       "  0.8595359296333499,\n",
       "  0.8582689913550043,\n",
       "  0.8584816035458712,\n",
       "  0.8565785422073147,\n",
       "  0.8561953593318056,\n",
       "  0.8533883984011363,\n",
       "  0.854374130203472,\n",
       "  0.8554751121900915,\n",
       "  0.8546133139511434,\n",
       "  0.855892132816276,\n",
       "  0.8551875767184467,\n",
       "  0.8567769798806043,\n",
       "  0.8566903946845512,\n",
       "  0.8585331430280112,\n",
       "  0.8555210357516762,\n",
       "  0.8547962845825567,\n",
       "  0.8548181989813238,\n",
       "  0.8561702207094286,\n",
       "  0.8586859803616516,\n",
       "  0.8581331532175948,\n",
       "  0.8588609978920076,\n",
       "  0.8566899460747959,\n",
       "  0.8580665681662598,\n",
       "  0.8576585506278325,\n",
       "  0.8634016098288017,\n",
       "  0.8680261214816474,\n",
       "  0.8619853161214813,\n",
       "  0.8597455600170585,\n",
       "  0.8606337576619978,\n",
       "  0.8625644602911259,\n",
       "  0.8587674382256298,\n",
       "  0.8628415637384586,\n",
       "  0.8542166731464185,\n",
       "  0.8565347741047541,\n",
       "  0.8556019546297507,\n",
       "  0.8546756329090615,\n",
       "  0.8526840678802351,\n",
       "  0.8495452984319469,\n",
       "  0.848483326958447,\n",
       "  0.8467106355157324,\n",
       "  0.845880512784167,\n",
       "  0.8479510986708044,\n",
       "  0.8506470526137003,\n",
       "  0.850082006032874,\n",
       "  0.8483523016538077,\n",
       "  0.8476054019075099,\n",
       "  0.8484580905699148,\n",
       "  0.850964163740476,\n",
       "  0.8525833980581625,\n",
       "  0.8522312857998096,\n",
       "  0.8532502853531179,\n",
       "  0.8565563647969952,\n",
       "  0.8547998747932233,\n",
       "  0.8556870398967247,\n",
       "  0.85586639717827,\n",
       "  0.8534532082759267,\n",
       "  0.853821212925562,\n",
       "  0.8536528936488842,\n",
       "  0.8530937747015217,\n",
       "  0.851934684849367,\n",
       "  0.848702664176623,\n",
       "  0.8487730841568815,\n",
       "  0.847942725066247,\n",
       "  0.8484439871660093,\n",
       "  0.8493560479908455,\n",
       "  0.8490727112545231,\n",
       "  0.8514273246613945,\n",
       "  0.8510568994816726,\n",
       "  0.8491202283923219,\n",
       "  0.850511495659991,\n",
       "  0.8519358132185975,\n",
       "  0.848549081058037,\n",
       "  0.8498498940370917,\n",
       "  0.8508573338025953,\n",
       "  0.8487497595994453,\n",
       "  0.8508973332439981,\n",
       "  0.8519263016741451,\n",
       "  0.8481976755750857,\n",
       "  0.8467540092826858,\n",
       "  0.8480644587094222,\n",
       "  0.8502305501360234,\n",
       "  0.8487623325450634,\n",
       "  0.8479150767491116,\n",
       "  0.8464639120227923,\n",
       "  0.8471920613835497,\n",
       "  0.8450597084876967,\n",
       "  0.845068019822361,\n",
       "  0.8455289662853489,\n",
       "  0.8481780932686194,\n",
       "  0.8495158086705014,\n",
       "  0.8480772941820021,\n",
       "  0.848085013831534,\n",
       "  0.8494739172662177,\n",
       "  0.850719495396304,\n",
       "  0.8498529348431564,\n",
       "  0.8488943896642546,\n",
       "  0.8475848906408481,\n",
       "  0.8477260512791998,\n",
       "  0.8479055728369612,\n",
       "  0.8454486922762259,\n",
       "  0.8471742598264198,\n",
       "  0.8454867643796331,\n",
       "  0.8448660464548483,\n",
       "  0.8442296268251853,\n",
       "  0.8431422074393529,\n",
       "  0.8432945673300968,\n",
       "  0.8432588674188629,\n",
       "  0.8443786581841911,\n",
       "  0.8450143522605663,\n",
       "  0.8458133308383508,\n",
       "  0.8436563224569569,\n",
       "  0.8433193970259613,\n",
       "  0.842833229560193,\n",
       "  0.8442979541009035,\n",
       "  0.8431911910694789,\n",
       "  0.8445524655706514,\n",
       "  0.8428662021954855,\n",
       "  0.8440084341095715,\n",
       "  0.8451556530667514,\n",
       "  0.8459465116020141,\n",
       "  0.8462700167807137,\n",
       "  0.848055909561917,\n",
       "  0.8463193640960911,\n",
       "  0.8489372822327342,\n",
       "  0.8499075018051194,\n",
       "  0.8495987979619484,\n",
       "  0.8507248664774546,\n",
       "  0.8540514850519537,\n",
       "  0.8529291324983768,\n",
       "  0.8494463531467004,\n",
       "  0.8468714515125848,\n",
       "  0.8428345395297538,\n",
       "  0.8402722924947739,\n",
       "  0.839186614485291,\n",
       "  0.8399124477452379,\n",
       "  0.8398903247302141,\n",
       "  0.8413208380220382,\n",
       "  0.8408804658224912,\n",
       "  0.8425830439096544,\n",
       "  0.8414047370112039,\n",
       "  0.8432506586962599,\n",
       "  0.8423481063871849,\n",
       "  0.8447683677683032,\n",
       "  0.8437952373812838,\n",
       "  0.8427367169197982,\n",
       "  0.8438781421116697,\n",
       "  0.8435111572829689,\n",
       "  0.8431338974373127,\n",
       "  0.8434710929064246,\n",
       "  0.8432063067831644,\n",
       "  0.8440881502337572,\n",
       "  0.8449881647418185,\n",
       "  0.8431872829673736,\n",
       "  0.8437590733533953,\n",
       "  0.8447123752619193,\n",
       "  0.8458931467882017,\n",
       "  0.8443065857499595,\n",
       "  0.8417122401599961,\n",
       "  0.8431572041860441,\n",
       "  0.8433386819149421,\n",
       "  0.8439851824830218,\n",
       "  0.844358280664537,\n",
       "  0.8430098640482601,\n",
       "  0.8435635684224648,\n",
       "  0.8418169330532957,\n",
       "  0.8408109706834079,\n",
       "  0.8402254753481082,\n",
       "  0.8381860142316275,\n",
       "  0.8400036846476842,\n",
       "  0.8396394697388982,\n",
       "  0.8402151712557164,\n",
       "  0.8415095299724641,\n",
       "  0.8424831672412593,\n",
       "  0.8433625866484836,\n",
       "  0.8427640256600651,\n",
       "  0.8413564990448757,\n",
       "  0.8416555004148949,\n",
       "  0.8400575569490107,\n",
       "  0.841231946295839,\n",
       "  0.8415506656576948,\n",
       "  0.8430977629694513,\n",
       "  0.8455301741274391,\n",
       "  0.8445857111516037,\n",
       "  0.8449244860226546,\n",
       "  0.8452870770683133,\n",
       "  0.8456323347925171,\n",
       "  0.8451367030298806,\n",
       "  0.8446769230976338,\n",
       "  0.8451377928741579,\n",
       "  0.8485001295320387,\n",
       "  0.849092800563913,\n",
       "  0.8530909220377604,\n",
       "  0.8502759595469731],\n",
       " 'val_VAL_loss': [1.9914055328651015,\n",
       "  1.8426645794525522,\n",
       "  1.7263006958468208,\n",
       "  1.6854352017341576,\n",
       "  1.6713212640414685,\n",
       "  1.664353321338522,\n",
       "  1.6570660868301768,\n",
       "  1.6511939875598025,\n",
       "  1.647788932170774,\n",
       "  1.6449018311617998,\n",
       "  1.6435245457541179,\n",
       "  1.6415610248819361,\n",
       "  1.6402093330627592,\n",
       "  1.638307082829217,\n",
       "  1.636388815095272,\n",
       "  1.6349050470173652,\n",
       "  1.634092044360532,\n",
       "  1.633084690629555,\n",
       "  1.6323134993097466,\n",
       "  1.631978179433663,\n",
       "  1.6311550927279619,\n",
       "  1.6308085871447484,\n",
       "  1.6305659750999488,\n",
       "  1.6297468563606,\n",
       "  1.6296460681360931,\n",
       "  1.6290369174750567,\n",
       "  1.6287794858951288,\n",
       "  1.6288012636118923,\n",
       "  1.628156386572739,\n",
       "  1.6280541050023045,\n",
       "  1.6277130372418558,\n",
       "  1.6273323050860702,\n",
       "  1.6275309124603647,\n",
       "  1.6273279847769901,\n",
       "  1.6267224479778646,\n",
       "  1.6270783176563057,\n",
       "  1.6266723025608532,\n",
       "  1.6264279282151772,\n",
       "  1.6266936833048102,\n",
       "  1.6263544124922729,\n",
       "  1.6260327306287041,\n",
       "  1.6264738795792528,\n",
       "  1.625900787672973,\n",
       "  1.6261901350444175,\n",
       "  1.6257997070040022,\n",
       "  1.6257976187861025,\n",
       "  1.6259302247333995,\n",
       "  1.6252990773158709,\n",
       "  1.6256758544245378,\n",
       "  1.6257096898966823,\n",
       "  1.625185260631768,\n",
       "  1.625377064855228,\n",
       "  1.6252385700864744,\n",
       "  1.6246138300214494,\n",
       "  1.6252614365422666,\n",
       "  1.6253988014653398,\n",
       "  1.6251528621307147,\n",
       "  1.6249670718103795,\n",
       "  1.6248117368209538,\n",
       "  1.6249287633473062,\n",
       "  1.6251158221014614,\n",
       "  1.6256340783217857,\n",
       "  1.6253432969154396,\n",
       "  1.6247480250344488,\n",
       "  1.6250551551433619,\n",
       "  1.6246180957174066,\n",
       "  1.6246960926525698,\n",
       "  1.624795026967091,\n",
       "  1.6246003376439286,\n",
       "  1.6246234037606,\n",
       "  1.6246619371357809,\n",
       "  1.6245067624622964,\n",
       "  1.6247704299212677,\n",
       "  1.624483678141251,\n",
       "  1.6249880479474372,\n",
       "  1.6249993005996854,\n",
       "  1.6245946302789773,\n",
       "  1.6247051290690606,\n",
       "  1.6246170339913204,\n",
       "  1.624295471924279,\n",
       "  1.624304847764264,\n",
       "  1.624145235334124,\n",
       "  1.624538968936563,\n",
       "  1.6242260932922363,\n",
       "  1.6238036578511956,\n",
       "  1.6241168535401669,\n",
       "  1.6242689346445018,\n",
       "  1.6248581714818042,\n",
       "  1.6242513727084757,\n",
       "  1.6236718840199738,\n",
       "  1.6237770694817228,\n",
       "  1.6244346285101228,\n",
       "  1.6239577338026074,\n",
       "  1.6239930115309842,\n",
       "  1.6239933709205665,\n",
       "  1.6240506383585813,\n",
       "  1.6239770568650345,\n",
       "  1.623944313068108,\n",
       "  1.6238657717634304,\n",
       "  1.6240514463979034,\n",
       "  1.623947609234326,\n",
       "  1.6240039911176183,\n",
       "  1.6243182215197334,\n",
       "  1.6246512047762942,\n",
       "  1.6245612011754453,\n",
       "  1.6245674593695278,\n",
       "  1.6241225503348364,\n",
       "  1.6246076428831504,\n",
       "  1.6249296641702136,\n",
       "  1.6252726463261495,\n",
       "  1.625398141997201,\n",
       "  1.625374937879628,\n",
       "  1.625155601595423,\n",
       "  1.6247108282126816,\n",
       "  1.6250901603933625,\n",
       "  1.6245363251916294,\n",
       "  1.6249268060834536,\n",
       "  1.6248219500621552,\n",
       "  1.624971794377407,\n",
       "  1.6250248695241993,\n",
       "  1.6246795607317845,\n",
       "  1.624828655731502,\n",
       "  1.6249449388146988,\n",
       "  1.6252968599056374,\n",
       "  1.6253675756783321,\n",
       "  1.625573622769323,\n",
       "  1.6253914897665014,\n",
       "  1.6254332793757247,\n",
       "  1.625401936141141,\n",
       "  1.6255075432396875,\n",
       "  1.626274589834542,\n",
       "  1.6263143293963278,\n",
       "  1.626423716545105,\n",
       "  1.6266406386943872,\n",
       "  1.6270841542135905,\n",
       "  1.62693241253275,\n",
       "  1.6264795652164028,\n",
       "  1.6266759416739929,\n",
       "  1.6267706643184419,\n",
       "  1.626983531002928,\n",
       "  1.6273845610360207,\n",
       "  1.6274448087062743,\n",
       "  1.6277710915786292,\n",
       "  1.628577377408596,\n",
       "  1.6308406503329724,\n",
       "  1.6339024446280719,\n",
       "  1.6372833287187398,\n",
       "  1.6404410594789853,\n",
       "  1.6408477387404794,\n",
       "  1.6422028905652426,\n",
       "  1.641581611093042,\n",
       "  1.6378824657994537,\n",
       "  1.6345072338733766,\n",
       "  1.633109810904329,\n",
       "  1.6312181057013901,\n",
       "  1.6325760997574905,\n",
       "  1.631823187391159,\n",
       "  1.630793146899181,\n",
       "  1.6304837753032815,\n",
       "  1.6294256272574363,\n",
       "  1.629881841795785,\n",
       "  1.63137018269506,\n",
       "  1.6317756228846283,\n",
       "  1.6342955128899936,\n",
       "  1.6358040359807131,\n",
       "  1.6357276815498991,\n",
       "  1.6380063459790986,\n",
       "  1.639985633013871,\n",
       "  1.6417799859211362,\n",
       "  1.6432549959333071,\n",
       "  1.643309114014574,\n",
       "  1.64299897372429,\n",
       "  1.641139136746599,\n",
       "  1.6364525157242573,\n",
       "  1.6330143101696897,\n",
       "  1.6291702562952277,\n",
       "  1.6270129786336363,\n",
       "  1.6252031367400597,\n",
       "  1.625593349851411,\n",
       "  1.6296517385050582,\n",
       "  1.6394461493186763,\n",
       "  1.6508288142716356,\n",
       "  1.6518696117870912,\n",
       "  1.6404229320328811,\n",
       "  1.631837628745093,\n",
       "  1.632731897490365,\n",
       "  1.6351493667499186,\n",
       "  1.6342122325756279,\n",
       "  1.63233352118525,\n",
       "  1.6313816778765524,\n",
       "  1.6310300363108443,\n",
       "  1.630546184008932,\n",
       "  1.629048397975602,\n",
       "  1.6278413804293854,\n",
       "  1.626302529438376,\n",
       "  1.6255580128120084,\n",
       "  1.6254136145408518,\n",
       "  1.6250602277041657,\n",
       "  1.6248554167489113,\n",
       "  1.6245806822048618,\n",
       "  1.6247797787483103,\n",
       "  1.625141715181285,\n",
       "  1.6256266039580547,\n",
       "  1.6255968474402216,\n",
       "  1.6255387173497617,\n",
       "  1.6260297462858002,\n",
       "  1.62630514264694,\n",
       "  1.6262872483342738,\n",
       "  1.6254006647711317,\n",
       "  1.6249321986889016,\n",
       "  1.6247523993694137,\n",
       "  1.6250328078058554,\n",
       "  1.6251557161068093,\n",
       "  1.6255115664064004,\n",
       "  1.6260504663871427,\n",
       "  1.626323137964521,\n",
       "  1.6269619969898843,\n",
       "  1.6268892934169676,\n",
       "  1.6278086624709256,\n",
       "  1.6283054439892322,\n",
       "  1.6294214743111521,\n",
       "  1.6290496211921053,\n",
       "  1.6291227099930712,\n",
       "  1.629964588898156,\n",
       "  1.6295783472765843,\n",
       "  1.6289611796440162,\n",
       "  1.6287863383739454,\n",
       "  1.6277428518962391,\n",
       "  1.6276510055429243,\n",
       "  1.6274691227034395,\n",
       "  1.6268997174765676,\n",
       "  1.6263215823713781,\n",
       "  1.6259771726401568,\n",
       "  1.6260746118470366,\n",
       "  1.6263968592206832,\n",
       "  1.6267805387233865,\n",
       "  1.6268548883240799,\n",
       "  1.627748142909534,\n",
       "  1.627617183577251,\n",
       "  1.6271558172009848,\n",
       "  1.627152242684012,\n",
       "  1.62654070079033,\n",
       "  1.6254665669549275,\n",
       "  1.6255456473439784,\n",
       "  1.6251428620568638,\n",
       "  1.6254998827215486,\n",
       "  1.6257376788284978,\n",
       "  1.627084006229645,\n",
       "  1.6279130181655508,\n",
       "  1.6296523134109422,\n",
       "  1.6304521378625203,\n",
       "  1.6298326682574644,\n",
       "  1.6296647182239101,\n",
       "  1.629375550547257,\n",
       "  1.629499185261468,\n",
       "  1.6288047929115483,\n",
       "  1.6279267900683023,\n",
       "  1.6281692288779273,\n",
       "  1.6275657368410985,\n",
       "  1.6272109341738847,\n",
       "  1.6265898666945584,\n",
       "  1.6265078906355233,\n",
       "  1.6263958286182048,\n",
       "  1.6266388346996214,\n",
       "  1.6263498784286048,\n",
       "  1.6266500591644513,\n",
       "  1.627003194663325,\n",
       "  1.6265750694744692,\n",
       "  1.627880091737644,\n",
       "  1.6271879954878332,\n",
       "  1.6266364722416318,\n",
       "  1.6268676096582648,\n",
       "  1.626739989947803,\n",
       "  1.6268118619918823,\n",
       "  1.6266705302769326,\n",
       "  1.6263009562280966,\n",
       "  1.6264915031752563,\n",
       "  1.6268022747462607,\n",
       "  1.6264403899902193,\n",
       "  1.6273942070054304,\n",
       "  1.6281615583767444,\n",
       "  1.6276797255859,\n",
       "  1.6282768766281053,\n",
       "  1.627888418183538,\n",
       "  1.6286864544957729,\n",
       "  1.6291809886547144,\n",
       "  1.6296646131083297,\n",
       "  1.6297580009610781,\n",
       "  1.6314660428192815,\n",
       "  1.633969671033286,\n",
       "  1.6339560253866787,\n",
       "  1.6330151399368136,\n",
       "  1.6328035587160459,\n",
       "  1.6325746909738175,\n",
       "  1.632833256510091,\n",
       "  1.6332389732887005,\n",
       "  1.6326865421727372,\n",
       "  1.630431875219486,\n",
       "  1.6309201641035784,\n",
       "  1.6316552608471198],\n",
       " 'val_ATT_acc': [30.89430894308943,\n",
       "  50.609756097560975,\n",
       "  57.31707317073171,\n",
       "  63.41463414634146,\n",
       "  67.07317073170732,\n",
       "  70.32520325203252,\n",
       "  73.3739837398374,\n",
       "  76.21951219512195,\n",
       "  77.84552845528455,\n",
       "  78.86178861788618,\n",
       "  80.08130081300813,\n",
       "  82.72357723577235,\n",
       "  84.14634146341463,\n",
       "  83.53658536585365,\n",
       "  85.77235772357723,\n",
       "  85.97560975609755,\n",
       "  86.17886178861788,\n",
       "  86.78861788617886,\n",
       "  87.1951219512195,\n",
       "  86.78861788617886,\n",
       "  87.8048780487805,\n",
       "  87.60162601626017,\n",
       "  88.00813008130082,\n",
       "  88.82113821138212,\n",
       "  88.82113821138212,\n",
       "  90.2439024390244,\n",
       "  90.44715447154472,\n",
       "  90.65040650406505,\n",
       "  91.26016260162602,\n",
       "  91.66666666666667,\n",
       "  92.07317073170732,\n",
       "  92.47967479674797,\n",
       "  92.07317073170732,\n",
       "  92.88617886178862,\n",
       "  92.88617886178862,\n",
       "  93.29268292682927,\n",
       "  92.88617886178862,\n",
       "  93.08943089430895,\n",
       "  93.29268292682927,\n",
       "  93.69918699186992,\n",
       "  94.10569105691057,\n",
       "  93.90243902439025,\n",
       "  93.90243902439025,\n",
       "  94.10569105691057,\n",
       "  94.3089430894309,\n",
       "  94.10569105691057,\n",
       "  93.90243902439025,\n",
       "  94.10569105691057,\n",
       "  93.90243902439025,\n",
       "  94.10569105691057,\n",
       "  94.71544715447155,\n",
       "  94.3089430894309,\n",
       "  94.51219512195122,\n",
       "  94.71544715447155,\n",
       "  94.71544715447155,\n",
       "  95.1219512195122,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.1219512195122,\n",
       "  95.52845528455285,\n",
       "  95.1219512195122,\n",
       "  95.32520325203252,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.32520325203252,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.32520325203252,\n",
       "  96.13821138211382,\n",
       "  95.1219512195122,\n",
       "  95.32520325203252,\n",
       "  95.9349593495935,\n",
       "  95.1219512195122,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.1219512195122,\n",
       "  95.73170731707317,\n",
       "  95.1219512195122,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.32520325203252,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.1219512195122,\n",
       "  95.32520325203252,\n",
       "  95.32520325203252,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.1219512195122,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.32520325203252,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  95.9349593495935,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  96.34146341463415,\n",
       "  96.54471544715447,\n",
       "  96.54471544715447,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.32520325203252,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  96.54471544715447,\n",
       "  96.7479674796748,\n",
       "  96.54471544715447,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.95121951219512,\n",
       "  96.54471544715447,\n",
       "  96.7479674796748,\n",
       "  96.13821138211382,\n",
       "  96.54471544715447,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  96.54471544715447,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  95.9349593495935,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  96.54471544715447,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.54471544715447,\n",
       "  96.54471544715447,\n",
       "  96.54471544715447,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  96.54471544715447,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.7479674796748,\n",
       "  96.34146341463415,\n",
       "  96.34146341463415,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317],\n",
       " 'val_VAL_acc': [71.42857142857143,\n",
       "  88.66995073891626,\n",
       "  93.5960591133005,\n",
       "  95.56650246305419,\n",
       "  97.04433497536945,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  96.05911330049261,\n",
       "  95.56650246305419,\n",
       "  97.53694581280789,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  97.53694581280789,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  97.53694581280789,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463],\n",
       " 'val_VAL_jac': [0.09770114962103332,\n",
       "  0.2454844014397983,\n",
       "  0.4991789827205865,\n",
       "  0.5968801352778091,\n",
       "  0.6288998397113067,\n",
       "  0.6469622386500166,\n",
       "  0.6642036179603614,\n",
       "  0.6896551794606477,\n",
       "  0.7085385956787711,\n",
       "  0.7192118320559046,\n",
       "  0.7142857189836174,\n",
       "  0.7192118273580016,\n",
       "  0.7348111697605678,\n",
       "  0.7512315341404506,\n",
       "  0.75862070139993,\n",
       "  0.75862070139993,\n",
       "  0.7684729181486984,\n",
       "  0.7717569896153041,\n",
       "  0.769293935428112,\n",
       "  0.7619047728665357,\n",
       "  0.7660098592636033,\n",
       "  0.7758620760123718,\n",
       "  0.7725780068947177,\n",
       "  0.7610837508892191,\n",
       "  0.7594417139814404,\n",
       "  0.7643678223558248,\n",
       "  0.7594417139814404,\n",
       "  0.749589497232672,\n",
       "  0.7684729134507955,\n",
       "  0.7660098592636033,\n",
       "  0.7569786597942484,\n",
       "  0.7660098569146518,\n",
       "  0.7660098592636033,\n",
       "  0.7619047681686326,\n",
       "  0.7643678223558248,\n",
       "  0.7627257830990947,\n",
       "  0.7651888372862867,\n",
       "  0.7643678223558248,\n",
       "  0.7717569849174011,\n",
       "  0.770114945660671,\n",
       "  0.770114945660671,\n",
       "  0.7668308741940654,\n",
       "  0.7643678200068732,\n",
       "  0.7619047658196811,\n",
       "  0.7692939283812574,\n",
       "  0.7758620760123718,\n",
       "  0.7709359676379876,\n",
       "  0.7684729134507955,\n",
       "  0.7635468050764112,\n",
       "  0.7610837508892191,\n",
       "  0.7717569825684496,\n",
       "  0.7643678200068732,\n",
       "  0.7619047658196811,\n",
       "  0.7733990218251797,\n",
       "  0.7643678200068732,\n",
       "  0.7569786574452969,\n",
       "  0.7668308741940654,\n",
       "  0.7668308741940654,\n",
       "  0.7742200367556417,\n",
       "  0.7766830909428338,\n",
       "  0.7717569825684496,\n",
       "  0.7717569825684496,\n",
       "  0.7586206931785997,\n",
       "  0.7586206931785997,\n",
       "  0.7586206931785997,\n",
       "  0.7536945848042155,\n",
       "  0.7536945848042155,\n",
       "  0.7545156032581047,\n",
       "  0.7536945848042155,\n",
       "  0.7536945848042155,\n",
       "  0.7512315306170233,\n",
       "  0.7512315306170233,\n",
       "  0.7504105156865614,\n",
       "  0.7487684764298312,\n",
       "  0.7561576389914075,\n",
       "  0.7512315306170233,\n",
       "  0.7463054222426391,\n",
       "  0.7536945848042155,\n",
       "  0.7561576389914075,\n",
       "  0.7561576389914075,\n",
       "  0.7536945848042155,\n",
       "  0.7561576389914075,\n",
       "  0.7528735698737534,\n",
       "  0.7536945848042155,\n",
       "  0.754515602083629,\n",
       "  0.7512315306170233,\n",
       "  0.7512315306170233,\n",
       "  0.7553366240609456,\n",
       "  0.7536945848042155,\n",
       "  0.7487684764298312,\n",
       "  0.7463054222426391,\n",
       "  0.7536945848042155,\n",
       "  0.7463054222426391,\n",
       "  0.7512315306170233,\n",
       "  0.7487684764298312,\n",
       "  0.7487684764298312,\n",
       "  0.7471264395220526,\n",
       "  0.7487684764298312,\n",
       "  0.7487684764298312,\n",
       "  0.7463054222426391,\n",
       "  0.7487684764298312,\n",
       "  0.7438423704043985,\n",
       "  0.7438423704043985,\n",
       "  0.7487684787787827,\n",
       "  0.7463054245915907,\n",
       "  0.7487684787787827,\n",
       "  0.7463054245915907,\n",
       "  0.7463054245915907,\n",
       "  0.7454844073121771,\n",
       "  0.7536945883276427,\n",
       "  0.7536945883276427,\n",
       "  0.7487684787787827,\n",
       "  0.7487684787787827,\n",
       "  0.7487684787787827,\n",
       "  0.7487684764298312,\n",
       "  0.7471264395220526,\n",
       "  0.7471264395220526,\n",
       "  0.7495894937092448,\n",
       "  0.7520525478964368,\n",
       "  0.754515602083629,\n",
       "  0.754515602083629,\n",
       "  0.754515602083629,\n",
       "  0.754515602083629,\n",
       "  0.7577996747247104,\n",
       "  0.7561576389914075,\n",
       "  0.754515602083629,\n",
       "  0.754515602083629,\n",
       "  0.7487684787787827,\n",
       "  0.7438423704043985,\n",
       "  0.7487684787787827,\n",
       "  0.754515602083629,\n",
       "  0.7643678188323975,\n",
       "  0.7610837497147434,\n",
       "  0.7438423704043985,\n",
       "  0.7438423704043985,\n",
       "  0.7339901513066786,\n",
       "  0.7438423704043985,\n",
       "  0.73399015365563,\n",
       "  0.7364532078428222,\n",
       "  0.7282430303507837,\n",
       "  0.723316914929545,\n",
       "  0.7266009863961507,\n",
       "  0.7397372804839035,\n",
       "  0.7323481120499484,\n",
       "  0.7348111697605678,\n",
       "  0.7282430268273565,\n",
       "  0.7233169161040207,\n",
       "  0.7142857213325688,\n",
       "  0.7224958988246072,\n",
       "  0.7216748815451937,\n",
       "  0.7249589553607508,\n",
       "  0.7249589577097023,\n",
       "  0.7307060845379759,\n",
       "  0.7364532125407252,\n",
       "  0.7454844131845558,\n",
       "  0.7536945906765943,\n",
       "  0.7561576413403591,\n",
       "  0.7586206955275512,\n",
       "  0.7619047681686326,\n",
       "  0.7520525514198642,\n",
       "  0.7512315364894021,\n",
       "  0.7520525537688156,\n",
       "  0.7536945906765943,\n",
       "  0.7463054281150179,\n",
       "  0.7405583001122686,\n",
       "  0.7323481179223272,\n",
       "  0.7323481179223272,\n",
       "  0.7315271006429137,\n",
       "  0.7233169208019238,\n",
       "  0.7266009969664324,\n",
       "  0.7249589577097023,\n",
       "  0.7233169208019238,\n",
       "  0.7183908124275395,\n",
       "  0.7151067362630309,\n",
       "  0.729064048804673,\n",
       "  0.7438423739278258,\n",
       "  0.7487684799532585,\n",
       "  0.744663387683812,\n",
       "  0.7454844061377013,\n",
       "  0.7348111721095193,\n",
       "  0.7077175737014545,\n",
       "  0.666666669798602,\n",
       "  0.6715927805219378,\n",
       "  0.7134647017042038,\n",
       "  0.7134646993552523,\n",
       "  0.7405582930654141,\n",
       "  0.7364532043193949,\n",
       "  0.7282430233039292,\n",
       "  0.7307060798400729,\n",
       "  0.7274220083734672,\n",
       "  0.7274220083734672,\n",
       "  0.7372742274711872,\n",
       "  0.7257799714656886,\n",
       "  0.7233169172784965,\n",
       "  0.7364532090172979,\n",
       "  0.7430213496015576,\n",
       "  0.7380952412271734,\n",
       "  0.7397372804839035,\n",
       "  0.7446633888582878,\n",
       "  0.7380952412271734,\n",
       "  0.7430213496015576,\n",
       "  0.7454844037887498,\n",
       "  0.7413793126937791,\n",
       "  0.7454844037887498,\n",
       "  0.7495894948837205,\n",
       "  0.7520525490709127,\n",
       "  0.759441711632489,\n",
       "  0.7561576401658834,\n",
       "  0.7528735663503262,\n",
       "  0.7536945859786912,\n",
       "  0.7528735686992777,\n",
       "  0.7577996770736619,\n",
       "  0.7504105145120855,\n",
       "  0.7479474603248935,\n",
       "  0.7553366205375183,\n",
       "  0.7553366228864697,\n",
       "  0.7504105145120855,\n",
       "  0.7553366228864697,\n",
       "  0.7446633888582878,\n",
       "  0.7397372804839035,\n",
       "  0.7471264430454799,\n",
       "  0.7463054257660664,\n",
       "  0.7463054257660664,\n",
       "  0.7348111721095193,\n",
       "  0.7372742239477599,\n",
       "  0.739737278134952,\n",
       "  0.7356321870399813,\n",
       "  0.7307060786655971,\n",
       "  0.7356321870399813,\n",
       "  0.7315270982939621,\n",
       "  0.7463054257660664,\n",
       "  0.749589492534769,\n",
       "  0.7422003299731926,\n",
       "  0.7389162585065869,\n",
       "  0.7380952412271734,\n",
       "  0.7372742215988084,\n",
       "  0.7331691305038377,\n",
       "  0.738095243576125,\n",
       "  0.7380952388782219,\n",
       "  0.7389162608555385,\n",
       "  0.7364532066683463,\n",
       "  0.7364532066683463,\n",
       "  0.7339901571790574,\n",
       "  0.7339901571790574,\n",
       "  0.738095243576125,\n",
       "  0.7561576401658834,\n",
       "  0.7389162608555385,\n",
       "  0.7454844037887498,\n",
       "  0.7331691328527892,\n",
       "  0.7298850613861836,\n",
       "  0.727422009547943,\n",
       "  0.7298850637351351,\n",
       "  0.7372742239477599,\n",
       "  0.73891626320449,\n",
       "  0.7413793173916822,\n",
       "  0.739737278134952,\n",
       "  0.7471264430454799,\n",
       "  0.7512315341404506,\n",
       "  0.7520525514198642,\n",
       "  0.7446633888582878,\n",
       "  0.7487684799532585,\n",
       "  0.7471264406965283,\n",
       "  0.7520525490709127,\n",
       "  0.7520525490709127,\n",
       "  0.7471264406965283,\n",
       "  0.7372742239477599,\n",
       "  0.739737278134952,\n",
       "  0.7372742239477599,\n",
       "  0.7479474556269904,\n",
       "  0.7446633865093363,\n",
       "  0.7430213472526062,\n",
       "  0.7479474556269904,\n",
       "  0.7471264383475769,\n",
       "  0.7405582930654141,\n",
       "  0.7446633865093363,\n",
       "  0.7438423692299228,\n",
       "  0.7430213519505092,\n",
       "  0.7454844061377013,\n",
       "  0.7430213519505092,\n",
       "  0.7389162585065869,\n",
       "  0.7479474603248935,\n",
       "  0.7356321893889328,\n",
       "  0.740558297763317,\n",
       "  0.7389162655534416,\n",
       "  0.7339901571790574,\n",
       "  0.7339901548301058,\n",
       "  0.7249589553607508,\n",
       "  0.7266009922685295,\n",
       "  0.7101806278886467,\n",
       "  0.7134647064021068,\n",
       "  0.7019704503966082,\n",
       "  0.7249589600586539,\n",
       "  0.727422014245846,\n",
       "  0.7266009922685295,\n",
       "  0.7290640441067701,\n",
       "  0.7290640441067701,\n",
       "  0.7266009899195779,\n",
       "  0.7323481155733756,\n",
       "  0.7216748838941452,\n",
       "  0.7372742262967115],\n",
       " 'val_VAL_acc_1': [23.645320197044335,\n",
       "  55.172413793103445,\n",
       "  66.50246305418719,\n",
       "  68.96551724137932,\n",
       "  71.42857142857143,\n",
       "  71.92118226600985,\n",
       "  73.39901477832512,\n",
       "  76.84729064039409,\n",
       "  78.32512315270937,\n",
       "  77.33990147783251,\n",
       "  76.84729064039409,\n",
       "  77.83251231527093,\n",
       "  78.81773399014779,\n",
       "  78.81773399014779,\n",
       "  79.3103448275862,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  82.26600985221675,\n",
       "  81.77339901477832,\n",
       "  82.75862068965517,\n",
       "  82.26600985221675,\n",
       "  82.75862068965517,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  82.75862068965517,\n",
       "  81.2807881773399,\n",
       "  82.26600985221675,\n",
       "  81.77339901477832,\n",
       "  80.78817733990148,\n",
       "  82.26600985221675,\n",
       "  83.2512315270936,\n",
       "  82.26600985221675,\n",
       "  81.77339901477832,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  82.75862068965517,\n",
       "  82.26600985221675,\n",
       "  82.75862068965517,\n",
       "  81.2807881773399,\n",
       "  81.77339901477832,\n",
       "  82.75862068965517,\n",
       "  82.26600985221675,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  81.77339901477832,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  81.77339901477832,\n",
       "  81.2807881773399,\n",
       "  82.26600985221675,\n",
       "  83.2512315270936,\n",
       "  83.74384236453201,\n",
       "  82.75862068965517,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  83.2512315270936,\n",
       "  83.2512315270936,\n",
       "  81.2807881773399,\n",
       "  83.2512315270936,\n",
       "  82.26600985221675,\n",
       "  82.75862068965517,\n",
       "  82.26600985221675,\n",
       "  82.75862068965517,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  81.77339901477832,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  79.3103448275862,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  79.3103448275862,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  78.81773399014779,\n",
       "  79.3103448275862,\n",
       "  78.81773399014779,\n",
       "  78.32512315270937,\n",
       "  78.32512315270937,\n",
       "  78.32512315270937,\n",
       "  78.32512315270937,\n",
       "  76.84729064039409,\n",
       "  76.35467980295566,\n",
       "  77.83251231527093,\n",
       "  76.84729064039409,\n",
       "  77.83251231527093,\n",
       "  76.35467980295566,\n",
       "  76.84729064039409,\n",
       "  78.32512315270937,\n",
       "  78.81773399014779,\n",
       "  78.32512315270937,\n",
       "  77.33990147783251,\n",
       "  76.35467980295566,\n",
       "  76.84729064039409,\n",
       "  76.84729064039409,\n",
       "  77.33990147783251,\n",
       "  77.83251231527093,\n",
       "  77.33990147783251,\n",
       "  77.33990147783251,\n",
       "  76.84729064039409,\n",
       "  76.84729064039409,\n",
       "  76.35467980295566,\n",
       "  76.84729064039409,\n",
       "  77.33990147783251,\n",
       "  77.83251231527093,\n",
       "  76.84729064039409,\n",
       "  78.32512315270937,\n",
       "  77.83251231527093,\n",
       "  78.81773399014779,\n",
       "  78.32512315270937,\n",
       "  76.84729064039409,\n",
       "  76.35467980295566,\n",
       "  76.84729064039409,\n",
       "  75.86206896551724,\n",
       "  76.35467980295566,\n",
       "  75.86206896551724,\n",
       "  76.84729064039409,\n",
       "  77.33990147783251,\n",
       "  77.33990147783251,\n",
       "  77.33990147783251,\n",
       "  78.32512315270937,\n",
       "  79.3103448275862,\n",
       "  79.3103448275862,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  79.80295566502463,\n",
       "  77.33990147783251,\n",
       "  72.9064039408867,\n",
       "  71.42857142857143,\n",
       "  76.84729064039409,\n",
       "  79.3103448275862,\n",
       "  75.86206896551724,\n",
       "  76.84729064039409,\n",
       "  77.83251231527093,\n",
       "  78.81773399014779,\n",
       "  76.84729064039409,\n",
       "  76.84729064039409,\n",
       "  77.83251231527093,\n",
       "  77.83251231527093,\n",
       "  77.83251231527093,\n",
       "  78.32512315270937,\n",
       "  79.80295566502463,\n",
       "  78.81773399014779,\n",
       "  79.3103448275862,\n",
       "  78.81773399014779,\n",
       "  78.81773399014779,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  78.32512315270937,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  78.81773399014779,\n",
       "  78.81773399014779,\n",
       "  77.83251231527093,\n",
       "  77.33990147783251,\n",
       "  77.33990147783251,\n",
       "  76.35467980295566,\n",
       "  75.86206896551724,\n",
       "  75.86206896551724,\n",
       "  77.33990147783251,\n",
       "  76.84729064039409,\n",
       "  76.84729064039409,\n",
       "  77.33990147783251,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  79.3103448275862,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  77.33990147783251,\n",
       "  77.83251231527093,\n",
       "  78.81773399014779,\n",
       "  77.83251231527093,\n",
       "  78.81773399014779,\n",
       "  79.3103448275862,\n",
       "  80.29556650246306,\n",
       "  77.83251231527093,\n",
       "  79.3103448275862,\n",
       "  79.3103448275862,\n",
       "  79.3103448275862,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  81.77339901477832,\n",
       "  81.77339901477832,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  81.77339901477832,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  79.3103448275862,\n",
       "  78.81773399014779,\n",
       "  78.81773399014779,\n",
       "  79.3103448275862,\n",
       "  79.80295566502463,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  79.3103448275862,\n",
       "  80.78817733990148,\n",
       "  78.81773399014779,\n",
       "  79.80295566502463,\n",
       "  79.3103448275862,\n",
       "  78.32512315270937,\n",
       "  77.33990147783251,\n",
       "  77.83251231527093,\n",
       "  77.83251231527093,\n",
       "  77.83251231527093,\n",
       "  78.81773399014779,\n",
       "  77.83251231527093,\n",
       "  78.32512315270937,\n",
       "  78.32512315270937,\n",
       "  77.83251231527093,\n",
       "  77.33990147783251,\n",
       "  76.84729064039409],\n",
       " 'test_loss': 5.786125828680503,\n",
       " 'test_ATT_loss': 0.8526348501266116,\n",
       " 'test_VAL_loss': 1.6444969928512971,\n",
       " 'test_ATT_acc': 96.42147117296223,\n",
       " 'test_VAL_acc': 99.47916666666667,\n",
       " 'test_VAL_jac': 0.740451397995154,\n",
       " 'test_VAL_acc_1': 78.64583333333333,\n",
       " 'model_filename': 'model_storage/HGT/model.pth'}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "states = pd.DataFrame(train_state['val_VAL_acc'], columns=['val_VAL'])\n",
    "states['train_VAL'] = pd.DataFrame(train_state['train_VAL_acc'])\n",
    "states['val_ATT'] = pd.DataFrame(train_state['val_ATT_acc'])\n",
    "states['train_ATT'] = pd.DataFrame(train_state['train_ATT_acc'])\n",
    "states['val_VAL_1'] = pd.DataFrame(train_state['val_VAL_acc_1'])\n",
    "states['train_VAL_1'] = pd.DataFrame(train_state['train_VAL_acc_1'])\n",
    "states['train_VAL_jac'] = pd.DataFrame(train_state['train_VAL_jac'])\n",
    "states['val_VAL_jac'] = pd.DataFrame(train_state['val_VAL_jac'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>val_VAL</th>\n",
       "      <th>train_VAL</th>\n",
       "      <th>val_ATT</th>\n",
       "      <th>train_ATT</th>\n",
       "      <th>val_VAL_1</th>\n",
       "      <th>train_VAL_1</th>\n",
       "      <th>train_VAL_jac</th>\n",
       "      <th>val_VAL_jac</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>71.428571</td>\n",
       "      <td>74.238227</td>\n",
       "      <td>30.894309</td>\n",
       "      <td>28.254848</td>\n",
       "      <td>23.645320</td>\n",
       "      <td>26.315789</td>\n",
       "      <td>0.096953</td>\n",
       "      <td>0.097701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>88.669951</td>\n",
       "      <td>89.473684</td>\n",
       "      <td>50.609756</td>\n",
       "      <td>47.922438</td>\n",
       "      <td>55.172414</td>\n",
       "      <td>49.030471</td>\n",
       "      <td>0.242382</td>\n",
       "      <td>0.245484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>93.596059</td>\n",
       "      <td>91.412742</td>\n",
       "      <td>57.317073</td>\n",
       "      <td>59.833795</td>\n",
       "      <td>66.502463</td>\n",
       "      <td>55.678670</td>\n",
       "      <td>0.415512</td>\n",
       "      <td>0.499179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>95.566502</td>\n",
       "      <td>94.182825</td>\n",
       "      <td>63.414634</td>\n",
       "      <td>72.853186</td>\n",
       "      <td>68.965517</td>\n",
       "      <td>63.988920</td>\n",
       "      <td>0.522161</td>\n",
       "      <td>0.596880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>97.044335</td>\n",
       "      <td>94.182825</td>\n",
       "      <td>67.073171</td>\n",
       "      <td>76.731302</td>\n",
       "      <td>71.428571</td>\n",
       "      <td>64.819945</td>\n",
       "      <td>0.538319</td>\n",
       "      <td>0.628900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295</th>\n",
       "      <td>98.029557</td>\n",
       "      <td>99.722992</td>\n",
       "      <td>95.731707</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>78.325123</td>\n",
       "      <td>86.703601</td>\n",
       "      <td>0.876731</td>\n",
       "      <td>0.729064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>296</th>\n",
       "      <td>98.029557</td>\n",
       "      <td>99.722992</td>\n",
       "      <td>95.528455</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>78.325123</td>\n",
       "      <td>85.872576</td>\n",
       "      <td>0.883657</td>\n",
       "      <td>0.726601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>297</th>\n",
       "      <td>98.029557</td>\n",
       "      <td>99.722992</td>\n",
       "      <td>95.731707</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>77.832512</td>\n",
       "      <td>88.919668</td>\n",
       "      <td>0.892890</td>\n",
       "      <td>0.732348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>98.029557</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>95.528455</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>77.339901</td>\n",
       "      <td>88.365651</td>\n",
       "      <td>0.883195</td>\n",
       "      <td>0.721675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>98.029557</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>95.731707</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>76.847291</td>\n",
       "      <td>86.980609</td>\n",
       "      <td>0.873961</td>\n",
       "      <td>0.737274</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>300 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       val_VAL   train_VAL    val_ATT   train_ATT  val_VAL_1  train_VAL_1  \\\n",
       "0    71.428571   74.238227  30.894309   28.254848  23.645320    26.315789   \n",
       "1    88.669951   89.473684  50.609756   47.922438  55.172414    49.030471   \n",
       "2    93.596059   91.412742  57.317073   59.833795  66.502463    55.678670   \n",
       "3    95.566502   94.182825  63.414634   72.853186  68.965517    63.988920   \n",
       "4    97.044335   94.182825  67.073171   76.731302  71.428571    64.819945   \n",
       "..         ...         ...        ...         ...        ...          ...   \n",
       "295  98.029557   99.722992  95.731707  100.000000  78.325123    86.703601   \n",
       "296  98.029557   99.722992  95.528455  100.000000  78.325123    85.872576   \n",
       "297  98.029557   99.722992  95.731707  100.000000  77.832512    88.919668   \n",
       "298  98.029557  100.000000  95.528455  100.000000  77.339901    88.365651   \n",
       "299  98.029557  100.000000  95.731707  100.000000  76.847291    86.980609   \n",
       "\n",
       "     train_VAL_jac  val_VAL_jac  \n",
       "0         0.096953     0.097701  \n",
       "1         0.242382     0.245484  \n",
       "2         0.415512     0.499179  \n",
       "3         0.522161     0.596880  \n",
       "4         0.538319     0.628900  \n",
       "..             ...          ...  \n",
       "295       0.876731     0.729064  \n",
       "296       0.883657     0.726601  \n",
       "297       0.892890     0.732348  \n",
       "298       0.883195     0.721675  \n",
       "299       0.873961     0.737274  \n",
       "\n",
       "[300 rows x 8 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['KMP_DUPLICATE_LIB_OK']='True'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAr5klEQVR4nO3deXxcdb3/8ddnJpNM9j1p0iRNW0qXdAltgGJFLS2LUCkgaBGweFEe98r1Al4U0Kvg/bmg3ut2VbSo2Adia62UIiJaagHZm9KdtqRLmmbf98xklu/vjzkZGpq0aSZpMpPP8/HI48ycmTPnc3KSd775nnO+R4wxKKWUiiy2sS5AKaXUyNNwV0qpCKThrpRSEUjDXSmlIpCGu1JKRaCosS4AICMjwxQWFo51GUopFVZ27NjRaIzJHOi1cRHuhYWFlJaWjnUZSikVVkTk+GCvabeMUkpFIA13pZSKQBruSikVgTTclVIqAmm4K6VUBDpjuIvIb0SkXkT2nTQvTUS2iEiZNU096bUHReSwiBwSkStHq3CllFKDG0rL/bfAVe+b9wCw1RgzA9hqPUdE5gCrgCJrmZ+LiH3EqlVKKTUkZzzP3RjzsogUvm/2SuAj1uO1wIvA/db89cYYN3BMRA4DFwGvj1C95967f4PKUii4GMQOx18b64qUUpEkazbMvWHEP3a4FzFlG2NqAIwxNSKSZc2fDLxx0vsqrXmnEJE7gTsBCgoKhlnGKHrlh3DwL1C5fYAX5ZyXo5SKUHNvGFfhPpiBUm/Au4EYY9YAawBKSkrG1x1DDv0VXngYsorgww/Akv+Al/8HjA+WfhWiYsa6QqWUOq3hhnudiORYrfYcoN6aXwnkn/S+PKA6lALPqXf/DlU74PWfBoL9zhchKjrw2vKHxrQ0pZQ6G8M9FfIZYLX1eDWw+aT5q0QkRkSmAjOAt0Ir8Rzw9MAzX4Df3wQvPQKT5sEtf3wv2JVSKsycseUuIusIHDzNEJFK4CHgEWCDiNwBVAA3ARhj9ovIBuAdwAvcZYzxjVLtoas/CJs/H5h6uuDS/4Ql90BMIoj2qyulwpeMhxtkl5SUmHM+KqS3F35YBMYfOJgx6xqY9pFzW4NSSoVARHYYY0oGem1cDPk7Jo69BF31cPN6mPnRsa5GKaVG1MQdfmD/JohJgumXjXUlSqlBeHx+xkPvQjiamC13nzdwDvvMq/W0RqXGCZfHx9M7q8hOdvKR8zPp9flZ+dNXSYp1cNOiPPLT4lg8LX2sywQCtf7+zQpWLMghK9E51uUMaGKGe80ucLXC+VeMdSVqnKhu7WHLO3XkpcZy2awsZBQPqPf0+vjL3hp6PD6unJNNVtLA4fDioXqON3Wz5LwMzstKYMfxZvZVtXPZrCxq213EOux0ur3ER0fR4fbwbm0HF09LZ3ZO0ojUWdPWw97KNj48M5OYqP6jiLR29/L6kSY+OCODRKcDr8/PT/5xmN+8cgy/MTz8sSLiYuwsnZlFfMyZY6ahw80n17zO0YYuABZPSyM3JZaDtR2IwFvHmkmOdXBBQQopsQ5++MliRASvz8+XNu6huauX39x+IXbb2e23mrYefvD3d3ntSBPXXZBLZkKgsXe4oZONOyrp9fpxOuzcungKCwtS+drmfVxZlE1VSw/bDjXwh+0nuPWSKXzk/Ex2V7ayZHoGqfHj4yy7iXlA9eXvwz++CV86CvHjoyUwXPur25iSHk9CTBSH6zvJS43F6ej/i2iM4d4/7GLxtHRWXVRARVM3cTF2MhLe+6+lqrUHn89QkB4XnFfb5sLj85OfFpjX5fbS2Olme3kLf9tfy9dXzOFQbQcZiTEU56ecsVa318eh2g6KcpP7/RIaY9hf3c7snCQO1LQza1IiUfb3egwP1rYzPTMBh31kehFPNHdzsLaDqRnxJDqjaO/x8K+/28ERK1iyEmMozIjn3uXnc6yxizUvHyEtPppbLp5CUqyDSUlO5uUlD3l9Xp+fP+6o5Km3K5k3OYXn9tZQ2+4CICbKRk6yk2vm51Ccn4pNoKQwjWd2VfG1zfsBsNuE2xZP4XdvHMfrN6TGOWh3efH5T/3djYu28/vPLSY3xYnHZ3DYhQ6Xl+mZCTR39dLY6eb87EQAnt1TzeZd1Xx9xRx++1o5h2o7WD47iz/vqWFubhLrt5/A7fVTkBbHr1eX8Pd36vjTjko8fj/Nnb109fpIiIkiLzWW+XnJbCit5KqiSdR3uHi7ohWAmy8q4Ds3zAPgnep28tNiOdrQxYzsBOKio2jr8XD3+p3srGil1+vn57cs5ERLNz9+oYymrl4+vjCPa+ZPoqbNxUOb9+O1tvniqWlUt/Xg9vip73ADUDIllbqOwPd19qQkFuSn8M+yBqZmJPDq4UYMhoyEGOZNTubdug5mZifyh9IT+P0wLy+ZHcdbgt9Hu01YWZzL5JRYjjd18+c91RgT+Nlo6urFGMMnL8xn865qunvfOyFw7uQknvzsYqpbe8hNiSU51sG+qjampMeR6HQM+PNhjBl2Y+J0B1QnZrg/fg242+Ff/3nu1hmi1w438vu3Krj38vNJiXWQFh/Nm8eaWbXmDTISovnXD0/n288dYNGUVO67YiabdlYxLTOeT19SyNsVLXzqsTfJS43l6buWsOx/X2LmpER+tboEuwi17S5ufPQ12l1eFhakYLcJfgO7KlpJjnPwX9fM5lhjF/ur23npUAN2m9Dj6X+Ga1FuEonOQAstNS6aRVNS2VfVxvUL8/jNK8dwe31UNHVT3eaiMD2OScmB1mpMlJ30+Gie2lnFrEmJHKzt4NoFufzok8XYbMLv36zgK5v2csvFBdy19DxaunuxiTA9M4HoqP5h39jppq7dxbSMBGKj7fzf1jJ6PD6uKJrE5l1VZCc58Xj9/HhrGV6/QQQcNhu9Pj8Aj96ykK5eH68dbuSNo01UtwWCojg/hZbuXo43dQfX9f0b53PDwjzaejxUtnTzv39/F4dd+MD0DHZXtrIgLyU4XfdWBWX1nUxOiaWqtYcFecnc/9FZZCXG8Ls3Kihv6uLFQw3Bz3Y6bLg8fi6blcU3ri3iW385wPP7a0mOdfDorQv5/JNvU5AWx9Xzckh0RtHT6yPGYWfJ9HRWP/4WnS4vUXYbbo+P+JgoattdXF88mbcrWihv6uaa+Tl88fLzufVXb1JjbaNNIMr6XvStf2VxLpfPyearm/bR1uMBYMl56WQnOomNtnPpjExeereel99tpKq1h6UzM/nN7RfS6fby038cprKlh+f21fDB8zL48PmZfPMvB4KfXTIllZyUWPZUtlLd2sOK+bmsujCfi61uF4/PT6fLS0qcIxh8f95djQF+8eIRKpq7uWxWFlE24cKpaTy7p5rS8haWz8nGJsJf9lTjN4HvpcdnWDYri4SYKN6uaOF4czeTkpzUtLm4dkEuX7pyJvlpcbS7PPh8gTyMjrL1+4/jnep2nnq7ks9eOo2k2Cj8BhJionB5fFS2dPP0zmriYuzBnwOXx09KnIO81Fj2VbUzd3ISX19RxNO7qpiU5KShw01afDQnmruZnpXAXUvPG1YuaLj38Xlh2zcD48Zceh8s+9rorzNEh2o7+M5fDwR/+fPTYqls6eETi/LZUdFCT6+P+Bg779Z1kuiMotPtxZhAi9Dt9TM5JRanwxZslc6bnMzeqjYAUuICLQmP109stJ2r5+VwsLYjuO4kp4MXDtT1qyfabkMEvnTlTPZXt/PJC/PZV9XGCwfq6GtIHq7vpLmrN7hMRkIM0zLjiY+2s+S8DF56twG3NxCoNW09nGjuYWZ2Ioes1tShug6KcpPITIzhxUMNJMc6aHcFwqXvx3VKehz3Lj+fw/Wd1LS5cHl9bNlfR6/PT0ZCDDcsnMyal48O+D392IJc/mVJIVveqaPL7SXR6UAE/vOKmcH3uDw+fvtaOcebunjoY0XYRCir78Dvh+/97SD/LGsMhrXTYSMhxoHdBnXt7uD3vm86NSOe+6+ayZVFk2js7CU9Phrb+7oPTjR309bjob3Hw5/ermJObhK3LZ5CdJSNXq+fbz93gA+el8HyOdm0dXuIjbaf8scNoLyxixt/8Roen8HpsNHt9nH9wsms334Cm8AnS/LZUFoZ/OO8+pIptLu8/NtHplPd2sNze2v46jVz8PkNaVb3ws6KFh7751E+fUnhgH3ex5u6+PELZXzxivPJS33vP79Ot5f/2rSX1482UdfuJifZySXT00mIieKJN46TGBPFnNwkPrNkKlcWTRpwXw2k0+3FbwxJJ7WEXR4fLo+PlLhAzZt3VbGzopUvXTkTj88fnN/3RyPRGUW7yxvcxpGyt7KNx187xoysRPZVtdHY6WZ2ThJPvHEcn98E9+fJ039feh7/sWzGsNan4Q6BVHjieji6DRbdDlc9Ao7Y0V1niJ7bW8MX1u0kPtrOv192Hk6Hna9v3k9GQgyNnW6cDhtrbith1qREvrhhN7d/oJCcFCdVLT1cWJjGgZp2fvRCGdVtPdz+gUJ+sOVdunt93LV0Ok+8Hrhp+kVT03E6bNx3xUwKM+L7rd8Yw4r/e4XD9Z3kJAdaOn+750MAp7z3ZJ1uL8cauqhtd/HYy0f5/k3zmZI+8Pu9Pj+7TrRyQUEq71S3MzsnkWf31PCLl47Q4/GxckEut11SyN3rdzJvcjILp6TS6fKy5uWjHKrrwCaQkxzYj0vOS+eDMzJ54vVytpe3kJ8Wy6O3LKKqNfDH47F/BsL+v1fOPeu+2ZP19Pr46bYyth9rYU5uEnsqW/nBJ4qZlOzkYG0HsyYlcqDmvW6muZOTR6xLaSjq2l14/YZouw2Xx0d+Whw1bT109/qYnplAXbuLH71QRltPLz+9eeEpf2hGWlVrD/dt2M3nl07n0hmZQKA7cXJKbDB0I92h2g6ON3VxQUEqbT29JMU66HB5SYiJInuQYy5DoeEO0HIcfjw/MBDY0gdHd13D8NaxZr729D7+a8Vsth9r5sk3K2ju7mVRQSqPfbqE1Pho/H7DPw7Wc/G0NF5+t5FFU1KD3RtDseN4C06HjaLcZPZVtREXbWdaZsJplznW2EVTp5vMxBjq2t1cNDUt1E0dET6/4aV36ylIi+e8rP7bYIzhlcON5KbEMv0M26dUONNwBzjwZ/jDrfDZrZA34PdiVLg8Pk40dzMjO5GeXh+/efUY28ub+fQlU1hyXgYnmnvIT4vl6h//M9h1AnDFnGxm5SRxx5KpJMcNfCBGKTWx6RWqADV7QGyQNeecrdLl8fGZx7fz+tEmFk1JpbKlm7p2N+nx0fzLb0tJdEbRYfX/dbi8/L+VRRys7eCmkvwhnX2ilFKDmTjhXrsXMs6H6LgzvzcEfac1+fyB0w9fP9rEJ0ryOFTXyaxJSfzfzedxQUHgDIpXDzcyNzdwgHPVRflcNit7VGtTSk0cEyjc98CUJaP28X6/4cGn9rL9eDNfWzGHX750hDeONvNf18zms5dOO+X9n76kkE9fUjhq9SilJraJEe6d9dBeBTnzR+XjH3v5KI++dITmrl4cduEzj28nNc7Bt6+fx6cuHoe3EFRKRbyJEe4nrPuF5F04oh/721ePcaShi6d3VpGfFscXLz+f2TlJvHmsiVsXT+l3Hq5SSp1LEyPcK98CmwNyikfsI3/3xnEe/vM7weePfHwe8/NSAFg0JXXE1qOUUsMxQcK9NNAl4xiZ0dv+ureGr23ex7JZWVw4NY3aNlcw2JVSajyI/HD39kLV24GrUkfA60eauHv9LhYVpPLTTy0kNtp+5oWUUuoci/ybdRx9Ebw9I3ILvU63l3v+sJOC9Dh+tbpEg10pNW6FFO4icreI7BOR/SJyjzUvTUS2iEiZNR3bDuh3noaYZJi+NKSPae7q5f6Ne6jvcPP9G+dPmDExlFLhadjhLiJzgc8BFwELgBUiMgN4ANhqjJkBbLWejw2fFw4+C7NCu+PS5l1VfPh72/jrvhruWXY+FxToAVOl1PgWSp/7bOANY0w3gIi8BFwPrAQ+Yr1nLfAicH8I6xm+9kpwtYV08ZIxhu/+9SD5aXH8eFUxM6wbHSil1HgWSrfMPuBDIpIuInHA1UA+kG2MqQGwplkDLSwid4pIqYiUNjQ0DPSW0LVVBabJecP+iHLrBhM3X1ygwa6UChvDDndjzAHgu8AW4HlgN+A9i+XXGGNKjDElmZmZwy3j9NqrA9OkycP+iNeONAKwZHp4345PKTWxhHRA1Rjza2PMQmPMh4BmoAyoE5EcAGtaH3qZw9ReGZgm5Q5r8cdfPcbPtx0hJ9nJ1NPcnEIppcabUM+WybKmBcANwDrgGWC19ZbVwOZQ1hGS9mpwJkPM2d+wobbNxbefOwDAnR+aNuwb2Cql1FgI9SKmP4lIOuAB7jLGtIjII8AGEbkDqABuCrXIYWuvHnaXzOOvHcPnN6y/czH5aaM7TLBSSo20kMLdGHPpAPOagGWhfO6Iaas863C/74+7aehw88rhRq5dkKvBrpQKS5E9/EB7NeQsGNJbGzvdNHX2snFHoJ9+fl4y37x+3mhWp5RSoyZyw93bC131Q2q5769uY9Uv38Dt9SMCW+79MPlpscRE6fACSqnwFLnh3nAwME079S5IJ+t0e/ns2lKc0XZ6PD6WTM/gvKyzPwCrlFLjSeSGe6V1g478wW/Q4fcbvvf8QWrbXTz1bx8gJspORoKOGaOUCn+RG+4ntkN8FqRMGfDlHcebeeiZ/eyraue2xVN0vBilVESJ3HCvfAvyL4IBzk/3+Q23P76dhJgofvjJBaxcMPwrWJVSajyKzPHce1qg+ShMXjTgyxXN3XS4vNx7+flcf0EeNpteoKSUiiyRGe7tNYFpauGAL5fVdQAwQw+cKqUiVGSGe5c1ymR8xoAvl9V3AuhZMUqpiBWZfe7dgZEcie8/2qQxhi+s28m2g/XkJjtJdDrGoDillBp9EdpybwpM4/q33MvqO3l2Tw1dvT4yk5xjUJhSSp0bERruDYBAXFq/2a8ebgw+np6pQ/gqpSJX5HbLxKWBrf/wAa8daaIgLY6f37KQ/FQdEEwpFbkitOXeeEqXjN9veONoEx+Yns7cyckkx2l/u1IqckVmuHc3nXIwtbbdRYfLy7y85DEqSimlzp3IDPeuBojvf8/T8qYuAKakaV+7UiryRWi4n9otU9HUDcCUdO1rV0pFvsgLd583MPzA+7plypu6cdiF3JTYMSpMKaXOnVBvkH2viOwXkX0isk5EnCKSJiJbRKTMmp7b4RZ7mgFzytWpx5u6yE+Lw67jyCilJoBhh7uITAb+AygxxswF7MAq4AFgqzFmBrDVen7uBIce6N9yP97UTWG69rcrpSaGULtlooBYEYkC4oBqYCWw1np9LXBdiOs4O511gWlCVnCWMYbjTV0U6M2ulVITxLDD3RhTBfwPUAHUAG3GmL8D2caYGus9NUDWQMuLyJ0iUioipQ0NDcMt41Sd1mclZAdnHaztoKvXx+ycxJFbj1JKjWOhdMukEmilTwVygXgRuXWoyxtj1hhjSowxJZmZmWdeYKj6Wu4ndctsPRCYt3TmgH9nlFIq4oTSLbMcOGaMaTDGeICngA8AdSKSA2BN60Mv8yx01kFULMS810rfcqCeBfkpZOlgYUqpCSKUcK8AFotInIgIsAw4ADwDrLbesxrYHFqJZ6mrARIyg7fXq27tYfeJVpbP0la7UmriGPbAYcaYN0VkI/A24AV2AmuABGCDiNxB4A/ATSNR6JB11vXrb396VxUA1xbnntMylFJqLIU0KqQx5iHgoffNdhNoxY+Nzobg7fWMMWx6u4pFU1KZoqdBKqUmkMi7QrWzLnga5InmHsrqO7l2gbbalVITS2SFu88bGBHSCvcdFc0AXDwt7XRLKaVUxImscO9uBMx74X68hcSYKGZk6fntSqmJJbLCvdM66zI+EO6l5S0UF6ToeDJKqQknMsM9IZsut5dDdR0sLDi345YppdR4EGHh3jeuTCblTV0YAzMnaZeMUmriiaxw73qvW+ZEcw+A3ghbKTUhRVa4d9aDIx5iEqhsCdx5SUeCVEpNRJEX7sFz3LtJdEaRHOcY46KUUurci7Bwf+8Cpormbu2SUUpNWJEV7l0N77XcW3rIT9P7pSqlJqbICvfOOojPwhhDZYu23JVSE1fkhLu3F3paICGbhk43Lo+ffD2YqpSaoCIn3PtujJ2QSXWrC4C8VO2WUUpNTJET7q62wNSZQnVr4Bz33BQNd6XUxBQ54e5zB6ZRTg13pdSEFznh7u0L9xiqWntIiIkiyRnSvUiUUipsRWS4V7f2kJviRERHg1RKTUwRGO5Oqltd2iWjlJrQhh3uIjJTRHad9NUuIveISJqIbBGRMmt6bsbc7etzt0dbLXcNd6XUxDXscDfGHDLGFBtjioFFQDewCXgA2GqMmQFstZ6PPqvl7sZBU1cvkzXclVIT2Eh1yywDjhhjjgMrgbXW/LXAdSO0jtPzBs5trwsMBkluivOcrFYppcajkQr3VcA663G2MaYGwJpmDbSAiNwpIqUiUtrQ0BB6BVbLvabLAJCbrC13pdTEFXK4i0g0cC3wx7NZzhizxhhTYowpyczMDLWMYLhXd/oBPcddKTWxjUTL/aPA28YY6x531IlIDoA1rR+BdZyZdUC1st2PCExK1m4ZpdTENRLhfjPvdckAPAOsth6vBjaPwDrOzGq5n2j3kp3oxGGPnLM8lVLqbIWUgCISB1wOPHXS7EeAy0WkzHrtkVDWMWReF9iiqGrv1YOpSqkJL6Tr840x3UD6++Y1ETh75tzy9gYvYCrKTTrnq1dKqfEkcvouvC6MNa6MnuOulJroIifcfW78tmh6vX49U0YpNeFFTrh73Xht0QBkJ2mfu1JqYouocPfgAPQ0SKWUiqhw77XCPTspZoyLUUqpsRVB4e7CZRyIQEaChrtSamKLnHD39dLjjyI9PkYvYFJKTXiRk4JeF13+KLIStdWulFIRFO69dHlt2t+ulFJEVLi7aPfa9TRIpZQigsLdWOGepeGulFKRE+5+r5te49BuGaWUIoLCHY8bNw7S46PHuhKllBpzERPu4gtcxJTodIx1KUopNeYiI9yNwebvxY2DRGdIoxgrpVREiIxw9/UC4DZRJMRouCulVGSEu9cFYLXctVtGKaUiJNwD9091E63dMkopRej3UE0RkY0iclBEDojIJSKSJiJbRKTMmqaOVLGDssLdLw6cDvuor04ppca7UFvuPwaeN8bMAhYAB4AHgK3GmBnAVuv56LLCXRx6AZNSSkEI4S4iScCHgF8DGGN6jTGtwEpgrfW2tcB1oZU4BL5AuNujNdyVUgpCa7lPAxqAx0Vkp4j8SkTigWxjTA2ANc0aaGERuVNESkWktKGhIYQyCB5Q1XBXSqmAUMI9ClgIPGqMuQDo4iy6YIwxa4wxJcaYkszMzBDKALyBUyGjovXG2EopBaGFeyVQaYx503q+kUDY14lIDoA1rQ+txCGwWu4ObbkrpRQQQrgbY2qBEyIy05q1DHgHeAZYbc1bDWwOqcKhsA6oOpzacldKKQh0rYTiC8CTIhINHAU+Q+APxgYRuQOoAG4KcR1nZh1QdcbEjfqqlFIqHIQU7saYXUDJAC8tC+Vzz7oOjwsBYmK15a6UUhAhV6i63YE+9xinttyVUgoiJdxdXQDExmq4K6UUREi497p6AIiNix/jSpRSanyIjHB3B8I9Pk5b7kopBRES7h4Nd6WU6idCwt2F20SRGKv3T1VKKYiQcPd7XHqLPaWUOklEhLsvGO56FyallIIICXd/r4teHMTpjTqUUgqIkHA3XhceicZmk7EuRSmlxoUICXc3PtEuGaWU6hMR4S5eNz6bnimjlFJ9IiPcfb34bDFjXYZSSo0bERHuNr8bv11b7kop1Sciwt3ud2Ps2nJXSqk+ERLuHojScFdKqT4REe4O04touCulVFDYh7vL48OBB3HoXZiUUqpP2Id7p9tLDB7sDm25K6VUn5BG2hKRcqAD8AFeY0yJiKQBfwAKgXLgE8aYltDKHFyHy0sGXuzRztFahVJKhZ2RaLkvNcYUG2P6bpT9ALDVGDMD2Go9HzVdbi8x9GJzaLgrpVSf0eiWWQmstR6vBa4bhXUEebxeosWHRGm4K6VUn1DD3QB/F5EdInKnNS/bGFMDYE2zBlpQRO4UkVIRKW1oaBh2AT6PO/BAL2JSSqmgUO9uscQYUy0iWcAWETk41AWNMWuANQAlJSVmuAX4e12BB9oto5RSQSG13I0x1da0HtgEXATUiUgOgDWtD7XI0/F7A+Gu3TJKKfWeYYe7iMSLSGLfY+AKYB/wDLDaettqYHOoRZ6O6e0Ld+2WUUqpPqF0y2QDm0Sk73N+b4x5XkS2AxtE5A6gArgp9DIH5/P0AGDTi5iUUipo2OFujDkKLBhgfhOwLJSizqoOby8AohcxKaVUUNhfoWp8gXC3abeMUkoFhX24+62Wu01PhVRKqaAICHcvAPaoUM/qVEqpyBH+4W51y9gd2nJXSqk+YR/uxtfXcneMcSVKKTV+REC4ewCwa5+7UkoFhX24o90ySil1irAP92C3jEMPqCqlVJ+wD3e/1S0TpfdQVUqpoLAPd6yWu00PqCqlVFD4h7s/0HLHpuGulFJ9wj/crW4Z7BruSinVJ/zDPdhy1wOqSinVJ+zDve9sGQ13pZR6T9iHu/itcNduGaWUCgr7cNcDqkopdaqwD/dgy91mH9tClFJqHImIcPcQBYHb/SmllGIEwl1E7CKyU0SetZ6nicgWESmzpqmhl3ma9fs9+NBWu1JKnWwkWu53AwdOev4AsNUYMwPYaj0fNeL34RMNd6WUOllI4S4iecA1wK9Omr0SWGs9XgtcF8o6zliD8eAb/n2+lVIqIoXacv8R8GXAf9K8bGNMDYA1zQpxHadl83u15a6UUu8z7HAXkRVAvTFmxzCXv1NESkWktKGhYbhlIMaLT7TlrpRSJwul5b4EuFZEyoH1wGUi8jugTkRyAKxp/UALG2PWGGNKjDElmZmZwy7C5vdqt4xSSr3PsFPRGPMg8CCAiHwEuM8Yc6uIfB9YDTxiTTeHXubgxPjwa8tdqXHL4/FQWVmJy+Ua61LCltPpJC8vD4dj6BdrjkYqPgJsEJE7gArgplFYR5DNePBrn7tS41ZlZSWJiYkUFhYiej3KWTPG0NTURGVlJVOnTh3yciMS7saYF4EXrcdNwLKR+NyhsBsffh00TKlxy+VyabCHQERIT0/nbI9Nhv0Vqjbj1XBXapzTYA/NcL5/YR/uduPVPnellHqfsA93m4a7UkqdIuzD3W58GB0RUik1QhISEs74nh/+8Ic4nU7a2tpoamqiuLiY4uJiJk2axOTJkykuLsZutzNnzhyKi4tJS0tj6tSpFBcXs3z58nOwFaNztsw5ZceLER3LXalw8I0/7+ed6vYR/cw5uUk89LGiEf3MM1m3bh0XXnghmzZt4vbbb2fXrl0APPzwwyQkJHDffff1e//tt9/OihUruPHGG89ZjRHRctcDqkqpwdx///38/Oc/Dz5/+OGH+cY3vsGyZctYuHAh8+bNY/PmoV+Oc+TIETo7O/nmN7/JunXrRqPkERH2qWhHz5ZRKlyc6xY2wKpVq7jnnnv4/Oc/D8CGDRt4/vnnuffee0lKSqKxsZHFixdz7bXXDumslHXr1nHzzTdz6aWXcujQIerr68nKGtUhtIYl7FvuUfgwGu5KqUFccMEF1NfXU11dze7du0lNTSUnJ4evfOUrzJ8/n+XLl1NVVUVdXd2QPm/9+vWsWrUKm83GDTfcwB//+MdR3oLhCetUNMYQZby49f6pSqnTuPHGG9m4cSO1tbWsWrWKJ598koaGBnbs2IHD4aCwsHBIwyPs2bOHsrIyLr/8cgB6e3uZNm0ad91112hvwlkL65a7x2ewix+05a6UOo1Vq1axfv16Nm7cyI033khbWxtZWVk4HA62bdvG8ePHh/Q569at4+GHH6a8vJzy8nKqq6upqqoa8vLnUliHu89vcGi3jFLqDIqKiujo6GDy5Mnk5ORwyy23UFpaSklJCU8++SSzZs0a0uesX7+e66+/vt+866+/nvXr149G2SERY8xY10BJSYkpLS096+XaXR683ymkNu9q5nzusVGoTCkVqgMHDjB79uyxLiPsDfR9FJEdxpiSgd4f1i13r88QhQ/s2nJXSqmThXUqen1+YvGBHlBVSo2gvXv3ctttt/WbFxMTw5tvvjlGFZ29sA53j99gxwd2DXel1MiZN29e8KrTcBXW3TI+r59o8enZMkop9T5hHe4enwcA0Za7Ukr1E9bh7vP0AhruSin1fmEd7l4r3LXPXSml+ht2uIuIU0TeEpHdIrJfRL5hzU8TkS0iUmZNU0eu3P583kC3jC1Kw10pNbDW1tZ+o0IO1dVXX01ra+uw1un1esnIyODBBx8E4Fvf+lZwzHe73R58LCIUFxczZ84cYmNjg/M3btw4rPWebNgXMUlg+LR4Y0yniDiAV4C7gRuAZmPMIyLyAJBqjLn/dJ813IuYmupOkP7oXFqXfoeUD39+GFuhlBpt/S6++esDULt3ZFcwaR589JFBXy4vL2fFihXs27ev33yfz4fdPjo3+nnuuef41re+RW1tLYcPH+432mRCQgKdnZ1DqvFk5+wiJhPQV6HD+jLASmCtNX8tcN1w13Em6c5A+SkJcaO1CqVUmHvggQc4cuQIxcXFXHjhhSxdupRPfepTzJs3D4DrrruORYsWUVRUxJo1a4LLFRYW0tjYSHl5ObNnz+Zzn/scRUVFXHHFFfT09Jx2nevWrePuu++moKCAN954Y1S3b1DGmGF/AXZgF9AJfNea1/q+97QMsuydQClQWlBQYIal+ZgxDyUZ8/bvhre8UmrUvfPOO2O6/mPHjpmioiJjjDHbtm0zcXFx5ujRo8HXm5qajDHGdHd3m6KiItPY2GiMMWbKlCmmoaHBHDt2zNjtdrNz505jjDE33XSTeeKJJwZdX3d3t8nJyTFdXV3ml7/8pfnCF77Q7/X4+PjT1jiYgb6PQKkZJJ9DOqBqjPEZY4qBPOAiEZl7FsuuMcaUGGNKMjMzh1eAzxuY6gFVpdQQXXTRRUydOjX4/Cc/+QkLFixg8eLFnDhxgrKyslOW6bv/KcCiRYsoLy8f9POfffZZli5dSlxcHB//+MfZtGkTPp9vpDfjjEbkbBljTCvwInAVUCciOQDWtH4k1jEgf+CAql7EpJQaqvj4+ODjF198kRdeeIHXX3+d3bt3c8EFFww4rntMTEzwsd1ux+v1Dvr569at44UXXqCwsJBFixbR1NTEtm3bRnYjhiCUs2UyRSTFehwLLAcOAs8Aq623rQaGfnPCs2VdxKQtd6XUYBITE+no6Bjwtba2NlJTU4mLi+PgwYMh94+3t7fzyiuvUFFRERzz/Wc/+9mY3Gs1lCZvDrBWROwE/khsMMY8KyKvAxtE5A6gArhpBOocmN/666ktd6XUINLT01myZAlz584lNjaW7Ozs4GtXXXUVv/jFL5g/fz4zZ85k8eLFIa3rqaee4rLLLuvX0l+5ciVf/vKXcbvd/eaPtrAez52mI7D1v+HSL0LOgpEvTCkVMh3PfWSc7amQ4d3kTZ8On1h75vcppdQEE97hrpRSY+Suu+7i1Vdf7Tfv7rvv5jOf+cwYVdSfhrtSatQZY/pdpRkJfvazn52zdQ2n+zysBw5TSo1/TqeTpqamYQWUCgR7U1MTTqfzrJbTlrtSalTl5eVRWVlJQ0PDWJcStpxOJ3l5eWe1jIa7UmpUORyOfleEqnNDu2WUUioCabgrpVQE0nBXSqkINC6uUBWRBuB4CB+RATSOUDljKVK2A3RbxivdlvFpuNsyxRgz4LC64yLcQyUipYNdghtOImU7QLdlvNJtGZ9GY1u0W0YppSKQhrtSSkWgSAn3NWd+S1iIlO0A3ZbxSrdlfBrxbYmIPnellFL9RUrLXSml1Ek03JVSKgKFdbiLyFUickhEDovIA2Ndz9kSkXIR2Ssiu0Sk1JqXJiJbRKTMmqaOdZ0DEZHfiEi9iOw7ad6gtYvIg9Z+OiQiV45N1QMbZFseFpEqa9/sEpGrT3ptXG6LiOSLyDYROSAi+0Xkbmt+2O2X02xLOO4Xp4i8JSK7rW35hjV/dPeLMSYsvwA7cASYBkQDu4E5Y13XWW5DOZDxvnnfAx6wHj8AfHes6xyk9g8BC4F9Z6odmGPtnxhgqrXf7GO9DWfYloeB+wZ477jdFgL3NV5oPU4E3rXqDbv9cpptCcf9IkCC9dgBvAksHu39Es4t94uAw8aYo8aYXmA9sHKMaxoJK4G+eweuBa4bu1IGZ4x5GWh+3+zBal8JrDfGuI0xx4DDBPbfuDDItgxm3G6LMabGGPO29bgDOABMJgz3y2m2ZTDjeVuMMabTeuqwvgyjvF/COdwnAydOel7J6Xf+eGSAv4vIDhG505qXbYypgcAPOJA1ZtWdvcFqD9d99e8issfqtun7lzkstkVECoELCLQSw3q/vG9bIAz3i4jYRWQXUA9sMcaM+n4J53Af6J5d4XZe5xJjzELgo8BdIvKhsS5olITjvnoUmA4UAzXA/1rzx/22iEgC8CfgHmNM++neOsC88b4tYblfjDE+Y0wxkAdcJCJzT/P2EdmWcA73SiD/pOd5QPUY1TIsxphqa1oPbCLwr1ediOQAWNP6savwrA1We9jtK2NMnfUL6Qce471/i8f1toiIg0AYPmmMecqaHZb7ZaBtCdf90scY0wq8CFzFKO+XcA737cAMEZkqItHAKuCZMa5pyEQkXkQS+x4DVwD7CGzDauttq4HNY1PhsAxW+zPAKhGJEZGpwAzgrTGob8j6fuks1xPYNzCOt0UCd6D+NXDAGPODk14Ku/0y2LaE6X7JFJEU63EssBw4yGjvl7E+khziUeirCRxFPwJ8dazrOcvapxE4Ir4b2N9XP5AObAXKrGnaWNc6SP3rCPxb7CHQ0rjjdLUDX7X20yHgo2Nd/xC25QlgL7DH+mXLGe/bAnyQwL/ve4Bd1tfV4bhfTrMt4bhf5gM7rZr3AV+35o/qftHhB5RSKgKFc7eMUkqpQWi4K6VUBNJwV0qpCKThrpRSEUjDXSmlIpCGu1JKRSANd6WUikD/H/cZzkXibZmFAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_ATT','train_ATT']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAyMUlEQVR4nO3deXxU1f3/8ddnJhvZyB52EpAlhJ2ACAoiiEstgkoLVmr9WrWt1qXfttJVrLbVuv2+tnZR269+bQsi7iur1H0Jq0DCmgAhIRshK1kmc35/3MkkIQlkmZjc8fN8PPKYmTsz956Tm7znzLnnnivGGJRSSvkXR08XQCmllO9puCullB/ScFdKKT+k4a6UUn5Iw10ppfxQQE8XACAuLs4kJSX1dDGUUspWtmzZUmSMiW/tuV4R7klJSaSnp/d0MZRSylZE5HBbz2m3jFJK+SENd6WU8kMa7kop5Yc03JVSyg9puCullB86a7iLyD9EpEBEdjVZFiMi60Vkv+c2uslzPxORAyKyV0Qu6a6CK6WUalt7Wu7PAJeetmw5sNEYMwLY6HmMiIwBlgCpnvf8WUScPiutUkqpdjnrOHdjzHsiknTa4iuBCz33nwU2A3d7lq8yxtQAWSJyAJgGfOyj8vrM6zty2Z9f3mL5sBPvM6PPERIigsk5eYpAp5AYEdIDJfQICIap34VdL8L4b0BwRONzxsDnT0NFATicMOk66Duo5Toy3oDYcyBhtPV4/wYIi4MBE+HQfyAwFAZP9W25K4sg802YtAwcnjZETTnsfB4mXw/OQN9ta9eLUJDpu/UljoHURb5bny9lvgm525svG3UpDJzS+PjwR3Dw3ZbvHTARoobCnldbX3f0UBh2IWz7J7jrfVTgJhxOmPxt2L8eSnN8v36w6jj6a9b9wr3W38aXMa25IwCmfAcy37D+Hycva/1/sanMtyB3GySkwNirfF6kzp7ElGiMyQMwxuSJSIJn+UDgkyavy/Esa0FEbgZuBhgyZEgni9E5tS43dz2/HZfbINK4fABFbAq6m2BxYRAGfKmlaouBL16Awkw4cQgu+W3jU5lvwFs/bnycux2uXdX87QUZsHoZJKTCLe9B2TFYdS1EJMJ/rbXuB4XB7dusW19Z+wvYuQqCw2Hs1day/zwIH/0R6l0w/Xu+2c7xXbDmRsAAcrZXt4MBcUDCGIgf5YP1+ZCrxqqr6xSNdTVwcBPctNF6WFMOz18HVcU0/30YcARagVOSRcvflScA40ZB0d5WnvcFA7tftv6WoRu24anjD9Oh7xB46WbI294N22lj27tfaqxb3o6W/4tNlRyG1d8Gd531/9GLwr0trf0WW/3YNMY8CTwJkJaW9qVeMeTIiSqWyVssG1XLsPgmgZa3E1eug5nVf+R3N1zG9f/4DIAPl1/EwKg+X2YRG635L6v1IQ747CmorcT7ibR/A8SNhO9/DB/9D2z8DbzyA6u13yDnc+s2/wt44Xooz7P+oE4egf+9zFpfbYUVCNFJvimz22W10MUB634F2R9Yraft/7aW/edBT4D4wNHPISQS7tgBfaLP/vqzqSyG/5kAm+6Db/7TWla4Fz57Eoy7c+sMCIHZd0OfqMZl7nrY/ABUFTUuEyec+z2IHQ6f/tVqRSekND5/9DMr2JeuglGXWcs23Q/vPwLVpda3sPS/W8F+06bmrfnSY/DHyVawL34WUhc2L2PdKXh8krVf5v4aLvjvztX1TDY/AJt/bwXvD9Ob/536QlmuVYcXboCoIVawL/wrTFzq2+20ZsMK+OAxiBkG45fA5t+1/F9sKm+H9U3mzp0Q2T3NyM6Ge76I9Pe02vsDBZ7lOcDgJq8bBOR2pYDdoSRjM/cEPocrLxKKmv/yCyffzrEPY7n3td3eZW/syGXRpIHEhAUhIhRX1BAVGkRQgANjDPVuQ4DT6nowxlBYUUNkSCAhgc0PNxRV1OB2G8KCAwgLDuBEZS2u+jMHhnPaT4nMz6TyvB8T/t59ODJeBwSHgFuclF30ALWVLhjzHaJ3vYZz39pm7zcIVbN/Q1D2uwQd+RgBXLOW4zq+h6CjH3Bq6q04TpUQfGgtcvwLBOvTuOEKXSLiXQa0fB5BpPnzgNXynf0TeOfnkPG6tazvILj4N/DO8sZlXSZWGPki2AHCYmHGD61/zpwtMGgKfPg47FgJoTGdW2dlkfWh1vRb15FP4L0/QEhUYxfVqZNQfABm3Gb9jgamwXc3NH6YH9psfQAMndm4nmEXwnsPwa6XYO3Pre2k/VfzYAfoOxAu/Jn1YT/mypZlDOwDl/zO+lA510ffqk533q1wYAPMvMP3wQ5WSF64HD5+AkqPWr+b8d/w/XZaM/MO68P1wuWQdD7sexv2rzvze2bf3W3BDiDtucyep8/9DWPMWM/jh4BiY8wDIrIciDHG/FREUoF/Y/WzD8A62DrCGHPGDry0tDTzZc4tk/fYbOTkYUJ+tJ2ovlHNnnO7DdN/v5GC8hrShkZTXFlLVlElAHNGxRMeEsjrO3KZmhTNC9+bwe/eyuDjg8W8dttMRIRH1u3lj5sOMDCqDx/cPQfx/GP+69PD/OJla8BRaJCTn12ewq9e2UVn3b9wLL99M4NTde3vG/3BhcP56aWjWfCnD9iZU9ri+XkpCTx9/VRuX7mN13ZYn8kTBkfx6q0z+e2be3hvXxHv3HkBj23Yz+Mb9wMQFx7MR8svYs2WHB5Zt5ePfnYRwQE2P4ZeUw7/M9E6vpG6CHautkL+G//XufW98gP4Yg2MvAScQTBvBWx7zgrln2Y1tug/fsIK6LAEqD4J9bWw5N+NfchPzbVaezc2CQ1XDTyYZH0zqq+F2z63Wv7qK0FEthhj0lp77qwtdxFZiXXwNE5EcoB7gAeA1SJyI3AEWAxgjNktIquBPYALuPVswf6lq6+jf+l2nnJczU2nBTuAwyE8fX0aXxwr5bxhsVTXudl2tIT39hWyIaOAAIfQJ9DJ59klHC+tZsvhEr44VsqhokqGx4fzxs48AI6dPEVRRS3xEVYL5Y0deQyJCeWaKYN4dP0+Hnw7k/iIYO6cN6LDVXhs/T4eeDuTU3X1/Hj+SKLDgs76nqffz2L70ZMcKa5iZ04pV00eyJShja3dzXsLeTezgKKKGtbvyWf2yHhiw4J4adsxjp6o4tXtuRSU17A3v5w3duYydmAk5w2L5an3s/gs6wSvbD9GcWUth4urGJkYcYaS2EBwBFz6gNV6/+BRa9mwLnRTzPm51bVTtB9OHLSWlR6FAZObd9Wk3Wj1n5flwRWPwcvfs1rro79mtepzt8KsnzRfd0AwTP++dXBu7NUa7MqrPaNl2uqwmtvG638L/La153qFykIA3OFtfx0aPyiK8YOivI/HDIhkdL8I1u7Op95t+OXXUrj/zQw2ZRZ4W/UbM/IRIKuoknkpCWzIsJ6Ljwim9FQdn2ef4KZZw7htzjn885PDFJTX8PUJA/jWuUM7XIUdR0+yOj2HAX1DuHXOOd5vB2eyJbuEjw8VszEzH4A75o5gaGzj8YYRCRGs35PPg54PjRtmJjEkJpSXth3j8Y37KSivAeDv72dxqLCSFV8fwzenDuH/Pj7MS1tz2HK4BIBDhZX2D3eA8Yutfuk/TbX6qYdd2Pl19R3UeMBz42+sPnJxwPk/av66wBC47sXGx5sGWv3I4Dlu4W69HHN/bf0o1USvmPK3uxljuPf1PeSUnGJwzX7uAUKiEju0jomDo4kJC6Kmrp5l5w3lmY+yeWlrDicqawH4+wdZ3q6M714wjA0ZBby0NYen3z9k9a27DfNSEnA4hLkpCaz87ChzRyecaZNtumh0IqvTc7goJaFdwQ4wLD6Ml7Yd442deZyTEN4s2AEmD4kiKjSQF7bkEBrkZPqwWEICnQyLC+OFLTmIwNCYUF7YYg1hm5uSSJ8gJzOGx/LStmPe9TR82B0srODhtXupq7e6/fr2CeT2uefwh7V7CXQI9y0ci9sNv35tF5U19QQHOrjnijEkRHbfsNNXPOVcOGkgr+/I5dXtZz8cNDr4ViZEfsTzrxWDnACgf98Q7l2QisNh/e6Pnqji929nUOs6exdnH/csbgj9lCBTw8qsseQ/a3VHBjiEuy4eyah+TT4YI/pbB8DBasEHhln98B7/+CCLjw4Wex9fMb4/Cye1OjitV3t03V725LUclrx02mDmpnTs/1Q1+kqE++7cMp75KJuhsaH0F6vlOnZUx7pDnA7hhxedQ1VtPcEBTuaOTuDZj62plL82vj9ZhZW43bBk6mCmJsUQFOBg1edH6RPoJDkujPljEpk42OoG+da5QymprOP8EXGdqs/skfFckprItdPa3+pPjgsHYMvhEm6ZNazF8wFOB9+fPZxXt+dySWo/78HgW2YP49mPDjMtOYbJQ6P5238OMnZAXwbHhALwnZnJFFbUMCgqlPTDJzhUWAHAPz85zPo9+YxMjKCu3s3+ggoOFFaw4+hJAGaeE0dtvZtXt+cyul8EmcfLGTugL9+/sHu6Fdxuw/1v7gGEBRMG8NDavZRX19G/75lHQeUymk3O0VBaDUBVrYsNGfksmjyQyUOs/bnysyO8s+s4o/tFtqMkTn4ZvNy6WwlwCoADBRVEhwXx+6vGNb40cgAc9IwqyvoPJM2EAKsL7lRtPQ++k0lUaCCxYcEcL6tmT24pV04c0O4P/N7gVG09j286QL/IEGKadC/mlFSRX1at4d4FX4lw35hRgAi8+P0ZxB3Ig1dgypiRHV7PDTOTvfcvSkn0hvuPLh7J8PjwZq9Nig1lX34FiyYP5HeLxjV7buzAvvx12WmjGTqgT5CTvy1r9RhKm5LjGlvqF7XxjeGW2cO5ZXbzcP3m1CF8c2rjeQgLJjTvzpo9Mp7ZI60LwXzjbx+TVVSJMYaNGQVcMCKO/71hGq56N5PvW8+OoycZOzCSovJaNmYUUFfvZnBMH96+4wKu+OMHbMzI77Zw35FzkqIK61vWi1tzOHKiivsXjuW66R3rFiutqmPy/evZmJHvDfdNmQVMS45h1c3ndbp83//nFjZl5mPM2MZwjugPFfnWMMeifTCuceTHhweKqHG5eWTxRM4fEcfznx/h7he/ICOvnDED2vMh0ztkF1vf9H55RQpXjG/823ri3QM8tHYv+WXVJHbjtzl/5pfhfqCgnGFx4ezJK6Ooooa3d+UxaXAUceHB1tljYI1I6ILpw2IIDXJS43IzxNOKbSo5Lox9+RWd7nrxtaQ4q4x9+wQ2O5DqS8Piwnhn93Fe2nqMIyequNnzDSHA6eDCUQm8tiOXeSmJFJTX8PLWY7iNYem0IYgIc1MS+dOm/azdfZzggLPPiuEQIS0pmtAg6084q6iSw56gmDg4iqjQIHJKqjhQYH2TeGNnHk6HYIzhd29lADA3peP7pm9oIGlDo3ln13GmJsVQXu0i83g5v7g85exvPoOLRifw9q7jvLAlhwUTBhAS6OSEM44YU28NsQNITCW7qJLs4kpWpx8lLMjJtGRreOYcz9/Zc58c5pLUlq3d4fHh3m9bTVXVuigsr2nRTXc2xhg+zTqBQ4SpSdGICJnHyxiVGIGIsO1ICaWn6gh0OpiWHEOgs/V9eqjQ2mdNGx8A81ISeWjtXjZlFrB02pd7kmNrjDFsOVxCRY2LoAAH5ybH4nS0/xvSkeIqYsODCAv+8iLX78I99+Qp5j36HvPHJLI+I9975vHPLvOcel9ZCAF9unw2ZnCAk3kpiRwsrGj1D3fsgL58fLCYGcM71/Xia6FBAQyPD2PK0GjvmHxfGzMgklWfH+W/X9hBgOfYQoPLxvbj9Z25zB/Tj8KKGv796REA5nuCaP6YRB7fuJ9bntvS7u3ddEEyv/jaGOrq3Sz684ecrKoDrL7nPy6dxLK/f+Y9BgAwY3gsAB8dLGbCoL5n7ZJpyyWp/fjNG3v4zv9aJ4g5BOaN6Vr3wZzRCQQ5Hfx0zU6OlZziplnD+Om6Qp4OAg6sB6A+PoVr/vKR9xvI1ycMIMjzQZgQEcKUodGs/OwIKz870mL9g6L78P5P57Tosnlk3T5WfnaEz38xr0PBs35PPjd79tVzN04jJNDJ4r9+zF+vm8zgmFAW/fkj72vvuzKVZecltbqerCLrwzfptA+XkYnhDIzqw8aM/F4R7jtySrnmr42zqPzPkolcObF9xzeq6+r52uPvc9Xkgdx75djuKmILfhfueaVWH+a6PVbf+lPfTiMhIrjxq2plIYTHgw/6JR+4ehy1rtZPQrpl9nCWnjuEPkG9Z8z3mu/NaHFilS9dO20IEwdH4XIbYkKDmoXnpWP78cHd1pm+xhjeuv0CHA68/dRjB/Zl/V2zKK9xtWtbD76dybo9+fz88hQ+zz7Byao6fvm1FLYdOcnmvYXsySsjq6iSH150jrdV29B1drCwokWYdMS3zxtKWlI0LnfjweLTW54dFRcezIYfzeaHq7axfk8+Kf0jOG4837AObISgcLaVRVBUUcvPLhvN1OQYRp02Kunv16dxqMmHWYP39xXx2IZ9LbpsjDGs3X2cqtp6PjxQxPzUfu0u77o9+UQEB1DndrN+Tz5BngbDut35DIkNRQT+eeO5/OLlL1i3J7/NcD9UVEm/yJAWHywiwryUBJ5PP0p1XX23/t22x77j1gHfvy2bwi9e3sX6PfntDvePDhZRXuNi3Z58VixI/dKOifhduBeU1XjvD4sP4+IxidZcJg1foSoKutwl0yA0KIDQNoaYBwU4rG6gXqQ94+G7IsDpaDaEtCkR8U7hICKt9guP6MAQyivG9+dXr+7mYGElGzMKCHI6WDptCENjw3jzizweeNua4+Pac4e0aKE39JV31pnq2RVDYkO5bGw/Hng7k39+coR84zkjtuwYDJrGhswiAhzC0nOHEBnScuK1qNAgJg9puY8HRffhsQ372JiR3+z3vi+/gpwSqzG0MaOg3eFe7za8m1nAnNEJVNXWW79/zzeId/cW0L9vHyYNjmLmOXFcPCaRZz86TEWNi/BWvhkcKqxsPgVIEw3HtT46WMRFo3v2wOqhokqCnA7mpSSyMSOft3cdp67e3WZ3U1MbMqyu4LzSavbklZE6oG93Fxfww3DPL6v23p87OsE62/D/jYf591mzJlYWQt/BZ1iDsoOLUhL51au7+cbfPqaixsV5w2MJCw7g/HPiCA5w8P7+IlIHRHa666WnzEtJ4IG3M/ngQBFCJLUSRJCphX5j2ZSZz7TkmFaD/UwSIkKYMDiKP28+yKrPj3qXN5zdPC0phpe3HeODA0VtraIZtzEUV9YyNyWBU7X1bMiwviVPS47hs6wTlFTV8ZNLrEnX5qYk8tT7Wcx5eLO3dd/U8bJqvjm19f/H6cNiCAtycueq7UR0sM5nMii6D//87rneYN5yuIT/Xr2dunpDVGgg93w9leUv7STQ4eD/bpxGYmQIhworGBobitNhHR9anZ7DpswC/rTpAOXVdTzyjQmkDujLdU9/Sl5pdbPtFZbXMDUpmvTDJVz39Kfe40QN5qUkdEt3jd+Fe0F5DQ6B2+eOYElqqHXyx6kTsOe1xnAfOLmni6m6aGBUH5ZfNpr9+RWIWC10sEYSrViQSnp2CVdO7B3zenbE8Phw7po3kpySKjZk5PP7vvdyz4xgjiXOYd8HGXwjrXMNk7svGcWLW4+1WD6qXzgzz4njmQ+zcXdg+r6IkADmj+mHy+1md24ZbmO4fe4I/rL5IDUuN4vTrOlupybFcMvsYRSV17a6HofAt85tvU89OMDal58cOtH+gp1FcWUNm/cWsuVwCdOHWcdgXkg/SkF5DXNGJfDmF3n8aPV27zeat7/I4zszk8kqqvR2vZ1/ThxBTgf3vbHH+7oX0nMoH+si/XAJc0cnENXkK71D4NvnJfFpVjEZrYznP6ebTvrzu3DPL6uhX2QId86Igz80Dl0k+wM4edQK94j+PVdA5TPfm936sMml04b0ioNwnSEi3OGZkuKW59L5qGgsTJ3Fug+zAGsUSWfMOCeOGee0fXD/ocUTOrVecHLfwsZW54oFqc2fdQg/u6zzI4kWpw1mcSc/0FpTUeNi8m+soazTh8Xidhs2ZRYwZ1QCf7p2Eum/P0FOySlmDI/leFk1GzMLWHZeEoeLq7jIM0AgLDiA6cNjeW9fIXHhQUxNimFjZgEBTmtqkie+NbnVYwTjBn053TEN/O4aqgXl1cRHhsDxnY0LA0OhrtKa11wcMGFJzxVQqXaKCAmkvNoaAbQps4Dh8WEkdfHA7VddeHAA5w6L4c2defxl80EeXJtJQXkNF422zvZu6Nufm5LIvJREPjlUzKPr91Jb72ZYk9/9PE/QzxmVwMVjEiksr2H15zmcPyKuxw/+NvC/cC+rITEiGPL3NC6ccTs4g62rnkz5jjXnslK9XERIAOXV1uih7UdO9pphtXa3cOJAckurefCdTP72n0P07RPoHVG1aNJAokMDuSQ1kSvG98cYeOLdgwQ5Hd4zzAHmj+lHTFgQiyYNZM6oBPr2CaTO7e5VXYF+1y1TUF7N1ORoKNgNYfHw3Y3WadzjFltdMgMm9XQRlWqXiJBAymtcVNfVU17jIiGid42+squrpwziign9vefABDjEe+7HtOQYtv16PgCDokPZ85tLcRuDQ8Q7IgigX98Qtv7qYu/jLb+cR70xvWq6a78K9xpXPSVVddY1Tw/uti4YEe05vTzuHOtHKZuIDLH+PRsO2kV181DWr5L2hnBQO86WBmt4bG8LU7/qlin0TEubGBFgXTA5MfUs71Cq92oY8njkhHViUnSoDy8qrvyeX4V7vucEpsHOEutak73tAsdKdUCEp+V+pLgKgOi2zphTqhV+Fe6F5dbJA/3EM8d15KAeLI1SXRPZp6Hl7umW0Za76gC/CveGlnus23PSQ6SOZ1f25W25e7tltOWu2s+vwr2gvJoAhxBe45nWtxuvLK5Ud4vw9rlrt4zqOP8K97Ia4iOCcVQct6b1DYnq6SIp1WmR3pZ7FcEBjl41w6jq/fwq3PPLa6yxwGW5VpeMjS43ptTpGlru1XXuZpegU6o9/CrcC8qqrQssl+VChHbJKHsLCnAQ5mmtR2mXjOog/wr3hpZ7ea72tyu/0DBzoY5xVx3lN+Fe63JzorLWmlem/LiOlFF+oWEmwqKKmrO8Uqnm/CbcCz1//IODq6C+VrtllF+Y65mlsMRzfVil2qu3TYfQaSc8Fw3u5yixFmjLXfmBfn1D+OXXUkhLiunpoiib6VLLXUTuEJFdIrJbRO70LFshIsdEZLvn53KflPQsTlRZ4R7n9pydqi135Se+e8EwJg6O6uliKJvpdMtdRMYCNwHTgFrgHRF50/P0Y8aYh31QvnY76Qn3vi7PdSD1gKpS6iusK90yKcAnxpgqABH5D7DIJ6XqhJJKK9zDa/Ktqy2F9+zV0pVSqid1pVtmFzBLRGJFJBS4HGi42OFtIrJTRP4hItGtvVlEbhaRdBFJLyws7EIxLA0HnEKqCyAsAZx+czhBKaU6rNPhbozJAB4E1gPvADsAF/AXYDgwEcgDHmnj/U8aY9KMMWnx8fGdLYbXyapaIkMCcJTn6cFUpdRXXpcOqBpj/m6MmWyMmQWcAPYbY/KNMfXGGDfwFFaffLcrqaqzTtEuz4PIgV/GJpVSqtfq6miZBM/tEOAqYKWING02L8Lqvul2JVW11inaZbkQoS13pdRXW1c7pl8UkVigDrjVGFMiIs+JyETAANnALV3cRruUVNUyLTAbqk9qt4xS6iuvS+FujLmglWXLurLOzgqqOMava26zHsQM64kiKKVUr+E30w8EnfKMb5/1U0hZ0LOFUUqpHuYX4V7jqsfpqrAeDLsQHHpRA6XUV5tfhPvJqjrCsS6OTXBEzxZGKaV6Ab8I99JTdYRjXSGe4PCeLYxSSvUCfhHu1XX1hElDuEf2bGGUUqoX8Itwr3G5m7TctVtGKaX8ItxrXW7CpRq3IxACgnu6OEop1eP8ItxrXPWEcwp3oPa3K6UU+Eu417kJl1O4gzTclVIK/CTca+utPnej4a6UUoCfhHtNneeAqh5MVUopwF/C3VVPmFRruCullIefhLvVchcNd6WUAvwp3OUUjhANd6WUAj8J91pPy90ZomenKqUUdP1iHb1CbV0dYVIDGu5KKQX4Scvd1Hqm+9WhkEopBfhJuDtqyq07ekBVKaUAPwl3aWi563S/SikF+Em4B9aWWnf6xPRsQZRSqpfwi3APri2x7oRquCulFPhLuNedtO5oy10ppQA/Cfc+Lk+3TGhszxZEKaV6Cb8I91BXKbUEQVBoTxdFKaV6Bf8I9/pSKpx9e7oYSinVa3Qp3EXkDhHZJSK7ReROz7IYEVkvIvs9t9E+KekZhNeXUenUs1OVUqpBp8NdRMYCNwHTgAnAFSIyAlgObDTGjAA2eh53q3B3GVUB2nJXSqkGXWm5pwCfGGOqjDEu4D/AIuBK4FnPa54FFnaphO0Q6S6jWsNdKaW8uhLuu4BZIhIrIqHA5cBgINEYkwfguU1o7c0icrOIpItIemFhYReKAZGUUx0Y1aV1KKWUP+l0uBtjMoAHgfXAO8AOwNWB9z9pjEkzxqTFx8d3thjgrifSVFATFNX5dSillJ/p0gFVY8zfjTGTjTGzgBPAfiBfRPoDeG4Lul7MM6guxSmGOg13pZTy6upomQTP7RDgKmAl8Bpwvecl1wOvdmUbZ1NfWQxAbXC3D8pRSinb6OrFOl4UkVigDrjVGFMiIg8Aq0XkRuAIsLirhTyTulPlOEHncldKqSa6FO7GmAtaWVYMzO3KejvCVWd18wc4/eKiUkop5RO2P0O11lUHgDPA2cMlUUqp3sP24V7nqge05a6UUk35Qbhb3TKBgdpyV0qpBrYPd5cn3J3acldKKS/bh7u73uqWcTi05a6UUg1sH+7GWOEuDttXRSmlfMb2iWjcDeGuLXellGrgB+Hutu5ouCullJftw52GlrvYvypKKeUrtk9EY6yWu3bLKKVUI9uHe8NoGT2gqpRSjeyfiJ6Wu0O05a6UUg1sH+7e0TJODXellGpg+3DH2+du/6oopZSv2D4RjdtzZT89oKqUUl72D/eGlrv2uSullJftwx3PSUw6t4xSSjWyfbg3Tj9g+6oopZTP2D4RG09i0il/lVKqge3DXdwNU/5KD5dEKaV6D9uHu7uh5a4X61BKKS/bhzue+dz1DFWllGpk/3B3N7Tc7V8VpZTyFfsnorbclVKqBfuHu1v73JVS6nS2D3fvNVT1Yh1KKeXVpUQUkbtEZLeI7BKRlSISIiIrROSYiGz3/Fzuq8K2qmG0TIB2yyilVINO92WIyEDgdmCMMeaUiKwGlniefswY87AvCnhWnm4Zp/a5K6WUV1f7MgKAPiISAIQCuV0vUgc1XKxD53NXSimvToe7MeYY8DBwBMgDSo0x6zxP3yYiO0XkHyIS3dr7ReRmEUkXkfTCwsLOFsM7WkavoaqUUo06He6e0L4SSAYGAGEich3wF2A4MBEr9B9p7f3GmCeNMWnGmLT4+PjOFqOx5a4ThymllFdXEnEekGWMKTTG1AEvATOMMfnGmHpjzej1FDDNFwVtk7ueeiM4ROeWUUqpBl0J9yPAdBEJFREB5gIZItK/yWsWAbu6UsCzMcZNPQ6dOEwppZro9GgZY8ynIrIG2Aq4gG3Ak8DTIjIRMEA2cEvXi9k2MW4MDjTblVKqUZdO6zTG3APcc9riZV1ZZ8cLUY8b7ZZRSqmm7H8U0u3pltFwV0opL/uHu3HjRtBsV0qpRn4S7g6c2umulFJetg938bTctVtGKaUa2T7caRgKqdmulFJefhDu9RgciLbclVLKyw/C3Wq5K6WUamT7VBRTj0Fb7Uop1ZTtwx1jtOWulFKnsX0qastdKaVasn24W+PcdS53pZRqyvbh3jDOXSmlVCPbhzvGjVvsXw2llPIl26ei9rkrpVRLfhDuRvvclVLqNPYPd7TlrpRSp7N9uGufu1JKtWT7VBTPlL9KKaUa2T4VG66hqpRSqpHtU1HHuSulVEv2D3fcGO1zV0qpZmyfitoto5RSLflBKmrLXSmlTmf7VLROYrJ9NZRSyqdsn4o6/YBSSrXUpXAXkbtEZLeI7BKRlSISIiIxIrJeRPZ7bqN9VdhWy4DBLTr9gFJKNdXpcBeRgcDtQJoxZizgBJYAy4GNxpgRwEbP427j0Ja7Ukq10NVumQCgj4gEAKFALnAl8Kzn+WeBhV3cxlkYjLbclVKqmU6HuzHmGPAwcATIA0qNMeuARGNMnuc1eUCCLwraFoepx4i23JVSqqmudMtEY7XSk4EBQJiIXNeB998sIukikl5YWNjZYiAYHeeulFKn6UoqzgOyjDGFxpg64CVgBpAvIv0BPLcFrb3ZGPOkMSbNGJMWHx/f6UKI0XHuSil1uq6k4hFguoiEiogAc4EM4DXges9rrgde7VoRz0zQWSGVUup0AZ19ozHmUxFZA2wFXMA24EkgHFgtIjdifQAs9kVB2+IwbtCWu1JKNdPpcAcwxtwD3HPa4hqsVvyXwpo4TEfLKKVUU7Zv8jqMW8e5K6XUaWwf7tpyV0qplmwf7g7cOs5dKaVOY/twF2MwaMtdKaWasn24a8tdKaVasn24C27QPnellGrG9uHu0CsxKaVUC7ZPRTFGw10ppU5j+1R0UI8fVEMppXzK9qnoQFvuSil1OtunogM3OGxfDaWU8inbp6LO566UUi3ZPhWdOhRSKaVasHe4u93Wrfa5K6VUM/ZORWOFu1vDXSmlmrF3Kpp6AETDXSmlmrF3Knpa7jrlr1JKNWfvcHdbLXd04jCllGrG3uHe0HJ3aMtdKaWasnm4e1ruNq+GUkr5mr1T0RgARM9QVUqpZuydip4+dz2gqpRSzdk73I2exKSUUq2xdyo29LlruCulVDMBPV2ALmlouetoGaV6pbq6OnJycqiuru7pothaSEgIgwYNIjAwsN3vsXe4e/vcteWuVG+Uk5NDREQESUlJiJ6P0inGGIqLi8nJySE5Obnd7+t0KorIKBHZ3uSnTETuFJEVInKsyfLLO7uNs/K03EUPqCrVK1VXVxMbG6vB3gUiQmxsbIe//XS65W6M2QtM9GzcCRwDXgZuAB4zxjzc2XW3vxB6QFWp3k6Dves68zv0VSrOBQ4aYw77aH3to+GulFKt8lUqLgFWNnl8m4jsFJF/iEh0a28QkZtFJF1E0gsLCzu1UeN2eVam4a6UUk11ORVFJAhYALzgWfQXYDhWl00e8Ehr7zPGPGmMSTPGpMXHx3dq2263jpZRSvlOeHh4m88lJyezd+/eZsvuvPNO/vCHPwCwbds2RIS1a9e2e53dyRejZS4Dthpj8gEabgFE5CngDR9so1XuehdOdPoBpezg3td3sye3zKfrHDMgknu+nurTdbZlyZIlrFq1invuuQewGpdr1qzhww8/BGDlypWcf/75rFy5kksuueRLKdOZ+CIVl9KkS0ZE+jd5bhGwywfbaJXRPnel1Bncfffd/PnPf/Y+XrFiBffeey9z585l8uTJjBs3jldffbVd61q6dCmrVq3yPn7vvfdISkpi6NChGGNYs2YNzzzzDOvWresV4/q71HIXkVDgYuCWJov/ICITAQNkn/acTxnvNVS1W0ap3u7LamE3tWTJEu68805+8IMfALB69Wreeecd7rrrLiIjIykqKmL69OksWLDgrCNSxo8fj8PhYMeOHUyYMIFVq1axdOlSAD788EOSk5MZPnw4F154IW+99RZXXXVVt9fvTLrU5DXGVBljYo0xpU2WLTPGjDPGjDfGLDDG5HW9mG1s33NAVS+zp5RqzaRJkygoKCA3N5cdO3YQHR1N//79+fnPf8748eOZN28ex44dIz8//+wro7H17nK5ePXVV1m8eDFgdcksWbIEsD5QVq5ceabVfClsfYZqXfQIrq1ZwZVRE3u6KEqpXuqaa65hzZo1HD9+nCVLlvCvf/2LwsJCtmzZQmBgIElJSe3uRlm6dCnz589n9uzZjB8/noSEBOrr63nxxRd57bXX+O1vf+s9o7S8vJyIiIhurl3bbN3kdQdFsNWMxBXct6eLopTqpRoOhK5Zs4ZrrrmG0tJSEhISCAwM5N133+Xw4fafnjN8+HBiY2NZvny5t0tmw4YNTJgwgaNHj5Kdnc3hw4e5+uqreeWVV7qpRu1j63A3not1OPQEOKVUG1JTUykvL2fgwIH079+fb33rW6Snp5OWlsa//vUvRo8e3aH1LV26lMzMTBYtWgRYXTIN9xtcffXV/Pvf/wagqqqKQYMGeX8effRR31TsLKQhIHtSWlqaSU9P7/D7TlTWMvm+9dy7IJXrZyT5vmBKqS7JyMggJSWlp4vhF1r7XYrIFmNMWmuvt3XL3a0td6WUapWtD6i63Z5rqOrEREopH/niiy9YtmxZs2XBwcF8+umnPVSizrF3uHt6lJzadFdK+ci4cePYvn17Txejy7RbRiml/JBfhLt2yyilVHO2DveGgT4ODXellGrG1uFe79ZuGaWUao2tw72xz13TXSnV0smTJ5vNCtlel19+OSdPnuzQe5555hnvWasNioqKiI+Pp6amBoArr7yS8847r9lrVqxYwcMP+/6qpH4xWsahTXeler+3l8PxL3y7zn7j4LIH2ny6IdwbZoVsUF9fj9PZ9myyb731VoeLctVVV/HjH/+YqqoqQkNDAVizZg0LFiwgODiYkydPsnXrVsLDw8nKyiI5ObnD2+gIW7fcdfoBpdSZLF++nIMHDzJx4kSmTp3KnDlzuPbaaxk3bhwACxcuZMqUKaSmpvLkk09635eUlERRURHZ2dmkpKRw0003kZqayvz58zl16lSr24qMjGTWrFm8/vrr3mVNpwV+8cUX+frXv+6d66bbGWN6/GfKlCmmMzLzyszQu98wb+7M7dT7lVLda8+ePT26/aysLJOammqMMebdd981oaGh5tChQ97ni4uLjTHGVFVVmdTUVFNUVGSMMWbo0KGmsLDQZGVlGafTabZt22aMMWbx4sXmueeea3N7q1evNgsXLjTGGHPs2DHTv39/43K5jDHGzJ0717z33ntm7969Zty4cd733HPPPeahhx46a11a+10C6aaNXLV1y13HuSulOmLatGnNukMef/xxJkyYwPTp0zl69Cj79+9v8Z7k5GQmTpwIwJQpU8jOzm5z/VdccQUffPABZWVlrF69mmuuuQan00l+fj4HDhzg/PPPZ+TIkQQEBLBrV7ddpA6webdMvU4/oJTqgLCwMO/9zZs3s2HDBj7++GN27NjBpEmTWp3XPTg42Hvf6XTicrnaXH+fPn249NJLefnll5t1yTz//POUlJSQnJxMUlIS2dnZ3d41Y+twbxjn7tRwV0q1IiIigvLy8lafKy0tJTo6mtDQUDIzM/nkk098ss2lS5fy6KOPkp+fz/Tp0wFrWuB33nmH7OxssrOz2bJli4b7mXi7ZWxdC6VUd4mNjWXmzJmMHTuWn/zkJ82eu/TSS3G5XIwfP55f/epX3iDuqvnz55Obm8s3v/lNRITs7GyOHDnSbP3JyclERkZ6JyO7//77m8357gu2ns89u6iSh9bu5fsXDmfsQL0ak1K9jc7n7jsdnc/d1uPck+LCeOJbk3u6GEop1evYOtyVUqon3HrrrXz44YfNlt1xxx3ccMMNPVSiljTclVLdyhjjdyPannjiiS91e53pPtdDkUqpbhMSEkJxcXGnwklZjDEUFxcTEhLSofdpy10p1W0GDRpETk4OhYWFPV0UWwsJCenwKBoNd6VUtwkMDOz2CbJU6zrdLSMio0Rke5OfMhG5U0RiRGS9iOz33Eb7ssBKKaXOrtPhbozZa4yZaIyZCEwBqoCXgeXARmPMCGCj57FSSqkvka8OqM4FDhpjDgNXAs96lj8LLPTRNpRSSrWTr/rclwArPfcTjTF5AMaYPBFJaO0NInIzcLPnYYWI7O3C9uOAoi68v7fwl3qA1qW30rr0Tp2ty9C2nujy9AMiEgTkAqnGmHwROWmMiWryfIkxplv73UUkva1TcO3EX+oBWpfeSuvSO3VHXXzRLXMZsNUYk+95nC8i/QE8twU+2IZSSqkO8EW4L6WxSwbgNeB6z/3rgVd9sA2llFId0KVwF5FQ4GLgpSaLHwAuFpH9nufavnqt7zx59pfYgr/UA7QuvZXWpXfyeV16xZS/SimlfEvnllFKKT+k4a6UUn7I1uEuIpeKyF4ROSAitjsTVkSyReQLz/QN6Z5ltpi+QUT+ISIFIrKrybI2yy4iP/Psp70icknPlLp1bdRlhYgcazK9xuVNnuuVdRGRwSLyrohkiMhuEbnDs9x2++UMdbHjfgkRkc9EZIenLvd6lnfvfjHG2PIHcAIHgWFAELADGNPT5epgHbKBuNOW/QFY7rm/HHiwp8vZRtlnAZOBXWcrOzDGs3+CgWTPfnP2dB3OUpcVwI9beW2vrQvQH5jsuR8B7POU13b75Qx1seN+ESDccz8Q+BSY3t37xc4t92nAAWPMIWNMLbAKa+oDu7PF9A3GmPeAE6ctbqvsVwKrjDE1xpgs4ADW/usV2qhLW3ptXYwxecaYrZ775UAGMBAb7pcz1KUtvbkuxhhT4XkY6PkxdPN+sXO4DwSONnmcw5l3fm9kgHUissUzHQOcNn0D0Or0Db1UW2W36766TUR2erptGr4y26IuIpIETMJqJdp6v5xWF7DhfhERp4hsxzqpc70xptv3i53DvbXrdtltXOdMY8xkrLN8bxWRWT1doG5ix331F2A4MBHIAx7xLO/1dRGRcOBF4E5jTNmZXtrKst5eF1vuF2NMvbFm0B0ETBORsWd4uU/qYudwzwEGN3k8CGuOG9swxuR6bguwpkuehr2nb2ir7LbbV8aYfM8/pBt4isavxb26LiISiBWG/zLGNJxcaMv90lpd7LpfGhhjTgKbgUvp5v1i53D/HBghIsmeycuWYE19YAsiEiYiEQ33gfnALuw9fUNbZX8NWCIiwSKSDIwAPuuB8rVbwz+dxyKsfQO9uC4iIsDfgQxjzKNNnrLdfmmrLjbdL/EiEuW53weYB2TS3fulp48kd/Eo9OVYR9EPAr/o6fJ0sOzDsI6I7wB2N5QfiMW6yMl+z21MT5e1jfKvxPpaXIfV0rjxTGUHfuHZT3uBy3q6/O2oy3PAF8BOzz9b/95eF+B8rK/vO4Htnp/L7bhfzlAXO+6X8cA2T5l3Ab/2LO/W/aLTDyillB+yc7eMUkqpNmi4K6WUH9JwV0opP6ThrpRSfkjDXSml/JCGu1JK+SENd6WU8kP/H3khVeA0+ilQAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_VAL','train_VAL']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABLqUlEQVR4nO2dd3hUZfbHP+9Mei8kEBJCQq+hBQQLKGDDAiquoKus6+q6ltUtKqu7ir/Vtbv2dS2rWBFRBBuCiNJ7QHpNIAkhDRJC+sy8vz/emTQSMgkpjJzP8+S5M3fu3Dl3buZ7zz3vOedVWmsEQRAEz8PS3gYIgiAIzUMEXBAEwUMRARcEQfBQRMAFQRA8FBFwQRAED8WrLT+sQ4cOOiEhoS0/UhAEwePZsGFDntY6qu76NhXwhIQE1q9f35YfKQiC4PEopQ7Ut15CKIIgCB6KCLggCIKHIgIuCILgoYiAC4IgeCgi4IIgCB6KCLggCIKHIgIuCILgoYiAC4JweuBwwIaZUF7U3pZ4DCLggiCcHmSsgy//CHNvb29LapOxHr6f0d5W1IsIuCCcyWgNc/8A39zf3pbA0TSz3PkVVJa1qym1WP0aLP83FGW3tyUnIAIuCGcymz6CzR/B2v/CT0/D0XorttuGI/uqH39xO1SWtp8tACVHIHMDpC41z7O3tK899SACLgieiMMBq1+HnJ3N38fG92H+3RA/CjoPhSWPw1vjYfMnsGdR0/Z1JBXWvmk8+uaSvw/C4mHcw7BtLqR80Px9tQTf/BXeHAvFueb5YRFwQRBaggMrYMED8NpZkLa8eftY8i+IHQbXfwK3LISb5kFJPsy9DT6c3DQxXvqMEbzcU7igHNkHEd3hvL8YId//Y8Pb1rVNa3NRq/ua1s2/qGRvr37sEwyHtzZvP62ICLggeCI75ptlcAx88QeoKG7a+4vzoegQ9L0C/ELB6g3dzocrXoSwrmabvN0n34fDDhkbjOe882uzbvu8ptnhQmvI3w+R3c3zxDHmwuSwm+eFGdWPS47Aa6Ng5StQkA67voWXhsD/hcN3D8E7E2DB38z2H14L3z3ovh3LnoOZV5iLQWEGRPeDc/8EieeJBy4IQgtgt8GOr6DP5TD5f1BwEN6b1LRBNlc8t9PA2uuH3mg8caiO/TbE57fBW2Ph5aFQVgDegbB5Fqx6zfz9/Kn73m9JPpQXGg8cjICXFcDhn+FYFrw0FJY/b/a3YDrk7oDvHzHC/fEUI9a9LoFVr8DBlWbgcfZNsHcRrH0Djh1q3AatYf075rhzd0BFEZz1exg/w3xP+XtOuxRHtwRcKXWPUmqrUmqbUupe57oIpdQipdQe5zK8VS0VBMFkZ7w/yXjPg6ZA17Ph2neNdzjvDvcF0xUOqCvgABGJJoSx86tqr7cuuxbA1jlw1u3Giw+NN0J3NBW++5v5+/x3JtTjDps+NMuYJLPsNgaUxcTCd34F9nITY599I/z8CYy4DYI6Qc8L4defwx+Wm4tZdH8Y8muIGWze132sOYb1/2vchkMboTDdPN76mVl2dH4/Xc8G7YADq9w7njZC6UZOuFJqADALGAFUAAuAPwC3Ake01k8qpaYD4VrrB062r+TkZC0TOgjCKZDyAcy7Ey5/AZJvrl6/5r/w7f0m/HHZc0bYTsbnvzcx5r/uqv/1pc/AD49BtwvgmrcgsEP1a1rDmxdA2TG4YzV4+Zh1ShkP1WEHewW8OgISzoXrGhmMLEiHV5Khx3izrVJm/Sc3QtoyiOwJ2VuhsgSUFS76J4y8wwiqxVp7Xw4HWCzGhvIiEx5670ooOWpEviFWvw5LnzZ3AmC+x8J0+Fsm+ASYjJgnu0KnAeAbAtfPNsfdRiilNmitk+uud8cD7wus1lqXaK1twE/AVcBEYKZzm5nApBayVRBOjbw91QNavyS0NkIT3R+G/ab2a8NvhfMfhOK86nj0ycjZZsSoIUbfB1e+DAdWmvBMTUcvYx0cSoGRf6gWMZfo+gaDfxgERRsbd35tbDoZa98AeyVc8kT1fsB496VHIWOt+axxD8PN38CoO812dcUbjHiDec0/zGzX5SxzvA2NExzaZOLk4Qlw8b/A4gUFB0xmjk+A2cbbH7qMMGmF+5dA6k8nP6Y2wp0p1bYCjyulIoFSYAKwHuiotc4C0FpnKaWi63uzUuo24DaA+Pj4FjFaEBpk17cmJjrqLrj48fa2puWwVZgQSfYWuPKV2kIHRrjOf8CIS24DXnVNjh406YMnY+hNxuOddweseAEqSozo715osjIGTT35+7ueYwpg8vfW9uBrUlEMG2eaMExYHX1whYdKjsCAa4wgN4fYZOOtH9oECeeYGLfFy+zfXgnz7zL2/foz8A835fx5u6DflbX30/MiSF9rLg4rX4byY8audqRRD1xrvQN4CliECZ9sBmzufoDW+g2tdbLWOjkq6oQ5OQXBPY6mwVd/MtkTx3PhizvND7smR1LNNspqBrE+vBb2fN8u5p6UvD3w9V/NYKS7rHgBtnwKF/wdBt/Q8HYdehnxORllhWbAMDSu8c8dOBkCo00p+dKnYc5vTQZMnwngG3Ty97qyWU5WHHRgpbFn2LQTX1MK+l8Fw29pvngDxDkjDxnrTObKzCtg1vXm4vH9DDN+cNlzRrwBonqZZd86Aj7yDrh3C/S62Hjgc35r/ufaEbcGMbXWb2uth2qtRwNHgD1AtlIqBsC5zGk9M4UzGocDvrjDDEQteMCI86YPqlPpwKSyvTHGxElvmmd+fDk7TD7zjq/az/b6SPkA1r3ZuNC6yNlpqiQHXANj7qsOE9RHVB8Txz1Z2KIw0yzdEXAvXxNWSL4Ffr/UGVs+dqK41YfLoy44iYAf/tksY4c1vr/mEtjBhEd2fWty36P7mdDM6+earJWh08wdgIshN8LZd5vB3JpYvSC4I5xzrxm0hZPf7RxNg1eGm1TLVsLdLJRo5zIeuBr4GJgPuC6b04BmJoAKQiNseMdkM8SNMF7oqlfN+v3OOKTDAfP/CBq47UeTs/urmXDnWojsAcuePbUKwZYm0/mDdqfoxWE3t/i+wXDJU41v7/IeN31kBhnrozDDLEO7NL4/gKRr4fLnIWYQ9L7UhE96jGv8fd5+JlPkZB744a1G6P1C3bOluQydBumrobIYJv3HZN8UpMOlz5jc95r0uhgueqzhfXUeDLcvM49Pdg4PrjG59G9faEJgrYC7eeCfKaW2A18Cd2qtjwJPAhcqpfYAFzqfC0LLUXQYvp0Oix4xecE3f2PSx7TdeFGpS40wb3wXDiyHix+DiG7V7/cJMHm8h1JM7PJ0wGE39oB7ser0tebW/8JHIciNEGRUH7Nc9A9TGl8fhQfN0l0Br8mVr8At35lBPXcIT2jEA99SnarXmpx9t4mF9xhvBPj62SaD5qzbThxPcAf/MHNxyttt8u/XOAdia1KUZZbaDnu+O9UjqBd3Qyjnaa37aa0Haa0XO9fla63Haa17OpdHGtuP8Atj+b9Nbm5zacwrXvY8rHkdwroYL8nqDROegenpZpCyJA9ePw8WPAiJo82tb10GTQWrD+xyIzOjLcjdCRXHnY/dEPDcHWbZ7QL39h8SCz0vNo8Prq5/m8IMsHhDUEf39lmTwEjo2N/97cO7NuyBV5SY8vn6ctFbGqu3aRcwdZZ5HtIZOvQ4tX1G9TLn84vb4dv7TqxCLcwA7wBT4NRYUVQzkUrMM5WKYnghqXZ82GGH/46Gz26tTrkqygZbefU2xflQftwMwC19Dn58ouFij5Ox4iXzWQ21DS0rNMUdSdfBHatqxyN9AswgWv+rzQ+xzwSY+Gr9npRvkBGIlohDlhWa2OmpkOGsg+jQy00B32UEwJ14NZjv4IbZcPYfIWd77XPnojDDfG8ni6W3FGFd4VjGid4pGO9bO06eztiSWKxGyFuKDr1MOGzfD2D1NRWiz/eDI/vN64UZ5o6w66jqcF8LIwJ+pnJ4i7m13Tizel3uLsjaDFtmm54Qmz6GFwbCJ7823nLJEfjP2aZAY8M7ptS4JN9kEjSFrM2w+FEzgOWqeAMoLYB1b5ky7Fk3GE/1rN/Xvw//cLj2HSNWk/93YgpaTWKTTZVdU7I+6mIrh7cvhhcHw97Fzd/P4Z9NDLnXJSa9rj6b7DbY+rlZ5u4ynl5Tb/Pjkk0xTX39Owozmhc+aQ4RiUakXaJWk93fOtP5zmkbW1oa18Dr0JtMzLw4F45lmqIqcH7PcebuMG+XCQm2MO7kgQu/RFw/7H1LjGfpFwqZTu/QP9yI1MpXICAC9iw0/5Tpa8w/aXAn03kOjOexba4ZOHShtSl3rig2KWAu0pabTJLdCyEg0lS0rXrFPD6wwniMe51pf94BMPE1iB166scal2z6XefuaPrtusNhCk22zzPvD4uH2dNg7N9NdsPAyU3b3+GtxuOM6g2OShOPrhm3B1OiPvf3ppAmd5cRgKYS60yd2/dDdRqdi4KDzdtnc+hyllmmLTPH7EJr2D4fEs4z/2OeSNIUiBsOHXqa40kcbfq1bHzfZEPlbIf4kdXfdeYG6HNZi5ogAv5L5sh++PJe48Uu/7cJfVzxgvmnOrzF9JpwVBpxGnqTub33CzNhizWvm31c8aJ5vMDZJWHcw5B4Prw93gwkdkqClPeNJ+4anHPYzW0zmNvMxPPMP/i3D5h4aPxIuPzfxiv+9Dfw8XXVNo+fAcNuBi8/k8XQErg8pU0fmZQ4d71ZrU188+dPjHCPvs/kYP/n7OrvoykC7nCYkvDB15tYNZhGTXUF3BVLXfa86XlSU/jcJTTWiOOSx82dzNiHTRpcRYnxEut+ZmsR0Q1C4kwMePjvqtdnbzXx77Pvahs7WgOLxYg3mP+p6D6mc+HhrbDxPbM+NM78Rv6yyzg+LYwIuCdht5lR7479Gt/WlVqXtswUHXgHmpHzub+HP6wyAt71HGcxw6Oms13mBiN2ccONaLtub7uPNZ5yRDfoN9Hsf+KrEBhlmgbt/hZ2fGlyab2cottpAKx728xxOO0rU1KdvRWueKm6aCOsC0z70twFJJwD6etM7La+EulTIaKbuSitfg18Ao337A7bvzDiPfo+uOChauGf/D/TS6TC2ffDXXsL0oyYdhxgYtBwYpe88iJz9xPaxTSGAoju697+63LDHBOXXfGiGcgd+/fqfbaVgCtlPNDdC8z/pFLmf2/92+Z/pe/EtrGjrYjuC3eshKe7GacmpLP5/2gF8QYRcM9iwzsmdNFvkvlRDPuNEYQ1b5g84eTfmt4UlWXGs01bBgOvhS1zTBpaVG9ThbZxprm9S74FhtwA/zkHVr5k1vW5rPqWOza5utru3D/VtmXw9dWPf/M1OGwmT7gmMYPM572YZF4PjIKkX9XeJuFc8wfmQtEaKAVXOeOSy/9tLkKNhVLsNlj4sPGexkyv7bX3vhQunGGqPouy3B9grNkB0CXgRXUEPH2t6bx3+Qtw3Bkz7THevf3XxdvP3HEd2WeKWMb+3RQ8QXXf7bag54Vm2rbPbzV3Y1/eY9YPudFktfwSGT/DzHYUeYqZLo0gAu5JuFKRdn1rvEN7hRlYXPq0WV96BC540IQ0dn8LFz9hmgBd8lT1DyWqj+k0ZyszQt2xvxH2DTPNYFPccJM50OWsE8W2IRpKK0scbdL9Un+CC/9pQi7u5g+3NErBJU+aO4F1bxthOxm7vjbx6UueMKGHurgq8VwDVe6Qttx4ndF9zffgG3KiB15W6Nx/LPRspnDXpdv5sPj/TAsC17yTEW0o4P0mwQX7YMljpno2MNpcXM6+u+1saGuG3mR6p7SS5+1CBLw9Wf5v86Oe+okRCYfdtK2sr8dE+XETY+5/FUx+x/T5WPx/5ta416XgF2LKrTe+Z4Q4dhiMusO8t6aX0/dKI/hBHU3YBIynvcnZ8jN2mBG7Wxa2zDGeTg2lAiKMt+9OTu6a/5q4d+9L63/dJdquqsbGcDhMmKn7uOqLWHCMiUfXxJW+6RPo3n7dIXGMWaYtMx54YJT5f2krLBbTAiBnmxnwHvcPI3C/dFpZvEHSCNsPh8OIxN7vYdXLZt3Cf8CzvUzMsmZq3uJ/whOxpj9xbLIR2CtfNrfBZQUw8na49GkYc78RluPZcNYf6v9cVwzbFW4BiHMO8kV089yMAHdJHG280JMJb9bPJitmxG0Nx7dDnYOQrgkAGiNzgwmX1OxwF9LZDGLWpErAG2kU1RRiBhtvf/d3pvlSW3rfNbnsefOXNKV9Pv8XiHjg7UXmehM/De5sOqKV5JsYN8Cih80A4l/3mEqv5c9Xvy9uuFmGxMAtiyB7W3XM+oIHzYBb5obq9K26dBoAv10InYecuE/X8peMyxvdvaB2VoQLh8PcGXkHmJldGsI32GTsuOuB711ksn56XVK9LiTWpPnVpMI5ZVdLCrjVy4yFpLxvOjUObqQNbGsREFE7rVQ4ZcQDb2kaKw93vb5jvilnvu1Hczu58mWTO/rbBXDDZ2bQ7+s/w3sTza38la8Yga05UOjtf2KOr9XbpOmdLFUu/qwq7/tQQSmODn2MZ9r/6qYfr6cR3c9kgXxzv5kWrC5f3QPbPjeTCbjaizZEaBf3BTxjnfnsmm1RQ2LM3VLNYp6KYvN/0dKzvZx1uxkzsXrDeX9t2X0L7YZ44C1J7i749GZT4DHmATO45/rBag1zbzf9O66fDWkrjJcc3NGEQ7qea27HYwaZbUPjTbwwZhDc+IXxXobW0+vjFFi5N48b3l7DXy/qzZ3TvmzRfZ+2WCymKdZLQ02+de8aHrGt3EzKO/gGk+/eGGHxJiXOXnnyEm2Hw5TyD7iq9vqQzqbRUXFOdVZKRXHLxr9dRPUyOfBRvavDP4LHIx54SzLrepP6lb4W3p1gBBtM6fh3D8HPs0zMe+VL5odf03sedB2MdnpGSpHa6UJseGG/8rVWiUuXVtj50+xNaA2z16ejtcbu0Ez731rOf2YJC7Zm8efZm/giJbPxnXkafqEmG8eVkeHi8Fbjpfa8yL1inyE3mEyVeXc13DgKTMl8eWF1daQLf+d5LS2oXldR3LLhk5qMurP5KYnCaYl44C3BurdMNkj+XjJGzsDeawIhSx4kJG0lVofDzB6TvcVUxlm9TfaIdkBcMml5xWQVljGqe+182AfyLyO7vD+PH++EM0uagpIKlu7JY0zPKH7cncOVgzpTYXfww44cLuzXES+r+9fjhdsPk32snKuHxPJ5SiYbDx6lrNLBT7tzAfh4bTo/7c5lc3oBUcG+xIX7k1VYRlSwL92jWklg2pLIbiYdsyYZ68yybliqIfpcZuLkKR+Yni6/X1p/kZWrRUHd/boyQcpr9O0uL2p8phtBcCIC3hxsFTg+/Q1lPS7DP2ki6tvppiQd+OPqENLWHWR8WSJPe39v+odkbzHFIGMeME3lXQNXscn8bdYWtmQWsunhC7EohcWi2J97nLUZZUAnPk/J4NyeZj7BV5fs5c1l1VM4dY8KYsXePJ74did/HNuDP19Uu+S6pMKG1hDoe+Jp/nxjJrFh/syY2J9FO7L551c7iI8IINjXi8HxYSzbY4R8X24xN7y1hqhgX44UVxAZ6MNnfzibYD8vQvy8sVia0Uv5dCCiu+nrUnasWkgz15sezyFNCDFMfNWc2zfGwNd/gd9+e+I2OdvNBb5Dr9rrfZ2TGLhyv6H1QijCLxIJoTQD+/IXsOz6mv1fPs3n8z6vEu8jKoyNZR05WlLBJoepwNIr/m3e1PNCE3+NH2Wq+0K7kOkIY3VqPsfLbdzx4UbGP/8TR4or+HJzFkrB2D7RfLvlMOlHSgDYdqj2DCtrU48wNyUTpeDlJXs5kF896/a+3OMkzVjI4P9byFc/1y4WySkqY9meXCYN6UyInzdPXD2QTekFzN98iEsHdmJIlzAczrFWL4siNsyfo8UVRAX5UlBSyXlPL2Hw/y3iL59ubo2vt21wlZLXDKNkbjReclM7/4V1MfMlHlxpimXqcvSAKY6qm5LounDUnDlHBFxoAuKBN5XifFj2LIU6gAGWNI7u/QQbXhzWYay09Scq2I83b0rmq03pFK33J3j3d6Z9aMxg836lzHRf5UV8kZJZlZSycHs2ANM/+5ljZZUM6BzKo1f2Z8KLy7j3k018dOtZbDx4lMnD4pgwsBOPzN/G28tTySwo5Yaz4vlwzUE2pRfQNdL8+DekHcXm0HTrEMj0z7bQLyaEN5ftp6zSQbCfFw4NVw0xxSiXJ3UmwMdK+pFSLh3YiQ1ppud1eIA379w8gphQPzKOltAxxI/conI2pRewbE8eX/+cxYwr+hMa4M2Xmw+hlNmXR+AqJT+y36RU2ipMn5D+V538fQ3R7QL44Z+QtvTEmcoLDpiJDWqQfqSEz5dncw+Y+LiLimLTnVFoNm8t28+GA0e5emgcF/arPWnFFymZ2ByaycPcrJ49zTnjBFxrzab0AipsDkYkRqDc9LZ2HS4ip6iMszLewcdezt8t03lZP8l5FctZ6+jD0qEvUoYP98dFMbhLGHaHZuHaZK7yWklln0mUlWusVhvllXYiI7qhtWbux0sZnhBOblE5afklDE8IZ+H2bLwsimlnJ9AlIoC/TejLg3O3MHNlGmWVDsb1iWZsn458kXKI+ZsPERbgzb3je/Hp+gy2Zx1j4mBz+7896xgBPlbeuXk4Fzz7I79/fwN7co7j62Wh3OYgKS6UHtHVsdaxfar/0fvEGM+wT6cQBncJA6BjiGlSFRcewJD4cIYnRPDDzhy+2nKIntHB3DMrhUBfL0Z1M+ITGeR7wnd4vNzG0eIKukQENP3EtTThzgkiljxhLrAR3cy4RHN7hMQMMsUyqfUI+NEDJwxg/uenfXy2Jpd7/KjjgRe1mAe+N+c4WYWlDE+IwM+72vvPO16Ov7e13tDa6cD+3OMkRAY2Kzz3RUomj329A18vCzuyjjG+b3TVb3zN/nz+PHsTGugc6kf/2FA2pRfQp1Nw1f+3p3F6nsFW5O3lqTz2tZmm6j83DOXSgTH1b1h61BRqZKzHNvcPdM7PIAawqgqWOwYSOvRyskozydyxhv/ZLuH+cwbQrcbgXp9OwUy2/YHM8//NDztzyH5xKb5eFrytFhb+aTRbM4+xN+c4/7pqIDuyjlFccZjnfzWY0c8swebQjEg0GQoTB3fmn19t5/lFu7EoGO5cf37vKOZvPsQTVw0kKtiXHtFB7Mgqqvr8HVnH6N0pmK6RgZzbM4qlu3OJDfPnsasG8Nt31zFleMMTIHSNCKBDkA/DujacB92/cwi9Owbz35/2Y7M7CPT1oqjMxnlPLyEm1I/v/zzmhIvj0wt28kVKJmseHI+/Twt3HGwqPgEw/FaT8/3dg9WT2Da3StHq7Ny47weTNuia7aas0FTL1vDAy212vv45i3K8sSsvrOUtH0IpKqvkyleWU1JhZ2S3CD783UisFkXG0RImvLiMs7pF8uZNbg7WtiHzNmVyz6xN3HpeIg9d5kbXzRpU2Bz831fbGdY1nGuHxTH98y2kpBcwND6co8UV3PvJJuIjArBaFPd+sokgPy/25xYTFuDNt/ecR0xoO/XpOQXcnZX+T0qpbUqprUqpj5VSfkqpCKXUIqXUHueykaqH9mdrZiFPLdjJ+L7RdArxY86G6iKMQwWlTHx1Bfc+9QqrN22BpxKo/OBXVLx9KUePFTHbfj4/+F3E/2wX82jljVw1JI6wa15kqv1RsmIvqiXeYAYOu0YE8OXmQ2xKLyCrsIy0/BL25BwnJb2AD9ccwMdq4bKBMTw4oS/f3nMeXSICOKe7GbAcnhBRtZ+L+3ekrNLBH8f1pIPTs500OJblD1xQdQHqExPM0t25/PqtNRwqKGXn4SL6dDKe9NVDjFc+aUhnLugdzbqHxjN1RMMzslgsigX3juausQ13UlNK8c9JA8g4WkLu8XLev+UsOob4UlJhZ19uMRsPFpzwnuV78zhWZmPh9tozk3y7JYvfzVxPhc3R4Oe1Cpc9axp+5e+Bdc65PU+lS9+Aq81kCTWrK13zQYZVC/jXP2dRWFoJKMqsQSfGwE8hC0Vrzc3vrOVPn2yipMLOr0fGs3r/EV5bsheABz77mWNlNn7YmUNu0YnTrb3w/W6e+HZHsz/fxbq0I1z331UcL7exen8+4577kSteXl5rnCbjaAk3vLWarZkmhJRTVMZDc7cS4GPlzWWprE1t2jS7P+3O5UhxBXde0J3LkmLw9bLwwaoDPDh3Cxc89yN5x8t5eepQXp46lIKSSlLzinn0yv5U2Bw8+Hk9MxfVYMmuHH43cx0ORyOFem1Mox64UioW+CPQT2tdqpSaDUwB+gGLtdZPKqWmA9OBB1rV2mZQbrOzaHs25/WI4u6PU+gQ5Mszkwfx+tJ9vL0slZcW78GiYNH2bEJy1vGCZQarvkgCwHvfQo7qICYcf5iOsfHcO64X9763noTIAIbGh6GU4rFJA2qFImoyeVgczy7cjdWieHySmffvkfnbeHjeVrZmHuO35yQSGmAKQFwe6fRL+7Dx4FEiAqsr8f5yUW/6xoRwy7nV80JaLIq48OpQRK+OwYARyX8v2k1haSX9Ysy6Swd2Yn9uD246OwGg6iJwMtzZZkRiBC9PHYq3VTG4SxjPXjuI7GPl/P2LLTz73S7O6RHJwLgwxvSKIqeojP255sf7+cbMqlDPvtzj/Hn2Zkor7SzZlcPF/asbABWWVvLjrhzO7x3Nd9sOM3loXMtnvfSfBAv/bvLzfUNPLf7cz7mvNf+p7iR4NM0swxMA4yjMmL+NpDiTgVJ01J9AlwfusJtq3FPIA88sKGXJLjOQmtghkH9OHEBRmY0XFu+hY6gfK/bmM3lYHHM2ZDB/86Fa/1PHy228/tM+7A7NOd074OdtrboTbCpvLt3PmtQjfLj6AG8tT8XP20JufjF//DiFT28/Gx8vCx+tOciKvfnc/XEKX//xXL5IyeR4uY2v7j6XG95aw4drDjT4+d9vz2ZAbCidQqtDH59vzKBDkA/n9YzC22rhN2cn8N+lZiq3cX2imTIinoHO7/31G4dSVGZj4uBYsgrLeGvZforLbSeElbIKS9mTfZw56zP4fkcO6UdLqsaZTgfcDaF4Af5KqUogADgE/A043/n6TOBHTkMBf3/VAR77egedQvzIKSrjo1tHEh7ow6+Su/DeygMs/H4B93p9xnd6Ck/Fr4RDMIqfAfjZkcju3rcTnBXLred1Y0zvKHpGB3HDWfFV4YHrThKKuH1MdzZnFBIR4MOUEWa7LZmFfLjmIMMTwrn/khNnWhkQG8qA2NBa67pEBPD7MSf3DM/r2YHXlngR4u/Np847i2FdzT+/r5f1hBTDluKypOoQ1Hk9owDYlH6UD1YfZNX+fAJ9rKz7+3h+corK2D7R/LAzh2+2ZDGsazh3f5SCn7eFAB8rczdmcnH/ThSWVFJhd3D/nM1VYgQQF+7P2c47lBbDy9f05/jxCTN/Y1MzUGrtywcGTTVtESpKTJjGNYGCM4Ty1rJUyiodvDx1CK//tJ8jeX50LDuGgmZ1IrQ7NNYaF7WaYbTbRnercjJSDhZw/5yfsSi4/+Le7M89zqtL9jKmVxTdOgTi0Jq5KZmUVZq7oJv+txaA1CcmuD1OBHC0uILC0kqW7MoB4Ilvd+LjZeH9W85hf24xd3y4kWe+28lto7szb9Mh4iMCSM0r5rONmXy+MZNBXcIYEBvKZUkxfL4xg+PlNoLqiOrB/BJ+9956zu4eyUe3jgSgsKSSxTtyuGFkPN7Oeog/X9SLzRkFdAzx44XrBtc6jppjPqO6R/L6T/tIOVhQlbILkHOsjBvfXsvenOMEO23YkXWMuPCAWt95e9KogGutM5VSzwIHgVJgodZ6oVKqo9Y6y7lNllIqur73K6VuA24DiI8/ycSzrcRnGzOxKDh8rIw/juvJSOcgW/eoILbe2xvLf6ahbGWMCzuCyko3/ZptZWxzdOW5hDd4Z+pwrlFUnfxFfx7j9md7WS28eVMyukZ/lMevGsiMK/vjZVFN+mE0Rv/OoWx+5CLmpmTy59mbuW10N/p1bsOWoTV4bNJAHrmiP+tSj3D9W2u4/OXl7M8tJtDHystThzD59VXc8eHGqu3fuimZlfvyeX91GjPmb+PdlWlVr3XrEMj+PCNsa1OPtLyAg+nMuOy5lmm+Hz8SVrwAWZtMyuiWORDVF/zDsdkdzN+cybi+0XSNDKRfTDAFKf5UFhfgA03uRFhQUsF5Ty/hnxMHMMkZJtuRdQylYOuMi6u8yWA/b16aOoTJ/1nJ2T06EB3ix9OTk7ji5RWMf/4nrh4Sy7EyG9/vyCY+IoAAHys7D5sLQcbRUrcHnedsyOCvNVJLL+7fke+2ZfOPy/rSp1MIfTqFMHVEPG8uS62qZ3hxymBeW7KPV3/Yy+FjZTx6pektf/WQWD5ac5AfdpqCtZrMdVYHHy2pnun+6y1ZVNgdXD2kOrvE18vKx7eObPR3NqxrOBYFa1PzqwT8oblb+HDNwarreVG56Vfz3MLdPDR3K7NvH3VaFLS5E0IJByYCiUAB8KlS6iRt2mqjtX4DeAMgOTm5TQNIWzML2ZF1jIcm9KVHdBCje0WZH8ne76HPFVi3fWYmNrjkSdSC6RB/NiRdC1/9ifBeZ/PK5CEtcste9x/IuwkVk039nEmDY4kK9q26ULUX3lYLI7tFEhvmz/7cYqaOiGfS4M4E+nrx/i0j+G7bYRwOTWKHIM7t2YGBcaHMTcng3ZVpnN87inF9ookK9mNMryhWp+bzzIJdzNt0iMLSSu66oEe9WS7NJigarv+kVpy62biyTTKc1ZeHf4bLX6gaf8k7XsFVTrHtHhVEEQFUltQW8M+2FjAhyd7oQO+a1CMUldl4Z2UaO7KOccWgzuw8fIyuEQEnhAIGdwlj7h3nVIUcekQH8+nto3js6+0s3plDWaWd83tHcf/FfQj28+LnjELu/Ggja1KPuC3gKQePEuzrxf2X9CYq2I8L+kSxLvUo5/So/l+ccWU/hieEU1xuw9/Hi8uTOpNVWMaT3+6ke1Qg1yYbAU6KC8OiYPfhIqjRv81mdzA3xdxhltvsPLdwFzsPF7E1s5Ce0UEMiK3ttLjjJAX5ejEgNpQV+/L5M2Yg9cM1B7lmaByTh8Xx/KJdrEs7SpCvF3tyjgNw90cpfHn3ue3uibsTQhkPpGqtcwGUUp8DZwPZSqkYp/cdA+S0op1NotLuYPX+fJ78didhAd5MHhZHeKAPHM+B10aZhlJTZ5mOgHHDzaw1Pcab1LLyY/DDY3QePhH8TtKg6DTFYlFVoYz2xmJRPHBpHzYeOMrDl/eruhh2CPLlhrNqi2XHED9evX4oH6w5wJPXJBFS47u/oHc0S3bm8N6qA6TmFZOWV8zb04Y36eK6P/c4ceEB+Hg1cPFswnRuhSWVVDoc9Y8TBEWZJleZ682fc5LoV2fvYF3aEcb2ieb83uZmNaFDICsJQJc5+804W8l+u7uI6ANHqs7j9kPH6BsTfIIYuQb5NqcXsDm9gC83m4KtpLiweu12xX9dDIgN5Zqhcdw3x4QMp46Ir7priw3zJzzAm2+2ZDE8IdytuG9afjHdooO4cVRC1bqaIQkwXvHVQ2vnYF87LI4tmYXcPbYHAT5Gkny8LHSJCCDVOei5O7uIhMhAXvlhD2n5JXTrEMiB/BJe/mEvnUP9CA/w4fbzuzf7rvbypBj+9c1Onlu4i3dWpDGsazhPXTMQL6uFu+w9mbcpk3Kbg69/zuKsxAjWpB5hxd484xQ6cTg0Ow8Xtemdrzuu4EFgpFIqQJlvZxywA5gPOGenZRowr3VMbDofrTnIjW+vZXvWMZ6ZPMiIN5j5JEvyzOOdX5kmU32dDfY79DSpYAERcP/+hmdiEZrElYM6M+PK/m6J7dk9OvDaDcNqibeL83ubH8roXlEs2ZXL/1aknrBNQ6zcm8e453/inlkptcJZzeX2DzZw0b+Xkn2srP4N4obD/p9gx1cwbBqFNm8W78hhyvB4/veb4VUXkU4hfpSoALwqnYOYTg+8GD+yj5kMkRV785jw0rKqHjU1WZt6hO5Rgfh7WxnfN5q84xUcKixjcHyY28dyVmK1d+zKfAJz8T2nRwd+2JnDdf9djd2N7Iu0PCOsTSUyyJdXrx9alTXlIiEykLS8Yg7ml3Dpi8uY8eU2Xlmyl6uHxnL7mO5VNr1yw1C+uee8E0ItTeGWc7sxslsEL/+wF6tF8eKUwVW9hcb0iuL5Xw1mWHw4/t5WXpgymBA/r6pQjos5GzOY8NIy1qc1LXvmVHAnBr5GKTUH2AjYgBRMSCQImK2UugUj8te2pqEnY+XePF5cvId3bx6Bv4+VORsy6NMpmDdvSq6+/du3xLQPHfcwpHwImz8x63td3F5mC03ggt7RrJw+lphQP257fwNPfLuTN5eZDANfLytvT0umpzMTpyblNtN1McDbyrdbDzNv06GqeHFTeX7RbnYfLmLV/nwA/v7F1vpzqc/6gwnToWH476ris9fU8TwtFoXVPxSf8hKTO+4U8BLtW3Vx+HS9mfFnc3phlecOcLiwjG2HCrlrbE9uPjuBsABvso+Vk1tUTp+YE7+HhugS4U+nED9C/L1qZT4BPHVNEkPiw/nnV9tP8DbrUlZpJ7OglIQWzNBI7BDIujTTLsLu0Hy05iAAd4/tWZUC6e9tZWCdQf/mYLUo3r/lLLYfOkaXiIATvguAm0Z15YpBnYkK9uWypM6maGhSdebKnPUmtPPUgp3kF1fgZVG8OGUIfWNazyN3KwtFa/0I8Eid1eUYb7zdmZuSyZrUIyzakU2/mGC2ZBbyj8v71Y7drXrVTKY66i4zZdaRfaZxUd0GQ8JpiVKKzmGm0OKZyUm89uM+jpVWojV8sj6d77Ydpkd0EHM2ZHDJgE4s3JbN+L4d2Z1TRPaxcv5zw1CeW7SbD1YfOEHAF+/IJrFD4Am5/C5SDh7F22rhpcV7qtZNGNiJ77eb2HHNKkcAugyHP6w0kz2ExTM3ZSU96onPAvgGhWMp1yZ8Uml63hgPvIzj5Ta+22ZaLOw8fAy7Q/PeqjRnlkeuCUcMia26w+wU6lcrrc7d7/WfkwbgW09oKdDXixvOiufF73fz+caMkwr4gXxje0KHlquyTewQSEmFnTeW7sPf20pppZ0h8WEkdggk0Nd858O6hrfYmJK31cIgZ+VxfXhZLUQFm7DZNUNj+XjtQWatS6fC5qC43MbatCP4e1tZl3aUzqF+VNg1d320kS/vPrcqNNTS/CIqMdc6b1k+35jBzpgQrBZV+3Yqf5+Z0ur8v5m0sU4DzKzuiaNPLW1MaBfCAnx4cELfquebMwpYk3qEUd2Pct+cn/liUyYr9uZz38XVqZMju0Vy9dBYnl6wiwP5xVUx3Z9253LLzPV0DvXjo1tH0jUyoFYcdU92EVPfXF1LJCYN7szlSZ35ZsthNqcXcFZ9A8ahceRaosjKKGBd2lHuv6R3vfHZgJBIyAdbSQHWihIUUIYvhwvLWLD1MKWVdmLD/NmRdYwfd+Xw6JfbARMjfvqaJBKaEbKoS91+ITXx8zYx6/dWpTFxSCyjukVWXbC01lTYHfh6WUl1ZgoltoA9LlzHVlxh519XDeSNpfv4jbOWISrIl6S4UC5PaqCSupUZ1jWc+IgA/vnV9qp1IX5ePHVNEk8u2MmLU4ZQXG7j12+v4Z9fbeeJq5NaxQ6PF/DDhWUcyC+hQ5Avy/bksSWjkNE9O1RdKYHqvs+uOQ47Ob/MxNFta6zQKoxIjGDOhgxW7jWhjRXO5drUI2igd8dgwgN9mDQ4lme+28XclEzuHd+LnKIy/jJ7E10jAzhUUMr5z/7IhIGdePX6oSilKKu0c9dHKVTYHFX50T/8ZQzdooIoLKlEKfMZ9Qn4w/O28t4qU4WplKmcrY/Q8HBIhbz8fIKKiwgCSrUP2UXlzE3JID4igKuHxvLi4j18sPoAEYE+rPrbWHyslhZNQz0Zf724Nz/uyuHmd9aREBnAD385H4tF8fpP+3l24S6W3X8BK/bmoRQtckFx0T3K7CshMoCpI7pw/VnVachKKebfdW5Db211lFJcNcSclyevHsh1w7tUra/ZnuMPY7rz2o/7OLdHVK2aiZbC4wXc5X0/96tB3PfpZnKKyrmqTqyR1J9Mjm+oc333sXDR4yc2HRI8khGJEby36gBvLN2PUtXTjq7Ym4eXVXHtMPPj6hzmz6hukcxNyaS00lToFpXZ+OjWkZRW2Jmbksm7K9O47o3VTBuVwOr9+ezKLuI/Nwzlz7M3E+jrVeVhhgZ407tjMMv35tGvcwgH8kv4rbOq8btth3lv1QGuHhpLctcI4sL9q8I/dQkKNh0oCguPUFlQQBAQEBjMvpzjFFfY+OPYnvSLCUFrWLIrl2mjuuLr1bZ9ZIJ8vZh12yjeXLaft5ensnp/Pmf36MAHqw9gd2iue2MV6UdK+fXI+HoHoJtLXHgAM387guSu4W12sWoKd17Qg5HdIhnZreGmeH+6sBer9ucz/fOfSYoLbfFGbh4v4Huyi7AoOKd7JK/dMJSP16ZzkeuWsLwIVr8OexZCco3ZsK3ecPZd7WOw0OKc26MDwX6mmdZlSTHY7Zru0YG8umQfNoeu5flcNSSW++b8zH9/2k+/mBCe/9XgqjYESXGhWJTiu22H+eOsFOwOza3nJXLpwBgyC0pRqnbx1RWDOvPMd7vYcOAoDq0Z0zsKL4vizaX76RYVyFPXJDUanw0IDgOg+FghFJs0wj5dolm0M6/K3vAAH87uHkm5zVErRa8t6RTqx30X92b2unQ+T8lkaFfTRbNbVCDBft4M6RLO35vYfModxpwk7t7e+HhZTphJqy7eVgsvTRnCtP+tJaeoTAS8LocLy+gQ5IuX1UJyQgTJNVKh2D4flji7zEm45BdLWIAPT16dxJ0fbeSKpBguGRBDTlEZry7ZxzVD42oVNV06MIZ/fbOD0b2iTiivVkrx8BX9uHtsDy5/eTkdgny47+I+APzuvG4nfO7vR3fjp1257M09zpHiCia8uAyN6Yp3/yW93RpcCw4xHnjZ8QJ8y0so1970jglj0c48hsaHVYUkXCXj7Ymft5XLB8Xw2cZMBsaGUmF38OClfRl/khi6YFphLPrzmFYp+vF4Ac8uKm945L3QpGCRfIuZEUf4xXJZUgzDE8ZVjX1EB/ux9sFxtcdCMOGAH++7gBA/rwZve8MDffjuT6PxtqqGC38wWQkf3noWpZV2bnp7LZvSCwjy9aLS7mgw5l2XkFAj4OUlhdjLiynFh16dzB3BCaHA04A/X9ibRdtzmPHlNpSqnTsuNExrVWx6voAXlhEf2cBtSWG6SRW8/Pm2NUpoF6LrNOWv+9xFqH/jcdq6DZQawttqerz/66qB7DxserDvyT7eYMy7Lv5BJoe5suQYjooSyvDlwr4d+etFvZh8Ggp4VLAv7948nE/WpdOzY1BVN02hffB8AS8qY3hiA63ICzPMfIWC0Mr06xxSVULdv7P7hSXK13jb9rJjUFlKufLF38fKXWN7toqdLUF9HTOF9sGjJzUuq7RTUFJJp4amQyrMqM48EYTTES8fKvBGlx1HVZZSafHMqb2E9sGjBdxVbnzCfHZ2m+lzIgIueABllgCoOI7FVopNBFxoAh4u4KYfQpxPcXVvE4BNH8J/R5tWsaESQhFObyqsgVgrj2Oxl2H3EgEX3MejY+CHnR54z4w5sPYZMx3WjvlwPLt6I/HAhdMcm1cg3iXFeFvKqPALa29zBA/CswW8sBSAkGLnxLHfPQh5u2pvFNL8FpOC0BY4vAPxc5TgRRnlXp43M7rQfni0gG9OLyQm1A/vwjSzoqZ4X/KkWXYadML7BOG0wjeIQHUIX8op8WnZSj3hl43HCrjWmjWpRzi3RyTq4L7qF0LjoTgX+l8FwZ0a3oEgnCZ4+YcSxD78KadABFxoAh4r4Kl5xeQdL+ecOG/YmV/9wti/Q9/LmzSztyC0J2Hh4egDpfhTgVUEXGgCHpuFsmKvafYzKqzArHBNJhuXLOIteBQ+/qEEUYq/qsDbT/53BffxSA/8cGEZzy/azYDYEGIdZiJXLv4XFGVBZPf2NU4QmopvMIHKpMRafcUDF9zHIz3w91alUVRm48UpQ1DZW8HiDTGDoP+k9jZNEJqOb/VUbkFBbTejueD5NCrgSqneSqlNNf6OKaXuVUpFKKUWKaX2OJcNNCRpefblHqdrZADdo4IgdamZBdxbCiAED8WnWsA7hIe1nx2Cx9GogGutd2mtB2utBwPDgBJgLjAdWKy17gksdj5vE1Lzis3MKKVH4dAm6DamrT5aEFoevxpet7fkgQvu09QQyjhgn9b6ADARmOlcPxOY1IJ2NYjDoTmQX2IEPG0FoGWyBsGziRlc/dhbYuCC+zRVwKcAHzsfd9RaZwE4l9H1vUEpdZtSar1San1ubm7zLXWSdayMcpvDzFSStQmUFWKHnfJ+BaHdiKgx24944EITcFvAlVI+wJXAp035AK31G1rrZK11clTUqc9vl5pbDEBiZCAUpJtSeS/fRt4lCKcxSkGQc1oyaWYlNIGmeOCXAhu11q5OUdlKqRgA5zKnpY2rj9R8I+AJHQKlXazwy2HIjWYpHrjQBJoi4FOpDp8AzAemOR9PA+a1lFEnIy2vGD9vi5nEoTBdBFz4ZXDBQ/C7xdB5cHtbIngQbgm4UioAuBD4vMbqJ4ELlVJ7nK892fLmnUhqXjEJkYFYcMCxQyLgwi8Di8VUEQtCE3CrElNrXQJE1lmXj8lKaVPS8orp1TEYjueAo1IEXBCEMxaPqsS02R0cPFJCYpQz/g2m+6AgCMIZiEcJeGZBKTaHNhkohelmpXjggiCcoXiUgO/Pq5mB4hLw2Ha0SBAEof3wKAFPcwp4YodAyN8LAR3AL7SdrRIEQWgfPErADxWU4udtoUOQD+Tugqje7W2SIAhCu+FRAl5aaSfAxwsFIuCCIJzxeJSAl1c68PWymBTCsgLoIAIuCMKZi2cJuM0p4K7Z58UDFwThDMbDBNyOr5fVhE9ABFwQhDMaDxNwB77eFtj5NQR3huCY9jZJEASh3fAsAa900J102L8Eht9i2nAKgiCcoXiWgNvsjK5YDsoCw25ub3MEQRDaFQ8TcAchFIFvCARGNv4GQRCEXzAeJ+ABlINPYHubIgiC0O54lICXVdrx0+Uy8asgCAIeJuDlNgf+lIKPCLggCIJnCXiVBy4hFEEQBM8ScJsDX10mHrggCAIeJOBaa8ptDnwcZRIDFwRBwIMEvMLuAMDHUSpZKIIgCLg/K32YUmqOUmqnUmqHUmqUUipCKbVIKbXHuQxvTUPLbUbAvR1lIuCCIAi474G/CCzQWvcBBgE7gOnAYq11T2Cx83mrUV5pBNzLXiohFEEQBNwQcKVUCDAaeBtAa12htS4AJgIznZvNBCa1jomGcpsdCw68HFLIIwiCAO554N2AXOAdpVSKUuotpVQg0FFrnQXgXEbX92al1G1KqfVKqfW5ubnNNtTkgJebJ+KBC4IguCXgXsBQ4D9a6yFAMU0Il2it39BaJ2utk6OioppppgmhBLgEXNIIBUEQ3BLwDCBDa73G+XwORtCzlVIxAM5lTuuYaCiz2fFXLg9cQiiCIAiNCrjW+jCQrpRyTX8zDtgOzAemOddNA+a1ioVOxAMXBEGojZeb290NfKiU8gH2AzdjxH+2UuoW4CBwbeuYaCi32QmgzDwRD1wQBME9AddabwKS63lpXItacxLKbQ4ClHjggiAILjymEtP0And54CLggiAIniPglfbqNELJAxcEQfAgAa8ZQhEPXBAEwcMEXLJQBEEQqvAgAa8RQpEsFEEQBA8S8EoTQtEWL/DyaW9zBEEQ2h3PEXCbg2BLOUoGMAVBEACPEnA7IaoUfEPb2xRBEITTAo8RcLtDE6xKwTeovU0RBEE4LfAoAQ+kDHyD29sUQRCE0wKPEXCH1gSpUhFwQRAEJx4j4HaHJlCLgAuCILjwIAGHIEpEwAVBEJx4jIBrrQmgFHxEwAVBEMCDBNzhsJluhOKBC4IgAB4k4F62EvNABFwQBAHwIAH3tRc7H4iAC4IggAcJuLcIuCAIQi08RsB9JIQiCIJQC7fmxFRKpQFFgB2waa2TlVIRwCdAApAG/EprfbR1zARfh3jggiAINWmKB36B1nqw1to1ufF0YLHWuiew2Pm81fCREIogCEItTiWEMhGY6Xw8E5h0ytacBD8RcEEQhFq4K+AaWKiU2qCUus25rqPWOgvAuYyu741KqduUUuuVUutzc3ObbaivwxkD95FuhIIgCOBmDBw4R2t9SCkVDSxSSu109wO01m8AbwAkJyfrZtgISBqhIAhCXdzywLXWh5zLHGAuMALIVkrFADiXOa1lJIC/o4QKfMDq3ZofIwiC4DE0KuBKqUClVLDrMXARsBWYD0xzbjYNmNdaRgJ46QrKLb6t+RGCIAgehTshlI7AXKWUa/uPtNYLlFLrgNlKqVuAg8C1rWcmKG3HgbU1P0IQBMGjaFTAtdb7gUH1rM8HxrWGUfVh0XYcSgRcEATBhcdUYlrEAxcEQaiFBwm4Dbt44IIgCFV4joAjIRRBEISaeI6AaztaBFwQBKEKjxJwh3K37kgQBOGXj2cJuAxiCoIgVOE5Ai4xcEEQhFp4jIBbJQYuCIJQC48RcAt2HBYRcEEQBBceI+BeMogpCIJQC48RcCsSQhEEQaiJxwi4RQRcEAShFh4j4FZpZiUIglALzxFw7GiLTOYgCILgwoME3CEhFEEQhBp4kIDb0RbJQhEEQXDhWQIuHrggCEIVHiHgWmtnCEU8cEEQBBceIeAODV7YQSoxBUEQqnBbwJVSVqVUilLqK+fzCKXUIqXUHucyvLWMtDu0MwYuAi4IguCiKR74PcCOGs+nA4u11j2Bxc7nrYJDa7wkhCIIglALtwRcKRUHXAa8VWP1RGCm8/FMYFKLWlYDh9aShSIIglAHdz3wF4D7AUeNdR211lkAzmV0y5pWjd1hPHAkC0UQBKGKRgVcKXU5kKO13tCcD1BK3aaUWq+UWp+bm9ucXeBwmDRCxAMXBEGowh0P/BzgSqVUGjALGKuU+gDIVkrFADiXOfW9WWv9htY6WWudHBUV1Swj7Q4HXsohWSiCIAg1aFTAtdZ/01rHaa0TgCnAD1rrXwPzgWnOzaYB81rLSLvdZh6IBy4IglDFqeSBPwlcqJTaA1zofN4qOOyVADKIKQiCUIMmKaLW+kfgR+fjfGBcy5t0Ig6beOCCIAh18YhKTLutwjwQARcEQajCIwRcO4wHrmQQUxAEoQqPEHCHzcTAxQMXBEGoxiMEXLuyUKwi4IIgCC48QsAdVWmEMqWaIAiCC88QcFcMXDxwQRCEKjxCwLXEwAVBEE7AMwS8KgtFBFwQBMGFRwi4qxJTBFwQBKEajxBwyUIRBEE4EQ8RcDsgHrggCEJNPEIRq0IoVqnEFIS2oLKykoyMDMrKytrblDMKPz8/4uLi8PZ2L2XaIwScqkFMyQMXhLYgIyOD4OBgEhISUEq1tzlnBFpr8vPzycjIIDEx0a33eEQIxVXIY5EYuCC0CWVlZURGRop4tyFKKSIjI5t01+MRAu5KI5RBTEFoO0S8256mfuceIeBUeeASQhEEQXDhEQLu8sAlhCIIglCNRwm4pBEKgtAQQUFBDb6WmJjIrl27aq279957efrppwFISUlBKcV3333n9j5rsnTpUoYOHYqXlxdz5sxpouXNxzMU0SXgXp5hriD8knj0y21sP3SsRffZr3MIj1zRv0X3eTKmTJnCrFmzeOSRRwBwOBzMmTOHFStWAPDxxx9z7rnn8vHHH3PxxRc3ef/x8fG8++67PPvssy1qd2N4hAdeFQMXD1wQzhgeeOABXnvttarnM2bM4NFHH2XcuHEMHTqUgQMHMm/ePLf2NXXqVGbNmlX1fOnSpSQkJNC1a1e01syZM4d3332XhQsXNiv3PSEhgaSkJCyWtpXURhVRKeUHLAV8ndvP0Vo/opSKAD4BEoA04Fda66OtYqXDVcgjg5iC0Na0padckylTpnDvvfdyxx13ADB79mwWLFjAn/70J0JCQsjLy2PkyJFceeWVjWZvuMR18+bNDBo0iFmzZjF16lQAVqxYQWJiIt27d+f888/nm2++4eqrr27142sJ3LlclANjtdaDgMHAJUqpkcB0YLHWuiew2Pm8VdCSBy4IZxxDhgwhJyeHQ4cOsXnzZsLDw4mJieHBBx8kKSmJ8ePHk5mZSXZ2tlv7c3nhNpuNefPmce211wImfDJlyhTAXDQ+/vjjVjumlqZRRdRaa+C486m3808DE4HznetnAj8CD7S4hQAO0wvF4iUeuCCcSUyePJk5c+Zw+PBhpkyZwocffkhubi4bNmzA29ubhIQEt0MeU6dO5aKLLmLMmDEkJSURHR2N3W7ns88+Y/78+Tz++ONV1ZBFRUUEBwe38tGdOm4FbJRSVqXUJiAHWKS1XgN01FpnATiX0Q289zal1Hql1Prc3NzmWSlphIJwRuIafJwzZw6TJ0+msLCQ6OhovL29WbJkCQcOHHB7X927dycyMpLp06dXhU++//57Bg0aRHp6OmlpaRw4cIBrrrmGL774opWOqGVxS8C11nat9WAgDhihlBrg7gdord/QWidrrZOjoqKaZ2WVgPs07/2CIHgk/fv3p6ioiNjYWGJiYrjhhhtYv349ycnJfPjhh/Tp06dJ+5s6dSo7d+7kqquuAkz4xPXYxTXXXMNHH30EQElJCXFxcVV/zz//fL37XbduHXFxcXz66af8/ve/p3//thk3UCZC0oQ3KPUIUAzcCpyvtc5SSsUAP2qte5/svcnJyXr9+vVNNnLDR48wbPcL5PwxleiIiCa/XxCEprFjxw769u3b3mackdT33SulNmitk+tu26gHrpSKUkqFOR/7A+OBncB8YJpzs2mAe/k8zUA5Y+BWyUIRBEGowp2gcgwwUyllxQj+bK31V0qpVcBspdQtwEHg2laz0hlCscogpiAIJ2HLli3ceOONtdb5+vqyZs2aFtn/448/zqefflpr3bXXXstDDz3UIvtvKu5kofwMDKlnfT4wrjWMOgGHDYdWWGRCB0EQTsLAgQPZtGlTq+3/oYceajexrg+PqMRU2oYNC1ZpbykIglCFRwg4Djt2rFhEwAVBEKrwCAFXDjs2rLRxmwFBEITTGs+QRG3DLiEUQRCEWniEgCuHDRtWrBYRcEE4UygoKKjVjdBdJkyYQEFBQZPe8+6771ZVZ7rIy8sjKiqK8vJyACZOnMioUaNqbTNjxgy3W8j+9re/JTo6mgED3K6DbBSPqE1fGn8n7x0YzzrxwAWh7fl2Ohze0rL77DQQLn3ypJu4BNzVjdCF3W7HepKMtG+++abJ5lx99dX89a9/paSkhICAAADmzJnDlVdeia+vLwUFBWzcuJGgoCBSU1PdnjW+Jr/5zW+46667uOmmm5r83obwCA+81BLIUVNLJAjCGcL06dPZt28fgwcPZvjw4VxwwQVcf/31DBw4EIBJkyYxbNgw+vfvzxtvvFH1voSEBPLy8khLS6Nv377ceuut9O/fn4suuojS0tJ6PyskJITRo0fz5ZdfVq2r2XL2s88+44orrqjqzdIcRo8eTURLV5Jrrdvsb9iwYbo5/Oub7brnQ980672CIDSd7du3t7cJOjU1Vffv319rrfWSJUt0QECA3r9/f9Xr+fn5WmutS0pKdP/+/XVeXp7WWuuuXbvq3NxcnZqaqq1Wq05JSdFaa33ttdfq999/v8HPmz17tp40aZLWWuvMzEwdExOjbTab1lrrcePG6aVLl+pdu3bpgQMHVr3nkUce0c8880yzjqkh6vvugfW6Hk31CA/c4dAygCkIZzgjRoyoFbp46aWXGDRoECNHjiQ9PZ09e/ac8J7ExEQGDx4MwLBhw0hLS2tw/5dffjnLly/n2LFjzJ49m8mTJ2O1WsnOzmbv3r2ce+659OrVCy8vL7Zu3drSh9csPELA7Q5kAFMQznACAwOrHv/44498//33rFq1is2bNzNkyJB6+4L7+vpWPbZardhstgb37+/vzyWXXMLcuXNrhU8++eQTjh49SmJiIgkJCaSlpTU7jNLSeISAO7RG9FsQziyCg4MpKiqq97XCwkLCw8MJCAhg586drF69ukU+c+rUqTz//PNkZ2czcuRIwLScXbBgAWlpaaSlpbFhwwYR8KZgd2jxwAXhDCMyMpJzzjmHAQMGcN9999V67ZJLLsFms5GUlMQ//vGPKrE9VS666CIOHTrEddddh1KKtLQ0Dh48WGv/iYmJhISEVDXIeuyxx2r1DG+IqVOnMmrUKHbt2kVcXBxvv/32Kdvb5H7gp0Jz+4HPWnuQlIMFPDU5qRWsEgShLtIPvP1oSj9wj8gDnzIinikj4tvbDEEQhNMKjxBwQRCEluLOO+9kxYoVtdbdc8893Hzzzae87/z8fMaNO7HL9uLFi4mMjDzl/ddFBFwQhHrRWqN+gem7r776aqvtOzIy8pT6kTc1pO0Rg5iCILQtfn5+5OfnN1lQhOajtSY/Px8/Pz+33yMeuCAIJxAXF0dGRga5ubntbcoZhZ+f30kzWeoiAi4Iwgl4e3s3q2GT0LZICEUQBMFDEQEXBEHwUETABUEQPJQ2rcRUSuUCB5r59g5AXgua057IsZyeyLGcnsixQFetdVTdlW0q4KeCUmp9faWknogcy+mJHMvpiRxLw0gIRRAEwUMRARcEQfBQPEnA32h8E49BjuX0RI7l9ESOpQE8JgYuCIIg1MaTPHBBEAShBiLggiAIHopHCLhS6hKl1C6l1F6l1PT2tqepKKXSlFJblFKblFLrnesilFKLlFJ7nMvw9razPpRS/1NK5SilttZY16DtSqm/Oc/TLqXUxe1j9Yk0cBwzlFKZzvOySSk1ocZrp+VxACiluiilliildiiltiml7nGu98Tz0tCxeNy5UUr5KaXWKqU2O4/lUef61jsvWuvT+g+wAvuAboAPsBno1952NfEY0oAOddY9DUx3Pp4OPNXedjZg+2hgKLC1MduBfs7z4wskOs+btb2P4STHMQP4az3bnrbH4bQvBhjqfBwM7Hba7InnpaFj8bhzAyggyPnYG1gDjGzN8+IJHvgIYK/Wer/WugKYBUxsZ5tagonATOfjmcCk9jOlYbTWS4EjdVY3ZPtEYJbWulxrnQrsxZy/dqeB42iI0/Y4ALTWWVrrjc7HRcAOIBbPPC8NHUtDnM7HorXWx51PvZ1/mlY8L54g4LFAeo3nGZz8BJ+OaGChUmqDUuo257qOWussMP/EQHS7Wdd0GrLdE8/VXUqpn50hFtetrccch1IqARiC8fY8+rzUORbwwHOjlLIqpTYBOcAirXWrnhdPEPD65nTytNzHc7TWQ4FLgTuVUqPb26BWwtPO1X+A7sBgIAt4zrneI45DKRUEfAbcq7U+drJN61l3Wh1PPcfikedGa23XWg8G4oARSqkBJ9n8lI/FEwQ8A+hS43kccKidbGkWWutDzmUOMBdzm5StlIoBcC5z2s/CJtOQ7R51rrTW2c4fnAN4k+rb19P+OJRS3hjB+1Br/blztUeel/qOxZPPDYDWugD4EbiEVjwvniDg64CeSqlEpZQPMAWY3842uY1SKlApFex6DFwEbMUcwzTnZtOAee1jYbNoyPb5wBSllK9SKhHoCaxtB/vcwvWjcnIV5rzAaX4cysw0/DawQ2v9fI2XPO68NHQsnnhulFJRSqkw52N/YDywk9Y8L+09cuvm6O4EzOj0PuCh9ranibZ3w4w0bwa2uewHIoHFwB7nMqK9bW3A/o8xt7CVGI/hlpPZDjzkPE+7gEvb2/5GjuN9YAvws/PHFHO6H4fTtnMxt9o/A5ucfxM89Lw0dCwed26AJCDFafNW4GHn+lY7L1JKLwiC4KF4QghFEARBqAcRcEEQBA9FBFwQBMFDEQEXBEHwUETABUEQPBQRcEEQBA9FBFwQBMFD+X9pj/DKK9XxeAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_VAL_1','train_VAL_1']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABG3ElEQVR4nO3deXhU1fnA8e/JZE8IhJCwJEDCvoclICAICgrigggqaK3aVsWqrVr9aWvr0mrrVlsVl+JSqyLI4oKKoggIggJhJ6whCZAEspGN7DNzfn+cycokmcSEMMn7eZ48mblz595zZ5J3zrz3nPcqrTVCCCHcn0dLN0AIIUTTkIAuhBCthAR0IYRoJSSgCyFEKyEBXQghWgnPltpxp06ddGRkZEvtXggh3NL27dsztdahzh5rsYAeGRlJbGxsS+1eCCHcklLqWG2PScpFCCFaCQnoQgjRSkhAF0KIVkICuhBCtBIS0IUQopVwKaArpaYrpQ4ppeKVUo84eTxYKfWJUmqPUmqrUmpI0zdVCCFEXeoN6EopC/AqcDkwCJinlBpUY7U/Abu01sOAXwIvNXVDhRBC1M2VHvoYIF5rnaC1LgWWADNrrDMI+A5Aa30QiFRKdW7SlgohhCtStsPap+HQV823D7sNEr6HvcvBVtZ8+2kgVyYWhQMnqtxPBi6osc5u4FrgB6XUGKAnEAGkVV1JKXUHcAdAjx49GtlkIYSoRcp2ePdKKCsEiw/8fjcEdW267ZcVw+ZXYP+nkLbPLCvMggvurP+52UnQrit4+jRde2pwpYeunCyreVWMZ4BgpdQu4F5gJ2A960laL9Rax2itY0JDnc5cFUL8HHY77Hgfdn9kbrcEWxnEfQK5KfDZPbDxn2Atbb79aQ3f/RWW3ATvXQMBneBXq8FuhU1NnP2N/xbWPWVuz1oI3UbC1oXOX2trCXz9J0jbD3knYcEYWPNk07anBld66MlA9yr3I4DUqitorfOA2wCUUgpIdPwIIc6VsiL46GYTdAASN8A1rzZsG0XZJjgFhJj71lLIiofONU+bOVFaCDv+B/knTSD1DjQ9ZW0HrwAYO9/1dmgNy26FwDCY8jj4BNa+7prHzf58O0BIb7jhA2gfAcNvNME26iKIiDHb+rlSdoCHJ/zmO/DyNcs+uQMSv4feF1euZy2Brx+B2Heg9Az4h4CtBHa8B5MfAd+gn98WJ1zpoW8D+iqlopRS3sBcYGXVFZRSHRyPAfwG2OAI8kKIc+XjOyB+Dcx4AUbdCnuWQEGm68/f9DI8GwkvDoDvnzfB59u/wOvjYNeH9T9/639MENv0EnTsZQLZ9GcgbBAc+Lxhx3L0O5PW2LoQvnm0jvXWmf2Nug0eToI71ptgDjDtaejQHZbMg9fGQXEDQpLNCkU5Zy9P3WmOpzyYD5ppPrjiPjb37TZY9X/wfB8TzL38TS4/9h0IGwyl+bDzA9fb0UD1BnSttRW4B1gNHACWaq3jlFLzlVLlH7kDgTil1EHMaJjfN1eDhRBOZB6BAyvhoodgzO0w5k6TctiztP7n2qyQuBHWPgV9pkLX4SatsPJeE1AtPvDpXfDW1NqDorUUtvwHugyDIXPgls/h/xJNbnnAlXB8s+sfLlqbPHVgFxg8C/Z/BvHfwU+vw6m9leud2gvLb4NO/WD6P0DVyA77tofbvoJpf4fCTPhkPmx7q8axl0Fusrltt8PpBDj2I7w40Hy4/fiq6ZW/EmPSWKk7oduIyud7+UK/6XDgC/M6Jm8zH2y9JsFNK8yHa0E6lOTBtf+B7mPN43aba69FA7lUbVFrvQpYVWPZG1Vu/wj0bdqmCSFctv1dkwoY/Rtzv/Mgk9/d+QGMvevsYFfVt3+Bn14DnyC4egEEhELuCfjifkhYD79eDUk/wDd/gfXPwPS/m+dpDTnHoH0P+Oohk2q5egH0nVp9+4Ouhg3PQex/YdJD9R/LjvfMfqf9HTr0MPn4D66tfHzkLXDJX2DRdaYHfONS8PJzvq2gbjDubjix1fT4D31pPrSCI82H0OIb4NhmuG+v2e/av4FfsHkt+l4Gq/8EHXvD6aMmtQLVAzrA4Gtg33JzjB6OkHrVy+Df0XyIWXzMN6YuQ817sewWOPw1DLii/teigVqsfK4QoomU5MPO901PuF2V0cIjboIv/wAnd50dhMppDQe/hJ4XwrVvVo4I6RgFcxfB6UToMsQ8P/MI/PQqHFkNNyyCXR+YnnR4DKTEwsQ/QJ8pZ++jy1AYeLU5OdprMnQfXfuxHPkWVj1o1rtgvslFe/qZHPQtK2Hb26YN+z8z5wx+s8a0tT7XvG6+Lfx3BuxcBNFz4Zs/w9G15vG4T2Drm2DxNucRrn/P9KZfH2fOIYy7B3zamf33mlR92/2mw6Br4PtnTfDuPMQEczAnaO/ZVpkGGnAldOgJp/Y1S0BXWtccsHJuxMTEaKmHLkQT2PyKCU63r4XwUZXLi7Lhhf7mpOCM550Hvqyj8MpIkxoYc3vd+ykrMj3+Dc+bbdtKoX1305uf8ABMeaz2bwI5J+DNi6EgwwTAa16vDHrlclNgQQyE9IFfflb5ePwaCAqHsIHmA2jTS+ZDKnoe9Jvm8ssEwP+uMieLAVDmdfnpNZNqAZi3BEIHVL5WiRvhywfg5k8qg7IzWptRNoe+NOmuGc/Vvm5ZUe3fKFyglNqutY5x9pj00IVoSVqbIDnkWvAOaNzzty6EyInVgzmY1MGE+2DDC/D2ZTD/h+o9eDDBEkwaoj5efibo97oYtr1pAtzo2x2jYAbXndbp0B3u3WHyx2ufMicJL3qw+jprnzJ5/xverx7sq7ZNKXNMjTX1Sdi7zJzY7DbcfHsoyITvn4FhN0DfaeBR5dRi1ETTw66PUiaPn3HQnCity88I5vU2Q3roQrSglO3w5iVw5b8g5lcNf35mPCwYVffz0+LgzSmmp35TjZOk714J+afg3nP4v/jySNPbnruocllxrjkJOeZOuPyZc9cWMLn0jAPQNfrc7reR6uqhS7VFIVpS+kHz+9S+xj0/YZ353Wty7et0HgyTHza57yU3wad3m+W5KeZk59A5jdt3Y4WPNCNHqjqxzYxX73/5uW0LgKe32wTz+khAF62f1mbI2ZFvW7olZ8s8ZH6nxTXu+QnrzUiQ4HpODI6+3ZxYPPiFOZmZtMmMzEDD0Osat+/G6jYS8lPN7MlyxzeDspgJQKLRJIcuGs9WZiaP+AU33TaLss1JvphfmRoZm142X8/H3V0993hyjxl/XD7Bw5mfXjfjhnOOw/EfzbIxd8D0Z6vnSVtSRpWAbrfX3S67DTa/DL2nQNdhlePHB8+sO38NZqbl7LfMa7H2aTNVPi8VeowzsyvPpfCR5nfKdgi60tw+/pPpJTfmPIKoIAFdNMzRtZC83XxNX/pLyEuBB+NNIDrwhRn+dc1rDStAVJwLmxfA+HtMOuDQl2bMcnGuGWe8b7k5cXX5MyaofXY37F5sAtuNH4HF6+xtpu40sxYDwqBdF3My7EyaGdFg8TazCM8HGYfM2OXSfDOmu+ZIlNOJlSdNN79ijjvuUzMj8uQuKMk1Jyld0fsS89vD07yGYEZ5nGtdo8G/kzmeAVeYESbJsZVj6EWjSUAXrss8Akt+AWUFlQWKwFSd8/SFlfeYHnanfjDyZlNaNOeY6Qle+2b1ehwlZ8xkGJ92cCbdTMo4sNKMEhh1m/kHH/ELmPgArHnCjKqI+RWk7TVBrd90Mzlj4cUw6w0zVjp5u9l/2EBY/aj55nBvrJkxCCb1YrfCjwvMbMhh5zjVUFNZkanA12eqqb9yam/1gG4thdcvNK/31oVmtmH3sXDiJ/PBWp6HjprkdPO1Gn6TeW5WvJk8c655+cHUJ8zfy5uXmL8P7wDXKhaKOskoF2Gc2Gp6rt2G177O+7NMz/fGZSag5J+E7f81Y4TzUkwOtMc4kw/1DzFjjj19wVpsxilPfbxyW+v+biZiACgPE2zR0H8GzP2wegqhINOMjAgfafapNfz2R5MPXvV/5mTakGthyxtVGqtg5gLzoVCVtRTenWGmaA+aCRf9n/kwaAkHV5k6I7PfNtPsh98EV7xQ+Xh+Gvyzn5m0sv9TMzTxpmWwYDSgzAeVUjB/Y+P2r3X9qZrmYrebiUZH14K3P0x6GLqPaZm2uBkZhy7qln0M/ne1qQY3+U+OmXAKOvUxv/06QMZh8893yZ/NTL/y2X7HfzJDvobMMXnuTv3g89+Z4v+/WWu+Xq+8x3y9DupmJqMMm2vqfvS/wvTQ9yyBy5+D0gIzRbpmkAnoBJP+zxRpUh6mmp6HxQTk0AFmssiWN8zJvVG3mhN+PcaePaMPzIiGWz6H758zs/6yEuCuHxr3upXkmxojnj4NH51hLTHTyjv1N8exZ6kpSFVt+466KQOuhPG/M++Hl5+ZxfjfyyH3uJkQ1FgtFczBpOgmPeRaKQDhMgnobdn6Z6FTXxNMlIKBV5lUSnk6RVlMfrr/DJM6sXjDyFurb2PQTNhdYMZBl5cEnfOOyXV7WMz9aX83aZlVjokka58yZVUn/gFC+5mRDSNvMcG2NmPuMAG0z9TqU8dD+8P9ceZbgE87syxyQt3H7eVnvi14+sL6f5iqen4dXHjBqkiLgw9vMLMkAe74vu5vNzUd2wTZiWYKvcXLHNeR1Saf3LGXWac41/z2DYKIKpOGwkfC/E3mONqHN6zdolWTgN4W5Rw3+dv1fwcPL7CXwaV/M/Uq1v7VfBX372gq6xVmwuFvzIm0y5+DwBoXJpn8iKnwZ6nxp1QezMFs65bPTW/WWmIq3k1+pDJI1TflHEywv/iPzh+zeDk/MVqfnuMBDSe21D2FvCDTnD/IOGBO+g6ba2qJlBaYqeLLf23qgDSk9njKdvO7/MOn/NvE8Z+qBPQc87v8HEBVnfq4vi/RZkhAb2s2PA/r/mF622ACdad+pgqch4c5WdUQSp0dzJ3xC66cwDLipobto7lExJgPtGObaw/o5RdaSHLkqf06QuJvze1JD5tUS/QNpuDTtKdd7+mn7ISQvpXrB3Uzv6uWmK3ooTsJ6EI4cZ4MxhU/S+Hpyttam5ETdpvpVdrKzNC3TS/Bt4+ZdEePcWAtMifZ7txgqtg1pofr7rz8TBXBY5trX+fgFyaYD55lvsE8sN8M//P0NaNxwNQAsZVUVu6rj9amOmH5eGwwF0nw8DSjdMqV1x6XgC5c5FIPXSk1HXgJsABvaa2fqfF4e+ADoIdjmy9orf/bxG0VzqTuMhcemHCfKTf6/iw4tceMTU5YD8OuN6MpSvPN+j3Gm0p2CevMhJLyr/dtVc9x8ONrzivgndgGH99pyqFe+2blh968j8yY9vJSsxGjTc/98Goz2qY+eanm+VWLaSllvsVUC+jSQxcNU29AV0pZgFeBSzHXF92mlFqptd5fZbW7gf1a66uUUqHAIaXUIq11M14Zto2xWSFxvRlzXLU3/f2zJge+4QXYv9L0zvtfYSbnAOz5yPy+fZ3pGXbqY1IkfS8910dwfuox3nx7SY41lfWq2vC8OdH6i4+rv+ae3qZ6YDkPixnPfeQb8z7Vl4JKdYwf7zay+nJnAV1ZzOQqIVzgSsplDBCvtU5wBOglQM36kBpo57hAdCBwGrA2aUvbmvLUCZia1St+DR/MNvnc0kKzPO5TOLQKJtxvvvbnnzIz/67/H0x8EK5yXPG8z6Xm633EKOnt1dTjAkBVlgYoZyszI1EGzDi75KwzA6+CotPmA3bZrdUDc00p2016pcvQ6sudBfTyseZCuMCVlEs4cKLK/WTgghrrLMBcODoVaAfcoLW219yQUuoO4A6AHj16NKa9rUNBlsmhlp+I09qMkti71Hy1L8yCzMPmOo5do+HkbrNe/xkmp/vyCDPc8PiP0P0Cc3KuZrpgyl/Mb1tZ3ZX42jq/YFMb++AXZqx3eW2YlO2mTo2rr12/6RAUYWa8gplsVVt5gZQdJo1Tsw6NX7BJx5QryZMPYNEgrvTQnXUPak4vnQbsAroBw4EFSqmgs56k9UKtdYzWOiY0NLTmw23Hhufhw+vNVHUwBZe+esjMXnxtLLw1Bb580My2zDhkZlneEwvzFsOtq0xvuyTf1I6+8aO6C+aPud0Ef1G7CfeZD83ya0aCmRiFMieOXWHxhDGOWiTdRpqJU1WrCZaz281s2/CRZz/mF1z9SvPlPXQhXORKDz0ZqJIwJALTE6/qNuAZbeoIxCulEoEBwNYmaWVrojUc/srcXv8PuPhPpvLdwKvN9Ri//qMZeZG600xdHzSz+pjuyAvNj2g6w643E4S++ysc+hr6T4fE701Fw5qXSavLuHtNesvTx1xKbf+nZjhoVVnxpudd8+pCUHvKRQgXuRLQtwF9lVJRQAowF7ixxjrHgSnARqVUZ6A/kNCUDXVrse+YSoRj7zIXiM1OgrDBpiBT8jYI7AJXv2z+oYf/wkyPP7bZBHjJn54b439naqZ/96Q5OXpi69nBuD4Wz8q6MJ2HmklINbdR2wlRMO9/ab5Jk1m8TEDvFNbwYxFtVr0pF621FbgHWA0cAJZqreOUUvOVUvMdq/0NGK+U2gt8Bzystc50vsU2JPsYfHIXfHG/mQG4aA587PhaPm+xqdFRkgfX/qeyprjF0xQr6jtVgvm5ZPEy6an0/aZ0r/1nnnsYfI2ZgZqbXH15ynbwCjAlC2oq/xsoT7tID100kEsTi7TWq7TW/bTWvbXWTzuWvaG1fsNxO1VrfZnWeqjWeojW+oPmbHRz2JeSS3p+cdNu9NO7TC9t9G/IuzeO3O5TIP0ATPsHBPeE6/4Hv99TZ+2RhIwzvPzdEfKKy5q2beJsA68yv7951Myk7TGu8dsaPMv8jvu0+vKU7SalVjWNVq4ioDvSLsV54Nuh8W0QbU6bnSn6U0IWmWdKAEjPK2bWa5u4/N8b2ZOc07ANFWWbet5AelIc2W/PRn9wHRzfYoa9TX0cPeMF7l52mOFHbmPNFT/AODN1vNAGsTnmCi2H0/L53eKdFJZWjvaMTTrN9H9v5MVvD/PbD3bw9b6T2OwtU+64TWjXBcIdVUnH3mW+KTVWSG8zQmnPElNq4e3LTMXKU3shfITz55SXASjKNmmXsgLpoYsGaXO1XLTWPPZZHO//dIx2vp688YtRxCZlU2bTaOD51Yd4/9fVR2WuP5ROmU0zdWAYqmoaRGt4bybkpmCPnEjY/k+wa4VSmlMJuwn1DcYy8pesjktj45FMOvj7cNeKo8zPhAcu7ccra+N5ff1Rlt45juXbT7Bydyoje3TglvGRpOQU8eTn++kU6M0t4yN55uuD/BCfyWs3jWTG0K7n9kVrS+a8Y2ZxNkVt7sGzzMU5Tu01NWM+vN6cH3GWP4fqPXSZ9i8aoc0F9G1J2bz/0zHmju7O1sTT/PHjvRSW2rioXyjDwtvz2vp40vOL2ZaYTc8QfwZ3C+LW/24D4PejA+iV/g1rT/nh1+sCbg7ayWDHGHGP/Z/whvUqrNE38+uDv6aLLY0nCn9D6KaTLN56nH6dA1n0m7H8Y9UBXlkbD8CyWJNf/esXcRzLNJOF3vohkY6BPvxu8U4A/nVDNLNGRDBrRDjTX9rI6rhTEtCd+GxXCn3D2jGo21mjZRsmuKf5aQoxvza/B1xprq70zZ/NMMh+052vXy2g55jbPj/zeESb0qYCen5xGe/8kEiQryePXTWIrYmnufW/22jn68nD0/vj4+nBgnXx3PvhTrYknibI15PXru1JOwq5xGMHt+95h0BVzEwF1gQPPJWdBHsXHrfeSlRAKSd6Xs47c0ajfnyYoiPfk8r1vLvaXAT4w9svILSdD/+8PhoUFUF99sgIPt6ZjNbwqwujeGdTIn9csYceHf35+6yhXNgnBICwIF+mDgzjq72nOHG6EH9vCyGBDbhuZyt26FQ+v1+yC29PD965ZTQT+nZqtn1ZbXb++e1hdhzL5r1fj8HH00kuvJxvkJnFC9Cxt+mZd7+g9tIAVQN6SfP20E8XlJJdWErv0MD6VxZuo80E9MTMAq5e8AP5xVZ+PSEKf29PJvcP45lrhxLdvQMDu5qe0E0X9GDx1uPE9AzmdEYq/VdMI9anAB9Vxlb7AJ6w38GKi9JIy0jneMfxdIqK5k9BXSueD8D4e/Ebfy8Lge8PZ5CWW8z43ibIKKV4fk400REd2JeSyz+uHcqt4yPZlnSaW8dHsi81l62Jp7l/XM+zAtO0wV1YGpvMxOfWAfD4VYO47cIocovKaOfjiYfH2aNivtiTSnREB7p39KfMZmfF9mRScoq4sE8nUrKLiAoNYGSPYIrLbPh4eqCU4sTpQkpt9hb9Z88pLKW4zEw2Pngqj39+c5i+nQOZO7oHvUID6FTlw+z9n5Lw9vQgyNeLRVuONVtA33E8m8c/i2Nviima9fW+U8wc7uIFJjw86p8/4NMeUI4eevMV5rLbNbe9u424lFz+fMVAbhkfiVKKNzcksDE+k7d+ac4j/BCfQecgX/qEBZKcXXRO/x7W7E+jb+dAeoYEnLN9tgZt5pqid74fy8YjmTxx9WCmD+lCkG8t5WLXPElep2h8Bk4j9T/X0S3rR75gItdMnczrJdMICvDjl+Mim62dh07l8+K3h3huTjTt/aq30W7XfLXvFAUlVlbtO8n3hzNo7+dFTmEZAd4Wgvy8uGZEOKMjg+kZEsDJnGJ+8fYWenT0Z/n8cdz+Xiy7k3OrbdPTQ9EjxJ/EzAIGdAniVxdG8tcv9pNfbOXakeG8MCeal747QlZBCfdP7Ud+sZXITo37J9t8NJM/LN1Nez8vXrtpJL2cBIjTBaU89/VBPoo9QdU/zfAOfuQWlXGmxJw0fmhaf+6+uA/HswqZ9u8NzBjaFbvWbDySwbZHp1Y/19EEFm05xqOf7KNzkA9/vmIQ//zmEH7envx6QhSb4zNJyCzgf7eNob3/zyxD/GykuZxf1ERY+ktzZaImvubp4q3H+ePHe+kbFsiR9DP0Cg1Aa9PpAXjsykHsS8nl450pKAUDugRxOC2fNQ9MIqqR731DFJXaGPrEamYM7crL82o5gVzDydwivjuQTlSnAC7sYz7QD5zMY3XcKYZ0a8/UQS7U43ETbf6aosezClkdl8Z9U/tyfUz32lc8uRt+eJEg3/awcwhRpzfxhP02DkTcwOyJ47jnHLS1f5d2/Odmp+8VHh6KK4aZ/PmV0V15dV08OYVlRAT7k5ZXTEpOEa+vP8rrjvU9PRRh7XxIyyvm8pc2klVQyvNzhnHlsG4s2XackEAfNsdnknmmhMsGdWHx1uM8tNyke64b1Z13NiVSUmZn1b6TaA1f7T1FdmEpE/qGEhniT0iAD/tScwnv4EefsEBScop48LL+7DyezcrdqQwJb8+hU/nMHd2drh38eOarg9i1Jj2/hF+8tYXB4ab36ePpwZSBYWQXlPHSd0coKLFyy7hI+ncxl5Tztnhw2eDOeCjF2oPpfLYrlX9+c4joiA7889tDeFoU91/al41HMvlkZwqJmQVOPywaq6jUxovfHGZMVEfeuXU0gT6elFrtPLh8Nw8u2007H0/OlFpZsO4Ij14xyOXtZheU8u81h/nF2J707ey4fF75bNFm6KHb7Jo/f7qPxVuPM7JHB5beOY43Nyay+Wgmvl4WxvbqyLGsQp5bfZDiMju/HNeT3cm57D6RA8Cy2BM8cGk/difnEBHsTztfT/y9aw8hG49k8M9vDvO3mUMYGuH6cexOzsFq12xLOo3Wut4P50On8rnxzZ/IKijF39vC/VP7kZxdyKe7UsktMsN9X71xZMX/zqncYrYfy2bG0C6k55fw7f40hnfvwJBw520sKrXx+Z5UZo+MwOLkW/D5pE0E9E1HzRynq6K7nf3g6URTmKnneNj6lilVWlZs6ozP+g/D7RO5vL3v2c9rYf7enjw0bcBZy49nFZJZUML+VNM7+c3EXuQWlfG7xTu5sE8Ic0ZFoJTitgujALi6ymsyf1Ivjp8upE9YIH5eFiwe8ObGRDr4e9HO15O0vBKuHRlBXGoeWxOzKC6z0zcssGIUEEBSZgGb4jPJK64cfvn2D4kVt5+bPYzeYYE8/eV+krOLAMg6U8IXe0zdk3G9QvjrzMGVAa6Gq6K7Mbl/KFcv2MTN72xBa/PPGhHsz+hIM03/pe+OcN/UfqRkF9Gtg+/PDu4fbTtOVkEpr1/Wn0Af8y8ze1QEUwd2JuNMMZEhAfzx4728uzmJ2aMiGNCl/hOZVpud+R9sZ0viaT7dlUrX9r7849qhjKgI6OU5dNdPiqbnF+PnZaGdrxeLtx4n0MeTET068I9VByvSaSt2JHPHRb144NJ+eFo8uGtyb+6a3LtiG2l5xTz6yV4SMgp4aFp/7HbYm5LLfzclsnx7MmU2O29uNO+nt6cHa+6fRI+Qs4d3xqfnc8s7W7FreGzlPj6+a7zL35q2HzPj8E/mFjPz1U3cOKYHc8c4L+ZXZrNz1wfbsXgoFt48it8u2sHTqw4AEOzvxbf3X8TDK/bwyMd7uHhAKP7enjz5eRxf7TvFJQPCOJZVwNGMAjwU/HXmEOJSc+kZEkBxmY3N8Vn8cnxP4tPP8O81R2jv58W0wV1cfj9aQptIudy7eCdbE7P46Y9TzB+VzWoKMWUeMReDAEwNMm0KXo34hanh0T7inLTvXPjxaBb9u7SjY0AdF2J2YveJHDwtikAfT/KLrRW9mLziMgpLbHRp70tiZgEp2UVsjM/gP98nEN7Bjxevj6awzEa/zu1YezCdvKIyCkqs3H9pP7ws1ac/2OyaAyfz8FCKgV3bufSPn5Bxhl+8tYV5Y3pw7xRTfExrzaX/2kB8+hmiu3fgQGoewyLas/yu8dWeu+tEDp4eqtYeWVWlVjuTn19HeLAfy+aPr3W9zDMlTP/3RjoGePHFvRPx9qx7ise6g+nc9u427r2kD7tO5LD9WDaXDAhjgf1pcwWqPlNhw/Pox7L413dHmdi3U8UHljM5haVMffF7gny9uGF0d/7x1UG8LR508Pcir7is4nzEfVP7ct/UfvUed02b4zO56W3zAXrJgDAu7NOJp77cz31T+vGLsT1YdyiDq6K7Vpwkfm19PM99fYg/XNqPf357mF9dGMX8yb0Ia2c6R6VWO4WlVpKzi+gTFoivV+XJ5V+9u40tCVkUlNoA6NUpgO/+MKna38XRjDO8+M1hEjML2H8yj3dujeGSAZ15+4dEEjPPVBxjp0AfYpNOM+eNH7lzUi8GdQ3iwWW7GdAliMTMAkqsNl6ZN5InVsZxKq8Yb08PSq3mteoc5ENaXgneFg9KbXYm9u1EzxB/JvULO3sIcwO48q2jLnWlXFp9QNdaM/rpNUzsG8q/bhhuFu5cBJ/91lzcoNdkM0Nw60JTlXD07XVffV7UqdRqx9NDOT1B29Sc/WPY7ZoF6+J58dvDFctuHR/JJQPCuKhfKFprxj+zFptds+Ku8QT6eBJcx4fckq3HeeTjvbx722gm96+7rsqa/Wn85r1YnrpmCL8YW/fQx8c+28ey2GR2PnYpvl4WnlgZx4dbjrN32DJ8Tu0wF8zYs4TMe44Q89QaAFbcNY5RPTtWHPtDy/cwtpf51vXIij0s256Mh4Iym2Zsr47sT83Datcsnz+e+IwzFJRYmVdLT9cVb21M4K2NiSy/axwRwf7MW/gTR9LzKbXaySu2csmAMF6eN4J1B9N564dESq12vrx3Ak98Hsd7Px4DoGt7X24e15OFGxLIKTTpkIhgP2J6BjN9SFeT7ntjMzOGdmXZ9sqyCat+N7FiSOqfPtnLh1vMN5BAH08GdwvirVtiag2SWmtmvPwDB07mVSz75v6LCO/gR3ZhKRHB/vxwJJO3fkjgH9cOxdfTQnZhKT1DAnho+W4+2ZnCoK5BxKVWPv8Pl/bjzkm96/3gdtaWaf/ewOyREdw5qXf9T3CiTQf0w2n5XPavDTw3Z5jJn9vtsGCUuRLNHd9LvZRWKCO/hPHPfMegrkEcSsunuMxOWDsfNvzfxRxOy+fqBZsq1u3VKYBVv59YrYdYLiWniMv/vYE+YYGscCFloLVm9uubOX66kCkDOjMqMpg5IyPO+nDTWjPp+fX0DQvk7VtHA6b0xJWv/MC3A76g76lVZqz68c1snfk91//HXHxjWER7Xr1xJKHtfIhLzWP265uJDPHni99NZOTfvuW6URHMHB5OcZmNCX06sd/xrednj82v0fby12HF9mT+sGw3F/YJ4YKoEF789jBh7XxIzzczsO+c1Is/Xj4QoOJbyCc7k9mXkke39r78akIU7f28WBp7gqSsQjIczwtr58PSO8fh4+WBXcNFz63jqmFdeeG6aLYmnubGt7Ywb0x3Hri0P6HtXBu6G5t0mg1HMokM8SfrTCm3X+TapRftds3JvGLS8oq5/X+xPDt7GF/sSeXTXalYPBSLbx/LmKj6K3La7Zo3NyYQ1SmAO97fzt9nDeXGCxr34dqmT4pujjf58/G9zXhukrfC6QS49i0J5q1UaDsf3r1tDN2D/TmZW8ShtHwzO/jHY+QWlWHxUFw3KoKTucV8fziDl787wv9Nr34+wmbX3L9kFza75l83DHfpK7JSij/OGMjDK/bw3cE0Poo9gY+nB6MjO9K1vW/FNhIyCzh+urBaUBnYNQhviwdppX70Lc41eXTf9iRknAHg3kv68MraeCY+t45+nQMJCTCBLCmrkJe/O0Kp1c5V0d2qBRdXUkoNVfV1uHZkONHd29M7NBClFFabnZfXxjO5fyjbj2Vz5dDK8zPDu3dgePcOXB8TwStr45kzKoJ+jvMk18V0x2qz8/meVDLyS7h8SFe6d6zMy99xUS9eX3+UxMwC0vNLCO/gx+NXDXb6IVybmMiOxNSRsqqNh4civIMf4R38iP2zGT01oW8nRkd15F/fHmbBunieChpC945+Fa9Nmc3ON3FpWO12Fm89Tk5hGeEd/PjuYHpFj35Cn+YZWtv6A/rRLHp09Cci2PEHcuBzU3ip/GpBolUqH7rWI8SfC3qFsO5gOs+vPoSPlwdje3XkmdnDAHho2W5eW3+UYREdmNi3E1lnSlm2/QRf7jlJQmYBL14f3aCx0KMjO7L2D5Ox2zXTX9rAQ8v2UGqzExniz00X9OTWCyNZfygDgMn9Ki/yYvFQ9Azx50SxD6BNfXa/jiRkFuDt6cE9l/Rhf2oeIYHerDmQzuG0M9x4QQ9WbE9m4YYE/L0txPQMbroX0AVKKfqEVZ68vv/SfswcEU6vTmYYpLO0WztfL/40Y+BZyz0tHswa4fyc1cPTBzCgSzue/vIAwf7ePH/dsAYF86ZSHrB9vSzcdEFPsgtKeeGbw1z0/DreviWGKQM7U1xm49rXNrPfkd4J7+BHRLAJ5oE+npwpsdK9o5/TE8lNoVUGdK01WxJPMyyiPT8lZFVOldcaDqyEXhc3aPSAcH//umE4c974ER9PD565dljF8r9dM4RDafn8+dN99A4NYEviacCMtrl+dHdmjXBx4lANHh6Kh6YN4O4Pd/CrC6PYl5rL06sOkJBZQHK2GUlUtRcK0Cs0gKQURz4/OwmCoziafoaokAB8PC0V6ZkSq42CEhvB/l6M6xXCnz7Zy5xREXhaWrbWnlKqYvJRU3/5nTk83PVJXOfIL8dHcvx0IUtjk1l/KIMpAzsTl5rL/pN5/PmKgYzoEcygrkH4eVs4eCoPhWLavzdwYe/mm8ncKgP694czuPW/2+je0Y+8YiuT+zt6Qge/hJzjcPGjLdtAcc518Pfmq99PxNNDVUsb+HpZeHj6AG56awuZZ0qYPTKCGUO7MGXgz5+IcumgzsQ9Oa1iVM+zXx/k9fVHAfjNhKiz1u8VGsiRg17gBZQVmpRLSgEDulQfwunjaakYTXJVdDeukNo+LSLI14vn5kSTllfCTwlZABw4mQ/A5UO7Et6h8tKQ5UNZX79pJMO6d2i2NrXKgP7RthN4eihOnC7i2hHhZuyo3Q5rn4KQPmYmnmhzag6XLDe+dwhDwoNIzyvh6VlDmvTrfNV9PnRZf+x2zcKNCcwYdnYQ7tUpgC32yvTO8v35JOYXMGNo3WOfz8WIIlG7sb1CePbrgwx+7Gt6hQbSzteTbrXMXbm8mT98XQroSqnpwEuABXhLa/1MjccfAm6qss2BQKjW+nQTttUl2QWlrDmQxi3jI5k5vBsDO3mhlt0KfS+FjAMw89XaiyOJNkkpxdu3jKakzN6suVkPD3PS9IHL+jkt6tUrNJAcKidBJRd5MW1wZ266oImqP4pmMaFPJ54FCkpt7E3JZUxkxyYvPeGqepNuSikL8CpwOTAImKeUqja/WWv9vNZ6uNZ6OPBH4PuWCOYAWxKzKLNpZgztyrCIDnid3G4u1vvlg2aFvnIyVJytc5Bvs52oqqm2Co2RIf6k6w4V97N0EPPG9KBbla/u4vwzNKI9n98zgbmjTVmR/l2cz3I+F1w5izIGiNdaJ2itS4ElwMw61p8HLG6KxjXGzuM5eFs8GBLuOOmZst38thaZC/cGhtb+ZCFaUMcAb0osAbw2/DN2jH2Fj20TK2ZWivPb0Ij23OAI6BWxpwW4knsIB05UuZ8MXOBsRaWUPzAdnNexUkrdAdwB0KNH42es1WXniRwGhwdV9oKSY0FZQNug98XNsk8hmoJSirB2vsSXdiAgLJIC4ggLkpr37mJEj2CWzR9HdESHFmuDKz10Z8mg2qaXXgVsqi3dorVeqLWO0VrHhIY2fU/ZarOzJzmH4VXPIqfsgEFXw4W/h5hfNfk+hWhKYUE+pOeVkJZXjKeHoqO/lKFwJ6MjOza4HEBTcqWHngxUrTkbAaTWsu5cWjDdcjjtDMVl9sqAnnMc8lPNVWLG3tVSzRLCZZ3b+XI04wzp+b50CvSRESyiQVz5KNkG9FVKRSmlvDFBe2XNlZRS7YFJwGdN20TXHU4zY0AHlV89KO4T87vvZS3UIiEaJizI1K9Pzy+RdItosHoDutbaismJrwYOAEu11nFKqflKqflVVp0FfKO1LmieptYvPv2MYwq1YyzvnmUQHgMhjatqJsS51jnIl7xiK8mnC+WEqGgwlwZka61XAatqLHujxv13gXebqmGNcSQ9n8gQf5PDStkBaXvh8udasklCNEiYo3pgQmYBY8sLygnhopYt/tDEjqSfoU+YY2LG98+BbweInteibRKiIToHVfbKw1wsDStEuVYT0Eutdo5lFdI3rB1kHYXDX8G4u6UIl3ArVfPmdV2hSAhnWs0c+KSsAmx2bXroid+YhUNmt2yjhGigiGB//L0t3DimR0UJYCFc1WoC+i7HlckHdwuCTT9BQCh0dO2qJEKcLwJ9PNnz+GUtXgpXuKdW81cTm3SaDv5eph7z8R+hx1i5IpFwSxLMRWO1mr+c2KRsYnoG43HmJOQcgx7jWrpJQghxTrWKgJ55poSEzAJzzcDjP5mFEtCFEG1MqwjosUnZAIyODDYB3SsAugyr51lCCNG6tJKAfhpvTw9zlfPjmyEiRi5iIYRoc1pHQD+WTXREe3ysZyAtTtItQog2ye0DelGpjX0puSZ/nhYH2m566EII0ca4fUDfm5KL1a6J6RkM2UlmoYw/F0K0QW4f0JOyTHHHPmGBJqArD2jfve4nCSFEK+T2AT35dCEeCnMh3ewkCAoHT7nKixCi7XH7gH78dCFd2/vhZfGA7GMQHNnSTRJCiBbh9gH9RHYREcF+5k52EnTo2aLtEUKIluJSQFdKTVdKHVJKxSulHqllnclKqV1KqTil1PdN28zanThdSPeO/lBWBGdOSQ9dCNFm1Tv7RillAV4FLsVcMHqbUmql1np/lXU6AK8B07XWx5VSYc3U3mqKy2yk55fQPdjfXBAaIFh66EKItsmVHvoYIF5rnaC1LgWWADNrrHMj8LHW+jiA1jq9aZvpXHJ2EQDdO/pBXqpZGBR+LnYthBDnHVcCejhwosr9ZMeyqvoBwUqp9Uqp7UqpXzrbkFLqDqVUrFIqNiMjo3EtruJkrgno4R38oDDLLAyQiwIIIdomVwK6s6LiusZ9T2AUcAUwDfiLUqrfWU/SeqHWOkZrHRMaGtrgxtZ0ptgKQJCfV2VA95cL6woh2iZXKlglA1Vn6kQAqU7WydRaFwAFSqkNQDRwuElaWYvCUhsA/t4WKMgEFPgFN+cuhRDivOVKD30b0FcpFaWU8gbmAitrrPMZMFEp5amU8gcuAA40bVPPVlhmArqfl8X00P07goeluXcrhBDnpXp76Fprq1LqHmA1YAHe0VrHKaXmOx5/Q2t9QCn1NbAHsANvaa33NWfDAYodPXQ/bwsUZkq6RQjRprlUNFxrvQpYVWPZGzXuPw8833RNq19lysUTCrLAX06ICiHaLreeKVpYZsXb0wOLhzIplwDpoQsh2i63DuhFpTZzQhQk5SKEaPPcOqAXltrMCVG7HQpPS8pFCNGmuXVALyqzmROixTmgbTKpSAjRprl3QC9PucikIiGEcO+AXlhqxd/Ls7KOiwR0IUQb5tYBvajUkXJJWAfKAuEjW7pJQgjRYtw6oFecFD24CiInyLR/IUSb5tYBvajMRg9OQuYhGHBFSzdHCCFalHsH9FIbQ4q3mzt9L23ZxgghRAtz64BeWGqjb+FOaN8dgqNaujlCCNGi3Dag2+2a4rIyIvN3QOREUM7KtgshRNvhtgG92Gqjn0rGz5oLURNbujlCCNHi3DagF5XaiFCOy9iF9m/ZxgghxHnAbQN6YakNP0rNHa+Alm2MEEKcB1wK6Eqp6UqpQ0qpeKXUI04en6yUylVK7XL8PNb0Ta2uqMyGnyoxd7z8mnt3Qghx3qv3AhdKKQvwKnAp5tqh25RSK7XW+2usulFrfWUztNGpwlIbvhU9dP9ztVshhDhvudJDHwPEa60TtNalwBJgZvM2q36FpVb8kB66EEKUcyWghwMnqtxPdiyraZxSardS6iul1GBnG1JK3aGUilVKxWZkZDSiuZWKy6rm0CWgCyGEKwHd2QBvXeP+DqCn1joaeAX41NmGtNYLtdYxWuuY0NDQBjW0pjKbxleVYrf4gIflZ21LCCFaA1cCejLQvcr9CCC16gpa6zyt9RnH7VWAl1KqWa82YbNrfClFe/o2526EEMJtuBLQtwF9lVJRSilvYC6wsuoKSqkuSpmpmkqpMY7tZjV1Y6uy2jV+lKA9Jd0ihBDgwigXrbVVKXUPsBqwAO9oreOUUvMdj78BzAHuUkpZgSJgrta6ZlqmSVltdvxUKdpTRrgIIQS4ENChIo2yqsayN6rcXgAsaNqm1c1q1wRSgpYTokIIAbjxTNHyHDpekkMXQghw44ButWszU1QmFQkhBODGAd1ms5tx6JJyEUIIwI0DevkoFyUBXQghADcP6L6qFOUtKRchhAA3Dui2ih66BHQhhAA3DuhWm8aXMjykhy6EEICL49DPRzabDX9VAhLQhRACcOMeOrZi81tOigohBODGAd2jrDygSw9dCCHAjQO6shWZG9JDF0IIwJ0DelmhuSE9dCGEANw4oHtYHSkXqYcuhBCAGwd0i6RchBCiGrcN6B42OSkqhBBVuW1At9hKzA0pnyuEEICLAV0pNV0pdUgpFa+UeqSO9UYrpWxKqTlN18Ra9mUvNTcsPs29KyGEcAv1BnSllAV4FbgcGATMU0oNqmW9ZzGXqmt2ylZmbli8z8XuhBDivOdKD30MEK+1TtBalwJLgJlO1rsXWAGkN2H7alXZQ/c6F7sTQojznisBPRw4UeV+smNZBaVUODALeIM6KKXuUErFKqViMzIyGtrWajzs0kMXQoiqXAnoyskyXeP+v4GHtda2ujaktV6otY7RWseEhoa62MRaGiUBXQghqnGl2mIy0L3K/QggtcY6McASpRRAJ2CGUsqqtf60KRrpTGUPXVIuQggBrgX0bUBfpVQUkALMBW6suoLWOqr8tlLqXeCL5gzmICkXIYSoqd6ArrW2KqXuwYxesQDvaK3jlFLzHY/XmTdvLhbpoQshRDUuXeBCa70KWFVjmdNArrW+9ec3q34eugwbHlg8LOdid0IIcd5z25miHvYyrEp650IIUc5tA7rFXoZNue0V9IQQosm5b0DXZViVnBAVQohybhzQrdJDF0KIKtw4oJdhkxy6EEJUcPOALj10IYQo57YB3VNbsXtID10IIcq5dUCXlIsQQlRy24BuoUx66EIIUYXbBnRJuQghRHVuG9C9KMMuKRchhKjgxgFdeuhCCFGVWwZ0rTWe2oqWgC6EEBXcMqDbtfTQhRCiJrcM6Fa7HW9lxS4XtxBCiApuGdBtdo0XVpAeuhBCVHApoCulpiulDiml4pVSjzh5fKZSao9SapdSKlYpNaHpm1rJ6gjo0kMXQohK9RZDUUpZgFeBSzEXjN6mlFqptd5fZbXvgJVaa62UGgYsBQY0R4MBbDaNFzbpoQshRBWu9NDHAPFa6wStdSmwBJhZdQWt9RmttXbcDQA0zchq13hjRUsPXQghKrgS0MOBE1XuJzuWVaOUmqWUOgh8CfzK2YaUUnc4UjKxGRkZjWkvAFabzeTQ5QLRQghRwZWArpwsO6sHrrX+RGs9ALgG+JuzDWmtF2qtY7TWMaGhoQ1qaFVWqxUPpUF66EIIUcGVgJ4MdK9yPwJIrW1lrfUGoLdSqtPPbFut7GUl5ob00IUQooIrAX0b0FcpFaWU8gbmAiurrqCU6qOUUo7bIwFvIKupG1vOVhHQpYcuhBDl6h3lorW2KqXuAVYDFuAdrXWcUmq+4/E3gNnAL5VSZUARcEOVk6RNzm4tNTckoAshRAWXruGmtV4FrKqx7I0qt58Fnm3aptXOZpUeuhBC1OSWM0XtZaaHrjwloAshRDn3DOiSchFCiLO4aUA3KRcP6aELIUQFtwzo2iopFyGEqMmtA7r00IUQopJbBnS7zdFDlxy6EEJUcMuAXtlD92nhlgghxPnDPQO6Y6aoh5f00IUQopxbBvTyiUWe3tJDF0KIcm4Z0O1lxQB4evu1cEuEEOL84ZYBXTt66F7evi3cEiGEOH+4Z0B35NC9fCSgCyFEObcM6NjKA7qkXIQQopxbBvTyHrq3BHQhhKjgUvnc846jh+7pJSkXIZpCWVkZycnJFBcXt3RThIOvry8RERF4ebl+ZTaXArpSajrwEuYCF29prZ+p8fhNwMOOu2eAu7TWu11uRUNZSyjVFrw93PILhhDnneTkZNq1a0dkZCSOi4+JFqS1Jisri+TkZKKiolx+Xr0RUSllAV4FLgcGAfOUUoNqrJYITNJaD8NcIHqhyy1oBA9bCWVKricqRFMpLi4mJCREgvl5QilFSEhIg78xudLFHQPEa60TtNalwBJgZtUVtNabtdbZjrs/YS4k3WyUvZRSZJaoEE1Jgvn5pTHvhysBPRw4UeV+smNZbX4NfOXsAaXUHUqpWKVUbEZGhuutrLkdWyllSA9dCCGqciWgO/uYcHoBaKXUxZiA/rCzx7XWC7XWMVrrmNDQUNdbWXM/tlKsknIRQohqXAnoyUD3KvcjgNSaKymlhgFvATO11llN0zznLHbJoQvRlgUGBtb6WFRUFIcOHaq27L777uO5554DYOfOnSilWL16tcvbrOqNN97gvffea2CLzw1XRrlsA/oqpaKAFGAucGPVFZRSPYCPgZu11oebvJU1eNjKsCrJoQvRHJ78PI79qXlNus1B3YJ4/KrBTbrN2sydO5clS5bw+OOPA2C321m+fDmbNm0CYPHixUyYMIHFixczbdq0Bm9//vz5TdreplRvD11rbQXuAVYDB4ClWus4pdR8pVT5kT0GhACvKaV2KaVim63FgEWXYvOQHroQrcXDDz/Ma6+9VnH/iSee4Mknn2TKlCmMHDmSoUOH8tlnn7m0rXnz5rFkyZKK+xs2bCAyMpKePXuitWb58uW8++67fPPNN40ad//EE0/wwgsvAPDmm28yevRooqOjmT17NoWFhQCkpaUxa9YsoqOjiY6OZvPmzQ3eT6NorVvkZ9SoUbqx9j01Xsc9PaHRzxdCVLd///4W3f+OHTv0RRddVHF/4MCB+tixYzo3N1drrXVGRobu3bu3ttvtWmutAwIC6tzeoEGD9K5du7TWWt955516wYIFWmutN27cqC+55BKttdbz5s3TK1asqHhOfdss9/jjj+vnn39ea611ZmZmxfJHH31Uv/zyy1prra+//nr9r3/9S2uttdVq1Tk5OS5tuyZn7wsQq2uJq245M8dTl2LzkJSLEK3FiBEjSE9PJzU1ld27dxMcHEzXrl3505/+xLBhw5g6dSopKSmkpaW5tL3yXrrVauWzzz7juuuuA0y6Ze7cuYBJzSxevPhntXvfvn1MnDiRoUOHsmjRIuLi4gBYu3Ytd911FwAWi4X27dv/rP24yi2n/nvayyiSgC5EqzJnzhyWL1/OqVOnmDt3LosWLSIjI4Pt27fj5eVFZGSkyymSefPmcdlllzFp0iSGDRtGWFgYNpuNFStWsHLlSp5++umK2Zj5+fm0a9euUW2+9dZb+fTTT4mOjubdd99l/fr1jdpOU3HTHnoZdrlAtBCtSvnJzOXLlzNnzhxyc3MJCwvDy8uLdevWcezYMZe31bt3b0JCQnjkkUeYN28eAGvWrCE6OpoTJ06QlJTEsWPHmD17Np9++mmj25yfn0/Xrl0pKytj0aJFFcunTJnC66+/DoDNZiMvr2lPMtfGbQO6lh66EK3K4MGDyc/PJzw8nK5du3LTTTcRGxtLTEwMixYtYsCAAQ3a3rx58zh48CCzZs0CTLql/Ha52bNn8+GHHwJQWFhIRERExc+LL75Y67bLZ3H+7W9/44ILLuDSSy+t1r6XXnqJdevWMXToUEaNGlWRimluyuTYz72YmBgdG9u4wTAZT0SS1HECo3/3QRO3Soi26cCBAwwcOLClm+EW7r33XkaOHMltt93W7Pty9r4opbZrrWOcre+WPXQvytCSchFCnGN/+ctf2LJlC1dffXVLN8Uptzwp6qUloAvR1u3du5ebb7652jIfHx+2bNnSJNt/+umnWbZsWbVl1113HVu3bm2S7TcHtwvoWmu8sYLFp6WbIoRoQUOHDmXXrl3Ntv1HH32URx99tNm23xzcLuVitVrxUja0BHQhhKjG7QJ6SUmRueEpAV0IIapyu4BeVmImFnh4Sg5dCCGqctuAjqdcIFoIIapyw4BuUi7KSwK6EK1FTk5OtWqLrpoxYwY5OTkNes67775bMXu0XGZmJqGhoZSUlAAwc+ZMxo0bV22dqlUW6zN+/PgGtampuN0oF2upCegeXpJDF6JZfPUInNrbtNvsMhQuf6bWh8sD+m9/+9tqy202GxaLpdbnrVq1qsFNufbaa3nwwQcpLCzE398fgOXLl3P11Vfj4+NDTk4OO3bsIDAwkMTERKKiohq8j3NWLrcG9+uhl5bn0CWgC9FaPPLIIxw9epThw4czevRoLr74Ym688UaGDh0KwDXXXMOoUaMYPHgwCxcurHheZGQkmZmZJCUlMXDgQG6//XYGDx7MZZddRlFRkdN9BQUFcdFFF/H5559XLFuyZElFr33FihVcddVVFbVlGqP86kdnzpyptab7e++9x7Bhw4iOjj5rPH2j1VZXt7l/GlsP/WDsWq0fD9K71yxu1POFEGdr6XroiYmJevDgwVprrdetW6f9/f11QkJCxeNZWVlaa60LCwv14MGDK+qQ9+zZU2dkZOjExERtsVj0zp07tdZaX3fddfr999+vdX9Lly7V11xzjdZa65SUFN21a1dttVq11lpPmTJFb9iwQR86dEgPHTq04jlV66DXp7y2ellZmdOa7vv27dP9+vXTGRkZ1Y6vpmaph66Umq6UOqSUildKPeLk8QFKqR+VUiVKqQeb5qPGOWt5D93brzl3I4RoQWPGjKmW6nj55ZeJjo5m7NixnDhxgiNHjpz1nKioKIYPHw7AqFGjSEpKqnX7V155JT/88AN5eXksXbqUOXPmYLFYSEtLIz4+ngkTJtCvXz88PT3Zt29fo49Da+20pvvatWuZM2cOnTp1AqBjx46N3kdV9QZ0pZQFeBW4HBgEzFNKDaqx2mngd4BrZwx+BluZOWnh6S0pFyFaq4CAgIrb69evZ82aNfz444/s3r2bESNGOK2L7uNTGRMsFgtWq7XW7fv5+TF9+nQ++eSTaumWjz76iOzsbKKiooiMjCQpKanRaRegWk33Xbt20blzZ4qLi9FaV1RsbEqu9NDHAPFa6wStdSmwBJhZdQWtdbrWehtQ1uQtrMHm6KFbZJSLEK1Gu3btyM/Pd/pYbm4uwcHB+Pv7c/DgQX766acm2ee8efN48cUXSUtLY+zYsYApsfv111+TlJREUlIS27dv/1kBvbaa7lOmTGHp0qVkZWUBcPr06Z9/QLgW0MOBE1XuJzuWNZhS6g6lVKxSKjYjI6Mxm0BbTUD39JGUixCtRUhICBdeeCFDhgzhoYceqvbY9OnTsVqtDBs2jL/85S8Vwffnuuyyy0hNTeWGG25AKUVSUhLHjx+vtv2oqCiCgoIqCn499dRT1Wqm16a8911bTffBgwfz6KOPMmnSJKKjo3nggQea5JjqrYeulLoOmKa1/o3j/s3AGK31vU7WfQI4o7WuN/XS2HroJ3avo3jjK3Sa8yLBXSIb/HwhxNmkHnrTycrKYuTIkQ26wlJtGloP3ZVx6MlA9yr3I4DURrfwZ+oefTFEX9xSuxdCiFqlpqYyefJkHnywWceG1MqVgL4N6KuUigJSgLnAjc3aKiGEaAJ33303mzZtqrbs97//fZNcbSgrK4spU6actfzHH38kJCTkZ2+/MeoN6Fprq1LqHmA1YAHe0VrHKaXmOx5/QynVBYgFggC7Uuo+YJDW+txcGVUI8bM118iLlvTqq68227ZDQkKatR57felwZ1ya+q+1XgWsqrHsjSq3T2FSMUIIN+Tr60tWVhYhISGtLqi7I601WVlZ+Po2bDSf29VyEUI0vYiICJKTk2ns6DPR9Hx9fescSeOMBHQhBF5eXo0qQiXOL25XnEsIIYRzEtCFEKKVkIAuhBCtRL0zRZttx0plAI2dStUJyGzC5rQkOZbzkxzL+UmOBXpqrUOdPdBiAf3nUErF1jb11d3IsZyf5FjOT3IsdZOUixBCtBIS0IUQopVw14C+sP5V3IYcy/lJjuX8JMdSB7fMoQshhDibu/bQhRBC1CABXQghWgm3C+hKqelKqUNKqXil1CMt3Z6GUkolKaX2KqV2KaViHcs6KqW+VUodcfwObul2OqOUekcpla6U2ldlWa1tV0r90fE+HVJKTWuZVjtXy7E8oZRKcbw3u5RSM6o8dl4ei1Kqu1JqnVLqgFIqTin1e8dyt3tf6jgWd3xffJVSW5VSux3H8qRjefO+L1prt/nB1GM/CvQCvIHdmLrrLd62BhxDEtCpxrLngEcctx8Bnm3pdtbS9ouAkcC++toODHK8Pz5AlON9s7T0MdRzLE8ADzpZ97w9FqArMNJxux1w2NFet3tf6jgWd3xfFBDouO0FbAHGNvf74m499DFAvNY6QWtdCiwBZrZwm5rCTOB/jtv/A65puabUTmu9Aah5efLa2j4TWKK1LtFaJwLxmPfvvFDLsdTmvD0WrfVJrfUOx+184ADmIu5u977UcSy1OZ+PRWutzzjuejl+NM38vrhbQA8HTlS5n0zdb/j5SAPfKKW2K6XucCzrrLU+CeaPGghrsdY1XG1td9f36h6l1B5HSqb867BbHItSKhIYgekNuvX7UuNYwA3fF6WURSm1C0gHvtVaN/v74m4B3dmlVNxt3OWFWuuRwOXA3Uqpi1q6Qc3EHd+r14HewHDgJPBPx/Lz/liUUoHACuA+XfelH93xWNzyfdFa27TWwzFXcxujlBpSx+pNcizuFtCTge5V7kcAqS3UlkbRWqc6fqcDn2C+VqUppboCOH6nt1wLG6y2trvde6W1TnP8E9qBN6n8ynteH4tSygsTABdprT92LHbL98XZsbjr+1JOa50DrAem08zvi7sF9G1AX6VUlFLKG5gLrGzhNrlMKRWglGpXfhu4DNiHOYZbHKvdAnzWMi1slNravhKYq5TyUUpFAX2BrS3QPpeV/6M5zMK8N3AeH4syFwB9GzigtX6xykNu977Udixu+r6EKqU6OG77AVOBgzT3+9LSZ4MbcfZ4Bubs91Hg0ZZuTwPb3gtzJns3EFfefiAE+A444vjdsaXbWkv7F2O+8pZhehS/rqvtwKOO9+kQcHlLt9+FY3kf2AvscfyDdT3fjwWYgPlqvgfY5fiZ4Y7vSx3H4o7vyzBgp6PN+4DHHMub9X2Rqf9CCNFKuFvKRQghRC0koAshRCshAV0IIVoJCehCCNFKSEAXQohWQgK6EEK0EhLQhRCilfh/RiCWM2f8ZGEAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_VAL_jac','train_VAL_jac']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "unlabel_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', ~(data['all'].train_mask + data['all'].val_mask + data['all'].test_mask)),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def predict(model, loader):\n",
    "    model.eval()\n",
    "    seed_everything(args.seed)\n",
    "    all_preds = []\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch['all'].batch_size\n",
    "        new_dict = {}\n",
    "        for edge_type in [edge_type for edge_type in batch.edge_index_dict if 'all' in edge_type and not 'simp_link' in edge_type]:\n",
    "            edge_index = batch.edge_index_dict[edge_type]\n",
    "            edge_index = to_undirected(edge_index)\n",
    "            new_dict[edge_type] = edge_index\n",
    "        batch.edge_index_dict = new_dict\n",
    "        \n",
    "        out = model(batch.x_dict, batch.edge_index_dict)[:batch_size]\n",
    "        out_att = out[:,:9].softmax(axis=1)\n",
    "        out_val = out[:,9:].softmax(axis=1)\n",
    "        IDs = batch['all'].n_id[:batch_size].unsqueeze(dim=-1).int()\n",
    "        \n",
    "        now = torch.hstack([IDs, out_att, out_val])\n",
    "        all_preds.append(now)\n",
    "    \n",
    "    final = torch.vstack(all_preds)\n",
    "        \n",
    "    return final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 24.14it/s]\n"
     ]
    }
   ],
   "source": [
    "pred_train = predict(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 54.26it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 50.36it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 38/38 [00:00<00:00, 50.20it/s]\n"
     ]
    }
   ],
   "source": [
    "pred_val = predict(model, val_loader)\n",
    "pred_test = predict(model, test_loader)\n",
    "pred_unlab = predict(model, unlabel_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = torch.vstack([pred_train, pred_val, pred_test, pred_unlab]).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([2.0000, 2.0000, 2.0000,  ..., 2.0000, 2.0000, 2.0000])"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds[:,1:].sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df = pd.DataFrame(preds).sort_values(0).set_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.012029</td>\n",
       "      <td>0.088683</td>\n",
       "      <td>0.011687</td>\n",
       "      <td>0.006054</td>\n",
       "      <td>0.118311</td>\n",
       "      <td>0.131774</td>\n",
       "      <td>0.054826</td>\n",
       "      <td>0.006914</td>\n",
       "      <td>0.569722</td>\n",
       "      <td>0.062617</td>\n",
       "      <td>0.313355</td>\n",
       "      <td>0.248413</td>\n",
       "      <td>0.226336</td>\n",
       "      <td>0.020531</td>\n",
       "      <td>0.110596</td>\n",
       "      <td>0.003042</td>\n",
       "      <td>0.002350</td>\n",
       "      <td>0.002812</td>\n",
       "      <td>0.002456</td>\n",
       "      <td>0.007490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.004837</td>\n",
       "      <td>0.028246</td>\n",
       "      <td>0.002863</td>\n",
       "      <td>0.001764</td>\n",
       "      <td>0.034731</td>\n",
       "      <td>0.046041</td>\n",
       "      <td>0.005025</td>\n",
       "      <td>0.001421</td>\n",
       "      <td>0.875072</td>\n",
       "      <td>0.137637</td>\n",
       "      <td>0.398767</td>\n",
       "      <td>0.201193</td>\n",
       "      <td>0.121985</td>\n",
       "      <td>0.012093</td>\n",
       "      <td>0.108119</td>\n",
       "      <td>0.003789</td>\n",
       "      <td>0.002976</td>\n",
       "      <td>0.002760</td>\n",
       "      <td>0.002394</td>\n",
       "      <td>0.008287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.036089</td>\n",
       "      <td>0.078575</td>\n",
       "      <td>0.031075</td>\n",
       "      <td>0.019435</td>\n",
       "      <td>0.185393</td>\n",
       "      <td>0.265998</td>\n",
       "      <td>0.122452</td>\n",
       "      <td>0.022095</td>\n",
       "      <td>0.238887</td>\n",
       "      <td>0.138922</td>\n",
       "      <td>0.255579</td>\n",
       "      <td>0.274190</td>\n",
       "      <td>0.166238</td>\n",
       "      <td>0.014719</td>\n",
       "      <td>0.129615</td>\n",
       "      <td>0.003438</td>\n",
       "      <td>0.002693</td>\n",
       "      <td>0.002419</td>\n",
       "      <td>0.003353</td>\n",
       "      <td>0.008832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.051361</td>\n",
       "      <td>0.150938</td>\n",
       "      <td>0.045759</td>\n",
       "      <td>0.013143</td>\n",
       "      <td>0.010101</td>\n",
       "      <td>0.123450</td>\n",
       "      <td>0.538207</td>\n",
       "      <td>0.053582</td>\n",
       "      <td>0.013460</td>\n",
       "      <td>0.058045</td>\n",
       "      <td>0.042164</td>\n",
       "      <td>0.138819</td>\n",
       "      <td>0.068568</td>\n",
       "      <td>0.021752</td>\n",
       "      <td>0.636689</td>\n",
       "      <td>0.009049</td>\n",
       "      <td>0.003480</td>\n",
       "      <td>0.003555</td>\n",
       "      <td>0.005968</td>\n",
       "      <td>0.011910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.180169</td>\n",
       "      <td>0.345325</td>\n",
       "      <td>0.006466</td>\n",
       "      <td>0.055198</td>\n",
       "      <td>0.008491</td>\n",
       "      <td>0.354537</td>\n",
       "      <td>0.034503</td>\n",
       "      <td>0.006942</td>\n",
       "      <td>0.008367</td>\n",
       "      <td>0.202020</td>\n",
       "      <td>0.097755</td>\n",
       "      <td>0.087355</td>\n",
       "      <td>0.212235</td>\n",
       "      <td>0.012933</td>\n",
       "      <td>0.368726</td>\n",
       "      <td>0.003895</td>\n",
       "      <td>0.002591</td>\n",
       "      <td>0.001858</td>\n",
       "      <td>0.002564</td>\n",
       "      <td>0.008069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2946.0</th>\n",
       "      <td>0.017616</td>\n",
       "      <td>0.439280</td>\n",
       "      <td>0.135162</td>\n",
       "      <td>0.017472</td>\n",
       "      <td>0.004883</td>\n",
       "      <td>0.022320</td>\n",
       "      <td>0.309937</td>\n",
       "      <td>0.038139</td>\n",
       "      <td>0.015191</td>\n",
       "      <td>0.340245</td>\n",
       "      <td>0.229997</td>\n",
       "      <td>0.144590</td>\n",
       "      <td>0.165742</td>\n",
       "      <td>0.018018</td>\n",
       "      <td>0.067797</td>\n",
       "      <td>0.006549</td>\n",
       "      <td>0.004812</td>\n",
       "      <td>0.004153</td>\n",
       "      <td>0.006711</td>\n",
       "      <td>0.011385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2947.0</th>\n",
       "      <td>0.002321</td>\n",
       "      <td>0.031558</td>\n",
       "      <td>0.256148</td>\n",
       "      <td>0.008119</td>\n",
       "      <td>0.001889</td>\n",
       "      <td>0.003627</td>\n",
       "      <td>0.648931</td>\n",
       "      <td>0.043604</td>\n",
       "      <td>0.003803</td>\n",
       "      <td>0.293560</td>\n",
       "      <td>0.207870</td>\n",
       "      <td>0.170516</td>\n",
       "      <td>0.201816</td>\n",
       "      <td>0.020935</td>\n",
       "      <td>0.074092</td>\n",
       "      <td>0.006173</td>\n",
       "      <td>0.003816</td>\n",
       "      <td>0.003362</td>\n",
       "      <td>0.006331</td>\n",
       "      <td>0.011528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2948.0</th>\n",
       "      <td>0.046939</td>\n",
       "      <td>0.525092</td>\n",
       "      <td>0.069220</td>\n",
       "      <td>0.015932</td>\n",
       "      <td>0.013705</td>\n",
       "      <td>0.025892</td>\n",
       "      <td>0.262478</td>\n",
       "      <td>0.031094</td>\n",
       "      <td>0.009648</td>\n",
       "      <td>0.323923</td>\n",
       "      <td>0.192555</td>\n",
       "      <td>0.196003</td>\n",
       "      <td>0.186708</td>\n",
       "      <td>0.020813</td>\n",
       "      <td>0.040270</td>\n",
       "      <td>0.007749</td>\n",
       "      <td>0.006391</td>\n",
       "      <td>0.005237</td>\n",
       "      <td>0.008875</td>\n",
       "      <td>0.011475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2949.0</th>\n",
       "      <td>0.308479</td>\n",
       "      <td>0.270680</td>\n",
       "      <td>0.056931</td>\n",
       "      <td>0.021725</td>\n",
       "      <td>0.006670</td>\n",
       "      <td>0.272194</td>\n",
       "      <td>0.026415</td>\n",
       "      <td>0.026180</td>\n",
       "      <td>0.010726</td>\n",
       "      <td>0.425284</td>\n",
       "      <td>0.223206</td>\n",
       "      <td>0.119197</td>\n",
       "      <td>0.161414</td>\n",
       "      <td>0.009501</td>\n",
       "      <td>0.035536</td>\n",
       "      <td>0.004139</td>\n",
       "      <td>0.003767</td>\n",
       "      <td>0.003120</td>\n",
       "      <td>0.004360</td>\n",
       "      <td>0.010476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2950.0</th>\n",
       "      <td>0.028779</td>\n",
       "      <td>0.728906</td>\n",
       "      <td>0.033079</td>\n",
       "      <td>0.020635</td>\n",
       "      <td>0.008293</td>\n",
       "      <td>0.123867</td>\n",
       "      <td>0.036656</td>\n",
       "      <td>0.008795</td>\n",
       "      <td>0.010990</td>\n",
       "      <td>0.318885</td>\n",
       "      <td>0.175638</td>\n",
       "      <td>0.215040</td>\n",
       "      <td>0.193284</td>\n",
       "      <td>0.017065</td>\n",
       "      <td>0.042960</td>\n",
       "      <td>0.006359</td>\n",
       "      <td>0.007399</td>\n",
       "      <td>0.004439</td>\n",
       "      <td>0.007931</td>\n",
       "      <td>0.011001</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2951 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              1         2         3         4         5         6         7   \\\n",
       "0                                                                              \n",
       "0.0     0.012029  0.088683  0.011687  0.006054  0.118311  0.131774  0.054826   \n",
       "1.0     0.004837  0.028246  0.002863  0.001764  0.034731  0.046041  0.005025   \n",
       "2.0     0.036089  0.078575  0.031075  0.019435  0.185393  0.265998  0.122452   \n",
       "3.0     0.051361  0.150938  0.045759  0.013143  0.010101  0.123450  0.538207   \n",
       "4.0     0.180169  0.345325  0.006466  0.055198  0.008491  0.354537  0.034503   \n",
       "...          ...       ...       ...       ...       ...       ...       ...   \n",
       "2946.0  0.017616  0.439280  0.135162  0.017472  0.004883  0.022320  0.309937   \n",
       "2947.0  0.002321  0.031558  0.256148  0.008119  0.001889  0.003627  0.648931   \n",
       "2948.0  0.046939  0.525092  0.069220  0.015932  0.013705  0.025892  0.262478   \n",
       "2949.0  0.308479  0.270680  0.056931  0.021725  0.006670  0.272194  0.026415   \n",
       "2950.0  0.028779  0.728906  0.033079  0.020635  0.008293  0.123867  0.036656   \n",
       "\n",
       "              8         9         10        11        12        13        14  \\\n",
       "0                                                                              \n",
       "0.0     0.006914  0.569722  0.062617  0.313355  0.248413  0.226336  0.020531   \n",
       "1.0     0.001421  0.875072  0.137637  0.398767  0.201193  0.121985  0.012093   \n",
       "2.0     0.022095  0.238887  0.138922  0.255579  0.274190  0.166238  0.014719   \n",
       "3.0     0.053582  0.013460  0.058045  0.042164  0.138819  0.068568  0.021752   \n",
       "4.0     0.006942  0.008367  0.202020  0.097755  0.087355  0.212235  0.012933   \n",
       "...          ...       ...       ...       ...       ...       ...       ...   \n",
       "2946.0  0.038139  0.015191  0.340245  0.229997  0.144590  0.165742  0.018018   \n",
       "2947.0  0.043604  0.003803  0.293560  0.207870  0.170516  0.201816  0.020935   \n",
       "2948.0  0.031094  0.009648  0.323923  0.192555  0.196003  0.186708  0.020813   \n",
       "2949.0  0.026180  0.010726  0.425284  0.223206  0.119197  0.161414  0.009501   \n",
       "2950.0  0.008795  0.010990  0.318885  0.175638  0.215040  0.193284  0.017065   \n",
       "\n",
       "              15        16        17        18        19        20  \n",
       "0                                                                   \n",
       "0.0     0.110596  0.003042  0.002350  0.002812  0.002456  0.007490  \n",
       "1.0     0.108119  0.003789  0.002976  0.002760  0.002394  0.008287  \n",
       "2.0     0.129615  0.003438  0.002693  0.002419  0.003353  0.008832  \n",
       "3.0     0.636689  0.009049  0.003480  0.003555  0.005968  0.011910  \n",
       "4.0     0.368726  0.003895  0.002591  0.001858  0.002564  0.008069  \n",
       "...          ...       ...       ...       ...       ...       ...  \n",
       "2946.0  0.067797  0.006549  0.004812  0.004153  0.006711  0.011385  \n",
       "2947.0  0.074092  0.006173  0.003816  0.003362  0.006331  0.011528  \n",
       "2948.0  0.040270  0.007749  0.006391  0.005237  0.008875  0.011475  \n",
       "2949.0  0.035536  0.004139  0.003767  0.003120  0.004360  0.010476  \n",
       "2950.0  0.042960  0.006359  0.007399  0.004439  0.007931  0.011001  \n",
       "\n",
       "[2951 rows x 20 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df.to_csv(args.save_dir + 'preds.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "states.to_csv(args.save_dir + 'train_state.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Per-Class Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.012029</td>\n",
       "      <td>0.088683</td>\n",
       "      <td>0.011687</td>\n",
       "      <td>0.006054</td>\n",
       "      <td>0.118311</td>\n",
       "      <td>0.131774</td>\n",
       "      <td>0.054826</td>\n",
       "      <td>0.006914</td>\n",
       "      <td>0.569722</td>\n",
       "      <td>0.062617</td>\n",
       "      <td>0.313355</td>\n",
       "      <td>0.248413</td>\n",
       "      <td>0.226336</td>\n",
       "      <td>0.020531</td>\n",
       "      <td>0.110596</td>\n",
       "      <td>0.003042</td>\n",
       "      <td>0.002350</td>\n",
       "      <td>0.002812</td>\n",
       "      <td>0.002456</td>\n",
       "      <td>0.007490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.004837</td>\n",
       "      <td>0.028246</td>\n",
       "      <td>0.002863</td>\n",
       "      <td>0.001764</td>\n",
       "      <td>0.034731</td>\n",
       "      <td>0.046041</td>\n",
       "      <td>0.005025</td>\n",
       "      <td>0.001421</td>\n",
       "      <td>0.875072</td>\n",
       "      <td>0.137637</td>\n",
       "      <td>0.398767</td>\n",
       "      <td>0.201193</td>\n",
       "      <td>0.121985</td>\n",
       "      <td>0.012093</td>\n",
       "      <td>0.108119</td>\n",
       "      <td>0.003789</td>\n",
       "      <td>0.002976</td>\n",
       "      <td>0.002760</td>\n",
       "      <td>0.002394</td>\n",
       "      <td>0.008287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.036089</td>\n",
       "      <td>0.078575</td>\n",
       "      <td>0.031075</td>\n",
       "      <td>0.019435</td>\n",
       "      <td>0.185393</td>\n",
       "      <td>0.265998</td>\n",
       "      <td>0.122452</td>\n",
       "      <td>0.022095</td>\n",
       "      <td>0.238887</td>\n",
       "      <td>0.138923</td>\n",
       "      <td>0.255579</td>\n",
       "      <td>0.274190</td>\n",
       "      <td>0.166238</td>\n",
       "      <td>0.014719</td>\n",
       "      <td>0.129615</td>\n",
       "      <td>0.003438</td>\n",
       "      <td>0.002693</td>\n",
       "      <td>0.002419</td>\n",
       "      <td>0.003353</td>\n",
       "      <td>0.008832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.051361</td>\n",
       "      <td>0.150938</td>\n",
       "      <td>0.045759</td>\n",
       "      <td>0.013143</td>\n",
       "      <td>0.010101</td>\n",
       "      <td>0.123450</td>\n",
       "      <td>0.538207</td>\n",
       "      <td>0.053582</td>\n",
       "      <td>0.013460</td>\n",
       "      <td>0.058045</td>\n",
       "      <td>0.042164</td>\n",
       "      <td>0.138819</td>\n",
       "      <td>0.068568</td>\n",
       "      <td>0.021752</td>\n",
       "      <td>0.636689</td>\n",
       "      <td>0.009049</td>\n",
       "      <td>0.003480</td>\n",
       "      <td>0.003555</td>\n",
       "      <td>0.005968</td>\n",
       "      <td>0.011910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.180170</td>\n",
       "      <td>0.345325</td>\n",
       "      <td>0.006466</td>\n",
       "      <td>0.055198</td>\n",
       "      <td>0.008491</td>\n",
       "      <td>0.354537</td>\n",
       "      <td>0.034503</td>\n",
       "      <td>0.006942</td>\n",
       "      <td>0.008367</td>\n",
       "      <td>0.202020</td>\n",
       "      <td>0.097755</td>\n",
       "      <td>0.087355</td>\n",
       "      <td>0.212235</td>\n",
       "      <td>0.012933</td>\n",
       "      <td>0.368726</td>\n",
       "      <td>0.003895</td>\n",
       "      <td>0.002591</td>\n",
       "      <td>0.001858</td>\n",
       "      <td>0.002564</td>\n",
       "      <td>0.008069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2946.0</th>\n",
       "      <td>0.017616</td>\n",
       "      <td>0.439280</td>\n",
       "      <td>0.135162</td>\n",
       "      <td>0.017472</td>\n",
       "      <td>0.004883</td>\n",
       "      <td>0.022320</td>\n",
       "      <td>0.309937</td>\n",
       "      <td>0.038139</td>\n",
       "      <td>0.015191</td>\n",
       "      <td>0.340245</td>\n",
       "      <td>0.229997</td>\n",
       "      <td>0.144590</td>\n",
       "      <td>0.165742</td>\n",
       "      <td>0.018018</td>\n",
       "      <td>0.067797</td>\n",
       "      <td>0.006549</td>\n",
       "      <td>0.004812</td>\n",
       "      <td>0.004153</td>\n",
       "      <td>0.006711</td>\n",
       "      <td>0.011385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2947.0</th>\n",
       "      <td>0.002321</td>\n",
       "      <td>0.031558</td>\n",
       "      <td>0.256148</td>\n",
       "      <td>0.008119</td>\n",
       "      <td>0.001889</td>\n",
       "      <td>0.003627</td>\n",
       "      <td>0.648931</td>\n",
       "      <td>0.043604</td>\n",
       "      <td>0.003803</td>\n",
       "      <td>0.293560</td>\n",
       "      <td>0.207870</td>\n",
       "      <td>0.170516</td>\n",
       "      <td>0.201816</td>\n",
       "      <td>0.020935</td>\n",
       "      <td>0.074092</td>\n",
       "      <td>0.006173</td>\n",
       "      <td>0.003816</td>\n",
       "      <td>0.003362</td>\n",
       "      <td>0.006331</td>\n",
       "      <td>0.011528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2948.0</th>\n",
       "      <td>0.046939</td>\n",
       "      <td>0.525092</td>\n",
       "      <td>0.069220</td>\n",
       "      <td>0.015932</td>\n",
       "      <td>0.013705</td>\n",
       "      <td>0.025892</td>\n",
       "      <td>0.262478</td>\n",
       "      <td>0.031094</td>\n",
       "      <td>0.009648</td>\n",
       "      <td>0.323923</td>\n",
       "      <td>0.192555</td>\n",
       "      <td>0.196003</td>\n",
       "      <td>0.186708</td>\n",
       "      <td>0.020813</td>\n",
       "      <td>0.040270</td>\n",
       "      <td>0.007749</td>\n",
       "      <td>0.006391</td>\n",
       "      <td>0.005237</td>\n",
       "      <td>0.008875</td>\n",
       "      <td>0.011475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2949.0</th>\n",
       "      <td>0.308479</td>\n",
       "      <td>0.270680</td>\n",
       "      <td>0.056931</td>\n",
       "      <td>0.021725</td>\n",
       "      <td>0.006670</td>\n",
       "      <td>0.272194</td>\n",
       "      <td>0.026415</td>\n",
       "      <td>0.026180</td>\n",
       "      <td>0.010726</td>\n",
       "      <td>0.425284</td>\n",
       "      <td>0.223206</td>\n",
       "      <td>0.119197</td>\n",
       "      <td>0.161414</td>\n",
       "      <td>0.009501</td>\n",
       "      <td>0.035536</td>\n",
       "      <td>0.004139</td>\n",
       "      <td>0.003767</td>\n",
       "      <td>0.003120</td>\n",
       "      <td>0.004360</td>\n",
       "      <td>0.010476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2950.0</th>\n",
       "      <td>0.028779</td>\n",
       "      <td>0.728906</td>\n",
       "      <td>0.033079</td>\n",
       "      <td>0.020635</td>\n",
       "      <td>0.008293</td>\n",
       "      <td>0.123867</td>\n",
       "      <td>0.036656</td>\n",
       "      <td>0.008795</td>\n",
       "      <td>0.010990</td>\n",
       "      <td>0.318885</td>\n",
       "      <td>0.175638</td>\n",
       "      <td>0.215040</td>\n",
       "      <td>0.193284</td>\n",
       "      <td>0.017065</td>\n",
       "      <td>0.042960</td>\n",
       "      <td>0.006359</td>\n",
       "      <td>0.007399</td>\n",
       "      <td>0.004439</td>\n",
       "      <td>0.007931</td>\n",
       "      <td>0.011001</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2951 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               1         2         3         4         5         6         7  \\\n",
       "0                                                                              \n",
       "0.0     0.012029  0.088683  0.011687  0.006054  0.118311  0.131774  0.054826   \n",
       "1.0     0.004837  0.028246  0.002863  0.001764  0.034731  0.046041  0.005025   \n",
       "2.0     0.036089  0.078575  0.031075  0.019435  0.185393  0.265998  0.122452   \n",
       "3.0     0.051361  0.150938  0.045759  0.013143  0.010101  0.123450  0.538207   \n",
       "4.0     0.180170  0.345325  0.006466  0.055198  0.008491  0.354537  0.034503   \n",
       "...          ...       ...       ...       ...       ...       ...       ...   \n",
       "2946.0  0.017616  0.439280  0.135162  0.017472  0.004883  0.022320  0.309937   \n",
       "2947.0  0.002321  0.031558  0.256148  0.008119  0.001889  0.003627  0.648931   \n",
       "2948.0  0.046939  0.525092  0.069220  0.015932  0.013705  0.025892  0.262478   \n",
       "2949.0  0.308479  0.270680  0.056931  0.021725  0.006670  0.272194  0.026415   \n",
       "2950.0  0.028779  0.728906  0.033079  0.020635  0.008293  0.123867  0.036656   \n",
       "\n",
       "               8         9        10        11        12        13        14  \\\n",
       "0                                                                              \n",
       "0.0     0.006914  0.569722  0.062617  0.313355  0.248413  0.226336  0.020531   \n",
       "1.0     0.001421  0.875072  0.137637  0.398767  0.201193  0.121985  0.012093   \n",
       "2.0     0.022095  0.238887  0.138923  0.255579  0.274190  0.166238  0.014719   \n",
       "3.0     0.053582  0.013460  0.058045  0.042164  0.138819  0.068568  0.021752   \n",
       "4.0     0.006942  0.008367  0.202020  0.097755  0.087355  0.212235  0.012933   \n",
       "...          ...       ...       ...       ...       ...       ...       ...   \n",
       "2946.0  0.038139  0.015191  0.340245  0.229997  0.144590  0.165742  0.018018   \n",
       "2947.0  0.043604  0.003803  0.293560  0.207870  0.170516  0.201816  0.020935   \n",
       "2948.0  0.031094  0.009648  0.323923  0.192555  0.196003  0.186708  0.020813   \n",
       "2949.0  0.026180  0.010726  0.425284  0.223206  0.119197  0.161414  0.009501   \n",
       "2950.0  0.008795  0.010990  0.318885  0.175638  0.215040  0.193284  0.017065   \n",
       "\n",
       "              15        16        17        18        19        20  \n",
       "0                                                                   \n",
       "0.0     0.110596  0.003042  0.002350  0.002812  0.002456  0.007490  \n",
       "1.0     0.108119  0.003789  0.002976  0.002760  0.002394  0.008287  \n",
       "2.0     0.129615  0.003438  0.002693  0.002419  0.003353  0.008832  \n",
       "3.0     0.636689  0.009049  0.003480  0.003555  0.005968  0.011910  \n",
       "4.0     0.368726  0.003895  0.002591  0.001858  0.002564  0.008069  \n",
       "...          ...       ...       ...       ...       ...       ...  \n",
       "2946.0  0.067797  0.006549  0.004812  0.004153  0.006711  0.011385  \n",
       "2947.0  0.074092  0.006173  0.003816  0.003362  0.006331  0.011528  \n",
       "2948.0  0.040270  0.007749  0.006391  0.005237  0.008875  0.011475  \n",
       "2949.0  0.035536  0.004139  0.003767  0.003120  0.004360  0.010476  \n",
       "2950.0  0.042960  0.006359  0.007399  0.004439  0.007931  0.011001  \n",
       "\n",
       "[2951 rows x 20 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = pd.read_csv(args.save_dir + 'preds.csv', sep='\\t', index_col='0')\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = torch.tensor(np.array(preds)).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_k_confusion_matrix(y, y_pred, k=3):\n",
    "    dim = y.shape[-1]\n",
    "    y = y.topk(k=k, axis=1)[1]\n",
    "    y_pred = y_pred.topk(k=k, axis=1)[1]\n",
    "    conf = np.zeros((dim, dim))\n",
    "    for i in range(k):\n",
    "        for j in range(k):\n",
    "            conf = np.add(conf, confusion_matrix(y[:,i], y_pred[:,j], labels = range(dim)))\n",
    "    return conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ATT_conf = confusion_matrix(data['all'].y[(data['all'].att_lab) * data['all'].test_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                                 pred[(data['all'].att_lab) * data['all'].test_mask][:,:9].argmax(axis=1).cpu(), labels = range(9))\n",
    "test_VAL_conf = confusion_matrix(data['all'].y[(data['all'].val_lab) * data['all'].test_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                                 pred[(data['all'].val_lab) * data['all'].test_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "test_VAL_conf_k = (top_k_confusion_matrix(data['all'].y[(data['all'].val_lab) * data['all'].test_mask][:,9:].cpu(),  \n",
    "                                 pred[(data['all'].val_lab) * data['all'].test_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 64,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  1, 142,   0,   0,   0,   1,   1,   0,   0],\n",
       "       [  0,   0,  10,   0,   0,   0,   0,   1,   0],\n",
       "       [  1,   0,   0,  16,   0,   1,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,  70,   0,   2,   0,   0],\n",
       "       [  0,   2,   0,   0,   0, 100,   0,   0,   0],\n",
       "       [  0,   3,   0,   0,   3,   0,  71,   0,   0],\n",
       "       [  0,   0,   2,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,  12]], dtype=int64)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[55,  2,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 5, 14,  3,  1,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 2,  6, 21,  2,  0,  4,  0,  0,  0,  0,  0],\n",
       "       [ 1,  3,  2, 25,  0,  1,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 2,  2,  2,  3,  0, 35,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  1,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0]], dtype=int64)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 91,  60,  41,  46,   0,  44,   0,   0,   0,   0,   0],\n",
       "       [ 49,  87,  70,  59,   2,  42,   0,   0,   0,   0,   0],\n",
       "       [ 55,  82, 106,  90,   1,  74,   0,   0,   0,   0,   0],\n",
       "       [ 56,  80,  79, 118,   2,  64,   0,   0,   0,   0,   0],\n",
       "       [  0,   3,   2,   3,   1,   0,   0,   0,   0,   0,   0],\n",
       "       [ 45,  42,  70,  68,   0,  84,   0,   0,   0,   0,   0],\n",
       "       [  1,   0,   1,   0,   0,   2,   1,   0,   0,   1,   0],\n",
       "       [  0,   0,   0,   0,   0,   1,   1,   0,   0,   1,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   1,   1,   0,   0,   1,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0]])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_ATT_conf = confusion_matrix(data['all'].y[(data['all'].att_lab) * data['all'].val_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                                 pred[(data['all'].att_lab) * data['all'].val_mask][:,:9].argmax(axis=1).cpu(), labels = range(9))\n",
    "val_VAL_conf = confusion_matrix(data['all'].y[(data['all'].val_lab) * data['all'].val_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                                 pred[(data['all'].val_lab) * data['all'].val_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "val_VAL_conf_k = (top_k_confusion_matrix(data['all'].y[(data['all'].val_lab) * data['all'].val_mask][:,9:].cpu(),  \n",
    "                                 pred[(data['all'].val_lab) * data['all'].val_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 61,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0, 144,   0,   0,   0,   1,   1,   0,   0],\n",
       "       [  0,   0,   8,   0,   0,   0,   0,   0,   0],\n",
       "       [  1,   1,   0,  20,   0,   0,   0,   0,   0],\n",
       "       [  0,   1,   0,   0,  73,   0,   1,   0,   0],\n",
       "       [  0,   0,   0,   0,   0, 103,   0,   0,   0],\n",
       "       [  0,   2,   0,   0,   0,   0,  60,   0,   0],\n",
       "       [  0,   0,   2,   0,   0,   0,   2,   1,   0],\n",
       "       [  0,   1,   0,   0,   2,   0,   0,   0,   7]], dtype=int64)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[64,  3,  1,  0,  0,  1,  0,  0,  0,  0,  0],\n",
       "       [ 2,  9,  4,  3,  0,  1,  0,  0,  0,  0,  0],\n",
       "       [ 2,  0, 21,  2,  0,  6,  0,  0,  0,  0,  0],\n",
       "       [ 2,  1,  0, 30,  0,  1,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 4,  4,  4,  1,  0, 37,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0]], dtype=int64)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[106,  66,  50,  55,   0,  47,   0,   0,   0,   0,   0],\n",
       "       [ 60,  85,  64,  69,   3,  40,   0,   0,   0,   0,   0],\n",
       "       [ 57,  60, 111,  90,   0,  75,   0,   0,   0,   0,   0],\n",
       "       [ 69,  82,  93, 118,   3,  61,   0,   0,   0,   0,   0],\n",
       "       [  0,   8,   5,   8,   3,   0,   0,   0,   0,   0,   0],\n",
       "       [ 53,  35,  91,  65,   0,  95,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0]])"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat([pd.DataFrame(val_ATT_conf),pd.DataFrame(test_ATT_conf)],axis=1).to_csv(args.save_dir+'confusion_matrix_ATT.csv')\n",
    "pd.concat([pd.DataFrame(val_VAL_conf),pd.DataFrame(test_VAL_conf)],axis=1).to_csv(args.save_dir+'confusion_matrix_VAL.csv')\n",
    "pd.concat([pd.DataFrame(val_VAL_conf_k),pd.DataFrame(test_VAL_conf_k)],axis=1).to_csv(args.save_dir+'confusion_matrix_VAL_k.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics(confusion_matrix, classes):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val/(row.sum()+0.000000001)\n",
    "        recall = val/(col.sum()+0.000000001)\n",
    "        F1 = 2*(precision*recall)/(precision+recall+0.000000001)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics_k(confusion_matrix, classes, k=3):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val*k/(row.sum()+0.000000001)\n",
    "        recall = val*k/(col.sum()+0.000000001)\n",
    "        F1 = 2*(precision*recall)/(precision+recall+0.000000001)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = ['Criterion i', 'Criterion ii', 'Criterion iii', 'Criterion iv', 'Criterion v', 'Criterion vi', \n",
    "              'Criterion vii', 'Criterion viii', 'Criterion ix', 'Criterion x', 'Others']\n",
    "categories = ['Building Elements',\n",
    " 'Urban Form Elements',\n",
    " 'Gastronomy',\n",
    " 'Interior Scenery',\n",
    " 'Natural Features and Land-scape Scenery',\n",
    " 'Monuments and Buildings',\n",
    " 'Peoples Activity and Association',\n",
    " 'Artifact Products',\n",
    " 'Urban Scenery']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_dict = {}\n",
    "metrics_dict['test_ATT'] = per_class_metrics(test_ATT_conf, categories)\n",
    "metrics_dict['val_ATT'] = per_class_metrics(val_ATT_conf, categories)\n",
    "metrics_dict['test_VAL'] = per_class_metrics(test_VAL_conf, classes)\n",
    "metrics_dict['val_VAL'] = per_class_metrics(val_VAL_conf, classes)\n",
    "metrics_dict['test_VAL_k'] = per_class_metrics_k(test_VAL_conf_k, classes)\n",
    "metrics_dict['val_VAL_k'] = per_class_metrics_k(val_VAL_conf_k, classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df = pd.DataFrame.from_dict({(i,j): metrics_dict[i][j] \n",
    "                           for i in metrics_dict.keys() \n",
    "                           for j in metrics_dict[i].keys()},\n",
    "                       orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df.to_csv(args.save_dir+'per_class_metrics.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TEM links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [3 *args.sample_nodes] * 2 for key in data.edge_types if 'TEM_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].train_mask),\n",
    ")\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [3* args.sample_nodes] * 2 for key in data.edge_types if 'TEM_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].val_mask),\n",
    ")\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [3* args.sample_nodes] * 2 for key in data.edge_types if 'TEM_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=1027,\n",
       "    x=[1027, 1753],\n",
       "    y=[1027, 20],\n",
       "    node_type=[1027],\n",
       "    att_lab=[1027],\n",
       "    val_lab=[1027],\n",
       "    train_mask=[1027],\n",
       "    val_mask=[1027],\n",
       "    test_mask=[1027],\n",
       "    n_id=[1027],\n",
       "    batch_size=32\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 51708],\n",
       "    edge_attr=[51708]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_hetero_data = next(iter(train_loader))\n",
    "batch = sampled_hetero_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HGT_L(\n",
       "  (lin_dict): ModuleDict(\n",
       "    (all): Linear(1753, 32, bias=True)\n",
       "  )\n",
       "  (convs): ModuleList(\n",
       "    (0): HGTConv(-1, 32, heads=2)\n",
       "    (1): HGTConv(-1, 32, heads=2)\n",
       "    (2): HGTConv(-1, 32, heads=2)\n",
       "  )\n",
       "  (lin1): Linear(1753, 32, bias=True)\n",
       "  (lin2): Linear(64, 20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:04<00:00,  2.85it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7255107517717948,\n",
       " 1.5993303985146605,\n",
       " 100.0,\n",
       " 100.0,\n",
       " 0.8716528250570112,\n",
       " 89.75069252077563)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 11.07it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8410670141863629,\n",
       " 1.6250746361727786,\n",
       " 96.95121951219512,\n",
       " 98.0295566502463,\n",
       " 0.7348111721095193,\n",
       " 79.3103448275862)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, val_loader,'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 43.75it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8529867110385099,\n",
       " 1.6444494693229597,\n",
       " 96.22266401590457,\n",
       " 99.47916666666667,\n",
       " 0.7317708432674408,\n",
       " 78.125)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, test_loader,'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 48.79it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 77.44it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 77.05it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 76.06it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 80.16it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 75.23it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 80.19it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 75.14it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 78.71it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 78.16it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 80.12it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 73.00it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 78.32it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 75.60it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 77.84it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 78.08it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 78.04it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 75.15it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 75.52it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 77.39it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test(model, val_loader,'val'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test(model, test_loader,'val'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>8.410675e-01</td>\n",
       "      <td>1.625075e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>6.327162e-07</td>\n",
       "      <td>9.332467e-08</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.497956e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>8.410664e-01</td>\n",
       "      <td>1.625075e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>8.410670e-01</td>\n",
       "      <td>1.625075e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>8.410676e-01</td>\n",
       "      <td>1.625075e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>8.410680e-01</td>\n",
       "      <td>1.625075e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>8.410685e-01</td>\n",
       "      <td>1.625075e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ATT_loss      VAL_loss       ATT_acc     VAL_k_acc  VAL_k_jac  \\\n",
       "count  1.000000e+01  1.000000e+01  1.000000e+01  1.000000e+01  10.000000   \n",
       "mean   8.410675e-01  1.625075e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "std    6.327162e-07  9.332467e-08  1.497956e-14  1.497956e-14   0.000000   \n",
       "min    8.410664e-01  1.625075e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "25%    8.410670e-01  1.625075e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "50%    8.410676e-01  1.625075e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "75%    8.410680e-01  1.625075e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "max    8.410685e-01  1.625075e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "\n",
       "          VAL_1_acc  \n",
       "count  1.000000e+01  \n",
       "mean   7.931034e+01  \n",
       "std    1.497956e-14  \n",
       "min    7.931034e+01  \n",
       "25%    7.931034e+01  \n",
       "50%    7.931034e+01  \n",
       "75%    7.931034e+01  \n",
       "max    7.931034e+01  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>8.529866e-01</td>\n",
       "      <td>1.644450e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>78.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>5.912551e-07</td>\n",
       "      <td>3.124121e-07</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>8.529856e-01</td>\n",
       "      <td>1.644450e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>78.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>8.529863e-01</td>\n",
       "      <td>1.644450e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>78.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>8.529867e-01</td>\n",
       "      <td>1.644450e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>78.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>8.529870e-01</td>\n",
       "      <td>1.644450e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>78.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>8.529872e-01</td>\n",
       "      <td>1.644451e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>78.125</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ATT_loss      VAL_loss    ATT_acc     VAL_k_acc  VAL_k_jac  \\\n",
       "count  1.000000e+01  1.000000e+01  10.000000  1.000000e+01  10.000000   \n",
       "mean   8.529866e-01  1.644450e+00  96.222664  9.947917e+01   0.731771   \n",
       "std    5.912551e-07  3.124121e-07   0.000000  1.497956e-14   0.000000   \n",
       "min    8.529856e-01  1.644450e+00  96.222664  9.947917e+01   0.731771   \n",
       "25%    8.529863e-01  1.644450e+00  96.222664  9.947917e+01   0.731771   \n",
       "50%    8.529867e-01  1.644450e+00  96.222664  9.947917e+01   0.731771   \n",
       "75%    8.529870e-01  1.644450e+00  96.222664  9.947917e+01   0.731771   \n",
       "max    8.529872e-01  1.644451e+00  96.222664  9.947917e+01   0.731771   \n",
       "\n",
       "       VAL_1_acc  \n",
       "count     10.000  \n",
       "mean      78.125  \n",
       "std        0.000  \n",
       "min       78.125  \n",
       "25%       78.125  \n",
       "50%       78.125  \n",
       "75%       78.125  \n",
       "max       78.125  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'TEM_val_metrics_transfer.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'TEM_test_metrics_transfer.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SPA links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [3*args.sample_nodes] * 2 for key in data.edge_types if 'SPA_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].train_mask),\n",
    ")\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [3*args.sample_nodes] * 2 for key in data.edge_types if 'SPA_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].val_mask),\n",
    ")\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [3*args.sample_nodes] * 2 for key in data.edge_types if 'SPA_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=2163,\n",
       "    x=[2163, 1753],\n",
       "    y=[2163, 20],\n",
       "    node_type=[2163],\n",
       "    att_lab=[2163],\n",
       "    val_lab=[2163],\n",
       "    train_mask=[2163],\n",
       "    val_mask=[2163],\n",
       "    test_mask=[2163],\n",
       "    n_id=[2163],\n",
       "    batch_size=32\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 67657],\n",
       "    edge_attr=[67657]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_hetero_data = next(iter(train_loader))\n",
    "batch = sampled_hetero_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 26.16it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7255342998993364,\n",
       " 1.599371404528948,\n",
       " 100.0,\n",
       " 100.0,\n",
       " 0.8702677835057647,\n",
       " 90.02770083102493)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.33it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.841040769243628,\n",
       " 1.6250538796626877,\n",
       " 96.95121951219512,\n",
       " 98.0295566502463,\n",
       " 0.7348111721095193,\n",
       " 79.3103448275862)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, val_loader, 'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.18it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8529594320189171,\n",
       " 1.6444498679290216,\n",
       " 96.22266401590457,\n",
       " 99.47916666666667,\n",
       " 0.7317708432674408,\n",
       " 78.64583333333333)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, test_loader, 'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 59.11it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 58.22it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 59.99it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 58.87it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 52.54it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 59.97it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 57.42it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 54.17it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 56.17it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 55.43it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 61.15it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 60.45it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 55.51it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 54.51it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 59.73it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 59.81it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 60.60it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 60.41it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 60.69it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 58.59it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test(model, val_loader, 'val'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test(model, test_loader, 'val'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.841042</td>\n",
       "      <td>1.625054e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.735796</td>\n",
       "      <td>79.359606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000001</td>\n",
       "      <td>7.171497e-07</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.001272</td>\n",
       "      <td>0.155777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.841039</td>\n",
       "      <td>1.625053e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>79.310345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.841041</td>\n",
       "      <td>1.625054e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>79.310345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.841042</td>\n",
       "      <td>1.625054e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>79.310345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.841042</td>\n",
       "      <td>1.625055e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.737274</td>\n",
       "      <td>79.310345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.841044</td>\n",
       "      <td>1.625056e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.737274</td>\n",
       "      <td>79.802956</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss      VAL_loss       ATT_acc     VAL_k_acc  VAL_k_jac  \\\n",
       "count  10.000000  1.000000e+01  1.000000e+01  1.000000e+01  10.000000   \n",
       "mean    0.841042  1.625054e+00  9.695122e+01  9.802956e+01   0.735796   \n",
       "std     0.000001  7.171497e-07  1.497956e-14  1.497956e-14   0.001272   \n",
       "min     0.841039  1.625053e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "25%     0.841041  1.625054e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "50%     0.841042  1.625054e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "75%     0.841042  1.625055e+00  9.695122e+01  9.802956e+01   0.737274   \n",
       "max     0.841044  1.625056e+00  9.695122e+01  9.802956e+01   0.737274   \n",
       "\n",
       "       VAL_1_acc  \n",
       "count  10.000000  \n",
       "mean   79.359606  \n",
       "std     0.155777  \n",
       "min    79.310345  \n",
       "25%    79.310345  \n",
       "50%    79.310345  \n",
       "75%    79.310345  \n",
       "max    79.802956  "
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.852962</td>\n",
       "      <td>1.644450e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731944</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000002</td>\n",
       "      <td>8.763981e-07</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.000366</td>\n",
       "      <td>1.497956e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.852958</td>\n",
       "      <td>1.644448e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.852960</td>\n",
       "      <td>1.644449e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.852962</td>\n",
       "      <td>1.644450e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.852962</td>\n",
       "      <td>1.644450e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.852964</td>\n",
       "      <td>1.644451e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.732639</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss      VAL_loss    ATT_acc     VAL_k_acc  VAL_k_jac  \\\n",
       "count  10.000000  1.000000e+01  10.000000  1.000000e+01  10.000000   \n",
       "mean    0.852962  1.644450e+00  96.222664  9.947917e+01   0.731944   \n",
       "std     0.000002  8.763981e-07   0.000000  1.497956e-14   0.000366   \n",
       "min     0.852958  1.644448e+00  96.222664  9.947917e+01   0.731771   \n",
       "25%     0.852960  1.644449e+00  96.222664  9.947917e+01   0.731771   \n",
       "50%     0.852962  1.644450e+00  96.222664  9.947917e+01   0.731771   \n",
       "75%     0.852962  1.644450e+00  96.222664  9.947917e+01   0.731771   \n",
       "max     0.852964  1.644451e+00  96.222664  9.947917e+01   0.732639   \n",
       "\n",
       "          VAL_1_acc  \n",
       "count  1.000000e+01  \n",
       "mean   7.864583e+01  \n",
       "std    1.497956e-14  \n",
       "min    7.864583e+01  \n",
       "25%    7.864583e+01  \n",
       "50%    7.864583e+01  \n",
       "75%    7.864583e+01  \n",
       "max    7.864583e+01  "
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'SPA_val_metrics_transfer.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'SPA_test_metrics_transfer.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SOC links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [3*args.sample_nodes] * 2 for key in data.edge_types if 'SOC_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].train_mask),\n",
    ")\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [3*args.sample_nodes] * 2 for key in data.edge_types if 'SOC_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].val_mask),\n",
    ")\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [3*args.sample_nodes] * 2 for key in data.edge_types if 'SOC_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=1761,\n",
       "    x=[1761, 1753],\n",
       "    y=[1761, 20],\n",
       "    node_type=[1761],\n",
       "    att_lab=[1761],\n",
       "    val_lab=[1761],\n",
       "    train_mask=[1761],\n",
       "    val_mask=[1761],\n",
       "    test_mask=[1761],\n",
       "    n_id=[1761],\n",
       "    batch_size=32\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 47162],\n",
       "    edge_attr=[47162]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_hetero_data = next(iter(train_loader))\n",
    "batch = sampled_hetero_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HGT_L(\n",
       "  (lin_dict): ModuleDict(\n",
       "    (all): Linear(1753, 32, bias=True)\n",
       "  )\n",
       "  (convs): ModuleList(\n",
       "    (0): HGTConv(-1, 32, heads=2)\n",
       "    (1): HGTConv(-1, 32, heads=2)\n",
       "    (2): HGTConv(-1, 32, heads=2)\n",
       "  )\n",
       "  (lin1): Linear(1753, 32, bias=True)\n",
       "  (lin2): Linear(64, 20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 35.44it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7255760066396972,\n",
       " 1.599351929825759,\n",
       " 100.0,\n",
       " 100.0,\n",
       " 0.8702677835057647,\n",
       " 90.02770083102493)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 41.24it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8410872680142643,\n",
       " 1.6250287664347682,\n",
       " 96.95121951219512,\n",
       " 98.0295566502463,\n",
       " 0.7348111721095193,\n",
       " 79.80295566502463)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, val_loader, 'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 47.69it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8530556848698535,\n",
       " 1.6444088034331799,\n",
       " 96.22266401590457,\n",
       " 99.47916666666667,\n",
       " 0.7317708432674408,\n",
       " 78.125)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, test_loader, 'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 47.60it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 59.44it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 59.54it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 63.63it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 64.95it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 66.95it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 66.79it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 67.94it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 67.06it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 69.65it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 69.94it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 69.94it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 70.59it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 67.60it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 68.14it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 66.79it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 68.37it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 70.40it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 69.20it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 57.16it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test(model, val_loader, 'val'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test(model, test_loader, 'val'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>8.410878e-01</td>\n",
       "      <td>1.625029e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.980296e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>6.125743e-07</td>\n",
       "      <td>5.314976e-07</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.497956e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>8.410868e-01</td>\n",
       "      <td>1.625029e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.980296e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>8.410873e-01</td>\n",
       "      <td>1.625029e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.980296e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>8.410881e-01</td>\n",
       "      <td>1.625029e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.980296e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>8.410883e-01</td>\n",
       "      <td>1.625030e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.980296e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>8.410884e-01</td>\n",
       "      <td>1.625030e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.980296e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ATT_loss      VAL_loss       ATT_acc     VAL_k_acc  VAL_k_jac  \\\n",
       "count  1.000000e+01  1.000000e+01  1.000000e+01  1.000000e+01  10.000000   \n",
       "mean   8.410878e-01  1.625029e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "std    6.125743e-07  5.314976e-07  1.497956e-14  1.497956e-14   0.000000   \n",
       "min    8.410868e-01  1.625029e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "25%    8.410873e-01  1.625029e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "50%    8.410881e-01  1.625029e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "75%    8.410883e-01  1.625030e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "max    8.410884e-01  1.625030e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "\n",
       "          VAL_1_acc  \n",
       "count  1.000000e+01  \n",
       "mean   7.980296e+01  \n",
       "std    1.497956e-14  \n",
       "min    7.980296e+01  \n",
       "25%    7.980296e+01  \n",
       "50%    7.980296e+01  \n",
       "75%    7.980296e+01  \n",
       "max    7.980296e+01  "
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>8.530563e-01</td>\n",
       "      <td>1.644410e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>78.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>7.405013e-07</td>\n",
       "      <td>7.480199e-07</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>8.530550e-01</td>\n",
       "      <td>1.644408e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>78.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>8.530559e-01</td>\n",
       "      <td>1.644410e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>78.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>8.530565e-01</td>\n",
       "      <td>1.644410e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>78.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>8.530566e-01</td>\n",
       "      <td>1.644410e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>78.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>8.530573e-01</td>\n",
       "      <td>1.644411e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>78.125</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ATT_loss      VAL_loss    ATT_acc     VAL_k_acc  VAL_k_jac  \\\n",
       "count  1.000000e+01  1.000000e+01  10.000000  1.000000e+01  10.000000   \n",
       "mean   8.530563e-01  1.644410e+00  96.222664  9.947917e+01   0.731771   \n",
       "std    7.405013e-07  7.480199e-07   0.000000  1.497956e-14   0.000000   \n",
       "min    8.530550e-01  1.644408e+00  96.222664  9.947917e+01   0.731771   \n",
       "25%    8.530559e-01  1.644410e+00  96.222664  9.947917e+01   0.731771   \n",
       "50%    8.530565e-01  1.644410e+00  96.222664  9.947917e+01   0.731771   \n",
       "75%    8.530566e-01  1.644410e+00  96.222664  9.947917e+01   0.731771   \n",
       "max    8.530573e-01  1.644411e+00  96.222664  9.947917e+01   0.731771   \n",
       "\n",
       "       VAL_1_acc  \n",
       "count     10.000  \n",
       "mean      78.125  \n",
       "std        0.000  \n",
       "min       78.125  \n",
       "25%       78.125  \n",
       "50%       78.125  \n",
       "75%       78.125  \n",
       "max       78.125  "
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'SOC_val_metrics_transfer.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'SOC_test_metrics_transfer.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NO TEM links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 3 for key in data.edge_types if 'SOC_link' in key or 'SPA_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].train_mask),\n",
    ")\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 3 for key in data.edge_types if 'SOC_link' in key or 'SPA_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].val_mask),\n",
    ")\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 3 for key in data.edge_types if 'SOC_link' in key or 'SPA_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=2943,\n",
       "    x=[2943, 1753],\n",
       "    y=[2943, 20],\n",
       "    node_type=[2943],\n",
       "    att_lab=[2943],\n",
       "    val_lab=[2943],\n",
       "    train_mask=[2943],\n",
       "    val_mask=[2943],\n",
       "    test_mask=[2943],\n",
       "    n_id=[2943],\n",
       "    batch_size=32\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 63929],\n",
       "    edge_attr=[63929]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 68216],\n",
       "    edge_attr=[68216]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_hetero_data = next(iter(train_loader))\n",
    "batch = sampled_hetero_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HGT_L(\n",
       "  (lin_dict): ModuleDict(\n",
       "    (all): Linear(1753, 32, bias=True)\n",
       "  )\n",
       "  (convs): ModuleList(\n",
       "    (0): HGTConv(-1, 32, heads=2)\n",
       "    (1): HGTConv(-1, 32, heads=2)\n",
       "    (2): HGTConv(-1, 32, heads=2)\n",
       "  )\n",
       "  (lin1): Linear(1753, 32, bias=True)\n",
       "  (lin2): Linear(64, 20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 23.08it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7254844600143855,\n",
       " 1.5993549268991993,\n",
       " 100.0,\n",
       " 100.0,\n",
       " 0.872576187852347,\n",
       " 90.02770083102493)"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.05it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8409768699145899,\n",
       " 1.6250564453049834,\n",
       " 96.95121951219512,\n",
       " 98.0295566502463,\n",
       " 0.7348111721095193,\n",
       " 79.3103448275862)"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, val_loader, 'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.82it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.85284924305694,\n",
       " 1.6444347966462374,\n",
       " 96.22266401590457,\n",
       " 99.47916666666667,\n",
       " 0.7343750099341074,\n",
       " 78.64583333333333)"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, test_loader, 'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 32.69it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 34.40it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 35.11it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 33.81it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 35.79it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 34.77it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 34.78it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 34.98it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 36.00it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 35.29it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 34.44it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 36.76it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 36.61it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 35.72it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 35.89it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 35.46it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 36.75it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.29it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.17it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 34.11it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test(model, val_loader, 'val'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test(model, test_loader, 'val'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.840975</td>\n",
       "      <td>1.625058e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000002</td>\n",
       "      <td>8.522315e-07</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.497956e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.840972</td>\n",
       "      <td>1.625056e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.840973</td>\n",
       "      <td>1.625057e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.840975</td>\n",
       "      <td>1.625057e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.840976</td>\n",
       "      <td>1.625058e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.840978</td>\n",
       "      <td>1.625059e+00</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.734811</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss      VAL_loss       ATT_acc     VAL_k_acc  VAL_k_jac  \\\n",
       "count  10.000000  1.000000e+01  1.000000e+01  1.000000e+01  10.000000   \n",
       "mean    0.840975  1.625058e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "std     0.000002  8.522315e-07  1.497956e-14  1.497956e-14   0.000000   \n",
       "min     0.840972  1.625056e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "25%     0.840973  1.625057e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "50%     0.840975  1.625057e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "75%     0.840976  1.625058e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "max     0.840978  1.625059e+00  9.695122e+01  9.802956e+01   0.734811   \n",
       "\n",
       "          VAL_1_acc  \n",
       "count  1.000000e+01  \n",
       "mean   7.931034e+01  \n",
       "std    1.497956e-14  \n",
       "min    7.931034e+01  \n",
       "25%    7.931034e+01  \n",
       "50%    7.931034e+01  \n",
       "75%    7.931034e+01  \n",
       "max    7.931034e+01  "
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.852848</td>\n",
       "      <td>1.644436e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.734635</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000002</td>\n",
       "      <td>7.253492e-07</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.000824</td>\n",
       "      <td>1.497956e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.852845</td>\n",
       "      <td>1.644435e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.734375</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.852846</td>\n",
       "      <td>1.644435e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.734375</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.852846</td>\n",
       "      <td>1.644436e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.734375</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.852850</td>\n",
       "      <td>1.644436e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.734375</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.852851</td>\n",
       "      <td>1.644437e+00</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.736979</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss      VAL_loss    ATT_acc     VAL_k_acc  VAL_k_jac  \\\n",
       "count  10.000000  1.000000e+01  10.000000  1.000000e+01  10.000000   \n",
       "mean    0.852848  1.644436e+00  96.222664  9.947917e+01   0.734635   \n",
       "std     0.000002  7.253492e-07   0.000000  1.497956e-14   0.000824   \n",
       "min     0.852845  1.644435e+00  96.222664  9.947917e+01   0.734375   \n",
       "25%     0.852846  1.644435e+00  96.222664  9.947917e+01   0.734375   \n",
       "50%     0.852846  1.644436e+00  96.222664  9.947917e+01   0.734375   \n",
       "75%     0.852850  1.644436e+00  96.222664  9.947917e+01   0.734375   \n",
       "max     0.852851  1.644437e+00  96.222664  9.947917e+01   0.736979   \n",
       "\n",
       "          VAL_1_acc  \n",
       "count  1.000000e+01  \n",
       "mean   7.864583e+01  \n",
       "std    1.497956e-14  \n",
       "min    7.864583e+01  \n",
       "25%    7.864583e+01  \n",
       "50%    7.864583e+01  \n",
       "75%    7.864583e+01  \n",
       "max    7.864583e+01  "
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'NO_TEM_val_metrics_transfer.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'NO_TEM_test_metrics_transfer.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NO SPA links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 3 for key in data.edge_types if 'SOC_link' in key or 'TEM_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].train_mask),\n",
    ")\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 3 for key in data.edge_types if 'SOC_link' in key or 'TEM_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].val_mask),\n",
    ")\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 3 for key in data.edge_types if 'SOC_link' in key or 'TEM_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=2946,\n",
       "    x=[2946, 1753],\n",
       "    y=[2946, 20],\n",
       "    node_type=[2946],\n",
       "    att_lab=[2946],\n",
       "    val_lab=[2946],\n",
       "    train_mask=[2946],\n",
       "    val_mask=[2946],\n",
       "    test_mask=[2946],\n",
       "    n_id=[2946],\n",
       "    batch_size=32\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 53785],\n",
       "    edge_attr=[53785]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 54310],\n",
       "    edge_attr=[54310]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_hetero_data = next(iter(train_loader))\n",
    "batch = sampled_hetero_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HGT_L(\n",
       "  (lin_dict): ModuleDict(\n",
       "    (all): Linear(1753, 32, bias=True)\n",
       "  )\n",
       "  (convs): ModuleList(\n",
       "    (0): HGTConv(-1, 32, heads=2)\n",
       "    (1): HGTConv(-1, 32, heads=2)\n",
       "    (2): HGTConv(-1, 32, heads=2)\n",
       "  )\n",
       "  (lin1): Linear(1753, 32, bias=True)\n",
       "  (lin2): Linear(64, 20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 23.01it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7254592820878174,\n",
       " 1.599312377108101,\n",
       " 100.0,\n",
       " 100.0,\n",
       " 0.8711911463011005,\n",
       " 89.75069252077563)"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 41.09it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8410046713623575,\n",
       " 1.6250814951112118,\n",
       " 96.95121951219512,\n",
       " 98.0295566502463,\n",
       " 0.7372742262967115,\n",
       " 79.3103448275862)"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, val_loader, 'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 42.80it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.852870823611796,\n",
       " 1.6444365698844194,\n",
       " 96.42147117296223,\n",
       " 99.47916666666667,\n",
       " 0.7369791766007742,\n",
       " 78.64583333333333)"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, test_loader, 'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 40.87it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 40.89it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 42.02it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 41.41it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 43.21it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 42.90it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 42.64it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 42.86it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 43.49it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 43.62it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 43.89it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 42.48it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 42.37it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 43.92it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 44.43it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 44.37it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 42.31it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 42.40it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 43.82it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 42.53it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test(model, val_loader, 'val'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test(model, test_loader, 'val'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.841006</td>\n",
       "      <td>1.625080</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.737274</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.497956e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.841002</td>\n",
       "      <td>1.625078</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.737274</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.841005</td>\n",
       "      <td>1.625079</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.737274</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.841005</td>\n",
       "      <td>1.625080</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.737274</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.841007</td>\n",
       "      <td>1.625081</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.737274</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.841010</td>\n",
       "      <td>1.625082</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.737274</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss       ATT_acc     VAL_k_acc  VAL_k_jac  \\\n",
       "count  10.000000  10.000000  1.000000e+01  1.000000e+01  10.000000   \n",
       "mean    0.841006   1.625080  9.695122e+01  9.802956e+01   0.737274   \n",
       "std     0.000002   0.000001  1.497956e-14  1.497956e-14   0.000000   \n",
       "min     0.841002   1.625078  9.695122e+01  9.802956e+01   0.737274   \n",
       "25%     0.841005   1.625079  9.695122e+01  9.802956e+01   0.737274   \n",
       "50%     0.841005   1.625080  9.695122e+01  9.802956e+01   0.737274   \n",
       "75%     0.841007   1.625081  9.695122e+01  9.802956e+01   0.737274   \n",
       "max     0.841010   1.625082  9.695122e+01  9.802956e+01   0.737274   \n",
       "\n",
       "          VAL_1_acc  \n",
       "count  1.000000e+01  \n",
       "mean   7.931034e+01  \n",
       "std    1.497956e-14  \n",
       "min    7.931034e+01  \n",
       "25%    7.931034e+01  \n",
       "50%    7.931034e+01  \n",
       "75%    7.931034e+01  \n",
       "max    7.931034e+01  "
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.852874</td>\n",
       "      <td>1.644435</td>\n",
       "      <td>96.322068</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>7.369792e-01</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.104781</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>1.170278e-16</td>\n",
       "      <td>1.497956e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.852872</td>\n",
       "      <td>1.644433</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>7.369792e-01</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.852873</td>\n",
       "      <td>1.644435</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>7.369792e-01</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.852874</td>\n",
       "      <td>1.644436</td>\n",
       "      <td>96.322068</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>7.369792e-01</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.852875</td>\n",
       "      <td>1.644436</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>7.369792e-01</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.852876</td>\n",
       "      <td>1.644438</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>7.369792e-01</td>\n",
       "      <td>7.864583e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc     VAL_k_acc     VAL_k_jac  \\\n",
       "count  10.000000  10.000000  10.000000  1.000000e+01  1.000000e+01   \n",
       "mean    0.852874   1.644435  96.322068  9.947917e+01  7.369792e-01   \n",
       "std     0.000001   0.000001   0.104781  1.497956e-14  1.170278e-16   \n",
       "min     0.852872   1.644433  96.222664  9.947917e+01  7.369792e-01   \n",
       "25%     0.852873   1.644435  96.222664  9.947917e+01  7.369792e-01   \n",
       "50%     0.852874   1.644436  96.322068  9.947917e+01  7.369792e-01   \n",
       "75%     0.852875   1.644436  96.421471  9.947917e+01  7.369792e-01   \n",
       "max     0.852876   1.644438  96.421471  9.947917e+01  7.369792e-01   \n",
       "\n",
       "          VAL_1_acc  \n",
       "count  1.000000e+01  \n",
       "mean   7.864583e+01  \n",
       "std    1.497956e-14  \n",
       "min    7.864583e+01  \n",
       "25%    7.864583e+01  \n",
       "50%    7.864583e+01  \n",
       "75%    7.864583e+01  \n",
       "max    7.864583e+01  "
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'NO_SPA_val_metrics_transfer.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'NO_SPA_test_metrics_transfer.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NO SOC links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'TEM_link' in key or 'SPA_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].train_mask),\n",
    ")\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'TEM_link' in key or 'SPA_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].val_mask),\n",
    ")\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'TEM_link' in key or 'SPA_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=2929,\n",
       "    x=[2929, 1753],\n",
       "    y=[2929, 20],\n",
       "    node_type=[2929],\n",
       "    att_lab=[2929],\n",
       "    val_lab=[2929],\n",
       "    train_mask=[2929],\n",
       "    val_mask=[2929],\n",
       "    test_mask=[2929],\n",
       "    n_id=[2929],\n",
       "    batch_size=32\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 23343],\n",
       "    edge_attr=[23343]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 22314],\n",
       "    edge_attr=[22314]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_hetero_data = next(iter(train_loader))\n",
    "batch = sampled_hetero_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HGT_L(\n",
       "  (lin_dict): ModuleDict(\n",
       "    (all): Linear(1753, 32, bias=True)\n",
       "  )\n",
       "  (convs): ModuleList(\n",
       "    (0): HGTConv(-1, 32, heads=2)\n",
       "    (1): HGTConv(-1, 32, heads=2)\n",
       "    (2): HGTConv(-1, 32, heads=2)\n",
       "  )\n",
       "  (lin1): Linear(1753, 32, bias=True)\n",
       "  (lin2): Linear(64, 20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 31.01it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7254116338706083,\n",
       " 1.5993375355517105,\n",
       " 100.0,\n",
       " 100.0,\n",
       " 0.8711911463011005,\n",
       " 90.02770083102493)"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 67.37it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8409538775440154,\n",
       " 1.6251165326592958,\n",
       " 96.95121951219512,\n",
       " 98.0295566502463,\n",
       " 0.7372742262967115,\n",
       " 79.3103448275862)"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, val_loader, 'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 65.23it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8527654360824266,\n",
       " 1.644487414509058,\n",
       " 96.42147117296223,\n",
       " 99.47916666666667,\n",
       " 0.7369791766007742,\n",
       " 78.125)"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, test_loader, 'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 66.53it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 63.23it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 68.36it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 65.02it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 69.97it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 53.82it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 65.81it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 64.69it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 61.54it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 65.38it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 60.75it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 65.55it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 69.31it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 70.02it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 68.23it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 66.88it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 71.73it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 66.52it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 68.50it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 70.35it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test(model, val_loader, 'val'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test(model, test_loader, 'val'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.840953</td>\n",
       "      <td>1.625117</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.737356</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.000260</td>\n",
       "      <td>1.497956e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.840947</td>\n",
       "      <td>1.625114</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.737274</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.840950</td>\n",
       "      <td>1.625117</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.737274</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.840953</td>\n",
       "      <td>1.625117</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.737274</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.840955</td>\n",
       "      <td>1.625118</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.737274</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.840958</td>\n",
       "      <td>1.625120</td>\n",
       "      <td>9.695122e+01</td>\n",
       "      <td>9.802956e+01</td>\n",
       "      <td>0.738095</td>\n",
       "      <td>7.931034e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss       ATT_acc     VAL_k_acc  VAL_k_jac  \\\n",
       "count  10.000000  10.000000  1.000000e+01  1.000000e+01  10.000000   \n",
       "mean    0.840953   1.625117  9.695122e+01  9.802956e+01   0.737356   \n",
       "std     0.000004   0.000002  1.497956e-14  1.497956e-14   0.000260   \n",
       "min     0.840947   1.625114  9.695122e+01  9.802956e+01   0.737274   \n",
       "25%     0.840950   1.625117  9.695122e+01  9.802956e+01   0.737274   \n",
       "50%     0.840953   1.625117  9.695122e+01  9.802956e+01   0.737274   \n",
       "75%     0.840955   1.625118  9.695122e+01  9.802956e+01   0.737274   \n",
       "max     0.840958   1.625120  9.695122e+01  9.802956e+01   0.738095   \n",
       "\n",
       "          VAL_1_acc  \n",
       "count  1.000000e+01  \n",
       "mean   7.931034e+01  \n",
       "std    1.497956e-14  \n",
       "min    7.931034e+01  \n",
       "25%    7.931034e+01  \n",
       "50%    7.931034e+01  \n",
       "75%    7.931034e+01  \n",
       "max    7.931034e+01  "
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.852765</td>\n",
       "      <td>1.644489</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.737760</td>\n",
       "      <td>78.177083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.001258</td>\n",
       "      <td>0.164702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.852763</td>\n",
       "      <td>1.644487</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.736979</td>\n",
       "      <td>78.125000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.852764</td>\n",
       "      <td>1.644488</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.736979</td>\n",
       "      <td>78.125000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.852766</td>\n",
       "      <td>1.644489</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.736979</td>\n",
       "      <td>78.125000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.852767</td>\n",
       "      <td>1.644491</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.738932</td>\n",
       "      <td>78.125000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.852769</td>\n",
       "      <td>1.644493</td>\n",
       "      <td>96.421471</td>\n",
       "      <td>9.947917e+01</td>\n",
       "      <td>0.739583</td>\n",
       "      <td>78.645833</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc     VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  1.000000e+01  10.000000  10.000000\n",
       "mean    0.852765   1.644489  96.421471  9.947917e+01   0.737760  78.177083\n",
       "std     0.000002   0.000002   0.000000  1.497956e-14   0.001258   0.164702\n",
       "min     0.852763   1.644487  96.421471  9.947917e+01   0.736979  78.125000\n",
       "25%     0.852764   1.644488  96.421471  9.947917e+01   0.736979  78.125000\n",
       "50%     0.852766   1.644489  96.421471  9.947917e+01   0.736979  78.125000\n",
       "75%     0.852767   1.644491  96.421471  9.947917e+01   0.738932  78.125000\n",
       "max     0.852769   1.644493  96.421471  9.947917e+01   0.739583  78.645833"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'NO_SOC_val_metrics_transfer.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'NO_SOC_test_metrics_transfer.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Masking Visual and Textual Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = HGT_L(data.metadata(), hidden_channels=32, out_channels=data.y_dict['all'].shape[-1],\n",
    "                  num_layers=3, num_heads = 2, group='mean').to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load(args.save_dir+'HGT_best_model/model.pth',map_location=torch.device('cpu')))\n",
    "model.eval()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HGT_L(\n",
       "  (lin_dict): ModuleDict(\n",
       "    (all): Linear(1753, 32, bias=True)\n",
       "  )\n",
       "  (convs): ModuleList(\n",
       "    (0): HGTConv(-1, 32, heads=2)\n",
       "    (1): HGTConv(-1, 32, heads=2)\n",
       "    (2): HGTConv(-1, 32, heads=2)\n",
       "  )\n",
       "  (lin1): Linear(1753, 32, bias=True)\n",
       "  (lin2): Linear(64, 20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def test_Mask(model, loader, mask='vis'):\n",
    "    model.eval()\n",
    "\n",
    "    total_examples_att = total_examples_val = 0\n",
    "    running_loss_1 = running_loss_2 = 0.\n",
    "    running_1_acc = 0.\n",
    "    running_k_acc = 0.\n",
    "    running_k_jac = 0.\n",
    "    running_1_val = 0.\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        loss_1 = 0\n",
    "        acc_1_t = 0\n",
    "        loss_2 = 0\n",
    "        acc_1_val = 0\n",
    "        acc_k_t = 0\n",
    "        jac_k_t = 0\n",
    "\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch['all'].batch_size\n",
    "        \n",
    "        if mask == 'vis':\n",
    "            batch['all'].x[:batch_size,:982] = torch.zeros(batch_size,982)\n",
    "        elif mask == 'tex':\n",
    "            batch['all'].x[:batch_size,982:] = torch.zeros(batch_size,771)\n",
    "                \n",
    "        new_dict = {}\n",
    "        for edge_type in [edge_type for edge_type in batch.edge_index_dict if 'all' in edge_type and not 'simp_link' in edge_type]:\n",
    "            edge_index = batch.edge_index_dict[edge_type]\n",
    "            edge_index = to_undirected(edge_index)\n",
    "            new_dict[edge_type] = edge_index\n",
    "        batch.edge_index_dict = new_dict\n",
    "\n",
    "        out = model(batch.x_dict, batch.edge_index_dict)[:batch_size]\n",
    "        out_att = out[:,:9]\n",
    "        out_val = out[:,9:]\n",
    "        att_node = (batch['all'].att_lab[:batch_size]).nonzero().squeeze()\n",
    "        val_node = (batch['all'].val_lab[:batch_size]).nonzero().squeeze()\n",
    "\n",
    "        #print(type_node)\n",
    "\n",
    "        #pred_att = out_att.argmax(dim=-1)\n",
    "        #pred_val = out_val.argmax(dim=-1)\n",
    "\n",
    "        y = batch.y_dict['all']\n",
    "        y_att = y[:,:9]\n",
    "        y_val = y[:,9:]\n",
    "\n",
    "        if not att_node.shape[0]==0:\n",
    "            loss_1 = F.cross_entropy(out_att[att_node], y_att[:batch_size][att_node])\n",
    "            acc_1_t = compute_1_accuracy(y_att[:batch_size][att_node], out_att[att_node])\n",
    "\n",
    "        if not val_node.shape[0]==0:\n",
    "            loss_2 = F.cross_entropy(out_val[val_node], y_val[val_node])\n",
    "            acc_1_val = compute_1_accuracy(y_val[val_node], out_val[val_node])\n",
    "            acc_k_t = compute_k_accuracy(y_val[val_node], out_val[val_node], args.k)\n",
    "            jac_k_t = compute_jaccard_index(y_val[val_node], F.softmax(out_val[val_node],dim=-1), args.k)\n",
    "            #loss_3 = loss_1 + loss_2\n",
    "\n",
    "        total_examples_att += att_node.shape[0]\n",
    "        total_examples_val += val_node.shape[0]\n",
    "        #total_correct_att += int((pred_att == y_att[:batch_size]).sum())\n",
    "        #total_correct_val += int((pred_val == y_val[:batch_size]).sum())\n",
    "\n",
    "        running_loss_1 += float(loss_1) * att_node.shape[0]\n",
    "        running_loss_2 += float(loss_2) * val_node.shape[0]\n",
    "        running_1_acc += float(acc_1_t) * att_node.shape[0]\n",
    "        running_1_val += float(acc_1_val) * val_node.shape[0]\n",
    "        running_k_acc += float(acc_k_t) * val_node.shape[0]\n",
    "        running_k_jac += float(jac_k_t) * val_node.shape[0]\n",
    "\n",
    "    return running_loss_1/total_examples_att, running_loss_2/total_examples_val, running_1_acc/ total_examples_att, running_k_acc/ total_examples_val, running_k_jac/ total_examples_val, running_1_val/total_examples_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:05<00:00,  2.40it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.9956734160637262,\n",
       " 1.6647645836061389,\n",
       " 23.822714681440445,\n",
       " 99.7229916897507,\n",
       " 0.6980609523952833,\n",
       " 77.5623268698061)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, train_loader, 'vis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 26.61it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7889452987760718,\n",
       " 1.92601609692349,\n",
       " 100.0,\n",
       " 77.5623268698061,\n",
       " 0.05632502311154416,\n",
       " 32.40997229916898)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, train_loader, 'tex')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 41.12it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2.1549280154995802,\n",
       " 1.6708702688733932,\n",
       " 17.276422764227643,\n",
       " 99.01477832512315,\n",
       " 0.7068965540730895,\n",
       " 77.33990147783251)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, val_loader, 'vis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 33.56it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8652394693799135,\n",
       " 1.968609806939299,\n",
       " 96.7479674796748,\n",
       " 59.60591133004926,\n",
       " 0.04433497536945813,\n",
       " 21.67487684729064)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, val_loader, 'tex')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 45.84it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2.146076456926926,\n",
       " 1.6830203402787447,\n",
       " 14.115308151093439,\n",
       " 98.4375,\n",
       " 0.7031250074505806,\n",
       " 76.04166666666667)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, test_loader, 'vis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 47.71it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8747231437719128,\n",
       " 1.9744189474731684,\n",
       " 97.4155069582505,\n",
       " 62.5,\n",
       " 0.01996527782951792,\n",
       " 22.395833333333332)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, test_loader, 'tex')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 48.40it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 50.75it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 50.21it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 50.96it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 47.46it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 47.75it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 50.22it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 50.78it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 51.43it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 49.84it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 49.89it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 51.08it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 48.59it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 50.48it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 47.90it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 51.32it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 50.62it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 51.75it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 51.28it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 45.11it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 51.26it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 52.03it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 48.15it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 46.35it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 48.20it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 52.33it/s]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 50.79it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 49.86it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 46.94it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 51.62it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 45.01it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 48.08it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 52.02it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 50.82it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 48.41it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 46.87it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 51.98it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 52.60it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 50.30it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 49.46it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers_vis = []\n",
    "val_numbers_tex = []\n",
    "test_numbers_vis = []\n",
    "test_numbers_tex = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers_vis.append(test_Mask(model, val_loader, 'vis'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers_tex.append(test_Mask(model, val_loader, 'tex'))\n",
    "    \n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers_vis.append(test_Mask(model, test_loader, 'vis'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers_tex.append(test_Mask(model, test_loader, 'tex'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df_vis = pd.DataFrame(val_numbers_vis, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "val_df_tex = pd.DataFrame(val_numbers_tex, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df_vis = pd.DataFrame(test_numbers_vis, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df_tex = pd.DataFrame(test_numbers_tex, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.154920</td>\n",
       "      <td>1.670870</td>\n",
       "      <td>17.276423</td>\n",
       "      <td>9.901478e+01</td>\n",
       "      <td>7.068966e-01</td>\n",
       "      <td>7.733990e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>1.170278e-16</td>\n",
       "      <td>1.497956e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.154904</td>\n",
       "      <td>1.670866</td>\n",
       "      <td>17.276423</td>\n",
       "      <td>9.901478e+01</td>\n",
       "      <td>7.068966e-01</td>\n",
       "      <td>7.733990e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.154912</td>\n",
       "      <td>1.670868</td>\n",
       "      <td>17.276423</td>\n",
       "      <td>9.901478e+01</td>\n",
       "      <td>7.068966e-01</td>\n",
       "      <td>7.733990e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.154920</td>\n",
       "      <td>1.670871</td>\n",
       "      <td>17.276423</td>\n",
       "      <td>9.901478e+01</td>\n",
       "      <td>7.068966e-01</td>\n",
       "      <td>7.733990e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.154927</td>\n",
       "      <td>1.670871</td>\n",
       "      <td>17.276423</td>\n",
       "      <td>9.901478e+01</td>\n",
       "      <td>7.068966e-01</td>\n",
       "      <td>7.733990e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.154939</td>\n",
       "      <td>1.670873</td>\n",
       "      <td>17.276423</td>\n",
       "      <td>9.901478e+01</td>\n",
       "      <td>7.068966e-01</td>\n",
       "      <td>7.733990e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc     VAL_k_acc     VAL_k_jac  \\\n",
       "count  10.000000  10.000000  10.000000  1.000000e+01  1.000000e+01   \n",
       "mean    2.154920   1.670870  17.276423  9.901478e+01  7.068966e-01   \n",
       "std     0.000011   0.000002   0.000000  1.497956e-14  1.170278e-16   \n",
       "min     2.154904   1.670866  17.276423  9.901478e+01  7.068966e-01   \n",
       "25%     2.154912   1.670868  17.276423  9.901478e+01  7.068966e-01   \n",
       "50%     2.154920   1.670871  17.276423  9.901478e+01  7.068966e-01   \n",
       "75%     2.154927   1.670871  17.276423  9.901478e+01  7.068966e-01   \n",
       "max     2.154939   1.670873  17.276423  9.901478e+01  7.068966e-01   \n",
       "\n",
       "          VAL_1_acc  \n",
       "count  1.000000e+01  \n",
       "mean   7.733990e+01  \n",
       "std    1.497956e-14  \n",
       "min    7.733990e+01  \n",
       "25%    7.733990e+01  \n",
       "50%    7.733990e+01  \n",
       "75%    7.733990e+01  \n",
       "max    7.733990e+01  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df_vis.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.865238</td>\n",
       "      <td>1.968593</td>\n",
       "      <td>96.747967</td>\n",
       "      <td>5.960591e+01</td>\n",
       "      <td>0.044335</td>\n",
       "      <td>2.167488e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.489778e-15</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.744889e-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.865229</td>\n",
       "      <td>1.968580</td>\n",
       "      <td>96.747967</td>\n",
       "      <td>5.960591e+01</td>\n",
       "      <td>0.044335</td>\n",
       "      <td>2.167488e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.865236</td>\n",
       "      <td>1.968587</td>\n",
       "      <td>96.747967</td>\n",
       "      <td>5.960591e+01</td>\n",
       "      <td>0.044335</td>\n",
       "      <td>2.167488e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.865237</td>\n",
       "      <td>1.968594</td>\n",
       "      <td>96.747967</td>\n",
       "      <td>5.960591e+01</td>\n",
       "      <td>0.044335</td>\n",
       "      <td>2.167488e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.865240</td>\n",
       "      <td>1.968600</td>\n",
       "      <td>96.747967</td>\n",
       "      <td>5.960591e+01</td>\n",
       "      <td>0.044335</td>\n",
       "      <td>2.167488e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.865245</td>\n",
       "      <td>1.968604</td>\n",
       "      <td>96.747967</td>\n",
       "      <td>5.960591e+01</td>\n",
       "      <td>0.044335</td>\n",
       "      <td>2.167488e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc     VAL_k_acc  VAL_k_jac     VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  1.000000e+01  10.000000  1.000000e+01\n",
       "mean    0.865238   1.968593  96.747967  5.960591e+01   0.044335  2.167488e+01\n",
       "std     0.000005   0.000008   0.000000  7.489778e-15   0.000000  3.744889e-15\n",
       "min     0.865229   1.968580  96.747967  5.960591e+01   0.044335  2.167488e+01\n",
       "25%     0.865236   1.968587  96.747967  5.960591e+01   0.044335  2.167488e+01\n",
       "50%     0.865237   1.968594  96.747967  5.960591e+01   0.044335  2.167488e+01\n",
       "75%     0.865240   1.968600  96.747967  5.960591e+01   0.044335  2.167488e+01\n",
       "max     0.865245   1.968604  96.747967  5.960591e+01   0.044335  2.167488e+01"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df_tex.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.0000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.146078</td>\n",
       "      <td>1.683017</td>\n",
       "      <td>14.234592</td>\n",
       "      <td>98.4375</td>\n",
       "      <td>0.703125</td>\n",
       "      <td>7.604167e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.102664</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.497956e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.146073</td>\n",
       "      <td>1.683013</td>\n",
       "      <td>14.115308</td>\n",
       "      <td>98.4375</td>\n",
       "      <td>0.703125</td>\n",
       "      <td>7.604167e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.146076</td>\n",
       "      <td>1.683016</td>\n",
       "      <td>14.115308</td>\n",
       "      <td>98.4375</td>\n",
       "      <td>0.703125</td>\n",
       "      <td>7.604167e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.146077</td>\n",
       "      <td>1.683018</td>\n",
       "      <td>14.314115</td>\n",
       "      <td>98.4375</td>\n",
       "      <td>0.703125</td>\n",
       "      <td>7.604167e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.146081</td>\n",
       "      <td>1.683019</td>\n",
       "      <td>14.314115</td>\n",
       "      <td>98.4375</td>\n",
       "      <td>0.703125</td>\n",
       "      <td>7.604167e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.146086</td>\n",
       "      <td>1.683020</td>\n",
       "      <td>14.314115</td>\n",
       "      <td>98.4375</td>\n",
       "      <td>0.703125</td>\n",
       "      <td>7.604167e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac     VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000    10.0000  10.000000  1.000000e+01\n",
       "mean    2.146078   1.683017  14.234592    98.4375   0.703125  7.604167e+01\n",
       "std     0.000004   0.000002   0.102664     0.0000   0.000000  1.497956e-14\n",
       "min     2.146073   1.683013  14.115308    98.4375   0.703125  7.604167e+01\n",
       "25%     2.146076   1.683016  14.115308    98.4375   0.703125  7.604167e+01\n",
       "50%     2.146077   1.683018  14.314115    98.4375   0.703125  7.604167e+01\n",
       "75%     2.146081   1.683019  14.314115    98.4375   0.703125  7.604167e+01\n",
       "max     2.146086   1.683020  14.314115    98.4375   0.703125  7.604167e+01"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df_vis.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.874717</td>\n",
       "      <td>1.974425</td>\n",
       "      <td>97.415507</td>\n",
       "      <td>62.5</td>\n",
       "      <td>1.996528e-02</td>\n",
       "      <td>2.239583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000003</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.657118e-18</td>\n",
       "      <td>3.744889e-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.874713</td>\n",
       "      <td>1.974413</td>\n",
       "      <td>97.415507</td>\n",
       "      <td>62.5</td>\n",
       "      <td>1.996528e-02</td>\n",
       "      <td>2.239583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.874715</td>\n",
       "      <td>1.974419</td>\n",
       "      <td>97.415507</td>\n",
       "      <td>62.5</td>\n",
       "      <td>1.996528e-02</td>\n",
       "      <td>2.239583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.874715</td>\n",
       "      <td>1.974427</td>\n",
       "      <td>97.415507</td>\n",
       "      <td>62.5</td>\n",
       "      <td>1.996528e-02</td>\n",
       "      <td>2.239583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.874718</td>\n",
       "      <td>1.974430</td>\n",
       "      <td>97.415507</td>\n",
       "      <td>62.5</td>\n",
       "      <td>1.996528e-02</td>\n",
       "      <td>2.239583e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.874722</td>\n",
       "      <td>1.974439</td>\n",
       "      <td>97.415507</td>\n",
       "      <td>62.5</td>\n",
       "      <td>1.996528e-02</td>\n",
       "      <td>2.239583e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc     VAL_k_jac     VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000       10.0  1.000000e+01  1.000000e+01\n",
       "mean    0.874717   1.974425  97.415507       62.5  1.996528e-02  2.239583e+01\n",
       "std     0.000003   0.000008   0.000000        0.0  3.657118e-18  3.744889e-15\n",
       "min     0.874713   1.974413  97.415507       62.5  1.996528e-02  2.239583e+01\n",
       "25%     0.874715   1.974419  97.415507       62.5  1.996528e-02  2.239583e+01\n",
       "50%     0.874715   1.974427  97.415507       62.5  1.996528e-02  2.239583e+01\n",
       "75%     0.874718   1.974430  97.415507       62.5  1.996528e-02  2.239583e+01\n",
       "max     0.874722   1.974439  97.415507       62.5  1.996528e-02  2.239583e+01"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df_tex.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df_vis.to_csv(args.save_dir + 'vis_masked_val_metrics.csv', sep='\\t')\n",
    "val_df_tex.to_csv(args.save_dir + 'tex_masked_val_metrics.csv', sep='\\t')\n",
    "test_df_vis.to_csv(args.save_dir + 'vis_masked_test_metrics.csv', sep='\\t')\n",
    "test_df_tex.to_csv(args.save_dir + 'tex_masked_test_metrics.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Direct Prediction on VEN-XL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cpu\")\n",
    "#device = torch.device(\"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_XL = VEN_XL_Links('dataset/Venice_XL_links')\n",
    "data_XL = dataset_XL[0].to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_XL = VEN_XL_Links('dataset/Venice_XL_links')[0].to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=80963,\n",
       "    x=[80963, 1753],\n",
       "    y=[80963, 20],\n",
       "    node_type=[80963],\n",
       "    att_lab=[80963],\n",
       "    val_lab=[80963],\n",
       "    train_mask=[80963],\n",
       "    val_mask=[80963],\n",
       "    test_mask=[80963],\n",
       "    n_id=[80963]\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 76422265],\n",
       "    edge_attr=[76422265]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 202173159],\n",
       "    edge_attr=[202173159]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 71135671],\n",
       "    edge_attr=[71135671]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 290091503],\n",
       "    edge_attr=[290091503]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_XL['all'].n_id = torch.arange(data_XL.num_nodes)\n",
    "#data_XL = data_XL.to(device)\n",
    "data_XL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = HGT_L(data_XL.metadata(), hidden_channels=32, out_channels=data_XL.y_dict['all'].shape[-1],\n",
    "                  num_layers=3, num_heads = 2, group='mean').to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load(args.save_dir+'HGT_best_model/model.pth',map_location=torch.device('cpu')))\n",
    "model.eval()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HGT_L(\n",
       "  (lin_dict): ModuleDict(\n",
       "    (all): Linear(1753, 32, bias=True)\n",
       "  )\n",
       "  (convs): ModuleList(\n",
       "    (0): HGTConv(-1, 32, heads=2)\n",
       "    (1): HGTConv(-1, 32, heads=2)\n",
       "    (2): HGTConv(-1, 32, heads=2)\n",
       "  )\n",
       "  (lin1): Linear(1753, 32, bias=True)\n",
       "  (lin2): Linear(64, 20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.loader import NeighborLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data_XL,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data_XL.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data_XL['all'].train_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "val_loader = NeighborLoader(\n",
    "    data_XL,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data_XL.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data_XL['all'].val_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "test_loader = NeighborLoader(\n",
    "    data_XL,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data_XL.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data_XL['all'].test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "unlabel_loader = NeighborLoader(\n",
    "    data_XL,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data_XL.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', ~(data_XL['all'].train_mask + data_XL['all'].val_mask + data_XL['all'].test_mask)),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def predict(model, loader):\n",
    "    model.eval()\n",
    "    seed_everything(args.seed)\n",
    "    all_preds = []\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch['all'].batch_size\n",
    "        new_dict = {}\n",
    "        for edge_type in [edge_type for edge_type in batch.edge_index_dict if 'all' in edge_type and not 'simp_link' in edge_type]:\n",
    "            edge_index = batch.edge_index_dict[edge_type]\n",
    "            edge_index = to_undirected(edge_index)\n",
    "            new_dict[edge_type] = edge_index\n",
    "        batch.edge_index_dict = new_dict\n",
    "        \n",
    "        out = model(batch.x_dict, batch.edge_index_dict)[:batch_size]\n",
    "        out_att = out[:,:9].softmax(axis=1)\n",
    "        out_val = out[:,9:].softmax(axis=1)\n",
    "        IDs = batch['all'].n_id[:batch_size].unsqueeze(dim=-1).int()\n",
    "        \n",
    "        now = torch.hstack([IDs, out_att, out_val])\n",
    "        all_preds.append(now)\n",
    "    \n",
    "    final = torch.vstack(all_preds)\n",
    "        \n",
    "    return final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 362/362 [08:21<00:00,  1.39s/it]\n"
     ]
    }
   ],
   "source": [
    "pred_train = predict(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 624/624 [13:51<00:00,  1.33s/it]\n"
     ]
    }
   ],
   "source": [
    "pred_val = predict(model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 624/624 [13:43<00:00,  1.32s/it]\n"
     ]
    }
   ],
   "source": [
    "pred_test = predict(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 921/921 [19:17<00:00,  1.26s/it]\n"
     ]
    }
   ],
   "source": [
    "pred_unlab = predict(model, unlabel_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([4.0000e+00, 3.6224e-03, 9.1489e-01, 3.9630e-03, 1.8197e-03, 1.3219e-03,\n",
       "        5.5136e-02, 7.6846e-03, 1.3151e-03, 1.0246e-02, 1.7893e-02, 1.2161e-01,\n",
       "        1.6164e-01, 4.4398e-01, 1.1871e-01, 9.0284e-02, 8.5269e-03, 4.6221e-03,\n",
       "        1.1879e-02, 5.8970e-03, 1.4952e-02])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = torch.vstack([pred, pred_unlab]).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = torch.vstack([pred_train, pred_val, pred_test]).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = torch.vstack([pred_train, pred_val, pred_test, pred_unlab]).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[4.0000e+00, 2.6310e-03, 9.3372e-01,  ..., 7.3632e-03, 4.8628e-03,\n",
       "         1.2648e-02],\n",
       "        [5.0000e+00, 7.4130e-03, 9.0296e-01,  ..., 6.7272e-03, 5.7115e-03,\n",
       "         1.2796e-02],\n",
       "        [8.0000e+00, 3.3455e-03, 8.2061e-01,  ..., 4.6515e-03, 7.1458e-03,\n",
       "         8.4681e-03],\n",
       "        ...,\n",
       "        [8.0955e+04, 2.2351e-02, 5.4676e-01,  ..., 6.5204e-03, 5.7883e-03,\n",
       "         1.2591e-02],\n",
       "        [8.0960e+04, 7.2552e-01, 1.0158e-01,  ..., 1.8194e-03, 2.8158e-03,\n",
       "         7.7784e-03],\n",
       "        [8.0962e+04, 5.5097e-01, 1.5797e-01,  ..., 2.8654e-03, 3.3919e-03,\n",
       "         1.0750e-02]], dtype=torch.float64)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([2.0000, 2.0000, 2.0000,  ..., 2.0000, 2.0000, 2.0000])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds[:,1:].sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df = pd.DataFrame(preds).sort_values(0).set_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.590949</td>\n",
       "      <td>0.045722</td>\n",
       "      <td>0.019597</td>\n",
       "      <td>0.036334</td>\n",
       "      <td>0.006759</td>\n",
       "      <td>0.234161</td>\n",
       "      <td>0.023612</td>\n",
       "      <td>0.017325</td>\n",
       "      <td>0.025541</td>\n",
       "      <td>0.254323</td>\n",
       "      <td>0.110100</td>\n",
       "      <td>0.157668</td>\n",
       "      <td>0.360964</td>\n",
       "      <td>0.027947</td>\n",
       "      <td>0.065056</td>\n",
       "      <td>0.004690</td>\n",
       "      <td>0.003224</td>\n",
       "      <td>0.003101</td>\n",
       "      <td>0.004269</td>\n",
       "      <td>0.008657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.224058</td>\n",
       "      <td>0.007389</td>\n",
       "      <td>0.000600</td>\n",
       "      <td>0.003432</td>\n",
       "      <td>0.000876</td>\n",
       "      <td>0.756404</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000756</td>\n",
       "      <td>0.005767</td>\n",
       "      <td>0.172469</td>\n",
       "      <td>0.098551</td>\n",
       "      <td>0.113473</td>\n",
       "      <td>0.485997</td>\n",
       "      <td>0.025722</td>\n",
       "      <td>0.089703</td>\n",
       "      <td>0.002135</td>\n",
       "      <td>0.001354</td>\n",
       "      <td>0.001644</td>\n",
       "      <td>0.001947</td>\n",
       "      <td>0.007005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.010895</td>\n",
       "      <td>0.313081</td>\n",
       "      <td>0.016462</td>\n",
       "      <td>0.013692</td>\n",
       "      <td>0.455680</td>\n",
       "      <td>0.016656</td>\n",
       "      <td>0.087140</td>\n",
       "      <td>0.008610</td>\n",
       "      <td>0.077785</td>\n",
       "      <td>0.113593</td>\n",
       "      <td>0.104693</td>\n",
       "      <td>0.133691</td>\n",
       "      <td>0.131544</td>\n",
       "      <td>0.041658</td>\n",
       "      <td>0.372583</td>\n",
       "      <td>0.028839</td>\n",
       "      <td>0.018939</td>\n",
       "      <td>0.016940</td>\n",
       "      <td>0.014645</td>\n",
       "      <td>0.022876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.009238</td>\n",
       "      <td>0.242052</td>\n",
       "      <td>0.035775</td>\n",
       "      <td>0.020205</td>\n",
       "      <td>0.033555</td>\n",
       "      <td>0.094468</td>\n",
       "      <td>0.470154</td>\n",
       "      <td>0.029190</td>\n",
       "      <td>0.065364</td>\n",
       "      <td>0.042914</td>\n",
       "      <td>0.174083</td>\n",
       "      <td>0.210648</td>\n",
       "      <td>0.218459</td>\n",
       "      <td>0.088202</td>\n",
       "      <td>0.200309</td>\n",
       "      <td>0.015257</td>\n",
       "      <td>0.010063</td>\n",
       "      <td>0.014033</td>\n",
       "      <td>0.008566</td>\n",
       "      <td>0.017468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.002631</td>\n",
       "      <td>0.933718</td>\n",
       "      <td>0.005695</td>\n",
       "      <td>0.001904</td>\n",
       "      <td>0.001124</td>\n",
       "      <td>0.033052</td>\n",
       "      <td>0.008198</td>\n",
       "      <td>0.001211</td>\n",
       "      <td>0.012467</td>\n",
       "      <td>0.016775</td>\n",
       "      <td>0.144019</td>\n",
       "      <td>0.176092</td>\n",
       "      <td>0.419996</td>\n",
       "      <td>0.137657</td>\n",
       "      <td>0.071999</td>\n",
       "      <td>0.005330</td>\n",
       "      <td>0.003258</td>\n",
       "      <td>0.007363</td>\n",
       "      <td>0.004863</td>\n",
       "      <td>0.012648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80958.0</th>\n",
       "      <td>0.044957</td>\n",
       "      <td>0.352496</td>\n",
       "      <td>0.012269</td>\n",
       "      <td>0.015928</td>\n",
       "      <td>0.008300</td>\n",
       "      <td>0.409278</td>\n",
       "      <td>0.056828</td>\n",
       "      <td>0.011234</td>\n",
       "      <td>0.088709</td>\n",
       "      <td>0.093243</td>\n",
       "      <td>0.331395</td>\n",
       "      <td>0.165370</td>\n",
       "      <td>0.290632</td>\n",
       "      <td>0.013885</td>\n",
       "      <td>0.087260</td>\n",
       "      <td>0.002615</td>\n",
       "      <td>0.002840</td>\n",
       "      <td>0.002756</td>\n",
       "      <td>0.002504</td>\n",
       "      <td>0.007499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80959.0</th>\n",
       "      <td>0.886580</td>\n",
       "      <td>0.029885</td>\n",
       "      <td>0.001693</td>\n",
       "      <td>0.043460</td>\n",
       "      <td>0.012236</td>\n",
       "      <td>0.008936</td>\n",
       "      <td>0.009009</td>\n",
       "      <td>0.006306</td>\n",
       "      <td>0.001894</td>\n",
       "      <td>0.235541</td>\n",
       "      <td>0.250493</td>\n",
       "      <td>0.204294</td>\n",
       "      <td>0.184092</td>\n",
       "      <td>0.009316</td>\n",
       "      <td>0.095288</td>\n",
       "      <td>0.002649</td>\n",
       "      <td>0.003574</td>\n",
       "      <td>0.002417</td>\n",
       "      <td>0.002847</td>\n",
       "      <td>0.009487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80960.0</th>\n",
       "      <td>0.725524</td>\n",
       "      <td>0.101577</td>\n",
       "      <td>0.002193</td>\n",
       "      <td>0.055688</td>\n",
       "      <td>0.010050</td>\n",
       "      <td>0.079787</td>\n",
       "      <td>0.013755</td>\n",
       "      <td>0.007198</td>\n",
       "      <td>0.004228</td>\n",
       "      <td>0.270172</td>\n",
       "      <td>0.137503</td>\n",
       "      <td>0.240337</td>\n",
       "      <td>0.161904</td>\n",
       "      <td>0.004919</td>\n",
       "      <td>0.165948</td>\n",
       "      <td>0.002785</td>\n",
       "      <td>0.004019</td>\n",
       "      <td>0.001819</td>\n",
       "      <td>0.002816</td>\n",
       "      <td>0.007778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80961.0</th>\n",
       "      <td>0.508486</td>\n",
       "      <td>0.321462</td>\n",
       "      <td>0.011151</td>\n",
       "      <td>0.032964</td>\n",
       "      <td>0.011639</td>\n",
       "      <td>0.027733</td>\n",
       "      <td>0.060230</td>\n",
       "      <td>0.020961</td>\n",
       "      <td>0.005375</td>\n",
       "      <td>0.157282</td>\n",
       "      <td>0.279329</td>\n",
       "      <td>0.191692</td>\n",
       "      <td>0.180926</td>\n",
       "      <td>0.008948</td>\n",
       "      <td>0.161806</td>\n",
       "      <td>0.002692</td>\n",
       "      <td>0.003054</td>\n",
       "      <td>0.002269</td>\n",
       "      <td>0.002939</td>\n",
       "      <td>0.009065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80962.0</th>\n",
       "      <td>0.550968</td>\n",
       "      <td>0.157973</td>\n",
       "      <td>0.007150</td>\n",
       "      <td>0.152399</td>\n",
       "      <td>0.010979</td>\n",
       "      <td>0.073657</td>\n",
       "      <td>0.018118</td>\n",
       "      <td>0.016144</td>\n",
       "      <td>0.012613</td>\n",
       "      <td>0.154999</td>\n",
       "      <td>0.367239</td>\n",
       "      <td>0.174375</td>\n",
       "      <td>0.160868</td>\n",
       "      <td>0.010337</td>\n",
       "      <td>0.108932</td>\n",
       "      <td>0.002700</td>\n",
       "      <td>0.003543</td>\n",
       "      <td>0.002865</td>\n",
       "      <td>0.003392</td>\n",
       "      <td>0.010750</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80963 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               1         2         3         4         5         6         7   \\\n",
       "0                                                                               \n",
       "0.0      0.590949  0.045722  0.019597  0.036334  0.006759  0.234161  0.023612   \n",
       "1.0      0.224058  0.007389  0.000600  0.003432  0.000876  0.756404  0.000719   \n",
       "2.0      0.010895  0.313081  0.016462  0.013692  0.455680  0.016656  0.087140   \n",
       "3.0      0.009238  0.242052  0.035775  0.020205  0.033555  0.094468  0.470154   \n",
       "4.0      0.002631  0.933718  0.005695  0.001904  0.001124  0.033052  0.008198   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80958.0  0.044957  0.352496  0.012269  0.015928  0.008300  0.409278  0.056828   \n",
       "80959.0  0.886580  0.029885  0.001693  0.043460  0.012236  0.008936  0.009009   \n",
       "80960.0  0.725524  0.101577  0.002193  0.055688  0.010050  0.079787  0.013755   \n",
       "80961.0  0.508486  0.321462  0.011151  0.032964  0.011639  0.027733  0.060230   \n",
       "80962.0  0.550968  0.157973  0.007150  0.152399  0.010979  0.073657  0.018118   \n",
       "\n",
       "               8         9         10        11        12        13        14  \\\n",
       "0                                                                               \n",
       "0.0      0.017325  0.025541  0.254323  0.110100  0.157668  0.360964  0.027947   \n",
       "1.0      0.000756  0.005767  0.172469  0.098551  0.113473  0.485997  0.025722   \n",
       "2.0      0.008610  0.077785  0.113593  0.104693  0.133691  0.131544  0.041658   \n",
       "3.0      0.029190  0.065364  0.042914  0.174083  0.210648  0.218459  0.088202   \n",
       "4.0      0.001211  0.012467  0.016775  0.144019  0.176092  0.419996  0.137657   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80958.0  0.011234  0.088709  0.093243  0.331395  0.165370  0.290632  0.013885   \n",
       "80959.0  0.006306  0.001894  0.235541  0.250493  0.204294  0.184092  0.009316   \n",
       "80960.0  0.007198  0.004228  0.270172  0.137503  0.240337  0.161904  0.004919   \n",
       "80961.0  0.020961  0.005375  0.157282  0.279329  0.191692  0.180926  0.008948   \n",
       "80962.0  0.016144  0.012613  0.154999  0.367239  0.174375  0.160868  0.010337   \n",
       "\n",
       "               15        16        17        18        19        20  \n",
       "0                                                                    \n",
       "0.0      0.065056  0.004690  0.003224  0.003101  0.004269  0.008657  \n",
       "1.0      0.089703  0.002135  0.001354  0.001644  0.001947  0.007005  \n",
       "2.0      0.372583  0.028839  0.018939  0.016940  0.014645  0.022876  \n",
       "3.0      0.200309  0.015257  0.010063  0.014033  0.008566  0.017468  \n",
       "4.0      0.071999  0.005330  0.003258  0.007363  0.004863  0.012648  \n",
       "...           ...       ...       ...       ...       ...       ...  \n",
       "80958.0  0.087260  0.002615  0.002840  0.002756  0.002504  0.007499  \n",
       "80959.0  0.095288  0.002649  0.003574  0.002417  0.002847  0.009487  \n",
       "80960.0  0.165948  0.002785  0.004019  0.001819  0.002816  0.007778  \n",
       "80961.0  0.161806  0.002692  0.003054  0.002269  0.002939  0.009065  \n",
       "80962.0  0.108932  0.002700  0.003543  0.002865  0.003392  0.010750  \n",
       "\n",
       "[80963 rows x 20 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df.to_csv(args.save_dir + 'preds_XL_trans.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.590949</td>\n",
       "      <td>0.045722</td>\n",
       "      <td>0.019597</td>\n",
       "      <td>0.036334</td>\n",
       "      <td>0.006759</td>\n",
       "      <td>0.234161</td>\n",
       "      <td>0.023612</td>\n",
       "      <td>0.017325</td>\n",
       "      <td>0.025541</td>\n",
       "      <td>0.254323</td>\n",
       "      <td>0.110100</td>\n",
       "      <td>0.157668</td>\n",
       "      <td>0.360964</td>\n",
       "      <td>0.027947</td>\n",
       "      <td>0.065056</td>\n",
       "      <td>0.004690</td>\n",
       "      <td>0.003224</td>\n",
       "      <td>0.003101</td>\n",
       "      <td>0.004269</td>\n",
       "      <td>0.008657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.224058</td>\n",
       "      <td>0.007389</td>\n",
       "      <td>0.000600</td>\n",
       "      <td>0.003432</td>\n",
       "      <td>0.000876</td>\n",
       "      <td>0.756404</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000756</td>\n",
       "      <td>0.005767</td>\n",
       "      <td>0.172469</td>\n",
       "      <td>0.098551</td>\n",
       "      <td>0.113473</td>\n",
       "      <td>0.485997</td>\n",
       "      <td>0.025722</td>\n",
       "      <td>0.089703</td>\n",
       "      <td>0.002135</td>\n",
       "      <td>0.001354</td>\n",
       "      <td>0.001644</td>\n",
       "      <td>0.001947</td>\n",
       "      <td>0.007005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.010895</td>\n",
       "      <td>0.313081</td>\n",
       "      <td>0.016462</td>\n",
       "      <td>0.013692</td>\n",
       "      <td>0.455680</td>\n",
       "      <td>0.016656</td>\n",
       "      <td>0.087140</td>\n",
       "      <td>0.008610</td>\n",
       "      <td>0.077785</td>\n",
       "      <td>0.113593</td>\n",
       "      <td>0.104693</td>\n",
       "      <td>0.133691</td>\n",
       "      <td>0.131544</td>\n",
       "      <td>0.041658</td>\n",
       "      <td>0.372583</td>\n",
       "      <td>0.028839</td>\n",
       "      <td>0.018939</td>\n",
       "      <td>0.016940</td>\n",
       "      <td>0.014645</td>\n",
       "      <td>0.022876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.009238</td>\n",
       "      <td>0.242052</td>\n",
       "      <td>0.035775</td>\n",
       "      <td>0.020205</td>\n",
       "      <td>0.033555</td>\n",
       "      <td>0.094468</td>\n",
       "      <td>0.470154</td>\n",
       "      <td>0.029190</td>\n",
       "      <td>0.065364</td>\n",
       "      <td>0.042914</td>\n",
       "      <td>0.174083</td>\n",
       "      <td>0.210648</td>\n",
       "      <td>0.218459</td>\n",
       "      <td>0.088202</td>\n",
       "      <td>0.200309</td>\n",
       "      <td>0.015257</td>\n",
       "      <td>0.010063</td>\n",
       "      <td>0.014033</td>\n",
       "      <td>0.008566</td>\n",
       "      <td>0.017468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.002631</td>\n",
       "      <td>0.933718</td>\n",
       "      <td>0.005695</td>\n",
       "      <td>0.001904</td>\n",
       "      <td>0.001124</td>\n",
       "      <td>0.033052</td>\n",
       "      <td>0.008198</td>\n",
       "      <td>0.001211</td>\n",
       "      <td>0.012467</td>\n",
       "      <td>0.016775</td>\n",
       "      <td>0.144019</td>\n",
       "      <td>0.176092</td>\n",
       "      <td>0.419996</td>\n",
       "      <td>0.137657</td>\n",
       "      <td>0.071999</td>\n",
       "      <td>0.005330</td>\n",
       "      <td>0.003258</td>\n",
       "      <td>0.007363</td>\n",
       "      <td>0.004863</td>\n",
       "      <td>0.012648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80958.0</th>\n",
       "      <td>0.044957</td>\n",
       "      <td>0.352496</td>\n",
       "      <td>0.012269</td>\n",
       "      <td>0.015928</td>\n",
       "      <td>0.008300</td>\n",
       "      <td>0.409278</td>\n",
       "      <td>0.056828</td>\n",
       "      <td>0.011234</td>\n",
       "      <td>0.088709</td>\n",
       "      <td>0.093243</td>\n",
       "      <td>0.331395</td>\n",
       "      <td>0.165370</td>\n",
       "      <td>0.290632</td>\n",
       "      <td>0.013885</td>\n",
       "      <td>0.087260</td>\n",
       "      <td>0.002615</td>\n",
       "      <td>0.002840</td>\n",
       "      <td>0.002756</td>\n",
       "      <td>0.002504</td>\n",
       "      <td>0.007499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80959.0</th>\n",
       "      <td>0.886580</td>\n",
       "      <td>0.029885</td>\n",
       "      <td>0.001693</td>\n",
       "      <td>0.043460</td>\n",
       "      <td>0.012236</td>\n",
       "      <td>0.008936</td>\n",
       "      <td>0.009009</td>\n",
       "      <td>0.006306</td>\n",
       "      <td>0.001894</td>\n",
       "      <td>0.235541</td>\n",
       "      <td>0.250493</td>\n",
       "      <td>0.204294</td>\n",
       "      <td>0.184092</td>\n",
       "      <td>0.009316</td>\n",
       "      <td>0.095288</td>\n",
       "      <td>0.002649</td>\n",
       "      <td>0.003574</td>\n",
       "      <td>0.002417</td>\n",
       "      <td>0.002847</td>\n",
       "      <td>0.009487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80960.0</th>\n",
       "      <td>0.725524</td>\n",
       "      <td>0.101577</td>\n",
       "      <td>0.002193</td>\n",
       "      <td>0.055688</td>\n",
       "      <td>0.010050</td>\n",
       "      <td>0.079787</td>\n",
       "      <td>0.013755</td>\n",
       "      <td>0.007198</td>\n",
       "      <td>0.004228</td>\n",
       "      <td>0.270172</td>\n",
       "      <td>0.137503</td>\n",
       "      <td>0.240337</td>\n",
       "      <td>0.161904</td>\n",
       "      <td>0.004919</td>\n",
       "      <td>0.165948</td>\n",
       "      <td>0.002785</td>\n",
       "      <td>0.004019</td>\n",
       "      <td>0.001819</td>\n",
       "      <td>0.002816</td>\n",
       "      <td>0.007778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80961.0</th>\n",
       "      <td>0.508486</td>\n",
       "      <td>0.321462</td>\n",
       "      <td>0.011151</td>\n",
       "      <td>0.032964</td>\n",
       "      <td>0.011639</td>\n",
       "      <td>0.027733</td>\n",
       "      <td>0.060230</td>\n",
       "      <td>0.020961</td>\n",
       "      <td>0.005375</td>\n",
       "      <td>0.157282</td>\n",
       "      <td>0.279329</td>\n",
       "      <td>0.191692</td>\n",
       "      <td>0.180926</td>\n",
       "      <td>0.008948</td>\n",
       "      <td>0.161806</td>\n",
       "      <td>0.002692</td>\n",
       "      <td>0.003054</td>\n",
       "      <td>0.002269</td>\n",
       "      <td>0.002939</td>\n",
       "      <td>0.009065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80962.0</th>\n",
       "      <td>0.550968</td>\n",
       "      <td>0.157973</td>\n",
       "      <td>0.007150</td>\n",
       "      <td>0.152399</td>\n",
       "      <td>0.010979</td>\n",
       "      <td>0.073657</td>\n",
       "      <td>0.018118</td>\n",
       "      <td>0.016144</td>\n",
       "      <td>0.012613</td>\n",
       "      <td>0.154999</td>\n",
       "      <td>0.367239</td>\n",
       "      <td>0.174375</td>\n",
       "      <td>0.160868</td>\n",
       "      <td>0.010337</td>\n",
       "      <td>0.108932</td>\n",
       "      <td>0.002700</td>\n",
       "      <td>0.003543</td>\n",
       "      <td>0.002865</td>\n",
       "      <td>0.003392</td>\n",
       "      <td>0.010750</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80963 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                1         2         3         4         5         6         7  \\\n",
       "0                                                                               \n",
       "0.0      0.590949  0.045722  0.019597  0.036334  0.006759  0.234161  0.023612   \n",
       "1.0      0.224058  0.007389  0.000600  0.003432  0.000876  0.756404  0.000719   \n",
       "2.0      0.010895  0.313081  0.016462  0.013692  0.455680  0.016656  0.087140   \n",
       "3.0      0.009238  0.242052  0.035775  0.020205  0.033555  0.094468  0.470154   \n",
       "4.0      0.002631  0.933718  0.005695  0.001904  0.001124  0.033052  0.008198   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80958.0  0.044957  0.352496  0.012269  0.015928  0.008300  0.409278  0.056828   \n",
       "80959.0  0.886580  0.029885  0.001693  0.043460  0.012236  0.008936  0.009009   \n",
       "80960.0  0.725524  0.101577  0.002193  0.055688  0.010050  0.079787  0.013755   \n",
       "80961.0  0.508486  0.321462  0.011151  0.032964  0.011639  0.027733  0.060230   \n",
       "80962.0  0.550968  0.157973  0.007150  0.152399  0.010979  0.073657  0.018118   \n",
       "\n",
       "                8         9        10        11        12        13        14  \\\n",
       "0                                                                               \n",
       "0.0      0.017325  0.025541  0.254323  0.110100  0.157668  0.360964  0.027947   \n",
       "1.0      0.000756  0.005767  0.172469  0.098551  0.113473  0.485997  0.025722   \n",
       "2.0      0.008610  0.077785  0.113593  0.104693  0.133691  0.131544  0.041658   \n",
       "3.0      0.029190  0.065364  0.042914  0.174083  0.210648  0.218459  0.088202   \n",
       "4.0      0.001211  0.012467  0.016775  0.144019  0.176092  0.419996  0.137657   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80958.0  0.011234  0.088709  0.093243  0.331395  0.165370  0.290632  0.013885   \n",
       "80959.0  0.006306  0.001894  0.235541  0.250493  0.204294  0.184092  0.009316   \n",
       "80960.0  0.007198  0.004228  0.270172  0.137503  0.240337  0.161904  0.004919   \n",
       "80961.0  0.020961  0.005375  0.157282  0.279329  0.191692  0.180926  0.008948   \n",
       "80962.0  0.016144  0.012613  0.154999  0.367239  0.174375  0.160868  0.010337   \n",
       "\n",
       "               15        16        17        18        19        20  \n",
       "0                                                                    \n",
       "0.0      0.065056  0.004690  0.003224  0.003101  0.004269  0.008657  \n",
       "1.0      0.089703  0.002135  0.001354  0.001644  0.001947  0.007005  \n",
       "2.0      0.372583  0.028839  0.018939  0.016940  0.014645  0.022876  \n",
       "3.0      0.200309  0.015257  0.010063  0.014033  0.008566  0.017468  \n",
       "4.0      0.071999  0.005330  0.003258  0.007363  0.004863  0.012648  \n",
       "...           ...       ...       ...       ...       ...       ...  \n",
       "80958.0  0.087260  0.002615  0.002840  0.002756  0.002504  0.007499  \n",
       "80959.0  0.095288  0.002649  0.003574  0.002417  0.002847  0.009487  \n",
       "80960.0  0.165948  0.002785  0.004019  0.001819  0.002816  0.007778  \n",
       "80961.0  0.161806  0.002692  0.003054  0.002269  0.002939  0.009065  \n",
       "80962.0  0.108932  0.002700  0.003543  0.002865  0.003392  0.010750  \n",
       "\n",
       "[80963 rows x 20 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = pd.read_csv(args.save_dir + 'preds_XL_trans.csv', sep='\\t', index_col='0')\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = torch.tensor(np.array(preds.reset_index())).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = torch.tensor(np.array(preds)).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df = pd.DataFrame(('val_ATT_loss', 'val_VAL_loss', 'val_ATT_acc', 'val_VAL_acc', 'val_VAL_acc_k', 'val_VAL_jac_k'), columns=['name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ATT_loss = F.cross_entropy(data_XL['all'].y[data_XL['all'].train_mask][:,:9], \n",
    "                pred[data_XL['all'].train_mask][:,:9]).cpu().detach().item()\n",
    "train_VAL_loss = F.cross_entropy(data_XL['all'].y[data_XL['all'].train_mask][:,9:], \n",
    "                pred[data_XL['all'].train_mask][:,9:]).cpu().detach().item()\n",
    "\n",
    "train_ATT_acc = compute_1_accuracy(data_XL['all'].y[data_XL['all'].train_mask][:,:9], \n",
    "                pred[data_XL['all'].train_mask][:,:9])\n",
    "train_VAL_acc = compute_1_accuracy(data_XL['all'].y[data_XL['all'].train_mask][:,9:], \n",
    "                pred[data_XL['all'].train_mask][:,9:])\n",
    "train_VAL_acc_k = compute_k_accuracy(data_XL['all'].y[data_XL['all'].train_mask][:,9:].cpu(),  \n",
    "                pred[data_XL['all'].train_mask][:,9:].cpu(),3)\n",
    "train_VAL_jac_k = compute_jaccard_index(data_XL['all'].y[data_XL['all'].train_mask][:,9:].cpu(),  \n",
    "                pred[data_XL['all'].train_mask][:,9:].cpu(),3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df['train'] = pd.DataFrame((train_ATT_loss, train_VAL_loss, train_ATT_acc, train_VAL_acc, train_VAL_acc_k, train_VAL_jac_k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_ATT_loss = F.cross_entropy(data_XL['all'].y[(data_XL['all'].att_lab) * data_XL['all'].val_mask][:,:9], \n",
    "                pred[(data_XL['all'].att_lab) * data_XL['all'].val_mask][:,:9]).cpu().detach().item()\n",
    "val_VAL_loss = F.cross_entropy(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:], \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:]).cpu().detach().item()\n",
    "\n",
    "val_ATT_acc = compute_1_accuracy(data_XL['all'].y[(data_XL['all'].att_lab) * data_XL['all'].val_mask][:,:9], \n",
    "                pred[(data_XL['all'].att_lab) * data_XL['all'].val_mask][:,:9])\n",
    "val_VAL_acc = compute_1_accuracy(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:], \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:])\n",
    "val_VAL_acc_k = compute_k_accuracy(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:].cpu(),3)\n",
    "val_VAL_jac_k = compute_jaccard_index(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:].cpu(),3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df['val'] = pd.DataFrame((val_ATT_loss, val_VAL_loss, val_ATT_acc, val_VAL_acc, val_VAL_acc_k, val_VAL_jac_k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ATT_loss = F.cross_entropy(data_XL['all'].y[(data_XL['all'].att_lab) * data_XL['all'].test_mask][:,:9], \n",
    "                pred[(data_XL['all'].att_lab) * data_XL['all'].test_mask][:,:9]).cpu().detach().item()\n",
    "test_VAL_loss = F.cross_entropy(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:], \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:]).cpu().detach().item()\n",
    "\n",
    "test_ATT_acc = compute_1_accuracy(data_XL['all'].y[(data_XL['all'].att_lab) * data_XL['all'].test_mask][:,:9], \n",
    "                pred[(data_XL['all'].att_lab) * data_XL['all'].test_mask][:,:9])\n",
    "test_VAL_acc = compute_1_accuracy(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:], \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:])\n",
    "test_VAL_acc_k = compute_k_accuracy(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:].cpu(),3)\n",
    "test_VAL_jac_k = compute_jaccard_index(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:].cpu(),3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df['test'] = pd.DataFrame((test_ATT_loss, test_VAL_loss, test_ATT_acc, test_VAL_acc, test_VAL_acc_k, test_VAL_jac_k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>train</th>\n",
       "      <th>val</th>\n",
       "      <th>test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>val_ATT_loss</td>\n",
       "      <td>1.758587</td>\n",
       "      <td>1.778306</td>\n",
       "      <td>1.780264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>val_VAL_loss</td>\n",
       "      <td>2.245272</td>\n",
       "      <td>2.243987</td>\n",
       "      <td>2.243471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>val_ATT_acc</td>\n",
       "      <td>91.356211</td>\n",
       "      <td>95.399798</td>\n",
       "      <td>95.253288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>val_VAL_acc</td>\n",
       "      <td>77.578010</td>\n",
       "      <td>78.341794</td>\n",
       "      <td>78.917018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>val_VAL_acc_k</td>\n",
       "      <td>97.951422</td>\n",
       "      <td>98.040045</td>\n",
       "      <td>98.199719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>val_VAL_jac_k</td>\n",
       "      <td>0.726554</td>\n",
       "      <td>0.724760</td>\n",
       "      <td>0.723887</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            name      train        val       test\n",
       "0   val_ATT_loss   1.758587   1.778306   1.780264\n",
       "1   val_VAL_loss   2.245272   2.243987   2.243471\n",
       "2    val_ATT_acc  91.356211  95.399798  95.253288\n",
       "3    val_VAL_acc  77.578010  78.341794  78.917018\n",
       "4  val_VAL_acc_k  97.951422  98.040045  98.199719\n",
       "5  val_VAL_jac_k   0.726554   0.724760   0.723887"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df.to_csv(args.save_dir+'eval_metrics_XL_trans.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Per-Class Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_k_confusion_matrix(y, y_pred, k=3):\n",
    "    dim = y.shape[-1]\n",
    "    y = y.topk(k=k, axis=1)[1]\n",
    "    y_pred = y_pred.topk(k=k, axis=1)[1]\n",
    "    conf = np.zeros((dim, dim))\n",
    "    for i in range(k):\n",
    "        for j in range(k):\n",
    "            conf = np.add(conf, confusion_matrix(y[:,i], y_pred[:,j], labels = range(dim)))\n",
    "    return conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ATT_conf = confusion_matrix(data_XL['all'].y[(data_XL['all'].att_lab) * data_XL['all'].test_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                pred[(data_XL['all'].att_lab) * data_XL['all'].test_mask][:,:9].argmax(axis=1).cpu())\n",
    "test_VAL_conf = confusion_matrix(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "test_VAL_conf_k = (top_k_confusion_matrix(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1348,    7,    0,   11,    1,   15,    6,    1,    0],\n",
       "       [   5, 3151,    0,    6,    0,   11,   38,    0,    0],\n",
       "       [   0,    0,  252,    0,    0,    0,   13,    0,    0],\n",
       "       [  31,    5,    0,  586,    0,    1,   13,    1,    0],\n",
       "       [   7,   20,    0,    3, 2636,    0,   35,    0,    3],\n",
       "       [  16,   27,    0,    2,    1, 1533,    4,    0,    0],\n",
       "       [   2,   32,    0,    1,   10,    0, 2548,    0,    0],\n",
       "       [  12,    3,  107,   15,    0,    0,  135,   41,    0],\n",
       "       [   0,    5,    0,    0,    4,    1,    0,    0,  146]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 582,   15,   25,    6,    0,   52,    0,    0,    0,    0,    0],\n",
       "       [  41,  518,  135,   29,    0,  126,    0,    0,    0,    0,    0],\n",
       "       [  27,   76, 1119,   36,    0,  349,    0,    0,    0,    0,    0],\n",
       "       [  29,   25,   72,  323,    0,   87,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [  33,   22,  272,   23,    0, 3065,    0,    0,    0,    0,    0],\n",
       "       [   1,    0,    2,    0,    0,    6,    1,    0,    0,    2,    0],\n",
       "       [   0,    0,    0,    0,    0,    1,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    1,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    2,    0,    0,    3,    1,    0,    0,    3,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1360,  827, 1151,  703,    0,  909,    3,    0,    0,    0,    0],\n",
       "       [ 888, 1790, 2244, 1552,    4, 1730,    0,    0,    0,    0,    0],\n",
       "       [ 952, 1892, 5786, 4237,    5, 5208,    4,    0,    0,    0,    0],\n",
       "       [ 844, 1652, 4763, 4243,    5, 4296,    1,    0,    0,    0,    0],\n",
       "       [   0,    8,   17,   16,    3,   10,    0,    0,    0,    0,    0],\n",
       "       [ 834, 1379, 5289, 3894,    1, 5270,    4,    0,    0,    0,    0],\n",
       "       [   9,    2,   25,    8,    0,   22,   13,    1,    3,   13,    0],\n",
       "       [   3,    0,    3,    0,    0,    6,    5,    0,    1,    3,    0],\n",
       "       [   1,    0,   11,    1,    0,    7,    9,    1,    2,   10,    0],\n",
       "       [   2,    1,   13,    1,    0,   11,   12,    1,    3,   13,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_ATT_conf = confusion_matrix(data_XL['all'].y[(data_XL['all'].att_lab) * data_XL['all'].val_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                pred[(data_XL['all'].att_lab) * data_XL['all'].val_mask][:,:9].argmax(axis=1).cpu())\n",
    "val_VAL_conf = confusion_matrix(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "val_VAL_conf_k = (top_k_confusion_matrix(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1322,    3,    0,    9,    1,    7,   10,    0,    0],\n",
       "       [   7, 3237,    0,    3,    0,    8,   44,    0,    0],\n",
       "       [   0,    0,  269,    0,    0,    0,   11,    0,    0],\n",
       "       [  32,    7,    0,  629,    0,    0,   10,    2,    0],\n",
       "       [   5,   34,    0,    3, 2604,    3,   36,    0,    2],\n",
       "       [  18,   28,    0,    1,    3, 1436,    2,    0,    0],\n",
       "       [   0,   24,    1,    1,   15,    0, 2589,    0,    0],\n",
       "       [  13,    4,   84,    7,    0,    0,  138,   40,    0],\n",
       "       [   0,   10,    0,    0,    3,    2,    1,    0,  151]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 541,   32,   33,   18,    0,   43,    0,    0,    0,    0,    0],\n",
       "       [  47,  504,  117,   53,    0,  108,    0,    0,    0,    0,    0],\n",
       "       [  33,   61, 1167,   43,    0,  339,    0,    0,    0,    1,    0],\n",
       "       [  37,   28,   89,  367,    0,   77,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    2,    2,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [  29,   34,  280,   16,    0, 2974,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    1,    0,    0,    4,    0,    0,    0,    0,    0],\n",
       "       [   1,    0,    0,    0,    0,    1,    0,    0,    0,    1,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    1,    0],\n",
       "       [   0,    0,    1,    0,    0,    1,    3,    0,    0,    3,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1365,  823, 1148,  730,    1,  947,    2,    0,    0,    0,    0],\n",
       "       [ 935, 1807, 2336, 1648,    7, 1802,    0,    0,    0,    0,    0],\n",
       "       [ 969, 1886, 5769, 4182,   12, 5146,    5,    0,    0,    1,    0],\n",
       "       [ 849, 1654, 4675, 4178,   14, 4196,    1,    0,    0,    0,    0],\n",
       "       [   0,    9,   20,   22,    6,    9,    0,    0,    0,    0,    0],\n",
       "       [ 812, 1315, 5212, 3867,    5, 5229,    5,    0,    0,    1,    0],\n",
       "       [   9,    7,   35,   15,    2,   32,   10,    1,    1,    8,    0],\n",
       "       [   1,    1,    1,    1,    0,    3,    1,    0,    0,    1,    0],\n",
       "       [   3,    2,   10,    2,    1,   10,    9,    1,    1,    9,    0],\n",
       "       [   4,    2,   12,    1,    0,   11,    9,    1,    1,   10,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ATT_conf = confusion_matrix(data_XL['all'].y[data_XL['all'].train_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                                  pred[data_XL['all'].train_mask][:,:9].argmax(axis=1).cpu())\n",
    "train_VAL_conf = confusion_matrix(data_XL['all'].y[data_XL['all'].train_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                                 pred[data_XL['all'].train_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "train_VAL_conf_k = (top_k_confusion_matrix(data_XL['all'].y[data_XL['all'].train_mask][:,9:].cpu(),  \n",
    "                                pred[data_XL['all'].train_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1438,   19,    0,    9,    1,   25,    9,    0,    0],\n",
       "       [   1, 2621,    0,    1,    0,    9,    4,    0,    0],\n",
       "       [   2,    2,  121,    0,    0,    0,   14,    0,    0],\n",
       "       [  31,   14,    0,  422,    0,    1,   11,    1,    0],\n",
       "       [   6,   21,    0,    1, 2010,    1,    8,    2,    2],\n",
       "       [   4,   31,    0,    0,    5, 1467,    0,    0,    0],\n",
       "       [   2,   74,    0,    3,   29,    0, 2348,    1,    0],\n",
       "       [  27,   12,  141,   34,    0,    0,  422,   49,    0],\n",
       "       [   0,   14,    0,    0,    3,    3,    0,    0,   93]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 825,   34,   50,    9,    0,   94,    0,    0,    0,    0,    0],\n",
       "       [  57,  795,  235,   73,    0,  218,    0,    0,    0,    0,    0],\n",
       "       [  42,   89, 1895,   55,    0,  734,    0,    0,    0,    0,    0],\n",
       "       [  51,   59,  150,  651,    0,  132,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    0,    2,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [  44,   31,  368,   45,    0, 4795,    0,    0,    0,    0,    0],\n",
       "       [   1,    0,    1,    1,    0,    8,    8,    0,    0,    2,    0],\n",
       "       [   1,    0,    0,    0,    0,    0,    1,    0,    0,    0,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    1,    0,    0,    0,    0],\n",
       "       [   0,    0,    2,    0,    0,    1,    3,    0,    0,    6,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1997, 1140, 1683, 1082,    0, 1484,    2,    0,    0,    1,    0],\n",
       "       [1267, 3002, 3941, 2757,    5, 3140,    0,    0,    0,    0,    0],\n",
       "       [1384, 3100, 9510, 6985,   10, 8601,    2,    0,    0,    0,    0],\n",
       "       [1273, 2663, 7766, 6978,   14, 7039,    1,    0,    0,    0,    0],\n",
       "       [   0,   25,   50,   53,   10,   24,    0,    0,    0,    0,    0],\n",
       "       [1210, 2145, 8480, 6351,    3, 8571,    3,    0,    0,    0,    0],\n",
       "       [  17,    2,   43,   14,    0,   44,   29,    1,    5,   19,    0],\n",
       "       [   8,    0,    9,    2,    0,   14,   14,    1,    0,    6,    0],\n",
       "       [   1,    0,   13,    0,    0,    9,   16,    0,    5,   13,    0],\n",
       "       [   4,    1,   17,    0,    0,   15,   23,    1,    5,   18,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat([pd.DataFrame(train_ATT_conf),pd.DataFrame(val_ATT_conf),pd.DataFrame(test_ATT_conf)],axis=1).to_csv(args.save_dir+'confusion_matrix_ATT_XL_trans.csv')\n",
    "pd.concat([pd.DataFrame(train_VAL_conf),pd.DataFrame(val_VAL_conf),pd.DataFrame(test_VAL_conf)],axis=1).to_csv(args.save_dir+'confusion_matrix_VAL_XL_trans.csv')\n",
    "pd.concat([pd.DataFrame(train_VAL_conf_k),pd.DataFrame(val_VAL_conf_k),pd.DataFrame(test_VAL_conf_k)],axis=1).to_csv(args.save_dir+'confusion_matrix_VAL_k_XL_trans.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics(confusion_matrix, classes):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val/(row.sum()+0.000000001)\n",
    "        recall = val/(col.sum()+0.000000001)\n",
    "        F1 = 2*(precision*recall)/(precision+recall+0.000000001)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics_k(confusion_matrix, classes, k=3):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val*k/(row.sum()+0.000000001)\n",
    "        recall = val*k/(col.sum()+0.000000001)\n",
    "        F1 = 2*(precision*recall)/(precision+recall+0.000000001)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = ['Criterion i', 'Criterion ii', 'Criterion iii', 'Criterion iv', 'Criterion v', 'Criterion vi', \n",
    "              'Criterion vii', 'Criterion viii', 'Criterion ix', 'Criterion x', 'Others']\n",
    "categories = ['Building Elements',\n",
    " 'Urban Form Elements',\n",
    " 'Gastronomy',\n",
    " 'Interior Scenery',\n",
    " 'Natural Features and Land-scape Scenery',\n",
    " 'Monuments and Buildings',\n",
    " 'Peoples Activity and Association',\n",
    " 'Artifact Products',\n",
    " 'Urban Scenery']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_dict = {}\n",
    "metrics_dict['test_ATT'] = per_class_metrics(test_ATT_conf, categories)\n",
    "metrics_dict['val_ATT'] = per_class_metrics(val_ATT_conf, categories)\n",
    "metrics_dict['test_VAL'] = per_class_metrics(test_VAL_conf, classes)\n",
    "metrics_dict['val_VAL'] = per_class_metrics(val_VAL_conf, classes)\n",
    "metrics_dict['test_VAL_k'] = per_class_metrics_k(test_VAL_conf_k, classes)\n",
    "metrics_dict['val_VAL_k'] = per_class_metrics_k(val_VAL_conf_k, classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df = pd.DataFrame.from_dict({(i,j): metrics_dict[i][j] \n",
    "                           for i in metrics_dict.keys() \n",
    "                           for j in metrics_dict[i].keys()},\n",
    "                       orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">test_ATT</th>\n",
       "      <th>Building Elements</th>\n",
       "      <td>0.970482</td>\n",
       "      <td>0.948628</td>\n",
       "      <td>0.959431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Urban Form Elements</th>\n",
       "      <td>0.981314</td>\n",
       "      <td>0.969538</td>\n",
       "      <td>0.975391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gastronomy</th>\n",
       "      <td>0.950943</td>\n",
       "      <td>0.701950</td>\n",
       "      <td>0.807692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Interior Scenery</th>\n",
       "      <td>0.919937</td>\n",
       "      <td>0.939103</td>\n",
       "      <td>0.929421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Natural Features and Land-scape Scenery</th>\n",
       "      <td>0.974852</td>\n",
       "      <td>0.993967</td>\n",
       "      <td>0.984317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">val_VAL_k</th>\n",
       "      <th>Criterion vii</th>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.370370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Criterion viii</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Criterion ix</th>\n",
       "      <td>0.062500</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.117647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Criterion x</th>\n",
       "      <td>0.588235</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.740741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Others</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>62 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   precision    recall  \\\n",
       "test_ATT  Building Elements                         0.970482  0.948628   \n",
       "          Urban Form Elements                       0.981314  0.969538   \n",
       "          Gastronomy                                0.950943  0.701950   \n",
       "          Interior Scenery                          0.919937  0.939103   \n",
       "          Natural Features and Land-scape Scenery   0.974852  0.993967   \n",
       "...                                                      ...       ...   \n",
       "val_VAL_k Criterion vii                             0.250000  0.714286   \n",
       "          Criterion viii                            0.000000  0.000000   \n",
       "          Criterion ix                              0.062500  1.000000   \n",
       "          Criterion x                               0.588235  1.000000   \n",
       "          Others                                    0.000000  0.000000   \n",
       "\n",
       "                                                         F1  \n",
       "test_ATT  Building Elements                        0.959431  \n",
       "          Urban Form Elements                      0.975391  \n",
       "          Gastronomy                               0.807692  \n",
       "          Interior Scenery                         0.929421  \n",
       "          Natural Features and Land-scape Scenery  0.984317  \n",
       "...                                                     ...  \n",
       "val_VAL_k Criterion vii                            0.370370  \n",
       "          Criterion viii                           0.000000  \n",
       "          Criterion ix                             0.117647  \n",
       "          Criterion x                              0.740741  \n",
       "          Others                                   0.000000  \n",
       "\n",
       "[62 rows x 3 columns]"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df.to_csv(args.save_dir+'per_class_metrics_XL_trans.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}