HeteroGNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HeteroGNN Models to Compute Heritage Values and Attributes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import os.path as osp\n",
    "from itertools import product\n",
    "from typing import Callable, List, Optional\n",
    "\n",
    "import numpy as np\n",
    "import scipy.sparse as sp\n",
    "\n",
    "from torch_geometric.data import (\n",
    "    HeteroData,\n",
    "    InMemoryDataset,\n",
    "    download_url,\n",
    "    extract_zip,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from argparse import Namespace\n",
    "from collections import Counter\n",
    "import json\n",
    "import os\n",
    "import re\n",
    "import string\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import random\n",
    "\n",
    "import torch\n",
    "torch.manual_seed(1337)\n",
    "torch.backends.cudnn.benchmark = True\n",
    "\n",
    "from torch_geometric.transforms import RandomLinkSplit, ToUndirected\n",
    "import torch_geometric.transforms as T\n",
    "from torch_geometric.nn import GATConv, Linear, to_hetero\n",
    "from torch_geometric.utils import to_undirected\n",
    "\n",
    "import torch.nn.functional as F\n",
    "from torch_geometric import seed_everything\n",
    "\n",
    "from tqdm import tqdm\n",
    "import torch.optim as optim\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.nn import HeteroConv, Linear, SAGEConv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'D:\\\\surf\\\\TUD\\\\Paper\\\\Venice_Graph'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch version 1.10.2\n",
      "GPU-enabled installation? True\n"
     ]
    }
   ],
   "source": [
    "print(\"PyTorch version {}\".format(torch.__version__))\n",
    "print(\"GPU-enabled installation? {}\".format(torch.cuda.is_available()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "#device = torch.device(\"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed_everywhere(seed, cuda):\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    random.seed(seed)\n",
    "    if cuda:\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "\n",
    "def handle_dirs(dirpath):\n",
    "    if not os.path.exists(dirpath):\n",
    "        os.makedirs(dirpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = Namespace(\n",
    "    # Data and Path information\n",
    "    path = 'dataset/Venice',\n",
    "    save_dir='model_storage/HeteroGNN/',\n",
    "    model_state_file='model.pth',\n",
    "    \n",
    "    # Model hyper parameters\n",
    "    hidden_channels = 32,\n",
    "    num_layers = 3,\n",
    "    k=3,\n",
    "    use_gdc=True,\n",
    "    \n",
    "    # Training hyper parameters\n",
    "    sample_nodes = 25,\n",
    "    batch_size=32,\n",
    "    early_stopping_criteria=100,\n",
    "    learning_rate=0.001,\n",
    "    l2=2e-4,\n",
    "    dropout_p=0,\n",
    "    num_epochs=300,\n",
    "    seed=42,\n",
    "    \n",
    "    # Runtime options\n",
    "    catch_keyboard_interrupt=True,\n",
    "    cuda=True,\n",
    "    expand_filepaths_to_save_dir=True,\n",
    "    reload_from_files=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expanded filepaths: \n",
      "\tmodel_storage/HeteroGNN/model.pth\n"
     ]
    }
   ],
   "source": [
    "if args.expand_filepaths_to_save_dir:\n",
    "    args.model_state_file = os.path.join(args.save_dir,\n",
    "                                         args.model_state_file)\n",
    "    \n",
    "    print(\"Expanded filepaths: \")\n",
    "    print(\"\\t{}\".format(args.model_state_file))\n",
    "\n",
    "# handle dirs\n",
    "handle_dirs(args.save_dir)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Constructing Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VEN(InMemoryDataset):\n",
    "    r\"\"\"A subset of Flickr post collected in Venice annotated with Heritage \n",
    "    Values and Attributes, as collected in the `\"Heri-Graphs: A Workflow of \n",
    "    Creating Datasets for Multi-modal Machine Learning on Graphs of Heritage \n",
    "    Values and Attributes with Social Media\" <https://arxiv.org/abs/2205.07545>`\n",
    "    paper.\n",
    "    VEN is a heterogeneous graph containing two types of nodes - nodes with only \n",
    "    visual features 'vis_only' (1,190 nodes), nodes with both visual and textual\n",
    "    features 'vis_tex' (1,761 nodes) and four types of links - social similarity\n",
    "    'SOC' (488,103 links), spatial similarity (445,779 links), temporal similarity\n",
    "    (501,191 links), and simple composed link (1,071,977 links).\n",
    "    Vis_only nodes are represented with 982-dimensional visual features and are\n",
    "    divided into 9 heritage attribute categories \n",
    "    ('architectural elements', 'form', 'gastronomy', 'interior',\n",
    "    'landscape scenery and natural features', 'monuments', 'people', 'product', \n",
    "    'urban scenery').\n",
    "    Vis_text nodes are represented with 1753-dimensional visual and textual \n",
    "    features and are divided into 9 heritage attribute categories plus 11 \n",
    "    heritage value categories ('criterion i-x', 'other').\n",
    "    Both types of nodes are also merged into a single type of node 'all' with \n",
    "    1753-dimensional features and 20-dimensional label categories.\n",
    "\n",
    "    Args:\n",
    "        root (string): Root directory where the dataset should be saved.\n",
    "        transform (callable, optional): A function/transform that takes in an\n",
    "            :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            every access. (default: :obj:`None`)\n",
    "        pre_transform (callable, optional): A function/transform that takes in\n",
    "            an :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            being saved to disk. (default: :obj:`None`)\n",
    "    \n",
    "    Stats:\n",
    "            * - #nodes\n",
    "              - #edges\n",
    "              - #features\n",
    "              - #classes\n",
    "            * - 2,951\n",
    "              - 1,071,977\n",
    "              - 1753\n",
    "              - 20\n",
    "    \"\"\"\n",
    "\n",
    "    url = 'https://drive.google.com/uc?export=download&id=1sxcKiZr1YGDv06wr03nsk5HVZledgzi9'\n",
    "\n",
    "    def __init__(self, root: str, transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "        \n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            'A_simp.npz', 'A_SOC.npz', 'A_SPA.npz', 'A_TEM.npz', 'labels.npz',\n",
    "            'node_types.npy', 'Textual_Features.npy', 'train_val_test_idx.npz',\n",
    "            'Visual_Features.npy'\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        os.remove(path)\n",
    "\n",
    "    def process(self):\n",
    "        data = HeteroData()\n",
    "\n",
    "        node_types = ['vis_only', 'vis_tex']\n",
    "        link_types = ['SOC', 'SPA', 'TEM', 'simp']\n",
    "\n",
    "        vis = np.load(osp.join(self.raw_dir, 'Visual_Features.npy'),allow_pickle=True)[:,2:].astype(float)\n",
    "        tex = np.load(osp.join(self.raw_dir, 'Textual_Features.npy'),allow_pickle=True)[:,5:].astype(float)\n",
    "\n",
    "        x = np.hstack([vis,np.nan_to_num(tex)])\n",
    "\n",
    "\n",
    "        node_type_idx = np.load(osp.join(self.raw_dir, 'node_types.npy'))\n",
    "        node_type_idx = torch.from_numpy(node_type_idx).to(torch.long)\n",
    "\n",
    "        data['vis_only'].num_nodes = int((node_type_idx == 0).sum())\n",
    "        data['vis_tex'].num_nodes = int((node_type_idx == 1).sum())\n",
    "        data['all'].num_nodes = len(node_type_idx)\n",
    "\n",
    "        data['vis_only'].x = torch.from_numpy(vis[node_type_idx==0]).to(torch.float)\n",
    "        data['vis_tex'].x = torch.from_numpy(x[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].x = torch.from_numpy(x).to(torch.float)\n",
    "\n",
    "\n",
    "        y_s = np.load(osp.join(self.raw_dir, 'labels.npz'), allow_pickle=True)\n",
    "        att_lab = y_s['ATT_LAB'][:,1:10].astype(float)\n",
    "        val_lab = np.nan_to_num(y_s['VAL_LAB'][:,2:13].astype(float))\n",
    "        ys = np.hstack([att_lab, val_lab])\n",
    "\n",
    "        data['vis_only'].y = torch.from_numpy(att_lab[node_type_idx==0]).to(torch.float)\n",
    "        data['vis_tex'].y = torch.from_numpy(ys[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].y = torch.from_numpy(ys).to(torch.float)\n",
    "\n",
    "        data.node_type = node_type_idx\n",
    "\n",
    "        split = np.load(osp.join(self.raw_dir, 'train_val_test_idx.npz'))\n",
    "        for name in ['train', 'val', 'test']:\n",
    "            idx = split[f'{name}_idx']\n",
    "            idx = torch.from_numpy(idx).to(torch.long)\n",
    "            mask = torch.zeros(data['all'].num_nodes, dtype=torch.bool)\n",
    "            mask[idx] = True\n",
    "            data['all'][f'{name}_mask'] = mask\n",
    "            data['vis_only'][f'{name}_mask'] = mask[node_type_idx==0]\n",
    "            data['vis_tex'][f'{name}_mask'] = mask[node_type_idx==1]\n",
    "\n",
    "        \n",
    "        s = {}\n",
    "        s['vis_only'] = np.arange(len(x))[node_type_idx==0]\n",
    "        s['vis_tex'] = np.arange(len(x))[node_type_idx==1]\n",
    "\n",
    "        for link in link_types:\n",
    "            A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz')).tocoo()\n",
    "            if A_sub.nnz>0:\n",
    "                row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                data['all', f'{link}_link', 'all'].edge_index = torch.stack([row, col], dim=0)\n",
    "                data['all', f'{link}_link', 'all'].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "        for src, dst in product(node_types, node_types):\n",
    "            for link in link_types:\n",
    "                A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz'))[s[src]][:,s[dst]].tocoo()\n",
    "                if A_sub.nnz>0:\n",
    "                    row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                    col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                    data[src, f'{link}_link', dst].edge_index = torch.stack([row, col], dim=0)\n",
    "                    data[src, f'{link}_link', dst].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "\n",
    "        if self.pre_transform is not None:\n",
    "            data = self.pre_transform(data)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.__class__.__name__}()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VEN_XL(InMemoryDataset):\n",
    "    r\"\"\"A large subset of Flickr post collected in Venice annotated with Heritage \n",
    "    Values and Attributes, as collected in the `\"Heri-Graphs: A Workflow of \n",
    "    Creating Datasets for Multi-modal Machine Learning on Graphs of Heritage \n",
    "    Values and Attributes with Social Media\" <https://arxiv.org/abs/2205.07545>`\n",
    "    paper.\n",
    "    VEN_XL is a heterogeneous graph containing two types of nodes - nodes with only \n",
    "    visual features 'vis_only' (31,140 nodes), nodes with both visual and textual\n",
    "    features 'vis_tex' (49,823 nodes) and four types of links - social similarity\n",
    "    'SOC' (76,422,265 links), spatial similarity (202,173,159 links), temporal similarity\n",
    "    (71,135,671 links), and simple composed link (290,091,503 links).\n",
    "    Vis_only nodes are represented with 982-dimensional visual features and are\n",
    "    divided into 9 heritage attribute categories \n",
    "    ('architectural elements', 'form', 'gastronomy', 'interior',\n",
    "    'landscape scenery and natural features', 'monuments', 'people', 'product', \n",
    "    'urban scenery').\n",
    "    Vis_text nodes are represented with 1753-dimensional visual and textual \n",
    "    features and are divided into 9 heritage attribute categories plus 11 \n",
    "    heritage value categories ('criterion i-x', 'other').\n",
    "    Both types of nodes are also merged into a single type of node 'all' with \n",
    "    1753-dimensional features and 20-dimensional label categories.\n",
    "\n",
    "    Args:\n",
    "        root (string): Root directory where the dataset should be saved.\n",
    "        transform (callable, optional): A function/transform that takes in an\n",
    "            :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            every access. (default: :obj:`None`)\n",
    "        pre_transform (callable, optional): A function/transform that takes in\n",
    "            an :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            being saved to disk. (default: :obj:`None`)\n",
    "    \n",
    "    Stats:\n",
    "            * - #nodes\n",
    "              - #edges\n",
    "              - #features\n",
    "              - #classes\n",
    "            * - 80,963\n",
    "              - 290,091,503\n",
    "              - 1753\n",
    "              - 20\n",
    "    \"\"\"\n",
    "\n",
    "    url = 'https://drive.google.com/uc?export=download&id=1QZ5tyUWs6jYjh7mJrsnpou76iy-vb0CA'\n",
    "\n",
    "    def __init__(self, root: str, transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "        \n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            'A_simp.npz', 'A_SOC.npz', 'A_SPA.npz', 'A_TEM.npz', 'labels.npz',\n",
    "            'node_types.npy', 'Textual_Features.npy', 'train_val_test_idx.npz',\n",
    "            'Visual_Features.npy'\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        os.remove(path)\n",
    "\n",
    "    def process(self):\n",
    "        data = HeteroData()\n",
    "\n",
    "        node_types = ['vis_only', 'vis_tex']\n",
    "        link_types = ['SOC', 'SPA', 'TEM', 'simp']\n",
    "\n",
    "        vis = np.load(osp.join(self.raw_dir, 'Visual_Features.npy'),allow_pickle=True)[:,2:].astype(float)\n",
    "        tex = np.load(osp.join(self.raw_dir, 'Textual_Features.npy'),allow_pickle=True)[:,5:].astype(float)\n",
    "\n",
    "        x = np.hstack([vis,np.nan_to_num(tex)])\n",
    "\n",
    "\n",
    "        node_type_idx = np.load(osp.join(self.raw_dir, 'node_types.npy'))\n",
    "        node_type_idx = torch.from_numpy(node_type_idx).to(torch.long)\n",
    "\n",
    "        data['vis_only'].num_nodes = int((node_type_idx == 0).sum())\n",
    "        data['vis_tex'].num_nodes = int((node_type_idx == 1).sum())\n",
    "        data['all'].num_nodes = len(node_type_idx)\n",
    "\n",
    "        data['vis_only'].x = torch.from_numpy(vis[node_type_idx==0]).to(torch.float)\n",
    "        data['vis_tex'].x = torch.from_numpy(x[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].x = torch.from_numpy(x).to(torch.float)\n",
    "\n",
    "\n",
    "        y_s = np.load(osp.join(self.raw_dir, 'labels.npz'), allow_pickle=True)\n",
    "        att_lab = y_s['ATT_LAB'][:,1:10].astype(float)\n",
    "        val_lab = np.nan_to_num(y_s['VAL_LAB'][:,2:13].astype(float))\n",
    "        ys = np.hstack([att_lab, val_lab])\n",
    "\n",
    "        data['vis_only'].y = torch.from_numpy(att_lab[node_type_idx==0]).to(torch.float)\n",
    "        data['vis_tex'].y = torch.from_numpy(ys[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].y = torch.from_numpy(ys).to(torch.float)\n",
    "\n",
    "        data.node_type = node_type_idx\n",
    "\n",
    "        split = np.load(osp.join(self.raw_dir, 'train_val_test_idx.npz'))\n",
    "        for name in ['train', 'val', 'test']:\n",
    "            idx = split[f'{name}_idx']\n",
    "            idx = torch.from_numpy(idx).to(torch.long)\n",
    "            mask = torch.zeros(data['all'].num_nodes, dtype=torch.bool)\n",
    "            mask[idx] = True\n",
    "            data['all'][f'{name}_mask'] = mask\n",
    "            data['vis_only'][f'{name}_mask'] = mask[node_type_idx==0]\n",
    "            data['vis_tex'][f'{name}_mask'] = mask[node_type_idx==1]\n",
    "\n",
    "        \n",
    "        s = {}\n",
    "        s['vis_only'] = np.arange(len(x))[node_type_idx==0]\n",
    "        s['vis_tex'] = np.arange(len(x))[node_type_idx==1]\n",
    "\n",
    "        for link in link_types:\n",
    "            A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz')).tocoo()\n",
    "            if A_sub.nnz>0:\n",
    "                row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                data['all', f'{link}_link', 'all'].edge_index = torch.stack([row, col], dim=0)\n",
    "                data['all', f'{link}_link', 'all'].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "        for src, dst in product(node_types, node_types):\n",
    "            for link in link_types:\n",
    "                A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz'))[s[src]][:,s[dst]].tocoo()\n",
    "                if A_sub.nnz>0:\n",
    "                    row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                    col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                    data[src, f'{link}_link', dst].edge_index = torch.stack([row, col], dim=0)\n",
    "                    data[src, f'{link}_link', dst].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "\n",
    "        if self.pre_transform is not None:\n",
    "            data = self.pre_transform(data)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.__class__.__name__}()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VEN_links(InMemoryDataset):\n",
    "    r\"\"\"A subset of Flickr post collected in Venice annotated with Heritage \n",
    "    Values and Attributes, as collected in the `\"Heri-Graphs: A Workflow of \n",
    "    Creating Datasets for Multi-modal Machine Learning on Graphs of Heritage \n",
    "    Values and Attributes with Social Media\" <https://arxiv.org/abs/2205.07545>`\n",
    "    paper.\n",
    "    VEN is a heterogeneous graph containing two types of nodes - nodes with only \n",
    "    visual features 'vis_only' (1,190 nodes), nodes with both visual and textual\n",
    "    features 'vis_tex' (1,761 nodes) and four types of links - social similarity\n",
    "    'SOC' (488,103 links), spatial similarity (445,779 links), temporal similarity\n",
    "    (501,191 links), and simple composed link (1,071,977 links).\n",
    "    Vis_only nodes are represented with 982-dimensional visual features and are\n",
    "    divided into 9 heritage attribute categories \n",
    "    ('architectural elements', 'form', 'gastronomy', 'interior',\n",
    "    'landscape scenery and natural features', 'monuments', 'people', 'product', \n",
    "    'urban scenery').\n",
    "    Vis_text nodes are represented with 1753-dimensional visual and textual \n",
    "    features and are divided into 9 heritage attribute categories plus 11 \n",
    "    heritage value categories ('criterion i-x', 'other').\n",
    "    Both types of nodes are also merged into a single type of node 'all' with \n",
    "    1753-dimensional features and 20-dimensional label categories.\n",
    "\n",
    "    Args:\n",
    "        root (string): Root directory where the dataset should be saved.\n",
    "        transform (callable, optional): A function/transform that takes in an\n",
    "            :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            every access. (default: :obj:`None`)\n",
    "        pre_transform (callable, optional): A function/transform that takes in\n",
    "            an :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            being saved to disk. (default: :obj:`None`)\n",
    "    \n",
    "    Stats:\n",
    "            * - #nodes\n",
    "              - #edges\n",
    "              - #features\n",
    "              - #classes\n",
    "            * - 2,951\n",
    "              - 1,071,977\n",
    "              - 1753\n",
    "              - 20\n",
    "    \"\"\"\n",
    "\n",
    "    url = 'https://drive.google.com/uc?export=download&id=1sxcKiZr1YGDv06wr03nsk5HVZledgzi9'\n",
    "\n",
    "    def __init__(self, root: str, transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "        \n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            'A_simp.npz', 'A_SOC.npz', 'A_SPA.npz', 'A_TEM.npz', 'labels.npz',\n",
    "            'node_types.npy', 'Textual_Features.npy', 'train_val_test_idx.npz',\n",
    "            'Visual_Features.npy'\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        os.remove(path)\n",
    "\n",
    "    def process(self):\n",
    "        data = HeteroData()\n",
    "\n",
    "        node_types = ['all']\n",
    "        link_types = ['SOC', 'SPA', 'TEM', 'simp']\n",
    "\n",
    "        vis = np.load(osp.join(self.raw_dir, 'Visual_Features.npy'),allow_pickle=True)[:,2:].astype(float)\n",
    "        tex = np.load(osp.join(self.raw_dir, 'Textual_Features.npy'),allow_pickle=True)[:,5:].astype(float)\n",
    "\n",
    "        x = np.hstack([vis,np.nan_to_num(tex)])\n",
    "\n",
    "\n",
    "        node_type_idx = np.load(osp.join(self.raw_dir, 'node_types.npy'))\n",
    "        node_type_idx = torch.from_numpy(node_type_idx).to(torch.long)\n",
    "\n",
    "        #data['vis_only'].num_nodes = int((node_type_idx == 0).sum())\n",
    "        #data['vis_tex'].num_nodes = int((node_type_idx == 1).sum())\n",
    "        data['all'].num_nodes = len(node_type_idx)\n",
    "\n",
    "        #data['vis_only'].x = torch.from_numpy(vis[node_type_idx==0]).to(torch.float)\n",
    "        #data['vis_tex'].x = torch.from_numpy(x[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].x = torch.from_numpy(x).to(torch.float)\n",
    "\n",
    "\n",
    "        y_s = np.load(osp.join(self.raw_dir, 'labels.npz'), allow_pickle=True)\n",
    "        att_lab = y_s['ATT_LAB'][:,1:10].astype(float)\n",
    "        val_lab = np.nan_to_num(y_s['VAL_LAB'][:,2:13].astype(float))\n",
    "        ys = np.hstack([att_lab, val_lab])\n",
    "\n",
    "        #data['vis_only'].y = torch.from_numpy(att_lab[node_type_idx==0]).to(torch.float)\n",
    "        #data['vis_tex'].y = torch.from_numpy(ys[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].y = torch.from_numpy(ys).to(torch.float)\n",
    "\n",
    "        data['all'].node_type = node_type_idx\n",
    "        \n",
    "        data['all'].att_lab = torch.tensor(y_s['ATT_LAB'][:,-1].astype(bool))\n",
    "        data['all'].val_lab = torch.tensor(y_s['VAL_LAB'][:,-1].astype(bool))\n",
    "\n",
    "        split = np.load(osp.join(self.raw_dir, 'train_val_test_idx.npz'))\n",
    "        for name in ['train', 'val', 'test']:\n",
    "            idx = split[f'{name}_idx']\n",
    "            idx = torch.from_numpy(idx).to(torch.long)\n",
    "            mask = torch.zeros(data['all'].num_nodes, dtype=torch.bool)\n",
    "            mask[idx] = True\n",
    "            data['all'][f'{name}_mask'] = mask\n",
    "            #data['vis_only'][f'{name}_mask'] = mask[node_type_idx==0]\n",
    "            #data['vis_tex'][f'{name}_mask'] = mask[node_type_idx==1]\n",
    "\n",
    "        \n",
    "        s = {}\n",
    "        #s['vis_only'] = np.arange(len(x))[node_type_idx==0]\n",
    "        #s['vis_tex'] = np.arange(len(x))[node_type_idx==1]\n",
    "\n",
    "        for link in link_types:\n",
    "            A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz')).tocoo()\n",
    "            if A_sub.nnz>0:\n",
    "                row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                data['all', f'{link}_link', 'all'].edge_index = torch.stack([row, col], dim=0)\n",
    "                data['all', f'{link}_link', 'all'].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "        if self.pre_transform is not None:\n",
    "            data = self.pre_transform(data)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.__class__.__name__}()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VEN_XL_Links(InMemoryDataset):\n",
    "    r\"\"\"A large subset of Flickr post collected in Venice annotated with Heritage \n",
    "    Values and Attributes, as collected in the `\"Heri-Graphs: A Workflow of \n",
    "    Creating Datasets for Multi-modal Machine Learning on Graphs of Heritage \n",
    "    Values and Attributes with Social Media\" <https://arxiv.org/abs/2205.07545>`\n",
    "    paper.\n",
    "    VEN_XL is a heterogeneous graph containing two types of nodes - nodes with only \n",
    "    visual features 'vis_only' (31,140 nodes), nodes with both visual and textual\n",
    "    features 'vis_tex' (49,823 nodes) and four types of links - social similarity\n",
    "    'SOC' (76,422,265 links), spatial similarity (202,173,159 links), temporal similarity\n",
    "    (71,135,671 links), and simple composed link (290,091,503 links).\n",
    "    Vis_only nodes are represented with 982-dimensional visual features and are\n",
    "    divided into 9 heritage attribute categories \n",
    "    ('architectural elements', 'form', 'gastronomy', 'interior',\n",
    "    'landscape scenery and natural features', 'monuments', 'people', 'product', \n",
    "    'urban scenery').\n",
    "    Vis_text nodes are represented with 1753-dimensional visual and textual \n",
    "    features and are divided into 9 heritage attribute categories plus 11 \n",
    "    heritage value categories ('criterion i-x', 'other').\n",
    "    Both types of nodes are also merged into a single type of node 'all' with \n",
    "    1753-dimensional features and 20-dimensional label categories.\n",
    "\n",
    "    Args:\n",
    "        root (string): Root directory where the dataset should be saved.\n",
    "        transform (callable, optional): A function/transform that takes in an\n",
    "            :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            every access. (default: :obj:`None`)\n",
    "        pre_transform (callable, optional): A function/transform that takes in\n",
    "            an :obj:`torch_geometric.data.HeteroData` object and returns a\n",
    "            transformed version. The data object will be transformed before\n",
    "            being saved to disk. (default: :obj:`None`)\n",
    "    \n",
    "    Stats:\n",
    "            * - #nodes\n",
    "              - #edges\n",
    "              - #features\n",
    "              - #classes\n",
    "            * - 80,963\n",
    "              - 290,091,503\n",
    "              - 1753\n",
    "              - 20\n",
    "    \"\"\"\n",
    "\n",
    "    url = 'https://drive.google.com/uc?export=download&id=1QZ5tyUWs6jYjh7mJrsnpou76iy-vb0CA'\n",
    "\n",
    "    def __init__(self, root: str, transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "        \n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            'A_simp.npz', 'A_SOC.npz', 'A_SPA.npz', 'A_TEM.npz', 'labels.npz',\n",
    "            'node_types.npy', 'Textual_Features.npy', 'train_val_test_idx.npz',\n",
    "            'Visual_Features.npy'\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        os.remove(path)\n",
    "\n",
    "    def process(self):\n",
    "        data = HeteroData()\n",
    "\n",
    "        node_types = ['all']\n",
    "        link_types = ['SOC', 'SPA', 'TEM', 'simp']\n",
    "\n",
    "        vis = np.load(osp.join(self.raw_dir, 'Visual_Features.npy'),allow_pickle=True)[:,2:].astype(float)\n",
    "        tex = np.load(osp.join(self.raw_dir, 'Textual_Features.npy'),allow_pickle=True)[:,5:].astype(float)\n",
    "\n",
    "        x = np.hstack([vis,np.nan_to_num(tex)])\n",
    "\n",
    "\n",
    "        node_type_idx = np.load(osp.join(self.raw_dir, 'node_types.npy'))\n",
    "        node_type_idx = torch.from_numpy(node_type_idx).to(torch.long)\n",
    "\n",
    "        #data['vis_only'].num_nodes = int((node_type_idx == 0).sum())\n",
    "        #data['vis_tex'].num_nodes = int((node_type_idx == 1).sum())\n",
    "        data['all'].num_nodes = len(node_type_idx)\n",
    "\n",
    "        #data['vis_only'].x = torch.from_numpy(vis[node_type_idx==0]).to(torch.float)\n",
    "        #data['vis_tex'].x = torch.from_numpy(x[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].x = torch.from_numpy(x).to(torch.float)\n",
    "\n",
    "\n",
    "        y_s = np.load(osp.join(self.raw_dir, 'labels.npz'), allow_pickle=True)\n",
    "        att_lab = y_s['ATT_LAB'][:,1:10].astype(float)\n",
    "        val_lab = np.nan_to_num(y_s['VAL_LAB'][:,2:13].astype(float))\n",
    "        ys = np.hstack([att_lab, val_lab])\n",
    "\n",
    "        #data['vis_only'].y = torch.from_numpy(att_lab[node_type_idx==0]).to(torch.float)\n",
    "        #data['vis_tex'].y = torch.from_numpy(ys[node_type_idx==1]).to(torch.float)\n",
    "        data['all'].y = torch.from_numpy(ys).to(torch.float)\n",
    "\n",
    "        data['all'].node_type = node_type_idx\n",
    "        \n",
    "        data['all'].att_lab = torch.tensor(y_s['ATT_LAB'][:,-1].astype(bool))\n",
    "        data['all'].val_lab = torch.tensor(y_s['VAL_LAB'][:,-1].astype(bool))\n",
    "\n",
    "        split = np.load(osp.join(self.raw_dir, 'train_val_test_idx.npz'))\n",
    "        for name in ['train', 'val', 'test']:\n",
    "            idx = split[f'{name}_idx']\n",
    "            idx = torch.from_numpy(idx).to(torch.long)\n",
    "            mask = torch.zeros(data['all'].num_nodes, dtype=torch.bool)\n",
    "            mask[idx] = True\n",
    "            data['all'][f'{name}_mask'] = mask\n",
    "            #data['vis_only'][f'{name}_mask'] = mask[node_type_idx==0]\n",
    "            #data['vis_tex'][f'{name}_mask'] = mask[node_type_idx==1]\n",
    "\n",
    "        \n",
    "        s = {}\n",
    "        #s['vis_only'] = np.arange(len(x))[node_type_idx==0]\n",
    "        #s['vis_tex'] = np.arange(len(x))[node_type_idx==1]\n",
    "\n",
    "        for link in link_types:\n",
    "            A_sub = sp.load_npz(osp.join(self.raw_dir, f'A_{link}.npz')).tocoo()\n",
    "            if A_sub.nnz>0:\n",
    "                row = torch.from_numpy(A_sub.row).to(torch.long)\n",
    "                col = torch.from_numpy(A_sub.col).to(torch.long)\n",
    "                data['all', f'{link}_link', 'all'].edge_index = torch.stack([row, col], dim=0)\n",
    "                data['all', f'{link}_link', 'all'].edge_attr = torch.from_numpy(A_sub.data).to(torch.long)\n",
    "\n",
    "        if self.pre_transform is not None:\n",
    "            data = self.pre_transform(data)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.__class__.__name__}()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#path = osp.join(os.getcwd(), '../../data/VEN')\n",
    "dataset = VEN_links('dataset/Venice_links')\n",
    "data = dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=2951,\n",
       "    x=[2951, 1753],\n",
       "    y=[2951, 20],\n",
       "    node_type=[2951],\n",
       "    att_lab=[2951],\n",
       "    val_lab=[2951],\n",
       "    train_mask=[2951],\n",
       "    val_mask=[2951],\n",
       "    test_mask=[2951],\n",
       "    n_id=[2951]\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 488103],\n",
       "    edge_attr=[488103]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 445779],\n",
       "    edge_attr=[445779]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 501191],\n",
       "    edge_attr=[501191]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 1071977],\n",
       "    edge_attr=[1071977]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['all'].n_id = torch.arange(data.num_nodes)\n",
    "data = data.to(device)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(361, device='cuda:0')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['all']['train_mask'].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Loader for Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.loader import NeighborLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('all', 'SOC_link', 'all'),\n",
       " ('all', 'SPA_link', 'all'),\n",
       " ('all', 'TEM_link', 'all'),\n",
       " ('all', 'simp_link', 'all')]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.edge_types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].train_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].val_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=2948,\n",
       "    x=[2948, 1753],\n",
       "    y=[2948, 20],\n",
       "    node_type=[2948],\n",
       "    att_lab=[2948],\n",
       "    val_lab=[2948],\n",
       "    train_mask=[2948],\n",
       "    val_mask=[2948],\n",
       "    test_mask=[2948],\n",
       "    n_id=[2948],\n",
       "    batch_size=32\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 26922],\n",
       "    edge_attr=[26922]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 28515],\n",
       "    edge_attr=[28515]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 27098],\n",
       "    edge_attr=[27098]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_hetero_data = next(iter(train_loader))\n",
    "batch = sampled_hetero_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(361, device='cuda:0')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch['all']['train_mask'].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_train_state(args):\n",
    "    return {'stop_early': False,\n",
    "            'early_stopping_step': 0,\n",
    "            'early_stopping_best_ATT_acc_val': 0,\n",
    "            'early_stopping_best_VAL_acc_val': 0,\n",
    "            'early_stopping_best_ATT_acc_val_2': 0,\n",
    "            'early_stopping_lowest_loss': 1000,\n",
    "            'learning_rate': args.learning_rate,\n",
    "            'epoch_index': 0,\n",
    "            'train_loss': [],\n",
    "            'train_ATT_loss': [],\n",
    "            'train_VAL_loss':[],\n",
    "            'train_ATT_acc': [],\n",
    "            'train_VAL_acc': [],\n",
    "            'train_VAL_jac': [],\n",
    "            'train_VAL_acc_1':[], \n",
    "            'val_loss': [],\n",
    "            'val_ATT_loss': [],\n",
    "            'val_VAL_loss':[],\n",
    "            'val_ATT_acc': [],\n",
    "            'val_VAL_acc': [],\n",
    "            'val_VAL_jac': [],\n",
    "            'val_VAL_acc_1': [],\n",
    "            'test_loss': -1,\n",
    "            'test_ATT_loss': -1,\n",
    "            'test_VAL_loss':-1,\n",
    "            'test_ATT_acc': -1,\n",
    "            'test_VAL_acc': -1,\n",
    "            'test_VAL_jac': -1,\n",
    "            'test_VAL_acc_1': -1,\n",
    "            'model_filename': args.model_state_file}\n",
    "\n",
    "def update_train_state(args, model, train_state):\n",
    "    \"\"\"Handle the training state updates.\n",
    "\n",
    "    Components:\n",
    "     - Early Stopping: Prevent overfitting.\n",
    "     - Model Checkpoint: Model is saved if the model is better\n",
    "\n",
    "    :param args: main arguments\n",
    "    :param model: model to train\n",
    "    :param train_state: a dictionary representing the training state values\n",
    "    :returns:\n",
    "        a new train_state\n",
    "    \"\"\"\n",
    "\n",
    "    # Save one model at least\n",
    "    if train_state['epoch_index'] == 0:\n",
    "        torch.save(model.state_dict(), train_state['model_filename'])\n",
    "        train_state['stop_early'] = False\n",
    "\n",
    "    # Save model if performance improved\n",
    "    elif train_state['epoch_index'] >= 1:\n",
    "        ATT_acc_tm1, ATT_acc_t = train_state['val_ATT_acc'][-2:]\n",
    "        #ATT_acc_2_tm1, ATT_acc_2_t = train_state['val_ATT_acc_2'][-2:]\n",
    "        VAL_acc_tm1, VAL_acc_t = train_state['val_VAL_acc'][-2:]\n",
    "        loss_tm1, loss_t = train_state['val_loss'][-2:]\n",
    "\n",
    "        # If accuracy worsened\n",
    "        #if loss_t >= train_state['early_stopping_lowest_loss']:\n",
    "        #    train_state['early_stopping_step'] += 1\n",
    "        \n",
    "        if ATT_acc_t <= train_state['early_stopping_best_ATT_acc_val'] and VAL_acc_t <= train_state['early_stopping_best_VAL_acc_val']:# and ATT_acc_2_t <= train_state['early_stopping_best_ATT_acc_val_2']:\n",
    "            # Update step\n",
    "            train_state['early_stopping_step'] += 1\n",
    "        # Loss decreased\n",
    "        else:\n",
    "            # Save the best model from sklearn\n",
    "            if VAL_acc_t > train_state['early_stopping_best_VAL_acc_val']:\n",
    "                train_state['early_stopping_best_VAL_acc_val'] = VAL_acc_t\n",
    "                \n",
    "            if ATT_acc_t > train_state['early_stopping_best_ATT_acc_val']:\n",
    "                train_state['early_stopping_best_ATT_acc_val'] = ATT_acc_t\n",
    "            \n",
    "            #if ATT_acc_2_t > train_state['early_stopping_best_ATT_acc_val_2']:\n",
    "            #    train_state['early_stopping_best_ATT_acc_val_2'] = ATT_acc_2_t\n",
    "                \n",
    "            if loss_t < train_state['early_stopping_lowest_loss']:\n",
    "                train_state['early_stopping_lowest_loss'] = loss_t\n",
    "                torch.save(model.state_dict(), train_state['model_filename'])\n",
    "                \n",
    "            # Reset early stopping step\n",
    "            train_state['early_stopping_step'] = 0\n",
    "\n",
    "        # Stop early ?\n",
    "        train_state['stop_early'] = \\\n",
    "            train_state['early_stopping_step'] >= args.early_stopping_criteria\n",
    "\n",
    "    return train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_cross_entropy(y_pred, y_target):\n",
    "    y_target = y_target.cpu().float()\n",
    "    y_pred = y_pred.cpu().float()\n",
    "    criterion = nn.BCEWithLogitsLoss()\n",
    "    return criterion(y_target, y_pred)\n",
    "\n",
    "def compute_1_accuracy(y_pred, y_target):\n",
    "    y_target_indices = y_target.max(dim=1)[1]\n",
    "    y_pred_indices = y_pred.max(dim=1)[1]\n",
    "    n_correct = torch.eq(y_pred_indices, y_target_indices).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_k_accuracy(y_pred, y_target, k=3):\n",
    "    y_pred_indices = y_pred.topk(k, dim=1)[1]\n",
    "    y_target_indices = y_target.max(dim=1)[1]\n",
    "    n_correct = torch.tensor([y_pred_indices[i] in y_target_indices[i] for i in range(len(y_pred))]).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_k_jaccard_index(y_pred, y_target, k=3):\n",
    "    y_target_indices = y_target.topk(k, dim=1)[1]\n",
    "    y_pred_indices = y_pred.max(dim=1)[1]\n",
    "    jaccard = torch.tensor([len(np.intersect1d(y_target_indices[i], y_pred_indices[i]))/\n",
    "                            len(np.union1d(y_target_indices[i], y_pred_indices[i]))\n",
    "                            for i in range(len(y_pred))]).sum().item()\n",
    "    return jaccard / len(y_pred_indices)\n",
    "\n",
    "def compute_jaccard_index(y_pred, y_target, k=3, multilabel=False):\n",
    "    \n",
    "    threshold = 1.0/(k+1)\n",
    "    threshold_2 = 0.5\n",
    "    \n",
    "    if multilabel:\n",
    "        y_pred_indices = y_pred.gt(threshold_2)\n",
    "    else:\n",
    "        y_pred_indices = y_pred.gt(threshold)\n",
    "    \n",
    "    y_target_indices = y_target.gt(threshold)\n",
    "        \n",
    "    jaccard = ((y_target_indices*y_pred_indices).sum(axis=1)/((y_target_indices+y_pred_indices).sum(axis=1)+1e-8)).sum().item()\n",
    "    return jaccard / len(y_pred_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy(pred, soft_targets):\n",
    "    logsoftmax = nn.LogSoftmax(dim=1)\n",
    "    return torch.mean(torch.sum(- soft_targets * logsoftmax(pred), 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Searched Best Hyper-Parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'42/hyperdict.p', 'rb') as fp:\n",
    "    hyperdict= pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(hyperdict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "hyper_df = pd.DataFrame(hyperdict).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>stop_early</th>\n",
       "      <th>early_stopping_step</th>\n",
       "      <th>early_stopping_best_ATT_acc_val</th>\n",
       "      <th>early_stopping_best_VAL_acc_val</th>\n",
       "      <th>early_stopping_best_ATT_acc_val_2</th>\n",
       "      <th>early_stopping_lowest_loss</th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>epoch_index</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_ATT_loss</th>\n",
       "      <th>...</th>\n",
       "      <th>val_VAL_jac</th>\n",
       "      <th>val_VAL_acc_1</th>\n",
       "      <th>test_loss</th>\n",
       "      <th>test_ATT_loss</th>\n",
       "      <th>test_VAL_loss</th>\n",
       "      <th>test_ATT_acc</th>\n",
       "      <th>test_VAL_acc</th>\n",
       "      <th>test_VAL_jac</th>\n",
       "      <th>test_VAL_acc_1</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">0</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
       "      <th>32</th>\n",
       "      <th>0.0100</th>\n",
       "      <td>True</td>\n",
       "      <td>100</td>\n",
       "      <td>86.043165</td>\n",
       "      <td>98.53499</td>\n",
       "      <td>0</td>\n",
       "      <td>4.307965</td>\n",
       "      <td>0.01</td>\n",
       "      <td>150</td>\n",
       "      <td>[5.701295952002208, 3.371192216873169, 2.93144...</td>\n",
       "      <td>[1.8001848581425042, 1.278290778315959, 0.9924...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.20119904201483452, 0.2581687443901413, 0.28...</td>\n",
       "      <td>[26.183562241116203, 38.57684761281884, 48.260...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HeteroGNN/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <th>0.0001</th>\n",
       "      <td>True</td>\n",
       "      <td>100</td>\n",
       "      <td>89.640288</td>\n",
       "      <td>98.116416</td>\n",
       "      <td>0</td>\n",
       "      <td>4.212483</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>271</td>\n",
       "      <td>[4.122023801008861, 3.6332812507947287, 3.4577...</td>\n",
       "      <td>[1.8689053507722977, 1.7542523825928116, 1.637...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.030630041421408325, 0.19073468597724808, 0....</td>\n",
       "      <td>[24.10202746893394, 31.659036407237853, 40.239...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HeteroGNN/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">3</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">32</th>\n",
       "      <th>0.0010</th>\n",
       "      <td>True</td>\n",
       "      <td>100</td>\n",
       "      <td>89.352518</td>\n",
       "      <td>98.744277</td>\n",
       "      <td>0</td>\n",
       "      <td>4.226059</td>\n",
       "      <td>0.001</td>\n",
       "      <td>157</td>\n",
       "      <td>[3.9881125489870706, 3.4168097178141275, 3.183...</td>\n",
       "      <td>[1.7477501148662409, 1.5433839830335156, 1.320...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.19956834549006744, 0.2770612624945764, 0.26...</td>\n",
       "      <td>[46.34314366688468, 54.34139960758666, 49.0752...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HeteroGNN/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.0001</th>\n",
       "      <td>False</td>\n",
       "      <td>45</td>\n",
       "      <td>87.769784</td>\n",
       "      <td>98.744277</td>\n",
       "      <td>0</td>\n",
       "      <td>4.215985</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>299</td>\n",
       "      <td>[4.394130706787109, 4.11641263961792, 3.931373...</td>\n",
       "      <td>[2.019101794076428, 1.9337231571324314, 1.8795...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.0, 0.011510791366906475, 0.0680183126226291...</td>\n",
       "      <td>[15.289295836058425, 18.41726618705036, 21.160...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HeteroGNN/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <th>32</th>\n",
       "      <th>0.0001</th>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>87.769784</td>\n",
       "      <td>98.53499</td>\n",
       "      <td>0</td>\n",
       "      <td>4.230353</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>299</td>\n",
       "      <td>[4.4204078912734985, 4.052810370922089, 3.7973...</td>\n",
       "      <td>[2.0110542879870725, 1.9153890008741468, 1.861...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.0, 0.058940483976455196, 0.1642816659092824...</td>\n",
       "      <td>[58.74776542402442, 64.80575539568345, 67.7357...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HeteroGNN/model.pth</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              stop_early early_stopping_step early_stopping_best_ATT_acc_val  \\\n",
       "0 2 32 0.0100       True                 100                       86.043165   \n",
       "    64 0.0001       True                 100                       89.640288   \n",
       "  3 32 0.0010       True                 100                       89.352518   \n",
       "       0.0001      False                  45                       87.769784   \n",
       "  5 32 0.0001      False                   3                       87.769784   \n",
       "\n",
       "              early_stopping_best_VAL_acc_val  \\\n",
       "0 2 32 0.0100                        98.53499   \n",
       "    64 0.0001                       98.116416   \n",
       "  3 32 0.0010                       98.744277   \n",
       "       0.0001                       98.744277   \n",
       "  5 32 0.0001                        98.53499   \n",
       "\n",
       "              early_stopping_best_ATT_acc_val_2 early_stopping_lowest_loss  \\\n",
       "0 2 32 0.0100                                 0                   4.307965   \n",
       "    64 0.0001                                 0                   4.212483   \n",
       "  3 32 0.0010                                 0                   4.226059   \n",
       "       0.0001                                 0                   4.215985   \n",
       "  5 32 0.0001                                 0                   4.230353   \n",
       "\n",
       "              learning_rate epoch_index  \\\n",
       "0 2 32 0.0100          0.01         150   \n",
       "    64 0.0001        0.0001         271   \n",
       "  3 32 0.0010         0.001         157   \n",
       "       0.0001        0.0001         299   \n",
       "  5 32 0.0001        0.0001         299   \n",
       "\n",
       "                                                      train_loss  \\\n",
       "0 2 32 0.0100  [5.701295952002208, 3.371192216873169, 2.93144...   \n",
       "    64 0.0001  [4.122023801008861, 3.6332812507947287, 3.4577...   \n",
       "  3 32 0.0010  [3.9881125489870706, 3.4168097178141275, 3.183...   \n",
       "       0.0001  [4.394130706787109, 4.11641263961792, 3.931373...   \n",
       "  5 32 0.0001  [4.4204078912734985, 4.052810370922089, 3.7973...   \n",
       "\n",
       "                                                  train_ATT_loss  ...  \\\n",
       "0 2 32 0.0100  [1.8001848581425042, 1.278290778315959, 0.9924...  ...   \n",
       "    64 0.0001  [1.8689053507722977, 1.7542523825928116, 1.637...  ...   \n",
       "  3 32 0.0010  [1.7477501148662409, 1.5433839830335156, 1.320...  ...   \n",
       "       0.0001  [2.019101794076428, 1.9337231571324314, 1.8795...  ...   \n",
       "  5 32 0.0001  [2.0110542879870725, 1.9153890008741468, 1.861...  ...   \n",
       "\n",
       "                                                     val_VAL_jac  \\\n",
       "0 2 32 0.0100  [0.20119904201483452, 0.2581687443901413, 0.28...   \n",
       "    64 0.0001  [0.030630041421408325, 0.19073468597724808, 0....   \n",
       "  3 32 0.0010  [0.19956834549006744, 0.2770612624945764, 0.26...   \n",
       "       0.0001  [0.0, 0.011510791366906475, 0.0680183126226291...   \n",
       "  5 32 0.0001  [0.0, 0.058940483976455196, 0.1642816659092824...   \n",
       "\n",
       "                                                   val_VAL_acc_1 test_loss  \\\n",
       "0 2 32 0.0100  [26.183562241116203, 38.57684761281884, 48.260...        -1   \n",
       "    64 0.0001  [24.10202746893394, 31.659036407237853, 40.239...        -1   \n",
       "  3 32 0.0010  [46.34314366688468, 54.34139960758666, 49.0752...        -1   \n",
       "       0.0001  [15.289295836058425, 18.41726618705036, 21.160...        -1   \n",
       "  5 32 0.0001  [58.74776542402442, 64.80575539568345, 67.7357...        -1   \n",
       "\n",
       "              test_ATT_loss test_VAL_loss test_ATT_acc test_VAL_acc  \\\n",
       "0 2 32 0.0100            -1            -1           -1           -1   \n",
       "    64 0.0001            -1            -1           -1           -1   \n",
       "  3 32 0.0010            -1            -1           -1           -1   \n",
       "       0.0001            -1            -1           -1           -1   \n",
       "  5 32 0.0001            -1            -1           -1           -1   \n",
       "\n",
       "              test_VAL_jac test_VAL_acc_1                     model_filename  \n",
       "0 2 32 0.0100           -1             -1  model_storage/HeteroGNN/model.pth  \n",
       "    64 0.0001           -1             -1  model_storage/HeteroGNN/model.pth  \n",
       "  3 32 0.0010           -1             -1  model_storage/HeteroGNN/model.pth  \n",
       "       0.0001           -1             -1  model_storage/HeteroGNN/model.pth  \n",
       "  5 32 0.0001           -1             -1  model_storage/HeteroGNN/model.pth  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hyper_df[(hyper_df.early_stopping_best_VAL_acc_val + hyper_df.early_stopping_best_VAL_acc_val)>196]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>stop_early</th>\n",
       "      <th>early_stopping_step</th>\n",
       "      <th>early_stopping_best_ATT_acc_val</th>\n",
       "      <th>early_stopping_best_VAL_acc_val</th>\n",
       "      <th>early_stopping_best_ATT_acc_val_2</th>\n",
       "      <th>early_stopping_lowest_loss</th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>epoch_index</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_ATT_loss</th>\n",
       "      <th>...</th>\n",
       "      <th>val_VAL_jac</th>\n",
       "      <th>val_VAL_acc_1</th>\n",
       "      <th>test_loss</th>\n",
       "      <th>test_ATT_loss</th>\n",
       "      <th>test_VAL_loss</th>\n",
       "      <th>test_ATT_acc</th>\n",
       "      <th>test_VAL_acc</th>\n",
       "      <th>test_VAL_jac</th>\n",
       "      <th>test_VAL_acc_1</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"6\" valign=\"top\">0</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
       "      <th>128</th>\n",
       "      <th>0.0001</th>\n",
       "      <td>True</td>\n",
       "      <td>100</td>\n",
       "      <td>89.352518</td>\n",
       "      <td>77.549597</td>\n",
       "      <td>0</td>\n",
       "      <td>4.195606</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>292</td>\n",
       "      <td>[4.032147785027822, 3.5648412505785623, 3.3649...</td>\n",
       "      <td>[1.8548441715848083, 1.70860315888212, 1.54358...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.18747329442465202, 0.24502725225714314, 0.3...</td>\n",
       "      <td>[39.44800523217789, 45.29670808807499, 39.2047...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HeteroGNN/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256</th>\n",
       "      <th>0.0001</th>\n",
       "      <td>False</td>\n",
       "      <td>9</td>\n",
       "      <td>90.503597</td>\n",
       "      <td>94.977109</td>\n",
       "      <td>0</td>\n",
       "      <td>4.190596</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>299</td>\n",
       "      <td>[3.8349266250928244, 3.401456912358602, 3.1584...</td>\n",
       "      <td>[1.7400510050253195, 1.5369147234015847, 1.349...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.2583765721534212, 0.33895211571360145, 0.33...</td>\n",
       "      <td>[60.17876607804667, 55.57444953128406, 41.0882...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HeteroGNN/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">3</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">128</th>\n",
       "      <th>0.0010</th>\n",
       "      <td>False</td>\n",
       "      <td>25</td>\n",
       "      <td>90.503597</td>\n",
       "      <td>92.24504</td>\n",
       "      <td>0</td>\n",
       "      <td>4.195897</td>\n",
       "      <td>0.001</td>\n",
       "      <td>299</td>\n",
       "      <td>[3.9379881024360657, 3.30867866675059, 3.00129...</td>\n",
       "      <td>[1.7698697393290554, 1.376316657026719, 1.1517...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.2788431093462521, 0.33746239571730396, 0.31...</td>\n",
       "      <td>[38.61085676913015, 41.297580117724, 35.844778...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HeteroGNN/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.0001</th>\n",
       "      <td>False</td>\n",
       "      <td>50</td>\n",
       "      <td>89.208633</td>\n",
       "      <td>90.977981</td>\n",
       "      <td>0</td>\n",
       "      <td>4.196027</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>299</td>\n",
       "      <td>[4.013673742612203, 3.5879997611045837, 3.4103...</td>\n",
       "      <td>[1.8652556794502069, 1.731206324952461, 1.5743...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.08609112709832133, 0.24117142825961396, 0.2...</td>\n",
       "      <td>[59.15500327011116, 40.27381730978855, 39.2047...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HeteroGNN/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256</th>\n",
       "      <th>0.0010</th>\n",
       "      <td>False</td>\n",
       "      <td>9</td>\n",
       "      <td>90.215827</td>\n",
       "      <td>94.977109</td>\n",
       "      <td>0</td>\n",
       "      <td>4.197124</td>\n",
       "      <td>0.001</td>\n",
       "      <td>299</td>\n",
       "      <td>[4.122933804988861, 3.1471897959709167, 2.8036...</td>\n",
       "      <td>[1.6379114300260254, 1.2345053076413859, 0.989...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.3065126117725427, 0.3338609126015949, 0.364...</td>\n",
       "      <td>[36.68192718552431, 41.9141050795727, 43.40178...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HeteroGNN/model.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <th>128</th>\n",
       "      <th>0.0001</th>\n",
       "      <td>False</td>\n",
       "      <td>43</td>\n",
       "      <td>89.208633</td>\n",
       "      <td>88.036625</td>\n",
       "      <td>0</td>\n",
       "      <td>4.192393</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>299</td>\n",
       "      <td>[4.073585728804271, 3.665245294570923, 3.53939...</td>\n",
       "      <td>[1.8968283849409742, 1.810092810448517, 1.7256...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.11103553520819706, 0.19264879039151486, 0.2...</td>\n",
       "      <td>[13.614998909962935, 30.182690211467193, 35.23...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>model_storage/HeteroGNN/model.pth</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               stop_early early_stopping_step early_stopping_best_ATT_acc_val  \\\n",
       "0 2 128 0.0001       True                 100                       89.352518   \n",
       "    256 0.0001      False                   9                       90.503597   \n",
       "  3 128 0.0010      False                  25                       90.503597   \n",
       "        0.0001      False                  50                       89.208633   \n",
       "    256 0.0010      False                   9                       90.215827   \n",
       "  5 128 0.0001      False                  43                       89.208633   \n",
       "\n",
       "               early_stopping_best_VAL_acc_val  \\\n",
       "0 2 128 0.0001                       77.549597   \n",
       "    256 0.0001                       94.977109   \n",
       "  3 128 0.0010                        92.24504   \n",
       "        0.0001                       90.977981   \n",
       "    256 0.0010                       94.977109   \n",
       "  5 128 0.0001                       88.036625   \n",
       "\n",
       "               early_stopping_best_ATT_acc_val_2 early_stopping_lowest_loss  \\\n",
       "0 2 128 0.0001                                 0                   4.195606   \n",
       "    256 0.0001                                 0                   4.190596   \n",
       "  3 128 0.0010                                 0                   4.195897   \n",
       "        0.0001                                 0                   4.196027   \n",
       "    256 0.0010                                 0                   4.197124   \n",
       "  5 128 0.0001                                 0                   4.192393   \n",
       "\n",
       "               learning_rate epoch_index  \\\n",
       "0 2 128 0.0001        0.0001         292   \n",
       "    256 0.0001        0.0001         299   \n",
       "  3 128 0.0010         0.001         299   \n",
       "        0.0001        0.0001         299   \n",
       "    256 0.0010         0.001         299   \n",
       "  5 128 0.0001        0.0001         299   \n",
       "\n",
       "                                                       train_loss  \\\n",
       "0 2 128 0.0001  [4.032147785027822, 3.5648412505785623, 3.3649...   \n",
       "    256 0.0001  [3.8349266250928244, 3.401456912358602, 3.1584...   \n",
       "  3 128 0.0010  [3.9379881024360657, 3.30867866675059, 3.00129...   \n",
       "        0.0001  [4.013673742612203, 3.5879997611045837, 3.4103...   \n",
       "    256 0.0010  [4.122933804988861, 3.1471897959709167, 2.8036...   \n",
       "  5 128 0.0001  [4.073585728804271, 3.665245294570923, 3.53939...   \n",
       "\n",
       "                                                   train_ATT_loss  ...  \\\n",
       "0 2 128 0.0001  [1.8548441715848083, 1.70860315888212, 1.54358...  ...   \n",
       "    256 0.0001  [1.7400510050253195, 1.5369147234015847, 1.349...  ...   \n",
       "  3 128 0.0010  [1.7698697393290554, 1.376316657026719, 1.1517...  ...   \n",
       "        0.0001  [1.8652556794502069, 1.731206324952461, 1.5743...  ...   \n",
       "    256 0.0010  [1.6379114300260254, 1.2345053076413859, 0.989...  ...   \n",
       "  5 128 0.0001  [1.8968283849409742, 1.810092810448517, 1.7256...  ...   \n",
       "\n",
       "                                                      val_VAL_jac  \\\n",
       "0 2 128 0.0001  [0.18747329442465202, 0.24502725225714314, 0.3...   \n",
       "    256 0.0001  [0.2583765721534212, 0.33895211571360145, 0.33...   \n",
       "  3 128 0.0010  [0.2788431093462521, 0.33746239571730396, 0.31...   \n",
       "        0.0001  [0.08609112709832133, 0.24117142825961396, 0.2...   \n",
       "    256 0.0010  [0.3065126117725427, 0.3338609126015949, 0.364...   \n",
       "  5 128 0.0001  [0.11103553520819706, 0.19264879039151486, 0.2...   \n",
       "\n",
       "                                                    val_VAL_acc_1 test_loss  \\\n",
       "0 2 128 0.0001  [39.44800523217789, 45.29670808807499, 39.2047...        -1   \n",
       "    256 0.0001  [60.17876607804667, 55.57444953128406, 41.0882...        -1   \n",
       "  3 128 0.0010  [38.61085676913015, 41.297580117724, 35.844778...        -1   \n",
       "        0.0001  [59.15500327011116, 40.27381730978855, 39.2047...        -1   \n",
       "    256 0.0010  [36.68192718552431, 41.9141050795727, 43.40178...        -1   \n",
       "  5 128 0.0001  [13.614998909962935, 30.182690211467193, 35.23...        -1   \n",
       "\n",
       "               test_ATT_loss test_VAL_loss test_ATT_acc test_VAL_acc  \\\n",
       "0 2 128 0.0001            -1            -1           -1           -1   \n",
       "    256 0.0001            -1            -1           -1           -1   \n",
       "  3 128 0.0010            -1            -1           -1           -1   \n",
       "        0.0001            -1            -1           -1           -1   \n",
       "    256 0.0010            -1            -1           -1           -1   \n",
       "  5 128 0.0001            -1            -1           -1           -1   \n",
       "\n",
       "               test_VAL_jac test_VAL_acc_1                     model_filename  \n",
       "0 2 128 0.0001           -1             -1  model_storage/HeteroGNN/model.pth  \n",
       "    256 0.0001           -1             -1  model_storage/HeteroGNN/model.pth  \n",
       "  3 128 0.0010           -1             -1  model_storage/HeteroGNN/model.pth  \n",
       "        0.0001           -1             -1  model_storage/HeteroGNN/model.pth  \n",
       "    256 0.0010           -1             -1  model_storage/HeteroGNN/model.pth  \n",
       "  5 128 0.0001           -1             -1  model_storage/HeteroGNN/model.pth  \n",
       "\n",
       "[6 rows x 30 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hyper_df[(hyper_df.early_stopping_lowest_loss<4.2)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Re-run Model and get Inference Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HeteroGNN_L(torch.nn.Module):\n",
    "    def __init__(self, metadata, hidden_channels, out_channels, num_layers, in_channels=None):\n",
    "        super().__init__()\n",
    "\n",
    "        self.convs = torch.nn.ModuleList()\n",
    "        for _ in range(num_layers):\n",
    "            conv = HeteroConv({\n",
    "                edge_type: SAGEConv((-1, -1), hidden_channels)\n",
    "                for edge_type in metadata[1] if 'all' in edge_type and not 'simp_link' in edge_type\n",
    "            })\n",
    "            self.convs.append(conv)\n",
    "\n",
    "        self.lin1 = Linear(in_channels, hidden_channels)\n",
    "        self.lin2 = Linear(2*hidden_channels, out_channels)\n",
    "\n",
    "    def forward(self, x_dict, edge_index_dict):\n",
    "        x_0 = self.lin1(x_dict['all']).relu()\n",
    "        for conv in self.convs:\n",
    "            x_dict = conv(x_dict, edge_index_dict)\n",
    "            x_dict = {key: F.leaky_relu(x) for key, x in x_dict.items()}\n",
    "        x = self.lin2(torch.hstack([x_0,x_dict['all']]))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def init_params(model, train_loader):\n",
    "    # Initialize lazy parameters via forwarding a single batch to the model:\n",
    "    batch = next(iter(train_loader))\n",
    "    batch = batch.to(device)\n",
    "    batch = batch.to(device, 'edge_index')\n",
    "    out = model(batch.x_dict, batch.edge_index_dict)\n",
    "\n",
    "def train(model, optimizer, train_loader):\n",
    "    model.train()\n",
    "\n",
    "    total_examples = total_loss = 0\n",
    "    for batch in tqdm(train_loader):\n",
    "        optimizer.zero_grad()\n",
    "        batch = batch.to(device)\n",
    "        batch_size = args.batch_size\n",
    "        new_dict = {}\n",
    "        for edge_type in [edge_type for edge_type in batch.edge_index_dict if 'all' in edge_type and not 'simp_link' in edge_type]:\n",
    "            edge_index = batch.edge_index_dict[edge_type]\n",
    "            edge_index = to_undirected(edge_index)\n",
    "            new_dict[edge_type] = edge_index\n",
    "        batch.edge_index_dict = new_dict\n",
    "        out = model(batch.x_dict, batch.edge_index_dict)[:batch_size]\n",
    "        out_att = out[:,:9]\n",
    "        out_val = out[:,9:]\n",
    "        y = batch.y_dict['all']\n",
    "        y_att = y[:,:9]\n",
    "        y_val = y[:,9:]\n",
    "        \n",
    "        loss = F.cross_entropy(out_att, y_att[:batch_size]) + F.cross_entropy(out_val, y_val[:batch_size])\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        total_examples += batch_size\n",
    "        total_loss += float(loss) * batch_size\n",
    "\n",
    "    return total_loss / total_examples\n",
    "\n",
    "@torch.no_grad()\n",
    "def test(model, loader, mode='train'):\n",
    "    model.eval()\n",
    "\n",
    "    total_examples_att = total_examples_val = 0\n",
    "    running_loss_1 = running_loss_2 = 0.\n",
    "    running_1_acc = 0.\n",
    "    running_k_acc = 0.\n",
    "    running_k_jac = 0.\n",
    "    running_1_val = 0.\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        loss_1 = 0\n",
    "        acc_1_t = 0\n",
    "        loss_2 = 0\n",
    "        acc_1_val = 0\n",
    "        acc_k_t = 0\n",
    "        jac_k_t = 0\n",
    "\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch['all'].batch_size\n",
    "        new_dict = {}\n",
    "        for edge_type in [edge_type for edge_type in batch.edge_index_dict if 'all' in edge_type and not 'simp_link' in edge_type]:\n",
    "            edge_index = batch.edge_index_dict[edge_type]\n",
    "            edge_index = to_undirected(edge_index)\n",
    "            new_dict[edge_type] = edge_index\n",
    "        batch.edge_index_dict = new_dict\n",
    "\n",
    "        out = model(batch.x_dict, batch.edge_index_dict)[:batch_size]\n",
    "        out_att = out[:,:9]\n",
    "        out_val = out[:,9:]\n",
    "        att_node = (batch['all'].att_lab[:batch_size]).nonzero().squeeze()\n",
    "        val_node = (batch['all'].val_lab[:batch_size]).nonzero().squeeze()\n",
    "\n",
    "        #print(type_node)\n",
    "\n",
    "        #pred_att = out_att.argmax(dim=-1)\n",
    "        #pred_val = out_val.argmax(dim=-1)\n",
    "\n",
    "        y = batch.y_dict['all']\n",
    "        y_att = y[:,:9]\n",
    "        y_val = y[:,9:]\n",
    "\n",
    "        if not att_node.shape[0]==0:\n",
    "            loss_1 = F.cross_entropy(out_att[att_node], y_att[:batch_size][att_node])\n",
    "            acc_1_t = compute_1_accuracy(y_att[:batch_size][att_node], out_att[att_node])\n",
    "\n",
    "        if not val_node.shape[0]==0:\n",
    "            loss_2 = F.cross_entropy(out_val[val_node], y_val[val_node])\n",
    "            acc_1_val = compute_1_accuracy(y_val[val_node], out_val[val_node])\n",
    "            acc_k_t = compute_k_accuracy(y_val[val_node], out_val[val_node], args.k)\n",
    "            jac_k_t = compute_jaccard_index(y_val[val_node], F.softmax(out_val[val_node],dim=-1), args.k)\n",
    "            #loss_3 = loss_1 + loss_2\n",
    "\n",
    "        total_examples_att += att_node.shape[0]\n",
    "        total_examples_val += val_node.shape[0]\n",
    "        #total_correct_att += int((pred_att == y_att[:batch_size]).sum())\n",
    "        #total_correct_val += int((pred_val == y_val[:batch_size]).sum())\n",
    "\n",
    "        running_loss_1 += float(loss_1) * att_node.shape[0]\n",
    "        running_loss_2 += float(loss_2) * val_node.shape[0]\n",
    "        running_1_acc += float(acc_1_t) * att_node.shape[0]\n",
    "        running_1_val += float(acc_1_val) * val_node.shape[0]\n",
    "        running_k_acc += float(acc_k_t) * val_node.shape[0]\n",
    "        running_k_jac += float(jac_k_t) * val_node.shape[0]\n",
    "\n",
    "    return running_loss_1/total_examples_att, running_loss_2/total_examples_val, running_1_acc/ total_examples_att, running_k_acc/ total_examples_val, running_k_jac/ total_examples_val, running_1_val/total_examples_val\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialization():\n",
    "    set_seed_everywhere(args.seed, args.cuda)\n",
    "    #transform = T.Compose([T.ToSparseTensor()])\n",
    "    dataset = VEN_links('dataset/Venice_links')\n",
    "    data = dataset[0].to(device)\n",
    "    \n",
    "    train_loader = NeighborLoader(\n",
    "        data,\n",
    "        # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "        num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "        # Use a batch size of 128 for sampling training nodes of type paper\n",
    "        batch_size=args.batch_size,\n",
    "        input_nodes=('all', data['all'].train_mask),\n",
    "    )\n",
    "    val_loader = NeighborLoader(\n",
    "        data,\n",
    "        # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "        num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "        # Use a batch size of 128 for sampling training nodes of type paper\n",
    "        batch_size=args.batch_size,\n",
    "        input_nodes=('all', data['all'].val_mask),\n",
    "    )\n",
    "    test_loader = NeighborLoader(\n",
    "        data,\n",
    "        # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "        num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "        # Use a batch size of 128 for sampling training nodes of type paper\n",
    "        batch_size=args.batch_size,\n",
    "        input_nodes=('all', data['all'].test_mask),\n",
    "    )\n",
    " \n",
    "    model = HeteroGNN_L(data.metadata(), hidden_channels=32, out_channels=data.y_dict['all'].shape[-1],\n",
    "                  num_layers=3, in_channels = data.x_dict['all'].shape[-1]).to(device)\n",
    "    return data, model, train_loader, val_loader, test_loader\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_loop(verbose=False):\n",
    "    \n",
    "    _, model, train_loader, val_loader, test_loader = initialization()\n",
    "    if torch.cuda.device_count() > 1:\n",
    "        print(\"Use {} GPUs !\".format(torch.cuda.device_count()))\n",
    "        model = DataParallel(model)\n",
    "    \n",
    "    init_params(model, train_loader)\n",
    "    optimizer = optim.Adam(model.parameters(), lr=0.0001, weight_decay=args.l2)\n",
    "    #scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer,\n",
    "    #                                           mode='min', factor=0.5,\n",
    "    #                                           patience=1)\n",
    "\n",
    "    train_state = make_train_state(args)\n",
    "\n",
    "    try:\n",
    "        for epoch in range(args.num_epochs):\n",
    "            train_state['epoch_index'] = epoch\n",
    "            \n",
    "            loss = train(model, optimizer, train_loader)\n",
    "            train_loss_att, train_loss_val, train_att_acc, train_val_acc, train_val_jac, train_val_1 = test(model, train_loader)\n",
    "            val_loss_att, val_loss_val, val_att_acc, val_val_acc, val_val_jac, val_val_1 = test(model, val_loader)\n",
    "            if verbose:\n",
    "                print(f'Epoch: {epoch:02d}, Loss: {loss:.4f}, Train_ATT: {train_att_acc:.4f}, Train_VAL: {train_val_acc:.4f}, Val_vis_tex_ATT: {val_att_acc:.4f}, Val_vis_tex_VAL: {val_val_acc:.4f}')\n",
    "            \n",
    "            train_state['train_loss'].append(loss)\n",
    "            train_state['train_ATT_loss'].append(train_loss_att)\n",
    "            train_state['train_VAL_loss'].append(train_loss_val)\n",
    "            train_state['train_ATT_acc'].append(train_att_acc)\n",
    "            train_state['train_VAL_acc'].append(train_val_acc)\n",
    "            train_state['train_VAL_jac'].append(train_val_jac)\n",
    "            train_state['train_VAL_acc_1'].append(train_val_1)\n",
    "            \n",
    "            train_state['val_ATT_loss'].append(val_loss_att)\n",
    "            train_state['val_VAL_loss'].append(val_loss_val)\n",
    "            train_state['val_loss'].append(val_loss_att + 3*val_loss_val)\n",
    "            train_state['val_ATT_acc'].append(val_att_acc)\n",
    "            train_state['val_VAL_acc'].append(val_val_acc)\n",
    "            train_state['val_VAL_jac'].append(val_val_jac)\n",
    "            train_state['val_VAL_acc_1'].append(val_val_1)\n",
    "            \n",
    "            train_state = update_train_state(args=args, model=model,\n",
    "                                                train_state=train_state)\n",
    "            if train_state['stop_early']:\n",
    "                break\n",
    "\n",
    "    except KeyboardInterrupt:\n",
    "        print(\"Exiting loop\")\n",
    "        pass\n",
    "    \n",
    "    return train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_state = training_loop(verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = HeteroGNN_L(data.metadata(), hidden_channels=32, out_channels=data.y_dict['all'].shape[-1],\n",
    "                  num_layers = 3, in_channels = data.x_dict['all'].shape[-1]).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load(args.save_dir+'Hetero_best_model/model.pth',map_location=torch.device('cpu')))\n",
    "model.eval()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroGNN_L(\n",
       "  (convs): ModuleList(\n",
       "    (0): HeteroConv(num_relations=3)\n",
       "    (1): HeteroConv(num_relations=3)\n",
       "    (2): HeteroConv(num_relations=3)\n",
       "  )\n",
       "  (lin1): Linear(1753, 32, bias=True)\n",
       "  (lin2): Linear(64, 20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 24.13it/s]\n"
     ]
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test_loss_att, test_loss_val, test_att_acc, test_val_acc, test_val_jac, test_val_1 = test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 19.26it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7261433411503102,\n",
       " 1.6014879826363435,\n",
       " 100.0,\n",
       " 100.0,\n",
       " 0.8550323211585386,\n",
       " 88.08864265927978)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 32.12it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8145147678328724,\n",
       " 1.620982464898396,\n",
       " 96.54471544715447,\n",
       " 99.50738916256158,\n",
       " 0.7651888443331413,\n",
       " 84.72906403940887)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 25.48it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8253275398942632,\n",
       " 1.6446616904189189,\n",
       " 95.62624254473161,\n",
       " 99.47916666666667,\n",
       " 0.7118055646618208,\n",
       " 77.08333333333333)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 23.03it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.41it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.61it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.47it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.90it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.91it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.95it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.25it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.83it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.36it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.95it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.32it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.42it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.48it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.90it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.47it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.15it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.38it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.52it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.84it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test(model, val_loader))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test(model, test_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.814461</td>\n",
       "      <td>1.621137</td>\n",
       "      <td>96.626016</td>\n",
       "      <td>99.113300</td>\n",
       "      <td>0.773317</td>\n",
       "      <td>84.729064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000261</td>\n",
       "      <td>0.000234</td>\n",
       "      <td>0.257096</td>\n",
       "      <td>0.207703</td>\n",
       "      <td>0.006294</td>\n",
       "      <td>1.206645</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.814182</td>\n",
       "      <td>1.620698</td>\n",
       "      <td>96.138211</td>\n",
       "      <td>99.014778</td>\n",
       "      <td>0.765189</td>\n",
       "      <td>82.758621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.814262</td>\n",
       "      <td>1.621013</td>\n",
       "      <td>96.544715</td>\n",
       "      <td>99.014778</td>\n",
       "      <td>0.768473</td>\n",
       "      <td>83.866995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.814409</td>\n",
       "      <td>1.621122</td>\n",
       "      <td>96.646341</td>\n",
       "      <td>99.014778</td>\n",
       "      <td>0.771757</td>\n",
       "      <td>84.975369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.814638</td>\n",
       "      <td>1.621296</td>\n",
       "      <td>96.747967</td>\n",
       "      <td>99.014778</td>\n",
       "      <td>0.776683</td>\n",
       "      <td>85.591133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.814974</td>\n",
       "      <td>1.621454</td>\n",
       "      <td>96.951220</td>\n",
       "      <td>99.507389</td>\n",
       "      <td>0.783251</td>\n",
       "      <td>86.206897</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    0.814461   1.621137  96.626016  99.113300   0.773317  84.729064\n",
       "std     0.000261   0.000234   0.257096   0.207703   0.006294   1.206645\n",
       "min     0.814182   1.620698  96.138211  99.014778   0.765189  82.758621\n",
       "25%     0.814262   1.621013  96.544715  99.014778   0.768473  83.866995\n",
       "50%     0.814409   1.621122  96.646341  99.014778   0.771757  84.975369\n",
       "75%     0.814638   1.621296  96.747967  99.014778   0.776683  85.591133\n",
       "max     0.814974   1.621454  96.951220  99.507389   0.783251  86.206897"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.826263</td>\n",
       "      <td>1.644597</td>\n",
       "      <td>95.646123</td>\n",
       "      <td>99.114583</td>\n",
       "      <td>0.717361</td>\n",
       "      <td>77.864583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000505</td>\n",
       "      <td>0.000228</td>\n",
       "      <td>0.317124</td>\n",
       "      <td>0.351536</td>\n",
       "      <td>0.004430</td>\n",
       "      <td>0.368285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.825291</td>\n",
       "      <td>1.644249</td>\n",
       "      <td>95.228628</td>\n",
       "      <td>98.437500</td>\n",
       "      <td>0.710069</td>\n",
       "      <td>77.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.826116</td>\n",
       "      <td>1.644440</td>\n",
       "      <td>95.427435</td>\n",
       "      <td>98.958333</td>\n",
       "      <td>0.714410</td>\n",
       "      <td>77.604167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.826271</td>\n",
       "      <td>1.644601</td>\n",
       "      <td>95.626243</td>\n",
       "      <td>98.958333</td>\n",
       "      <td>0.717448</td>\n",
       "      <td>78.125000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.826411</td>\n",
       "      <td>1.644749</td>\n",
       "      <td>95.974155</td>\n",
       "      <td>99.479167</td>\n",
       "      <td>0.720052</td>\n",
       "      <td>78.125000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.827022</td>\n",
       "      <td>1.644931</td>\n",
       "      <td>96.023857</td>\n",
       "      <td>99.479167</td>\n",
       "      <td>0.724826</td>\n",
       "      <td>78.125000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    0.826263   1.644597  95.646123  99.114583   0.717361  77.864583\n",
       "std     0.000505   0.000228   0.317124   0.351536   0.004430   0.368285\n",
       "min     0.825291   1.644249  95.228628  98.437500   0.710069  77.083333\n",
       "25%     0.826116   1.644440  95.427435  98.958333   0.714410  77.604167\n",
       "50%     0.826271   1.644601  95.626243  98.958333   0.717448  78.125000\n",
       "75%     0.826411   1.644749  95.974155  99.479167   0.720052  78.125000\n",
       "max     0.827022   1.644931  96.023857  99.479167   0.724826  78.125000"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'val_metrics.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'test_metrics.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_state['test_ATT_loss']=test_loss_att\n",
    "train_state['test_VAL_loss']=test_loss_val\n",
    "train_state['test_loss']=test_loss_att + 3*test_loss_val\n",
    "train_state['test_ATT_acc']=test_att_acc\n",
    "train_state['test_VAL_acc_1']=test_val_1\n",
    "train_state['test_VAL_acc']=test_val_acc\n",
    "train_state['test_VAL_jac']=test_val_jac"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': False,\n",
       " 'early_stopping_step': 36,\n",
       " 'early_stopping_best_ATT_acc_val': 97.15447154471545,\n",
       " 'early_stopping_best_VAL_acc_val': 100.0,\n",
       " 'early_stopping_best_ATT_acc_val_2': 0,\n",
       " 'early_stopping_lowest_loss': 5.6760863951857985,\n",
       " 'learning_rate': 0.001,\n",
       " 'epoch_index': 299,\n",
       " 'train_loss': [4.388954063256581,\n",
       "  4.111910243829091,\n",
       "  3.9167340795199075,\n",
       "  3.7769437630971274,\n",
       "  3.663265665372213,\n",
       "  3.571526805559794,\n",
       "  3.4576874375343323,\n",
       "  3.3690043489138284,\n",
       "  3.297426422437032,\n",
       "  3.2279419898986816,\n",
       "  3.1783733566602073,\n",
       "  3.0797815521558127,\n",
       "  3.0304313699404397,\n",
       "  2.984511991341909,\n",
       "  2.927460332711538,\n",
       "  2.8726566433906555,\n",
       "  2.8256458242734275,\n",
       "  2.8014265497525535,\n",
       "  2.747178097565969,\n",
       "  2.724012811978658,\n",
       "  2.7161844968795776,\n",
       "  2.6957101424535117,\n",
       "  2.6611533761024475,\n",
       "  2.654522637526194,\n",
       "  2.6134462555249534,\n",
       "  2.6218040386835733,\n",
       "  2.591593027114868,\n",
       "  2.602132042249044,\n",
       "  2.571110506852468,\n",
       "  2.5524837970733643,\n",
       "  2.551305890083313,\n",
       "  2.52798322836558,\n",
       "  2.5358527501424155,\n",
       "  2.505404233932495,\n",
       "  2.5157782435417175,\n",
       "  2.494962910811106,\n",
       "  2.4864739179611206,\n",
       "  2.50363423426946,\n",
       "  2.47866952419281,\n",
       "  2.479182183742523,\n",
       "  2.45834352572759,\n",
       "  2.466731905937195,\n",
       "  2.4625399907430015,\n",
       "  2.470397432645162,\n",
       "  2.4687440991401672,\n",
       "  2.4646003047625222,\n",
       "  2.445659657319387,\n",
       "  2.4416032632191977,\n",
       "  2.440070609251658,\n",
       "  2.4383193453152976,\n",
       "  2.4302775859832764,\n",
       "  2.4370516737302146,\n",
       "  2.438392241795858,\n",
       "  2.438637912273407,\n",
       "  2.433453540007273,\n",
       "  2.429096003373464,\n",
       "  2.4261916677157083,\n",
       "  2.42717844247818,\n",
       "  2.4249561031659446,\n",
       "  2.4230575561523438,\n",
       "  2.4229254523913064,\n",
       "  2.409611145655314,\n",
       "  2.405556579430898,\n",
       "  2.408200482527415,\n",
       "  2.402225434780121,\n",
       "  2.3983307083447776,\n",
       "  2.4048211375872293,\n",
       "  2.393370747566223,\n",
       "  2.409110168615977,\n",
       "  2.408299148082733,\n",
       "  2.405722419420878,\n",
       "  2.399186849594116,\n",
       "  2.4169872204462686,\n",
       "  2.406339446703593,\n",
       "  2.396828909715017,\n",
       "  2.3978665669759116,\n",
       "  2.3838562766710916,\n",
       "  2.394664446512858,\n",
       "  2.3869577248891196,\n",
       "  2.386576473712921,\n",
       "  2.3770355383555093,\n",
       "  2.4006502827008567,\n",
       "  2.3859001994132996,\n",
       "  2.37542732556661,\n",
       "  2.390003820260366,\n",
       "  2.375158727169037,\n",
       "  2.3815301060676575,\n",
       "  2.3926325043042502,\n",
       "  2.383542458216349,\n",
       "  2.381210466225942,\n",
       "  2.365354537963867,\n",
       "  2.3795762062072754,\n",
       "  2.3901172478993735,\n",
       "  2.3835468888282776,\n",
       "  2.379439870516459,\n",
       "  2.3778444131215415,\n",
       "  2.378092428048452,\n",
       "  2.381461481253306,\n",
       "  2.38859494527181,\n",
       "  2.386697769165039,\n",
       "  2.3806606332461038,\n",
       "  2.3688963452974954,\n",
       "  2.3765501777331033,\n",
       "  2.389418641726176,\n",
       "  2.3829174836476645,\n",
       "  2.3702133297920227,\n",
       "  2.3828011751174927,\n",
       "  2.3826940457026162,\n",
       "  2.3792535265286765,\n",
       "  2.3759764234224954,\n",
       "  2.3671969175338745,\n",
       "  2.3569746216138205,\n",
       "  2.362920582294464,\n",
       "  2.357337931791941,\n",
       "  2.355326771736145,\n",
       "  2.353691021601359,\n",
       "  2.360222081343333,\n",
       "  2.364189326763153,\n",
       "  2.370778719584147,\n",
       "  2.359314223130544,\n",
       "  2.3646690249443054,\n",
       "  2.363721231619517,\n",
       "  2.349721829096476,\n",
       "  2.3649836579958596,\n",
       "  2.3626747528711953,\n",
       "  2.35607647895813,\n",
       "  2.364254673322042,\n",
       "  2.364100376764933,\n",
       "  2.3537800113360086,\n",
       "  2.350101868311564,\n",
       "  2.3663147687911987,\n",
       "  2.3571947614351907,\n",
       "  2.3679378827412925,\n",
       "  2.3674731453259787,\n",
       "  2.3665674130121865,\n",
       "  2.3783408602078757,\n",
       "  2.3608392079671225,\n",
       "  2.363680819670359,\n",
       "  2.360964318116506,\n",
       "  2.371244966983795,\n",
       "  2.359252452850342,\n",
       "  2.3607866764068604,\n",
       "  2.353879431883494,\n",
       "  2.3622482816378274,\n",
       "  2.3713524540265403,\n",
       "  2.3503873546918235,\n",
       "  2.3601442774136863,\n",
       "  2.358242074648539,\n",
       "  2.3634556333223977,\n",
       "  2.3446944753328958,\n",
       "  2.368248720963796,\n",
       "  2.346737722555796,\n",
       "  2.3579296271006265,\n",
       "  2.36068058013916,\n",
       "  2.349623123804728,\n",
       "  2.3711888591448465,\n",
       "  2.3673854072888694,\n",
       "  2.3646881779034934,\n",
       "  2.3593584299087524,\n",
       "  2.3729857206344604,\n",
       "  2.3659945130348206,\n",
       "  2.363038659095764,\n",
       "  2.378166615962982,\n",
       "  2.361967066923777,\n",
       "  2.360795875390371,\n",
       "  2.357949455579122,\n",
       "  2.363940417766571,\n",
       "  2.359562317530314,\n",
       "  2.348058541615804,\n",
       "  2.361478626728058,\n",
       "  2.3607571919759116,\n",
       "  2.3890658617019653,\n",
       "  2.3782591223716736,\n",
       "  2.388572712739309,\n",
       "  2.3701412280400596,\n",
       "  2.3636367122332254,\n",
       "  2.3620920379956565,\n",
       "  2.3568396965662637,\n",
       "  2.33955051501592,\n",
       "  2.356592059135437,\n",
       "  2.3445999026298523,\n",
       "  2.3512706756591797,\n",
       "  2.3624726136525473,\n",
       "  2.3429580330848694,\n",
       "  2.3561543027559915,\n",
       "  2.339993973573049,\n",
       "  2.3511575063069663,\n",
       "  2.353710134824117,\n",
       "  2.3540356159210205,\n",
       "  2.3563989798227944,\n",
       "  2.3426860173543296,\n",
       "  2.3530708154042563,\n",
       "  2.349029461542765,\n",
       "  2.358958423137665,\n",
       "  2.3449621001879373,\n",
       "  2.3535136381785073,\n",
       "  2.3625789880752563,\n",
       "  2.3411711851755777,\n",
       "  2.3465145428975425,\n",
       "  2.3473063906033835,\n",
       "  2.3365597128868103,\n",
       "  2.3522019386291504,\n",
       "  2.3385072350502014,\n",
       "  2.3181092540423074,\n",
       "  2.344865063826243,\n",
       "  2.3435508410135903,\n",
       "  2.3529452284177146,\n",
       "  2.3372586766878762,\n",
       "  2.353265364964803,\n",
       "  2.3432264924049377,\n",
       "  2.336401800314585,\n",
       "  2.3531691233317056,\n",
       "  2.3542412519454956,\n",
       "  2.3543742299079895,\n",
       "  2.3459666768709817,\n",
       "  2.3539301554361978,\n",
       "  2.3513548970222473,\n",
       "  2.3498521645863852,\n",
       "  2.3453265031178794,\n",
       "  2.351103583971659,\n",
       "  2.3299291729927063,\n",
       "  2.3465545376141868,\n",
       "  2.338423709074656,\n",
       "  2.358804245789846,\n",
       "  2.353646159172058,\n",
       "  2.3563027381896973,\n",
       "  2.341228942076365,\n",
       "  2.3427202502886453,\n",
       "  2.3277524511019387,\n",
       "  2.3292542894681296,\n",
       "  2.3454113602638245,\n",
       "  2.3559965093930564,\n",
       "  2.342159907023112,\n",
       "  2.331003189086914,\n",
       "  2.356131931145986,\n",
       "  2.338758130868276,\n",
       "  2.337250530719757,\n",
       "  2.3414183060328164,\n",
       "  2.3319870034853616,\n",
       "  2.3442893822987876,\n",
       "  2.3469204902648926,\n",
       "  2.333474338054657,\n",
       "  2.3447211384773254,\n",
       "  2.3606010476748147,\n",
       "  2.336021979649862,\n",
       "  2.3362440864245095,\n",
       "  2.3375486532847085,\n",
       "  2.3539971113204956,\n",
       "  2.3519998590151467,\n",
       "  2.328289190928141,\n",
       "  2.3384411136309304,\n",
       "  2.342794676621755,\n",
       "  2.3438648184140525,\n",
       "  2.3375807801882424,\n",
       "  2.3559675017992654,\n",
       "  2.3490860064824424,\n",
       "  2.347822288672129,\n",
       "  2.354279855887095,\n",
       "  2.345888296763102,\n",
       "  2.345495601495107,\n",
       "  2.34063192208608,\n",
       "  2.351053754488627,\n",
       "  2.354772925376892,\n",
       "  2.3462391098340354,\n",
       "  2.33652400970459,\n",
       "  2.350505828857422,\n",
       "  2.34730197985967,\n",
       "  2.344653864701589,\n",
       "  2.344192147254944,\n",
       "  2.3500117858250937,\n",
       "  2.347766021887461,\n",
       "  2.3380969365437827,\n",
       "  2.343236207962036,\n",
       "  2.342330892880758,\n",
       "  2.330157736937205,\n",
       "  2.3546345829963684,\n",
       "  2.3482478062311807,\n",
       "  2.3424277106920877,\n",
       "  2.3406275113423667,\n",
       "  2.338111956914266,\n",
       "  2.3488920529683432,\n",
       "  2.339644412199656,\n",
       "  2.3393059571584067,\n",
       "  2.3444071412086487,\n",
       "  2.328494211037954,\n",
       "  2.3418864210446677,\n",
       "  2.353987395763397,\n",
       "  2.3414581219355264,\n",
       "  2.3510565559069314,\n",
       "  2.3435968160629272,\n",
       "  2.3473441998163858,\n",
       "  2.347361743450165,\n",
       "  2.3536466360092163,\n",
       "  2.3396398027737937,\n",
       "  2.345754782358805,\n",
       "  2.3437530597050986,\n",
       "  2.3427784045537314,\n",
       "  2.3556066751480103,\n",
       "  2.350013474623362,\n",
       "  2.33803258339564],\n",
       " 'train_ATT_loss': [2.013716212270002,\n",
       "  1.9292624089196118,\n",
       "  1.8740108680196745,\n",
       "  1.8224977730383833,\n",
       "  1.7698265895288736,\n",
       "  1.7113243067363624,\n",
       "  1.655225843934141,\n",
       "  1.5871561012770001,\n",
       "  1.526644576289317,\n",
       "  1.469591087911928,\n",
       "  1.411108181086934,\n",
       "  1.3575446176396844,\n",
       "  1.2958572386374434,\n",
       "  1.239402277317734,\n",
       "  1.1913312607524797,\n",
       "  1.1475651333536798,\n",
       "  1.107105476374111,\n",
       "  1.0736328667220647,\n",
       "  1.04269411458203,\n",
       "  1.0168249392443416,\n",
       "  0.9909687666351445,\n",
       "  0.9727944512116281,\n",
       "  0.9532103617765897,\n",
       "  0.9354077765816137,\n",
       "  0.924855170487697,\n",
       "  0.9133421876093687,\n",
       "  0.9021582933674228,\n",
       "  0.8894303515347087,\n",
       "  0.8769720340369481,\n",
       "  0.8661544725835488,\n",
       "  0.8595056431445388,\n",
       "  0.8494709165472734,\n",
       "  0.8425004353483628,\n",
       "  0.8380791384757721,\n",
       "  0.8296126417836324,\n",
       "  0.8232896123236236,\n",
       "  0.8192988792614924,\n",
       "  0.8160468551260612,\n",
       "  0.8099731750105227,\n",
       "  0.8081161913118864,\n",
       "  0.8027032658333924,\n",
       "  0.7999918036513711,\n",
       "  0.7974516846796812,\n",
       "  0.7948331540641362,\n",
       "  0.7931661744527209,\n",
       "  0.7889862146403981,\n",
       "  0.7871941996743475,\n",
       "  0.7856767983317705,\n",
       "  0.7826934242182492,\n",
       "  0.7820739434036191,\n",
       "  0.7791931939587369,\n",
       "  0.7780541052778672,\n",
       "  0.7759075741028192,\n",
       "  0.7751471347425783,\n",
       "  0.7734121634028955,\n",
       "  0.7714464266544564,\n",
       "  0.7704315230126526,\n",
       "  0.7692999029093502,\n",
       "  0.7682299130180866,\n",
       "  0.7664091218871754,\n",
       "  0.7654371449821874,\n",
       "  0.7639962827730047,\n",
       "  0.7633402996446287,\n",
       "  0.762214810729357,\n",
       "  0.7611957375692859,\n",
       "  0.7608208875907095,\n",
       "  0.7600750508731092,\n",
       "  0.7595955650231845,\n",
       "  0.7578546147267244,\n",
       "  0.7583975562428503,\n",
       "  0.7560182927057684,\n",
       "  0.7553857872030412,\n",
       "  0.7548312265787098,\n",
       "  0.7542897649418945,\n",
       "  0.753670724970482,\n",
       "  0.7533007249277385,\n",
       "  0.7524530192491421,\n",
       "  0.751844017459415,\n",
       "  0.7506378685998785,\n",
       "  0.7507066256121585,\n",
       "  0.750237072108525,\n",
       "  0.7509043926677545,\n",
       "  0.7493643777192134,\n",
       "  0.7481526540917373,\n",
       "  0.74755275431102,\n",
       "  0.7471168977401924,\n",
       "  0.7471037833974632,\n",
       "  0.7467401321574921,\n",
       "  0.7459299402553949,\n",
       "  0.7451879269858807,\n",
       "  0.7458559028659831,\n",
       "  0.7452379262018072,\n",
       "  0.7474148630105227,\n",
       "  0.7442024284122393,\n",
       "  0.7440294989918738,\n",
       "  0.7442102673311313,\n",
       "  0.7435939082478552,\n",
       "  0.7433865805081713,\n",
       "  0.7429864248080267,\n",
       "  0.741524022536925,\n",
       "  0.7411276886337682,\n",
       "  0.7408990539672302,\n",
       "  0.7410722051300831,\n",
       "  0.741221820383521,\n",
       "  0.740551427956103,\n",
       "  0.7405770318660049,\n",
       "  0.7421172427999015,\n",
       "  0.7406581788842368,\n",
       "  0.7391769648258706,\n",
       "  0.7400281358293549,\n",
       "  0.7388306438427552,\n",
       "  0.7384694685896348,\n",
       "  0.7383638768975425,\n",
       "  0.7383963648962513,\n",
       "  0.7374222676509635,\n",
       "  0.7373706054489368,\n",
       "  0.73865170112277,\n",
       "  0.7395264671449846,\n",
       "  0.7374073050689169,\n",
       "  0.737463810272164,\n",
       "  0.7366233758649007,\n",
       "  0.7363399438910867,\n",
       "  0.7353283036448619,\n",
       "  0.7356301062325031,\n",
       "  0.7368253976354309,\n",
       "  0.735954833658118,\n",
       "  0.7354073691236015,\n",
       "  0.7357141733829995,\n",
       "  0.7352189682527263,\n",
       "  0.7346818265492235,\n",
       "  0.7346998269868359,\n",
       "  0.7344435083568922,\n",
       "  0.734362738119268,\n",
       "  0.7351270659808637,\n",
       "  0.7336494319987099,\n",
       "  0.734882205972381,\n",
       "  0.7341047077958274,\n",
       "  0.7337511687067407,\n",
       "  0.7330161253831393,\n",
       "  0.73374850086228,\n",
       "  0.7337954097177183,\n",
       "  0.7334633879053956,\n",
       "  0.7342620085811351,\n",
       "  0.7326602394230808,\n",
       "  0.7321469323126563,\n",
       "  0.7327229953538678,\n",
       "  0.7319975024775455,\n",
       "  0.7323025511242346,\n",
       "  0.7320066436175825,\n",
       "  0.7320261107257199,\n",
       "  0.7319339114212924,\n",
       "  0.731472403223825,\n",
       "  0.7316216398804471,\n",
       "  0.7319422308097586,\n",
       "  0.7320703388581316,\n",
       "  0.7326672912634641,\n",
       "  0.7333793688018566,\n",
       "  0.7336947019740815,\n",
       "  0.7337837460298617,\n",
       "  0.7334459262211237,\n",
       "  0.7349501054042594,\n",
       "  0.7338839473816827,\n",
       "  0.7330467978342748,\n",
       "  0.7355112549010406,\n",
       "  0.7364733379303253,\n",
       "  0.7422202744642453,\n",
       "  0.7407309591274842,\n",
       "  0.7403392165981831,\n",
       "  0.7456844145241206,\n",
       "  0.749564654609173,\n",
       "  0.7516348817672095,\n",
       "  0.7428210514734326,\n",
       "  0.7353639343438716,\n",
       "  0.7456515362057989,\n",
       "  0.73670703833123,\n",
       "  0.7359616814227645,\n",
       "  0.7323251305524662,\n",
       "  0.7306799355306124,\n",
       "  0.7299480195520988,\n",
       "  0.7295467870057124,\n",
       "  0.7299379253981847,\n",
       "  0.729655166577104,\n",
       "  0.7292345773810495,\n",
       "  0.7294351555964292,\n",
       "  0.7286053319717047,\n",
       "  0.7285287702182653,\n",
       "  0.7286091247093645,\n",
       "  0.7284000661234447,\n",
       "  0.728435194558384,\n",
       "  0.7288877138470679,\n",
       "  0.7282057444144484,\n",
       "  0.7282541019434414,\n",
       "  0.7284225456602356,\n",
       "  0.7285376688119778,\n",
       "  0.7280211120100893,\n",
       "  0.7278754892771925,\n",
       "  0.7275262965389896,\n",
       "  0.7275948669771739,\n",
       "  0.727412667961332,\n",
       "  0.7274898759546042,\n",
       "  0.7271428626660165,\n",
       "  0.7270479131273285,\n",
       "  0.7271431430224897,\n",
       "  0.7272295332681439,\n",
       "  0.7273318995399158,\n",
       "  0.727294868874748,\n",
       "  0.7271689160048467,\n",
       "  0.7270450255216985,\n",
       "  0.7268707557398196,\n",
       "  0.7269711136157493,\n",
       "  0.7268194668510944,\n",
       "  0.7268461144201643,\n",
       "  0.7264318646156227,\n",
       "  0.726433675540121,\n",
       "  0.7268175373777458,\n",
       "  0.7265645787326253,\n",
       "  0.7262813752708013,\n",
       "  0.7262754304917566,\n",
       "  0.7266141278591843,\n",
       "  0.7267321268937594,\n",
       "  0.7263679628226896,\n",
       "  0.7267914214622941,\n",
       "  0.7262867238382884,\n",
       "  0.7265098809865703,\n",
       "  0.7259966083840981,\n",
       "  0.7261470861712321,\n",
       "  0.726517616051386,\n",
       "  0.7264827222374998,\n",
       "  0.7260291299001002,\n",
       "  0.7258123426886477,\n",
       "  0.7264922068059609,\n",
       "  0.7256785026877871,\n",
       "  0.7258344117624278,\n",
       "  0.7258245492575902,\n",
       "  0.7260482373990511,\n",
       "  0.7259881607053023,\n",
       "  0.7256774055330377,\n",
       "  0.7252895374707568,\n",
       "  0.7258222857670771,\n",
       "  0.7260018393933938,\n",
       "  0.725725130361203,\n",
       "  0.725758491790856,\n",
       "  0.7255208365144492,\n",
       "  0.7257675789399821,\n",
       "  0.7253567711798438,\n",
       "  0.7257261111135298,\n",
       "  0.7269755433470919,\n",
       "  0.7269543072523503,\n",
       "  0.7271165768525607,\n",
       "  0.7291132835800298,\n",
       "  0.7301957222563408,\n",
       "  0.7304850397347743,\n",
       "  0.73127476313768,\n",
       "  0.7305319816121765,\n",
       "  0.7286757290198201,\n",
       "  0.726854551201712,\n",
       "  0.7263583446803846,\n",
       "  0.7284630229928817,\n",
       "  0.7315887404280687,\n",
       "  0.729933500289917,\n",
       "  0.728988694517236,\n",
       "  0.727874883819485,\n",
       "  0.7274452778110874,\n",
       "  0.7261452851533229,\n",
       "  0.7259075177673487,\n",
       "  0.7259169011234907,\n",
       "  0.7253479582781277,\n",
       "  0.7249385346336048,\n",
       "  0.7244664136722808,\n",
       "  0.7251919894337324,\n",
       "  0.7246838884010209,\n",
       "  0.7249365863377367,\n",
       "  0.7245938971102073,\n",
       "  0.7244610122696514,\n",
       "  0.7244204294318307,\n",
       "  0.7245200396904985,\n",
       "  0.7247234351747254,\n",
       "  0.724474257874687,\n",
       "  0.724198644676367,\n",
       "  0.7242959012945603,\n",
       "  0.7242971123750851,\n",
       "  0.7247079358206562,\n",
       "  0.7264176400744684,\n",
       "  0.7267018257746075,\n",
       "  0.7259161049639419,\n",
       "  0.7247415901551286,\n",
       "  0.7246312370260667,\n",
       "  0.7244392507954648,\n",
       "  0.7250122787879775,\n",
       "  0.7240217869631802,\n",
       "  0.7250120819770729,\n",
       "  0.7256077308403818,\n",
       "  0.7255091449230332,\n",
       "  0.7248866300503634,\n",
       "  0.725155030591336,\n",
       "  0.7265659486487962,\n",
       "  0.7261343377118625,\n",
       "  0.7254034586230144,\n",
       "  0.7248637047170602,\n",
       "  0.7242272423575129],\n",
       " 'train_VAL_loss': [2.2222460306912577,\n",
       "  2.082051289708991,\n",
       "  1.9718510701715781,\n",
       "  1.9009597958620235,\n",
       "  1.8382733591375588,\n",
       "  1.7870791648563586,\n",
       "  1.752872230933974,\n",
       "  1.7357921765451616,\n",
       "  1.7205303081845313,\n",
       "  1.7120863212442794,\n",
       "  1.7033715267921088,\n",
       "  1.698463977538978,\n",
       "  1.6955102193388583,\n",
       "  1.6894552043930646,\n",
       "  1.687384534080273,\n",
       "  1.6831912046654403,\n",
       "  1.680815609538324,\n",
       "  1.6795323456423434,\n",
       "  1.6758869255678805,\n",
       "  1.6750399633788007,\n",
       "  1.673066265695313,\n",
       "  1.6703960964884454,\n",
       "  1.666399406264033,\n",
       "  1.665786721699786,\n",
       "  1.6680595316715188,\n",
       "  1.6709399345508904,\n",
       "  1.6694374721796559,\n",
       "  1.665520405835392,\n",
       "  1.6625410552830577,\n",
       "  1.660370935033233,\n",
       "  1.65894886133083,\n",
       "  1.6564619739299995,\n",
       "  1.6529346872895048,\n",
       "  1.6525023439254127,\n",
       "  1.6510872305925532,\n",
       "  1.6496499377274447,\n",
       "  1.647987165939775,\n",
       "  1.6456848950927607,\n",
       "  1.6454474780698232,\n",
       "  1.6437008569115086,\n",
       "  1.6432391137627684,\n",
       "  1.6421204683192878,\n",
       "  1.6413917908047706,\n",
       "  1.6398726632390326,\n",
       "  1.6389778026913673,\n",
       "  1.638641254063128,\n",
       "  1.6378411556545056,\n",
       "  1.6366541702661488,\n",
       "  1.6358094192277692,\n",
       "  1.635381904665453,\n",
       "  1.63507055410718,\n",
       "  1.6355134024184166,\n",
       "  1.6344032766416132,\n",
       "  1.6335238211703103,\n",
       "  1.6322661050468932,\n",
       "  1.632239038263992,\n",
       "  1.6316786056740462,\n",
       "  1.6305661957680024,\n",
       "  1.6304751462883567,\n",
       "  1.6294365952880099,\n",
       "  1.629178220545486,\n",
       "  1.627963634739292,\n",
       "  1.6277740572958441,\n",
       "  1.6268903951565645,\n",
       "  1.627024052545965,\n",
       "  1.6266243451189797,\n",
       "  1.6256999652471569,\n",
       "  1.625633789561792,\n",
       "  1.6249862427196344,\n",
       "  1.6246919506474544,\n",
       "  1.62411492154869,\n",
       "  1.6241580832367788,\n",
       "  1.6233019521692122,\n",
       "  1.6231248926918262,\n",
       "  1.6234206605155712,\n",
       "  1.6234512996145232,\n",
       "  1.6221040566542142,\n",
       "  1.6218584482029204,\n",
       "  1.6212714088260303,\n",
       "  1.6207318061606706,\n",
       "  1.621033073132058,\n",
       "  1.6210623609722485,\n",
       "  1.6201202410409985,\n",
       "  1.6198030046478864,\n",
       "  1.6192983438433703,\n",
       "  1.6188038709751458,\n",
       "  1.619446947634055,\n",
       "  1.6184015746261935,\n",
       "  1.6180992839739263,\n",
       "  1.6176856660446632,\n",
       "  1.6177208634955071,\n",
       "  1.6176163271853798,\n",
       "  1.6173355932711233,\n",
       "  1.616740956531007,\n",
       "  1.6164853685120135,\n",
       "  1.6163807377590698,\n",
       "  1.6165363431967528,\n",
       "  1.6157711783934827,\n",
       "  1.615814203370641,\n",
       "  1.615522005881629,\n",
       "  1.614891273163032,\n",
       "  1.6146965175454306,\n",
       "  1.61430954470859,\n",
       "  1.614467287657994,\n",
       "  1.614464201755471,\n",
       "  1.614142335683025,\n",
       "  1.6136772110851847,\n",
       "  1.6144180869131537,\n",
       "  1.6142633066943477,\n",
       "  1.6138743148946366,\n",
       "  1.6143058386535856,\n",
       "  1.6133747041390543,\n",
       "  1.6129524674772227,\n",
       "  1.6126607058781335,\n",
       "  1.6123754591822954,\n",
       "  1.6123587289345231,\n",
       "  1.6120640884806245,\n",
       "  1.6118901584947538,\n",
       "  1.6118343269395696,\n",
       "  1.612486410339123,\n",
       "  1.6118645512826555,\n",
       "  1.6114296163548394,\n",
       "  1.6109391939607023,\n",
       "  1.610854459601426,\n",
       "  1.6109376280591758,\n",
       "  1.6109797660663847,\n",
       "  1.610473711404774,\n",
       "  1.610077418779072,\n",
       "  1.6103364179669324,\n",
       "  1.6102657582290945,\n",
       "  1.6103265724684064,\n",
       "  1.6108684387893888,\n",
       "  1.611015225381402,\n",
       "  1.6104583591635537,\n",
       "  1.6099896751282288,\n",
       "  1.6100514268610946,\n",
       "  1.6108409186149237,\n",
       "  1.6111410030697852,\n",
       "  1.6102639139524126,\n",
       "  1.610408315367976,\n",
       "  1.6109844915094138,\n",
       "  1.610244239796562,\n",
       "  1.6105967743575078,\n",
       "  1.6089630638793564,\n",
       "  1.6088339975335921,\n",
       "  1.6096035351053168,\n",
       "  1.6088660001094321,\n",
       "  1.6088567552804287,\n",
       "  1.6096271233545445,\n",
       "  1.6085606377540864,\n",
       "  1.6086702036395297,\n",
       "  1.6108178745676607,\n",
       "  1.6102436414385766,\n",
       "  1.611487170996098,\n",
       "  1.6135391363476783,\n",
       "  1.6132309502842024,\n",
       "  1.6143044041794754,\n",
       "  1.616552829412212,\n",
       "  1.6197333355689643,\n",
       "  1.619232157590977,\n",
       "  1.6187722474914508,\n",
       "  1.6183985577395748,\n",
       "  1.6136347610204174,\n",
       "  1.6138555389362028,\n",
       "  1.6146711840854127,\n",
       "  1.617735922831908,\n",
       "  1.6201298022204158,\n",
       "  1.619525725821709,\n",
       "  1.6189473263114442,\n",
       "  1.6198861390930133,\n",
       "  1.6187952112953419,\n",
       "  1.6136241025211409,\n",
       "  1.6082106016349265,\n",
       "  1.6142057187009056,\n",
       "  1.612640212777579,\n",
       "  1.611891142549277,\n",
       "  1.606793025524002,\n",
       "  1.6061628515370334,\n",
       "  1.6058354417372938,\n",
       "  1.6055038282415544,\n",
       "  1.6055999832470331,\n",
       "  1.605427740683516,\n",
       "  1.6050000996470781,\n",
       "  1.6049182675882059,\n",
       "  1.6050747395222207,\n",
       "  1.6050207340816383,\n",
       "  1.6045673108166936,\n",
       "  1.6044306088022249,\n",
       "  1.604307260869943,\n",
       "  1.604621660676359,\n",
       "  1.6045079333630294,\n",
       "  1.604451770267328,\n",
       "  1.6042481855672481,\n",
       "  1.6041041506954838,\n",
       "  1.6039239224304453,\n",
       "  1.6040890497514086,\n",
       "  1.6040065338076648,\n",
       "  1.6039515684185925,\n",
       "  1.6036304479820906,\n",
       "  1.603920403609976,\n",
       "  1.6037807150890953,\n",
       "  1.6036131051768887,\n",
       "  1.6032172950020787,\n",
       "  1.6033573698799366,\n",
       "  1.603377716032752,\n",
       "  1.603384768863794,\n",
       "  1.6032372435704494,\n",
       "  1.603046252786948,\n",
       "  1.6031367910865932,\n",
       "  1.6034203886655558,\n",
       "  1.6033743382160683,\n",
       "  1.6033824692142307,\n",
       "  1.6036871173044982,\n",
       "  1.6031862593093407,\n",
       "  1.6035213886535729,\n",
       "  1.6031745200011869,\n",
       "  1.602696544906109,\n",
       "  1.6025641624947333,\n",
       "  1.602601377917789,\n",
       "  1.6026472510393306,\n",
       "  1.6027935740029713,\n",
       "  1.6028676825547152,\n",
       "  1.602687147845852,\n",
       "  1.6028547230849965,\n",
       "  1.6031067503456264,\n",
       "  1.6029010984706087,\n",
       "  1.6025768402871003,\n",
       "  1.6024213781647405,\n",
       "  1.602168932185609,\n",
       "  1.6022070183978516,\n",
       "  1.6026497590574862,\n",
       "  1.6023426623885981,\n",
       "  1.6018475247882409,\n",
       "  1.6023528542875254,\n",
       "  1.6024672532675999,\n",
       "  1.6019712369527843,\n",
       "  1.6020423405058166,\n",
       "  1.6017584952621249,\n",
       "  1.6017768406141497,\n",
       "  1.6020286766776088,\n",
       "  1.6018479590270658,\n",
       "  1.6016245664321815,\n",
       "  1.6018616588492143,\n",
       "  1.6021936537486365,\n",
       "  1.6019154404008817,\n",
       "  1.601731565190154,\n",
       "  1.60209569640437,\n",
       "  1.602281123977619,\n",
       "  1.602807899591335,\n",
       "  1.603519999419553,\n",
       "  1.6031949397929817,\n",
       "  1.6027564814216213,\n",
       "  1.602798616787073,\n",
       "  1.6027110187630904,\n",
       "  1.6024982922625344,\n",
       "  1.6030885727451778,\n",
       "  1.6054277674313067,\n",
       "  1.6097425271929797,\n",
       "  1.6115714116770146,\n",
       "  1.614103135639941,\n",
       "  1.6093715974168434,\n",
       "  1.6052242183949479,\n",
       "  1.6035013472934838,\n",
       "  1.6014210913650218,\n",
       "  1.6012930843638582,\n",
       "  1.6014032132737854,\n",
       "  1.602047997828666,\n",
       "  1.6020458873949552,\n",
       "  1.6019841518428517,\n",
       "  1.6012965922870794,\n",
       "  1.6019631885095316,\n",
       "  1.6025890260522055,\n",
       "  1.6033098532552534,\n",
       "  1.6031471692293966,\n",
       "  1.6027914909774907,\n",
       "  1.603808624922734,\n",
       "  1.6039029924493087,\n",
       "  1.6033769647830742,\n",
       "  1.601691103377831,\n",
       "  1.601065093130286,\n",
       "  1.6006986933071528,\n",
       "  1.601327504147453,\n",
       "  1.601162511886322,\n",
       "  1.6020330065175106,\n",
       "  1.6024710230549948,\n",
       "  1.6015752999736332,\n",
       "  1.6023863857802922,\n",
       "  1.6032112840139965,\n",
       "  1.603700801276104,\n",
       "  1.6025517482176381,\n",
       "  1.6009848784872038,\n",
       "  1.6006441208794506,\n",
       "  1.6010594387794135,\n",
       "  1.6008530102278058,\n",
       "  1.6019480439764642,\n",
       "  1.6029538243100914,\n",
       "  1.6041565605147723,\n",
       "  1.6043514888372448,\n",
       "  1.603629171022748,\n",
       "  1.6051836974733094],\n",
       " 'train_ATT_acc': [20.498614958448755,\n",
       "  21.329639889196677,\n",
       "  30.47091412742382,\n",
       "  43.21329639889197,\n",
       "  49.86149584487535,\n",
       "  57.89473684210526,\n",
       "  59.2797783933518,\n",
       "  63.988919667590025,\n",
       "  67.59002770083103,\n",
       "  69.52908587257618,\n",
       "  73.40720221606648,\n",
       "  77.5623268698061,\n",
       "  79.77839335180056,\n",
       "  82.54847645429363,\n",
       "  83.93351800554017,\n",
       "  84.7645429362881,\n",
       "  85.87257617728532,\n",
       "  87.81163434903047,\n",
       "  89.75069252077563,\n",
       "  90.30470914127424,\n",
       "  92.24376731301939,\n",
       "  92.797783933518,\n",
       "  93.35180055401662,\n",
       "  93.90581717451524,\n",
       "  93.90581717451524,\n",
       "  93.62880886426593,\n",
       "  94.18282548476455,\n",
       "  95.29085872576178,\n",
       "  95.56786703601108,\n",
       "  96.1218836565097,\n",
       "  97.22991689750693,\n",
       "  96.95290858725762,\n",
       "  98.06094182825485,\n",
       "  97.78393351800554,\n",
       "  98.06094182825485,\n",
       "  98.61495844875347,\n",
       "  98.61495844875347,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0],\n",
       " 'train_VAL_acc': [74.51523545706371,\n",
       "  77.5623268698061,\n",
       "  77.00831024930748,\n",
       "  79.50138504155125,\n",
       "  86.14958448753463,\n",
       "  88.08864265927978,\n",
       "  90.85872576177286,\n",
       "  91.68975069252078,\n",
       "  91.41274238227147,\n",
       "  93.07479224376732,\n",
       "  91.13573407202216,\n",
       "  91.68975069252078,\n",
       "  91.41274238227147,\n",
       "  92.5207756232687,\n",
       "  92.5207756232687,\n",
       "  93.07479224376732,\n",
       "  91.96675900277009,\n",
       "  92.797783933518,\n",
       "  93.90581717451524,\n",
       "  93.90581717451524,\n",
       "  93.90581717451524,\n",
       "  95.29085872576178,\n",
       "  95.8448753462604,\n",
       "  95.8448753462604,\n",
       "  96.95290858725762,\n",
       "  97.22991689750693,\n",
       "  97.22991689750693,\n",
       "  97.50692520775624,\n",
       "  98.06094182825485,\n",
       "  98.06094182825485,\n",
       "  98.06094182825485,\n",
       "  98.06094182825485,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  98.61495844875347,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  98.61495844875347,\n",
       "  99.16897506925208,\n",
       "  98.61495844875347,\n",
       "  99.16897506925208,\n",
       "  98.89196675900277,\n",
       "  99.16897506925208,\n",
       "  98.89196675900277,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  98.89196675900277,\n",
       "  98.89196675900277,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  98.89196675900277,\n",
       "  99.44598337950139,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  98.89196675900277,\n",
       "  99.44598337950139,\n",
       "  99.16897506925208,\n",
       "  99.44598337950139,\n",
       "  98.89196675900277,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.16897506925208,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.44598337950139,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.44598337950139,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.16897506925208,\n",
       "  99.16897506925208,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.16897506925208,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  99.44598337950139,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  100.0,\n",
       "  99.7229916897507,\n",
       "  99.44598337950139,\n",
       "  99.16897506925208],\n",
       " 'train_VAL_jac': [0.00554016620498615,\n",
       "  0.05817174515235457,\n",
       "  0.13942751659911096,\n",
       "  0.18374884623900015,\n",
       "  0.23638042386549002,\n",
       "  0.3268698060941828,\n",
       "  0.38411819373471584,\n",
       "  0.4164358308110541,\n",
       "  0.44829178384796736,\n",
       "  0.4963065673109567,\n",
       "  0.49722991953926404,\n",
       "  0.5277008415919592,\n",
       "  0.5406278943090888,\n",
       "  0.5397045262302388,\n",
       "  0.5498615064119038,\n",
       "  0.5701754456411768,\n",
       "  0.567405367822198,\n",
       "  0.5614035255030582,\n",
       "  0.5812557833346633,\n",
       "  0.5895660326421426,\n",
       "  0.5877193123349853,\n",
       "  0.5937211572958822,\n",
       "  0.6172668583835591,\n",
       "  0.6089566090760798,\n",
       "  0.6131117337298195,\n",
       "  0.6094182878319907,\n",
       "  0.6214219830372988,\n",
       "  0.6315789526519353,\n",
       "  0.6338873517150033,\n",
       "  0.6380424816522572,\n",
       "  0.6394275232035037,\n",
       "  0.6500461768575653,\n",
       "  0.6722068416775099,\n",
       "  0.6680517170237702,\n",
       "  0.6708217948427491,\n",
       "  0.6735918779452421,\n",
       "  0.6846722050717002,\n",
       "  0.695752548048701,\n",
       "  0.6994459833795014,\n",
       "  0.6929824543791795,\n",
       "  0.7003693461748371,\n",
       "  0.706832875175159,\n",
       "  0.7137580882149059,\n",
       "  0.7192982491363779,\n",
       "  0.7165281713173991,\n",
       "  0.7234533737901175,\n",
       "  0.7303785868298645,\n",
       "  0.7437673183029048,\n",
       "  0.7336103434047541,\n",
       "  0.7382271468144044,\n",
       "  0.734072027444179,\n",
       "  0.7276084984438571,\n",
       "  0.7294552240345287,\n",
       "  0.7386888414208579,\n",
       "  0.7460757226494871,\n",
       "  0.7460757226494871,\n",
       "  0.7373037945860971,\n",
       "  0.7548476559963914,\n",
       "  0.7423822767516582,\n",
       "  0.759464464689556,\n",
       "  0.7516158888544733,\n",
       "  0.7543859772404805,\n",
       "  0.7479224482401586,\n",
       "  0.750692536626166,\n",
       "  0.7580794231383093,\n",
       "  0.7516158888544733,\n",
       "  0.7673130299576102,\n",
       "  0.7580794231383093,\n",
       "  0.7659279884063637,\n",
       "  0.7714681546113498,\n",
       "  0.7622345425085347,\n",
       "  0.7673130299576102,\n",
       "  0.7673130299576102,\n",
       "  0.770544791816014,\n",
       "  0.7599261381619524,\n",
       "  0.7733148802020213,\n",
       "  0.7867036116750616,\n",
       "  0.7830101710607471,\n",
       "  0.7917820938406228,\n",
       "  0.7811634401865614,\n",
       "  0.7802400773912256,\n",
       "  0.7691597502647675,\n",
       "  0.7719298333672605,\n",
       "  0.7834718498166579,\n",
       "  0.7825484817378079,\n",
       "  0.790858731045287,\n",
       "  0.7917820938406228,\n",
       "  0.7927054566359586,\n",
       "  0.7927054566359586,\n",
       "  0.7867036116750616,\n",
       "  0.7857802488797259,\n",
       "  0.8019390687387736,\n",
       "  0.7991689909197948,\n",
       "  0.8037857890459309,\n",
       "  0.8047091518412667,\n",
       "  0.8042474730853559,\n",
       "  0.8070175509043348,\n",
       "  0.8037857890459309,\n",
       "  0.8107109968021636,\n",
       "  0.8079409136996705,\n",
       "  0.8097876340068279,\n",
       "  0.8051708305971774,\n",
       "  0.8019390687387736,\n",
       "  0.8074792349437597,\n",
       "  0.8107109968021636,\n",
       "  0.8074792349437597,\n",
       "  0.8139427639440816,\n",
       "  0.8236380548028074,\n",
       "  0.8153278054953282,\n",
       "  0.8167128417630605,\n",
       "  0.8111726808415886,\n",
       "  0.8167128417630605,\n",
       "  0.818559567353732,\n",
       "  0.8291782157242793,\n",
       "  0.8222530079680467,\n",
       "  0.8171745205189713,\n",
       "  0.815789484251239,\n",
       "  0.8194829248655536,\n",
       "  0.8107109968021636,\n",
       "  0.8176362045583963,\n",
       "  0.8120960436369243,\n",
       "  0.8148661214559032,\n",
       "  0.815327800211814,\n",
       "  0.8319482988267725,\n",
       "  0.8231763707633825,\n",
       "  0.8194829248655536,\n",
       "  0.8236380495192932,\n",
       "  0.8351800659686904,\n",
       "  0.8231763654798682,\n",
       "  0.8347183819292655,\n",
       "  0.8347183819292655,\n",
       "  0.8310249360314367,\n",
       "  0.8277931741730328,\n",
       "  0.8310249413149509,\n",
       "  0.8397968640948267,\n",
       "  0.8296398997637043,\n",
       "  0.8254847698264505,\n",
       "  0.8204062876608893,\n",
       "  0.8259464485823613,\n",
       "  0.8319482988267725,\n",
       "  0.8236380548028074,\n",
       "  0.8236380495192932,\n",
       "  0.8194829301490678,\n",
       "  0.8374884650317586,\n",
       "  0.8314866200708616,\n",
       "  0.8254847698264505,\n",
       "  0.8310249360314367,\n",
       "  0.8337950191339297,\n",
       "  0.8301015732361009,\n",
       "  0.8291782210077936,\n",
       "  0.8314866200708616,\n",
       "  0.8148661214559032,\n",
       "  0.8047091518412667,\n",
       "  0.8111726808415886,\n",
       "  0.8000923431481021,\n",
       "  0.8000923484316163,\n",
       "  0.7862419276356367,\n",
       "  0.7783933623675825,\n",
       "  0.7654663043669386,\n",
       "  0.7516158888544733,\n",
       "  0.753000935689234,\n",
       "  0.7659279831228494,\n",
       "  0.7899353682499513,\n",
       "  0.7848568860843902,\n",
       "  0.7802400826747398,\n",
       "  0.7650046308945421,\n",
       "  0.7640812628156921,\n",
       "  0.7599261381619524,\n",
       "  0.7571560550594594,\n",
       "  0.7617728584691098,\n",
       "  0.7543859719569663,\n",
       "  0.7825484817378079,\n",
       "  0.8277931741730328,\n",
       "  0.7885503372657332,\n",
       "  0.7862419329191509,\n",
       "  0.7894736947775548,\n",
       "  0.8439519887485663,\n",
       "  0.8430286259532305,\n",
       "  0.8457987143392378,\n",
       "  0.8476454399299094,\n",
       "  0.842105268441409,\n",
       "  0.8453370355833271,\n",
       "  0.8568790467492101,\n",
       "  0.8527239273789847,\n",
       "  0.8462603930951486,\n",
       "  0.8365651022364228,\n",
       "  0.8527239220954704,\n",
       "  0.8596491298517032,\n",
       "  0.8527239220954704,\n",
       "  0.8471837558904843,\n",
       "  0.8587257670563674,\n",
       "  0.8462603930951486,\n",
       "  0.8490304761976416,\n",
       "  0.8541089636467171,\n",
       "  0.8499538389929774,\n",
       "  0.8374884597482444,\n",
       "  0.8554940051979636,\n",
       "  0.8545706424026278,\n",
       "  0.8605724873635248,\n",
       "  0.8554940051979636,\n",
       "  0.8499538389929774,\n",
       "  0.8651892907731751,\n",
       "  0.8596491298517032,\n",
       "  0.8619575341982855,\n",
       "  0.8522622433395597,\n",
       "  0.8587257723398817,\n",
       "  0.8665743376079359,\n",
       "  0.8670360163638466,\n",
       "  0.8545706424026278,\n",
       "  0.851800564583649,\n",
       "  0.8490304761976416,\n",
       "  0.8527239220954704,\n",
       "  0.8494921549535525,\n",
       "  0.8531856008513813,\n",
       "  0.866112658852025,\n",
       "  0.8628808917101071,\n",
       "  0.8619575289147713,\n",
       "  0.8642659332613536,\n",
       "  0.874884586915415,\n",
       "  0.863342575749532,\n",
       "  0.866112658852025,\n",
       "  0.8647276173007785,\n",
       "  0.8725761825688327,\n",
       "  0.8734995453641685,\n",
       "  0.8582640883004566,\n",
       "  0.8601108086076139,\n",
       "  0.8610341714029497,\n",
       "  0.8642659279778393,\n",
       "  0.8730378613247436,\n",
       "  0.8711911410175862,\n",
       "  0.8578024042610316,\n",
       "  0.8559556839538743,\n",
       "  0.8670360163638466,\n",
       "  0.8531856008513813,\n",
       "  0.853647279607292,\n",
       "  0.8698060994663397,\n",
       "  0.872114503812922,\n",
       "  0.872114503812922,\n",
       "  0.8716528250570112,\n",
       "  0.8670360163638466,\n",
       "  0.8619575289147713,\n",
       "  0.8656509800961143,\n",
       "  0.8734995453641685,\n",
       "  0.8633425810330462,\n",
       "  0.8624192129541962,\n",
       "  0.874884586915415,\n",
       "  0.8651892960566894,\n",
       "  0.8582640883004566,\n",
       "  0.8467220771345736,\n",
       "  0.8434903152761697,\n",
       "  0.842105268441409,\n",
       "  0.8531856008513813,\n",
       "  0.8614958501588604,\n",
       "  0.8573407255051209,\n",
       "  0.8795013903250655,\n",
       "  0.8790397062856404,\n",
       "  0.8481071134023058,\n",
       "  0.8310249307479224,\n",
       "  0.8167128417630605,\n",
       "  0.7913204203682263,\n",
       "  0.8107109968021636,\n",
       "  0.8199446036214644,\n",
       "  0.8411819056460732,\n",
       "  0.863342575749532,\n",
       "  0.8776546700179082,\n",
       "  0.8758079444272366,\n",
       "  0.8591874510957924,\n",
       "  0.8651892960566894,\n",
       "  0.8651892960566894,\n",
       "  0.8582640883004566,\n",
       "  0.8467220824180878,\n",
       "  0.8411819109295874,\n",
       "  0.851800564583649,\n",
       "  0.857340730788635,\n",
       "  0.8453370355833271,\n",
       "  0.8324099828661974,\n",
       "  0.8324099828661974,\n",
       "  0.8614958501588604,\n",
       "  0.872114503812922,\n",
       "  0.8859649193253873,\n",
       "  0.8767313125060866,\n",
       "  0.8698060994663397,\n",
       "  0.872114503812922,\n",
       "  0.8601108086076139,\n",
       "  0.8527239168119563,\n",
       "  0.8679593738756681,\n",
       "  0.8610341661194355,\n",
       "  0.8476454346463951,\n",
       "  0.8351800606851763,\n",
       "  0.8407202268901625,\n",
       "  0.8568790467492101,\n",
       "  0.8559556839538743,\n",
       "  0.8619575341982855,\n",
       "  0.8707294622616755,\n",
       "  0.8614958501588604,\n",
       "  0.8444136727879913,\n",
       "  0.8301015785196151,\n",
       "  0.8277931688895186,\n",
       "  0.844875351543902,\n",
       "  0.8439519887485663],\n",
       " 'train_VAL_acc_1': [32.40997229916898,\n",
       "  34.903047091412745,\n",
       "  33.51800554016621,\n",
       "  40.16620498614959,\n",
       "  50.692520775623265,\n",
       "  52.35457063711911,\n",
       "  54.29362880886426,\n",
       "  54.016620498614955,\n",
       "  55.95567867036011,\n",
       "  57.89473684210526,\n",
       "  58.448753462603875,\n",
       "  58.72576177285318,\n",
       "  57.340720221606645,\n",
       "  59.2797783933518,\n",
       "  57.89473684210526,\n",
       "  59.00277008310249,\n",
       "  59.2797783933518,\n",
       "  59.556786703601105,\n",
       "  60.11080332409972,\n",
       "  61.49584487534626,\n",
       "  61.21883656509695,\n",
       "  62.880886426592795,\n",
       "  63.988919667590025,\n",
       "  63.71191135734072,\n",
       "  64.26592797783934,\n",
       "  62.880886426592795,\n",
       "  63.988919667590025,\n",
       "  65.37396121883657,\n",
       "  65.37396121883657,\n",
       "  64.81994459833795,\n",
       "  67.03601108033241,\n",
       "  65.65096952908587,\n",
       "  67.03601108033241,\n",
       "  69.25207756232687,\n",
       "  68.14404432132964,\n",
       "  70.3601108033241,\n",
       "  70.91412742382272,\n",
       "  72.57617728531856,\n",
       "  72.02216066481995,\n",
       "  71.46814404432133,\n",
       "  74.23822714681441,\n",
       "  73.6842105263158,\n",
       "  72.57617728531856,\n",
       "  73.40720221606648,\n",
       "  73.6842105263158,\n",
       "  76.17728531855956,\n",
       "  72.29916897506925,\n",
       "  75.62326869806094,\n",
       "  73.9612188365651,\n",
       "  76.45429362880887,\n",
       "  74.23822714681441,\n",
       "  72.57617728531856,\n",
       "  75.62326869806094,\n",
       "  73.9612188365651,\n",
       "  76.73130193905817,\n",
       "  76.17728531855956,\n",
       "  77.00831024930748,\n",
       "  75.90027700831025,\n",
       "  76.17728531855956,\n",
       "  77.00831024930748,\n",
       "  77.28531855955679,\n",
       "  78.39335180055402,\n",
       "  77.8393351800554,\n",
       "  78.94736842105263,\n",
       "  80.60941828254848,\n",
       "  77.00831024930748,\n",
       "  78.39335180055402,\n",
       "  78.67036011080333,\n",
       "  80.88642659279779,\n",
       "  78.67036011080333,\n",
       "  80.88642659279779,\n",
       "  80.05540166204986,\n",
       "  79.50138504155125,\n",
       "  78.67036011080333,\n",
       "  79.50138504155125,\n",
       "  79.77839335180056,\n",
       "  81.99445983379502,\n",
       "  82.27146814404432,\n",
       "  81.4404432132964,\n",
       "  80.88642659279779,\n",
       "  82.27146814404432,\n",
       "  83.65650969529086,\n",
       "  83.65650969529086,\n",
       "  83.65650969529086,\n",
       "  82.54847645429363,\n",
       "  81.7174515235457,\n",
       "  81.16343490304709,\n",
       "  81.4404432132964,\n",
       "  83.65650969529086,\n",
       "  83.93351800554017,\n",
       "  84.21052631578948,\n",
       "  83.93351800554017,\n",
       "  83.65650969529086,\n",
       "  85.31855955678671,\n",
       "  83.93351800554017,\n",
       "  84.7645429362881,\n",
       "  85.0415512465374,\n",
       "  84.7645429362881,\n",
       "  84.7645429362881,\n",
       "  84.21052631578948,\n",
       "  82.82548476454294,\n",
       "  84.7645429362881,\n",
       "  84.7645429362881,\n",
       "  84.21052631578948,\n",
       "  86.14958448753463,\n",
       "  85.0415512465374,\n",
       "  85.31855955678671,\n",
       "  85.0415512465374,\n",
       "  85.0415512465374,\n",
       "  85.59556786703601,\n",
       "  85.0415512465374,\n",
       "  85.87257617728532,\n",
       "  86.98060941828255,\n",
       "  86.42659279778394,\n",
       "  84.48753462603878,\n",
       "  83.93351800554017,\n",
       "  85.87257617728532,\n",
       "  87.25761772853186,\n",
       "  85.87257617728532,\n",
       "  86.98060941828255,\n",
       "  87.81163434903047,\n",
       "  86.42659279778394,\n",
       "  85.59556786703601,\n",
       "  86.42659279778394,\n",
       "  86.42659279778394,\n",
       "  86.42659279778394,\n",
       "  83.93351800554017,\n",
       "  85.0415512465374,\n",
       "  85.31855955678671,\n",
       "  87.25761772853186,\n",
       "  86.42659279778394,\n",
       "  85.0415512465374,\n",
       "  83.37950138504155,\n",
       "  83.93351800554017,\n",
       "  84.7645429362881,\n",
       "  83.93351800554017,\n",
       "  83.10249307479225,\n",
       "  82.82548476454294,\n",
       "  83.10249307479225,\n",
       "  84.21052631578948,\n",
       "  82.82548476454294,\n",
       "  82.82548476454294,\n",
       "  83.65650969529086,\n",
       "  83.65650969529086,\n",
       "  84.21052631578948,\n",
       "  84.21052631578948,\n",
       "  84.21052631578948,\n",
       "  83.93351800554017,\n",
       "  83.65650969529086,\n",
       "  84.48753462603878,\n",
       "  83.93351800554017,\n",
       "  81.99445983379502,\n",
       "  84.21052631578948,\n",
       "  82.54847645429363,\n",
       "  79.50138504155125,\n",
       "  80.60941828254848,\n",
       "  79.50138504155125,\n",
       "  78.94736842105263,\n",
       "  77.5623268698061,\n",
       "  77.00831024930748,\n",
       "  76.45429362880887,\n",
       "  76.45429362880887,\n",
       "  81.16343490304709,\n",
       "  79.77839335180056,\n",
       "  80.60941828254848,\n",
       "  79.22437673130194,\n",
       "  78.94736842105263,\n",
       "  78.11634349030471,\n",
       "  78.94736842105263,\n",
       "  78.67036011080333,\n",
       "  78.39335180055402,\n",
       "  80.60941828254848,\n",
       "  86.14958448753463,\n",
       "  83.10249307479225,\n",
       "  84.21052631578948,\n",
       "  82.82548476454294,\n",
       "  87.25761772853186,\n",
       "  86.70360110803324,\n",
       "  87.25761772853186,\n",
       "  87.53462603878117,\n",
       "  88.08864265927978,\n",
       "  87.53462603878117,\n",
       "  87.53462603878117,\n",
       "  89.19667590027701,\n",
       "  87.53462603878117,\n",
       "  86.14958448753463,\n",
       "  88.08864265927978,\n",
       "  87.53462603878117,\n",
       "  87.81163434903047,\n",
       "  90.58171745152355,\n",
       "  89.19667590027701,\n",
       "  88.9196675900277,\n",
       "  90.30470914127424,\n",
       "  87.53462603878117,\n",
       "  90.58171745152355,\n",
       "  88.6426592797784,\n",
       "  87.53462603878117,\n",
       "  86.70360110803324,\n",
       "  88.08864265927978,\n",
       "  89.47368421052632,\n",
       "  87.25761772853186,\n",
       "  88.36565096952909,\n",
       "  88.36565096952909,\n",
       "  89.75069252077563,\n",
       "  90.85872576177286,\n",
       "  89.47368421052632,\n",
       "  90.02770083102493,\n",
       "  91.13573407202216,\n",
       "  88.36565096952909,\n",
       "  87.53462603878117,\n",
       "  87.25761772853186,\n",
       "  87.25761772853186,\n",
       "  84.7645429362881,\n",
       "  88.36565096952909,\n",
       "  86.98060941828255,\n",
       "  87.25761772853186,\n",
       "  88.36565096952909,\n",
       "  87.25761772853186,\n",
       "  88.6426592797784,\n",
       "  89.19667590027701,\n",
       "  89.19667590027701,\n",
       "  87.53462603878117,\n",
       "  88.36565096952909,\n",
       "  88.6426592797784,\n",
       "  89.47368421052632,\n",
       "  88.9196675900277,\n",
       "  88.9196675900277,\n",
       "  89.47368421052632,\n",
       "  91.68975069252078,\n",
       "  88.08864265927978,\n",
       "  88.9196675900277,\n",
       "  88.9196675900277,\n",
       "  88.6426592797784,\n",
       "  89.75069252077563,\n",
       "  90.02770083102493,\n",
       "  90.58171745152355,\n",
       "  87.53462603878117,\n",
       "  91.68975069252078,\n",
       "  89.75069252077563,\n",
       "  90.58171745152355,\n",
       "  91.13573407202216,\n",
       "  90.30470914127424,\n",
       "  91.68975069252078,\n",
       "  90.02770083102493,\n",
       "  91.41274238227147,\n",
       "  90.85872576177286,\n",
       "  90.30470914127424,\n",
       "  88.08864265927978,\n",
       "  90.58171745152355,\n",
       "  89.75069252077563,\n",
       "  90.30470914127424,\n",
       "  89.75069252077563,\n",
       "  90.58171745152355,\n",
       "  89.19667590027701,\n",
       "  88.36565096952909,\n",
       "  86.14958448753463,\n",
       "  83.10249307479225,\n",
       "  83.10249307479225,\n",
       "  80.60941828254848,\n",
       "  80.60941828254848,\n",
       "  81.4404432132964,\n",
       "  83.93351800554017,\n",
       "  85.31855955678671,\n",
       "  88.36565096952909,\n",
       "  89.47368421052632,\n",
       "  89.19667590027701,\n",
       "  87.25761772853186,\n",
       "  86.42659279778394,\n",
       "  89.19667590027701,\n",
       "  88.6426592797784,\n",
       "  87.53462603878117,\n",
       "  88.6426592797784,\n",
       "  87.25761772853186,\n",
       "  86.14958448753463,\n",
       "  85.59556786703601,\n",
       "  85.0415512465374,\n",
       "  85.0415512465374,\n",
       "  86.14958448753463,\n",
       "  87.25761772853186,\n",
       "  87.81163434903047,\n",
       "  88.6426592797784,\n",
       "  87.81163434903047,\n",
       "  88.08864265927978,\n",
       "  86.98060941828255,\n",
       "  85.59556786703601,\n",
       "  87.81163434903047,\n",
       "  85.0415512465374,\n",
       "  85.59556786703601,\n",
       "  85.87257617728532,\n",
       "  85.31855955678671,\n",
       "  89.19667590027701,\n",
       "  88.08864265927978,\n",
       "  90.58171745152355,\n",
       "  91.13573407202216,\n",
       "  91.13573407202216,\n",
       "  88.36565096952909,\n",
       "  89.19667590027701,\n",
       "  87.53462603878117,\n",
       "  89.19667590027701,\n",
       "  88.36565096952909],\n",
       " 'val_loss': [8.693429539460166,\n",
       "  8.195884168682761,\n",
       "  7.806548549765948,\n",
       "  7.557963584320074,\n",
       "  7.332777340324818,\n",
       "  7.129387312513783,\n",
       "  6.9614313655623565,\n",
       "  6.857662414730065,\n",
       "  6.7615880207277055,\n",
       "  6.67892670210913,\n",
       "  6.609913864538789,\n",
       "  6.548871487035326,\n",
       "  6.502360352161597,\n",
       "  6.443190966499569,\n",
       "  6.391090649793201,\n",
       "  6.341579265186805,\n",
       "  6.30447566446465,\n",
       "  6.266460146040179,\n",
       "  6.227068808552351,\n",
       "  6.206159141005539,\n",
       "  6.177135500838337,\n",
       "  6.152727246186015,\n",
       "  6.11641616616281,\n",
       "  6.08892262547658,\n",
       "  6.07023906259577,\n",
       "  6.063057408496487,\n",
       "  6.051930903605444,\n",
       "  6.020481466798823,\n",
       "  6.014004290858847,\n",
       "  5.996990675399294,\n",
       "  5.971286041656801,\n",
       "  5.964936479866431,\n",
       "  5.95367177576513,\n",
       "  5.940648073407134,\n",
       "  5.931660918202081,\n",
       "  5.921250913646512,\n",
       "  5.912352130808868,\n",
       "  5.905766830005245,\n",
       "  5.895651398639073,\n",
       "  5.886621590970639,\n",
       "  5.877164671218645,\n",
       "  5.876970750578528,\n",
       "  5.868661913977658,\n",
       "  5.864111479310466,\n",
       "  5.858305512613302,\n",
       "  5.847508792053585,\n",
       "  5.846891273847163,\n",
       "  5.840416701096901,\n",
       "  5.831583998424471,\n",
       "  5.828110128200103,\n",
       "  5.825933606144477,\n",
       "  5.81580611805417,\n",
       "  5.816867713998596,\n",
       "  5.815553172179707,\n",
       "  5.800679974070853,\n",
       "  5.805436654542816,\n",
       "  5.800773762928895,\n",
       "  5.79977819488014,\n",
       "  5.79632404173393,\n",
       "  5.792232894777865,\n",
       "  5.790704266698981,\n",
       "  5.785183242624267,\n",
       "  5.781591623220643,\n",
       "  5.783374756728005,\n",
       "  5.782460792088108,\n",
       "  5.780856314064686,\n",
       "  5.773723924718357,\n",
       "  5.769535753890144,\n",
       "  5.769893472140316,\n",
       "  5.772089027696981,\n",
       "  5.765890887866862,\n",
       "  5.7583873851636636,\n",
       "  5.763323474521297,\n",
       "  5.759401613202116,\n",
       "  5.753931978186531,\n",
       "  5.75578228149212,\n",
       "  5.7558213196185495,\n",
       "  5.74918743377251,\n",
       "  5.750122793767487,\n",
       "  5.745184671335619,\n",
       "  5.7443955637199196,\n",
       "  5.742881658471692,\n",
       "  5.736924488617081,\n",
       "  5.738235419030432,\n",
       "  5.735238235037568,\n",
       "  5.7342120679564985,\n",
       "  5.737047868907497,\n",
       "  5.730328237916353,\n",
       "  5.732094200685393,\n",
       "  5.727900484678499,\n",
       "  5.72927069050863,\n",
       "  5.728571022532361,\n",
       "  5.734089213248162,\n",
       "  5.72740437594204,\n",
       "  5.721989490574899,\n",
       "  5.725676440142641,\n",
       "  5.7213529308376785,\n",
       "  5.725165297274399,\n",
       "  5.721670083401128,\n",
       "  5.722095437954522,\n",
       "  5.722379817792419,\n",
       "  5.72074643279579,\n",
       "  5.717715487516711,\n",
       "  5.719521796251811,\n",
       "  5.719433318645883,\n",
       "  5.7191406819539585,\n",
       "  5.7175552153009415,\n",
       "  5.714921980278073,\n",
       "  5.711991457422641,\n",
       "  5.712296659122327,\n",
       "  5.714348363906898,\n",
       "  5.709017048926958,\n",
       "  5.70955657880865,\n",
       "  5.707597195618148,\n",
       "  5.707362772619306,\n",
       "  5.710199090458996,\n",
       "  5.71038919268148,\n",
       "  5.709145049277971,\n",
       "  5.7058423701592424,\n",
       "  5.709775082570965,\n",
       "  5.709962467655717,\n",
       "  5.707966761025517,\n",
       "  5.705247955342389,\n",
       "  5.703813531472392,\n",
       "  5.706647505399233,\n",
       "  5.70668760337008,\n",
       "  5.708803543567567,\n",
       "  5.706586550713669,\n",
       "  5.70666091381964,\n",
       "  5.703854872049336,\n",
       "  5.70520422534166,\n",
       "  5.7090853248026505,\n",
       "  5.713541462072154,\n",
       "  5.711223104708693,\n",
       "  5.709211949155649,\n",
       "  5.708777174044464,\n",
       "  5.712287352593122,\n",
       "  5.7107200548927075,\n",
       "  5.708449274587389,\n",
       "  5.710992069258397,\n",
       "  5.712542311122771,\n",
       "  5.711787301113491,\n",
       "  5.714073441433001,\n",
       "  5.706780615058175,\n",
       "  5.702934477251934,\n",
       "  5.708631670311725,\n",
       "  5.7045764612719,\n",
       "  5.704969806229091,\n",
       "  5.708483194416494,\n",
       "  5.699204654096507,\n",
       "  5.698901536754108,\n",
       "  5.701545188771222,\n",
       "  5.701328623443549,\n",
       "  5.701877283663986,\n",
       "  5.712125827763148,\n",
       "  5.709993548999513,\n",
       "  5.711092074963961,\n",
       "  5.7112991228425765,\n",
       "  5.717887792995745,\n",
       "  5.717286772875898,\n",
       "  5.718745879540384,\n",
       "  5.721858094445653,\n",
       "  5.711772821871972,\n",
       "  5.716453551088925,\n",
       "  5.725927138336072,\n",
       "  5.748102932516123,\n",
       "  5.7551727851807195,\n",
       "  5.746928935093289,\n",
       "  5.748685406251141,\n",
       "  5.750959871846548,\n",
       "  5.742631699249099,\n",
       "  5.717966581336001,\n",
       "  5.69596419112789,\n",
       "  5.729388779071399,\n",
       "  5.716111347108519,\n",
       "  5.713673719053938,\n",
       "  5.692681914601606,\n",
       "  5.691024399773728,\n",
       "  5.687138069736007,\n",
       "  5.688974116236715,\n",
       "  5.684580021425011,\n",
       "  5.683257314793277,\n",
       "  5.683174003857201,\n",
       "  5.6854691713853835,\n",
       "  5.681013863066004,\n",
       "  5.68437597084434,\n",
       "  5.681825064193993,\n",
       "  5.684101675531547,\n",
       "  5.681992012015991,\n",
       "  5.68126753127827,\n",
       "  5.681398249946863,\n",
       "  5.68103796044878,\n",
       "  5.681649200480111,\n",
       "  5.682595428912033,\n",
       "  5.682889420534814,\n",
       "  5.686840592640374,\n",
       "  5.683135935281995,\n",
       "  5.6790891186645975,\n",
       "  5.681929678022039,\n",
       "  5.682020164377183,\n",
       "  5.68467927534315,\n",
       "  5.6827028430728275,\n",
       "  5.680854582188341,\n",
       "  5.6801660465897665,\n",
       "  5.6813532715953725,\n",
       "  5.682576350928322,\n",
       "  5.6812474220212925,\n",
       "  5.679285928642795,\n",
       "  5.6799450175500015,\n",
       "  5.683484188143179,\n",
       "  5.679851454860901,\n",
       "  5.6826563667538945,\n",
       "  5.682357359112844,\n",
       "  5.685419125176363,\n",
       "  5.68523478433636,\n",
       "  5.683932555883657,\n",
       "  5.682241217130257,\n",
       "  5.682549469472243,\n",
       "  5.685069565177966,\n",
       "  5.684063160089799,\n",
       "  5.681989459093601,\n",
       "  5.687499714742039,\n",
       "  5.687594747589526,\n",
       "  5.687398629027835,\n",
       "  5.688077576261813,\n",
       "  5.686670545902617,\n",
       "  5.682429561328437,\n",
       "  5.683423868856759,\n",
       "  5.68288870097565,\n",
       "  5.68378832237271,\n",
       "  5.683095483252061,\n",
       "  5.68142447338977,\n",
       "  5.682419072181515,\n",
       "  5.683939967914235,\n",
       "  5.681053971989076,\n",
       "  5.682853536227305,\n",
       "  5.683182759239784,\n",
       "  5.679149989195512,\n",
       "  5.679506597539833,\n",
       "  5.680942917861738,\n",
       "  5.683869706776452,\n",
       "  5.6808633917030384,\n",
       "  5.683482745636297,\n",
       "  5.6835278369970625,\n",
       "  5.683802385433253,\n",
       "  5.681296302307698,\n",
       "  5.68535373970461,\n",
       "  5.6831357548328665,\n",
       "  5.683628631869174,\n",
       "  5.687366619229321,\n",
       "  5.686325561605679,\n",
       "  5.684647097305161,\n",
       "  5.689438459163048,\n",
       "  5.691583066248336,\n",
       "  5.691258475948924,\n",
       "  5.689500205300253,\n",
       "  5.6987150406329246,\n",
       "  5.7205202087931255,\n",
       "  5.73144493432716,\n",
       "  5.735482092876497,\n",
       "  5.710046611579773,\n",
       "  5.691586754516987,\n",
       "  5.683290801232471,\n",
       "  5.6760863951857985,\n",
       "  5.679451869122067,\n",
       "  5.679852247986529,\n",
       "  5.676608362665375,\n",
       "  5.6753644264267376,\n",
       "  5.6714117387521155,\n",
       "  5.675563558699792,\n",
       "  5.674570936252355,\n",
       "  5.675326048100858,\n",
       "  5.676726709363352,\n",
       "  5.6803329619662755,\n",
       "  5.678792610636791,\n",
       "  5.677796339328304,\n",
       "  5.680455605652709,\n",
       "  5.684378256248841,\n",
       "  5.678691354661201,\n",
       "  5.678040128278381,\n",
       "  5.6801891838219,\n",
       "  5.6826402722411835,\n",
       "  5.682559809143849,\n",
       "  5.686166751935091,\n",
       "  5.68509163918787,\n",
       "  5.681501247764894,\n",
       "  5.687364905727078,\n",
       "  5.688284261752591,\n",
       "  5.686252357469762,\n",
       "  5.68399516809528,\n",
       "  5.6828935127163005,\n",
       "  5.678981630274309,\n",
       "  5.68298361563674,\n",
       "  5.6794591311372375,\n",
       "  5.686231453515817,\n",
       "  5.690644668493889,\n",
       "  5.699785512856934,\n",
       "  5.7030057855774885,\n",
       "  5.702611591868799,\n",
       "  5.713136592443267],\n",
       " 'val_ATT_loss': [2.0628629072410303,\n",
       "  1.9926818265178339,\n",
       "  1.930031727242276,\n",
       "  1.8815406289527086,\n",
       "  1.8281794638653113,\n",
       "  1.7681751963568897,\n",
       "  1.703275931559927,\n",
       "  1.6399110109825445,\n",
       "  1.5864936799053255,\n",
       "  1.5364096210739477,\n",
       "  1.49033897242895,\n",
       "  1.4503923705438289,\n",
       "  1.4094390563848542,\n",
       "  1.363439965296567,\n",
       "  1.3263687237006863,\n",
       "  1.291207428627867,\n",
       "  1.2597112796170924,\n",
       "  1.2307452595815427,\n",
       "  1.2041816954932563,\n",
       "  1.189667251051926,\n",
       "  1.162913224803723,\n",
       "  1.146044554870303,\n",
       "  1.1256014321877705,\n",
       "  1.1030359313013107,\n",
       "  1.0848153361944648,\n",
       "  1.0695633141975092,\n",
       "  1.0604948202768962,\n",
       "  1.0440516901936958,\n",
       "  1.0402397914388315,\n",
       "  1.0289866959418708,\n",
       "  1.015850187195995,\n",
       "  1.0103642156695931,\n",
       "  1.004153801052551,\n",
       "  0.9915825631318054,\n",
       "  0.9856627610156207,\n",
       "  0.982313059936694,\n",
       "  0.9755656606540447,\n",
       "  0.9740223402414865,\n",
       "  0.9642707538313982,\n",
       "  0.9588522124823516,\n",
       "  0.9508410729770738,\n",
       "  0.9508315050989632,\n",
       "  0.945832598984726,\n",
       "  0.9424832953912455,\n",
       "  0.9408338911891952,\n",
       "  0.9321288585178251,\n",
       "  0.9312737397546691,\n",
       "  0.9292619347087736,\n",
       "  0.9212983946005503,\n",
       "  0.9176339333861824,\n",
       "  0.9157498029431677,\n",
       "  0.9120339166100432,\n",
       "  0.914070880025383,\n",
       "  0.91096814425011,\n",
       "  0.9018004689274765,\n",
       "  0.9031248679006003,\n",
       "  0.9015705693301147,\n",
       "  0.9021207411599353,\n",
       "  0.9016220511459723,\n",
       "  0.8982637735401712,\n",
       "  0.8949619736371002,\n",
       "  0.894901902452717,\n",
       "  0.8921808866950555,\n",
       "  0.8920210121366067,\n",
       "  0.8920826492755394,\n",
       "  0.892038471693915,\n",
       "  0.8856680710625843,\n",
       "  0.8845787474779578,\n",
       "  0.8815214543080911,\n",
       "  0.8843220140391249,\n",
       "  0.87814670425605,\n",
       "  0.8729857574633466,\n",
       "  0.8755320311804128,\n",
       "  0.8756943188789414,\n",
       "  0.8724922365773984,\n",
       "  0.8747765063270321,\n",
       "  0.8713814783387068,\n",
       "  0.868887147404314,\n",
       "  0.8691213277539587,\n",
       "  0.8642848879098892,\n",
       "  0.8635706813112507,\n",
       "  0.8633936207468916,\n",
       "  0.8584286955798545,\n",
       "  0.8605160255257677,\n",
       "  0.8576688514492377,\n",
       "  0.8572755835162915,\n",
       "  0.8595346029454131,\n",
       "  0.8562147375044784,\n",
       "  0.8570252489752885,\n",
       "  0.8538406372554903,\n",
       "  0.8545555983617054,\n",
       "  0.8556447444650216,\n",
       "  0.8595295324315869,\n",
       "  0.8545046997506444,\n",
       "  0.8510372284951249,\n",
       "  0.8517539042040585,\n",
       "  0.8503212965116268,\n",
       "  0.8507200027626705,\n",
       "  0.8515025396657184,\n",
       "  0.8507378143750555,\n",
       "  0.8485134276675015,\n",
       "  0.8475986738515094,\n",
       "  0.8467819594997701,\n",
       "  0.8459157046021485,\n",
       "  0.8472239611352362,\n",
       "  0.8485210208388848,\n",
       "  0.8468395002731463,\n",
       "  0.8454602371628691,\n",
       "  0.843536935322653,\n",
       "  0.8427983710194021,\n",
       "  0.8449811983883866,\n",
       "  0.842844994208677,\n",
       "  0.8410631739269427,\n",
       "  0.8408426214766697,\n",
       "  0.8397692422556683,\n",
       "  0.8386390301270213,\n",
       "  0.839403674733348,\n",
       "  0.8403635795523481,\n",
       "  0.8367377263985998,\n",
       "  0.8389020500386634,\n",
       "  0.8387475798769695,\n",
       "  0.8382372656246511,\n",
       "  0.8371837091882054,\n",
       "  0.835548485197672,\n",
       "  0.8353191949729997,\n",
       "  0.8360605254405882,\n",
       "  0.8363656618488513,\n",
       "  0.8358083882225238,\n",
       "  0.8353082706046299,\n",
       "  0.8331300084426151,\n",
       "  0.8323133700504536,\n",
       "  0.8336777523523424,\n",
       "  0.8338251626346169,\n",
       "  0.8343382279078165,\n",
       "  0.8333627556155367,\n",
       "  0.8341989036255736,\n",
       "  0.83389320690942,\n",
       "  0.8330852958729597,\n",
       "  0.8307460976567694,\n",
       "  0.8328846636099544,\n",
       "  0.8332051130814281,\n",
       "  0.8316002436042801,\n",
       "  0.8351767299378791,\n",
       "  0.8320141513415469,\n",
       "  0.8313658585151037,\n",
       "  0.8339744733601082,\n",
       "  0.8328436525856576,\n",
       "  0.8320665114778814,\n",
       "  0.8326356850746202,\n",
       "  0.8315367864641717,\n",
       "  0.8309758969438754,\n",
       "  0.8305672585237317,\n",
       "  0.8322391885567487,\n",
       "  0.8321702007840319,\n",
       "  0.8329555027610888,\n",
       "  0.8355592933854437,\n",
       "  0.8351526253107118,\n",
       "  0.8344842424479927,\n",
       "  0.8341164691903726,\n",
       "  0.8338942245496967,\n",
       "  0.8358023074584279,\n",
       "  0.8310883649723316,\n",
       "  0.831746141963858,\n",
       "  0.831519808105337,\n",
       "  0.8312727369186355,\n",
       "  0.8373666893902832,\n",
       "  0.8371940703411412,\n",
       "  0.8367526491240758,\n",
       "  0.8428891596755361,\n",
       "  0.8445068654732976,\n",
       "  0.8434746325985203,\n",
       "  0.8337006493797147,\n",
       "  0.8263408565908913,\n",
       "  0.8417300579509115,\n",
       "  0.8299241061133098,\n",
       "  0.829059043550879,\n",
       "  0.824550705712016,\n",
       "  0.8190613879663188,\n",
       "  0.8192622335703392,\n",
       "  0.8207914406448845,\n",
       "  0.8195052148123098,\n",
       "  0.8198490857593412,\n",
       "  0.818221726189784,\n",
       "  0.8202195569751708,\n",
       "  0.8190342569496574,\n",
       "  0.8191942599246173,\n",
       "  0.8195590739085422,\n",
       "  0.8203470706939697,\n",
       "  0.8181565832800981,\n",
       "  0.8181649137560915,\n",
       "  0.8176119360255032,\n",
       "  0.8193241264519653,\n",
       "  0.818523293103629,\n",
       "  0.819024873821716,\n",
       "  0.8185681376030775,\n",
       "  0.8201910853385925,\n",
       "  0.8186278472828671,\n",
       "  0.8188082854437634,\n",
       "  0.8196305181921982,\n",
       "  0.8193038131405668,\n",
       "  0.8204985983003446,\n",
       "  0.8192961907725993,\n",
       "  0.8186216504593206,\n",
       "  0.8176002073578719,\n",
       "  0.8186494285255913,\n",
       "  0.8197528646001971,\n",
       "  0.8189425736181135,\n",
       "  0.818244110883736,\n",
       "  0.8190224554722871,\n",
       "  0.8190906450031249,\n",
       "  0.8174414532940563,\n",
       "  0.8179333567377028,\n",
       "  0.8186604997249154,\n",
       "  0.8201153592123249,\n",
       "  0.8181195169444976,\n",
       "  0.8165379547006716,\n",
       "  0.8182078404882089,\n",
       "  0.8182220540153302,\n",
       "  0.8193465184147765,\n",
       "  0.8189820077845721,\n",
       "  0.8170667077467694,\n",
       "  0.8187360770818664,\n",
       "  0.8169556794370093,\n",
       "  0.8170737536942086,\n",
       "  0.8176673564726744,\n",
       "  0.8166837663185306,\n",
       "  0.8197578836262711,\n",
       "  0.8192586156168604,\n",
       "  0.8186974272253068,\n",
       "  0.8182432122104536,\n",
       "  0.8190806415992055,\n",
       "  0.8180359244588914,\n",
       "  0.8183380887518085,\n",
       "  0.8182805648179559,\n",
       "  0.8178638290099012,\n",
       "  0.8181699375069238,\n",
       "  0.8186126871079933,\n",
       "  0.8176615047987884,\n",
       "  0.8178533330196287,\n",
       "  0.818084153217998,\n",
       "  0.8195311854767605,\n",
       "  0.8191539778457424,\n",
       "  0.8186025978103886,\n",
       "  0.8184050201642804,\n",
       "  0.8181579392857667,\n",
       "  0.8183223127834196,\n",
       "  0.8191951885213696,\n",
       "  0.8164843818763408,\n",
       "  0.8165581437145791,\n",
       "  0.8182236564353229,\n",
       "  0.8183481467933189,\n",
       "  0.8175660617710129,\n",
       "  0.8169854866053031,\n",
       "  0.8173692450533069,\n",
       "  0.8183833404285151,\n",
       "  0.8157700925338559,\n",
       "  0.8165577583438982,\n",
       "  0.8247471104065577,\n",
       "  0.8282063048544938,\n",
       "  0.8259788431772371,\n",
       "  0.8222861004069568,\n",
       "  0.8199364768295754,\n",
       "  0.8153427330459037,\n",
       "  0.8136459723478411,\n",
       "  0.8152720570806565,\n",
       "  0.8157283351188753,\n",
       "  0.8159110691489243,\n",
       "  0.8180350907207504,\n",
       "  0.8156808393030632,\n",
       "  0.8177904279493704,\n",
       "  0.8167533786073933,\n",
       "  0.8169518365123407,\n",
       "  0.8161467248346748,\n",
       "  0.8184609771744022,\n",
       "  0.8177992135770922,\n",
       "  0.8178560594475366,\n",
       "  0.8179708422441793,\n",
       "  0.8181179766732503,\n",
       "  0.8159999968559761,\n",
       "  0.8165200951138163,\n",
       "  0.8174975124316487,\n",
       "  0.8172750841311323,\n",
       "  0.8183788660822845,\n",
       "  0.8190125964763688,\n",
       "  0.8170301272132532,\n",
       "  0.8160736033829247,\n",
       "  0.81675745328752,\n",
       "  0.8184584393733885,\n",
       "  0.8168806150192167,\n",
       "  0.8163011848199658,\n",
       "  0.8190413970288223,\n",
       "  0.8196091559844288,\n",
       "  0.8203037759152855,\n",
       "  0.8173867567041055,\n",
       "  0.8163313009147721,\n",
       "  0.8138945839269375,\n",
       "  0.8153857981286398,\n",
       "  0.814211673489431,\n",
       "  0.8144963085651398,\n",
       "  0.8142286321011986],\n",
       " 'val_VAL_loss': [2.2101888774063787,\n",
       "  2.067734114054976,\n",
       "  1.958838940841224,\n",
       "  1.892140985122455,\n",
       "  1.8348659588198357,\n",
       "  1.787070705385631,\n",
       "  1.75271847800081,\n",
       "  1.7392504679158403,\n",
       "  1.7250314469407932,\n",
       "  1.7141723603450607,\n",
       "  1.7065249640366127,\n",
       "  1.6994930388304987,\n",
       "  1.6976404319255811,\n",
       "  1.6932503337343339,\n",
       "  1.688240642030838,\n",
       "  1.6834572788529796,\n",
       "  1.681588128282519,\n",
       "  1.6785716288195456,\n",
       "  1.6742957043530318,\n",
       "  1.672163963317871,\n",
       "  1.6714074253448712,\n",
       "  1.6688942304385708,\n",
       "  1.663604911325013,\n",
       "  1.6619622313917564,\n",
       "  1.6618079088004352,\n",
       "  1.6644980314329927,\n",
       "  1.6638120277761825,\n",
       "  1.6588099255350424,\n",
       "  1.6579214998066718,\n",
       "  1.6560013264858078,\n",
       "  1.6518119514869352,\n",
       "  1.6515240880656126,\n",
       "  1.649839324904193,\n",
       "  1.6496885034251096,\n",
       "  1.648666052395487,\n",
       "  1.6463126179032725,\n",
       "  1.6455954900516079,\n",
       "  1.6439148299212527,\n",
       "  1.643793548269225,\n",
       "  1.642589792829429,\n",
       "  1.6421078660805237,\n",
       "  1.642046415159855,\n",
       "  1.6409431049976442,\n",
       "  1.6405427279730735,\n",
       "  1.639157207141369,\n",
       "  1.638459977845253,\n",
       "  1.6385391780308314,\n",
       "  1.6370515887960424,\n",
       "  1.6367618679413067,\n",
       "  1.636825398271307,\n",
       "  1.6367279344004364,\n",
       "  1.6345907338147092,\n",
       "  1.6342656113244043,\n",
       "  1.6348616759765324,\n",
       "  1.6329598350477923,\n",
       "  1.6341039288807384,\n",
       "  1.6330677311995934,\n",
       "  1.6325524845734019,\n",
       "  1.631567330195986,\n",
       "  1.6313230404125645,\n",
       "  1.6319140976872937,\n",
       "  1.6300937800571835,\n",
       "  1.6298035788418623,\n",
       "  1.6304512481971327,\n",
       "  1.6301260476041897,\n",
       "  1.6296059474569236,\n",
       "  1.629351951218591,\n",
       "  1.6283190021373954,\n",
       "  1.6294573392774083,\n",
       "  1.6292556712192854,\n",
       "  1.6292480612036042,\n",
       "  1.6284672092334391,\n",
       "  1.6292638144469613,\n",
       "  1.6279024314410582,\n",
       "  1.6271465805363772,\n",
       "  1.6270019250550294,\n",
       "  1.6281466137599476,\n",
       "  1.626766762122732,\n",
       "  1.627000488671176,\n",
       "  1.6269665944752434,\n",
       "  1.626941627469556,\n",
       "  1.6264960125749335,\n",
       "  1.6261652643457423,\n",
       "  1.6259064645015548,\n",
       "  1.6258564611961102,\n",
       "  1.6256454948134023,\n",
       "  1.6258377553206946,\n",
       "  1.6247045001372915,\n",
       "  1.625022983903368,\n",
       "  1.6246866158076696,\n",
       "  1.6249050307156416,\n",
       "  1.62430875935578,\n",
       "  1.6248532269388585,\n",
       "  1.6242998920637985,\n",
       "  1.6236507540265914,\n",
       "  1.624640845312861,\n",
       "  1.6236772114420173,\n",
       "  1.6248150981705765,\n",
       "  1.6233891812451366,\n",
       "  1.623785874526489,\n",
       "  1.6246221300416392,\n",
       "  1.6243825863147605,\n",
       "  1.6236445093389802,\n",
       "  1.6245353638832205,\n",
       "  1.6240697858368822,\n",
       "  1.6235398870383577,\n",
       "  1.623571905009265,\n",
       "  1.6231539143717348,\n",
       "  1.6228181740333294,\n",
       "  1.6231660960343084,\n",
       "  1.6231223885061705,\n",
       "  1.6220573515727603,\n",
       "  1.622831134960569,\n",
       "  1.6222515247138263,\n",
       "  1.6225311767878792,\n",
       "  1.6238533534439914,\n",
       "  1.623661839316044,\n",
       "  1.6229271565752077,\n",
       "  1.6230348812535478,\n",
       "  1.623624344177434,\n",
       "  1.6237382959262492,\n",
       "  1.6232431651336219,\n",
       "  1.6226880820513947,\n",
       "  1.6227550154249069,\n",
       "  1.623776103475411,\n",
       "  1.6235423593098306,\n",
       "  1.6241459605729052,\n",
       "  1.6235927208303818,\n",
       "  1.6237842144050034,\n",
       "  1.6235749545355735,\n",
       "  1.6242969517637356,\n",
       "  1.6251358574834363,\n",
       "  1.6265720998125124,\n",
       "  1.6256282922669585,\n",
       "  1.625283064513371,\n",
       "  1.6248594234729636,\n",
       "  1.6261313818945673,\n",
       "  1.6258782530065827,\n",
       "  1.6259010589768734,\n",
       "  1.6260358018828143,\n",
       "  1.6264457326804476,\n",
       "  1.626729019169737,\n",
       "  1.6262989038317075,\n",
       "  1.6249221545722097,\n",
       "  1.6238562062456103,\n",
       "  1.6248857323172057,\n",
       "  1.6239109362287474,\n",
       "  1.6243010982504031,\n",
       "  1.625282503113958,\n",
       "  1.622555955877445,\n",
       "  1.6226418799367444,\n",
       "  1.6236593100824968,\n",
       "  1.6230298116289337,\n",
       "  1.6232356942933182,\n",
       "  1.6263901083340198,\n",
       "  1.6248114185380231,\n",
       "  1.6253131498844164,\n",
       "  1.6256049601315277,\n",
       "  1.6279237746017907,\n",
       "  1.6277975161087337,\n",
       "  1.6276478573606519,\n",
       "  1.630256576491107,\n",
       "  1.6266755599693712,\n",
       "  1.628311247661196,\n",
       "  1.6315514671391453,\n",
       "  1.6369120810419469,\n",
       "  1.6393262382798595,\n",
       "  1.6367254286564041,\n",
       "  1.6352654155252015,\n",
       "  1.6354843354577502,\n",
       "  1.6330523555501928,\n",
       "  1.6280886439854287,\n",
       "  1.6232077781789995,\n",
       "  1.629219573706829,\n",
       "  1.6287290803317367,\n",
       "  1.628204891834353,\n",
       "  1.6227104029631967,\n",
       "  1.6239876706024696,\n",
       "  1.6226252787218893,\n",
       "  1.6227275585306102,\n",
       "  1.6216916022042336,\n",
       "  1.6211360763446452,\n",
       "  1.6216507592224723,\n",
       "  1.6217498714700709,\n",
       "  1.6206598687054488,\n",
       "  1.621727236973241,\n",
       "  1.6207553300951503,\n",
       "  1.6212515349458592,\n",
       "  1.6212784762452976,\n",
       "  1.621034205840726,\n",
       "  1.6212621046404534,\n",
       "  1.6205712779989383,\n",
       "  1.621041969125494,\n",
       "  1.6211901850301056,\n",
       "  1.6214404276439123,\n",
       "  1.6222165024339272,\n",
       "  1.6215026959997092,\n",
       "  1.6200936110736115,\n",
       "  1.6207663866099467,\n",
       "  1.6209054504122054,\n",
       "  1.6213935590142687,\n",
       "  1.6211355507667429,\n",
       "  1.6207443105763402,\n",
       "  1.620855279743965,\n",
       "  1.6209012810232604,\n",
       "  1.620941162109375,\n",
       "  1.6207682828010597,\n",
       "  1.6203472725863528,\n",
       "  1.6203075206925717,\n",
       "  1.6214645143800182,\n",
       "  1.620803333855615,\n",
       "  1.621574336672064,\n",
       "  1.621232286462643,\n",
       "  1.6217679219880128,\n",
       "  1.6223717557972874,\n",
       "  1.622464867060995,\n",
       "  1.621344458880683,\n",
       "  1.6214424718189708,\n",
       "  1.6219076822543967,\n",
       "  1.6216937174350756,\n",
       "  1.6216409171156108,\n",
       "  1.6229212125533907,\n",
       "  1.623546356050839,\n",
       "  1.6234416251112087,\n",
       "  1.6234700732630463,\n",
       "  1.6233289265280286,\n",
       "  1.6208905592340555,\n",
       "  1.6213884177466331,\n",
       "  1.6213970912501143,\n",
       "  1.6218483700540853,\n",
       "  1.621338280550952,\n",
       "  1.6211295163102926,\n",
       "  1.6213603278099022,\n",
       "  1.6218864676987597,\n",
       "  1.6210633809930586,\n",
       "  1.6215611995734605,\n",
       "  1.6215233573772636,\n",
       "  1.6204961614655744,\n",
       "  1.620551088173401,\n",
       "  1.6209529215479133,\n",
       "  1.6214461737665637,\n",
       "  1.6205698046190986,\n",
       "  1.6216267159419695,\n",
       "  1.6217076056109274,\n",
       "  1.6218814820491623,\n",
       "  1.6209913298414258,\n",
       "  1.6220528503944134,\n",
       "  1.6222171243188417,\n",
       "  1.622356829384865,\n",
       "  1.6230476542646661,\n",
       "  1.622659138270787,\n",
       "  1.6223603451780497,\n",
       "  1.6241509908525815,\n",
       "  1.624737940398343,\n",
       "  1.6242917118401363,\n",
       "  1.6245767042554657,\n",
       "  1.6273857607630087,\n",
       "  1.6319243661288558,\n",
       "  1.634412876490889,\n",
       "  1.6365010832330864,\n",
       "  1.6292535037242721,\n",
       "  1.6238834258958037,\n",
       "  1.622649356062189,\n",
       "  1.6208134742793192,\n",
       "  1.62139327068047,\n",
       "  1.6213746376225513,\n",
       "  1.62023243117215,\n",
       "  1.6191097785686623,\n",
       "  1.6185769664830174,\n",
       "  1.6192577102501404,\n",
       "  1.6192725192149873,\n",
       "  1.6194580705295056,\n",
       "  1.620193328176226,\n",
       "  1.6206239949306245,\n",
       "  1.620331132353233,\n",
       "  1.6199800932935893,\n",
       "  1.6208282544695098,\n",
       "  1.6220867598585307,\n",
       "  1.620897119268408,\n",
       "  1.6205066777215216,\n",
       "  1.6208972237967505,\n",
       "  1.6217883960366837,\n",
       "  1.6213936476871884,\n",
       "  1.622384718486241,\n",
       "  1.6226871706582056,\n",
       "  1.6218092147939898,\n",
       "  1.6235358174798524,\n",
       "  1.6232752741264005,\n",
       "  1.6231239141501816,\n",
       "  1.6225646610917717,\n",
       "  1.6212840385624927,\n",
       "  1.6197908247632933,\n",
       "  1.6208932799071514,\n",
       "  1.6206907914777107,\n",
       "  1.623300050867015,\n",
       "  1.6255833615223174,\n",
       "  1.6281332382427647,\n",
       "  1.629598037362686,\n",
       "  1.62937176110122,\n",
       "  1.632969320114023],\n",
       " 'val_ATT_acc': [12.804878048780488,\n",
       "  13.414634146341463,\n",
       "  20.121951219512194,\n",
       "  29.471544715447155,\n",
       "  35.97560975609756,\n",
       "  50.8130081300813,\n",
       "  56.707317073170735,\n",
       "  64.83739837398375,\n",
       "  66.869918699187,\n",
       "  69.71544715447155,\n",
       "  70.52845528455285,\n",
       "  73.3739837398374,\n",
       "  74.59349593495935,\n",
       "  75.60975609756098,\n",
       "  76.42276422764228,\n",
       "  77.23577235772358,\n",
       "  76.6260162601626,\n",
       "  77.84552845528455,\n",
       "  78.86178861788618,\n",
       "  78.2520325203252,\n",
       "  80.28455284552845,\n",
       "  79.67479674796748,\n",
       "  80.6910569105691,\n",
       "  81.91056910569105,\n",
       "  82.72357723577235,\n",
       "  82.3170731707317,\n",
       "  82.72357723577235,\n",
       "  84.14634146341463,\n",
       "  84.14634146341463,\n",
       "  84.55284552845528,\n",
       "  84.55284552845528,\n",
       "  84.7560975609756,\n",
       "  85.36585365853658,\n",
       "  85.36585365853658,\n",
       "  85.97560975609755,\n",
       "  85.97560975609755,\n",
       "  85.5691056910569,\n",
       "  85.5691056910569,\n",
       "  86.3821138211382,\n",
       "  86.78861788617886,\n",
       "  86.58536585365853,\n",
       "  87.39837398373983,\n",
       "  87.8048780487805,\n",
       "  86.58536585365853,\n",
       "  87.60162601626017,\n",
       "  87.1951219512195,\n",
       "  87.8048780487805,\n",
       "  87.1951219512195,\n",
       "  87.60162601626017,\n",
       "  88.82113821138212,\n",
       "  89.02439024390245,\n",
       "  88.41463414634147,\n",
       "  88.41463414634147,\n",
       "  88.21138211382114,\n",
       "  89.63414634146342,\n",
       "  89.63414634146342,\n",
       "  89.63414634146342,\n",
       "  89.22764227642277,\n",
       "  89.02439024390245,\n",
       "  89.22764227642277,\n",
       "  90.04065040650407,\n",
       "  90.2439024390244,\n",
       "  89.4308943089431,\n",
       "  89.4308943089431,\n",
       "  89.22764227642277,\n",
       "  89.63414634146342,\n",
       "  91.0569105691057,\n",
       "  91.0569105691057,\n",
       "  90.2439024390244,\n",
       "  90.2439024390244,\n",
       "  91.46341463414635,\n",
       "  91.46341463414635,\n",
       "  91.26016260162602,\n",
       "  91.26016260162602,\n",
       "  91.869918699187,\n",
       "  91.66666666666667,\n",
       "  91.46341463414635,\n",
       "  92.07317073170732,\n",
       "  91.869918699187,\n",
       "  92.27642276422765,\n",
       "  92.88617886178862,\n",
       "  93.08943089430895,\n",
       "  93.4959349593496,\n",
       "  93.08943089430895,\n",
       "  93.08943089430895,\n",
       "  93.4959349593496,\n",
       "  93.08943089430895,\n",
       "  92.6829268292683,\n",
       "  93.08943089430895,\n",
       "  93.90243902439025,\n",
       "  92.88617886178862,\n",
       "  93.08943089430895,\n",
       "  92.6829268292683,\n",
       "  92.88617886178862,\n",
       "  93.69918699186992,\n",
       "  93.69918699186992,\n",
       "  94.71544715447155,\n",
       "  93.90243902439025,\n",
       "  93.4959349593496,\n",
       "  93.4959349593496,\n",
       "  93.90243902439025,\n",
       "  93.4959349593496,\n",
       "  94.71544715447155,\n",
       "  93.08943089430895,\n",
       "  93.08943089430895,\n",
       "  93.90243902439025,\n",
       "  94.3089430894309,\n",
       "  94.3089430894309,\n",
       "  94.10569105691057,\n",
       "  93.90243902439025,\n",
       "  94.3089430894309,\n",
       "  94.51219512195122,\n",
       "  93.90243902439025,\n",
       "  94.51219512195122,\n",
       "  94.3089430894309,\n",
       "  94.3089430894309,\n",
       "  94.10569105691057,\n",
       "  94.3089430894309,\n",
       "  94.3089430894309,\n",
       "  94.71544715447155,\n",
       "  93.69918699186992,\n",
       "  94.10569105691057,\n",
       "  94.10569105691057,\n",
       "  94.71544715447155,\n",
       "  94.3089430894309,\n",
       "  94.71544715447155,\n",
       "  94.3089430894309,\n",
       "  94.3089430894309,\n",
       "  94.3089430894309,\n",
       "  94.10569105691057,\n",
       "  94.71544715447155,\n",
       "  94.51219512195122,\n",
       "  94.3089430894309,\n",
       "  94.51219512195122,\n",
       "  94.51219512195122,\n",
       "  94.71544715447155,\n",
       "  95.1219512195122,\n",
       "  94.91869918699187,\n",
       "  94.3089430894309,\n",
       "  94.3089430894309,\n",
       "  94.51219512195122,\n",
       "  95.52845528455285,\n",
       "  94.91869918699187,\n",
       "  94.51219512195122,\n",
       "  94.3089430894309,\n",
       "  94.91869918699187,\n",
       "  94.51219512195122,\n",
       "  95.1219512195122,\n",
       "  94.91869918699187,\n",
       "  94.51219512195122,\n",
       "  94.71544715447155,\n",
       "  94.71544715447155,\n",
       "  94.91869918699187,\n",
       "  95.32520325203252,\n",
       "  94.51219512195122,\n",
       "  94.91869918699187,\n",
       "  94.51219512195122,\n",
       "  94.10569105691057,\n",
       "  94.51219512195122,\n",
       "  94.91869918699187,\n",
       "  94.91869918699187,\n",
       "  94.3089430894309,\n",
       "  94.71544715447155,\n",
       "  93.90243902439025,\n",
       "  94.51219512195122,\n",
       "  93.69918699186992,\n",
       "  93.29268292682927,\n",
       "  93.4959349593496,\n",
       "  92.88617886178862,\n",
       "  92.88617886178862,\n",
       "  92.27642276422765,\n",
       "  93.90243902439025,\n",
       "  96.34146341463415,\n",
       "  95.73170731707317,\n",
       "  93.69918699186992,\n",
       "  94.3089430894309,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.32520325203252,\n",
       "  94.91869918699187,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.32520325203252,\n",
       "  95.73170731707317,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.9349593495935,\n",
       "  94.91869918699187,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  95.32520325203252,\n",
       "  95.32520325203252,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.32520325203252,\n",
       "  95.9349593495935,\n",
       "  94.71544715447155,\n",
       "  95.32520325203252,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  95.1219512195122,\n",
       "  95.1219512195122,\n",
       "  94.71544715447155,\n",
       "  95.32520325203252,\n",
       "  94.91869918699187,\n",
       "  95.1219512195122,\n",
       "  94.51219512195122,\n",
       "  95.1219512195122,\n",
       "  94.91869918699187,\n",
       "  95.1219512195122,\n",
       "  95.52845528455285,\n",
       "  95.1219512195122,\n",
       "  95.32520325203252,\n",
       "  95.1219512195122,\n",
       "  95.1219512195122,\n",
       "  95.32520325203252,\n",
       "  95.1219512195122,\n",
       "  95.73170731707317,\n",
       "  95.1219512195122,\n",
       "  95.32520325203252,\n",
       "  95.32520325203252,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.1219512195122,\n",
       "  95.52845528455285,\n",
       "  95.9349593495935,\n",
       "  94.91869918699187,\n",
       "  94.3089430894309,\n",
       "  93.90243902439025,\n",
       "  95.1219512195122,\n",
       "  95.73170731707317,\n",
       "  96.7479674796748,\n",
       "  97.15447154471545,\n",
       "  95.52845528455285,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.1219512195122,\n",
       "  95.32520325203252,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.1219512195122,\n",
       "  94.91869918699187,\n",
       "  95.32520325203252,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.32520325203252,\n",
       "  95.52845528455285,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317,\n",
       "  96.13821138211382,\n",
       "  95.73170731707317,\n",
       "  95.52845528455285,\n",
       "  95.32520325203252,\n",
       "  95.9349593495935,\n",
       "  95.52845528455285,\n",
       "  95.1219512195122,\n",
       "  94.51219512195122,\n",
       "  95.32520325203252,\n",
       "  94.91869918699187,\n",
       "  95.32520325203252,\n",
       "  96.13821138211382,\n",
       "  96.34146341463415,\n",
       "  95.9349593495935,\n",
       "  95.73170731707317,\n",
       "  95.73170731707317],\n",
       " 'val_VAL_acc': [76.35467980295566,\n",
       "  78.81773399014779,\n",
       "  76.84729064039409,\n",
       "  79.3103448275862,\n",
       "  86.69950738916256,\n",
       "  87.6847290640394,\n",
       "  89.65517241379311,\n",
       "  89.65517241379311,\n",
       "  88.17733990147784,\n",
       "  92.11822660098522,\n",
       "  91.13300492610837,\n",
       "  93.10344827586206,\n",
       "  93.5960591133005,\n",
       "  92.61083743842364,\n",
       "  94.58128078817734,\n",
       "  95.07389162561576,\n",
       "  93.5960591133005,\n",
       "  96.05911330049261,\n",
       "  94.08866995073892,\n",
       "  96.55172413793103,\n",
       "  96.55172413793103,\n",
       "  96.55172413793103,\n",
       "  98.0295566502463,\n",
       "  96.55172413793103,\n",
       "  98.0295566502463,\n",
       "  97.53694581280789,\n",
       "  97.04433497536945,\n",
       "  98.52216748768473,\n",
       "  96.55172413793103,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  98.0295566502463,\n",
       "  97.53694581280789,\n",
       "  97.53694581280789,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  100.0,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  100.0,\n",
       "  100.0,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  96.55172413793103,\n",
       "  97.53694581280789,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  98.0295566502463,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.0295566502463,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  97.53694581280789,\n",
       "  97.04433497536945,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315,\n",
       "  99.50738916256158,\n",
       "  99.50738916256158,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  99.01477832512315,\n",
       "  98.52216748768473,\n",
       "  98.0295566502463,\n",
       "  98.52216748768473,\n",
       "  99.01477832512315],\n",
       " 'val_VAL_jac': [0.0,\n",
       "  0.04433497536945813,\n",
       "  0.14285714285714285,\n",
       "  0.15270935960591134,\n",
       "  0.23973727578600051,\n",
       "  0.28571428806323723,\n",
       "  0.4006568164073775,\n",
       "  0.3981937633946611,\n",
       "  0.4458128125796764,\n",
       "  0.4958949136029323,\n",
       "  0.5344827609696412,\n",
       "  0.5648604308443116,\n",
       "  0.5779967214086373,\n",
       "  0.5607553397493409,\n",
       "  0.5599343224699274,\n",
       "  0.5845648643418486,\n",
       "  0.5747126452441286,\n",
       "  0.5919540269034249,\n",
       "  0.5944170810906171,\n",
       "  0.6297208569907202,\n",
       "  0.6174055872292354,\n",
       "  0.6280788200829417,\n",
       "  0.6403940910189022,\n",
       "  0.6486042755577952,\n",
       "  0.6502463077676708,\n",
       "  0.6559934334214685,\n",
       "  0.6461412190216516,\n",
       "  0.6724137978013514,\n",
       "  0.6830870318295333,\n",
       "  0.6871921252734555,\n",
       "  0.6954023074633969,\n",
       "  0.6945812878350319,\n",
       "  0.6806239799912928,\n",
       "  0.7019704527455598,\n",
       "  0.7019704503966082,\n",
       "  0.6995073962094162,\n",
       "  0.706896556422041,\n",
       "  0.7093596082602815,\n",
       "  0.6954023051144455,\n",
       "  0.7093596082602815,\n",
       "  0.7192118273580016,\n",
       "  0.7093596106092331,\n",
       "  0.7200328446374151,\n",
       "  0.7093596082602815,\n",
       "  0.7216748815451937,\n",
       "  0.7266009899195779,\n",
       "  0.7134646993552523,\n",
       "  0.7151067386119824,\n",
       "  0.7323481179223272,\n",
       "  0.7216748791962422,\n",
       "  0.7356321870399813,\n",
       "  0.7356321893889328,\n",
       "  0.7348111697605678,\n",
       "  0.7323481132244242,\n",
       "  0.7430213496015576,\n",
       "  0.7413793150427306,\n",
       "  0.7348111721095193,\n",
       "  0.7471264430454799,\n",
       "  0.7331691352017408,\n",
       "  0.7315270982939621,\n",
       "  0.7397372804839035,\n",
       "  0.7356321917378844,\n",
       "  0.7331691328527892,\n",
       "  0.7307060786655971,\n",
       "  0.7372742262967115,\n",
       "  0.7331691398996437,\n",
       "  0.7257799726401644,\n",
       "  0.7495894948837205,\n",
       "  0.7389162608555385,\n",
       "  0.7389162585065869,\n",
       "  0.7372742239477599,\n",
       "  0.7430213496015576,\n",
       "  0.7372742239477599,\n",
       "  0.7594417139814404,\n",
       "  0.7545156056070562,\n",
       "  0.7610837532381706,\n",
       "  0.7586206990509785,\n",
       "  0.7512315364894021,\n",
       "  0.7569786597942484,\n",
       "  0.7446633888582878,\n",
       "  0.7495894995816236,\n",
       "  0.7610837485402676,\n",
       "  0.7627257877969976,\n",
       "  0.7528735733971808,\n",
       "  0.7561576401658834,\n",
       "  0.768472915799747,\n",
       "  0.7643678223558248,\n",
       "  0.7627257924949007,\n",
       "  0.7643678247047763,\n",
       "  0.7561576425148349,\n",
       "  0.7536945906765943,\n",
       "  0.7742200391045932,\n",
       "  0.759441716330392,\n",
       "  0.7643678270537277,\n",
       "  0.7577996770736619,\n",
       "  0.7651888466820929,\n",
       "  0.7692939283812574,\n",
       "  0.7709359652890361,\n",
       "  0.7619047658196811,\n",
       "  0.7660098616125548,\n",
       "  0.7569786621431999,\n",
       "  0.7602627312608541,\n",
       "  0.7660098616125548,\n",
       "  0.7668308765430168,\n",
       "  0.7643678223558248,\n",
       "  0.7676518938224304,\n",
       "  0.7635468050764112,\n",
       "  0.7668308765430168,\n",
       "  0.7676518938224304,\n",
       "  0.7569786597942484,\n",
       "  0.7610837485402676,\n",
       "  0.7619047658196811,\n",
       "  0.7627257854480461,\n",
       "  0.7643678200068732,\n",
       "  0.7619047681686326,\n",
       "  0.7569786621431999,\n",
       "  0.7627257854480461,\n",
       "  0.7577996770736619,\n",
       "  0.7619047658196811,\n",
       "  0.7528735710482292,\n",
       "  0.7536945859786912,\n",
       "  0.7430213519505092,\n",
       "  0.7536945836297397,\n",
       "  0.7561576401658834,\n",
       "  0.7561576401658834,\n",
       "  0.7561576378169318,\n",
       "  0.7668308788919683,\n",
       "  0.7545156056070562,\n",
       "  0.7561576425148349,\n",
       "  0.7479474603248935,\n",
       "  0.760262735958757,\n",
       "  0.748768477604307,\n",
       "  0.7356321940868359,\n",
       "  0.751231531791499,\n",
       "  0.7536945953744972,\n",
       "  0.7586206967020269,\n",
       "  0.7536945930255458,\n",
       "  0.747126447743383,\n",
       "  0.7315271029918652,\n",
       "  0.7446633912072393,\n",
       "  0.7446633935561908,\n",
       "  0.7471264453944314,\n",
       "  0.7504105192099886,\n",
       "  0.7553366252354213,\n",
       "  0.751231531791499,\n",
       "  0.7643678247047763,\n",
       "  0.7446633912072393,\n",
       "  0.770935969986939,\n",
       "  0.7684729111018439,\n",
       "  0.7553366252354213,\n",
       "  0.7627257854480461,\n",
       "  0.7602627312608541,\n",
       "  0.7479474603248935,\n",
       "  0.7676518961713819,\n",
       "  0.7463054281150179,\n",
       "  0.7676518961713819,\n",
       "  0.7512315364894021,\n",
       "  0.7651888396352383,\n",
       "  0.7569786621431999,\n",
       "  0.7635468027274597,\n",
       "  0.7651888419841898,\n",
       "  0.7684729134507955,\n",
       "  0.7561576495616894,\n",
       "  0.7323481226202302,\n",
       "  0.7183908100785881,\n",
       "  0.7266009899195779,\n",
       "  0.7175697927991745,\n",
       "  0.7331691352017408,\n",
       "  0.7307060857124517,\n",
       "  0.7405583012867444,\n",
       "  0.7389162655534416,\n",
       "  0.7651888431586655,\n",
       "  0.7520525514198642,\n",
       "  0.7216748862430967,\n",
       "  0.7101806325865496,\n",
       "  0.7323481155733756,\n",
       "  0.7668308741940654,\n",
       "  0.7520525490709127,\n",
       "  0.7586206990509785,\n",
       "  0.7635468050764112,\n",
       "  0.740558297763317,\n",
       "  0.7512315294425476,\n",
       "  0.7553366252354213,\n",
       "  0.7479474650227965,\n",
       "  0.7536945836297397,\n",
       "  0.7577996794226134,\n",
       "  0.759441716330392,\n",
       "  0.7832512432718511,\n",
       "  0.7709359723358906,\n",
       "  0.7602627336098056,\n",
       "  0.7660098592636033,\n",
       "  0.7758620783613233,\n",
       "  0.7660098569146518,\n",
       "  0.7619047681686326,\n",
       "  0.7594417139814404,\n",
       "  0.7430213566484123,\n",
       "  0.7816092016661695,\n",
       "  0.7676518985203334,\n",
       "  0.7668308788919683,\n",
       "  0.7775041176180534,\n",
       "  0.7586206990509785,\n",
       "  0.7725780045457662,\n",
       "  0.7610837532381706,\n",
       "  0.7586206967020269,\n",
       "  0.7725780068947177,\n",
       "  0.7627257901459492,\n",
       "  0.7569786574452969,\n",
       "  0.7758620760123718,\n",
       "  0.760262735958757,\n",
       "  0.7668308765430168,\n",
       "  0.7635468074253627,\n",
       "  0.7635468050764112,\n",
       "  0.7733990218251797,\n",
       "  0.7750410610819097,\n",
       "  0.7881773469483324,\n",
       "  0.7676518985203334,\n",
       "  0.7733990241741312,\n",
       "  0.7824302236434861,\n",
       "  0.7725780068947177,\n",
       "  0.779967169456294,\n",
       "  0.760262735958757,\n",
       "  0.7651888443331413,\n",
       "  0.7610837532381706,\n",
       "  0.7479474626738449,\n",
       "  0.7471264453944314,\n",
       "  0.7553366228864697,\n",
       "  0.7717569825684496,\n",
       "  0.7577996794226134,\n",
       "  0.7742200438024962,\n",
       "  0.770935969986939,\n",
       "  0.7684729111018439,\n",
       "  0.7627257830990947,\n",
       "  0.7660098616125548,\n",
       "  0.7520525537688156,\n",
       "  0.7660098616125548,\n",
       "  0.7635468074253627,\n",
       "  0.7610837532381706,\n",
       "  0.7701149527075255,\n",
       "  0.7619047705175841,\n",
       "  0.7536945859786912,\n",
       "  0.7586206967020269,\n",
       "  0.7668308788919683,\n",
       "  0.7446633888582878,\n",
       "  0.7545156103049593,\n",
       "  0.7545156056070562,\n",
       "  0.7561576425148349,\n",
       "  0.7520525514198642,\n",
       "  0.7545156032581047,\n",
       "  0.7594417092835375,\n",
       "  0.7619047681686326,\n",
       "  0.7520525490709127,\n",
       "  0.7643678200068732,\n",
       "  0.7504105215589402,\n",
       "  0.749589497232672,\n",
       "  0.7438423715788742,\n",
       "  0.7397372828328551,\n",
       "  0.7389162679023931,\n",
       "  0.7044335045838004,\n",
       "  0.718390814776491,\n",
       "  0.749589501930575,\n",
       "  0.7413793162172064,\n",
       "  0.7446633900327636,\n",
       "  0.7586206943530754,\n",
       "  0.7775041105711988,\n",
       "  0.7627257877969976,\n",
       "  0.7701149527075255,\n",
       "  0.7725780045457662,\n",
       "  0.7955665095099087,\n",
       "  0.7766830909428338,\n",
       "  0.7766830956407369,\n",
       "  0.7709359652890361,\n",
       "  0.770114950358574,\n",
       "  0.7717569872663526,\n",
       "  0.7553366275843728,\n",
       "  0.7676518961713819,\n",
       "  0.7619047705175841,\n",
       "  0.7561576425148349,\n",
       "  0.7676518985203334,\n",
       "  0.7701149550564771,\n",
       "  0.7742200461514478,\n",
       "  0.7676519008692849,\n",
       "  0.7807881937825621,\n",
       "  0.7865353100405538,\n",
       "  0.7627257901459492,\n",
       "  0.7561576425148349,\n",
       "  0.7824302236434861,\n",
       "  0.7504105168610371,\n",
       "  0.7758620760123718,\n",
       "  0.7701149527075255,\n",
       "  0.7758620783613233,\n",
       "  0.7725780068947177,\n",
       "  0.7619047705175841,\n",
       "  0.7561576425148349,\n",
       "  0.7668308765430168,\n",
       "  0.7594417104580132,\n",
       "  0.7397372793094278,\n",
       "  0.7110016451680602,\n",
       "  0.717569790450223,\n",
       "  0.7282430244784049,\n",
       "  0.7142857166346658],\n",
       " 'val_VAL_acc_1': [30.049261083743843,\n",
       "  33.99014778325123,\n",
       "  37.4384236453202,\n",
       "  36.45320197044335,\n",
       "  48.275862068965516,\n",
       "  47.783251231527096,\n",
       "  52.216748768472904,\n",
       "  50.24630541871921,\n",
       "  49.75369458128079,\n",
       "  57.142857142857146,\n",
       "  57.142857142857146,\n",
       "  59.60591133004926,\n",
       "  60.59113300492611,\n",
       "  59.11330049261084,\n",
       "  61.083743842364534,\n",
       "  60.59113300492611,\n",
       "  60.59113300492611,\n",
       "  66.50246305418719,\n",
       "  62.5615763546798,\n",
       "  66.50246305418719,\n",
       "  65.02463054187191,\n",
       "  65.51724137931035,\n",
       "  68.96551724137932,\n",
       "  66.99507389162562,\n",
       "  69.95073891625616,\n",
       "  69.45812807881774,\n",
       "  67.98029556650246,\n",
       "  73.89162561576354,\n",
       "  72.41379310344827,\n",
       "  72.9064039408867,\n",
       "  74.38423645320196,\n",
       "  72.9064039408867,\n",
       "  73.39901477832512,\n",
       "  73.89162561576354,\n",
       "  74.8768472906404,\n",
       "  76.35467980295566,\n",
       "  74.8768472906404,\n",
       "  78.32512315270937,\n",
       "  77.83251231527093,\n",
       "  77.83251231527093,\n",
       "  76.84729064039409,\n",
       "  74.8768472906404,\n",
       "  75.36945812807882,\n",
       "  77.33990147783251,\n",
       "  77.33990147783251,\n",
       "  76.35467980295566,\n",
       "  75.86206896551724,\n",
       "  76.84729064039409,\n",
       "  76.35467980295566,\n",
       "  77.33990147783251,\n",
       "  79.3103448275862,\n",
       "  77.83251231527093,\n",
       "  76.84729064039409,\n",
       "  79.80295566502463,\n",
       "  78.81773399014779,\n",
       "  79.3103448275862,\n",
       "  78.81773399014779,\n",
       "  79.80295566502463,\n",
       "  76.35467980295566,\n",
       "  76.84729064039409,\n",
       "  77.83251231527093,\n",
       "  79.3103448275862,\n",
       "  79.3103448275862,\n",
       "  78.32512315270937,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  81.2807881773399,\n",
       "  77.83251231527093,\n",
       "  79.3103448275862,\n",
       "  78.32512315270937,\n",
       "  78.81773399014779,\n",
       "  78.81773399014779,\n",
       "  80.29556650246306,\n",
       "  80.29556650246306,\n",
       "  78.32512315270937,\n",
       "  79.80295566502463,\n",
       "  79.3103448275862,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  79.3103448275862,\n",
       "  79.3103448275862,\n",
       "  80.78817733990148,\n",
       "  79.80295566502463,\n",
       "  80.78817733990148,\n",
       "  78.81773399014779,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  79.80295566502463,\n",
       "  80.29556650246306,\n",
       "  80.78817733990148,\n",
       "  81.77339901477832,\n",
       "  82.26600985221675,\n",
       "  81.2807881773399,\n",
       "  81.77339901477832,\n",
       "  80.29556650246306,\n",
       "  83.2512315270936,\n",
       "  81.2807881773399,\n",
       "  82.75862068965517,\n",
       "  81.77339901477832,\n",
       "  79.80295566502463,\n",
       "  82.26600985221675,\n",
       "  81.2807881773399,\n",
       "  82.75862068965517,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  81.2807881773399,\n",
       "  81.2807881773399,\n",
       "  80.78817733990148,\n",
       "  82.26600985221675,\n",
       "  82.75862068965517,\n",
       "  80.78817733990148,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  83.74384236453201,\n",
       "  80.78817733990148,\n",
       "  82.75862068965517,\n",
       "  81.77339901477832,\n",
       "  82.26600985221675,\n",
       "  81.2807881773399,\n",
       "  81.77339901477832,\n",
       "  83.74384236453201,\n",
       "  83.74384236453201,\n",
       "  81.77339901477832,\n",
       "  82.26600985221675,\n",
       "  81.77339901477832,\n",
       "  82.75862068965517,\n",
       "  83.2512315270936,\n",
       "  83.2512315270936,\n",
       "  82.26600985221675,\n",
       "  81.77339901477832,\n",
       "  82.26600985221675,\n",
       "  83.2512315270936,\n",
       "  81.77339901477832,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  82.75862068965517,\n",
       "  82.26600985221675,\n",
       "  81.77339901477832,\n",
       "  82.26600985221675,\n",
       "  81.2807881773399,\n",
       "  82.26600985221675,\n",
       "  80.78817733990148,\n",
       "  80.29556650246306,\n",
       "  79.3103448275862,\n",
       "  78.81773399014779,\n",
       "  80.29556650246306,\n",
       "  78.81773399014779,\n",
       "  80.78817733990148,\n",
       "  77.33990147783251,\n",
       "  76.35467980295566,\n",
       "  76.84729064039409,\n",
       "  78.32512315270937,\n",
       "  77.83251231527093,\n",
       "  78.32512315270937,\n",
       "  77.83251231527093,\n",
       "  77.83251231527093,\n",
       "  79.3103448275862,\n",
       "  82.75862068965517,\n",
       "  75.86206896551724,\n",
       "  79.3103448275862,\n",
       "  80.29556650246306,\n",
       "  83.2512315270936,\n",
       "  82.75862068965517,\n",
       "  84.23645320197045,\n",
       "  82.75862068965517,\n",
       "  82.26600985221675,\n",
       "  84.23645320197045,\n",
       "  83.2512315270936,\n",
       "  84.23645320197045,\n",
       "  82.75862068965517,\n",
       "  83.74384236453201,\n",
       "  82.75862068965517,\n",
       "  84.72906403940887,\n",
       "  84.72906403940887,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  82.26600985221675,\n",
       "  84.23645320197045,\n",
       "  84.72906403940887,\n",
       "  84.23645320197045,\n",
       "  82.26600985221675,\n",
       "  84.23645320197045,\n",
       "  85.71428571428571,\n",
       "  85.22167487684729,\n",
       "  84.72906403940887,\n",
       "  83.74384236453201,\n",
       "  85.71428571428571,\n",
       "  84.72906403940887,\n",
       "  84.72906403940887,\n",
       "  85.22167487684729,\n",
       "  84.23645320197045,\n",
       "  86.69950738916256,\n",
       "  85.22167487684729,\n",
       "  84.72906403940887,\n",
       "  85.22167487684729,\n",
       "  85.71428571428571,\n",
       "  83.2512315270936,\n",
       "  85.71428571428571,\n",
       "  85.22167487684729,\n",
       "  84.72906403940887,\n",
       "  84.72906403940887,\n",
       "  85.22167487684729,\n",
       "  86.20689655172414,\n",
       "  84.72906403940887,\n",
       "  84.23645320197045,\n",
       "  85.22167487684729,\n",
       "  82.75862068965517,\n",
       "  84.23645320197045,\n",
       "  81.77339901477832,\n",
       "  82.26600985221675,\n",
       "  81.2807881773399,\n",
       "  82.75862068965517,\n",
       "  83.2512315270936,\n",
       "  85.71428571428571,\n",
       "  83.74384236453201,\n",
       "  83.74384236453201,\n",
       "  83.2512315270936,\n",
       "  84.72906403940887,\n",
       "  83.74384236453201,\n",
       "  83.2512315270936,\n",
       "  83.74384236453201,\n",
       "  83.74384236453201,\n",
       "  82.75862068965517,\n",
       "  84.72906403940887,\n",
       "  83.2512315270936,\n",
       "  82.75862068965517,\n",
       "  84.23645320197045,\n",
       "  83.74384236453201,\n",
       "  83.2512315270936,\n",
       "  82.75862068965517,\n",
       "  83.74384236453201,\n",
       "  80.78817733990148,\n",
       "  81.2807881773399,\n",
       "  82.26600985221675,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  82.75862068965517,\n",
       "  80.78817733990148,\n",
       "  80.78817733990148,\n",
       "  83.2512315270936,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  79.80295566502463,\n",
       "  78.32512315270937,\n",
       "  77.33990147783251,\n",
       "  77.83251231527093,\n",
       "  80.29556650246306,\n",
       "  81.2807881773399,\n",
       "  84.23645320197045,\n",
       "  86.20689655172414,\n",
       "  87.19211822660098,\n",
       "  85.22167487684729,\n",
       "  85.71428571428571,\n",
       "  86.20689655172414,\n",
       "  85.71428571428571,\n",
       "  84.72906403940887,\n",
       "  84.23645320197045,\n",
       "  84.23645320197045,\n",
       "  83.2512315270936,\n",
       "  83.2512315270936,\n",
       "  83.74384236453201,\n",
       "  84.23645320197045,\n",
       "  84.23645320197045,\n",
       "  84.72906403940887,\n",
       "  85.71428571428571,\n",
       "  87.19211822660098,\n",
       "  86.20689655172414,\n",
       "  84.72906403940887,\n",
       "  85.22167487684729,\n",
       "  84.23645320197045,\n",
       "  85.22167487684729,\n",
       "  84.72906403940887,\n",
       "  83.2512315270936,\n",
       "  84.23645320197045,\n",
       "  85.71428571428571,\n",
       "  85.71428571428571,\n",
       "  84.72906403940887,\n",
       "  82.26600985221675,\n",
       "  83.74384236453201,\n",
       "  81.77339901477832,\n",
       "  80.29556650246306,\n",
       "  79.80295566502463,\n",
       "  75.86206896551724,\n",
       "  78.32512315270937,\n",
       "  79.3103448275862],\n",
       " 'test_loss': 5.760235344531907,\n",
       " 'test_ATT_loss': 0.8260694216070308,\n",
       " 'test_VAL_loss': 1.6447219743082921,\n",
       " 'test_ATT_acc': 95.42743538767395,\n",
       " 'test_VAL_acc': 99.47916666666667,\n",
       " 'test_VAL_jac': 0.7196180621782938,\n",
       " 'test_VAL_acc_1': 77.60416666666667,\n",
       " 'model_filename': 'model_storage/HeteroGNN/model.pth'}"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'Hetero_best_model/best_config.p', 'wb') as fp:\n",
    "    pickle.dump(train_state,fp, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'Hetero_best_model/best_config.p', 'rb') as fp:\n",
    "    train_state = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "states = pd.DataFrame(train_state['val_VAL_acc'], columns=['val_VAL'])\n",
    "states['train_VAL'] = pd.DataFrame(train_state['train_VAL_acc'])\n",
    "states['val_ATT'] = pd.DataFrame(train_state['val_ATT_acc'])\n",
    "states['train_ATT'] = pd.DataFrame(train_state['train_ATT_acc'])\n",
    "states['val_VAL_1'] = pd.DataFrame(train_state['val_VAL_acc_1'])\n",
    "states['train_VAL_1'] = pd.DataFrame(train_state['train_VAL_acc_1'])\n",
    "states['train_VAL_jac'] = pd.DataFrame(train_state['train_VAL_jac'])\n",
    "states['val_VAL_jac'] = pd.DataFrame(train_state['val_VAL_jac'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>val_VAL</th>\n",
       "      <th>train_VAL</th>\n",
       "      <th>val_ATT</th>\n",
       "      <th>train_ATT</th>\n",
       "      <th>val_VAL_1</th>\n",
       "      <th>train_VAL_1</th>\n",
       "      <th>train_VAL_jac</th>\n",
       "      <th>val_VAL_jac</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>76.354680</td>\n",
       "      <td>74.515235</td>\n",
       "      <td>12.804878</td>\n",
       "      <td>20.498615</td>\n",
       "      <td>30.049261</td>\n",
       "      <td>32.409972</td>\n",
       "      <td>0.005540</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>78.817734</td>\n",
       "      <td>77.562327</td>\n",
       "      <td>13.414634</td>\n",
       "      <td>21.329640</td>\n",
       "      <td>33.990148</td>\n",
       "      <td>34.903047</td>\n",
       "      <td>0.058172</td>\n",
       "      <td>0.044335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>76.847291</td>\n",
       "      <td>77.008310</td>\n",
       "      <td>20.121951</td>\n",
       "      <td>30.470914</td>\n",
       "      <td>37.438424</td>\n",
       "      <td>33.518006</td>\n",
       "      <td>0.139428</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>79.310345</td>\n",
       "      <td>79.501385</td>\n",
       "      <td>29.471545</td>\n",
       "      <td>43.213296</td>\n",
       "      <td>36.453202</td>\n",
       "      <td>40.166205</td>\n",
       "      <td>0.183749</td>\n",
       "      <td>0.152709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>86.699507</td>\n",
       "      <td>86.149584</td>\n",
       "      <td>35.975610</td>\n",
       "      <td>49.861496</td>\n",
       "      <td>48.275862</td>\n",
       "      <td>50.692521</td>\n",
       "      <td>0.236380</td>\n",
       "      <td>0.239737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295</th>\n",
       "      <td>99.014778</td>\n",
       "      <td>99.722992</td>\n",
       "      <td>96.138211</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>80.295567</td>\n",
       "      <td>88.365651</td>\n",
       "      <td>0.844414</td>\n",
       "      <td>0.739737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>296</th>\n",
       "      <td>98.522167</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>96.341463</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>79.802956</td>\n",
       "      <td>89.196676</td>\n",
       "      <td>0.830102</td>\n",
       "      <td>0.711002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>297</th>\n",
       "      <td>98.029557</td>\n",
       "      <td>99.722992</td>\n",
       "      <td>95.934959</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>75.862069</td>\n",
       "      <td>87.534626</td>\n",
       "      <td>0.827793</td>\n",
       "      <td>0.717570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>98.522167</td>\n",
       "      <td>99.445983</td>\n",
       "      <td>95.731707</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>78.325123</td>\n",
       "      <td>89.196676</td>\n",
       "      <td>0.844875</td>\n",
       "      <td>0.728243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>99.014778</td>\n",
       "      <td>99.168975</td>\n",
       "      <td>95.731707</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>79.310345</td>\n",
       "      <td>88.365651</td>\n",
       "      <td>0.843952</td>\n",
       "      <td>0.714286</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>300 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       val_VAL   train_VAL    val_ATT   train_ATT  val_VAL_1  train_VAL_1  \\\n",
       "0    76.354680   74.515235  12.804878   20.498615  30.049261    32.409972   \n",
       "1    78.817734   77.562327  13.414634   21.329640  33.990148    34.903047   \n",
       "2    76.847291   77.008310  20.121951   30.470914  37.438424    33.518006   \n",
       "3    79.310345   79.501385  29.471545   43.213296  36.453202    40.166205   \n",
       "4    86.699507   86.149584  35.975610   49.861496  48.275862    50.692521   \n",
       "..         ...         ...        ...         ...        ...          ...   \n",
       "295  99.014778   99.722992  96.138211  100.000000  80.295567    88.365651   \n",
       "296  98.522167  100.000000  96.341463  100.000000  79.802956    89.196676   \n",
       "297  98.029557   99.722992  95.934959  100.000000  75.862069    87.534626   \n",
       "298  98.522167   99.445983  95.731707  100.000000  78.325123    89.196676   \n",
       "299  99.014778   99.168975  95.731707  100.000000  79.310345    88.365651   \n",
       "\n",
       "     train_VAL_jac  val_VAL_jac  \n",
       "0         0.005540     0.000000  \n",
       "1         0.058172     0.044335  \n",
       "2         0.139428     0.142857  \n",
       "3         0.183749     0.152709  \n",
       "4         0.236380     0.239737  \n",
       "..             ...          ...  \n",
       "295       0.844414     0.739737  \n",
       "296       0.830102     0.711002  \n",
       "297       0.827793     0.717570  \n",
       "298       0.844875     0.728243  \n",
       "299       0.843952     0.714286  \n",
       "\n",
       "[300 rows x 8 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['KMP_DUPLICATE_LIB_OK']='True'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAs7UlEQVR4nO3deXxU9b3/8ddnJnsm+wIhARL2RSBIWNwQBZeqrVqXYqVFry29V3tLbXsV2/6uem9t7W17vfpoq+JSvS5wLa611oIU3AEDIoQ1LCEkIfu+Tmbm+/tjhkg0YckkmcyZz/Px4DEz55w58/nmkPd88z2bGGNQSillLbZAF6CUUqr/abgrpZQFabgrpZQFabgrpZQFabgrpZQFhQW6AIDU1FSTnZ0d6DKUUiqobN26tdoYk9bTvCER7tnZ2eTn5we6DKWUCioicqS3eToso5RSFqThrpRSFqThrpRSFqThrpRSFqThrpRSFnTKcBeRp0WkUkQKTpiWLCLrRKTQ95h0wrx7ROSAiOwTkcsGqnCllFK9O52e+zPA5V+YtgJYb4wZD6z3vUZEpgCLgam+9/xRROz9Vq1SSqnTcsrj3I0x74lI9hcmXw0s8D1/FtgI3O2bvtoY0wEcFpEDwBzg436q17ra6mHnn+Gs68DdCbtegTELYP/b4GwNdHVKqYGSPhnO+nq/r7avJzENM8YcAzDGHBORdN/0TGDTCcuV+KZ9iYgsA5YBjBo1qo9lBAm3C0o+AY+r+/SSLbDtOQiPhvAYKM2Hd+4D44HOEwNdBrNapdRgOuvrQyrce9NTCvV4NxBjzEpgJUBeXp517xji8cBL34Z9f+15/pgF0FLjDfYL74bmSrDZYcJX4NAGmLnE+82ulFJnoK/hXiEiGb5eewZQ6ZteAow8YbksoMyfAoNKSzUceAd2vwH73qLb99qFKyD7/O7LRyfC8GngcUPtIUgd333++EUDXbFSyqL6Gu5vAEuBB32Pr58w/UUR+W9gBDAe2OJvkUOax+0bRmmDP30Fqvd7h1jm/QtExnmXScqBGYtBehlesdm/HOxKKeWHU4a7iKzCu/M0VURKgHvxhvpLInIbUAzcAGCM2SUiLwG7ARdwhzHGPUC1Dw1//RHseg0cw6DmICxeBdnnQVRCoCtTSoWw0zla5qZeZi3sZfkHgAf8KSoodDRDYxlsXwURMeBqg5tWw4RLA12ZUqoXxhikt7+gLWZIXPI3KL18m/cwRYBbNkD6lN6HXZRSAdXp9nD3mh1sLa5j7Z3ziQyz/uk3evmBvqg97A321AmQuwSGTdVgV2esrL4NY6x7oNhQ8sj6Ql75tJQjNa3kF9UFtJbyhnbanAM/Wq3h3hdbVoLY4duvwzV/CHQ1Kggda2jj3Af/wU/+vKNrmjGGnSUNtHd6f/EPVTVT1+Jkz7FGmto7T3vdR2tbKa1vA6CpvZPKxvZel23pcHG4uqXHec0dLt7dX8X2o/Xdauxt+ePz9xxrHLQvrdL6No7Wfn5OSENbZ1fbj2t1uvjfj48wf0Ia4Xbhvf1V/fb5xxraWL76Uz46WN017UBlc9c2PJExhp++upNzHlzPNX/4sFvdA0GHZc7U7tdh06Mw82aIHxHoapQfKpvaaW53MSbNMeif/dnRBgBe3lbCTXNGMj0rkZuf3MQnRXVcNnUY/3n1WVz+8Pt4PAaXxzA7O4n0+Cimjohn5sikbgE2JSOeVqeLpz44TGFlMwermnFEhPGbG2bwX3/fS7vTzcZ/u4ji2ha2+z4XvGHz+HuHOFLTwpp/PpcZIxM51tDGMx8W8e7+Kkrr2mjq8J54970Lx3Dnogk8vL6QRzce5KXvnYMI/ObtfVw6dRhTRyTw0Dv7iYsMY/3eSn56xSSWzR97yp9DflEtOamxpDgiu033eAz//kYBY1Id/NP5OXS43Nz/l93kF9WeUD8crGomKtzOo0tmkZkYzdKnt9DqdPHhiouJifDG24ubi2lo6+QHF4/D6XKzdncFBthd1sgNeVl8bcYIRITaFiefldRz4fg0bDZh86EafvnWHjwGZmcnU1DawO0XjeVobStrtpXS5nRR0dhBQ1snb+08xncvGMPE4XH88P+2Mz0zgadvmd2tXev3VPLi5mK+OmMEG/ZWcvHvNpKdEsuCiWn87MopZ/6f6BRkKPxZmJeXZ4LiNnsbH4SNv4IRM+GWt7w7UlVQ2HqkluYONxeMS8VmE4wxfPX3H3C4qoU3f3ABOamxXcserW3ls5J6LpkyrGts1hjDUx8cZt3uCp66ZTaOSP/6RQ+t28/D6wsBWDZ/DOePS+XbT2/h/HGpfHCgmknD49hX0cSSuaNxG8OLm4tPuc7EmHDm5iQzLt3B2l0VFFY2d82bOSqRT4vrv/SeuKgwYiLs1Ld2EhcVRl1rJ8YYzhuXyoiEaK6akcFbO8tZtaWYmAg7rb7hhJmjEtld1khEmI2mdu8XQFxkGE0dLtLiImnvdPPjSyYwd0wK97yyk2tnZhIXFcaCielsOVxLZmI06/ZU8Mj6QjITo0mNiyQzMYqpIxJYv6cCp9tDQWkjIjAtM4HD1S00tbu4eFI6kWGfDzhkp8aycV8VhRVNxETYMUBTu4v/vHoq3zonm33lTXzt9x8wb0wKz9w6mz99WMR/vLkbm8CIxGhK6tqYm5NMUU0LdS2dON0erskdwQXj07jn1Z0Mi48kzRHJtuJ64qLCuto6LTOBrKRoIsJsfGP2SF7cXMxbO4/hMZCVFE11cwcZCdE8fcts/v31AqaMiOftgnLsNuHvP5xPRWM7z35UREldG7NGJ/GdC8b06f+RiGw1xuT1OE/D/TS11cPvJsG4hXDdk95LBijAG4YfH6rhxryR3abXtzp5c8cxbp476qRHKLjcHsLstq7HE3W6Pfw5v4RLpw4j1RGJMYaXt5WSOzKBbUfqmT4ygUnD47uW/9vOY9hswqLJwzDG0Nju4tVPS/nlW3twewwXTUzjqaWz2VJUy+KVmxCBcWkOnv/OXJ547xAfHayh1emiqKaVkcnR/PiSieyvaKKgrLHrz/npWQnUNDt55KZcZo1O5sn3D/HbtftwRIbzP9/IpaimhRvysgi32bDZem73sv/N50BlM4kx4dhtwoRhcbz6aSlbf34JP3ttJ69sK2XR5GE8udT7e/vi5mLGpTvYW95ITEQYc7KTEQG3x7BhXyURYTaunZnZ1Vttc7p59dNSYiPtPLK+kINVLdx2fg43zx1F+Ak/46TYCErr2nhh8xHcHkNSTATfmD2Skcmfd1yMMXx8sIa/7yonITqcwzWt/OWzMkYlx/DaHedRUtfK2l0VfOuc0URH2KloaOeGxz+mvrUTEW8P+7isJG+gHnfZ1GFsK64nJTaCYw3tNLR1Mi7dQW2Lk3PGplBU3UKr0805Y1O4cEIal00d/qWfZWN7J7c/v42y+jb+dOtsfrB6O7UtHby9fD53rdnBhwerWXfnhaTFef//lDW0ExNuJyE6nF+/vZdnPirikinDyEyMxmYTHt14EPB+gT29dDZJsRE0d7jodHl46oPDLJiYxqzRSV/6P32kpoWXt5Vy7cxMals6uOVPn2CMd3gLwBEZxjO3ziYvO7mX34Qzp+HeHzY/Dn+7C5a9CyNyA13NkPK95/L5+64Knv2nOVw44fMbsT/4t7089u5B1vzzOeRlJ1NS18qT7x8G4JqZmWQmRnPvGwWs213BP52XwzMfFZGXnUR2Sixzx6Sw42g9nx6tZ+uROm6YlcXPrpzMK9tK+Y83d2O3CW6PITbCzpXTM7CJkOqI5PcbDgAwOiWG5nYXNS1OAOZPSGNOdhK/XbufBRPT2FfeRHunm/++MZfvv7gNm01oandxPIt/esVkHt14kJoWJ3abkOaI5NbzstlR2sBfdxzDERmGy+Nh0eRhvLnjGBdOSGPPsUYqmzoAWDgpnW3FdTzx7bwef5kv+K9/MD0zkeEJUTz7URHx0d5e96NLZuHxGF7bXso5Y1PISPC/E7G7rJFjDW0snDzM73UBFJQ28KOXtvPbG2YwPSuxx2WcLg/5RbXc8+pO7lw0gab2TmpbOnnonf3Mzk7i6txMxqc7mJOTjMeA3Sa43B7aXR5iIz4/ksUY77EKpzp80RiDMXQNp9z0xCYunTKcf+yr5Oa5o7j3q1N7fa/bY7Cf8CVcWNHE+4XVLJ4zsuvLsi/W7ipn2XNbuWB8Kl8/O5MpGQlMHB7X5/X1RMO9rzav9F7fZfg0yP8TJI6E7/4j0FUFxIZ9lewvb+J7F34+jurxGN4/UM0tf9qCMTBxWBwv334u+8qb+MtnZbxdUE55Yzs/WDieG2Zlcf1jH1HX2oldhLZON7ERdtzGMDw+iqKaVkYkRBEZbqe6qYOmDhcRYTbSHJEkx0awt7wREcHp8jAnOxkEZmcnkV9UR1FNC51uQ22LkwnDHHz/4vG8uPkIjshwzhmbwpSMeOaN8Qbsf7y5m7d2HiMnNZYfXzqR2dnJ7C1v5KF1+zEG7v7KJCobOzhnbApHa1vZUdLAoinpXcMzDW2dbDlcy8xRidz6p0/YWdrAknmjuP9rZ5FfVMsv39oDwGcl3rHtGVkJvHr7ed168E3tnUy7by0/uXQCOakO7nhxGwBPfjuPRVP6J4CHqg37Kpk5MpHEmIgB/Zzf/6OQ367dD8Cb/3o+Z2UG5qTCDw9UMzkjnuTYgWmvhntfbH8RXvsXSBwF9cUQ4YDvrIf0SYGubEB4PIaH1xfi9hjuvGQCT7x/iJrmDu66fBIdLg8LfrOB6mYn7/zoQsale3dA3vPKTlZtKSbCbuP+q6fy89cKyEiIoqqpgw6XB4CIMBtTR8QzPD6Kd/dX8crt55KVFMP/fXKUd3ZXcNflE0l1RHL/X3bzo0smMGVEPE6Xhy2Ha5k6Ip6k2AgOVTVzyUPvMSMrgVvPy+HiSenEfmHM2xjD1iN1jEqOIT0+alB+Zq1OF58dbWDemORuPcvD1S386q09TM6I5+H1hVw5LcN7nPVXJjE2zcGGvZXc+swnPHfbHManxzHvV+s5f1wqz39n7qDUHSqe/aiI/RVN/OKasyx74pKG+5kq2eq9TsyoubDkVe/lesOjLTscU1DawK/f3sv7hd7DudLiIqnyDS8smJhGRkIUq7YcJdwujEyOYWyag5vnjuLWZz7hurOzWDZ/DBOGxbFhXyVPf3CY5NgILp86nI8O1hAbGcbj7x3EGLhz0QSWL+rbNXSO1LQwPCEqqE4+Mcbw4Nt7efzdQ4h4/7L5/sXj+PhgDWu2lvDZvZcSFW6nsKKJnNTYL+1vUOpUNNzPRFM5rFwA9ghYthFi+m/nx1BUUtfKFQ+/j90mLF84npiIMN4trGLWqCSiwu38/LWdeIz3iI6OTjerPzmKMeB0e0h1RLD+RwtIiAk/6fp/t3Y/STER/OSyCX6NYQaro7WtHKhsZtlz+XS6vb9v8yek8b//NCfAlalgp+F+Jp77OhR/DLetg+FnBbqafuVye7DbBBFh7a5ymtq9x0YfrW3lrz+4gFEpXz608/3CKsrq27gxbyQe4z16ZVdZA5sO1XJDXhbpcYMzBGIF9a1OHlq3n2c/PsK/XTaROy4aF+iSVJA7WbiHXjfqZDqa4dBGOG+5JYK9vtVJTYuTkUkxtHW6ufr3HwBw1fQR/HHjATwGosJtPLZkVo/BDnDB+M+PfrEL2G12Zo1OZtZoa/9FMxASYyL4+VVTmDg8nq/OyAh0OcriNNxPVLoVjBtGnxfoSvrkSE0LtS1OZmQl4nR7WPi7d6lpcTIsPpKkmAhK6tqYnBHP7zccYHRKDD+/cgpZSdFMzog/9cpVvwi32/jmXIvfVlINCRruJyreBAiMnB3oSs5YXYuTSx56D6fLw7fPGc2cnGRqWpz8YOF4CkobqGhs51dfn8b1s7LYUdJAenxkvxxDrZQamjTcT1T8sffSvUP8RhvtnW7e21/FBePTiPad8LF+b6X3GPCcZJ7fdIQth2vJSIhi+cLx3U7QAJgxMjEAVSulBpMee3Xcnr94T1gaQjfb8HhM17/jKhrbufA3G1j23FZ+/fberul/31VORkIUf7z5bFIdkewtb+LGvJFfCnalVGjQnjt4z3H+608gI9d7I+shYH+F94JH7Z0eIsNs/L+rprBk3mh+t3YfdS2dXDQxjec3HSHcLoxLd/De/ioWzx5JqiOSd//tIhraOhkWH3nqD1JKWZKGO0DtIWguhwUrIDwwh/a9t7+KDw9U8/2LxxEXFc7r20txujwsXzierUfq+PlrBTy/6Qj7Kpr4zvk5LJs/lpuf3MQzHxXR6TaMTonpujRAdIS9a7hGKRWaNNzBtyMVGDUvIB//dkE5//z8VgBK6tr47vwxvF1QzpycZO68ZAIut4enPzzMX3cc43vzx7J84XiiI+ysvfNC2jvdrN9TybwxyV+6JrZSKnRpuAMc3eTdiZo6MSAfv2pLMZmJ0Vw/K4uH1xfy153HAFgybzQAYXYby+aP7fHmB1Hh3qsiKqXUiTTcAYo3w8i5YBu4/curtxSzfm8lv71hBjtK6rGJ8Ni7B7nrskm8X1jF7QvG8cNF47l4UjoHKpv56GAN1+RmDlg9Silr03BvrYXqfTDjGwP2ES/lH2XFKzsBuOGxj9hf0Uy4Xeh0GzYd+hCbCNfNykJEmDEykRkjE7luVtaA1aOUsj4N96ObvY8j+3+8fVdZA+/tr+apDw6RNzqJ4tpW9lc0k5kYjcvj4RuzR/Hi5iP81/XTu93mTSml/KXhXrwJbOGQeXa/rja/qJalT2+hxXfPyaeWzuavO4+x8r1DPLrkbKZlJiAi3LlovGWvNa2UChwN9+JNkDHD73uirt9TwfObjjAmzcGPL53Aild2kuKI5JkbZ9Dp8jBjZCJj0mK5YHxqt1uTabArpQZCaId7UzmU5sO5/9rnVdS1OFm/t5Kfv7aThOhwNuyr4vXtZVQ3d/DYklnMPuH+mXFR4d2usqiUUgMltMP90+fA44KZ3+rT21udLm54/GMOVDaTFhfJX75/PvlH6liztYSpI+K5bKq174eplBq6QjfcXR2Q/wzkXAgpXz5+/GQ+La7jvf3VbNhXycGqZn7/zZksmJiOIzKMK6ZlcMU0Pe5cKRVYoRvu+U9DYwl87eHTfsv7hVX8raCcP+cfpdNtyEyM5rfXz+Cq6SMGsFCllDpzoRnu7k5477eQMx/GLjyttxhj+NmrBZQ3tHP+uFQe+kYuCdHhukNUKTUkhWa4F70PrdUw53twmuG8t7yJ4tpWfnntNL2TjlJqyAvNcN/9BoTHwrhT99qP1rZiswlrd1UgAoumpA9CgUop5Z/QC3djYN9bMP6SUx7b3up0cf1jH9Ha4cZjDHOyk0mPC8wlgZVS6kyEXrg3V3j/ncZNsJ98/zAVjR2MSo4hNjKMR26aOQgFKqWU/0Iv3Kt8t6ZLm3DSxZwuD898VMSiyek8/q08AL1lnVIqaPh1jVsRuVNEdolIgYisEpEoEUkWkXUiUuh7TOqvYvtF1X7vY9qkky727v4qaluc3DRnFHabaLArpYJKn8NdRDKBHwB5xpizADuwGFgBrDfGjAfW+14PHVV7ITIBHCc/e/TlrSWkOiKYP0EvF6CUCj7+DsuEAdEi0gnEAGXAPcAC3/xngY3A3X5+Tv+p3g9pE3s9BPLBv+1FBNbtqeC283MItw/cDTyUUmqg9DncjTGlIvJboBhoA9YaY9aKyDBjzDHfMsdEZGgdO1i1FyZc1uOsDpebx949CHjH15eemz2IhSmlVP/xZ1gmCbgayAFGALEisuQM3r9MRPJFJL+qqqqvZZyZpnJoqYL0KT3O3l3W2PX8xryRZCb6dxlgpZQKFH+GZRYBh40xVQAi8gpwLlAhIhm+XnsGUNnTm40xK4GVAHl5ecaPOk5f2Xbv44gv35jjL5+V8c6eCgA+vudihsfr8exKqeDlT7gXA/NEJAbvsMxCIB9oAZYCD/oeX/e3yH5T9imIDYZP6za5pK6V5as/xeP7islI0B67Uiq4+TPmvllE1gDbABfwKd6euAN4SURuw/sFcEN/FNovyj6F1IkQ6eg2+dmPirwXADOGC/XoGKWUBfh1tIwx5l7g3i9M7sDbix9ajPGG+7hF3Sa3d7r5v0+O8pWzhnPXZZOIjw6987qUUtYTOknW0QgtlZA+GQCX28O/vLANgMZ2F4tnj2JUSkwgK1RKqX4TOuHeUOp9TMgCYPUnR1m327sDdXh8FOeMTQlUZUop1e9CJ9wbfeEen0lzh4v/eWc/Z49KxCbCVdMz9PICSilLCb1wT8hk5bsHqW528uTS2eSOTAxoWUopNRBC59z6hlIQG3W2ZJ54/zBXTc/QYFdKWVbohHtjGTiG82J+GW2dbr5/8bhAV6SUUgMmhMK9BBOfyf9+XMQF41OZNDw+0BUppdSACZ1wbyilypZCRWMHN88dHehqlFJqQIVGuBsDjWUUNDlIjAnn4klD60KVSinV30Ij3NvrobOFzTXRfHX6CCLCQqPZSqnQFRop5zuB6ag7iVmjh9Zd/5RSaiCERrg3lgFwzKQwNs1xioWVUir4hUi4lwBwzCQzJi02wMUopdTAC5FwL8ONnbD44cRGhs5JuUqp0BUa4d5QSq0tiZxhCYGuRCmlBkVIhLtpKKHEnazj7UqpkBES4d5UVUypJ1kv66uUChmWD3eXy01ESxluRwaXThkW6HKUUmpQWD7cS44eJAonI8dM9t4nVSmlQoDlw71h3wcAxI47N8CVKKXU4LF8uMvRTbSYSDIm5gW6FKWUGjSWD/fkmm3stk0gPiY60KUopdSgsXa4O1vJaD/IUce0QFeilFKDytLhbuqKsOPBlTwx0KUopdSgsnS415QdAMAxfEyAK1FKqcFl6XA/VrQPgOyxkwJciVJKDS5Lh3tT+SHaTTjjx4wNdClKKTWoLB3u1BdTEzaM8DB7oCtRSqlBZdlw73C5iWsro8ORFehSlFJq0Fk23KubnYyQKjzxIwNdilJKDTrLhntLUyMp0oQrXnvuSqnQY9lwd9YdBcDEjwhwJUopNfgsG+6dDeUA2OMzAlyJUkoNPsuGu6fJG+4RCcMDXIlSSg0+y4a7NFUCEJWkPXelVOixbLjbWytwGjuxiWmBLkUppQadZcM9rK2KahKIjYwIdClKKTXo/Ap3EUkUkTUisldE9ojIOSKSLCLrRKTQ95jUX8Weicj2ampIwmbTW+sppUKPvz33h4G3jTGTgBnAHmAFsN4YMx5Y73s96GKc1dTbAvK9opRSAdfncBeReGA+8BSAMcZpjKkHrgae9S32LHCNfyX2jaOzlsaw5EB8tFJKBZw/PfcxQBXwJxH5VESeFJFYYJgx5hiA7zG9pzeLyDIRyReR/KqqKj/K6IHbhcNdT1N4av+uVymlgoQ/4R4GnA08aoyZCbRwBkMwxpiVxpg8Y0xeWlo/H9FSfwQbhtZIDXelVGjyJ9xLgBJjzGbf6zV4w75CRDIAfI+V/pXYB/veAuBwwpxB/2illBoK+hzuxphy4KiIHL9B6UJgN/AGsNQ3bSnwul8V9sXuN9gvOXQ4Rg36Ryul1FAQ5uf7/xV4QUQigEPArXi/MF4SkduAYuAGPz/jzLTWQskW1pobcUSGD+pHK6XUUOFXuBtjtgN5Pcxa6M96/dJcAcBe13DGRPn73aWUUsHJemeodjQB0GSiiYvUcFdKhSYLhnsj4A13h/bclVIhynrh3u4Ld2JIitHryiilQpP1wt03LNNsokmK0R2qSqnQZN1wJ5rkWO25K6VCk4XDPYpEHZZRSoUoC4Z7Ix32WAw2EnVYRikVoqwZ7rYY4qLCCLdbr3lKKXU6rJd+HU20SKyOtyulQpolw72ZaB1vV0qFNOuFe3sjTSaKZB1vV0qFMOuFe0cT9e4oPYFJKRXSLBnute4oknTMXSkVwiwX7qaj0ddz12EZpVTosla4e9yIs1l3qCqlQp61wt3ZDECTiSE20h7gYpRSKnCsFe6+K0I2E01kmIa7Uip0WSzcGwBoMLFE6NmpSqkQZq0EbK8HoIFYIsOt1TSllDoT1krAtnoAGk2sDssopUKaxcK9DvD13MOs1TSllDoT1kpA37BMvYklQsNdKRXCrJWAbfV4xO47WsZaTVNKqTNhrQRsq6MzLA4QIsN1zF0pFbqsFe7t9XSExwNoz10pFdKslYBt9bSHecNdx9yVUqHMWgnYVkebPQ7QnrtSKrRZKwHb67vCXc9QVUqFMmslYFs9rTYHkWE2RCTQ1SilVMBYJ9w9Hmivp9kWp0MySqmQZ50UdDaB8dAkDiL00gNKqRBnnXD3XVemGYf23JVSIc86Kei7UUczUXpFSKVUyLNOCna2A9DqCdcrQiqlQp51wt3VBkCLCdcTmJRSIc86Kdit526dZimlVF/4nYIiYheRT0XkTd/rZBFZJyKFvsck/8s8Db6ee5Nbw10ppfojBZcDe054vQJYb4wZD6z3vR54vp57s1vH3JVSyq9wF5Es4ErgyRMmXw0863v+LHCNP59x2o6PueuwjFJK+d1z/x/gLsBzwrRhxphjAL7H9J7eKCLLRCRfRPKrqqr8LIOunnuT267hrpQKeX1OQRG5Cqg0xmzty/uNMSuNMXnGmLy0tLS+lvE5X8+9sTNMj3NXSoW8MD/eex7wNRG5AogC4kXkeaBCRDKMMcdEJAOo7I9CT8nXc290hemYu1Iq5PW5i2uMuccYk2WMyQYWA/8wxiwB3gCW+hZbCrzud5Wnw9UGtnDa3aLHuSulQt5ApOCDwCUiUghc4ns98DrbMeHRdLjcOuaulAp5/gzLdDHGbAQ2+p7XAAv7Y71nxNUGYVF4jN6FSSmlrJOCne2YsCgAHXNXSoU864S7q60r3HXMXSkV6qyTgp1teOzHe+7WaZZSSvWFdVKwsw23PRKAqHAdllFKhTbrhLurHad4wz0hOjzAxSilVGBZJ9w723FKBAAJMRruSqnQZp1wd7XRgTfcE7XnrpQKcdYJ98522oyv567hrpQKcdYJd1cbrR5vqGu4K6VCnXXCvbOdFk8YjsgwwuzWaZZSSvWFNVLQGHC10ewJ1167UkphlXB3d4Lx0OwKI1GPlFFKKYuEu+9GHQ2uMO25K6UUVgn3E27UoT13pZSySrj7eu51TjsJ0REBLkYppQLPGuHu67nXOW06LKOUUlgm3FsAaPRE6rCMUkphlXDvaAaglSjtuSulFFYJd6e3595sooiP0nBXSimLhLu3595CNLGRei13pZSyVribKGIj++We30opFdSsEe4dx3vuUcREaM9dKaWsEe6+MfdWInFoz10ppawS7s102qMx2IiJ0HBXSilrhHtHE532GADdoaqUUlgl3J0tdNhiEIHocA13pZSyxhiGs5l2iSY2IgwRCXQ1SqkTdHZ2UlJSQnt7e6BLCVpRUVFkZWURHn765/FYJNxbaJdoPVJGqSGopKSEuLg4srOztfPVB8YYampqKCkpIScn57TfZ41hmY4mWiVaj3FXaghqb28nJSVFg72PRISUlJQz/svHGuHubKHFROrOVKWGKA12//Tl52eRcG+m2UTpYZBKKeVjkXBvoclEEatj7kopBVgh3I0BZzONnkgdc1dK+c3hcJxymYceeoioqCgaGhqoqakhNzeX3Nxchg8fTmZmJrm5udjtdqZMmUJubi7Jycnk5OSQm5vLokWLBqEVVjhaprMVjId6dySxOiyj1JB2/192sbussV/XOWVEPPd+dWq/rvNUVq1axezZs3n11Ve55ZZb2L59OwD33XcfDoeDn/zkJ92Wv+WWW7jqqqu4/vrrB63G4O+5+64r0+CKIEZ3qCqlvuDuu+/mj3/8Y9fr++67j/vvv5+FCxdy9tlnM23aNF5//fXTXt/Bgwdpbm7mF7/4BatWrRqIkvtF8Hd1O5oAqHVFkKnDMkoNaYPdwwZYvHgxP/zhD7n99tsBeOmll3j77be58847iY+Pp7q6mnnz5vG1r33ttI5KWbVqFTfddBMXXHAB+/bto7KykvT09IFuxhnrc89dREaKyAYR2SMiu0RkuW96soisE5FC32NS/5Xbg9rDADSYGD1aRin1JTNnzqSyspKysjI+++wzkpKSyMjI4Kc//SnTp09n0aJFlJaWUlFRcVrrW716NYsXL8Zms/H1r3+dP//5zwPcgr7xJw1dwI+NMdtEJA7YKiLrgFuA9caYB0VkBbACuNv/UntgDGz8JW5HBh9UT+NCHZZRSvXg+uuvZ82aNZSXl7N48WJeeOEFqqqq2Lp1K+Hh4WRnZ5/WSUI7duygsLCQSy65BACn08mYMWO44447BroJZ6zPPXdjzDFjzDbf8yZgD5AJXA0861vsWeAaP2vs3aGNULqV+rn/RgcRukNVKdWjxYsXs3r1atasWcP1119PQ0MD6enphIeHs2HDBo4cOXJa61m1ahX33XcfRUVFFBUVUVZWRmlp6Wm/fzD1yw5VEckGZgKbgWHGmGPg/QIABm4waswCWPIy5TnXAHq5X6VUz6ZOnUpTUxOZmZlkZGRw8803k5+fT15eHi+88AKTJk06rfWsXr2aa6+9ttu0a6+9ltWrVw9E2X4RY4x/KxBxAO8CDxhjXhGRemNM4gnz64wxXxp3F5FlwDKAUaNGzfLnm++jg9V884nNvPjduZw7NrXP61FK9b89e/YwefLkQJcR9Hr6OYrIVmNMXk/L+9VzF5Fw4GXgBWPMK77JFSKS4ZufAVT29F5jzEpjTJ4xJi8tLc2fMmhs6wQgMTrCr/UopZRV9HmQWrzHDD0F7DHG/PcJs94AlgIP+h5P/wDSPmrwhXtCzOlf61gppXqzc+dOvvWtb3WbFhkZyebNmwNU0ZnzZw/kecC3gJ0ist037ad4Q/0lEbkNKAZu8KvC09AV7tEa7kop/02bNq3rrNNg1edwN8Z8APR2xP/Cvq63LxraOrHbRC8cppRSPsF/+QGgvrWThOhwvWa0Ukr5WCLcG9o6dUhGKaVOoOGulFIWZIlwb9RwV0r1or6+vttVIU/XFVdcQX19fZ8+0+VykZqayj333APAAw880HXNd7vd3vVcRMjNzWXKlClER0d3TV+zZk2fPvdEfp/E1B/y8vJMfn5+n9+/4DcbmJ6VyCM3zezHqpRS/aHbyTd/WwHlO/v3A4ZPg6882OvsoqIirrrqKgoKCrpNd7vd2O0DcxDGW2+9xQMPPEB5eTkHDhzotj/Q4XDQ3Nx8WjWeaFBPYhoqdFhGKdWbFStWcPDgQXJzc5k9ezYXXXQR3/zmN5k2bRoA11xzDbNmzWLq1KmsXLmy633Z2dlUV1dTVFTE5MmT+e53v8vUqVO59NJLaWtrO+lnrlq1iuXLlzNq1Cg2bdo0oO3rTdBfacvjMRruSgWLk/SwB8qDDz5IQUEB27dvZ+PGjVx55ZUUFBSQk5MDwNNPP01ycjJtbW3Mnj2b6667jpSUlG7rKCwsZNWqVTzxxBPceOONvPzyyyxZsqTHz2tra2P9+vU8/vjj1NfXs2rVKs4555wBb+cXBX3PvdnpwmP0BCal1OmZM2dOV7ADPPLII8yYMYN58+Zx9OhRCgsLv/Se4/c/BZg1axZFRUW9rv/NN9/koosuIiYmhuuuu45XX30Vt9vd3804paDvuTe06tmpSqnTFxsb2/V848aNvPPOO3z88cfExMSwYMGCHq/rHhkZ2fXcbrefdFhm1apVfPjhh2RnZwNQU1PDhg0bBu3G2McFfc998+FaANLiIk+xpFIqFMXFxdHU1NTjvIaGBpKSkoiJiWHv3r1+j483NjbywQcfUFxc3HXN9z/84Q8BuddqUPfci6pbuPf1AvJGJ3HBeL3Ur1Lqy1JSUjjvvPM466yziI6OZtiwYV3zLr/8ch577DGmT5/OxIkTmTdvnl+f9corr3DxxRd36+lfffXV3HXXXXR0dHSbPtCC+lDI4ppWfvbaTh68bjqZidEDUJlSyl96Pff+caaHQgZ1z31USgzP3TY30GUopdSQE9ThrpRSgXLHHXfw4Ycfdpu2fPlybr311gBV1J2Gu1JqwBljLHfV1j/84Q+D9ll9GT4P+qNllFJDW1RUFDU1NX0KKOUN9pqaGqKios7ofdpzV0oNqKysLEpKSqiqqgp0KUErKiqKrKysM3qPhrtSakCFh4d3OyNUDQ4dllFKKQvScFdKKQvScFdKKQsaEmeoikgVcMSPVaQC1f1UTiBZpR2gbRmqtC1DU1/bMtoYk9bTjCER7v4SkfzeTsENJlZpB2hbhipty9A0EG3RYRmllLIgDXellLIgq4T7ylMvEhSs0g7QtgxV2pahqd/bYokxd6WUUt1ZpeeulFLqBBruSillQUEd7iJyuYjsE5EDIrIi0PWcKREpEpGdIrJdRPJ905JFZJ2IFPoekwJdZ09E5GkRqRSRghOm9Vq7iNzj2077ROSywFTds17acp+IlPq2zXYRueKEeUOyLSIyUkQ2iMgeEdklIst904Nuu5ykLcG4XaJEZIuIfOZry/2+6QO7XYwxQfkPsAMHgTFABPAZMCXQdZ1hG4qA1C9M+y9ghe/5CuDXga6zl9rnA2cDBaeqHZji2z6RQI5vu9kD3YZTtOU+4Cc9LDtk2wJkAGf7nscB+331Bt12OUlbgnG7CODwPQ8HNgPzBnq7BHPPfQ5wwBhzyBjjBFYDVwe4pv5wNfCs7/mzwDWBK6V3xpj3gNovTO6t9quB1caYDmPMYeAA3u03JPTSlt4M2bYYY44ZY7b5njcBe4BMgnC7nKQtvRnKbTHGmGbfy3DfP8MAb5dgDvdM4OgJr0s4+cYfigywVkS2isgy37Rhxphj4P0PDqQHrLoz11vtwbqtvi8iO3zDNsf/ZA6KtohINjATby8xqLfLF9oCQbhdRMQuItuBSmCdMWbAt0swh3tP9+wKtuM6zzPGnA18BbhDROYHuqABEozb6lFgLJALHAN+55s+5NsiIg7gZeCHxpjGky3aw7Sh3pag3C7GGLcxJhfIAuaIyFknWbxf2hLM4V4CjDzhdRZQFqBa+sQYU+Z7rARexfunV4WIZAD4HisDV+EZ6632oNtWxpgK3y+kB3iCz/8sHtJtEZFwvGH4gjHmFd/koNwuPbUlWLfLccaYemAjcDkDvF2COdw/AcaLSI6IRACLgTcCXNNpE5FYEYk7/hy4FCjA24alvsWWAq8HpsI+6a32N4DFIhIpIjnAeGBLAOo7bcd/6XyuxbttYAi3Rbx3oH4K2GOM+e8TZgXddumtLUG6XdJEJNH3PBpYBOxloLdLoPck+7kX+gq8e9EPAj8LdD1nWPsYvHvEPwN2Ha8fSAHWA4W+x+RA19pL/avw/lncibencdvJagd+5ttO+4CvBLr+02jLc8BOYIfvly1jqLcFOB/vn+87gO2+f1cE43Y5SVuCcbtMBz711VwA/Ltv+oBuF738gFJKWVAwD8sopZTqhYa7UkpZkIa7UkpZkIa7UkpZkIa7UkpZkIa7UkpZkIa7UkpZ0P8HTBX4icKJcEoAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_ATT','train_ATT']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA6MUlEQVR4nO3dd3xUdbr48c93Sia9JxDS6Z3QQRALiugqoKKCrtdd/al3dQvu6uruvba76127W3Wvu2tZr6JcLFixLUpHQXpNgBBCQiqpkzLl+/vjTCYJSSCVSSbP+/XKa2ZOm+fkzDzzPc/5nnOU1hohhBD+xeTrAIQQQnQ/Se5CCOGHJLkLIYQfkuQuhBB+SJK7EEL4IYuvAwCIjY3VaWlpvg5DCCH6lG3bthVrreNaG9crkntaWhpbt271dRhCCNGnKKWOtTVOyjJCCOGHJLkLIYQfkuQuhBB+SJK7EEL4IUnuQgjhh86a3JVSLymlCpVSe5oMi1ZKfa6UyvQ8RjUZ9yulVJZS6qBS6rKeClwIIUTb2tNyfwWYf9qwB4AvtdbDgC89r1FKjQaWAGM88zyvlDJ3W7RCCCHa5az93LXWa5VSaacNXghc6Hn+KvAVcL9n+Jta6zrgqFIqC5gGbOqmePsVl1uzYutxrp6YyLvbT7AoI5GgADNaa/656RjlNQ5unJ5CbKjt7As7+AlEpUH8KAC+PlTEtuxSLh41gIzkSDi6Do6uhcEXgCUQDn1qzJc8HcIGUrp1BRU1TtJiQlouOyoN0ufA9v8F7W45PigKJiyBb/8BrvrG9dOafXkVjEqOwzL9dvR3/2Rf9gmGx4diDbDBlFth77tQVdjh/123ih4MabNg++vN1q/G4WJLgSJ78I3cMmswSimoq4Jdb8GkfwOztfly3G747lUYtxgCQmH7azDqKgiMhG2vQEVe52Mcdik4a43t6CsWG0z9f7D3HRhzDexcDvbSltMpE0y8CSJTYOebUHK4fcsfegm46ny7jl2hFIy9FkqPNPsu9pTOnsQ0QGudD6C1zldKxXuGJwKbm0yX6xnWglLqDuAOgJSUlE6G4d82HynhV+/sZuPhEj7YmYfTrbl5Riq7cst5+P29gPEDcM+lw8+8oLIceOv7ED8a7lyLW8M9b+2gtLqerw8VseqOifB/t4C9xEimgeFwYpsxb3AMDJpIdNYXRGqFVqCaLdxzP4CB4+Dkbk4f6x2/840W403AGA2mwxoOrEKd3M0YQGcqY77973vmoZXlnittr18QmguBOw64mDXsFwwbEAYb/gBrnzS+yFNubb6o7LXw4TIoO2Yk9fd/Avm7YPqdxnCgc+upYcfr4LBDzalOLqM7aNj3nvF/2vrSGbadhsK9cMH98O6dbUzTyrJ3vA6OGqgpbcf0vZGGfe9DSab3u4jqufXo7jNUW4u01buBaK1fBF4EmDJlitwxpBXHSuwAfLDTaNFtyCzm5oHH2Xg4BoDEyCA2ZBW3ndzzdhito41/ArcTTu6CtU9zQscSUG3m/EiIzV9P3SdvY7OXwIgr4ODHoMww++dG6+KDn6KPfMUKfQn3193Km3fMYMbgmMb3qK2A58YaX+Tz74W5DzaPwe2G56cb48dcDde94h3163d2s/ybHD6O/ROjT26iPGIUEwr+k59cPIxf1PzR2BOISIGffteyFXyu1JQ1rt8F98NFv/aOWvLX9TyR90PusrzPseJ/Z1gk8M3/GCM3/AEm/huYm3zFcjztnm//AWbP3tb21yBsoPH87m8h7iw/1K3J/BxeX2w8v+1zSJ7W8WV0hzeWwKFPjOcnd0PscLhrC5hOq/5++RtY9wxUFhh7MPfsMfbuziTzC3j9WuP5rZ9ByvTuj7+nbXkRPrnPeH5yFxz+0tgb6SGd7S1ToJRKAPA8Nuw35wLJTaZLArqwr9m/5ZTam70uObwVXvkeMTteYFRCOAszBrH9eBmVtY6WM5fnwt8vgRX/Bt/9E8bfYCT6Nb8l+atl/Mb6Mi9YnuE56/PYtr8EaefDrGXGvNoFqecZf4ByO9noMJLO+szi5u8TGA4z7wZbBMz4Ucs4TCaY80swWY0fjCY2ZBnLeqZ2AVqZ2ZB0G6CM9Z51j5EAz/+57xI7QFAkzLwLAiNg2p3ewdV1TrYdr2B74lIyTIepOL4bDnwEteUw4244lQ3H1jdf1rGNEBwLdRWw/lkIijbKVF8/YewhxQ7rXIxDL4HEyTD4It8ldoA59xoNgwseMLb3nF+2TOxgfE4CQiH3G5h2x9kTO8DQuZA4BQZf2DcTO8CkmyE8ESZ+H8IGwbpne/TtOttyfx+4BXjc87iqyfA3lFLPAoOAYcA3XQ2yL9p2rJS/rT1KamwwV40fxF/WZOFyt38HJUGVcLgu0vt6VEI4Ywt3gxUuqXiHmtGzyYhx81e3i/teWo0pbCA/mxnJuwdqOVV4nKtOvcpstwOy1+FG8XDp5ZRH3sSNs4Nx/+sxZts3ElRVx1/1NWwInUcYqVx6Mo6rLYHgrIOkqUZCC44FezHb9AhGDAhjfVYx9142gm+zS/n7uiNoDUpfxJDRF3NfcAyZBZX8/otDxITYuGPOYF7dmM0v519L+aAL+MvGIu6f7yIowExOiZ2cUjujEsL5Mj+Vo3fvZu03JcBxckrtuKIzeG78h1yXNpo3Pt7PdVOSeX3LMU6W13LXhUMZlxTR4n/20vqjjEwI47whsd2wBRvtGPLvvJIzG/vbR0mMOsmSqSn85sN9OFyaxKkL4P0/UHd4HdsyjzApMAJ1wX2w+Xk4tglC4o1SAhpyt8LEm3CXHMZ0+EuqUudyJL+E8eX/omrAVF747CBXT0xkaHxYm7G88NVhtuecYum0FC4a6amGKgU/+MioZTexek8+73x3gllDY7nlvLROr39OiZ0nPj1AaICF/1o0BpuljT4SSVPg/qPG56bhBxGjdPjE6gMsmZrM4LhQCImFZbuMH8LIdpZklYIffNhiHTtjX14Ff/pXJnFhNh6+agxmU8uCQ3FVHf/1wT4sJsW9l43gqU8P4taah64cTYjNwuOfHOBHFw5hQHhgi3m11vz2o/2cOFXDjy4cwoTkSGOENQju/sY4pvXNi/DpryBnS4/9WJ01uSullmMcPI1VSuUCD2Mk9RVKqduAHOA6z0rtVUqtAPYBTuBurbWrRyLv5f6+7iir954EYEdOGdtzyhgc18rByFaMcuzl0eoH+GH9fYxPupDYUBv3zhtBxT8fp7bWRrSq5Jb9t8N++N3A21hU+Brv5M8h7fA6LI4FPGT9kBBq2GyZxgTnLr61TubbqhhyT9VwrMrMyKpRnGdaA0D4hAUU5cSwJ6eK7flHWZQyA1VbYbRYAdLnkH9gE7FJw5gzPI4//yuT8hoHf1t7hHWZxaTGBFNV5+TT/TVcM7Oa1zcf4+Pdxno73Zrl3+Rw4Yh4dp2o4JWN2cwYHMP8sQNZ72m1/3L+CH748reszan37qkcL7WTWVjJnzcV8e3J3Ww5WsqO42VsOWocnAuxWXj6ugnN/mcVtQ4e+3g/80YP6Pbk/vLGY6zOrCY+3Mln+wrYl1fBtmOnmDM8jnFjMyj5IJqgk98SobIpTpxIXFAUDBxrlGi+frz5wtLOZ0fYhUw6/CWv5qfwUeFEPgxYwwbXaP6y5jD/3HiM3Y+23oO4stbB058dxOXWlFTXNyZ3MBLHaV5an8032aVsyCrmxukpWM2dS4z/t+04H+3KB2DJtGQmppyhpe1J6N5HYMfxU7y49ghOl+ahq0YbA4Ojjb+OaGUdO+O1zcf4ZI/xGb12UlJj8m1iXWYR73vKoYEBZt7dfgKAuaMGYLOYeGVjNnFhNu6+aGiLeTMLq/jH+qMABAWYee6GjMaRtlDjcfItsPYpYw/uxre6Zb1O157eMkvbGDW3jekfAx7rSlB9itbGbrY1xFtfdbk1Gw+XMDk1im3HTrHlaCmXjxnACzdPMWrQJpPRq0K7jNKD1fPrX28HpdDr1sJa+KnlXT5MmMuD3xtu9NIwH4Lxi2Dq7caBs/d/wg117wL1LDV/AcCtlk8IoQYufpAZ0++EqkLmhMQxJzCcJ1Yf4IWvDlOqhoENsAZz48IrudFs5fUtx/iPd/dw7MY/kBbd2BqpuOQJrtv+KYuGxjJ7aCx//DKTDVnFbDpSwsKMQTx+7XhySuzMeWoNG7KKWZ9VTJjNQmWd01t2WZ9VzO4TZYBRipk/diAbsooZGB7IhcPjSIkOZn1WiTe5F1fVcyC/EsCb0BseJ6VEsiGrGK210TvFY8uRUlxu3aKU1fXNq70x3z9/JOc9/i+2HC3lguFxvHqrUQLZEzyeS6s2EaLq+Fwt5FKAlPOM5B6ZClc8bSzMYoO02Xz88UF+Xfc4hwqSUCYzl9U/QaJ7DFBOZZ2zzVga1nFyahQ7POW4sMC2S1Y5pXbvtthxvIypaR1Mph7rMhu3aU6p/czJvRXrM0uAxjKcr23IKiYjOZIdx8tYn1XcanLPKanxPm9aijxeaqegotY7vLXk3jD9pJRI1rfyWQUgIMQoT615DE7uMRoD3UzOUO2qLx+Fx1Pg742/dXvzyimvcfDQwE1sDFzGEvO/eDb3BuMg21ODYcMf4XeJxnxPpBldozK/gP8eBL9LRu18CzcmJpqyeHD3fM90qVBdaNTBk6fC8HlG97yaUu+uqksrolWVEcSUW8EWBjFDjLo4MHuo0aLNUwNwhw0ySi+eenbDuLV5NB7gA7bka3Ld0cwaGktGciTBAWb++vVhKmudzPLMkxITTHJ0ECu35XK4qJql041d7YZE++X+Ar7NPgUYXyy3W7PhcDGzhsailGL2sFjWZxWRe6qGwbEh3ulONyQuhMWTk8kvr+VIcXWzcQ3T55TY0br7js8fLKikuKqeWUNjGRQZ5N37avh/AZTETCJE1QHwQVmaMTB1pvE462fGtho+z+hmajKzPquYAzoFNyYmJEVgjxjG3oJa7/KOt/EDtT6rmECriZ9cPBSXW7PlSCvdDD1qHS5OVtRy/dRkTKqVYyXtVF7jYFduGUumJZ8xtjNp2DYHCyoprKw9y9Q9q6EcuChjECMHhrX5g5NTamdgeCBBVrP3RzI21EZOid2717nt2Clq6lsWJjZkFZMWE8wNU5Mpqqwjs7Cq9WCm3Y4OCIX1z3Xb+jXVK67n3qc1dBnM38GHG7bz1MYyqmqN1teosrUEUMiPLe8RVFcMnz9kTPv5gxA3CjJuhH/9Ftb/Hgr3QUSS0fukPIejaUv4e2YI/zY5jlEJRnLGHADjrm9875SZsOdtSJ5O/tg7eGHVV/yX9VWIG9nqLu/k1ChsFhPjEiMwLXi92a5zakwIydFBPLX6IH9fZ+xSXuzZ7Q+ympmUGkmAxcT09GjWHCwC8CZ3gNlD41j+TQ4AizISeXVjNnVOo094w4f7/GGxrMss5vwn11BmdzB7WIxn3lje2GLMO3tYLEeKq71foKbzzfbsPQDc8D+bCbGZue+yEXyy+yRf7C8AoLLOybK3djA1LZrvz0jld5/sJzbERqm9no925bMwYxC/mDfCu+x/bsrmaHE1IwaEsftEOZNTo9h4uISnr5vA6j35PLhqrzdGgPOHxnKkqLrZupcOu47/PHyKlAExvJ+XxK/Ka0gYeRUsfhlGLWi2DYqr6jhwsrJxnYbF8fXBQnbmlnunueaFjYQEmPn1FaOYN2Yg//7aNvblV1BQUcu09GhmDokh0Gri3pU7CW/Scr9uchIOl5v3duQxcqBRtx+XGMG4pEj+tu6It7RwNoMiA3nttulYzSY2HynBreGSUQN4b0ceXx4oZNWOPO+2bY/cU3bv+l75x/UEWrt2XuO1k5L42SXtO/j8wc48nv7sIA2/9zUOIxnPHhbHibIa/rH+KHOeXNNsnvAgC/VONykxwZTbHRwsqCQ5OpigADPfZJdytLjauz6X/X4tf1o6kREDw7jjtW386IIhbD5SwqKJicweZtxD48a/bSY4wEi1SsFPLh7G4slJEBTF/wTfSUD9EE7rNNstJLl3VXmusetddozkr5bxjiObALOiYsBgAvIPAZCkmrQOogcbLfWLfgWjF8Kpo7D1ZUAbu++V+bDuGZImXkbUwNGkXDQUbG1sppSZ3seBU69mSOkA2PwqpMxodfJAq5nfLBpLclQwJMa0GP/ry0fx2T4jSR4qqOSNLTkMjAhkWnq09yDajy8eSlRwACMGhhEdEuCd99ZZadQ73QyMsDEqIYyU6GAyC6tYmDEIm8VEcICFOy8YzB++yKTO6SY4wMylo409hItGxHPzjFScbjd3zBnMm98cJ7+8lpEDw5g/diDXTkpi1Y4TXDl+ECkxwdxzyXCyS6pZl1nE77/IJKuwiqlpUUxMieLFtUdYtSOPfXkVLJ6cxMsbsokKtlJd56Kqzsk/Nx1j2SXDvQfRXt6QTU6pneSoII6V2tl0pIQjRdXcO28En+8rpKbexb3zhjMo0qj3/nBWOrGhxjo2uHTiEPJrfsSM0QPgzxvYkFVifHnHXtPif5xZYPzQ3TY7nYnJkdw4LYW9JxoT+w/OS6O8xsHXh4pYsTWXkQPDWb33JFPTopiSGsWSaSnYLGYeunIM32Y3ttx35Zbx6qZsHC5NeY3Du9eUHB3MLy4d3u7EXlxVx7rMYnYeL2NKWjQbsooJDjAzMSWKlOhgth07hUnBggmDWpYa2jA9PZofXzyUN789zsnyrrXcv80u5aPdee1O7m9+m0N1nYvzhzX+GCdHBTEkLoSbpqdyyu5o1tGhpt7lPVa2eHISZYFGck/xJPdtx4w90F9eNpKh8bm8viWHVTvyuGhkHGsPFZFXVkN1vfF+iZFB/OLS4c32MjcdLuGNLcdYPDmJyloHTxVO4UejhnTpf9IWSe5d4XZD+QmYeht62ytMcOygMHgoYWljCdv3XvNpz/+F0eVr9EKjtT3ySmP4BQ8YLXJLIEy82TgDzxKEbfQV/HJCyyPxzQwYA5f9N4y5GqUUt8yfBTHPGt3F2nD9lOQ2x10+LoHLxyUA8NXBQn7w8rfklNq5eUaqd5rJqdFMTm25VzBsQBjPXN94kDM1xkju09KjuWl64/yPXzu+xbxBAcaPTuN7RLHpSAnpsSEsu8Togvnjixu/zA1f7Ec/2MvLG7IBeOzqcbi15sW1RwBjb+Hj3fnUO90UVBglk/ljBrJ670n25pUzPimS3FN2jnq+eNmecwqOFBmvN2QVc7zUzuhB4c3eOy02hJ/MbZ5YYkNt/HzeCNxuTUxIABuyio3k3oqGssaQuFAuHBHvnR8gwGzi4atGo5Ti1+/u5v0deXx9qND7fxsSF+pdzo3TU7hxemNPkxVbj/PLlbuarSdASnQwcWE25gxv9U5sLZTZ65n4m89Zn1XMlLRo1mcVMz09mgCLiVRPch+fFMnvl0xs1/Kaun/+yA7Pc7rffriP/91yrPU69mlqHS6+zT7FzTNSefDK0S3Gp8WGtDgw73ZrJv/2c07ZHaREBxMeaHQzTokJ9u5xxIYGMDYxnHFJY8gsqGJDVjFWixFLVmEVSsHMwcaPyemflac/PcgLXx+motbBN55jKE33AruT1Ny7orrISMZR6ZRGGUmrYM6Txu54rGfXP32O8TjhRjjvJ0bXr9n3gMmzaxo2AC5/Ai591DiwGhgBF9zXeJD1TJQy+piHD2ocNvU2o87eRdPSo7GajQ9sZz58ydHBgJFcOmq2p5V1tnkbSiXxYTaGxYcaeyQ0nvT31KcHvc+VgnsvM34ovjpYhNut2ZhV0mz6po8bsorJKbV3KH6TSXHe0FjWZxVT52y9k1hOqR2zSZEQ0bh9Y0KNPaDY0ABvwpo9NJaqOicvfHWYhIhA77GItjTdRvdeNgKljHJabGjAGeZqKTI4gPGJEazLLOZocXWzElTDNp3dQ8moPVJigql1uCmqqkNrjb3e2ezP3aQV/m12KfVOt/fz1B4N2xCMz19KtLHHlhwd7P0sNBwrAuOzerCgklXb87yfn/GJEUQEt36ge9bQWO/xkoZjKJNSIzv6b2gXabl3wEOr9lBV5+TZ6zOMAeW5xmNkMp+FL+bUyQRum3Kh0Rvme08b12cZc7XRUu+GhHsuBQdYmJwaRWZBlbd+2xHpnmTU6rVozmL20Fie+vQgqWeZd/rgGKxmxWzPly3EZiEuzMaYQeHszi0nv7yWaWnR5FfUEBkUwND4MEYlhPPs54fYlVtOUICZuDAbIwd66u0pUWw+UsL5w+L4+lARJdX1Hf5xOn9oLB/szGP8I5/xxc8v8CbEnBI7V/5pHbGhNhIjg7A06ZbY0HKPDWu8RtDMwTGYFOSV17J4ctJZW6mJnoO9AWYTQ+NDGZ8YQZ3T3e7SSVOzh8XylzWHuejpr7yvAdJiG5ObrzT9fz7z6SHe2nq82fhJKZG8c9cswLh8h8WkmNbBXkLnD43lo135pMYEE+lJ0mkxwQR5Wu7NjzUZz09W1HLtpCTe/i73jP+fSamRBFnNrM8sYsPhEqalx7R93kAXSXLvgO05ZdQ6mrTIyj0frIgkPnXEUBw3krsaNlT6nMZWe9KUcxtoN/ntonFU1DowtXKSx9lcOymJgeGB3i9jR4xPiuCv35901lJCqM3Cyz+YxpD4xh+BF26aRFyYjfzyWnYeL+PCEfHUOV3eL9BTi8fz1KcH+epgIaGBFi4aEc9P5w6jpKqO+LBA8stryC6pblbW6IgFGYNYm1nEh7vyyS6p9q7/F/sLqKh1UlHrbNHybUjqTS8AFxUSwF+/P5ljJXauGJ/Qrvf+45KJmDzJ/KnrJlDfgYOeTd02ezBRwQG43Nrz42cc0L98bAIBZjMzBneuS2V3aNgeR4ur+XhPPtPSopk7yihv7cwt4+PdJ8kvryEhIoi8sloGRgQS0tYxqzZcMymJEJuFjORI3Br+tHQis4bEopTxfP7Yxt5kYwaF8/R1EyivcbB4UhLzxw4844+JzWJmWno0H+0+SXFVHTecoUzaVZLcO6C4qq75WaYNLfeIJHJKd3aqhdubDY0PPftEbQixWZg3ZuDZJ2yFUor5Y9uX0E7f5Z7i+WKlxoQ0vwaOx9jECG4/fzBfHyqizO5g1tBY0mNDvHsaKTHBJEY1nizT0R+nQKuZn84dxoe78imvabwsRNMud6cvs6F0EhPSvITS0f/f2MTG3k/DB3T+sxgdEsD/O39wi+GBVjPfa+cPTU9JjAxCKfh4dz6VtU6+PzOVBROMsuS+vAo+3n3Se0C7uKqufVdMPU2AxcRVnmWaFd7n0Pw5GJ/VpsdXLh094KzLnz00lq8Ptexx1t2k5t6G8hoH1XVOymscVNY60FpTUlXPoJpDRr/U9c/h2v8ROiAUd0AEuaU1nWqlinNvSprRJRRarx8nRQWTFtP5YwYRQcaufJndSO778yvYfKTE+56nL7O1soxoXaDVzMDwwMbuuEMaf8BHDgwjNjTA+0NaVNm55N7TGhJ6TEhAjzYIpeXehtv/uZWkqCCKKuswmxR/WDKRepebhwL+AV9kAWAGtpozGFRZR73L3alEIM69QKuZmUNiOFlu7La35qKR8azakdfhA5LQmNzLaxxsO1bKtS8YtzNYdskwfv9FJsMHNN8jGhAe6O2NIs5uaHwo+eW1jEuMIKZJ8jaZFOcNifWeFVpSXW/cq6CXGTkwjIHhgZw3JKZTJc/2kuTehqPF1Thcboqr6igor+N4qZ1A6hinjlI95W4CL/1PZv7uS4qq4QlPS0GSe9/x3PUZOFxt16Tvu2wEt81O79QByUCrmQCLiYoaB2sOFGE2Kf556zRmDo7hyvEJzbo0gvFj8NmyOc3KQaJtz16fwb78Cka0UnqaPTSW93fmceBkJaXV9b2y5W4yKd656zxCA3s2/Upyb4XbrSmtrifQaqK4sp56l5vVe04y0ZSFVbkojJ9GcWE9hbXGbnbD2ZWS3PuOqJAzt8iDAyzeswo7IyLISnmNg/0nK8lIjvTuird1xce0s3R1FI3iwmxcENb6wfZZnmMwH+7Kw+XWndrzOhcaTorrSVJzb8Upez0utya/rNZ7uvJ7O04wRR3ErRVb3cP5u+eqb1HBVnYcL8Okzs0GE31DRJCV46fs7M4t82nXwf4mMTKI9NgQVu0wrujYn49jSHJvRXGVcZ9PZ5OeMbmn7Fxs3sFBncx9Hx7jg515TE6N8h6pH5UQ3ulLqgr/ExFkZcuRUtwan3Yd7I9mDTUubw0QE9J/k7uUZVpRUlXX7PWUlEgsuRuZaMriIcct1LvcXDIqnudvmozFpLj7oqFtnpEm+qeIIKu3cXB6jV30rNlDY/nfzUapNC6sd5ZlzgVJ7q0oOi25P8dTJAesoUSHs8J1IWDUTgM8XdviW7kbi+jfGnrM2Cwm4nrhQT1/NnNwLCYFbk2vPKB6rkhyb0VDWQZgvDpMcuEaVjOT1SELqa0zPixy8FScSUNyT44O7tHubqKliGAr45Ii2XuivNklkfsbSe6tKG7Scr/d8hE6MIID4x4j0RJC2MZjVNY5JbmLM2pI7vI58Y0bpyWzLjOoX/+wSnJvRUlVHdEhAZRW1zPenI0aMpdl35sMwHvb8yS5i7Pyttyl77pP3DA1hRumtvPm235Kune0oriqnoSIQCICTSRQDJGNF/eJCLIal2yNlDq7aFvTsowQviAt9yae+/wQGw8Xsz+/ksmpUQTWFRNQ7YSI5sl9UGSgdHsUZyRlGeFrktybePNbo/vU+KQIFk9OwnqyEDbRLLnfOD2l2dX+hGjN1LRoFk9OYsaQllemFOJckOTeRFWtk6XTUvjPhltyWbYYjxGNl/Q8/ZKfQrQmItja4hZuQpxLUlvwcLk11fWu5hf2b3K9diGE6Euk5Q78dPl2xnludBDW9EptZcfBFg5Bkb4JTAghOkmSO/CvA4WcrKgFjFu3AUZiP7lLWu1CiD6p35dl3G5Ndb2TE54LDXnLMm9cDzmbIHaYD6MTQojO6ffJ3e5woTWNLfdAC1QXQ+E+mP4jWPBnH0cohBAd1++Te3WdE8B74+swm8VosQOMuRoCw30VmhBCdFq/T+6Vtc5mr0NsFsjZDJZAGJThm6CEEKKLupTclVI/U0rtUUrtVUot8wx7RCl1Qim1w/N3RbdE2kMaWu4NQhuSe+JksPTfy4UKIfq2TveWUUqNBW4HpgH1wGql1Eee0c9prZ/uhvh6VE6J3VtrbxBqs0BZDoy43EdRCSFE13WlK+QoYLPW2g6glPoauLpbojoH6pwu5jy1psXwEKsCezGEDvBBVEII0T26UpbZA8xRSsUopYKBK4CGi7D8WCm1Syn1klIqqrWZlVJ3KKW2KqW2FhUVdSGMztmeU9ZimM1iIqCuBLQbwiS5CyH6rk4nd631fuAJ4HNgNbATcAIvAEOADCAfeKaN+V/UWk/RWk+Ji4vrbBidtiGruMWwUJsFqgo8LyS5CyH6ri4dUNVa/0NrPUlrPQcoBTK11gVaa5fW2g38DaMm3+usby25B1qgqtDzQpK7EKLv6mpvmXjPYwpwDbBcKZXQZJKrMco3vYrWmr0nKloMb95yjz/HUQkhRPfp6rVl3lZKxQAO4G6t9Sml1GtKqQxAA9nAnV18j25X43BR73ITZDVT43BhNSscLm30ca88aUwkLXchRB/WpeSutT6/lWE3d2WZ50LDzTaGxIew50QF8WGB5JXXGGenVhWCLQKscu9LIUTf1S/PUG1I7oNjQwEID7ISHmj11NwLpCQjhOjz+uUlf8vtnpZ7nJHcQ21mbhsfwKTw45B5WEoyQog+r38m9yZlGTAOpP70+N1QesSYIOMmX4UmhBDdol8n94ayTFgAUHrUSOrjr4eEDN8FJ4QQ3aBfJXetNcve2uFN7omRQdgsJhJNpYCGlJkw+EKfxiiEEN2hXyX3gwWVrNqRB4BSxv1S//N7o5hh2g8HkVvqCSH8Rr/qLbM+s/Gs1PBAKyaT4uaZaQwLLDcGRqb4KDIhhOhe/Sq5N72eTESQtXFE+XHjMXzQOY5ICCF6hv8nd7cbVt2NO/c7thwtRSljcESQFVb/CjI/h7LjEBInJy4JIfyG/9fcq4tg+/9SH5yIvX4Cw+JDySysIiWgEjY/D/ZSYxqptwsh/Ij/t9ztJQDUV5cCMC4xAoAMfcAYX55r/EUktzq7EEL0Rf0mubs8yX2sJ7mPdu41xpcdg1PZcjBVCOFX/L8sYzcOorrtZQCMTAjjWts3jK3aaIxvOJgaN8IHwQkhRM/oNy13VVsGQHQgPG36ExG1J2DA2MbpYiW5CyH8Rz9I7kY5xlRXBkC0qxilXbDgTzD34cbpYof5IDghhOgZ/aAsY7TcLXXGiUrhdZ6bcUSmQIjn0r4hcRAc7YvohBCiR/h9cndWFmMBrPXlWM0KW7Vx+QEiko2kDhA73GfxCSFET/D75O6uLgLAphzEB7pR5SeMEeGDjJOWQgfCwHE+jFAIIbqffyd3rb1lGYDEwDooz2l+Nuqtq6UkI4TwO/6b3Ouq4KmhBDhrsGsbwaqOFdW3wnfAoEmN00Wn+yxEIYToKf7bW6ZwPzhrAMjRp90TNTjGBwEJIcS547/JvfiQ96kTc/NxrvpzHIwQQpxbfpzcD4LJwrGJv+QxZ5N7ol7+JFzxtO/iEkKIc8CPk3smxAzjyIjb2edObRw+/U6Ik66PQgj/5r/JveggxA2nzuminBA+ClrIoQUf+DoqIYQ4J/yzt4yzzrjS49hrqHW4AcWoW59ncFyoryMTQohzwj9b7lWFoF0QkUyd0wWAzWo+y0xCCOE//DO511cbj7YwT8sdAi3+uapCCNEa/8x49VXGY0CotNyFEP2Snyf3EGm5CyH6pS5lPKXUz5RSe5RSe5VSyzzDopVSnyulMj2PUd0SaUc0lGUCQqhzujCbFBazJHchRP/R6YynlBoL3A5MAyYAVyqlhgEPAF9qrYcBX3pen1ve5B5KrcMtrXYhRL/Tlaw3CtistbZrrZ3A18DVwELgVc80rwKLuhRhZzSUZWxGzV3q7UKI/qYryX0PMEcpFaOUCgauAJKBAVrrfADPY3xrMyul7lBKbVVKbS0qKupCGK1oUpaRlrsQoj/qdNbTWu8HngA+B1YDOwFnB+Z/UWs9RWs9JS4urrNhtM6T3Dcfr6Gq1iktdyFEv9OlJq3W+h9a60la6zlAKZAJFCilEgA8j4VdD7NjaqrKsWsbS/7+Lav3nsQmLXchRD/T1d4y8Z7HFOAaYDnwPnCLZ5JbgFVdeY/OqLNXUI3N+1pa7kKI/qar15Z5WykVAziAu7XWp5RSjwMrlFK3ATnAdV0NsqOctZXYdaD3tdTchRD9TZeSu9b6/FaGlQBzu7LcrnLXVmGnMblLy10I0d/4ZZNW11dTTSAp0cGAtNyFEP2PX2Y9VV+NXdsYEhcCSMtdCNH/+GVyNzmqqCaItFgjuUvLXQjR3/hl1rO47NSbg4gPM+ruLq19HJEQQpxbfpncra4anJZgYkMDAKiqbfe5VUII4Rf8MrkHuGpwW0KIDTP6ulfVSXIXQvQv/pfcnfVYcUBACJFBVkCSuxCi//G/5O5ovGhYuCe5u6XmLoToZ7p6hmrvU2dc7tdkC2VwbAh3XzSExZOTfRyUEEKcW36X3HXNKRSggiJRSnHfZSN9HZIQQpxzfleWqa86BYA55Nzf3U8IIXoLv0vu9ooSAKwh0T6ORAghfMfvknt5qXFXp8iYAT6ORAghfMfvkntVmXFvkAEDBvo4EiGE8B2/S+41FSU4tYnE+G6+dZ8QQvQhfpfc66tOUalCCQzwu45AQgjRbn6X3Kk5RY05zNdRCCGET/ldcjfXleOwhvs6DCGE8Cm/Su61DheBrgp0kPRxF0L0b36V3Asr6oigGlNQpK9DEUIIn/Kr5F7vchGpqnHZInwdihBC+JRfJXeny0U41Thtkb4ORQghfMqvkru7thKz0ril5S6E6Of8KrnrmnIASe5CiH7Pr5K721FjPLEE+jYQIYTwMf9K7k4HACaL1ceRCCGEb/llcldmSe5CiP7Nz5J7HQAmS4CPIxFCCN/ys+TuKctIy10I0c91Kbkrpe5RSu1VSu1RSi1XSgUqpR5RSp1QSu3w/F3RXcGejXbVA9JyF0KITl8XVymVCPwUGK21rlFKrQCWeEY/p7V+ujsC7Ai3ywnIAVUhhOhqWcYCBCmlLEAwkNf1kDpPe8oyZknuQoh+rtPJXWt9AngayAHygXKt9Wee0T9WSu1SSr2klDpnl2jUroaukFKWEUL0b51O7p6kvRBIBwYBIUqp7wMvAEOADIyk/0wb89+hlNqqlNpaVFTU2TCaaai5S8tdCNHfdaUscwlwVGtdpLV2AO8A52mtC7TWLq21G/gbMK21mbXWL2qtp2itp8TFdc/9Thta7mZpuQsh+rmuJPccYIZSKlgppYC5wH6lVEKTaa4G9nQlwI7wJnerJHchRP/W6d4yWustSqmVwHeAE9gOvAj8XSmVAWggG7iz62G2kze5287ZWwohRG/U6eQOoLV+GHj4tME3d2WZXaHdRldIqbkLIfo7vzpDVXla7hZpuQsh+jm/Su4NNXeLtNyFEP2cXyV33A1nqMoBVSFE/+Znyd1ouSMXDhNC9HN+ldyVy4FbKzCZfR2KEEL4lF8ld9xOnEoSuxBC+FVyV+56nF3r3SmEEH7Br5K7ye3EibTchRDCr5I7bicuSe5CCOFfyd3kduJUUpYRQgi/Su5KO3BJzV0IIfwruZvcTlzSW0YIIfwruSu3U1ruQgiBnyV3k3bikpq7EEL4Y3KXsowQQvhdcndLy10IIfwvuUtZRggh/C25u6XlLoQQ4GfJ3YwkdyGEAH9L7tqJ2yTJXQgh/Cy5u6TlLoQQ+F1yd+I2yV2YhBDCr5K7BSdaWu5CCOFfyV1a7kIIYfCr5G7BhZYDqkII4V/J3YxTkrsQQuBnyd2CC6QsI4QQfpbctbTchRAC/C2540JLy10IIfwouWuNRblBWu5CCNG15K6UukcptVcptUcptVwpFaiUilZKfa6UyvQ8RnVXsGfictYbT8zSchdCiE4nd6VUIvBTYIrWeixgBpYADwBfaq2HAV96Xvc4p6POeCJlGSGE6HJZxgIEKaUsQDCQBywEXvWMfxVY1MX3aBenw2E8MUtZRgghOp3ctdYngKeBHCAfKNdafwYM0Frne6bJB+Jbm18pdYdSaqtSamtRUVFnw/DylmWk5S6EEF0qy0RhtNLTgUFAiFLq++2dX2v9otZ6itZ6SlxcXGfD8HI6jOSupOYuhBBdKstcAhzVWhdprR3AO8B5QIFSKgHA81jY9TDPzi0HVIUQwqsryT0HmKGUClZKKWAusB94H7jFM80twKquhdg+TqdRc5eWuxBCGAdEO0VrvUUptRL4DnAC24EXgVBghVLqNowfgOu6I9CzxlNfazyx2M7F2wkhRK/Wpa4lWuuHgYdPG1yH0Yo/p1x1FUZMAaHn+q2FEKLX8ZszVB3VRnI3B4X7OBIhhPA9v0nu9fZyAAKCI3wciRBC+J5fJPfCylqcNZUABIZIchdCiD6f3I+X2pnx319y5EQ+AEFhkb4NSAgheoE+n9zzympwaygvOwVAUKi03IUQos8n96o6JwDumgpqtZWw4GAfRySEEL7nN8nd5KiiiiACrX1+lYQQosv6/CUUG5J7qKrBroKIVcrHEQkhGjgcDnJzc6mtrfV1KH1aYGAgSUlJWK3tPwO/7yf3WiO5h1BLrZKSjBC9SW5uLmFhYaSlpaGk4dUpWmtKSkrIzc0lPT293fP1+RpGQ8s9TNVQa5LkLkRvUltbS0xMjCT2LlBKERMT0+G9H79J7iHUUG+W5C5EbyOJves68z/s+8ndU5YJpQaHRa4rI4QQ4AfJvbrOwQLTRqJUFU5LiK/DEUKIXqHPJ/dB5dv5Y8CfiVTVuOWKkEKILggNbTuHpKenc/DgwWbDli1bxpNPPgnA9u3bUUrx6aeftnuZPanP95apdTi9z7VVkrsQvdWjH+xlX15Fty5z9KBwHr5qTLcusy1LlizhzTff5OGHjaucu91uVq5cyYYNGwBYvnw5s2fPZvny5Vx22WXnJKYz6fMtd11vb3xuC/NhJEKI3ub+++/n+eef975+5JFHePTRR5k7dy6TJk1i3LhxrFrVvpvFLV26lDfffNP7eu3ataSlpZGamorWmpUrV/LKK6/w2Wef9Yp+/X2+5U6T5B5qqvNhIEKIMzlXLeymlixZwrJly7jrrrsAWLFiBatXr+aee+4hPDyc4uJiZsyYwYIFC87aI2X8+PGYTCZ27tzJhAkTePPNN1m6dCkAGzZsID09nSFDhnDhhRfy8ccfc8011/T4+p1Jn2+5K2eTlntQjA8jEUL0NhMnTqSwsJC8vDx27txJVFQUCQkJ/PrXv2b8+PFccsklnDhxgoKCgnYtr6H17nQ6WbVqFdddZ9xFdPny5SxZsgQwflCWL1/eY+vUXn265a61xuywgxV+UH8f1wy7wdchCSF6mcWLF7Ny5UpOnjzJkiVLeP311ykqKmLbtm1YrVbS0tLaXUZZunQp8+bN44ILLmD8+PHEx8fjcrl4++23ef/993nssce8Z5RWVlYSFua7UnGfbrnXOFwEYpRiLpx3NRePSvBxREKI3qbhQOjKlStZvHgx5eXlxMfHY7VaWbNmDceOHWv3soYMGUJMTAwPPPCAtyTzxRdfMGHCBI4fP052djbHjh3j2muv5b333uuhNWqfPp3cq2qdBCsjuf9gzihCbX16R0QI0QPGjBlDZWUliYmJJCQkcNNNN7F161amTJnC66+/zsiRIzu0vKVLl3LgwAGuvvpqwCjJNDxvcO211/LGG28AYLfbSUpK8v49++yz3bNiZ6G01ufkjc5kypQpeuvWrR2e70hRFV/84Q5utf0Ly4MneyAyIURX7N+/n1GjRvk6DL/Q2v9SKbVNaz2lten7dsu9zkkwtbgtQb4ORQghepU+XcdIiAgiLCUYU5VcdkAI0T12797NzTff3GyYzWZjy5YtPoqoc/p0co8LsxEXATjkzFQhRPcYN24cO3bs8HUYXdanyzKAcRJTgFzqVwghmur7yd1hB6skdyGEaKrvJ/f6agiQmrsQQjTV95O7tNyFEKKFvp/cpeYuhGhDWVlZs6tCttcVV1xBWVlZh+Z55ZVXvGetNiguLiYuLo66OuNky4ULFzJz5sxm0zzyyCM8/fTTHY7xbDrdW0YpNQJ4q8mgwcBDQCRwO1DkGf5rrfXHnX2fs3JUg1XKMkL0ep88ACd3d+8yB46Dyx9vc3RDcm+4KmQDl8uF2Wxuc76PP+54yrrmmmu49957sdvtBAcbDc6VK1eyYMECbDYbZWVlfPfdd4SGhnL06FHS09M7/B4d0emWu9b6oNY6Q2udAUwG7MC7ntHPNYzr0cQO0nIXQrTpgQce4PDhw2RkZDB16lQuuugibrzxRsaNGwfAokWLmDx5MmPGjOHFF1/0zpeWlkZxcTHZ2dmMGjWK22+/nTFjxjBv3jxqampafa/w8HDmzJnDBx984B3W9LLAb7/9NldddZX3Wjc9Tmvd5T9gHrDB8/wR4N6OzD958mTdKc56rR8O1/qrJzs3vxCiR+3bt8+n73/06FE9ZswYrbXWa9as0cHBwfrIkSPe8SUlJVprre12ux4zZowuLi7WWmudmpqqi4qK9NGjR7XZbNbbt2/XWmt93XXX6ddee63N91uxYoVetGiR1lrrEydO6ISEBO10OrXWWs+dO1evXbtWHzx4UI8bN847z8MPP6yfeuqps65La/9LYKtuI692V819CdD0AsY/VkrtUkq9pJSKam0GpdQdSqmtSqmtRUVFrU1ydvXVxqO03IUQ7TBt2rRm5ZA//vGPTJgwgRkzZnD8+HEyMzNbzJOenk5GRgYAkydPJjs7u83lX3nllaxfv56KigpWrFjB4sWLMZvNFBQUkJWVxezZsxk+fDgWi4U9e/Z09+o10+XkrpQKABYA/+cZ9AIwBMgA8oFnWptPa/2i1nqK1npKXFxc597c4blRh/SWEUK0Q0hI4/G5r776ii+++IJNmzaxc+dOJk6c2Op13W02m/e52WzG6XS2mKZBUFAQ8+fP5913321Wknnrrbc4deoU6enppKWlkZ2d3eOlme5ouV8OfKe1LgDQWhdorV1aazfwN2BaN7xH6xpusSf93IUQrQgLC6OysrLVceXl5URFRREcHMyBAwfYvHlzt7zn0qVLefbZZykoKGDGjBmAcVng1atXk52dTXZ2Ntu2besTyX0pTUoySqmmd8y4Gui5fQ+HpywjLXchRCtiYmKYNWsWY8eO5b777ms2bv78+TidTsaPH8+DDz7oTcRdNW/ePPLy8rjhhhtQSpGdnU1OTk6z5aenpxMeHu69GNlvf/vbZtd87w5dup67UioYOA4M1lqXe4a9hlGS0UA2cKfWOv9My+ns9dwpOQxf/hec/3NImNDx+YUQPUqu5959Ono99y5dFVJrbQdiTht2cxuTd7+YIXD9q+fs7YQQoq/o05f8FUIIX7j77rvZsGFDs2E/+9nP+OEPf+ijiFqS5C6E6FFaa5RSvg6jW/3lL385p+/XmfJ537+2jBCi1woMDKSkpKRTyUkYtNaUlJQQGBjYofmk5S6E6DFJSUnk5ubS6RMVBWD8SHa0F40kdyFEj7FarT1+gSzROinLCCGEH5LkLoQQfkiSuxBC+KEunaHabUEoVQQc68IiYoHibgrHl/xlPUDWpbeSdemdOrsuqVrrVq+82CuSe1cppba2dQpuX+Iv6wGyLr2VrEvv1BPrImUZIYTwQ5LchRDCD/lLcn/x7JP0Cf6yHiDr0lvJuvRO3b4uflFzF0II0Zy/tNyFEEI0IcldCCH8UJ9O7kqp+Uqpg0qpLKXUA76Op6OUUtlKqd1KqR1Kqa2eYdFKqc+VUpmexyhfx9kapdRLSqlCpdSeJsPajF0p9SvPdjqolLrMN1G3ro11eUQpdcKzbXYopa5oMq5XrotSKlkptUYptV8ptVcp9TPP8D63Xc6wLn1xuwQqpb5RSu30rMujnuE9u1201n3yDzADh4HBQACwExjt67g6uA7ZQOxpw54EHvA8fwB4wtdxthH7HGASsOdssQOjPdvHBqR7tpvZ1+twlnV5BLi3lWl77boACcAkz/Mw4JAn3j63Xc6wLn1xuygg1PPcCmwBZvT0dunLLfdpQJbW+ojWuh54E1jo45i6w0Kg4d6BrwKLfBdK27TWa4HS0wa3FftC4E2tdZ3W+iiQhbH9eoU21qUtvXZdtNb5WuvvPM8rgf1AIn1wu5xhXdrSm9dFa62rPC+tnj9ND2+XvpzcEzFuzt0glzNv/N5IA58ppbYppe7wDBugPTcU9zzG+yy6jmsr9r66rX6slNrlKds07DL3iXVRSqUBEzFaiX16u5y2LtAHt4tSyqyU2gEUAp9rrXt8u/Tl5N7afbv6Wr/OWVrrScDlwN1KqTm+DqiH9MVt9QIwBMgA8oFnPMN7/boopUKBt4FlWuuKM03ayrDevi59crtorV1a6wwgCZimlBp7hsm7ZV36cnLPBZKbvE4C8nwUS6dorfM8j4XAuxi7XgVKqQQAz2Oh7yLssLZi73PbSmtd4PlCuoG/0bhb3KvXRSllxUiGr2ut3/EM7pPbpbV16avbpYHWugz4CphPD2+XvpzcvwWGKaXSlVIBwBLgfR/H1G5KqRClVFjDc2AesAdjHW7xTHYLsMo3EXZKW7G/DyxRStmUUunAMOAbH8TXbg1fOo+rMbYN9OJ1UcZdqP8B7NdaP9tkVJ/bLm2tSx/dLnFKqUjP8yDgEuAAPb1dfH0kuYtHoa/AOIp+GPgPX8fTwdgHYxwR3wnsbYgfiAG+BDI9j9G+jrWN+Jdj7BY7MFoat50pduA/PNvpIHC5r+Nvx7q8BuwGdnm+bAm9fV2A2Ri777uAHZ6/K/ridjnDuvTF7TIe2O6JeQ/wkGd4j24XufyAEEL4ob5clhFCCNEGSe5CCOGHJLkLIYQfkuQuhBB+SJK7EEL4IUnuQgjhhyS5CyGEH/r/+uRy3HbPz08AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_VAL','train_VAL']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABRGElEQVR4nO2dd3xUVfr/32dm0ntCGukQEAi9N1EEEVBXUFFYu2vZ1d1Vtyi2Vfe3fld3XXfdta27unYUsWBFimJBQFroJSE9pPeeTOb8/jh3MklISAJpQ8779crrzj1zy3Nzk8997nOe8xwhpUSj0Wg0zoeprw3QaDQazemhBVyj0WicFC3gGo1G46RoAddoNBonRQu4RqPROCmW3jzZoEGDZGxsbG+eUqPRaJyeXbt2FUopg1u396qAx8bGsnPnzt48pUaj0Tg9Qoj0ttp1CEWj0WicFC3gGo1G46RoAddoNBonRQu4RqPROClawDUajcZJ0QKu0Wg0TooWcI1Go3FStIBrNJozozwHDn9y6m3qq2HPm9C8fHVjA+z8n1pqTgst4BqN5szY/gK8ex001LS/zXdPwdo74cinjrajX8Cnd8PBj+CrP0FVYU9betbRqyMxNRpNH/P9PyBkJAy/qPuOWXQckFBVAP7RbW9TX6WWxSlQlqUEG2HY9DTkH4LyE7Dk+e6zq7cpToX1D4EwwZIXwM27x0+pPXCN5mzn41/BgQ+gtgw2PgJvX9W9xy9OUcuqgva3cfVSy4pc+PIB2LsK9r6t2vIPqWXiW3BiT/fa1ltICZ/9Vr1hHP64166jUwIuhLhLCHFACHFQCHG30RYohNgghEgylgE9aqlGM1CpzIdnp0LeQbUupfJoX5wN6VtPvW9JGux+HXa/Bimbu982m80h4JVtCPiRz+DZKVCaqdb3roJDa8Hk0nK70NHgFQxfrISaUvjXJDj+lTp+XQW8eC4kbeh++7uL9B/g+CaY/DO1XpbVK6ftUMCFEKOBW4GpwDjgEiHEMGAlsElKOQzYZKxrNJruJmcfFB5VYliRB0/GKFHO3Q/JHYiaXfSydsLRdeqzu1/32VZxAqy16nNrD9xaB+/8FAqPQeY21VZTAq7esOBPaj32XLUcOhcueFhtt/VZKEqGN5bC0yNg9Q2Qu0/FyvsrOYlqee5v1bK8nwg4MBLYJqWsllJagW+ApcBlwGvGNq8BS3rEQo1moGMXg4xtkPadCoUc+Uy1FRw99b7JG9WyvtIRsqgtU1kh3YHd+waoynd8bqiFD25zrJdmOD4PXwhTboFlr8HcB1Rb7Lkw6jL12f6gAajMU54tOB4C/ZGSdHD1Ad/B4BnUfzxw4AAwRwgRJITwBBYDUUColDIHwFiGtLWzEOI2IcROIcTOgoJTxMg0Gk3b2MUgawdkGCGT7N1qWXCk5bbpW5WXCyo9L/U7GNasw3Laz9Wy/MTp21ORqx4mYHRgGjQPoex8BQ59BDN/dfL+E68DswUSlkDMTLjlKxi2ADz8lfjl7XdsO3Y5mCwQEKu88s5mquQdUuGj3qI0Q3XgCgF+kVCW3Sun7VDApZSHgSeBDcA6YC9g7ewJpJQvSSknSyknBwefVI9co9HYaaiFXa+CrbFlu10M6sphx3+NbZtldVjr1OfqYnh1MWx7Qa3n7lfbjbsa/GNg5E9gxCXqu86+4ucdUhkjzb3iz38Hr14MBcdUp527nzp+8xDKsXUQPALm/1EJMMD4a+GeQzDk/JbniJykhA8gcIhaeofBPQdh6Ytw1z64zMhOyfyxY5ttNnhhBjwzDtK2qLb0HyBje+euuSvUlELi21CaDgExqs03EoqPw46XVWrljpfVve0BOtWJKaV8WUo5UUo5BygGkoA8IUQ4gLHMP9UxNBpNBxz6CD65S4VJmlOepQSyLaRNeaagxE3aVMzZvg4QNR1+tUuFLPwiVFtnXvHrq+GtK+Hbv8JHP1edp1WFKn/bZlXfJW+EOfeCTxikfa9yvT//PaR+A8MuBJNJiTGAZ6Dj/O0ROFQt/aOVJyuE2idiErh4wdHPO7Y7p1kGyHdPqeX/FsErCxxvJ93FN0/CR79QmTT2e+QXoR6sn/1GZdx89hvY80b3ntegs1koIcYyGrgcWAV8DNxgbHIDsLYnDNRoBgw5e1su7ZRlQcREFWqImQ0xs1S7xUMt8w+rpT1GbI9LZ25T3qBfBJhdlJj62gW8E6/4256D8mwYfaUSvrJM2P6iEu9zf6fEdfgimHqbyiCpzIW978KPL6n94y9US59mAt4Rdg88oNUDy8UdEpbCwQ8dOeXtkbQBEOr8rfsIvn1Kecw7Xj71MZI3wYZHOra3OfYceL9IR9v+NWq5582uHauTdDYP/H0hxCHgE+BOKWUJ8ARwoRAiCbjQWNdoNKdL7v6WS1Beb/kJJQqRk+Cmz2DoBeq76OnKK7XHxe0ed1EKrPopHPkcoqe1PIfFDbxCWnYqAnz/d/UD8MOzKmxy6GP1sLDHzT+9R3njo5bAvIfhrr3w03fA4uoIk0y+GX66GibdCNEzVJtvuFp6BnX8Owhq5oG3ZsI1qjO2o2H7yRuVxx4zUz2AKnId3+17Fzb9P9j855bD+luz6Y+w5R+Qe+DU57I1iybbHzr2ByuosBeoLJXm97Wb6NRITCnluW20FQHzut0izcDGZlOe4kDh+New4WG4+k2VKgcqbbAkDV5fAgufUGl6vs28OruHFxADLh6QtB6s9ZC9S2VC1FfA0c9UZsf0O04+Z/Q0SPpSdXKajXzsjY+q5cy7lHDVVUBjvfK0Q0cBQglj9Ey4og3v1V7PZMI1ED6u5UhPn8Fq2SUBbyNkFD1DpSBm74Zxy9s/RuExGHOVisGDyicH9eCzfwb1pmI/X3Ny9zvSAldfrzJLrl8LJvPJ21YXOT77Gtc5dhmUZUD+EZXmOWQumF1P7tvoBgbQf4qm37P3HfjrEKgt72tLegdrvYqP5u6H929R6X0+g6EoSXUalqTCqqvVts09UnsYxGcwxM9X3vSB95XQJyxxbHf5fyBy8snnHX+t6nBMWn/yd0c+Vd9Za1U8PXqaGkU5aJj6fsrPVAZJaxb/RXU0ho87+Tu7B+7RiRBK+Hi4+G8w+vKTvxMCAuNapi62pr5a/R59wyH4HNVmT6WceEPLbdvrEP3xP0pwh5yvOiPTvoMTiSo89PQodd/sVBXCoHNg8VPKdgCPAJXnHj1drQ85D65ZDYPHn/LSTwct4Jq+paEGCpPU520vqFhr1ikyDaTskVfRXsNap+Ky1jr48DYlRkPnqRRBgInXK+E8+KFjn+ELIb7Zy27gEMAQs2FGnHnz/6nluBVqGTbGIZytiZ+vOhbXP6TCEc1DDF8ZA2zMbuockVPUesRkJcD2LJbW+EUq77st7HFtu4d6KoRQOeJuPu0ca6gS1faoyFFLn8HY/KKRZjcVzwYloJFTIGwsuPm1nVeeewD2vIGcdCONS/9jvG0I5Uln/GCEZJqlYFYXKS9+6q2OTBo79odnRBsP0W5CC7im9yjPceQP29nyT3hhlkrzsr+2nird6+gXagh54ts9ZmaPsv3f8NxUVY/k4Idw4f+Da9bAtR/A9R+rGDIocQkZBbd+DVe/pWLXdvwi4PZvIeFy5ZlHTTfykGMgaiq4+6uUwfYwW+Cq19XD8t1rVZjATuFRiJwKwxeoOLJ91OZFj8MtG1VnYlcZcYmyt3XH5OkQOEQNmmmvBG2TgIex/L87yXGNhdpSQKg3lqtehxXvqN+TPcWwOT/8E1x9eNmynAtePEhjwhVKiJPWOzz/5hk8VYXth4bizoNbv4LY2ad5sR2jBVzTe2x4WOUP2z1uUK+njXWw7n4QZgiIg8xTCLh94EoP9er3OJV5apmyGcb9FGb9WsX84+epV22fUDVoBVQMN2Ji2yGL8LGO9gnXqmXUNBXT/uVOmH3PSbt8l1TA8YJKtRI9DX61W4UJsnepNpNFPRAufwmW/huu+8Cxs2dg2/HizmAytx1aOR2ChoJsPLkT1k65EvDsxgB+TCtmnbR34krV2eo7WD0Ah16gQlXNB/vYGlW45ZyFrE+pJ72omh1pxeoNKXu3o0PTnsEjpfLAvQa1bYsQ6iHY2jPvRrSAa3oH+z+Hzape3UF5UVk71eecRBUzjJ+v2opTYe0v4cOfK3G3e1z21+fM7W0XT+rvNBrx05BRMO8PbW8TZcRO7Z1wHZGwRG07yvC6vYMdnZPN+PWqPTz5RbORm56BqoiUPZPi2g/UoJnAOBX37s6aKd2FPU+8vTi4Ed74MlOJ5iuV09vezh56al4g68QeqC6iceh89meXAbDuQC7EzAAk1BSr7eyDoGrLwNbQuc7ZHkILuKb72PuO6gBqi+xd6pU9aJh6HW2oVbFsaw1NdaGHXai80IYqVQQp8W2VNbDtedWJBCp+7OanRCfl6964qjMjezds+IMjZa26WL1l3LG1/Rh11FS1tHfCdYSbD9y5HUZe2u4m5bUNlFQ3kJhZimyePmf39kHlaxve4jMbk3joo37Y12B/CyhqJw5ekQsuXnx2VL1pZFn9KZtwO5+F3ckfPznU7Djx6tqPfeloS9oAwkSSz1RqGhrxcjWz/mAuMmKSqvFtx+6B2zNQPNvxwHsBLeCa7mPrsyrHtrGNSgvHvlT/BNN/rjrpio87sgBGXKyW8Requh2eQWpkW8JSuO4j9V1puhLBgqMw5gqVLtc6nt4fObQWtjzjyAeuKel4QMvIn6jBM3FzOn2aPRkl1NQ70tQOnSinotYRJ84sVsWr8ivqyClTw7pTCiopcQlzHMRLlbpoaLTxypZU3t+VjbXR1uG5K+usHDrRS5lDXsEqz7oss+3vy08gfcM5mFPOhGh/AB6uXsGdabN4ZUsqGw4ZISwhVKph8gbINDqQkzdAxCR2FaiH2E+nRXOirJbsarN6UwFVBrcsCxJXIf8zF4CDpSe/7fQWWsA1p0bKUw94sGNrVLHtujIV3rDnvG5+Et68EvavVoJkDw8UHFFFi7yCYd4jMPchCE1QccqxRurchGsc6XOlGaqDqq5chR8iJ586Vt5fqKtQS3umR02JSjM7Fd7BcOXLnRu5CBRW1nHFCz9w7/sqj7zeauPyF7bw5DpHuCSz2DHdWWJmKfkVtSx9/gf+sUvVUbFipsqkZpDZllJEWU0DNQ2NHM2r6PD8//s+lSXPbaG6vtMlkk4fIdSbgr2zsjUVOdR5hFLbYGNhgno4fbz3BOOj/BkW4s3TG445tp31a/AOhXUrVTguezcMW8D2lGIGebty6TiVNZOYWepICYyconLoP/o5olaFWVZ+mcPBE2U9dcWnRAu45tS8eC588xf1uaak/aI8pemOutCvLoZnJ6thxN88qTyb0gyVfxwUrzzxgqPqJ3gEBA+H837v6Ow597cqrzbufDUtlWeQOr59+HPISPUPlXew/+eM2z1ve/W/muKOBbyL7MkoxSbhk70n2HQ4j/SiKmobbKw7kIe10cajHx/k1R9SATCbBD+mFvPoxwcpq2lgV5kvAIXSl7+uV53Ln+7NwWJS9yIxs5TP9uXw7FdJbZ8cOF5QSX2jjZSCDoa4dxe+g5s6K0+iLItSswppTIwJINjHDRez4C9XjuXqKVEcziknvciw081HOQ/ZO1UNFyT1cRfw1ZF85o8MZUSYL64WE//alMwT5ReRcu7f2VlmpDdGTiXXVTkXRfipWHkfoAVc0z6VBcpLPvaFKlT0zDj49xw4/KnqwAH1Onl0nWNmGDej46u2DN7/mRLgiElGatslKg0tIE7V7yg41nac12uQyqu1j8j0j1ZV+r57Snnn0TONOLFU/3yg6ofYOn7d73XsDxi7x1hT0rkBLQbpRVWkFla1jFsDZTWO8EhiZglmk2BYiDcPfXSAPZmlgPLM731/H6/+kMa2lGJ83CwsTAjjzW3pfL4/lwBPF44WWymQvpQKP17bmsYr36fy3q5Mrp4SRaCXK4kZpdz59m6eWn+M9sgsUd59cr6KO9tskqLKuja3ba+9S/iEt+2BV+RCeTapLipOHh/szS/OG8rjS8YwPNSHiwyPvIXYjluhBuAkfQneoXxfFUFlnZWLRofhajERGeDB0bwKXkys45bEIWwpMqaGW/hnrjc/wfPhjxMdN0wLuKYfkmsvrrRP5Qt7DlIFi969RqUD2hphzc1qtOBaY8j2bV+rMqC/2Kryl3/+vcpzvmWTGvYNyutO/UaFWzqTaWEfVu3mC5c9p4TdPuotZ5/K5/33HEg8zdTCzB97rgB/UwglR/2+ass67YEfyC7jvL9uZu5Tm9meWtzUvjOtmMl/2sDzm1UVwsTMUkaE+fDXZePILa/l/z5Xxa3cLCY+2J2Nm0X9m7tYTPzh0lF4uJoZEebDnXPjaWiUHLbF4Dl4BOG+7vzx00MM8nbj3oUjmBDlz7qDDmEqq24799oeX7cL+JrdWcx84ivyylu+rW1JLmTK4xtJzu84LHNK7CGU1qE9I6S2yzaMQd6uBHi5cvPsOK6aEgVAVKAnYyL8eHdHJnVWI8RnMsENH8PVbyFv+IQ3tmXi425h1lDlxU+JUQ9bs0mQUlDFf1nCxQ1PkuY+kmMlYBm5iAWjwkjKrySrpJsmyegCWsA1LbHZVI513kEljqDybmtKYPFf4deJsOgvKoNkzxsqNzZ0jGP/oKFqVJ5PqPK4/aNVLHdQvGObweMdHnxnMi3scfC4OY70OM9A8ItS9UN2v67adjcr2Smluo6qopbH2rcaNj6mOlvzDqp6zq9dqh5QNpvKerHPPdkd2EMoJ/Y4qvR5BJBfUcvzm5Ox2SSl1fX8a1MShZV1/GPjsSZxOZDtiKseM2LRDY027nt/Hw2Nkn9sSCI5v5J9mWWMj/JnfJQ/U2MDKa1uIMLfgw/umMlL103i/V/MBKC4qp5QX3c+//W5vHv7DOIGKW/yFw13U37hP1j7y9m8eO0kPvnVbPw8XLhjbjyVdY649mf7c/hwT8sHXW1DI/kVyqu2C/gPyYXUWW18ebClV5qYqUI921KKqaqz8vSGYy3eJDqN72AVrmtdGjZjO5jd+LZiMEOD254R/rcLhpNSWMXPXt3JJ3uNsJa7Hzs9ZvLLDVV8fbSAu+YNw9V46D10yUjW3zOHaXFKyK+dfQ4HG6N4fWs6AOOjAhgZrsJQaYW9L+CdKmalGSBY6+CjO+DAGlXlzj9adTJWFahiSkPOV4Mypt6mxPGbv6qUvyk3q5lfOps3PPVW+Ppx9bkzHrg9B7z1YJCwsSq0U1uuvNqsH7HmHsYSNlLlla+9E85/AM6/T22fvAk+uFUNGEKqofsjLlZicGKP6mjd8Ad13GvXdO5aOsIeQjn8iaOKnmcg7/6Yyd82HGN2/CD2Z5fxtw3H+OZYATvTSxgb6ccFI0JJzq/EzWJCSoeXe+hEOccLqrh/0Qj+/MURXv4+hYo6K2Mi1O9+4egwtqcWMyzUm4TBfiQMVu0Xjw3n/OEqyyQq0LPFsgoPIkOD8Pd0ZeFoR1bKpJgA7p43nA2HczmQXc5jnxzEbBIsGR+BMPorsozwicUkSMqvwGaTqtMPFaq4fkYs1kYbQogmgU/MLMXbzcI/NyWRW1bDX67s4iAfHyP9siK3ZUdv5naImEjKiQbmj2w7THX+OSH84vyhvLE1nX1ZpSwcHUZZTQO3vL6TBquNxWPCuGlWnONU7i74uLtw06w4TEJw67lDeGHzcVb9mIGbxcSYCD+KqtQDLFN74JoepaFWDZt+9RJ4/bKTy5auWqHEe87vlSddcFh1Fk68XomgvRqbEGoSWvuAhuARsOx/cOk/OmeHRwBc+YoaYu3ViVmaptwCsedyNOoqJv9pA9mlRkZF+Fg1stFaQ+niF7BKE6889yd+OF4ISUYBo8xtKnTx4c/hw9tV/P3BHPjNYfVWsH81hCSonOC9q9Tx7FUBu4O6NjpZPQKaRC4xs5SkPCVsO9OVR/n29gwm/HE9m47kMzTYm6hAj6YskiRDBOePCiXU1431B1Va3LBQ5XHaBXhYSEsP9LmfTmTZ5KgWbZEBKqTl42bBz6PtVLi75g/jrVtUBkad1UZ1fSPFVY5iTnbRmhwbwPGCKkY/+iVpRdX4e7qwLaWIFzYfZ9QfvmTcY+vZnlLUdM3261+9M4styYUk5VUQu/KzpvZT0iTgzWqSSAn5h7CGjaWwsp5wP4+29wXuWziCp68aR3mtla3Hi/jjJ4eormvkoztn8fw1kzCbTh45eeGoUN68ZRqBXq4MCfaipqGR84YH4+FqJtzPA4tJ8Om+E0z+00byy9vp6O8BtIAPJDK2qrzkunJVe2TXa47vqgrV5LFz7oULHoKbPoept8OUW+En/1Ii3pyoZnWmB7UfBsksrmZ3RhuzoIy+Apa/1blhxoPi4cZP2ZhupbCynsP2nOPQhCZbdpon8pVtAktN37HreIFjtvbMHez6ag3sXUWDb7SaosvixpY8C/lXfKiu78I/quJP9toYlXlq9vczRUpHDLwZ23JsDgHPKHUMbzfYeDifkuoGUguriA/xJirQs0kok/MrcTELYgI9iQ/xpsgQ0/hglR0R7ufByzdM5tZzh3RonqerhUHerkQGejZ51G3h5+GCr7vjZd3eaQmQZbwZPHTxKH5/0TlUG7noj16agJerhSfXHSE6yJPKOisnympxMQuOF1TybVIB46L8iRvkxf0f7OeFb9TAnK+OnHpir2+PFVBmMQbONM9EqSqEhmoq3FWp3TA/tzb2djBneDCermb+7/PDfLz3BHfOjWdYaDsFtFoxPsofcDwszSbBYH8PtiQXUVhZ17mHUDehBXwgkfkjIOCGT1QtiKT1jo4g+9Bke/U5Nx9VInTIeW0fyy7gnoPAq/2hxH9bf5TbXt/VLebb/zFy7B7O0Hkw89dw9VskF1SyuvF8gkUZEw4+rqr7BQ6F+gp8vn2MYunNE6FPQ/R0iirruP6VH3nmuxNw8VMwbL56i7A1i8d2hxdeX6X6D1px3xdZFFXVYzYJEjNLSc6vZMggL3zcLMwc2vJ3GR/iTVSAZ4uOwtggLyxmE/FGnDfYxw0/T4cHPW9kKCG+nSs6NS0uiKmxHXeq2sMt4AjnNNokH+7JJsjLlVHhvtw5N57fX3QO/p4uXDgqlD9cOopB3m68eO1ERoQpcbx07GCkhJSCKqbHBfLny8eQUVzNB7ub1Rdph/LaBm7434+8eageEGqCCXtRs1IVky60hAIQdgoPHMDdxcylYwdzJLeCCdH+/OL8ztd5mTcilEHebswbEdrs9+M4X3KrB3JPogV8IJG5TXmt7n5q2HppumM+RXuNkc4WLPIapIbFh4w85WbZpTUUVtbxyvep/HZ1y6nCNh3O46b//dhiBGFzdqQVc/W/t1JZZ0VKR2x1b2YpV724lawqYMH/A+9gkvMr2e85jZ2es5ld9gkEDaPu0ucAGG7KZpfvfD4/XISUko2H82i0SfZkqOM12iSvJ7cSvNbTmp0Ohvf9ReB1yKUvqQkWgBKphPeihFBSCqvIKavlikmR7H/sIi4brwaP2L085YF7UF5rpaymgeMFlcQb4ZGmZTsddp3huWsm8thlozvcLirAE1ezkgv728CaXZnszijlwYtHYjLCDnfOjWf3Qxfi5WZh2eQofnxgHvEhPk3e6tVTopri9WMj/Zk+JIgVUx21znNbhR9qGxq54ZUf2Z1RQkZRteoPKG+Ey54FV0/V9wJNAp6NIeCdeIA9ccUY9j+6gPd/PrOp07IzXDw2nJ0PzW/x0IwKcDzg7LH+3kB3Yp7NSAl5B9QwYGlTQ4bHLlPf2ecrPPIZzL5b1ZYQ5ransmqPq15Xhe+bUVRZR32jrSkGaf+H/PuGY1TUWblz7lBCfd0pqqznn18lszezlMc+OchFo8MI9nZjtPHPXVPfyLIXVW753sxSKuusFBjZDp/uO0Ftg40Pdmczb2QIo8J9lRcb4sf6wU/wv+0f8Lslt3C01MyH9ffw21n+1ATOJ2dtOnuzyvjCyNk9mlfBsbwKPtyTzTepHlzvBlbPULC4YjtxgJLyWlzMJgK9Wl5jpzHi31/k+jF31JW4j1jMM6+voirVh1+fP5QlEyL4fL+yxS7Gi8aEk15UzY2zYvnfljTOGx7Mt8dU0a6PE7NJL6ri0rHhxj4+LfbtSW6dE8cFI0N44osjHM2tIK2wik2H84kO9GTphJYTFZuaxZDtn6+bHkNDo42JMQG8etMU/vNdKnNHqP6PBxaPwNfDwhf7c5uG+SflVTAk2Jv92WV8c6yAUYN9GRep/jZyy2tVBcY9bzYJNyVqmWYNAqoI8+tYwIUQ+Lh3zzB4+xuKEFrANd2BzaZmy973jioNGjpaTbVlH8oeEKPCIIlvw6y7VAjFP7rNKnbtEjqqxWptQyNX/VuJ7sbfqNBLXpkS3QojHW3dwVwOZpfz2f4cQn1VnPKdHZm8syMTs0nw/X1zCffzYG2iY9LdZ79KZqvRAebv6UKpkY/8wubjPL3hGC9cM5Hk/EqWTohgaKgvLzVM5rNnE/F0NePiNpO4hfMJrWvE9dNM/vz5YXall3BOqA9H8ypY8PdvATh/6Ghs2YLD9cHUVkv8647x8/9sIzbIi5dvnNKV33yzX4gS8HI8qK5vxN3Lh80NCUwfYuY3C1S/wY0zY3n1hzRGGalovu4u3LtQZebcZyztsdmH16r0xlFGZsk5YT64WUyMM7z1nmRSTCCTYgJ5c1s6axNP8NXhfNxcTMyOH3TK+LmdIG83fn/RiKbPKxc5so983F24f9FI0grVoKUjueUseuY7/nH1ePLL1d9PUl4lgZ7qQZpriDz+0aovB5SQewSSWW3Gw8XcImbfG4wM98FsEpw7bBA7UouRUnbq93Km6BDK2UrGViXeJovKk7bPPtJ8ktvx16gC/lk7kcXHSbGFsi+rtMVhntmYxFdHOteh9+I3xzleUMXxgir+vuEYT284Rn2zYkgmAV8eyFVZIkBeeR33LxrBp7+azUvXTaLRJpuyKnallxDo5YqXq5mtKUW4u5jYcM8cZgxxxIhrGlTo5eXvU6msszIs1LuFN1pd38j8kaG4mE34ebpwx9yhbE8txtfDhWd/OqFpu1W3Tue/N59LaeA4vqsdQlajP261+RwvqGJHWjE2WydqwbSF4YFXSg+qjAdYXlktoc1e7x+5dBQ/rLygRYy5NfEh3mz8zXl8eMdMPvv1bBaMUmGCQC9Xvrt3Lpe38oB7kspadR0VdapDeXw3PjzCfN3JKavl8305SAkHT5Q3hc2OF1Q2hW6awiz+MWqGnMYGVaohIIbcslrC/dx7RTybM/ecELbcdwHzRoZSVd/IibLeyUTRAu5kSCmpbejE5KhFqnZFw7hrIX2L8rS9Q1tOFpuwVA19//pxbIXH+bbIlze3pTd9nVNWw983HuPudxLJr1B/kLUNjU0V6uqsjs82m+Tt7RlMiglACPjnV8n866vkFiYtGhPO/uwyApqFJCZEBzA6wo8FCWEMC/Hm030nKK9VZU/HR/kTb3ifYyP8GRbq0/RqPDkmgPkjQ5gzPJid6Woo+cyhQSQM9mNhQhhv3zqNRaPDuGGm43rvOD+eZZMi+fvV4xkW6sM106J5/pqJzBgahMVsIuCXX1M28wEGhccSSikgKa+1kmqvndFVDAGvwJOahkYabZK8ijrCm73eC6EyGDoiPsSbCdEBJAz2axGiCPF1b7He06xcNIIVU6Ob0g7HR3dfXZcwPw8qaq28b3RoJudXNgl4elFVU62V0uoGSqvrsflFqdBgWZYKofhHk1ve8gHZWwghCPNzZ6TRWXuw2SCsnkQLuJPx8vepjHh4HSXNcnHbpOg40uTK7K2TsZrcVT3uqGkt0/bcfeG8eyHla0wNVWy2jWNvpuMPz+4NV9c38ujHB6mut7L4me+4/4P91FkbWfLcD9xjdEzuySwhv6KO62fENA0/tjMsxJtAL1cuHBmKzchAAHA1mxgd4du03aLRYexIK2Hso+tJyq9UAm500I03SoPaO6emDwnivzdM4ZbZatDFbXOGEB/ig7uLmRevm8TMoYN44dpJjI30bzq+q8XEX5eN4zxjQMvjS8eweIyjJrcwmbh/8UhmTRiNm2hghK8K1SQanZ0tOPghbH3u1PfACKFUSE+q6qwUVdbRaJOd6mDrryxICOPPl49hwahQ3F1MjAzvXOpdZ7Cn/mWX1uBqNrErvYTs0hrGRvphk/B9cmHTn+/4P27giywjVTD1Wyg+jgwbS3pRdYsHZG+TMNgPi5Fd1BtoAXcy7F5tUWoivL5ETXTw6iUnz05TnEKFZyR5BPK/+gsA2FM3mLKaBqSUvLEtncLKOlabFlE99nr+4PEAm20TOJZfwZvb0kktrOKLAzkMD/XmnguH8/n+XH76n+2kFFbx+f4c/rExicM55WxJLkRKyRf7c3ExC+aOCOHpq8fx/DUTm0z5+9XjefvWaS3CG7+6IJ73fj4DT1dHrPK284by9FWOUXnjo/yb9rG/qts9cHv7ucMG8frNU/nNhcO749cLgMmYfPfVK6PwdrOwJ7ONPPb3boQvH3CUzW0Da416GFYaMXD7q39feIjdzQOLR/Le7TNxs5i77ZjNfy83zoptGmZ/S7Oc9hFhjgf+V3nGm8vmPwOCw8GLKaysY8bQ9tNaexoPVzMjwn36l4ALIe4RQhwUQhwQQqwSQrgLIQKFEBuEEEnGsntrZGpasn8NPBlHbY3yXk3JG9SMNOtWqnklv/97y+2LU8h3UYMa/mZdxsc+V3ProXG8vyuLI7kVPPzRAe5+J5F7PzrCE+bbeaMkgVnxQUgJD310gKv+vZVtKcVcNj6C2+YMYc7wYI7lVXDusEFU1TfywubjBHq5UlxVz7aUYt7+MYP5I0PxdXchMsCThQlh+LhbMJsEI8N9GRHmy9Bg7yYPKmGw70mdb95uFi6fGMmL105keKg3E6L9OXfYIIaHejfVopgQFUB8iDdTjHUhBHOGB+Ni7kZfxBjpF/bR1fwt4H0+2nPC0XHWmtXXqzoqbVBToQpQVaJi4PYMi1ONEnQWArxcGRPZvVOujQjzZcggL165cTJjjWPHh3izYFQoQ4O98HG3sHSCY2b7zTkuSJNFFbYaOpdP0k2YTYILR4W2d4peYXyUP/uyyk6/76QLdNhVK4SIAH4NjJJS1gghVgPLgVHAJinlE0KIlcBK4L4etXYA0/jRnZgbawkVJWTIUIQ9bztfVZ6rO7KOqdvOw2w2sfaOGUQVp5DqlcCQYC/cLb78OucyQA2G8XBVXtP3yaoz0V506KrJUWxJVtkeBRV1DAn24mez43Axm3j9ZjXNV73VxqQ/bcBiEjyzfDzXvfwjd7ylBuo8eLEjJ9xkEoyP8ic5v7JpaLKHq5nIADUsPDKg/U67haPDWThaiejoCD/W3+MYTBQd5NmU4dJj2Kc6qy5kvm09Nttl/PmLwzyzfAJPfHGEspp6/ogFF6xw5FNV37y2XIWkmtFQnEGODMSGiZqGRooqVdgrtINRggOVQC9Xvvrd+QAczVU59ItGh+HuYmbTb1V7VZ2V//tcTVRRVGOj4NJ/EFKTAmOvZsObecwYomq69CXjowJ4c1sGezJLmRTTs35tZ3NtLICHEKIB8AROAPcD5xvfvwZsRgt4l8gtq6Wkur6pmllRZZ0R8/M/aVtzo/Legigng1Bcy43OxtpSANxKjzO+fhff2MZx4OgRoqy17K8NIj7CmzERfhzKUfHYxMxSPFxavvbmGalak2ICePqqcUyMDmB/dhmjI/xwb7Wtq8XEP1dMwNfdwrhIf9xdTJRUN/DopaNOEuX7F42koFX95/hgbzKLa06ZddHneDsKOplrS7hnZAV/O5hLdb2VtYnZFFfV8zuzB0HCGCYvbWrkZ/y8FocxFaeQZlPHqqpr5JtjBUQHehLsrQW8I4aHevP40tFc3KyPAsDLzcI/rh6Pn4cLN726gy2ec1k66zoaGm2kFBxvmoWnL7lwpBqp+cjHB/jojllYuvPtsBUdHllKmQ08BWQAOUCZlHI9ECqlzDG2yQFC2tpfCHGbEGKnEGJnQYETziLeg1z1760seuY7GoxMjvlPf8NPnt1y8obN4qyDhIqr+lQ5skU452LyLYN5xO1tPE0NeG9+BIDvy8OID/HmknGDcbWYmDEkiIziar46ms/YSD983CxNBY08Xc0M9vPg8omRxA7y4tJxg5vKjbZm7jkhTIoJxGI2MXPoIKbGBXLdjNiTths12Lepw9DOlLhA4gZ5tVs8qV9gaebBCRML3fZT22Dj3R2Z5JTVUmdtxA8Vymrwi1UeeBvTu7lXpJMm1et8blkNPxwvZNHosF5PcXNGhBBcMy2mTW96yYQI5gwPxsvV3NTBnFNai022HNLeV/h5urBy0QgOZJf3eCy8QwE3YtuXAXHAYMBLCNF20K8NpJQvSSknSyknBwd3ovLcWc67OzL41yaV4pdh1JR4+KMDPPDhfkqMASr2nOEtyYVc+q/v+cMrHzbt//ySKIYFmPFtaPYwDBnB37iGITKT/3o8y7n13/PnhhXslsOICfIkbpAX+x5ZwF3zhwEqPDL3nBB2PDSfX89TbUODvU8rHe2l6ybx1i3T2qzg1hY/nzOUDfd0frLePiN4hBr6HjOLyLQPGOxp4/99qmY196IWi7Dxt4YrOXDJx6o8QdqWlnU8astwry8mTSqP8NP9OTQ0Si4a3fce4tmA2SQYE+nXJJD2HPGoU4TmehP7wKzCyjqySqqZ+9Rmth4v6mCvrtMZ334+kCqlLJBSNgAfADOBPCFEOICxPHUZMQ2gSoW+9G0KNpskxMcxEvHt7RlN29iHjL+/O4v92WWUpTiKQbnUFDHaw4hTS9XRc7wxhPcqx1HlFsLMxh0ct4XzoccVXDMthgtHKcFwdzEzMTqAqyZHsmh0GEsmRODuYj6prkZXsZhNXepANJlEj75Sdht3bFNFv+Y+iKg4wV8Hf4O9TyrCXYWz8gggv95NlcVN/x42PAxZu+C7p9V0cUCZRxRuFhMpBVWYTYLxbYTHNKfH+KgADuWUU9vQ2FRgq7+E5uzlF4qq6vnyYB6phVU9kt7YmRh4BjBdCOEJ1ADzgJ1AFXAD8ISxXNvt1p1lSCk5XlBFZZ2V5ILKpiHhPu4Whof6sMuoB11YWUfsIC8SM0uZOTSIGZlHqZAeeFgElqp8JpnVH+vmxnEss3zLfV9XYcNE1cir8Ur8F2saz+OCkaE8vnRMi/O7WkwnFc8fFuKNm8VEwuCWHXADHnuYI2YGDL2AGaWbgblE+HswO6gCsqFMeqmH7Zx7oTQTfnhWzQ9alNSUyeI1+By8si3UWesJ8nLt1UE3Zzvjo/xpaJQcPFFOZkk1ZpPo0xzw5gR4qRBhcWU93yYVMCLMh9h2QpJnQocCLqXcLoRYA+wGrMAe4CXAG1gthPgZSuSXdbt1/ZxXt6QyMtyXaUM6l3eaW17bNEXVV0fyqW+08dsLh3P5pEj8PVz4PrmQ29/YRUFFHWXVDcQUfc+jli34uhxjS/1oLvApgsOfsqIij0O2GJ5tXIJEEDBsGh/OTyAkYBIZBam8c/x87u3kEGcfdxfW3zPnrEht6zFiz8W06TF2/2YCjR5BeGRtgXehHEPATSYS425hfOKbUJREVuB0Ikp2IIDBcSPwLMinuEqVfdV0H/axAYmZpWQW1zDY373fvN25Wcx4u1k4mlfBzvQS7jJCld1Np7JQpJSPAI+0aq5DeeMDEiklj36iYqJbVl7A4E7UX2hepcxeEW9YqDcRxlDqCcZow4LKOvZmlfJT8yZiSnYDkBd6PS6eWyFjK43CjavrH+ay6SP4IH8Uz1w5oWkQhNvVLzPy3UQuGNFmn3KbxAR1v2dwVmHUPg8s2Qshi0Cq+yjd/Sk0smxu/6yIvzeOYorpKEtO3MDN56wgM3kfS2MH45moQl6DdPZJtxLm5050oCffJRVQVtPQb+LfdgK9XPn2WAFSqgFnPUH/eFw5IVXNaljPeuIrHv/scIf72KfOGhnuy16j86X56LMgLzdMAgor6khMzWWW6SBSqDS+G667BWFMP5bjNYoKPFk6IZJ3bpvR4hihvu68fev0s2K0X78hYiKYXByZJsZkumavQAoq6pBSUlXXyB+sN/Ir6134DhrMX44G856cz5gIv6bRptoD734Wjg5jS3IhyXmVRPeT+LedQC9Xyo3iX9GBPeMk6XKyp4m9o3HF1GiO5VXwUWI29y8eidkk+OF4ITvTSjhveHCL0YbJBZX4ebhwxcQI/vSZystuHrowmwSBXq4UVNZhStmMp6iDJS+Bdwj4RYCr6mgcPHYu/4ue0uODBDQGLh4weAIc+RzmPtSUe+/uE0hhZR3pRdVU1lm5eN5cZg8bREyQJ5/vyyF2kBcerma83NRDWHvg3c9FCWG89G0KjTYry6d2oZZ9L2DvyPRwMTPIu2cGF2kP/DSxvzovHhPGjTNjKaysb+qEvP+D/Ty94Rj3rmk5Ldfu9BJGhvtwUbPBBk03tiwb0rcyyNuNzNRj3Fz+PPVmLxh5qZpAGKAsEwCXqEnM7UKIRNMNzPyVKr27+1WoKQVhJiQoiKT8Sn5MU0PmL0oIY0psICE+7tw4K47zz1H3SHvgPceEKH8SBqvp3LqztG13YBfwyACPHsv91x74aWL3wAd5uzEhOgBXi4kvD+YyNtKPjOJqfN1VB0ZqYRXv7sgkOb9S1SC5ZFSLVCdLwSE1mULiW5CymYjQ1Vya8TKhphJKlqwi1LXZa+H4a1Tdk+jpvX25mpGXqrzwrx5XD1SPAOaNDGPVjixe/OY4Hi5mhoe2nYrp5Wr3wPt2iPfZiMkk+OzX5/a1GW0SZAh4T6Y2agE/DZLyKkgxJi4N9nHD283CmAg/DmSXkVJQhZTwywvi+b/Pj/Dhnmxe+yGNmoZGzjPtZalrBRDH+7+YoWYbee9SlXYmTCBtjK74nkWmHWzxW8wFYy5oeeLxK2Dc8s7N5K7pXoSAi/4P/j0HDrwPQfHMHjYIL1czKQVVXDMtut0MCA/tgQ9I7HXvowJ6LsNLC3gXqbfaWPLcFqrqGzEJCDCG+sYHe7PpSF7TjNRzhgdj2/Eyqd9+R03DuQzydmOl+JTArdUwdTmT7DWzvzHEWNrA7Mo95X8BAWMuuaNtA7R49x3hY2H+I2qGoxGX4O5iZuHocH44Xsh9zaYIa43dA9c1UAYWgdoD738czilvykAJ8nZrGkIeH+LNuzsz2ZFajElA3CAvYsTnHBQufOl5AV//7jy8niuB0jw1BZR97sn6anD1gWm3q/n9Mn6gYdgigodNa88ETV8y+x71Y/B/l4+m3mo75eS4nm7aAx+IBDXFwLWA9xuaF6fxaTZxarwR//zyYC4xQV64mQRUZTPMK4y3Qz/BZ/0nqm4xUk3/dHgtnNij2mbfA/Mehpy9kPINLtN/oT1tJ8HNYu5wUoMFo0KpqrP27wJemm5nckwgl0+IaDGPa3ejBbyLNBfwgnJHqVT71F/5FXUsiPKHylxorMdPluFXsxv2HAaMYhrFKZDyDaR+o9b91MQLhI9TP5qzitERfoyO6N7JDzT9Hz9PF56+enyPnkOnEXaRxMzSptzuCmNYPNA0mhLU/IyUGsWpasvUpKuy2dRbxcehItexbhdwjUaj6QLaA+8CmcXVpBZW8eDikfi6W1g6IaLpO5NJcPXkKEJ93ZgcGwh7m9XrNgZ+ACrbpOi4EU4x8HUcR6PRaDqLFvBTUNvQyOJ/fsfvFpzD4jHhTVOPLUgI5dY5Q1pufPQLnpwRChFj1XppBidhcoGQEZC7H+rKHe3aA9doNKeBDqG0wddH8qmpb+TgCZXXveFQHgDrDuQyMty37eJPq5bDf+ZCQ41aL007eRu/SDVRQPZOR5ub70lzKWo0Gk1n0ALeipyyGm56dQe/WrWHPcZ0TYmZpaq+SXoJl44LP3kna7N5H7c+p9IEc/eDW7OOq4TLYcI1EDYGbEbsfNwKmHh9z12MRqM5q9EhlFZUGtXDNh7Ow91FPd9SC6v4/Xv7iAny5OZZcSfvVJ7t+Lz/PSg4olICz1sJ3zyh2i98DPyj4fjXjm3P/S0M6pk6wRqN5uxHe+CtaJ5ZkphZ2jT4Iru0hj9fPuakWdoBVYgKYNgCJd7716jiR+fdCxj53N5qctsWaYI+en5EjUZz+mgBb4XdAwfIKqnhuukxuLuYWDE1iplDWxVlryxQ4RO7Bz7pJuMLCVNuBZMZPALA3R8sxig8z0DwjVSjL918evx6NBrN2YsOobSispkHDvCz2XFcMSmS0NbDoKVUhY2ipqi4NqgqdYFDwD8GAmJUm9cglTrYnKgpajCPRqPRnAFawFthF/DREb48fPEovNwseLm18Wsqy4SKE3Borcrr9gxShf9v+FQt7Qwarjzx5lz8NFhre/AqNBrNQEALeCvsIZQ3fzYNf89T1G/O3a+WFg/IOwBhRv63X6tBOVf89+R9PQO7wVKNRjPQ0THwVtg98Da97ubk7FOhkYufUuvtDcZx8WjpkWs0Gk03oT3wVlTVWXGzmHBppzh/E7n7IGiYmiUnYxvEzu4dAzUajcZAC3grKuqsLcrEtkmjFbJ3K9EWAi57tneM02g0mmZ0GEIRQpwjhEhs9lMuhLhbCBEohNgghEgylmfFFOmVtVa8Owqf7PqfKhebsLR3jNJoNJo26FDApZRHpZTjpZTjgUlANfAhsBLYJKUcBmwy1p2eyjor3qfywKWEb/6iJrgdcXHvGabRaDSt6Gon5jzguJQyHbgMeM1ofw1Y0o129So19Y3c/sZODp4oUwJ+Kg+8qhCq8pV461lzNBpNH9LVGPhyYJXxOVRKmQMgpcwRQoS0tYMQ4jbgNoDo6OjTtbNHOZxbzpcH8/jyYB7DQryJCTrFHHbFx9UycGjvGKfRaDTt0GkPXAjhCvwEeK8rJ5BSviSlnCylnBwcHNxV+3qFvDLHoJqk/Mr2PfC970DyJvU5SAu4RqPpW7rigS8Cdksp84z1PCFEuOF9hwP53W9ez9LQaOOfm5KaZpa302YMvK4SPrxdfRZmVVlQo9Fo+pCuxMBX4AifAHwM3GB8vgFY211G9Rb7skr511fJvL41HVeLiZlD1ezR3m5tzB7evHaJfzSY9QzjGo2mb+mUgAshPIELgQ+aNT8BXCiESDK+e6L7zetZCirURAzFVfWE+bo3zRxuMbXROdlcwHX4RKPR9AM6FUKRUlYDQa3ailBZKU5LQWV90+cwX3fiQ7wBSC2qOnlje+clAoLie8E6jUajOTUDeiSm3QMHCPVz5/xzVCfr5RPamCW+KAW8w+CSpx2FqzQajaYP0QJuEO7nToiPO2lPtDM4p/i4qvWtB+9oNJp+woCuRlhY2cwD93Vve6OkDVBTomLgQUN6yTKNRqPpmAHvgccEeZJdUsM5oW1Mb1ZVBG9dqaZKq8xT1Qc1Go2mnzBgPPDS6nruXbOX3LJafv/eXkqq6imsrGNidAA/PjifWfFBJ+9UlKyWiW+rZdTU3jNYo9FoOmDAeOA/phazemcWVXWNfLY/Bw9XMwUVdQT7uBHo1c7MO/bUwcY6MLnA4Am9Z7BGo9F0wIDxwO0z7VQYy72ZpdRZbQzyPsW0aU2pg0D4OD2zjkaj6VcMGA+8yhDulIJKAPZmlQEQ3Hq2eYDaMrDWqcmK3fygrgyip/earRqNRtMZBoyA2z3v7NKaFu0jwnxbblhVCH9PcMwaP3QeTLweomf0hpkajUbTaQZOCMWYbV5KR9v8kaGMDG8l4Ok/KPEOiFPrbj6QsAR8QnvHUI1Go+kkA0bA7SEUgPgQbx5fOpp/rhh/8oaZ28HsBle/odaHnNc7Bmo0Gk0XGXAhFIAQHzeumRbj+NJaDz/+W80wn7ldZZuEjYH70sHNt42jaTQaTd8zYATcHkIBHGmD9dWw+c9QXQyJb6r494lEmHGH+t7Dv9ft1Gg0ms4ycAS8rg0BP/YF/PBPx0Z7V4GtAWJm97J1Go1G03UGZAy8ScCTNqrlhGshcqoaLm92g1gt4BqNpv8zYAS8orWA22yQvAFGXwmXPQdjr1Jfxs4G11NMaqzRaDT9hAEj4JW1Vtxd1OUGerlC3gGoKoBhF6oNYmap5fCL+shCjUaj6RoDRsCr6qwMDVYz7oT6ukNOovoicopaho6C6z9WlQc1Go3GCRgQnZiNNklVfSPzRoby+4vOYXJMABzcB64+jgE7oHO+NRqNUzEgBLyqXsW/fd0tnH9OiGrM3adyvU0D5iVEo9GcZQwI9bLngHu7WaCuAtY9oAbshOu5LTUajfNyVgt4Q6ONrceLmnLAvd0tcOQz2Pac2kBPTqzRaJyYszqE8tGebH6/Zh+XjA0HDA88abv6cvLNcM6iPrROo9FozoxOCbgQwh/4LzAakMDNwFHgXSAWSAOuklKW9ISRXSU5v4J/bExqqvX96b4cwBDwjO2qROwlf+9LEzUajeaM6awH/gywTkp5pRDCFfAEHgA2SSmfEEKsBFYC9/WQnV3i9a3pfLovhwBPF1wtJuaNCMEkBCMCJOQfglGX9bWJGo1Gc8Z0KOBCCF9gDnAjgJSyHqgXQlwGnG9s9hqwmX4g4Dab5MuDuQCUVDewMCGMF66dpL7c8yYgIUZPzqDRaJyfznRiDgEKgP8JIfYIIf4rhPACQqWUOQDGMqStnYUQtwkhdgohdhYUFHSb4e2RmFVKXnld03p8iBq8Q205fPUniJiki1VpNJqzgs4IuAWYCLwgpZwAVKHCJZ1CSvmSlHKylHJycHDwaZrZeXanqzD84nh3fmb+jPhgDyg/Aa8uhsp8WPikzv3WaDRnBZ1RsiwgS0pppG+wBiXoeUKIcABjmd8zJnaNrJIavN0s3OC7k4dd3mJy5dfw3/lQnArXrIaoKX1tokaj0XQLHQq4lDIXyBRCnGM0zQMOAR8DNxhtNwBre8TCLhKQ9RXPuD7PJJcMACK2/wnKs+HGTyF+fh9bp9FoNN1HZ7NQfgW8ZWSgpAA3ocR/tRDiZ0AGsKxnTOwCR9dxV/5D6vMhLwBEVb4asDN4Qh8aptFoNN1PpwRcSpkITG7jq3ndas2Z0FCL/OJecmUQ4aIIGqoc39lLxmo0Gs1ZxNnTm7f/PURpOr9ruI0S73jVFpKglsN0jW+NRnP2cfYMpc/dR6OLD1tqR1MVkkpAZTJc8BC4eUP0tL62TqPRaLqds8YDr84+SLIcDAjEsAvV3JYREyFuTl+bptFoND3C2eOBFxxlb20Cs+KDCJ4yFSZeBG4+fW2VRqPR9Bhnh4BXF+NZX0iSjOC1m6ZiMZvAosVbo9Gc3ZwdIZTCYwCkiSgl3hqNRjMAODvUruAIAFmW6D42RKPRaHqPsyOEUnCUeuFOmUtoX1ui0Wg0vcZZIuBHyHGNxt3Vpa8t0Wg0ml7jLAmhHCXbEo2nq7mvLdFoNJpew/kFvLYcyrNJN0Xi6XJ2vFBoNBpNZ3BuAS/Lhs9+A0AKkXhoD1yj0QwgnFvAtzwD+98DIMkWqUMoGo1mQOHcAl6SppYTruO4dZD2wDUazYDCuQW84AiMvgIue5YqK9oD12g0AwrnFfD6KijNgOARAFTXW/Fw0QKu0WgGDs4r4IVJgITgc7DZJLUNNjxcdRaKRqMZODivgBccVcvgEdQ0NAI6hKLRaAYWzivg+YewCRcuej2LiloroAVco9EMLJxXwHP3kSQjOVpQy9G8CgAdA9doNAMK5xRwKSFnH2muQwE4kF0GgKeOgWs0mgGEcwp4RQ5UF5LnORyAfVmlgA6haDSagYVzCnjufgBSzMoD35+lPHA9kEej0QwkOhVzEEKkARVAI2CVUk4WQgQC7wKxQBpwlZSypGfMbEXeAQD2NUYBVk6U1QLaA9doNAOLrnjgc6WU46WUk431lcAmKeUwYJOx3jvUloHFndyals8fLeAajWYgcSYhlMuA14zPrwFLztiazmKtA7MbJdUNRPh7NDXrgTwajWYg0VkBl8B6IcQuIcRtRluolDIHwFiGtLWjEOI2IcROIcTOgoKCM7cYwFqHtLhR09DIiqlRTc3eWsA1Gs0AorOKN0tKeUIIEQJsEEIc6ewJpJQvAS8BTJ48WZ6GjSdjrcNmdgPA39OVXQ/NZ1tKMX6eeko1jUYzcOiUBy6lPGEs84EPgalAnhAiHMBY5veUkSdhraXR5AqAv6cLQd5uXDw2vNdOr9FoNP2BDgVcCOElhPCxfwYWAAeAj4EbjM1uANb2lJEnYa2jQShvO8DTtddOq9FoNP2JzoRQQoEPhRD27d+WUq4TQuwAVgshfgZkAMt6zsyWNDbUUtWoTPfz0GETjUYzMOlQwKWUKcC4NtqLgHk9YVRHZBWUkFumClj567i3RqMZoDjnSExrHXVSCbe/DqFoNJoBilMKuCsN1OHCFRMj8dKDdzQazQDFKROnTbZ6sLjxt6tOiuxoNBrNgMEpPXCLrQ6rcOtrMzQajaZPcVIBr6fRpDsvNRrNwMYpBdwsG7CadOelRqMZ2DilgLvIehpNOoSi0WgGNs4n4FLiIuux6RCKRqMZ4DifgNusmLA1FbPSaDSagYrzCbi1DgCb2b2PDdFoNJq+xYkFXHvgGo1mYOOEAq7mv8SsY+AajWZg47QCLnUIRaPRDHCcT8Ab6wGQFh1C0Wg0AxvnE3DDAxdawDUazQDHCQVcdWJi0SEUjUYzsHFaARdawDUazQDH6QRc2kMoLlrANRrNwMbpBNxaXwOAsOhiVhqNZmDjdALeWK88cJP2wDUazQDH6QTcqgVco9FoACcUcLsHbnb16GNLNBqNpm9xOgG3GTFws6v2wDUazcCm0wIuhDALIfYIIT411gOFEBuEEEnGMqDnzHRga9AeuEaj0UDXPPC7gMPN1lcCm6SUw4BNxnqP02gIuEXHwDUazQCnUwIuhIgELgb+26z5MuA14/NrwJJutawdZEMNddIFVxeni/5oNBpNt9JZFfwHcC9ga9YWKqXMATCWIW3tKIS4TQixUwixs6Cg4ExsVdRXUYUbrmbzmR9Lo9FonJgOBVwIcQmQL6XcdTonkFK+JKWcLKWcHBwcfDqHaEl9FdW442rRHrhGoxnYWDqxzSzgJ0KIxYA74CuEeBPIE0KESylzhBDhQH5PGmpHNFRRJbWAazQ9SUNDA1lZWdTW1va1KQMKd3d3IiMjcXHp3IQ1HQq4lPJ+4H4AIcT5wO+klNcKIf4K3AA8YSzXnqbNXcLUoDxwN7MWcI2mp8jKysLHx4fY2FiEEH1tzoBASklRURFZWVnExcV1ap8zUcEngAuFEEnAhcZ6j2NqqKJSe+AaTY9SW1tLUFCQFu9eRAhBUFBQl956OhNCaUJKuRnYbHwuAuZ1Zf/uQHngfrhpAddoehQt3r1PV3/nTqeCZms1VbjjokMoGo1mgONUKphWWIWtrpJq6aZDKBqNZsDjVCr48zd34dZYTSUeWsA1Gk0LvL292/0uLi6Oo0ePtmi7++67+ctf/gLAnj17EELw5ZdfdvqYzfn222+ZOHEiFouFNWvWdNHy06dLMfC+xmptwF00UC3dcdUhFI2mV3jsk4McOlHercccNdiXRy5N6NZjnorly5fzzjvv8MgjjwBgs9lYs2YNW7ZsAWDVqlXMnj2bVatWcdFFF3X5+NHR0bz66qs89dRT3Wp3RziVCg72UANBq3DDxaw7WDSas5n77ruP559/vmn90Ucf5bHHHmPevHlMnDiRMWPGsHZt57KXV6xYwTvvvNO0/u233xIbG0tMTAxSStasWcOrr77K+vXrTyv3PTY2lrFjx2Iy9a6kOpUHXl1Zppa46x5yjaaX6E1PuTnLly/n7rvv5o477gBg9erVrFu3jnvuuQdfX18KCwuZPn06P/nJTzrUA7u47t27l3HjxvHOO++wYsUKALZs2UJcXBxDhw7l/PPP5/PPP+fyyy/v8evrDpzKA6+trgDgj8um9bElGo2mp5kwYQL5+fmcOHGCvXv3EhAQQHh4OA888ABjx45l/vz5ZGdnk5eX16nj2b1wq9XK2rVrWbZsGaDCJ8uXLwfUQ2PVqlU9dk3djdN44DX1jYj6SnADFw/fvjZHo9H0AldeeSVr1qwhNzeX5cuX89Zbb1FQUMCuXbtwcXEhNja20yGPFStWsGDBAs477zzGjh1LSEgIjY2NvP/++3z88cc8/vjjTaMhKyoq8PHx6eGrO3OcxgMvrKzDSxg3ytWrb43RaDS9gr3zcc2aNVx55ZWUlZUREhKCi4sLX3/9Nenp6Z0+1tChQwkKCmLlypVN4ZONGzcybtw4MjMzSUtLIz09nSuuuIKPPvqoh66oe3EaAS/Oz2KG6ZBa0QKu0QwIEhISqKioICIigvDwcK655hp27tzJ5MmTeeuttxgxYkSXjrdixQqOHDnC0qVLARU+sX+2c8UVV/D2228DUF1dTWRkZNPP008/3eZxd+zYQWRkJO+99x633347CQm9028gpJS9ciKAyZMny507d57Wvulv3EnM8TfVyp0/QvA53WiZRqNpzuHDhxk5cmRfmzEgaet3L4TYJaWc3Hpbp/HAXcpSHCuunUuu12g0mrMZp+nEdK/McqzoEIpGo2mD/fv3c91117Voc3NzY/v27d1y/Mcff5z33nuvRduyZct48MEHu+X4XcU5BNxmw7cux7GuBVyj0bTBmDFjSExM7LHjP/jgg30m1m3hHCGUyjwsssGxbu7cbBUajUZzNuMcAl6qUoX+5/NzWPZaHxuj0Wg0/QMnEfAMANL8p0LCkr61RaPRaPoJziHgJcoDt/nF9LEhGo1G039wCgGXJWnkS3/8nGBoq0aj6R5KS0tbVCPsLIsXL6a0tLRL+7z66qtNozPtFBYWEhwcTF1dHQCXXXYZM2bMaLHNo48+2ukSsjfffDMhISGMHj26S7adCqfIQim74Al+sm0at3q59rUpGs3A44uVkLu/e48ZNgYWnXoedLuA26sR2mlsbMRsNre73+eff95lcy6//HJ+97vfUV1djaenJwBr1qzhJz/5CW5ubpSWlrJ79268vb1JTU3t9Kzxzbnxxhv55S9/yfXXX9/lfdvDKTzwojpBLkEEaQHXaAYMK1eu5Pjx44wfP54pU6Ywd+5cfvrTnzJmzBgAlixZwqRJk0hISOCll15q2i82NpbCwkLS0tIYOXIkt956KwkJCSxYsICampo2z+Xr68ucOXP45JNPmtqal5x9//33ufTSS5tqs5wOc+bMITAw8LT2bRcpZa/9TJo0SZ4OP6YWyZj7PpWbj+af1v4ajaZrHDp0qK9NkKmpqTIhIUFKKeXXX38tPT09ZUpKStP3RUVFUkopq6urZUJCgiwsLJRSShkTEyMLCgpkamqqNJvNcs+ePVJKKZctWybfeOONds+3evVquWTJEimllNnZ2TI8PFxarVYppZTz5s2T3377rTx69KgcM2ZM0z6PPPKI/Otf/3pa19Qebf3ugZ2yDU11Cg+8uKoeQHvgGs0AZurUqS1CF//85z8ZN24c06dPJzMzk6SkpJP2iYuLY/z48QBMmjSJtLS0do9/ySWX8P3331NeXs7q1au58sorMZvN5OXlkZyczOzZsxk+fDgWi4UDBw509+WdFh0KuBDCXQjxoxBirxDioBDiMaM9UAixQQiRZCwDespIu4AHagHXaAYsXl6OEdibN29m48aNbN26lb179zJhwoQ264K7ubk1fTabzVit1naP7+HhwcKFC/nwww9bhE/effddSkpKiIuLIzY2lrS0tNMOo3Q3nfHA64ALpJTjgPHAQiHEdGAlsElKOQzYZKz3CFrANZqBh4+PDxUVFW1+V1ZWRkBAAJ6enhw5coRt27Z1yzlXrFjB008/TV5eHtOnTwdUydl169aRlpZGWloau3btch4BN0Iwlcaqi/EjgcsA+7DI14AlPWEgKAH3dDXj7tJ+z7NGozm7CAoKYtasWYwePZrf//73Lb5buHAhVquVsWPH8vDDDzeJ7ZmyYMECTpw4wdVXX40QgrS0NDIyMlocPy4uDl9f36YCWX/6059a1AxvjxUrVjBjxgyOHj1KZGQkL7/88hnb26l64EIIM7ALiAeek1LeJ4QolVL6N9umREp5UhhFCHEbcBtAdHT0pK7MoGHnnR8z2JNRypNXju3yvhqNpuvoeuB9R7fXA5dSNkopxwORwFQhRKcz0aWUL0kpJ0spJwcHB3d2txYsnxqtxVuj0Wha0aWBPFLKUiHEZmAhkCeECJdS5gghwoH8njBQo9FoupM777yTLVu2tGi76667uOmmm8742EVFRcybN++k9k2bNhEUFHTGx29NhwIuhAgGGgzx9gDmA08CHwM3AE8Yy7Xdbp1Go+kzpJQIIfrajG7nueee67FjBwUFnVE98s6EtJvTGQ88HHjNiIObgNVSyk+FEFuB1UKInwEZwLKuGqvRaPon7u7uFBUVERQUdFaKeH9ESklRURHu7u6d3qdDAZdS7gMmtNFeBJz8rqDRaJyeyMhIsrKyKCgo6GtTBhTu7u6nzGRpjVMUs9JoNL2Li4vLaRVs0vQuTjGUXqPRaDQnowVco9FonBQt4BqNRuOkdGokZredTIgCoOtDMRWDgMJuNKcv0dfSP9HX0j/R1wIxUsqTRkL2qoCfCUKInW0NJXVG9LX0T/S19E/0tbSPDqFoNBqNk6IFXKPRaJwUZxLwlzrexGnQ19I/0dfSP9HX0g5OEwPXaDQaTUucyQPXaDQaTTO0gGs0Go2T4hQCLoRYKIQ4KoRIFkL02NybPYUQIk0IsV8IkSiE2Gm09dqk0GeCEOIVIUS+EOJAs7Z2bRdC3G/cp6NCiIv6xuqTaec6HhVCZBv3JVEIsbjZd/3yOgCEEFFCiK+FEIeNicbvMtqd8b60dy1Od29OZwL4M74WKWW//gHMwHFgCOAK7AVG9bVdXbyGNGBQq7a/ACuNzyuBJ/vaznZsnwNMBA50ZDswyrg/bkCccd/MfX0Np7iOR4HftbFtv70Ow75wYKLx2Qc4ZtjsjPelvWtxunsDCMDb+OwCbAem9+R9cQYPfCqQLKVMkVLWA++gJlR2dnptUugzQUr5LVDcqrk92y8D3pFS1kkpU4Fk1P3rc9q5jvbot9cBIKXMkVLuNj5XAIeBCJzzvrR3Le3Rn69Fyq5NAH/G1+IMAh4BZDZbz+LUN7g/IoH1QohdxiTPAKFSyhxQf8RASJ9Z13Xas90Z79UvhRD7jBCL/dXWaa5DCBGLqte/HSe/L62uBZzw3gghzEKIRNQUkxuklD16X5xBwNuaDsTZch9nSSknAouAO4UQc/raoB7C2e7VC8BQYDyQA/zNaHeK6xBCeAPvA3dLKctPtWkbbf3qetq4Fqe8N7JrE8Cf8bU4g4BnAVHN1iOBE31ky2khpTxhLPOBD1GvSXnGZNA44aTQ7dnuVPdKSpln/MPZgP/geH3t99chhHBBCd5bUsoPjGanvC9tXYsz3xtQE8ADm2k2ATx0/31xBgHfAQwTQsQJIVyB5agJlZ0CIYSXEMLH/hlYABzAMSk0ON+k0O3Z/jGwXAjhJoSIA4YBP/aBfZ3C/k9lsBR1X6CfX4dQk1S+DByWUj7d7Cunuy/tXYsz3hshRLAQwt/4bJ8A/gg9eV/6uue2k727i1G908eBB/vani7aPgTV07wXOGi3HwgCNgFJxjKwr21tx/5VqFfYBpTH8LNT2Q48aNyno8Civra/g+t4A9gP7DP+mcL7+3UYts1GvWrvAxKNn8VOel/auxanuzfAWGCPYfMB4A9Ge4/dFz2UXqPRaJwUZwihaDQajaYNtIBrNBqNk6IFXKPRaJwULeAajUbjpGgB12g0GidFC7hGo9E4KVrANRqNxkn5/2KWCnlxoOJmAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_VAL_1','train_VAL_1']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABFaklEQVR4nO3dd3hUVf7H8ffJpFdCCi1AQocAoYSigIBYABUQUMCydn/uWtd1V1bXRVdde8G1Yi8IIoiConSkqPQaegkQAum9Z+b8/jiTkEACAZJMZvi+nidPpty5c24ufO655557jtJaI4QQwvm5OboAQgghaocEuhBCuAgJdCGEcBES6EII4SIk0IUQwkW4O+qLQ0NDdWRkpKO+XgghnNLGjRtTtdZhVb3nsECPjIxkw4YNjvp6IYRwSkqpw9W9J00uQgjhIiTQhRDCRUigCyGEi5BAF0IIFyGBLoQQLkICXQghXIQEuhBCuAgJdCGEqE5RLmz4FKylji5JjUigCyGck80GpcV1+x0bPoEfH4GtX9ft99QSCXQhhHOadSs8FwYL/n7m5eLXwOJ/Q9qBc/+Ond+b3ytfBWvJuX++nkmgCyHObvtseLUD5KWZ5wUZsOgp+O5e0yxRpjDLhOf+pVWvp6QAtn0Lm7+CXT+a5wA2K/z8uAlfmxXe6Q8rXqq+PDYrHFhuHu9fcuayL3ka1kyFT4afW9NJ5hE4thGiLoPMw7DiBdg5z5wZ1ERJAdTzjHAS6EJcjKwlcHQdpB+q+n2tTwaX1rDqNchNgq0zTJj+8gT89hZs+wb2/nLyc6teM+H51VjYZw/aEzvMugqz4N3+8N3d8MP98M3NMP0GE3xbpsPa92HhE3BwOaTsMusvyKi6fKl7oSQPAiMg43D1teecJEhYD6EdIS8ZEjfX/G+09Rvz+7qp0HmU2bZZt0Lcd+bAsPFzKM6r+rMZh+GVdrDh45p/Xy2QQBfC1a1+w9R+SwqgON8E9De3wMdXmp+q2qHXfwQvRZrf66ZB8k5w94Flz8KzoaZN+dKHwDfkZKAnxcHaaRB9Pfg0hm0zTY37/QEm+A8sg4x4GPshPLzVBGX8ahPqi6eAZwAc3wILnwQPXyjOhd/fPb1sWsOxTeZx9xtAW02AVmXPT4CGa14DFHx7O0yNgek3njw7qIq1xGx722HQuA1c+yYMegwCW5iDz+4fYf5D5iylKr++ZMr/68tn/p5aJoEuhDPQGta8Bce3mueHf4OsY9Uv+8sTsOQZE0yr3zC135fbwMdXwZ6fTQi3uxLyUk5vstAa1n1oAumnv8HP/4DmvWDUW2Dxgl5/gu4TYfA/oP1VsP1b+OhK+GAwePnDFU9D5+tg9wL47X9mnXHfmRq7dxBEj4XgSOh9uwna+FXgFQB3/mzCM2U39LjZLLfmTUjedbJsNht8di388BfzvMNw8zttf9V/iwPLIKgVRA4E38aQnQCe/rBvoWlCqajigW3fIsg9Af3+zzz3C4FhT5lyHVhuwh5MDTxhY+X1HP7dnMm0HmDOarZ9U3XZ6oDDhs8VQlQjPx0+GgZdRsOwKaCUaXNe/JQJp9vnwxejod0VMGmGCRS/EBOSa6fB/sUmkADS9pmmjlaXQkm+qQHPucs0QUz4Et7sBhs/hQ5Xg3IDazGk7IHUPTDyVWjRG4qyIfIycHOD7jdWLmv0WBNehVkm/AY8Av5hZrlNn8Pen02AHlhuQrvt5WCpEDt97oKm3SG0Pfg0gr/8Aan7TLAX55nml6XPwiR7L5Pt38Lh1eZxsxgI7WDfzmoC/dgmaHWJ+RsO+aepXf/pB3uT0dvQ/y8Q0NS0+2/6Eu5fZ8p/fBugIGpw5fX1vs0cpOJXQceRZru2fAURvSE3xTQpFWVDo9Zm37w/EPYtNgeveiCBLkRDs20WpB80NWuLJwz6Gyx6EsI6mbD9ZIQJ3n2LYOtM+P4vpmZ807ew9D9QWmAOBm7usGOOWcfN34K7N7zdG7IS4Pr3wcMHYu+CX1+ED4eCT7BpYw5qaT4TPdYcKM6kw1Xw2D7wCzOhWSZyIEyaCXsWQIcRMHMSFKSbx6dq2efkY3cvaNrVPPb0NWcDv70NOSfAL9zUqpvFmGYbTz9T6/YJhvQqerBkH4fsY+agBND3HvMDMPCvJog3f2UOCmummtd3zIH+95k2+uDW4OFdeZ1BEebMZOkz0G28+RvGzYXhL5laf34qdLoWhj5pzkbaDIG4H0ybu6Xu41YCXQhHKswyp/qr34CdP5jT+k2fQ/Oepha98hWweJjlrp9mmlxW/NeEbtZRmPt/0KQbFGaa0CzOgfGfQtexUFIIhdng38QEPsCE6SZ0WvQyz4dMhrAOpt06eZcJt4x4s46zhXkZ//CqX+84wvwA3LnQHCSa9Ti3v0+v20zY/vqSaV7JOATjPoawjieXCWlnDnSnSrS3s5cFekWh7SBykPlbezcy2+3uZc42ygK9rPZ/qksfMu91HAEefuYgcGCZabrybwoTvjp5cGszBDZ9Yc6MImLNAVPboElX8321TAJduJ68VHOaf2rtqqHJSjBNHtoGymIuuM21t9le/4E5pT+4HJY9Z8IwapBps/bwMV3p1n9kaqlD/mmCafG/zWcjB5rfHt5wy+zKXefKar9llIKu46D91aZHSWAL0zRTdgCoLa36n9/nQtpC7zvMDT6bvjBnAp1HVV6mZT/T5l9SYP42ZY5tMn/Xpt2qXvfAv8JX44Ajpi3fWgK/TIaknaYJp82Qqj9ncYfO15rHbS83ZwhbZ8DBFaZ2XvFMJWowoEzYh7Y31zCsxdDvPhhxhm6Z50kuigrnlJdmLpCVFFbu2laQAW/HwoLHzvz5zdNNQDhS+kET5i37w32r4d4V0GWMqYHGTATvQLjkfrNs60tNeLu5wYCHoHkPGP22CQWfRtDtRtMGHtbp9BpzxYCpjpc/NGpp1l/bYX6hrn0DbvzCHOCueAbcPSu/32YIWIvgyO+VXz+2EZp0MU03VWk3DK5+3rThd58IXcebA8DKV6C0sPoaekXunqZXz87vzVlSp5GV3/cLNQec3T+aHj3WYhj6L4iZVMONPzcS6KLh0xp2fAdFOeZ59nF4syusfBne6GK61824yYT5Gnvf5a0zzXJljqyF1P0n17fkaZj3kDlVzj4Oc+4xXdoKMk3Qpu6r++3KSzW/r33DBI9fCNz4uWmbLRN7p7lA2H3CmdcV2MzUOMsOAK5EKXNNYMKX0PPm099vfSm4eZy80QjMwT5xU9XNLRVdcj/ct8ocxPzDzIXmuO/MezUJdDi5b3rcYg46p+p8LZzYDhs/M90xyw7IdUACXThO6n6YMcnckXcmiZtg9h3mgh+Yi1kl+aZdNT/N9CDYtwje6WfaWyMHga3U9AqxlsKRP+CzkTB9vGmvTt1nbjKxeMCcu+HT4bBrnrlz8bVO8FZP+OTqmt8RWFNlfZvLboIpC3S/KidwN7wC4KHN0OOms69/2L/NRcSLjaefqQVXvDs1/aC57nC2QD9Vv3tNE0rjNtAkumafadUf/vyb6Vdf1dlQJ3vzzL5F5uBTB23nZSTQRd3LPGJOY23Wyq+ves30gpj30Jlvkd6/zPze8Km5eWXTF+ZiVNnFpWvfhDt/Mb0Kut8IE7827crbv4XZt8OsP5n3Mg6ZuxN/edys78YvTZNN9nG4bT7ctRBi7zAXHPPTIGlH7f4dDq00/brLapL5qYAyPTXEhek0EpLjTJCDaW6Bcw/0dlfA4/HmIOodWPPPNYmuvhdL4ygY9T9zkKijppYyEuii7i3+t7mwt+Xrk6GeFAc7ZkNw1MkLf2kH4OfJpuaafvBkyB9Yapbz8IX3BkDmUdN+3Lit6XGglOlB8MB60x3POxCGPA6XPwW75puxRm770fSYOLrONLOA6Xt923y4bR607Gv+8w9/Ae5ecvJ7a1OOvQko66j5nZdiwtzNUrvfczEqqwXv+tH8TlhvDvphnRxXpop6/ckcJCo2p9UB6eUi6lbaAdMdz83dDEP68+PmH/fa98x/uJtnw5o3YNWrpsaONjdtJO0wTSnaZkJ44COmrfKXySaYo8eYrnlnMuhvpmYe1tG0UY96C0a+Ym7XDmlnPxBUUYMLaArh0eYUfuBfa+9vUR7oCeZ3Xir4htbe+i9mwa1Nb5Y9P5t/X9tmmYueF9nBskaBrpQaDkwFLMBHWusXT3k/CPgKaGVf56ta609ruayiIdq7yLQNthlsasIxE01Qxn1vumplxJsbWm780nQ9O7HdhHnbYaY27R8O1041t0mf2G7Cbtc8Uxvf+JnpftiqvzlVDW0Pt86tedmUOnkjSRl3Lxj58tk/22mkGTL115dN8NbkM2eTc8L8zrbfsp+Xeub2c3FuWl1izgLXfgBFWXDZWYbVdUFnDXSllAV4B7gSSADWK6Xmaa13VljsfmCn1vo6pVQYsEcpNV1rXcejz4t6VVJgxhApyDDNFplH4MQ2c/Fp/YdmmeSdJoRX/Pfk5659E9pfYX5S9ppbzYdMNrVnMG2PZRf9cpLMAWDQ3+DYBtOmGdC0XjcTMHdQrn4Tlj9vnsdMOPf22FOVBXpZDT0/FcI7X9g6xUlNos34M+s/NBWEZt0dXaJ6V5Mael9gv9b6IIBSaiYwGqgY6BoIUEopwB9IB5xjziZxdlu+Nncoxq8+eaHQzcPUoq1FcMt3pmlkzwIz5CmY5pGBj5oeKhUvBIV1MO3U1QloAuPsB4dwB7Z/BjYzF0j3/mL6vK940RyELiTUT2tDlxp6rWpiv2kqL8XcjHQRqkmgtwCOVnieAPQ7ZZm3gXlAIhAATNBan9bnSyl1L3AvQKtWrc6nvKK+aW3arQuzTM153MemJuTdyLRPFmScvA273RXQItZckBrxkmnecGQoX6gRL8NVz5txVNZNM71UHtt3br0fKipvckk03ScL0qUNvTaFdQIUoKH1JVUuklVQQlJ2IR2aBNRr0epLTXq5VHWb2al9zK4GtgDNgR7A20qp0/7Va62naa1jtdaxYWFSM3GI3T9VPxVXVoK5hfr7++HrCWYA/1T7aH2XPAD3LDdX6cM7mxqsf3jlMTWUMjd+XPdmnfa1rTdKmTsBr37B3KlYWnhyFMNTnW16MpvNDKXqE2z6yKfYh4T1k0CvNV7+pougcoOIPlUuMnXJPka/vYbCEmuV7zu7mgR6AtCywvMITE28ojuA77SxHzgEOHHVzAWd2G6aTGbebHqbnKo4z4xpveAx08yQstsM4L/YPoB/v/8zPUUuRhZ36HSdGXgproqLsttnw0tR5vpCdfLTTJC3iDXPV79hfjfQJpftCVm8tmgPup6nULtgkQPNj1fVNfDNRzMoKLGy9WjmOa/aatNsPJzeoP8mNQn09UB7pVSUUsoTmIhpXqnoCDAMQCnVBOgIHKzNgooLkJ9uxmX+7BpAm6aDpLiT72/8HL65FXISTS+Sv+8341I3izHhHtYZGl3kTWRubub2832LTp8d59BKM8rh1xMgcUvVny9rP+861oykGDcX2gw1g23V0Ndrj7AvKeeMy2TmF5NbdOGXr6Yu3cv/lu3nWGYB2YUl/Lo35YLXWS+ufRNuqbonVKnVxq7j2QCsj08/51X/Y/Y2xr33O+sOmVBfvjsZq61hhftZA11rXQo8ACwEdgGztNZxSqn7lFL32Rd7FrhUKbUdWAo8rrVOratCi3O0q8Lxd8g/zVRiH1xmmlfyUu13Ly4141+3vdw0NXj4wJ2LzA05E6c7ruwNyYCHzOBNi/5V+fUT280Qtt6NzOh9hVmnf7bsQmhoR9N09effzcXk6gaOOvXj+SU8MXc7L/68+4zLxT63hEEvmRunTmQVkpRdWKP1n/pdZQG+6Ugmz/24k9s+Wcd7Kw5wy0drG3ZzhZul0h2bmfnFHEnLB+BASh6FJebS3ur9qeSdw4Fv3aF05mwyvZO2JWSx9lA6d3y2nhV7kmux8BeuRneKaq0XaK07aK3baq2ft7/2vtb6ffvjRK31VVrrblrrrlrrr+qy0OIc7fjO3HY8JdP01LjjJzMu9YoXTV9vW4kZ7W/8J5U/5+ENUYNYm9WIpbuSHFDwBiYowgzmtGveyfFnrKWmq2abwaZffX6qmaHmVGXjdYfab2hq0sXU+mso7rg5SKzYm0JKThE2m+ZPn6zjo1WVT4RLbZqM/BIy84u56aM/ePDrzeQWlZJdaNr484tLyco3jz9efYg/f1V5+rT0vGKemR9HiVWjFPyw+RhzNpl+8y/9spvV+1PZdLiaiZuBwhIrH/x6gONZ9TeP5pn8Z/5Oxr63hlKrje3HzN9wQLsQ/jiYzqUvLmN/cu4ZP5+YWcD7vx5g+trD+Hu5E+TjQVxiFjsTTU1/7aF0hr22go9XH+LK13/lneX7sTmw1i63/ruK1P3w7qXmRh9rqbl1fu595iJo/CpT+y4bOKhFbxj6TxM+K140Axs17UZSThEZeaffOvD8gl089u1WSq0nOy7FJWbx4IzNPPrNlmrbFLXW5bU5q02fU42ovn3xezz3T9902n9wm01TYrVhs2mKS20nR/uL+x6AwhO7zcXSpt3NDVAVJ02uKHUvBDQ/2ff+HJUFiNWm+WHLMZbuTmbl3hSmLt1Hjj2sy0Ib4N8/xHEwJY9NRzK449N13PzhWrTWPPrNVka/s5pSq405GxP4eccJjmWeDN8nvtvOvK2JTIhtSa9WwSzdnYynxY2JfU5eRttQRaDvOJZFr2cX879l+3jh591c8sIyPv8tvsomiVKrjRcW7DprmF4orTVrDqSSmlvM1KX7eOmX3QT5eDB1Yk9eGd8ddzfFpA//4Jq3VjHgxWXkF5dSUuHf+K97U7j+3TW8+PNuftiSyFXRTejdOpi4xGz22pu+vtt0jAMpeTz7404Op+XzysI9fL3uLIPN1SG59d+ZZSeasU2ueQ2W/9fMH7ngb2Zm93x7i9eOOaaP+Km3ybe53LSRu7nDqP9hs2kmfPA7uUWlvDmhJ14ebjT286RFIx92JmZTatPM35ZIl2ZBpOUWcdfnG7BpTVGpDT8vd3w9LUwe0YmV+1J5ZeFuiktthAV4sfFwBqNjWjBvayIazeK/DqZlY9PMkJxdyI7ELC5tG4q3hwWtNeosY3fnFJZw80drefTKDgzpWM1MOefgSFo+Vq35ePUhDqfls+1YJr8+NhQ3N1OON5bs5ZcdJxjTswUz1h3hszv6EBwUTeMdc/g58Aa2/DibJ4Djvh1o5mYxbeJ7FphhA8qarwBb8m4S3VsSWmLF28Pcjp6RV8zEaX/w8vjuxLRsRHJ2IaH+XuXfXVFcYjbhAV5EBPvw1R+HaeTrSSNfDzLzS/jg14M8dnVHEisE87ytiShlauzr400Ar9mfxrLdyRRbbcxYf5RdJ8xB4v7pm3BTcPuAKBbuPMH9Q9rx2NUdeXpeHBsPZ/DiuG5c1705fx7Slnu/2MiP2xLx9nDj1v6R+HiabVmw/TjpecW8/6s5Y+gb1Zgp8+LYfiwLm03zl6Ht+ODXA/RrE0Kp1cYHKw+y+Wgm39zbn4ISK7d+vI57BkUxvGuzM+6vvUk5+Hm506KRzxmXAziSnk9SdhEA/1u2n7Zhfrx+Yw9C/b24IbYl7ZsE8OaSvazYY5qXHp+znRW7k/ny7n4cyyjg/q830aqxLyO7NWXB9hOMimnOxsMZrNiTjMW+j1Jzi8q39/HhHfnP/J18uuYQ18U05+aP/uCBoe0Z3rX+boxTjrpiGxsbqzds2OCQ73YqWpsLmE2iTx+ac/1Hpv3bzd30oOhzj7lLLrAF9LuPjOSjBG+dRppvG1ZfOZ/rYlpUCotPVh3kg1UHGRXTnGGdmzBx2h/4elrILza16qaB3rw8vjt/+mRd+Wd8PS34eroT7OvBF3f1ZcIHf3Ak3bRRPjmyM1OX7iMswAsfDwv7U3Jp1diX/cm5XB3dhCW7khke3RSNJi4xm8P2ts0WjXy4e1AU7604wO0DIll3KJ0p10UTFep32p/j0zWHeGb+Tga1D+XLu8ztEEt3JdGtRRDhgSdnKCqx2nB3U2c9QIyYugqbTXMoNY9Gvh4k5xTRL6oxSdmFPHlNF95YvJedx7NpHeLL4bR82oT6MSzjG570+JrBRa/zlM9selq3M33gYh66srOZ8f2L0eaGq773mjOj39+G3T/yaenV+I5+lQl9zAXmRXEnuPfLjXRuFshdA6N47NutTJ3Yg9E9WlQqo82mufrNlUQE+zCmZwsenrkFgNduiGHVvhS+35LIDb0j6BvVmL/P3sYr47uTkFFAx6YB3P/1JrQGdzdFqb22HOjtTnahOVsqe91NgU2Dl7sbqx+/nLAAL7LyS4g7bg64Zf7+7Va+3Wjaknu1asTX9/TH28PCNW+tIs5+FjGmR3PemNCDf32/g+lrTW21Xbh/eY1cKfDxMP/Opt3am13Hc3hjyV66tgjkxwcHnbaPdh3PZuqSfQxoH8rUJftoF+7HzHur7mde0awNR/nH7G1EhfpxOC2PeQ8MpGuL08+QCkus9PjPovL29aaB3oT4e5JfbOWXR0x5ftufxpCOYSyMS+K+U5qp2oT5sexvQwCYuzmBv36zlaEdw1i+J4VQf0+WPjqEIF+Ps5a3ppRSG7XWsVW9JzX0hkpr0188/SB8fYOZkqzDcDM7TfJumHWr6ftcFuZdx5ubeYIjocNwcvxbM/jpuSz3msF7WZfw0Tdb2ZKQhbeHhcEdwmgd4stzC3bh6+nO578dZl9yLn6eFlb+Yyir96eSkFHAKwv3lId5bOtgNhzOwOKmSMsr4pPbY2kW5MOrN8Swen8q321K4PkFuwgL8GL63f1oGuhNfokVdzfF3qQcukc04i/TN/LT9uME+3pwSdsQboxtSZtQP6bMi+OZ+TtRCl7+xbQ1+3nu4Z2bzbyXyTmFfLPuKDlFpfxor3mu3p/K1CX7iI0M5q7PNzC6R3OmTuwJwOG0PCZ88AfdI4J45+ZeeFhOb1k8kJLLgeTc8l4PAA9e3o7nftrF2kPpBHi58+CMTaaZBcoPPgdT8yikP096fM1/W2/m0tRNLPK+kjlbTvDgFZ1QrS8xw68uf94E+bpp5evfr1sQvzWxPNCTc0ztbtfxbB6fsw2APw6mMSqmOevjM+jWIogT2YXc88UG9ifnMrZXBCO6NuO9pge4tG0o43pHMKZnC5oEefPJ6kPlQXtZhzCa2A9uXZsHkVdcyqiY5ry5ZB+NfD14a2LP8v1618Aolu5O5rM7+rDreA4RwT6EBZh7CIJ8PSqFOcCIbk35eccJJvZpyUerD7F0VzJ9oxoTl5hNq8a+HEnPZ3DHMJRSTLkumm4tgvji98PsPJ5NqL8n/74umi9/j+evV3bgqe938Mz8naTbm/nyiqyk5BRxPKuATYcziI1sTFGplZs/WkthiY1FO09g05CeV0RKTlF5Oavzx4E0gn09+PBPsRxJz6syzAG8PSz0iwrh170pjO8dwfytiZzILuSJkZ3wcjdnIEM7mbPBwR3Cyg+CYQFepOQU0bnpyVtuRnZrxge/HmT5npTySsCM9Ue4b3DbM5a1tkigN1S7fzJjdze3T+Y77yFT6xvzvpk/MnWveb3P3dia98at43Bzhf/SBwDYfiCVbPy4tOh/XNKhORODfPh0TTwAa/anMqxTE2wa3rulF7d9so4Ve1K4e2AUIf5e5TXEA8m5fLf5WHlt3KZh9/FsErMK6R7RCDCnmn2jGtOrVSPmbU3k8eGdysPE38v88ypb9q9XdMDdzY1/DO9IRPDJ3h1twvx5a9k+7rusLcv3JJOUXcj0tUdYOWUhz47pylM/7CC3qBRvdwvubop/XdOF537ayRtL9uJpD+ufd5zgmfxi1uxP4+n5ceQXlbJoZxL/mb+TZ8dUnkdzW0ImN3249rTufT1bBXNNt2ZsPprJc2O6cvNHayu9r5Q5zo4d2p/DW7tyadLXKG2F2Bs4vCKf+LR8c1bh6WumNoscZIYILi2EXfPZa4tg44E0knMKCQ/w5qj9zAZMrTDQx4PdJ3J4/qddfLT6EJd3CmdbQiY2bWrjo3s0x93ixs8PDyo/87C4Kf45ojPZBaXMWHcEpSDM/2TQvTGhB2BqyFd0boK7RdGpaSAvjO1GYmYBf7uqI5NHdEIpVWmfVOfyTk3Y/vRV2DT8sDWReVuPEZ+WB8DrN8bwx8E0hkebZhNPdzcm9m2FBv753Xaujm7KqJjmjIppDsATIztz1+cbaNXYl6Edw/j898Nc/uoKcuz7xcvdDa2hRbAPT4zszD1fbMDbw43CEhuzNyZw72Vtyps+ymitsWlzrWHJriSu6NKEduH+tAs/87R6I7o2ZUN8Ov8Y3pHBHcKYtvIg43pFnLacj6eFWfddwqPfbGFMzxa8uWQfnZqe7PPu5W7hg1t78+evNjF5RCeenh/H+kPpEugXJZsNDq8GiyclexbhAWYslJD2Jszd3OF7e0/RAQ9D3Pdkth/LgK9yeO3GIoZXyK3tCeaK/qf3DKJny2CKrTb+OJiGxU2xLSGL+NQ8BrYLZVD7MKZcF427RXFT38p9zV+7MYZrY5oR5OOBr6f5pxIbWfVkDEM6hp+1Tbt9kwDemtTztNc7Ng3gnZvMgatbRBBZBSWUWG0s3pnEY99uxaY1Cx4aROdmJ2tC/aIas2RXEm8u2UdkiC/xafl88fth/rdsH+3DA3j1jr7M3ZzAh6tMMA7tFE6p1caMdUf474LdNPbzpG24P2H+nizfk4LWmnbh/rw0vjulVo23hxuRIabG2TbMn33JudzavzW/7k3h4Sva49HsEVgyBbqMoUXXwbBiDXGJWZWbiToONz82K4+88h7HvTtjyyxk0+EMhndtxtGMfBr7edIm1I9/DO/EwrgTfLLmEJuPZNIm1I9lu5MJ8HJn7v0DKgVSVc1IgzuEMmPdEbSmUrNaxc9VrKFOqrCvz9YsdSqlFBYF13Rrxtdrj7BmfxpXdA4nNrJxlf8+RnRtytxNx7ipX+V/X5d3CuetST2JbR3M/uRcPv/9MDlFpbw4thtdWwTxysI9NAvy5tGrOhAe4M2kvi1pHx7ArA1HeemX3azcm8Lnd/bF0/3kGdg/v9vO8j3JvDiuO9mFpYw8S5t8mQl9WnJdTHP8vNy5LqY519kPOlXp1SqYFX8fyp4TOby5ZB89WwVXer91iB8LHjZNNbGtg1m0MwmbTVd5baS2SaDXt0OrzAzho/532ljNub88jf+6qeaJW4UZ62PvMN3lMg6b8cRjJpjJhK/8D+t3JpFXvIF3Vxzg6uimWG2aD1YeZP7WRCKCfcpPmX2wsOxvQ0jPL6b/f83EDU9da+78vO3SyCqLqpTi8k5NanXzayLIx4OXx8fwxuK9TF26j+HRTSuFOZhw6tg0gMNp+dwY25LnftrJ28v3U2LVPD0qmi7NA2kb3pFFO5P4cNVBvvzjMCv2JGPTptvaazf0oGmQN1prRr29hrzi0vKLlfZfPHZ1R3Ycy8bDoijYfIxnRkWfDL/uN5gfoH2paVqKS8zm2u6nB0F+qeaHzCjuGtCMj1Yf4mCqqdEeSc+ne0QQn93RFzBd5LQ22z/nz5fy7E87Gdsz4qy1S4BL29X/EAJ3Dohi85EMTmQXMnlE9aNGNvL1ZNZ9p7d5K6XKa+u+9ourQzqGMdF+sPn8zr6Vln9hrBk9cUzPFszacJQXf97Nk3O38+K47uU19ZnrTX//yXO24edpYWD7mv1dlFL4eZ1bHHZsGsCqfwwtv8hfldjIxszakMDB1Fzahdf9+DES6PVtzZtmnPCwjuYGn9g7zY0QKXvwWfc286396em2nwhbKvOsl3C5ZxxpoYP423u/8cgVHRh400y01mw5mkl08yDiEk1NfFtCFnd8th6LUizdbW52GHjKf3I3N0WovxfT/tSbFo186di0YQ9QdIu9Rnz/0HZVvu9hcStvUhgV05wXfjbd0nq1agSY09/hXZvy0apDWG2aKzqHM7FPK4Z1Di8PZqUUz43pSlHp6fOHXtu9Odd2b06p1cafh7Sttibr5W6hfZOA8ouCFa3el8otH5umm96tg5m3NZFDKfZAT8unV4XaXbcIU4Oe0KclwX6evH5jj7P/kewCvT24pX8r2tdDaJRpFeLLDw8MrJV1NfL15K1JPenZstFZl23s58l9g9uSX2zlraX72Hw0k7sHRnFjbEu83N0oKrWRmV/CC2O7lR+k68qZwhxMDR1gQ3yGBLrLyUs7OZ/k4n8DsHzBTB5Vj/G+97tEa09+7ziZvSdW8XDOG3zmMZEnSpvRfUU+Gw5ncMdn67hzYBTBvp68+PPu8sCOCPahTZg/R9LzOZiSx1VdmrBoZxJXRVddu3ZErft8hAV48f39A2q07LX2QL+sQxjuFS6CXt4xnA/sXen+M7orzavo7hZzlhBxt7hVWmdVopsHsmJPcnnXy+JSG8t2J/PRqoOE+HnSPSKIS9qGEBXqx8HUPLLyS8guLKVVhUBoG+bPtFt7M+A8a9vPjel2Xp9rKEadoZmjKo9e2YGIYB/eXLyXT9fEc1mHMIpKbTw+vBPjerWo1OvJUaJC/fDztLD7xJmHbKgtEuj1accc0Fa2tLyVgCNL2eg7kBsLZvF5o4/onrmKt61juGloL/y8+vLs4iE83Lszd3y6jt8OpPHQsPYkZOQzbeVBtIb24f6sOZCK1uY/QlnbdHJ2oelyVlBCoHftdZVq6Fo08uGtST3pdkpPht6tgwny8aBjk4Aqw7y2dGsRxOyNCRxNL6BViC/frD/CUz+Y8XKeurYLdw2MAswF4IVxJziYarrwnVrDuyraAZN5OLEbY1uy50QOX689wiF7U1b3iKAGEeZgzgDbhvtzIMXs771JOTRv5FPeYaC2SaDXk49//p0J657BL6If/y2+iXVFI6AIurTIonvaQmyNIrlm3LNE2QPpmUmXAbDk0cFsPpLJqB7N8bC4cc+gNny3KYF7BrXhpV/2MGdTAq1DToZC2T/kRr6e9b+RDlZVDc/d4sYnt8fS2K9uh/Md2jGcKcQxZd4OjqTnl/+H/fOQtpUuNrcJ9SM9r5g1+82NX9HNz3NsdVGuVWNfCkqsbLDfQBVZxf0LjtQ2zJ8/DqZxIquQa95axS39WzPluug6+S4J9DqQmV9MVkEJNg0JiYkUphyi2W+v4OFWxJroKWyan1l+I0fJNVPB7QBuLfsRZTm9Rt0mzJ82YScvinVuFsiT15iLmVNGdcFNwQ29W572OXFS79ZV98ypTa1CfOnaIpDle06OSnjHgEgeH155FOk2YSZs5m4+Roj9TlxxYVo2Nn/DX/cm4+XuRrMGUjsv0y7cn7mbjzFt5UFKrJoF24/z1DVd6qTXiwR6HXjkmy1sPpJJpxALTyU/Sle3eLDA67abmLuqiFKb5plR0aTlFRMT2RTcata16lSB3h68ckNM7RZenLeR3Zqx41g2nZsFsut4Npd3Or0bZ69WwXha3DiQksdQ+w044sKUXYfYdCSTTk0D6qV74Lloa6+QfbLmEEE+HiRlF7H5aEadVDQk0GuJzaaxas2+pNzysSHGnXibLpbDzCgdSp9WASQG3sXRLUmE+nsxoU/LOr8CL+rXnQOiiIloRKemAczZlHDaXZYAwX6eXN21KfO3JtLNfsOVuDAVb4g6tWdXQ1Cx2+l7t/Ti9k/Ws2D7CQn0hmr1vlT+PnsrzRv50DbMXNW+MXgPN2b9yvf+E5iScT3rbhnG30qs9GkbzrDOTSTMXZC3h6W8h8q9l1V/Z+Ct/Vszf2si/aPqvinoYlDx/9KIbud3tluXWof40qKRD+N6R3Bp21DeubnXaRfva4sMznWetidk8fvBVJo38uHJuTvIKjBDl3p7uDG2SxDPHruLEndfcm5bxtEcW6X+xkIkZhbUaa+bi03k5J8AOPjfkQ2uyQWo0UiiNSWDc9WBf87dxo5j5kYSD4viy9tiiJ/+MEHk0bs4AkveCSx3LcY7OIgwyXJxCgnz2jX/gYEUW60NMszh3IdXOF8S6DWktebuzzew7VgW9wyKYtfxHO4ZFEX78AD8vNwZtPsZBrkvMQsfAi59ECKqPIgKIWpZ2V22FzsJ9BpatDOJpbuTCfX35IWfd6M19I0K4couTSAnCb7/lrSud1KSn03Tgn0w9ElHF1kIcZGRQK+BnYnZ/Ov7HbQL9+dvV3bgz9M3ARDT0l4r2PwF2EoJGXK/mTPSZjun+SKFEKI2SKCfRanVxj1fbMCiFO/d3IuWjX3x87TQyNeT8ABvsFlhw2fQZogJc5AwF0I4hAT6WSzemcSxzAI+uLU37ZuY0dIeGtb+5Cw4exdCdgIMf8GBpRRCCAn0s/pq7WFaNPLhis4nRyj8P8+FkJsEO3vBz5PNbO4dRzqwlEIIIYF+RlabZuPhDG7q27ryVFcL/3nycXg0jP6fGdNcCCEcSFLoDI6m51NYYqs0ZyDZx83v5r2gx03Q+w4JcyFEgyBJVI2cwpLyGWgqzeyTuNn8Hv4itOrngJIJIUTVJNCrcDQ9n1FvryYj39zO375JhTkdEzeBskBT554dRgjheiTQT/HN+iO8tXR/eZgD5TPeA5CwHsI7g+eZ5xIUQoj6Jh2mK9iWkMnjc7YT4u/Juzf3AiDUv8LMPxmH4dBKaH+Vg0oohBDVkxp6BS8s2E2InyfT7+5HgLcHn9/Zl2ZB9tlP8lJh+fOAgj53ObScQghRFQl0u31JOfx+MI0nRnYiwD658uAOYebNYxth+o2Qn2p6tQRFOLCkQghRtYs+0HcmZtM6xJfZmxKwuCmu73lKWC+eAr+9ZUL85uXQvKdjCiqEEGdxUQd6camNkW+tonOzQNLzihjSIYywgAqzw6fshTVvQtdxMPJV8JUZZoQQDddFfVE0MbMAgF3Hs0nKLmJi31aVF/jjHbB4wfCXJMyFEA3eRR3oRzPyyx+3bOzDsJIVkLTTvFCUA9tmQfcbwT/MMQUUQohzcFE3uRxNNzX0NqF+PDS4JW5zrzdvXPkf09xSkg+9bnNgCYUQouZqFOhKqeHAVMACfKS1frGKZYYAbwIeQKrWenCtlbKOHM3Ix8OiWPzoYCzHN518Y/G/ze/QjjKNnBDCaZw10JVSFuAd4EogAVivlJqntd5ZYZlGwLvAcK31EaVUeB2Vt1YdTc+neSMfM5Li8W3mxXZXmJ+CDGjZD+ppclchhLhQNamh9wX2a60PAiilZgKjgZ0VlrkJ+E5rfQRAa51c2wWtC0czCmgZbL+F//hW8A6Cm2dLiAshnFJNLoq2AI5WeJ5gf62iDkCwUmqFUmqjUupPVa1IKXWvUmqDUmpDSkrK+ZW4lhSVWjmSlkfLxvZAP7ENmnaXMBdCOK2aBHpVCadPee4O9AauAa4GnlJKdTjtQ1pP01rHaq1jw8Ic13OkxGpj9NtryMgv4aqAeIj73tTQpb1cCOHEatLkkgC0rPA8AkisYplUrXUekKeUWgnEAHtrpZS1KD41j6TsQnafyOGtyz0YsuZGQEOjVjDgYUcXTwghzltNAn090F4pFQUcAyZi2swr+gF4WynlDngC/YA3arOgtSEjr5ghr64ATMvKiOPvobyD4LLHzAiKPsGOLaAQQlyAswa61rpUKfUAsBDTbfETrXWcUuo++/vva613KaV+AbYBNkzXxh11WfCayi8u5bFvt/LwsA6k5BSVvz6omRWPQ8tg6JNw6YMOLKEQQtSOGvVD11ovABac8tr7pzx/BXil9opWOxbvTGLB9hM0D/KhSaB3+evXhyRAOtBmqOMKJ4QQtcjl7xSdt8U096/en0pME3fWej+IpXk3Aj1Cwd0bmsU4uIRCCFE7XDrQswpKWLkvhSAfD3afyCEqew9NSIPEFeaybkQfcPc822qEEMIpuPTgXHtO5FBi1dw5IAqAdkVxaBRca79eK7VzIYQLceka+qHUXADG9GyOm4KhG45Q7N0er9g7ITgSmvVwaPmEEKI2uXSgH0zNw9PiRkSwLw8ObQvr90DUGPNm28sdWjYhhKhtLt3kcjAlj9YhvmbwrYPLoDALoi5zdLGEEKJOuHQN/VBqHm1Dfc1EFeumgX9T6DzK0cUSQog64bI19FKrjcNpeVzmvR++uwcS1kP/+6RXixDCZblsDf1YZgElVk2/3GXg4Qu3zZeLoEIIl+a6gZ5RgDultE5aDB1HykiKQgiX57JNLscyC+ioEvAoyoBOIx1dHCGEqHMuHeht3Oyj/IZ1cmxhhBCiHrhsoCdmFtDVKwVQ0LiNo4sjhBB1zoUDvZBOHkkQ1BI8fBxdHCGEqHMuG+jHMguI5DiEtHV0UYQQol64ZKBrrTmWmU+T0gQIaefo4gghRL1wyUBPzcrjMb7C25oHoe0dXRwhhKgXLhnoWYc2ca/7T2QFdzVzhQohxEXAJQM9L/0YACcGPg+NoxxcGiGEqB8uGejFmab/eWBYCweXRAgh6o9LBrotOwmA4PAIB5dECCHqj0sGulteMhkE4O0t/c+FEBcPlwx0j4IUMt2CHV0MIYSoVy4Z6L7FqeS4hzi6GEIIUa9cMtADStMp9Ap1dDGEEKJeuV6ga01jWwalvmGOLokQQtQrlwv0/Jx0vFQJ2j/c0UURQoh65XKBnpmUAIB7QFMHl0QIIeqXywV6XvIBADxDWzu4JEIIUb9cLtBLUvYD4NtUBuUSQlxcXC7Q3dIPkau95S5RIcRFx+UC3SsnnsO6CSH+3o4uihBC1CuXC/SAvCMkWppjcVOOLooQQtQr1wp0aymNio+T5iXNLUKIi49rBXrWEdyxkuPT0tElEUKIeudagZ5+EIDCwEjHlkMIIRygRoGulBqulNqjlNqvlJp8huX6KKWsSqnxtVfEmtNppg+6btzGEV8vhBAOddZAV0pZgHeAEUAXYJJSqks1y70ELKztQtZU+tE95GkvfIObO6oIQgjhMDWpofcF9mutD2qti4GZwOgqlnsQmAMk12L5amxfUg5bt27ksG5KWKB0WRRCXHxqEugtgKMVnifYXyunlGoBXA+8f6YVKaXuVUptUEptSElJOdeynlFSdhGtVRJHVVP6RclY6EKIi09NAr2qDt36lOdvAo9rra1nWpHWeprWOlZrHRsWVrvD2xYWFdFSJdOrR2+aBkkNXQhx8XGvwTIJQMV+gBFA4inLxAIzlVIAocBIpVSp1vr72ihkjWQn4Kms2ILlgqgQ4uJUk0BfD7RXSkUBx4CJwE0VF9BaR5U9Vkp9BvxYr2EOWLLNsLmWxjLKohDi4nTWQNdalyqlHsD0XrEAn2it45RS99nfP2O7eb3JTwXAI1AmthBCXJxqUkNHa70AWHDKa1UGudb69gsv1rmzFGQA4BUoU88JIS5OLnOnqHthGiCBLoS4eLlMoHsUZ5CtfVHuno4uihBCOITLBLpXUQZZKtDRxRBCCIdxmUD3LsmUQBdCXNRcJtB9SjPJcQtydDGEEMJhXCbQ/UozyXWXQBdCXLxcI9C1xt+aRb57I0eXRAghHMY1Ar04F09KKPAIdnRJhBDCYVwj0PNNH/Qij0aOLYcQQjiQawR6ngn0Yk+poQshLl6uEeiFmQBYveWiqBDi4uUagV6ca357+ju2HEII4UCuEehFJtCVV4CDCyKEEI7jEoFuLcwBwE0CXQhxEXOJQC8pyAbA3UcCXQhx8XKJQLcWZFOiLXh6+zi6KEII4TCuEeiFueThjY9njebrEEIIl+QSgW4ryiEXH3w8LI4uihBCOIxLBLouyiVfe+HtKYEuhLh4uUig55CHDwFe0uQihLh4uUSg2wpzydXeNAn0dnRRhBDCYVwi0FWxqaGHBXg5uihCCOEwLhHolpI8ii2+eMtFUSHERcwlAt3Dmo/28HN0MYQQwqFcItA9bQUob7lLVAhxcXP+QC8twoNSLBLoQoiLnNMHui4yA3N5+AY6uCRCCOFYTh/o2ZkZAHj7yeQWQoiLm9MHenpmOgC+AY0cWxAhhHAwpw/0shq6nwS6EOIi5/SBXppvb3LxlwmihRAXN6cPdFtuKgCeQU0dXBIhhHAspw90lZcCgE+jcAeXRAghHMvpA929MJU87YVfgHRbFEJc3Jw/0AvSSCcIL3cZx0UIcXGrUaArpYYrpfYopfYrpSZX8f7NSqlt9p/flFIxtV/UqnkXp5GhpA+6EEKcNdCVUhbgHWAE0AWYpJTqcspih4DBWuvuwLPAtNouaHV8ijPIcmtUX18nhBANVk1q6H2B/Vrrg1rrYmAmMLriAlrr37TWGfanfwARtVvM6vmVZpDrLl0WhRCiJoHeAjha4XmC/bXq3AX8XNUbSql7lVIblFIbUlJSal7K6ths+FszyfeQQBdCiJoEuqriNV3lgkoNxQT641W9r7WeprWO1VrHhoWF1byU1SnMxB0rBR6NL3xdQgjh5Goyq3IC0LLC8wgg8dSFlFLdgY+AEVrrtNop3lnY+6AXe4fUy9cJIURDVpMa+nqgvVIqSinlCUwE5lVcQCnVCvgOuFVrvbf2i1kNe6CXSqALIcTZa+ha61Kl1APAQsACfKK1jlNK3Wd//33g30AI8K5SCqBUax1bd8W2yzkBgNWvSZ1/lRBCNHQ1aXJBa70AWHDKa+9XeHw3cHftFu3sbNnHcQNs/jKOixBC1CjQG6rSzGNYtScevnJjkRBCOPWt/9bs45zQwfh7ezq6KEII4XBOHejkHCeZYPy8ZBwXIYRw6kB3yzlBkg4mwNupW46EEKJWOG+ga417fhIndGP8PCXQhRDCeQO9MAuLtZAsSwhdmstY6EII4bSBfjzhEAAd2rcnwNvDwaURQgjHc9pATzhyAIBunTo5uCRCCNEwOG2gW/MzAfAJrIVBvoQQwgU47dVEXZAFgKd/I8cWRAgXUFJSQkJCAoWFhY4uirDz9vYmIiICD4+aNyk7b6AX5QDgEyB3iQpxoRISEggICCAyMhL7eEzCgbTWpKWlkZCQQFRUVI0/57RNLm5F2di0wltu+xfighUWFhISEiJh3kAopQgJCTnnMyanDXRVnEMe3rhZ5C5RIWqDhHnDcj77w2kD3VKSS77ydXQxhBCiwXDaQPcoySVf+Tm6GEII0WA4b6CX5lLgJjV0IS5G/v7+1b4XFRXFnj17Kr32yCOP8PLLLwOwefNmlFIsXLiwxuus6P333+eLL744xxLXD6ft5eJlzSXbIrf8C1Hbnpkfx87E7FpdZ5fmgUy5LrpW11mdiRMnMnPmTKZMmQKAzWZj9uzZrFmzBoAZM2YwcOBAZsyYwdVXX33O67/vvvtqtby1yWlr6F62fIrdpclFCFfw+OOP8+6775Y/f/rpp3nmmWcYNmwYvXr1olu3bvzwww81WtekSZOYOXNm+fOVK1cSGRlJ69at0Voze/ZsPvvsMxYtWnRe/e6ffvppXn31VQA+/PBD+vTpQ0xMDOPGjSM/Px+ApKQkrr/+emJiYoiJieG333475+85H05bQ/ex5VMigS5ErauvmnRFEydO5JFHHuEvf/kLALNmzeKXX37hr3/9K4GBgaSmptK/f39GjRp11t4f3bt3x83Nja1btxITE8PMmTOZNGkSAGvWrCEqKoq2bdsyZMgQFixYwNixY8+73GPHjuWee+4B4F//+hcff/wxDz74IA899BCDBw9m7ty5WK1WcnNzz/s7zoXT1tB9dR6l7gGOLoYQohb07NmT5ORkEhMT2bp1K8HBwTRr1ownnniC7t27c8UVV3Ds2DGSkpJqtL6yWnppaSk//PADN9xwA2CaWyZOnAiYg8iMGTMuqNw7duxg0KBBdOvWjenTpxMXFwfAsmXL+POf/wyAxWIhKKh+7pdxzhq6tRQfirB51uwihhCi4Rs/fjyzZ8/mxIkTTJw4kenTp5OSksLGjRvx8PAgMjKyxk0kkyZN4qqrrmLw4MF0796d8PBwrFYrc+bMYd68eTz//PPld2Pm5OQQEHB+lcPbb7+d77//npiYGD777DNWrFhxXuupLc5ZQy8yF2xsXnJRVAhXUXYxc/bs2YwfP56srCzCw8Px8PBg+fLlHD58uMbratu2LSEhIUyePLm8uWXJkiXExMRw9OhR4uPjOXz4MOPGjeP7778/7zLn5OTQrFkzSkpKmD59evnrw4YN47333gPAarWSnV27F5mr45SBXlJg/jjaU5pchHAV0dHR5OTk0KJFC5o1a8bNN9/Mhg0biI2NZfr06XQ6x6GyJ02axO7du7n++usB09xS9rjMuHHj+PrrrwHIz88nIiKi/Of111+vdt1l7fjPPvss/fr148orr6xUvqlTp7J8+XK6detG7969y5ti6prSWtfLF50qNjZWb9iw4bw+m3N4CwGfDmZJt1e4Yty9tVwyIS4+u3btonPnzo4uhlN48MEH6dWrF3fccUedf1dV+0UptVFrHVvV8k5ZQy/KzQDAzUcG5hJC1J+nnnqKtWvXMmrUKEcXpUpOeVG0OM+MhW7xkTZ0IS5W27dv59Zbb630mpeXF2vXrq2V9T///PN8++23lV674YYbWLduXa2svy44ZaBb89IAcPcLdXBJhBCO0q1bN7Zs2VJn63/yySd58skn62z9dcEpm1xsuSkAWGT6OSGEKOeUgU5eMoXaAx+Z3EIIIco5ZaDr3BRSCSLI19PRRRFCiAbDKQNd5aWSpgNpEujt6KIIIUSD4ZSB7lGYRpZbED6eMv2cEK4gMzOz0miLNTVy5EgyMzPP6TOfffZZ+d2jZVJTUwkLC6OoqAiA0aNHc8kll1RapuIoi2dz6aWXnlOZaotT9nLxLk4n3yPC0cUQwjX9PBlObK/ddTbtBiNerPbtskAvG22xjNVqxXKGeYMXLFhwzkUZO3Ysjz32GPn5+fj6mklyZs+ezahRo/Dy8iIzM5NNmzbh7+/PoUOHiIqKOufvqK/hck/lfDV0rQmwZlDiLV0WhXAVkydP5sCBA/To0YM+ffowdOhQbrrpJrp16wbAmDFj6N27N9HR0UybNq38c5GRkaSmphIfH0/nzp255557iI6O5qqrrqKgoKDK7woMDOSyyy5j/vz55a9VHGJ3zpw5XHfddeVjy5yPstmPcnNzqx3T/YsvvqB79+7ExMSc1p/+vGmtHfLTu3dvfV7yM7SeEqi/f/ef5/d5IcRpdu7c6dDvP3TokI6OjtZaa718+XLt6+urDx48WP5+Wlqa1lrr/Px8HR0drVNTU7XWWrdu3VqnpKToQ4cOaYvFojdv3qy11vqGG27QX375ZbXfN2vWLD1mzBittdbHjh3TzZo106WlpVprrYcNG6ZXrlyp9+zZo7t161b+mSlTpuhXXnmlRtvj5+entda6pKREZ2Vlaa21TklJ0W3bttU2m03v2LFDd+jQQaekpFTavlNVtV+ADbqaXHW6GnpZH3T3AOmDLoSr6tu3b6WmjrfeeouYmBj69+/P0aNH2bdv32mfiYqKokePHgD07t2b+Pj4atd/7bXXsnr1arKzs5k1axbjx4/HYrGQlJTE/v37GThwIB06dMDd3Z0dO3ac93Zorasc033ZsmWMHz+e0FDT0tC4cePz/o6KahToSqnhSqk9Sqn9SqnJVbyvlFJv2d/fppTqVSulq0JWaiIAnkFN6uorhBAO5ud3cjayFStWsGTJEn7//Xe2bt1Kz549qxwX3cvLq/yxxWKhtLS02vX7+PgwfPhw5s6dW6m55ZtvviEjI4OoqCgiIyOJj48/72YXoNKY7lu2bKFJkyYUFhaitT7rzEvn46yBrpSyAO8AI4AuwCSlVJdTFhsBtLf/3Au8V8vlLJeTdhwA3+BmdfUVQoh6FhAQQE5OTpXvZWVlERwcjK+vL7t37+aPP/6ole+cNGkSr7/+OklJSfTv3x8wQ+z+8ssvxMfHEx8fz8aNGy8o0Ksb033YsGHMmjWLtDQzjEl6evqFbxA1q6H3BfZrrQ9qrYuBmcDoU5YZDXxhb+L5A2iklKqTxE3wjOI/JbcS2LRNXaxeCOEAISEhDBgwgK5du/L3v/+90nvDhw+ntLSU7t2789RTT5WH74W66qqrSExMZMKECSiliI+P58iRI5XWHxUVRWBgYPmAX88991ylMdOrU1b7rm5M9+joaJ588kkGDx5MTEwMjz76aK1s01nHQ1dKjQeGa63vtj+/FeintX6gwjI/Ai9qrVfbny8FHtdabzhlXfdiavC0atWq97nMQFJmQ3w6H606xH/GRBMeIDcWCVEbZDz02pOWlkavXr3OaYal6pzreOg16YdeVUPPqUeBmiyD1noaMA3MBBc1+O7TxEY2Jjaydi4gCCFEbUpMTGTIkCE89thjDvn+mgR6AtCywvMIIPE8lhFCiHp1//33s2bNmkqvPfzww7Uy21BaWhrDhg077fXff/+dkJCQC17/+ahJoK8H2iulooBjwETgplOWmQc8oJSaCfQDsrTWx2u1pEKIOlVXPS8c6Z133qmzdYeEhNTpeOxnaw6vylkDXWtdqpR6AFgIWIBPtNZxSqn77O+/DywARgL7gXyg7ifbE0LUGm9vb9LS0ggJCXG5UHdGWmvS0tLw9j6364ROOUm0EKJ2lZSUkJCQUGX/buEY3t7eRERE4OHhUen1C70oKoRwcR4eHuc1CJVoWJzu1n8hhBBVk0AXQggXIYEuhBAuwmEXRZVSKcD53koVCqTWYnEcSbalYZJtaZhkW6C11rrK4WYdFugXQim1obqrvM5GtqVhkm1pmGRbzkyaXIQQwkVIoAshhItw1kCfdvZFnIZsS8Mk29IwybacgVO2oQshhDids9bQhRBCnEICXQghXITTBfrZJqxu6JRS8Uqp7UqpLUqpDfbXGiulFiul9tl/Bzu6nFVRSn2ilEpWSu2o8Fq1ZVdK/dO+n/Yopa52TKmrVs22PK2UOmbfN1uUUiMrvNcgt0Up1VIptVwptUspFaeUetj+utPtlzNsizPuF2+l1Dql1Fb7tjxjf71u94vW2ml+MMP3HgDaAJ7AVqCLo8t1jtsQD4Se8trLwGT748nAS44uZzVlvwzoBew4W9kxE4pvBbyAKPt+szh6G86yLU8Dj1WxbIPdFqAZ0Mv+OADYay+v0+2XM2yLM+4XBfjbH3sAa4H+db1fnK2GXpMJq53RaOBz++PPgTGOK0r1tNYrgVOnJ6+u7KOBmVrrIq31IcxY+X3ro5w1Uc22VKfBbovW+rjWepP9cQ6wC2iBE+6XM2xLdRrytmitda79qYf9R1PH+8XZAr0FcLTC8wTOvMMbIg0sUkpttE+aDdBE22d4sv8Od1jpzl11ZXfWffWAUmqbvUmm7HTYKbZFKRUJ9MTUBp16v5yyLeCE+0UpZVFKbQGSgcVa6zrfL84W6DWajLqBG6C17gWMAO5XSl3m6ALVEWfcV+8BbYEewHHgNfvrDX5blFL+wBzgEa119pkWreK1hr4tTrlftNZWrXUPzBzLfZVSXc+weK1si7MFutNPRq21TrT/TgbmYk6rkpRSzQDsv5MdV8JzVl3ZnW5faa2T7P8JbcCHnDzlbdDbopTywATgdK31d/aXnXK/VLUtzrpfymitM4EVwHDqeL84W6CXT1itlPLETFg9z8FlqjGllJ9SKqDsMXAVsAOzDbfZF7sN+MExJTwv1ZV9HjBRKeVln2C8PbDOAeWrsbL/aHbXY/YNNOBtUWYC0I+BXVrr1yu85XT7pbptcdL9EqaUamR/7ANcAeymrveLo68Gn8fV45GYq98HgCcdXZ5zLHsbzJXsrUBcWfmBEGApsM/+u7Gjy1pN+WdgTnlLMDWKu85UduBJ+37aA4xwdPlrsC1fAtuBbfb/YM0a+rYAAzGn5tuALfafkc64X86wLc64X7oDm+1l3gH82/56ne4XufVfCCFchLM1uQghhKiGBLoQQrgICXQhhHAREuhCCOEiJNCFEMJFSKALIYSLkEAXQggX8f/nlOtoML0meQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states[['val_VAL_jac','train_VAL_jac']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "unlabel_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', ~(data['all'].train_mask + data['all'].val_mask + data['all'].test_mask)),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def predict(model, loader):\n",
    "    model.eval()\n",
    "    seed_everything(args.seed)\n",
    "    all_preds = []\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch['all'].batch_size\n",
    "        new_dict = {}\n",
    "        for edge_type in [edge_type for edge_type in batch.edge_index_dict if 'all' in edge_type and not 'simp_link' in edge_type]:\n",
    "            edge_index = batch.edge_index_dict[edge_type]\n",
    "            edge_index = to_undirected(edge_index)\n",
    "            new_dict[edge_type] = edge_index\n",
    "        batch.edge_index_dict = new_dict\n",
    "        \n",
    "        out = model(batch.x_dict, batch.edge_index_dict)[:batch_size]\n",
    "        out_att = out[:,:9].softmax(axis=1)\n",
    "        out_val = out[:,9:].softmax(axis=1)\n",
    "        IDs = batch['all'].n_id[:batch_size].unsqueeze(dim=-1).int()\n",
    "        \n",
    "        now = torch.hstack([IDs, out_att, out_val])\n",
    "        all_preds.append(now)\n",
    "    \n",
    "    final = torch.vstack(all_preds)\n",
    "        \n",
    "    return final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 27.32it/s]\n"
     ]
    }
   ],
   "source": [
    "pred_train = predict(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 41.74it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 45.45it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 38/38 [00:00<00:00, 48.49it/s]\n"
     ]
    }
   ],
   "source": [
    "pred_val = predict(model, val_loader)\n",
    "pred_test = predict(model, test_loader)\n",
    "pred_unlab = predict(model, unlabel_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = torch.vstack([pred_train, pred_val, pred_test, pred_unlab]).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([2.0000, 2.0000, 2.0000,  ..., 2.0000, 2.0000, 2.0000])"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds[:,1:].sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df = pd.DataFrame(preds).sort_values(0).set_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.016423</td>\n",
       "      <td>0.050291</td>\n",
       "      <td>0.012526</td>\n",
       "      <td>0.008743</td>\n",
       "      <td>0.110640</td>\n",
       "      <td>0.087991</td>\n",
       "      <td>0.036223</td>\n",
       "      <td>0.004730</td>\n",
       "      <td>0.672433</td>\n",
       "      <td>0.083771</td>\n",
       "      <td>0.332159</td>\n",
       "      <td>0.204477</td>\n",
       "      <td>0.234954</td>\n",
       "      <td>0.016103</td>\n",
       "      <td>0.101231</td>\n",
       "      <td>0.005806</td>\n",
       "      <td>0.003253</td>\n",
       "      <td>0.003801</td>\n",
       "      <td>0.003791</td>\n",
       "      <td>0.010653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.007282</td>\n",
       "      <td>0.021287</td>\n",
       "      <td>0.007297</td>\n",
       "      <td>0.004733</td>\n",
       "      <td>0.031022</td>\n",
       "      <td>0.035716</td>\n",
       "      <td>0.010356</td>\n",
       "      <td>0.002186</td>\n",
       "      <td>0.880121</td>\n",
       "      <td>0.145999</td>\n",
       "      <td>0.416139</td>\n",
       "      <td>0.177033</td>\n",
       "      <td>0.117582</td>\n",
       "      <td>0.012862</td>\n",
       "      <td>0.094682</td>\n",
       "      <td>0.006688</td>\n",
       "      <td>0.004986</td>\n",
       "      <td>0.004848</td>\n",
       "      <td>0.005506</td>\n",
       "      <td>0.013674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.039838</td>\n",
       "      <td>0.067240</td>\n",
       "      <td>0.025460</td>\n",
       "      <td>0.021443</td>\n",
       "      <td>0.145253</td>\n",
       "      <td>0.285262</td>\n",
       "      <td>0.100707</td>\n",
       "      <td>0.012931</td>\n",
       "      <td>0.301865</td>\n",
       "      <td>0.157047</td>\n",
       "      <td>0.288716</td>\n",
       "      <td>0.207685</td>\n",
       "      <td>0.171816</td>\n",
       "      <td>0.009803</td>\n",
       "      <td>0.137863</td>\n",
       "      <td>0.005051</td>\n",
       "      <td>0.003682</td>\n",
       "      <td>0.003071</td>\n",
       "      <td>0.003312</td>\n",
       "      <td>0.011954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.044647</td>\n",
       "      <td>0.171641</td>\n",
       "      <td>0.054057</td>\n",
       "      <td>0.013017</td>\n",
       "      <td>0.012908</td>\n",
       "      <td>0.070298</td>\n",
       "      <td>0.576590</td>\n",
       "      <td>0.045380</td>\n",
       "      <td>0.011461</td>\n",
       "      <td>0.062484</td>\n",
       "      <td>0.026733</td>\n",
       "      <td>0.193096</td>\n",
       "      <td>0.063548</td>\n",
       "      <td>0.018841</td>\n",
       "      <td>0.590197</td>\n",
       "      <td>0.011869</td>\n",
       "      <td>0.009724</td>\n",
       "      <td>0.005058</td>\n",
       "      <td>0.009313</td>\n",
       "      <td>0.009137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.240394</td>\n",
       "      <td>0.270734</td>\n",
       "      <td>0.008813</td>\n",
       "      <td>0.069668</td>\n",
       "      <td>0.012651</td>\n",
       "      <td>0.328453</td>\n",
       "      <td>0.045598</td>\n",
       "      <td>0.011963</td>\n",
       "      <td>0.011726</td>\n",
       "      <td>0.205327</td>\n",
       "      <td>0.084273</td>\n",
       "      <td>0.090498</td>\n",
       "      <td>0.212623</td>\n",
       "      <td>0.010540</td>\n",
       "      <td>0.375556</td>\n",
       "      <td>0.004466</td>\n",
       "      <td>0.003712</td>\n",
       "      <td>0.002396</td>\n",
       "      <td>0.002357</td>\n",
       "      <td>0.008251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2946.0</th>\n",
       "      <td>0.012299</td>\n",
       "      <td>0.352459</td>\n",
       "      <td>0.196328</td>\n",
       "      <td>0.019530</td>\n",
       "      <td>0.004680</td>\n",
       "      <td>0.019436</td>\n",
       "      <td>0.325722</td>\n",
       "      <td>0.056517</td>\n",
       "      <td>0.013029</td>\n",
       "      <td>0.307784</td>\n",
       "      <td>0.166586</td>\n",
       "      <td>0.258492</td>\n",
       "      <td>0.167580</td>\n",
       "      <td>0.022277</td>\n",
       "      <td>0.036914</td>\n",
       "      <td>0.007928</td>\n",
       "      <td>0.004127</td>\n",
       "      <td>0.003969</td>\n",
       "      <td>0.011200</td>\n",
       "      <td>0.013142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2947.0</th>\n",
       "      <td>0.001303</td>\n",
       "      <td>0.015353</td>\n",
       "      <td>0.350792</td>\n",
       "      <td>0.007368</td>\n",
       "      <td>0.001894</td>\n",
       "      <td>0.001741</td>\n",
       "      <td>0.567814</td>\n",
       "      <td>0.051717</td>\n",
       "      <td>0.002019</td>\n",
       "      <td>0.302574</td>\n",
       "      <td>0.133512</td>\n",
       "      <td>0.271232</td>\n",
       "      <td>0.188981</td>\n",
       "      <td>0.020561</td>\n",
       "      <td>0.044853</td>\n",
       "      <td>0.006896</td>\n",
       "      <td>0.004195</td>\n",
       "      <td>0.004115</td>\n",
       "      <td>0.010768</td>\n",
       "      <td>0.012314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2948.0</th>\n",
       "      <td>0.037760</td>\n",
       "      <td>0.610750</td>\n",
       "      <td>0.054277</td>\n",
       "      <td>0.017287</td>\n",
       "      <td>0.019161</td>\n",
       "      <td>0.025083</td>\n",
       "      <td>0.187835</td>\n",
       "      <td>0.035659</td>\n",
       "      <td>0.012188</td>\n",
       "      <td>0.213285</td>\n",
       "      <td>0.251156</td>\n",
       "      <td>0.273004</td>\n",
       "      <td>0.172218</td>\n",
       "      <td>0.019935</td>\n",
       "      <td>0.030302</td>\n",
       "      <td>0.005923</td>\n",
       "      <td>0.004361</td>\n",
       "      <td>0.004487</td>\n",
       "      <td>0.013026</td>\n",
       "      <td>0.012303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2949.0</th>\n",
       "      <td>0.204585</td>\n",
       "      <td>0.350784</td>\n",
       "      <td>0.063568</td>\n",
       "      <td>0.026831</td>\n",
       "      <td>0.020202</td>\n",
       "      <td>0.270606</td>\n",
       "      <td>0.027623</td>\n",
       "      <td>0.026520</td>\n",
       "      <td>0.009281</td>\n",
       "      <td>0.351371</td>\n",
       "      <td>0.209657</td>\n",
       "      <td>0.178272</td>\n",
       "      <td>0.188575</td>\n",
       "      <td>0.010959</td>\n",
       "      <td>0.028735</td>\n",
       "      <td>0.004664</td>\n",
       "      <td>0.004071</td>\n",
       "      <td>0.003191</td>\n",
       "      <td>0.008932</td>\n",
       "      <td>0.011574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2950.0</th>\n",
       "      <td>0.028532</td>\n",
       "      <td>0.716678</td>\n",
       "      <td>0.032341</td>\n",
       "      <td>0.019462</td>\n",
       "      <td>0.010945</td>\n",
       "      <td>0.128355</td>\n",
       "      <td>0.039859</td>\n",
       "      <td>0.010576</td>\n",
       "      <td>0.013251</td>\n",
       "      <td>0.297182</td>\n",
       "      <td>0.153409</td>\n",
       "      <td>0.249300</td>\n",
       "      <td>0.206405</td>\n",
       "      <td>0.016053</td>\n",
       "      <td>0.036428</td>\n",
       "      <td>0.007929</td>\n",
       "      <td>0.004760</td>\n",
       "      <td>0.003839</td>\n",
       "      <td>0.011723</td>\n",
       "      <td>0.012971</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2951 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              1         2         3         4         5         6         7   \\\n",
       "0                                                                              \n",
       "0.0     0.016423  0.050291  0.012526  0.008743  0.110640  0.087991  0.036223   \n",
       "1.0     0.007282  0.021287  0.007297  0.004733  0.031022  0.035716  0.010356   \n",
       "2.0     0.039838  0.067240  0.025460  0.021443  0.145253  0.285262  0.100707   \n",
       "3.0     0.044647  0.171641  0.054057  0.013017  0.012908  0.070298  0.576590   \n",
       "4.0     0.240394  0.270734  0.008813  0.069668  0.012651  0.328453  0.045598   \n",
       "...          ...       ...       ...       ...       ...       ...       ...   \n",
       "2946.0  0.012299  0.352459  0.196328  0.019530  0.004680  0.019436  0.325722   \n",
       "2947.0  0.001303  0.015353  0.350792  0.007368  0.001894  0.001741  0.567814   \n",
       "2948.0  0.037760  0.610750  0.054277  0.017287  0.019161  0.025083  0.187835   \n",
       "2949.0  0.204585  0.350784  0.063568  0.026831  0.020202  0.270606  0.027623   \n",
       "2950.0  0.028532  0.716678  0.032341  0.019462  0.010945  0.128355  0.039859   \n",
       "\n",
       "              8         9         10        11        12        13        14  \\\n",
       "0                                                                              \n",
       "0.0     0.004730  0.672433  0.083771  0.332159  0.204477  0.234954  0.016103   \n",
       "1.0     0.002186  0.880121  0.145999  0.416139  0.177033  0.117582  0.012862   \n",
       "2.0     0.012931  0.301865  0.157047  0.288716  0.207685  0.171816  0.009803   \n",
       "3.0     0.045380  0.011461  0.062484  0.026733  0.193096  0.063548  0.018841   \n",
       "4.0     0.011963  0.011726  0.205327  0.084273  0.090498  0.212623  0.010540   \n",
       "...          ...       ...       ...       ...       ...       ...       ...   \n",
       "2946.0  0.056517  0.013029  0.307784  0.166586  0.258492  0.167580  0.022277   \n",
       "2947.0  0.051717  0.002019  0.302574  0.133512  0.271232  0.188981  0.020561   \n",
       "2948.0  0.035659  0.012188  0.213285  0.251156  0.273004  0.172218  0.019935   \n",
       "2949.0  0.026520  0.009281  0.351371  0.209657  0.178272  0.188575  0.010959   \n",
       "2950.0  0.010576  0.013251  0.297182  0.153409  0.249300  0.206405  0.016053   \n",
       "\n",
       "              15        16        17        18        19        20  \n",
       "0                                                                   \n",
       "0.0     0.101231  0.005806  0.003253  0.003801  0.003791  0.010653  \n",
       "1.0     0.094682  0.006688  0.004986  0.004848  0.005506  0.013674  \n",
       "2.0     0.137863  0.005051  0.003682  0.003071  0.003312  0.011954  \n",
       "3.0     0.590197  0.011869  0.009724  0.005058  0.009313  0.009137  \n",
       "4.0     0.375556  0.004466  0.003712  0.002396  0.002357  0.008251  \n",
       "...          ...       ...       ...       ...       ...       ...  \n",
       "2946.0  0.036914  0.007928  0.004127  0.003969  0.011200  0.013142  \n",
       "2947.0  0.044853  0.006896  0.004195  0.004115  0.010768  0.012314  \n",
       "2948.0  0.030302  0.005923  0.004361  0.004487  0.013026  0.012303  \n",
       "2949.0  0.028735  0.004664  0.004071  0.003191  0.008932  0.011574  \n",
       "2950.0  0.036428  0.007929  0.004760  0.003839  0.011723  0.012971  \n",
       "\n",
       "[2951 rows x 20 columns]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df.to_csv(args.save_dir + 'preds.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "states.to_csv(args.save_dir + 'train_state.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Per-Class Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.016423</td>\n",
       "      <td>0.050291</td>\n",
       "      <td>0.012526</td>\n",
       "      <td>0.008743</td>\n",
       "      <td>0.110640</td>\n",
       "      <td>0.087991</td>\n",
       "      <td>0.036223</td>\n",
       "      <td>0.004730</td>\n",
       "      <td>0.672433</td>\n",
       "      <td>0.083771</td>\n",
       "      <td>0.332159</td>\n",
       "      <td>0.204477</td>\n",
       "      <td>0.234954</td>\n",
       "      <td>0.016103</td>\n",
       "      <td>0.101231</td>\n",
       "      <td>0.005806</td>\n",
       "      <td>0.003253</td>\n",
       "      <td>0.003801</td>\n",
       "      <td>0.003791</td>\n",
       "      <td>0.010653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.007282</td>\n",
       "      <td>0.021287</td>\n",
       "      <td>0.007297</td>\n",
       "      <td>0.004733</td>\n",
       "      <td>0.031022</td>\n",
       "      <td>0.035716</td>\n",
       "      <td>0.010356</td>\n",
       "      <td>0.002186</td>\n",
       "      <td>0.880121</td>\n",
       "      <td>0.145999</td>\n",
       "      <td>0.416139</td>\n",
       "      <td>0.177033</td>\n",
       "      <td>0.117582</td>\n",
       "      <td>0.012862</td>\n",
       "      <td>0.094682</td>\n",
       "      <td>0.006688</td>\n",
       "      <td>0.004986</td>\n",
       "      <td>0.004848</td>\n",
       "      <td>0.005506</td>\n",
       "      <td>0.013674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.039838</td>\n",
       "      <td>0.067240</td>\n",
       "      <td>0.025460</td>\n",
       "      <td>0.021443</td>\n",
       "      <td>0.145254</td>\n",
       "      <td>0.285262</td>\n",
       "      <td>0.100707</td>\n",
       "      <td>0.012931</td>\n",
       "      <td>0.301865</td>\n",
       "      <td>0.157047</td>\n",
       "      <td>0.288716</td>\n",
       "      <td>0.207685</td>\n",
       "      <td>0.171816</td>\n",
       "      <td>0.009803</td>\n",
       "      <td>0.137863</td>\n",
       "      <td>0.005051</td>\n",
       "      <td>0.003682</td>\n",
       "      <td>0.003071</td>\n",
       "      <td>0.003312</td>\n",
       "      <td>0.011954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.044647</td>\n",
       "      <td>0.171641</td>\n",
       "      <td>0.054057</td>\n",
       "      <td>0.013017</td>\n",
       "      <td>0.012908</td>\n",
       "      <td>0.070298</td>\n",
       "      <td>0.576590</td>\n",
       "      <td>0.045380</td>\n",
       "      <td>0.011461</td>\n",
       "      <td>0.062484</td>\n",
       "      <td>0.026733</td>\n",
       "      <td>0.193096</td>\n",
       "      <td>0.063548</td>\n",
       "      <td>0.018841</td>\n",
       "      <td>0.590197</td>\n",
       "      <td>0.011869</td>\n",
       "      <td>0.009724</td>\n",
       "      <td>0.005058</td>\n",
       "      <td>0.009313</td>\n",
       "      <td>0.009137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.240394</td>\n",
       "      <td>0.270734</td>\n",
       "      <td>0.008813</td>\n",
       "      <td>0.069668</td>\n",
       "      <td>0.012651</td>\n",
       "      <td>0.328453</td>\n",
       "      <td>0.045598</td>\n",
       "      <td>0.011963</td>\n",
       "      <td>0.011726</td>\n",
       "      <td>0.205327</td>\n",
       "      <td>0.084273</td>\n",
       "      <td>0.090498</td>\n",
       "      <td>0.212623</td>\n",
       "      <td>0.010540</td>\n",
       "      <td>0.375556</td>\n",
       "      <td>0.004466</td>\n",
       "      <td>0.003712</td>\n",
       "      <td>0.002396</td>\n",
       "      <td>0.002357</td>\n",
       "      <td>0.008251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2946.0</th>\n",
       "      <td>0.012299</td>\n",
       "      <td>0.352459</td>\n",
       "      <td>0.196328</td>\n",
       "      <td>0.019530</td>\n",
       "      <td>0.004680</td>\n",
       "      <td>0.019436</td>\n",
       "      <td>0.325722</td>\n",
       "      <td>0.056517</td>\n",
       "      <td>0.013029</td>\n",
       "      <td>0.307784</td>\n",
       "      <td>0.166586</td>\n",
       "      <td>0.258492</td>\n",
       "      <td>0.167580</td>\n",
       "      <td>0.022277</td>\n",
       "      <td>0.036914</td>\n",
       "      <td>0.007928</td>\n",
       "      <td>0.004127</td>\n",
       "      <td>0.003969</td>\n",
       "      <td>0.011200</td>\n",
       "      <td>0.013142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2947.0</th>\n",
       "      <td>0.001303</td>\n",
       "      <td>0.015353</td>\n",
       "      <td>0.350792</td>\n",
       "      <td>0.007368</td>\n",
       "      <td>0.001894</td>\n",
       "      <td>0.001741</td>\n",
       "      <td>0.567814</td>\n",
       "      <td>0.051717</td>\n",
       "      <td>0.002019</td>\n",
       "      <td>0.302574</td>\n",
       "      <td>0.133512</td>\n",
       "      <td>0.271232</td>\n",
       "      <td>0.188981</td>\n",
       "      <td>0.020561</td>\n",
       "      <td>0.044853</td>\n",
       "      <td>0.006896</td>\n",
       "      <td>0.004195</td>\n",
       "      <td>0.004115</td>\n",
       "      <td>0.010768</td>\n",
       "      <td>0.012314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2948.0</th>\n",
       "      <td>0.037760</td>\n",
       "      <td>0.610750</td>\n",
       "      <td>0.054277</td>\n",
       "      <td>0.017287</td>\n",
       "      <td>0.019161</td>\n",
       "      <td>0.025083</td>\n",
       "      <td>0.187835</td>\n",
       "      <td>0.035659</td>\n",
       "      <td>0.012188</td>\n",
       "      <td>0.213285</td>\n",
       "      <td>0.251156</td>\n",
       "      <td>0.273004</td>\n",
       "      <td>0.172218</td>\n",
       "      <td>0.019935</td>\n",
       "      <td>0.030302</td>\n",
       "      <td>0.005923</td>\n",
       "      <td>0.004361</td>\n",
       "      <td>0.004487</td>\n",
       "      <td>0.013026</td>\n",
       "      <td>0.012303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2949.0</th>\n",
       "      <td>0.204585</td>\n",
       "      <td>0.350784</td>\n",
       "      <td>0.063568</td>\n",
       "      <td>0.026831</td>\n",
       "      <td>0.020202</td>\n",
       "      <td>0.270606</td>\n",
       "      <td>0.027623</td>\n",
       "      <td>0.026520</td>\n",
       "      <td>0.009281</td>\n",
       "      <td>0.351371</td>\n",
       "      <td>0.209657</td>\n",
       "      <td>0.178272</td>\n",
       "      <td>0.188575</td>\n",
       "      <td>0.010959</td>\n",
       "      <td>0.028735</td>\n",
       "      <td>0.004664</td>\n",
       "      <td>0.004071</td>\n",
       "      <td>0.003191</td>\n",
       "      <td>0.008932</td>\n",
       "      <td>0.011574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2950.0</th>\n",
       "      <td>0.028532</td>\n",
       "      <td>0.716678</td>\n",
       "      <td>0.032341</td>\n",
       "      <td>0.019462</td>\n",
       "      <td>0.010945</td>\n",
       "      <td>0.128355</td>\n",
       "      <td>0.039859</td>\n",
       "      <td>0.010576</td>\n",
       "      <td>0.013251</td>\n",
       "      <td>0.297182</td>\n",
       "      <td>0.153409</td>\n",
       "      <td>0.249300</td>\n",
       "      <td>0.206405</td>\n",
       "      <td>0.016053</td>\n",
       "      <td>0.036428</td>\n",
       "      <td>0.007929</td>\n",
       "      <td>0.004760</td>\n",
       "      <td>0.003839</td>\n",
       "      <td>0.011723</td>\n",
       "      <td>0.012971</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2951 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               1         2         3         4         5         6         7  \\\n",
       "0                                                                              \n",
       "0.0     0.016423  0.050291  0.012526  0.008743  0.110640  0.087991  0.036223   \n",
       "1.0     0.007282  0.021287  0.007297  0.004733  0.031022  0.035716  0.010356   \n",
       "2.0     0.039838  0.067240  0.025460  0.021443  0.145254  0.285262  0.100707   \n",
       "3.0     0.044647  0.171641  0.054057  0.013017  0.012908  0.070298  0.576590   \n",
       "4.0     0.240394  0.270734  0.008813  0.069668  0.012651  0.328453  0.045598   \n",
       "...          ...       ...       ...       ...       ...       ...       ...   \n",
       "2946.0  0.012299  0.352459  0.196328  0.019530  0.004680  0.019436  0.325722   \n",
       "2947.0  0.001303  0.015353  0.350792  0.007368  0.001894  0.001741  0.567814   \n",
       "2948.0  0.037760  0.610750  0.054277  0.017287  0.019161  0.025083  0.187835   \n",
       "2949.0  0.204585  0.350784  0.063568  0.026831  0.020202  0.270606  0.027623   \n",
       "2950.0  0.028532  0.716678  0.032341  0.019462  0.010945  0.128355  0.039859   \n",
       "\n",
       "               8         9        10        11        12        13        14  \\\n",
       "0                                                                              \n",
       "0.0     0.004730  0.672433  0.083771  0.332159  0.204477  0.234954  0.016103   \n",
       "1.0     0.002186  0.880121  0.145999  0.416139  0.177033  0.117582  0.012862   \n",
       "2.0     0.012931  0.301865  0.157047  0.288716  0.207685  0.171816  0.009803   \n",
       "3.0     0.045380  0.011461  0.062484  0.026733  0.193096  0.063548  0.018841   \n",
       "4.0     0.011963  0.011726  0.205327  0.084273  0.090498  0.212623  0.010540   \n",
       "...          ...       ...       ...       ...       ...       ...       ...   \n",
       "2946.0  0.056517  0.013029  0.307784  0.166586  0.258492  0.167580  0.022277   \n",
       "2947.0  0.051717  0.002019  0.302574  0.133512  0.271232  0.188981  0.020561   \n",
       "2948.0  0.035659  0.012188  0.213285  0.251156  0.273004  0.172218  0.019935   \n",
       "2949.0  0.026520  0.009281  0.351371  0.209657  0.178272  0.188575  0.010959   \n",
       "2950.0  0.010576  0.013251  0.297182  0.153409  0.249300  0.206405  0.016053   \n",
       "\n",
       "              15        16        17        18        19        20  \n",
       "0                                                                   \n",
       "0.0     0.101231  0.005806  0.003253  0.003801  0.003791  0.010653  \n",
       "1.0     0.094682  0.006688  0.004986  0.004848  0.005506  0.013674  \n",
       "2.0     0.137863  0.005051  0.003682  0.003071  0.003312  0.011954  \n",
       "3.0     0.590197  0.011869  0.009724  0.005058  0.009313  0.009137  \n",
       "4.0     0.375556  0.004466  0.003712  0.002396  0.002357  0.008251  \n",
       "...          ...       ...       ...       ...       ...       ...  \n",
       "2946.0  0.036914  0.007928  0.004127  0.003969  0.011200  0.013142  \n",
       "2947.0  0.044853  0.006896  0.004195  0.004115  0.010768  0.012314  \n",
       "2948.0  0.030302  0.005923  0.004361  0.004487  0.013026  0.012303  \n",
       "2949.0  0.028735  0.004664  0.004071  0.003191  0.008932  0.011574  \n",
       "2950.0  0.036428  0.007929  0.004760  0.003839  0.011723  0.012971  \n",
       "\n",
       "[2951 rows x 20 columns]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = pd.read_csv(args.save_dir + 'preds.csv', sep='\\t', index_col='0')\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = torch.tensor(np.array(preds)).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_k_confusion_matrix(y, y_pred, k=3):\n",
    "    dim = y.shape[-1]\n",
    "    y = y.topk(k=k, axis=1)[1]\n",
    "    y_pred = y_pred.topk(k=k, axis=1)[1]\n",
    "    conf = np.zeros((dim, dim))\n",
    "    for i in range(k):\n",
    "        for j in range(k):\n",
    "            conf = np.add(conf, confusion_matrix(y[:,i], y_pred[:,j], labels = range(dim)))\n",
    "    return conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ATT_conf = confusion_matrix(data['all'].y[(data['all'].att_lab) * data['all'].test_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                                 pred[(data['all'].att_lab) * data['all'].test_mask][:,:9].argmax(axis=1).cpu(), labels = range(9))\n",
    "test_VAL_conf = confusion_matrix(data['all'].y[(data['all'].val_lab) * data['all'].test_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                                 pred[(data['all'].val_lab) * data['all'].test_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "test_VAL_conf_k = (top_k_confusion_matrix(data['all'].y[(data['all'].val_lab) * data['all'].test_mask][:,9:].cpu(),  \n",
    "                                 pred[(data['all'].val_lab) * data['all'].test_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 63,   0,   0,   0,   0,   1,   0,   0,   0],\n",
       "       [  1, 141,   0,   0,   0,   1,   2,   0,   0],\n",
       "       [  0,   0,  11,   0,   0,   0,   0,   0,   0],\n",
       "       [  2,   0,   0,  14,   0,   2,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,  71,   0,   1,   0,   0],\n",
       "       [  0,   3,   0,   0,   0,  99,   0,   0,   0],\n",
       "       [  0,   4,   0,   0,   2,   0,  71,   0,   0],\n",
       "       [  0,   0,   1,   0,   0,   0,   1,   0,   0],\n",
       "       [  1,   1,   0,   0,   0,   0,   0,   0,  10]], dtype=int64)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[51,  2,  4,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 2, 19,  1,  1,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 2,  5, 22,  3,  0,  3,  0,  0,  0,  0,  0],\n",
       "       [ 1,  3,  2, 26,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 5,  3,  1,  5,  0, 30,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0]], dtype=int64)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 91,  54,  54,  43,   0,  40,   0,   0,   0,   0,   0],\n",
       "       [ 52,  90,  75,  64,   0,  28,   0,   0,   0,   0,   0],\n",
       "       [ 59,  83, 117,  90,   0,  59,   0,   0,   0,   0,   0],\n",
       "       [ 58,  78,  94, 117,   0,  52,   0,   0,   0,   0,   0],\n",
       "       [  0,   3,   3,   3,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [ 48,  37,  73,  73,   0,  78,   0,   0,   0,   0,   0],\n",
       "       [  1,   0,   2,   0,   0,   2,   0,   0,   0,   1,   0],\n",
       "       [  0,   0,   1,   0,   0,   1,   0,   0,   0,   1,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   1,   0,   0,   1,   0,   0,   0,   1,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0]])"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_ATT_conf = confusion_matrix(data['all'].y[(data['all'].att_lab) * data['all'].val_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                                 pred[(data['all'].att_lab) * data['all'].val_mask][:,:9].argmax(axis=1).cpu(), labels = range(9))\n",
    "val_VAL_conf = confusion_matrix(data['all'].y[(data['all'].val_lab) * data['all'].val_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                                 pred[(data['all'].val_lab) * data['all'].val_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "val_VAL_conf_k = (top_k_confusion_matrix(data['all'].y[(data['all'].val_lab) * data['all'].val_mask][:,9:].cpu(),  \n",
    "                                 pred[(data['all'].val_lab) * data['all'].val_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 61,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0, 144,   0,   0,   0,   1,   1,   0,   0],\n",
       "       [  0,   0,   8,   0,   0,   0,   0,   0,   0],\n",
       "       [  1,   1,   0,  20,   0,   0,   0,   0,   0],\n",
       "       [  0,   1,   0,   0,  74,   0,   0,   0,   0],\n",
       "       [  0,   1,   0,   0,   0, 102,   0,   0,   0],\n",
       "       [  0,   1,   1,   0,   0,   0,  60,   0,   0],\n",
       "       [  0,   0,   1,   0,   0,   0,   4,   0,   0],\n",
       "       [  0,   1,   0,   0,   2,   0,   0,   0,   7]], dtype=int64)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[66,  3,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 1, 17,  0,  1,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  1, 26,  2,  0,  2,  0,  0,  0,  0,  0],\n",
       "       [ 1,  1,  0, 31,  0,  1,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 5,  4,  4,  1,  0, 36,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0]], dtype=int64)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[104,  63,  54,  60,   0,  43,   0,   0,   0,   0,   0],\n",
       "       [ 62,  91,  61,  70,   1,  36,   0,   0,   0,   0,   0],\n",
       "       [ 59,  58, 114,  91,   0,  71,   0,   0,   0,   0,   0],\n",
       "       [ 70,  80,  99, 120,   1,  56,   0,   0,   0,   0,   0],\n",
       "       [  0,   8,   7,   8,   1,   0,   0,   0,   0,   0,   0],\n",
       "       [ 53,  42,  88,  68,   0,  88,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0]])"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat([pd.DataFrame(val_ATT_conf),pd.DataFrame(test_ATT_conf)],axis=1).to_csv(args.save_dir+'confusion_matrix_ATT.csv')\n",
    "pd.concat([pd.DataFrame(val_VAL_conf),pd.DataFrame(test_VAL_conf)],axis=1).to_csv(args.save_dir+'confusion_matrix_VAL.csv')\n",
    "pd.concat([pd.DataFrame(val_VAL_conf_k),pd.DataFrame(test_VAL_conf_k)],axis=1).to_csv(args.save_dir+'confusion_matrix_VAL_k.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics(confusion_matrix, classes):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val/(row.sum()+0.000000001)\n",
    "        recall = val/(col.sum()+0.000000001)\n",
    "        F1 = 2*(precision*recall)/(precision+recall+0.000000001)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics_k(confusion_matrix, classes, k=3):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val*k/(row.sum()+0.000000001)\n",
    "        recall = val*k/(col.sum()+0.000000001)\n",
    "        F1 = 2*(precision*recall)/(precision+recall+0.000000001)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = ['Criterion i', 'Criterion ii', 'Criterion iii', 'Criterion iv', 'Criterion v', 'Criterion vi', \n",
    "              'Criterion vii', 'Criterion viii', 'Criterion ix', 'Criterion x', 'Others']\n",
    "categories = ['Building Elements',\n",
    " 'Urban Form Elements',\n",
    " 'Gastronomy',\n",
    " 'Interior Scenery',\n",
    " 'Natural Features and Land-scape Scenery',\n",
    " 'Monuments and Buildings',\n",
    " 'Peoples Activity and Association',\n",
    " 'Artifact Products',\n",
    " 'Urban Scenery']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_dict = {}\n",
    "metrics_dict['test_ATT'] = per_class_metrics(test_ATT_conf, categories)\n",
    "metrics_dict['val_ATT'] = per_class_metrics(val_ATT_conf, categories)\n",
    "metrics_dict['test_VAL'] = per_class_metrics(test_VAL_conf, classes)\n",
    "metrics_dict['val_VAL'] = per_class_metrics(val_VAL_conf, classes)\n",
    "metrics_dict['test_VAL_k'] = per_class_metrics_k(test_VAL_conf_k, classes)\n",
    "metrics_dict['val_VAL_k'] = per_class_metrics_k(val_VAL_conf_k, classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df = pd.DataFrame.from_dict({(i,j): metrics_dict[i][j] \n",
    "                           for i in metrics_dict.keys() \n",
    "                           for j in metrics_dict[i].keys()},\n",
    "                       orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df.to_csv(args.save_dir+'per_class_metrics.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TEM links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 6 for key in data.edge_types if 'TEM_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].train_mask),\n",
    ")\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 6 for key in data.edge_types if 'TEM_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].val_mask),\n",
    ")\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 6 for key in data.edge_types if 'TEM_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=1840,\n",
       "    x=[1840, 1753],\n",
       "    y=[1840, 20],\n",
       "    node_type=[1840],\n",
       "    att_lab=[1840],\n",
       "    val_lab=[1840],\n",
       "    train_mask=[1840],\n",
       "    val_mask=[1840],\n",
       "    test_mask=[1840],\n",
       "    n_id=[1840],\n",
       "    batch_size=32\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 40865],\n",
       "    edge_attr=[40865]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_hetero_data = next(iter(train_loader))\n",
    "batch = sampled_hetero_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroGNN_L(\n",
       "  (convs): ModuleList(\n",
       "    (0): HeteroConv(num_relations=3)\n",
       "    (1): HeteroConv(num_relations=3)\n",
       "    (2): HeteroConv(num_relations=3)\n",
       "  )\n",
       "  (lin1): Linear(1753, 32, bias=True)\n",
       "  (lin2): Linear(64, 20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 20.91it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.1346464411373614,\n",
       " 1.791048172108024,\n",
       " 83.10249307479225,\n",
       " 93.35180055401662,\n",
       " 0.5415512623879388,\n",
       " 54.016620498614955)"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 39.11it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.2173465841669378,\n",
       " 1.7932460578204377,\n",
       " 84.34959349593495,\n",
       " 88.66995073891626,\n",
       " 0.5566502557012248,\n",
       " 53.69458128078818)"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.68it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.2452885824927753,\n",
       " 1.8158961217850447,\n",
       " 80.9145129224652,\n",
       " 87.5,\n",
       " 0.5212673656642437,\n",
       " 56.25)"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.51it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 66.53it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 68.52it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 69.60it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 69.86it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 70.12it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 70.29it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 68.68it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 57.59it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 63.88it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 68.04it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 68.81it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 65.69it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 55.59it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 69.42it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 70.98it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 71.37it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 64.99it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 66.67it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 68.55it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test(model, val_loader))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test(model, test_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.216514</td>\n",
       "      <td>1.793131</td>\n",
       "      <td>84.390244</td>\n",
       "      <td>8.866995e+01</td>\n",
       "      <td>0.553038</td>\n",
       "      <td>54.039409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000174</td>\n",
       "      <td>0.000216</td>\n",
       "      <td>0.299945</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.002514</td>\n",
       "      <td>0.616584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.216272</td>\n",
       "      <td>1.792778</td>\n",
       "      <td>83.739837</td>\n",
       "      <td>8.866995e+01</td>\n",
       "      <td>0.549261</td>\n",
       "      <td>53.201970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.216383</td>\n",
       "      <td>1.792933</td>\n",
       "      <td>84.349593</td>\n",
       "      <td>8.866995e+01</td>\n",
       "      <td>0.551724</td>\n",
       "      <td>53.694581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.216483</td>\n",
       "      <td>1.793219</td>\n",
       "      <td>84.349593</td>\n",
       "      <td>8.866995e+01</td>\n",
       "      <td>0.553777</td>\n",
       "      <td>53.694581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.216677</td>\n",
       "      <td>1.793257</td>\n",
       "      <td>84.502033</td>\n",
       "      <td>8.866995e+01</td>\n",
       "      <td>0.554187</td>\n",
       "      <td>54.556650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.216766</td>\n",
       "      <td>1.793409</td>\n",
       "      <td>84.959350</td>\n",
       "      <td>8.866995e+01</td>\n",
       "      <td>0.556650</td>\n",
       "      <td>55.172414</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc     VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  1.000000e+01  10.000000  10.000000\n",
       "mean    1.216514   1.793131  84.390244  8.866995e+01   0.553038  54.039409\n",
       "std     0.000174   0.000216   0.299945  1.497956e-14   0.002514   0.616584\n",
       "min     1.216272   1.792778  83.739837  8.866995e+01   0.549261  53.201970\n",
       "25%     1.216383   1.792933  84.349593  8.866995e+01   0.551724  53.694581\n",
       "50%     1.216483   1.793219  84.349593  8.866995e+01   0.553777  53.694581\n",
       "75%     1.216677   1.793257  84.502033  8.866995e+01   0.554187  54.556650\n",
       "max     1.216766   1.793409  84.959350  8.866995e+01   0.556650  55.172414"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.244264</td>\n",
       "      <td>1.816401</td>\n",
       "      <td>81.312127</td>\n",
       "      <td>87.708333</td>\n",
       "      <td>0.521224</td>\n",
       "      <td>56.145833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000207</td>\n",
       "      <td>0.000193</td>\n",
       "      <td>0.209561</td>\n",
       "      <td>0.439205</td>\n",
       "      <td>0.002831</td>\n",
       "      <td>0.219603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.243927</td>\n",
       "      <td>1.815987</td>\n",
       "      <td>81.113320</td>\n",
       "      <td>86.979167</td>\n",
       "      <td>0.517361</td>\n",
       "      <td>55.729167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.244172</td>\n",
       "      <td>1.816305</td>\n",
       "      <td>81.113320</td>\n",
       "      <td>87.500000</td>\n",
       "      <td>0.518989</td>\n",
       "      <td>56.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.244201</td>\n",
       "      <td>1.816427</td>\n",
       "      <td>81.312127</td>\n",
       "      <td>88.020833</td>\n",
       "      <td>0.520399</td>\n",
       "      <td>56.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.244376</td>\n",
       "      <td>1.816547</td>\n",
       "      <td>81.461233</td>\n",
       "      <td>88.020833</td>\n",
       "      <td>0.524197</td>\n",
       "      <td>56.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.244604</td>\n",
       "      <td>1.816600</td>\n",
       "      <td>81.709742</td>\n",
       "      <td>88.020833</td>\n",
       "      <td>0.524740</td>\n",
       "      <td>56.250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    1.244264   1.816401  81.312127  87.708333   0.521224  56.145833\n",
       "std     0.000207   0.000193   0.209561   0.439205   0.002831   0.219603\n",
       "min     1.243927   1.815987  81.113320  86.979167   0.517361  55.729167\n",
       "25%     1.244172   1.816305  81.113320  87.500000   0.518989  56.250000\n",
       "50%     1.244201   1.816427  81.312127  88.020833   0.520399  56.250000\n",
       "75%     1.244376   1.816547  81.461233  88.020833   0.524197  56.250000\n",
       "max     1.244604   1.816600  81.709742  88.020833   0.524740  56.250000"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'TEM_val_metrics_transfer.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'TEM_test_metrics_transfer.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SPA links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 6 for key in data.edge_types if 'SPA_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].train_mask),\n",
    ")\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 6 for key in data.edge_types if 'SPA_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].val_mask),\n",
    ")\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 6 for key in data.edge_types if 'SPA_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=2702,\n",
       "    x=[2702, 1753],\n",
       "    y=[2702, 20],\n",
       "    node_type=[2702],\n",
       "    att_lab=[2702],\n",
       "    val_lab=[2702],\n",
       "    train_mask=[2702],\n",
       "    val_mask=[2702],\n",
       "    test_mask=[2702],\n",
       "    n_id=[2702],\n",
       "    batch_size=32\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 64239],\n",
       "    edge_attr=[64239]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_hetero_data = next(iter(train_loader))\n",
    "batch = sampled_hetero_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 19.27it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.2329906126138577,\n",
       " 1.7856713189973066,\n",
       " 72.57617728531856,\n",
       " 88.36565096952909,\n",
       " 0.4893813569129669,\n",
       " 47.64542936288089)"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 28.85it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.4011133589395663,\n",
       " 1.7888418289240946,\n",
       " 64.02439024390245,\n",
       " 81.77339901477832,\n",
       " 0.46469622644884834,\n",
       " 45.320197044334975)"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 27.67it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.4115610653553048,\n",
       " 1.817808320124944,\n",
       " 63.02186878727634,\n",
       " 86.45833333333333,\n",
       " 0.4440104228754838,\n",
       " 51.041666666666664)"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.34it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.44it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.08it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.93it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.67it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.30it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.68it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 37.83it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.09it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.33it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.34it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.61it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.46it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.77it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.12it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.28it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.89it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 39.33it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.65it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 38.99it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test(model, val_loader,'val'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test(model, test_loader,'val'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.401637</td>\n",
       "      <td>1.788999</td>\n",
       "      <td>64.126016</td>\n",
       "      <td>82.019704</td>\n",
       "      <td>0.469951</td>\n",
       "      <td>45.369458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000578</td>\n",
       "      <td>0.000306</td>\n",
       "      <td>0.239535</td>\n",
       "      <td>0.348328</td>\n",
       "      <td>0.004365</td>\n",
       "      <td>0.279629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.400590</td>\n",
       "      <td>1.788591</td>\n",
       "      <td>63.821138</td>\n",
       "      <td>81.773399</td>\n",
       "      <td>0.465517</td>\n",
       "      <td>44.827586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.401324</td>\n",
       "      <td>1.788725</td>\n",
       "      <td>63.871951</td>\n",
       "      <td>81.773399</td>\n",
       "      <td>0.466338</td>\n",
       "      <td>45.320197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.401658</td>\n",
       "      <td>1.789015</td>\n",
       "      <td>64.227642</td>\n",
       "      <td>81.773399</td>\n",
       "      <td>0.469622</td>\n",
       "      <td>45.320197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.401967</td>\n",
       "      <td>1.789168</td>\n",
       "      <td>64.227642</td>\n",
       "      <td>82.266010</td>\n",
       "      <td>0.472701</td>\n",
       "      <td>45.320197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.402671</td>\n",
       "      <td>1.789471</td>\n",
       "      <td>64.430894</td>\n",
       "      <td>82.758621</td>\n",
       "      <td>0.477833</td>\n",
       "      <td>45.812808</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    1.401637   1.788999  64.126016  82.019704   0.469951  45.369458\n",
       "std     0.000578   0.000306   0.239535   0.348328   0.004365   0.279629\n",
       "min     1.400590   1.788591  63.821138  81.773399   0.465517  44.827586\n",
       "25%     1.401324   1.788725  63.871951  81.773399   0.466338  45.320197\n",
       "50%     1.401658   1.789015  64.227642  81.773399   0.469622  45.320197\n",
       "75%     1.401967   1.789168  64.227642  82.266010   0.472701  45.320197\n",
       "max     1.402671   1.789471  64.430894  82.758621   0.477833  45.812808"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.410991</td>\n",
       "      <td>1.817671</td>\n",
       "      <td>63.121272</td>\n",
       "      <td>8.645833e+01</td>\n",
       "      <td>0.449826</td>\n",
       "      <td>51.510417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000676</td>\n",
       "      <td>0.000258</td>\n",
       "      <td>0.285034</td>\n",
       "      <td>1.497956e-14</td>\n",
       "      <td>0.004312</td>\n",
       "      <td>0.573180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.410006</td>\n",
       "      <td>1.817257</td>\n",
       "      <td>62.624254</td>\n",
       "      <td>8.645833e+01</td>\n",
       "      <td>0.442274</td>\n",
       "      <td>50.520833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.410490</td>\n",
       "      <td>1.817515</td>\n",
       "      <td>63.021869</td>\n",
       "      <td>8.645833e+01</td>\n",
       "      <td>0.447483</td>\n",
       "      <td>51.041667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.411199</td>\n",
       "      <td>1.817665</td>\n",
       "      <td>63.121272</td>\n",
       "      <td>8.645833e+01</td>\n",
       "      <td>0.449653</td>\n",
       "      <td>51.562500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.411329</td>\n",
       "      <td>1.817803</td>\n",
       "      <td>63.220676</td>\n",
       "      <td>8.645833e+01</td>\n",
       "      <td>0.452257</td>\n",
       "      <td>52.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.412103</td>\n",
       "      <td>1.818112</td>\n",
       "      <td>63.618290</td>\n",
       "      <td>8.645833e+01</td>\n",
       "      <td>0.457899</td>\n",
       "      <td>52.083333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc     VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  1.000000e+01  10.000000  10.000000\n",
       "mean    1.410991   1.817671  63.121272  8.645833e+01   0.449826  51.510417\n",
       "std     0.000676   0.000258   0.285034  1.497956e-14   0.004312   0.573180\n",
       "min     1.410006   1.817257  62.624254  8.645833e+01   0.442274  50.520833\n",
       "25%     1.410490   1.817515  63.021869  8.645833e+01   0.447483  51.041667\n",
       "50%     1.411199   1.817665  63.121272  8.645833e+01   0.449653  51.562500\n",
       "75%     1.411329   1.817803  63.220676  8.645833e+01   0.452257  52.083333\n",
       "max     1.412103   1.818112  63.618290  8.645833e+01   0.457899  52.083333"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'SPA_val_metrics_transfer.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'SPA_test_metrics_transfer.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SOC links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 6 for key in data.edge_types if 'SOC_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].train_mask),\n",
    ")\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 6 for key in data.edge_types if 'SOC_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].val_mask),\n",
    ")\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 6 for key in data.edge_types if 'SOC_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=2351,\n",
       "    x=[2351, 1753],\n",
       "    y=[2351, 20],\n",
       "    node_type=[2351],\n",
       "    att_lab=[2351],\n",
       "    val_lab=[2351],\n",
       "    train_mask=[2351],\n",
       "    val_mask=[2351],\n",
       "    test_mask=[2351],\n",
       "    n_id=[2351],\n",
       "    batch_size=32\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 55179],\n",
       "    edge_attr=[55179]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_hetero_data = next(iter(train_loader))\n",
    "batch = sampled_hetero_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroGNN_L(\n",
       "  (convs): ModuleList(\n",
       "    (0): HeteroConv(num_relations=3)\n",
       "    (1): HeteroConv(num_relations=3)\n",
       "    (2): HeteroConv(num_relations=3)\n",
       "  )\n",
       "  (lin1): Linear(1753, 32, bias=True)\n",
       "  (lin2): Linear(64, 20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 27.53it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.26060254421921,\n",
       " 1.6852095067666177,\n",
       " 76.17728531855956,\n",
       " 95.56786703601108,\n",
       " 0.6172668636670734,\n",
       " 68.69806094182826)"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 27.22it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.4632786357305883,\n",
       " 1.6922608925203972,\n",
       " 61.78861788617886,\n",
       " 96.05911330049261,\n",
       " 0.6658456525191885,\n",
       " 71.42857142857143)"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 42.40it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.4651099559564003,\n",
       " 1.7077636308968067,\n",
       " 64.01590457256461,\n",
       " 96.35416666666667,\n",
       " 0.626736119389534,\n",
       " 70.83333333333333)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 27.21it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 44.35it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 44.45it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 45.09it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 44.83it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 44.85it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 44.70it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 44.53it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 42.99it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 44.24it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 44.84it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 44.20it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 43.24it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 41.08it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 43.60it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 43.99it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 43.92it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 44.03it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 42.50it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 43.47it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test(model, val_loader))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test(model, test_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.462812</td>\n",
       "      <td>1.692444</td>\n",
       "      <td>62.337398</td>\n",
       "      <td>95.665025</td>\n",
       "      <td>0.667241</td>\n",
       "      <td>71.773399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000374</td>\n",
       "      <td>0.000268</td>\n",
       "      <td>0.271848</td>\n",
       "      <td>0.388577</td>\n",
       "      <td>0.003506</td>\n",
       "      <td>0.405553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.462320</td>\n",
       "      <td>1.692182</td>\n",
       "      <td>61.991870</td>\n",
       "      <td>95.073892</td>\n",
       "      <td>0.660920</td>\n",
       "      <td>71.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.462455</td>\n",
       "      <td>1.692241</td>\n",
       "      <td>62.195122</td>\n",
       "      <td>95.566502</td>\n",
       "      <td>0.667693</td>\n",
       "      <td>71.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.462832</td>\n",
       "      <td>1.692363</td>\n",
       "      <td>62.296748</td>\n",
       "      <td>95.566502</td>\n",
       "      <td>0.668309</td>\n",
       "      <td>71.674877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.463087</td>\n",
       "      <td>1.692525</td>\n",
       "      <td>62.550813</td>\n",
       "      <td>96.059113</td>\n",
       "      <td>0.668309</td>\n",
       "      <td>71.921182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.463352</td>\n",
       "      <td>1.693015</td>\n",
       "      <td>62.804878</td>\n",
       "      <td>96.059113</td>\n",
       "      <td>0.670772</td>\n",
       "      <td>72.413793</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    1.462812   1.692444  62.337398  95.665025   0.667241  71.773399\n",
       "std     0.000374   0.000268   0.271848   0.388577   0.003506   0.405553\n",
       "min     1.462320   1.692182  61.991870  95.073892   0.660920  71.428571\n",
       "25%     1.462455   1.692241  62.195122  95.566502   0.667693  71.428571\n",
       "50%     1.462832   1.692363  62.296748  95.566502   0.668309  71.674877\n",
       "75%     1.463087   1.692525  62.550813  96.059113   0.668309  71.921182\n",
       "max     1.463352   1.693015  62.804878  96.059113   0.670772  72.413793"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.464345</td>\n",
       "      <td>1.707703</td>\n",
       "      <td>64.055666</td>\n",
       "      <td>96.406250</td>\n",
       "      <td>0.626997</td>\n",
       "      <td>70.885417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000621</td>\n",
       "      <td>0.000166</td>\n",
       "      <td>0.278014</td>\n",
       "      <td>0.295649</td>\n",
       "      <td>0.006015</td>\n",
       "      <td>0.623552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.463021</td>\n",
       "      <td>1.707392</td>\n",
       "      <td>63.618290</td>\n",
       "      <td>95.833333</td>\n",
       "      <td>0.621528</td>\n",
       "      <td>70.312500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.464027</td>\n",
       "      <td>1.707655</td>\n",
       "      <td>63.866799</td>\n",
       "      <td>96.354167</td>\n",
       "      <td>0.622179</td>\n",
       "      <td>70.442708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.464511</td>\n",
       "      <td>1.707746</td>\n",
       "      <td>64.214712</td>\n",
       "      <td>96.354167</td>\n",
       "      <td>0.624566</td>\n",
       "      <td>70.833333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.464718</td>\n",
       "      <td>1.707804</td>\n",
       "      <td>64.214712</td>\n",
       "      <td>96.354167</td>\n",
       "      <td>0.629123</td>\n",
       "      <td>70.833333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.465175</td>\n",
       "      <td>1.707895</td>\n",
       "      <td>64.413519</td>\n",
       "      <td>96.875000</td>\n",
       "      <td>0.638021</td>\n",
       "      <td>72.395833</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    1.464345   1.707703  64.055666  96.406250   0.626997  70.885417\n",
       "std     0.000621   0.000166   0.278014   0.295649   0.006015   0.623552\n",
       "min     1.463021   1.707392  63.618290  95.833333   0.621528  70.312500\n",
       "25%     1.464027   1.707655  63.866799  96.354167   0.622179  70.442708\n",
       "50%     1.464511   1.707746  64.214712  96.354167   0.624566  70.833333\n",
       "75%     1.464718   1.707804  64.214712  96.354167   0.629123  70.833333\n",
       "max     1.465175   1.707895  64.413519  96.875000   0.638021  72.395833"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'SOC_val_metrics_transfer.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'SOC_test_metrics_transfer.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NO TEM links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 3 for key in data.edge_types if 'SOC_link' in key or 'SPA_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].train_mask),\n",
    ")\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 3 for key in data.edge_types if 'SOC_link' in key or 'SPA_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].val_mask),\n",
    ")\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 3 for key in data.edge_types if 'SOC_link' in key or 'SPA_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=2945,\n",
       "    x=[2945, 1753],\n",
       "    y=[2945, 20],\n",
       "    node_type=[2945],\n",
       "    att_lab=[2945],\n",
       "    val_lab=[2945],\n",
       "    train_mask=[2945],\n",
       "    val_mask=[2945],\n",
       "    test_mask=[2945],\n",
       "    n_id=[2945],\n",
       "    batch_size=32\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 64047],\n",
       "    edge_attr=[64047]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 68310],\n",
       "    edge_attr=[68310]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_hetero_data = next(iter(train_loader))\n",
    "batch = sampled_hetero_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroGNN_L(\n",
       "  (convs): ModuleList(\n",
       "    (0): HeteroConv(num_relations=3)\n",
       "    (1): HeteroConv(num_relations=3)\n",
       "    (2): HeteroConv(num_relations=3)\n",
       "  )\n",
       "  (lin1): Linear(1753, 32, bias=True)\n",
       "  (lin2): Linear(64, 20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 15.92it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.0241467265871422,\n",
       " 1.6519027872429,\n",
       " 90.58171745152355,\n",
       " 96.39889196675901,\n",
       " 0.6445060106525791,\n",
       " 69.25207756232687)"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 18.66it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.227570243240372,\n",
       " 1.6624594938578865,\n",
       " 71.7479674796748,\n",
       " 94.08866995073892,\n",
       " 0.6535303792342764,\n",
       " 73.89162561576354)"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 17.61it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.2317539114601332,\n",
       " 1.6801333762705326,\n",
       " 72.96222664015905,\n",
       " 94.79166666666667,\n",
       " 0.6276041766007742,\n",
       " 71.875)"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 17.60it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.00it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.19it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.08it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.79it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.18it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.97it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.93it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.46it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.85it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.07it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.06it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.19it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.06it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.83it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.59it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.67it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.90it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.99it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.94it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test(model, val_loader))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test(model, test_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.227966</td>\n",
       "      <td>1.662532</td>\n",
       "      <td>72.276423</td>\n",
       "      <td>93.349754</td>\n",
       "      <td>0.644910</td>\n",
       "      <td>73.448276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.001294</td>\n",
       "      <td>0.000163</td>\n",
       "      <td>0.290618</td>\n",
       "      <td>0.706266</td>\n",
       "      <td>0.006386</td>\n",
       "      <td>0.912771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.226522</td>\n",
       "      <td>1.662300</td>\n",
       "      <td>71.747967</td>\n",
       "      <td>91.625616</td>\n",
       "      <td>0.633826</td>\n",
       "      <td>71.921182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.226974</td>\n",
       "      <td>1.662429</td>\n",
       "      <td>72.154472</td>\n",
       "      <td>93.103448</td>\n",
       "      <td>0.642036</td>\n",
       "      <td>72.906404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.227592</td>\n",
       "      <td>1.662481</td>\n",
       "      <td>72.256098</td>\n",
       "      <td>93.596059</td>\n",
       "      <td>0.643268</td>\n",
       "      <td>73.152709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.228927</td>\n",
       "      <td>1.662637</td>\n",
       "      <td>72.560976</td>\n",
       "      <td>93.596059</td>\n",
       "      <td>0.650657</td>\n",
       "      <td>74.261084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.230421</td>\n",
       "      <td>1.662828</td>\n",
       "      <td>72.560976</td>\n",
       "      <td>94.088670</td>\n",
       "      <td>0.654351</td>\n",
       "      <td>74.876847</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    1.227966   1.662532  72.276423  93.349754   0.644910  73.448276\n",
       "std     0.001294   0.000163   0.290618   0.706266   0.006386   0.912771\n",
       "min     1.226522   1.662300  71.747967  91.625616   0.633826  71.921182\n",
       "25%     1.226974   1.662429  72.154472  93.103448   0.642036  72.906404\n",
       "50%     1.227592   1.662481  72.256098  93.596059   0.643268  73.152709\n",
       "75%     1.228927   1.662637  72.560976  93.596059   0.650657  74.261084\n",
       "max     1.230421   1.662828  72.560976  94.088670   0.654351  74.876847"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.230208</td>\n",
       "      <td>1.679670</td>\n",
       "      <td>73.200795</td>\n",
       "      <td>95.000000</td>\n",
       "      <td>0.633160</td>\n",
       "      <td>72.291667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000689</td>\n",
       "      <td>0.000209</td>\n",
       "      <td>0.466715</td>\n",
       "      <td>0.364170</td>\n",
       "      <td>0.004669</td>\n",
       "      <td>0.329404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.229123</td>\n",
       "      <td>1.679351</td>\n",
       "      <td>72.564612</td>\n",
       "      <td>94.270833</td>\n",
       "      <td>0.627604</td>\n",
       "      <td>71.875000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.229724</td>\n",
       "      <td>1.679541</td>\n",
       "      <td>72.962227</td>\n",
       "      <td>94.791667</td>\n",
       "      <td>0.628472</td>\n",
       "      <td>72.005208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.230443</td>\n",
       "      <td>1.679656</td>\n",
       "      <td>73.161034</td>\n",
       "      <td>95.052083</td>\n",
       "      <td>0.634549</td>\n",
       "      <td>72.395833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.230591</td>\n",
       "      <td>1.679814</td>\n",
       "      <td>73.310139</td>\n",
       "      <td>95.312500</td>\n",
       "      <td>0.637370</td>\n",
       "      <td>72.395833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.231208</td>\n",
       "      <td>1.680043</td>\n",
       "      <td>74.155070</td>\n",
       "      <td>95.312500</td>\n",
       "      <td>0.638889</td>\n",
       "      <td>72.916667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    1.230208   1.679670  73.200795  95.000000   0.633160  72.291667\n",
       "std     0.000689   0.000209   0.466715   0.364170   0.004669   0.329404\n",
       "min     1.229123   1.679351  72.564612  94.270833   0.627604  71.875000\n",
       "25%     1.229724   1.679541  72.962227  94.791667   0.628472  72.005208\n",
       "50%     1.230443   1.679656  73.161034  95.052083   0.634549  72.395833\n",
       "75%     1.230591   1.679814  73.310139  95.312500   0.637370  72.395833\n",
       "max     1.231208   1.680043  74.155070  95.312500   0.638889  72.916667"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'NO_TEM_val_metrics_transfer.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'NO_TEM_test_metrics_transfer.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NO SPA links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 3 for key in data.edge_types if 'SOC_link' in key or 'TEM_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].train_mask),\n",
    ")\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 3 for key in data.edge_types if 'SOC_link' in key or 'TEM_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].val_mask),\n",
    ")\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 3 for key in data.edge_types if 'SOC_link' in key or 'TEM_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=2934,\n",
       "    x=[2934, 1753],\n",
       "    y=[2934, 20],\n",
       "    node_type=[2934],\n",
       "    att_lab=[2934],\n",
       "    val_lab=[2934],\n",
       "    train_mask=[2934],\n",
       "    val_mask=[2934],\n",
       "    test_mask=[2934],\n",
       "    n_id=[2934],\n",
       "    batch_size=32\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 53615],\n",
       "    edge_attr=[53615]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 54054],\n",
       "    edge_attr=[54054]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_hetero_data = next(iter(train_loader))\n",
    "batch = sampled_hetero_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroGNN_L(\n",
       "  (convs): ModuleList(\n",
       "    (0): HeteroConv(num_relations=3)\n",
       "    (1): HeteroConv(num_relations=3)\n",
       "    (2): HeteroConv(num_relations=3)\n",
       "  )\n",
       "  (lin1): Linear(1753, 32, bias=True)\n",
       "  (lin2): Linear(64, 20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 18.78it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.9038548145928211,\n",
       " 1.6474158152979166,\n",
       " 98.06094182825485,\n",
       " 98.33795013850416,\n",
       " 0.695752548048701,\n",
       " 75.34626038781164)"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 26.42it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.028695691891802,\n",
       " 1.6605011612323706,\n",
       " 90.65040650406505,\n",
       " 99.01477832512315,\n",
       " 0.6888341621812342,\n",
       " 74.38423645320196)"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, val_loader,'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 25.56it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.0472979101224638,\n",
       " 1.6775725428014994,\n",
       " 89.86083499005964,\n",
       " 98.95833333333333,\n",
       " 0.6553819527228674,\n",
       " 72.39583333333333)"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, test_loader,'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 20.09it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 26.66it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 26.52it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 26.81it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 26.30it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 26.76it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 26.92it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 26.63it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 26.16it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 26.38it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 26.24it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 26.84it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 25.64it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 26.61it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 26.25it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 26.29it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 24.87it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 26.22it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 24.17it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 25.76it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test(model, val_loader))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test(model, test_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.029012</td>\n",
       "      <td>1.660474</td>\n",
       "      <td>90.264228</td>\n",
       "      <td>98.866995</td>\n",
       "      <td>0.685961</td>\n",
       "      <td>74.581281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000410</td>\n",
       "      <td>0.000279</td>\n",
       "      <td>0.261521</td>\n",
       "      <td>0.467332</td>\n",
       "      <td>0.004648</td>\n",
       "      <td>0.415406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.028261</td>\n",
       "      <td>1.659906</td>\n",
       "      <td>89.837398</td>\n",
       "      <td>97.536946</td>\n",
       "      <td>0.678982</td>\n",
       "      <td>73.891626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.028819</td>\n",
       "      <td>1.660371</td>\n",
       "      <td>90.243902</td>\n",
       "      <td>99.014778</td>\n",
       "      <td>0.683292</td>\n",
       "      <td>74.384236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.029046</td>\n",
       "      <td>1.660558</td>\n",
       "      <td>90.243902</td>\n",
       "      <td>99.014778</td>\n",
       "      <td>0.683908</td>\n",
       "      <td>74.876847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.029073</td>\n",
       "      <td>1.660680</td>\n",
       "      <td>90.447154</td>\n",
       "      <td>99.014778</td>\n",
       "      <td>0.689450</td>\n",
       "      <td>74.876847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.029813</td>\n",
       "      <td>1.660774</td>\n",
       "      <td>90.650407</td>\n",
       "      <td>99.014778</td>\n",
       "      <td>0.693760</td>\n",
       "      <td>74.876847</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    1.029012   1.660474  90.264228  98.866995   0.685961  74.581281\n",
       "std     0.000410   0.000279   0.261521   0.467332   0.004648   0.415406\n",
       "min     1.028261   1.659906  89.837398  97.536946   0.678982  73.891626\n",
       "25%     1.028819   1.660371  90.243902  99.014778   0.683292  74.384236\n",
       "50%     1.029046   1.660558  90.243902  99.014778   0.683908  74.876847\n",
       "75%     1.029073   1.660680  90.447154  99.014778   0.689450  74.876847\n",
       "max     1.029813   1.660774  90.650407  99.014778   0.693760  74.876847"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.045348</td>\n",
       "      <td>1.677649</td>\n",
       "      <td>89.860835</td>\n",
       "      <td>99.010417</td>\n",
       "      <td>0.658247</td>\n",
       "      <td>72.447917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000584</td>\n",
       "      <td>0.000154</td>\n",
       "      <td>0.187437</td>\n",
       "      <td>0.164702</td>\n",
       "      <td>0.003592</td>\n",
       "      <td>0.670148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.044370</td>\n",
       "      <td>1.677341</td>\n",
       "      <td>89.463221</td>\n",
       "      <td>98.958333</td>\n",
       "      <td>0.653646</td>\n",
       "      <td>71.354167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.044903</td>\n",
       "      <td>1.677548</td>\n",
       "      <td>89.860835</td>\n",
       "      <td>98.958333</td>\n",
       "      <td>0.655599</td>\n",
       "      <td>71.875000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.045416</td>\n",
       "      <td>1.677684</td>\n",
       "      <td>89.860835</td>\n",
       "      <td>98.958333</td>\n",
       "      <td>0.658420</td>\n",
       "      <td>72.656250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.045831</td>\n",
       "      <td>1.677741</td>\n",
       "      <td>90.009940</td>\n",
       "      <td>98.958333</td>\n",
       "      <td>0.660590</td>\n",
       "      <td>72.916667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.046169</td>\n",
       "      <td>1.677864</td>\n",
       "      <td>90.059642</td>\n",
       "      <td>99.479167</td>\n",
       "      <td>0.663194</td>\n",
       "      <td>73.437500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    1.045348   1.677649  89.860835  99.010417   0.658247  72.447917\n",
       "std     0.000584   0.000154   0.187437   0.164702   0.003592   0.670148\n",
       "min     1.044370   1.677341  89.463221  98.958333   0.653646  71.354167\n",
       "25%     1.044903   1.677548  89.860835  98.958333   0.655599  71.875000\n",
       "50%     1.045416   1.677684  89.860835  98.958333   0.658420  72.656250\n",
       "75%     1.045831   1.677741  90.009940  98.958333   0.660590  72.916667\n",
       "max     1.046169   1.677864  90.059642  99.479167   0.663194  73.437500"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'NO_SPA_val_metrics_transfer.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'NO_SPA_test_metrics_transfer.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NO SOC links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 3 for key in data.edge_types if 'SPA_link' in key or 'TEM_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].train_mask),\n",
    ")\n",
    "val_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 3 for key in data.edge_types if 'SPA_link' in key or 'TEM_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].val_mask),\n",
    ")\n",
    "test_loader = NeighborLoader(\n",
    "    data,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 3 for key in data.edge_types if 'SPA_link' in key or 'TEM_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data['all'].test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=2950,\n",
       "    x=[2950, 1753],\n",
       "    y=[2950, 20],\n",
       "    node_type=[2950],\n",
       "    att_lab=[2950],\n",
       "    val_lab=[2950],\n",
       "    train_mask=[2950],\n",
       "    val_mask=[2950],\n",
       "    test_mask=[2950],\n",
       "    n_id=[2950],\n",
       "    batch_size=32\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 69424],\n",
       "    edge_attr=[69424]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 66296],\n",
       "    edge_attr=[66296]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 0],\n",
       "    edge_attr=[0]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_hetero_data = next(iter(train_loader))\n",
    "batch = sampled_hetero_data\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroGNN_L(\n",
       "  (convs): ModuleList(\n",
       "    (0): HeteroConv(num_relations=3)\n",
       "    (1): HeteroConv(num_relations=3)\n",
       "    (2): HeteroConv(num_relations=3)\n",
       "  )\n",
       "  (lin1): Linear(1753, 32, bias=True)\n",
       "  (lin2): Linear(64, 20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 15.13it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.9280028412546808,\n",
       " 1.7612723247496376,\n",
       " 94.45983379501385,\n",
       " 93.90581717451524,\n",
       " 0.5184672347726584,\n",
       " 53.46260387811634)"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.50it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.0008981227874756,\n",
       " 1.75920490971927,\n",
       " 91.26016260162602,\n",
       " 85.22167487684729,\n",
       " 0.4950738974979946,\n",
       " 49.75369458128079)"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, val_loader,'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 17.77it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.023624079601902,\n",
       " 1.7890314950297277,\n",
       " 89.06560636182903,\n",
       " 86.97916666666667,\n",
       " 0.492187508692344,\n",
       " 54.6875)"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "test(model, test_loader,'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.20it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 20.92it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.01it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.48it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.53it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.29it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.51it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.41it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.07it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.49it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 20.58it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.36it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.46it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.62it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.32it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.55it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.14it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.26it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.35it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.31it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers = []\n",
    "test_numbers = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers.append(test(model, val_loader,'val'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers.append(test(model, test_loader,'val'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.DataFrame(val_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df = pd.DataFrame(test_numbers, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.000845</td>\n",
       "      <td>1.759049</td>\n",
       "      <td>91.056911</td>\n",
       "      <td>85.517241</td>\n",
       "      <td>0.495484</td>\n",
       "      <td>49.704433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000570</td>\n",
       "      <td>0.000382</td>\n",
       "      <td>0.345462</td>\n",
       "      <td>0.529541</td>\n",
       "      <td>0.003510</td>\n",
       "      <td>0.882738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000276</td>\n",
       "      <td>1.758623</td>\n",
       "      <td>90.650407</td>\n",
       "      <td>84.729064</td>\n",
       "      <td>0.490148</td>\n",
       "      <td>48.275862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000374</td>\n",
       "      <td>1.758806</td>\n",
       "      <td>90.853659</td>\n",
       "      <td>85.221675</td>\n",
       "      <td>0.493432</td>\n",
       "      <td>49.261084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000622</td>\n",
       "      <td>1.758922</td>\n",
       "      <td>90.955285</td>\n",
       "      <td>85.467980</td>\n",
       "      <td>0.495074</td>\n",
       "      <td>49.261084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.001291</td>\n",
       "      <td>1.759124</td>\n",
       "      <td>91.260163</td>\n",
       "      <td>85.714286</td>\n",
       "      <td>0.498153</td>\n",
       "      <td>50.246305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.001743</td>\n",
       "      <td>1.759794</td>\n",
       "      <td>91.666667</td>\n",
       "      <td>86.699507</td>\n",
       "      <td>0.501642</td>\n",
       "      <td>51.231527</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    1.000845   1.759049  91.056911  85.517241   0.495484  49.704433\n",
       "std     0.000570   0.000382   0.345462   0.529541   0.003510   0.882738\n",
       "min     1.000276   1.758623  90.650407  84.729064   0.490148  48.275862\n",
       "25%     1.000374   1.758806  90.853659  85.221675   0.493432  49.261084\n",
       "50%     1.000622   1.758922  90.955285  85.467980   0.495074  49.261084\n",
       "75%     1.001291   1.759124  91.260163  85.714286   0.498153  50.246305\n",
       "max     1.001743   1.759794  91.666667  86.699507   0.501642  51.231527"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.023211</td>\n",
       "      <td>1.788929</td>\n",
       "      <td>89.085487</td>\n",
       "      <td>86.614583</td>\n",
       "      <td>0.486806</td>\n",
       "      <td>54.062500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000489</td>\n",
       "      <td>0.000440</td>\n",
       "      <td>0.317124</td>\n",
       "      <td>0.428788</td>\n",
       "      <td>0.001665</td>\n",
       "      <td>0.410839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.022142</td>\n",
       "      <td>1.788378</td>\n",
       "      <td>88.469185</td>\n",
       "      <td>85.937500</td>\n",
       "      <td>0.482639</td>\n",
       "      <td>53.645833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.022985</td>\n",
       "      <td>1.788670</td>\n",
       "      <td>88.866799</td>\n",
       "      <td>86.458333</td>\n",
       "      <td>0.486979</td>\n",
       "      <td>53.645833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.023216</td>\n",
       "      <td>1.788866</td>\n",
       "      <td>89.165010</td>\n",
       "      <td>86.458333</td>\n",
       "      <td>0.486979</td>\n",
       "      <td>54.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.023572</td>\n",
       "      <td>1.789026</td>\n",
       "      <td>89.264414</td>\n",
       "      <td>86.848958</td>\n",
       "      <td>0.487739</td>\n",
       "      <td>54.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.023779</td>\n",
       "      <td>1.790004</td>\n",
       "      <td>89.463221</td>\n",
       "      <td>87.500000</td>\n",
       "      <td>0.488715</td>\n",
       "      <td>54.687500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    1.023211   1.788929  89.085487  86.614583   0.486806  54.062500\n",
       "std     0.000489   0.000440   0.317124   0.428788   0.001665   0.410839\n",
       "min     1.022142   1.788378  88.469185  85.937500   0.482639  53.645833\n",
       "25%     1.022985   1.788670  88.866799  86.458333   0.486979  53.645833\n",
       "50%     1.023216   1.788866  89.165010  86.458333   0.486979  54.166667\n",
       "75%     1.023572   1.789026  89.264414  86.848958   0.487739  54.166667\n",
       "max     1.023779   1.790004  89.463221  87.500000   0.488715  54.687500"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv(args.save_dir + 'NO_SOC_val_metrics_transfer.csv', sep='\\t')\n",
    "test_df.to_csv(args.save_dir + 'NO_SOC_test_metrics_transfer.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Masking Textual and Visual Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = HeteroGNN_L(data.metadata(), hidden_channels=32, out_channels=data.y_dict['all'].shape[-1],\n",
    "                  num_layers = 3, in_channels = data.x_dict['all'].shape[-1]).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load(args.save_dir+'Hetero_best_model/model.pth',map_location=torch.device('cpu')))\n",
    "model.eval()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroGNN_L(\n",
       "  (convs): ModuleList(\n",
       "    (0): HeteroConv(num_relations=3)\n",
       "    (1): HeteroConv(num_relations=3)\n",
       "    (2): HeteroConv(num_relations=3)\n",
       "  )\n",
       "  (lin1): Linear(1753, 32, bias=True)\n",
       "  (lin2): Linear(64, 20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def test_Mask(model, loader, mask='vis'):\n",
    "    model.eval()\n",
    "\n",
    "    total_examples_att = total_examples_val = 0\n",
    "    running_loss_1 = running_loss_2 = 0.\n",
    "    running_1_acc = 0.\n",
    "    running_k_acc = 0.\n",
    "    running_k_jac = 0.\n",
    "    running_1_val = 0.\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        loss_1 = 0\n",
    "        acc_1_t = 0\n",
    "        loss_2 = 0\n",
    "        acc_1_val = 0\n",
    "        acc_k_t = 0\n",
    "        jac_k_t = 0\n",
    "\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch['all'].batch_size\n",
    "        \n",
    "        if mask == 'vis':\n",
    "            batch['all'].x[:batch_size,:982] = torch.zeros(batch_size,982)\n",
    "        elif mask == 'tex':\n",
    "            batch['all'].x[:batch_size,982:] = torch.zeros(batch_size,771)\n",
    "                \n",
    "        new_dict = {}\n",
    "        for edge_type in [edge_type for edge_type in batch.edge_index_dict if 'all' in edge_type and not 'simp_link' in edge_type]:\n",
    "            edge_index = batch.edge_index_dict[edge_type]\n",
    "            edge_index = to_undirected(edge_index)\n",
    "            new_dict[edge_type] = edge_index\n",
    "        batch.edge_index_dict = new_dict\n",
    "\n",
    "        out = model(batch.x_dict, batch.edge_index_dict)[:batch_size]\n",
    "        out_att = out[:,:9]\n",
    "        out_val = out[:,9:]\n",
    "        att_node = (batch['all'].att_lab[:batch_size]).nonzero().squeeze()\n",
    "        val_node = (batch['all'].val_lab[:batch_size]).nonzero().squeeze()\n",
    "\n",
    "        #print(type_node)\n",
    "\n",
    "        #pred_att = out_att.argmax(dim=-1)\n",
    "        #pred_val = out_val.argmax(dim=-1)\n",
    "\n",
    "        y = batch.y_dict['all']\n",
    "        y_att = y[:,:9]\n",
    "        y_val = y[:,9:]\n",
    "        \n",
    "        if not att_node.shape[0]==0:\n",
    "            loss_1 = F.cross_entropy(out_att[att_node], y_att[:batch_size][att_node])\n",
    "            acc_1_t = compute_1_accuracy(y_att[:batch_size][att_node], out_att[att_node])\n",
    "\n",
    "        if not val_node.shape[0]==0:\n",
    "            loss_2 = F.cross_entropy(out_val[val_node], y_val[val_node])\n",
    "            acc_1_val = compute_1_accuracy(y_val[val_node], out_val[val_node])\n",
    "            acc_k_t = compute_k_accuracy(y_val[val_node], out_val[val_node], args.k)\n",
    "            jac_k_t = compute_jaccard_index(y_val[val_node], F.softmax(out_val[val_node],dim=-1), args.k)\n",
    "            #loss_3 = loss_1 + loss_2\n",
    "\n",
    "        total_examples_att += att_node.shape[0]\n",
    "        total_examples_val += val_node.shape[0]\n",
    "        #total_correct_att += int((pred_att == y_att[:batch_size]).sum())\n",
    "        #total_correct_val += int((pred_val == y_val[:batch_size]).sum())\n",
    "\n",
    "        running_loss_1 += float(loss_1) * att_node.shape[0]\n",
    "        running_loss_2 += float(loss_2) * val_node.shape[0]\n",
    "        running_1_acc += float(acc_1_t) * att_node.shape[0]\n",
    "        running_1_val += float(acc_1_val) * val_node.shape[0]\n",
    "        running_k_acc += float(acc_k_t) * val_node.shape[0]\n",
    "        running_k_jac += float(jac_k_t) * val_node.shape[0]\n",
    "\n",
    "    return running_loss_1/total_examples_att, running_loss_2/total_examples_val, running_1_acc/ total_examples_att, running_k_acc/ total_examples_val, running_k_jac/ total_examples_val, running_1_val/total_examples_val\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 18.85it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2.1056877727984062,\n",
       " 1.6669259338828004,\n",
       " 16.62049861495845,\n",
       " 98.06094182825485,\n",
       " 0.6509695422946581,\n",
       " 68.42105263157895)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, train_loader, 'vis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 28.77it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.7522614801359309,\n",
       " 1.9097514446422335,\n",
       " 100.0,\n",
       " 73.6842105263158,\n",
       " 0.1481994459833795,\n",
       " 24.099722991689752)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, train_loader, 'tex')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 24.99it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2.1542519878565782,\n",
       " 1.676019734349744,\n",
       " 16.463414634146343,\n",
       " 96.55172413793103,\n",
       " 0.6617405637731693,\n",
       " 67.98029556650246)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, val_loader, 'vis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 24.77it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8239235200775348,\n",
       " 1.9393611265520745,\n",
       " 96.34146341463415,\n",
       " 67.98029556650246,\n",
       " 0.08866995073891626,\n",
       " 18.7192118226601)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, val_loader, 'tex')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.52it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2.162035334892349,\n",
       " 1.694229070097208,\n",
       " 13.916500994035784,\n",
       " 96.875,\n",
       " 0.6371527829517921,\n",
       " 64.0625)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, test_loader, 'vis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 22.57it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8315963697717867,\n",
       " 1.957061951359113,\n",
       " 96.02385685884691,\n",
       " 71.35416666666667,\n",
       " 0.09809027798473835,\n",
       " 20.3125)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_Mask(model, test_loader, 'tex')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:01<00:00, 21.87it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.41it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.76it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.05it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.94it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 32.53it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 32.06it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.96it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 32.31it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 32.22it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.77it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.44it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.85it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.85it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.97it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.33it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 32.25it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 32.21it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.46it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.62it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.87it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.15it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.73it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.13it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.59it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.93it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.94it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.32it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.18it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.83it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.12it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.88it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 32.32it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.57it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 30.54it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.16it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 29.98it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 32.11it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.10it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 22/22 [00:00<00:00, 31.18it/s]\n"
     ]
    }
   ],
   "source": [
    "val_numbers_vis = []\n",
    "val_numbers_tex = []\n",
    "test_numbers_vis = []\n",
    "test_numbers_tex = []\n",
    "for seed in [0,1,2,42,100,233,1024,1337,2333,4399]:\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers_vis.append(test_Mask(model, val_loader, 'vis'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    val_numbers_tex.append(test_Mask(model, val_loader, 'tex'))\n",
    "    \n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers_vis.append(test_Mask(model, test_loader, 'vis'))\n",
    "    set_seed_everywhere(seed, args.cuda)\n",
    "    test_numbers_tex.append(test_Mask(model, test_loader, 'tex'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df_vis = pd.DataFrame(val_numbers_vis, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "val_df_tex = pd.DataFrame(val_numbers_tex, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df_vis = pd.DataFrame(test_numbers_vis, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])\n",
    "test_df_tex = pd.DataFrame(test_numbers_tex, columns=['ATT_loss', 'VAL_loss', 'ATT_acc', 'VAL_k_acc', 'VAL_k_jac', 'VAL_1_acc'],\n",
    "            index = [0,1,2,42,100,233,1024,1337,2333,4399])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.153660</td>\n",
       "      <td>1.676738</td>\n",
       "      <td>15.182927</td>\n",
       "      <td>96.403941</td>\n",
       "      <td>0.657553</td>\n",
       "      <td>69.113300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.002757</td>\n",
       "      <td>0.000653</td>\n",
       "      <td>0.429028</td>\n",
       "      <td>0.405553</td>\n",
       "      <td>0.006808</td>\n",
       "      <td>0.616584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.150324</td>\n",
       "      <td>1.675918</td>\n",
       "      <td>14.430894</td>\n",
       "      <td>96.059113</td>\n",
       "      <td>0.646962</td>\n",
       "      <td>67.980296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.151220</td>\n",
       "      <td>1.676298</td>\n",
       "      <td>14.939024</td>\n",
       "      <td>96.059113</td>\n",
       "      <td>0.652915</td>\n",
       "      <td>68.596059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.153498</td>\n",
       "      <td>1.676622</td>\n",
       "      <td>15.243902</td>\n",
       "      <td>96.305419</td>\n",
       "      <td>0.657225</td>\n",
       "      <td>69.458128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.155457</td>\n",
       "      <td>1.677082</td>\n",
       "      <td>15.447154</td>\n",
       "      <td>96.551724</td>\n",
       "      <td>0.662151</td>\n",
       "      <td>69.458128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.158879</td>\n",
       "      <td>1.678108</td>\n",
       "      <td>15.853659</td>\n",
       "      <td>97.044335</td>\n",
       "      <td>0.669130</td>\n",
       "      <td>69.950739</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    2.153660   1.676738  15.182927  96.403941   0.657553  69.113300\n",
       "std     0.002757   0.000653   0.429028   0.405553   0.006808   0.616584\n",
       "min     2.150324   1.675918  14.430894  96.059113   0.646962  67.980296\n",
       "25%     2.151220   1.676298  14.939024  96.059113   0.652915  68.596059\n",
       "50%     2.153498   1.676622  15.243902  96.305419   0.657225  69.458128\n",
       "75%     2.155457   1.677082  15.447154  96.551724   0.662151  69.458128\n",
       "max     2.158879   1.678108  15.853659  97.044335   0.669130  69.950739"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df_vis.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.825238</td>\n",
       "      <td>1.939382</td>\n",
       "      <td>96.097561</td>\n",
       "      <td>67.093596</td>\n",
       "      <td>0.093103</td>\n",
       "      <td>17.635468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000671</td>\n",
       "      <td>0.001077</td>\n",
       "      <td>0.230750</td>\n",
       "      <td>0.979733</td>\n",
       "      <td>0.004313</td>\n",
       "      <td>0.688873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.824183</td>\n",
       "      <td>1.937483</td>\n",
       "      <td>95.731707</td>\n",
       "      <td>65.517241</td>\n",
       "      <td>0.088670</td>\n",
       "      <td>16.748768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.824754</td>\n",
       "      <td>1.938761</td>\n",
       "      <td>95.985772</td>\n",
       "      <td>66.256158</td>\n",
       "      <td>0.089286</td>\n",
       "      <td>17.241379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.825357</td>\n",
       "      <td>1.939430</td>\n",
       "      <td>96.138211</td>\n",
       "      <td>67.241379</td>\n",
       "      <td>0.092365</td>\n",
       "      <td>17.487685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.825592</td>\n",
       "      <td>1.939961</td>\n",
       "      <td>96.290650</td>\n",
       "      <td>67.857143</td>\n",
       "      <td>0.095443</td>\n",
       "      <td>18.226601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.826147</td>\n",
       "      <td>1.941189</td>\n",
       "      <td>96.341463</td>\n",
       "      <td>68.472906</td>\n",
       "      <td>0.100985</td>\n",
       "      <td>18.719212</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    0.825238   1.939382  96.097561  67.093596   0.093103  17.635468\n",
       "std     0.000671   0.001077   0.230750   0.979733   0.004313   0.688873\n",
       "min     0.824183   1.937483  95.731707  65.517241   0.088670  16.748768\n",
       "25%     0.824754   1.938761  95.985772  66.256158   0.089286  17.241379\n",
       "50%     0.825357   1.939430  96.138211  67.241379   0.092365  17.487685\n",
       "75%     0.825592   1.939961  96.290650  67.857143   0.095443  18.226601\n",
       "max     0.826147   1.941189  96.341463  68.472906   0.100985  18.719212"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df_tex.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.159370</td>\n",
       "      <td>1.694245</td>\n",
       "      <td>14.652087</td>\n",
       "      <td>96.979167</td>\n",
       "      <td>0.630642</td>\n",
       "      <td>65.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.001348</td>\n",
       "      <td>0.000459</td>\n",
       "      <td>0.386979</td>\n",
       "      <td>0.478613</td>\n",
       "      <td>0.006263</td>\n",
       "      <td>0.537914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.157010</td>\n",
       "      <td>1.693455</td>\n",
       "      <td>14.115308</td>\n",
       "      <td>96.354167</td>\n",
       "      <td>0.619792</td>\n",
       "      <td>64.062500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.158662</td>\n",
       "      <td>1.694174</td>\n",
       "      <td>14.363817</td>\n",
       "      <td>96.875000</td>\n",
       "      <td>0.627170</td>\n",
       "      <td>65.104167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.159237</td>\n",
       "      <td>1.694288</td>\n",
       "      <td>14.711730</td>\n",
       "      <td>96.875000</td>\n",
       "      <td>0.630642</td>\n",
       "      <td>65.104167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.160106</td>\n",
       "      <td>1.694434</td>\n",
       "      <td>14.910537</td>\n",
       "      <td>97.265625</td>\n",
       "      <td>0.635417</td>\n",
       "      <td>65.104167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.161866</td>\n",
       "      <td>1.695114</td>\n",
       "      <td>15.308151</td>\n",
       "      <td>97.916667</td>\n",
       "      <td>0.639757</td>\n",
       "      <td>65.625000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    2.159370   1.694245  14.652087  96.979167   0.630642  65.000000\n",
       "std     0.001348   0.000459   0.386979   0.478613   0.006263   0.537914\n",
       "min     2.157010   1.693455  14.115308  96.354167   0.619792  64.062500\n",
       "25%     2.158662   1.694174  14.363817  96.875000   0.627170  65.104167\n",
       "50%     2.159237   1.694288  14.711730  96.875000   0.630642  65.104167\n",
       "75%     2.160106   1.694434  14.910537  97.265625   0.635417  65.104167\n",
       "max     2.161866   1.695114  15.308151  97.916667   0.639757  65.625000"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df_vis.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ATT_loss</th>\n",
       "      <th>VAL_loss</th>\n",
       "      <th>ATT_acc</th>\n",
       "      <th>VAL_k_acc</th>\n",
       "      <th>VAL_k_jac</th>\n",
       "      <th>VAL_1_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.830668</td>\n",
       "      <td>1.956222</td>\n",
       "      <td>95.964215</td>\n",
       "      <td>70.989583</td>\n",
       "      <td>0.094444</td>\n",
       "      <td>20.208333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000588</td>\n",
       "      <td>0.000962</td>\n",
       "      <td>0.230517</td>\n",
       "      <td>0.886947</td>\n",
       "      <td>0.006402</td>\n",
       "      <td>0.591298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.829640</td>\n",
       "      <td>1.954506</td>\n",
       "      <td>95.427435</td>\n",
       "      <td>69.791667</td>\n",
       "      <td>0.085069</td>\n",
       "      <td>19.270833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.830184</td>\n",
       "      <td>1.955974</td>\n",
       "      <td>95.874751</td>\n",
       "      <td>70.312500</td>\n",
       "      <td>0.090278</td>\n",
       "      <td>19.791667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.830857</td>\n",
       "      <td>1.956240</td>\n",
       "      <td>96.023857</td>\n",
       "      <td>70.833333</td>\n",
       "      <td>0.092882</td>\n",
       "      <td>20.312500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.831115</td>\n",
       "      <td>1.956870</td>\n",
       "      <td>96.023857</td>\n",
       "      <td>71.354167</td>\n",
       "      <td>0.100694</td>\n",
       "      <td>20.312500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.831377</td>\n",
       "      <td>1.957708</td>\n",
       "      <td>96.222664</td>\n",
       "      <td>72.395833</td>\n",
       "      <td>0.103299</td>\n",
       "      <td>21.354167</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ATT_loss   VAL_loss    ATT_acc  VAL_k_acc  VAL_k_jac  VAL_1_acc\n",
       "count  10.000000  10.000000  10.000000  10.000000  10.000000  10.000000\n",
       "mean    0.830668   1.956222  95.964215  70.989583   0.094444  20.208333\n",
       "std     0.000588   0.000962   0.230517   0.886947   0.006402   0.591298\n",
       "min     0.829640   1.954506  95.427435  69.791667   0.085069  19.270833\n",
       "25%     0.830184   1.955974  95.874751  70.312500   0.090278  19.791667\n",
       "50%     0.830857   1.956240  96.023857  70.833333   0.092882  20.312500\n",
       "75%     0.831115   1.956870  96.023857  71.354167   0.100694  20.312500\n",
       "max     0.831377   1.957708  96.222664  72.395833   0.103299  21.354167"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df_tex.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df_vis.to_csv(args.save_dir + 'vis_masked_val_metrics.csv', sep='\\t')\n",
    "val_df_tex.to_csv(args.save_dir + 'tex_masked_val_metrics.csv', sep='\\t')\n",
    "test_df_vis.to_csv(args.save_dir + 'vis_masked_test_metrics.csv', sep='\\t')\n",
    "test_df_tex.to_csv(args.save_dir + 'tex_masked_test_metrics.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Direct Prediction on VEN-XL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cpu\")\n",
    "#device = torch.device(\"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_XL = VEN_XL_Links('dataset/Venice_XL_links')\n",
    "data_XL = dataset_XL[0].to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroData(\n",
       "  \u001b[1mall\u001b[0m={\n",
       "    num_nodes=80963,\n",
       "    x=[80963, 1753],\n",
       "    y=[80963, 20],\n",
       "    node_type=[80963],\n",
       "    att_lab=[80963],\n",
       "    val_lab=[80963],\n",
       "    train_mask=[80963],\n",
       "    val_mask=[80963],\n",
       "    test_mask=[80963],\n",
       "    n_id=[80963]\n",
       "  },\n",
       "  \u001b[1m(all, SOC_link, all)\u001b[0m={\n",
       "    edge_index=[2, 76422265],\n",
       "    edge_attr=[76422265]\n",
       "  },\n",
       "  \u001b[1m(all, SPA_link, all)\u001b[0m={\n",
       "    edge_index=[2, 202173159],\n",
       "    edge_attr=[202173159]\n",
       "  },\n",
       "  \u001b[1m(all, TEM_link, all)\u001b[0m={\n",
       "    edge_index=[2, 71135671],\n",
       "    edge_attr=[71135671]\n",
       "  },\n",
       "  \u001b[1m(all, simp_link, all)\u001b[0m={\n",
       "    edge_index=[2, 290091503],\n",
       "    edge_attr=[290091503]\n",
       "  }\n",
       ")"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_XL['all'].n_id = torch.arange(data_XL.num_nodes)\n",
    "#data_XL = data_XL.to(device)\n",
    "data_XL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = HeteroGNN_L(data_XL.metadata(), hidden_channels=32, out_channels=data_XL.y_dict['all'].shape[-1],\n",
    "                  num_layers = 3, in_channels = data_XL.x_dict['all'].shape[-1]).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load(args.save_dir+'Hetero_best_model/model.pth',map_location=torch.device('cpu')))\n",
    "model.eval()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HeteroGNN_L(\n",
       "  (convs): ModuleList(\n",
       "    (0): HeteroConv(num_relations=3)\n",
       "    (1): HeteroConv(num_relations=3)\n",
       "    (2): HeteroConv(num_relations=3)\n",
       "  )\n",
       "  (lin1): Linear(1753, 32, bias=True)\n",
       "  (lin2): Linear(64, 20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.loader import NeighborLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "train_loader = NeighborLoader(\n",
    "    data_XL,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data_XL.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data_XL['all'].train_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "val_loader = NeighborLoader(\n",
    "    data_XL,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data_XL.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data_XL['all'].val_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "test_loader = NeighborLoader(\n",
    "    data_XL,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data_XL.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', data_XL['all'].test_mask),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(args.seed)\n",
    "unlabel_loader = NeighborLoader(\n",
    "    data_XL,\n",
    "    # Sample 30 neighbors for each node and edge type for 2 iterations\n",
    "    num_neighbors={key: [args.sample_nodes] * 2 for key in data_XL.edge_types if 'all' in key and not 'simp_link' in key},\n",
    "    # Use a batch size of 128 for sampling training nodes of type paper\n",
    "    batch_size=args.batch_size,\n",
    "    input_nodes=('all', ~(data_XL['all'].train_mask + data_XL['all'].val_mask + data_XL['all'].test_mask)),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def predict(model, loader):\n",
    "    model.eval()\n",
    "    seed_everything(args.seed)\n",
    "    all_preds = []\n",
    "    \n",
    "    for batch in tqdm(loader):\n",
    "        batch = batch.to(device)\n",
    "        batch_size = batch['all'].batch_size\n",
    "        new_dict = {}\n",
    "        for edge_type in [edge_type for edge_type in batch.edge_index_dict if 'all' in edge_type and not 'simp_link' in edge_type]:\n",
    "            edge_index = batch.edge_index_dict[edge_type]\n",
    "            edge_index = to_undirected(edge_index)\n",
    "            new_dict[edge_type] = edge_index\n",
    "        batch.edge_index_dict = new_dict\n",
    "        \n",
    "        out = model(batch.x_dict, batch.edge_index_dict)[:batch_size]\n",
    "        out_att = out[:,:9].softmax(axis=1)\n",
    "        out_val = out[:,9:].softmax(axis=1)\n",
    "        IDs = batch['all'].n_id[:batch_size].unsqueeze(dim=-1).int()\n",
    "        \n",
    "        now = torch.hstack([IDs, out_att, out_val])\n",
    "        all_preds.append(now)\n",
    "    \n",
    "    final = torch.vstack(all_preds)\n",
    "        \n",
    "    return final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 362/362 [25:18<00:00,  4.19s/it]\n"
     ]
    }
   ],
   "source": [
    "pred_train = predict(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋              | 586/624 [32:25<02:08,  3.39s/it]"
     ]
    }
   ],
   "source": [
    "pred_val = predict(model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 624/624 [33:10<00:00,  3.19s/it]\n"
     ]
    }
   ],
   "source": [
    "pred_test = predict(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 921/921 [44:23<00:00,  2.89s/it]\n"
     ]
    }
   ],
   "source": [
    "pred_unlab = predict(model, unlabel_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([4.0000e+00, 9.5017e-03, 8.2870e-01, 2.9284e-02, 3.8555e-03, 3.9593e-03,\n",
       "        7.1498e-02, 2.8506e-02, 7.4202e-03, 1.7272e-02, 3.8777e-02, 9.6328e-02,\n",
       "        2.2508e-01, 3.7685e-01, 9.8926e-02, 9.1405e-02, 1.5600e-02, 1.2718e-02,\n",
       "        1.1601e-02, 1.5288e-02, 1.7429e-02])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = torch.vstack([pred, pred_test, pred_unlab]).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = torch.vstack([pred_train, pred_val, pred_test]).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = torch.vstack([pred_train, pred_val, pred_test, pred_unlab]).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0000e+00, 2.3903e-01, 6.9283e-02,  ..., 5.0324e-03, 6.3327e-03,\n",
       "         1.2382e-02],\n",
       "        [1.0000e+00, 1.7605e-01, 1.4921e-02,  ..., 4.0110e-03, 4.4668e-03,\n",
       "         1.2308e-02],\n",
       "        [4.0000e+00, 4.3924e-03, 8.8463e-01,  ..., 8.8102e-03, 1.4086e-02,\n",
       "         1.6274e-02],\n",
       "        ...,\n",
       "        [8.0946e+04, 6.1603e-02, 2.8324e-01,  ..., 3.7812e-02, 6.7982e-02,\n",
       "         2.3695e-02],\n",
       "        [8.0957e+04, 2.5587e-02, 4.4412e-01,  ..., 7.9951e-03, 8.4820e-03,\n",
       "         1.7054e-02],\n",
       "        [8.0961e+04, 5.3520e-01, 2.3284e-01,  ..., 4.5963e-03, 2.7339e-03,\n",
       "         1.0685e-02]], dtype=torch.float64)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([2.0000, 2.0000, 2.0000,  ..., 2.0000, 2.0000, 2.0000])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds[:,1:].sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df = pd.DataFrame(preds).sort_values(0).set_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.239032</td>\n",
       "      <td>0.069283</td>\n",
       "      <td>0.039445</td>\n",
       "      <td>0.055686</td>\n",
       "      <td>0.013506</td>\n",
       "      <td>0.498343</td>\n",
       "      <td>0.026811</td>\n",
       "      <td>0.041769</td>\n",
       "      <td>0.016125</td>\n",
       "      <td>0.235945</td>\n",
       "      <td>0.138339</td>\n",
       "      <td>0.136415</td>\n",
       "      <td>0.377030</td>\n",
       "      <td>0.016253</td>\n",
       "      <td>0.057162</td>\n",
       "      <td>0.008264</td>\n",
       "      <td>0.006845</td>\n",
       "      <td>0.005032</td>\n",
       "      <td>0.006333</td>\n",
       "      <td>0.012382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.176052</td>\n",
       "      <td>0.014921</td>\n",
       "      <td>0.002331</td>\n",
       "      <td>0.010017</td>\n",
       "      <td>0.003138</td>\n",
       "      <td>0.778988</td>\n",
       "      <td>0.003957</td>\n",
       "      <td>0.003257</td>\n",
       "      <td>0.007339</td>\n",
       "      <td>0.174502</td>\n",
       "      <td>0.101622</td>\n",
       "      <td>0.153769</td>\n",
       "      <td>0.445207</td>\n",
       "      <td>0.017809</td>\n",
       "      <td>0.075973</td>\n",
       "      <td>0.004658</td>\n",
       "      <td>0.005674</td>\n",
       "      <td>0.004011</td>\n",
       "      <td>0.004467</td>\n",
       "      <td>0.012308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.008293</td>\n",
       "      <td>0.438736</td>\n",
       "      <td>0.016735</td>\n",
       "      <td>0.016557</td>\n",
       "      <td>0.319096</td>\n",
       "      <td>0.018162</td>\n",
       "      <td>0.120364</td>\n",
       "      <td>0.024655</td>\n",
       "      <td>0.037401</td>\n",
       "      <td>0.091521</td>\n",
       "      <td>0.087596</td>\n",
       "      <td>0.229066</td>\n",
       "      <td>0.141877</td>\n",
       "      <td>0.043438</td>\n",
       "      <td>0.258959</td>\n",
       "      <td>0.044236</td>\n",
       "      <td>0.022986</td>\n",
       "      <td>0.019999</td>\n",
       "      <td>0.035320</td>\n",
       "      <td>0.025003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.007731</td>\n",
       "      <td>0.212674</td>\n",
       "      <td>0.060881</td>\n",
       "      <td>0.027692</td>\n",
       "      <td>0.098404</td>\n",
       "      <td>0.064098</td>\n",
       "      <td>0.402679</td>\n",
       "      <td>0.053342</td>\n",
       "      <td>0.072499</td>\n",
       "      <td>0.044819</td>\n",
       "      <td>0.173665</td>\n",
       "      <td>0.247423</td>\n",
       "      <td>0.209091</td>\n",
       "      <td>0.067114</td>\n",
       "      <td>0.156775</td>\n",
       "      <td>0.025087</td>\n",
       "      <td>0.015049</td>\n",
       "      <td>0.017127</td>\n",
       "      <td>0.020303</td>\n",
       "      <td>0.023548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.004392</td>\n",
       "      <td>0.884633</td>\n",
       "      <td>0.013877</td>\n",
       "      <td>0.005151</td>\n",
       "      <td>0.005875</td>\n",
       "      <td>0.053898</td>\n",
       "      <td>0.013674</td>\n",
       "      <td>0.006233</td>\n",
       "      <td>0.012267</td>\n",
       "      <td>0.021860</td>\n",
       "      <td>0.144453</td>\n",
       "      <td>0.233614</td>\n",
       "      <td>0.357309</td>\n",
       "      <td>0.107493</td>\n",
       "      <td>0.073830</td>\n",
       "      <td>0.014845</td>\n",
       "      <td>0.007426</td>\n",
       "      <td>0.008810</td>\n",
       "      <td>0.014086</td>\n",
       "      <td>0.016274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80958.0</th>\n",
       "      <td>0.059333</td>\n",
       "      <td>0.213964</td>\n",
       "      <td>0.010122</td>\n",
       "      <td>0.021720</td>\n",
       "      <td>0.009755</td>\n",
       "      <td>0.564988</td>\n",
       "      <td>0.037177</td>\n",
       "      <td>0.011433</td>\n",
       "      <td>0.071507</td>\n",
       "      <td>0.090714</td>\n",
       "      <td>0.342257</td>\n",
       "      <td>0.161905</td>\n",
       "      <td>0.266439</td>\n",
       "      <td>0.012688</td>\n",
       "      <td>0.098222</td>\n",
       "      <td>0.004578</td>\n",
       "      <td>0.003098</td>\n",
       "      <td>0.004052</td>\n",
       "      <td>0.003347</td>\n",
       "      <td>0.012698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80959.0</th>\n",
       "      <td>0.932363</td>\n",
       "      <td>0.010110</td>\n",
       "      <td>0.001268</td>\n",
       "      <td>0.031923</td>\n",
       "      <td>0.004079</td>\n",
       "      <td>0.008335</td>\n",
       "      <td>0.005243</td>\n",
       "      <td>0.005432</td>\n",
       "      <td>0.001247</td>\n",
       "      <td>0.230146</td>\n",
       "      <td>0.339425</td>\n",
       "      <td>0.145487</td>\n",
       "      <td>0.170682</td>\n",
       "      <td>0.007845</td>\n",
       "      <td>0.081629</td>\n",
       "      <td>0.002546</td>\n",
       "      <td>0.005387</td>\n",
       "      <td>0.004262</td>\n",
       "      <td>0.003147</td>\n",
       "      <td>0.009444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80960.0</th>\n",
       "      <td>0.789606</td>\n",
       "      <td>0.044373</td>\n",
       "      <td>0.003783</td>\n",
       "      <td>0.065795</td>\n",
       "      <td>0.006891</td>\n",
       "      <td>0.058347</td>\n",
       "      <td>0.014418</td>\n",
       "      <td>0.012272</td>\n",
       "      <td>0.004515</td>\n",
       "      <td>0.298882</td>\n",
       "      <td>0.172716</td>\n",
       "      <td>0.186109</td>\n",
       "      <td>0.148433</td>\n",
       "      <td>0.004076</td>\n",
       "      <td>0.163437</td>\n",
       "      <td>0.004744</td>\n",
       "      <td>0.004990</td>\n",
       "      <td>0.003501</td>\n",
       "      <td>0.002442</td>\n",
       "      <td>0.010670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80961.0</th>\n",
       "      <td>0.535197</td>\n",
       "      <td>0.232845</td>\n",
       "      <td>0.020557</td>\n",
       "      <td>0.047099</td>\n",
       "      <td>0.012990</td>\n",
       "      <td>0.035292</td>\n",
       "      <td>0.083062</td>\n",
       "      <td>0.026299</td>\n",
       "      <td>0.006660</td>\n",
       "      <td>0.171584</td>\n",
       "      <td>0.326464</td>\n",
       "      <td>0.154456</td>\n",
       "      <td>0.182478</td>\n",
       "      <td>0.007839</td>\n",
       "      <td>0.132046</td>\n",
       "      <td>0.003085</td>\n",
       "      <td>0.004032</td>\n",
       "      <td>0.004596</td>\n",
       "      <td>0.002734</td>\n",
       "      <td>0.010685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80962.0</th>\n",
       "      <td>0.644504</td>\n",
       "      <td>0.063732</td>\n",
       "      <td>0.006310</td>\n",
       "      <td>0.157164</td>\n",
       "      <td>0.005757</td>\n",
       "      <td>0.089713</td>\n",
       "      <td>0.012814</td>\n",
       "      <td>0.012926</td>\n",
       "      <td>0.007080</td>\n",
       "      <td>0.175824</td>\n",
       "      <td>0.340701</td>\n",
       "      <td>0.194704</td>\n",
       "      <td>0.161014</td>\n",
       "      <td>0.008808</td>\n",
       "      <td>0.084242</td>\n",
       "      <td>0.004327</td>\n",
       "      <td>0.006544</td>\n",
       "      <td>0.007175</td>\n",
       "      <td>0.004352</td>\n",
       "      <td>0.012310</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80963 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               1         2         3         4         5         6         7   \\\n",
       "0                                                                               \n",
       "0.0      0.239032  0.069283  0.039445  0.055686  0.013506  0.498343  0.026811   \n",
       "1.0      0.176052  0.014921  0.002331  0.010017  0.003138  0.778988  0.003957   \n",
       "2.0      0.008293  0.438736  0.016735  0.016557  0.319096  0.018162  0.120364   \n",
       "3.0      0.007731  0.212674  0.060881  0.027692  0.098404  0.064098  0.402679   \n",
       "4.0      0.004392  0.884633  0.013877  0.005151  0.005875  0.053898  0.013674   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80958.0  0.059333  0.213964  0.010122  0.021720  0.009755  0.564988  0.037177   \n",
       "80959.0  0.932363  0.010110  0.001268  0.031923  0.004079  0.008335  0.005243   \n",
       "80960.0  0.789606  0.044373  0.003783  0.065795  0.006891  0.058347  0.014418   \n",
       "80961.0  0.535197  0.232845  0.020557  0.047099  0.012990  0.035292  0.083062   \n",
       "80962.0  0.644504  0.063732  0.006310  0.157164  0.005757  0.089713  0.012814   \n",
       "\n",
       "               8         9         10        11        12        13        14  \\\n",
       "0                                                                               \n",
       "0.0      0.041769  0.016125  0.235945  0.138339  0.136415  0.377030  0.016253   \n",
       "1.0      0.003257  0.007339  0.174502  0.101622  0.153769  0.445207  0.017809   \n",
       "2.0      0.024655  0.037401  0.091521  0.087596  0.229066  0.141877  0.043438   \n",
       "3.0      0.053342  0.072499  0.044819  0.173665  0.247423  0.209091  0.067114   \n",
       "4.0      0.006233  0.012267  0.021860  0.144453  0.233614  0.357309  0.107493   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80958.0  0.011433  0.071507  0.090714  0.342257  0.161905  0.266439  0.012688   \n",
       "80959.0  0.005432  0.001247  0.230146  0.339425  0.145487  0.170682  0.007845   \n",
       "80960.0  0.012272  0.004515  0.298882  0.172716  0.186109  0.148433  0.004076   \n",
       "80961.0  0.026299  0.006660  0.171584  0.326464  0.154456  0.182478  0.007839   \n",
       "80962.0  0.012926  0.007080  0.175824  0.340701  0.194704  0.161014  0.008808   \n",
       "\n",
       "               15        16        17        18        19        20  \n",
       "0                                                                    \n",
       "0.0      0.057162  0.008264  0.006845  0.005032  0.006333  0.012382  \n",
       "1.0      0.075973  0.004658  0.005674  0.004011  0.004467  0.012308  \n",
       "2.0      0.258959  0.044236  0.022986  0.019999  0.035320  0.025003  \n",
       "3.0      0.156775  0.025087  0.015049  0.017127  0.020303  0.023548  \n",
       "4.0      0.073830  0.014845  0.007426  0.008810  0.014086  0.016274  \n",
       "...           ...       ...       ...       ...       ...       ...  \n",
       "80958.0  0.098222  0.004578  0.003098  0.004052  0.003347  0.012698  \n",
       "80959.0  0.081629  0.002546  0.005387  0.004262  0.003147  0.009444  \n",
       "80960.0  0.163437  0.004744  0.004990  0.003501  0.002442  0.010670  \n",
       "80961.0  0.132046  0.003085  0.004032  0.004596  0.002734  0.010685  \n",
       "80962.0  0.084242  0.004327  0.006544  0.007175  0.004352  0.012310  \n",
       "\n",
       "[80963 rows x 20 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df.to_csv(args.save_dir + 'preds_XL_trans.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.239032</td>\n",
       "      <td>0.069283</td>\n",
       "      <td>0.039445</td>\n",
       "      <td>0.055686</td>\n",
       "      <td>0.013506</td>\n",
       "      <td>0.498343</td>\n",
       "      <td>0.026811</td>\n",
       "      <td>0.041769</td>\n",
       "      <td>0.016125</td>\n",
       "      <td>0.235945</td>\n",
       "      <td>0.138339</td>\n",
       "      <td>0.136415</td>\n",
       "      <td>0.377030</td>\n",
       "      <td>0.016253</td>\n",
       "      <td>0.057162</td>\n",
       "      <td>0.008264</td>\n",
       "      <td>0.006845</td>\n",
       "      <td>0.005032</td>\n",
       "      <td>0.006333</td>\n",
       "      <td>0.012382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.176052</td>\n",
       "      <td>0.014921</td>\n",
       "      <td>0.002331</td>\n",
       "      <td>0.010017</td>\n",
       "      <td>0.003138</td>\n",
       "      <td>0.778988</td>\n",
       "      <td>0.003957</td>\n",
       "      <td>0.003257</td>\n",
       "      <td>0.007339</td>\n",
       "      <td>0.174502</td>\n",
       "      <td>0.101622</td>\n",
       "      <td>0.153769</td>\n",
       "      <td>0.445207</td>\n",
       "      <td>0.017809</td>\n",
       "      <td>0.075973</td>\n",
       "      <td>0.004658</td>\n",
       "      <td>0.005674</td>\n",
       "      <td>0.004011</td>\n",
       "      <td>0.004467</td>\n",
       "      <td>0.012308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.008293</td>\n",
       "      <td>0.438736</td>\n",
       "      <td>0.016735</td>\n",
       "      <td>0.016557</td>\n",
       "      <td>0.319096</td>\n",
       "      <td>0.018162</td>\n",
       "      <td>0.120364</td>\n",
       "      <td>0.024655</td>\n",
       "      <td>0.037401</td>\n",
       "      <td>0.091521</td>\n",
       "      <td>0.087596</td>\n",
       "      <td>0.229066</td>\n",
       "      <td>0.141877</td>\n",
       "      <td>0.043438</td>\n",
       "      <td>0.258959</td>\n",
       "      <td>0.044236</td>\n",
       "      <td>0.022986</td>\n",
       "      <td>0.019999</td>\n",
       "      <td>0.035320</td>\n",
       "      <td>0.025003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.007731</td>\n",
       "      <td>0.212674</td>\n",
       "      <td>0.060881</td>\n",
       "      <td>0.027692</td>\n",
       "      <td>0.098404</td>\n",
       "      <td>0.064098</td>\n",
       "      <td>0.402679</td>\n",
       "      <td>0.053342</td>\n",
       "      <td>0.072499</td>\n",
       "      <td>0.044819</td>\n",
       "      <td>0.173665</td>\n",
       "      <td>0.247423</td>\n",
       "      <td>0.209091</td>\n",
       "      <td>0.067114</td>\n",
       "      <td>0.156775</td>\n",
       "      <td>0.025087</td>\n",
       "      <td>0.015049</td>\n",
       "      <td>0.017127</td>\n",
       "      <td>0.020303</td>\n",
       "      <td>0.023548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.004392</td>\n",
       "      <td>0.884633</td>\n",
       "      <td>0.013877</td>\n",
       "      <td>0.005151</td>\n",
       "      <td>0.005875</td>\n",
       "      <td>0.053898</td>\n",
       "      <td>0.013674</td>\n",
       "      <td>0.006233</td>\n",
       "      <td>0.012267</td>\n",
       "      <td>0.021860</td>\n",
       "      <td>0.144453</td>\n",
       "      <td>0.233614</td>\n",
       "      <td>0.357309</td>\n",
       "      <td>0.107493</td>\n",
       "      <td>0.073830</td>\n",
       "      <td>0.014845</td>\n",
       "      <td>0.007426</td>\n",
       "      <td>0.008810</td>\n",
       "      <td>0.014086</td>\n",
       "      <td>0.016274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80958.0</th>\n",
       "      <td>0.059333</td>\n",
       "      <td>0.213964</td>\n",
       "      <td>0.010122</td>\n",
       "      <td>0.021720</td>\n",
       "      <td>0.009755</td>\n",
       "      <td>0.564988</td>\n",
       "      <td>0.037177</td>\n",
       "      <td>0.011433</td>\n",
       "      <td>0.071507</td>\n",
       "      <td>0.090714</td>\n",
       "      <td>0.342257</td>\n",
       "      <td>0.161905</td>\n",
       "      <td>0.266439</td>\n",
       "      <td>0.012688</td>\n",
       "      <td>0.098222</td>\n",
       "      <td>0.004578</td>\n",
       "      <td>0.003098</td>\n",
       "      <td>0.004052</td>\n",
       "      <td>0.003347</td>\n",
       "      <td>0.012698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80959.0</th>\n",
       "      <td>0.932363</td>\n",
       "      <td>0.010110</td>\n",
       "      <td>0.001268</td>\n",
       "      <td>0.031923</td>\n",
       "      <td>0.004079</td>\n",
       "      <td>0.008335</td>\n",
       "      <td>0.005243</td>\n",
       "      <td>0.005432</td>\n",
       "      <td>0.001247</td>\n",
       "      <td>0.230146</td>\n",
       "      <td>0.339425</td>\n",
       "      <td>0.145487</td>\n",
       "      <td>0.170682</td>\n",
       "      <td>0.007845</td>\n",
       "      <td>0.081629</td>\n",
       "      <td>0.002546</td>\n",
       "      <td>0.005387</td>\n",
       "      <td>0.004262</td>\n",
       "      <td>0.003147</td>\n",
       "      <td>0.009444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80960.0</th>\n",
       "      <td>0.789606</td>\n",
       "      <td>0.044373</td>\n",
       "      <td>0.003783</td>\n",
       "      <td>0.065795</td>\n",
       "      <td>0.006891</td>\n",
       "      <td>0.058347</td>\n",
       "      <td>0.014418</td>\n",
       "      <td>0.012272</td>\n",
       "      <td>0.004515</td>\n",
       "      <td>0.298882</td>\n",
       "      <td>0.172716</td>\n",
       "      <td>0.186109</td>\n",
       "      <td>0.148433</td>\n",
       "      <td>0.004076</td>\n",
       "      <td>0.163437</td>\n",
       "      <td>0.004744</td>\n",
       "      <td>0.004990</td>\n",
       "      <td>0.003501</td>\n",
       "      <td>0.002442</td>\n",
       "      <td>0.010670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80961.0</th>\n",
       "      <td>0.535197</td>\n",
       "      <td>0.232845</td>\n",
       "      <td>0.020557</td>\n",
       "      <td>0.047099</td>\n",
       "      <td>0.012990</td>\n",
       "      <td>0.035292</td>\n",
       "      <td>0.083062</td>\n",
       "      <td>0.026299</td>\n",
       "      <td>0.006660</td>\n",
       "      <td>0.171584</td>\n",
       "      <td>0.326464</td>\n",
       "      <td>0.154456</td>\n",
       "      <td>0.182478</td>\n",
       "      <td>0.007839</td>\n",
       "      <td>0.132046</td>\n",
       "      <td>0.003085</td>\n",
       "      <td>0.004032</td>\n",
       "      <td>0.004596</td>\n",
       "      <td>0.002734</td>\n",
       "      <td>0.010685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80962.0</th>\n",
       "      <td>0.644504</td>\n",
       "      <td>0.063732</td>\n",
       "      <td>0.006310</td>\n",
       "      <td>0.157164</td>\n",
       "      <td>0.005757</td>\n",
       "      <td>0.089713</td>\n",
       "      <td>0.012814</td>\n",
       "      <td>0.012926</td>\n",
       "      <td>0.007080</td>\n",
       "      <td>0.175824</td>\n",
       "      <td>0.340701</td>\n",
       "      <td>0.194704</td>\n",
       "      <td>0.161014</td>\n",
       "      <td>0.008808</td>\n",
       "      <td>0.084242</td>\n",
       "      <td>0.004327</td>\n",
       "      <td>0.006544</td>\n",
       "      <td>0.007175</td>\n",
       "      <td>0.004352</td>\n",
       "      <td>0.012310</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80963 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                1         2         3         4         5         6         7  \\\n",
       "0                                                                               \n",
       "0.0      0.239032  0.069283  0.039445  0.055686  0.013506  0.498343  0.026811   \n",
       "1.0      0.176052  0.014921  0.002331  0.010017  0.003138  0.778988  0.003957   \n",
       "2.0      0.008293  0.438736  0.016735  0.016557  0.319096  0.018162  0.120364   \n",
       "3.0      0.007731  0.212674  0.060881  0.027692  0.098404  0.064098  0.402679   \n",
       "4.0      0.004392  0.884633  0.013877  0.005151  0.005875  0.053898  0.013674   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80958.0  0.059333  0.213964  0.010122  0.021720  0.009755  0.564988  0.037177   \n",
       "80959.0  0.932363  0.010110  0.001268  0.031923  0.004079  0.008335  0.005243   \n",
       "80960.0  0.789606  0.044373  0.003783  0.065795  0.006891  0.058347  0.014418   \n",
       "80961.0  0.535197  0.232845  0.020557  0.047099  0.012990  0.035292  0.083062   \n",
       "80962.0  0.644504  0.063732  0.006310  0.157164  0.005757  0.089713  0.012814   \n",
       "\n",
       "                8         9        10        11        12        13        14  \\\n",
       "0                                                                               \n",
       "0.0      0.041769  0.016125  0.235945  0.138339  0.136415  0.377030  0.016253   \n",
       "1.0      0.003257  0.007339  0.174502  0.101622  0.153769  0.445207  0.017809   \n",
       "2.0      0.024655  0.037401  0.091521  0.087596  0.229066  0.141877  0.043438   \n",
       "3.0      0.053342  0.072499  0.044819  0.173665  0.247423  0.209091  0.067114   \n",
       "4.0      0.006233  0.012267  0.021860  0.144453  0.233614  0.357309  0.107493   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "80958.0  0.011433  0.071507  0.090714  0.342257  0.161905  0.266439  0.012688   \n",
       "80959.0  0.005432  0.001247  0.230146  0.339425  0.145487  0.170682  0.007845   \n",
       "80960.0  0.012272  0.004515  0.298882  0.172716  0.186109  0.148433  0.004076   \n",
       "80961.0  0.026299  0.006660  0.171584  0.326464  0.154456  0.182478  0.007839   \n",
       "80962.0  0.012926  0.007080  0.175824  0.340701  0.194704  0.161014  0.008808   \n",
       "\n",
       "               15        16        17        18        19        20  \n",
       "0                                                                    \n",
       "0.0      0.057162  0.008264  0.006845  0.005032  0.006333  0.012382  \n",
       "1.0      0.075973  0.004658  0.005674  0.004011  0.004467  0.012308  \n",
       "2.0      0.258959  0.044236  0.022986  0.019999  0.035320  0.025003  \n",
       "3.0      0.156775  0.025087  0.015049  0.017127  0.020303  0.023548  \n",
       "4.0      0.073830  0.014845  0.007426  0.008810  0.014086  0.016274  \n",
       "...           ...       ...       ...       ...       ...       ...  \n",
       "80958.0  0.098222  0.004578  0.003098  0.004052  0.003347  0.012698  \n",
       "80959.0  0.081629  0.002546  0.005387  0.004262  0.003147  0.009444  \n",
       "80960.0  0.163437  0.004744  0.004990  0.003501  0.002442  0.010670  \n",
       "80961.0  0.132046  0.003085  0.004032  0.004596  0.002734  0.010685  \n",
       "80962.0  0.084242  0.004327  0.006544  0.007175  0.004352  0.012310  \n",
       "\n",
       "[80963 rows x 20 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = pd.read_csv(args.save_dir + 'preds_XL_trans.csv', sep='\\t', index_col='0')\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = torch.tensor(np.array(preds.reset_index()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = torch.tensor(np.array(preds)).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df = pd.DataFrame(('val_ATT_loss', 'val_VAL_loss', 'val_ATT_acc', 'val_VAL_acc', 'val_VAL_acc_k', 'val_VAL_jac_k'), columns=['name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ATT_loss = F.cross_entropy(data_XL['all'].y[data_XL['all'].train_mask][:,:9], \n",
    "                pred[data_XL['all'].train_mask][:,:9]).cpu().detach().item()\n",
    "train_VAL_loss = F.cross_entropy(data_XL['all'].y[data_XL['all'].train_mask][:,9:], \n",
    "                pred[data_XL['all'].train_mask][:,9:]).cpu().detach().item()\n",
    "\n",
    "train_ATT_acc = compute_1_accuracy(data_XL['all'].y[data_XL['all'].train_mask][:,:9], \n",
    "                pred[data_XL['all'].train_mask][:,:9])\n",
    "train_VAL_acc = compute_1_accuracy(data_XL['all'].y[data_XL['all'].train_mask][:,9:], \n",
    "                pred[data_XL['all'].train_mask][:,9:])\n",
    "train_VAL_acc_k = compute_k_accuracy(data_XL['all'].y[data_XL['all'].train_mask][:,9:].cpu(),  \n",
    "                pred[data_XL['all'].train_mask][:,9:].cpu(),3)\n",
    "train_VAL_jac_k = compute_jaccard_index(data_XL['all'].y[data_XL['all'].train_mask][:,9:].cpu(),  \n",
    "                pred[data_XL['all'].train_mask][:,9:].cpu(),3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df['train'] = pd.DataFrame((train_ATT_loss, train_VAL_loss, train_ATT_acc, train_VAL_acc, train_VAL_acc_k, train_VAL_jac_k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_ATT_loss = F.cross_entropy(data_XL['all'].y[(data_XL['all'].att_lab) * data_XL['all'].val_mask][:,:9], \n",
    "                pred[(data_XL['all'].att_lab) * data_XL['all'].val_mask][:,:9]).cpu().detach().item()\n",
    "val_VAL_loss = F.cross_entropy(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:], \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:]).cpu().detach().item()\n",
    "\n",
    "val_ATT_acc = compute_1_accuracy(data_XL['all'].y[(data_XL['all'].att_lab) * data_XL['all'].val_mask][:,:9], \n",
    "                pred[(data_XL['all'].att_lab) * data_XL['all'].val_mask][:,:9])\n",
    "val_VAL_acc = compute_1_accuracy(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:], \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:])\n",
    "val_VAL_acc_k = compute_k_accuracy(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:].cpu(),3)\n",
    "val_VAL_jac_k = compute_jaccard_index(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:].cpu(),3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df['val'] = pd.DataFrame((val_ATT_loss, val_VAL_loss, val_ATT_acc, val_VAL_acc, val_VAL_acc_k, val_VAL_jac_k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ATT_loss = F.cross_entropy(data_XL['all'].y[(data_XL['all'].att_lab) * data_XL['all'].test_mask][:,:9], \n",
    "                pred[(data_XL['all'].att_lab) * data_XL['all'].test_mask][:,:9]).cpu().detach().item()\n",
    "test_VAL_loss = F.cross_entropy(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:], \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:]).cpu().detach().item()\n",
    "\n",
    "test_ATT_acc = compute_1_accuracy(data_XL['all'].y[(data_XL['all'].att_lab) * data_XL['all'].test_mask][:,:9], \n",
    "                pred[(data_XL['all'].att_lab) * data_XL['all'].test_mask][:,:9])\n",
    "test_VAL_acc = compute_1_accuracy(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:], \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:])\n",
    "test_VAL_acc_k = compute_k_accuracy(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:].cpu(),3)\n",
    "test_VAL_jac_k = compute_jaccard_index(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:].cpu(),3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df['test'] = pd.DataFrame((test_ATT_loss, test_VAL_loss, test_ATT_acc, test_VAL_acc, test_VAL_acc_k, test_VAL_jac_k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>train</th>\n",
       "      <th>val</th>\n",
       "      <th>test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>val_ATT_loss</td>\n",
       "      <td>1.783117</td>\n",
       "      <td>1.770349</td>\n",
       "      <td>1.772480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>val_VAL_loss</td>\n",
       "      <td>2.250756</td>\n",
       "      <td>2.248278</td>\n",
       "      <td>2.248172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>val_ATT_acc</td>\n",
       "      <td>90.656064</td>\n",
       "      <td>95.096744</td>\n",
       "      <td>94.615205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>val_VAL_acc</td>\n",
       "      <td>77.119889</td>\n",
       "      <td>78.807107</td>\n",
       "      <td>78.481013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>val_VAL_acc_k</td>\n",
       "      <td>98.487337</td>\n",
       "      <td>98.406655</td>\n",
       "      <td>98.410689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>val_VAL_jac_k</td>\n",
       "      <td>0.706277</td>\n",
       "      <td>0.705349</td>\n",
       "      <td>0.698453</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            name      train        val       test\n",
       "0   val_ATT_loss   1.783117   1.770349   1.772480\n",
       "1   val_VAL_loss   2.250756   2.248278   2.248172\n",
       "2    val_ATT_acc  90.656064  95.096744  94.615205\n",
       "3    val_VAL_acc  77.119889  78.807107  78.481013\n",
       "4  val_VAL_acc_k  98.487337  98.406655  98.410689\n",
       "5  val_VAL_jac_k   0.706277   0.705349   0.698453"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df.to_csv(args.save_dir+'eval_metrics_XL_trans.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Per-Class Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_k_confusion_matrix(y, y_pred, k=3):\n",
    "    dim = y.shape[-1]\n",
    "    y = y.topk(k=k, axis=1)[1]\n",
    "    y_pred = y_pred.topk(k=k, axis=1)[1]\n",
    "    conf = np.zeros((dim, dim))\n",
    "    for i in range(k):\n",
    "        for j in range(k):\n",
    "            conf = np.add(conf, confusion_matrix(y[:,i], y_pred[:,j], labels = range(dim)))\n",
    "    return conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ATT_conf = confusion_matrix(data_XL['all'].y[(data_XL['all'].att_lab) * data_XL['all'].test_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                pred[(data_XL['all'].att_lab) * data_XL['all'].test_mask][:,:9].argmax(axis=1).cpu())\n",
    "test_VAL_conf = confusion_matrix(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "test_VAL_conf_k = (top_k_confusion_matrix(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].test_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1323,   10,    0,   12,    1,   39,    4,    0,    0],\n",
       "       [   1, 3158,    0,    3,    0,   16,   33,    0,    0],\n",
       "       [   0,    0,  248,    0,    0,    0,   17,    0,    0],\n",
       "       [  43,    6,    0,  561,    0,    1,   25,    1,    0],\n",
       "       [   7,   18,    0,    0, 2642,    3,   32,    0,    2],\n",
       "       [   1,   20,    0,    2,    2, 1557,    1,    0,    0],\n",
       "       [   1,   32,    2,    0,   14,    0, 2544,    0,    0],\n",
       "       [  19,    7,   85,   19,    2,    1,  160,   20,    0],\n",
       "       [   0,   15,    0,    0,   13,   22,    0,    0,  106]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 588,   14,   19,    6,    0,   53,    0,    0,    0,    0,    0],\n",
       "       [  52,  547,   99,   26,    0,  125,    0,    0,    0,    0,    0],\n",
       "       [  55,   55, 1157,   32,    0,  308,    0,    0,    0,    0,    0],\n",
       "       [  34,   29,   73,  316,    0,   84,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [  45,   29,  359,   17,    0, 2965,    0,    0,    0,    0,    0],\n",
       "       [   1,    0,    2,    0,    0,    5,    0,    0,    0,    4,    0],\n",
       "       [   0,    0,    1,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [   1,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    1,    0,    0,    1,    0,    0,    0,    7,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1436,  793, 1142,  708,    0,  874,    0,    0,    0,    0,    0],\n",
       "       [ 974, 1699, 2228, 1661,    1, 1645,    0,    0,    0,    0,    0],\n",
       "       [1211, 1493, 5800, 4465,    2, 5112,    0,    0,    0,    1,    0],\n",
       "       [1050, 1387, 4743, 4419,    2, 4203,    0,    0,    0,    0,    0],\n",
       "       [   1,    9,   18,   16,    1,    9,    0,    0,    0,    0,    0],\n",
       "       [1052, 1006, 5282, 4084,    0, 5245,    0,    0,    0,    2,    0],\n",
       "       [  14,    0,   29,    7,    0,   23,    6,    0,    1,   16,    0],\n",
       "       [   2,    0,    6,    0,    0,    6,    1,    0,    0,    6,    0],\n",
       "       [   5,    0,   13,    0,    0,    6,    5,    0,    1,   12,    0],\n",
       "       [   6,    0,   17,    0,    0,   10,    6,    0,    1,   17,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_ATT_conf = confusion_matrix(data_XL['all'].y[(data_XL['all'].att_lab) * data_XL['all'].val_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                pred[(data_XL['all'].att_lab) * data_XL['all'].val_mask][:,:9].argmax(axis=1).cpu())\n",
    "val_VAL_conf = confusion_matrix(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "val_VAL_conf_k = (top_k_confusion_matrix(data_XL['all'].y[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:].cpu(),  \n",
    "                pred[(data_XL['all'].val_lab) * data_XL['all'].val_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1304,    5,    0,    6,    2,   28,    7,    0,    0],\n",
       "       [   1, 3251,    1,    2,    0,   15,   29,    0,    0],\n",
       "       [   0,    0,  264,    0,    0,    0,   16,    0,    0],\n",
       "       [  47,    9,    0,  597,    0,    3,   22,    2,    0],\n",
       "       [   8,   24,    0,    0, 2617,    4,   33,    0,    1],\n",
       "       [   2,   18,    0,    1,    2, 1465,    0,    0,    0],\n",
       "       [   1,   23,    2,    0,   11,    0, 2593,    0,    0],\n",
       "       [  17,    3,   71,   16,    2,    1,  150,   26,    0],\n",
       "       [   1,   14,    0,    0,   13,   17,    1,    0,  121]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 585,   23,   12,   10,    0,   37,    0,    0,    0,    0,    0],\n",
       "       [  58,  519,  109,   45,    0,   98,    0,    0,    0,    0,    0],\n",
       "       [  53,   48, 1217,   30,    0,  296,    0,    0,    0,    0,    0],\n",
       "       [  49,   40,   91,  340,    0,   78,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    3,    1,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [  36,   39,  328,    8,    0, 2922,    0,    0,    0,    0,    0],\n",
       "       [   2,    0,    3,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [   1,    0,    1,    0,    0,    0,    0,    0,    0,    1,    0],\n",
       "       [   0,    0,    1,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    2,    0,    0,    0,    0,    0,    0,    6,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1449,  785, 1170,  737,    0,  875,    0,    0,    0,    0,    0],\n",
       "       [1036, 1719, 2343, 1759,    1, 1677,    0,    0,    0,    0,    0],\n",
       "       [1193, 1559, 5772, 4418,    1, 5025,    1,    0,    0,    1,    0],\n",
       "       [1007, 1440, 4660, 4345,    2, 4113,    0,    0,    0,    0,    0],\n",
       "       [   0,   11,   22,   22,    2,    9,    0,    0,    0,    0,    0],\n",
       "       [1011, 1014, 5203, 4052,    0, 5164,    1,    0,    0,    1,    0],\n",
       "       [  20,    0,   38,   13,    2,   35,    2,    0,    0,   10,    0],\n",
       "       [   2,    0,    3,    0,    0,    1,    0,    0,    0,    3,    0],\n",
       "       [   7,    0,   15,    1,    1,   10,    2,    0,    0,   12,    0],\n",
       "       [   8,    0,   16,    1,    0,   11,    3,    0,    0,   12,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]])"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ATT_conf = confusion_matrix(data_XL['all'].y[data_XL['all'].train_mask][:,:9].argmax(axis=1).cpu(), \n",
    "                                  pred[data_XL['all'].train_mask][:,:9].argmax(axis=1).cpu())\n",
    "train_VAL_conf = confusion_matrix(data_XL['all'].y[data_XL['all'].train_mask][:,9:].argmax(axis=1).cpu(), \n",
    "                                 pred[data_XL['all'].train_mask][:,9:].argmax(axis=1).cpu(), labels=range(11))\n",
    "train_VAL_conf_k = (top_k_confusion_matrix(data_XL['all'].y[data_XL['all'].train_mask][:,9:].cpu(),  \n",
    "                                pred[data_XL['all'].train_mask][:,9:].cpu(),3)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1413,   18,    0,   12,    0,   53,    5,    0,    0],\n",
       "       [   0, 2606,    0,    1,    0,   14,   15,    0,    0],\n",
       "       [   0,    2,  126,    1,    0,    0,   10,    0,    0],\n",
       "       [  35,   12,    0,  418,    0,    2,   12,    1,    0],\n",
       "       [   9,   27,    1,    1, 1954,   10,   46,    0,    3],\n",
       "       [   2,   29,    0,    0,    2, 1474,    0,    0,    0],\n",
       "       [   0,   43,    1,    5,   17,    0, 2391,    0,    0],\n",
       "       [  52,   56,   91,   49,    9,    0,  396,   32,    0],\n",
       "       [   0,   16,    0,    0,    5,   18,    0,    0,   74]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ATT_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 861,   32,   30,    8,    0,   81,    0,    0,    0,    0,    0],\n",
       "       [  79,  831,  217,   59,    0,  192,    0,    0,    0,    0,    0],\n",
       "       [  63,   70, 1927,   49,    0,  706,    0,    0,    0,    0,    0],\n",
       "       [  60,   57,  143,  659,    0,  124,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    1,    1,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [  63,   34,  513,   41,    0, 4632,    0,    0,    0,    0,    0],\n",
       "       [   3,    0,    2,    0,    0,    7,    1,    0,    0,    8,    0],\n",
       "       [   1,    0,    1,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    1,    0],\n",
       "       [   0,    0,    0,    0,    0,    1,    0,    0,    0,   11,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_VAL_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2113, 1091, 1666, 1094,    0, 1423,    0,    0,    0,    2,    0],\n",
       "       [1409, 2749, 3928, 2991,    0, 3035,    0,    0,    0,    0,    0],\n",
       "       [1788, 2529, 9521, 7307,    1, 8445,    1,    0,    0,    0,    0],\n",
       "       [1613, 2292, 7742, 7136,    1, 6950,    0,    0,    0,    0,    0],\n",
       "       [   0,   32,   54,   52,    1,   23,    0,    0,    0,    0,    0],\n",
       "       [1552, 1613, 8482, 6579,    0, 8535,    1,    0,    0,    1,    0],\n",
       "       [  22,    1,   53,   13,    0,   39,   14,    0,    3,   29,    0],\n",
       "       [   9,    1,   17,    1,    0,   11,    3,    0,    0,   12,    0],\n",
       "       [   4,    0,   15,    0,    0,    7,   10,    0,    3,   18,    0],\n",
       "       [   7,    0,   25,    0,    0,   11,   13,    0,    3,   25,    0],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0]])"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_VAL_conf_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat([pd.DataFrame(train_ATT_conf),pd.DataFrame(val_ATT_conf),pd.DataFrame(test_ATT_conf)],axis=1).to_csv(args.save_dir+'confusion_matrix_ATT_XL_trans.csv')\n",
    "pd.concat([pd.DataFrame(train_VAL_conf),pd.DataFrame(val_VAL_conf),pd.DataFrame(test_VAL_conf)],axis=1).to_csv(args.save_dir+'confusion_matrix_VAL_XL_trans.csv')\n",
    "pd.concat([pd.DataFrame(train_VAL_conf_k),pd.DataFrame(val_VAL_conf_k),pd.DataFrame(test_VAL_conf_k)],axis=1).to_csv(args.save_dir+'confusion_matrix_VAL_k_XL_trans.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics(confusion_matrix, classes):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val/(row.sum()+0.000000001)\n",
    "        recall = val/(col.sum()+0.000000001)\n",
    "        F1 = 2*(precision*recall)/(precision+recall+0.000000001)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics_k(confusion_matrix, classes, k=3):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val*k/(row.sum()+0.000000001)\n",
    "        recall = val*k/(col.sum()+0.000000001)\n",
    "        F1 = 2*(precision*recall)/(precision+recall+0.000000001)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = ['Criterion i', 'Criterion ii', 'Criterion iii', 'Criterion iv', 'Criterion v', 'Criterion vi', \n",
    "              'Criterion vii', 'Criterion viii', 'Criterion ix', 'Criterion x', 'Others']\n",
    "categories = ['Building Elements',\n",
    " 'Urban Form Elements',\n",
    " 'Gastronomy',\n",
    " 'Interior Scenery',\n",
    " 'Natural Features and Land-scape Scenery',\n",
    " 'Monuments and Buildings',\n",
    " 'Peoples Activity and Association',\n",
    " 'Artifact Products',\n",
    " 'Urban Scenery']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_dict = {}\n",
    "metrics_dict['test_ATT'] = per_class_metrics(test_ATT_conf, categories)\n",
    "metrics_dict['val_ATT'] = per_class_metrics(val_ATT_conf, categories)\n",
    "metrics_dict['test_VAL'] = per_class_metrics(test_VAL_conf, classes)\n",
    "metrics_dict['val_VAL'] = per_class_metrics(val_VAL_conf, classes)\n",
    "metrics_dict['test_VAL_k'] = per_class_metrics_k(test_VAL_conf_k, classes)\n",
    "metrics_dict['val_VAL_k'] = per_class_metrics_k(val_VAL_conf_k, classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df = pd.DataFrame.from_dict({(i,j): metrics_dict[i][j] \n",
    "                           for i in metrics_dict.keys() \n",
    "                           for j in metrics_dict[i].keys()},\n",
    "                       orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">test_ATT</th>\n",
       "      <th>Building Elements</th>\n",
       "      <td>0.952484</td>\n",
       "      <td>0.948387</td>\n",
       "      <td>0.950431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Urban Form Elements</th>\n",
       "      <td>0.983494</td>\n",
       "      <td>0.966932</td>\n",
       "      <td>0.975143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gastronomy</th>\n",
       "      <td>0.935849</td>\n",
       "      <td>0.740299</td>\n",
       "      <td>0.826667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Interior Scenery</th>\n",
       "      <td>0.880691</td>\n",
       "      <td>0.939698</td>\n",
       "      <td>0.909238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Natural Features and Land-scape Scenery</th>\n",
       "      <td>0.977071</td>\n",
       "      <td>0.988033</td>\n",
       "      <td>0.982521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">val_VAL_k</th>\n",
       "      <th>Criterion vii</th>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.093023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Criterion viii</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Criterion ix</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Criterion x</th>\n",
       "      <td>0.705882</td>\n",
       "      <td>0.923077</td>\n",
       "      <td>0.800000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Others</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>62 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   precision    recall  \\\n",
       "test_ATT  Building Elements                         0.952484  0.948387   \n",
       "          Urban Form Elements                       0.983494  0.966932   \n",
       "          Gastronomy                                0.935849  0.740299   \n",
       "          Interior Scenery                          0.880691  0.939698   \n",
       "          Natural Features and Land-scape Scenery   0.977071  0.988033   \n",
       "...                                                      ...       ...   \n",
       "val_VAL_k Criterion vii                             0.050000  0.666667   \n",
       "          Criterion viii                            0.000000  0.000000   \n",
       "          Criterion ix                              0.000000  0.000000   \n",
       "          Criterion x                               0.705882  0.923077   \n",
       "          Others                                    0.000000  0.000000   \n",
       "\n",
       "                                                         F1  \n",
       "test_ATT  Building Elements                        0.950431  \n",
       "          Urban Form Elements                      0.975143  \n",
       "          Gastronomy                               0.826667  \n",
       "          Interior Scenery                         0.909238  \n",
       "          Natural Features and Land-scape Scenery  0.982521  \n",
       "...                                                     ...  \n",
       "val_VAL_k Criterion vii                            0.093023  \n",
       "          Criterion viii                           0.000000  \n",
       "          Criterion ix                             0.000000  \n",
       "          Criterion x                              0.800000  \n",
       "          Others                                   0.000000  \n",
       "\n",
       "[62 rows x 3 columns]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df.to_csv(args.save_dir+'per_class_metrics_XL_trans.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}