21-train_RNN_RNN_OnlyNER_Wikipedia_geo_ratio_sc_0.5.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "606fa719",
   "metadata": {},
   "source": [
    "# Train 19-train_RNN_RNN_Wikipedia_geo_ratio_sc_0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a8a5bd39",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn.utils import clip_grad_norm_\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "from rouge_score import rouge_scorer\n",
    "\n",
    "from utils_french_sum_ner.EmbMgr import EmbMgr\n",
    "from utils_french_sum_ner.Dataset import Dataset\n",
    "from utils_french_sum_ner.DataLoader import DataLoader\n",
    "from utils_french_sum_ner.preprocess_df import preprocess_df\n",
    "from utils_french_sum_ner.accuracy_nb_sent_per_doc import accuracy_nb_sent_per_doc_fn\n",
    "from utils_french_sum_ner.accuracy_prop_sent_per_doc import accuracy_prop_sent_per_doc_fn\n",
    "from utils_french_sum_ner.split_all_docs import split_all_docs\n",
    "\n",
    "from models.RNN_RNN import RNN_RNN\n",
    "\n",
    "from time import time\n",
    "\n",
    "import json\n",
    "import argparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "818410ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_notebook() -> bool:\n",
    "    try:\n",
    "        shell = get_ipython().__class__.__name__\n",
    "        if shell == 'ZMQInteractiveShell':\n",
    "            return True   # Jupyter notebook or qtconsole\n",
    "        elif shell == 'TerminalInteractiveShell':\n",
    "            return False  # Terminal running IPython\n",
    "        else:\n",
    "            return False  # Other type (?)\n",
    "    except NameError:\n",
    "        return False      # Probably standard Python interpreter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "690fa032",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "b2636e0b",
   "metadata": {},
   "source": [
    "## (Hyper-)parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6e7aad39",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse args if script mode\n",
    "parser = argparse.ArgumentParser(description='extractive summary')\n",
    "\n",
    "parser.add_argument('-is_graphic',type=int,default=1,choices=[0,1])\n",
    "parser.add_argument('-gpu_num',type=int,default=1)\n",
    "parser.add_argument('-batch_size',type=int,default=8)\n",
    "parser.add_argument('-epochs',type=int,default=100)\n",
    "\n",
    "args = None\n",
    "\n",
    "if is_notebook():\n",
    "    args = parser.parse_args(\"\")\n",
    "else:\n",
    "    args = parser.parse_args()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "824eacf5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "parse:\n",
      "is_graphic: True\n",
      "cuda_num: 1\n",
      "epochs 100\n",
      "batch_size 8\n"
     ]
    }
   ],
   "source": [
    "# parameters\n",
    "is_graphic = args.is_graphic != 0\n",
    "cuda_num = args.gpu_num\n",
    "embed_name = \"cc.fr.300.vec\"\n",
    "\n",
    "# hyper-parameters\n",
    "vocab_size = 150000\n",
    "batch_size = args.batch_size\n",
    "epochs = args.epochs\n",
    "learning_rate = 1e-3\n",
    "early_stopping = 3\n",
    "model_name = \"21-train_RNN_RNN_OnlyNER_Wikipedia_geo_ratio_sc_0.5\"\n",
    "sub_folder_name = \"model_name__{}__time__{}__embed_name__{}__lr__{}__batch_size__{}__vocab_size__{}__cuda_num__{}\".format(model_name, time(), embed_name, learning_rate, batch_size, vocab_size, cuda_num)\n",
    "checkpoints_folder = \"./checkpoints/\" + sub_folder_name\n",
    "average_number_of_sentences_per_document = 3\n",
    "loss_sum_coef = 0.5\n",
    "loss_ner_coef = 0.5\n",
    "\n",
    "# print\n",
    "print(\"parse:\")\n",
    "print(\"is_graphic:\", is_graphic)\n",
    "print(\"cuda_num:\", cuda_num)\n",
    "print(\"epochs\", epochs)\n",
    "print(\"batch_size\", batch_size)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "c8ddd4d5",
   "metadata": {},
   "source": [
    "## PyTorch initialisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d09bc3ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of available GPUs: 5\n",
      "GPU 0: NVIDIA GeForce GTX 1080 Ti\n",
      "GPU 1: NVIDIA GeForce GTX 1080 Ti\n",
      "GPU 2: NVIDIA GeForce GTX 1080\n",
      "GPU 3: NVIDIA GeForce GTX 1080\n",
      "GPU 4: NVIDIA GeForce GTX 1080\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/raoufdine/.local/lib/python3.9/site-packages/torch/cuda/__init__.py:497: UserWarning: Can't initialize NVML\n",
      "  warnings.warn(\"Can't initialize NVML\")\n"
     ]
    }
   ],
   "source": [
    "# Check if a GPU is available\n",
    "if torch.cuda.is_available():\n",
    "    # Display the number of available GPUs\n",
    "    print(f\"Number of available GPUs: {torch.cuda.device_count()}\")\n",
    "    # Display the name of each GPU\n",
    "    for i in range(torch.cuda.device_count()):\n",
    "        print(f\"GPU {i}: {torch.cuda.get_device_name(i)}\")\n",
    "else:\n",
    "    print(\"No GPU available.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b0c5786e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=1)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if torch.cuda.is_available():  \n",
    "  dev = \"cuda:\" + str(cuda_num) \n",
    "else:  \n",
    "  dev = \"cpu\" \n",
    "\n",
    "device = torch.device(dev)\n",
    "device"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "3393fe4e",
   "metadata": {},
   "source": [
    "## Chargement des données"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "97644745",
   "metadata": {},
   "outputs": [],
   "source": [
    "embmgr = EmbMgr(\"./data/cc.fr.300.vec\", vocab_size=vocab_size, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fe745c8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df_train = pd.read_json(\"./data/wiki_geo_ratio_sc_0.5/train.json\")\n",
    "train_dataset = Dataset(preprocess_df(df_train, glovemgr=embmgr, is_sep_n=False, doc_column_name=\"flat_contents\", labels_column_name=\"labels_sentences\", labels_ner_col_name=\"labels_entities\", remove_stop_word=True, stemming=False, trunc_sent=50, padding_sent=50, trunc_doc=100))\n",
    "train_iter = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f8b39f3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_val = pd.read_json(\"./data/wiki_geo_ratio_sc_0.5/val.json\")\n",
    "val_dataset = Dataset(preprocess_df(df_val, glovemgr=embmgr, is_sep_n=False, doc_column_name=\"flat_contents\", labels_column_name=\"labels_sentences\", labels_ner_col_name=\"labels_entities\", remove_stop_word=True, stemming=False, trunc_sent=50, padding_sent=50, trunc_doc=100))\n",
    "val_iter = DataLoader(dataset=val_dataset, batch_size=1, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fb052a32",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = pd.read_json(\"./data/wiki_geo_ratio_sc_0.5/test.json\")\n",
    "test_dataset = Dataset(preprocess_df(df_test, glovemgr=embmgr, is_sep_n=False, doc_column_name=\"flat_contents\", labels_column_name=\"labels_sentences\", labels_ner_col_name=\"labels_entities\", remove_stop_word=True, stemming=False, trunc_sent=50, padding_sent=50, trunc_doc=100))\n",
    "test_iter = DataLoader(dataset=test_dataset, batch_size=1, shuffle=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "fc41eb07",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b756c45f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from models.BasicModel import BasicModel\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class RNN_RNN(BasicModel):\n",
    "    def __init__(self, device, vocab_size, word_embed = None, dim_word_embed = 100):\n",
    "        super(RNN_RNN, self).__init__(device)\n",
    "\n",
    "        self.device = device\n",
    "\n",
    "        self.word_embedding = nn.Embedding(vocab_size+2, dim_word_embed, padding_idx=0)\n",
    "        # Load word embedding if specified\n",
    "        if word_embed is not None:\n",
    "            self.word_embedding = torch.nn.Embedding.from_pretrained(torch.from_numpy(word_embed).float())\n",
    "\n",
    "        # 100 : word2vec embedding size\n",
    "        self.word_GRU = nn.GRU(input_size = dim_word_embed, hidden_size = 200, batch_first = True, bidirectional = True)\n",
    "        self.sent_GRU = nn.GRU(input_size = 2*200, hidden_size=200, batch_first = True, bidirectional = True)\n",
    "\n",
    "        # 10: relative position range size, with segment size = 10\n",
    "        self.rel_pos_emb = nn.Embedding(11, 100)\n",
    "        self.abs_pos_emb = nn.Embedding(100, 100)\n",
    "\n",
    "        self.Wdoc = nn.Linear(2*200,2*200,bias=True)\n",
    "\n",
    "        self.Wcontent = nn.Linear(2*200,1,bias=False)\n",
    "        self.Wsalience = nn.Bilinear(2*200,2*200,1,bias=False)\n",
    "        self.Wnovelty = nn.Bilinear(2*200,2*200,1,bias=False)\n",
    "        self.Wabs_pos = nn.Linear(100,1,bias=False)\n",
    "        self.Wrel_pos = nn.Linear(100,1,bias=False)\n",
    "        self.bias = nn.Parameter(torch.empty(1).uniform_(-0.1, 0.1))\n",
    "\n",
    "        self.w_ner = nn.Linear(400, 1)\n",
    "\n",
    "    def avg_pool1d(self,x,seq_lens):\n",
    "        out = []\n",
    "        for index,t in enumerate(x):\n",
    "            if seq_lens[index] == 0:\n",
    "                t = t[:1]\n",
    "            else:\n",
    "                t = t[:seq_lens[index],:]\n",
    "            t = torch.t(t).unsqueeze(0)\n",
    "            out.append(F.avg_pool1d(t,t.size(2)))\n",
    "        \n",
    "        out = torch.cat(out).squeeze(2)\n",
    "        return out\n",
    "\n",
    "    def forward(self, arr_x, doc_lens):\n",
    "        probs = []\n",
    "\n",
    "        original_arr_x = arr_x\n",
    "\n",
    "        max_doc_lens = max(doc_lens)\n",
    "        sent_lens = torch.sum(torch.sign(arr_x),dim=1).data\n",
    "\n",
    "        arr_x = self.word_embedding(arr_x)\n",
    "        arr_x = self.word_GRU(arr_x)[0]\n",
    "\n",
    "        o_ner = arr_x\n",
    "        o_ner = o_ner.reshape(-1, arr_x.size(-1))[original_arr_x.view(-1) != 0, :]\n",
    "        o_ner = torch.sigmoid(self.w_ner(o_ner)).squeeze(-1)\n",
    "\n",
    "        arr_x = self.avg_pool1d(arr_x, sent_lens)\n",
    "\n",
    "        arr_x = self.pad_doc(arr_x, doc_lens)\n",
    "\n",
    "        arr_x = self.sent_GRU(arr_x)[0]\n",
    "        sents_all_docs = arr_x.reshape(len(doc_lens) * max_doc_lens, 2*200) # flat docs\n",
    "        \n",
    "        docs = self.avg_pool1d(arr_x, doc_lens)\n",
    "        docs = torch.tanh(self.Wdoc(docs))\n",
    "        docs_for_each_sent = docs.repeat_interleave(max_doc_lens, dim=0) # repeat doc n times\n",
    "        \n",
    "        contents = self.Wcontent(sents_all_docs)\n",
    "        saliences = self.Wsalience(sents_all_docs, docs_for_each_sent)\n",
    "\n",
    "        # Compute position embedding\n",
    "        abs_poses = torch.tensor([[position] for position in range(max_doc_lens)], dtype=torch.long)\n",
    "        abs_poses = abs_poses.to(self.device)\n",
    "        abs_poses = self.abs_pos_emb(abs_poses).squeeze(1)\n",
    "        aps = self.Wabs_pos(abs_poses)\n",
    "\n",
    "        # Compute relative position embedding\n",
    "        rel_poses = torch.tensor([[int(round(position / 10))] for position in range(max_doc_lens)], dtype=torch.long)\n",
    "        rel_poses = rel_poses.to(self.device)\n",
    "        rel_poses = self.rel_pos_emb(rel_poses).squeeze(1)\n",
    "        rps = self.Wrel_pos(rel_poses)\n",
    "\n",
    "        # doc_len (number) s\n",
    "        ses = torch.zeros(len(doc_lens), 2*200)\n",
    "        ses = ses.to(self.device)\n",
    "\n",
    "        # for each document, compute probabilities\n",
    "        for position in range(max_doc_lens):\n",
    "            contents_for_position = contents[position::max_doc_lens]\n",
    "            saliences_for_position = saliences[position::max_doc_lens]\n",
    "            novelties_for_position = -1 * self.Wnovelty(sents_all_docs[position::max_doc_lens], torch.tanh(ses))\n",
    "            prob = contents_for_position + saliences_for_position + novelties_for_position + aps[position] + rps[position] + self.bias\n",
    "            prob = torch.sigmoid(prob)\n",
    "            probs.append(prob)\n",
    "\n",
    "            ses = ses + torch.mul(sents_all_docs[position::max_doc_lens], prob)\n",
    "        probs = torch.cat(probs).squeeze()\n",
    "        if len(probs.shape) == 0:\n",
    "            probs = probs.unsqueeze(0)\n",
    "        probs_filtered = []\n",
    "        for idx, doc_len in enumerate(doc_lens):\n",
    "            prob = probs[idx::len(doc_lens)]\n",
    "            prob = prob[:doc_len]\n",
    "            probs_filtered.append(prob)\n",
    "        probs_filtered = torch.cat(probs_filtered).squeeze()\n",
    "        if len(probs_filtered.shape) == 0:\n",
    "            probs_filtered = probs_filtered.unsqueeze(0)\n",
    "        #probs = probs.to(self.device)\n",
    "        return probs_filtered, o_ner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "380f8ba1",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = RNN_RNN(device=device, vocab_size=vocab_size, word_embed=embmgr.getEmbeddings(), dim_word_embed=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2ae1a236",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNN_RNN(\n",
       "  (word_embedding): Embedding(150001, 300)\n",
       "  (word_GRU): GRU(300, 200, batch_first=True, bidirectional=True)\n",
       "  (sent_GRU): GRU(400, 200, batch_first=True, bidirectional=True)\n",
       "  (rel_pos_emb): Embedding(11, 100)\n",
       "  (abs_pos_emb): Embedding(100, 100)\n",
       "  (Wdoc): Linear(in_features=400, out_features=400, bias=True)\n",
       "  (Wcontent): Linear(in_features=400, out_features=1, bias=False)\n",
       "  (Wsalience): Bilinear(in1_features=400, in2_features=400, out_features=1, bias=False)\n",
       "  (Wnovelty): Bilinear(in1_features=400, in2_features=400, out_features=1, bias=False)\n",
       "  (Wabs_pos): Linear(in_features=100, out_features=1, bias=False)\n",
       "  (Wrel_pos): Linear(in_features=100, out_features=1, bias=False)\n",
       "  (w_ner): Linear(in_features=400, out_features=1, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4bc411ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = nn.BCELoss()\n",
    "optimizer = torch.optim.Adamax(model.parameters(),lr=learning_rate)\n",
    "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.5)\n",
    "mae_fn = nn.L1Loss()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "1852182a",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "fef6d114",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(checkpoints_folder):\n",
    "    os.makedirs(checkpoints_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5c803c4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_val[\"doc_splitted\"] = split_all_docs(df_val[\"flat_contents\"], is_sep_n = False)\n",
    "val_set = df_val\n",
    "scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0991d3c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1: 100%|██████████| 13/13 [00:01<00:00,  8.39batch/s, accuracy_ner=0.842, loss=0.503, loss_ner=0.503]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 : val loss = 0.367, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2: 100%|██████████| 13/13 [00:01<00:00,  8.50batch/s, accuracy_ner=0.894, loss=0.344, loss_ner=0.344]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2 : val loss = 0.361, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3: 100%|██████████| 13/13 [00:01<00:00,  9.24batch/s, accuracy_ner=0.897, loss=0.328, loss_ner=0.328]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3 : val loss = 0.359, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4: 100%|██████████| 13/13 [00:01<00:00,  9.33batch/s, accuracy_ner=0.898, loss=0.325, loss_ner=0.325]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4 : val loss = 0.358, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5: 100%|██████████| 13/13 [00:01<00:00,  8.87batch/s, accuracy_ner=0.897, loss=0.326, loss_ner=0.326]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6: 100%|██████████| 13/13 [00:01<00:00,  9.05batch/s, accuracy_ner=0.897, loss=0.325, loss_ner=0.325]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 7: 100%|██████████| 13/13 [00:01<00:00,  9.13batch/s, accuracy_ner=0.895, loss=0.33, loss_ner=0.33]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 8: 100%|██████████| 13/13 [00:01<00:00,  9.21batch/s, accuracy_ner=0.897, loss=0.324, loss_ner=0.324]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 9: 100%|██████████| 13/13 [00:01<00:00,  9.65batch/s, accuracy_ner=0.898, loss=0.322, loss_ner=0.322]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 10: 100%|██████████| 13/13 [00:01<00:00,  9.45batch/s, accuracy_ner=0.896, loss=0.327, loss_ner=0.327]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 11: 100%|██████████| 13/13 [00:01<00:00,  9.42batch/s, accuracy_ner=0.896, loss=0.327, loss_ner=0.327]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 12: 100%|██████████| 13/13 [00:01<00:00,  9.34batch/s, accuracy_ner=0.897, loss=0.325, loss_ner=0.325]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 13: 100%|██████████| 13/13 [00:01<00:00,  8.58batch/s, accuracy_ner=0.894, loss=0.331, loss_ner=0.331]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 14: 100%|██████████| 13/13 [00:01<00:00,  8.87batch/s, accuracy_ner=0.895, loss=0.327, loss_ner=0.327]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 15: 100%|██████████| 13/13 [00:01<00:00,  9.22batch/s, accuracy_ner=0.896, loss=0.326, loss_ner=0.326]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 15 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 16: 100%|██████████| 13/13 [00:01<00:00,  8.88batch/s, accuracy_ner=0.892, loss=0.334, loss_ner=0.334]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 16 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 17: 100%|██████████| 13/13 [00:01<00:00,  9.07batch/s, accuracy_ner=0.893, loss=0.331, loss_ner=0.331]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 17 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 18: 100%|██████████| 13/13 [00:01<00:00,  9.05batch/s, accuracy_ner=0.896, loss=0.326, loss_ner=0.326]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 19: 100%|██████████| 13/13 [00:01<00:00,  8.79batch/s, accuracy_ner=0.894, loss=0.33, loss_ner=0.33]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 19 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 20: 100%|██████████| 13/13 [00:01<00:00,  8.88batch/s, accuracy_ner=0.896, loss=0.327, loss_ner=0.327]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 20 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 21: 100%|██████████| 13/13 [00:01<00:00,  9.34batch/s, accuracy_ner=0.9, loss=0.318, loss_ner=0.318]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 21 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 22: 100%|██████████| 13/13 [00:01<00:00,  8.56batch/s, accuracy_ner=0.896, loss=0.326, loss_ner=0.326]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 22 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 23: 100%|██████████| 13/13 [00:01<00:00,  9.02batch/s, accuracy_ner=0.895, loss=0.328, loss_ner=0.328]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 23 : val loss = 0.357, val mae = 0.000, val accuracy = 0.000, r1 = 0.000, r2 = 0.000, rL = 0.000\n",
      "Training duration = 96.55654907226562\n"
     ]
    }
   ],
   "source": [
    "arr_train_loss = []\n",
    "arr_train_loss_sum = []\n",
    "arr_train_loss_ner = []\n",
    "arr_train_mae = []\n",
    "arr_train_acc = []\n",
    "arr_train_acc_ner = []\n",
    "arr_val_loss = []\n",
    "arr_val_mae = []\n",
    "arr_val_acc = []\n",
    "arr_val_acc_ner = []\n",
    "\n",
    "t1 = time()\n",
    "for epoch in range(1, epochs+1):\n",
    "    # train 1 epoch\n",
    "    model.train()\n",
    "    nb_batch_train = 0\n",
    "    total_train_loss = 0\n",
    "    total_train_mae = 0\n",
    "    total_train_acc = 0\n",
    "    total_train_loss_sum = 0\n",
    "    total_train_loss_ner = 0\n",
    "    total_train_acc_ner = 0\n",
    "    with tqdm(train_iter, unit=\"batch\", total=len(train_iter)) as tepoch:\n",
    "        for batch in tepoch:\n",
    "            tepoch.set_description(f\"Epoch {epoch}\")\n",
    "            features = []\n",
    "            doc_lens = []\n",
    "\n",
    "            for j in range(batch_size):\n",
    "                doc_lens.append(len(batch[j][\"docs\"]))\n",
    "                features = features + batch[j][\"docs\"]\n",
    "            \n",
    "            features = torch.tensor(features, dtype=torch.long).to(device)\n",
    "\n",
    "            #targets = [torch.tensor(batch[j][\"labels\"], dtype=torch.float) for j in range(batch_size)]\n",
    "            #targets = torch.cat(targets)\n",
    "            #targets = targets.to(device)\n",
    "\n",
    "            targets_ner = [torch.tensor(batch[j][\"labels_ner\"][k], dtype=torch.float) for j in range(batch_size) for k in range(len(batch[j][\"labels_ner\"]))]\n",
    "            targets_ner = torch.cat(targets_ner)\n",
    "            targets_ner = targets_ner.to(device)\n",
    "\n",
    "            probs, y_ner_pred = model(features, doc_lens)\n",
    "            \n",
    "            #loss_sum = loss_fn(probs, targets)\n",
    "            loss_ner = loss_fn(y_ner_pred, targets_ner)\n",
    "\n",
    "            loss = loss_ner # loss_sum_coef * loss_sum + loss_ner_coef * loss_ner\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            clip_grad_norm_(model.parameters(), 1.0)\n",
    "            optimizer.step()\n",
    "            nb_batch_train += 1\n",
    "            total_train_loss += loss.item()\n",
    "            #total_train_loss_sum += loss_sum.item()\n",
    "            total_train_loss_ner += loss_ner.item()\n",
    "            #total_train_mae += mae_fn(probs, targets).item()\n",
    "            #total_train_acc += accuracy_prop_sent_per_doc_fn(probs=probs.cpu().detach().numpy(), targets=targets.cpu().detach().numpy(), doc_lens=doc_lens)\n",
    "            total_train_acc_ner += torch.sum(((y_ner_pred > 0.5).float() == targets_ner).float()).item() / targets_ner.shape[0]\n",
    "            tepoch.set_postfix(loss=total_train_loss/nb_batch_train, loss_ner=total_train_loss_ner/nb_batch_train, accuracy_ner=total_train_acc_ner/nb_batch_train)\n",
    "\n",
    "    # Save model\n",
    "    model.save(checkpoints_folder + \"/\" + model_name + \"-\" + str(epoch) + \".pt\")\n",
    "\n",
    "    # Show train and val score\n",
    "    model.eval()\n",
    "    nb_batch_val = 0\n",
    "    total_val_loss = 0\n",
    "    total_val_mae = 0\n",
    "    total_val_acc = 0\n",
    "    total_val_loss_sum = 0\n",
    "    total_val_loss_ner = 0\n",
    "    total_val_acc_ner = 0\n",
    "    total_r1 = 0\n",
    "    total_r2 = 0\n",
    "    total_rl = 0\n",
    "\n",
    "    for i, batch in enumerate(val_iter):\n",
    "        features = []\n",
    "        doc_lens = []\n",
    "\n",
    "        for j in range(1):\n",
    "            doc_lens.append(len(batch[j][\"docs\"]))\n",
    "            features = features + batch[j][\"docs\"]\n",
    "            \n",
    "        features = torch.tensor(features, dtype=torch.long).to(device)\n",
    "\n",
    "        #targets = [torch.tensor(batch[j][\"labels\"], dtype=torch.float) for j in range(1)]\n",
    "        #targets = torch.cat(targets)\n",
    "        #targets = targets.to(device)\n",
    "\n",
    "        targets_ner = [torch.tensor(batch[j][\"labels_ner\"][k], dtype=torch.float) for j in range(1) for k in range(len(batch[j][\"labels_ner\"]))]\n",
    "        targets_ner = torch.cat(targets_ner)\n",
    "        targets_ner = targets_ner.to(device)\n",
    "\n",
    "        y_pred, y_ner_pred = model(features, doc_lens)\n",
    "        \n",
    "        #loss_sum = loss_fn(y_pred, targets)\n",
    "        loss_ner = loss_fn(y_ner_pred, targets_ner)\n",
    "\n",
    "        loss = loss_ner # loss_sum_coef * loss_sum + loss_ner_coef * loss_ner\n",
    "\n",
    "        nb_batch_val += 1\n",
    "        total_val_loss += loss.item()\n",
    "        #total_val_loss_sum += loss_sum.item()\n",
    "        total_val_loss_ner += loss_ner.item()\n",
    "\n",
    "        total_val_acc_ner += torch.sum(((y_ner_pred > 0.5).float() == targets_ner).float()).item() / targets_ner.shape[0]\n",
    "\n",
    "    arr_train_loss.append(total_train_loss / nb_batch_train)\n",
    "    arr_train_acc_ner.append(total_train_acc_ner / nb_batch_train)\n",
    "\n",
    "    arr_val_loss.append(total_val_loss / nb_batch_val)\n",
    "    arr_val_acc_ner.append(total_val_acc_ner / nb_batch_val)\n",
    "    \n",
    "    print(\"Epoch {} : val loss = {:.3f}, val mae = {:.3f}, val accuracy = {:.3f}, r1 = {:.3f}, r2 = {:.3f}, rL = {:.3f}\".format(epoch, total_val_loss / nb_batch_val, total_val_mae / nb_batch_val, total_val_acc / nb_batch_val, total_r1 / nb_batch_val, total_r2 / nb_batch_val, total_rl / nb_batch_val))\n",
    "\n",
    "    if len(arr_val_loss) >= early_stopping+1:\n",
    "        if min(arr_val_loss[-early_stopping:]) >= arr_val_loss[-(early_stopping+1)]:\n",
    "            break\n",
    "\n",
    "    # update lr\n",
    "    scheduler.step()\n",
    "\n",
    "t2 = time()\n",
    "print(\"Training duration =\", t2-t1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2f39ceed",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_metrics = {}\n",
    "training_metrics[\"duration\"]   = t2 - t1\n",
    "training_metrics[\"train_loss\"] = arr_train_loss\n",
    "training_metrics[\"train_mae\"]  = arr_train_mae\n",
    "training_metrics[\"train_acc\"]  = arr_train_acc\n",
    "training_metrics[\"train_acc_ner\"] = arr_train_acc_ner\n",
    "training_metrics[\"val_loss\"]   = arr_val_loss\n",
    "training_metrics[\"val_mae\"]    = arr_val_mae\n",
    "training_metrics[\"val_acc\"]    = arr_val_acc\n",
    "training_metrics[\"val_acc_ner\"] = arr_val_acc_ner\n",
    "\n",
    "# Save to file in JSON format\n",
    "\n",
    "with open(checkpoints_folder + \"/training_metrics.json\", 'w') as fp:\n",
    "    json.dump(training_metrics, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "8cd8fbee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAHHCAYAAABEEKc/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAA9hAAAPYQGoP6dpAABdN0lEQVR4nO3deVxU5eI/8M/MwMywD4iAIMqm4AoqQqR5NTHIcim7aVEqmfatbJGsm91yvTe8WV6zLLulv9ab3jYtLUpxKQ2xQMoMUVHEhUXZhkWGYeb8/gAGh0URZubMwOf9ep0XcOY5Z57DOM6HZzsSQRAEEBEREZGBVOwKEBEREVkbBiQiIiKiFhiQiIiIiFpgQCIiIiJqgQGJiIiIqAUGJCIiIqIWGJCIiIiIWmBAIiIiImqBAYmIiIioBQYkIiIiohYYkIio23n//fchkUjw66+/il0VIrJRDEhERERELTAgEREREbXAgEREPdKRI0dw++23w9XVFc7Ozpg4cSIOHTpkVEar1WLFihUYMGAAlEolevXqhbFjx2LXrl2GMoWFhUhMTETfvn2hUCjQp08fTJs2DXl5eRa+IiIyJTuxK0BEZGnHjh3DLbfcAldXVzz33HOwt7fHO++8g/Hjx2P//v2Ijo4GACxfvhzJycl4+OGHERUVBbVajV9//RWZmZmYNGkSAGDGjBk4duwYnnjiCQQEBKC4uBi7du1Cfn4+AgICRLxKIuoKiSAIgtiVICIypffffx+JiYn45ZdfEBkZ2erxu+66C99++y2ys7MRFBQEACgoKEBoaChGjBiB/fv3AwAiIiLQt29f7Nixo83nKS8vh7u7O9asWYPFixeb74KIyOLYxUZEPYpOp8MPP/yA6dOnG8IRAPTp0wf3338/Dhw4ALVaDQBQqVQ4duwYTp482ea5HBwcIJfLsW/fPpSVlVmk/kRkGQxIRNSjXLp0CTU1NQgNDW312KBBg6DX63Hu3DkAwMqVK1FeXo6BAwdi2LBhePbZZ/H7778byisUCvzrX//Cd999B29vb4wbNw6vvPIKCgsLLXY9RGQeDEhERO0YN24ccnNzsXnzZgwdOhTvvfceRo4ciffee89Q5umnn8aJEyeQnJwMpVKJl156CYMGDcKRI0dErDkRdRUDEhH1KL1794ajoyNycnJaPXb8+HFIpVL4+/sb9nl4eCAxMRGffvopzp07h+HDh2P58uVGxwUHB+OZZ57BDz/8gD/++AN1dXV47bXXzH0pRGRGDEhE1KPIZDLcdttt2L59u9FU/KKiIvz3v//F2LFj4erqCgAoKSkxOtbZ2RkhISHQaDQAgJqaGtTW1hqVCQ4OhouLi6EMEdkmTvMnom5r8+bNSElJabV/+fLl2LVrF8aOHYvHHnsMdnZ2eOedd6DRaPDKK68Yyg0ePBjjx4/HqFGj4OHhgV9//RWff/45Fi5cCAA4ceIEJk6ciHvvvReDBw+GnZ0dvvrqKxQVFWHWrFkWu04iMj1O8yeibqdpmn97zp07h0uXLmHJkiU4ePAg9Ho9oqOj8c9//hMxMTGGcv/85z/x9ddf48SJE9BoNOjfvz8efPBBPPvss7C3t0dJSQmWLVuG1NRUnDt3DnZ2dggLC8MzzzyDv/71r5a4VCIyEwYkIiIiohY4BomIiIioBQYkIiIiohYYkIiIiIhaYEAiIiIiaoEBiYiIiKgFBiQiIiKiFrhQZCfp9XpcvHgRLi4ukEgkYleHiIiIOkAQBFRWVsLX1xdSafvtRAxInXTx4kWj+zURERGR7Th37hz69u3b7uMMSJ3k4uICoOEX3HTfJiIiIrJuarUa/v7+hs/x9jAgdVJTt5qrqysDEhERkY253vAYDtImIiIiaoEBiYiIiKgFBiQiIiKiFjgGyYz0ej3q6urEroZNsre3h0wmE7saRETUQzEgmUldXR3OnDkDvV4vdlVslkqlgo+PD9eZIiIii2NAMgNBEFBQUACZTAZ/f/9rLkRFrQmCgJqaGhQXFwMA+vTpI3KNiIiop2FAMoP6+nrU1NTA19cXjo6OYlfHJjk4OAAAiouL4eXlxe42IiKyKDZtmIFOpwMAyOVykWti25rCpVarFbkmRETU0zAgmRHHznQNf39ERCQWBiQiIiKiFhiQyCwCAgKwbt06satBRETUKRykTQbjx49HRESESYLNL7/8Aicnp65XioiISAQMSFamXqeHTi/AXiaFVGpdY3AEQYBOp4Od3fX/2fTu3dsCNSIiIjIPdrFZmVPFVcgpqkRtvc6izzt37lzs378fr7/+OiQSCSQSCd5//31IJBJ89913GDVqFBQKBQ4cOIDc3FxMmzYN3t7ecHZ2xujRo7F7926j87XsYpNIJHjvvfdw1113wdHREQMGDMDXX39t0WskIiLqKAYkCxAEATV19R3a6nR61Gp1qKzVdviYa22CIHSojq+//jpiYmIwf/58FBQUoKCgAP7+/gCA559/HqtXr0Z2djaGDx+OqqoqTJ48GampqThy5Aji4+MxZcoU5OfnX/M5VqxYgXvvvRe///47Jk+ejISEBJSWlnb590tERGRqonexbdiwAWvWrEFhYSHCw8PxxhtvICoqqs2y77//PhITE432KRQK1NbWGn4WBAHLli3Du+++i/LycowZMwZvv/02BgwYYChTWlqKJ554At988w2kUilmzJiB119/Hc7Ozma5xitaHQYv/d4s576eP1fGwVF+/ZfZzc0Ncrkcjo6O8PHxAQAcP34cALBy5UpMmjTJUNbDwwPh4eGGn1etWoWvvvoKX3/9NRYuXNjuc8ydOxf33XcfAODll1/G+vXrcfjwYcTHx3fq2oiIiMxF1BakrVu3IikpCcuWLUNmZibCw8MRFxdnuMVEW1xdXQ0tHAUFBTh79qzR46+88grWr1+PjRs3Ij09HU5OToiLizMKUQkJCTh27Bh27dqFHTt24Mcff8SCBQvMdp22LjIy0ujnqqoqLF68GIMGDYJKpYKzszOys7Ov24I0fPhww/dOTk5wdXW95mtNREQkFlFbkNauXYv58+cbWoU2btyInTt3YvPmzXj++efbPEYikRhaOFoSBAHr1q3Diy++iGnTpgEAPvzwQ3h7e2Pbtm2YNWsWsrOzkZKSgl9++cXwwf/GG29g8uTJePXVV+Hr62vy63Swl+HPlXEdKnuh7ArKaurg5aKEl6vCJM/dVS1noy1evBi7du3Cq6++ipCQEDg4OOCee+5BXV3dNc9jb29v9LNEIuHNfImIyCqJ1oJUV1eHjIwMxMbGNldGKkVsbCzS0tLaPa6qqgr9+/eHv78/pk2bhmPHjhkeO3PmDAoLC43O6ebmhujoaMM509LSoFKpjFpFYmNjIZVKkZ6e3u7zajQaqNVqo62jJBIJHOV2HdpclHZQ2ssgt5N2+JhrbTeyGrVcLjfcJuVaDh48iLlz5+Kuu+7CsGHD4OPjg7y8vA4/DxERkbUTLSBdvnwZOp0O3t7eRvu9vb1RWFjY5jGhoaHYvHkztm/fjo8//hh6vR4333wzzp8/DwCG4651zsLCQnh5eRk9bmdnBw8Pj3afFwCSk5Ph5uZm2JoGMJuarHFqv07fscHVphQQEID09HTk5eXh8uXL7bbuDBgwAF9++SWysrLw22+/4f7772dLEBERdSs2NYstJiYGs2fPRkREBP7yl7/gyy+/RO/evfHOO++Y/bmXLFmCiooKw3bu3DmzPI9M2vCS1IsQkBYvXgyZTIbBgwejd+/e7Y4pWrt2Ldzd3XHzzTdjypQpiIuLw8iRIy1cWyIiIvMRbQySp6cnZDIZioqKjPYXFRW1O8aoJXt7e4wYMQKnTp0CAMNxRUVF6NOnj9E5IyIiDGVaDgyur69HaWnpNZ9XoVBAoej6mKDrEbMFaeDAga26N+fOnduqXEBAAPbs2WO07/HHHzf6uWWXW1vLDZSXl3eqnkREROYmWguSXC7HqFGjkJqaatin1+uRmpqKmJiYDp1Dp9Ph6NGjhjAUGBgIHx8fo3Oq1Wqkp6cbzhkTE4Py8nJkZGQYyuzZswd6vR7R0dGmuLQusRMxIBEREVEDUWexJSUlYc6cOYiMjERUVBTWrVuH6upqw6y22bNnw8/PD8nJyQAa1uO56aabEBISgvLycqxZswZnz57Fww8/DKBhMPTTTz+Nf/zjHxgwYAACAwPx0ksvwdfXF9OnTwcADBo0CPHx8Zg/fz42btwIrVaLhQsXYtasWWaZwXajmluQOKaHiIhILKIGpJkzZ+LSpUtYunQpCgsLERERgZSUFMMg6/z8fEilzY1cZWVlmD9/PgoLC+Hu7o5Ro0bh559/xuDBgw1lnnvuOVRXV2PBggUoLy/H2LFjkZKSAqVSaSjzySefYOHChZg4caJhocj169db7sKv4eouNkEQbmgWGhEREZmGROjovSjIiFqthpubGyoqKuDq6mr0WG1tLc6cOYPAwECjYNYRekHAHxcqAACD+7jCTmZT4+hNqiu/RyIiorZc6/P7aj3309dKSSUSSBtbjXTMrkRERKJgQLJCYs5kIyIiIgYkq8SAREREJC4GJCvEqf5ERETiYkCyQk0tSGKspk1EREQMSFbJVrvYAgICsG7dOrGrQURE1GUMSFbIVgMSERFRd8GAZIU4BomIiEhcDEhWSNa4erglA9J//vMf+Pr6Qt/iFifTpk3DQw89hNzcXEybNg3e3t5wdnbG6NGjsXv3bovVj4iIyJIYkCxBEIC66g5vsvoaSLQ10Gk6fky7WwcXm/zrX/+KkpIS7N2717CvtLQUKSkpSEhIQFVVFSZPnozU1FQcOXIE8fHxmDJlCvLz8831WyMiIhKNqPdi6zG0NcDLHb8RrhuAYaZ67hcuAnKn6xZzd3fH7bffjv/+97+YOHEiAODzzz+Hp6cnJkyYAKlUivDwcEP5VatW4auvvsLXX3+NhQsXmqq2REREVoEtSGSQkJCAL774AhqNBkDDTX1nzZoFqVSKqqoqLF68GIMGDYJKpYKzszOys7PZgkRERN0SW5Aswd6xoSWng7Q6PY4XVkICYIivKySN92br9HN30JQpUyAIAnbu3InRo0fjp59+wr///W8AwOLFi7Fr1y68+uqrCAkJgYODA+655x7U1dV1vm5ERERWigHJEiSSDnVzNZHpBQj2OggA9PaOhkHb5qZUKnH33Xfjk08+walTpxAaGoqRI0cCAA4ePIi5c+firrvuAgBUVVUhLy/PIvUiIiKyNAYkKySVSiCVSKAXBNTrBcgs2BGakJCAO++8E8eOHcMDDzxg2D9gwAB8+eWXmDJlCiQSCV566aVWM96IiIi6C45BslJiLRZ56623wsPDAzk5Obj//vsN+9euXQt3d3fcfPPNmDJlCuLi4gytS0RERN0NW5CslEwqgVZn+YAklUpx8WLr8VIBAQHYs2eP0b7HH3/c6Gd2uRERUXfBFiQrxduNEBERiYcByUrxdiNERETiYUCyUk0tSPUMSERERBbHgGSl2MVGREQkHgYkMxI6eB+0tjAgde33R0RE1BUMSGYgk8kAoEurTHMMElBTUwMAsLe3F7kmRETU03CavxnY2dnB0dERly5dgr29PaSdWAlbp9VCqK+DRqNDba3MDLW0XoIgoKamBsXFxVCpVIbASUREZCkMSGYgkUjQp08fnDlzBmfPnu3UOTT1OlyqrIO9TAK9WmniGtoGlUoFHx8fsatBREQ9EAOSmcjlcgwYMKDT3WynL1Vh+de/QuVgjy8eG2Pi2lk/e3t7thwREZFoGJDMSCqVQqnsXOuPhytwoVKHwmo9FAoFJBKJiWtHRERE7eEgbSulcmwYmKzTC6jS1ItcGyIiop6FAclKKe1lUNo3vDzlNVqRa0NERNSzMCBZMZWDHAADEhERkaUxIFmxpm628iudX0+JiIiIbhwDkhVzc2gMSGxBIiIisigGJCvW3ILEgERERGRJDEhWzN2xcQxSNbvYiIiILEn0gLRhwwYEBARAqVQiOjoahw8f7tBxW7ZsgUQiwfTp0432SySSNrc1a9YYygQEBLR6fPXq1aa8LJNwYwsSERGRKEQNSFu3bkVSUhKWLVuGzMxMhIeHIy4uDsXFxdc8Li8vD4sXL8Ytt9zS6rGCggKjbfPmzZBIJJgxY4ZRuZUrVxqVe+KJJ0x6babAWWxERETiEDUgrV27FvPnz0diYiIGDx6MjRs3wtHREZs3b273GJ1Oh4SEBKxYsQJBQUGtHvfx8THatm/fjgkTJrQq6+LiYlTOycnJ5NfXVU1jkCo4i42IiMiiRAtIdXV1yMjIQGxsbHNlpFLExsYiLS2t3eNWrlwJLy8vzJs377rPUVRUhJ07d7ZZdvXq1ejVqxdGjBiBNWvWoL7+2qtVazQaqNVqo83cVJzFRkREJArR7sV2+fJl6HQ6eHt7G+339vbG8ePH2zzmwIED2LRpE7Kysjr0HB988AFcXFxw9913G+1/8sknMXLkSHh4eODnn3/GkiVLUFBQgLVr17Z7ruTkZKxYsaJDz2sqqsZB2mU1bEEiIiKyJJu5WW1lZSUefPBBvPvuu/D09OzQMZs3b0ZCQkKrG8YmJSUZvh8+fDjkcjkeeeQRJCcnQ6FQtHmuJUuWGB2nVqvh7+/fiSvpuOYuNrYgERERWZJoAcnT0xMymQxFRUVG+4uKiuDj49OqfG5uLvLy8jBlyhTDPr1eDwCws7NDTk4OgoODDY/99NNPyMnJwdatW69bl+joaNTX1yMvLw+hoaFtllEoFO2GJ3MxrINUo4UgCJBIJBZ9fiIiop5KtDFIcrkco0aNQmpqqmGfXq9HamoqYmJiWpUPCwvD0aNHkZWVZdimTp2KCRMmICsrq1VrzqZNmzBq1CiEh4dfty5ZWVmQSqXw8vLq+oWZUNMstnq9gOo6nci1ISIi6jlE7WJLSkrCnDlzEBkZiaioKKxbtw7V1dVITEwEAMyePRt+fn5ITk6GUqnE0KFDjY5XqVQA0Gq/Wq3GZ599htdee63Vc6alpSE9PR0TJkyAi4sL0tLSsGjRIjzwwANwd3c3z4V2ktJeCrmdFHX1epTX1MFZYTM9okRERDZN1E/cmTNn4tKlS1i6dCkKCwsRERGBlJQUw8Dt/Px8SKU33si1ZcsWCIKA++67r9VjCoUCW7ZswfLly6HRaBAYGIhFixYZjS+yFhKJBO6O9ihSa1Beo0Vf68pvRERE3ZZEEARB7ErYIrVaDTc3N1RUVMDV1dVszxP37x+RU1SJj+dFY+yAjg1OJyIiorZ19PNb9FuN0LU1326EU/2JiIgshQHJynGxSCIiIstjQLJyXAuJiIjI8hiQrJx702ra1exiIyIishQGJCvXPAaJLUhERESWwoBk5ZoWi+QYJCIiIsthQLJyzWOQ2MVGRERkKQxIVo6z2IiIiCyPAcnKqZoGaTMgERERWQwDkpW7uouNi54TERFZBgOSlWsKSFqdgJo6nci1ISIi6hkYkKycg70MclnDy8Sp/kRERJbBgGTlJBJJ81pINZzJRkREZAkMSDbA3ZEz2YiIiCyJAckGcLFIIiIiy2JAsgHNtxthFxsREZElMCDZAC4WSUREZFkMSDageS0kBiQiIiJLYECyAYbVtKvZxUZERGQJDEg2QGUYg8QWJCIiIktgQLIBTbPYKjgGiYiIyCIYkGyAirPYiIiILIoByQa4cRYbERGRRTEg2QB3p+aFIgVBELk2RERE3R8Dkg1oWgepTqfHFa1O5NoQERF1fwxINsBRLoO9TAKA3WxERESWwIBkAyQSCdx4PzYiIiKLYUCyEZzJRkREZDkMSDbC3ZEz2YiIiCyFAclGsIuNiIjIchiQbAS72IiIiCyHAclGNE315+1GiIiIzI8ByUZcvVgkERERmRcDko1out1IWQ272IiIiMxN9IC0YcMGBAQEQKlUIjo6GocPH+7QcVu2bIFEIsH06dON9s+dOxcSicRoi4+PNypTWlqKhIQEuLq6QqVSYd68eaiqqjLVJZlF8xgktiARERGZm6gBaevWrUhKSsKyZcuQmZmJ8PBwxMXFobi4+JrH5eXlYfHixbjlllvafDw+Ph4FBQWG7dNPPzV6PCEhAceOHcOuXbuwY8cO/Pjjj1iwYIHJrsscVI2z2DgGiYiIyPxEDUhr167F/PnzkZiYiMGDB2Pjxo1wdHTE5s2b2z1Gp9MhISEBK1asQFBQUJtlFAoFfHx8DJu7u7vhsezsbKSkpOC9995DdHQ0xo4dizfeeANbtmzBxYsXTX6NpsJZbERERJYjWkCqq6tDRkYGYmNjmysjlSI2NhZpaWntHrdy5Up4eXlh3rx57ZbZt28fvLy8EBoaikcffRQlJSWGx9LS0qBSqRAZGWnYFxsbC6lUivT09C5elfmouFAkERGRxdiJ9cSXL1+GTqeDt7e30X5vb28cP368zWMOHDiATZs2ISsrq93zxsfH4+6770ZgYCByc3Pxwgsv4Pbbb0daWhpkMhkKCwvh5eVldIydnR08PDxQWFjY7nk1Gg00Go3hZ7Va3YGrNB2VY0MXm6Zejyt1OjjIZRZ9fiIiop5EtIB0oyorK/Hggw/i3XffhaenZ7vlZs2aZfh+2LBhGD58OIKDg7Fv3z5MnDix08+fnJyMFStWdPr4rnKSy2AnlaBeL6D8Sh0c5A6i1YWIiKi7E62LzdPTEzKZDEVFRUb7i4qK4OPj06p8bm4u8vLyMGXKFNjZ2cHOzg4ffvghvv76a9jZ2SE3N7fN5wkKCoKnpydOnToFAPDx8Wk1CLy+vh6lpaVtPm+TJUuWoKKiwrCdO3fuRi+5SyQSCbvZiIiILES0gCSXyzFq1CikpqYa9un1eqSmpiImJqZV+bCwMBw9ehRZWVmGberUqZgwYQKysrLg7+/f5vOcP38eJSUl6NOnDwAgJiYG5eXlyMjIMJTZs2cP9Ho9oqOj262vQqGAq6ur0WZpTWshMSARERGZl6hdbElJSZgzZw4iIyMRFRWFdevWobq6GomJiQCA2bNnw8/PD8nJyVAqlRg6dKjR8SqVCgAM+6uqqrBixQrMmDEDPj4+yM3NxXPPPYeQkBDExcUBAAYNGoT4+HjMnz8fGzduhFarxcKFCzFr1iz4+vpa7uI7wd1RDqAaFZzJRkREZFaiBqSZM2fi0qVLWLp0KQoLCxEREYGUlBTDwO38/HxIpR1v5JLJZPj999/xwQcfoLy8HL6+vrjtttuwatUqKBQKQ7lPPvkECxcuxMSJEyGVSjFjxgysX7/e5Ndnak1dbGVsQSIiIjIriSAIgtiVsEVqtRpubm6oqKiwWHfbM//7DV9knsff4sPw6PhgizwnERFRd9LRz2/RbzVCHcfFIomIiCyDAcmGqBoHafN2I0RERObFgGRDVE4Ni0VyFhsREZF5MSDZkKYWpLIadrERERGZEwOSDWkag1RxhS1IRERE5sSAZENUDuxiIyIisgQGJBvCWWxERESWwYBkQ5oCUq1Wj1qtTuTaEBERdV8MSDbEWWEHmVQCgN1sRERE5sSAZEMkEolhJhu72YiIiMyHAcnGuDWNQ2ILEhERkdkwINkYQwsSAxIREZHZMCDZGHfHpqn+7GIjIiIyFwYkG2PoYuNikURERGbDgGRjuFgkERGR+TEg2Zjm242wi42IiMhcGJBsjIqz2IiIiMyOAcnGqBoHaZdxkDYREZHZMCDZGE7zJyIiMj8GJBvTPAaJAYmIiMhcGJBsDGexERERmR8Dko1pWgfpilaHWq1O5NoQERF1TwxINsZVaQeZVAKA3WxERETmwoBkYyQSCdw4UJuIiMisGJBsUPNMNk71JyIiMgcGJBvE+7ERERGZFwOSDWpqQapgFxsREZFZMCDZIHeupk1ERGRWDEg2iF1sRERE5sWAZIO4WCQREZF5MSDZoObbjbCLjYiIyBwYkGxQU0BiCxIREZF5MCDZIJVhkDYDEhERkTkwINmg5mn+7GIjIiIyBwYkG6TiLDYiIiKzEj0gbdiwAQEBAVAqlYiOjsbhw4c7dNyWLVsgkUgwffp0wz6tVou//e1vGDZsGJycnODr64vZs2fj4sWLRscGBARAIpEYbatXrzblZZlV0yy2mjodNPU6kWtDRETU/YgakLZu3YqkpCQsW7YMmZmZCA8PR1xcHIqLi695XF5eHhYvXoxbbrnFaH9NTQ0yMzPx0ksvITMzE19++SVycnIwderUVudYuXIlCgoKDNsTTzxh0mszJxelHaSShu8r2IpERERkcqIGpLVr12L+/PlITEzE4MGDsXHjRjg6OmLz5s3tHqPT6ZCQkIAVK1YgKCjI6DE3Nzfs2rUL9957L0JDQ3HTTTfhzTffREZGBvLz843Kuri4wMfHx7A5OTmZ5RrNQSqVwM2BM9mIiIjMRbSAVFdXh4yMDMTGxjZXRipFbGws0tLS2j1u5cqV8PLywrx58zr0PBUVFZBIJFCpVEb7V69ejV69emHEiBFYs2YN6uvrr3kejUYDtVpttImpaSYbAxIREZHp2Yn1xJcvX4ZOp4O3t7fRfm9vbxw/frzNYw4cOIBNmzYhKyurQ89RW1uLv/3tb7jvvvvg6upq2P/kk09i5MiR8PDwwM8//4wlS5agoKAAa9eubfdcycnJWLFiRYee1xKaW5A4k42IiMjURAtIN6qyshIPPvgg3n33XXh6el63vFarxb333gtBEPD2228bPZaUlGT4fvjw4ZDL5XjkkUeQnJwMhULR5vmWLFlidJxarYa/v38nr6brOJONiIjIfEQLSJ6enpDJZCgqKjLaX1RUBB8fn1blc3NzkZeXhylTphj26fV6AICdnR1ycnIQHBwMoDkcnT17Fnv27DFqPWpLdHQ06uvrkZeXh9DQ0DbLKBSKdsOTGJrXQmJAIiIiMjXRxiDJ5XKMGjUKqamphn16vR6pqamIiYlpVT4sLAxHjx5FVlaWYZs6dSomTJiArKwsQ2tOUzg6efIkdu/ejV69el23LllZWZBKpfDy8jLdBZpZ82ra7GIjIiIyNVG72JKSkjBnzhxERkYiKioK69atQ3V1NRITEwEAs2fPhp+fH5KTk6FUKjF06FCj45sGXjft12q1uOeee5CZmYkdO3ZAp9OhsLAQAODh4QG5XI60tDSkp6djwoQJcHFxQVpaGhYtWoQHHngA7u7ulrv4LmIXGxERkfmIGpBmzpyJS5cuYenSpSgsLERERARSUlIMA7fz8/MhlXa8kevChQv4+uuvAQARERFGj+3duxfjx4+HQqHAli1bsHz5cmg0GgQGBmLRokVG44tsAbvYiIiIzEciCIIgdiVskVqthpubGyoqKq47xskcth25gKe3ZmFMSC988vBNFn9+IiIiW9TRz2/RbzVCnePmyIUiiYiIzIUByUa5c6FIIiIis2FAslEqLhRJRERkNgxINqppFlt1nQ519XqRa0NERNS9MCDZKBelPSSShu8rONWfiIjIpBiQbJRMKoGrsnGq/xV2sxEREZkSA5INc2/sZivjQG0iIiKTYkCyYW6cyUZERGQWDEg2jDPZiIiIzIMByYY1zWTjIG0iIiLT6lRA+uCDD7Bz507Dz8899xxUKhVuvvlmnD171mSVo2trbkFiQCIiIjKlTgWkl19+GQ4ODgCAtLQ0bNiwAa+88go8PT2xaNEik1aQ2qdqHINUxi42IiIik7LrzEHnzp1DSEgIAGDbtm2YMWMGFixYgDFjxmD8+PGmrB9dQ1MXWzm72IiIiEyqUy1Izs7OKCkpAQD88MMPmDRpEgBAqVTiypUrpqsdXZNhDBK72IiIiEyqUy1IkyZNwsMPP4wRI0bgxIkTmDx5MgDg2LFjCAgIMGX96BpUDo3T/LlQJBERkUl1qgVpw4YNiImJwaVLl/DFF1+gV69eAICMjAzcd999Jq0gtc/NkYO0iYiIzKFTLUgqlQpvvvlmq/0rVqzocoWo49y5UCQREZFZdKoFKSUlBQcOHDD8vGHDBkREROD+++9HWVmZySpH19Y0zb9KUw+tTi9ybYiIiLqPTgWkZ599Fmq1GgBw9OhRPPPMM5g8eTLOnDmDpKQkk1aQ2ufaGJAALhZJRERkSp3qYjtz5gwGDx4MAPjiiy9w55134uWXX0ZmZqZhwDaZn0wqgavSDuraepTXaOHprBC7SkRERN1Cp1qQ5HI5ampqAAC7d+/GbbfdBgDw8PAwtCyRZbg7NYxDquBMNiIiIpPpVAvS2LFjkZSUhDFjxuDw4cPYunUrAODEiRPo27evSStI16ZysMdZAGXV7GIjIiIylU61IL355puws7PD559/jrfffht+fn4AgO+++w7x8fEmrSBdm1vTTDaOQSIiIjKZTrUg9evXDzt27Gi1/9///neXK0Q3pvmGtexiIyIiMpVOBSQA0Ol02LZtG7KzswEAQ4YMwdSpUyGTyUxWObo+w+1G2IJERERkMp0KSKdOncLkyZNx4cIFhIaGAgCSk5Ph7++PnTt3Ijg42KSVpPapuFgkERGRyXVqDNKTTz6J4OBgnDt3DpmZmcjMzER+fj4CAwPx5JNPmrqOdA1NXWxl7GIjIiIymU61IO3fvx+HDh2Ch4eHYV+vXr2wevVqjBkzxmSVo+tjFxsREZHpdaoFSaFQoLKystX+qqoqyOXyLleKOk7FG9YSERGZXKcC0p133okFCxYgPT0dgiBAEAQcOnQI//d//4epU6eauo50DW4OTdP82cVGRERkKp0KSOvXr0dwcDBiYmKgVCqhVCpx8803IyQkBOvWrTNxFela3JtakLhQJBERkcl0agySSqXC9u3bcerUKcM0/0GDBiEkJMSklaPra5rFVqmph1anh72sU5mXiIiIrtLhgJSUlHTNx/fu3Wv4fu3atZ2vEd0QV2XzS6i+okUv3rCWiIioyzockI4cOdKhchKJpNOVoRtnJ5PCRWmHytp6lDMgERERmUSH+2P27t3boW3Pnj03VIENGzYgICAASqUS0dHROHz4cIeO27JlCyQSCaZPn260XxAELF26FH369IGDgwNiY2Nx8uRJozKlpaVISEiAq6srVCoV5s2bh6qqqhuqtzXhTDYiIiLTEnXAytatW5GUlIRly5YhMzMT4eHhiIuLQ3Fx8TWPy8vLw+LFi3HLLbe0euyVV17B+vXrsXHjRqSnp8PJyQlxcXGora01lElISMCxY8ewa9cu7NixAz/++CMWLFhg8uuzFHfDatqcyUZERGQKogaktWvXYv78+UhMTMTgwYOxceNGODo6YvPmze0eo9PpkJCQgBUrViAoKMjoMUEQsG7dOrz44ouYNm0ahg8fjg8//BAXL17Etm3bAADZ2dlISUnBe++9h+joaIwdOxZvvPEGtmzZgosXL5rzcs3GzYEtSERERKYkWkCqq6tDRkYGYmNjmysjlSI2NhZpaWntHrdy5Up4eXlh3rx5rR47c+YMCgsLjc7p5uaG6OhowznT0tKgUqkQGRlpKBMbGwupVIr09HRTXJrFGe7HxtW0iYiITKJT0/xN4fLly9DpdPD29jba7+3tjePHj7d5zIEDB7Bp0yZkZWW1+XhhYaHhHC3P2fRYYWEhvLy8jB63s7ODh4eHoUxbNBoNNBqN4We1Wt1uWUtruh9bBbvYiIiITMJmFs2prKzEgw8+iHfffReenp4Wf/7k5GS4ubkZNn9/f4vXoT2GQdpsQSIiIjIJ0VqQPD09IZPJUFRUZLS/qKgIPj4+rcrn5uYiLy8PU6ZMMezT6/UAGlqAcnJyDMcVFRWhT58+RueMiIgAAPj4+LQaBF5fX4/S0tI2n7fJkiVLjNaCUqvVVhOSmrrYyjgGiYiIyCREa0GSy+UYNWoUUlNTDfv0ej1SU1MRExPTqnxYWBiOHj2KrKwswzZ16lRMmDABWVlZ8Pf3R2BgIHx8fIzOqVarkZ6ebjhnTEwMysvLkZGRYSizZ88e6PV6REdHt1tfhUIBV1dXo81aqAyDtNnFRkREZAqitSABDatzz5kzB5GRkYiKisK6detQXV2NxMREAMDs2bPh5+eH5ORkKJVKDB061Oh4lUoFAEb7n376afzjH//AgAEDEBgYiJdeegm+vr6G9ZIGDRqE+Ph4zJ8/Hxs3boRWq8XChQsxa9Ys+Pr6WuS6Ta2pi62CXWxEREQmIWpAmjlzJi5duoSlS5eisLAQERERSElJMQyyzs/Ph1R6Y41czz33HKqrq7FgwQKUl5dj7NixSElJgVKpNJT55JNPsHDhQkycOBFSqRQzZszA+vXrTXptlsSFIomIiExLIgiCIHYlbJFarYabmxsqKipE7247VVyF2LX74aq0w+/L40StCxERkTXr6Oe3zcxio/a5N7YgqWvrUa/Ti1wbIiIi28eA1A00raQNNIQkIiIi6hoGpG7ATiaFi6JhOBlnshEREXUdA1I34cbFIomIiEyGAambMEz150w2IiKiLmNA6ibcDatps4uNiIioqxiQugk3B66FREREZCoMSN0Eb1hLRERkOgxI3YTKoaGLrYJdbERERF3GgNRNsAWJiIjIdBiQugmVYZA2AxIREVFXMSB1EyqHpmn+7GIjIiLqKgakboJdbERERKbDgNRNGAISu9iIiIi6jAGpm3BrnMWmrtVCpxdErg0REZFtY0DqJppakAQBULObjYiIqEsYkLoJe5kUzgo7AByHRERE1FUMSN1I8+1GOJONiIioKxiQuhHOZCMiIjINBqRupCkgVXAmGxERUZcwIHUjzatps4uNiIioKxiQuhGVA9dCIiIiMgUGpG7E0MXGMUhERERdwoDUjagaF4vkLDYiIqKuYUDqRtw4i42IiMgkGJC6EXfDIG0GJCIioq5gQOpGmqf5s4uNiIioKxiQuhHDLDZ2sREREXUJA1I34nbVLDa9XhC5NkRERLaLAakbaZrFJghAZW29yLUhIiKyXQxI3YjcTgonuQwAV9MmIiLqCgakbqbpdiMch0RERNR5DEjdjJvhdiNsQSIiIuosBqRuhrcbISIi6joGpG6mKSDxhrVERESdJ3pA2rBhAwICAqBUKhEdHY3Dhw+3W/bLL79EZGQkVCoVnJycEBERgY8++siojEQiaXNbs2aNoUxAQECrx1evXm22a7QklWE1bXaxERERdZadmE++detWJCUlYePGjYiOjsa6desQFxeHnJwceHl5tSrv4eGBv//97wgLC4NcLseOHTuQmJgILy8vxMXFAQAKCgqMjvnuu+8wb948zJgxw2j/ypUrMX/+fMPPLi4uZrhCyzMsFskWJCIiok4TNSCtXbsW8+fPR2JiIgBg48aN2LlzJzZv3oznn3++Vfnx48cb/fzUU0/hgw8+wIEDBwwBycfHx6jM9u3bMWHCBAQFBRntd3FxaVW2O+AYJCIioq4TrYutrq4OGRkZiI2Nba6MVIrY2FikpaVd93hBEJCamoqcnByMGzeuzTJFRUXYuXMn5s2b1+qx1atXo1evXhgxYgTWrFmD+vprL6yo0WigVquNNmvUtFgkZ7ERERF1nmgtSJcvX4ZOp4O3t7fRfm9vbxw/frzd4yoqKuDn5weNRgOZTIa33noLkyZNarPsBx98ABcXF9x9991G+5988kmMHDkSHh4e+Pnnn7FkyRIUFBRg7dq17T5vcnIyVqxYcQNXKI6m241wHSQiIqLOE7WLrTNcXFyQlZWFqqoqpKamIikpCUFBQa263wBg8+bNSEhIgFKpNNqflJRk+H748OGQy+V45JFHkJycDIVC0ebzLlmyxOg4tVoNf39/01yUCbk3LRTJMUhERESdJlpA8vT0hEwmQ1FRkdH+oqKia44NkkqlCAkJAQBEREQgOzsbycnJrQLSTz/9hJycHGzduvW6dYmOjkZ9fT3y8vIQGhraZhmFQtFueLImzdP82cVGRETUWaKNQZLL5Rg1ahRSU1MN+/R6PVJTUxETE9Ph8+j1emg0mlb7N23ahFGjRiE8PPy658jKyoJUKm1z5pytaZrFVnFFC71eELk2REREtknULrakpCTMmTMHkZGRiIqKwrp161BdXW2Y1TZ79mz4+fkhOTkZQMM4oMjISAQHB0Oj0eDbb7/FRx99hLffftvovGq1Gp999hlee+21Vs+ZlpaG9PR0TJgwAS4uLkhLS8OiRYvwwAMPwN3d3fwXbWaujQFJLwCVmnrDrUeIiIio40QNSDNnzsSlS5ewdOlSFBYWIiIiAikpKYaB2/n5+ZBKmxu5qqur8dhjj+H8+fNwcHBAWFgYPv74Y8ycOdPovFu2bIEgCLjvvvtaPadCocCWLVuwfPlyaDQaBAYGYtGiRUbji2yZ0l4GB3sZrmh1qKjRMiARERF1gkQQBPbDdIJarYabmxsqKirg6uoqdnWM3JyciosVtdj++BiE+6vErg4REZHV6Ojnt+i3GiHTc2uaycap/kRERJ3CgNQNNd9uhDPZiIiIOoMBqRvi7UaIiIi6hgGpG2peC4kBiYiIqDMYkLohVeMYpDJ2sREREXUKA5K1Ob0fOJUKdGFyoWGxSLYgERERdQoDkjXR1QPfLgY+vhv4cCpwIaNTp1HxhrVERERdwoBkTXQaIGQSIJMDZ34E3r0V+N9s4PKpGzqNm0PTDWvZxUZERNQZDEjWRO4ExL8MPJEBhN8PQAL8uR3YEAV88xSgLujQadw5SJuIiKhLGJCskaofcNfbwKM/AwNvBwQdkPE+sH4EsGsZcKXs2odzoUgiIqIuYUCyZt6Dgfu3AIkpgP9NQP0V4OA64PUI4MA6QHulzcOap/nXQa/nnWSIiIhuFAOSLegfAzyUAty3Beg9CKgtB3YvA9aPBDI+aBjcfZWmG9TqBaCqrr6NExIREdG1MCDZCokECL0dePQgMP1twM0fqLwIfPMk8HYM8OfXhqUBlPYyKO0bXlpO9SciIrpxDEi2RioDIu4HFv4KxL0MOHgAl08A/3sQeG9iw+w3AO5cLJKIiKjTGJBslb0SiHkceOo3YNxzgL1Tw7pJH0wBProbI+3zAXAmGxERUWcwINk6pStw69+Bp7KA0fMBqR2Qm4oNVU/jdfs3UXcpV+waEhER2RyJIHThnhY9mFqthpubGyoqKuDq6ip2dZqVngb2/BP443MAgF5iB2lkIjD6YaB3aMNYJiIioh6qo5/fDEidZLUBqdH6jz9HeM7r+Ivs9+adTl5A4C1AwC1A4DjAI4iBiYiIepSOfn7bWbBOZEE1vYZijvZ5rBxUitm6r4CzB4HqYuCPLxo2AHD1aw5Lgbc0LFBJREREDEjdVdNikVl2wzD7/geBeg1w/hfgzE9A3k/AucOA+gLw+5aGDQBU/RvD0riG4OTaR8QrICIiEg8DUjelalws0rAOkp0CCBjbsGEJUFcDnEtvCEtnfgQuZALlZ4EjHzVsANBrQEPLUlNgcvIU52KIiIgsjAGpmzLcbqS9+7HJHYHgCQ0bAGgqgfxDwJn9Da1MBb8BJScbtl83N5TxGtwclvqObghMUpkFroaIiMiyGJC6KTeHxhvWdnShSIULMGBSwwY03BD37M/NXXJFfwDFfzZs6RsbD5IADu6AY6+rNo+Gr06ebe9XuHJgOBERWT0GpG7K3anphrWdXCjSwR0Iu6NhA4Dqy0DegeYuucsnAAjAldKGreRkx84rtW8dmpo2e4eGrkCZvGGzUwAye0DWuM9O3uL7q8vJjTcpl/giIqLOY0DqplRNLUhXtBAEAZKutto4eQJDpjdsAKDTNrQy1ZQYb9UlrffVlAI1lwFtDaDXAlWFDZs5Se0bwpVE1tANKJU1f9/WPqkdIJFef59E1rBPImncpG1vkFxVruXjkhblGl8bieSqnzvzFcbnQ+PPhnN3YJ/R/rb2Xa2NfV0q15Yb+HdrCy2TtlBHImsy9B7AQSXKUzMgdVNNY5B0egFVmnq4KO1N+wQye8DZq2HrqLqahtamq4NT9eXmn+trAV1dw4w7nRbQaYD6uoZ9Rt/XXVWu8Xt9vfFz6bUNGxER2a7A8QxIZFpKexkUdlJo6vUor9GaPiB1htyxYXPra/pz63VXBaerQpSgb3hMXw8IuobvBR2g1zd+rW9jX8vy+uYygr7FJjRuLfc3bhCuKtfyuKvLCFd9RYufb/RrE8Hoi/G+NsoZ7W9rXxvnMdrVlXJtuYE1bE2+3q0Z1s/lmrwmwt9jjyJ3Eu2pGZC6MZWjPYrUGlRc0cJf7MqYm1QGSB0axjERERF1EUeydmPujg3jkMo6OpONiIiIADAgdWtuDl2cyUZERNRDMSB1Y9ddLJKIiIjaxIDUjTVN9a9gFxsREdENYUDqxgwtSOxiIyIiuiEMSN2YyjBImwGJiIjoRogekDZs2ICAgAAolUpER0fj8OHD7Zb98ssvERkZCZVKBScnJ0REROCjjz4yKjN37lxIJBKjLT4+3qhMaWkpEhIS4OrqCpVKhXnz5qGqqsos1yemphakiivsYiMiIroRogakrVu3IikpCcuWLUNmZibCw8MRFxeH4uLiNst7eHjg73//O9LS0vD7778jMTERiYmJ+P77743KxcfHo6CgwLB9+umnRo8nJCTg2LFj2LVrF3bs2IEff/wRCxYsMNt1ikXFWWxERESdIhEE8ZZ3jY6OxujRo/Hmm28CAPR6Pfz9/fHEE0/g+eef79A5Ro4ciTvuuAOrVq0C0NCCVF5ejm3btrVZPjs7G4MHD8Yvv/yCyMhIAEBKSgomT56M8+fPw9fXt0PPq1ar4ebmhoqKCri6unboGEv7Ofcy7n83HSFeztid9Bexq0NERCS6jn5+i9aCVFdXh4yMDMTGxjZXRipFbGws0tLSrnu8IAhITU1FTk4Oxo0bZ/TYvn374OXlhdDQUDz66KMoKSkxPJaWlgaVSmUIRwAQGxsLqVSK9PT0dp9Po9FArVYbbdbOcMNatiARERHdENFuNXL58mXodDp4e3sb7ff29sbx48fbPa6iogJ+fn7QaDSQyWR46623MGnSJMPj8fHxuPvuuxEYGIjc3Fy88MILuP3225GWlgaZTIbCwkJ4eRnfYNXOzg4eHh4oLGz/DvPJyclYsWJFJ69WHO5OTV1sdRAEARLeSZyIiKhDbO5ebC4uLsjKykJVVRVSU1ORlJSEoKAgjB8/HgAwa9YsQ9lhw4Zh+PDhCA4Oxr59+zBx4sROP++SJUuQlJRk+FmtVsPf37rvcNbUglSvF1Bdp4OzwuZebiIiIlGI9onp6ekJmUyGoqIio/1FRUXw8fFp9zipVIqQkBAAQEREBLKzs5GcnGwISC0FBQXB09MTp06dwsSJE+Hj49NqEHh9fT1KS0uv+bwKhQIKhaKDV2cdlPZSyO2kqKvXo7ymjgGJiIiog0QbgySXyzFq1CikpqYa9un1eqSmpiImJqbD59Hr9dBoNO0+fv78eZSUlKBPnz4AgJiYGJSXlyMjI8NQZs+ePdDr9YiOju7ElVgviUTCmWxERESdIGqTQlJSEubMmYPIyEhERUVh3bp1qK6uRmJiIgBg9uzZ8PPzQ3JyMoCGcUCRkZEIDg6GRqPBt99+i48++ghvv/02AKCqqgorVqzAjBkz4OPjg9zcXDz33HMICQlBXFwcAGDQoEGIj4/H/PnzsXHjRmi1WixcuBCzZs3q8Aw2W6JytEdxpQYVvB8bERFRh4kakGbOnIlLly5h6dKlKCwsREREBFJSUgwDt/Pz8yGVNjdyVVdX47HHHsP58+fh4OCAsLAwfPzxx5g5cyYAQCaT4ffff8cHH3yA8vJy+Pr64rbbbsOqVauMusc++eQTLFy4EBMnToRUKsWMGTOwfv16y168hTSvps3FIomIiDpK1HWQbJktrIMEAAs+/BU//FmEf0wfigdu6i92dYiIiERl9esgkWU0326EXWxEREQdxYDUzTV1sZWzi42IiKjDGJC6OTfOYiMiIrphDEjdnHtjC1JpNVuQiIiIOooBqZsL7u0EANh/4hKOnq8QuTZERES2gQGpm4sK9MDkYT6o1wtY9L8s1Gp1YleJiIjI6jEgdXMSiQT/mD4Mns4KnCquwqvf54hdJSIiIqvHgNQDeDjJ8a8ZwwAAmw6eQVpuicg1IiIism4MSD3ExEHemDXaH4IALP7sN1TWclYbERFRexiQepAX7xyMvu4OuFB+Bat2/Cl2dYiIiKwWA1IP4qyww9p7IyCRAP/79Tx2/VkkdpWIiIisEgNSDxMV6IEFtwQBAJZ8+TtKqjQi14iIiMj6MCD1QIsmDUSotwsuV9Xhha+OgvcrJiIiMsaA1AMp7WVYOzMc9jIJvj9WhK+OXBC7SkRERFaFAamHGuLrhqdjBwIAlm0/hovlV0SuERERkfVgQOrBHhkXhBH9VKjU1OPZz3+DXs+uNiIiIoABqUezk0mx9t4IONjLcPBUCT5MyxO7SkRERFaBAamHC/R0wguTwwAAyd8dx6niKpFrREREJD4GJMIDN/XHLQM8oanX45n/ZaFepxe7SkRERKJiQCJIJBK8cs9wuCrt8Nv5Cry1L1fsKhEREYmKAYkAAH3cHLBq+lAAwPrUkzh6vkLkGhEREYmHAYkMpob74o5hfVCvF7Dof1mo1erErhIREZEo7MSuAFkPiUSCVdOH4nBeKU4VV+HV73Pw4p2Dxa4WEdkYvV7Ap7/kY/OBM3B1sMdwPzcM66vC8L5uCO7tDJlUInYVia5LIvA+E52iVqvh5uaGiooKuLq6il0dk9pzvAgPvf8rJBLgvw/fhJjgXmJXiYhsxKniSiz58ih+yStr83EHexmG+LpiWF83DPNzw/C+bgj0ZGgiy+no5zcDUid154AENNzI9tPD5+CnckDK07fARWkvdpWIyIpp6nV4a28u3tp3ClqdAEe5DEmTBqK3iwK/n6/A0QsVOHahAtV1rbvuneQyDPFrDkzD/NwQ0MsJUoYmMgMGJDPr7gGpSlOP21//EedKr+DeyL545Z5wsatERFbq8JlSLPnyd+ReqgYA3BrmhVXTh8JP5WBUTqcXcOZyFY5eqGgITecrcOyiGlfaGO/oorDDED9XDO+rwrDG8NS/lyMkEoYm6hoGJDPr7gEJaPhPb+Z/0iAIwLuzIzFpsLfYVSIiK1JxRYvV3x3Hp4fzAQCezgosnzoYdwzr0+Ego9MLyL1U1RiYyhtami6qoalvvR6bq9IOUYEeeOCm/vjLwN4MS9QpDEhm1hMCEgAkf5uNd348DU9nOb5/ehx6OSvErhKRTTl0ugSvpBxHfukV3DLAE3FDvDFuYG84ym13jowgCPj2aCGWf3MMlyo1AID7ovzxfPwguDl2vTu+XqfHyeIqHG3smvv9QgWyC9Souyo0hXg546Exgbh7pB+U9rIuPyf1HAxIZtZTAlKtVodpbx5ETlEl4oZ4Y+MDo/hXG1EH5F6qQvK3x7E7u6jVY0p7KW4Z0BtxQ3wQO8gLKke5CDXsnIvlV/DStj+QerwYABDU2wnJdw1DdJB5J3NodXrkFFbiqyMXsPWXc6jS1AMA3B3tkRDdH7Nj+sPLVWnWOlxL3uVqbM+6iK9/u4CKK1rcHOyJW8O8MG5gb3g42c7r2xMwIJlZTwlIAHDsYgWmbzgIrU7A2nvDcffIvmJXyebVanXYc7wY6ita3D6sD9wcOAi+uyitrsPru0/gk/R81OsFyKQS3B/VD3FDfLAvpxjf/1mIc6VXDOVlUgmiAz0QN8QHtw3xRh83h2ucXTw6vYAP0/Lw6vc5qK7TwV4mwaPjQ/DY+GCLt+BU1mqx9ZdzeP/nPJwva/hd2sskmDLcFw+NDcRQPzeL1ONSpQY7fr+IbVkX8du58jbLSCVAhL8Kt4Z5YXyoF4b4uvKPTJExIJlZTwpIALBh7yms+T4HLgo7pCwa12rwJV2fXi/g0JkSbDtyAd8dLURl41/ATnIZ7ovqh8Sxgfy92rBarQ4f/JyHN/eeQmVtw2s7McwLSyaHIcTLxVBOEARkF1Ti+2OF+P5YIY4XVhqdJ7yvG24b4oO4IT4I8XK26DW0J7tAjee/PGoIAZH93ZF89zAM8Ha59oFmptML+OFYITYdOINfzzYvKxAd6IF5YwMxcZC3yZcPqNLU4/s/CrEt6wIOnroMfeMnqFQCjB3QG9MjfOGncsD+E5ew53hxq9fXy0WBCaFemBDmhbEDPOGssN2uVlvFgGRmPS0g1ev0+Os7aTiSX46bg3vh43nRnILbQccL1fjqyAV8nXURBRW1hv1+Kgc4yGU4VVwFALCTSnDn8D5YMC4Yg327/7+p7kIQBHzzewFeSTluaM0Y3McVL94xCDeHeF73+LMl1fjhWBFSjhUiM78MV/+PHNzbCXGNYWl4XzeLtzzUanV4PfUk3v3xNOr1AlwUdvjb7WG4P6qf1b3/fztXjk0HzuDbowWob0wt/Xs5Yu7NAfhrpH+XgkhdvR77T1zCtqwL2P1nkdEA8gh/FaZH+OKO4b7o7dJ6jGZBxRXsPd4Qlg6eumw0Y89eJkFUoIchMAV5OrF1yQIYkMyspwUkADhzuRqTX/8JV7Q6LJsyGIljAsWuktUqqLiCr7Mu4qsjF4z+gnRR2uHO4X0wPcIPowM8IJEA+3Iu4Z0fc3HodKmh3C0DPLFgXBDGhnjyP0wr9mteKf6xMxtZjS0rPq5KLI4Lxd0j/DoVIIora7HrzyJ8f6wIabmXodU1//fcx02J2wZ7I26ID6ICPWAnM++dog6cvIy/bzuKsyU1AID4IT5YMW0IvEUc59MRBRVX8MHPZ/Hp4XxUXNECaHjfzRrtjzk3B6Cvu2OHzqPXC/glrxTbsi7i26MFhnMBDeOupkf4YWq4LwI8nTpcN029DumnS7E3pxh7jxcjr/F326R/L0dDWIoO9OiRg88FQUBpdR3Ol13B+bIriBvibfJ/6wxIZtYTAxIAfJSWh5e2H4NUAoT6uDbeQqBhcbdQHxco7HreG7qJulaLlD8Kse3IBaSdLjG0BNjLJLg1zAt3jfDD+FCvdv/T+/18Of7z42l8e7TA0Gw/uI8rFowLwh3D+8DezB+IllKsrkWlph7G//M0/NC07+qHmve1KHNVIRelHfq6O1gsTJ4tqcbq747juz8KAQCOchn+7y/BmH9LEBzkpnkPqGu12Hu8GN8fK8S+nEuouWqBRZWjPSaGeSPEyxkeTvZwd5TDw0kOdyc5PBzlcHOw73QLT1l1Hf6xMxtfZJ4H0BD6VkwbgrghPia5LkupqavHF5kX8P8OnMHpyw3rM8mkEsQP8cFDYwMxqr97m8dlF6ixLesCvsm6iItXtfh6uSgwNdwX0yL8MNTPNOOIzlyuxp7jxdiXU4z006Wo0zW3TDnYyzAmpBfGNwam7tL9LggCKq5oca70Cs6X1eB82RWca/za9PPV/9YP/G1Ch0NtRzEgmVlPDUiCIODJLVn45reLrR6zl0kQ6uOCYX4qw2q4A71dILfrHh/sbamr1+PHE5fwVRtN71EBHpg+wg+Th/nc0Cylc6U12HTgDLb+cs7QHO/rpsRDYwMxK6qfzY5ZOHq+Aq+nnsDu7GKznN/TWY4R/dwxsp87RvV3x/C+bib/C7y8pg5v7DmFD9PyoNUJkEqAmaP9sSh2oFlnUNVqdThw8jK+P1aI3dlFKKvRXrO8VAKoHOVQOdrDw7E5OLk7yY0Clarxq4ejHC5KO3z920Ws3PEnSqvrIJEAs2/qj8VxoTa9kr5eL2DfiWJsOnAGB0+VGPZH+Kswb2wgbh/qg0J1bcMMtKyLyCm6qsVXYYfbh/lgWoQfbgrqZdbboVRr6nHg1GXsyynGnuPFKFJrjB6fGemPldOH2MQfoRVXtDhfVmMUgpq/XjHMQLwWb1cF/N0dsXrGMKMxfKZgMwFpw4YNWLNmDQoLCxEeHo433ngDUVFRbZb98ssv8fLLL+PUqVPQarUYMGAAnnnmGTz44IMAAK1WixdffBHffvstTp8+DTc3N8TGxmL16tXw9fU1nCcgIABnz541OndycjKef/75Dte7pwYkoCEkFVTUNt4+oNxwG4HyNv7TlttJMcjHpaGVyU+FYX3dMMDL2ezdA+YkCAIy88ux7cgF7Pj9otGHVXBvJ9w9si+mhvvC36Nrf/WU19Th40Nn8f7PebhcVQegoaUkIbo/EscEWH1XR5PfzpXj9dST2NM4LVwiAVyv+sBt+kNcYvhZYvTz1WWa9rY8pqymzqg7CmgY0zXE1xUj+zeHJt9O/hVeV6/HR4fOYn3qSUNXy7iBvfHC5DCE+Vj2/V+v0+OXvDL8ePISitS1KKuuQ2mNFmXVdSirrjMM/r9RUgkMLZcDvZ2RfPfwdltZbFV2gRqbD5zB9qyLhtYalaO90f9dcpkUt4Z5YVqELyaEtd/ia05NA/mbuuIyGsemjervjo0PjGpzrJOYThVX4vXUUzhVXIXzZTWGSQrX0ttFgb7uDvB3d0Rfdwf0bfzq7+EIX5XSrEHQJgLS1q1bMXv2bGzcuBHR0dFYt24dPvvsM+Tk5MDLy6tV+X379qGsrAxhYWGQy+XYsWMHnnnmGezcuRNxcXGoqKjAPffcg/nz5yM8PBxlZWV46qmnoNPp8OuvvxrOExAQgHnz5mH+/PmGfS4uLnBy6nhfck8OSG0RBAHny64030KgMTi19UZR2Ekx2Ne1zTt86/UCquvqUa3RoUpTj+rGrUpTj+q6elRpdMb7NA1lK1uW1dSjXidAKZfBwV4GR7kMSvuG7x0a9yntZXCQSxv22cuglMvg2Ph4W2UFAdiVXYRtRy4gv7R57ICnswLTInxx1wg/s0zhrdXq8NWRC3j3p9M43XgrB3uZBNMj/DB/XBAGijyTqD1H8svweupJ7Mu5BKDhA3j6CD8snBCCoN6mnZ1Vq9Xh2MUKZJ4tR8bZMmTklxkWMLyaj6sSo/q7Y0Q/FUb1d8cQX7drtnAKgoCUPwqxOuW4YSxOqLcLXrhjEP4ysLdJr8FU6ur1KL9Sh7JqLUqr61BWU9fwtboOpTV1KK9pvb/p/mhyOymevDUEC8YFd+uW30uVGnx86Cw+PnQWJY2tZTcF9sL0Eb6IH2p9y27sP3EJC/+bicraevi6KfHunEgM8bXMUgbXs+3IBSz58mir28V4Osvh1xR6DCGoIQD5qRxEHV9lEwEpOjoao0ePxptvvgkA0Ov18Pf3xxNPPNHh1pyRI0fijjvuwKpVq9p8/JdffkFUVBTOnj2Lfv36AWgISE8//TSefvrpTtedAen6BEHA2ZIaHL3QuBru+XL8cUHdZvOq0l4KqURi1PdszRzlMsQP8cH0EX64ObiXRVrE9HoBqceL8Z8fc43ulD4htDcWjAvGTUEeVjGgO+NsQzD68URDMJJJG8LcwltDEHgDA1q7oimwZ+aXIfNsGTLzy/FngRo6vfF/d3I7KYb7uTWGJneM7K+Cl0tDy1zWuXL8c+efht91bxcFnpk0EH+N9O92d57X1OtQXqOFo1xm091pN6pWq0Pm2TIE9XaGj5t1t8jmXqrCwx/8ijOXq+FgL8O/Z4Yjfmgf0epTq9VhxTd/Gm4zMyakF+aNDYS/uyP83B2seqV4qw9IdXV1cHR0xOeff47p06cb9s+ZMwfl5eXYvn37NY8XBAF79uzB1KlTsW3bNkyaNKnNcrt378Ztt92G8vJywy8iICAAtbW10Gq16NevH+6//34sWrQIdnbtv6AajQYaTfNfpGq1Gv7+/gxIN0ivF3CmpBp/XHWzyj8uVrQKRlIJ4KSwg4vCDk6Nm7PCDk4K2VXfN3xt/l5mVNZOKkGtVo8rWh1qtTpcqdPhirZxq2v+WttiX61Wh5o2HtfU6xHeV4W7R/ph0mBvUf8DyMwvw3/2n8b3fxYaBiuH93XD/HFBuG2wjyh//f+aV4rXU0/ip5OXATQEo7tHNASj/r0sE4yupaauHr+fr0DG2TIcyS9DxtmyNsfy+Hs4oK/KEWmnG8arKO2lWHBLEB75SzCcbHT8F3UPFTVaLPw00/AeS5o0EE/cGmLxP4zOllTj0Y8z8WeBGhIJ8MStA/DUxAE284eD1Qekixcvws/PDz///DNiYmIM+5977jns378f6enpbR5XUVEBPz8/aDQayGQyvPXWW3jooYfaLFtbW4sxY8YgLCwMn3zyiWH/2rVrMXLkSHh4eODnn3/GkiVLkJiYiLVr17Zb3+XLl2PFihVt1ocBqWt0egH5pTWQAIZwo7SXWkVriLU7c7kamw6cxme/njcMEHewlyEywB03BfVCTHAvDPNzM+sMuMNnSvF66gnDAFg7qQQzRvbF4xNC0K+XaWefmJIgCMgrqUHG2TJDS1NOUaUhcEokwIyRffHMbQOtdnVr6nnqdXr889ts/L+DeQCAO4b1wat/DTfZ7MnrSfmjAM9+9jsqNfXwcJJj3cwIjLPS7ub2dNuApNfrcfr0aVRVVSE1NRWrVq3Ctm3bMH78eKNyWq0WM2bMwPnz57Fv375r/hI2b96MRx55BFVVVVAo2h78xhYksmYlVRp8mHYWn6Tn43KV8dgbJ7kMkQEeiAnuhZigXhji62qSLsFDp0vw+u6ThpYWO6kEf43si8fGh3R5gLpYKmu1yDpXjpNFVYgO8rCacR5ELW05nI+Xtv8BrU7AEF9XvDs7stOTEDqirl6P1d8dx+aDZwA0rKb+xv0jbPKPB6sPSF3tYmvy8MMP49y5c/j+++8N+7RaLe69916cPn0ae/bsQa9e176J4rFjxzB06FAcP34coaGhHXpejkEia6TXCzhZXIW03MtIO12C9DOlrWYXuijsMDrQAzFBvXBTUC8M9nW9oabxtNwSvJ56wrCwpb1MgntG+eOx8cE2G4yIbNHhM6X4v48zUFpdB09nBf4zexRG9jP9zMML5Vew8L+ZOJJfDgBYMC4Iz8aF2uzabB39/BatQ10ul2PUqFFITU01BCS9Xo/U1FQsXLiww+fR6/VGLTtN4ejkyZPYu3fvdcMRAGRlZUEqlbY5c47IlkilDWtRhfq4YO6YQOj1Ao4XViLtdAkOnS5B+ukSqGvrsed4sWHavavSDlGBvXBTUEMr0yAf11aLDAqCgLTcEqxLPYnDZ5qD0b2R/nhsQki3WcSOyJZEBXpg++NjMP/DX3G8sBKz3jmE5LuHYcYo091QfG9OMRZtzUJ5jRauSju8+tdw3GZji4Z2lqgjDpOSkjBnzhxERkYiKioK69atQ3V1NRITEwEAs2fPhp+fH5KTkwE0rFUUGRmJ4OBgaDQafPvtt/joo4/w9ttvA2gIR/fccw8yMzOxY8cO6HQ6FBY2rHTr4eEBuVyOtLQ0pKenY8KECXBxcUFaWhoWLVqEBx54AO7u3WvNDyKpVILBvq4Y7OuKeWMDodMLyC5QIy23ITAdPlMKdW09dmcXYXd2EYCGdWGiAz0MY5guV9bh9dQThtlccpkUM0f749HxwWZt0iei6/P3cMQXj96Mp7dmYdefRXjms99woqgSz8WHdWnQdL1Oj3/vPoENe3MBAMP83PBWwsge1Uos+kKRb775pmGhyIiICKxfvx7R0dEAgPHjxyMgIADvv/8+AODFF1/E1q1bcf78eTg4OCAsLAxPPfUUZs6cCQDIy8tDYGDb9wfbu3cvxo8fj8zMTDz22GM4fvw4NBoNAgMD8eCDDyIpKand8UdtYRcbdQf1Oj2OXVTj0OkSpJ0uwS9nSg1r4rQkl0kxK6ohGNniuAOi7kyvF7B21wm8ufcUAODWMC+8PiuiU8s2FKtr8eSWI4Zu9Adv6o8X7xxkE6t4d4TVj0GydQxI1B1pdXr8caECaadLkJZbgl/zyqATBNwf1Q//95dgq18rhqin+/q3i3j2s9+gqdcjxMsZ782OvKEb6v6cexlPfpqFy1UaOMllSJ4xHFPDfa9/oA1hQDIzBiTqCbQ6PQQB3XpVZaLu5vfz5Zj/4a8oUmugcrTHW/ePxM0hntc8Rq8X8Na+U1i76wT0QsOK8W89MBLBJl713hp09POb/+sRUbvsZVKGIyIbM7yvCl8vHItwfxXKa7R4cPNhfJSW12750uo6JL7/C179oSEc/XVUX2x7fEy3DEc3gv/zERERdTPerkpsXXATpkf4QqcX8NL2Y3hx21FoG2/S2yTjbBnuWP8T9p+4BKW9FK/cMxxrLLjwpDXjuvlERETdkNJehn/PjECojyte+f44Pj6Uj9ziaryVMBIqR3tsOnAGq787jnq9gCBPJ7z1wEiE+XDISBOOQeokjkEiIiJbsfvPIjy15Qiq63To5+GIgd7O2J3dsBbancP7YPWM4XDuIfca5BgkIiIiAgDEDvbGl4+Ngb+HA/JLa7A7uxhymRSrpg3BG/eN6DHh6EYwIBEREfUAoT4u2P74WEwM80KYjws+fzQGD8YE8Mbg7WBkJCIi6iE8nOTYNHe02NWwCWxBIiIiImqBAYmIiIioBQYkIiIiohYYkIiIiIhaYEAiIiIiaoEBiYiIiKgFBiQiIiKiFhiQiIiIiFpgQCIiIiJqgQGJiIiIqAUGJCIiIqIWGJCIiIiIWmBAIiIiImqBAYmIiIioBTuxK2CrBEEAAKjVapFrQkRERB3V9Lnd9DneHgakTqqsrAQA+Pv7i1wTIiIiulGVlZVwc3Nr93GJcL0IRW3S6/W4ePEiBEFAv379cO7cObi6uopdLWqkVqvh7+/P18UK8bWxTnxdrBdfG9MSBAGVlZXw9fWFVNr+SCO2IHWSVCpF3759DU11rq6u/Idrhfi6WC++NtaJr4v14mtjOtdqOWrCQdpERERELTAgEREREbXAgNRFCoUCy5Ytg0KhELsqdBW+LtaLr4114utivfjaiIODtImIiIhaYAsSERERUQsMSEREREQtMCARERERtcCARERERNQCA1IXbNiwAQEBAVAqlYiOjsbhw4fFrlKPt3z5ckgkEqMtLCxM7Gr1OD/++COmTJkCX19fSCQSbNu2zehxQRCwdOlS9OnTBw4ODoiNjcXJkyfFqWwPc73XZu7cua3eQ/Hx8eJUtgdJTk7G6NGj4eLiAi8vL0yfPh05OTlGZWpra/H444+jV69ecHZ2xowZM1BUVCRSjbs/BqRO2rp1K5KSkrBs2TJkZmYiPDwccXFxKC4uFrtqPd6QIUNQUFBg2A4cOCB2lXqc6upqhIeHY8OGDW0+/sorr2D9+vXYuHEj0tPT4eTkhLi4ONTW1lq4pj3P9V4bAIiPjzd6D3366acWrGHPtH//fjz++OM4dOgQdu3aBa1Wi9tuuw3V1dWGMosWLcI333yDzz77DPv378fFixdx9913i1jrbk6gTomKihIef/xxw886nU7w9fUVkpOTRawVLVu2TAgPDxe7GnQVAMJXX31l+Fmv1ws+Pj7CmjVrDPvKy8sFhUIhfPrppyLUsOdq+doIgiDMmTNHmDZtmij1oWbFxcUCAGH//v2CIDS8R+zt7YXPPvvMUCY7O1sAIKSlpYlVzW6NLUidUFdXh4yMDMTGxhr2SaVSxMbGIi0tTcSaEQCcPHkSvr6+CAoKQkJCAvLz88WuEl3lzJkzKCwsNHr/uLm5ITo6mu8fK7Fv3z54eXkhNDQUjz76KEpKSsSuUo9TUVEBAPDw8AAAZGRkQKvVGr1vwsLC0K9fP75vzIQBqRMuX74MnU4Hb29vo/3e3t4oLCwUqVYEANHR0Xj//feRkpKCt99+G2fOnMEtt9yCyspKsatGjZreI3z/WKf4+Hh8+OGHSE1Nxb/+9S/s378ft99+O3Q6ndhV6zH0ej2efvppjBkzBkOHDgXQ8L6Ry+VQqVRGZfm+MR87sStAZEq333674fvhw4cjOjoa/fv3x//+9z/MmzdPxJoR2YZZs2YZvh82bBiGDx+O4OBg7Nu3DxMnThSxZj3H448/jj/++IPjJ0XGFqRO8PT0hEwmazV7oKioCD4+PiLVitqiUqkwcOBAnDp1SuyqUKOm9wjfP7YhKCgInp6efA9ZyMKFC7Fjxw7s3bsXffv2Nez38fFBXV0dysvLjcrzfWM+DEidIJfLMWrUKKSmphr26fV6pKamIiYmRsSaUUtVVVXIzc1Fnz59xK4KNQoMDISPj4/R+0etViM9PZ3vHyt0/vx5lJSU8D1kZoIgYOHChfjqq6+wZ88eBAYGGj0+atQo2NvbG71vcnJykJ+fz/eNmbCLrZOSkpIwZ84cREZGIioqCuvWrUN1dTUSExPFrlqPtnjxYkyZMgX9+/fHxYsXsWzZMshkMtx3331iV61HqaqqMmpxOHPmDLKysuDh4YF+/frh6aefxj/+8Q8MGDAAgYGBeOmll+Dr64vp06eLV+ke4lqvjYeHB1asWIEZM2bAx8cHubm5eO655xASEoK4uDgRa939Pf744/jvf/+L7du3w8XFxTCuyM3NDQ4ODnBzc8O8efOQlJQEDw8PuLq64oknnkBMTAxuuukmkWvfTYk9jc6WvfHGG0K/fv0EuVwuREVFCYcOHRK7Sj3ezJkzhT59+ghyuVzw8/MTZs6cKZw6dUrsavU4e/fuFQC02ubMmSMIQsNU/5deeknw9vYWFAqFMHHiRCEnJ0fcSvcQ13ptampqhNtuu03o3bu3YG9vL/Tv31+YP3++UFhYKHa1u722XhMAwv/7f//PUObKlSvCY489Jri7uwuOjo7CXXfdJRQUFIhX6W5OIgiCYPlYRkRERGS9OAaJiIiIqAUGJCIiIqIWGJCIiIiIWmBAIiIiImqBAYmIiIioBQYkIiIiohYYkIiIiIhaYEAiIjKBffv2QSKRtLpXFhHZJgYkIiIiohYYkIiIiIhaYEAiom5Br9cjOTkZgYGBcHBwQHh4OD7//HMAzd1fO3fuxPDhw6FUKnHTTTfhjz/+MDrHF198gSFDhkChUCAgIACvvfaa0eMajQZ/+9vf4O/vD4VCgZCQEGzatMmoTEZGBiIjI+Ho6Iibb74ZOTk55r1wIjILBiQi6haSk5Px4YcfYuPGjTh27BgWLVqEBx54APv37zeUefbZZ/Haa6/hl19+Qe/evTFlyhRotVoADcHm3nvvxaxZs3D06FEsX74cL730Et5//33D8bNnz8ann36K9evXIzs7G++88w6cnZ2N6vH3v/8dr732Gn799VfY2dnhoYcessj1E5Fp8Wa1RGTzNBoNPDw8sHv3bsTExBj2P/zww6ipqcGCBQswYcIEbNmyBTNnzgQAlJaWom/fvnj//fdx7733IiEhAZcuXcIPP/xgOP65557Dzp07cezYMZw4cQKhoaHYtWsXYmNjW9Vh3759mDBhAnbv3o2JEycCAL799lvccccduHLlCpRKpZl/C0RkSmxBIiKbd+rUKdTU1GDSpElwdnY2bB9++CFyc3MN5a4OTx4eHggNDUV2djYAIDs7G2PGjDE675gxY3Dy5EnodDpkZWVBJpPhL3/5yzXrMnz4cMP3ffr0AQAUFxd3+RqJyLLsxK4AEVFXVVVVAQB27twJPz8/o8cUCoVRSOosBweHDpWzt7c3fC+RSAA0jI8iItvCFiQisnmDBw+GQqFAfn4+QkJCjDZ/f39DuUOHDhm+Lysrw4kTJzBo0CAAwKBBg3Dw4EGj8x48eBADBw6ETCbDsGHDoNfrjcY0EVH3xRYkIrJ5Li4uWLx4MRYtWgS9Xo+xY8eioqICBw8ehKurK/r37w8AWLlyJXr16gVvb2/8/e9/h6enJ6ZPnw4AeOaZZzB69GisWrUKM2fORFpaGt5880289dZbAICAgADMmTMHDz30ENavX4/w8HCcPXsWxcXFuPfee8W6dCIyEwYkIuoWVq1ahd69eyM5ORmnT5+GSqXCyJEj8cILLxi6uFavXo2nnnoKJ0+eREREBL755hvI5XIAwMiRI/G///0PS5cuxapVq9CnTx+sXLkSc+fONTzH22+/jRdeeAGPPfYYSkpK0K9fP7zwwgtiXC4RmRlnsRFRt9c0w6ysrAwqlUrs6hCRDeAYJCIiIqIWGJCIiIiIWmAXGxEREVELbEEiIiIiaoEBiYiIiKgFBiQiIiKiFhiQiIiIiFpgQCIiIiJqgQGJiIiIqAUGJCIiIqIWGJCIiIiIWmBAIiIiImrh/wOPglY7rLb2pgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# draw Losses\n",
    "if is_graphic:\n",
    "    plt.plot(list(range(1, len(arr_train_loss) + 1)), arr_train_loss, label=\"train\")\n",
    "    plt.plot(list(range(1, len(arr_val_loss) + 1)), arr_val_loss, label=\"val\")\n",
    "\n",
    "    plt.title(\"Loss\")\n",
    "    plt.xlabel(\"epoch\")\n",
    "    plt.ylabel(\"loss\")\n",
    "    plt.legend(loc=\"upper left\")\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8bda887e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAksAAAHHCAYAAACvJxw8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvUUlEQVR4nO3de1TVdb7/8dfmjhLsVGSLYtjkJJaX1FBs1nJKCtNMTUfFuzk6nTSbvOQ1PemcQ1NZahc9nqbjcgQ1rZxSsyGsxhRvaOUFnWq8lQGpASoKBN/fH/7c0074iLg3m63Px1rfpXy+n8/+vj+fRe5X38veNsuyLAEAAKBCft4uAAAAoDYjLAEAABgQlgAAAAwISwAAAAaEJQAAAAPCEgAAgAFhCQAAwICwBAAAYEBYAgAAMCAsAQAAGBCWAFzXli5dKpvNJpvNps8+++yy/ZZlKSYmRjabTQ899NBl+/Pz8xUSEiKbzabs7OwKjzFixAjnMX65hYSEuH1OAGpWgLcLAICaEBISorS0NP3mN79xaf/000/17bffKjg4uMJxq1evls1mk8PhUGpqqv70pz9V2C84OFhvvPHGZe3+/v7XXjwAryIsAbghdO/eXatXr9bChQsVEPDvf/rS0tLUvn17nTx5ssJxy5cvV/fu3XXLLbcoLS2t0rAUEBCgIUOGeKR2AN7FZTgAN4Tk5GSdOnVK6enpzraSkhKtWbNGgwYNqnDMsWPHtHnzZg0cOFADBw7U4cOHtXXr1poqGUAtQVgCcEOIjY1VQkKCVqxY4Wz74IMPVFBQoIEDB1Y4ZsWKFapbt64eeughxcfH61e/+pVSU1MrPcbJkycv2woLC90+FwA1i7AE4IYxaNAgrV27VufPn5ckpaamqkuXLoqOjq6wf2pqqnr16qXQ0FBJ0oABA/TWW2/pp59+uqzvuXPnFBkZednWv39/z00IQI0gLAG4YfTv31/nz5/XunXrdObMGa1bt67SS3Bffvml9u7dq+TkZGdbcnKyTp48qQ8//PCy/iEhIUpPT79se+655zw2HwA1gxu8AdwwIiMjlZiYqLS0NBUVFamsrEz9+vWrsO/y5ctVt25d3Xrrrfr6668lXQxEsbGxSk1NVY8ePVz6+/v7KzEx0eNzAFDzCEsAbiiDBg3S6NGjlZOTowcffFB2u/2yPpZlacWKFTp37pxatmx52f68vDydPXtWYWFhNVAxAG8jLAG4ofTp00d/+MMftG3bNq1atarCPpc+e2nOnDmKi4tz2ffjjz9qzJgxWrt2LR8VANwgCEsAbihhYWFatGiRjhw5op49e1bY59IluMmTJ1f4CdwvvPCCUlNTCUvADYKwBOCGM3z48Er3FRcX6+2339b9999f6VeVPPzww1qwYIHy8vLUsGFDSdJPP/2k5cuXV9i/T58+qlu37rUXDsArCEsA8DPr169Xfn5+pWedJKlnz56aN2+eVq5cqfHjx0u6GLKGDh1aYf/Dhw8TlgAfZrMsy/J2EQAAALUVn7MEAABgQFgCAAAwICwBAAAYEJYAAAAMCEsAAAAGhCUAAAADPmfJDcrLy3XixAnddNNNstls3i4HAABUgWVZOnPmjKKjo+XnV/n5I8KSG5w4cUIxMTHeLgMAAFTD8ePH1aRJk0r3E5bc4KabbpJ0cbHDw8O9XA0AAKiKwsJCxcTEON/HK0NYcoNLl97Cw8MJSwAA+Jgr3ULDDd4AAAAGhCUAAAADwhIAAIAB9ywBAFCLlZWVqbS01Ntl+KTAwED5+/tf8+sQlgAAqIUsy1JOTo7y8/O9XYpPs9vtcjgc1/Q5iIQlAABqoUtBqWHDhqpTpw4fenyVLMtSUVGR8vLyJEmNGjWq9msRlgAAqGXKysqcQal+/freLsdnhYaGSpLy8vLUsGHDal+S4wZvAABqmUv3KNWpU8fLlfi+S2t4Lfd9EZYAAKiluPR27dyxhoQlAAAAA8ISAAColWJjYzV//nxvl8EN3gAAwH1++9vfqm3btm4JOTt37lTdunWvvahrRFgCAAA1xrIslZWVKSDgyhEkMjKyBiq6Mi7DAQAAtxgxYoQ+/fRTLViwQDabTTabTUuXLpXNZtMHH3yg9u3bKzg4WJ999pm++eYb9erVS1FRUQoLC9Pdd9+tjz76yOX1fnkZzmaz6Y033lCfPn1Up04dNW/eXO+9957H50VYAgDAB1iWpaKSn2p8syyryjUuWLBACQkJGj16tL7//nt9//33iomJkSRNnTpVzz33nLKzs9W6dWudPXtW3bt3V0ZGhvbs2aNu3bqpZ8+eOnbsmPEYzz77rPr3768vv/xS3bt31+DBg3X69OlrWtsr4TIcAAA+4HxpmVrO+rDGj3tgTpLqBFUtLkRERCgoKEh16tSRw+GQJB08eFCSNGfOHN1///3OvvXq1VObNm2cP8+dO1fvvvuu3nvvPY0bN67SY4wYMULJycmSpP/+7//WwoULtWPHDnXr1u2q51ZVnFkCAAAe16FDB5efz549q0mTJikuLk52u11hYWHKzs6+4pml1q1bO/9et25dhYeHO7/SxFM4swQAgA8IDfTXgTlJXjmuO/zyqbZJkyYpPT1dL774om677TaFhoaqX79+KikpMb5OYGCgy882m03l5eVuqbEyhCUAAHyAzWar8uUwbwoKClJZWdkV+23ZskUjRoxQnz59JF0803TkyBEPV1c9XIYDAABuExsbq+3bt+vIkSM6efJkpWd9mjdvrnfeeUeff/65vvjiCw0aNMjjZ4iqi7AEAADcZtKkSfL391fLli0VGRlZ6T1IL730km6++WZ17txZPXv2VFJSktq1a1fD1VaNzbqaZwJRocLCQkVERKigoEDh4eHeLgcA4OMuXLigw4cPq1mzZgoJCfF2OT7NtJZVff/mzBIAAIABYQkAAMCAsAQAAGBAWAIAADAgLAEAABgQlgAAAAwISwAAAAaEJQAAAAPCEgAAgAFhCQAA1BqxsbGaP3++t8twQVgCAAAwICwBAAAYEJYAAIBbLFmyRNHR0SovL3dp79Wrlx599FF988036tWrl6KiohQWFqa7775bH330kZeqrTrCEgAAvsCypJJzNb9ZVpVL/N3vfqdTp07p448/dradPn1aGzdu1ODBg3X27Fl1795dGRkZ2rNnj7p166aePXvq2LFjnlgxtwnwdgEAAKAKSouk/46u+eNOPyEF1a1S15tvvlkPPvig0tLS1LVrV0nSmjVr1KBBA917773y8/NTmzZtnP3nzp2rd999V++9957GjRvnkfLdgTNLAADAbQYPHqy3335bxcXFkqTU1FQNHDhQfn5+Onv2rCZNmqS4uDjZ7XaFhYUpOzubM0sAAMANAutcPMvjjeNehZ49e8qyLK1fv1533323Nm/erJdfflmSNGnSJKWnp+vFF1/UbbfdptDQUPXr108lJSWeqNxtCEsAAPgCm63Kl8O8KSQkRI888ohSU1P19ddf6/bbb1e7du0kSVu2bNGIESPUp08fSdLZs2d15MgRL1ZbNYQlAADgVoMHD9ZDDz2k/fv3a8iQIc725s2b65133lHPnj1ls9n0zDPPXPbkXG3EPUsAAMCt7rvvPtWrV0+HDh3SoEGDnO0vvfSSbr75ZnXu3Fk9e/ZUUlKS86xTbcaZJQAA4FZ+fn46ceLy+6tiY2O1adMml7axY8e6/FwbL8v53Jml1157TbGxsQoJCVHHjh21Y8cOY//Vq1erRYsWCgkJUatWrbRhw4ZK+z722GOy2Wy17jtpAACA9/hUWFq1apUmTJig2bNna/fu3WrTpo2SkpKUl5dXYf+tW7cqOTlZo0aN0p49e9S7d2/17t1b+/btu6zvu+++q23btik62gufYQEAAGotnwpLL730kkaPHq2RI0eqZcuWWrx4serUqaM333yzwv4LFixQt27dNHnyZMXFxWnu3Llq166dXn31VZd+3333nZ544gmlpqYqMDCwJqYCAAB8hM+EpZKSEmVlZSkxMdHZ5ufnp8TERGVmZlY4JjMz06W/JCUlJbn0Ly8v19ChQzV58mTdcccdnikeAAD4LJ+5wfvkyZMqKytTVFSUS3tUVJQOHjxY4ZicnJwK++fk5Dh//vOf/6yAgACNHz++yrUUFxc7P5lUkgoLC6s8FgCAqrKu4nvZUDF3rKHPnFnyhKysLC1YsEBLly6VzWar8riUlBRFREQ4t5iYGA9WCQC40Vy6JaSoqMjLlfi+S2t4LbfZ+MyZpQYNGsjf31+5ubku7bm5uXI4HBWOcTgcxv6bN29WXl6emjZt6txfVlamiRMnav78+ZU+vjht2jRNmDDB+XNhYSGBCQDgNv7+/rLb7c4HmOrUqXNV/1OPi2eUioqKlJeXJ7vdLn9//2q/ls+EpaCgILVv314ZGRnq3bu3pIv3G2VkZFT6TcUJCQnKyMjQH//4R2dbenq6EhISJElDhw6t8J6moUOHauTIkZXWEhwcrODg4GubEAAABpf+x76yJ75RNXa7vdKTKlXlM2FJkiZMmKDhw4erQ4cOio+P1/z583Xu3DlnsBk2bJgaN26slJQUSdKTTz6pLl26aN68eerRo4dWrlypXbt2acmSJZKk+vXrq379+i7HCAwMlMPh0O23316zkwMA4GdsNpsaNWqkhg0bqrS01Nvl+KTAwMBrOqN0iU+FpQEDBuiHH37QrFmzlJOTo7Zt22rjxo3Om7iPHTsmP79/34bVuXNnpaWlaebMmZo+fbqaN2+utWvX6s477/TWFAAAuCr+/v5uecNH9dksbrW/ZoWFhYqIiFBBQYHCw8O9XQ4AAKiCqr5/39BPwwEAAFwJYQkAAMCAsAQAAGBAWAIAADAgLAEAABgQlgAAAAwISwAAAAaEJQAAAAPCEgAAgAFhCQAAwICwBAAAYEBYAgAAMCAsAQAAGBCWAAAADAhLAAAABoQlAAAAA8ISAACAAWEJAADAgLAEAABgQFgCAAAwICwBAAAYEJYAAAAMCEsAAAAGhCUAAAADwhIAAIABYQkAAMCAsAQAAGBAWAIAADAgLAEAABgQlgAAAAwISwAAAAaEJQAAAAPCEgAAgAFhCQAAwICwBAAAYEBYAgAAMCAsAQAAGBCWAAAADAhLAAAABoQlAAAAA8ISAACAAWEJAADAgLAEAABgQFgCAAAwICwBAAAYEJYAAAAMCEsAAAAGhCUAAAADwhIAAIABYQkAAMCAsAQAAGBAWAIAADAgLAEAABgQlgAAAAwISwAAAAaEJQAAAAOfC0uvvfaaYmNjFRISoo4dO2rHjh3G/qtXr1aLFi0UEhKiVq1aacOGDc59paWlmjJlilq1aqW6desqOjpaw4YN04kTJzw9DQAA4CN8KiytWrVKEyZM0OzZs7V79261adNGSUlJysvLq7D/1q1blZycrFGjRmnPnj3q3bu3evfurX379kmSioqKtHv3bj3zzDPavXu33nnnHR06dEgPP/xwTU4LAADUYjbLsixvF1FVHTt21N13361XX31VklReXq6YmBg98cQTmjp16mX9BwwYoHPnzmndunXOtk6dOqlt27ZavHhxhcfYuXOn4uPjdfToUTVt2rRKdRUWFioiIkIFBQUKDw+vxswAAEBNq+r7t8+cWSopKVFWVpYSExOdbX5+fkpMTFRmZmaFYzIzM136S1JSUlKl/SWpoKBANptNdrvdLXUDAADfFuDtAqrq5MmTKisrU1RUlEt7VFSUDh48WOGYnJycCvvn5ORU2P/ChQuaMmWKkpOTjQmzuLhYxcXFzp8LCwurOg0AAOBjfObMkqeVlpaqf//+sixLixYtMvZNSUlRRESEc4uJiamhKgEAQE3zmbDUoEED+fv7Kzc316U9NzdXDoejwjEOh6NK/S8FpaNHjyo9Pf2K9x1NmzZNBQUFzu348ePVmBEAAPAFPhOWgoKC1L59e2VkZDjbysvLlZGRoYSEhArHJCQkuPSXpPT0dJf+l4LSV199pY8++kj169e/Yi3BwcEKDw932QAAwPXJZ+5ZkqQJEyZo+PDh6tChg+Lj4zV//nydO3dOI0eOlCQNGzZMjRs3VkpKiiTpySefVJcuXTRv3jz16NFDK1eu1K5du7RkyRJJF4NSv379tHv3bq1bt05lZWXO+5nq1aunoKAg70wUAADUGj4VlgYMGKAffvhBs2bNUk5Ojtq2bauNGzc6b+I+duyY/Pz+fbKsc+fOSktL08yZMzV9+nQ1b95ca9eu1Z133ilJ+u677/Tee+9Jktq2betyrI8//li//e1va2ReAACg9vKpz1mqrficJQAAfM919zlLAAAA3kBYAgAAMCAsAQAAGBCWAAAADAhLAAAABoQlAAAAA8ISAACAAWEJAADAgLAEAABgQFgCAAAwICwBAAAYEJYAAAAMCEsAAAAGhCUAAAADwhIAAIABYQkAAMCAsAQAAGBAWAIAADAgLAEAABgQlgAAAAwISwAAAAaEJQAAAAPCEgAAgAFhCQAAwICwBAAAYEBYAgAAMCAsAQAAGBCWAAAADAhLAAAABoQlAAAAA8ISAACAAWEJAADAgLAEAABgQFgCAAAwICwBAAAYEJYAAAAMCEsAAAAGhCUAAAADwhIAAIABYQkAAMCAsAQAAGBAWAIAADAgLAEAABgQlgAAAAwISwAAAAaEJQAAAAPCEgAAgAFhCQAAwICwBAAAYFDtsPTXv/5V99xzj6Kjo3X06FFJ0vz58/W3v/3NbcUBAAB4W7XC0qJFizRhwgR1795d+fn5KisrkyTZ7XbNnz/fnfUBAAB4VbXC0iuvvKL//d//1YwZM+Tv7+9s79Chg/bu3eu24gAAALytWmHp8OHDuuuuuy5rDw4O1rlz5665KAAAgNqiWmGpWbNm+vzzzy9r37hxo+Li4q61JgAAgFojoDqDJkyYoLFjx+rChQuyLEs7duzQihUrlJKSojfeeMPdNQIAAHhNtcLS73//e4WGhmrmzJkqKirSoEGDFB0drQULFmjgwIHurhEAAMBrbJZlWdfyAkVFRTp79qwaNmzorpp8TmFhoSIiIlRQUKDw8HBvlwMAAKqgqu/f1/yhlHXq1KnRoPTaa68pNjZWISEh6tixo3bs2GHsv3r1arVo0UIhISFq1aqVNmzY4LLfsizNmjVLjRo1UmhoqBITE/XVV195cgoAAMCHVDssrVmzRv3791enTp3Url07l81TVq1apQkTJmj27NnavXu32rRpo6SkJOXl5VXYf+vWrUpOTtaoUaO0Z88e9e7dW71799a+ffucfZ5//nktXLhQixcv1vbt21W3bl0lJSXpwoULHpsHAADwHdUKSwsXLtTIkSMVFRWlPXv2KD4+XvXr19e//vUvPfjgg+6u0emll17S6NGjNXLkSLVs2VKLFy9WnTp19Oabb1bYf8GCBerWrZsmT56suLg4zZ07V+3atdOrr74q6eJZpfnz52vmzJnq1auXWrdurWXLlunEiRNau3atx+YBAAB8R7XC0uuvv64lS5bolVdeUVBQkJ5++mmlp6dr/PjxKigocHeNkqSSkhJlZWUpMTHR2ebn56fExERlZmZWOCYzM9OlvyQlJSU5+x8+fFg5OTkufSIiItSxY8dKX1OSiouLVVhY6LIBAIDrU7XC0rFjx9S5c2dJUmhoqM6cOSNJGjp0qFasWOG+6n7m5MmTKisrU1RUlEt7VFSUcnJyKhyTk5Nj7H/pz6t5TUlKSUlRRESEc4uJibnq+QAAAN9QrbDkcDh0+vRpSVLTpk21bds2SRfP1Fzjw3U+Ydq0aSooKHBux48f93ZJAADAQ6oVlu677z699957kqSRI0fqqaee0v33368BAwaoT58+bi3wkgYNGsjf31+5ubku7bm5uXI4HBWOcTgcxv6X/rya15Qufq1LeHi4ywYAAK5P1QpLS5Ys0YwZMyRJY8eO1f/93/8pLi5Oc+bM0eLFi91a4CVBQUFq3769MjIynG3l5eXKyMhQQkJChWMSEhJc+ktSenq6s3+zZs3kcDhc+hQWFmr79u2VviYAALixVOsTvP38/FRSUqLdu3crLy/P+flE0sXvh+vZs6dbi7xkwoQJGj58uDp06KD4+HjNnz9f586d08iRIyVJw4YNU+PGjZWSkiJJevLJJ9WlSxfNmzdPPXr00MqVK7Vr1y4tWbJEkmSz2fTHP/5Rf/rTn9S8eXM1a9ZMzzzzjKKjo9W7d2+PzAEAAPiWaoWljRs3aujQoTp16tRl+2w2m8rKyq65sIoMGDBAP/zwg2bNmqWcnBy1bdtWGzdudN6gfezYMfn5/ftkWefOnZWWlqaZM2dq+vTpat68udauXas777zT2efpp5/WuXPnNGbMGOXn5+s3v/mNNm7cqJCQEI/MAQAA+JZqfd1J8+bN9cADD2jWrFmXPUl2I+LrTgAA8D0e/bqT3NxcTZgwgaAEAACue9UKS/369dMnn3zi5lIAAABqn2pdhisqKtLvfvc7RUZGqlWrVgoMDHTZP378eLcV6Au4DAcAgO+p6vt3tW7wXrFihf7+978rJCREn3zyiWw2m3OfzWa74cISAAC4flUrLM2YMUPPPvuspk6d6vL0GQAAwPWmWkmnpKREAwYMICgBAIDrXrXSzvDhw7Vq1Sp31wIAAFDrVOsyXFlZmZ5//nl9+OGHat269WU3eL/00ktuKQ4AAMDbqhWW9u7dq7vuukuStG/fPpd9P7/ZGwAAwNdVKyx9/PHH7q4DAACgVuIObQAAAAPCEgAAgAFhCQAAwICwBAAAYEBYAgAAMCAsAQAAGBCWAAAADAhLAAAABoQlAAAAA8ISAACAAWEJAADAgLAEAABgQFgCAAAwICwBAAAYEJYAAAAMCEsAAAAGhCUAAAADwhIAAIABYQkAAMCAsAQAAGBAWAIAADAgLAEAABgQlgAAAAwISwAAAAaEJQAAAAPCEgAAgAFhCQAAwICwBAAAYEBYAgAAMCAsAQAAGBCWAAAADAhLAAAABoQlAAAAA8ISAACAAWEJAADAgLAEAABgQFgCAAAwICwBAAAYEJYAAAAMCEsAAAAGhCUAAAADwhIAAIABYQkAAMCAsAQAAGBAWAIAADAgLAEAABj4TFg6ffq0Bg8erPDwcNntdo0aNUpnz541jrlw4YLGjh2r+vXrKywsTH379lVubq5z/xdffKHk5GTFxMQoNDRUcXFxWrBggaenAgAAfIjPhKXBgwdr//79Sk9P17p16/SPf/xDY8aMMY556qmn9P7772v16tX69NNPdeLECT3yyCPO/VlZWWrYsKGWL1+u/fv3a8aMGZo2bZpeffVVT08HAAD4CJtlWZa3i7iS7OxstWzZUjt37lSHDh0kSRs3blT37t317bffKjo6+rIxBQUFioyMVFpamvr16ydJOnjwoOLi4pSZmalOnTpVeKyxY8cqOztbmzZtqnJ9hYWFioiIUEFBgcLDw6sxQwAAUNOq+v7tE2eWMjMzZbfbnUFJkhITE+Xn56ft27dXOCYrK0ulpaVKTEx0trVo0UJNmzZVZmZmpccqKChQvXr1jPUUFxersLDQZQMAANcnnwhLOTk5atiwoUtbQECA6tWrp5ycnErHBAUFyW63u7RHRUVVOmbr1q1atWrVFS/vpaSkKCIiwrnFxMRUfTIAAMCneDUsTZ06VTabzbgdPHiwRmrZt2+fevXqpdmzZ+uBBx4w9p02bZoKCgqc2/Hjx2ukRgAAUPMCvHnwiRMnasSIEcY+t956qxwOh/Ly8lzaf/rpJ50+fVoOh6PCcQ6HQyUlJcrPz3c5u5Sbm3vZmAMHDqhr164aM2aMZs6cecW6g4ODFRwcfMV+AADA93k1LEVGRioyMvKK/RISEpSfn6+srCy1b99ekrRp0yaVl5erY8eOFY5p3769AgMDlZGRob59+0qSDh06pGPHjikhIcHZb//+/brvvvs0fPhw/dd//ZcbZgUAAK4nPvE0nCQ9+OCDys3N1eLFi1VaWqqRI0eqQ4cOSktLkyR999136tq1q5YtW6b4+HhJ0n/8x39ow4YNWrp0qcLDw/XEE09IunhvknTx0tt9992npKQkvfDCC85j+fv7VynEXcLTcAAA+J6qvn979czS1UhNTdW4cePUtWtX+fn5qW/fvlq4cKFzf2lpqQ4dOqSioiJn28svv+zsW1xcrKSkJL3++uvO/WvWrNEPP/yg5cuXa/ny5c72W265RUeOHKmReQEAgNrNZ84s1WacWQIAwPdcV5+zBAAA4C2EJQAAAAPCEgAAgAFhCQAAwICwBAAAYEBYAgAAMCAsAQAAGBCWAAAADAhLAAAABoQlAAAAA8ISAACAAWEJAADAgLAEAABgQFgCAAAwICwBAAAYEJYAAAAMCEsAAAAGhCUAAAADwhIAAIABYQkAAMCAsAQAAGBAWAIAADAgLAEAABgQlgAAAAwISwAAAAaEJQAAAAPCEgAAgAFhCQAAwICwBAAAYEBYAgAAMCAsAQAAGBCWAAAADAhLAAAABoQlAAAAA8ISAACAAWEJAADAgLAEAABgQFgCAAAwICwBAAAYEJYAAAAMCEsAAAAGhCUAAAADwhIAAIABYQkAAMCAsAQAAGBAWAIAADAgLAEAABgQlgAAAAwISwAAAAaEJQAAAAPCEgAAgAFhCQAAwICwBAAAYEBYAgAAMCAsAQAAGBCWAAAADHwmLJ0+fVqDBw9WeHi47Ha7Ro0apbNnzxrHXLhwQWPHjlX9+vUVFhamvn37Kjc3t8K+p06dUpMmTWSz2ZSfn++BGQAAAF/kM2Fp8ODB2r9/v9LT07Vu3Tr94x//0JgxY4xjnnrqKb3//vtavXq1Pv30U504cUKPPPJIhX1HjRql1q1be6J0AADgw2yWZVneLuJKsrOz1bJlS+3cuVMdOnSQJG3cuFHdu3fXt99+q+jo6MvGFBQUKDIyUmlpaerXr58k6eDBg4qLi1NmZqY6derk7Lto0SKtWrVKs2bNUteuXfXjjz/KbrdXub7CwkJFRESooKBA4eHh1zZZAABQI6r6/u0TZ5YyMzNlt9udQUmSEhMT5efnp+3bt1c4JisrS6WlpUpMTHS2tWjRQk2bNlVmZqaz7cCBA5ozZ46WLVsmP7+qLUdxcbEKCwtdNgAAcH3yibCUk5Ojhg0burQFBASoXr16ysnJqXRMUFDQZWeIoqKinGOKi4uVnJysF154QU2bNq1yPSkpKYqIiHBuMTExVzchAADgM7walqZOnSqbzWbcDh486LHjT5s2TXFxcRoyZMhVjysoKHBux48f91CFAADA2wK8efCJEydqxIgRxj633nqrHA6H8vLyXNp/+uknnT59Wg6Ho8JxDodDJSUlys/Pdzm7lJub6xyzadMm7d27V2vWrJEkXbp9q0GDBpoxY4aeffbZCl87ODhYwcHBVZkiAADwcV4NS5GRkYqMjLxiv4SEBOXn5ysrK0vt27eXdDHolJeXq2PHjhWOad++vQIDA5WRkaG+fftKkg4dOqRjx44pISFBkvT222/r/PnzzjE7d+7Uo48+qs2bN+tXv/rVtU4PAABcB7walqoqLi5O3bp10+jRo7V48WKVlpZq3LhxGjhwoPNJuO+++05du3bVsmXLFB8fr4iICI0aNUoTJkxQvXr1FB4erieeeEIJCQnOJ+F+GYhOnjzpPN7VPA0HAACuXz4RliQpNTVV48aNU9euXeXn56e+fftq4cKFzv2lpaU6dOiQioqKnG0vv/yys29xcbGSkpL0+uuve6N8AADgo3zic5ZqOz5nCQAA33Ndfc4SAACAtxCWAAAADAhLAAAABoQlAAAAA8ISAACAAWEJAADAgLAEAABgQFgCAAAwICwBAAAYEJYAAAAMCEsAAAAGhCUAAAADwhIAAIABYQkAAMCAsAQAAGBAWAIAADAgLAEAABgQlgAAAAwISwAAAAaEJQAAAAPCEgAAgAFhCQAAwICwBAAAYEBYAgAAMCAsAQAAGBCWAAAADAhLAAAABoQlAAAAA8ISAACAAWEJAADAgLAEAABgQFgCAAAwICwBAAAYEJYAAAAMCEsAAAAGhCUAAAADwhIAAIABYQkAAMCAsAQAAGBAWAIAADAgLAEAABgQlgAAAAwISwAAAAYB3i7gemBZliSpsLDQy5UAAICquvS+fel9vDKEJTc4c+aMJCkmJsbLlQAAgKt15swZRUREVLrfZl0pTuGKysvLdeLECd10002y2WzeLserCgsLFRMTo+PHjys8PNzb5Vy3WOeaw1rXDNa5ZrDOrizL0pkzZxQdHS0/v8rvTOLMkhv4+fmpSZMm3i6jVgkPD+c/xBrAOtcc1rpmsM41g3X+N9MZpUu4wRsAAMCAsAQAAGBAWIJbBQcHa/bs2QoODvZ2Kdc11rnmsNY1g3WuGaxz9XCDNwAAgAFnlgAAAAwISwAAAAaEJQAAAAPCEgAAgAFhCVft9OnTGjx4sMLDw2W32zVq1CidPXvWOObChQsaO3as6tevr7CwMPXt21e5ubkV9j116pSaNGkim82m/Px8D8zAN3hinb/44gslJycrJiZGoaGhiouL04IFCzw9lVrltddeU2xsrEJCQtSxY0ft2LHD2H/16tVq0aKFQkJC1KpVK23YsMFlv2VZmjVrlho1aqTQ0FAlJibqq6++8uQUfII717m0tFRTpkxRq1atVLduXUVHR2vYsGE6ceKEp6dR67n79/nnHnvsMdlsNs2fP9/NVfsgC7hK3bp1s9q0aWNt27bN2rx5s3XbbbdZycnJxjGPPfaYFRMTY2VkZFi7du2yOnXqZHXu3LnCvr169bIefPBBS5L1448/emAGvsET6/yXv/zFGj9+vPXJJ59Y33zzjfXXv/7VCg0NtV555RVPT6dWWLlypRUUFGS9+eab1v79+63Ro0dbdrvdys3NrbD/li1bLH9/f+v555+3Dhw4YM2cOdMKDAy09u7d6+zz3HPPWREREdbatWutL774wnr44YetZs2aWefPn6+padU67l7n/Px8KzEx0Vq1apV18OBBKzMz04qPj7fat29fk9OqdTzx+3zJO++8Y7Vp08aKjo62Xn75ZQ/PpPYjLOGqHDhwwJJk7dy509n2wQcfWDabzfruu+8qHJOfn28FBgZaq1evdrZlZ2dbkqzMzEyXvq+//rrVpUsXKyMj44YOS55e5597/PHHrXvvvdd9xddi8fHx1tixY50/l5WVWdHR0VZKSkqF/fv372/16NHDpa1jx47WH/7wB8uyLKu8vNxyOBzWCy+84Nyfn59vBQcHWytWrPDADHyDu9e5Ijt27LAkWUePHnVP0T7IU+v87bffWo0bN7b27dtn3XLLLYQly7K4DIerkpmZKbvdrg4dOjjbEhMT5efnp+3bt1c4JisrS6WlpUpMTHS2tWjRQk2bNlVmZqaz7cCBA5ozZ46WLVtm/ELDG4En1/mXCgoKVK9ePfcVX0uVlJQoKyvLZX38/PyUmJhY6fpkZma69JekpKQkZ//Dhw8rJyfHpU9ERIQ6duxoXPPrmSfWuSIFBQWy2Wyy2+1uqdvXeGqdy8vLNXToUE2ePFl33HGHZ4r3QTf2OxKuWk5Ojho2bOjSFhAQoHr16iknJ6fSMUFBQZf9oxYVFeUcU1xcrOTkZL3wwgtq2rSpR2r3JZ5a51/aunWrVq1apTFjxril7trs5MmTKisrU1RUlEu7aX1ycnKM/S/9eTWveb3zxDr/0oULFzRlyhQlJyffsF8G66l1/vOf/6yAgACNHz/e/UX7MMISJElTp06VzWYzbgcPHvTY8adNm6a4uDgNGTLEY8eoDby9zj+3b98+9erVS7Nnz9YDDzxQI8cErlVpaan69+8vy7K0aNEib5dzXcnKytKCBQu0dOlS2Ww2b5dTqwR4uwDUDhMnTtSIESOMfW699VY5HA7l5eW5tP/00086ffq0HA5HheMcDodKSkqUn5/vctYjNzfXOWbTpk3au3ev1qxZI+niE0aS1KBBA82YMUPPPvtsNWdWu3h7nS85cOCAunbtqjFjxmjmzJnVmouvadCggfz9/S97CrOi9bnE4XAY+1/6Mzc3V40aNXLp07ZtWzdW7zs8sc6XXApKR48e1aZNm27Ys0qSZ9Z58+bNysvLczm7X1ZWpokTJ2r+/Pk6cuSIeyfhS7x90xR8y6Ubj3ft2uVs+/DDD6t04/GaNWucbQcPHnS58fjrr7+29u7d69zefPNNS5K1devWSp/suJ55ap0ty7L27dtnNWzY0Jo8ebLnJlBLxcfHW+PGjXP+XFZWZjVu3Nh4Q+xDDz3k0paQkHDZDd4vvviic39BQQE3eLt5nS3LskpKSqzevXtbd9xxh5WXl+eZwn2Mu9f55MmTLv8O792714qOjramTJliHTx40HMT8QGEJVy1bt26WXfddZe1fft267PPPrOaN2/u8kj7t99+a91+++3W9u3bnW2PPfaY1bRpU2vTpk3Wrl27rISEBCshIaHSY3z88cc39NNwluWZdd67d68VGRlpDRkyxPr++++d243y5rNy5UorODjYWrp0qXXgwAFrzJgxlt1ut3JycizLsqyhQ4daU6dOdfbfsmWLFRAQYL344otWdna2NXv27Ao/OsBut1t/+9vfrC+//NLq1asXHx3g5nUuKSmxHn74YatJkybW559/7vK7W1xc7JU51gae+H3+JZ6Gu4iwhKt26tQpKzk52QoLC7PCw8OtkSNHWmfOnHHuP3z4sCXJ+vjjj51t58+ftx5//HHr5ptvturUqWP16dPH+v777ys9BmHJM+s8e/ZsS9Jl2y233FKDM/OuV155xWratKkVFBRkxcfHW9u2bXPu69KlizV8+HCX/m+99Zb161//2goKCrLuuOMOa/369S77y8vLrWeeecaKioqygoODra5du1qHDh2qianUau5c50u/6xVtP//9vxG5+/f5lwhLF9ks6//fHAIAAIDL8DQcAACAAWEJAADAgLAEAABgQFgCAAAwICwBAAAYEJYAAAAMCEsAAAAGhCUAcLNPPvlENptN+fn53i4FgBsQlgAAAAwISwAAAAaEJQDXnfLycqWkpKhZs2YKDQ1VmzZttGbNGkn/vkS2fv16tW7dWiEhIerUqZP27dvn8hpvv/227rjjDgUHBys2Nlbz5s1z2V9cXKwpU6YoJiZGwcHBuu222/SXv/zFpU9WVpY6dOigOnXqqHPnzjp06JBnJw7AIwhLAK47KSkpWrZsmRYvXqz9+/frqaee0pAhQ/Tpp586+0yePFnz5s3Tzp07FRkZqZ49e6q0tFTSxZDTv39/DRw4UHv37tV//ud/6plnntHSpUud44cNG6YVK1Zo4cKFys7O1v/8z/8oLCzMpY4ZM2Zo3rx52rVrlwICAvToo4/WyPwBuBdfpAvgulJcXKx69erpo48+UkJCgrP997//vYqKijRmzBjde++9WrlypQYMGCBJOn36tJo0aaKlS5eqf//+Gjx4sH744Qf9/e9/d45/+umntX79eu3fv1///Oc/dfvttys9PV2JiYmX1fDJJ5/o3nvv1UcffaSuXbtKkjZs2KAePXro/PnzCgkJ8fAqAHAnziwBuK58/fXXKioq0v3336+wsDDntmzZMn3zzTfOfj8PUvXq1dPtt9+u7OxsSVJ2drbuuecel9e955579NVXX6msrEyff/65/P391aVLF2MtrVu3dv69UaNGkqS8vLxrniOAmhXg7QIAwJ3Onj0rSVq/fr0aN27ssi84ONglMFVXaGholfoFBgY6/26z2SRdvJ8KgG/hzBKA60rLli0VHBysY8eO6bbbbnPZYmJinP22bdvm/PuPP/6of/7zn4qLi5MkxcXFacuWLS6vu2XLFv3617+Wv7+/WrVqpfLycpd7oABcvzizBOC6ctNNN2nSpEl66qmnVF5ert/85jcqKCjQli1bFB4erltuuUWSNGfOHNWvX19RUVGaMWOGGjRooN69e0uSJk6cqLvvvltz587VgAEDlJmZqVdffVWvv/66JCk2NlbDhw/Xo48+qoULF6pNmzY6evSo8vLy1L9/f29NHYCHEJYAXHfmzp2ryMhIpaSk6F//+pfsdrvatWun6dOnOy+DPffcc3ryySf11VdfqW3btnr//fcVFBQkSWrXrp3eeustzZo1S3PnzlWjRo00Z84cjRgxwnmMRYsWafr06Xr88cd16tQpNW3aVNOnT/fGdAF4GE/DAbihXHpS7ccff5Tdbvd2OQB8APcsAQAAGBCWAAAADLgMBwAAYMCZJQAAAAPCEgAAgAFhCQAAwICwBAAAYEBYAgAAMCAsAQAAGBCWAAAADAhLAAAABoQlAAAAg/8H0EpXF+8nRDMAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# draw mae\n",
    "if is_graphic:\n",
    "    plt.plot(list(range(1, len(arr_train_mae) + 1)), arr_train_mae, label=\"train\")\n",
    "    plt.plot(list(range(1, len(arr_val_mae) + 1)), arr_val_mae, label=\"val\")\n",
    "\n",
    "    plt.title(\"MAE\")\n",
    "    plt.xlabel(\"epoch\")\n",
    "    plt.ylabel(\"mae\")\n",
    "    plt.legend(loc=\"upper right\")\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1cc21154",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAksAAAHHCAYAAACvJxw8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2kElEQVR4nO3deVyVZf7/8fdhRxFQQRAFNTO1chk3xBanpDDN3DVGc52cSs3cxl0nnRnaLDUrp8d3/Pp1wjTN/Jo1zhfRllFy11zAMcddAZcAFQWE6/dHP89EwC3igcPR1/PxOA/huq/7nM91DdN5P65z3fexGWOMAAAAUCw3ZxcAAABQmRGWAAAALBCWAAAALBCWAAAALBCWAAAALBCWAAAALBCWAAAALBCWAAAALBCWAAAALBCWAAAALBCWAFRa77//vmw2myIjI51dCoC7mI3vhgNQWT300EM6c+aMjh07psOHD+vee+91dkkA7kKsLAGolI4ePaotW7bo7bffVnBwsOLj451dUrGuXLni7BIAlDPCEoBKKT4+XtWrV1fXrl3Vp0+fYsNSRkaGxo4dq/r168vb21t169bVoEGDdP78eXufa9eu6Q9/+IPuu+8++fj4qHbt2urVq5eOHDkiSfrqq69ks9n01VdfFXruY8eOyWazacmSJfa2IUOGyM/PT0eOHFGXLl1UrVo1DRgwQJL07bffqm/fvoqIiJC3t7fCw8M1duxYXb16tUjdKSkp6tevn4KDg+Xr66vGjRtr2rRpkqRNmzbJZrPps88+K3LesmXLZLPZlJSUdMvzCaDsPJxdAAAUJz4+Xr169ZKXl5diY2P1wQcfaPv27Wrbtq0k6fLly3rkkUeUnJysYcOGqVWrVjp//rzWrl2rU6dOKSgoSPn5+Xr66aeVmJioZ599VmPGjNGlS5eUkJCg/fv3q2HDhrdc1/Xr1xUTE6OHH35Yb731lqpUqSJJWrlypbKzs/Xiiy+qZs2a2rZtm959912dOnVKK1eutJ///fff65FHHpGnp6dGjBih+vXr68iRI/r888/1pz/9Sb/+9a8VHh6u+Ph49ezZs8icNGzYUFFRUbcxswBumQGASmbHjh1GkklISDDGGFNQUGDq1q1rxowZY+8zc+ZMI8msXr26yPkFBQXGGGMWL15sJJm33367xD6bNm0yksymTZsKHT969KiRZP77v//b3jZ48GAjyUyePLnI82VnZxdpi4uLMzabzRw/ftze9uijj5pq1aoVavt5PcYYM2XKFOPt7W0yMjLsbenp6cbDw8PMmjWryOsAKF98DAeg0omPj1dISIgee+wxSZLNZlP//v21fPly5efnS5I+/fRTtWjRosjqy43+N/oEBQVp9OjRJfYpixdffLFIm6+vr/3nK1eu6Pz58+rQoYOMMdq9e7ck6dy5c/rmm280bNgwRURElFjPoEGDlJOTo1WrVtnbVqxYoevXr2vgwIFlrhtA2RCWAFQq+fn5Wr58uR577DEdPXpUP/zwg3744QdFRkYqLS1NiYmJkqQjR47owQcftHyuI0eOqHHjxvLwcNyOAw8PD9WtW7dI+4kTJzRkyBDVqFFDfn5+Cg4OVseOHSVJmZmZkqR///vfknTTups0aaK2bdsW2qcVHx+v9u3bc0Ug4ATsWQJQqWzcuFFnz57V8uXLtXz58iLH4+Pj9eSTTzrs9UpaYbqxgvVL3t7ecnNzK9L3iSee0MWLFzVp0iQ1adJEVatW1enTpzVkyBAVFBTccl2DBg3SmDFjdOrUKeXk5Oi7777TwoULb/l5ANw+whKASiU+Pl61atXSe++9V+TY6tWr9dlnn2nRokVq2LCh9u/fb/lcDRs21NatW5WXlydPT89i+1SvXl3ST1fW/dzx48dLXfO+ffv0r3/9S//zP/+jQYMG2dsTEhIK9bvnnnsk6aZ1S9Kzzz6rcePG6eOPP9bVq1fl6emp/v37l7omAI7Dx3AAKo2rV69q9erVevrpp9WnT58ij1GjRunSpUtau3atevfurb179xZ7ib35//fa7d27t86fP1/sisyNPvXq1ZO7u7u++eabQsfff//9Utft7u5e6Dlv/Dx//vxC/YKDg/Xoo49q8eLFOnHiRLH13BAUFKSnnnpKH330keLj49W5c2cFBQWVuiYAjsPKEoBKY+3atbp06ZKeeeaZYo+3b9/efoPKZcuWadWqVerbt6+GDRum1q1b6+LFi1q7dq0WLVqkFi1aaNCgQVq6dKnGjRunbdu26ZFHHtGVK1e0YcMGvfTSS+revbsCAgLUt29fvfvuu7LZbGrYsKHWrVun9PT0UtfdpEkTNWzYUBMmTNDp06fl7++vTz/9VD/++GORvgsWLNDDDz+sVq1aacSIEWrQoIGOHTumL774Qnv27CnUd9CgQerTp48kac6cOaWfSACO5cxL8QDg57p162Z8fHzMlStXSuwzZMgQ4+npac6fP28uXLhgRo0aZerUqWO8vLxM3bp1zeDBg8358+ft/bOzs820adNMgwYNjKenpwkNDTV9+vQxR44csfc5d+6c6d27t6lSpYqpXr26+d3vfmf2799f7K0DqlatWmxdBw8eNNHR0cbPz88EBQWZ559/3uzdu7fIcxhjzP79+03Pnj1NYGCg8fHxMY0bNzYzZswo8pw5OTmmevXqJiAgwFy9erWUswjA0fhuOACopK5fv66wsDB169ZNf/3rX51dDnDXYs8SAFRSa9as0blz5wptGgdQ8VhZAoBKZuvWrfr+++81Z84cBQUFadeuXc4uCbirsbIEAJXMBx98oBdffFG1atXS0qVLnV0OcNdjZQkAAMACK0sAAAAWCEsAAAAWuCmlAxQUFOjMmTOqVq3abX2TOQAAqDjGGF26dElhYWFFvvPx5whLDnDmzBmFh4c7uwwAAFAGJ0+eVN26dUs8TlhygGrVqkn6abL9/f2dXA0AACiNrKwshYeH29/HS0JYcoAbH735+/sTlgAAcDE320LDBm8AAAALhCUAAAALhCUAAAAL7FmqIPn5+crLy3N2GS7J09NT7u7uzi4DAHCXIiyVM2OMUlNTlZGR4exSXFpgYKBCQ0O5jxUAoMIRlsrZjaBUq1YtValShTf7W2SMUXZ2ttLT0yVJtWvXdnJFAIC7DWGpHOXn59uDUs2aNZ1djsvy9fWVJKWnp6tWrVp8JAcAqFBs8C5HN/YoValSxcmVuL4bc8i+LwBARSMsVQA+ert9zCEAwFkISwAAABYISyh39evX17x585xdBgAAZcIGbxTr17/+tVq2bOmQkLN9+3ZVrVr19osCAMAJCEsoE2OM8vPz5eFx8z+h4ODgCqgIAIDywcdwKGLIkCH6+uuvNX/+fNlsNtlsNi1ZskQ2m01///vf1bp1a3l7e+uf//ynjhw5ou7duyskJER+fn5q27atNmzYUOj5fvkxnM1m03/913+pZ8+eqlKliho1aqS1a9dW8CgBACgdwlIFM8YoO/e6Ux7GmFLVOH/+fEVFRen555/X2bNndfbsWYWHh0uSJk+erNdee03Jyclq3ry5Ll++rC5duigxMVG7d+9W586d1a1bN504ccLyNV599VX169dP33//vbp06aIBAwbo4sWLtz2/AAA4Gh/DVbCrefm6f+Y/nPLaB2fHqIrXzf8nDwgIkJeXl6pUqaLQ0FBJUkpKiiRp9uzZeuKJJ+x9a9SooRYtWth/nzNnjj777DOtXbtWo0aNKvE1hgwZotjYWEnSn//8Zy1YsEDbtm1T586dyzQ2AADKCytLuCVt2rQp9Pvly5c1YcIENW3aVIGBgfLz81NycvJNV5aaN29u/7lq1ary9/e3f6UJAACVCStLFczX010HZ8c47bVv1y+vapswYYISEhL01ltv6d5775Wvr6/69Omj3Nxcy+fx9PQs9LvNZlNBQcFt1wcAgKMRliqYzWYr1Udhzubl5aX8/Pyb9tu8ebOGDBminj17SvpppenYsWPlXB0AABWHj+FQrPr162vr1q06duyYzp8/X+KqT6NGjbR69Wrt2bNHe/fu1W9+8xtWiAAAdxTCEoo1YcIEubu76/7771dwcHCJe5DefvttVa9eXR06dFC3bt0UExOjVq1aVXC1AACUH5sp7fXkKFFWVpYCAgKUmZkpf39/e/u1a9d09OhRNWjQQD4+Pk6s0PUxlwAARyvp/fuXWFkCAACwQFgCAACwQFgCAACwQFgCAACwQFgCAACwQFgCAACwQFgCAACwQFgCAACwQFgCAACwQFhCuahfv77mzZvn7DIAALhthCUAAAALhCUAAAALhCUU8eGHHyosLEwFBQWF2rt3765hw4bpyJEj6t69u0JCQuTn56e2bdtqw4YNTqoWAIDyRViqaMZIuVec8zCmVCX27dtXFy5c0KZNm+xtFy9e1Pr16zVgwABdvnxZXbp0UWJionbv3q3OnTurW7duOnHiRHnNGgAATuPh7ALuOnnZ0p/DnPPaU89IXlVv2q169ep66qmntGzZMnXq1EmStGrVKgUFBemxxx6Tm5ubWrRoYe8/Z84cffbZZ1q7dq1GjRpVbuUDAOAMrCyhWAMGDNCnn36qnJwcSVJ8fLyeffZZubm56fLly5owYYKaNm2qwMBA+fn5KTk5mZUlAMAdiZWliuZZ5acVHme9dil169ZNxhh98cUXatu2rb799lu98847kqQJEyYoISFBb731lu699175+vqqT58+ys3NLa/KAQBwGsJSRbPZSvVRmLP5+PioV69eio+P1w8//KDGjRurVatWkqTNmzdryJAh6tmzpyTp8uXLOnbsmBOrBQCg/BCWUKIBAwbo6aef1oEDBzRw4EB7e6NGjbR69Wp169ZNNptNM2bMKHLlHAAAdwr2LKFEjz/+uGrUqKFDhw7pN7/5jb397bffVvXq1dWhQwd169ZNMTEx9lUnAADuNKwsoURubm46c6bo/qr69etr48aNhdpGjhxZ6Hc+lgMA3ClcbmXpvffeU/369eXj46PIyEht27bNsv/KlSvVpEkT+fj4qFmzZvryyy9L7PvCCy/IZrPxnWYAAMDOpcLSihUrNG7cOM2aNUu7du1SixYtFBMTo/T09GL7b9myRbGxsRo+fLh2796tHj16qEePHtq/f3+Rvp999pm+++47hYU56R5IAACgUnKpsPT222/r+eef19ChQ3X//fdr0aJFqlKlihYvXlxs//nz56tz586aOHGimjZtqjlz5qhVq1ZauHBhoX6nT5/W6NGjFR8fL09Pz4oYCgAAcBEuE5Zyc3O1c+dORUdH29vc3NwUHR2tpKSkYs9JSkoq1F+SYmJiCvUvKCjQc889p4kTJ+qBBx4on+IBAIDLcpkN3ufPn1d+fr5CQkIKtYeEhCglJaXYc1JTU4vtn5qaav/99ddfl4eHh15++eVS15KTk2O/s7UkZWVlWfY3pfxONpSMOQQAOIvLrCyVh507d2r+/PlasmSJbDZbqc+Li4tTQECA/REeHl5svxsf6WVnZzuk3rvZjTnkY1IAQEVzmZWloKAgubu7Ky0trVB7WlqaQkNDiz0nNDTUsv+3336r9PR0RURE2I/n5+dr/PjxmjdvXomXv0+ZMkXjxo2z/56VlVVsYHJ3d1dgYKB9A3qVKlVuKZThpxWl7OxspaenKzAwUO7u7s4uCQBwl3GZsOTl5aXWrVsrMTFRPXr0kPTTfqPExMQSv+k+KipKiYmJeuWVV+xtCQkJioqKkiQ999xzxe5peu655zR06NASa/H29pa3t3ep6r4RzEq6Yg+lExgYWGIoBgCgPLlMWJKkcePGafDgwWrTpo3atWunefPm6cqVK/ZgM2jQINWpU0dxcXGSpDFjxqhjx46aO3euunbtquXLl2vHjh368MMPJUk1a9ZUzZo1C72Gp6enQkND1bhxY4fUbLPZVLt2bdWqVUt5eXkOec67jaenJytKAACncamw1L9/f507d04zZ85UamqqWrZsqfXr19s3cZ84cUJubv/ZhtWhQwctW7ZM06dP19SpU9WoUSOtWbNGDz74YIXX7u7uzhs+AAAuyGa4zOi2ZWVlKSAgQJmZmfL393d2OQAAoBRK+/59V18NBwAAcDOEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAsuF5bee+891a9fXz4+PoqMjNS2bdss+69cuVJNmjSRj4+PmjVrpi+//NJ+LC8vT5MmTVKzZs1UtWpVhYWFadCgQTpz5kx5DwMAALgIlwpLK1as0Lhx4zRr1izt2rVLLVq0UExMjNLT04vtv2XLFsXGxmr48OHavXu3evTooR49emj//v2SpOzsbO3atUszZszQrl27tHr1ah06dEjPPPNMRQ4LAABUYjZjjHF2EaUVGRmptm3bauHChZKkgoIChYeHa/To0Zo8eXKR/v3799eVK1e0bt06e1v79u3VsmVLLVq0qNjX2L59u9q1a6fjx48rIiKiVHVlZWUpICBAmZmZ8vf3L8PIAABARSvt+7fLrCzl5uZq586dio6Otre5ubkpOjpaSUlJxZ6TlJRUqL8kxcTElNhfkjIzM2Wz2RQYGOiQugEAgGvzcHYBpXX+/Hnl5+crJCSkUHtISIhSUlKKPSc1NbXY/qmpqcX2v3btmiZNmqTY2FjLhJmTk6OcnBz771lZWaUdBgAAcDEus7JU3vLy8tSvXz8ZY/TBBx9Y9o2Li1NAQID9ER4eXkFVAgCAiuYyYSkoKEju7u5KS0sr1J6WlqbQ0NBizwkNDS1V/xtB6fjx40pISLjpvqMpU6YoMzPT/jh58mQZRgQAAFyBy4QlLy8vtW7dWomJifa2goICJSYmKioqqthzoqKiCvWXpISEhEL9bwSlw4cPa8OGDapZs+ZNa/H29pa/v3+hBwAAuDO5zJ4lSRo3bpwGDx6sNm3aqF27dpo3b56uXLmioUOHSpIGDRqkOnXqKC4uTpI0ZswYdezYUXPnzlXXrl21fPly7dixQx9++KGkn4JSnz59tGvXLq1bt075+fn2/Uw1atSQl5eXcwYKAAAqDZcKS/3799e5c+c0c+ZMpaamqmXLllq/fr19E/eJEyfk5vafxbIOHTpo2bJlmj59uqZOnapGjRppzZo1evDBByVJp0+f1tq1ayVJLVu2LPRamzZt0q9//esKGRcAAKi8XOo+S5UV91kCAMD13HH3WQIAAHAGwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAIAFwhIAAICFMoWlTZs2OboOAACASqlMYalz585q2LCh/vjHP+rkyZOOrgkAAKDSKFNYOn36tEaNGqVVq1bpnnvuUUxMjD755BPl5uY6uj4AAACnKlNYCgoK0tixY7Vnzx5t3bpV9913n1566SWFhYXp5Zdf1t69ex1dJwAAgFPc9gbvVq1aacqUKRo1apQuX76sxYsXq3Xr1nrkkUd04MABR9QIAADgNGUOS3l5eVq1apW6dOmievXq6R//+IcWLlyotLQ0/fDDD6pXr5769u3ryFoBAAAqnM0YY271pNGjR+vjjz+WMUbPPfecfvvb3+rBBx8s1Cc1NVVhYWEqKChwWLGVVVZWlgICApSZmSl/f39nlwMAAEqhtO/fHmV58oMHD+rdd99Vr1695O3tXWyfoKAgbjEAAABcXplWllAYK0sAALie0r5/l2nPUlxcnBYvXlykffHixXr99dfL8pQAAACVUpnC0l/+8hc1adKkSPsDDzygRYsW3XZRAAAAlUWZwlJqaqpq165dpD04OFhnz5697aIAAAAqizKFpfDwcG3evLlI++bNmxUWFnbbRQEAAFQWZboa7vnnn9crr7yivLw8Pf7445KkxMRE/f73v9f48eMdWiAAAIAzlSksTZw4URcuXNBLL71k/z44Hx8fTZo0SVOmTHFogQAAAM50W7cOuHz5spKTk+Xr66tGjRqVeM+lOx23DgAAwPWU600pb/Dz81Pbtm1v5ykAAAAqtTKHpR07duiTTz7RiRMn7B/F3bB69erbLgwAAKAyKNPVcMuXL1eHDh2UnJyszz77THl5eTpw4IA2btyogIAAR9cIAADgNGUKS3/+85/1zjvv6PPPP5eXl5fmz5+vlJQU9evXTxEREY6uEQAAwGnKFJaOHDmirl27SpK8vLx05coV2Ww2jR07Vh9++KFDCwQAAHCmMoWl6tWr69KlS5KkOnXqaP/+/ZKkjIwMZWdnO646AAAAJyvTBu9HH31UCQkJatasmfr27asxY8Zo48aNSkhIUKdOnRxdIwAAgNOUKSwtXLhQ165dkyRNmzZNnp6e2rJli3r37q3p06c7tEAAAABnuuWwdP36da1bt04xMTGSJDc3N02ePNnhhQEAAFQGt7xnycPDQy+88IJ9Zamivffee6pfv758fHwUGRmpbdu2WfZfuXKlmjRpIh8fHzVr1kxffvlloePGGM2cOVO1a9eWr6+voqOjdfjw4fIcAgAAcCFl2uDdrl077dmzx8Gl3NyKFSs0btw4zZo1S7t27VKLFi0UExOj9PT0Yvtv2bJFsbGxGj58uHbv3q0ePXqoR48e9g3pkvTGG29owYIFWrRokbZu3aqqVasqJibGaWEQAABULmX6brhPPvlEU6ZM0dixY9W6dWtVrVq10PHmzZs7rMCfi4yMVNu2bbVw4UJJUkFBgcLDwzV69OhiPwrs37+/rly5onXr1tnb2rdvr5YtW2rRokUyxigsLEzjx4/XhAkTJEmZmZkKCQnRkiVL9Oyzz5aqLr4bDgAA11Ou3w13I0S8/PLL9jabzSZjjGw2m/Lz88vytJZyc3O1c+dOTZkyxd7m5uam6OhoJSUlFXtOUlKSxo0bV6gtJiZGa9askSQdPXpUqampio6Oth8PCAhQZGSkkpKSSgxLOTk5ysnJsf+elZVV1mEBAIBKrkxh6ejRo46u46bOnz+v/Px8hYSEFGoPCQlRSkpKseekpqYW2z81NdV+/EZbSX2KExcXp1dfffWWxwAAAFxPmcJSvXr1HF2HS5kyZUqhFausrCyFh4c7sSIAAFBeyhSWli5danl80KBBZSrGSlBQkNzd3ZWWllaoPS0tTaGhocWeExoaatn/xr9paWmqXbt2oT4tW7YssRZvb295e3uXZRgAAMDFlCksjRkzptDveXl5ys7OlpeXl6pUqVIuYcnLy0utW7dWYmKievToIemnDd6JiYkaNWpUsedERUUpMTFRr7zyir0tISFBUVFRkqQGDRooNDRUiYmJ9nCUlZWlrVu36sUXX3T4GAAAgOspU1j68ccfi7QdPnxYL774oiZOnHjbRZVk3LhxGjx4sNq0aaN27dpp3rx5unLlioYOHSrppxWtOnXqKC4uTtJPoa5jx46aO3euunbtquXLl2vHjh32L/u12Wx65ZVX9Mc//lGNGjVSgwYNNGPGDIWFhdkDGQAAuLuVKSwVp1GjRnrttdc0cODAEjdc367+/fvr3LlzmjlzplJTU9WyZUutX7/evkH7xIkTcnP7z62jOnTooGXLlmn69OmaOnWqGjVqpDVr1ujBBx+09/n973+vK1euaMSIEcrIyNDDDz+s9evXy8fHp1zGAAAAXEuZ7rNUkj179ujRRx+96y6l5z5LAAC4nnK9z9LatWsL/W6M0dmzZ7Vw4UI99NBDZXlKAACASqlMYemX+3lsNpuCg4P1+OOPa+7cuY6oCwAAoFIoU1gqKChwdB0AAACVUpm+SBcAAOBuUaaw1Lt3b73++utF2t944w317dv3tosCAACoLMoUlr755ht16dKlSPtTTz2lb7755raLAgAAqCzKFJYuX74sLy+vIu2enp533W0DAADAna1MYalZs2ZasWJFkfbly5fr/vvvv+2iAAAAKosyXQ03Y8YM9erVS0eOHNHjjz8uSUpMTNTHH3+slStXOrRAAAAAZypTWOrWrZvWrFmjP//5z1q1apV8fX3VvHlzbdiwQR07dnR0jQAAAE7j0K87uVvxdScAALie0r5/l2nP0vbt27V169Yi7Vu3btWOHTvK8pQAAACVUpnC0siRI3Xy5Mki7adPn9bIkSNvuygAAIDKokxh6eDBg2rVqlWR9l/96lc6ePDgbRcFAABQWZQpLHl7eystLa1I+9mzZ+XhUaY94wAAAJVSmcLSk08+qSlTpigzM9PelpGRoalTp+qJJ55wWHEAAADOVqZloLfeekuPPvqo6tWrp1/96leSpD179igkJER/+9vfHFogAACAM5UpLNWpU0fff/+94uPjtXfvXvn6+mro0KGKjY2Vp6eno2sEAABwmjJvMKpataoefvhhRUREKDc3V5L097//XZL0zDPPOKY6AAAAJytTWPr3v/+tnj17at++fbLZbDLGyGaz2Y/n5+c7rEAAAABnKtMG7zFjxqhBgwZKT09XlSpVtH//fn399ddq06aNvvrqKweXCAAA4DxlWllKSkrSxo0bFRQUJDc3N7m7u+vhhx9WXFycXn75Ze3evdvRdQIAADhFmVaW8vPzVa1aNUlSUFCQzpw5I0mqV6+eDh065LjqAAAAnKxMK0sPPvig9u7dqwYNGigyMlJvvPGGvLy89OGHH+qee+5xdI0AAABOU6awNH36dF25ckWSNHv2bD399NN65JFHVLNmTa1YscKhBQIAADiTzRhjHPFEFy9eVPXq1QtdFXe3yMrKUkBAgDIzM+Xv7+/scgAAQCmU9v3bYV/kVqNGDUc9FQAAQKVRpg3eAAAAdwvCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAWXCUsXL17UgAED5O/vr8DAQA0fPlyXL1+2POfatWsaOXKkatasKT8/P/Xu3VtpaWn243v37lVsbKzCw8Pl6+urpk2bav78+eU9FAAA4EJcJiwNGDBABw4cUEJCgtatW6dvvvlGI0aMsDxn7Nix+vzzz7Vy5Up9/fXXOnPmjHr16mU/vnPnTtWqVUsfffSRDhw4oGnTpmnKlClauHBheQ8HAAC4CJsxxji7iJtJTk7W/fffr+3bt6tNmzaSpPXr16tLly46deqUwsLCipyTmZmp4OBgLVu2TH369JEkpaSkqGnTpkpKSlL79u2Lfa2RI0cqOTlZGzduLHV9WVlZCggIUGZmpvz9/cswQgAAUNFK+/7tEitLSUlJCgwMtAclSYqOjpabm5u2bt1a7Dk7d+5UXl6eoqOj7W1NmjRRRESEkpKSSnytzMxM1ahRw7KenJwcZWVlFXoAAIA7k0uEpdTUVNWqVatQm4eHh2rUqKHU1NQSz/Hy8lJgYGCh9pCQkBLP2bJli1asWHHTj/fi4uIUEBBgf4SHh5d+MAAAwKU4NSxNnjxZNpvN8pGSklIhtezfv1/du3fXrFmz9OSTT1r2nTJlijIzM+2PkydPVkiNAACg4nk488XHjx+vIUOGWPa55557FBoaqvT09ELt169f18WLFxUaGlrseaGhocrNzVVGRkah1aW0tLQi5xw8eFCdOnXSiBEjNH369JvW7e3tLW9v75v2AwAArs+pYSk4OFjBwcE37RcVFaWMjAzt3LlTrVu3liRt3LhRBQUFioyMLPac1q1by9PTU4mJierdu7ck6dChQzpx4oSioqLs/Q4cOKDHH39cgwcP1p/+9CcHjAoAANxJXOJqOEl66qmnlJaWpkWLFikvL09Dhw5VmzZttGzZMknS6dOn1alTJy1dulTt2rWTJL344ov68ssvtWTJEvn7+2v06NGSftqbJP300dvjjz+umJgYvfnmm/bXcnd3L1WIu4Gr4QAAcD2lff926srSrYiPj9eoUaPUqVMnubm5qXfv3lqwYIH9eF5eng4dOqTs7Gx72zvvvGPvm5OTo5iYGL3//vv246tWrdK5c+f00Ucf6aOPPrK316tXT8eOHauQcQEAgMrNZVaWKjNWlgAAcD131H2WAAAAnIWwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYMFlwtLFixc1YMAA+fv7KzAwUMOHD9fly5ctz7l27ZpGjhypmjVrys/PT71791ZaWlqxfS9cuKC6devKZrMpIyOjHEYAAABckcuEpQEDBujAgQNKSEjQunXr9M0332jEiBGW54wdO1aff/65Vq5cqa+//lpnzpxRr169iu07fPhwNW/evDxKBwAALsxmjDHOLuJmkpOTdf/992v79u1q06aNJGn9+vXq0qWLTp06pbCwsCLnZGZmKjg4WMuWLVOfPn0kSSkpKWratKmSkpLUvn17e98PPvhAK1as0MyZM9WpUyf9+OOPCgwMLHV9WVlZCggIUGZmpvz9/W9vsAAAoEKU9v3bJVaWkpKSFBgYaA9KkhQdHS03Nzdt3bq12HN27typvLw8RUdH29uaNGmiiIgIJSUl2dsOHjyo2bNna+nSpXJzK9105OTkKCsrq9ADAADcmVwiLKWmpqpWrVqF2jw8PFSjRg2lpqaWeI6Xl1eRFaKQkBD7OTk5OYqNjdWbb76piIiIUtcTFxengIAA+yM8PPzWBgQAAFyGU8PS5MmTZbPZLB8pKSnl9vpTpkxR06ZNNXDgwFs+LzMz0/44efJkOVUIAACczcOZLz5+/HgNGTLEss8999yj0NBQpaenF2q/fv26Ll68qNDQ0GLPCw0NVW5urjIyMgqtLqWlpdnP2bhxo/bt26dVq1ZJkm5s3woKCtK0adP06quvFvvc3t7e8vb2Ls0QAQCAi3NqWAoODlZwcPBN+0VFRSkjI0M7d+5U69atJf0UdAoKChQZGVnsOa1bt5anp6cSExPVu3dvSdKhQ4d04sQJRUVFSZI+/fRTXb161X7O9u3bNWzYMH377bdq2LDh7Q4PAADcAZwalkqradOm6ty5s55//nktWrRIeXl5GjVqlJ599ln7lXCnT59Wp06dtHTpUrVr104BAQEaPny4xo0bpxo1asjf31+jR49WVFSU/Uq4Xwai8+fP21/vVq6GAwAAdy6XCEuSFB8fr1GjRqlTp05yc3NT7969tWDBAvvxvLw8HTp0SNnZ2fa2d955x943JydHMTExev/9951RPgAAcFEucZ+lyo77LAEA4HruqPssAQAAOAthCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwAJhCQAAwIKHswu4ExhjJElZWVlOrgQAAJTWjfftG+/jJSEsOcClS5ckSeHh4U6uBAAA3KpLly4pICCgxOM2c7M4hZsqKCjQmTNnVK1aNdlsNmeX41RZWVkKDw/XyZMn5e/v7+xy7ljMc8VhrisG81wxmOfCjDG6dOmSwsLC5OZW8s4kVpYcwM3NTXXr1nV2GZWKv78//0esAMxzxWGuKwbzXDGY5/+wWlG6gQ3eAAAAFghLAAAAFghLcChvb2/NmjVL3t7ezi7ljsY8VxzmumIwzxWDeS4bNngDAABYYGUJAADAAmEJAADAAmEJAADAAmEJAADAAmEJt+zixYsaMGCA/P39FRgYqOHDh+vy5cuW51y7dk0jR45UzZo15efnp969eystLa3YvhcuXFDdunVls9mUkZFRDiNwDeUxz3v37lVsbKzCw8Pl6+urpk2bav78+eU9lErlvffeU/369eXj46PIyEht27bNsv/KlSvVpEkT+fj4qFmzZvryyy8LHTfGaObMmapdu7Z8fX0VHR2tw4cPl+cQXIIj5zkvL0+TJk1Ss2bNVLVqVYWFhWnQoEE6c+ZMeQ+j0nP03/PPvfDCC7LZbJo3b56Dq3ZBBrhFnTt3Ni1atDDfffed+fbbb829995rYmNjLc954YUXTHh4uElMTDQ7duww7du3Nx06dCi2b/fu3c1TTz1lJJkff/yxHEbgGspjnv/617+al19+2Xz11VfmyJEj5m9/+5vx9fU17777bnkPp1JYvny58fLyMosXLzYHDhwwzz//vAkMDDRpaWnF9t+8ebNxd3c3b7zxhjl48KCZPn268fT0NPv27bP3ee2110xAQIBZs2aN2bt3r3nmmWdMgwYNzNWrVytqWJWOo+c5IyPDREdHmxUrVpiUlBSTlJRk2rVrZ1q3bl2Rw6p0yuPv+YbVq1ebFi1amLCwMPPOO++U80gqP8ISbsnBgweNJLN9+3Z729///ndjs9nM6dOniz0nIyPDeHp6mpUrV9rbkpOTjSSTlJRUqO/7779vOnbsaBITE+/qsFTe8/xzL730knnsscccV3wl1q5dOzNy5Ej77/n5+SYsLMzExcUV279fv36ma9euhdoiIyPN7373O2OMMQUFBSY0NNS8+eab9uMZGRnG29vbfPzxx+UwAtfg6HkuzrZt24wkc/z4cccU7YLKa55PnTpl6tSpY/bv32/q1atHWDLG8DEcbklSUpICAwPVpk0be1t0dLTc3Ny0devWYs/ZuXOn8vLyFB0dbW9r0qSJIiIilJSUZG87ePCgZs+eraVLl1p+oeHdoDzn+ZcyMzNVo0YNxxVfSeXm5mrnzp2F5sfNzU3R0dElzk9SUlKh/pIUExNj73/06FGlpqYW6hMQEKDIyEjLOb+Tlcc8FyczM1M2m02BgYEOqdvVlNc8FxQU6LnnntPEiRP1wAMPlE/xLujufkfCLUtNTVWtWrUKtXl4eKhGjRpKTU0t8RwvL68i/1ELCQmxn5OTk6PY2Fi9+eabioiIKJfaXUl5zfMvbdmyRStWrNCIESMcUndldv78eeXn5yskJKRQu9X8pKamWva/8e+tPOedrjzm+ZeuXbumSZMmKTY29q79MtjymufXX39dHh4eevnllx1ftAsjLEGSNHnyZNlsNstHSkpKub3+lClT1LRpUw0cOLDcXqMycPY8/9z+/fvVvXt3zZo1S08++WSFvCZwu/Ly8tSvXz8ZY/TBBx84u5w7ys6dOzV//nwtWbJENpvN2eVUKh7OLgCVw/jx4zVkyBDLPvfcc49CQ0OVnp5eqP369eu6ePGiQkNDiz0vNDRUubm5ysjIKLTqkZaWZj9n48aN2rdvn1atWiXppyuMJCkoKEjTpk3Tq6++WsaRVS7OnucbDh48qE6dOmnEiBGaPn16mcbiaoKCguTu7l7kKszi5ueG0NBQy/43/k1LS1Pt2rUL9WnZsqUDq3cd5THPN9wISsePH9fGjRvv2lUlqXzm+dtvv1V6enqh1f38/HyNHz9e8+bN07Fjxxw7CFfi7E1TcC03Nh7v2LHD3vaPf/yjVBuPV61aZW9LSUkptPH4hx9+MPv27bM/Fi9ebCSZLVu2lHhlx52svObZGGP2799vatWqZSZOnFh+A6ik2rVrZ0aNGmX/PT8/39SpU8dyQ+zTTz9dqC0qKqrIBu+33nrLfjwzM5MN3g6eZ2OMyc3NNT169DAPPPCASU9PL5/CXYyj5/n8+fOF/ju8b98+ExYWZiZNmmRSUlLKbyAugLCEW9a5c2fzq1/9ymzdutX885//NI0aNSp0SfupU6dM48aNzdatW+1tL7zwgomIiDAbN240O3bsMFFRUSYqKqrE19i0adNdfTWcMeUzz/v27TPBwcFm4MCB5uzZs/bH3fLms3z5cuPt7W2WLFliDh48aEaMGGECAwNNamqqMcaY5557zkyePNnef/PmzcbDw8O89dZbJjk52cyaNavYWwcEBgaa//3f/zXff/+96d69O7cOcPA85+bmmmeeecbUrVvX7Nmzp9Dfbk5OjlPGWBmUx9/zL3E13E8IS7hlFy5cMLGxscbPz8/4+/uboUOHmkuXLtmPHz161EgymzZtsrddvXrVvPTSS6Z69eqmSpUqpmfPnubs2bMlvgZhqXzmedasWUZSkUe9evUqcGTO9e6775qIiAjj5eVl2rVrZ7777jv7sY4dO5rBgwcX6v/JJ5+Y++67z3h5eZkHHnjAfPHFF4WOFxQUmBkzZpiQkBDj7e1tOnXqZA4dOlQRQ6nUHDnPN/7Wi3v8/O//buTov+dfIiz9xGbM/98cAgAAgCK4Gg4AAMACYQkAAMACYQkAAMACYQkAAMACYQkAAMACYQkAAMACYQkAAMACYQkAHOyrr76SzWZTRkaGs0sB4ACEJQAAAAuEJQAAAAuEJQB3nIKCAsXFxalBgwby9fVVixYttGrVKkn/+Yjsiy++UPPmzeXj46P27dtr//79hZ7j008/1QMPPCBvb2/Vr19fc+fOLXQ8JydHkyZNUnh4uLy9vXXvvffqr3/9a6E+O3fuVJs2bVSlShV16NBBhw4dKt+BAygXhCUAd5y4uDgtXbpUixYt0oEDBzR27FgNHDhQX3/9tb3PxIkTNXfuXG3fvl3BwcHq1q2b8vLyJP0Ucvr166dnn31W+/bt0x/+8AfNmDFDS5YssZ8/aNAgffzxx1qwYIGSk5P1l7/8RX5+foXqmDZtmubOnasdO3bIw8NDw4YNq5DxA3AsvkgXwB0lJydHNWrU0IYNGxQVFWVv/+1vf6vs7GyNGDFCjz32mJYvX67+/ftLki5evKi6detqyZIl6tevnwYMGKBz587p//7v/+zn//73v9cXX3yhAwcO6F//+pcaN26shIQERUdHF6nhq6++0mOPPaYNGzaoU6dOkqQvv/xSXbt21dWrV+Xj41POswDAkVhZAnBH+eGHH5Sdna0nnnhCfn5+9sfSpUt15MgRe7+fB6kaNWqocePGSk5OliQlJyfroYceKvS8Dz30kA4fPqz8/Hzt2bNH7u7u6tixo2UtzZs3t/9cu3ZtSVJ6evptjxFAxfJwdgEA4EiXL1+WJH3xxReqU6dOoWPe3t6FAlNZ+fr6lqqfp6en/WebzSbpp/1UAFwLK0sA7ij333+/vL29deLECd17772FHuHh4fZ+3333nf3nH3/8Uf/617/UtGlTSVLTpk21efPmQs+7efNm3XfffXJ3d1ezZs1UUFBQaA8UgDsXK0sA7ijVqlXThAkTNHbsWBUUFOjhhx9WZmamNm/eLH9/f9WrV0+SNHv2bNWsWVMhISGaNm2agoKC1KNHD0nS+PHj1bZtW82ZM0f9+/dXUlKSFi5cqPfff1+SVL9+fQ0ePFjDhg3TggUL1KJFCx0/flzp6enq16+fs4YOoJwQlgDccebMmaPg4GDFxcXp3//+twIDA9WqVStNnTrV/jHYa6+9pjFjxujw4cNq2bKlPv/8c3l5eUmSWrVqpU8++UQzZ87UnDlzVLt2bc2ePVtDhgyxv8YHH3ygqVOn6qWXXtKFCxcUERGhqVOnOmO4AMoZV8MBuKvcuFLtxx9/VGBgoLPLAeAC2LMEAABggbAEAABggY/hAAAALLCyBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYIGwBAAAYOH/AeX2OuILvXjIAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# draw Accuracies\n",
    "if is_graphic:\n",
    "    plt.plot(list(range(1, len(arr_train_acc) + 1)), arr_train_acc, label=\"train\")\n",
    "    plt.plot(list(range(1, len(arr_val_acc) + 1)), arr_val_acc, label=\"val\")\n",
    "\n",
    "    plt.title(\"Accuracy\")\n",
    "    plt.xlabel(\"epoch\")\n",
    "    plt.ylabel(\"accuracy\")\n",
    "    plt.legend(loc=\"upper left\")\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "0133ff45",
   "metadata": {},
   "source": [
    "# Eval model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e095b271",
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils_french.split_all_docs import split_all_docs\n",
    "from rouge_score import rouge_scorer\n",
    "from utils_french.threshold_probs import threshold_probs_by_prop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "273c6eb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = RNN_RNN(device=device, vocab_size=vocab_size, word_embed=embmgr.getEmbeddings(), dim_word_embed=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "4fc41994",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_epoch = np.argmin(np.array(arr_val_loss)) + 1\n",
    "model.load(checkpoints_folder + \"/\" + model_name + \"-\" + str(best_epoch) + \".pt\")\n",
    "model.to(device)\n",
    "model.eval()\n",
    "best_epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "16b5ecbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_set = df_test\n",
    "test_set[\"doc_splited\"] = split_all_docs(test_set[\"flat_contents\"], False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "bb8ab07a",
   "metadata": {},
   "outputs": [],
   "source": [
    "scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)\n",
    "criterion = nn.BCELoss()\n",
    "mae_fn = nn.L1Loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "67659b8a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Eval model: 100%|██████████| 100/100 [00:02<00:00, 35.75batch/s, accuracy_ner=0.885, loss=0.348, loss_ner=0.348]\n"
     ]
    }
   ],
   "source": [
    "arr_rouge1 = []\n",
    "arr_rouge2 = []\n",
    "arr_rougeL = []\n",
    "arr_loss = []\n",
    "arr_loss_sum = []\n",
    "arr_loss_ner = []\n",
    "accuracy_sum = []\n",
    "accuracy_ner = []\n",
    "mae = []\n",
    "\n",
    "idx = 0\n",
    "with tqdm(test_iter, unit=\"batch\", total=len(test_iter)) as tepoch:\n",
    "    for batch in tepoch:\n",
    "        tepoch.set_description(\"Eval model\")\n",
    "        features = []\n",
    "        doc_lens = []\n",
    "\n",
    "        for j in range(1):\n",
    "            doc_lens.append(len(batch[j][\"docs\"]))\n",
    "            features = features + batch[j][\"docs\"]\n",
    "                \n",
    "        features = torch.tensor(features, dtype=torch.long).to(device)\n",
    "\n",
    "        #targets = [torch.tensor(batch[j][\"labels\"], dtype=torch.float) for j in range(1)]\n",
    "        #targets = torch.cat(targets)\n",
    "        #targets = targets.to(device)\n",
    "\n",
    "        targets_ner = [torch.tensor(batch[j][\"labels_ner\"][k], dtype=torch.float) for j in range(1) for k in range(len(batch[j][\"labels_ner\"]))]\n",
    "        targets_ner = torch.cat(targets_ner)\n",
    "        targets_ner = targets_ner.to(device)\n",
    "\n",
    "        y_pred, y_ner_pred = model(features, doc_lens)\n",
    "        \n",
    "        #loss_sum = loss_fn(y_pred, targets)\n",
    "        loss_ner = loss_fn(y_ner_pred, targets_ner)\n",
    "\n",
    "        loss = loss_ner # loss_sum_coef * loss_sum + loss_ner_coef * loss_ner\n",
    "\n",
    "        arr_loss.append(loss.item())\n",
    "        #arr_loss_sum.append(loss_sum.item())\n",
    "        arr_loss_ner.append(loss_ner.item())\n",
    "        \n",
    "        accuracy_ner.append(torch.sum(((y_ner_pred > 0.5).float() == targets_ner).float()).item() / targets_ner.shape[0])\n",
    "        \n",
    "        tepoch.set_postfix(loss=np.mean(arr_loss), loss_ner=np.mean(arr_loss_ner), accuracy_ner=np.mean(accuracy_ner))\n",
    "        idx += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "5c1fd15f",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_metrics = {}\n",
    "test_metrics[\"accuracy_ner\"] = np.mean(accuracy_ner)\n",
    "\n",
    "# Save to file in JSON format\n",
    "\n",
    "with open(checkpoints_folder + \"/test_metrics.json\", 'w') as fp:\n",
    "    json.dump(test_metrics, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e4f4796c",
   "metadata": {},
   "outputs": [
    {
     "ename": "SystemExit",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "An exception has occurred, use %tb to see the full traceback.\n",
      "\u001b[0;31mSystemExit\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/raoufdine/anaconda3/lib/python3.9/site-packages/IPython/core/interactiveshell.py:3465: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
      "  warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "# Does not excute this cell if you want to execute the following cells.\n",
    "import sys\n",
    "sys.exit()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "4f810f0bde7d7b5dc90f9e6832822547e13e234c53731a97fc91254323942f40"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}