oe-imagenet-transformer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os \n",
    "import sys\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"MIG-GPU-8ab9a0c8-909c-3f13-97e6-7376d6d4a029/0/0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytorch_ood.model import WideResNet\n",
    "from pytorch_ood.detector import EnergyBased, MaxSoftmax\n",
    "from pytorch_ood.utils import OODMetrics, ToUnknown\n",
    "from pytorch_ood.dataset.img import Textures\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "from torchvision.transforms import Compose, ToPILImage, Normalize, ToTensor, Resize, ToPILImage\n",
    "import torch \n",
    "from torchvision import transforms \n",
    "from pytorch_ood.utils import ToRGB\n",
    "from torchvision.datasets import ImageNet\n",
    "from torchvision.models import resnet18, resnet101\n",
    "from torch.utils.data import DataLoader \n",
    "import tqdm.notebook as tqdm \n",
    "import torchmetrics \n",
    "import pandas as pd \n",
    "from matplotlib import rcParams\n",
    "rcParams['pgf.texsystem'] = \"pdflatex\"\n",
    "\n",
    "import seaborn as sb \n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "sb.set_style(\"whitegrid\")\n",
    "sb.set_context(\"paper\")\n",
    "\n",
    "\n",
    "from torchmetrics import Accuracy\n",
    "from torchvision.datasets import ImageFolder\n",
    "from pytorch_ood.dataset.img import ImageNetR, ImageNetO, ImageNetA\n",
    "from pytorch_ood.utils import OODMetrics\n",
    "from pytorch_ood.detector import MaxSoftmax\n",
    "from pytorch_ood.model import VisionTransformer\n",
    "\n",
    "\n",
    "preprocess = transforms.Compose([\n",
    "    ToRGB(),\n",
    "    transforms.Resize((224, 224)),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n",
    "])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from torch.optim import Adam, SGD\n",
    "from pytorch_ood.loss import OutlierExposureLoss\n",
    "from itertools import cycle\n",
    "\n",
    "\n",
    "def train(outlier_set=10, iters=1000, model=None):\n",
    "    dataset_out_train = ImageFolder(root=f\"/data_slow/kirchheim/gan_oe/imagenet/{outlier_set}/\", target_transform=ToUnknown(), transform=preprocess)\n",
    "    train_out_loader = DataLoader(dataset_out_train, batch_size=64, num_workers=20, shuffle=True)\n",
    "\n",
    "    dataset_in_train = ImageNet(split=\"train\", root=\"/data_fast/kirchheim/datasets/imagenet-2012/\", transform=preprocess)\n",
    "    imagenet_in_loader = DataLoader(dataset_in_train, batch_size=64, num_workers=20, shuffle=True)\n",
    "\n",
    "    # in test set \n",
    "    dataset_in_test = ImageNet(split=\"val\", root=\"/data_fast/kirchheim/datasets/imagenet-2012/\", transform=preprocess)\n",
    "    test_in_loader = DataLoader(dataset_in_test, batch_size=256, num_workers=20)\n",
    "\n",
    "    # ood test sets \n",
    "    imagenet_r = ImageNetR(root=\"/data_fast/kirchheim/datasets/\", \n",
    "                           download=True, transform=preprocess, target_transform=ToUnknown())\n",
    "    imagenet_r_loader = DataLoader(imagenet_r + dataset_in_test, batch_size=128, num_workers=12)\n",
    "    \n",
    "    imagenet_o = ImageNetO(root=\"/data_fast/kirchheim/datasets/\", \n",
    "                           download=True, transform=preprocess, target_transform=ToUnknown())\n",
    "    imagenet_o_loader = DataLoader(imagenet_o + dataset_in_test, batch_size=128, num_workers=12)\n",
    "\n",
    "    imagenet_a = ImageNetA(root=\"/data_fast/kirchheim/datasets/\", \n",
    "                           download=True, transform=preprocess, target_transform=ToUnknown())\n",
    "    imagenet_a_loader = DataLoader(imagenet_a + dataset_in_test, batch_size=128, num_workers=12)\n",
    "    \n",
    "    if model is None:\n",
    "        model = VisionTransformer(image_size=(224, 224), pretrained=\"b16-im21k-224\", num_classes=1000) # resnet101(pretrained=True)\n",
    "        model._from_pretrained(\"b16-im21k-224\")\n",
    "        \n",
    "    model = model.cuda()\n",
    "\n",
    "    opti = SGD(model.parameters(), lr=0.001)\n",
    "\n",
    "    criterion = OutlierExposureLoss(alpha=0.5)\n",
    "\n",
    "    mav = 0.0\n",
    "    mavs = []\n",
    "    perf_metrics = []\n",
    "\n",
    "    current_auroc = None \n",
    "\n",
    "    it_train_in = iter(imagenet_in_loader)\n",
    "    it_train_out = iter(train_out_loader)\n",
    "\n",
    "    criterion = OutlierExposureLoss()\n",
    "    bar = tqdm.tqdm(range(iters))\n",
    "\n",
    "    for i in bar:\n",
    "        try:\n",
    "            x1, y1 = next(it_train_in)\n",
    "            x2, y2 = next(it_train_out)\n",
    "        except:\n",
    "            it_train_in = iter(imagenet_in_loader)\n",
    "            it_train_out = iter(train_out_loader)\n",
    "\n",
    "        x = torch.cat([x1, x2])\n",
    "        y = torch.cat([y1, y2])\n",
    "\n",
    "        x = x.cuda()\n",
    "        y = y.cuda()\n",
    "\n",
    "        y_hat = model(x)\n",
    "        loss = criterion(y_hat, y)\n",
    "        opti.zero_grad()\n",
    "        loss.backward()\n",
    "        opti.step()\n",
    "\n",
    "        mav = 0.2 * loss.item() + 0.8 * mav\n",
    "        mavs.append(mav)\n",
    "\n",
    "        bar.set_postfix({\"loss\": mav, \"AUROC\": current_auroc})\n",
    "    \n",
    "    print(f\"Evaluating... \")\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        softmax = MaxSoftmax(model)\n",
    "        energy = EnergyBased(model)\n",
    "        \n",
    "        ###\n",
    "        metrics = OODMetrics()\n",
    "        metrics_energy = OODMetrics()\n",
    "        \n",
    "        for x, y in imagenet_a_loader:\n",
    "            logits = model(x.cuda())\n",
    "            metrics.update(softmax.score(logits), y)\n",
    "            metrics_energy.update(energy.score(logits), y)\n",
    "\n",
    "        m = metrics.compute()\n",
    "        m.update({\n",
    "            \"Dataset\": \"ImageNetA\",\n",
    "            \"Iteration\": i,\n",
    "            \"Method\": \"Softmax\"\n",
    "        })\n",
    "        perf_metrics.append(m)\n",
    "        \n",
    "        m = metrics_energy.compute()\n",
    "        m.update({\n",
    "            \"Dataset\": \"ImageNetA\",\n",
    "            \"Iteration\": i,\n",
    "            \"Method\": \"Energy\"\n",
    "        })\n",
    "        perf_metrics.append(m)\n",
    "        \n",
    "\n",
    "        ###\n",
    "        metrics = OODMetrics()\n",
    "        metrics_energy = OODMetrics()\n",
    "        \n",
    "        for x, y in imagenet_o_loader:\n",
    "            logits = model(x.cuda())\n",
    "            metrics.update(softmax.score(logits), y)\n",
    "            metrics_energy.update(energy.score(logits), y)\n",
    "\n",
    "        m = metrics.compute()\n",
    "        m.update({\n",
    "            \"Dataset\": \"ImageNetO\",\n",
    "            \"Iteration\": i,\n",
    "            \"Method\": \"Softmax\"\n",
    "        })\n",
    "        perf_metrics.append(m)\n",
    "        \n",
    "        m = metrics_energy.compute()\n",
    "        m.update({\n",
    "            \"Dataset\": \"ImageNetO\",\n",
    "            \"Iteration\": i,\n",
    "            \"Method\": \"Energy\"\n",
    "        })\n",
    "        perf_metrics.append(m)\n",
    "\n",
    "        ###\n",
    "        metrics = OODMetrics()\n",
    "        metrics_energy = OODMetrics()\n",
    "        \n",
    "        for x, y in imagenet_r_loader:\n",
    "            logits = model(x.cuda())\n",
    "            metrics.update(softmax.score(logits), y)\n",
    "            metrics_energy.update(energy.score(logits), y)\n",
    "\n",
    "        m = metrics.compute()\n",
    "        m.update({\n",
    "            \"Dataset\": \"ImageNetR\",\n",
    "            \"Iteration\": i,\n",
    "            \"Method\": \"Softmax\"\n",
    "        })\n",
    "        perf_metrics.append(m)\n",
    "        \n",
    "        m = metrics_energy.compute()\n",
    "        m.update({\n",
    "            \"Dataset\": \"ImageNetR\",\n",
    "            \"Iteration\": i,\n",
    "            \"Method\": \"Energy\"\n",
    "        })\n",
    "        perf_metrics.append(m)\n",
    "\n",
    "        print(f\"AUROC {i} -> {m['AUROC']:.3%}\")\n",
    "        current_auroc = m['AUROC']\n",
    "\n",
    "    model.train()\n",
    "\n",
    "    return perf_metrics\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cdb42947bda9447ca61fd322088d5d11",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kirchheim/anaconda3/envs/myenv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: Metric `AUROC` will save all targets and predictions in buffer. For large datasets this may lead to large memory footprint.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/home/kirchheim/anaconda3/envs/myenv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: Metric `PrecisionRecallCurve` will save all targets and predictions in buffer. For large datasets this may lead to large memory footprint.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/home/kirchheim/anaconda3/envs/myenv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: `pos_label` automatically set 1.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/home/kirchheim/anaconda3/envs/myenv/lib/python3.8/site-packages/torchmetrics/functional/classification/precision_recall_curve.py:156: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  thresholds = tensor(reversed(thresholds[sl]))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUROC 999 -> 89.106%\n"
     ]
    }
   ],
   "source": [
    "metrics = train(2.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrrrrr}\n",
      "\\toprule\n",
      "{} &  AUROC &  AUPR-IN &  AUPR-OUT &  ACC95TPR &  FPR95TPR &  Iteration \\\\\n",
      "Method  &        &          &           &           &           &            \\\\\n",
      "\\midrule\n",
      "Energy  &  90.35 &    62.10 &     96.77 &     67.99 &     39.16 &   99900.00 \\\\\n",
      "Softmax &  85.68 &    53.60 &     95.35 &     54.24 &     55.29 &   99900.00 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(metrics)\n",
    "# df = df[df[\"Variance\"] == 2.0]\n",
    "# df[\"Method\"] = \"MPS\"\n",
    "print((df.groupby(\"Method\").mean() * 100).to_latex(float_format=\"%.2f\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d88b03f6401d4cfda0b98c9c1f5e1cb7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 85.724%\n",
      "[{'AUROC': 0.8549557328224182, 'AUPR-IN': 0.5011280179023743, 'AUPR-OUT': 0.9733227491378784, 'ACC95TPR': 0.5501000285148621, 'FPR95TPR': 0.5097000002861023, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.89626145362854, 'AUPR-IN': 0.551554799079895, 'AUPR-OUT': 0.9815916419029236, 'ACC95TPR': 0.6658373475074768, 'FPR95TPR': 0.37665998935699463, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8300771713256836, 'AUPR-IN': 0.30460208654403687, 'AUPR-OUT': 0.9907640814781189, 'ACC95TPR': 0.4068653881549835, 'FPR95TPR': 0.6148599982261658, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.914482593536377, 'AUPR-IN': 0.5187401175498962, 'AUPR-OUT': 0.9955224990844727, 'ACC95TPR': 0.612500011920929, 'FPR95TPR': 0.4009999930858612, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.7990084886550903, 'AUPR-IN': 0.7307175993919373, 'AUPR-OUT': 0.8556296825408936, 'ACC95TPR': 0.5452874898910522, 'FPR95TPR': 0.6975399851799011, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8572423458099365, 'AUPR-IN': 0.8016011118888855, 'AUPR-OUT': 0.8955172300338745, 'ACC95TPR': 0.6085249781608582, 'FPR95TPR': 0.5963600277900696, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ae61a56155a4641a7c7f5d278187bf8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 86.065%\n",
      "[{'AUROC': 0.8584320545196533, 'AUPR-IN': 0.5092251896858215, 'AUPR-OUT': 0.9740358591079712, 'ACC95TPR': 0.5577381253242493, 'FPR95TPR': 0.5009199976921082, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8976821899414062, 'AUPR-IN': 0.5520417094230652, 'AUPR-OUT': 0.9819735288619995, 'ACC95TPR': 0.6722400784492493, 'FPR95TPR': 0.3693000078201294, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.83307284116745, 'AUPR-IN': 0.30684834718704224, 'AUPR-OUT': 0.9909559488296509, 'ACC95TPR': 0.4075576961040497, 'FPR95TPR': 0.614139974117279, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9168974161148071, 'AUPR-IN': 0.5234997272491455, 'AUPR-OUT': 0.9956931471824646, 'ACC95TPR': 0.6190000176429749, 'FPR95TPR': 0.39423999190330505, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8032436370849609, 'AUPR-IN': 0.7372796535491943, 'AUPR-OUT': 0.8585947751998901, 'ACC95TPR': 0.5485374927520752, 'FPR95TPR': 0.6923400163650513, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8606493473052979, 'AUPR-IN': 0.8074601888656616, 'AUPR-OUT': 0.8984131813049316, 'ACC95TPR': 0.6151124835014343, 'FPR95TPR': 0.5858200192451477, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f8de74026aad4376ba5eae3641e3b82c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 86.856%\n",
      "[{'AUROC': 0.8648132681846619, 'AUPR-IN': 0.5225027799606323, 'AUPR-OUT': 0.9753068089485168, 'ACC95TPR': 0.5696911811828613, 'FPR95TPR': 0.4871799945831299, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9003571271896362, 'AUPR-IN': 0.5556070804595947, 'AUPR-OUT': 0.9826620221138, 'ACC95TPR': 0.6863853931427002, 'FPR95TPR': 0.35304000973701477, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8350672721862793, 'AUPR-IN': 0.2977432310581207, 'AUPR-OUT': 0.9911048412322998, 'ACC95TPR': 0.41653844714164734, 'FPR95TPR': 0.6047999858856201, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9184238314628601, 'AUPR-IN': 0.5184578895568848, 'AUPR-OUT': 0.9958339929580688, 'ACC95TPR': 0.6191923022270203, 'FPR95TPR': 0.39403998851776123, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8130560517311096, 'AUPR-IN': 0.7511273622512817, 'AUPR-OUT': 0.8654782772064209, 'ACC95TPR': 0.5610125064849854, 'FPR95TPR': 0.6723799705505371, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8685559034347534, 'AUPR-IN': 0.8191600441932678, 'AUPR-OUT': 0.9050354957580566, 'ACC95TPR': 0.6315125226974487, 'FPR95TPR': 0.5595800280570984, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6c77f90fe2a47bebe69849ea1e4adad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 88.538%\n",
      "[{'AUROC': 0.8776921033859253, 'AUPR-IN': 0.5390326976776123, 'AUPR-OUT': 0.9779415726661682, 'ACC95TPR': 0.5920661091804504, 'FPR95TPR': 0.4614599943161011, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9022055864334106, 'AUPR-IN': 0.5493008494377136, 'AUPR-OUT': 0.9834966063499451, 'ACC95TPR': 0.7066376805305481, 'FPR95TPR': 0.32975998520851135, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8359349966049194, 'AUPR-IN': 0.2502119541168213, 'AUPR-OUT': 0.9912350177764893, 'ACC95TPR': 0.4243461489677429, 'FPR95TPR': 0.5966799855232239, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9152414798736572, 'AUPR-IN': 0.463082492351532, 'AUPR-OUT': 0.9957894086837769, 'ACC95TPR': 0.655096173286438, 'FPR95TPR': 0.35670000314712524, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8387573957443237, 'AUPR-IN': 0.7847355604171753, 'AUPR-OUT': 0.8842461109161377, 'ACC95TPR': 0.5866624712944031, 'FPR95TPR': 0.6313400268554688, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8853753209114075, 'AUPR-IN': 0.8425549268722534, 'AUPR-OUT': 0.9192293882369995, 'ACC95TPR': 0.6636375188827515, 'FPR95TPR': 0.5081800222396851, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c46a08cb786843ceb3ba0726127351d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 88.845%\n",
      "[{'AUROC': 0.8800475001335144, 'AUPR-IN': 0.5392556190490723, 'AUPR-OUT': 0.9784841537475586, 'ACC95TPR': 0.5965723991394043, 'FPR95TPR': 0.456279993057251, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9006820917129517, 'AUPR-IN': 0.540584921836853, 'AUPR-OUT': 0.9833730459213257, 'ACC95TPR': 0.7070726156234741, 'FPR95TPR': 0.329259991645813, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8370836973190308, 'AUPR-IN': 0.240282341837883, 'AUPR-OUT': 0.991315484046936, 'ACC95TPR': 0.43140384554862976, 'FPR95TPR': 0.5893399715423584, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9143682718276978, 'AUPR-IN': 0.44650864601135254, 'AUPR-OUT': 0.9957617521286011, 'ACC95TPR': 0.6512884497642517, 'FPR95TPR': 0.36065998673439026, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8446429967880249, 'AUPR-IN': 0.7927661538124084, 'AUPR-OUT': 0.8887811303138733, 'ACC95TPR': 0.5934749841690063, 'FPR95TPR': 0.6204400062561035, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8884508013725281, 'AUPR-IN': 0.846937894821167, 'AUPR-OUT': 0.9220463037490845, 'ACC95TPR': 0.6683499813079834, 'FPR95TPR': 0.5006399750709534, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "22a82bfc144e4c1cbd0d1b4880623826",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 88.842%\n",
      "[{'AUROC': 0.8800886869430542, 'AUPR-IN': 0.5409218668937683, 'AUPR-OUT': 0.9784746766090393, 'ACC95TPR': 0.5956850647926331, 'FPR95TPR': 0.45730000734329224, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9008774757385254, 'AUPR-IN': 0.5419211983680725, 'AUPR-OUT': 0.983375608921051, 'ACC95TPR': 0.7029839158058167, 'FPR95TPR': 0.33395999670028687, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8384467363357544, 'AUPR-IN': 0.2457425445318222, 'AUPR-OUT': 0.9913726449012756, 'ACC95TPR': 0.44142308831214905, 'FPR95TPR': 0.5789200067520142, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9157702922821045, 'AUPR-IN': 0.4547658860683441, 'AUPR-OUT': 0.9958142042160034, 'ACC95TPR': 0.6617115139961243, 'FPR95TPR': 0.34981998801231384, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.844355583190918, 'AUPR-IN': 0.7935352325439453, 'AUPR-OUT': 0.8884483575820923, 'ACC95TPR': 0.5921375155448914, 'FPR95TPR': 0.6225799918174744, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8884163498878479, 'AUPR-IN': 0.8475001454353333, 'AUPR-OUT': 0.9219323396682739, 'ACC95TPR': 0.6683250069618225, 'FPR95TPR': 0.5006800293922424, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "92f058d6442d45eb9545bcef285ea9c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 88.974%\n",
      "[{'AUROC': 0.8806209564208984, 'AUPR-IN': 0.5417772531509399, 'AUPR-OUT': 0.9785786867141724, 'ACC95TPR': 0.5971291661262512, 'FPR95TPR': 0.4556399881839752, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9004120230674744, 'AUPR-IN': 0.5397694110870361, 'AUPR-OUT': 0.9833161234855652, 'ACC95TPR': 0.7049151659011841, 'FPR95TPR': 0.33173999190330505, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.839525580406189, 'AUPR-IN': 0.24675607681274414, 'AUPR-OUT': 0.9914252758026123, 'ACC95TPR': 0.4376923143863678, 'FPR95TPR': 0.5827999711036682, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9160491824150085, 'AUPR-IN': 0.45355385541915894, 'AUPR-OUT': 0.9958236813545227, 'ACC95TPR': 0.6590384840965271, 'FPR95TPR': 0.35260000824928284, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8463866114616394, 'AUPR-IN': 0.7969493269920349, 'AUPR-OUT': 0.8899459838867188, 'ACC95TPR': 0.5946375131607056, 'FPR95TPR': 0.6185799837112427, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8897412419319153, 'AUPR-IN': 0.849495530128479, 'AUPR-OUT': 0.9230333566665649, 'ACC95TPR': 0.6691874861717224, 'FPR95TPR': 0.4993000030517578, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "804698c347384c52b9d1b82380ae998b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 88.933%\n",
      "[{'AUROC': 0.880304217338562, 'AUPR-IN': 0.5439414978027344, 'AUPR-OUT': 0.9784905910491943, 'ACC95TPR': 0.5957720875740051, 'FPR95TPR': 0.45719999074935913, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9010281562805176, 'AUPR-IN': 0.5439707636833191, 'AUPR-OUT': 0.983361542224884, 'ACC95TPR': 0.7006176710128784, 'FPR95TPR': 0.33667999505996704, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8408435583114624, 'AUPR-IN': 0.2538803517818451, 'AUPR-OUT': 0.9914875030517578, 'ACC95TPR': 0.4391730725765228, 'FPR95TPR': 0.5812600255012512, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9175332188606262, 'AUPR-IN': 0.46419116854667664, 'AUPR-OUT': 0.9958847165107727, 'ACC95TPR': 0.6530384421348572, 'FPR95TPR': 0.3588399887084961, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8452218770980835, 'AUPR-IN': 0.7962118983268738, 'AUPR-OUT': 0.889009952545166, 'ACC95TPR': 0.5940999984741211, 'FPR95TPR': 0.6194400191307068, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8893251419067383, 'AUPR-IN': 0.8492816686630249, 'AUPR-OUT': 0.9225716590881348, 'ACC95TPR': 0.6704624891281128, 'FPR95TPR': 0.4972600042819977, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "908da35abac841d699952d18157b49a5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.013%\n",
      "[{'AUROC': 0.8804917335510254, 'AUPR-IN': 0.5448309779167175, 'AUPR-OUT': 0.9785182476043701, 'ACC95TPR': 0.5959112644195557, 'FPR95TPR': 0.457040011882782, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9008492231369019, 'AUPR-IN': 0.5442379117012024, 'AUPR-OUT': 0.9833254814147949, 'ACC95TPR': 0.7011744379997253, 'FPR95TPR': 0.3360399901866913, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.841846227645874, 'AUPR-IN': 0.2572593688964844, 'AUPR-OUT': 0.9915335178375244, 'ACC95TPR': 0.43876922130584717, 'FPR95TPR': 0.5816799998283386, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9179898500442505, 'AUPR-IN': 0.4660857617855072, 'AUPR-OUT': 0.9959009885787964, 'ACC95TPR': 0.6609615087509155, 'FPR95TPR': 0.350600004196167, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8462478518486023, 'AUPR-IN': 0.7981045842170715, 'AUPR-OUT': 0.8897367715835571, 'ACC95TPR': 0.5945249795913696, 'FPR95TPR': 0.6187599897384644, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8901275992393494, 'AUPR-IN': 0.8505668640136719, 'AUPR-OUT': 0.9231829643249512, 'ACC95TPR': 0.6716625094413757, 'FPR95TPR': 0.4953399896621704, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d94ac8fc12a43c989ee486d9f6659f5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.109%\n",
      "[{'AUROC': 0.8807674050331116, 'AUPR-IN': 0.5456359386444092, 'AUPR-OUT': 0.978567361831665, 'ACC95TPR': 0.5981557369232178, 'FPR95TPR': 0.4544599950313568, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9004227519035339, 'AUPR-IN': 0.5426433086395264, 'AUPR-OUT': 0.9832576513290405, 'ACC95TPR': 0.7000260949134827, 'FPR95TPR': 0.3373599946498871, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8426735401153564, 'AUPR-IN': 0.25854676961898804, 'AUPR-OUT': 0.9915732145309448, 'ACC95TPR': 0.4347115457057953, 'FPR95TPR': 0.5859000086784363, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9182308912277222, 'AUPR-IN': 0.46492433547973633, 'AUPR-OUT': 0.9959070682525635, 'ACC95TPR': 0.6635577082633972, 'FPR95TPR': 0.34790000319480896, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8477500081062317, 'AUPR-IN': 0.8004887700080872, 'AUPR-OUT': 0.8908641934394836, 'ACC95TPR': 0.596137523651123, 'FPR95TPR': 0.6161800026893616, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.891086995601654, 'AUPR-IN': 0.8518325686454773, 'AUPR-OUT': 0.9239793419837952, 'ACC95TPR': 0.6731500029563904, 'FPR95TPR': 0.49296000599861145, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96c388b3dd4d4806a339670eb32b1545",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.060%\n",
      "[{'AUROC': 0.8803032636642456, 'AUPR-IN': 0.546554446220398, 'AUPR-OUT': 0.9784643054008484, 'ACC95TPR': 0.5948847532272339, 'FPR95TPR': 0.4582200050354004, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9003738164901733, 'AUPR-IN': 0.5437584519386292, 'AUPR-OUT': 0.9832057356834412, 'ACC95TPR': 0.7017486095428467, 'FPR95TPR': 0.3353799879550934, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8440250754356384, 'AUPR-IN': 0.26460957527160645, 'AUPR-OUT': 0.9916560053825378, 'ACC95TPR': 0.4316730797290802, 'FPR95TPR': 0.5890600085258484, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9194319248199463, 'AUPR-IN': 0.47062939405441284, 'AUPR-OUT': 0.9959666728973389, 'ACC95TPR': 0.6636154055595398, 'FPR95TPR': 0.34784001111984253, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8467569351196289, 'AUPR-IN': 0.7996893525123596, 'AUPR-OUT': 0.8900883197784424, 'ACC95TPR': 0.5956375002861023, 'FPR95TPR': 0.6169800162315369, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8905965089797974, 'AUPR-IN': 0.85130774974823, 'AUPR-OUT': 0.9235351085662842, 'ACC95TPR': 0.6741750240325928, 'FPR95TPR': 0.49132001399993896, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec49deb33dd44b38ae02ee8099b54334",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.106%\n",
      "[{'AUROC': 0.8800691366195679, 'AUPR-IN': 0.5470024943351746, 'AUPR-OUT': 0.9784209132194519, 'ACC95TPR': 0.6024010181427002, 'FPR95TPR': 0.44958001375198364, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9000380039215088, 'AUPR-IN': 0.5434185862541199, 'AUPR-OUT': 0.9831289649009705, 'ACC95TPR': 0.6972770690917969, 'FPR95TPR': 0.3405199944972992, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.845200777053833, 'AUPR-IN': 0.2683603763580322, 'AUPR-OUT': 0.9917206168174744, 'ACC95TPR': 0.4365769326686859, 'FPR95TPR': 0.5839599967002869, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9200836420059204, 'AUPR-IN': 0.47299322485923767, 'AUPR-OUT': 0.9959932565689087, 'ACC95TPR': 0.6645192503929138, 'FPR95TPR': 0.34689998626708984, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.847152054309845, 'AUPR-IN': 0.8005456924438477, 'AUPR-OUT': 0.8904210925102234, 'ACC95TPR': 0.5965625047683716, 'FPR95TPR': 0.6154999732971191, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8910609483718872, 'AUPR-IN': 0.8520060777664185, 'AUPR-OUT': 0.9238772392272949, 'ACC95TPR': 0.6752874851226807, 'FPR95TPR': 0.4895400106906891, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0e866322ef3498296eccbfbc79de183",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.130%\n",
      "[{'AUROC': 0.8794347047805786, 'AUPR-IN': 0.5465433597564697, 'AUPR-OUT': 0.978293776512146, 'ACC95TPR': 0.6002609729766846, 'FPR95TPR': 0.4520399868488312, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8992354869842529, 'AUPR-IN': 0.5412511229515076, 'AUPR-OUT': 0.982952356338501, 'ACC95TPR': 0.6927533745765686, 'FPR95TPR': 0.34571999311447144, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8462982177734375, 'AUPR-IN': 0.2730332314968109, 'AUPR-OUT': 0.9917774200439453, 'ACC95TPR': 0.4439423084259033, 'FPR95TPR': 0.5763000249862671, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9208483099937439, 'AUPR-IN': 0.4761631488800049, 'AUPR-OUT': 0.9960203170776367, 'ACC95TPR': 0.6575192213058472, 'FPR95TPR': 0.3541800081729889, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8470031023025513, 'AUPR-IN': 0.8009544014930725, 'AUPR-OUT': 0.8901911973953247, 'ACC95TPR': 0.5944125056266785, 'FPR95TPR': 0.618939995765686, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8913030028343201, 'AUPR-IN': 0.8524847626686096, 'AUPR-OUT': 0.9239221811294556, 'ACC95TPR': 0.6770625114440918, 'FPR95TPR': 0.48669999837875366, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "500c34c1d15745feb742f7d4bb47bffd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.176%\n",
      "[{'AUROC': 0.8795067071914673, 'AUPR-IN': 0.5480359792709351, 'AUPR-OUT': 0.9782940149307251, 'ACC95TPR': 0.5992170572280884, 'FPR95TPR': 0.4532400071620941, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8992211818695068, 'AUPR-IN': 0.5426047444343567, 'AUPR-OUT': 0.9829419255256653, 'ACC95TPR': 0.6918312311172485, 'FPR95TPR': 0.346780002117157, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8472269177436829, 'AUPR-IN': 0.2754238545894623, 'AUPR-OUT': 0.9918295741081238, 'ACC95TPR': 0.44119229912757874, 'FPR95TPR': 0.5791599750518799, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9212720394134521, 'AUPR-IN': 0.4767622649669647, 'AUPR-OUT': 0.9960421323776245, 'ACC95TPR': 0.657730758190155, 'FPR95TPR': 0.3539600074291229, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8476837873458862, 'AUPR-IN': 0.8021396398544312, 'AUPR-OUT': 0.8906416893005371, 'ACC95TPR': 0.5958499908447266, 'FPR95TPR': 0.6166399717330933, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8917557001113892, 'AUPR-IN': 0.8531897068023682, 'AUPR-OUT': 0.9242169857025146, 'ACC95TPR': 0.6771000027656555, 'FPR95TPR': 0.48664000630378723, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7c6c3b45a724b43909dfb828c89315a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.114%\n",
      "[{'AUROC': 0.8786927461624146, 'AUPR-IN': 0.547706663608551, 'AUPR-OUT': 0.978130578994751, 'ACC95TPR': 0.5975293517112732, 'FPR95TPR': 0.45517998933792114, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8989191055297852, 'AUPR-IN': 0.5430038571357727, 'AUPR-OUT': 0.9828566312789917, 'ACC95TPR': 0.6897781491279602, 'FPR95TPR': 0.3491399884223938, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.848025918006897, 'AUPR-IN': 0.2802947461605072, 'AUPR-OUT': 0.9918687343597412, 'ACC95TPR': 0.4394230842590332, 'FPR95TPR': 0.5809999704360962, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9219504594802856, 'AUPR-IN': 0.48136430978775024, 'AUPR-OUT': 0.9960646033287048, 'ACC95TPR': 0.6553077101707458, 'FPR95TPR': 0.3564800024032593, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8466396331787109, 'AUPR-IN': 0.8011845350265503, 'AUPR-OUT': 0.8898186683654785, 'ACC95TPR': 0.5941625237464905, 'FPR95TPR': 0.6193400025367737, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8911416530609131, 'AUPR-IN': 0.8525431752204895, 'AUPR-OUT': 0.9236438870429993, 'ACC95TPR': 0.6753374934196472, 'FPR95TPR': 0.4894599914550781, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ece500619d704654a209a14a54407e53",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.111%\n",
      "[{'AUROC': 0.8781715631484985, 'AUPR-IN': 0.5465315580368042, 'AUPR-OUT': 0.9780192375183105, 'ACC95TPR': 0.5975815653800964, 'FPR95TPR': 0.4551199972629547, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.898484468460083, 'AUPR-IN': 0.5423520803451538, 'AUPR-OUT': 0.9827507734298706, 'ACC95TPR': 0.6888212561607361, 'FPR95TPR': 0.35023999214172363, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8488392233848572, 'AUPR-IN': 0.28317883610725403, 'AUPR-OUT': 0.991920530796051, 'ACC95TPR': 0.4374615252017975, 'FPR95TPR': 0.5830399990081787, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9225320816040039, 'AUPR-IN': 0.4827365279197693, 'AUPR-OUT': 0.9960924386978149, 'ACC95TPR': 0.6622499823570251, 'FPR95TPR': 0.34926000237464905, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8464798331260681, 'AUPR-IN': 0.8012731075286865, 'AUPR-OUT': 0.8896365165710449, 'ACC95TPR': 0.5934125185012817, 'FPR95TPR': 0.6205400228500366, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8911137580871582, 'AUPR-IN': 0.85265052318573, 'AUPR-OUT': 0.9235093593597412, 'ACC95TPR': 0.6749374866485596, 'FPR95TPR': 0.4900999963283539, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41dca418562a4cf6a6cb69da5a7ecb25",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.066%\n",
      "[{'AUROC': 0.8779491186141968, 'AUPR-IN': 0.5472238659858704, 'AUPR-OUT': 0.9779475331306458, 'ACC95TPR': 0.5955632925033569, 'FPR95TPR': 0.45743998885154724, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8983873724937439, 'AUPR-IN': 0.5438829660415649, 'AUPR-OUT': 0.982710063457489, 'ACC95TPR': 0.6902479529380798, 'FPR95TPR': 0.34860000014305115, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8493766784667969, 'AUPR-IN': 0.2847881615161896, 'AUPR-OUT': 0.9919601678848267, 'ACC95TPR': 0.43799999356269836, 'FPR95TPR': 0.5824800133705139, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9228745698928833, 'AUPR-IN': 0.4837758541107178, 'AUPR-OUT': 0.9961168169975281, 'ACC95TPR': 0.6639423370361328, 'FPR95TPR': 0.3474999964237213, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8462256193161011, 'AUPR-IN': 0.8010331988334656, 'AUPR-OUT': 0.8893517851829529, 'ACC95TPR': 0.5934500098228455, 'FPR95TPR': 0.6204800009727478, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8906607627868652, 'AUPR-IN': 0.8520479798316956, 'AUPR-OUT': 0.9231016635894775, 'ACC95TPR': 0.6738749742507935, 'FPR95TPR': 0.4918000102043152, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "435b6f2671744564a33cdd90ffa9edd5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.148%\n",
      "[{'AUROC': 0.8782721757888794, 'AUPR-IN': 0.5475477576255798, 'AUPR-OUT': 0.9780260920524597, 'ACC95TPR': 0.5966246128082275, 'FPR95TPR': 0.45622000098228455, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.898273229598999, 'AUPR-IN': 0.5429127812385559, 'AUPR-OUT': 0.9827061891555786, 'ACC95TPR': 0.6892387866973877, 'FPR95TPR': 0.3497599959373474, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8501060605049133, 'AUPR-IN': 0.28414011001586914, 'AUPR-OUT': 0.9920033812522888, 'ACC95TPR': 0.4377307593822479, 'FPR95TPR': 0.5827599763870239, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.922859787940979, 'AUPR-IN': 0.47841423749923706, 'AUPR-OUT': 0.9961163401603699, 'ACC95TPR': 0.6632884740829468, 'FPR95TPR': 0.34817999601364136, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8476136326789856, 'AUPR-IN': 0.8030778169631958, 'AUPR-OUT': 0.8903481960296631, 'ACC95TPR': 0.5946750044822693, 'FPR95TPR': 0.6185200214385986, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8914754390716553, 'AUPR-IN': 0.8531261086463928, 'AUPR-OUT': 0.9237074851989746, 'ACC95TPR': 0.675849974155426, 'FPR95TPR': 0.4886400103569031, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "894550fe29854d2caf873b6fbb80d79b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.181%\n",
      "[{'AUROC': 0.8782665729522705, 'AUPR-IN': 0.547940731048584, 'AUPR-OUT': 0.9780102968215942, 'ACC95TPR': 0.5945541262626648, 'FPR95TPR': 0.4586000144481659, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.898117184638977, 'AUPR-IN': 0.5433395504951477, 'AUPR-OUT': 0.9826611876487732, 'ACC95TPR': 0.686333179473877, 'FPR95TPR': 0.3531000018119812, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8505652546882629, 'AUPR-IN': 0.28463810682296753, 'AUPR-OUT': 0.9920286536216736, 'ACC95TPR': 0.43978846073150635, 'FPR95TPR': 0.5806199908256531, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9228695631027222, 'AUPR-IN': 0.4765337109565735, 'AUPR-OUT': 0.996116042137146, 'ACC95TPR': 0.6626923084259033, 'FPR95TPR': 0.34880000352859497, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8482120037078857, 'AUPR-IN': 0.8039688467979431, 'AUPR-OUT': 0.8907589912414551, 'ACC95TPR': 0.5953249931335449, 'FPR95TPR': 0.6174799799919128, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.891811728477478, 'AUPR-IN': 0.8535781502723694, 'AUPR-OUT': 0.9239222407341003, 'ACC95TPR': 0.67535001039505, 'FPR95TPR': 0.489439994096756, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7931877c74745c68b15c63d0e180e34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.189%\n",
      "[{'AUROC': 0.8778544664382935, 'AUPR-IN': 0.5467808842658997, 'AUPR-OUT': 0.977924644947052, 'ACC95TPR': 0.5936667919158936, 'FPR95TPR': 0.45961999893188477, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8976799249649048, 'AUPR-IN': 0.5422875881195068, 'AUPR-OUT': 0.9825575351715088, 'ACC95TPR': 0.6868377327919006, 'FPR95TPR': 0.3525199890136719, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.850874662399292, 'AUPR-IN': 0.2844182848930359, 'AUPR-OUT': 0.992046594619751, 'ACC95TPR': 0.4419230818748474, 'FPR95TPR': 0.5784000158309937, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9229873418807983, 'AUPR-IN': 0.47445762157440186, 'AUPR-OUT': 0.9961178302764893, 'ACC95TPR': 0.6620000004768372, 'FPR95TPR': 0.3495199978351593, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8482074737548828, 'AUPR-IN': 0.8042131066322327, 'AUPR-OUT': 0.8907171487808228, 'ACC95TPR': 0.5962250232696533, 'FPR95TPR': 0.6160399913787842, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8918884992599487, 'AUPR-IN': 0.853775680065155, 'AUPR-OUT': 0.9238981008529663, 'ACC95TPR': 0.6757749915122986, 'FPR95TPR': 0.48875999450683594, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "350eec14f7db453f827746c147d3df81",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.115%\n",
      "[{'AUROC': 0.8776047229766846, 'AUPR-IN': 0.5476850271224976, 'AUPR-OUT': 0.9778597354888916, 'ACC95TPR': 0.5928316712379456, 'FPR95TPR': 0.4605799913406372, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8979435563087463, 'AUPR-IN': 0.5447900295257568, 'AUPR-OUT': 0.982581615447998, 'ACC95TPR': 0.6820878386497498, 'FPR95TPR': 0.35798001289367676, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8510422706604004, 'AUPR-IN': 0.28782591223716736, 'AUPR-OUT': 0.9920521974563599, 'ACC95TPR': 0.4420384466648102, 'FPR95TPR': 0.578279972076416, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.92327880859375, 'AUPR-IN': 0.47775888442993164, 'AUPR-OUT': 0.9961235523223877, 'ACC95TPR': 0.6612692475318909, 'FPR95TPR': 0.3502799868583679, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.84716796875, 'AUPR-IN': 0.8029839992523193, 'AUPR-OUT': 0.8899682760238647, 'ACC95TPR': 0.5947625041007996, 'FPR95TPR': 0.6183800101280212, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8911529779434204, 'AUPR-IN': 0.8529039025306702, 'AUPR-OUT': 0.9232492446899414, 'ACC95TPR': 0.6745499968528748, 'FPR95TPR': 0.4907200038433075, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b41861937fd46f29d9a3e10c79ff78d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.138%\n",
      "[{'AUROC': 0.8772768974304199, 'AUPR-IN': 0.547160267829895, 'AUPR-OUT': 0.9778013229370117, 'ACC95TPR': 0.590291440486908, 'FPR95TPR': 0.4634999930858612, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8975977897644043, 'AUPR-IN': 0.5441298484802246, 'AUPR-OUT': 0.9825059771537781, 'ACC95TPR': 0.6792170405387878, 'FPR95TPR': 0.36127999424934387, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8513854146003723, 'AUPR-IN': 0.2880728840827942, 'AUPR-OUT': 0.9920747876167297, 'ACC95TPR': 0.4438653886318207, 'FPR95TPR': 0.5763800144195557, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9234027862548828, 'AUPR-IN': 0.4761330187320709, 'AUPR-OUT': 0.9961278438568115, 'ACC95TPR': 0.6628461480140686, 'FPR95TPR': 0.34863999485969543, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8473072052001953, 'AUPR-IN': 0.8032759428024292, 'AUPR-OUT': 0.89007568359375, 'ACC95TPR': 0.5945125222206116, 'FPR95TPR': 0.6187800168991089, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8913843631744385, 'AUPR-IN': 0.8531923294067383, 'AUPR-OUT': 0.9233630895614624, 'ACC95TPR': 0.6745374798774719, 'FPR95TPR': 0.49074000120162964, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "724138f6293a48a9a3349c6f44077487",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.189%\n",
      "[{'AUROC': 0.8775246143341064, 'AUPR-IN': 0.5471977591514587, 'AUPR-OUT': 0.9778541326522827, 'ACC95TPR': 0.5921009182929993, 'FPR95TPR': 0.4614199995994568, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8976644277572632, 'AUPR-IN': 0.5439581274986267, 'AUPR-OUT': 0.9825162887573242, 'ACC95TPR': 0.6796694397926331, 'FPR95TPR': 0.36076000332832336, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8519265055656433, 'AUPR-IN': 0.28650274872779846, 'AUPR-OUT': 0.9921085238456726, 'ACC95TPR': 0.44676923751831055, 'FPR95TPR': 0.573360025882721, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9233781099319458, 'AUPR-IN': 0.4697694778442383, 'AUPR-OUT': 0.9961285591125488, 'ACC95TPR': 0.6640961766242981, 'FPR95TPR': 0.3473399877548218, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8483068943023682, 'AUPR-IN': 0.8047105669975281, 'AUPR-OUT': 0.8907448053359985, 'ACC95TPR': 0.5950624942779541, 'FPR95TPR': 0.617900013923645, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8918912410736084, 'AUPR-IN': 0.8536997437477112, 'AUPR-OUT': 0.9237196445465088, 'ACC95TPR': 0.67535001039505, 'FPR95TPR': 0.489439994096756, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70f0462023424c0297698343c6d4becc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.067%\n",
      "[{'AUROC': 0.8762616515159607, 'AUPR-IN': 0.5443187355995178, 'AUPR-OUT': 0.9775940179824829, 'ACC95TPR': 0.5908482074737549, 'FPR95TPR': 0.46285998821258545, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8970969915390015, 'AUPR-IN': 0.5432442426681519, 'AUPR-OUT': 0.9823534488677979, 'ACC95TPR': 0.6731448173522949, 'FPR95TPR': 0.368259996175766, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8523832559585571, 'AUPR-IN': 0.2900218367576599, 'AUPR-OUT': 0.992143988609314, 'ACC95TPR': 0.44701921939849854, 'FPR95TPR': 0.5730999708175659, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9240837097167969, 'AUPR-IN': 0.4759574234485626, 'AUPR-OUT': 0.9961647391319275, 'ACC95TPR': 0.6651153564453125, 'FPR95TPR': 0.3462800085544586, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8463006019592285, 'AUPR-IN': 0.8021392822265625, 'AUPR-OUT': 0.889285683631897, 'ACC95TPR': 0.5928750038146973, 'FPR95TPR': 0.621399998664856, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8906707763671875, 'AUPR-IN': 0.8522207140922546, 'AUPR-OUT': 0.9226645231246948, 'ACC95TPR': 0.6743249893188477, 'FPR95TPR': 0.49107998609542847, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a9c303a585fc4a0cb20a236bb8d459c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.064%\n",
      "[{'AUROC': 0.8759234547615051, 'AUPR-IN': 0.5435724258422852, 'AUPR-OUT': 0.9775334000587463, 'ACC95TPR': 0.5923966765403748, 'FPR95TPR': 0.46108001470565796, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8964602947235107, 'AUPR-IN': 0.5411379933357239, 'AUPR-OUT': 0.9822239875793457, 'ACC95TPR': 0.6733710169792175, 'FPR95TPR': 0.36800000071525574, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8529400825500488, 'AUPR-IN': 0.2901917099952698, 'AUPR-OUT': 0.9921703338623047, 'ACC95TPR': 0.4503461420536041, 'FPR95TPR': 0.5696399807929993, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9242894053459167, 'AUPR-IN': 0.47405093908309937, 'AUPR-OUT': 0.9961711168289185, 'ACC95TPR': 0.6658461689949036, 'FPR95TPR': 0.3455199897289276, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8463491201400757, 'AUPR-IN': 0.8024282455444336, 'AUPR-OUT': 0.8893417716026306, 'ACC95TPR': 0.5929625034332275, 'FPR95TPR': 0.6212599873542786, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8906419277191162, 'AUPR-IN': 0.8521765470504761, 'AUPR-OUT': 0.9226667881011963, 'ACC95TPR': 0.6732875108718872, 'FPR95TPR': 0.4927400052547455, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "260ce2d9abca48f9be4667556194377d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.026%\n",
      "[{'AUROC': 0.8756884932518005, 'AUPR-IN': 0.5429226756095886, 'AUPR-OUT': 0.9774820804595947, 'ACC95TPR': 0.5920313000679016, 'FPR95TPR': 0.46149998903274536, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8964509963989258, 'AUPR-IN': 0.5414091348648071, 'AUPR-OUT': 0.9822129011154175, 'ACC95TPR': 0.6727968454360962, 'FPR95TPR': 0.36866000294685364, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8531582355499268, 'AUPR-IN': 0.2905195355415344, 'AUPR-OUT': 0.9921855926513672, 'ACC95TPR': 0.4526346027851105, 'FPR95TPR': 0.5672600269317627, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9243596196174622, 'AUPR-IN': 0.4717778265476227, 'AUPR-OUT': 0.996176540851593, 'ACC95TPR': 0.6685961484909058, 'FPR95TPR': 0.34266000986099243, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8460521697998047, 'AUPR-IN': 0.8020366430282593, 'AUPR-OUT': 0.8890702128410339, 'ACC95TPR': 0.5920875072479248, 'FPR95TPR': 0.6226599812507629, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8902631998062134, 'AUPR-IN': 0.8515541553497314, 'AUPR-OUT': 0.9223273992538452, 'ACC95TPR': 0.6725624799728394, 'FPR95TPR': 0.49390000104904175, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8760dbc7d87f43bbb762da97d5846aad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 88.999%\n",
      "[{'AUROC': 0.8748005628585815, 'AUPR-IN': 0.5416141152381897, 'AUPR-OUT': 0.9772987365722656, 'ACC95TPR': 0.5911091566085815, 'FPR95TPR': 0.4625599980354309, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.895964503288269, 'AUPR-IN': 0.540855348110199, 'AUPR-OUT': 0.9820902347564697, 'ACC95TPR': 0.6712135672569275, 'FPR95TPR': 0.3704800009727478, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8533856868743896, 'AUPR-IN': 0.29261234402656555, 'AUPR-OUT': 0.99219810962677, 'ACC95TPR': 0.4552307724952698, 'FPR95TPR': 0.5645599961280823, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9246443510055542, 'AUPR-IN': 0.47287794947624207, 'AUPR-OUT': 0.9961845874786377, 'ACC95TPR': 0.6689038276672363, 'FPR95TPR': 0.34233999252319336, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.845413327217102, 'AUPR-IN': 0.8014782667160034, 'AUPR-OUT': 0.8885468244552612, 'ACC95TPR': 0.5913375020027161, 'FPR95TPR': 0.6238600015640259, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8899878859519958, 'AUPR-IN': 0.8513371348381042, 'AUPR-OUT': 0.9220041036605835, 'ACC95TPR': 0.6719124913215637, 'FPR95TPR': 0.49494001269340515, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a5e7701fbfa64860a12872c13143d253",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 89.010%\n",
      "[{'AUROC': 0.8745662569999695, 'AUPR-IN': 0.5416324138641357, 'AUPR-OUT': 0.9772424697875977, 'ACC95TPR': 0.5904132127761841, 'FPR95TPR': 0.4633600115776062, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8958838582038879, 'AUPR-IN': 0.541359543800354, 'AUPR-OUT': 0.9820611476898193, 'ACC95TPR': 0.6724662780761719, 'FPR95TPR': 0.36904001235961914, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8533329963684082, 'AUPR-IN': 0.29276424646377563, 'AUPR-OUT': 0.9921923875808716, 'ACC95TPR': 0.4536346197128296, 'FPR95TPR': 0.5662199854850769, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9244581460952759, 'AUPR-IN': 0.470355749130249, 'AUPR-OUT': 0.9961728453636169, 'ACC95TPR': 0.6709423065185547, 'FPR95TPR': 0.34022000432014465, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8456774950027466, 'AUPR-IN': 0.8019573092460632, 'AUPR-OUT': 0.888678789138794, 'ACC95TPR': 0.5912749767303467, 'FPR95TPR': 0.623960018157959, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8900957107543945, 'AUPR-IN': 0.8514599800109863, 'AUPR-OUT': 0.9220366477966309, 'ACC95TPR': 0.6716625094413757, 'FPR95TPR': 0.4953399896621704, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33cc27b6ae384d2396c4b77a01a0ebaa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 88.878%\n",
      "[{'AUROC': 0.8737152218818665, 'AUPR-IN': 0.5402501821517944, 'AUPR-OUT': 0.9770632982254028, 'ACC95TPR': 0.5868290662765503, 'FPR95TPR': 0.46748000383377075, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8955491781234741, 'AUPR-IN': 0.5416485071182251, 'AUPR-OUT': 0.9819638133049011, 'ACC95TPR': 0.672431468963623, 'FPR95TPR': 0.3690800070762634, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8531098365783691, 'AUPR-IN': 0.2946438193321228, 'AUPR-OUT': 0.9921776056289673, 'ACC95TPR': 0.454153835773468, 'FPR95TPR': 0.5656800270080566, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9245708584785461, 'AUPR-IN': 0.4735734760761261, 'AUPR-OUT': 0.9961702823638916, 'ACC95TPR': 0.6687884330749512, 'FPR95TPR': 0.3424600064754486, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8438527584075928, 'AUPR-IN': 0.7995981574058533, 'AUPR-OUT': 0.8873234391212463, 'ACC95TPR': 0.5890250205993652, 'FPR95TPR': 0.627560019493103, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8887763023376465, 'AUPR-IN': 0.8497521281242371, 'AUPR-OUT': 0.9209631681442261, 'ACC95TPR': 0.6695625185966492, 'FPR95TPR': 0.49869999289512634, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e5c8f9585f44ac381c4b7abd6a259be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 99 -> 88.937%\n",
      "[{'AUROC': 0.873785674571991, 'AUPR-IN': 0.5406473875045776, 'AUPR-OUT': 0.977072536945343, 'ACC95TPR': 0.585454523563385, 'FPR95TPR': 0.4690600037574768, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8954386711120605, 'AUPR-IN': 0.5417337417602539, 'AUPR-OUT': 0.9819434881210327, 'ACC95TPR': 0.6722227334976196, 'FPR95TPR': 0.36932000517845154, 'Dataset': 'ImageNetA', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8536221981048584, 'AUPR-IN': 0.29453372955322266, 'AUPR-OUT': 0.9922043681144714, 'ACC95TPR': 0.4516538381576538, 'FPR95TPR': 0.5682799816131592, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.9246270060539246, 'AUPR-IN': 0.47011300921440125, 'AUPR-OUT': 0.9961745738983154, 'ACC95TPR': 0.6712499856948853, 'FPR95TPR': 0.3398999869823456, 'Dataset': 'ImageNetO', 'Iteration': 99, 'Method': 'Energy'}, {'AUROC': 0.8448301553726196, 'AUPR-IN': 0.8011481165885925, 'AUPR-OUT': 0.8879445791244507, 'ACC95TPR': 0.5896750092506409, 'FPR95TPR': 0.6265199780464172, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Softmax'}, {'AUROC': 0.8893666863441467, 'AUPR-IN': 0.8505759239196777, 'AUPR-OUT': 0.9213645458221436, 'ACC95TPR': 0.6701375246047974, 'FPR95TPR': 0.4977799952030182, 'Dataset': 'ImageNetR', 'Iteration': 99, 'Method': 'Energy'}]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>AUPR-IN</th>\n",
       "      <th>AUPR-OUT</th>\n",
       "      <th>ACC95TPR</th>\n",
       "      <th>FPR95TPR</th>\n",
       "      <th>Dataset</th>\n",
       "      <th>Iteration</th>\n",
       "      <th>Method</th>\n",
       "      <th>iteration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.854956</td>\n",
       "      <td>0.501128</td>\n",
       "      <td>0.973323</td>\n",
       "      <td>0.550100</td>\n",
       "      <td>0.50970</td>\n",
       "      <td>ImageNetA</td>\n",
       "      <td>99</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.896261</td>\n",
       "      <td>0.551555</td>\n",
       "      <td>0.981592</td>\n",
       "      <td>0.665837</td>\n",
       "      <td>0.37666</td>\n",
       "      <td>ImageNetA</td>\n",
       "      <td>99</td>\n",
       "      <td>Energy</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.830077</td>\n",
       "      <td>0.304602</td>\n",
       "      <td>0.990764</td>\n",
       "      <td>0.406865</td>\n",
       "      <td>0.61486</td>\n",
       "      <td>ImageNetO</td>\n",
       "      <td>99</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.914483</td>\n",
       "      <td>0.518740</td>\n",
       "      <td>0.995522</td>\n",
       "      <td>0.612500</td>\n",
       "      <td>0.40100</td>\n",
       "      <td>ImageNetO</td>\n",
       "      <td>99</td>\n",
       "      <td>Energy</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.799008</td>\n",
       "      <td>0.730718</td>\n",
       "      <td>0.855630</td>\n",
       "      <td>0.545287</td>\n",
       "      <td>0.69754</td>\n",
       "      <td>ImageNetR</td>\n",
       "      <td>99</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>0.895439</td>\n",
       "      <td>0.541734</td>\n",
       "      <td>0.981943</td>\n",
       "      <td>0.672223</td>\n",
       "      <td>0.36932</td>\n",
       "      <td>ImageNetA</td>\n",
       "      <td>99</td>\n",
       "      <td>Energy</td>\n",
       "      <td>2900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>0.853622</td>\n",
       "      <td>0.294534</td>\n",
       "      <td>0.992204</td>\n",
       "      <td>0.451654</td>\n",
       "      <td>0.56828</td>\n",
       "      <td>ImageNetO</td>\n",
       "      <td>99</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>2900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>0.924627</td>\n",
       "      <td>0.470113</td>\n",
       "      <td>0.996175</td>\n",
       "      <td>0.671250</td>\n",
       "      <td>0.33990</td>\n",
       "      <td>ImageNetO</td>\n",
       "      <td>99</td>\n",
       "      <td>Energy</td>\n",
       "      <td>2900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178</th>\n",
       "      <td>0.844830</td>\n",
       "      <td>0.801148</td>\n",
       "      <td>0.887945</td>\n",
       "      <td>0.589675</td>\n",
       "      <td>0.62652</td>\n",
       "      <td>ImageNetR</td>\n",
       "      <td>99</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>2900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>0.889367</td>\n",
       "      <td>0.850576</td>\n",
       "      <td>0.921365</td>\n",
       "      <td>0.670138</td>\n",
       "      <td>0.49778</td>\n",
       "      <td>ImageNetR</td>\n",
       "      <td>99</td>\n",
       "      <td>Energy</td>\n",
       "      <td>2900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>180 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        AUROC   AUPR-IN  AUPR-OUT  ACC95TPR  FPR95TPR    Dataset  Iteration  \\\n",
       "0    0.854956  0.501128  0.973323  0.550100   0.50970  ImageNetA         99   \n",
       "1    0.896261  0.551555  0.981592  0.665837   0.37666  ImageNetA         99   \n",
       "2    0.830077  0.304602  0.990764  0.406865   0.61486  ImageNetO         99   \n",
       "3    0.914483  0.518740  0.995522  0.612500   0.40100  ImageNetO         99   \n",
       "4    0.799008  0.730718  0.855630  0.545287   0.69754  ImageNetR         99   \n",
       "..        ...       ...       ...       ...       ...        ...        ...   \n",
       "175  0.895439  0.541734  0.981943  0.672223   0.36932  ImageNetA         99   \n",
       "176  0.853622  0.294534  0.992204  0.451654   0.56828  ImageNetO         99   \n",
       "177  0.924627  0.470113  0.996175  0.671250   0.33990  ImageNetO         99   \n",
       "178  0.844830  0.801148  0.887945  0.589675   0.62652  ImageNetR         99   \n",
       "179  0.889367  0.850576  0.921365  0.670138   0.49778  ImageNetR         99   \n",
       "\n",
       "      Method  iteration  \n",
       "0    Softmax          0  \n",
       "1     Energy          0  \n",
       "2    Softmax          0  \n",
       "3     Energy          0  \n",
       "4    Softmax          0  \n",
       "..       ...        ...  \n",
       "175   Energy       2900  \n",
       "176  Softmax       2900  \n",
       "177   Energy       2900  \n",
       "178  Softmax       2900  \n",
       "179   Energy       2900  \n",
       "\n",
       "[180 rows x 9 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = VisionTransformer(image_size=(224, 224), pretrained=\"b16-im21k-224\", num_classes=1000) # resnet101(pretrained=True)\n",
    "model._from_pretrained(\"b16-im21k-224\")\n",
    "\n",
    "r = []\n",
    "\n",
    "for i in range(30):\n",
    "    res = train(2.0, iters=100, model=model)\n",
    "    print(res)\n",
    "    for re in res:\n",
    "        re.update({\n",
    "            \"iteration\": i * 100\n",
    "        })\n",
    "        r.append(re)\n",
    "        \n",
    "df_vary_iterations = pd.DataFrame(r)\n",
    "df_vary_iterations\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variance: 0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02529910096b4e489a32d2202dfa102f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kirchheim/anaconda3/envs/myenv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: Metric `AUROC` will save all targets and predictions in buffer. For large datasets this may lead to large memory footprint.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/home/kirchheim/anaconda3/envs/myenv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: Metric `PrecisionRecallCurve` will save all targets and predictions in buffer. For large datasets this may lead to large memory footprint.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/home/kirchheim/anaconda3/envs/myenv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: `pos_label` automatically set 1.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/home/kirchheim/anaconda3/envs/myenv/lib/python3.8/site-packages/torchmetrics/functional/classification/precision_recall_curve.py:156: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  thresholds = tensor(reversed(thresholds[sl]))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUROC 999 -> 85.525%\n",
      "Variance: 0.25\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1a569f38d5d451ab21b78f5a2632c74",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 85.411%\n",
      "Variance: 0.5\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6fc1ff6741bc4e29a733c71654b68edd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 86.579%\n",
      "Variance: 0.75\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39f11de837034f4a867d5aa0663c6b5a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 88.206%\n",
      "Variance: 1.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2353942aaaaf40fe8fa4df10b53b18bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 86.847%\n",
      "Variance: 10.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64954f42d6314f5cabc359fa2baf415c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 86.679%\n",
      "Variance: 1.25\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "18c9044f90d54da58422d9ac3ce6eada",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 88.109%\n",
      "Variance: 1.5\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "452614c955ea4b2598bba1873f002e9f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 88.082%\n",
      "Variance: 1.75\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55018f9268f546c3bfea4e6c2ca9fea3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 88.886%\n",
      "Variance: 2.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "75f71319435e44c4b49b359a96d04224",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 89.055%\n",
      "Variance: 2.5\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "115ed17474514ad3b7acb5bdfb206e76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 89.321%\n",
      "Variance: 3.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c7519a11b604dab8f7d7052b9f8fd5d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 89.323%\n",
      "Variance: 4.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10447e79bc024343b63c48f5930ef74d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 88.746%\n",
      "Variance: 5.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03a7b4dc1e124d93a55df01e99e76088",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 88.195%\n",
      "Variance: 6.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b42f2d552f354402a255fec72b18fb15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 87.806%\n",
      "Variance: 7.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1944e5e291fe4013b200df92e8bbc643",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 87.503%\n",
      "Variance: 8.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1adf34f9c3f407bb2df02577ba3815b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 87.091%\n",
      "Variance: 9.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "910ee1392a1841eebe84ffe4fee7e7d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 86.958%\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd \n",
    "dfs = []\n",
    "\n",
    "for var in [0, 0.25, 0.5, 0.75, 1.0, 10.0, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]:\n",
    "    print(f\"Variance: {var}\")\n",
    "    metrics = train(var)\n",
    "    df = pd.DataFrame(metrics)\n",
    "    df[\"Variance\"] = var \n",
    "    dfs.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAEZCAYAAAAt5touAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAACAGElEQVR4nO2dd3hU1daH36mZTJJJ7wkJCSWh995BKQKChSrCBVGvvfdr4VNsWLCjWK4oXgEjKgqIiiAovUNCSyCk9zrJZMr5/jjJJEMSSEgF9vswT+a0fdY5nNm/s/Zee22FJEkSAoFAIBC0QpQtbYBAIBAIBLUhREogEAgErRYhUgKBQCBotQiREggEAkGrRYiUQCAQCFotQqQEAoFA0GpRt7QBjcHevXtb2gSBQCAQNIDevXvXuP6KECmo/QLrSmxsLNHR0Y1kzeWNuBeViHtRibgXlYh74UhD78eFHA3R3CcQCASCVosQKYFAIBC0Wq6Y5j6BQCBoDmw2G1arlYSEhJY2pdUgSVKd74dGoyEoKAilsm4+khApgUAgqAcpKSn4+fnh5+fX0qa0GkpKSnB2dq7TvgUFBaSkpBASElKn/UVzn0AgENQDs9mMm5tbS5tx2WIwGDCbzXXeX4iUQCAQCFotQqQEAkGd0Ov1LWtAWTGkH4Wdy+S/ZcUtZkpd+1MAOnfuzJQpUxg/fjw333wz69evv+gxv/32G2fPnm2IidX49NNPG7W8qtTnftQX0SclELRmyooh9wyc2QbhQ8AzHLQuTXMuSQKbFSSr41+rGYrSCEneCk7DwNlTXifZwGap+ZiLrbNZyo8/f50VbLbyv1XK1rhASG/49Fr5OIUSFmyC9GNgNYFKAyqt/FGqy79r5I9S47h8sX0Uigv/f5hL0JrzQelWXpbqgrfVw8ODtWvXAnD69GnuuOMOXFxcGDZsWK3H/Pbbb2i1WsLCwmrewWYFaxmYisDJtU52fPbZZyxYsOCC+9Sbcju05sI634/6IkRKILgATeo9WC1QmgcleeV/c6t8z5Mr44jh8Pn4yop53s+w82MoSq8uAudX+g4Vfh3WU8P8p4ZgmPIhrJiCqsKGOWth7b+hILnm61KqQaGSK6uKvw7f1XI5F1ynLv9evi6gGyQY5WsE+W/CVihIgZR9coVtNVd+bObz1pXJ11kXKgRMqakibBpwD4WRT0F6KgrXdvK+3u2hOFMWVIUCKBc4u9Ap5HubewZQEOmt4e5/zeCbLz9jWPe2/Pbndpb99xvKyswEBfqz5P+e4VT8Gf74/Tf27NqJi4uer5e/x2cr/seW7TswmUyMGj6Uhx58AGt6HI+/8gGxJ8+g1Oi4ff4cJo2/hkNHjvHKm+9SUlJCaEgwry76Dx8t/4K8vFyunzSR/n168tQj99ftXsg3u+bVCiWodZB1Arus+0aBsm4BFHVFiJRAUBPlHkxI8lbQDavdg7HZwJRfXWBKcmv+bl+XB2WFleUolKDzAGcP2VPReUCb/pC4w7FiTt4D0ROhMK1Kha6sLgqNtV6jh9gfHW3IiIO7dsjLNZXRVKQfhT/+r1KwO4wD/051P95mqy5eNYmZzVK+rkx+kbCWyfvpPCBlP+BfWaa5GHTu8j4ASOV1ulTlO7LwSfK66PaRfPzf/4HNRt8enRkz9G0APvlqFWtifmDu9OsZNaQfE0YPY9iA3oCFW6dey31zb8Bms/HvJ18m7vB+LAXppGVk8/MXS8AtkMK8HMpyk3ltyVt8+PKTuLu5snzl96z86kse/NcNrFn7Iz98ukS2pzir9vtUoyNZw0qNs3zPqmIqktc3IkKkBFc3FhOYCsFUIP/ATIVyJeXkBstHV3oP8zfAwW8h57Sj51NagMObppN7udB4lIuOp/zdK7Ly+/li5OwBWreaK/j0o/D7C5UVc+SY+lXMjUHbYfK5K2xoOxR0hua1AeQXhTu3wZnt5U2ftTSF1YZSCUonUDtdug2ufnDiWOWyk+HilbJSBe6V4daSW7HsmXmGkZoex/3PvUB2djYlJSUMGjQIvCPlcg1B4NMegH92r2f58sWUlZWRnZ3NqaQMhkb5kZKRxf+98zljrruBgUNHcPz4cY4nnOPWh14E5EjEgQMHgl+0LJR+UZd+7TVhLoHC1Cr3w7Vxy0eIlKA1crF+GEsZlBU5Cot9ubCGdeXLNa2znfcmqNZBuzEQ1NPRezj7jywOhqDqAlMhRjr3Rm+Pb3DF3Ig2WE9vRRU5vGVsAPkZ8O8sf1oKz3DwVyAZfFA4lffB1JPjx4/Ttm1bAF588UXuvfde+vfvz4YNG/jzzz+r7W8ymXjllVeIiYnB29ubRYsWUWax4R7Rhx9Xfc3Wnfv55LP/sufAYa655hq6du3KZ5991sALrSMqLfhGIZkKL/l+XAwhUoLWg80G2SdlAfl0TOWb+9x1sjeRfVoWFqvJ8TiVk/wG5+QmeyRObpXLTm6ysDgZquzjWr7sdt46N/kNF2QPZvNLlTa0v7b5PRhoHRVzuQ1Jpa6E+beQQLUWtC6gcaZM446Tpv4eWXx8PO+//z7PPPMMAEVFRfj6+mKz2fjhhx9wd3cHwMXFheJiOXrRZDKhVCoxGAzk5eWxefNmunTpQk5hCVoXbyZMnoJPQBBffPEFd9xxB0lJSRw/fpyOHTtiNBrJzMwkLCwMhUKBzWZr3Eg8pQqUzpTZlJd0P+qCEClBy2E1Q+pBOLtd9lQS/4GALhAx0tGLSdkHI5+Wm+YchKX8e0Oab2qjtXgPrQij0djSJrQabDZbnffNy8vj+uuvx2Qy4ebmxoMPPsjw4cMBuOuuu1i4cCEeHh706tWLwkK5n3LChAn85z//4aOPPuLrr7/muuuuY8KECQQEBNCjRw8A0tPTefLJJ5EkCZVKxbPPPotWq2XJkiU899xzGI1GJEni0UcfJSwsjMmTJzNp0iQGDx7MU0891WL3o74oJEmqJXTj8mHv3r1iqo5GpMnuRZlR7vg/+48sTEm75Q7nwB4QNhDCBkObgXLU2EdDKr2YO7e3jBcDnD17tvYw4KsM8RuRSUhIICAgoM5pgK4G6pMWCeR7WNHkCReuw5vUk1q1ahUxMTFoNBoWL15MaGiofdvXX3/Njz/+iEaj4cknn6Rz587873//47vvvkOlUtG1a1eefvrppjRP0BRU7U8KGwyWUoj9Cc7+LUdGKVUQ0hfCBsHQh+Tv50fNqTQt3w9TjvAeBIKWpclEKi8vj9WrV/PNN99w7NgxlixZwtKlSwHIysrip59+YuXKleTm5vLggw+yYsUKBg0axPTp01EoFDz44ION4iEJ6k+9xgZJkhzllnsGijLAxQc+vabSC7r1B3l91AQY+5LsNakv0rnaGvphBAJBq6DJROrQoUP069cPtVpNt27dHNK4Jycn065dO1QqFT4+PqSnp1NWVkabNm3s+2g0GtTqupsXGxvbIHtLS0sbXMbljoerE77KIoKTt2FRDyHT5kpekQmFpQRtUQqa4opPKtriFDTl61QWIxIKrB0nogruiaJKf5I19ShJvZ6QPZIi4OTpFr3G+iKei0rEvZCRJAlJkigpKWlpU1oN9b0fJSUldX6Wmkyk8vPz7ZEqIF9EBW3atOHo0aOUlJSQlpZGSkoK+fn5+Pr6ArB//36ys7Pp3r17nc/X0Lbyq7693VwKWcfh4xF2Lyhw7joC/3wc0g/L++h95KY3jzBo2xs8brAvK9xDUau15VFxL9rLUEUOv6wjwq7656IK4l7IJCQkoFAoRJ9UFerbJ+Xs7FytT6o2mkykDAYDx48fty9XDXv09PRk4cKFLFy4kKCgIDp06ICnpycgPwCvvPIKH374YVOZdvVS0V+UsBVCB0BeIhz9Ts5/5hYgD9o8P6ruujfkSDqPNnUbqNcaxvUIBIIrhiYTqe7du/PBBx9gtVqJi4urFiE1YcIEJkyYQGJiIu+88w5qtZqMjAweffRR3njjDby8vJrKtKuX7FMOnhLzfga/ztB+LAR2lzOf/PlylewGo+sfVSf6kwQCQSPSZIm2PDw8mDJlCrNnz+bll1/m4YcfJiYmhl27dgHw8MMPc+utt/L666/z+OOPA/D222+Tk5PDM888w5w5c9i+fXtTmXf1kXkcTvzq6CmlHYERj0PP2fL4JM+28tiga1+Rw76FFyQQtAqSkpK44447muVco0aN4s0337QvT5w4sdZ9CwoK+OmnnxzWVQwkPn26cfqfmzQEfebMmcycOdO+XNWbeuONN6rtv3jx4qY05+rl3G744W64/j34c3GlpxQ+xHE/kVlAILjq0el0bNu2jdtuuw2D4cI5GgsKCli3bh1jxoyxr/v555+ZM2cOP//8M/fdd1+D7REZJ650zu2CFTdAn3ly014d+ovE2CCBoO7kl5gpNddxGpBa0GlUuDtrLrrfnDlziIqK4ujRo0RGRhIYGMjWrVtp27YtL7/8MqdOneKFF17AZrNhMBhYunQpWq2W5cuXs379etq2bUt8fDzLly/H1dWV5557juRkecqV//u//yMsLAylUsnMmTP58ssvueeeeyqvMz+f//znP+Tl5eHk5MSrr77Kl19+yYEDB1iwYAEPPPAA/fr1Y9euXXz55ZfMnTtXiJTgIiTugK9uhL4LYMwL8hw3or9IIGg0LFYbQ175g0KTpUHluDmp2f/sNahVF++BGT58OE8//TRTp05l2LBh3HXXXcyePZuMjAxCQkL473//i1Kp5I033uC3336jX79+bNy4kVWrVlFcXGz3etasWUN0dDQvv/wyx48fZ+nSpfZmvilTpjBt2jT+9a9/2c/78ccfc/311zN69Gj++OMPPv/8c2699VbOnj3L22+/jbOzMwcOHKB79+5otVo6d+7MkSNH6NKlS4PujRCpK5Wz/8DXN0G/hTD6uQvPNioQCC4JtUrJtidGNYonVReBAujUSQ5m8vPzIyoqyv49Pz+foqIiXnnlFYxGI1lZWXh6epKUlETHjh1RqVQYDAbCw8MBOHnyJAcPHmTTpk0ADiHkGo2Gm266iZUrV9rXnTx5kr179/LFF19gsVho3759NdvWrVvHkSNHWLBgAcXFxWg0GiFSghpI2iN7UAP+DaOeEQIlEDQh7s6aOjXVNQWKKr9tSZJYuXIl06ZNY8yYMSxZsgRJkggODubEiRNYrVaMRiNnzpwBoF27dnTs2JEZM2YAUFZW5lD2TTfdxLRp0zCZ5FkHIiMjGTRoEEOHDrXvn5ubi8Uie5E2m40DBw6wZs0aexnTp09HkiQHO+tLE06jKWhWyorlgbT/vC9/v/59IVACwVXGyJEjefPNN/n3v/9NRkYGAL6+vowZM4Zp06bx/PPP4+/vj1ar5eabb+bw4cPceuutzJkzx8FrAnBycmLKlClkZcmz+N55552sWbOGuXPnMmfOHDZu3GhPwPDII4+wbNmyat5VREQEu3fvbtA1iSzo5Vz2o+nTj56XOXzbJfc9Xfb3ohER96IScS9kLscs6GazGY1GQ0FBAdOnT2f9+vWNWv5lmwVd0AzYrJDwlzwFRtUxUGe2iwAJgUAAwEcffcSuXbsoKiri/vvvb2lz6oUQqcsVmw2OfQ9/vgooYMp7jjPJnj8GSiAQXLXce++9LW3CJSNE6nKh6jxNIX0hdh3s/gQG3CUHSLSiOZgEAoGgsRAidTlgLoXs0/Dx8EpPaf6vMOQ+0HlU7ifGQAkEgisMIVKtgapeUpsBUFoI8X/I+fYyYsEQVEOG8v0Q2rdFzRYIBIKmRoSgtwZyz8iReesfk7OUI0FBKnhHwrBHYdzL8sy2ivL/LtHnJBBcVVwuCWbLyspYvHgxs2bNYubMmXz44Yc0NIBciFRr4Mw2Ry8pIxamfgjXLIIeMyGgqz1DOeNfFxnKBQJBk1GRYLagoOCi+1YkmK3g448/xtnZmZUrV/L1118THx/PDz/80CB7RHNfayB8iOwdXSgyT8zTJBC0TkrywNzAqeQ1zuDscdHdWnuC2XXr1rF27VpAnuj2rrvu4oUXXmDKlCmXfGuESLUGNHqYs1b2oNoOE16SQHC5YLXA213BdHGv44I4GeCxBFBdvEpuzQlmy8rK0Ol09mOCgoJIT09v0K0RItUaOBoDiTth9qqWtkQgENQHlRoeONw4nlQdBApad4JZrVZLaWmpXahSU1Px9/ev//2oghCp1kDcz9BxQktbIRAILgVnjzo11TUFrSnBLMCECRNYtmwZ999/P5Ik8eGHHzJ58uQGXaMInGhpClIgeS9ET2ppSwQCwWVOSyaYPXDgAHfeeSeFhYX26L7Q0NAG9UeBSDBrp8WSZ+76BHZ+BPfsaTUZy0Ui0UrEvahE3AsZkWC2OiLB7JVM3M8QNbHVCJRAILjyEAlmBZdGSR6c+QtGPt3SlggEgiuYyznBrOiTaklO/gp6HwhuWFOlQCAQXKkIkSpHr9c3/0nj1snpjpTiv0EgEAhqQtSO5dOuhyT/Is9uW1bcPOc1l8DJ3yDquuY5XzNhNFmISyvgi78TiEsrwGiyXPwggUAgqAUhUuXJXVW/PiEnec090zznjd8CShWED2ue8zUTiblGJiz9i+d/PMaEpX9xNsdInrGMwlIzJWVWzFbbRRNOCqETCBy5XBLMPvHEE9xwww3MmTOHhQsXUlRU1GB7RODE+cldT28G7/ag1jbteePWQftrm/48zUBBqZmd8Tkk5xopMlmwlWuQTYJNx9LZfiqLnQk5DseolQpUSgUalRK1SoFaqUStVBDkoeOxcVHM+mQHNgmUClh1x0D+iMvAJoHBWY2bToNBp8bgXP5Xp8HgrMFNp8ZZo3IY4NhQWqQZWCBoQSoSzN52220YDIYL7luRYLYi1RLACy+8QNeuXXn//ff58ccfmTVrVoPsESJ1fnLXwO7y5IJth0HPORDQpfHPabPC8fUw4fXGL7sZMFms7E/MY/upLLadyuJQUj6uTmomdAlger82vLnphF1grunkz029QzBZbFisNsxWCYvNhsUmYbFK8jqb/NdikzDoNBxJzncQur1nczHo1BxMyqeg1ExBiYXCUjMFpRYKSsxYbJWemVqpwM0uYLJwySJWIW6aKtvL1zmX76PT4KpTo1IqMJosJOYa+TvRwiCnAtp46tE7iZ+LoGVp7Qlmq1JQUICHh0eDr1n86jzD4c5tWE9vRRU5HAzB8hxO+1fIzX/DH4P2YyFptyxohiDQusrTtVe8sVedtDB8iFym1qX2c57bKSekbH9NM1xg/Tnfe7DZJI6lFrD9VBbbT2ezKyEbSYK+4V5c2ymAFyZ3pnOQu71yX3//UHbE5zAgwptQT+d6V+6eLhqUCuxCN6KjHx0D3GrcV5IkSs02CkrNFJaayS+xlH+XBayw1FIubGYyC02cziyyb6vYz1hmdSizna8r/zelM7OX7yy3IY6YuwZTUmYh2ENPsKczKqUY1yaQKSgroNRS2qAydGodBu2FvZYKWnOC2ZiYGJ577jn74N7GCH1vUpFatWoVMTExaDQaFi9eTGhoqH3b119/zY8//ohGo+HJJ5+kc+fO2Gw2XnjhBU6ePImfnx+vvPKKQ0bdJqF8CoykUlfC/Muzj3e5Qf7kJ0NhCnw6ptLTmrMW1v4bCpJBqQGPcJj0Jnx5feU+d2678JQasesgYgQ41VzxthRVvYcBmnyyisr4dvc5/j6dRX6Jma7B7gxu58OdwyLoFeaJTqOqVobeSU3HAAMdA+r2g6uJNp76akJXGwqFAmetCmetCn/DpT0rZquNwtJy76zEgkIBO+KzHby57aey2HMmh83HM9GqlYR762nr40KErysRVf56ulz+zbeCumOxWRi7ZixF5ob1vbhqXPlrxl+olRevkltzglmQm/uio6O54447SE9Px9XV9ZLuSQVNJlJ5eXmsXr2ab775hmPHjrFkyRKWLl0KQFZWFj/99BMrV64kNzeXBx98kBUrVrB161aUSiUrV65k+fLlfPfdd8yePbupTHTAaDRWX+keLPcdOUxIeAzmroPSPLCa5ezFZ/5y3OfM9tpFSpLkMoc+1CTX0RDOZBcz8d1tdg9m5cIBtPVxYVL3IAZGeOOu1zSLHY0hdPVBo1Li5aLFq4rAqFUKB29uTLQ/d42IJLu4jPjMYhKyiojPLOZkehG/Hk3jbLYRi03CU68hwte1XMBciPBxJcLXhTBvPU7q6qIuuLxRK9VsvGljo3hSdRGo82ltCWYrUKvV3HXXXbz77ru8/fbb9b4uh7IadPQFOHToEP369UOtVtOtWzcSEhLs25KTk2nXrh0qlQofHx/S09MpKytjz549jBgxApATJb733nvNJlK1cn6fVdvh4B3huI9S5bhP2KDay0s/CnmJrSbruSRJ7IjPYdupLJzUSgfv4XhaIY+M7diyBrYQFd7c9pOZDG7vR6inMwqFAh9XJ3xcnejX1sthf4vVxrncEuIzi0jIKuZ0ZjFbjmfyxfYzZBSaUCogxFNPhK+L3QOL9HGhra8LAQZdowZ7CJoXg9ZQ56a6pmbkyJG89NJLfPfdd7i5ueHp6emQYDY8PNwhwewLL7zAL7/8giRJjB49mnnz5tnLqkgw+8477wBygtlnn32W5cuXY7PZmDZtGtddJw+heeSRR1i4cKGDLb179+bVV18lMTGRNm3aXPI1NVmC2Z9++onU1FRuv/12ACZNmmQPVczNzWX+/PmsXLmStLQ0Jk2axObNm3nnnXe48cYb6dGjBxkZGTzxxBN89tlnFz3X3r17GxyFVXUOlKp4uDrhqypCcWY7UvgQMq0u5BWZat1HEdQdy4nfiQ+ciKSuXp7PkeW4pO/i7OiPG2TvpaJWq9FqtRSXlrLpeD4/xOaTmF/GDV28uWVoB6Z88I/de1h3zyA0xqwa35SuBtTqyne4htyD4jIbyQVmkgrK5L/5ZpIKzCQXmDFZJHRqBcEGDSEGDcHuGkIMWkLcNQQbNHh6uFOidGFnQg7923rhbCvGVFzYGJdXb2r7jVxtSJJEeHj4ZfViUTXB7Ny5c/n+++8btXxJkup1PxISEhz2NxqNzZ9g1mAwcPz4cfuyskpWBU9PTxYuXMjChQsJCgqiQ4cOeHp6YjAYKCiQZ7gsLCzE3d29zudraHbmi2Z4DuoGQGD5p9Z9SvPQrrufqJw4mPal7GVVZctO6DW9RbJJG00WzmQb+eNkJt1CvMnDwuTeHszuH4avm5M96KGq96B38mx2O1sTjZX5u08N6yRJIq2glPjMYuKzionPlJsQ/zpXSFJuCQEGHW9O82f28r/tLw5r7hzE8QIrKpUSD2cNHnotHnoN7s7yp6Z+wsbi7NmzhIWJWaMrKtjLKQv68uXL7QlmH3jggUa3vb5Z0J2dnatlQa+NJhOp7t2788EHH2C1WomLi6v2cE+YMIEJEyaQmJjIO++8g1qtpm/fvmzdupVhw4axdetW+vSp6afdytF5wOzVsHwM/PoMjHu5clvuWUg7DDf/t9nNstkkTmQUcsMHlRXeL/cNJSqwspmioi9IZ8olrJZoOkHjoVAoCHR3JtDdmcHtfBy2lZqtZBaa2Hg0zaEJdmdCNmn5Jv6JzyLPaCavxEyZxWY/TqdR4u6swcNZi3u5eMliVi5kei0e5YLmoS/fr3yMmbKWiEURjn/5czknmG2yJ83Dw4MpU6Ywe/Zs1Go1L730EjExMYSEhNCvXz8efvhhMjMzcXd359lnnwVg2LBh/PHHH8yaNQtfX19eeeWVpjKvafEMh1nfwhcTwSMMBtwpr4/7Gfw6gXdks5qTml/Cp9sScHfWnFfh5TiIVAU1BpEImhWdRkWol54h7X0cAjhGRflXC8cvNVvJM5rJLzGTZywjr0T+nm80k1dSRn6JmdT8UuLSCsv3kfcrNFmoaOxXKLB7Yx7lYuburKGNl57RUX7c9NHf9nD8tXcPpqTMilKpQKdWodMo0WlU5R/5u0bV+MlsKsRyR3w2AyK8hVheJTTp//DMmTOZOXOmfbmqN/XGG29U21+pVLJo0aKmNKn5CO4NN34Kq+aAewhET5QDJrrPvPixjcgPB5L5z9ojjIzy44aeIbxVZaDtgAjvZrVFUH/qEo6v06gIcFcR4F6//iKrTaKw1FwpcOUiV1AhZCVmXJ1U7ExwDMffeiKTvWdz2X4628GLq4paqXAQLft3tTxcwKlc3JyrbHPWqHA6b7niu0EnN2VOfm+bQ0tAxwC3FukbUoqk0M2GeA1pSqImwNiX5USy7sGg94KI4fLg3wsN9m0E8o1mnvnhCL8eTePp66KZMyCMkjJrnccfCVoHTRmOr1Iqyvu0Ljy2Ky6tgNc3Hq+SRSSAe0bJY2RsNolSi5VSs41Ss5VSs5USs7xsqvK91Gyl1GKlpMyKySIvl5SVrzNbyTWWlR9vq1JG5XLXYHd6hXk6iOWvx9JZtO4Yx1ILcNGqcdOpcXFS41r+cXFS4eqkwdVJhet52+Ttjsc4qZUXFTyjyUKJ2UphmYRNaUWrUoqB3U2MEKmmpv/tcG4XfDJKDlH/8+WLD/a9BKo2hfRu48XaA8mczS7m5/uG0s5PHkzX3OOPBFcGNYXjV6BUKtBr1VxE5xqFuLQC3v7tRJWxa36M6OhLrtFMsclCUamFIpP8KTZZKDRZyC42cTbbcX2RyUqRyUyp2dELVCsVjkJWLmBu5YIX5O7MsA6+nEktwKIvRZFfSqSfKzYJFMg2KRUKlAoFCmXl96uBpvQshUg1Byn76z7Y9xKpyD5ujwL79yCeGB/VJH0DgquL1hJQU1PTZ0P6pCxWG8UmK0Vl1QWu2rLJgkqlYGdCNv7luiMBReVpteT+veqjeRQKBTeN6sd1N0znjvsfRamAlHNnuX32jdz36FNMvuFmFFXETVlF3L5duYLpM2ah1ahJS03hicceYeU3/7MLn9kqUVxmwcVJfVGP7sEHH2TGjBn079//ku9XTVhtEmVWG8VN6FkKkWoOzh8QHD6k3kXU1GlsstjYmZBDWkEJxedlHz+clE+vNld3+LigcWnpgJrGbglQq5S465X1yqQSl1bAsePyMBkFYHDW4FeejkuSJGyShE2Sm0FtEtgkCU9PL07HHsbXTYskwdq/NtM2sh2q8pkAbDYJi03CJtkqj5ckVn61goHXTEat0ZKWXUyp2UpcWiEalZJQT2cSsoqRyu1o6+NCSn4p1vJKQKEABQr798JSM8l5Rk5lFJVvo3ybvFeFw2dfLr/Aasso7PuqlAr0GhWnM4tkO/JLae/vhur8YTcNRIhUc1CexJYz28sT0NZ/rMn5ntLKhQN48NsDmCw2xnbyZ3q/UN74VQRFCARNSRtPPbZAA37uOlx1GoeWCoVCgUqhQAVQpZ5WqZT07NGdM8eP0qtXL/bu3M7okSNwd9YQ5OHMmTNneOGFF8jPz8fT05NXX32VjRs3kpudxbP3LSAiIoKHH34YNRJfvPl/HDt2lO69+jD7rkcA+GPDT6z/7huUCgXTZs5m6k03I0kSH73/Lr9v+pWAwEAkScLNSR52IEkgIVH+rzzCU7J/lxflZVu56FUcU+EsSiBnqLHJ+1WsKzJZGn2snhCp5qA8iW1DmvjOT3h6OCmPVXcMJNjDGWUt2cdbAqPZSFJREnvS9tAnoA8hriHoNfXLBtIYZQgETYHeSZ6zzE2rwKkelfG4cePYsGEDnp6eBAQEOGTuWLRoES+++CLBwcGsX7+eDz74gGeffZZPP/2U1atX4+TkRFJSEqdPn+Ltt98iNDSUKVOnkpaUiMbJiZivv2D1mjXoneSksGNGDiMnJ4d9u3fyy8/ryMnJYdy4cRicNfi4OjXq/SgxW0kvKLV7dK5NMCRAiNRlQv+23g7jZYZ18CPUq7Libi1BEUlFSdz8083YJBtKhZL/Xfc/tCotFpucVqjivaui/d6+XP5XrVBjk2zM+HmGvYw1k9bQ3rPmjMsCQUtgs9Ucel8bffv2ZcmSJXh4eDB27FhOnToFQFFREfv27eOuu+6yl1t1toiqREZG2jOYR3WMQmPKJzeziGFDh+Dt6YFKqWD48OEcOnSIjIwMrrnmGjQaDf7+/k2WGEGrUtLe342iUnM1z7KxECJ1maAAvlrQnxMZhQyM8GlV4eNmm5kDGQeIz4+nwFSArTxIxCbZ2Jq0lR2pO9iTvqdOZfXx78OAwAEOZfyT8g9t3NrgpG7ct0CBoLlQqVR06dKF//3vf2zYsMEuUpIk4e/vzw8//HDRMrTayhBKlUqJSiFhcNbY+7bOp2o4fVONJZPPrUJhs9TLs6wPQqQuE3bEZ7PlRCaf/6tfS5sCQFZJFn8l/cVfyX/xT8o/mG1mrg27lplRM3nvwHt2L2hUm1HM7zofqNKRS/UfT9V1J/NO8sHBD+xlRHlFceNPN9Leoz2j2oxiWMgw3J3qntdRIGgNzJkzh169ejkkw3Zzc8NgMLBt2zaGDBmC2WwmMTGRyMhIXFxcKC4uxsmp9pezrl278vrrr1NUVIRCoWDr1q3MmTOHnJwcFi9ezNy5c8nJyWHPnj0sWLCgya6tvp5lfRAidZlwJKWg3hkFLpWaMsrbJBtHso7wV/JfbE3ayrHsYwS5BDE0ZCivDnuVvgF9cVY7YzQbWTNpDbvTdtM3oC/BrsFolPWbhyrENcShjEB9IE/1f4o/Ev/grb1v8ez2Z+kT0IfRbUYzMnQk/i7+jXXpAkGTERkZSWRk9ZRoS5Ys4fnnn+e1117DarWyYMECIiMjufnmm5k1axZRUVE88sgjNZbp7+/PwoUL7XNCzZs3j5CQEEJCQhgwYAATJ04kJCSEHj16NOWlNSlNNlVHc7J3795a07zXlcbKdt1UjHt7K7cMCOOWAU2XhboiYGFnyk76B/XHR+fDrrRdbE3ayvaU7RSYCujp35OhwUMZFjKMCPeIZk9JUyGWvyf+zh+Jf3Cm4Axdfboyqs0oRrUZRYR7xMULqQet/bloTsS9kElISCAgIOCyyoLe1NQ3C3pCQkK1LOjNPlWHoPEoNVs5mVFEl+CmbeI6P+jh42s+5tPDn9LRqyNP93+agUEDcdO2bHZ0pUJJN99udPPtxoO9HyQ+L57fE3/n97O/s3TfUtq6t2V0m9GMCh1FZ5/OKBViMLNAcDkjROoy4HiaPMldVBON9s8rzWNvxl7i8+IdAhZO5J7g20nftuqKPsIjggiPCBZ2W0hacRqbz23m98Tf+fzI53g7ezMqVPaw+gT0qXezo0AgaHmESF0GHE0poJ2va4MHyZ0//qjEXMLnRz9nS9IWunh34dG+j6I8oLR7UgMCB7RqgTqfAJcAZkbNZGbUTPJN+WxN2srvib9z3x/3oVVpGR4ynFFtRjEoaJAYdyUQXCYIkboMOJKST+fgho9/qqk5r41bGz4b+xk9fHtQYilhzaQ19j6pYNfgRrC+ZXB3cmdS5CQmRU6ixFLCPyn/8Hvi7zz/z/OUWkoZGDSQ0W1GMyJkBB46j5Y2VyAQ1IIQqcuAo8n5XN+j4YKxM3WnQ3Pe6bzTPNTnIft2vUZPe8/2aAu0hF1C6qbWirPa2R5YYbFZ2Je+j98Tf+f9A+/z/N/P08u/l70fK9A10OHYmiIdBQJB8yFEqpVjttqITSvk6QYGTUiSRCfvTigVlc15fQP61rhvSycSbUrUSjX9AvvRL7AfT/R7gmM5x/gj8Q/WnFjDK7teIdormtFtRjM+fDyltlJ2Fu2kf25/kZpJ0KIkJSVx44030qFDBwB8fHx46623Wtiq5kGIVCvnVEYRZRYbnYIa1ty3+dxmtiVtY9XEVexN32sfw3Q1o1Ao6Ozdmc7enbm3572cLTjLH4l/sD9jP939unPHpjvsgr560mo6eHZoaZMFVzE9evRg2bJlDSrDarWiUjVNZoimQohUK+dIcj4RPi4NStxYZi1jyZ4lTIyYSEevjnT06tiIFl45hBnC+FeXf/Ev/sXXsV87NI1uTtzM5nObCXMLY1DwIAxaMXGkQKawsJDS0tIGlaHT6XBzq1/07pw5c+jUqROxsbG4u7vz7rvvAvDqq69y5MgRLBYLTz75JN26deO6665j2LBhxMbG8t577/HAAw9gtVpp164diYmJvPrqq9x7772sWLECgKeffpobbrihweNPGwMhUq2coykFdG5gU983cd9gspqY13le4xh1FdA3oK9D0+jQ4KH8kvALr8S+Qr4pn17+vRgeMpwRoSNoY2jT0uYKWgiLxcLEiRMpLi5uUDkuLi78/vvvqNW1V8kHDhxgzpw5AHTuLM+oMHToUJ588knuvPNOjh8/TlpaGjabjRUrVpCTk8P999/PihUrKCkpYfLkyTz++ON89tlnDBkyhHnz5vH999+TmJiIh4cHPj4+nD59muDgYI4fP94qBAqESLV6jiTnc02nS0/7k1Oaw0cHP+Kp/k+JPpV6UJGaqWqk4yN9H+GhPg9xJOsIW5K28OPpH3l9z+u0dW/LiJARDA8dTnff7qiV4md1taBWq1m3bl2jeFIXEiio3txX4UkBBAUFkZ+fz8mTJ9m2bZtdzCrE08nJyZ4t5MyZM1x//fUAdOvWjQ0bNgBw00038f3339OhQweuvfbaBl1PYyJ+Ta0Yq03iWGoBD15z6X0h7+9/n7bubbku4rpGtOzKp7ZIx6oZL+7teS+pRalsSdrCn0l/8lXsV+g1eoYED2FEyAgGBw9u8QwdgqbHzc2t3k11TYEkSbRr145Ro0bx8MMPA1BWVgbg0A8VFhbGkSNH6N27N4cPH7avHzhwIO+++y6xsbG8/PLLzWv8BRAi1Yo5k12MscxK50sMmjiRe4LvTn7HF+O+uKwG5bYmLhbpGOgayIyoGcyImoHRbOSflH/4M+lPXt71MgWmAnr792Z46HBGhIwg1FDzPEECQV2o2txXW2b0ESNGsGfPHvt+3bp149FHH3XYZ9q0aTzwwANs3ryZ8PBwNBo5E4tSqWTIkCEcPnwYPz+/JryS+iFEqhVzJDmfYA9nPPTai+98HpIk8dru17g2/Fp6+PVofOME1dBr9IwOG83osNHYJBuHsw6z5dwW1p5ay2u7XyPCPcIuWN18u4lmQUGdCQkJYefOnbVuf/bZZ+3fa8qYvm7dOvt3V1dXli1bhlqt5vvvv7fPbQVyxOtNN93USFY3DuJX0oo5mlJAl0vMNLElaQsHMg7w05SfGtkqQV1QKpR09+1Od9/u3NfrPlKKUtiStIUt57bw1TG5WXBo8FCGhw5ncJBoFhQ0HxaLhblz56JQKFAqlSxZsgSAxYsXk5CQwJ133tnCFjoiRKoVcyQ5n4ER3vU+zmw1s2TPEuZ2nlstg4KgZQhyDbLnFSw2F8vNguf+5OWd5c2CAb3twRehbqJZUNB0aDQaVq5cWW39U0891QLWXBwhUs3A+Yld65K9QJIkjiTns3Bo/edH+ibuG4xmIwu6NN1MnIJLx0XjwpiwMYwJG4PVZuVI9hG2nNtCzKkYXt39KpHukQwLHcaIkBF09OxIcnFyvZ4dgeBKQohUM3B+Ytc1k9bQ3rP9hY/JLaGg1FLvxLKFZYXEnIzh/l73i8rsMkClVDk0CyYXJbPl3Ba2JG3h1zO/smjQIhZuWigyXwiuWoRINQN70vY4ZC/Ynbb7oiJ1JDkfPzcn/NzkKeMv5o1VbN+WvI1nBjxDJ+9Ol2yvSKracgS7BjMrehazomdRYilh1fFV1TJfxJyIQavW0tWnK119uuKv92/2GZIFguaiSUVq1apVxMTEoNFoWLx4MaGhlW3t33//PV999RVKpZKJEycyd+5czGYzDz/8MFlZWUiSxAsvvGBPqNjUNGXF3Nu/d50Su1blSEq+w0y853tjn177Kf87/j/MVjMhriFcE34NczfMrZe3dj5Gk4XEXCN/J1oY5FRAG089+gakYxI0DGe1MwODBjpmvggZyv6M/exI3cGPp34kuzQbH2cfu2B18elCF58uIhBDcMXQZDVQXl4eq1ev5ptvvuHYsWMsWbKEpUuX2rcvW7aMmJgYdDodEydOZObMmezevRtXV1feeecd9uzZw/Lly3nttdeaykSZsmLIPUNI8lbQDQPPcNC6NOopcktzWX7tcg5mHKRfYL86JXY9klxA95BKkTp/mo2j2UcZGTqSU3mn8NJ5sTd9b729tfNJzDEy4Z2/sEmgVMTx871D0WmU2ACVQoFSocDf3QmtSklGoQmFQl6vUipQq5S4OqkpKjWTlFvCjoRsBkR4C6FrIBWZL3an7bYnBe7k3YnZ0bORJIm04jQOZx3mSNYRtqds55PDn1BiKaGte1u7cHX16UoHzw5oVGJm4ssVq9XKf/7zHxISEgDo1atXtfFPFTz33HOcPn2a++67j5KSEoYPH96cpjY6TVZ7HDp0iH79+qFWq+nWrZv95lYQERFhHyip0+lQqVSEhoZiNpsBKCgowMvLq6nMqyT3DHw0BJVkA4US7twG/p0b9RSrT6wmzBBGtHc0b+19i8/GfnbB/SVJ4mhKPjP7VeaE6+bTzeGNelDQIAcROpF7gnf2v3NBb63CU9oRn82Att608dJTZLLwwZ+nAfB21WKT5H1tEmw5mcHmuEx2JuTYy1h37xCiAw30X/y7Q9ndQ9z54Z4hnMos5oYPtpcLHfxwzxCiAtzQqMRg4kuhIvNFTS8cCoWCQNdAAl0DuTZcTmNjsVmIz4/ncOZhDmcdZs2JNby862XUCjVR3lEOwhXqFiqaCS8Ttm3bhpOTE9988w0gOwG1sW/fPn766Sd27tzJli1bhEjVRn5+Pu7ulZ6AJEkO28eOHcuUKVNQqVQsWLAAlUpFQEAARqOR8ePHYzQa7Rl560JsbGy9bdTr9YQkb5UFCkCyYT39J0mlro02p5JVsrI9aTs9Inqgz9OzN30vG/dtJExf+6SC2UYLWUVl6EoyiI3NxSbZ+KVgPV+O+4r96QcZENQPndnF4Zp1bjq+nfAte9L30Me/D+piNbFpldsTC2z4+Adw87IdlQJy1yDM+RkkpGTSI9iNoe0CeWsT9u1D23kzxN+KaaABSQKbJEFeMsfzk/n65jZYy9dJEqiUCs6ePcveRIuD0G2OyyA1NY1QFxtbzxTR1V+Hp/Pl41mVlpZe0rPV0nSiE508OzHdczql1lISjAmcKjrFyfST/Hr6VzLLMnFVudLOtR3tXNrZ/xo0NQfqqNVq1Go1J0+exGKxNPPVtC4kSUKSJEpKSgA5K8n59YWbmxtOTk5kZ2c71H0KhQJvb29MJhOFhYX29Xq9/oJdDgqFglOnThEXF0dYWBhOTk4cO3aMF198EZvNRnh4OM8++yyvv/46iYmJzJo1C4VC/k3GxcXxn//8h4ceeogBAwZw4MABhg8fTl5eHgcPHmTIkCHcdddd7Nq1i48//piysjIiIiJ4/vnnOXPmDIsWLWLZsmX8+uuvHDlyhMcff7zGe1JxP+pCSUlJnX9XTVZbGAwGjh8/bl9WKivfpIuKili2bBkbNmxAq9Uyf/58xowZw19//UVYWBjvv/8+J0+e5Pnnn+ezzy7sdVRQkTyx3uiGyR5UuSelCuxKWNYW6DkblA2fd+VAxgFMkokb+96IXqOnd2pvTihOMC56XK3H/B6bjoc+leF9uqJQKPgn5R9W713Nuq3dCHGP5JPcc3xyax98gr1Qq5Q4qZUogJT8Erxt7mBx4VSBkR8P5hEd6MZdI9qxc3sCpxJyHQRk77k85g3qyH+7yFN3GE0W1t8/lO0nMxnc3o9QT2f0wZ71ut7BTgUoFXF2obu2sz+Rvq4UlVr48odtpORn0C3YnVFR/ozt4k9UQOue8iI2NvbSn61WRE96OixnlWRxJOuIvalw45mNFJYVEuwaXOlt+XYlyisKSZJIKkqyJ9sNcw27qiNHExISUCgUODs7A/Dll1/yySefOOzz5ptvMmzYMG677TaysrLs6318fNiwYQO7d+/moYcqZ8VeuHAhd9xxR63nHDZsGGlpaSxatIisrCweeOABfvrpJ5566im6dOnC888/z99//81zzz3H7t27WblyJTt37mTjxo32bBQmk4lbb72Vp59+mqFDh7J8+XKee+45xo8fz0MPPUTfvn3tXtdDDz3EiRMn6N69O9dffz0vv/wyCQkJ/Pe//60xJVNJSYn9ftQFZ2dn2rZta1/eu3dvrfs2mUh1796dDz74AKvValf/CpRKJRqNBr1ej1KpRKfTUVRUhM1mw9NTrhQNBgMFBQVNZV4lnuFw5zasp7eiihwOJblwNAZ6zARJCQ1sDtmWvI3efr3tP+qx4WNZcWwF9/a8t9amliPJBXQJcrdvXxe/jgh9f/bkSCTnyE1vv8VmsP1UFjsTcgh01/Hp3L5MfPcvuzh8c/sAwr30RAfKIjBvcFvi0gp449fj9n0GRvg4nFfvpKZjgAGdKZewgEvreG/jqWf9/UPZEZ/DgAhvQj2d0aiUeLpo2fb4SI6nF/JHXAab4zIoLDXzzMRO/HgwBY1SwZD2PrjpNI7NkqJfq0nwcfZhROgIRoSOAOR+zMSCRA5nyc2EG85s4O19b+Pr7MtLQ15iwa8LKsPgJ66mg5cIg6/glltu4YYbbnBYZzDIv7uvvvqqmicF0L9/f9avX29fX5fArWnTpjFt2jSys7OZMWMGGo2GLl26ANCzZ89qXSrnU1UYvL296dSpEwqFAoPBQElJCcePH+edd97BbDaTkpLC2LFj6d69OzfccANvvfUWjz32WK05A5uSJvvle3h4MGXKFGbPno1areall14iJiaGkJAQ+vXrx9SpU5k+fToKhYIePXrQoUMHgoODefjhh9m6dSslJSU8+OCDTWVeJVoX8O9MUqkrYf7lQho+WP67+l8QPRl8O8CZbRA+pN6BFduStzEuvNJrGhM2hpd3vUxcThzR3jW/oR9JybePjzKajWw6u4n7urzAvj22Sg+lkz+z+oVSYrahVCjYFJvm4CXFpRby8FjHyQ1rEpCaaEhTZ4XQdazBQ1IoFEQFGIgKMHDXiHb2H29sagH/25VIkcnC2M4B3DakLTd8+Lf9WtffP7TG8gSNh1KhJNw9nHD3cCZFTgLkyTKTCpPYmrzVMQz+3GY+PPghRouRCPcIIj0iaefRjrbubXF3atjcZ5cjLi4uuLjUXCf4+PjUuN7JyQlfX986nyMjIwO9Xo+rqytubm5oNBqCg4M5cuQIXbp0Yf/+/QwZMsThGI1G49A0e/5LcdVlSZJYtmwZjz32GNHR0Tz44IP23+eSJUv497//zapVq7jmmmvsAtxcNOnr6cyZM5k5c6Z9uao3NW/ePObNm+ewv4uLCx999FFTmlQrNVbMfebLfz8aYm8OrE9gRXZJNkezj/J/g//Pvs7H2Yc+/n3YeGZjrSJ1NDmfyd2DAHnad2e1M2HOPfl6oYYTaYV2ganqXQyI8EapqOxPGlBDOqULCUhLUPEjeXxcFI9c25ED53I5mV7EP/HZDoL7T3x2q7H5akKr0hLhEYFFsvCW4i27JzU8dDhRXlEcyzlGfF48O1J3cKbgDBabBV9nXyI8Imjn0c5BwK5G8WpM0tLSWLx4MSqVCovFwsKFC+natau9KS88PJxRo0Y5HNOhQwdOnTrFfffdV2skYFXGjRvHI488QkREhF2gtm3bRmZmJk899RQdOnTg+eef580332z8C7wACun8iIbLkL179zZ4Fsla+x52LoP1j1Uuj38d+t9epzLXxa/jrb1v8dtNvzm8taw6vorPjnzG+hvWV3u7yS4y0fvF39j8yAja+rhw5293Em4IJ+vseDycNTwzseZBukaThXO5Rgcv6VKbyFq6HyYurYAJSyubLr+6rT8/HUhhdLQ/wzv6NmukYEvfi9aA0WwkuSjZYQLI8/ukLDYL5wrPcTrvtPzJP018XjwJ+QmU2crw1nkT6REpf9wjifCQBcxL1wwRvI1MQkICAQEB9eqDudKpb59UQkJCtT6p2upw0dB/McKHOARWED7k4seUsz15O0OCh1QTojFhY3hp50scyz5GZx9Hr+xoSgGuTmrCvPRklWTxT8o/3NvjXu7YnM6DY2rvB2htXlJDcGyW9MImyR7Vg98e4Kd7hxDk4czpzCJ7f5ugaaltAsiqqJVq2rq3pa17W8aEjbGvt9gsJBclO4jXdye/Iz4/HpPVhJfOy+5xVRUwb523CI8XAEKkLk55YAVntssCVcc5gGySjb9T/ubp/k/XEAhgoF9APzae2VhNpI6k5NMpyIBSqeCX+F/k8VVe0Tw3yYeebTwa//paITUJ7qs3dWPRlM44qVXsjM9mxic7iA4wcFPvEK7vEYS3a/N36F5tXEpfpVqpJswQRpghjFFtKpujrDYrKUUpnM6XxSs+P561p9aSkJ9AiaUEDycPR/EqF7BcU65ItnuVUWuNW1hYSH5+PiEhIQ7rk5KS8PDwwNXVtcmNaxWUB1bg3xnSDsOHQ2H+Rgjpc8HDYnNiyTflMyBogJzJoUrz1fr7hzIufBwfH/qYB3s/6PDGeDRFjuwDublwUsQkbBJc08m/SS/zcsBJLQ8J6B/hzfbHR/H9/mRW7DjL4l9i+fCW3lzTyR+bTUKpFG/grR2VUkWoIZRQQ6g9whDkl7vU4tRKzyvvND/H/0xhWSFPD3iaOzbdYe8bWzlhJaFuoRicmtejliTpqh8r1hDMZnO1cbMXolaRWrRoEVOnTq0mUufOnePdd9/l1VdfvXQrL1cCukLf2+C722Tvyql2od6evJ1uvt0waA3ExCc4BALsiM/h+l6j+b8d/8fhrMN08+1mP+5ocj6jRvlxKvcUsTmxvD3ybZ6IOYyPqxNPjI9q6iu8bAjycObuke24a0Qk+xLziPBxoaDUzLi3tnJt5wBu6h1C5yCDaDK6zFAqlAS7BhPsGsywkGH29ZIk8XXs1w5RhtuSt7EjdQfZpdn08O1BD78edPftTlv3tigVTddv6efnx9mzZ0Ui5irUp09KqVTWa3r6WkUqPj6eQYMGVVs/cOBAFi9eXOcTXHGMeQEStsKGx+H692vdbXvydgYHyaHsfcO9qkXeeejcGBA4gI1nNtpFqqDUzJlsI12C3fkpfjV9/PsQ5BrEjvi4C/ZHXc0oFAp6h8lj60wWK09MiOa7vUlMfm8bHfzdeGBMe8Z1ERM/Xu4oFAr6BfZzSA02qs0oxoaPZW/6Xg5kHuCLo1+QkJ+Am9bNPv1JD78edPXpioum8fJxurq6olQqHTr+r3ZiY2Ob7H5csLmvNkwmU5MYc1mg0cENn8CX10N+MrhXTxZbUFbAwcyDPNZXjgoM9nDmx3uGsDkug2EdfPE3yP0nY8PH8sHBD3i4z8MoFUqOpRTgpFbS1seZn//6mbt63MW5HCNJuSX0j7j8oqCaGye1isndg5jcPYj0glK+35+MVi2/Ub/92wk6+rsxKtrP3mwouLyoKdmuXqMn3D2cGzvcCEBeaR6Hsg5xIOMAu9N289mRzzBZTbT3aG/3tHr49iDELUR42ZcJtYpU+/bt+fHHH5k8ebLD+p9++onIyMgmN6xVE9AFHjgMWj1IUrWsFDtTd+Lu5E60dzRxaQWsP5LGmj3nCPHU882uRD77V1889FpGtRnFoh2LOJR5iB5+PTiSnE90oIH9mXvJM+VxTdg1bDycQ6iXMyGeommhPvgbdNw5XH5OJUkiz2jmiZjDKBRwffcgbu4T6jAViqD1c6FkuxV46DwYFjLM3lRosVk4mXuSA5kHOJh5kPcPvE9yUTJeOi+7p9XdtzudvTujU+ua61IE9aBWkfrPf/7D3XffzXfffUd0dDQKhYKjR49SVFTEBx980Jw2tk60eohdB/v+CzO/hSq5Cbcnb2dQ0CCUCiUfbD7NyYxCkvNKSc4rBeDXo+nkGc30DfdiYOBANp7ZSA+/HnLQRLCBn05/y4jQEbhp3ZCkPK7vfvGpPQS1o1AoeH5yZ56cEMXvsRms2ZvE8r/ieXtGz8rJJQ2igroSUSvVRHtHE+0dzcwoObFAVkkWBzMOciDzAJsTN/P+/vexSTaivaPlZkI/2dsKcAmotVzRH9V81CpSAQEBfPfdd/zzzz+cOnUKgKFDh9bYT3XVEtwbfrwX/nkPBt8HyG/t25K38UDvBziXY+Tnw6m8Oa07D357wB484e6s5pnvj2C22RjQrQ8bz3zFo30fRamAHm30vB67ideGyfNo3dwntLazC+qJk1rFhK6BTOgaiK38P+PjrfGsO5TCsA6+3NQ7hDHR/ug0lc2BojK68vBx9mF02GhGh40G5PRPsTmxHMiQva3Xd71ORkkG/np/evj1oIev7G1FeUVhtpnlZLtFO+mf21+EwTcDdRr0U9F2K9pwz8MQCJPfgZObIPUgJO7glHcYGcYMBgUN4u2N8fQN9+SaaH++WtCf2LQCfo/N4HhaEd/fPYj/7TrHp38XURKYy7eHthLmHUy2tB8nlRODggeRml/CF9vP8MjYjmI+pkamIkx96YweLBwawZq953hm7RFe23CcLY+OoKjUQnJeiZil+CpAq9LaAy1AftFMLU7lYOZBDmQcYF38OpbsWUKwazCLBi9i/sb59uCNb6/7lgCXAFy1rqjrOIZSUD9qvatJSUnce++9uLm52dPC/Prrr7z66qu899571ULTr1qiJ4GLH3w8AiQb290NdArvhZvGg03H9vPyDV0xmq08vPogP9wzGHdnLW/8epyXpnZh4bAI5g0OZ8GGn3lh80pK067HOfR/TOo0Go1Sw/ZT6fxyJJUnJ1zdaXmaEoVCQdcQd7qGuPPUddGcyiiS5+7JLOJGe5LbOJHk9ipCoVAQ5BpEkGsQ49uOB+TUUImFiexI2eEQBr8laQs7UnewJ30Pblo33LXueDh54O7kbv9UXfZw8rDvY3Ay4KZ1a9Jw+SuBWkXq+eef54477mDcOMd5jzZu3Mhzzz3Hp59+2uTGXTakHpDTJgHbnHUM1nijUSn5/eHhOGtUHEzKJ6PQhJdey5hoPx7/7hCHkvLpHuqBRqUkUNUfldsnKLJGonI5iZck5wasmEFX0Dw4qVV0Lh9IfeBcnsPYtt9i0/FxdcLLRQuIVoWrDb1GT5RXFEqFkrf2VSbbHdlmJNe3u57s0mzyS/PJL8snz5RHvinf/kkqTCK/TP6eZ8qjwFSAhPxwKRVKDFpDNSGruq5C0DycPOzrFChIKkq6KrJvXNCTOl+gQJ5R96233mpSoy47yvP7GZHYp3Pi3+0msvyveGb1b4NCoSAp10iAQYdapcRDr+XNad0JdK/sqJ/eZRx7C74lqPcJko1RTIkeAMgiJcZHtQwDIx2zyg9p54unXsuhpHwe/PYA47sGML5LoBgwfJVREQZ/frLdCwVZnI9NslFYVmgXLbt4lRWQZ8ojrzSP7NJs4vPjHYSt0CwPC/LX+/PSkJe4fdPtdrFcMX4FmcZMAlwDCDeEN+q4sJamVpGSJAmj0Vit47ioqKheKS2uCjzDYcEmdh/5Gl3235TYevH59njmDQoHIDm3hJAqczdd3yOYMovNvhzp7carwxezJ203/QOnEOimJClXjI9qSSqS3FadpVipVNDGS8+/Bofzy+E0PvzzNMGezvzf9V0Y0bHuI+gFly91SbZ7MZQKpd1rakObOh9ntpkpMBVgspr4PfF3h2bH3Wm7icuJY+OZjUhI+Dn7yfODGcId/ga5BKFqhBnHm5NaRWrWrFnMnz+fRx55xN4nFRsby5tvvukwR5QAOb+f2ci2jD30Dx3E4z/Gs3BoW9TlwQ5JuSUEVxGp/BIzg1/5gx/uGUykryvppSnM3/gvbJKNDw5+wJpJa2jn0Y4/Hh4uxke1ELXNUuzpomXOwHDmDAwnq8jEpmPphHrpsdokpry/nd5hnozvEkCfcC9UIofgFUtDJga9VDRKDd7OcvN//8D+Dtk3hoUMY0HXBbw45EUSCxI5U3CGM/lnOFNwhg0JG0goSKCwrBCNUkMbtzbVBKw1T1hZq0jNnTuXwMBA3njjDU6fPo1CoSAiIoJ58+YxduzY5rTx8iCgK9vLMhlU7IuxzMK0vpWh40m5RrqGeNiX3Z01tPVxYdOxdCKHu7InbU+1tyJXRYiDsAlahgtVRj6uTszsJ78Jl1lszB0UzoYjqcz5dBcGZw23DW1rH1AsEDQmNWXfAHBSOdU44FmSJHJKczhbcNYuYKfzT/N74u8kFSZhkSx4OnkSZghzELC2hraEuoWiUWkuaE9TDtW4YMzktddey7XXXlttvclkapG57lszieZCzmk0eCbmc+vAcPTaylubnFfC+K6O+eOu7eTPxqNp3Dk8kj4BfRzeivoG9OXulfuY3T+MG3uLKMrLAa1ayU29Q7ipdwiFpWb+iMtAW+5J/9+6YxSWmhnfNZDBkT72VE0CwaVSl+wbVVEoFHg7e+Pt7E0v/14O28w2M8mFyZwpOMPZgrMk5CfwV/JfrDi2guzSbFQKFcGuwXbxCjOE0da9rb3vq6nHjV1QpNLT00lPTycqKgqtVktWVhYrVqxg7dq1bNmypVENudzZl76P64NH8O+gIZS1r3x7liSJpNwSQjwcvaKxXQJ4Y9MJMgpKq70VuSj92Jd4gndn9Tr/NILLADedhut7VGYJGdzOm+/3p3DP1/tQKhVc08mf12/qLpoDBa0CjVIjC5B7eLVtBWUFnM2Xva+E/ATOFJxhR+oOzhacxcPJo1oAx5pJa+osnHWlVpH69NNP+eSTTwgLC6OkpISbb76Z9957jxtuuIE1a9Y0qhGXO0azkTaGNmT59eSEf0faYALkt4k8oxljmbVa31J7P1em9Agix1iGn8Hg8Fa0Zm8Sbbz0BHuI5r4rgVFR/oyK8qfUbOWvk1nEphagUir49WgaPx5MYXyXQEZG+Tp43wJBa8CgNdDVtytdfbs6rLdJNnJKc/gl4ZdqXRXNJlJr1qxhw4YNeHh4kJqaytixY1m9ejUdO3ZsVAOuBM4VnuNf5YEP7x5YyurrVtHBW75PSbklKBQQ4O6YG06hUPD2jJ41lifP4Cui+q40dBoV13Tyt09gGeThjKuTmmfWHqbEbGV4B1+emhBNmPeVEz4suDJRKpT4OPswIHBAta6KxqZWkdLpdHh4eAAQGBhIRESEEKha+CflH4e3iV2pO6qIlDxGqqZ+iNziMp6MOcxrN3fDoKvsmOwT5kmol4jqu9LpEuzOKzd248UpXdiVkMMvR1Jx02nIM5bx8KqDjO0SwOgoPzKLTOUvLt4iPZOgVVHTuLHGptanPTk5mYcffhiQ+1VSUlLsywBvvPFGoxtzudLZu4vD20Rv78qZdpPzHMdIVcXdWcOes7lsjstw6MOY0a/uYycElz9qlZJB7XwY1M4HgIzCUsK8XVi54yzBHs7M+XSnfVDxT/cOwUuvxc+gE31aghanMcaNXYxaReq9995zWJ4xY0aTGHC5cyr3FD/G/8BHo77iYMY+Bvl1JfDkFgiUm/KScktq7Vuq6ET/9Wi6XaR+PpTKzoRsFl3fpdmuQdC68HPT8eykTkiSxOfbExzSM205nsmWE5nsS8wl2MOZUC89n9zaB5PFxj+nswj10tPGS4+b7sIhwwJBY9KU48ZqFal+/fo12UmvJL47+R0phan4asIJ1/rgm3UI9+1vwJD7QClnjoi6QGLSazv7c8/X+yg1W9FpVGw+noF4QRaA3G85qJ2PQ3qm0dF+TO8byplsI+dyjCTnleCkVnI6s4jnfzxGWoE8Z5mXi5YFQ9py98h2bDqWTk6xiTZeLrTx1hNwiV6YmLZE0BLUKlKjRo1yyEmmUCjw9PSkT58+3H333bi6ujaLga2ZMmsZa47/wL+iHqKdvxvt/N3A7AFt/7FPgpiUW8KYaP9ayxgU6Y1WreRYagG92niyIz6bh64R+foEMhXpmXbE5zAgwptQT2f0Tmq8XZ3oHeZp369zkDs7nhpNqdlKUm4J53KM+LjKYxmPpuSz/nAaiTlGSsxWNCoFWx8biauTmtc2HKeNl97ugbXx1uN6Xp+X0WQhMdcopi0RtAi1PmkbNmyoti4vL48ff/yR//znPyLJLPDDyY2UmK1E6vtXrtQ4g5MBCtOQXP1JPi8l0vk4qVXseGo0TmoV53Iq8vWJzOcCmYr0THWdJkSnUdHOz5V2fpUvkQ+M6cADYzogSRKZRSbO5Rjxc9ORU1yGsczKr8dkAUsvMDGpexDvzuzJin/OsOtMLl2CDAyI8GbqB9vFtCWCFqFWkdJqtdXW+fn5cdtttzF16tQmNepy4ZP93+BqHsC1nc7LCrHhCVAoKLjmbQpNlovm37PaJFbtOYenXkuEr4sYHyVoEhQKBX5uOvzc5OEQvm5OvDGtu317qdmKscwKQIiXnvisYgpKzGw7lenQL/brsXSW/5VAWkEp/gYd3ULcuXVgOOdyjGQUlsrnMDjhpL68EpkKWif19tlNJhNWq7VO+65atYqYmBg0Gg2LFy8mNLQyn93333/PV199hVKpZOLEicydOxeAP//8k88//xybzcaNN97IlClT6mtis3A8K4FU0xHu7fZR9akaQvrCjg9JypM7E4M8dDWUUInVJvHeH6f4+NbebHpweFOZLBBcEJ1GhU4jC8vIjn6MLM/sHpdWwBu/nrD3i43s6Ed8VhFns4ykF5aSZzQDsOlYOot/icVSrmgeeg1vTevByCg/Xv4lFoVCgb/BCT83HYEeOnq18USSJDHVieCC1CpS3377bbV1+fn5bNq0iWnTpl204Ly8PFavXs0333zDsWPHWLJkCUuXLrVvX7ZsGTExMeh0OiZOnMjMmTMpKioiJiaGzz77DJWqdb+FfXzgW9SWcOb3G1B9Y2g/WPcAaelp+Lld/I1SpVDwxs3d+O1YOpIEYV6izV/Qeqhp2pIuwdUzZs8f0pZ5g8LJNZaRUWgivaCUToGVzYIJWUXsTMgmo8CEVq1k8yMj2Hg0ncfWHMTfIHtffm46pvQMZngHX/acycEmYRc2SZJIzDWKMWNXGbX+D2dmZlZb5+npyVNPPUVGRsZFCz506BD9+vVDrVbTrVs3EhISHLZHRETYwxZ1Oh0qlYotW7ag1+tZuHAhWq2WZ599lqCgoPpeU5NjsVnYn/MrTw+91z4dhwO+UaB1w3x2NyGebS9aXmKukekf78AmwZubTog2f0GrorZpS2pCqVTg7eqEt6sT0VUE6skJ0Q77VcxJ16+tF+/N6kV6QSkZhSYyyqMTAT7eGs+fxzMps9oIdNfxxrTu3LK8cszYmjsHEZuWj4tWQ1SgG1EBBkwWK1qVUnhnVxC1itQ999xj/24ymfjzzz9Zv349H3/8Mf3797/odB35+fm4u1e+bZ0/UeLYsWOZMmUKKpWKBQsWoFKpyMjIIDU1lU8//ZRdu3bx2muv8fbbb9fpQmJjY+u0X22UlpbWuYwVx7eRV1pIpDm81mO8O84mNr0Eg8p8wXL1ej1/J1rwN+gI89ZzNtvI9pOZ6Ey5LTJnDdTvXlzpiHtRSWlpaZM8k76ArwvgAgSowJpFbGwWD/Z14YE+egpNNtDoOJSc59A3tiM+i8TMAv46lc3AUD0L+njz5vYM/owvxtNZhaezCi9nFXf088bXRc3Gk4Xl69T27epLCMVXq9Wo1WpOnjyJxWJpzFtx2dKUv5NaRaqsrMwuTIcOHWLgwIHs2bOHLVu21KkpzmAwcPz4cfuyUlnpcRQVFbFs2TI2bNiAVqtl/vz5jBkzBoPBwIABA1Cr1QwaNIiXX365zhdSMTHjpRIbG1unMqw2iY1/LaK99zB6dqk5955s0KvEfrmHaD9XoqOjLljmBJdsJgd5oTm3HXPoYKzunvj7tFzuvrrei6sBcS8qael74Z5WgHL98SpjxgLoONIxmen/BbUlMcdIZqGJzELZO+vWKQKNSsGmTTvILDSRXWxCkmBghDff3D6AmH1JfL8/GV83uVnR182Jazv5E+qlJynXiIdei4tWhUKhqAzHP5nJoPa+hIkmR6Dhz8bevXtr3Vbr3e3fvz9RUVHcd999LFmyBJVKxahRo+rcV9S9e3c++OADrFYrcXFxhIVVpsxQKpVoNBr0ej1KpRKdTkdRURH9+vWze06xsbEEBzd+HqiGsmr/Ecq0R3l44BMX3jH7NPPPPcXpDssuWqafNQ3lV6NAsoFCie3ObYBIMCsQVKWmMWPnE+Cuq5bMuYL19w8FwGK1kVNchski59ts6+NC/7ZeZBSaOJNVzK6EbLoGuxPqpWfye9vJKS7DWaOic7CBx8dGMf3jf+zh+DF3DSY9vxQnjZIO/m4EeThTarbipBZNjo1FrSL1yCOPsGHDBl588UWuvfZaxo0bV6+b7uHhwZQpU5g9ezZqtZqXXnqJmJgYQkJC6NevH1OnTmX69OkoFAp69OhBhw7yANbo6GhuueUWbDYbixYtavgVNiKSJPHBnm/wdA2jb2C3C+/s7MkA805UyhTgwv1SyjPbZIECkGwoz2wH/86NY7RAcIVQ3zFjtaFWKfEzVApZzzae9GzjWeO+fz46gowCE5mFJqySjQNJjk2O209lcTK9kD9PZHLfqPbMH9KWWz/bxd6zubg7a/Bw1uCh1/D1bQMoMln4aMtpeZ2LFk+9hkhfV6IDDRSUmlEpFOjLPbaLUeHRXQ1BJLVe1ezZs5k9ezZZWVls3LiRF198kczMTF5//XXGjBlDz54XaOoqZ+bMmcycOdO+XNWbmjdvHvPmzat2zF133cVdd91Vz8toHhJzislXb+fezrdd9EEqVLqRYQskzHgUGHzhgsOHgEJp96QIH9J4RgsEgkvGoNNg0Gnsg6N9XJ0c0lSNifbn7pHtHI55c1p3MgtN5BnN5JWUkVtsRqdRkmO0kVlo4kR6IfklZnKNZYyJ9ue5SZ15ZX0cK3cmolUpcdfL4vb+7F6093PlyZjDGMrFrmIspbuzhglL/7Lb8ct9Q4kKbLlgqxabPh7Ax8enmmC9+eabrFixosmMaq0kmw7h5GRkeqfrHdYbzUaSipLYk7aHPgF9CHENITnPwmFbe6bmHrh4wZ7hcOc2OLkJ2gyAJsomLBAIGkZN4fjnE+Kpr3EAf7CHM+/MrPnl/vFxUcwf3Jb8clHLNZbh6+qExSahVCpIyjVyJNlMrtHMNdF+eLlqqw2wfumXWG7qHcL1PYK5e+U+MgtMuDipcHFS4+6s4aWpXUnNL+Gngym4OKlxdVLjolUT4etChK8rWUUmbJKEi1Zdb4+uKVNm1au0qoJ1peHpWbO7X8HpzCKW/P0l14Rdg0Hr+MaSVJTEzT/d7DCFcmaBBye00agzdlz4xGXFsOk5GP0MRF0HZ7bLQiUQCFod9QnHrw/uzhrcnWvOXL94atdq6+LSCqolHh7azseegm1s5wDS80spMlkwllkwW2VFyyku47fYDIpNFopNFopMVmb0DeWRsR15/sejrDuUCoBCAS5aNevuHYKfwYmZH+/AxUltF7fOQQZuGxpBfFYxk9/b1qQps67MRsx6kF9SRIoxmZ2Zu+mv6UuQPhh358q8ZxVvCr8cPUm8cScPhzsGQsRmx7IrbVe1KZQ9LSPZ7zUeblt8YQN2fACnNsHYxeDTXv4IBIJWTUsND6mgtsTDFUzuXvP40s5B7qy6Y2CN2169sRtPXxdtF69ik8UehHJj7xBZ8ExWikyVYfc74rPPGxaQI0SqsUkxJjPj52myF7RfyTcTVvHVDiWhXi5M7h7E6cwi7vxqL+Fhp/FWdcBbXTk7cczJGD4+9DGvDXvNYdLD7r7d+f1gIf5e7pB1HJzcwKOGiQyLs2DbUpj0NqjLcyUeWgXH18PNnzfPDRAIBJcdjRVEUpUKT6kmbh0YXuP6Ie0dp5IZ0ATJsa96kdqZstvBC/or+U/OkcOpVB1tg8aQWmhgyWx/DuacpofPE6QVFtIhwI23977N17Ff8+KQF2nn0Y41k9awO2033X27s+rEKg4mu9LfdyJseBzaDISRT1Y/+dbXwTsSOt9Quc6zLcT+CMYc0IswdIFA0HqpSx9dQ7nqRap/UF8CjwcS7BpMclEyg4OGoFT8zZ9Jf/LIlg0sGryI2zfdbveS/nfdarYlb+OH0z/w6dhP6eHXA4D2nu1p7yk31Y0sGcnfB+IJae8MTv0gaVf1E9tskHUSrllkn3sKgJA+4BYEsT9B77nNcAcEAoHg0miqPrqqXPUi5e/sx9KRS9mZupP+gf3xdw5gYbeFLOy2kFJLKauOr8LX2ZdQt1DOFZ7j98RfuafnPfxw/Q946DxqLHNE6AgeyjZToDjMS2VJPJG0F5XN5ihGSiXc8p3cQ1kVhQI6T4GjMUKkBALBZUGLTB9/tZBlymTGzzMcIvO89HKkn06tY3jIcDp4deBgxkHu6H4H8bnxALUKFECxyUJOcRntvYNZfCaWHIOalzNPoPUvT4+UvA9+ew5uiQFVDRE9XW6AlP1gtYDqqv8vEggEVzFXfQ24J22PQ5/UH4l/8Nbet2jn2Y6Hej9Eflk+d2y6wy5iqyeuvmiZyXklAAxu040V/l+x6uiXHJaKiYv9mr4BfQlJ2ILeEFyzQAEE9YR56xrtGgUCgeBy5aoXqT4BfRwi84aFDKO3f28MTnLUzIHMAw4itid9Dx28OlywzOTcErxctOXRMqFM7HgT09ZNq/TWRn1I+67TL2xY2mE49iOMerpRrlMgEAguR656kQpxDWHNpDXsTNlJ/6D+BLsGo9dUjhYfEDjAQcT6BvS9aJlJuUaHKeD3pu50HEeVG0f7kEEXN+6vJdD3NnDzr/+FCQQCwRXAVS9Seo2e9p7t0RZoCashHVGFiO1O203fgL4Eu148M3tSXgkhVUIx+wT0dRS64Drk5vPvAl6RcOwH6H97va5JIBAIrhSuepGqoLbolAoRqwgvrwtJuY4iFaIPYM2ElWxO3sagoIEE6/0uXohCAZ2nylF+QqQEAsFVSg1znwsaSlJuiUNzn74ghfYrpqM5tZl1v/wbfX5y3QrqcgMk75UH9goEAsFViBCpJiA5t8QxC/KZbZB/joD4LfxqyUFK2Fa3gvyi4ZGTIvOEQCC4ahEi1ciUmq1kFZkI8aqSHqR8vqi+JaVkqtUk+EbUvUCFQs4+IRAIBFchok+qkUnKlcdIVW3uq5gvyufMdj5ycSUgoEfdCyzKgG9vgQcO15ykViAQCK5ghEg1Msl5Jbg7a3DTVRmoq3WRp4P378xg5EkS64xPe/DvCke/h8H3N7q9NdGUs2wKBAJBfRDNfY1MUq7RIbLvfP5J+Ydrv7sWq81a90K7TJVFqokxmo2cyD3B1qKtnMg9UT8xFQgEgiZAeFKNzPmRfefT2aczhWWFHM89TifvTnUrtPNU+H0R5J0Fj0ufWr6mae6d1c4oFApSClPIKs1izvo59vFc31z3DZ28O/FX0l+UWEpw07ph0Bpw07oR4haCUlH9Haemc1QdHC0QCAT1QYhUI1Mtsu88DFoDnbw6sSt1V91FyisC4+1/kmQzsSd25SVX/udPc//ptZ/y/oH3+Xzc5+SYctiVuqtaCqhO3p1YF7+OAxkHKCwrpNBcCMC+W/ZxquAUt/xyi128wgxhzO8y30Ho1kxaU68xZgKBQFAVIVKNTFKukR6hHhfcp39gf3ak7WBel3l1L1fnws3rKgVmzcTVtL9IDsGqSJJULZnu0eyjPNHvCQC6+HRBq9Ly7oF37ecYGChPM/3qsFft5VhtVootxWhUGkJcQ3h/9PsUlBVQWFaIXq3nQIZjrsO/kv4CoJ1HOxTnT0siEAgEF0GIVCOTlFtC8EVmpxweOhyjpX79PXvSHPP/bU/ZToRHJCql6oLHWWwWVhxbQUJ+Ard0usUhPdOgoEEOXk5NeQzPR6VUYdDKyXf1Gn21XIYnck84nKOnf0/u/f1ePHQerLxuZY1NhAKBQFAbQqQaEZPFSkah6YKBEwA9/XrS069nvcru5t3ZofLv7N2ZN/a8wWP9Hqv1mBO5J3h2+7OkFafx9ICnL5qH8GJ5DOtCTef4+rqvOZZ9DKVCydexX7MrdRc3tL+BwcGDUSvFI3i5IKI+BS2BqCEakZS8UoAL9klV8P1JOVpvavupF903qySL1ad+4JtRH3Ig8wB9fXviLSnxjpgMZUZWnf6RsRHjcXdytx+zPmE9T217igltJ7DsmmXytrJi2ltstJdcwWIDSbrEK62dmnId6jV6hoYMBaCXXy9O5p7ksa2P4apxZXK7ydzW9TZcNC6VhZQVQ34SZB4H347gHiKH8V+NlBVD7hk5a0n4EHnMXXPfi3IbQpK3gm5Yy9hQxY4WvRflCMFuPoRINSLJuSW4Oalxd65lMsMqpBvT2ZG6o04iFXMyhkOZB3m+/Sw6OYWCkze4+OCl96L4jxdZk/Atp/NOMSlyEvvT99I/aCC9vLqydPBihulD4Mzfcv4//07wySiQbKBQwoLfoDgdNC4Q0lcWrbwGVka1VSSSBGYj0a6hPD/oeR4Lm8zGhJ/54+wWtGYnzhWlsD+4C9cED0My5pBkzmOPKZk+Jh9C8kHv2xHO7Ya0g+BkACc3+ePXSU4bVZIrX4daWz97L0KjV0Y2G5iLQakGjTOkHYHiDDAVgakQyoqg3+2ySO/+FDqOg8/HV/6f3blNnmss9id5WaGQ/45+FrwiIOZ2QFG53qMNjHgCzv4Nh1ZVrlcoofMNEDYQ/vkACpIr1ysUMPo5KEyDI2sgdAB8di2qqjYUZ8G5nY7X1nMOGALhz1cd17v6QZ9/QcoBOLHRcVv7ayC4F+z9Qj5fVYY/DkXp8jYnNwjtB59eW3kvFm6GjGNgLQOlBlRaaDdafh5O/Va5TqWVj/ftAKUFUJovTziq0sp/1c51nwG7tQh2K6MpRVuIVCOSlGu8aH9UBf0D+7Ps0DJKLCU4q2s/xmKzsPrEahZ2XYjCryP4dXTY7jL4flb6RhPrH8nsDbfKzYH7l7Lmuv8x7LfXIWUvuPhC1EQoyZF/4CD/TfhTriRP/gr/3i5XlMuGVlZGt/4Ah9fA5Hdgz+ewfan8o1aq5c81L0DkKFizQK5oXQOgz3z4YkJlRTL3R1j/BGTGgc0CQx6CMc+h/+d9piZsYarOHbJWkeCk5o3MzXwa9xUvDHiWuX/8uzJIZNyXtAe5Qtr/lVyZV3ymfSlXdO/2AWMWqJzkCsk7Ehb8Cok7YNvb4ORaKWztx0L4YIjfAqaCyvVOBrmiVyihzOgo2B5hchlFmVCYKotJhQ1th4OrL2x+Wa4ATYVyuXpvmPS2PN3Khicr90eC8a/L2e1/fhgyY0HrVmlHn/myUDh7yEJQ9f/szHYI6Cp7mJKt/CPJFS6As5dcvn19+QuTJIFklUVSKt9uLZO3GbOgIKXKceXns5SAMRfO/FXdBrcAOLfL8WHtfIP8N3mP43rPtvLf4qzq24LKm70z4iAnvvoPoKwYkvfJLyMJ59uxFbIT5DKtZvl6/DuBsyd8t1B+3qxlYDFBYDe4Y6ssuusedDzHxLfke/7JaPkZU1URtwePys/umgVgCIIhD8J/J1b+Rm77DfatkP9fK34bwx8D91DY9J9yodTIf92DoccsyIiVn8uK9So1BPUCr7byi5ilpMpxagjsLl9DUVrlehTyb67iZbDi+WxumkG0hUg1IkkXCT+vShefLmiUGvan72dQcO0TIO5K3UVhWSHXRVxX8w5Obqi73sjhI184TqyYupP2M/8HOjf5jR0g/aj8w6r4gXUYD0Mfrixr5zLHSiDtMAy4S14OGyxXoDYr2MxyBeAZLm9rO0wWQLdASNrtWEbKQZi0VP6ucwfX8gkcb/rU4TKGAb/ZzBzOPMze9H2O15J1mPZ+3aH3XPlTE7f/KQtDhRBUNGXqPOQKqkI4cs9Acaa87eA3kLDVUTyeSJS9styzsGJKZWX0r/XQZgBsfxv+eU9++64QFa+2skhlHZcrESc3cIkAj1D5PIE9YPxr5fu7ymJYcR/mb5AF6XzcQ2DIA/L/2W/PV/6fhQ+RK+KwgTXfh/Gv1Lw+fLD8qYnRz9a83isCxjwr2/DH/1W3odPkmo+bvbrm9e3HyJ/62O0dCbNXyd/PtyNyDAyqZRjH4wmOy7byZ7L7LIieLIuXtQysFnDxlrdNelv2tKxlsujZzPL/jYsfDPi3vF/KPsfn++wO+fnKOiUfZzMDCnlbflJ5OeVi6dlWFqnM47Dns/L15ecZ9R/5Odr0LKQdqlwP8FwupB+B5aPlZUMwTPkQVkypvBdz1sLaf8veqFIFC/+Qhf3VtvI1KFWgUEFQD/n/5+C38OfLlesVShj5lPx/GnO7/MKgUMnblWr5ZTPrpPyyVXGMi4/8e1w+xtHL9u9c8//JJSJEqhFJPm+ywwuhUWro7d+bfRn7LihSA4MGsmbSGsc+mxroE9jfcWLFwAHgdt68VeU5BDmzvbwp7rzgiPJEuPYHP2Ik+EXJ23w7yJ+aqCoc6Ufht+eqVCSj5AqtDmiUGnr598JV48o7VULhO/t0paisCFftBd4UKwThfPyiwO+pmrdN/ajye0UznNYV1DqI++U8sT0gi9So/8CY5ys9lKrc/EXN5/EMq36vK7hYWP7F/s+ag3IbrKe3oooc3jI2VLHjku6FsjyqVKOTPzUR0LXm9S7e0GuO/P38F73IkbU/37U9D52nyJ+amL/ecbkiM01gD3gsQRYvhVL2CKs+n+nHYM73sshK1nLvVQHTv5TLkGzyXyc3+ZiQPjDqmUrP2WaVRQ2gw1jZ67VZ5bIqXvi0Fd0CVnmbRyic/ae6l93IIqWQpCboPS9n1apVxMTEoNFoWLx4MaGhlRXJ999/z1dffYVSqWTixInMnVtZ0f3www88//zz7N+/v07n2bt3L717926QrbGxsURHRzeojJs/+puxnQO4bWjdspznm/Jx07rVGpadWpRKmjGtTpGAxtI8kotT2Z26k76BAwh2CUCv86iP+XbX3aEyuuQ+qSoVST3LMJqNJBclszttNz39evLr2V+JORnDfT3vY0q7KRcNu28U0o/CR0Oq9AVtr7PYXqmcPXuWsLAWEqjWQmP8RhqD1vJ8NpIdF6zDpSYiNzdXuummmySz2SwdPHhQuu+++xy2jx07ViouLpasVqs0fvx4yWQySZIkSWVlZdKdd94pjR8/vs7n2rNnT4PtPXbsWIPLGLD4N2n94ZQ672+1WaX96fulorKiGre/tus16V8b/tVgu+rLmTNnmv2cF8JsNUtfH/taGrRykDT9p+lSmaWs6U9qKpKktCOSZfsHkpR2VF6+ymmM38iVQov/RsqfT2nHspZ9Phvpd3KhOrzJRlYeOnSIfv36oVar6datGwkJjm3EERERGI1GSktL0el0qFTy2/G3337LDTfcgFLZZKY1CWUWG+kFpQR71C/K5Z4/7mFn6s5q60ssJaw9tZbpHac3lol1xmhsXYll1Uo1s6Jn8fPUn5nXeR4alYZ96ftIKUppupOWZ65PCp4gvxmKCC5BFVr8N1Ixs0L/21v2+WyG30mT9Unl5+fj7l45bkc6r1Vx7NixTJkyBZVKxYIFC1CpVBiNRv766y+WLVvG0qVL63W+2NjYBtlbWlraoDJSC83YJDBmniO2sO6VZ5Q+ig3HNhBYHOiwfnPmZpSSkqDioAZfW31p6L1oSsIIIzY2li/iv2Bb9jYmBU5iSuAUdKpa+hkaSGlpactXSK2E1vxcNDfiXjjSlL+TJhMpg8HA8ePH7ctVPaOioiKWLVvGhg0b0Gq1zJ8/nzFjxrB27Vpmz559SedraH9SQ/ukck9lodem0K9H53rlqBujGMO3x7+tdu5Fpxcxo9MMunaupTO3CWmM/rmmZmnUUrYmbeX1Pa+z7dg2Hu33KOPCxzX6eS6He9FciHtRibgXjjT0fuzdu7fWbU3Wpta9e3d2796N1Wrl6NGjDh2uSqUSjUaDXq9Hq9Wi0+koKioiPj6e//73vyxYsIDk5GSefPLJpjKv0Ukqj+yrbxLV/oH9OZV3iqySLIf1rwx7hRkdZzSmiVcUCoWC4aHD+X7y99za+VayS7IByC3NbWHLBAJBY9JknpSHhwdTpkxh9uzZqNVqXnrpJWJiYggJCaFfv35MnTqV6dOno1Ao6NGjBx06dGDJkiX24ydOnMjLL7/cVOY1OvUZI1WVcEM4j/R5xCHC70jWETp6dUSjvHjmiqsdjUrD3M5yZKjRbGTS2kkMDR7KA70ewN/Fv4WtEwgEDaVJx0nNnDmTmTNn2perelPz5s1j3rx5tR67bt26pjSt0UnKNV5wssPaUCgU9koWZE9g7vq5fDjmQ/oF9mtME6949Bo9n1zzCa/ufpVJayexoMsC5naei07dNP1VAoGg6bm8QuhaMaVlVvqGe17SsbtSdzHr51kArD21liDXoGpTYAjqRrR3NJ+P/ZwXB79IzMkYNp/b3NImCQSCBiBEqhEwmizMGRhGRqGJuLQCjCZLvY4PcQvhcNZhzhWc49vj3zIjaoaYILABKBQKrg2/lh+n/sjY8LHE58Vz28bbiM0W0VgCweWGSIvUCCTmGJm9fCc2CZQKWH//UDoGGOp8fJBrED39evJj/I+olWomR9aSE01QL5xUTgC4ad3wd/Fn5s8zmdJuCvf2vBdvZ+8Wtk4gENQF4Uk1An+fzsZWPgzMJsGO+Jx6HW80G3mw94M4qZx4deirqBTNkPbnKsJX78tLQ17iy/FfcjLvJBO/n0iGMaOlzRIIBHVAeFKNQO8wT5QK7J7UgIj6vaUnFSUxb8M8bJKNdxXvsmbSGodJAwWNQzffbqwYv4K96Xvx0/txIOMAeaY8hocMF82rAkErRYhUI3Amq4hv7xjIsZQCBkR4E1rHTOgV7Enb4zg1RdpuIVJNhFKhtAelnMw7yau7XqWXXy8e6/sY7TzbtbB1AoHgfERzXyMQsz+F/Yl5zB0UTscAN/RO9dP+PgF97OOkqlaigqbl5g438+OUHzE4Gbj5p5t5bfdr1dJ3CQSClkV4Uo1AXFoBM/rWMp9RHQhxDWHNpDXsTttN34C+BLsGN6J1ggsR5BrEkuFL2Ju+l9jsWBQKBQn5CYS4haBRapp0WmyBQHBxhEg1kJziMtILTEQF1j2a73z0Gj3tPduLJr4WpLd/b3r790aSJB7f+jiDgwczps0Y9hXto39uf0JcQ9BrhGAJBM2NEKkGEpdWgLNGRRsvUYFdCSgUCj4f+zmn8k4x65dZ9tmBRTCLQNAyCJFqIHGphXQMcEOlFNFhVwouWheOZh+tFsxiNBsJdgvGx9mnhS0UCK4eROBEA4lLKyA60K2lzRA0MjUFs3x+9HOuWX0ND//5MLtSd4kgC4GgGRAi1UDi0gqJqkd2CcHlQUUwy+N9HmfNpDUEuwbz1oi3+Hzc52hVWu787U6e+OsJoPqEngKBoPEQzX0NwGqTOJ5WSFSA8KSuNCqCWbQFWsI8K7P39/DrQQ+/HjzW9zH7HFaPb30cnVrH9I7T6ezTuaVMFgiuSIQn1QASsooxWWzCk7qCqW1KbE+dp33w75T2U8gz5THrl1nMWDeDH0//2JwmCgRXNMKTagBxaQUEuetw14vJCa9mBgUNYlDQINKK0/ju5HccyjzE5MjJHMo8hKvWlQj3iJY2USC4bBEi1QDiUgsbND5KcGUR4BLA3T3uti//kvALX8d+Td+AvkzvOJ1RbUaJ2ZYFgnoimvsaQFxageiPEtTKE/2e4McpPxLlFcWifxZxww83YJNsItBCIKgHwpNqALGphUzuIVIYCWqnrXtbHuv7GPf1vI/jucdRKpR8evhTDmQeYFqHaQwOHmwPdRcIBNURv45LpKDUTHJeCdHCkxLUAZ1aR3ff7gAMCR6Ct86bh7c8zISYCXx6+FNKLaUtbKFA0DoRInWJHE8rRKtS0tbHpaVNEVxmdPTqyPODnue3m39jTqc57ErbhUapIakwiX3p+0RzoEBQBdHcd4nEpRbQ3t8VtUrovODSMGgNzI6ezezo2QDsTtvNoh2LaOvelmkdpjExYiKuWtcWtlIgaFlEDXuJxIpME4JGZmr7qWy6aRPjw8fz2ZHPGL16NOnF6ZzIPcHK2JWcyD2B0VzzuC2B4EpFeFKXSFxqARO6Bra0GYIrDB9nHxZ2W8j8LvM5nHmY/LJ8bv7pZns29hXjV2Cz2Wjv1R4XjWhqFlz5CE/qErDZJOLSCokWY6QETYRKqaKHfw/2pO1xyMa+K3UXHx78kLWn1gLwxZEvWHV8FbHZsZht5ha0WCBoGoQndQmcyzViLLOKMVKCJqciG3uFJzU8dDhzu8zFarMCkG5MZ2faTl7a+RJapZbe/r356JqPyCvNo9BcSIhrCAqFmEZGcPkiROoSiE0txNfNCW9Xp5Y2RXCFU5GNfXfabvoG9CXYNRiNUmPPXPF4v8cBKDYXcyz7GBnGDAA2n9vMs38/i4eTB118utDNpxvj244n3D28pS5FILgkhEhdAiLThKC5qMjGfrFZgV00LvQN6Gtfntp+KkOCh3A46zBHso6wL2Mfvfx7Ee4ezr82/AtfvS/dfLrRxacLUV5R6NS6pr4UgeCSECJ1CcSliv4oQevHV+/LqDajGNVmlH2dJElMbT+Vw5mH+Tn+Z97Y+wbtPdqzatIqjmQd4VTeKbr6dKWte9tqmTD0en1zX4JA0LQitWrVKmJiYtBoNCxevJjQ0FD7tu+//56vvvoKpVLJxIkTmTt3Lr/99hvLli1Dq9Xi7+/Pq6++ikbT+hJyxqUVcG1n/5Y2QyCoNwqFgsmRk5kcORkAk9VEVkkWAAn5CSw/vJyzBWdx0bjQxbsLj/Z9lFC3UM4VnmNX0S765/YnxDUEvUYIlqB5aDKRysvLY/Xq1XzzzTccO3aMJUuWsHTpUvv2ZcuWERMTg06nY+LEicycOZPo6GhWrlyJRqNhyZIlrF+/nsmTJzeViZdEscnC2RyjGCMluCJwUjkR7Crnn5wUOYlJkZPIN+VzJOsIh7MO46nzJKkoiWnrptmDN/477r/kluYyss1IzDYzaoVaBGcImowmE6lDhw7Rr18/1Go13bp1IyEhwWF7RESEfUI5nU6HSqUiOLgyWatGo0Gtbn2tkSfSC1EpFET6iTEqgisTdyd3BgcPZnDwYAB+O/ubQxj8vvR9dk9q+aHlrDi2grYebYlwjyDCPYLxbccT4BJgFzWBoCE0mQrk5+fj7u5uXz4/H9nYsWOZMmUKKpWKBQsWoFKp7NsSExPZtm0b//73v+t8vtjY2AbZW1paWqcyNp8oIMSgJv7kiQadrzVT13txNXC13wu1Wk1Pn54OYfCDAgehylURGxtLD2UPPNp6kFyaTHJ+MuvT1+Nf6k+YPowF+xbgrfUm2DmYYF0wkS6R9Pbs3dKX1Chc7c/F+TTl/WgykTIYDBw/fty+rFRWvlEVFRWxbNkyNmzYgFarZf78+YwZM4agoCCys7N59NFHef3119FqtXU+X3R0dIPsjY2NrVMZ/ztxhO7h2gafrzVT13txNSDuBRjNRtZMWsPOlJ30D+pPsGswep/KPqmBDKx2jNVm5X3v94nPjyc+L56EggRMVhO3RN/CimMrWBm7kggP2fNq696WIcFD8NP7NedlNQjxXDjS0Puxd+/eWrc1mUh1796dDz74AKvVSlxcHGFhYfZtSqUSjUaDXq9HqVSi0+koKiqiuLiYe++9lyeffJLw8PCmMq1BxKYWMjLq8vkxCQQNpSIMXlugJcwz7OIHIGfM6O3fm97+1T2nkaEj8dR5yuKVn8DWpK2EuoXip/dj6g9T8dR52sWro2dH+gT0AWSxTCpKYk/aHvoE9BEBHFcJTSZSHh4eTJkyhdmzZ6NWq3nppZeIiYkhJCSEfv36MXXqVKZPn45CoaBHjx506NCB999/n7Nnz/LGG28AcOONNzJlypSmMrHeSJJEbFoB/x4Z2dKmCATNTkUfckMJcQshxC2k2nqbZOOenveQkJ9AfF48P53+iT+1f9InoA+bzmzCU+fJgl8X2JsdV09aTQfPDo1ik6D10qSRCTNnzmTmzJn25are1Lx585g3b57D/nfffTd33313U5rUIFLySykstRAtIvsEgkZHqVAyus3oGrd19unMr2d/dQjg2JGygw6eHXhr71uEuIXQ1acrkR6R9mwcgiuD1hc+14qJSy3AU6/B3yDSIQkEzUmQaxCDggbxluItuyc1MGggVpuV7JJstpzbQnx+PFqVljFhY3hl6CukFadhsppo49ZGhMhfxgiRqgdx5XNIiQdeIGh+aspjqFKqeHHIi0Bl/sISSwkAP5z6gfcOvIdBa6Czd2e6+HTh5g43E+gqpti5nBAiVQ9iUwuIChQ5+wSCluBieQzPz194R/c7uKnDTRzNPmrPYVhqLaXMWsb1a6+no1dHuvh0oYtPFzp7d8ZNK37brREhUvUgLq2QYe19W9oMgUBQR7ydvRkWMoxhIcPs60xWE/f3vp8jmUf4K+kvPj70Mf0D+vPu6HfZmrSVpMIkuvh0oaNXR5xUNTftizyGzYcQqTpSarYSn1kkPCmB4DLHSeXEuPBxjAsfB4DFZqGwrBCQ5+dae2otr+9+HRTQwbMDbwx/g2DXYE7nnSbQJZDk4mR2Fu0UeQybCSFSdeRURhEA7f2ESAkEVxJqpRpPnScAN3e4mZs73EyppZS4nDiOZh/F29mb5KJk7vztTl4a8hK3b7rdHryxZtIalAol6cXpuGndMDgZcNO64a51R6VUXeTMgrogRKqOxKYWEO7jgrNWPHgCwZWOTq2jh18Pevj1AOSxXWuvX0vMqRiHMPjdabvRKDW8ve9tCssKkZDTv31z3Td08u7EuO/G4aJxwaCVxauDZwfu63Ufsdmx7E7bbRc2g9ZAhHsE3s7elFhKcFI5XTDv4dU0sFmIVDkXa2OOTS0U46MEgqsYV60rAwIHOOQx7BvQl/ae7bm5483YJBvF5mIKywrxcfZBkiQe7/c4BaYCCssKKTQX4q6V85mmFKfwe+LvFJQVUFAmb3+6/9Nc3+56Zv08i/j8eFw1rrhp3fDSefH1hK85W3CWz49+TpBLEAODBjJn/RwHj+5iE2Nerlz1ImU0WUjMNfJ3ooVBTgW08dSjd6p+W+LSChgY4d0CFgoEgtZCRRh81TyGFSgVSty0bg5RgrUNTh7dZnS1bRVJuN8Z9Q55pXl28SqxlNiHvVQI5K60XQ4e3dakrXx08CPyTfmEGcJoY2jDTR1uwlntjFWyXtYDnK96kUrMNTJh6V/YJFAq4vj5vqHVZt2VJAm9VkX3UI+WMVIgELQKLiWPYV2pEKJQt1BC3UKrbQ93D+e5gc8BcCL3BO/uf9fuSQ0OHkw7j3Yczz3O2YKzbDq7iZs73My5wnNcv/Z6gl2DaWNoQ5ghjIGBAxkeOpxiczE6la7V951d9SK1Iz4bW/ksIjYJNh1L5+udZwnx1HNNJ38CDTrOZBvpFuKBj6sWo8lSo6clEAiuHhorj+GlUtPA5iivKIaHDnfYL1AZyIrxKzhTcIbEwkTO5p/lTMEZhjOcV3a9wi/xvxDqFkqYIYwwQxi3dr4Vb503WSVZ+Dj71DlxQVOG5F/1te2ACG+UCso9KRgT7ceO+By+35/Mf/8+w1vTezDrkx3YJHj7txOsv38oHUXflEAgaEEuNrC5Aq1KS1ffrnT17Vpt24O9H2RKuykkFiTKIlaQiFKhJKski1GrR6FX6+1Nh4ODBjO1/VSyS7JRK9W4O8l9axUBHE0Zkn/Vi1QbTz3r7x/K9pOZDG7vR6inM52C3Jk/pC3ZRSZi9iU5eFo74nOESAkEgsseL50XXjqvatOp2CQbv974K2cLz3I2/yxnC89itMie48eHPmZl3Ercndzp6duT+V3nM2/DvCYN4LjqRUrvpKZjgAGdKZewAMcxUN6uTgzt4ItyfZzd0xoggicEAsEVjFKhJNA1kEDXQAYEDnDY9kjfR5gVPYuzBWcptZSyP2N/tZB8IVJNRG1tzBWe1o74HAZEeBPq6dzMlgkEAkHrQKPU2PuvQA7gWLpvqUNIfmMjROoiVHhaoolPIBAIHLlQSH5jIURKIBAIBJdEU4bkV1B73g2BQCAQCOpAU4bkC5ESCAQCQatFiJRAIBAIWi1CpAQCgUDQahEiJRAIBIJWixApgUAgELRahEgJBAKBoNWikComMbmM2bt3b0ubIBAIBIIG0Lt37xrXXxEiJRAIBIIrE9HcJxAIBIJWixApgUAgELRahEgJBAKBoNUiREogEAgErRYhUgKBQCBotQiREggEAkGrRYiUQCAQCFotQqSAVatWMWPGDObMmcO5c+da2pwWY//+/UyfPp1bbrmF22+/nYKCgpY2qcXZs2cPHTt2JCcnp6VNaVEOHTrE/PnzmTNnDsuXL29pc1qURYsWMWPGDKZNm8bOnTtb2pxmx2w2M2PGDPr06cOGDRsAyMnJ4bbbbmPmzJm8++67jXtC6SonNzdXuummmySz2SwdPHhQuu+++1rapBYjLS1NMhqNkiRJ0sqVK6UPPvighS1qee655x7phhtukLKzs1valBbDZDJJCxcutD8bVzMJCQnSrbfeKkmSJKWkpEizZs1qYYuaH5vNJqWnp0vvvPOOtH79ekmSJOmVV16RfvnlF0mSJGnhwoXSyZMnG+18V70ndejQIfr164daraZbt24kJCS0tEkthr+/P87OzgBoNBpUKlULW9SybN68md69e6PX61valBblwIED6HQ67rvvPubPn09cXFxLm9Ri+Pj4oNPpsFgsFBQU4OXl1dImNTsKhQI/Pz+Hdfv27WPkyJEAjBgxgt27dzfa+dSNVtJlSn5+Pu7u7vZlSWSJIjc3l5UrV17VzTo2m42VK1fy3nvv8fvvv7e0OS1KRkYGp06dYs2aNaSmpvLMM8/wzTfftLRZLYKLiwtBQUGMGzeO0tJS3nvvvZY2qVVgNBrR6XQAGAwGkpKSGq3sq96TMhgMDn0vSuXVfUtKSkq4//77eeaZZ67Kt8QKfvrpJ0aNGoWTk1NLm9LiGAwGevXqhV6vJzIykqKiopY2qcXYvn07eXl5/Prrr8TExLBo0aKWNqlV4OzsjMlkAqCwsNDhxb+hXN01MtC9e3d2796N1Wrl6NGjhIWFtbRJLYbFYuHBBx9kzpw59OrVq6XNaVFOnDjBxo0bWbBgAcePH+eRRx5paZNajO7du5OQkIDNZiMzMxOtVtvSJrUYNpsNd3d3lEolrq6uGI3GljapVdC7d2+2bNkCwNatW+nTp0+jlS2yoAPffPMNP/zwA2q1mpdeeumqFaq1a9fy4osvEh0dDcDw4cO57bbbWtiqlmfOnDksXbr0qvYs16xZQ0xMDBaLhUcffZS+ffu2tEktgtVq5YknniA5ORmTycTcuXOZPHlyS5vV7Nx///0cOXIEvV7P0KFDue2223jssccoLi5mwIAB3H///Y12LiFSAoFAIGi1XPXNfQKBQCBovQiREggEAkGrRYiUQCAQCFotQqQEAoFA0GoRIiUQCASCVosQKYFAIBC0Wq76tEgCweXC+vXr+fvvv8nPz+eee+6hQ4cOLW2SQNDkiHFSAsFlRmxsLNu2bWPhwoUtbYpA0OSI5j6B4DKiIvHtpEmTWtoUgaBZEM19AkEr5MCBAyxevJi8vDzOnj0LwBdffMHvv//OzJkzCQgIaGELBYLmQYiUQNDKMBqNPP7443zwwQdERkZy9913c91113HkyBGOHDlCWVkZgwcPZuzYsS1tqkDQ5AiREghaGdu2bWPgwIFERkYC0KFDBwoLC1m4cKHohxJcdYg+KYGglXH69Gnat29vX46Li7NnphcIrjaESAkErQyDwWCfon337t3k5OTQtWvXFrZKIGgZRAi6QNDKKCws5P777yczM5OgoCCee+45goKCWtosgaBFECIlEAgEglaLaO4TCAQCQatFiJRAIBAIWi1CpAQCgUDQahEiJRAIBIJWixApgUAgELRahEgJBAKBoNUiREogEAgErRYhUgKBQCBotQiREggEAkGr5f8B1XymOiJvxFoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = pd.concat(dfs)\n",
    "df.rename(columns = {\"Variance\": \"$\\sigma^2$\"}, inplace = True)\n",
    "d = df.groupby([\"$\\sigma^2$\", \"Dataset\", \"Method\"]).mean()\n",
    "\n",
    "# d[\"$\\sigma^2$\"] = d[\"Variance\"]\n",
    "sb.lineplot(data=d, x=\"$\\sigma^2$\", y=\"AUROC\", hue=\"Dataset\", style=\"Method\", marker=\"o\")\n",
    "# plt.title(\"ViT\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"img/imagenet-trans-auroc-vs-variance.png\", dpi=300)\n",
    "plt.savefig(\"img/imagenet-trans-auroc-vs-variance.pgf\")\n",
    "df.to_csv(\"results/imagenet-trans-auroc-vs-variance.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrrrrrr}\n",
      "\\toprule\n",
      "{} &  AUROC &  AUPR-IN &  AUPR-OUT &  ACC95TPR &  FPR95TPR &  Iteration &  Variance \\\\\n",
      "Method  &        &          &           &           &           &            &           \\\\\n",
      "\\midrule\n",
      "Energy  &  90.34 &    62.16 &     96.76 &     67.68 &     39.47 &   99900.00 &    200.00 \\\\\n",
      "Softmax &  85.65 &    53.57 &     95.33 &     54.24 &     55.31 &   99900.00 &    200.00 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = pd.concat(dfs)\n",
    "\n",
    "df = df[df[\"Variance\"] == 2.0]\n",
    "# df.groupby(\"Method\").mean()\n",
    "# df[\"Method\"] = \"MPS\"\n",
    "print((df.groupby(\"Method\").mean() * 100).to_latex(float_format=\"%.2f\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>AUPR-IN</th>\n",
       "      <th>AUPR-OUT</th>\n",
       "      <th>ACC95TPR</th>\n",
       "      <th>FPR95TPR</th>\n",
       "      <th>Iteration</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$\\sigma^2$</th>\n",
       "      <th>Dataset</th>\n",
       "      <th>Method</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">0.0</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">ImageNetA</th>\n",
       "      <th>Energy</th>\n",
       "      <td>0.811748</td>\n",
       "      <td>0.338904</td>\n",
       "      <td>0.966031</td>\n",
       "      <td>0.520296</td>\n",
       "      <td>0.54396</td>\n",
       "      <td>999.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Softmax</th>\n",
       "      <td>0.855323</td>\n",
       "      <td>0.471625</td>\n",
       "      <td>0.973294</td>\n",
       "      <td>0.545437</td>\n",
       "      <td>0.51506</td>\n",
       "      <td>999.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">ImageNetO</th>\n",
       "      <th>Energy</th>\n",
       "      <td>0.903127</td>\n",
       "      <td>0.429867</td>\n",
       "      <td>0.994872</td>\n",
       "      <td>0.557462</td>\n",
       "      <td>0.45824</td>\n",
       "      <td>999.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Softmax</th>\n",
       "      <td>0.879862</td>\n",
       "      <td>0.381769</td>\n",
       "      <td>0.993498</td>\n",
       "      <td>0.502058</td>\n",
       "      <td>0.51586</td>\n",
       "      <td>999.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ImageNetR</th>\n",
       "      <th>Energy</th>\n",
       "      <td>0.855252</td>\n",
       "      <td>0.775225</td>\n",
       "      <td>0.903317</td>\n",
       "      <td>0.639650</td>\n",
       "      <td>0.54656</td>\n",
       "      <td>999.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">10.0</th>\n",
       "      <th>ImageNetA</th>\n",
       "      <th>Softmax</th>\n",
       "      <td>0.856242</td>\n",
       "      <td>0.499444</td>\n",
       "      <td>0.973714</td>\n",
       "      <td>0.559391</td>\n",
       "      <td>0.49902</td>\n",
       "      <td>999.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">ImageNetO</th>\n",
       "      <th>Energy</th>\n",
       "      <td>0.924451</td>\n",
       "      <td>0.552155</td>\n",
       "      <td>0.996059</td>\n",
       "      <td>0.642654</td>\n",
       "      <td>0.36964</td>\n",
       "      <td>999.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Softmax</th>\n",
       "      <td>0.842178</td>\n",
       "      <td>0.324000</td>\n",
       "      <td>0.991507</td>\n",
       "      <td>0.418558</td>\n",
       "      <td>0.60270</td>\n",
       "      <td>999.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">ImageNetR</th>\n",
       "      <th>Energy</th>\n",
       "      <td>0.866793</td>\n",
       "      <td>0.821907</td>\n",
       "      <td>0.902226</td>\n",
       "      <td>0.626538</td>\n",
       "      <td>0.56754</td>\n",
       "      <td>999.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Softmax</th>\n",
       "      <td>0.809875</td>\n",
       "      <td>0.751021</td>\n",
       "      <td>0.862832</td>\n",
       "      <td>0.555050</td>\n",
       "      <td>0.68192</td>\n",
       "      <td>999.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>108 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 AUROC   AUPR-IN  AUPR-OUT  ACC95TPR  \\\n",
       "$\\sigma^2$ Dataset   Method                                            \n",
       "0.0        ImageNetA Energy   0.811748  0.338904  0.966031  0.520296   \n",
       "                     Softmax  0.855323  0.471625  0.973294  0.545437   \n",
       "           ImageNetO Energy   0.903127  0.429867  0.994872  0.557462   \n",
       "                     Softmax  0.879862  0.381769  0.993498  0.502058   \n",
       "           ImageNetR Energy   0.855252  0.775225  0.903317  0.639650   \n",
       "...                                ...       ...       ...       ...   \n",
       "10.0       ImageNetA Softmax  0.856242  0.499444  0.973714  0.559391   \n",
       "           ImageNetO Energy   0.924451  0.552155  0.996059  0.642654   \n",
       "                     Softmax  0.842178  0.324000  0.991507  0.418558   \n",
       "           ImageNetR Energy   0.866793  0.821907  0.902226  0.626538   \n",
       "                     Softmax  0.809875  0.751021  0.862832  0.555050   \n",
       "\n",
       "                              FPR95TPR  Iteration  \n",
       "$\\sigma^2$ Dataset   Method                        \n",
       "0.0        ImageNetA Energy    0.54396      999.0  \n",
       "                     Softmax   0.51506      999.0  \n",
       "           ImageNetO Energy    0.45824      999.0  \n",
       "                     Softmax   0.51586      999.0  \n",
       "           ImageNetR Energy    0.54656      999.0  \n",
       "...                                ...        ...  \n",
       "10.0       ImageNetA Softmax   0.49902      999.0  \n",
       "           ImageNetO Energy    0.36964      999.0  \n",
       "                     Softmax   0.60270      999.0  \n",
       "           ImageNetR Energy    0.56754      999.0  \n",
       "                     Softmax   0.68192      999.0  \n",
       "\n",
       "[108 rows x 6 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6fa9c29a604f4508a798f46ef2859d47",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kirchheim/anaconda3/envs/myenv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: Metric `AUROC` will save all targets and predictions in buffer. For large datasets this may lead to large memory footprint.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/home/kirchheim/anaconda3/envs/myenv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: Metric `PrecisionRecallCurve` will save all targets and predictions in buffer. For large datasets this may lead to large memory footprint.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/home/kirchheim/anaconda3/envs/myenv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: `pos_label` automatically set 1.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/home/kirchheim/anaconda3/envs/myenv/lib/python3.8/site-packages/torchmetrics/functional/classification/precision_recall_curve.py:156: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  thresholds = tensor(reversed(thresholds[sl]))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUROC 999 -> 89.009%\n"
     ]
    }
   ],
   "source": [
    "dfs_v2 = train(2.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrrrrr}\n",
      "\\toprule\n",
      "{} &  AUROC &  AUPR-IN &  AUPR-OUT &  ACC95TPR &  FPR95TPR &  Iteration \\\\\n",
      "Method  &        &          &           &           &           &            \\\\\n",
      "\\midrule\n",
      "Energy  &  90.30 &    62.04 &     96.74 &     67.61 &     39.57 &   99900.00 \\\\\n",
      "Softmax &  85.62 &    53.37 &     95.32 &     54.42 &     55.11 &   99900.00 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(dfs_v2)\n",
    "# df = df[df[\"Variance\"] == 2.0]\n",
    "# df[\"Method\"] = \"MPS\"\n",
    "print((df.groupby(\"Method\").mean() * 100).to_latex(float_format=\"%.2f\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>AUPR-IN</th>\n",
       "      <th>AUPR-OUT</th>\n",
       "      <th>ACC95TPR</th>\n",
       "      <th>FPR95TPR</th>\n",
       "      <th>Dataset</th>\n",
       "      <th>Iteration</th>\n",
       "      <th>Method</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.880068</td>\n",
       "      <td>0.544410</td>\n",
       "      <td>0.978448</td>\n",
       "      <td>0.597112</td>\n",
       "      <td>0.45566</td>\n",
       "      <td>ImageNetA</td>\n",
       "      <td>999</td>\n",
       "      <td>Softmax</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.900245</td>\n",
       "      <td>0.542860</td>\n",
       "      <td>0.983225</td>\n",
       "      <td>0.700757</td>\n",
       "      <td>0.33652</td>\n",
       "      <td>ImageNetA</td>\n",
       "      <td>999</td>\n",
       "      <td>Energy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.842565</td>\n",
       "      <td>0.258627</td>\n",
       "      <td>0.991574</td>\n",
       "      <td>0.439846</td>\n",
       "      <td>0.58056</td>\n",
       "      <td>ImageNetO</td>\n",
       "      <td>999</td>\n",
       "      <td>Softmax</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.918521</td>\n",
       "      <td>0.467750</td>\n",
       "      <td>0.995921</td>\n",
       "      <td>0.654673</td>\n",
       "      <td>0.35714</td>\n",
       "      <td>ImageNetO</td>\n",
       "      <td>999</td>\n",
       "      <td>Energy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.845949</td>\n",
       "      <td>0.797945</td>\n",
       "      <td>0.889470</td>\n",
       "      <td>0.595513</td>\n",
       "      <td>0.61718</td>\n",
       "      <td>ImageNetR</td>\n",
       "      <td>999</td>\n",
       "      <td>Softmax</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.890091</td>\n",
       "      <td>0.850567</td>\n",
       "      <td>0.923164</td>\n",
       "      <td>0.672850</td>\n",
       "      <td>0.49344</td>\n",
       "      <td>ImageNetR</td>\n",
       "      <td>999</td>\n",
       "      <td>Energy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      AUROC   AUPR-IN  AUPR-OUT  ACC95TPR  FPR95TPR    Dataset  Iteration  \\\n",
       "0  0.880068  0.544410  0.978448  0.597112   0.45566  ImageNetA        999   \n",
       "1  0.900245  0.542860  0.983225  0.700757   0.33652  ImageNetA        999   \n",
       "2  0.842565  0.258627  0.991574  0.439846   0.58056  ImageNetO        999   \n",
       "3  0.918521  0.467750  0.995921  0.654673   0.35714  ImageNetO        999   \n",
       "4  0.845949  0.797945  0.889470  0.595513   0.61718  ImageNetR        999   \n",
       "5  0.890091  0.850567  0.923164  0.672850   0.49344  ImageNetR        999   \n",
       "\n",
       "    Method  \n",
       "0  Softmax  \n",
       "1   Energy  \n",
       "2  Softmax  \n",
       "3   Energy  \n",
       "4  Softmax  \n",
       "5   Energy  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate model without additional training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "Accuracy: 0.8032799959182739\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kirchheim/anaconda3/envs/myenv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: Metric `AUROC` will save all targets and predictions in buffer. For large datasets this may lead to large memory footprint.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/home/kirchheim/anaconda3/envs/myenv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: Metric `PrecisionRecallCurve` will save all targets and predictions in buffer. For large datasets this may lead to large memory footprint.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/home/kirchheim/anaconda3/envs/myenv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: `pos_label` automatically set 1.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/home/kirchheim/anaconda3/envs/myenv/lib/python3.8/site-packages/torchmetrics/functional/classification/precision_recall_curve.py:156: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  thresholds = tensor(reversed(thresholds[sl]))\n"
     ]
    }
   ],
   "source": [
    "# in test set \n",
    "from pytorch_ood.utils import is_known\n",
    "from torchmetrics import Accuracy\n",
    "\n",
    "dataset_in_test = ImageNet(split=\"val\", root=\"/data_fast/kirchheim/datasets/imagenet-2012/\", transform=preprocess)\n",
    "test_in_loader = DataLoader(dataset_in_test, batch_size=256, num_workers=20)\n",
    "\n",
    "# ood test sets \n",
    "imagenet_r = ImageNetR(root=\"/data_fast/kirchheim/datasets/\", \n",
    "                       download=True, transform=preprocess, target_transform=ToUnknown())\n",
    "imagenet_r_loader = DataLoader(imagenet_r + dataset_in_test, batch_size=128, num_workers=12)\n",
    "\n",
    "imagenet_o = ImageNetO(root=\"/data_fast/kirchheim/datasets/\", \n",
    "                       download=True, transform=preprocess, target_transform=ToUnknown())\n",
    "imagenet_o_loader = DataLoader(imagenet_o + dataset_in_test, batch_size=128, num_workers=12)\n",
    "\n",
    "imagenet_a = ImageNetA(root=\"/data_fast/kirchheim/datasets/\", \n",
    "                       download=True, transform=preprocess, target_transform=ToUnknown())\n",
    "imagenet_a_loader = DataLoader(imagenet_a + dataset_in_test, batch_size=128, num_workers=12)\n",
    "\n",
    "\n",
    "model = VisionTransformer(image_size=(224, 224), pretrained=\"b16-im21k-224\", num_classes=1000)\n",
    "model._from_pretrained(\"b16-im21k-224\")\n",
    "\n",
    "model = model.cuda()\n",
    "\n",
    "perf_metrics = []\n",
    "\n",
    "print(f\"Evaluating... \")\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    \n",
    "    acc = Accuracy()\n",
    "    for x, y in test_in_loader:\n",
    "        logits = model(x.cuda())\n",
    "        acc.update(logits.cpu(), y)\n",
    "        \n",
    "    print(f\"Accuracy: {acc.compute()}\")\n",
    "\n",
    "    softmax = MaxSoftmax(model)\n",
    "    energy = EnergyBased(model)\n",
    "\n",
    "    ###\n",
    "    metrics = OODMetrics()\n",
    "    metrics_energy = OODMetrics()\n",
    "\n",
    "    for x, y in imagenet_a_loader:\n",
    "        logits = model(x.cuda())\n",
    "        metrics.update(softmax.score(logits), y)\n",
    "        metrics_energy.update(energy.score(logits), y)\n",
    "    \n",
    "\n",
    "    \n",
    "    m = metrics.compute()\n",
    "    m.update({\n",
    "        \"Dataset\": \"ImageNetA\",\n",
    "        \"Iteration\": 0,\n",
    "        \"Method\": \"Softmax\"\n",
    "    })\n",
    "    perf_metrics.append(m)\n",
    "\n",
    "    m = metrics_energy.compute()\n",
    "    m.update({\n",
    "        \"Dataset\": \"ImageNetA\",\n",
    "        \"Iteration\": 0,\n",
    "        \"Method\": \"Energy\"\n",
    "    })\n",
    "    perf_metrics.append(m)\n",
    "\n",
    "\n",
    "    ###\n",
    "    metrics = OODMetrics()\n",
    "    metrics_energy = OODMetrics()\n",
    "\n",
    "    for x, y in imagenet_o_loader:\n",
    "        logits = model(x.cuda())\n",
    "        metrics.update(softmax.score(logits), y)\n",
    "        metrics_energy.update(energy.score(logits), y)\n",
    "\n",
    "    m = metrics.compute()\n",
    "    m.update({\n",
    "        \"Dataset\": \"ImageNetO\",\n",
    "        \"Iteration\": 0,\n",
    "        \"Method\": \"Softmax\"\n",
    "    })\n",
    "    perf_metrics.append(m)\n",
    "\n",
    "    m = metrics_energy.compute()\n",
    "    m.update({\n",
    "        \"Dataset\": \"ImageNetO\",\n",
    "        \"Iteration\": 0,\n",
    "        \"Method\": \"Energy\"\n",
    "    })\n",
    "    perf_metrics.append(m)\n",
    "\n",
    "    ###\n",
    "    metrics = OODMetrics()\n",
    "    metrics_energy = OODMetrics()\n",
    "\n",
    "    for x, y in imagenet_r_loader:\n",
    "        logits = model(x.cuda())\n",
    "        metrics.update(softmax.score(logits), y)\n",
    "        metrics_energy.update(energy.score(logits), y)\n",
    "\n",
    "    m = metrics.compute()\n",
    "    m.update({\n",
    "        \"Dataset\": \"ImageNetR\",\n",
    "        \"Iteration\": 0,\n",
    "        \"Method\": \"Softmax\"\n",
    "    })\n",
    "    perf_metrics.append(m)\n",
    "\n",
    "    m = metrics_energy.compute()\n",
    "    m.update({\n",
    "        \"Dataset\": \"ImageNetR\",\n",
    "        \"Iteration\": 0,\n",
    "        \"Method\": \"Energy\"\n",
    "    })\n",
    "    perf_metrics.append(m)\n",
    "\n",
    "    # print(f\"AUROC {i} -> {m['AUROC']:.3%}\")\n",
    "    current_auroc = m['AUROC']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrrrrr}\n",
      "\\toprule\n",
      "{} &  AUROC &  AUPR-IN &  AUPR-OUT &  ACC95TPR &  FPR95TPR &  Iteration \\\\\n",
      "Method  &        &          &           &           &           &            \\\\\n",
      "\\midrule\n",
      "Energy  &  88.71 &    61.91 &     95.69 &     61.84 &     47.01 &       0.00 \\\\\n",
      "Softmax &  82.48 &    50.38 &     93.92 &     49.56 &     61.34 &       0.00 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(perf_metrics)\n",
    "# df = df[df[\"Variance\"] == 2.0]\n",
    "# df[\"Method\"] = \"MPS\"\n",
    "print((df.groupby(\"Method\").mean() * 100).to_latex(float_format=\"%.2f\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train on noise "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytorch_ood.dataset.img import UniformNoise\n",
    "\n",
    "def train_noise(outlier_set=10, iters=1000):\n",
    "    dataset_out_train = UniformNoise(length=80000, size=(224,224,3), \n",
    "                                     target_transform=ToUnknown(), transform=preprocess)\n",
    "    train_out_loader = DataLoader(dataset_out_train, batch_size=64, num_workers=20, shuffle=True)\n",
    "\n",
    "    dataset_in_train = ImageNet(split=\"train\", root=\"/data_fast/kirchheim/datasets/imagenet-2012/\", transform=preprocess)\n",
    "    imagenet_in_loader = DataLoader(dataset_in_train, batch_size=64, num_workers=20, shuffle=True)\n",
    "\n",
    "    # in test set \n",
    "    dataset_in_test = ImageNet(split=\"val\", root=\"/data_fast/kirchheim/datasets/imagenet-2012/\", transform=preprocess)\n",
    "    test_in_loader = DataLoader(dataset_in_test, batch_size=256, num_workers=20)\n",
    "\n",
    "    # ood test sets \n",
    "    imagenet_r = ImageNetR(root=\"/data_fast/kirchheim/datasets/\", \n",
    "                           download=True, transform=preprocess, target_transform=ToUnknown())\n",
    "    imagenet_r_loader = DataLoader(imagenet_r + dataset_in_test, batch_size=128, num_workers=12)\n",
    "    \n",
    "    imagenet_o = ImageNetO(root=\"/data_fast/kirchheim/datasets/\", \n",
    "                           download=True, transform=preprocess, target_transform=ToUnknown())\n",
    "    imagenet_o_loader = DataLoader(imagenet_o + dataset_in_test, batch_size=128, num_workers=12)\n",
    "\n",
    "    imagenet_a = ImageNetA(root=\"/data_fast/kirchheim/datasets/\", \n",
    "                           download=True, transform=preprocess, target_transform=ToUnknown())\n",
    "    imagenet_a_loader = DataLoader(imagenet_a + dataset_in_test, batch_size=128, num_workers=12)\n",
    "    \n",
    "    \n",
    "    model = VisionTransformer(image_size=(224, 224), pretrained=\"b16-im21k-224\", num_classes=1000) # resnet101(pretrained=True)\n",
    "    model._from_pretrained(\"b16-im21k-224\")\n",
    "    model = model.cuda()\n",
    "\n",
    "    opti = SGD(model.parameters(), lr=0.001)\n",
    "\n",
    "    criterion = OutlierExposureLoss(alpha=0.5)\n",
    "\n",
    "    mav = 0.0\n",
    "    mavs = []\n",
    "    perf_metrics = []\n",
    "\n",
    "    current_auroc = None \n",
    "\n",
    "    it_train_in = iter(imagenet_in_loader)\n",
    "    it_train_out = iter(train_out_loader)\n",
    "\n",
    "    criterion = OutlierExposureLoss()\n",
    "    bar = tqdm.tqdm(range(iters))\n",
    "\n",
    "    for i in bar:\n",
    "        try:\n",
    "            x1, y1 = next(it_train_in)\n",
    "            x2, y2 = next(it_train_out)\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "        except:\n",
    "            it_train_in = iter(imagenet_in_loader)\n",
    "            it_train_out = iter(train_out_loader)\n",
    "\n",
    "        x = torch.cat([x1, x2])\n",
    "        y = torch.cat([y1, y2])\n",
    "\n",
    "        x = x.cuda()\n",
    "        y = y.cuda()\n",
    "\n",
    "        y_hat = model(x)\n",
    "        loss = criterion(y_hat, y)\n",
    "        opti.zero_grad()\n",
    "        loss.backward()\n",
    "        opti.step()\n",
    "\n",
    "        mav = 0.2 * loss.item() + 0.8 * mav\n",
    "        mavs.append(mav)\n",
    "\n",
    "        bar.set_postfix({\"loss\": mav, \"AUROC\": current_auroc})\n",
    "    \n",
    "    print(f\"Evaluating... \")\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        softmax = MaxSoftmax(model)\n",
    "        energy = EnergyBased(model)\n",
    "        \n",
    "        ###\n",
    "        metrics = OODMetrics()\n",
    "        metrics_energy = OODMetrics()\n",
    "        \n",
    "        for x, y in imagenet_a_loader:\n",
    "            logits = model(x.cuda())\n",
    "            metrics.update(softmax.score(logits), y)\n",
    "            metrics_energy.update(energy.score(logits), y)\n",
    "\n",
    "        m = metrics.compute()\n",
    "        m.update({\n",
    "            \"Dataset\": \"ImageNetA\",\n",
    "            \"Iteration\": i,\n",
    "            \"Method\": \"Softmax\"\n",
    "        })\n",
    "        perf_metrics.append(m)\n",
    "        \n",
    "        m = metrics_energy.compute()\n",
    "        m.update({\n",
    "            \"Dataset\": \"ImageNetA\",\n",
    "            \"Iteration\": i,\n",
    "            \"Method\": \"Energy\"\n",
    "        })\n",
    "        perf_metrics.append(m)\n",
    "        \n",
    "\n",
    "        ###\n",
    "        metrics = OODMetrics()\n",
    "        metrics_energy = OODMetrics()\n",
    "        \n",
    "        for x, y in imagenet_o_loader:\n",
    "            logits = model(x.cuda())\n",
    "            metrics.update(softmax.score(logits), y)\n",
    "            metrics_energy.update(energy.score(logits), y)\n",
    "\n",
    "        m = metrics.compute()\n",
    "        m.update({\n",
    "            \"Dataset\": \"ImageNetO\",\n",
    "            \"Iteration\": i,\n",
    "            \"Method\": \"Softmax\"\n",
    "        })\n",
    "        perf_metrics.append(m)\n",
    "        \n",
    "        m = metrics_energy.compute()\n",
    "        m.update({\n",
    "            \"Dataset\": \"ImageNetO\",\n",
    "            \"Iteration\": i,\n",
    "            \"Method\": \"Energy\"\n",
    "        })\n",
    "        perf_metrics.append(m)\n",
    "\n",
    "        ###\n",
    "        metrics = OODMetrics()\n",
    "        metrics_energy = OODMetrics()\n",
    "        \n",
    "        for x, y in imagenet_r_loader:\n",
    "            logits = model(x.cuda())\n",
    "            metrics.update(softmax.score(logits), y)\n",
    "            metrics_energy.update(energy.score(logits), y)\n",
    "\n",
    "        m = metrics.compute()\n",
    "        m.update({\n",
    "            \"Dataset\": \"ImageNetR\",\n",
    "            \"Iteration\": i,\n",
    "            \"Method\": \"Softmax\"\n",
    "        })\n",
    "        perf_metrics.append(m)\n",
    "        \n",
    "        m = metrics_energy.compute()\n",
    "        m.update({\n",
    "            \"Dataset\": \"ImageNetR\",\n",
    "            \"Iteration\": i,\n",
    "            \"Method\": \"Energy\"\n",
    "        })\n",
    "        perf_metrics.append(m)\n",
    "\n",
    "        print(f\"AUROC {i} -> {m['AUROC']:.3%}\")\n",
    "        current_auroc = m['AUROC']\n",
    "\n",
    "    model.train()\n",
    "\n",
    "    return perf_metrics\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb7a857374a4473692d54579973532f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating... \n",
      "AUROC 999 -> 85.334%\n"
     ]
    }
   ],
   "source": [
    "noise_metrics = train_noise()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrrrrr}\n",
      "\\toprule\n",
      "{} &  AUROC &  AUPR-IN &  AUPR-OUT &  ACC95TPR &  FPR95TPR &  Iteration \\\\\n",
      "Method  &        &          &           &           &           &            \\\\\n",
      "\\midrule\n",
      "Energy  &  88.87 &    62.66 &     95.58 &     62.91 &     46.05 &   99900.00 \\\\\n",
      "Softmax &  82.78 &    50.93 &     93.89 &     50.25 &     60.63 &   99900.00 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(noise_metrics)\n",
    "# df = df[df[\"Variance\"] == 2.0]\n",
    "# df[\"Method\"] = \"MPS\"\n",
    "print((df.groupby(\"Method\").mean() * 100).to_latex(float_format=\"%.2f\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "myenv",
   "language": "python",
   "name": "myenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}