notebooks/tabsles.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:60% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "import scipy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "from matplotlib.colors import ListedColormap\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "import matplotlib\n",
    "\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', 5000)\n",
    "pd.set_option('display.max_columns', 5000)\n",
    "pd.set_option('display.width', 10000)\n",
    "\n",
    "from collections import defaultdict\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "import seaborn as sns\n",
    "sns.set_style('whitegrid')\n",
    "\n",
    "%matplotlib inline \n",
    "current_palette = sns.color_palette()\n",
    "cmap = current_palette.as_hex()\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:60% !important; }</style>\"))\n",
    "\n",
    "current_palette = sns.color_palette()\n",
    "cmap = ListedColormap(current_palette.as_hex())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "colors = {\n",
    "    'deepens': '#1f77b4', \n",
    "    'dropout': '#ff7f0e',\n",
    "    'kfacl': '#2ca02c',\n",
    "    'onenet': '#d62728',\n",
    "    'vi': '#9467bd',\n",
    "    'csgld':'#8c564b',\n",
    "    'sse':'#e377c2',\n",
    "    'swag': '#7f7f7f', \n",
    "    'fge': '#bcbd22'}\n",
    "\n",
    "names = {\n",
    "    'acc': 'Accuracy (%)',\n",
    "    'll': 'Log-Likelihood',\n",
    "    'brier': 'Brier',\n",
    "    'acc-aac': 'AURC'\n",
    "}\n",
    "\n",
    "names_ = {\n",
    "    'deepens': 'Deep ensemble', \n",
    "    'dropout': 'Dropout',\n",
    "    'kfacl': 'K-FAC-L',\n",
    "    'onenet': 'Single model',\n",
    "    'vi': 'FFG VI',\n",
    "    'csgld':'cSGLD',\n",
    "    'sse':'SSE',\n",
    "    'swag': 'SWAG', \n",
    "    'fge': 'FGE'}\n",
    "\n",
    "names_long = {\n",
    "    'deepens': 'Deep Ensembles', \n",
    "    'dropout': 'Dropout',\n",
    "    'kfacl': 'KFAC-Laplace',\n",
    "    'onenet': 'Single model',\n",
    "    'vi': 'Variational Inf. (FFG)',\n",
    "    'csgld':'Cyclic SGLD',\n",
    "    'sse':'Snapshot Ensembles',\n",
    "    'swag': 'SWA-Gaussian', \n",
    "    'fge': 'Fast Geometric Ens.', \n",
    "    'stochbn':'SBN'}\n",
    "\n",
    "    \n",
    "precition = {\n",
    "    'acc': '%.2f',\n",
    "    'acc@5': '%.2f',\n",
    "    'll':  '%.3f',\n",
    "    'brier': '%.3f',\n",
    "    'acc-aac': '%.4f',\n",
    "    'dee': '%.1f'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reading log-files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!unzip logs.zip > unzip_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = './logs/'\n",
    "df = pd.concat([pd.read_csv(path + f) for f in os.listdir(path)], sort=False)\n",
    "df.model = df.model.apply(lambda x: x.replace('do', '').replace('Bayes', ''))\n",
    "df.metric = df.metric.apply(lambda x: x.replace('_', '-'))\n",
    "df.method = df.method.apply(lambda x: x.lower().replace('_', '-').replace('kfaclaplace', 'kfacl'))\n",
    "df = df[df.metric.apply(lambda x: x in ['acc-aac', 'acc', 'll', 'brier'])]\n",
    "\n",
    "df.model = df.model.apply(\n",
    "    lambda x: x.replace('BN', '').replace('Pre', '').replace('28x10', ''))\n",
    "dfbase_ung = df\n",
    "dfbase = df.groupby(\n",
    "    ['dataset', 'model', 'method', 'n_samples', 'metric']\n",
    ").value.agg(['mean', 'std', len]).reset_index()\n",
    "dfbase = dfbase[dfbase.method.apply(lambda x: 'stochbn' not in x)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>model</th>\n",
       "      <th>method</th>\n",
       "      <th>n_samples</th>\n",
       "      <th>metric</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>33966</th>\n",
       "      <td>CIFAR10</td>\n",
       "      <td>VGG16</td>\n",
       "      <td>onenet</td>\n",
       "      <td>92</td>\n",
       "      <td>brier</td>\n",
       "      <td>0.102627</td>\n",
       "      <td>0.001480</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53971</th>\n",
       "      <td>CIFAR10</td>\n",
       "      <td>WideResNet</td>\n",
       "      <td>vi-augment</td>\n",
       "      <td>93</td>\n",
       "      <td>ll</td>\n",
       "      <td>-0.125742</td>\n",
       "      <td>0.000195</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69206</th>\n",
       "      <td>CIFAR100</td>\n",
       "      <td>ResNet164</td>\n",
       "      <td>deepens (ts)</td>\n",
       "      <td>2</td>\n",
       "      <td>brier</td>\n",
       "      <td>0.275663</td>\n",
       "      <td>0.002134</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        dataset       model        method  n_samples metric      mean       std  len\n",
       "33966   CIFAR10       VGG16        onenet         92  brier  0.102627  0.001480  5.0\n",
       "53971   CIFAR10  WideResNet    vi-augment         93     ll -0.125742  0.000195  5.0\n",
       "69206  CIFAR100   ResNet164  deepens (ts)          2  brier  0.275663  0.002134  5.0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfbase.sample(frac=0.0001).head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to read the table \n",
    "\n",
    "1. Methods (dfbase.method)\n",
    "    - Available methods: ```kfacl, vi, csgld, sse, dropout, fge, swag, deepens, onenet```\n",
    "    - All methods can be combined with ```'(ts)'``` substring that denotes a method with a temperature-scaling.\n",
    "    - All methods can be combined with ```'-augment'``` substring that denotes test-time augmentation---one augmentation of an image per one member of an ensemble.\n",
    "2. Datasets (dfbase.dataset column): \n",
    "    - Available datasets: `CIFAR100, CIFAR10, ImageNet`\n",
    "3. Models (dfbase.model):\n",
    "    - Available models for CIFAR-10/100: ```VGG16, ResNet110, WideResNet, ResNet164```\n",
    "    - Available model for ImageNet is ```ResNet50```\n",
    "4. Size of an ensemble \n",
    "    - Number of members of an ensemble (`dfbase.n_samples`) typically is 1-100 for CIFARs and 1-50 for ImageNet.\n",
    "5. Metrics:\n",
    "    - Available metrics: `acc, ll, brier, acc-aac`\n",
    "    - Where df.mean and df.std stores mean and standard deviation.\n",
    "    \n",
    "### How to work with data frame \n",
    "1. We find `df.query` quite usefull. There is an example: `dfbase.query(\"method=='deepens' and dataset=='ImageNet' and metric=='acc' and n_samples<5\")`  \n",
    "2. Iteration via a data frame is also very usefull. There is an example: `for row in dfbase.values[:3]: print (row)`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Accuracy (%)</th>\n",
       "      <th>cSGLD*</th>\n",
       "      <th>Deep ensemble*</th>\n",
       "      <th>FGE*</th>\n",
       "      <th>K-FAC-L*</th>\n",
       "      <th>Single model*</th>\n",
       "      <th>SSE*</th>\n",
       "      <th>SWAG*</th>\n",
       "      <th>FFG VI*</th>\n",
       "      <th>Dropout*</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ResNet110</th>\n",
       "      <td>18.07±0.16</td>\n",
       "      <td>16.97±0.07</td>\n",
       "      <td>19.64±0.15</td>\n",
       "      <td>22.14±0.29</td>\n",
       "      <td>22.66±0.31</td>\n",
       "      <td>18.48±0.25</td>\n",
       "      <td>20.69±0.19</td>\n",
       "      <td>21.86±0.07</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ResNet164</th>\n",
       "      <td>17.13±0.18</td>\n",
       "      <td>16.50±0.01</td>\n",
       "      <td>18.73±0.25</td>\n",
       "      <td>21.03±0.38</td>\n",
       "      <td>21.39±0.40</td>\n",
       "      <td>17.68±0.07</td>\n",
       "      <td>19.48±0.19</td>\n",
       "      <td>20.82±0.04</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VGG16</th>\n",
       "      <td>21.15±0.11</td>\n",
       "      <td>19.88±0.10</td>\n",
       "      <td>22.78±0.22</td>\n",
       "      <td>25.70±0.38</td>\n",
       "      <td>25.44±0.29</td>\n",
       "      <td>21.03±0.10</td>\n",
       "      <td>22.97±0.20</td>\n",
       "      <td>24.56±0.07</td>\n",
       "      <td>25.60±0.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WideResNet</th>\n",
       "      <td>16.29±0.10</td>\n",
       "      <td>15.77±0.04</td>\n",
       "      <td>17.12±0.16</td>\n",
       "      <td>19.43±0.21</td>\n",
       "      <td>19.31±0.24</td>\n",
       "      <td>16.07±0.07</td>\n",
       "      <td>17.08±0.19</td>\n",
       "      <td>18.74±0.08</td>\n",
       "      <td>19.22±0.15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Accuracy (%)      cSGLD* Deep ensemble*        FGE*    K-FAC-L* Single model*        SSE*       SWAG*     FFG VI*    Dropout*\n",
       "ResNet110     18.07±0.16     16.97±0.07  19.64±0.15  22.14±0.29    22.66±0.31  18.48±0.25  20.69±0.19  21.86±0.07            \n",
       "ResNet164     17.13±0.18     16.50±0.01  18.73±0.25  21.03±0.38    21.39±0.40  17.68±0.07  19.48±0.19  20.82±0.04            \n",
       "VGG16         21.15±0.11     19.88±0.10  22.78±0.22  25.70±0.38    25.44±0.29  21.03±0.10  22.97±0.20  24.56±0.07  25.60±0.17\n",
       "WideResNet    16.29±0.10     15.77±0.04  17.12±0.16  19.43±0.21    19.31±0.24  16.07±0.07  17.08±0.19  18.74±0.08  19.22±0.15"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = dfbase\n",
    "ts = True\n",
    "addlen = False\n",
    "if ts: df = dfbase[dfbase.method.apply(lambda x: 'ts' in x)]\n",
    "else:  df = dfbase[dfbase.method.apply(lambda x: 'ts' not in x)]\n",
    "latex = True\n",
    "td = defaultdict(lambda: dict())\n",
    "    \n",
    "for dataset_, ns_ in [('CIFAR100', 100)]:\n",
    "    for metric_ in ['acc']:\n",
    "        label = '%s_%s_%s' % (dataset_, metric_, ns_) \n",
    "        for i in df.values:\n",
    "            dataset, model, metod, ns, metric, val, std, len_ = i \n",
    "            if '-augment' in metod: continue\n",
    "            metod = metod.replace('-augment', '')\n",
    "            if dataset == dataset_  and metric == metric_ and ns == ns_:\n",
    "                opt_t = ' (ts)' in metod\n",
    "                metod = names_[metod.replace(' (ts)', '')]\n",
    "                # model += ' (%s samples)' % ns_\n",
    "                if opt_t: metod = metod+'*'\n",
    "                if metric == 'acc':\n",
    "                    td[metod][model] = '%.2f%s' % (100*(1-val), (('±%.2f' % (std*100)) if str(std) != 'nan' else ''))\n",
    "                elif metric == 'acc-aac':\n",
    "                    td[metod][model] = '%.4f±%.2f' % (val, std)\n",
    "                elif metric=='ll':\n",
    "                    td[metod][model] = '%.3f%s' % (-val, (('±%.3f' % std) if str(std) != 'nan' else ''))\n",
    "                td[metod][model].replace('nan', '')\n",
    "        \n",
    "        df_ = pd.DataFrame.from_dict(td) \n",
    "        df_ = df_.replace(np.nan, '', regex=True)\n",
    "        cap = '%s on %s dataset for diferent number of samples' % (('c' if ts and metric_ != 'acc' else '') + names[metric_], dataset_)\n",
    "\n",
    "        df_.columns.name = '%s' % (('Neg. ' if metric_ == 'll' else '') + names[metric_])\n",
    "        display(df_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Accuracy (%)</th>\n",
       "      <th>ResNet110</th>\n",
       "      <th>ResNet164</th>\n",
       "      <th>VGG16</th>\n",
       "      <th>WideResNet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cSGLD*</th>\n",
       "      <td>18.07±0.16</td>\n",
       "      <td>17.13±0.18</td>\n",
       "      <td>21.15±0.11</td>\n",
       "      <td>16.29±0.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Deep ensemble*</th>\n",
       "      <td>16.97±0.07</td>\n",
       "      <td>16.50±0.01</td>\n",
       "      <td>19.88±0.10</td>\n",
       "      <td>15.77±0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FGE*</th>\n",
       "      <td>19.64±0.15</td>\n",
       "      <td>18.73±0.25</td>\n",
       "      <td>22.78±0.22</td>\n",
       "      <td>17.12±0.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>K-FAC-L*</th>\n",
       "      <td>22.14±0.29</td>\n",
       "      <td>21.03±0.38</td>\n",
       "      <td>25.70±0.38</td>\n",
       "      <td>19.43±0.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Single model*</th>\n",
       "      <td>22.66±0.31</td>\n",
       "      <td>21.39±0.40</td>\n",
       "      <td>25.44±0.29</td>\n",
       "      <td>19.31±0.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SSE*</th>\n",
       "      <td>18.48±0.25</td>\n",
       "      <td>17.68±0.07</td>\n",
       "      <td>21.03±0.10</td>\n",
       "      <td>16.07±0.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SWAG*</th>\n",
       "      <td>20.69±0.19</td>\n",
       "      <td>19.48±0.19</td>\n",
       "      <td>22.97±0.20</td>\n",
       "      <td>17.08±0.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FFG VI*</th>\n",
       "      <td>21.86±0.07</td>\n",
       "      <td>20.82±0.04</td>\n",
       "      <td>24.56±0.07</td>\n",
       "      <td>18.74±0.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dropout*</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>25.60±0.17</td>\n",
       "      <td>19.22±0.15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Accuracy (%)     ResNet110   ResNet164       VGG16  WideResNet\n",
       "cSGLD*          18.07±0.16  17.13±0.18  21.15±0.11  16.29±0.10\n",
       "Deep ensemble*  16.97±0.07  16.50±0.01  19.88±0.10  15.77±0.04\n",
       "FGE*            19.64±0.15  18.73±0.25  22.78±0.22  17.12±0.16\n",
       "K-FAC-L*        22.14±0.29  21.03±0.38  25.70±0.38  19.43±0.21\n",
       "Single model*   22.66±0.31  21.39±0.40  25.44±0.29  19.31±0.24\n",
       "SSE*            18.48±0.25  17.68±0.07  21.03±0.10  16.07±0.07\n",
       "SWAG*           20.69±0.19  19.48±0.19  22.97±0.20  17.08±0.19\n",
       "FFG VI*         21.86±0.07  20.82±0.04  24.56±0.07  18.74±0.08\n",
       "Dropout*                                25.60±0.17  19.22±0.15"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = dfbase\n",
    "ts = True\n",
    "addlen = False\n",
    "if ts: df = dfbase[dfbase.method.apply(lambda x: 'ts' in x)]\n",
    "else:  df = dfbase[dfbase.method.apply(lambda x: 'ts' not in x)]\n",
    "latex = True\n",
    "td = defaultdict(lambda: dict())\n",
    "    \n",
    "for dataset_, ns_ in [('CIFAR100', 100)]:\n",
    "    for metric_ in ['acc']:\n",
    "        label = '%s_%s_%s' % (dataset_, metric_, ns_) \n",
    "        for i in df.values:\n",
    "            dataset, model, metod, ns, metric, val, std, len_ = i \n",
    "            if '-augment' in metod: continue\n",
    "            metod = metod.replace('-augment', '')\n",
    "            if dataset == dataset_  and metric == metric_ and ns == ns_:\n",
    "                opt_t = ' (ts)' in metod\n",
    "                metod = names_[metod.replace(' (ts)', '')]\n",
    "                # model += ' (%s samples)' % ns_\n",
    "                if opt_t: metod = metod+'*'\n",
    "                if metric == 'acc':\n",
    "                    td[model][metod] = '%.2f%s' % (100*(1-val), (('±%.2f' % (std*100)) if str(std) != 'nan' else ''))\n",
    "                elif metric == 'acc-aac':\n",
    "                    td[model][metod] = '%.4f±%.2f' % (val, std)\n",
    "                elif metric=='ll':\n",
    "                    td[model][metod] = '%.3f%s' % (-val, (('±%.3f' % std) if str(std) != 'nan' else ''))\n",
    "                td[model][metod].replace('nan', '')\n",
    "        \n",
    "        df_ = pd.DataFrame.from_dict(td) \n",
    "        df_ = df_.replace(np.nan, '', regex=True)\n",
    "        cap = '%s on %s dataset for diferent number of samples' % (('c' if ts and metric_ != 'acc' else '') + names[metric_], dataset_)\n",
    "\n",
    "        df_.columns.name = '%s' % (('Neg. ' if metric_ == 'll' else '') + names[metric_])\n",
    "        display(df_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Ensembling Method</th>\n",
       "      <th>ResNet50 (1)</th>\n",
       "      <th>ResNet50 (10)</th>\n",
       "      <th>ResNet50 (50)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Deep Ensembles</th>\n",
       "      <td>$0.935$ vs $1.292$±0.005↑</td>\n",
       "      <td>$0.805$ vs $0.793$±0.002↓</td>\n",
       "      <td>$0.788$ vs $0.739$±0.000↓</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fast Geometric Ens.</th>\n",
       "      <td>$0.929$ vs $1.288$±0.008↑</td>\n",
       "      <td>$0.916$ vs $0.842$±0.001↓</td>\n",
       "      <td>$0.904$ vs $0.793$±0.001↓</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KFAC-Laplace</th>\n",
       "      <td>$0.988$ vs $1.298$±0.008↑</td>\n",
       "      <td>$0.939$ vs $0.855$±0.005↓</td>\n",
       "      <td>$0.934$ vs $0.809$±0.004↓</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Single model</th>\n",
       "      <td>$0.938$ vs $1.292$±0.006↑</td>\n",
       "      <td>$0.938$ vs $0.851$±0.004↓</td>\n",
       "      <td>$0.938$ vs $0.805$±0.003↓</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Snapshot Ensembles</th>\n",
       "      <td>$0.983$ vs $1.361$±0.005↑</td>\n",
       "      <td>$0.843$ vs $0.815$±0.002↓</td>\n",
       "      <td>$0.830$ vs $0.763$±0.000↓</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variational Inf. (FFG)</th>\n",
       "      <td>$0.957$ vs $1.326$±0.007↑</td>\n",
       "      <td>$0.923$ vs $0.851$±0.001↓</td>\n",
       "      <td>$0.920$ vs $0.805$±0.001↓</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Ensembling Method                    ResNet50 (1)              ResNet50 (10)              ResNet50 (50)\n",
       "Deep Ensembles          $0.935$ vs $1.292$±0.005↑  $0.805$ vs $0.793$±0.002↓  $0.788$ vs $0.739$±0.000↓\n",
       "Fast Geometric Ens.     $0.929$ vs $1.288$±0.008↑  $0.916$ vs $0.842$±0.001↓  $0.904$ vs $0.793$±0.001↓\n",
       "KFAC-Laplace            $0.988$ vs $1.298$±0.008↑  $0.939$ vs $0.855$±0.005↓  $0.934$ vs $0.809$±0.004↓\n",
       "Single model            $0.938$ vs $1.292$±0.006↑  $0.938$ vs $0.851$±0.004↓  $0.938$ vs $0.805$±0.003↓\n",
       "Snapshot Ensembles      $0.983$ vs $1.361$±0.005↑  $0.843$ vs $0.815$±0.002↓  $0.830$ vs $0.763$±0.000↓\n",
       "Variational Inf. (FFG)  $0.957$ vs $1.326$±0.007↑  $0.923$ vs $0.851$±0.001↓  $0.920$ vs $0.805$±0.001↓"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{llll}\n",
      "\\toprule\n",
      "Ensembling Method &               ResNet50 (1) &              ResNet50 (10) &              ResNet50 (50) \\\\\n",
      "\\midrule\n",
      "Deep Ensembles         &  $0.935$ vs $1.292${\\tiny $\\pm0.005\\color{darkpink}{\\uparrow}$} &  $0.805$ vs $0.793${\\tiny $\\pm0.002\\color{darkpastelgreen}{\\downarrow}$} &  $0.788$ vs $0.739${\\tiny $\\pm0.000\\color{darkpastelgreen}{\\downarrow}$} \\\\\n",
      "Fast Geometric Ens.    &  $0.929$ vs $1.288${\\tiny $\\pm0.008\\color{darkpink}{\\uparrow}$} &  $0.916$ vs $0.842${\\tiny $\\pm0.001\\color{darkpastelgreen}{\\downarrow}$} &  $0.904$ vs $0.793${\\tiny $\\pm0.001\\color{darkpastelgreen}{\\downarrow}$} \\\\\n",
      "KFAC-Laplace           &  $0.988$ vs $1.298${\\tiny $\\pm0.008\\color{darkpink}{\\uparrow}$} &  $0.939$ vs $0.855${\\tiny $\\pm0.005\\color{darkpastelgreen}{\\downarrow}$} &  $0.934$ vs $0.809${\\tiny $\\pm0.004\\color{darkpastelgreen}{\\downarrow}$} \\\\\n",
      "Single model           &  $0.938$ vs $1.292${\\tiny $\\pm0.006\\color{darkpink}{\\uparrow}$} &  $0.938$ vs $0.851${\\tiny $\\pm0.004\\color{darkpastelgreen}{\\downarrow}$} &  $0.938$ vs $0.805${\\tiny $\\pm0.003\\color{darkpastelgreen}{\\downarrow}$} \\\\\n",
      "Snapshot Ensembles     &  $0.983$ vs $1.361${\\tiny $\\pm0.005\\color{darkpink}{\\uparrow}$} &  $0.843$ vs $0.815${\\tiny $\\pm0.002\\color{darkpastelgreen}{\\downarrow}$} &  $0.830$ vs $0.763${\\tiny $\\pm0.000\\color{darkpastelgreen}{\\downarrow}$} \\\\\n",
      "Variational Inf. (FFG) &  $0.957$ vs $1.326${\\tiny $\\pm0.007\\color{darkpink}{\\uparrow}$} &  $0.923$ vs $0.851${\\tiny $\\pm0.001\\color{darkpastelgreen}{\\downarrow}$} &  $0.920$ vs $0.805${\\tiny $\\pm0.001\\color{darkpastelgreen}{\\downarrow}$} \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = dfbase\n",
    "ts = True\n",
    "if ts: df = dfbase[dfbase.method.apply(lambda x: 'ts' in x)]\n",
    "else:  df = dfbase[dfbase.method.apply(lambda x: 'ts' not in x)]\n",
    "\n",
    "td = defaultdict(lambda: dict())\n",
    "for dataset_, ns_ in [('ImageNet', 1), ('ImageNet', 10), ('ImageNet', 50)]:\n",
    "    for metric_ in ['ll']:\n",
    "        label = '%s_%s_%s' % (dataset_, metric_, ns_) \n",
    "        for i in df.values:\n",
    "            dataset, model, metod, ns, metric, val, std, len_ = i \n",
    "            model += ' (%s)' % ns_\n",
    "            if '-augment' not in metod:\n",
    "                opt_t = ' (ts)' in metod\n",
    "                metod = names_long[metod.replace(' (ts)', '')]\n",
    "                if dataset == dataset_  and metric == metric_ and ns == ns_:\n",
    "                    metod, model = model, metod\n",
    "                    if metric=='ll':\n",
    "                        td[metod][model] = (-val)\n",
    "                    \n",
    "        for i in df.values:\n",
    "            dataset, model, metod, ns, metric, val, std, len_ = i \n",
    "            model += ' (%s)' % ns_\n",
    "            if '-augment' in metod:         \n",
    "                metod = metod.replace('-augment', '')\n",
    "                if dataset == dataset_  and metric == metric_ and ns == ns_:\n",
    "                    \n",
    "                    opt_t = ' (ts)' in metod\n",
    "                    if metric == 'll': val=-val\n",
    "                    metod = names_long[metod.replace(' (ts)', '')]\n",
    "                    prev = td[model][metod]\n",
    "                    cur = val if metric!='acc' else (100-100*(val))\n",
    "                    pm = precition[metric]\n",
    "                    \n",
    "                    if (pm % prev) == (pm % cur):\n",
    "                        s = '≈'\n",
    "                    elif prev > cur:\n",
    "                        s = '↓'\n",
    "                    else:\n",
    "                        s = '↑'\n",
    "                        \n",
    "                    vs = '$%s$ vs $%s$' % (precition[metric_], precition[metric_])\n",
    "                    #vs = '%s vs %s' % (precition[metric_], precition[metric_])\n",
    "                    metod, model = model, metod\n",
    "                    if metric == 'll':\n",
    "                        td[metod][model] = vs % (prev, cur)\n",
    "                    td[metod][model] += '±%.3f%s' % (std, s)\n",
    "                        \n",
    "                    \n",
    "        \n",
    "df_ = pd.DataFrame.from_dict(td)\n",
    "df_ = df_.replace(np.nan, '', regex=True)\n",
    "cap = '%s on %s dataset for diferent number of samples' % (('c' if ts and metric_ != 'acc' else '') + names[metric_], dataset_)\n",
    "\n",
    "df_.columns.name = 'Ensembling Method'\n",
    "display(df_)\n",
    "\n",
    "print(df_.to_latex().replace(\n",
    "    '±', '{\\\\tiny $\\pm').replace(\n",
    "    '↑', '\\color{darkpink}{\\\\uparrow}$}').replace(\n",
    "    '↓', '\\color{darkpastelgreen}{\\\\downarrow}$}').replace('\\$', '$'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'ImageNet'\n",
    "models = ['VGG16', 'ResNet110', 'ResNet164', 'WideResNet', 'ResNet50']\n",
    "methods = ['dropout', 'swag', 'csgld', 'fge', 'deepens', 'onenet', 'vi', 'kfacl', 'sse']\n",
    "\n",
    "rows, row_counter = defaultdict(lambda: dict()), 0\n",
    "for model in models:\n",
    "    for method in methods:\n",
    "        metrics = defaultdict(lambda: dict())\n",
    "        metrics_ = defaultdict(lambda: dict())\n",
    "        metrics_std = defaultdict(lambda: dict())\n",
    "        for line in df.values:\n",
    "            dataset_, model_, method_, ns_, metric_, val_, std_, len_ = line\n",
    "            if 'ts' not in method_: continue\n",
    "            if 'aug' not in method_:\n",
    "                method_ = method_.replace('-augment', '').replace(' (ts)', '')\n",
    "                if dataset_ == dataset and model_==model and method_ == method and metric_ in precition:\n",
    "                    if metric_ == 'acc':\n",
    "                        metrics[metric_][ns_] = '?'+ (precition[metric_] % (100*(1-val_))) + '?'\n",
    "                        std = precition[metric_] % (std_*100)\n",
    "                    if metric_ == 'll':\n",
    "                        metrics[metric_][ns_] = '?' + (precition[metric_] % -val_) + '?'\n",
    "                        std = precition[metric_] % std_\n",
    "\n",
    "            else:\n",
    "                method_ = method_.replace('-augment', '').replace(' (ts)', '')\n",
    "                if dataset_ == dataset and model_==model and method_ == method and metric_ in precition:\n",
    "                    if metric_ == 'acc':\n",
    "                        metrics_[metric_][ns_] = '?'+ (precition[metric_] % (100*(1-val_)))\n",
    "                        metrics_std[metric_][ns_] = precition[metric_] % (std_*100)\n",
    "                    if metric_ == 'll':\n",
    "                        metrics_[metric_][ns_] = '?' + (precition[metric_] % -val_)\n",
    "                        metrics_std[metric_][ns_] = precition[metric_] % std_\n",
    "\n",
    "        def get_arrow(x):\n",
    "            prev = metrics[x[0]][x[1]].replace('?', '')\n",
    "            cur = metrics_[x[0]][x[1]].replace('?', '')\n",
    "            std =  metrics_std[x[0]][x[1]]\n",
    "            if prev == cur:\n",
    "                s = '≈'\n",
    "            elif np.abs(float(cur)-float(prev)) <= float(std):\n",
    "                s = '≈'\n",
    "            elif float(prev) > float(cur):\n",
    "                s = '↓'\n",
    "            else:\n",
    "                s = '↑'\n",
    "            return s\n",
    "\n",
    "        f = lambda x: metrics[x[0]][x[1]]+vs+metrics_[x[0]][x[1]] + get_arrow(x)\n",
    "        if metrics: \n",
    "            if dataset != 'ImageNet':\n",
    "                rows[row_counter] = [model, names_long[method]]\n",
    "                rows[row_counter] += list(map(f, [('acc', 5), ('acc', 10), ('acc', 100), ('ll', 5), ('ll', 10), ('ll', 100)]))\n",
    "                columns = ['Model', 'Method', 'acc(5)', 'acc(10)', 'acc(100)', 'll(5)', 'll(10)', 'll(100)']\n",
    "            else:\n",
    "                rows[row_counter] = [model, names_long[method]]\n",
    "                rows[row_counter] += list(map(f, [('acc', 5), ('acc', 10), ('acc', 50), ('ll', 5), ('ll', 10), ('ll', 50)]))\n",
    "                columns = ['Model', 'Method', 'acc(5)', 'acc(10)', 'acc(50)', 'll(5)', 'll(10)', 'll(50)']\n",
    "            row_counter+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Method</th>\n",
       "      <th>acc(5)</th>\n",
       "      <th>acc(10)</th>\n",
       "      <th>acc(50)</th>\n",
       "      <th>ll(5)</th>\n",
       "      <th>ll(10)</th>\n",
       "      <th>ll(50)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>Fast Geometric Ens.</td>\n",
       "      <td>?23.61?$%.3f$ vs $%.3f$?22.21↓</td>\n",
       "      <td>?23.56?$%.3f$ vs $%.3f$?21.37↓</td>\n",
       "      <td>?23.28?$%.3f$ vs $%.3f$?20.67↓</td>\n",
       "      <td>?0.921?$%.3f$ vs $%.3f$?0.894↓</td>\n",
       "      <td>?0.916?$%.3f$ vs $%.3f$?0.842↓</td>\n",
       "      <td>?0.904?$%.3f$ vs $%.3f$?0.793↓</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>Deep Ensembles</td>\n",
       "      <td>?21.19?$%.3f$ vs $%.3f$?21.20≈</td>\n",
       "      <td>?20.90?$%.3f$ vs $%.3f$?20.16↓</td>\n",
       "      <td>?20.63?$%.3f$ vs $%.3f$?19.39↓</td>\n",
       "      <td>?0.823?$%.3f$ vs $%.3f$?0.855↑</td>\n",
       "      <td>?0.805?$%.3f$ vs $%.3f$?0.793↓</td>\n",
       "      <td>?0.788?$%.3f$ vs $%.3f$?0.739↓</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>Single model</td>\n",
       "      <td>?23.86?$%.3f$ vs $%.3f$?22.39↓</td>\n",
       "      <td>?23.86?$%.3f$ vs $%.3f$?21.60↓</td>\n",
       "      <td>?23.86?$%.3f$ vs $%.3f$?21.06↓</td>\n",
       "      <td>?0.938?$%.3f$ vs $%.3f$?0.900↓</td>\n",
       "      <td>?0.938?$%.3f$ vs $%.3f$?0.851↓</td>\n",
       "      <td>?0.938?$%.3f$ vs $%.3f$?0.805↓</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>Variational Inf. (FFG)</td>\n",
       "      <td>?23.82?$%.3f$ vs $%.3f$?22.58↓</td>\n",
       "      <td>?23.77?$%.3f$ vs $%.3f$?21.74↓</td>\n",
       "      <td>?23.67?$%.3f$ vs $%.3f$?21.10↓</td>\n",
       "      <td>?0.927?$%.3f$ vs $%.3f$?0.905↓</td>\n",
       "      <td>?0.923?$%.3f$ vs $%.3f$?0.851↓</td>\n",
       "      <td>?0.920?$%.3f$ vs $%.3f$?0.805↓</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>KFAC-Laplace</td>\n",
       "      <td>?24.19?$%.3f$ vs $%.3f$?22.50↓</td>\n",
       "      <td>?23.93?$%.3f$ vs $%.3f$?21.67↓</td>\n",
       "      <td>?23.86?$%.3f$ vs $%.3f$?21.04↓</td>\n",
       "      <td>?0.948?$%.3f$ vs $%.3f$?0.906↓</td>\n",
       "      <td>?0.939?$%.3f$ vs $%.3f$?0.855↓</td>\n",
       "      <td>?0.934?$%.3f$ vs $%.3f$?0.809↓</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>Snapshot Ensembles</td>\n",
       "      <td>?22.21?$%.3f$ vs $%.3f$?21.99↓</td>\n",
       "      <td>?21.75?$%.3f$ vs $%.3f$?20.81↓</td>\n",
       "      <td>?21.48?$%.3f$ vs $%.3f$?19.86↓</td>\n",
       "      <td>?0.865?$%.3f$ vs $%.3f$?0.879↑</td>\n",
       "      <td>?0.843?$%.3f$ vs $%.3f$?0.815↓</td>\n",
       "      <td>?0.830?$%.3f$ vs $%.3f$?0.763↓</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Model                  Method                          acc(5)                         acc(10)                         acc(50)                           ll(5)                          ll(10)                          ll(50)\n",
       "0  ResNet50     Fast Geometric Ens.  ?23.61?$%.3f$ vs $%.3f$?22.21↓  ?23.56?$%.3f$ vs $%.3f$?21.37↓  ?23.28?$%.3f$ vs $%.3f$?20.67↓  ?0.921?$%.3f$ vs $%.3f$?0.894↓  ?0.916?$%.3f$ vs $%.3f$?0.842↓  ?0.904?$%.3f$ vs $%.3f$?0.793↓\n",
       "1  ResNet50          Deep Ensembles  ?21.19?$%.3f$ vs $%.3f$?21.20≈  ?20.90?$%.3f$ vs $%.3f$?20.16↓  ?20.63?$%.3f$ vs $%.3f$?19.39↓  ?0.823?$%.3f$ vs $%.3f$?0.855↑  ?0.805?$%.3f$ vs $%.3f$?0.793↓  ?0.788?$%.3f$ vs $%.3f$?0.739↓\n",
       "2  ResNet50            Single model  ?23.86?$%.3f$ vs $%.3f$?22.39↓  ?23.86?$%.3f$ vs $%.3f$?21.60↓  ?23.86?$%.3f$ vs $%.3f$?21.06↓  ?0.938?$%.3f$ vs $%.3f$?0.900↓  ?0.938?$%.3f$ vs $%.3f$?0.851↓  ?0.938?$%.3f$ vs $%.3f$?0.805↓\n",
       "3  ResNet50  Variational Inf. (FFG)  ?23.82?$%.3f$ vs $%.3f$?22.58↓  ?23.77?$%.3f$ vs $%.3f$?21.74↓  ?23.67?$%.3f$ vs $%.3f$?21.10↓  ?0.927?$%.3f$ vs $%.3f$?0.905↓  ?0.923?$%.3f$ vs $%.3f$?0.851↓  ?0.920?$%.3f$ vs $%.3f$?0.805↓\n",
       "4  ResNet50            KFAC-Laplace  ?24.19?$%.3f$ vs $%.3f$?22.50↓  ?23.93?$%.3f$ vs $%.3f$?21.67↓  ?23.86?$%.3f$ vs $%.3f$?21.04↓  ?0.948?$%.3f$ vs $%.3f$?0.906↓  ?0.939?$%.3f$ vs $%.3f$?0.855↓  ?0.934?$%.3f$ vs $%.3f$?0.809↓\n",
       "5  ResNet50      Snapshot Ensembles  ?22.21?$%.3f$ vs $%.3f$?21.99↓  ?21.75?$%.3f$ vs $%.3f$?20.81↓  ?21.48?$%.3f$ vs $%.3f$?19.86↓  ?0.865?$%.3f$ vs $%.3f$?0.879↑  ?0.843?$%.3f$ vs $%.3f$?0.815↓  ?0.830?$%.3f$ vs $%.3f$?0.763↓"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_ = pd.DataFrame.from_dict(rows, columns=columns, orient='index')\n",
    "display(df_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{llllllll}\n",
      "\\toprule\n",
      "    Model &                  Method &                          acc(5) &                         acc(10) &                         acc(50) &                           ll(5) &                          ll(10) &                          ll(50) \\\\\n",
      "\\midrule\n",
      " ResNet50 &     Fast Geometric Ens. &  $23.61$\\$\\%.3f\\$ vs \\$\\%.3f\\$$22.21_{\\color{darkpastelgreen}{\\downarrow}}$ &  $23.56$\\$\\%.3f\\$ vs \\$\\%.3f\\$$21.37_{\\color{darkpastelgreen}{\\downarrow}}$ &  $23.28$\\$\\%.3f\\$ vs \\$\\%.3f\\$$20.67_{\\color{darkpastelgreen}{\\downarrow}}$ &  $0.921$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.894_{\\color{darkpastelgreen}{\\downarrow}}$ &  $0.916$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.842_{\\color{darkpastelgreen}{\\downarrow}}$ &  $0.904$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.793_{\\color{darkpastelgreen}{\\downarrow}}$ \\\\\n",
      " ResNet50 &          Deep Ensembles &  $21.19$\\$\\%.3f\\$ vs \\$\\%.3f\\$$21.20_{\\color{darkgray}{\\approx}}$ &  $20.90$\\$\\%.3f\\$ vs \\$\\%.3f\\$$20.16_{\\color{darkpastelgreen}{\\downarrow}}$ &  $20.63$\\$\\%.3f\\$ vs \\$\\%.3f\\$$19.39_{\\color{darkpastelgreen}{\\downarrow}}$ &  $0.823$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.855_{\\color{darkpink}{\\uparrow}}$ &  $0.805$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.793_{\\color{darkpastelgreen}{\\downarrow}}$ &  $0.788$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.739_{\\color{darkpastelgreen}{\\downarrow}}$ \\\\\n",
      " ResNet50 &            Single model &  $23.86$\\$\\%.3f\\$ vs \\$\\%.3f\\$$22.39_{\\color{darkpastelgreen}{\\downarrow}}$ &  $23.86$\\$\\%.3f\\$ vs \\$\\%.3f\\$$21.60_{\\color{darkpastelgreen}{\\downarrow}}$ &  $23.86$\\$\\%.3f\\$ vs \\$\\%.3f\\$$21.06_{\\color{darkpastelgreen}{\\downarrow}}$ &  $0.938$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.900_{\\color{darkpastelgreen}{\\downarrow}}$ &  $0.938$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.851_{\\color{darkpastelgreen}{\\downarrow}}$ &  $0.938$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.805_{\\color{darkpastelgreen}{\\downarrow}}$ \\\\\n",
      " ResNet50 &  Variational Inf. (FFG) &  $23.82$\\$\\%.3f\\$ vs \\$\\%.3f\\$$22.58_{\\color{darkpastelgreen}{\\downarrow}}$ &  $23.77$\\$\\%.3f\\$ vs \\$\\%.3f\\$$21.74_{\\color{darkpastelgreen}{\\downarrow}}$ &  $23.67$\\$\\%.3f\\$ vs \\$\\%.3f\\$$21.10_{\\color{darkpastelgreen}{\\downarrow}}$ &  $0.927$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.905_{\\color{darkpastelgreen}{\\downarrow}}$ &  $0.923$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.851_{\\color{darkpastelgreen}{\\downarrow}}$ &  $0.920$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.805_{\\color{darkpastelgreen}{\\downarrow}}$ \\\\\n",
      " ResNet50 &            KFAC-Laplace &  $24.19$\\$\\%.3f\\$ vs \\$\\%.3f\\$$22.50_{\\color{darkpastelgreen}{\\downarrow}}$ &  $23.93$\\$\\%.3f\\$ vs \\$\\%.3f\\$$21.67_{\\color{darkpastelgreen}{\\downarrow}}$ &  $23.86$\\$\\%.3f\\$ vs \\$\\%.3f\\$$21.04_{\\color{darkpastelgreen}{\\downarrow}}$ &  $0.948$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.906_{\\color{darkpastelgreen}{\\downarrow}}$ &  $0.939$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.855_{\\color{darkpastelgreen}{\\downarrow}}$ &  $0.934$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.809_{\\color{darkpastelgreen}{\\downarrow}}$ \\\\\n",
      " ResNet50 &      Snapshot Ensembles &  $22.21$\\$\\%.3f\\$ vs \\$\\%.3f\\$$21.99_{\\color{darkpastelgreen}{\\downarrow}}$ &  $21.75$\\$\\%.3f\\$ vs \\$\\%.3f\\$$20.81_{\\color{darkpastelgreen}{\\downarrow}}$ &  $21.48$\\$\\%.3f\\$ vs \\$\\%.3f\\$$19.86_{\\color{darkpastelgreen}{\\downarrow}}$ &  $0.865$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.879_{\\color{darkpink}{\\uparrow}}$ &  $0.843$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.815_{\\color{darkpastelgreen}{\\downarrow}}$ &  $0.830$\\$\\%.3f\\$ vs \\$\\%.3f\\$$0.763_{\\color{darkpastelgreen}{\\downarrow}}$ \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(df_.to_latex(index=False).replace(\n",
    "    '±', '{\\\\tiny $\\pm ').replace(\n",
    "    '!', '$}').replace(\n",
    "    '?', '$').replace('NAacc', '\\\\text{NA}\\hspace{5pt}').replace('NAll', '\\\\text{NA}\\hspace{8pt}').replace(\n",
    "    '↑', '_{\\color{darkpink}{\\\\uparrow}}$').replace(\n",
    "    '↓', '_{\\color{darkpastelgreen}{\\\\downarrow}}$').replace(\n",
    "    '≈', '_{\\color{darkgray}{\\\\approx}}$'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'ImageNet'\n",
    "models = ['VGG16', 'ResNet110', 'ResNet164', 'WideResNet', 'ResNet50']\n",
    "methods = ['dropout', 'swag', 'csgld', 'fge', 'deepens', 'onenet', 'vi', 'kfacl', 'sse']\n",
    "\n",
    "rows, row_counter = defaultdict(lambda: dict()), 0\n",
    "for model in models:\n",
    "    for method in methods:\n",
    "        metrics = defaultdict(lambda: dict())\n",
    "        for line in df.values:\n",
    "            dataset_, model_, method_, ns_, metric_, val_, std_, len_ = line\n",
    "            if 'ts' not in method_: continue\n",
    "            if 'aug' in method_: continue\n",
    "            method_ = method_.replace('-augment', '').replace(' (ts)', '')\n",
    "            if dataset_ == dataset and model_==model and method_ == method and metric_ in precition:\n",
    "                if metric_ == 'acc':\n",
    "                    metrics[metric_][ns_] = '?'+ (precition[metric_] % (100*(1-val_))) + '?'\n",
    "                    std = precition[metric_] % (std_*100)\n",
    "                if metric_ == 'll':\n",
    "                    metrics[metric_][ns_] = '?' + (precition[metric_] % -val_) + '?'\n",
    "                    std = precition[metric_] % std_\n",
    "                                \n",
    "                if (method == 'deepens' and '10' in dataset and ns_==100) or (\n",
    "                        dataset=='ImageNet' and method_ == 'sse') or (\n",
    "                        dataset=='ImageNet' and method_ == 'deepens' and ns_==50):\n",
    "                        std = ('NAacc' if metric_ == 'acc' else 'NAll')\n",
    "                \n",
    "                if metric_ in metrics and ns_ in metrics[metric_]: \n",
    "                    metrics[metric_][ns_] += ('±' + std + '!')\n",
    "                \n",
    "        if metrics: \n",
    "            if dataset != 'ImageNet':\n",
    "                rows[row_counter] = [\n",
    "                    model, names_long[method], \n",
    "                    metrics['acc'][1], metrics['acc'][5], metrics['acc'][10], metrics['acc'][100],\n",
    "                    metrics['ll'][1],  metrics['ll'][5], metrics['ll'][10],  metrics['ll'][100]]\n",
    "                columns = ['Model', 'Method', 'acc(1)', 'acc(5)', 'acc(10)', 'acc(100)', 'll(1)', 'll(5)', 'll(10)', 'll(100)']\n",
    "            else:\n",
    "                rows[row_counter] = [\n",
    "                    model, names_long[method], \n",
    "                    metrics['acc'][1], metrics['acc'][5], metrics['acc'][10], metrics['acc'][50],\n",
    "                    metrics['ll'][1],  metrics['ll'][5], metrics['ll'][10],  metrics['ll'][50]]\n",
    "                columns = ['Model', 'Method', 'acc(1)', 'acc(5)', 'acc(10)', 'acc(50)', 'll(1)', 'll(5)', 'll(10)', 'll(50)']\n",
    "            row_counter+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{llllllllll}\n",
      "\\toprule\n",
      "    Model &                  Method &          acc(1) &          acc(5) &         acc(10) &         acc(50) &           ll(1) &           ll(5) &          ll(10) &          ll(50) \\\\\n",
      "\\midrule\n",
      " ResNet50 &     Fast Geometric Ens. &   $23.71${\\tiny $\\pm 0.00$} &   $23.61${\\tiny $\\pm 0.00$} &   $23.56${\\tiny $\\pm 0.00$} &   $23.28${\\tiny $\\pm 0.00$} &  $0.929${\\tiny $\\pm 0.000$} &  $0.921${\\tiny $\\pm 0.000$} &  $0.916${\\tiny $\\pm 0.000$} &  $0.904${\\tiny $\\pm 0.000$} \\\\\n",
      " ResNet50 &          Deep Ensembles &   $23.79${\\tiny $\\pm 0.14$} &   $21.19${\\tiny $\\pm 0.14$} &   $20.90${\\tiny $\\pm 0.08$} &  $20.63${\\tiny $\\pm \\text{NA}\\hspace{5pt}$} &  $0.935${\\tiny $\\pm 0.007$} &  $0.823${\\tiny $\\pm 0.002$} &  $0.805${\\tiny $\\pm 0.000$} &   $0.788${\\tiny $\\pm \\text{NA}\\hspace{8pt}$} \\\\\n",
      " ResNet50 &            Single model &   $23.86${\\tiny $\\pm 0.20$} &   $23.86${\\tiny $\\pm 0.20$} &   $23.86${\\tiny $\\pm 0.20$} &   $23.86${\\tiny $\\pm 0.20$} &  $0.938${\\tiny $\\pm 0.006$} &  $0.938${\\tiny $\\pm 0.006$} &  $0.938${\\tiny $\\pm 0.006$} &  $0.938${\\tiny $\\pm 0.006$} \\\\\n",
      " ResNet50 &  Variational Inf. (FFG) &   $24.50${\\tiny $\\pm 0.06$} &   $23.82${\\tiny $\\pm 0.03$} &   $23.77${\\tiny $\\pm 0.04$} &   $23.67${\\tiny $\\pm 0.00$} &  $0.957${\\tiny $\\pm 0.001$} &  $0.927${\\tiny $\\pm 0.000$} &  $0.923${\\tiny $\\pm 0.001$} &  $0.920${\\tiny $\\pm 0.000$} \\\\\n",
      " ResNet50 &            KFAC-Laplace &   $25.01${\\tiny $\\pm 0.49$} &   $24.19${\\tiny $\\pm 0.29$} &   $23.93${\\tiny $\\pm 0.20$} &   $23.86${\\tiny $\\pm 0.16$} &  $0.988${\\tiny $\\pm 0.022$} &  $0.948${\\tiny $\\pm 0.013$} &  $0.939${\\tiny $\\pm 0.011$} &  $0.934${\\tiny $\\pm 0.008$} \\\\\n",
      " ResNet50 &      Snapshot Ensembles &  $24.92${\\tiny $\\pm \\text{NA}\\hspace{5pt}$} &  $22.21${\\tiny $\\pm \\text{NA}\\hspace{5pt}$} &  $21.75${\\tiny $\\pm \\text{NA}\\hspace{5pt}$} &  $21.48${\\tiny $\\pm \\text{NA}\\hspace{5pt}$} &   $0.983${\\tiny $\\pm \\text{NA}\\hspace{8pt}$} &   $0.865${\\tiny $\\pm \\text{NA}\\hspace{8pt}$} &   $0.843${\\tiny $\\pm \\text{NA}\\hspace{8pt}$} &   $0.830${\\tiny $\\pm \\text{NA}\\hspace{8pt}$} \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_ = pd.DataFrame.from_dict(rows, columns=columns, orient='index')\n",
    "print(df_.to_latex(index=False).replace(\n",
    "    '±', '{\\\\tiny $\\pm ').replace(\n",
    "    '!', '$}').replace(\n",
    "    '?', '$').replace('NAacc', '\\\\text{NA}\\hspace{5pt}').replace('NAll', '\\\\text{NA}\\hspace{8pt}'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bar plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!unzip df_dee_no_augment.csv.zip > unzip_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ars/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:12: UserWarning: Pandas doesn't allow columns to be created via a new attribute name - see https://pandas.pydata.org/pandas-docs/stable/indexing.html#attribute-access\n",
      "  if sys.path[0] == '':\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('./df_dee_no_augment.csv')\n",
    "df.model = df.model.apply(lambda x: x.replace('do', '').replace('Bayes', ''))\n",
    "df.metric = df.metric.apply(lambda x: x.replace('_', '-'))\n",
    "df.method = df.method.apply(lambda x: x.lower().replace('_', '-').replace('kfaclaplace', 'kfacl'))\n",
    "\n",
    "df.model = df.model.apply(\n",
    "    lambda x: x.replace('BN', '').replace('Pre', '').replace('28x10', ''))\n",
    "df = df[df.method.apply(lambda x: 'ts' in x)]\n",
    "df = df[df.method.apply(lambda x: 'aug' not in x)]\n",
    "df = df[df.method != 'stochbn']\n",
    "df = df[df.metric == 'll']\n",
    "df.value = df.dee\n",
    "df.metric = len(df.dee)*['dee']\n",
    "\n",
    "df.method = df.method.apply(lambda x: x.replace('-augment', '').replace(' (ts)', ''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>dataset</th>\n",
       "      <th>model</th>\n",
       "      <th>method</th>\n",
       "      <th>n_samples</th>\n",
       "      <th>metric</th>\n",
       "      <th>dee</th>\n",
       "      <th>lower</th>\n",
       "      <th>upper</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>700</th>\n",
       "      <td>700</td>\n",
       "      <td>CIFAR10</td>\n",
       "      <td>VGG16</td>\n",
       "      <td>dropout</td>\n",
       "      <td>1</td>\n",
       "      <td>dee</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>701</th>\n",
       "      <td>701</td>\n",
       "      <td>CIFAR10</td>\n",
       "      <td>VGG16</td>\n",
       "      <td>dropout</td>\n",
       "      <td>2</td>\n",
       "      <td>dee</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>702</th>\n",
       "      <td>702</td>\n",
       "      <td>CIFAR10</td>\n",
       "      <td>VGG16</td>\n",
       "      <td>dropout</td>\n",
       "      <td>3</td>\n",
       "      <td>dee</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.021788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>703</th>\n",
       "      <td>703</td>\n",
       "      <td>CIFAR10</td>\n",
       "      <td>VGG16</td>\n",
       "      <td>dropout</td>\n",
       "      <td>4</td>\n",
       "      <td>dee</td>\n",
       "      <td>1.006517</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.044008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>704</th>\n",
       "      <td>704</td>\n",
       "      <td>CIFAR10</td>\n",
       "      <td>VGG16</td>\n",
       "      <td>dropout</td>\n",
       "      <td>5</td>\n",
       "      <td>dee</td>\n",
       "      <td>1.023056</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.061072</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unnamed: 0  dataset  model   method  n_samples metric       dee  lower     upper\n",
       "700         700  CIFAR10  VGG16  dropout          1    dee  1.000000    1.0  1.000000\n",
       "701         701  CIFAR10  VGG16  dropout          2    dee  1.000000    1.0  1.000000\n",
       "702         702  CIFAR10  VGG16  dropout          3    dee  1.000000    1.0  1.021788\n",
       "703         703  CIFAR10  VGG16  dropout          4    dee  1.006517    1.0  1.044008\n",
       "704         704  CIFAR10  VGG16  dropout          5    dee  1.023056    1.0  1.061072"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'ImageNet'\n",
    "models = ['VGG16', 'ResNet110', 'ResNet164', 'WideResNet', 'ResNet50']\n",
    "methods = ['deepens', 'sse', 'csgld', 'swag', 'fge', 'dropout', 'vi', 'kfacl', 'onenet']\n",
    "\n",
    "rows, row_counter = defaultdict(lambda: dict()), 0\n",
    "for model in models:\n",
    "    for method in methods:\n",
    "        metrics = defaultdict(lambda: dict())\n",
    "        metrics_ = defaultdict(lambda: dict())\n",
    "        metrics_std = defaultdict(lambda: dict())\n",
    "        for line in df.values:\n",
    "            _, dataset_, model_, method_, ns_, metric_, val_, std_, len_ = line\n",
    "            method_ = method_.replace('-augment', '').replace(' (ts)', '')\n",
    "            if dataset_ == dataset and model_==model and method_ == method:\n",
    "                metrics[metric_][ns_] = '?' + (precition[metric_] % val_) + '?'\n",
    "                std = precition[metric_] % std_\n",
    "\n",
    "        f = lambda x: metrics[x[0]][x[1]]\n",
    "        if metrics: \n",
    "            if dataset != 'ImageNet':\n",
    "                rows[row_counter] = [model, names_long[method]]\n",
    "                rows[row_counter] += list(map(f, [('dee', 1), ('dee', 5), ('dee', 10), ('dee', 50), ('dee', 100)]))\n",
    "                columns = ['Model', 'Method', 'dee(1)', 'dee(5)', 'dee(10)', 'dee(50)', 'dee(100)']\n",
    "            else:\n",
    "                rows[row_counter] = [model, names_long[method]]\n",
    "                rows[row_counter] += list(map(f, [('dee', 1), ('dee', 5), ('dee', 10), ('dee', 25), ('dee', 50)]))\n",
    "                columns = ['Model', 'Method', 'dee(1)', 'dee(5)', 'dee(10)', 'dee(25)', 'dee(50)']\n",
    "            row_counter+=1\n",
    "df_ = pd.DataFrame.from_dict(rows, columns=columns, orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lllllll}\n",
      "\\toprule\n",
      "    Model &                  Method & dee(1) & dee(5) & dee(10) & dee(25) & dee(50) \\\\\n",
      "\\midrule\n",
      " ResNet50 &          Deep Ensembles &  $1.0$ &  $5.0$ &  $10.0$ &  $25.0$ &  $50.0$ \\\\\n",
      " ResNet50 &      Snapshot Ensembles &  $1.0$ &  $2.2$ &   $3.2$ &   $4.0$ &   $4.2$ \\\\\n",
      " ResNet50 &     Fast Geometric Ens. &  $1.1$ &  $1.2$ &   $1.3$ &   $1.4$ &   $1.5$ \\\\\n",
      " ResNet50 &  Variational Inf. (FFG) &  $1.0$ &  $1.1$ &   $1.2$ &   $1.2$ &   $1.2$ \\\\\n",
      " ResNet50 &            KFAC-Laplace &  $1.0$ &  $1.0$ &   $1.0$ &   $1.0$ &   $1.0$ \\\\\n",
      " ResNet50 &            Single model &  $1.0$ &  $1.0$ &   $1.0$ &   $1.0$ &   $1.0$ \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_ = pd.DataFrame.from_dict(rows, columns=columns, orient='index')\n",
    "print(df_.to_latex(index=False).replace('?', '$'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQcAAAC/CAYAAADzRRkLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAHsAAAB7AB1IKDYgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAauUlEQVR4nO3de3wV5bXw8V8SICESQAgcQMALviykCALHUy8IWD30FWxFaFG0XgFB6/EoogRFEbxAxKpHquK1SPUV4RWv2Fo9PYKCFUUQFFgqVAwKRkQDARJIss8fz+yQbGYnIdmXSbK+nw+fsGfPZc3MnjXPzDzzPCmhUAhjjImUmuwAjDHBZMnBGOPLkoMxxpclB2OMrybJDqC+E5HLgRxgO2577gF+rarFNZh2HqCqOrPC5ztU9SufcS9Q1RdE5DrgcmAXsERVZ4vI3cBgYK2qXh0x3TuqOriWq1dV7IOBh4E+qlrifR6sqnf4jHsm8Kmqfl9hWBZws6reJiJfAlu9r34LtAKeAULA9ar6UazjrxDHPKJsc59x7wduUtXSeMUTJFZyiI1ZqjpYVQcA7wODDmPaCSLSqQbjhQ/6nsB53vJmi0hXoJuqng7sFpFTDy/0OukAXFOD8S4DjogYdg2wQEQ6AEu99RnsJZApwFhgOHBrLAOuo//BJa9GwUoOsXcUsN874P8EZAIvqOofReQhoA9QwMEf2WxgFnBpeAYiMgi4G0gBbgZ+BpwkIpcAAswRkUxgAtAbWO5N+i5wCi5BVSIiHwDfA82Bt4HfAI8Az+PO0kd6o/4KOAF4DNgBdFHVXiJyvTdNIe5gB3gSuFJE/l+F5WQB8735vQ88CvxfIAsYWSGkM1Q1V0TO8tZtGfAXrxR1jKpu8ObXKmI9JgPnAvuBC4AewHRvO7+kqvdWsa6bvO2ZgSvh/aaKbb4aWAy0AJap6i3evOYDCyK3b0NkJYfYyBGR90XkG+ArVX0HmIwrrp4BnCUibYG+wDDgIVzRGeB1IFtETqkwv2m4A2oIcJuqPg6sUdU/A68BvwNuAu7FHXSF3nR7cD9mPx2BEcB63CXQmcAooAvwpKqeDawD+uPO3OfjDr5OItIeONsrGU3zlg2wD5fY7qywnLHAQu9SppX376/AjeERROQIIM37uBO4A1fa6isifaj8u0yJWI+zvdimAK2Bbt56neENr2pdAfao6i+A93D7IqzSNgf+D+5SZxDwGYCq7sOVlhoFSw6xMUtVTwVe5OCB2h2YJSLvAMcBXYGZwAvAGKDidetE4D4OHgg/wyWNJUDkJccTqlqoqp8Abb3lhYvsR+DuRfj5p6ruB37w/r8LSMedYS8UkT8B/4YrTR6pqlu9cRQ4FjjRW5d7gXbhmarqAtzZu3eF9b7eG/fnwDE+sbTGJQWADcBfVTUE/B1XMqpYM68sYtrbgCdwlxulwLe4Us7D3vpUta4AH3p/1+D2S1ilbe5t3/XAm0BvEQnvm0ZxvwEsOcTaZGC8iHQENgMTvDPoPNyP+GRVHYa7DCg/a6nqRtyP9lxv0GrcGewcXLEfAO8HulREmopIdyDfG/cMb5QzgI+jxBatKuxluGv+K7z5pQCFItLRO8MfD2wBlnvrMgF3wFQ0EZjq/X8zrsQ0GJiDO+uGqFwC+JGDJZxxwJXe/wfgDsgtItJTRLKBvRHL+pWqno9LEJfgLgUuA27nYJKsqtrvid7ffsDnFYZX2uYiciKwzStRdcOVsABKqph3g2LJIYa8YudM3A91FvCgd/17vKp+B7QWkZW4xPDXiMnv4OBZaTbu+vZ9XFIJG4srYbwHPA7c6t1l/1JEVgDtve8OxzLgZhF5D3fA/gswA3e9/QKuGL4d+Ni7L/BnYGPEeq8GXvY+Po5LkCuAoUAeLmE9WmH8vRz87c0DzvPmvUlVP8Ud8I8DfwHuiYh3p4h8jEtILwBvACtxSbRQRKq7jyZeqaafN/+wyG2+CbjCi2u3qn4tIum4xNYopNi7FSaSiFyGu0Qqw5UY+sZhGTm4R7HrYj3vKpY5mCiPW2s4/XlAhqq+EMu4gspKDsbPT8BSYAXuSUA8PIq74Vmf/AKXNBsFKzkYY3wlrJ7Dl19+GUpPT69+RGNMUuTn5y/u379/eV2UhCWH9PR0unTpUv2IxpikyM/P31Lxs91zMMb4suRgjPFl71aYpCsoKKCgoICUlMia0iZeQqEQrVq1olWrVlHHseRgkq6goIAuXbpYckigUChEXl5elcnBLitM0qWkpFhiSLCabHNLDqbe+u677yp93rVrF/v27YvLsuI575oqLi6moKDgkOHxis2SgwmMu+66iyuuuIILL7yQSZMmUVYW+UJmZQ8++GClz/Pnz+eHH35g7ty5tY5h69atzJkz55Dh4Xkn0xtvvMHGjRsPGR6L9fZj9xxMYEydOpUPPviAb775hhEjRrBlyxbmzZtHSUkJQ4cOZceOHaxZs4bdu3czbtw4PvvsMzZu3EiPHj0AWLVqFS1atOCrr74CYPjw4QwcOJAtW7bQvXt3du3axY033sjs2bMJhUJ07dqVSy91bexs3ryZJ554gtTUVDp06MC7777Lm2++yc6dO5k4cWL5vFu2bMmnn37Kt99+y913303btm0BmD17NkVFRRQWFpKbm0tOTg6zZs0iJyeHGTNmMH36dJo3b87GjRuZNWsWU6ZMQUTYs2cPHTt2JD09nVGjRpUnpr59+3LgwAHWrFnDgQMHGDFiBKtWrWL9+vXs37/fN7bwet95552kpaWRlZXF+eefz2233Ubv3r1p2bIlY8aMqfH+sJKDCawFCxaQnp5Oy5Yt+fjjj8nPz6dFixaMGjWK448/np49e5YnBoB+/fpx9tlnl3/u2rUrEydOJBQK8fvf/56ffvqJ5cuX891335GRkcFnn31WPu6iRYuYPHkyV17p3h7v1KkT55xzDp06dWLdunXl8+7ZsyeDBg0iMzOTTZs2lU8/ePBgTj75ZL744otD1uMf//gHAwcOZOrUqWRmZgLQq1cvcnJyKC4u5rrrrmPz5s0sWbKE4uJiMjMzWbt2LQBnnXUW48eP54MPPiiPIVpsAJ9//jkdO3bklltu4ccff6SwsJATTjiBG264gQ0bNhzW9q91yUFEBuLe7U/HtWw0Gte6z1Tv9WRj6qSsrIzRo0eTnZ3Nhx9+SPv27UlLS2Pu3LnlZ+wNGzbw6KOPcvrppx9ygy18IIb/pqSkUFZWxoABAxg1ahSvvfZa+bjh71JT3fnyueee47TTTqNXr16UlZWVz/vJJ59k+PDhiAjh95L27t3L008/zYQJE+jQwTUUFf6usLCQkpKS8s/h+WRmZtKkSROaNWtWaflDhw7llFNO4c0336SoqIiMjAyaNGlCKBQqnzZabOHlhj+HQiFCoRDNmzcHoEmTwzvc63JZ0RLX2MaJuHfgH/Lm931VExlTU6NGjeKBBx4gFAoxduxYNm3axPvvv09GRgbt2rWjuLiYwsJCHnroIQBefvllFiyounnHAQMGMGXKFNasWUPv3r3Lh19wwQXMnj2bpk2b0q5dO7Kzs1mxYgUFBQX07t2bzp07s2DBAlq3bs3SpUvJy8ujW7duwMGD7o033mD79u3s3r2b1NRUZsyYQVFREQMGDGDatGmsXr2a3bt3R41t2LBhTJ8+ncWLFzNs2DCKiooqfd+pUycWLlxIt27dfGMDEBEWLVpEbm4uHTp0ICsr6/A3vKdOb2WKSHNcaz9nAb1wr+BuVdW/RY77xRdfhDIyMmq9LNNwFRUVcfTRRyc7jLjZuXMnTz31FGlpaXTp0oXf/jYYDVhv2bKFisdkfn7+A/37958Y/lyXy4ojgVzc+/7ZQBGuHQDf+xgZGRn24pXxlZeXV168bog6dOjArbcGqYV9J/KYzM/Pr/R9XS4r/oBr/HQm8BKuua9duOa7jDH1XK2Tg6peGTHomTrGYowJEKvnYBq8H3bvY/e+/Yc9XVbzZrTNan7I8Dlz5rBx40aaN29Oamoqffr0YciQIbRr185nLgenOf/88+ncuTNAeT2IaKr7PhEsOZgGb/e+/Yyc/cphT/fiTef5JgeAKVOm0LlzZ9auXcuYMWMYNGgQ06dPJy0tjWOPPZZ+/frxzDPPsG/fPkaPHg3Aww8/TGlpKVdddVX5fB544AH27t1LixYtGD9+PPfccw/Nmzdn165o3Y8kjlWCMqYORIRhw1wXJNu3b6dHjx4MHDiQ1q1bc+6559K9e3c++sj1Azx27FhycnJYtGgR4CosrVu3jvT0dDZt2sR7773HgAEDyMnJIQhP9iw5GFMH69at4/XXX6e0tJSrr76a7Oxs7rvvPl555RW2bdvGSSedVF4BqlmzZqSlpZVXtCorK+PEE09k0qRJDBkyhKZNm5ZXYgrCW6p2WWEavKzmzXjxpvNqNV00M2fOJCMjg7KyMkaOHElaWhrPP/88mZmZ9OrVi+zsbFauXMm6devIzMzkiCOOYP78+YRCIS6++GIee+wxevTowbPPPsvtt99O+/btGTduHPfccw8fffQRJSXJ71grYU3T5+Xlhayeg/GTl5dndWCSIHK7r1q1qlIlKLusMMb4suRgjPFlycEY48tuSJoGr7RwB2VF0d+GjCY1I4u0FtlxiKh+sORgGryyot1se+TXhz1dx2te9U0OFWtIgmt5Kfz/SBVbtgJYvHgxRx11FD//+c99xw9CzcgwSw7G1EK4hiTAyJEjOfPMM9mzZw/XXnstM2bMID09nc6dO9OnTx9effXV8habwl566SU2bNhAYWEhU6dO5amnnmLv3r188sknyVqlQ9g9B2Nq4d577yUnJ4fc3Fw6d+7Mtddey/ffu3aOhg8fTt++fVm9ejUAv/zlL5k2bRrLli0rn37hwoU0a9aMUCjE+vXrKSgoYPLkyYhIUtbHj5UcjKmFm2++udJLVOBahNq4cSNvv/02o0eP5p133gEob6uiadOm5dO3atWKSZMmsXLlykp1DdLS0hK0BtWz5GAavNSMLDpe82qtpotm5syZ5fcZKjaSkpWVxfbt23nuuecoLi4G4K233mLr1q0MGjSIPXv2AHDaaacxdepUioqKmDlzJm3atCE3N5fPP//8sOOMF6shaZLOakgmh9WQNMbUiiUHk3QpKSmUlpYmO4xGpbS0tNo3P+2eg0m6Nm3a8M033wTiNeXGIhQKkZ1ddQWvWHVqcztQAtyvqsNqO0/TOGVmZtK1a9dkh2EixKpTm6FAe2BHLILKzc0FYPLkybGYXUwFOTZjYqkurU+/7nVqcy3wBfAwcEe08YuKisjLy6vRvN99910ALrrootqGFzdBjs2YWIpVpzaPAccCp4rIYFV9J3L8w+nUJtyMVhAfbwU5NmPqIl6d2jytqv9fROb5JQY/VTUXXubVvfgqvyDq9NGaDY+FIMdmTKLEslMbVPXymk5fVXPhmRkdAapsTryqZsPrqqrYWhUfSGpsxiRKIB9l7j1mQLJDiOpAK7urHk1tO48BK20FUSCTQ5AFOXElW207jwErbQWR1ZA0xviy5GCM8WXJwRjjy+45GJNEdbmJC/G9kWvJwZgkqstNXIjvjVy7rDDG+LLkYIzxZcnBGOPLkoMxxpclB2OMr0OeVohIBjAWOBnYD3wEPKmq1sifMY2I36PMCcBrqvpHABHpDlwDzElkYMaY5DokOajqgwAi0hIYAnysqpYYjGlkDrnnICLhJmmvB9YB/5HQiIwxgeB3WTHa68xzHTAK0IRGZIwJBL/k8BSuZekrgDxgfkIjMsYEgt+jzFnAaFWdCSwDpiU2JGNMEPiVHMqAYgBV3YwlB2MaJb/kcL+qfl1xgIh09RlWscertcBxQAvg96q6PU7xGmMSxC85nCEifXD3G0qAbsAK4OuI8Sr2eHWeql4mIsOBU4GXImca2alNSVpmnQIvKSmpcSc5hz3vAMcWZHXZbrbNajl9HLebXz2H50TkBdxBD/C4X+3IiB6vbhGRtsBw4Gq/BUV2alNVvw81CrxJE7p0ik/HMkGOLcjqst1sm9VOLLdbjTq1UdUSYHVVM4ro8SoDuBW4QVX3xSRSY0xS+b54JSK/qsG0fwC64nq8+ieQDcwRkcExi84YkzTRmom7xKsIVQCgqssiR4jo8ep3sQ/NGJNM0V7ZXgIcCRzj/TPGNDLRksMu4N9xL179kLhwjDFBEe2yYoiqXgIgIg/hShImAKw/SpMo0ZJDCxHpD4Rw9RlMQFh/lCZRoiWHOcApQApwY+LCMcYERbTkMAaYpKq7ExlMrOTm5gIwefLkJEdSWVDjCjrbbskRLTmUAv8lIilAKOKxZeAtX7482SH4CmpcQWfbLTmiJYefVPWWhEZymLKb7OPAjp3+X4bKADiw45++X6dmZJHWItv3u1iIGls1cUH8Ywuqqvbnqf/aB7DtlmjRksMwcbWgwpWgAldySD2wh22PDff97qSsLAC2PfJr3+87XvNqXH9I0WKrLq5ExBZUVe3Pi72/2x55Oer0jXW7xZNv0/Sq2qfC53r3NszYHsG8VRLUuIzx41cJ6nEAEbnX+3x34sIxxgRFVT1ehctoKYkIxBgTLH73HI4Qka64ilBdgbq1RmFMktWlVikEu2ZplTfmq1HdTVy/5LAW1/L0Z97ftbVasjEBUZdapRDsmqVV3citTnU3cf1agppeqyUZYxoU62XbGOPLkoMxxpdfPYe3gR+B3bgnFfWu+rQxpu78bkheDlyrqjkJjsUYEyB+NyS3AtUmhohObf4O9MW1/zDBryl7Y0z9Eu3dCgBE5FlVjdZ4bMVObZar6hEicinwC+CtyJFj3alNKBSq9bQHSkrYXkVHIA01tnh3HFOX2OqyzaDq7RbkjmOC/FurMjkAe6N9EdGpzUJv8DbgX/zGj3WnNikpta+42bRJE7p0iP7KSEONraoOUGLRZkJdYqvLNoOqt1uQOo6JFKTfWo06tRGR44CrgHy/fjK9cSp2ahPubLcj8G2tozVxVVVtuuXvvQvAgTGjok5vr0U3LtFKDhOAO4HmuE5rxviM8wegk/f9WyLyGJCGSyomgOrymjvYa9GNTbTk0ARoizvYU/xKDz6PN5+JQ3wmQex1chMpWiWoAuBSXDsbW3CPN40xjUi0ksNbwFhcJaiHVfWjxIVkjAmCqlqfHosrWfwRsORgjI+G3DJ2tOTQFGiNSw4ZiQvHmOCpy1Oe+vyEJ1pyuBOY4v3fmokzjVqQGzOOJ78Xr+ar6qXATUmIx5h6pSE/5fErOXTzqkGXU9X5CYrHGBMQfslhD/AV1rCsMY2aX3LYoKrLEh6JMSZQDqkEpar/mYxAjDHBYs3EGWN8WXIwxviy5GCM8WXJwRjjy5KDMcaXJQdjjC9LDsYYX5YcjDG+LDkYY3xV1zR9tUTkTOAC4FXgHFzT9Deo6jd1nbcxJnnqlBy8Juz74xqEGQgcAzQDdtQ5MmNMUtUpOajqZuA+EZkHvIRrIGY8cDLwXsVxrcermolnbPHsVQqCG1tD3Z+Q3B6vDsdNqvobEfkeaBP5pfV4VTPxjC2evUpBcGNrqPsTktDjVS39j4j8CSgFronhfI0xSRCT5KCql8diPsaY4LBHmcYYX5YcjDG+LDkYY3xZcjDG+LLkYIzxZcnBGOPLkoMxxpclB2OML0sOxhhflhyMMb4sORhjfFlyMMb4suRgjPFlycEY48uSgzHGlyUHY4wvSw7GGF+WHIwxviw5GGN8xbJTm+eAi4E0YKqqflfXeRtjkieWndqMBgq8eX5f99CMMckUy05t+gO/wJUizgb+VnFc69SmZuprxzEQ3Nga6v6E+tOpzTagCPgJn3sZ1qlNzdTXjmMguLE11P0J9adTmyeAecAuYGIM52uMSYJYd2qzJBbzM8Yknz3KNMb4suRgjPFlycEY48uSgzHGlyUHY4wvSw7GGF+WHIwxviw5GGN8pdS1TntNrVq16kVgS0IWZoypjaP79+8/MvwhYcnBGFO/2GWFMcaXJQdjjC9LDsYYX5YckkBEOiU7BmOqE+jkICJ3iMgx3v8XiMhV1Yw/LxFxxcDdyQ4giETklgQua7CIXJ6o5cWCiJwtIq+ISMdELC+Wjb3ES0sR+QvQCZgjIo8D+3GtTt0GzAbSiWiWLtZE5HRgLK4B3XbA+0BvYDxwLnAS0BK4DngZWAocq6pjROQuoAWuIZw3gX4i0lVVv45jvBcBpwLZQAmwF/gLsB24CMgC7lfVdfGKoSa8hH4FMA44IwHLOwP4HdAGWCoiDwJHAjnAECrvxyXAX3H77jbgTg7ux6dxDRytBH4EXgem4H4ft3tNKMbalV5sM0WkDPjaiy8u+zPQJQfPs8BNwGrcjngeWI5rs/JM4O+qOg74NM5xXIn7Ad8I7APuwx3oP/OGF+O25wnAj6p6F5AuIr2Af8Mls57eOqyOZ2LwdMI1+DsXlyDe8ZZ9DS5RFACnxDmGmlgBnAz8K5CIFssvAa7GHdwzcftlr7f8yP34tarOwjWYHLkfm+L24624hHKkN+wF4Ns4xf43oCMwBnjUGxa3/VkfksMs3E4DOAp39t4A7MSVfMKN6B0X5zjCy0kFUNW9wAHv84+qmoNrJu9b3M4Cd8ZOBT70vn9RVQ/EOc6wvwMLcWfAaUApcAfuB3wXcD+wPkGxVOVF3AGbX92IMZSC2x4h3Nl+LrCJQ/dj+PjIIGI/4vZ9xf28HbgH6IfroiFemnp/QxU+x2V/1ofksAJ3pm7n/W2JK4Z2At4ChorI07hLi3h6Cngcd7+gMOK7t0TkCVwSq9Qsv6quBdqLyFyguzf4KBHpGed4TwBuwP2IJ+OK7P8AHsEVhx8kfme4GlPVH4BjgZcStMg/4c66VwAzgGdwlwzfcuh+PE5E7gfaquonHLofK8rEJd9jgI/jHP+TuH1bRBz3p9WQNCYKEZlXoX3UQBCR83CXD22A2ar6ZbyWZcnBGOOrPlxWGGOSwJKDOSxWgavxsORgynmVzv5c4fOHIjI4YrS7ve/m1XCeNRrPBE99qARlEquViGQAnXFPh44WkQdwT4lux1XgGgQcLyJzcM/dLwSmAq1xlYCuwz1aKwWOT/wqmFiwkoOJtAQYCowAFlO5YlBHXMWfpUC+qv4HsNEbnq2q1+Nq7fUG8lR1EvBN4lfBxIIlBxPpv4HBQCtcp8h9IioGhe3y/pbgageGH3ulev8PVxorjW+4Jl4sOZhIJbiDO1xH/9aIikGFIjIiYppdwA4R+QPQ1pu2k1eBqENiwjaxZvUcjDG+rORgjPFlycEY48uSgzHGlyUHY4yv/wW/j/gBms7ztQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x200 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQ0AAAC/CAYAAADkZ4nCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAHsAAAB7AB1IKDYgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAe90lEQVR4nO3deXxU9dX48U8SICESQASFiKhFOUgRVB6rtghYra3gUxda0LYqCm5U+SmCBIsLuEDEuvGoVVxwqywVl8elLr9WquCKoihwrFAhIoiIhgSSQJJ5/vjeCclk7mQmk1mSnPfrxYvMXc/cO3Pme+/93nMzAoEAxhgTrcxUB2CMaV4saRhjYmJJwxgTE0saxpiYtEl1AC2ViIwBCoDNuO28A/i1qlZEMe88QFV1Zq3XN6jql2GmHa2qC0RkAjAG2A68qKqzReRmYBjwiapeGjLfG6o6rJFvL1Lsw4B7gIGqWum9HqaqN4SZ9gTgU1X9ttawPOBqVb1WRL4AvvJG/RboBDwKBIArVPWDpo6/Vhzz8NnmYaa9HZisqlWJiiedWEsjsWap6jBVHQy8DQyNYd5LRCQ/iumCyaAfcJq3vtki0gvorao/A0pE5LjYQo9Ld2B8FNOdB+wVMmw8MF9EugNLvPczzEssU4FxwOnAn5oy4Dj9E5fUWgVraSTP/sAuLxE8AuQCC1T1f0TkbmAgUMyeD99sYBZwbnABIjIUuBnIAK4GfgwcISLnAALMEZFc4BJgALDUm/VN4Fhc4qpDRN4FvgXaA68DvwHuBZ7C/arv7U3638BhwP3AVuAAVe0vIld485TikgDAg8AFIvLXWuvJAx7zlvc2cB/wKyAPGFkrpONVtVBETvTe27+Al71W10GqutpbXqeQ9zEFOBXYBYwG+gLTve38jKreGuG9rvW2Zw6uRfibCNv8I2Ax0AH4l6pe4y3rMWB+6PZtiaylkVgFIvK2iGwEvlTVN4ApuGbv8cCJIrIPcCQwArgb1wQHeAHoKiLH1lre9bgv2snAtar6ALBCVR8H/hf4AzAZuBX3ZSz15tuB+5CH0wM4E1iFO5Q6ARgFHAA8qKonASuBQbhf+jNwX8p8EdkXOMlrSV3vrRugDJfwbqy1nnHAQu+QqJP37+/AVcEJRGQvIMt7uQ24Adc6O1JEBlL385oR8j5O8mKbCnQGenvv63hveKT3CrBDVX8OvIXbF0F1tjlwKO6QaSjwGYCqluFaV62CJY3EmqWqxwFPs+cL3AeYJSJvAD8CegEzgQXAWKD2cfFE4Db2fEF+jEsmLwKhhy5zVbVUVT8G9vHWF2z674U71xHOf1R1F/Cd9/d2IBv3i3yWiDwC/ATXKt1bVb/yplHgYOBw773cCnQLLlRV5+N+7QfUet9XeNMeAxwUJpbOuGQBsBr4u6oGgH/gWlK1eyJWh8x7LTAXd9hSBXyNaxXd472fSO8V4H3v/xW4/RJUZ5t723cV8AowQESC+6ZVnM8ASxrJMgW4WER6AOuAS7xf3Hm4D/fRqjoCdzhR8yunqmtwH+ZTvUEf4X7xTsEdPgDgfXCXiEhbEekDbPGmPd6b5HjgQ5/Y/LoEn4c7p3C+t7wMoFREengtgkOA9cBS771cgvsi1TYRmOb9vQ7XwhoGzMH9Sgeo22L4nj0toguBC7y/B+O+qOtFpJ+IdAV2hqzrv1X1DFziOAd3SHEecB17kmek7s+He/8fBXxea3idbS4ihwObvBZYb1yLDKAywrJbFEsaSeA1X2fiPsCzgDu94+tDVPUboLOIvIdLGH8Pmf0G9vyKzcYdP7+NSzZB43AtkreAB4A/eWf9vxCRZcC+3rhY/Au4WkTewn2R9wNm4I7nF+Ca85uBD73zDo8Da0Le90fAs97LB3CJcxkwHCjCJbL7ak2/kz2fyXnAad6y16rqp7hE8ADwMnBLSLzbRORDXKJaALwEvIdLrqUi0tD5O/FaQUd5yw8K3eZrgfO9uEpUdYOIZOMSXquQYfeemGiJyHm4Q61qXAvjyASsowB3yXhlUy87wjqH4XNZOMr5TwNyVHVBU8aVrqylYWLxA7AEWIa7MpEI9+FOtDYnP8cl01bBWhrGmJikvJ/GF198EcjOzm54QmNMSmzZsmXxoEGDavrSpDxpZGdnc8ABBzQ8oTEmJbZs2bK+9ms7p2GMiUncLQ3vpqPRqnqJ93oMrvPODlWdJCKTgQOBjcEbsIwxzVdcSUNEfoTrXpxTa/BJqvoHEbnO6/p7qKpeJCL3i0iudy3emAYVFxdTXFxMRkZoj3GTSBkZGXTp0oXc3Nyw4+NKGqq6DrjNu404aLf3/yZct+At3uutQBdCevKVl5dTVFQUTximhSovL6dXr16WNJKsqqqKoqIi2rdvH3Z8Ik6EBnsv9sBd0w/eJdkV1+e/jpycHDsRasIqKirCrqylRvv27Wu+l1u2bKkzrslOhIrIGBERYJmI/A/QUVU/B4pEZA6wzutObUxCffPNN3Veb9++nbKyxHz0ErnsaFVUVFBcXFxveMJiCwQCKf23YcOGgDHhBD8bN954Y2DMmDGB0aNHB6666qpAVVVVxPkKCgrqvJ4zZ06gqKgocN999zU6lqKiosDdd99db3hw2am0ePHiwDvvvFNveDzvu/b38oMPPrg9UOs7m/J+GsY0ZNq0abz77rts3LiRM888k/Xr1zNv3jwqKysZPnw4W7duZcWKFZSUlHDhhRfy2WefsWbNGvr27QvA8uXL6dChA19++SUAp59+OkOGDGH9+vX06dOH7du3c9VVVzF79mwCgQC9evXi3HNd7aN169Yxd+5cMjMz6d69O2+++SavvPIK27ZtY+LEiTXL7tixI59++ilff/01N998M/vssw8As2fPpry8nNLSUgoLCykoKGDWrFkUFBQwY8YMpk+fTvv27VmzZg2zZs1i6tSpiAg7duygR48eZGdnM2rUKObMmQPAkUceye7du1mxYgW7d+/mzDPPZPny5axatYpdu3aFjS34vm+88UaysrLIy8vjjDPO4Nprr2XAgAF07NiRsWPHRr0/rJ+GaXbmz59PdnY2HTt25MMPP2TLli106NCBUaNGccghh9CvX7+ahAFw1FFHcdJJJ9W87tWrFxMnTiQQCPDHP/6RH374gaVLl/LNN9+Qk5PDZ599VjPtokWLmDJlChdc4O7Sz8/P55RTTiE/P5+VK1fWLLtfv34MHTqU3Nxc1q5dWzP/sGHDOProo/n3v/9d73288847DBkyhGnTptVcqejfvz8FBQVUVFQwYcIE1q1bx4svvkhFRQW5ubl88sknAJx44olcfPHFvPvuuzUx+MUG8Pnnn9OjRw+uueYavv/+e0pLSznssMO48sorWb16dUzb35KGaXaqq6s566yzuPTSS+nXrx/HHnssp5xyCk888UTNr+rq1auZMGECCxYsqHf1JfgFDf6fkZFBdXU1gwcPZtKkSQwePLhm2uC4zEz3VXnyyScpKyujf//+BAKBmmU/+OCDZGVlISIEvPu5du7cycMPP0yPHj3o3t0V9gqOKy0tpbKysuZ1cDm5ubm0adOGdu3a1Vn/8OHDufLKKznySHdjcU5ODm3atKkTg19swfUGXwcPM4JXR9q0ie2Aww5PTLMzatQo7rjjDgKBAOPGjWPt2rW8/fbb5OTk0K1bNyoqKigtLeXuu+8G4Nlnn2X+/MjlOwcPHszUqVNZsWIFAwYMqBk+evRoZs+eTdu2benWrRtdu3Zl2bJlFBcXM2DAAHr27Mn8+fPp3LkzS5YsoaioiN69ewN7vowvvfQSmzdvpqSkhMzMTGbMmEF5eTmDBw/m+uuv56OPPqKkpMQ3thEjRjB9+nQWL17MiBEjKC8vrzM+Pz+fhQsX0rt377CxAYgIixYtorCwkO7du5OXlxf7hvek/C7XoqKigF1yNeEUFRW16Mvx27Zt44EHHiAzM5MDDzyQ0aPTpyJA7W2/fPnyOwYNGjQxOM5aGsakSJcuXSgoKEh1GDGzcxrGmJhY0jDGxMQOT0yr9V1JGSVlu2KeL699O/bJq39fxpw5c1izZg3t27cnMzOTgQMHcvLJJ9OtW7cwS9kzzxlnnEHPnj0Bavpx+GlofDJY0jCtVknZLkbOfi7m+Z6efFrYpAEwdepUevbsySeffMLYsWMZOnQo06dPJysri4MPPpijjjqKRx99lLKyMs4++2wA7rnnHqqqqrjoootqlnPHHXewc+dOOnTowMUXX8wtt9xC+/bt2b7d7/E1yWOHJ8YkgIgwYoR7hM3mzZvp27cvQ4YMoXPnzpx66qn06dOHDz5wz68eN24cBQUFLFq0CHAdsVauXEl2djZr167lrbfeYvDgwRQUFJCTk+O7zmSxpGFMAqxcuZIXXniBqqoqLr30Urp27cptt93Gc889x6ZNmzjiiCNqOna1a9eOrKysmg5k1dXVHH744UyaNImTTz6Ztm3b1nTOSocyAXZ4YlqtvPbteHryaY2az8/MmTPJycmhurqakSNHkpWVxVNPPUVubi79+/ena9euvPfee6xcuZLc3Fz22msvHnvsMQKBAL///e+5//776du3L0888QTXXXcd++67LxdeeCG33HILH3zwAZWVqX+Qm3XuMmmrpXfuSmeROnfZ4YkxJibx1gg9AvfszAAwQVWLRWQm7rmfa1X1ZhF5Efecyy8a+9g7Y0z6iLelcTnu6d4PAaO8YQep6gVAuYj0BfKBXbinnxtjmrl4T4RmqmqFiGwChnjD/ikiD+ES0jvABcAKYCHwYugCrLCw8VNeXs6uXbF3vopWRvkPBCpKY58vuwOBnM4JiCh9RPpexps0ykSkLa6I8GZv2HeqOlZEbgKygHxV/UhEwj66wAoLGz9FRUU1dSUSYff2HWy+L/arJz3GP0+7jvvWG167Ryi4Sll+Fb1rVyIDWLx4Mfvvvz/HHHNM2OmT3RO09vcytLBwvEnjIeB+IBtXUFiAA0XkflwSeRt4UEROxlUmj0thYSEAU6ZMiXdRTS6dYzPJE+wRCjBy5EhOOOEEduzYwWWXXcaMGTPIzs6mZ8+eDBw4kOeff76mwlbQM888w+rVqyktLWXatGk89NBD7Ny5k48//jhVb6meeJ97shx3+FFncMjr8+JZR21Lly5tqkU1uXSOzSTPrbfeSm5uLnvvvTc9e/bksssuY9KkSYCrTbp582ZeffVVBg4cyC9/+UvOPvtsJk+ezHHHHQfAwoULGTRoEIFAgFWrVlFcXMy0adPYtGlTKt9WHfWShog8AvQGNgLVwH6qelLodK1RpBucqr3+Ll9uqV9KPsjvRifTclx99dV1bj4DV8FrzZo1vP7665x99tm88cYbADWHXm3btq2Zv1OnTkyaNIn33nuvzmF7VlZWkt5Bw+olDVU9X0Rmq+pkABH5c/LDSk+RbnDqVOEeLBfpBqhINzqZ5MvMyaPH+OcbNZ+fmTNn1pzHqH0uIC8vj82bN/Pkk09SUVEBwGuvvcZXX33F0KFD2bFjBwA//elPmTZtGuXl5cycOZMuXbpQWFjI559/HnOcieJ3eJIlIlcAGbg+GEnTXH/Nd3fqlfR1mvhkdehKVoeuTba8yy+/POzw4AnMe+65p87wcCc9Q4eNHz8eSK9zZWGThqpOFJEBQLWqfprMgJrrr/nOgwY3PJExLUDYzl0ichfwC2C4iKS24odptTIyMqiqqmp4QtOkSktLIz7WwG9Mtar+GUBEbktEYI1hhwCtS5cuXdi4cWNa3A7emrRp04b99tvPf7zP8B9E5G7c+Yx6T3pPFTsEaF1yc3Pp1ct+KNKNX9KYCfwKlzReSV44xph055c07gVexnUDvw2YkLSITIvT2AK+YH1b0pFf0ihT1acBRORnSYzHtECNLeAL1rclHfkljVVez9AM3P0jxhgD+CeNvwIrcYcnqa0HaIxJK35FeB4D+gAHef+MMQbwb2l8qaoPJzUSY0yz4HeXa2cRWQiUAnjl+4wxJmxL4y6gUzQzR1lYuBDIA5ar6kNNFLcxLVo8l6khsZeqwyWN/bx/0QgWFj4GV1h4Lq6w8NkicpV309sOVZ0iIo/jKn3VEVqLsDIrN8a3UFdlZWXCao6mc2zpLJ7t1lq3WUVWLqPvqFdSN2oLrhzBzh+2NmFEe4RLGmeFvA7gToyGE01h4a+94dXhFhBaIzTSbe/RaNOmDQfkJ6bmaDrHls7i2W62zRqnKbdbaI3QeldPVPV8XAm/acD4Bs5n+BYWxlX+2sieVovddWRMC+B3yfVOoBD4hYjMiTB/sLDwxUB2SGHhKlX9EndS9V7gn00XtjEmVfwuuVYBG1T1eRE5xW/maAoLB8sGGmNaBr+WRhFwgog8BmxLYjzGmDTn19J4GnclJBP4efLCMcakO7+WxnXA8cA9QPfkhWOMSXf1koaIDAHmA7OAfwGrkh2UMSZ9hWtpHIx70vvtQAV2w5oxppZw5zTycXe4BkXq3GWMaWXCPWFtZu3XInJQ0qIxxqQ9vxOhtc1IeBTGmGYjmqRhjDE1/OppBEv8ZQCHJjUiY0xaC/vU+ODfIvJTVV2W3JCMMemsocOTK5IShTGm2WgoaTyTlCiMMc1G2HtPRORvuISSISK/Beap6vNJjcwYk5b8WhpfA78BRgLrgaFJi8gYk9b87nLtDAzEXT3Zhz1VuerwKSw8xpt3f+A84G/A98AXqnpDUwZvjEk+v6QxAVcouC0uKXTxmS5cYeFfeeMycMkkH/gGeD/cAqywcMtnhYVjl86fNb+kcRbwM9wXvzzCowfCFRa+U1XfqfUogwuAFcBCoF55ZSss3PJZYeHYpdNnrcHCwp7DVPUcVf0DcFiE5YUrLDzJ+38rcCCQr6oBYGejozbGpA2/lkaeiIzEHV50jjB/sLBwNrDMKyz8uYjch7ut/m7gQRE5GVjSdGEbY1LFL2lcAvwCd3hypd/MPoWFrwl5fV6jozPGpJ2wSUNVd+Gdf/CKC5+bzKCMMenL7nI1xsQkmrtcj0tqRMaYtBbxLldjjAnldyLUNDPflZRRUrarUfPmtW/HPnntmzgi01KFOzzpFTpMVTckJxzTWCVluxg5+7lGzfv05NMsaZiohWtpnA+cgntgczXuoUlDwkxnjGmF6l09UdXpwJuqOlVV/wS8l/ywjDHpyu+cxpciMtf7O+yNZsaY1inSA6DXA2uBjckLxxiT7vw6d83CnddoCzyYvHASq7CwkMLCwlSHYUyz5tfS+B4oU9VPRKQsmQEl0tKlS1MdgjHNnl9L43Wgv4g8CzTuOp6JmrWATHNSp6UhIjnAvkBv4DJcN/KLgcXJD61xurYpY/fWbeFHBqoB2L31P2FHZ+bkkdWha6JC82UtoMYJJtopU6akOJLWJfTwJBsYAwzG1dEIAJ8lOaa4ZO7ewab7Tw877oi8PAA23fvrsON7jH8+oUnDN6E1kMwgdQkt1SL9CCx9600Ado8d5Tt/a91uiVQnaahqMTBDRA4Djsa1NPzqg0ZbWPgGIA9YHqFsYFKM61uSytX7JrSGkhkkPqGlq3h+BKD1brdE8jsRejOwAVgO/CTC/A0VFj4A2KGqU0TkcVylrzqaurBwIBBoeCIfuysr2RyhGGuiYosmmSUytkQX740ntkj7sym2W7pqjoWFtwL/C3TAXUnx01Bh4R64Z6iA65JeT1MXFs7IyGj0vG3btOGA7v7FWFtqbIku3htPbPFsM2h4u6Wr5lhY+G/AV8BJ+DzzxNNQYeHvcRXJwbU8jDEp1BRX6vxaGmeo6qW4w49IIhYW9vp5nCMi9+JugDPGJIHfCeSmOHnslzT6iMjTQDGAqoYWD8Yb3mBhYVWd7Lt2Y9JUc7+cm8iT7n5JYwSuGnkV8Gr0oRrTMrTUvjNNcQXRL2n8BXgeyAJuxz2m0ZgWJVK1s4GD3EXDSCckW2vFM7+ksU1VFwOIyOFJjMeYpIlc7cwVsHspQjW01lrxzK8a+d4i8iSwC3fZ1RhjgPAtjbuATskOxBjTPIRLGvuxp2+FMcbUES5pnBXyOgA8loRYjDHNQIMPSxKRvOSFY4xJd2GvnojIPODHQBHuprOjkxiTMSaN+V1y3Y6rE1oEjE5eOMaYdOeXNHYCW3Bdwht/r7kxpsXxSxqzVfU7ESkGViczIJN8EUskRsGqY7Uu4Tp33QW8BrwA/BcwFvh/SY7LJFGk6ljRsOpYrUu4ehpdVfUFAFV9GOvoZYypJVzSqBaRNgAiko0rMGyMMUD4cxp3Ak+JSCnQDrjVb2afwsIPAjnAscBpwNW4u2V/UNXLmjh+Y0yShevctRz4bZTz1yssrKrjROQQ4Heq+pmI9MadTF0ebgFWWDg6iYwtnrig+caWzsV70/mz5nf1JFrhCgsDXAlcKyIZwNWqukxE5onIIlUtrb0AKywcnUTGlujivekaWzoV7w2VTp+1aAsL1xCRQyOMrldY2DsPkqGq24Bc4BBv2pJo1meMSW/RfImPizAuWFj4YiDbKyzcB/g3gKruAI4WkTuAr1R1e5zxGmNSzO/ek3NrvSwXkS5ey6EOn8LCACtrTdNQRXNjTDPid07j18B83FWRMcAwYHxyQjKmeYinJ21z7kXr+4Q1Vf0bgIicCDS+j7ExLVQ8PWmbcy9av6Txsojcj+un8SJWJ9QY4/FLGmuB94FVwEZVXZ+8kIwx6czv6slEoB+unsafkxeOMSbd+SWNEqAc+A73GANjjAH8k8Zfgf2BucA9yQvHGJPuwtXTOA93qfUf3qDeQMt8sKUxJmbhWhpfAuuBDbj6oMOSGI8xJs3VSxqqugR3LuNy4C5VDdfj0xjTStVLGiJyHXATcB2wWkR6JT0qY0zaCtdPIwP4CDjT+zsAzEhmUMaY9BWuCM/0VARijGkerL6FMSYmljSMMTFpdLm/KIsKDwcOxN2/MrMJ4jXGpFg8NUIjFhUG/gMcqqoXicj9IpKrqjtDF2KFhaPTXIv3QvrG1lL3J6RvYeGIRYWBvXHPgwXYCnTBPSO2DissHJ3mWrwX0je2lro/IcWFhSNoqKjwt7jEAdAV12HMGNPMxZM0GioqvAsoEpE5wDpVLYs3WGNM6jX68CTKosKzGrt8Y0x6skuuxpiYWNIwxsTEkoYxJiaWNIwxMbGkYYyJiSUNY0xMLGkYY2JiScMYExNLGsaYmFjSMMbExJKGMSYmljSMMTGxpGGMiYklDWNMTCxpGGNiEk+5P7/iwr8BhgJtgStwhXqygB9U9bI44zXGpFhGPAVIReQhYDyuuLCo6lwReQz4HtiiqjeLyFvAamCZqj4Suozly5c/jXvgtDEmPR04aNCgkcEXcbU0CF9ceCBwFHC9VwLwalVdJiLzRGSRqpbWXkDtYIwx6S/ecxr1igsDRapaBfyAS0qHeMNLmmB9xpgUi/fwZBDwRyAbWAa8jmtx/AT4XlWv9goLVwKbVbUw/pCNMakUV9IwxrQ+driQRkQkP9UxGNOQZpk0ROQGETnI+3u+iFzUwPTzkhFXE7g51QGkIxG5JonrGiYiY5K1vqYgIieJyHMi0iMZ64v36kkqdRSRl4F8YI6IPADsAspxj4WcjTvX8moigxCRnwHjcH1RugFvAwNwD5E6FTgC6AhMAJ4FlgAHq+pYEbkJ6ABsB14BjhKRXqq6IYHx/g44DvfUu0rcozJfxp3I/h2QB9yuqit9F5IEXqI/H/e84OOTsL7jgT/gHh+6RETuxD0hsAA4mbr78UXg77h9dy1wI3v248PAXOA9XNeDF4CpuM/Hdaq6LgHhX+DFNlNEqoENXnwJ2Z/NsqXheQKYDHyE20FPAUuBQcAJwD9U9ULg0wTHcQHug30VUAbchksAP/aGV+C282G4k8M34Z5I1x93wrgc6Oe9h48SmTA8+UAx8Bdc4njDW/d4XAIpBo5NcAzRWAYcDfwX8E0S1ncOcCnuSz8Tt192eusP3Y8bvAeBfQuE7se2uP34J1yi2dsbtgD4OkGxv4q7gjkWuM8blrD92ZyTxizczgTYH/drvxrYhmtBBZ+A+6MExxFcTyaAqu4Ednuvv1fVAmAe7gMTfAB2pTf+fW/806q6O8FxBv0DWIj7xbweqAJuwH2wbwJuB1YlKZZInsZ9kbc0NGETysBtjwCudfAXYC3192Pwe5NDyH7E7fva+3kzcAuu79LvExh7W+//QK3XCdmfzTlpLMP9snfz/u+Ia87mA68Bw0XkYdwhSiI9BDyAOx9RGjLuNRGZi0tu39YeoaqfAPuKyF9wz8AF2F9E+iU43sOAK3Ef7im4pv87wL24ZvWdJO4XMWqq+h1wMPBMklb5CO5X+nxgBvAo7tDja+rvxx+JyO3APqr6MfX3Y225uKR8EPBhguN/ELdvy0ng/rRLrsbESETmqeqYVMdRm4ichjsM6QLMVtUvErUuSxrGmJg058MTY0wKWNIwTcY6p7UOljRMVLwOdY/Xev2+iAwLmexmb9y8KJcZ1XQmvTTnzl0m+TqJSA7QE3fF6kARuQN35eo6XOe0ocAh3o2KPYCzgGlAZ1wHpwm4y4BV7LkD2jQj1tIwsXgRGA6cCSymbqenHrhOTUtwBZguB9Z4w7uq6hW4nooDcOUTJgEbk/8WTLwsaZhY/H9gGNAJVy9lYEinp6Dt3v+VuB6RwUt0md7fwQ5xVYkN1ySCJQ0Ti0rclz54H8OfQjo9lYrImSHzbAe2isifgX28efO9zlHdkxO2aUrWT8MYExNraRhjYmJJwxgTE0saxpiYWNIwxsTk/wBFwrMQ4SlywQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x200 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = dfbase_ung\n",
    "\n",
    "q    = 'dataset == \"%s\" and model == \"%s\" and metric == \"%s\" and n_samples == 50'\n",
    "df  = df.query(q % ('ImageNet', 'ResNet50', 'acc'))\n",
    "df = df[df.method.apply(lambda x: 'ts' in x)]\n",
    "df['Test-time data augmentation'] = df.method.apply(lambda x: 'Enabled' if 'augment' in x else 'Disabled')\n",
    "df.method = df.method.apply(lambda x: x.replace('-augment', '').replace(' (ts)', ''))\n",
    "df['Top-1 Error (%)'] = 100*(1-df['value'].values)\n",
    "df['Method'] = df['method']\n",
    "\n",
    "mpl.rcParams['figure.dpi'] = 50\n",
    "plt.title('ResNet50 ImageNet (50 samples)')\n",
    "ax = sns.barplot(x=\"Method\", y=\"Top-1 Error (%)\", hue='Test-time data augmentation', data=df)\n",
    "\n",
    "plt.ylim(15, 27)\n",
    "plt.savefig('./pdf/barplot-err.pdf',bbox_inches='tight', pad_inches=0.05)\n",
    "\n",
    "plt.figure()\n",
    "df = dfbase_ung\n",
    "\n",
    "q    = 'dataset == \"%s\" and model == \"%s\" and metric == \"%s\" and n_samples == 50'\n",
    "df  = df.query(q % ('ImageNet', 'ResNet50', 'll'))\n",
    "df = df[df.method.apply(lambda x: 'ts' in x)]\n",
    "df['Test-time data augmentation'] = df.method.apply(lambda x: 'Enabled' if 'augment' in x else 'Disabled')\n",
    "df.method = df.method.apply(lambda x: x.replace('-augment', '').replace(' (ts)', ''))\n",
    "df['Neg. Calibrated Log-Likelihood'] = -df['value'].values\n",
    "df['Method'] = df['method']\n",
    "df = df[df.method != 'stochbn']\n",
    "\n",
    "mpl.rcParams['figure.dpi'] = 50\n",
    "plt.title('ResNet50 ImageNet (50 samples)')\n",
    "ax = sns.barplot(x=\"Method\", y=\"Neg. Calibrated Log-Likelihood\", hue='Test-time data augmentation', data=df)\n",
    "\n",
    "plt.ylim(0.65, 1.01)\n",
    "plt.savefig('./pdf/barplot-cll.pdf',bbox_inches='tight', pad_inches=0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQ8AAADGCAYAAADMvwX2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAHsAAAB7AB1IKDYgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAASw0lEQVR4nO3de3hV1ZnH8W8ghAAREKbiDVGsecWpl4KXovVGa9VWar2AjuKF1npldKZTO9NqnxhbW1scb6UtOlRph3ZmnnopTC9WbZ1aHHU6eKG19lVRGazWWOTiAQIJnPlj7dgQEwkra5+Tk/w+z5OHkJysd51kn99ee+191q4qFouIiGyvAeXugIhUJoWHiERReIhIFIWHiETpcXiY2bFmNtfMTjCzeWb2L2Y20MzON7Nvm9kNKTq6Hf05XrVUS7Xy16PwMLPxwCSgFpjh7hcAvwamAB9290uAtWa2T4972n2l/EWqlmr1t1pvq0pxqtbM5gNFd59pZscBY4APZf//NPCsuy9u/zPPPvtsMY/TxIVCgbq6uuTtqpZq9bdaGzZsuGfSpEmndfX96oS1Nmf/7gK82uH/D3V8cF1dHWPHjk1YXkRSWrJkyfJ3+37K8LjXzG4DBgIXAnuY2Ryg2d1fSFhHRHqBJOHh7udnn/6k3ZfvyD5EpA/SqVoRiZLysCU3K9/awFsbNiVvd4chNYzeYUjydkX6g4oIj7c2bOK02QuTt3v3lScrPEQi6bBFRKIoPEQkisJDRKIoPEQkisJDRKIoPEQkisJDRKIoPEQkisJDRKIoPEQkisJDRKIoPEQkisJDRKIoPEQkisJDRKIoPEQkSrLFgMzsTOBkYCXQAuwEFIGZ7t6Sqo6I9A4pVxL7GDADOAh4DJgHvNRVcDQ3N7NixYpuNdw6cGiqPm7dbmtrt/sgIltLGR5zge8CLwLXuPt1ZnadmdW7+3MdH1xbW9vt+7a83LQmYTf/orq6mrG76t4xIp1pamp61++nnPPYCTgXeBh4M/vaGmBQwhoi0kukHHmsAeYDW4CXzOxmoMXdn0lYQ0R6iWTh4e6/BH6Zqj0R6d10qlZEoig8RCSKwkNEoig8RCRKpxOmZjYEmAC8Crzh7ptL2isR6fW6Gnl8A7gMqAfuLF13RKRSdBUea4HX3f1hoLmE/RGRCtFVeDwFHGlmC4GlJeyPiFSIri4Su59wifky4I+l646IVIquRh7XAycS3pcyr3TdEZFK0VV4rALWuPtSYEMJ+yMiFeId4WFm5wIvALPMbDXwdMl7JSK9Xmcjj+XAUcD07GNSSXskIhXhHeHh7r8CVrn7fcCDQKHkvRKRXq+rsy0LzOw2whqkd5SwPyJSIToND3dfDCwucV9EpILojXEiEkXhISJR8rpviwP7Aevc/bOpaohI75Fy5NF235Y7gQZ3vwRYa2b7JKwhIr1EXvdtGZ197TVgDPB8xwfrpk8ilS1leLTdt2UK0LZ40C7AQ509WDd9EundtnXTp7zu23KZmc0Bmt39hYQ1RKSX0H1bRCSKTtWKSBSFh4hEUXiISBSFh4hEUXiISBSFh4hEUXiISBSFh4hEUXiISBSFh4hEUXiISBSFh4hEUXiISBSFh4hEUXiISBSFh4hEUXiISBSFh4hESbmGaZSWlhaampoYM2YM1dVl746ItNPY2FgF7AUMbGho2OouCGV5tTY2No489dRTWbZsGU1NTYwaNYrRo0crPER6gWKxyBtvvMHKlSvfB8wDBgE/pMMtVKqKxWJJOtTY2DgKOBqYDOx9wAEHDFm6dOlNwP80NDTkc28FEemWbIQxETgcOAxYP3HixLonnnjiS8AfGhoa3hEUpdzVnwDsBiwClpxyyikbTjnllBKWF5F3MQqYDjwJXA0snzp1anHq1Kld/kDJRh6daWxsHAZMIiTegoaGhj+XrTMispV28x2TgUJDQ8PC9t8v+SRDY2PjSOAD2cf+wHPAo8DqUvdFRLbW2Ng4AJhACIzJhPmOx4HHOj62lHMehwGnAXsDT0+bNq1x/Pjx1NbWbvNnX25aw2mzF27zcdvr7itPZs+dRiRvV6TStLS08OSTT/LKK68waNAgdt99d1avXv3Vhx9++KrO5jugtCOPLWTzHQ0NDRtWrFjR2J3gEJH8FYtFhg8fzpQpUxgxYgRVVVUsWbKktqvggBKGR0NDw29KVUtEtk9NTQ377rvvdv2MrjAVkSgKDxGJovAQkSgKDxGJovAQkSgKDxGJovAQkSgKDxGJovAQkSgKDxGJoqW7OiisXs3G9euStzt46DDqRo5M3q5IuSg8Oti4fh1zP3dF8nYv/votCg/pU3TYIiJRFB4iEkXhISJRFB4iEkXhISJRFB4iEkXhISJRFB4iEiXpRWJmdixwBvAj4HSgCFzs7ptT1hGR8ksWHmY2nnD3t1pghrvPMLNzgSnAAx0f39zczIoVK7rVduvAoam6uXW7ra3v6MOg4pZcarV0UkukkiULD3d/EbjBzOYDLdmXXwPGdPb42tpaxo4d2622X27K5z7Y1dXVjN116z6sfPWPudQaVF3NzrvulkvbInloamp61+/nNefRdpiyC/CnnGqISBnl9ca4e83sNmAgcGFONUSkjJKHh7ufn336k9Rti0jvoVO1IhJF4SEiURQeIhJF4SEiUbo1YWpmQ4AJwKvAG7piVES6O/L4BnAZUA/cmV93RKRSdDc81gKvu/vDQHOO/RGRCtHd8HgKONLMFgJLc+yPiFSI7l4kdj/wJrAMyOfNHyJSUbo78rgeOBEYBMzLrzsiUim6Gx6rgDXuvhTYkGN/RKRCbDM8sjU5XgBmmdlq4OnceyUivV53Rh7LgaOA6dnHpFx7JCIVYZvh4e6/Ala5+33Ag0Ah916JSK/X3bMtC7L1OYrAHTn2p19pLWxk88bW5O0OHFxNdd3g5O2KtNet8HD3xcDinPvS72ze2MqLcx9P3u74iw9TeEju9MY4EYmi8BCRKHmtYYqZnQRcAbwOfD27RkRE+ojcwgM4lHApewF4Psc6IlIGeYbHQuB3wNHAOcDt7b+pmz7BqEHDc6nV2tJKk24wJTnLMzwmEq5GXU14T8xWdNMn2LhyXS61qgdVM3bnrZ9Xc/NKNm1Kf4lOTU0dtbWjk7cr5betmz7lGR6vE0YbReCzOdaRbti0qcB9P5+WvN0Tjv+hwqOfyi083H0RsCiv9kWkvHSqVkSiKDxEJIrCQ0Si5DlhKv1UoVCguTn9Otm1tbXU1dUlb1fiKDwkuebmZubMmZO83VmzZik8ehEdtohIFIWHiERReIhIFIWHiERReIhIFIWHiERReIhIFIWHiETRRWJS0datbWZTc/rbV9TUVjNseG3ydvsShYdUtE3NrXz/aw8lb/fsfzyWYR0Welu5biWFHBZUqqupY/SwylsTReEh0k2FTQXOWnBW8nZ/MOMHCg8RSWfTm2/SWkg70qmuq6Nm1Kg0bSVpRUSSay0U+M2Mc5K2eciCf00WHjrbIiJR8rzp00HAZwgLIF/u7vksgS4iZZHnyONvgU8D3wGm51hHRMqgqlgs5tKwmd3p7jPNbB/gDHf/cvvvL1my5G5geeq6y5cvHzdu3Ljk7aqWavXDWuMmTZp0WpffLRaLuXzU19d/q76+flB9ff1R9fX1F+RVp5O6N6qWaqlW/h95nm35DnAbMBi4MMc6Hf1ctVRLtfKX22GLiPRtOlUrIlEUHtIvmFlV+3+l5/pFeJhZv3iePdXZ76mULzYzG55HTTPbi+xm6+7e54/TzWyome2Xd51+8aJy9y1mtqOZfSKP9s3sJDM7K/s8lxebmR1uZgeY2ZA82oe3f0+7mNnpZnZg9rWSvNjM7APAj83sGHcvJg78PwNTzOyChG1uk5kdZWbnlbJmZgsw28zOzfqRy+u8X4SHme0A3AAcYWZ7mNn+iUsMAK4xsw9mG37qPefJwJeA84FjcgyoEcDVwMyszoQ86nRSdzLwKeAW4DIz2ysLsh4/TzPbz93fApYBx5nZR3va5nZYAXzYzA7P+lKqUdwA4C7gVDMb4+5b8irSp5nZe4FxwB+AlcDZhBDp0WnqbCSzexZEk4HLgXPNbM9UAWJmNdlFduOBx4B/BjYAyUcf2Z7/MKAFmA3cA7wndZ1O6u4HvAD82t3vzurebGY1iUY955nZh4H/A34HzMwOY3JlZscBJwELgU+Z2fg8dixd1L0ZeAa4FriubRSZWp8Lj/Z/nGzYdibw98CjwO3AocBGd49afqpd+0cBnwCagVrg14Tz7QvMrLqnG76Z7Qw0AiOy9jcCBxKO3ZPcc9HMJpjZwWZ2IXABcDTw18BQ4KvAQSnqdFF7sJl9hhCInwfWZP04kPBcU40Ov0AIjvcQtvfL3P2lRG2/rcN2NxQ4A3jM3e8CXgauNbMhqQ8DO9Tdn7ATAzgWWAosAQ5IWbNNn7rOw8yq2v44ZrYbcCdwBTCSMCy+CZiQ/UGj28+OIc8ibOgPEUYDnwDWAM+5+4IePo/JwEXAXsA/AacBbxIu5//vFBt/ttF9kbCRPUI4rDsUOA54ANji7g/2tE4XtYcRnt8wwh7yBOBp4H7gemChu38vcc06oN7dn0jZbid1phP+TjsAfwLWAkcATe7+i8S12rbHOuBEws5rDjAfWAcc5u63pqzZXp8KDwAzqyUcs78O/JGwN/0mMA1Y2rbxtA+a7Wx/Z+BrhL3ZZsIVtA8QXtxr3f3FHvb//YS9x08JhysPAucA/+7uj/Wk7U5qTQcuJfT742Z2EnA8cGNbQMX+nt6l5k7APkCB8Ps7hjCSmkz4G5m7/zZVvVLI/mZNhDeC7gmMBRYRQuRCYI67/zSn2vWEN56OyfrwAGG0PRKY7e7P5FEX+lh4mNlg4CpgX8LhxM3A7sDu7v6tBO2PIAzrd3H3u83sVMI8wWvAN929JUGNjxFGMYsIe+dVwCPuPrenbXdR7xBgHvBfwB6Eof1n3H1ZDrX2JQTEcEL4LiLMrzwLfMXdN6aumads9DYemEvYoezGX0YdlwOnAsPc/U851R9EmFc5y92nmVlDVv8uYLC7r8yjbps+s5KYmY0HXgIGEjbMzxOWBbiHsPeOabP9YdBHCEHUBBxiZvcSgmQz8L0UwZG5jzDUbSUE4A/c/WeJ2u7M/wI3AlWEswOP5hQcnySMBmuyWhMJv8uL3H1V9piko5w8ZYeutYQg3I8wr3ALcCuwmvDcBucYHB8lBPEc4PdmNosQYDPdvUAY2eWqT0yYZnvrmYRJv0XAW4Q9Wg2w3N3Xx8xytwuOi4EGYBNhMm8v4HTgSMKEZrJTYe6+mXBmxYCv5Rwcbc/x+8B3gXnu/kjK9s2sysw+B1wJ7EoYXq8hzHEsr8TgyBxNmIsaQzg02Rn4CHA3YQGs8939z6mKdZgYrSeMTgcStsWbCYdL9e7+7VQ1t6UiD1s6jAj2AO4FphL+gPsTAuRDbROjPZjfqCKExg6EuY0HCcfrLYRh/v7Aj9teACmZ2YC8zs+XUnaq+QrgKcKw/ijgP4H/cPfXytm3nsi2jZmEeY4HCNvF3sA0d0+6tkbbtpCdxZlCuGbldsKZtxrgQHefk7Jmd1RkeEC4BgL4O8IM/RGEF/hPgMuAL7j7m9njovdo2RzHTHe/Ofv8TMIf72p3f74C95ZlkY0MlxH20JsIE7JN5e1Vz2WHLlcQrlNZD1S7ey5vj8/OHs4gXLN0L+Ew6b2Ew+Z/Sz1i7I6KDI9somgmYULqGcKx5nTCMeAMd38+Ya3rCWc+9iOcyrze3Z9L1X5/kO2ljwcOJgTH+r4SvNnVy7OAm9y9Oacakwmn7F9w938wszMJp2JXAb8t1/rAFRceZjaOMKfxFcIVo43AJwkLD92S/fv7VBtmFlRnAH9FmBMo9JVDilLKLpxrzT7vE8HRpv1zy6Ht9xNG2PMJI9+fAb8ADgfuL+fvsSLCI7t244OEi7GuAn5JmHQ7CVgMvNfdLzKzgdmEY5596VMbfqnp97d9zOwwYKy732VmxxJGIGennIyN1atP1bbb0HYjDHmHEs5jP004zlwM7Orus7MfyX00oA2/Z/T7225PALPMrAX4OPDt3hAcUDkjj+uAUcAhhLdXn0e4avTStok37dGkr8ou5z8YKLj7knL3p02vDw8z2xE4LzvjcQghNH5GeI9HWRZ+FZEKCA94+4zHj4EJwPuAL7r72vL2SqR/q5QrTL9ImPcYTLjGQsEhUmYVMfLoSPMbIuVXkeEhIuVXKYctItLLKDykR8xs13L3QcpD4SE9dR2Amc3vzoO7+zjp/Xr1FaZSXmZ2DWGJwH0IC+nWA48T3nq+I2Fx4YkW7ocz0sxuJawvcS7wN4Q3Ew4FLiEszjSMsPKa9AEaeci2zCesv/E4YUHpWwkrnK0nrAb2pLv/iLDy2RWEWw28j7D47izCBX3HAzu6+5WEFb2lD1B4yLYUCMGwkbB2xFuEUcRcwhodbafrCtnp81bCCldtK18NaPc5WRvSB+iwRbbXtYQlC4cSln1cZmaXdvK4R7PDmAGEFa/2N7MbCKMS6QN0nYeIRNFhi4hEUXiISBSFh4hEUXiISJT/B93/InIEt0dYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x200 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "f, (ax1, ax2) = plt.subplots(2, 1, sharex=True)\n",
    "f.subplots_adjust(hspace=0.05) \n",
    "\n",
    "df = pd.read_csv('./df_dee_no_augment.csv')\n",
    "df = df[df.method.apply(lambda x: 'ts' in x)]\n",
    "df = df[df.method.apply(lambda x: 'augment' not in x)]\n",
    "df.method = df.method.apply(lambda x: x.replace('-augment', '').replace(' (ts)', '').replace('kfaclaplace', 'kfacl'))\n",
    "df = df[df.method != 'stochbn']\n",
    "# df = df[df.method != 'deepens']\n",
    "\n",
    "q    = 'dataset == \"%s\" and model == \"%s\" and metric == \"%s\" and n_samples == 100'\n",
    "df  = df.query(q % ('CIFAR100', 'PreResNet110', 'll'))\n",
    "sns.barplot(x=\"method\", y=\"dee\", data=df, palette=colors, \n",
    "                 order=['deepens', 'csgld', 'sse', 'fge', 'swag', 'vi', 'kfacl', 'onenet'], ax=ax1)\n",
    "sns.barplot(x=\"method\", y=\"dee\", data=df, palette=colors, \n",
    "                 order=['deepens', 'csgld', 'sse', 'fge', 'swag', 'vi', 'kfacl', 'onenet'], ax=ax2)\n",
    "\n",
    "\n",
    "# zoom-in / limit the view to different portions of the data\n",
    "ax1.set_ylim(85.1, 100)  # outliers only\n",
    "ax2.set_ylim(0, 14.9)  # most of the data\n",
    "\n",
    "ax1.set_xlabel('')\n",
    "\n",
    "# # hide the spines between ax and ax2\n",
    "\n",
    "ax1.xaxis.tick_top()\n",
    "ax1.tick_params(labeltop=False)  # don't put tick labels at the top\n",
    "ax2.xaxis.tick_bottom()\n",
    "\n",
    "d = .015  # how big to make the diagonal lines in axes coordinates\n",
    "# arguments to pass to plot, just so we don't keep repeating them\n",
    "kwargs = dict(transform=ax1.transAxes, color='gray', alpha=0.7, clip_on=False)\n",
    "ax1.plot((-d, +d), (-d, +d), **kwargs)        # top-left diagonal\n",
    "ax1.plot((1 - d, 1 + d), (-d, +d), **kwargs)  # top-right diagonal\n",
    "\n",
    "kwargs.update(transform=ax2.transAxes)  # switch to the bottom axes\n",
    "ax2.plot((-d, +d), (1 - d, 1 + d), **kwargs)  # bottom-left diagonal\n",
    "ax2.plot((1 - d, 1 + d), (1 - d, 1 + d), **kwargs)  # bottom-right diagonal\n",
    "\n",
    "plt.xticks(rotation=40)\n",
    "\n",
    "ax1.spines['bottom'].set_visible(False)\n",
    "ax2.spines['top'].set_visible(False)\n",
    "plt.savefig('./pdf/barplot-dee.pdf',bbox_inches='tight', pad_inches=0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}