summarize_baseline_scorings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "from os import listdir\n",
    "from os.path import isfile, join, isdir\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simply for loading results generated by calc_baselines.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format(raw):\n",
    "    return pd.DataFrame(\n",
    "        [\n",
    "            [raw[x][2][1].detach().cpu().numpy(), raw[x][3][1].detach().cpu().numpy()]\n",
    "            for x in raw.keys()\n",
    "        ],\n",
    "        index=raw.keys(),\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Correlation Baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>M_C</th>\n",
       "      <th>Std_C</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.058117</td>\n",
       "      <td>0.001337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.096283</td>\n",
       "      <td>0.003553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.171475</td>\n",
       "      <td>0.004106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.229816</td>\n",
       "      <td>0.005856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.342012</td>\n",
       "      <td>0.006172</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        M_C     Std_C\n",
       "0  0.058117  0.001337\n",
       "1  0.096283  0.003553\n",
       "2  0.171475  0.004106\n",
       "3  0.229816  0.005856\n",
       "4  0.342012  0.006172"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Datsets of 5/7/10/12/15 vars.\n",
    "pd.read_csv(\"scores/compute_C.csv\", index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "synth = pickle.load(open(\"scores/corr_test_auroc.p\", \"rb\"))\n",
    "synth2 = pickle.load(open(\"scores/corr_test_scale_up_auroc.p\", \"rb\"))\n",
    "\n",
    "aero = pickle.load(open(\"scores/aerosol_corr_test_auroc.p\", \"rb\"))\n",
    "river = pickle.load(open(\"scores/river_corr_test_auroc.p\", \"rb\"))\n",
    "kura = pickle.load(open(\"scores/kuramoto_corr_test_auroc.p\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>SL</th>\n",
       "      <td>0.9993599</td>\n",
       "      <td>0.996957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ML</th>\n",
       "      <td>0.9988861</td>\n",
       "      <td>0.9965359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SNL</th>\n",
       "      <td>0.93239415</td>\n",
       "      <td>0.94297427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MNL</th>\n",
       "      <td>0.9324211</td>\n",
       "      <td>0.9370265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LNL</th>\n",
       "      <td>0.90118945</td>\n",
       "      <td>0.9457957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XLNL</th>\n",
       "      <td>0.9095715</td>\n",
       "      <td>0.9345094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>joint</th>\n",
       "      <td>0.9481106</td>\n",
       "      <td>0.99738514</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                0           1\n",
       "SL      0.9993599    0.996957\n",
       "ML      0.9988861   0.9965359\n",
       "SNL    0.93239415  0.94297427\n",
       "MNL     0.9324211   0.9370265\n",
       "LNL    0.90118945   0.9457957\n",
       "XLNL    0.9095715   0.9345094\n",
       "joint   0.9481106  0.99738514"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "format(synth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>0.9991704</td>\n",
       "      <td>0.9968045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>0.9985823</td>\n",
       "      <td>0.99296653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_3</th>\n",
       "      <td>0.9973241</td>\n",
       "      <td>0.97637033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_4</th>\n",
       "      <td>0.9966355</td>\n",
       "      <td>0.9613055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_5</th>\n",
       "      <td>0.9948303</td>\n",
       "      <td>0.91737527</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    0           1\n",
       "scale_up    0.9991704   0.9968045\n",
       "scale_up_2  0.9985823  0.99296653\n",
       "scale_up_3  0.9973241  0.97637033\n",
       "scale_up_4  0.9966355   0.9613055\n",
       "scale_up_5  0.9948303  0.91737527"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "format(synth2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.6020), tensor(1.0000), tensor(0.4527))"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kura[0][1],river[1], aero"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Var baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "synth = pickle.load(open(\"scores/var_test_auroc.p\", \"rb\"))\n",
    "synth2 = pickle.load(open(\"scores/var_test_scale_up_auroc.p\", \"rb\"))\n",
    "\n",
    "#aero = pickle.load(open(\"scores/aerosol_corr_test_auroc.p\", \"rb\"))\n",
    "#river = pickle.load(open(\"scores/river_var_test_auroc.p\", \"rb\"))\n",
    "#kura = pickle.load(open(\"scores/kuramoto_var_test_auroc.p\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>SL</th>\n",
       "      <td>0.9997213</td>\n",
       "      <td>0.99999857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ML</th>\n",
       "      <td>0.99950457</td>\n",
       "      <td>0.9999966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SNL</th>\n",
       "      <td>0.9287549</td>\n",
       "      <td>0.9464862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MNL</th>\n",
       "      <td>0.93359876</td>\n",
       "      <td>0.9396866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LNL</th>\n",
       "      <td>0.90360534</td>\n",
       "      <td>0.95426846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XLNL</th>\n",
       "      <td>0.9088663</td>\n",
       "      <td>0.9427805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>joint</th>\n",
       "      <td>0.86286294</td>\n",
       "      <td>0.92559755</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                0           1\n",
       "SL      0.9997213  0.99999857\n",
       "ML     0.99950457   0.9999966\n",
       "SNL     0.9287549   0.9464862\n",
       "MNL    0.93359876   0.9396866\n",
       "LNL    0.90360534  0.95426846\n",
       "XLNL    0.9088663   0.9427805\n",
       "joint  0.86286294  0.92559755"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "format(synth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>0.99954927</td>\n",
       "      <td>0.99999034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>0.9993863</td>\n",
       "      <td>0.9999916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_3</th>\n",
       "      <td>0.99940944</td>\n",
       "      <td>0.99996775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_4</th>\n",
       "      <td>0.99926454</td>\n",
       "      <td>0.9999354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_5</th>\n",
       "      <td>0.99931884</td>\n",
       "      <td>0.99986553</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     0           1\n",
       "scale_up    0.99954927  0.99999034\n",
       "scale_up_2   0.9993863   0.9999916\n",
       "scale_up_3  0.99940944  0.99996775\n",
       "scale_up_4  0.99926454   0.9999354\n",
       "scale_up_5  0.99931884  0.99986553"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "format(synth2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_res_folder(mypath = \"experimental_results/synthetic_data_1_grid_results/std/\"):\n",
    "\n",
    "    onlyfiles = [mypath + f for f in listdir(mypath) if isfile(join(mypath, f))]\n",
    "    onlyfiles = sorted([x for x in onlyfiles if \"summary\" in x])\n",
    "    res = pd.concat([pd.read_csv(x, index_col=0) for x in onlyfiles], axis=1).T\n",
    "    return  res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = load_res_folder(\"experimental_results/synthetic_data_1_grid_results/run_1/\")\n",
    "b = load_res_folder(\"experimental_results/synthetic_data_1_grid_results/run_2/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HP</th>\n",
       "      <th>Test1_Auroc</th>\n",
       "      <th>Test2_AUROC</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>data.ds_name</th>\n",
       "      <th>model.model_type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">base</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.999499</td>\n",
       "      <td>0.999961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.999336</td>\n",
       "      <td>0.999894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.999328</td>\n",
       "      <td>0.999933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.999534</td>\n",
       "      <td>0.999968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.999558</td>\n",
       "      <td>0.999948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">big_base_mirror</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.613235</td>\n",
       "      <td>0.626371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.506232</td>\n",
       "      <td>0.513293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.499207</td>\n",
       "      <td>0.498309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.600757</td>\n",
       "      <td>0.621380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.599960</td>\n",
       "      <td>0.621146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">big_nl</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.709532</td>\n",
       "      <td>0.748808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.503338</td>\n",
       "      <td>0.506151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.501747</td>\n",
       "      <td>0.498722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.518067</td>\n",
       "      <td>0.517684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.659285</td>\n",
       "      <td>0.688653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">big_poly</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.547126</td>\n",
       "      <td>0.555322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.518390</td>\n",
       "      <td>0.518896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.520201</td>\n",
       "      <td>0.524884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.537860</td>\n",
       "      <td>0.545076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.784930</td>\n",
       "      <td>0.828722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">nl</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.916400</td>\n",
       "      <td>0.936746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.912467</td>\n",
       "      <td>0.934696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.905332</td>\n",
       "      <td>0.932553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.915942</td>\n",
       "      <td>0.934492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.915071</td>\n",
       "      <td>0.936571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">poly</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.932779</td>\n",
       "      <td>0.948389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.931417</td>\n",
       "      <td>0.948634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.930141</td>\n",
       "      <td>0.950117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.934952</td>\n",
       "      <td>0.955290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.934503</td>\n",
       "      <td>0.954934</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "HP                                Test1_Auroc  Test2_AUROC\n",
       "data.ds_name    model.model_type                          \n",
       "base            bidirectional        0.999499     0.999961\n",
       "                convM                0.999336     0.999894\n",
       "                mlp                  0.999328     0.999933\n",
       "                transformer          0.999534     0.999968\n",
       "                unidirectional       0.999558     0.999948\n",
       "big_base_mirror bidirectional        0.613235     0.626371\n",
       "                convM                0.506232     0.513293\n",
       "                mlp                  0.499207     0.498309\n",
       "                transformer          0.600757     0.621380\n",
       "                unidirectional       0.599960     0.621146\n",
       "big_nl          bidirectional        0.709532     0.748808\n",
       "                convM                0.503338     0.506151\n",
       "                mlp                  0.501747     0.498722\n",
       "                transformer          0.518067     0.517684\n",
       "                unidirectional       0.659285     0.688653\n",
       "big_poly        bidirectional        0.547126     0.555322\n",
       "                convM                0.518390     0.518896\n",
       "                mlp                  0.520201     0.524884\n",
       "                transformer          0.537860     0.545076\n",
       "                unidirectional       0.784930     0.828722\n",
       "nl              bidirectional        0.916400     0.936746\n",
       "                convM                0.912467     0.934696\n",
       "                mlp                  0.905332     0.932553\n",
       "                transformer          0.915942     0.934492\n",
       "                unidirectional       0.915071     0.936571\n",
       "poly            bidirectional        0.932779     0.948389\n",
       "                convM                0.931417     0.948634\n",
       "                mlp                  0.930141     0.950117\n",
       "                transformer          0.934952     0.955290\n",
       "                unidirectional       0.934503     0.954934"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = load_res_folder(\"experimental_results/synthetic_data_1_grid_results/std/\")\n",
    "res.loc[res[\"model\"].isnull(), \"model\"] = \"small.yaml\"\n",
    "# Provide some summary statistics over the runs.\n",
    "res = res[[\"model.model_type\", \"data.ds_name\",'Test1_Auroc', 'Test2_AUROC']]\n",
    "res[\"Test1_Auroc\"] = res[\"Test1_Auroc\"].astype(float)\n",
    "res[\"Test2_AUROC\"] = res[\"Test2_AUROC\"].astype(float)\n",
    "\n",
    "res.groupby([\"data.ds_name\", \"model.model_type\"]).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment 1 - Joint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = load_res_folder(\"experimental_results/synthetic_data_1_joint_grid_results/splitted_run_1/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HP</th>\n",
       "      <th>Test1_Auroc</th>\n",
       "      <th>Test2_AUROC</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>model.model_type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">big.yaml</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.952198</td>\n",
       "      <td>0.962480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.867127</td>\n",
       "      <td>0.885812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.751937</td>\n",
       "      <td>0.760817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.969769</td>\n",
       "      <td>0.977339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.946323</td>\n",
       "      <td>0.959416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">medium.yaml</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.906445</td>\n",
       "      <td>0.920411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.869493</td>\n",
       "      <td>0.887307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.915602</td>\n",
       "      <td>0.928056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.938883</td>\n",
       "      <td>0.949564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.894770</td>\n",
       "      <td>0.906847</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "HP                            Test1_Auroc  Test2_AUROC\n",
       "model       model.model_type                          \n",
       "big.yaml    bidirectional        0.952198     0.962480\n",
       "            convM                0.867127     0.885812\n",
       "            mlp                  0.751937     0.760817\n",
       "            transformer          0.969769     0.977339\n",
       "            unidirectional       0.946323     0.959416\n",
       "medium.yaml bidirectional        0.906445     0.920411\n",
       "            convM                0.869493     0.887307\n",
       "            mlp                  0.915602     0.928056\n",
       "            transformer          0.938883     0.949564\n",
       "            unidirectional       0.894770     0.906847"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = load_res_folder(\"experimental_results/synthetic_data_1_joint_grid_results/std/\")\n",
    "\n",
    "# Provide some summary statistics over the runs.\n",
    "res = res[[\"model.model_type\", \"model\",'Test1_Auroc', 'Test2_AUROC']]\n",
    "res[\"Test1_Auroc\"] = res[\"Test1_Auroc\"].astype(float)\n",
    "res[\"Test2_AUROC\"] = res[\"Test2_AUROC\"].astype(float)\n",
    "\n",
    "res.groupby([\"model\", \"model.model_type\"]).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We dropped quite a bit of runs here somehow. So the results might be underreporting.\n",
    "d = load_res_folder(\"experimental_results/synthetic_data_2_grid_results/splitted_run_2/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>HP</th>\n",
       "      <th>model.optimizer_lr</th>\n",
       "      <th>data.batch_size</th>\n",
       "      <th>model.weight_decay</th>\n",
       "      <th>model.corr_regularization</th>\n",
       "      <th>corr_input</th>\n",
       "      <th>regression_head</th>\n",
       "      <th>hparams_search</th>\n",
       "      <th>max_lags</th>\n",
       "      <th>n_vars</th>\n",
       "      <th>Val_Auroc</th>\n",
       "      <th>Val_MSE</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>model.model_type</th>\n",
       "      <th>data.ds_name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"15\" valign=\"top\">medium.yaml</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">bidirectional</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">convM</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">mlp</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">transformer</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">unidirectional</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"10\" valign=\"top\">small.yaml</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">bidirectional</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">convM</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">mlp</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">transformer</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>71</td>\n",
       "      <td>71</td>\n",
       "      <td>71</td>\n",
       "      <td>71</td>\n",
       "      <td>71</td>\n",
       "      <td>71</td>\n",
       "      <td>71</td>\n",
       "      <td>71</td>\n",
       "      <td>71</td>\n",
       "      <td>71</td>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">unidirectional</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "HP                                            model.optimizer_lr   \n",
       "model       model.model_type data.ds_name                          \n",
       "medium.yaml bidirectional    big_base_mirror                  24  \\\n",
       "                             scale_up                         24   \n",
       "                             scale_up_2                       24   \n",
       "            convM            big_base_mirror                  24   \n",
       "                             scale_up                         24   \n",
       "                             scale_up_2                       24   \n",
       "            mlp              big_base_mirror                  24   \n",
       "                             scale_up                         24   \n",
       "                             scale_up_2                       24   \n",
       "            transformer      big_base_mirror                  24   \n",
       "                             scale_up                         24   \n",
       "                             scale_up_2                       24   \n",
       "            unidirectional   big_base_mirror                  24   \n",
       "                             scale_up                         24   \n",
       "                             scale_up_2                       24   \n",
       "small.yaml  bidirectional    big_base_mirror                  24   \n",
       "                             scale_up_2                       48   \n",
       "            convM            big_base_mirror                  24   \n",
       "                             scale_up_2                       72   \n",
       "            mlp              big_base_mirror                  24   \n",
       "                             scale_up_2                       48   \n",
       "            transformer      big_base_mirror                  24   \n",
       "                             scale_up_2                       71   \n",
       "            unidirectional   big_base_mirror                  24   \n",
       "                             scale_up_2                       48   \n",
       "\n",
       "HP                                            data.batch_size   \n",
       "model       model.model_type data.ds_name                       \n",
       "medium.yaml bidirectional    big_base_mirror               24  \\\n",
       "                             scale_up                      24   \n",
       "                             scale_up_2                    24   \n",
       "            convM            big_base_mirror               24   \n",
       "                             scale_up                      24   \n",
       "                             scale_up_2                    24   \n",
       "            mlp              big_base_mirror               24   \n",
       "                             scale_up                      24   \n",
       "                             scale_up_2                    24   \n",
       "            transformer      big_base_mirror               24   \n",
       "                             scale_up                      24   \n",
       "                             scale_up_2                    24   \n",
       "            unidirectional   big_base_mirror               24   \n",
       "                             scale_up                      24   \n",
       "                             scale_up_2                    24   \n",
       "small.yaml  bidirectional    big_base_mirror               24   \n",
       "                             scale_up_2                    48   \n",
       "            convM            big_base_mirror               24   \n",
       "                             scale_up_2                    72   \n",
       "            mlp              big_base_mirror               24   \n",
       "                             scale_up_2                    48   \n",
       "            transformer      big_base_mirror               24   \n",
       "                             scale_up_2                    71   \n",
       "            unidirectional   big_base_mirror               24   \n",
       "                             scale_up_2                    48   \n",
       "\n",
       "HP                                            model.weight_decay   \n",
       "model       model.model_type data.ds_name                          \n",
       "medium.yaml bidirectional    big_base_mirror                  24  \\\n",
       "                             scale_up                         24   \n",
       "                             scale_up_2                       24   \n",
       "            convM            big_base_mirror                  24   \n",
       "                             scale_up                         24   \n",
       "                             scale_up_2                       24   \n",
       "            mlp              big_base_mirror                  24   \n",
       "                             scale_up                         24   \n",
       "                             scale_up_2                       24   \n",
       "            transformer      big_base_mirror                  24   \n",
       "                             scale_up                         24   \n",
       "                             scale_up_2                       24   \n",
       "            unidirectional   big_base_mirror                  24   \n",
       "                             scale_up                         24   \n",
       "                             scale_up_2                       24   \n",
       "small.yaml  bidirectional    big_base_mirror                  24   \n",
       "                             scale_up_2                       48   \n",
       "            convM            big_base_mirror                  24   \n",
       "                             scale_up_2                       72   \n",
       "            mlp              big_base_mirror                  24   \n",
       "                             scale_up_2                       48   \n",
       "            transformer      big_base_mirror                  24   \n",
       "                             scale_up_2                       71   \n",
       "            unidirectional   big_base_mirror                  24   \n",
       "                             scale_up_2                       48   \n",
       "\n",
       "HP                                            model.corr_regularization   \n",
       "model       model.model_type data.ds_name                                 \n",
       "medium.yaml bidirectional    big_base_mirror                         24  \\\n",
       "                             scale_up                                24   \n",
       "                             scale_up_2                              24   \n",
       "            convM            big_base_mirror                         24   \n",
       "                             scale_up                                24   \n",
       "                             scale_up_2                              24   \n",
       "            mlp              big_base_mirror                         24   \n",
       "                             scale_up                                24   \n",
       "                             scale_up_2                              24   \n",
       "            transformer      big_base_mirror                         24   \n",
       "                             scale_up                                24   \n",
       "                             scale_up_2                              24   \n",
       "            unidirectional   big_base_mirror                         24   \n",
       "                             scale_up                                24   \n",
       "                             scale_up_2                              24   \n",
       "small.yaml  bidirectional    big_base_mirror                         24   \n",
       "                             scale_up_2                              48   \n",
       "            convM            big_base_mirror                         24   \n",
       "                             scale_up_2                              72   \n",
       "            mlp              big_base_mirror                         24   \n",
       "                             scale_up_2                              48   \n",
       "            transformer      big_base_mirror                         24   \n",
       "                             scale_up_2                              71   \n",
       "            unidirectional   big_base_mirror                         24   \n",
       "                             scale_up_2                              48   \n",
       "\n",
       "HP                                            corr_input  regression_head   \n",
       "model       model.model_type data.ds_name                                   \n",
       "medium.yaml bidirectional    big_base_mirror          24               24  \\\n",
       "                             scale_up                 24               24   \n",
       "                             scale_up_2               24               24   \n",
       "            convM            big_base_mirror          24               24   \n",
       "                             scale_up                 24               24   \n",
       "                             scale_up_2               24               24   \n",
       "            mlp              big_base_mirror          24               24   \n",
       "                             scale_up                 24               24   \n",
       "                             scale_up_2               24               24   \n",
       "            transformer      big_base_mirror          24               24   \n",
       "                             scale_up                 24               24   \n",
       "                             scale_up_2               24               24   \n",
       "            unidirectional   big_base_mirror          24               24   \n",
       "                             scale_up                 24               24   \n",
       "                             scale_up_2               24               24   \n",
       "small.yaml  bidirectional    big_base_mirror          24               24   \n",
       "                             scale_up_2               48               48   \n",
       "            convM            big_base_mirror          24               24   \n",
       "                             scale_up_2               72               72   \n",
       "            mlp              big_base_mirror          24               24   \n",
       "                             scale_up_2               48               48   \n",
       "            transformer      big_base_mirror          24               24   \n",
       "                             scale_up_2               71               71   \n",
       "            unidirectional   big_base_mirror          24               24   \n",
       "                             scale_up_2               48               48   \n",
       "\n",
       "HP                                            hparams_search  max_lags   \n",
       "model       model.model_type data.ds_name                                \n",
       "medium.yaml bidirectional    big_base_mirror              24        24  \\\n",
       "                             scale_up                     24        24   \n",
       "                             scale_up_2                   24        24   \n",
       "            convM            big_base_mirror              24        24   \n",
       "                             scale_up                     24        24   \n",
       "                             scale_up_2                   24        24   \n",
       "            mlp              big_base_mirror              24        24   \n",
       "                             scale_up                     24        24   \n",
       "                             scale_up_2                   24        24   \n",
       "            transformer      big_base_mirror              24        24   \n",
       "                             scale_up                     24        24   \n",
       "                             scale_up_2                   24        24   \n",
       "            unidirectional   big_base_mirror              24        24   \n",
       "                             scale_up                     24        24   \n",
       "                             scale_up_2                   24        24   \n",
       "small.yaml  bidirectional    big_base_mirror              24        24   \n",
       "                             scale_up_2                   48        48   \n",
       "            convM            big_base_mirror              24        24   \n",
       "                             scale_up_2                   72        72   \n",
       "            mlp              big_base_mirror              24        24   \n",
       "                             scale_up_2                   48        48   \n",
       "            transformer      big_base_mirror              24        24   \n",
       "                             scale_up_2                   71        71   \n",
       "            unidirectional   big_base_mirror              24        24   \n",
       "                             scale_up_2                   48        48   \n",
       "\n",
       "HP                                            n_vars  Val_Auroc  Val_MSE  \n",
       "model       model.model_type data.ds_name                                 \n",
       "medium.yaml bidirectional    big_base_mirror      24         24       24  \n",
       "                             scale_up             24         24       24  \n",
       "                             scale_up_2           24         24       24  \n",
       "            convM            big_base_mirror      24         24       24  \n",
       "                             scale_up             24         24       24  \n",
       "                             scale_up_2           24         24       24  \n",
       "            mlp              big_base_mirror      24         24       24  \n",
       "                             scale_up             24         24       24  \n",
       "                             scale_up_2           24         24       24  \n",
       "            transformer      big_base_mirror      24         24       24  \n",
       "                             scale_up             24         24       24  \n",
       "                             scale_up_2           24         24       24  \n",
       "            unidirectional   big_base_mirror      24         24       24  \n",
       "                             scale_up             24         24       24  \n",
       "                             scale_up_2           24         24       24  \n",
       "small.yaml  bidirectional    big_base_mirror      24         24       24  \n",
       "                             scale_up_2           48         48       48  \n",
       "            convM            big_base_mirror      24         24       24  \n",
       "                             scale_up_2           72         72       72  \n",
       "            mlp              big_base_mirror      24         24       24  \n",
       "                             scale_up_2           48         48       48  \n",
       "            transformer      big_base_mirror      24         24       24  \n",
       "                             scale_up_2           71         71       71  \n",
       "            unidirectional   big_base_mirror      24         24       24  \n",
       "                             scale_up_2           48         48       48  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d.groupby([ \"model\",\"model.model_type\", \"data.ds_name\"]).count()[40:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = load_res_folder(\"experimental_results/synthetic_data_2_grid_results/std/\")\n",
    "\n",
    "# Provide some summary statistics over the runs.\n",
    "#res = res[[\"model.model_type\", \"model\",'Test1_Auroc', 'Test2_AUROC']]\n",
    "#res[\"Test1_Auroc\"] = res[\"Test1_Auroc\"].astype(float)\n",
    "#res[\"Test2_AUROC\"] = res[\"Test2_AUROC\"].astype(float)\n",
    "\n",
    "#res.groupby([\"model\", \"model.model_type\"]).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>HP</th>\n",
       "      <th>seed</th>\n",
       "      <th>hparams_search</th>\n",
       "      <th>model.optimizer_lr</th>\n",
       "      <th>data.batch_size</th>\n",
       "      <th>model.weight_decay</th>\n",
       "      <th>model.corr_regularization</th>\n",
       "      <th>corr_input</th>\n",
       "      <th>regression_head</th>\n",
       "      <th>max_lags</th>\n",
       "      <th>n_vars</th>\n",
       "      <th>Test1_Auroc</th>\n",
       "      <th>Test1_MSE</th>\n",
       "      <th>Test2_AUROC</th>\n",
       "      <th>Test2_MSE</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>model.model_type</th>\n",
       "      <th>data.ds_name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"16\" valign=\"top\">big.yaml</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">bidirectional</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_3</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">convM</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_3</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">transformer</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_3</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">unidirectional</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_3</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"14\" valign=\"top\">medium.yaml</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">bidirectional</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">convM</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">mlp</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">transformer</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">unidirectional</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">small.yaml</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">bidirectional</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">unidirectional</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "HP                                            seed  hparams_search   \n",
       "model       model.model_type data.ds_name                            \n",
       "big.yaml    bidirectional    big_base_mirror     5               5  \\\n",
       "                             scale_up            5               5   \n",
       "                             scale_up_2          5               5   \n",
       "                             scale_up_3          5               5   \n",
       "            convM            big_base_mirror     5               5   \n",
       "                             scale_up            5               5   \n",
       "                             scale_up_2          5               5   \n",
       "                             scale_up_3          5               5   \n",
       "            transformer      big_base_mirror     5               5   \n",
       "                             scale_up            5               5   \n",
       "                             scale_up_2          5               5   \n",
       "                             scale_up_3          2               2   \n",
       "            unidirectional   big_base_mirror     5               5   \n",
       "                             scale_up            5               5   \n",
       "                             scale_up_2          5               5   \n",
       "                             scale_up_3          5               5   \n",
       "medium.yaml bidirectional    big_base_mirror     5               5   \n",
       "                             scale_up            5               5   \n",
       "                             scale_up_2          5               5   \n",
       "            convM            big_base_mirror     5               5   \n",
       "                             scale_up            5               5   \n",
       "                             scale_up_2          5               5   \n",
       "            mlp              big_base_mirror     5               5   \n",
       "                             scale_up            5               5   \n",
       "                             scale_up_2          5               5   \n",
       "            transformer      big_base_mirror     5               5   \n",
       "                             scale_up            5               5   \n",
       "            unidirectional   big_base_mirror     5               5   \n",
       "                             scale_up            5               5   \n",
       "                             scale_up_2          5               5   \n",
       "small.yaml  bidirectional    big_base_mirror     5               5   \n",
       "                             scale_up_2          5               5   \n",
       "            convM            big_base_mirror     5               5   \n",
       "            mlp              big_base_mirror     5               5   \n",
       "            transformer      big_base_mirror     5               5   \n",
       "            unidirectional   big_base_mirror     5               5   \n",
       "                             scale_up_2          5               5   \n",
       "\n",
       "HP                                            model.optimizer_lr   \n",
       "model       model.model_type data.ds_name                          \n",
       "big.yaml    bidirectional    big_base_mirror                   5  \\\n",
       "                             scale_up                          5   \n",
       "                             scale_up_2                        5   \n",
       "                             scale_up_3                        5   \n",
       "            convM            big_base_mirror                   5   \n",
       "                             scale_up                          5   \n",
       "                             scale_up_2                        5   \n",
       "                             scale_up_3                        5   \n",
       "            transformer      big_base_mirror                   5   \n",
       "                             scale_up                          5   \n",
       "                             scale_up_2                        5   \n",
       "                             scale_up_3                        2   \n",
       "            unidirectional   big_base_mirror                   5   \n",
       "                             scale_up                          5   \n",
       "                             scale_up_2                        5   \n",
       "                             scale_up_3                        5   \n",
       "medium.yaml bidirectional    big_base_mirror                   5   \n",
       "                             scale_up                          5   \n",
       "                             scale_up_2                        5   \n",
       "            convM            big_base_mirror                   5   \n",
       "                             scale_up                          5   \n",
       "                             scale_up_2                        5   \n",
       "            mlp              big_base_mirror                   5   \n",
       "                             scale_up                          5   \n",
       "                             scale_up_2                        5   \n",
       "            transformer      big_base_mirror                   5   \n",
       "                             scale_up                          5   \n",
       "            unidirectional   big_base_mirror                   5   \n",
       "                             scale_up                          5   \n",
       "                             scale_up_2                        5   \n",
       "small.yaml  bidirectional    big_base_mirror                   5   \n",
       "                             scale_up_2                        5   \n",
       "            convM            big_base_mirror                   5   \n",
       "            mlp              big_base_mirror                   5   \n",
       "            transformer      big_base_mirror                   5   \n",
       "            unidirectional   big_base_mirror                   5   \n",
       "                             scale_up_2                        5   \n",
       "\n",
       "HP                                            data.batch_size   \n",
       "model       model.model_type data.ds_name                       \n",
       "big.yaml    bidirectional    big_base_mirror                5  \\\n",
       "                             scale_up                       5   \n",
       "                             scale_up_2                     5   \n",
       "                             scale_up_3                     5   \n",
       "            convM            big_base_mirror                5   \n",
       "                             scale_up                       5   \n",
       "                             scale_up_2                     5   \n",
       "                             scale_up_3                     5   \n",
       "            transformer      big_base_mirror                5   \n",
       "                             scale_up                       5   \n",
       "                             scale_up_2                     5   \n",
       "                             scale_up_3                     2   \n",
       "            unidirectional   big_base_mirror                5   \n",
       "                             scale_up                       5   \n",
       "                             scale_up_2                     5   \n",
       "                             scale_up_3                     5   \n",
       "medium.yaml bidirectional    big_base_mirror                5   \n",
       "                             scale_up                       5   \n",
       "                             scale_up_2                     5   \n",
       "            convM            big_base_mirror                5   \n",
       "                             scale_up                       5   \n",
       "                             scale_up_2                     5   \n",
       "            mlp              big_base_mirror                5   \n",
       "                             scale_up                       5   \n",
       "                             scale_up_2                     5   \n",
       "            transformer      big_base_mirror                5   \n",
       "                             scale_up                       5   \n",
       "            unidirectional   big_base_mirror                5   \n",
       "                             scale_up                       5   \n",
       "                             scale_up_2                     5   \n",
       "small.yaml  bidirectional    big_base_mirror                5   \n",
       "                             scale_up_2                     5   \n",
       "            convM            big_base_mirror                5   \n",
       "            mlp              big_base_mirror                5   \n",
       "            transformer      big_base_mirror                5   \n",
       "            unidirectional   big_base_mirror                5   \n",
       "                             scale_up_2                     5   \n",
       "\n",
       "HP                                            model.weight_decay   \n",
       "model       model.model_type data.ds_name                          \n",
       "big.yaml    bidirectional    big_base_mirror                   5  \\\n",
       "                             scale_up                          5   \n",
       "                             scale_up_2                        5   \n",
       "                             scale_up_3                        5   \n",
       "            convM            big_base_mirror                   5   \n",
       "                             scale_up                          5   \n",
       "                             scale_up_2                        5   \n",
       "                             scale_up_3                        5   \n",
       "            transformer      big_base_mirror                   5   \n",
       "                             scale_up                          5   \n",
       "                             scale_up_2                        5   \n",
       "                             scale_up_3                        2   \n",
       "            unidirectional   big_base_mirror                   5   \n",
       "                             scale_up                          5   \n",
       "                             scale_up_2                        5   \n",
       "                             scale_up_3                        5   \n",
       "medium.yaml bidirectional    big_base_mirror                   5   \n",
       "                             scale_up                          5   \n",
       "                             scale_up_2                        5   \n",
       "            convM            big_base_mirror                   5   \n",
       "                             scale_up                          5   \n",
       "                             scale_up_2                        5   \n",
       "            mlp              big_base_mirror                   5   \n",
       "                             scale_up                          5   \n",
       "                             scale_up_2                        5   \n",
       "            transformer      big_base_mirror                   5   \n",
       "                             scale_up                          5   \n",
       "            unidirectional   big_base_mirror                   5   \n",
       "                             scale_up                          5   \n",
       "                             scale_up_2                        5   \n",
       "small.yaml  bidirectional    big_base_mirror                   5   \n",
       "                             scale_up_2                        5   \n",
       "            convM            big_base_mirror                   5   \n",
       "            mlp              big_base_mirror                   5   \n",
       "            transformer      big_base_mirror                   5   \n",
       "            unidirectional   big_base_mirror                   5   \n",
       "                             scale_up_2                        5   \n",
       "\n",
       "HP                                            model.corr_regularization   \n",
       "model       model.model_type data.ds_name                                 \n",
       "big.yaml    bidirectional    big_base_mirror                          5  \\\n",
       "                             scale_up                                 5   \n",
       "                             scale_up_2                               5   \n",
       "                             scale_up_3                               5   \n",
       "            convM            big_base_mirror                          5   \n",
       "                             scale_up                                 5   \n",
       "                             scale_up_2                               5   \n",
       "                             scale_up_3                               5   \n",
       "            transformer      big_base_mirror                          5   \n",
       "                             scale_up                                 5   \n",
       "                             scale_up_2                               5   \n",
       "                             scale_up_3                               2   \n",
       "            unidirectional   big_base_mirror                          5   \n",
       "                             scale_up                                 5   \n",
       "                             scale_up_2                               5   \n",
       "                             scale_up_3                               5   \n",
       "medium.yaml bidirectional    big_base_mirror                          5   \n",
       "                             scale_up                                 5   \n",
       "                             scale_up_2                               5   \n",
       "            convM            big_base_mirror                          5   \n",
       "                             scale_up                                 5   \n",
       "                             scale_up_2                               5   \n",
       "            mlp              big_base_mirror                          5   \n",
       "                             scale_up                                 5   \n",
       "                             scale_up_2                               5   \n",
       "            transformer      big_base_mirror                          5   \n",
       "                             scale_up                                 5   \n",
       "            unidirectional   big_base_mirror                          5   \n",
       "                             scale_up                                 5   \n",
       "                             scale_up_2                               5   \n",
       "small.yaml  bidirectional    big_base_mirror                          5   \n",
       "                             scale_up_2                               5   \n",
       "            convM            big_base_mirror                          5   \n",
       "            mlp              big_base_mirror                          5   \n",
       "            transformer      big_base_mirror                          5   \n",
       "            unidirectional   big_base_mirror                          5   \n",
       "                             scale_up_2                               5   \n",
       "\n",
       "HP                                            corr_input  regression_head   \n",
       "model       model.model_type data.ds_name                                   \n",
       "big.yaml    bidirectional    big_base_mirror           5                5  \\\n",
       "                             scale_up                  5                5   \n",
       "                             scale_up_2                5                5   \n",
       "                             scale_up_3                5                5   \n",
       "            convM            big_base_mirror           5                5   \n",
       "                             scale_up                  5                5   \n",
       "                             scale_up_2                5                5   \n",
       "                             scale_up_3                5                5   \n",
       "            transformer      big_base_mirror           5                5   \n",
       "                             scale_up                  5                5   \n",
       "                             scale_up_2                5                5   \n",
       "                             scale_up_3                2                2   \n",
       "            unidirectional   big_base_mirror           5                5   \n",
       "                             scale_up                  5                5   \n",
       "                             scale_up_2                5                5   \n",
       "                             scale_up_3                5                5   \n",
       "medium.yaml bidirectional    big_base_mirror           5                5   \n",
       "                             scale_up                  5                5   \n",
       "                             scale_up_2                5                5   \n",
       "            convM            big_base_mirror           5                5   \n",
       "                             scale_up                  5                5   \n",
       "                             scale_up_2                5                5   \n",
       "            mlp              big_base_mirror           5                5   \n",
       "                             scale_up                  5                5   \n",
       "                             scale_up_2                5                5   \n",
       "            transformer      big_base_mirror           5                5   \n",
       "                             scale_up                  5                5   \n",
       "            unidirectional   big_base_mirror           5                5   \n",
       "                             scale_up                  5                5   \n",
       "                             scale_up_2                5                5   \n",
       "small.yaml  bidirectional    big_base_mirror           5                5   \n",
       "                             scale_up_2                5                5   \n",
       "            convM            big_base_mirror           5                5   \n",
       "            mlp              big_base_mirror           5                5   \n",
       "            transformer      big_base_mirror           5                5   \n",
       "            unidirectional   big_base_mirror           5                5   \n",
       "                             scale_up_2                5                5   \n",
       "\n",
       "HP                                            max_lags  n_vars  Test1_Auroc   \n",
       "model       model.model_type data.ds_name                                     \n",
       "big.yaml    bidirectional    big_base_mirror         5       5            5  \\\n",
       "                             scale_up                5       5            5   \n",
       "                             scale_up_2              5       5            5   \n",
       "                             scale_up_3              5       5            5   \n",
       "            convM            big_base_mirror         5       5            5   \n",
       "                             scale_up                5       5            5   \n",
       "                             scale_up_2              5       5            5   \n",
       "                             scale_up_3              5       5            5   \n",
       "            transformer      big_base_mirror         5       5            5   \n",
       "                             scale_up                5       5            5   \n",
       "                             scale_up_2              5       5            5   \n",
       "                             scale_up_3              2       2            2   \n",
       "            unidirectional   big_base_mirror         5       5            5   \n",
       "                             scale_up                5       5            5   \n",
       "                             scale_up_2              5       5            5   \n",
       "                             scale_up_3              5       5            5   \n",
       "medium.yaml bidirectional    big_base_mirror         5       5            5   \n",
       "                             scale_up                5       5            5   \n",
       "                             scale_up_2              5       5            5   \n",
       "            convM            big_base_mirror         5       5            5   \n",
       "                             scale_up                5       5            5   \n",
       "                             scale_up_2              5       5            5   \n",
       "            mlp              big_base_mirror         5       5            5   \n",
       "                             scale_up                5       5            5   \n",
       "                             scale_up_2              5       5            5   \n",
       "            transformer      big_base_mirror         5       5            5   \n",
       "                             scale_up                5       5            5   \n",
       "            unidirectional   big_base_mirror         5       5            5   \n",
       "                             scale_up                5       5            5   \n",
       "                             scale_up_2              5       5            5   \n",
       "small.yaml  bidirectional    big_base_mirror         5       5            5   \n",
       "                             scale_up_2              5       5            5   \n",
       "            convM            big_base_mirror         5       5            5   \n",
       "            mlp              big_base_mirror         5       5            5   \n",
       "            transformer      big_base_mirror         5       5            5   \n",
       "            unidirectional   big_base_mirror         5       5            5   \n",
       "                             scale_up_2              5       5            5   \n",
       "\n",
       "HP                                            Test1_MSE  Test2_AUROC   \n",
       "model       model.model_type data.ds_name                              \n",
       "big.yaml    bidirectional    big_base_mirror          5            5  \\\n",
       "                             scale_up                 5            5   \n",
       "                             scale_up_2               5            5   \n",
       "                             scale_up_3               5            5   \n",
       "            convM            big_base_mirror          5            5   \n",
       "                             scale_up                 5            5   \n",
       "                             scale_up_2               5            5   \n",
       "                             scale_up_3               5            5   \n",
       "            transformer      big_base_mirror          5            5   \n",
       "                             scale_up                 5            5   \n",
       "                             scale_up_2               5            5   \n",
       "                             scale_up_3               2            2   \n",
       "            unidirectional   big_base_mirror          5            5   \n",
       "                             scale_up                 5            5   \n",
       "                             scale_up_2               5            5   \n",
       "                             scale_up_3               5            5   \n",
       "medium.yaml bidirectional    big_base_mirror          5            5   \n",
       "                             scale_up                 5            5   \n",
       "                             scale_up_2               5            5   \n",
       "            convM            big_base_mirror          5            5   \n",
       "                             scale_up                 5            5   \n",
       "                             scale_up_2               5            5   \n",
       "            mlp              big_base_mirror          5            5   \n",
       "                             scale_up                 5            5   \n",
       "                             scale_up_2               5            5   \n",
       "            transformer      big_base_mirror          5            5   \n",
       "                             scale_up                 5            5   \n",
       "            unidirectional   big_base_mirror          5            5   \n",
       "                             scale_up                 5            5   \n",
       "                             scale_up_2               5            5   \n",
       "small.yaml  bidirectional    big_base_mirror          5            5   \n",
       "                             scale_up_2               5            5   \n",
       "            convM            big_base_mirror          5            5   \n",
       "            mlp              big_base_mirror          5            5   \n",
       "            transformer      big_base_mirror          5            5   \n",
       "            unidirectional   big_base_mirror          5            5   \n",
       "                             scale_up_2               5            5   \n",
       "\n",
       "HP                                            Test2_MSE  \n",
       "model       model.model_type data.ds_name                \n",
       "big.yaml    bidirectional    big_base_mirror          5  \n",
       "                             scale_up                 5  \n",
       "                             scale_up_2               5  \n",
       "                             scale_up_3               5  \n",
       "            convM            big_base_mirror          5  \n",
       "                             scale_up                 5  \n",
       "                             scale_up_2               5  \n",
       "                             scale_up_3               5  \n",
       "            transformer      big_base_mirror          5  \n",
       "                             scale_up                 5  \n",
       "                             scale_up_2               5  \n",
       "                             scale_up_3               2  \n",
       "            unidirectional   big_base_mirror          5  \n",
       "                             scale_up                 5  \n",
       "                             scale_up_2               5  \n",
       "                             scale_up_3               5  \n",
       "medium.yaml bidirectional    big_base_mirror          5  \n",
       "                             scale_up                 5  \n",
       "                             scale_up_2               5  \n",
       "            convM            big_base_mirror          5  \n",
       "                             scale_up                 5  \n",
       "                             scale_up_2               5  \n",
       "            mlp              big_base_mirror          5  \n",
       "                             scale_up                 5  \n",
       "                             scale_up_2               5  \n",
       "            transformer      big_base_mirror          5  \n",
       "                             scale_up                 5  \n",
       "            unidirectional   big_base_mirror          5  \n",
       "                             scale_up                 5  \n",
       "                             scale_up_2               5  \n",
       "small.yaml  bidirectional    big_base_mirror          5  \n",
       "                             scale_up_2               5  \n",
       "            convM            big_base_mirror          5  \n",
       "            mlp              big_base_mirror          5  \n",
       "            transformer      big_base_mirror          5  \n",
       "            unidirectional   big_base_mirror          5  \n",
       "                             scale_up_2               5  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res.groupby([ \"model\",\"model.model_type\", \"data.ds_name\"]).count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment Kuramoto"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "e = load_res_folder(\"experimental_results/kuramoto_grid_results/splitted_run_1/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>HP</th>\n",
       "      <th>Test1_Auroc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model.model_type</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.852495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.954027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.824648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.888369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.905020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "HP                Test1_Auroc\n",
       "model.model_type             \n",
       "bidirectional        0.852495\n",
       "convM                0.954027\n",
       "mlp                  0.824648\n",
       "transformer          0.888369\n",
       "unidirectional       0.905020"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = load_res_folder(\"experimental_results/kuramoto_grid_results/std/\")\n",
    "\n",
    "# Provide some summary statistics over the runs.\n",
    "res = res[[\"model.model_type\",'Test1_Auroc']]\n",
    "res[\"Test1_Auroc\"] = res[\"Test1_Auroc\"].astype(float)\n",
    "\n",
    "res.groupby([ \"model.model_type\"]).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr = pickle.load(open(\"scores/corr_test_auroc.p\", \"rb\"))\n",
    "var = pickle.load(open(\"scores/var_test_auroc.p\", \"rb\"))\n",
    "# pcmci = pickle.load(open(\"scores/pcmci_test_auroc.p\", \"rb\"))\n",
    "# pcmci2 = pickle.load(open(\"scores/pcmci_test_auroc_joint.p\", \"rb\"))\n",
    "\n",
    "\n",
    "corr = pd.DataFrame(\n",
    "    [\n",
    "        [corr[x][2][1].detach().cpu().numpy(), corr[x][3][1].detach().cpu().numpy()]\n",
    "        for x in corr.keys()\n",
    "    ],\n",
    "    index=corr.keys(),\n",
    ")\n",
    "var = pd.DataFrame(\n",
    "    [\n",
    "        [var[x][2][1].detach().cpu().numpy(), var[x][3][1].detach().cpu().numpy()]\n",
    "        for x in var.keys()\n",
    "    ],\n",
    "    index=var.keys(),\n",
    ")\n",
    "# pcmci = pd.DataFrame([[pcmci[x][0][1].detach().cpu().numpy(), pcmci[x][1][1].detach().cpu().numpy()] for x in pcmci.keys()], index=pcmci.keys() )\n",
    "# pcmci2 = pd.DataFrame([[pcmci2[x][0][1].detach().cpu().numpy(), pcmci2[x][1][1].detach().cpu().numpy()] for x in pcmci2.keys()], index=pcmci2.keys() )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr = pickle.load(open(\"scores/corr_test_auroc.p\", \"rb\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[((tensor([0.0000, 0.0000, 0.0000,  ..., 1.0000, 1.0000, 1.0000]),\n",
       "   tensor([0.0000e+00, 8.6550e-05, 1.7310e-04,  ..., 1.0000e+00, 1.0000e+00,\n",
       "           1.0000e+00]),\n",
       "   tensor([1.0000e+00, 8.3439e-01, 8.1610e-01,  ..., 2.1808e-06, 2.0944e-06,\n",
       "           3.6692e-07])),\n",
       "  tensor(0.9396)),\n",
       " ((tensor([0.0000, 0.0000, 0.0000,  ..., 0.9997, 0.9999, 1.0000]),\n",
       "   tensor([0.0000e+00, 8.3752e-04, 1.6750e-03,  ..., 1.0000e+00, 1.0000e+00,\n",
       "           1.0000e+00]),\n",
       "   tensor([1.0000e+00, 4.4940e-01, 4.3376e-01,  ..., 3.3696e-05, 7.6904e-06,\n",
       "           2.3486e-06])),\n",
       "  tensor(0.9158)),\n",
       " ((tensor([0.0000, 0.0000, 0.0000,  ..., 0.9997, 0.9999, 1.0000]),\n",
       "   tensor([0.0000e+00, 8.4388e-04, 1.6878e-03,  ..., 1.0000e+00, 1.0000e+00,\n",
       "           1.0000e+00]),\n",
       "   tensor([1.0000e+00, 5.0259e-01, 4.9486e-01,  ..., 2.4641e-05, 1.5831e-05,\n",
       "           1.7236e-06])),\n",
       "  tensor(0.9031)),\n",
       " ((tensor([0.0000, 0.0000, 0.0000,  ..., 0.9997, 0.9999, 1.0000]),\n",
       "   tensor([0.0000e+00, 7.9681e-04, 1.5936e-03,  ..., 1.0000e+00, 1.0000e+00,\n",
       "           1.0000e+00]),\n",
       "   tensor([1.0000e+00, 8.1287e-01, 8.0800e-01,  ..., 1.6023e-05, 7.6446e-06,\n",
       "           5.7629e-06])),\n",
       "  tensor(0.9977))]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corr[\"LNL\"]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "causal_pretraining",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}